Nyaaori/rockbox
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Modernize toolchain script and add generic arm toolchain

Browse source 
I did some cleanup of redudant stuff, simplify some logic, also switch to bash
because sh in the pain when ones want to locally define variables for functions
calls. I also added support to download more stuff like alsa-lib and more recent
gccs that use a different naming convention, as well as linux kernels.

I also add some build to build linux toolchains without ct-ng. The biggest problem
with ct-ng is that they regularly drop support for old stuff and as a result it
rots really quickly for old toolchains. I add a new toolchain for generic linux arm
with the minimum requirements, it works fine on Sony NWZ and also on YP-R0.

Finally, rockboxdev.sh now understand options on its command line (see --help).
Notably --target to give the list of targets (useful for noninteractive scripts),
--restart to restart at a step, --makeflags and others (instead of the environment
variables)

Change-Id: I869760c1faeb00ab381796a4cda82ffbc9637123
This commit is contained in:
Amaury Pouly 2017-02-23 11:37:50 +01:00
parent 25a29e71d8
commit 142f80f07d
1 changed files with 488 additions and 92 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

580
tools/rockboxdev.sh
View file

@ -1,4 +1,4 @@
#!/bin/sh
#!/bin/bash
# Abort execution as soon as an error is encountered
# That way the script do not let the user think the process completed correctly
@ -33,6 +33,10 @@ if [ -z $GNU_MIRROR ] ; then
GNU_MIRROR=http://mirrors.kernel.org/gnu
fi
if [ -z $LINUX_MIRROR ] ; then
LINUX_MIRROR=http://www.kernel.org/pub/linux
fi
# These are the tools this script requires and depends upon.
reqtools="gcc bzip2 gzip make patch makeinfo automake libtool autoconf flex bison"
@ -53,45 +57,280 @@ findtool(){
done
}
findlib (){
lib="$1"
# on error, gcc will spit out "cannot find -lxxxx", but it may not be in
# english so grep for -lxxxx
if ! gcc -l$lib 2>&1 | grep -q -- "-l$lib"; then
echo "ok"
return
fi
}
# check if all the libraries in argument are installed, exit with error if not
checklib() {
for t in "$@"; do
lib=`findlib $t`
if test -z "$lib"; then
echo "ROCKBOXDEV: library \"$t\" is required for this script to work."
missingtools="yes"
fi
done
if [ -n "$missingtools" ]; then
echo "ROCKBOXDEV: Please install the missing libraries and re-run the script."
exit 1
fi
}
input() {
read response
echo $response
}
#$1 file
#$2 URL"root
getfile() {
if test -f $dlwhere/$1; then
echo "ROCKBOXDEV: Skipping download of $2/$1: File already exists"
return
fi
tool=`findtool curl`
if test -z "$tool"; then
tool=`findtool wget`
if test -n "$tool"; then
# wget download
echo "ROCKBOXDEV: Downloading $2/$1 using wget"
$tool -T 60 -O $dlwhere/$1 $2/$1
fi
else
# curl download
echo "ROCKBOXDEV: Downloading $2/$1 using curl"
$tool -Lo $dlwhere/$1 $2/$1
fi
if [ $? -ne 0 ] ; then
echo "ROCKBOXDEV: couldn't download the file!"
echo "ROCKBOXDEV: check your internet connection"
exit
fi
if test -z "$tool"; then
echo "ROCKBOXDEV: No downloader tool found!"
echo "ROCKBOXDEV: Please install curl or wget and re-run the script"
exit
fi
# compare two versions, return 1 if first version is strictly before the second
# version_lt ver1 ver2
version_lt() {
# use sort with natural version sorting
ltver=`printf "$1\n$2" | sort -V | head -n 1`
[ "$1" = "$ltver" ] && true || false
}
# Download a file from a server (unless it already exists locally in $dlwhere).
# The output file name is always $dlwhere/$1, and the function will try URLs
# one after the other
# $1 file
# $2 server file name
# $2,$3,... URL root list
getfile_ex() {
out_file="$dlwhere/$1"
srv_file="$2"
if test -f $out_file; then
echo "ROCKBOXDEV: Skipping download of $1: File already exists"
return
fi
# find tool (curl or wget) and build download command
tool=`findtool curl`
if test -z "$tool"; then
tool=`findtool wget`
if test -n "$tool"; then
# wget download
echo "ROCKBOXDEV: Downloading $1 using wget"
tool="$tool -T 60 -O "
else
echo "ROCKBOXDEV: No downloader tool found!"
echo "ROCKBOXDEV: Please install curl or wget and re-run the script"
exit
fi
else
# curl download
echo "ROCKBOXDEV: Downloading $1 using curl"
tool="$tool -fLo "
fi
# shift arguments so that $@ is the list of URLs
shift
shift
for url in "$@"; do
echo "ROCKBOXDEV: try $url/$srv_file"
if $tool "$out_file" "$url/$srv_file"; then
return
fi
done
echo "ROCKBOXDEV: couldn't download the file!"
echo "ROCKBOXDEV: check your internet connection"
exit
}
#$1 file
#$2 URL"root
# Output file name is the same as the server file name ($1)
# Does not download the file if it already exists locally in $dlwhere/
getfile() {
getfile_ex "$1" "$1" "$2"
}
# wrapper around getfile to abstract url
# getttool tool version
# the file is always downloaded to "$dlwhere/$toolname-$version.tar.bz2"
gettool() {
toolname="$1"
version="$2"
ext="tar.bz2"
file="$toolname-$version"
srv_file="$toolname-$version"
case $toolname in
gcc)
# before 4.7, the tarball was named gcc-core
if version_lt "$version" "4.7"; then
srv_file="gcc-core-$version"
fi
url="$GNU_MIRROR/gcc/gcc-$version"
;;
binutils)
url="$GNU_MIRROR/binutils"
;;
glibc)
url="$GNU_MIRROR/glibc"
;;
crosstool-ng)
url="http://crosstool-ng.org/download/crosstool-ng"
;;
alsa-lib)
url="ftp://ftp.alsa-project.org/pub/lib/"
;;
linux)
# FIXME linux kernel server uses an overcomplicated architecture,
# especially for longterm kernels, so we need to lookup several
# places. This will need fixing for non-4-part 2.6 versions but it
# is written in a way that should make it easy
case "$version" in
2.6.*.*)
# 4-part versions -> remove last digit for longterm
longterm_ver="${version%.*}"
top_dir="v2.6"
;;
*)
echo "ROCKBOXDEV: I don't know how to handle this kernel version: $version"
exit
;;
esac
base_url="http://www.kernel.org/pub/linux/kernel/$top_dir"
url="$base_url $base_url/longterm/v$longterm_ver $base_url/longterm"
ext="tar.gz"
;;
*)
echo "ROCKBOXDEV: Bad toolname $toolname"
exit
;;
esac
getfile_ex "$file.$ext" "$srv_file.$ext" $url
}
# extract file to the current directory
# extract file
extract() {
if [ -d "$1" ]; then
echo "ROCKBOXDEV: Skipping extraction of $1: already done"
return
fi
echo "ROCKBOXDEV: extracting $1"
if [ -f "$dlwhere/$1.tar.bz2" ]; then
tar xjf "$dlwhere/$1.tar.bz2"
elif [ -f "$dlwhere/$1.tar.gz" ]; then
tar xzf "$dlwhere/$1.tar.gz"
else
echo "ROCKBOXDEV: I don't know how to extract $1 (no bzip2 or gzip)"
exit
fi
}
# run a command, and a log command and output to a file (append)
# exit on error
# run_cmd logfile cmd...
run_cmd() {
logfile="$1"
shift
echo "Running '$@'" >>$logfile
if ! $@ >> "$logfile" 2>&1; then
echo "ROCKBOXDEV: an error occured, please see $logfile"
exit
fi
}
# check if the following should be executed or not, depending on RESTART variable:
# $1=tool
# If RESTART is empty, always perform step, otherwise only perform is there is
# an exact match. On the first match, RESTART is reset to "" so that next step
# are executed
check_restart() {
if [ "$1" = "$RESTART" ]; then
RESTART=""
true
fi
[ "$RESTART" = "" ] && true || false
}
# advanced tool building: create a build directory, run configure, run make
# and run make install. It is possible to customize all steps or disable them
# $1=tool
# $2=version
# $3=configure options, or "NO_CONFIGURE" to disable step
# $4=make options, or "NO_MAKE" to disable step
# $5=make install options (will be replaced by "install" if left empty)
# By default, the restary step is the toolname, but it can be changed by setting
# RESTART_STEP
buildtool() {
tool="$1"
version="$2"
toolname="$tool-$version"
config_opt="$3"
make_opts="$4"
install_opts="$5"
logfile="$builddir/build-$toolname.log"
stepname=${RESTART_STEP-$tool}
if ! check_restart "$stepname"; then
echo "ROCKBOXDEV: Skipping step '$stepname' as requested per RESTART"
return
fi
echo "ROCKBOXDEV: Starting step '$stepname'"
echo "ROCKBOXDEV: logging to $logfile"
rm -f "$logfile"
echo "ROCKBOXDEV: mkdir build-$toolname"
mkdir "build-$toolname"
echo "ROCKBOXDEV: cd build-$toolname"
cd "build-$toolname"
# in-tree/out-of-tree build
case "$tool" in
linux|alsa-lib)
# in-intree
echo "ROCKBOXDEV: copy $toolname for in-tree build"
# copy the source directory to the build directory
cp -r ../$toolname/* .
cfg_dir="."
;;
*)
# out-of-tree
cfg_dir="../$toolname";
;;
esac
if [ "$config_opt" != "NO_CONFIGURE" ]; then
echo "ROCKBOXDEV: $toolname/configure"
# NOTE glibc requires to be compiled with optimization
CFLAGS='-U_FORTIFY_SOURCE -fgnu89-inline -O2' run_cmd "$logfile" \
"$cfg_dir/configure" "--prefix=$prefix" \
--disable-docs $config_opt
fi
if [ "$make_opts" != "NO_MAKE" ]; then
echo "ROCKBOXDEV: $toolname/make"
run_cmd "$logfile" $make $make_opts
fi
if [ "$install_opts" = "" ]; then
install_opts="install"
fi
echo "ROCKBOXDEV: $toolname/make (install)"
run_cmd "$logfile" $make $install_opts
echo "ROCKBOXDEV: rm -rf build-$toolname $toolname"
cd ..
rm -rf build-$toolname
}
build() {
toolname="$1"
@ -103,27 +342,6 @@ build() {
patch_url="http://www.rockbox.org/gcc"
case $toolname in
gcc)
file="gcc-core-$version.tar.bz2"
url="$GNU_MIRROR/gcc/gcc-$version"
;;
binutils)
file="binutils-$version.tar.bz2"
url="$GNU_MIRROR/binutils"
;;
crosstool-ng)
file="crosstool-ng-$version.tar.bz2"
url="http://crosstool-ng.org/download/crosstool-ng"
;;
*)
echo "ROCKBOXDEV: Bad toolname $toolname"
exit
;;
esac
# create build directory
if test -d $builddir; then
if test ! -w $builddir; then
@ -135,21 +353,17 @@ build() {
fi
# download source tarball
if test ! -f "$dlwhere/$file"; then
getfile "$file" "$url"
fi
gettool "$toolname" "$version"
file="$toolname-$version"
# download patch
for p in $patch; do
if test ! -f "$dlwhere/$p"; then
getfile "$p" "$patch_url"
fi
getfile "$p" "$patch_url"
done
cd $builddir
echo "ROCKBOXDEV: extracting $file"
tar xjf $dlwhere/$file
extract "$toolname-$version"
# do we have a patch?
for p in $patch; do
@ -170,27 +384,21 @@ build() {
cd "gcc-$version"
if (echo $needs_libs | grep -q gmp && test ! -d gmp); then
echo "ROCKBOXDEV: Getting GMP"
if test ! -f $dlwhere/gmp-4.3.2.tar.bz2; then
getfile "gmp-4.3.2.tar.bz2" "$GNU_MIRROR/gmp"
fi
getfile "gmp-4.3.2.tar.bz2" "$GNU_MIRROR/gmp"
tar xjf $dlwhere/gmp-4.3.2.tar.bz2
ln -s gmp-4.3.2 gmp
fi
if (echo $needs_libs | grep -q mpfr && test ! -d mpfr); then
echo "ROCKBOXDEV: Getting MPFR"
if test ! -f $dlwhere/mpfr-2.4.2.tar.bz2; then
getfile "mpfr-2.4.2.tar.bz2" "$GNU_MIRROR/mpfr"
fi
getfile "mpfr-2.4.2.tar.bz2" "$GNU_MIRROR/mpfr"
tar xjf $dlwhere/mpfr-2.4.2.tar.bz2
ln -s mpfr-2.4.2 mpfr
fi
if (echo $needs_libs | grep -q mpc && test ! -d mpc); then
echo "ROCKBOXDEV: Getting MPC"
if test ! -f $dlwhere/mpc-0.8.1.tar.gz; then
getfile "mpc-0.8.1.tar.gz" "http://www.multiprecision.org/mpc/download"
fi
getfile "mpc-0.8.1.tar.gz" "http://www.multiprecision.org/mpc/download"
tar xzf $dlwhere/mpc-0.8.1.tar.gz
ln -s mpc-0.8.1 mpc
fi
@ -226,7 +434,6 @@ build() {
rm -rf build-$toolname $toolname-$version
}
make_ctng() {
if test -f "`which ct-ng 2>/dev/null`"; then
ctng="ct-ng"
@ -288,25 +495,170 @@ build_ctng() {
cd $builddir
rm -rf $builddir/build-$ctng_target
}
# build a cross compiler toolchain for linux
# $1=target
# $2=binutils version
# $3=binutils configure extra options
# $4=gcc version
# $5=gcc configure extra options
# $6=linux version
# $7=glibc version
# $8=glibc configure extra options
build_linux_toolchain () {
target="$1"
binutils_ver="$2"
binutils_opts="$3"
gcc_ver="$4"
gcc_opts="$5"
linux_ver="$6"
glibc_ver="$7"
glibc_opts="$8"
# where to put the sysroot
sysroot="$prefix/$target/sysroot"
# extract arch from target
arch=${target%%-*}
# check libraries:
# contrary to other toolchains that rely on a hack to avoid installing
# gmp, mpc and mpfr, we simply require that they are installed on the system
# this is not a huge requirement since virtually all systems these days
# provide dev packages for them
# FIXME: maybe add an option to download and install them automatically
checklib "mpc" "gmp" "mpfr"
# create build directory
if test -d $builddir; then
if test ! -w $builddir; then
echo "ROCKBOXDEV: No write permission for $builddir"
exit
fi
else
mkdir -p $builddir
fi
# download all tools
gettool "binutils" "$binutils_ver"
gettool "gcc" "$gcc_ver"
gettool "linux" "$linux_ver"
gettool "glibc" "$glibc_ver"
# extract them
cd $builddir
extract "binutils-$binutils_ver"
extract "gcc-$gcc_ver"
extract "linux-$linux_ver"
extract "glibc-$glibc_ver"
# we make it possible to restart a build on error by using the RESTART
# variable, the format is RESTART="tool" where tool is the toolname at which
# to restart (binutils, gcc)
# install binutils, with support for sysroot
buildtool "binutils" "$binutils_ver" "--target=$target --disable-werror \
--with-sysroot=$sysroot --disable-nls" "" ""
# build stage 1 compiler: disable headers, disable threading so that
# pthread headers are not included, pretend we use newlib so that libgcc
# doesn't get linked at the end
# NOTE there is some magic involved here
RESTART_STEP="gcc-stage1" \
buildtool "gcc" "$gcc_ver" "$gcc_opts --enable-languages=c --target=$target \
--without-headers --disable-threads --disable-libgomp --disable-libmudflap \
--disable-libssp --disable-libquadmath --disable-libquadmath-support \
--disable-shared --with-newlib --disable-libitm \
--disable-libsanitizer --disable-libatomic" "" ""
# install linux headers
# NOTE: we need to tell make where to put the build files, since buildtool
# switches to the builddir, "." will be the correct builddir when ran
linux_opts="O=. ARCH=$arch INSTALL_HDR_PATH=$sysroot/usr/"
RESTART_STEP="linux-headers" \
buildtool "linux" "$linux_ver" "NO_CONFIGURE" \
"$linux_opts headers_install" "$linux_opts headers_check"
# build glibc using the first stage cross compiler
# we need to set the prefix to /usr because the glibc runs on the actual
# target and is indeed installed in /usr
prefix="/usr" \
buildtool "glibc" "$glibc_ver" "--target=$target --host=$target --build=$MACHTYPE \
--with-__thread --with-headers=$sysroot/usr/include $glibc_opts" \
"" "install install_root=$sysroot"
# build stage 2 compiler
buildtool "gcc" "$gcc_ver" "$gcc_opts --enable-languages=c,c++ --target=$target \
--with-sysroot=$sysroot" "" ""
}
usage () {
echo "usage: rockboxdev.sh [options]"
echo "options:"
echo " --help Display this help"
echo " --target=LIST List of targets to build"
echo " --restart=STEP Restart build at given STEP (same as RESTART env var)"
echo " --prefix=PREFIX Set install prefix (same as RBDEV_PREFIX env var)"
echo " --dlwhere=DIR Set download directory (same as RBDEV_DOWNLOAD env var)"
echo " --builddir=DIR Set build directory (same as RBDEV_BUILD env var)"
echo " --makeflags=FLAGS Set make flags (same as MAKEFLAGS env var)"
exit 1
}
##############################################################################
# Code:
# Verify required tools
# Parse arguments
for i in "$@"
do
case $i in
--help)
usage
;;
--prefix=*)
prefix="${i#*=}"
shift
;;
--target=*)
RBDEV_TARGET="${i#*=}"
shift
;;
--restart=*)
RBDEV_RESTART="${i#*=}"
shift
;;
--dlwhere=*)
dlwhere="${i#*=}"
shift
;;
--builddir=*)
builddir="${i#*=}"
shift
;;
--makeflags=*)
export MAKEFLAGS="${i#*=}" # export so it's available in make
shift
;;
*)
echo "Unknown option '$i'"
exit 1
;;
esac
done
# Verify required tools and libraries
for t in $reqtools; do
tool=`findtool $t`
if test -z "$tool"; then
echo "ROCKBOXDEV: \"$t\" is required for this script to work."
echo "ROCKBOXDEV: Please install \"$t\" and re-run the script."
exit
missingtools="yes"
fi
done
if [ -n "$missingtools" ]; then
echo "ROCKBOXDEV: Please install the missing tools and re-run the script."
exit 1
fi
echo "Download directory : $dlwhere (set RBDEV_DOWNLOAD to change)"
echo "Install prefix : $prefix (set RBDEV_PREFIX to change)"
echo "Build dir : $builddir (set RBDEV_BUILD to change)"
echo "Make options : $MAKEFLAGS (set MAKEFLAGS to change)"
echo ""
echo "Download directory : $dlwhere (set RBDEV_DOWNLOAD or use --download= to change)"
echo "Install prefix : $prefix (set RBDEV_PREFIX or use --prefix= to change)"
echo "Build dir : $builddir (set RBDEV_BUILD or use --builddir= to change)"
echo "Make options : $MAKEFLAGS (set MAKEFLAGS or use --makeflags= to change)"
echo "Restart step : $RBDEV_RESTART (set RBDEV_RESTART or use --restart= to change)"
echo "Target arch : $RBDEV_TARGET (set RBDEV_TARGET or use --target to change)"
# Verify download directory
if test -d "$dlwhere"; then
@ -336,17 +688,21 @@ if test ! -w $prefix; then
exit
fi
echo "Select target arch:"
echo "s - sh (Archos models)"
echo "m - m68k (iriver h1x0/h3x0, iaudio m3/m5/x5 and mpio hd200)"
echo "a - arm (ipods, iriver H10, Sansa, D2, Gigabeat, etc)"
echo "i - mips (Jz4740 and ATJ-based players)"
echo "r - arm-app (Samsung ypr0)"
echo "separate multiple targets with spaces"
echo "(Example: \"s m a\" will build sh, m68k and arm)"
echo ""
selarch=`input`
if [ -z "$RBDEV_TARGET" ]; then
echo "Select target arch:"
echo "s - sh (Archos models)"
echo "m - m68k (iriver h1x0/h3x0, iaudio m3/m5/x5 and mpio hd200)"
echo "a - arm (ipods, iriver H10, Sansa, D2, Gigabeat, etc)"
echo "i - mips (Jz4740 and ATJ-based players)"
echo "r - arm-app (Samsung ypr0)"
echo "x - arm-linux (Generic Linux ARM: Samsung ypr0, Linux-based Sony NWZ)"
echo "separate multiple targets with spaces"
echo "(Example: \"s m a\" will build sh, m68k and arm)"
echo ""
selarch=`input`
else
selarch=$RBDEV_TARGET
fi
system=`uname -s`
# add target dir to path to ensure the new binutils are used in gcc build
@ -394,6 +750,46 @@ do
[Rr])
build_ctng "ypr0" "alsalib.tar.gz" "arm" "linux-gnueabi"
;;
[Xx])
# IMPORTANT NOTE
# This toolchain must support several targets and thus must support
# the oldest possible configuration.
#
# Samsung YP-R0/R1:
# ARM1176JZF-S, softfp EABI
# gcc: 4.9.4 is the latest 4.9.x stable branch, also the only one that
# compiles with GCC >6
# kernel: 2.6.27.59 is the same 2.6.x stable kernel as used by the
# original ct-ng toolchain, the device runs kernel 2.6.24
# glibc: 2.19 is the latest version that supports kernel 2.6.24 which
# is used on the device, but we need to support ABI 2.4 because
# the device uses glibc 2.4.2
#
# Sony NWZ:
# gcc: 4.9.4 is the latest 4.9.x stable branch, also the only one that
# compiles with GCC >6
# kernel: 2.6.32.68 is the latest 2.6.x stable kernel, the device
# runs kernel 2.6.23 or 2.6.35 or 3.x for the most recent
# glibc: 2.19 is the latest version that supports kernel 2.6.23 which
# is used on many Sony players, but we need to support ABI 2.7
# because the device uses glibc 2.7
#
# Thus the lowest common denominator is to use the latest 2.6.x stable
# kernel but compile glibc to support kernel 2.6.23 and glibc 2.4.
# We use a recent 2.26.1 binutils to avoid any build problems and
# avoid patches/bugs.
glibcopts="--enable-kernel=2.6.23 --enable-oldest-abi=2.4"
build_linux_toolchain "arm-rockbox-linux-gnueabi" "2.26.1" "" "4.9.4" \
"$gccopts" "2.6.32.68" "2.19" "$glibcopts"
# build alsa-lib
# we need to set the prefix to how it is on device (/usr) and then
# tweak install dir at make install step
alsalib_ver="1.0.19"
gettool "alsa-lib" "$alsalib_ver"
extract "alsa-lib-$alsalib_ver"
prefix="/usr" buildtool "alsa-lib" "$alsalib_ver" \
"--host=$target --disable-python" "" "install DESTDIR=$prefix/$target/sysroot"
;;
*)
echo "ROCKBOXDEV: Unsupported architecture option: $arch"
exit