rockbox/utils/common/deploy.py

#!/usr/bin/python
#             __________               __   ___.
#   Open      \______   \ ____   ____ |  | _\_ |__   _______  ___
#   Source     |       _//  _ \_/ ___\|  |/ /| __ \ /  _ \  \/  /
#   Jukebox    |    |   (  <_> )  \___|    < | \_\ (  <_> > <  <
#   Firmware   |____|_  /\____/ \___  >__|_ \|___  /\____/__/\_ \
#                     \/            \/     \/    \/            \/
# $Id$
#
# Copyright (c) 2009 Dominik Riebeling
#
# All files in this archive are subject to the GNU General Public License.
# See the file COPYING in the source tree root for full license agreement.
#
# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
# KIND, either express or implied.
#
#
# Automate building releases for deployment.
# Run from any folder to build
# - trunk
# - any tag (using the -t option)
# - any local folder (using the -p option)
# Will build a binary archive (tar.bz2 / zip) and source archive.
# The source archive won't be built for local builds. Trunk and
# tag builds will retrieve the sources directly from svn and build
# below the systems temporary folder.
#
# If the required Qt installation isn't in PATH use --qmake option.
# Tested on Linux and MinGW / W32
#
# requires upx.exe in PATH on Windows.
#

import re
import os
import sys
import tarfile
import zipfile
import shutil
import subprocess
import getopt
import time
import hashlib
import tempfile
from datetime import datetime
import multiprocessing
import gitscraper

CPUS = multiprocessing.cpu_count()
print("Info: %s cores found." % CPUS)

# == Global stuff ==
# DLL files to ignore when searching for required DLL files.
SYSTEMDLLS = [
        'advapi32.dll',
        'comdlg32.dll',
        'crypt32.dll',
        'd3d9.dll',
        'dwmapi.dll',
        'dxva2.dll',
        'evr.dll',
        'gdi32.dll',
        'imm32.dll',
        'imm32.dll',
        'iphlpapi.dll',
        'kernel32.dll',
        'mf.dll',
        'mfplat.dll',
        'msvcrt.dll',
        'msvcrt.dll',
        'netapi32.dll',
        'ole32.dll',
        'oleaut32.dll',
        'setupapi.dll',
        'shell32.dll',
        'user32.dll',
        'userenv.dll',
        'uxtheme.dll',
        'version.dll',
        'winmm.dll',
        'winspool.drv',
        'ws2_32.dll',
        'wtsapi32.dll'
        ]

gitrepo = os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))


# == Functions ==
def usage(myself):
    print("Usage: %s [options]" % myself)
    print("       -q, --qmake=<qmake>   path to qmake")
    print("       -p, --project=<pro>   path to .pro file for building with local tree")
    print("       -t, --tag=<tag>       use specified tag from svn")
    print("       -a, --add=<file>      add file to build folder before building")
    print("       -s, --source-only     only create source archive")
    print("       -b, --binary-only     only create binary archive")
    if nsisscript != "":
        print("       -n, --makensis=<file> path to makensis for building Windows setup program.")
    if sys.platform != "darwin":
        print("       -d, --dynamic         link dynamically instead of static")
    if sys.platform != "win32":
        print("       -x, --cross=          prefix to cross compile for win32")
    print("       -k, --keep-temp       keep temporary folder on build failure")
    print("       -h, --help            this help")
    print("  If neither a project file nor tag is specified trunk will get downloaded")
    print("  from svn.")


def which(executable):
    path = os.environ.get("PATH", "").split(os.pathsep)
    for p in path:
        fullpath = p + "/" + executable
        if os.path.exists(fullpath):
            return fullpath
    print("which: could not find " + executable)
    return ""


def getsources(treehash, filelist, dest):
    '''Get the files listed in filelist from svnsrv and put it at dest.'''
    gitscraper.scrape_files(gitrepo, treehash, filelist, dest)
    return 0


def getfolderrev(svnsrv):
    '''Get the most recent revision for svnsrv'''
    client = pysvn.Client()
    entries = client.info2(svnsrv, recurse=False)
    return entries[0][1].rev.number


def findversion(versionfile):
    '''figure most recent program version from version.h,
    returns version string.'''
    h = open(versionfile, "r")
    c = h.read()
    h.close()
    version = dict()
    for v in ['MAJOR', 'MINOR', 'MICRO']:
        r = re.compile("#define +VERSION_" + v + " +([0-9a-z]+)")
        m = re.search(r, c)
        version[v] = m.group(1)
    return "%s.%s.%s" % (version['MAJOR'], version['MINOR'], version['MICRO'])


def findqt(cross=""):
    '''Search for Qt4 installation. Return path to qmake.'''
    print("Searching for Qt")
    bins = [cross + "qmake", cross + "qmake-qt4"]
    for binary in bins:
        try:
            q = which(binary)
            if len(q) > 0:
                result = checkqt(q)
                if not result == "":
                    return result
        except:
            print(sys.exc_info()[1])

    return ""


def checkqt(qmakebin):
    '''Check if given path to qmake exists and is a suitable version.'''
    result = ""
    # check if binary exists
    if not os.path.exists(qmakebin):
        print("Specified qmake path does not exist!")
        return result
    # check version
    output = subprocess.Popen(
        [qmakebin, "-version"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    cmdout = output.communicate()
    # don't check the qmake return code here, Qt3 doesn't return 0 on -version.
    for ou in cmdout:
        r = re.compile(b'Qt[^0-9]+([0-9\.]+[a-z]*)')
        m = re.search(r, ou)
        if m is not None:
            print("Qt found: %s" % m.group(1).decode())
            s = re.compile(b'[45]\..*')
            n = re.search(s, m.group(1))
            if n is not None:
                result = qmakebin
    return result


def qmake(qmake, projfile, platform=sys.platform, wd=".", static=True, cross=""):
    print("Running qmake in %s..." % wd)
    command = [qmake, "-config", "release", "-config", "noccache"]
    if static == True:
        command.extend(["-config", "-static"])
    # special spec required?
    if len(qmakespec[platform]) > 0:
        command.extend(["-spec", qmakespec[platform]])
    # cross compiling prefix set?
    if len(cross) > 0:
        command.extend(["-config", "cross"])
    command.append(projfile)
    output = subprocess.Popen(command, stdout=subprocess.PIPE, cwd=wd)
    output.communicate()
    if not output.returncode == 0:
        print("qmake returned an error!")
        return -1
    return 0


def build(wd=".", platform=sys.platform, cross=""):
    # make
    print("Building ...")
    # use the current platforms make here, cross compiling uses the native make.
    command = [make[sys.platform]]
    if CPUS > 1:
        command.append("-j")
        command.append(str(CPUS))
    output = subprocess.Popen(command, stdout=subprocess.PIPE, cwd=wd)
    while True:
        c = output.stdout.readline()
        sys.stdout.write(".")
        sys.stdout.flush()
        if not output.poll() == None:
            sys.stdout.write("\n")
            sys.stdout.flush()
            if not output.returncode == 0:
                print("Build failed!")
                return -1
            break
    if platform != "darwin":
        # strip. OS X handles this via macdeployqt.
        print("Stripping binary.")
        output = subprocess.Popen([cross + "strip", progexe[platform]], \
                                  stdout=subprocess.PIPE, cwd=wd)
        output.communicate()
        if not output.returncode == 0:
            print("Stripping failed!")
            return -1
    return 0


def upxfile(wd=".", platform=sys.platform):
    # run upx on binary
    print("UPX'ing binary ...")
    output = subprocess.Popen(["upx", progexe[platform]], \
                              stdout=subprocess.PIPE, cwd=wd)
    output.communicate()
    if not output.returncode == 0:
        print("UPX'ing failed!")
        return -1
    return 0


def runnsis(versionstring, nsis, script, srcfolder):
    # run script through nsis to create installer.
    print("Running NSIS ...")

    # Assume the generated installer gets placed in the same folder the nsi
    # script lives in.  This seems to be a valid assumption unless the nsi
    # script specifies a path. NSIS expects files relative to source folder so
    # copy progexe. Additional files are injected into the nsis script.

    # FIXME: instead of copying binaries around copy the NSI file and inject
    # the correct paths.
    # Only win32 supported as target platform so hard coded.
    b = srcfolder + "/" + os.path.dirname(script) + "/" \
        + os.path.dirname(progexe["win32"])
    if not os.path.exists(b):
        os.mkdir(b)
    shutil.copy(srcfolder + "/" + progexe["win32"], b)
    output = subprocess.Popen([nsis, srcfolder + "/" + script], \
                              stdout=subprocess.PIPE)
    output.communicate()
    if not output.returncode == 0:
        print("NSIS failed!")
        return -1
    setupfile = program + "-" + versionstring + "-setup.exe"
    # find output filename in nsis script file
    nsissetup = ""
    for line in open(srcfolder + "/" + script):
        if re.match(r'^[^;]*OutFile\s+', line) != None:
            nsissetup = re.sub(r'^[^;]*OutFile\s+"(.+)"', r'\1', line).rstrip()
    if nsissetup == "":
        print("Could not retrieve output file name!")
        return -1
    shutil.copy(srcfolder + "/" + os.path.dirname(script) + "/" + nsissetup, \
                setupfile)
    return 0


def nsisfileinject(nsis, outscript, filelist):
    '''Inject files in filelist into NSIS script file after the File line
       containing the main binary. This assumes that the main binary is present
       in the NSIS script and that all additiona files (dlls etc) to get placed
       into $INSTDIR.'''
    output = open(outscript, "w")
    for line in open(nsis, "r"):
        output.write(line)
        # inject files after the progexe binary.
        # Match the basename only to avoid path mismatches.
        if re.match(r'^\s*File\s*.*' + os.path.basename(progexe["win32"]),
                    line, re.IGNORECASE):
            for f in filelist:
                injection = "    File /oname=$INSTDIR\\" + os.path.basename(f) \
                             + " " + os.path.normcase(f) + "\n"
                output.write(injection)
            output.write("    ; end of injected files\n")
    output.close()


def finddlls(program, extrapaths=[], cross=""):
    '''Check program for required DLLs. Find all required DLLs except ignored
       ones and return a list of DLL filenames (including path).'''
    # ask objdump about dependencies.
    output = subprocess.Popen([cross + "objdump", "-x", program], \
                              stdout=subprocess.PIPE)
    cmdout = output.communicate()

    # create list of used DLLs. Store as lower case as W32 is case-insensitive.
    dlls = []
    for line in cmdout[0].decode().split('\n'):
        if re.match(r'\s*DLL Name', line) != None:
            dll = re.sub(r'^\s*DLL Name:\s+([a-zA-Z_\-0-9\.\+]+).*$', r'\1', line)
            dlls.append(dll.lower())

    # find DLLs in extrapaths and PATH environment variable.
    dllpaths = []
    for file in dlls:
        if file in SYSTEMDLLS:
            print("System DLL:  " + file)
            continue
        dllpath = ""
        for path in extrapaths:
            if os.path.exists(path + "/" + file):
                dllpath = re.sub(r"\\", r"/", path + "/" + file)
                print(file + ": found at " + dllpath)
                dllpaths.append(dllpath)
                break
        if dllpath == "":
            try:
                dllpath = re.sub(r"\\", r"/", which(file))
                print(file + ": found at " + dllpath)
                dllpaths.append(dllpath)
            except:
                print("MISSING DLL: " + file)
    return dllpaths


def zipball(programfiles, versionstring, buildfolder, platform=sys.platform):
    '''package created binary'''
    print("Creating binary zipball.")
    archivebase = program + "-" + versionstring
    outfolder = buildfolder + "/" + archivebase
    archivename = archivebase + ".zip"
    # create output folder
    os.mkdir(outfolder)
    # move program files to output folder
    for f in programfiles:
        if re.match(r'^(/|[a-zA-Z]:)', f) != None:
            shutil.copy(f, outfolder)
        else:
            shutil.copy(buildfolder + "/" + f, outfolder)
    # create zipball from output folder
    zf = zipfile.ZipFile(archivename, mode='w', compression=zipfile.ZIP_DEFLATED)
    for root, dirs, files in os.walk(outfolder):
        for name in files:
            physname = os.path.normpath(os.path.join(root, name))
            filename = os.path.relpath(physname, buildfolder)
            zf.write(physname, filename)
    zf.close()
    # remove output folder
    shutil.rmtree(outfolder)
    return archivename


def tarball(programfiles, versionstring, buildfolder):
    '''package created binary'''
    print("Creating binary tarball.")
    archivebase = program + "-" + versionstring
    outfolder = buildfolder + "/" + archivebase
    archivename = archivebase + ".tar.bz2"
    # create output folder
    os.mkdir(outfolder)
    # move program files to output folder
    for f in programfiles:
        shutil.copy(buildfolder + "/" + f, outfolder)
    # create tarball from output folder
    tf = tarfile.open(archivename, mode='w:bz2')
    tf.add(outfolder, archivebase)
    tf.close()
    # remove output folder
    shutil.rmtree(outfolder)
    return archivename


def macdeploy(versionstring, buildfolder, platform=sys.platform):
    '''package created binary to dmg'''
    dmgfile = program + "-" + versionstring + ".dmg"
    appbundle = buildfolder + "/" + progexe[platform]

    # workaround to Qt issues when building out-of-tree. Copy files into bundle.
    sourcebase = buildfolder + re.sub('[^/]+.pro$', '', project) + "/"
    print(sourcebase)
    for src in bundlecopy:
        shutil.copy(sourcebase + src, appbundle + "/" + bundlecopy[src])
    # end of Qt workaround

    output = subprocess.Popen(["macdeployqt", progexe[platform], "-dmg"], \
                              stdout=subprocess.PIPE, cwd=buildfolder)
    output.communicate()
    if not output.returncode == 0:
        print("macdeployqt failed!")
        return -1
    # copy dmg to output folder
    shutil.copy(buildfolder + "/" + program + ".dmg", dmgfile)
    return dmgfile


def filehashes(filename):
    '''Calculate md5 and sha1 hashes for a given file.'''
    if not os.path.exists(filename):
        return ["", ""]
    m = hashlib.md5()
    s = hashlib.sha1()
    f = open(filename, 'rb')
    while True:
        d = f.read(65536)
        if d == b"":
            break
        m.update(d)
        s.update(d)
    return [m.hexdigest(), s.hexdigest()]


def filestats(filename):
    if not os.path.exists(filename):
        return
    st = os.stat(filename)
    print("%s\n%s" % (filename, "-" * len(filename)))
    print("Size:    %i bytes" % st.st_size)
    h = filehashes(filename)
    print("md5sum:  %s" % h[0])
    print("sha1sum: %s" % h[1])
    print("%s\n" % ("-" * len(filename)))


def tempclean(workfolder, nopro):
    if nopro == True:
        print("Cleaning up working folder %s" % workfolder)
        shutil.rmtree(workfolder)
    else:
        print("Project file specified or cleanup disabled!")
        print("Temporary files kept at %s" % workfolder)


def deploy():
    startup = time.time()

    try:
        opts, args = getopt.getopt(
            sys.argv[1:], "q:p:t:a:n:sbdkx:i:h",
            ["qmake=", "project=", "tag=", "add=", "makensis=", "source-only",
             "binary-only", "dynamic", "keep-temp", "cross=", "buildid=", "help"])
    except getopt.GetoptError as err:
        print(str(err))
        usage(sys.argv[0])
        sys.exit(1)
    qt = ""
    proj = ""
    svnbase = svnserver + "trunk/"
    tag = ""
    addfiles = []
    cleanup = True
    binary = True
    source = True
    keeptemp = False
    makensis = ""
    cross = ""
    buildid = None
    platform = sys.platform
    treehash = gitscraper.get_refs(gitrepo)['refs/remotes/origin/HEAD']
    if sys.platform != "darwin":
        static = True
    else:
        static = False
    for o, a in opts:
        if o in ("-q", "--qmake"):
            qt = a
        if o in ("-p", "--project"):
            proj = a
            cleanup = False
        if o in ("-a", "--add"):
            addfiles.append(a)
        if o in ("-n", "--makensis"):
            makensis = a
        if o in ("-s", "--source-only"):
            binary = False
        if o in ("-b", "--binary-only"):
            source = False
        if o in ("-d", "--dynamic") and sys.platform != "darwin":
            static = False
        if o in ("-k", "--keep-temp"):
            keeptemp = True
        if o in ("-t", "--tree"):
            treehash = a
        if o in ("-x", "--cross") and sys.platform != "win32":
            cross = a
            platform = "win32"
        if o in ("-i", "--buildid"):
            buildid = a
        if o in ("-h", "--help"):
            usage(sys.argv[0])
            sys.exit(0)

    if source == False and binary == False:
        print("Building build neither source nor binary means nothing to do. Exiting.")
        sys.exit(1)

    print("Building " + progexe[platform] + " for " + platform)
    # search for qmake
    if qt == "":
        qm = findqt(cross)
    else:
        qm = checkqt(qt)
    if qm == "":
        print("ERROR: No suitable Qt installation found.")
        sys.exit(1)

    # create working folder. Use current directory if -p option used.
    if proj == "":
        w = tempfile.mkdtemp()
        # make sure the path doesn't contain backslashes to prevent issues
        # later when running on windows.
        workfolder = re.sub(r'\\', '/', w)
        revision = gitscraper.describe_treehash(gitrepo, treehash)
        # try to find a version number from describe output.
        # WARNING: this is broken and just a temporary workaround!
        v = re.findall(b'([\d\.a-f]+)', revision)
        if v:
            if v[-1].decode().find('.') >= 0:
                revision = "v" + v[-1].decode()
            else:
                revision = v[-1].decode()
        if buildid == None:
            versionextra = ""
        else:
            versionextra = "-" + buildid
        sourcefolder = workfolder + "/" + program + "-" + str(revision) + versionextra + "/"
        archivename = program + "-" + str(revision) + versionextra + "-src.tar.bz2"
        ver = str(revision)
        os.mkdir(sourcefolder)
        print("Version: %s" % revision)
    else:
        workfolder = "."
        sourcefolder = "."
        archivename = ""
    # check if project file explicitly given. If yes, don't get sources from svn
    if proj == "":
        proj = sourcefolder + project
        # get sources and pack source tarball
        if getsources(treehash, svnpaths, sourcefolder) != 0:
            tempclean(workfolder, cleanup and not keeptemp)
            sys.exit(1)

        # replace version strings.
        print("Updating version information in sources")
        for f in regreplace:
            infile = open(sourcefolder + "/" + f, "r")
            incontents = infile.readlines()
            infile.close()

            outfile = open(sourcefolder + "/" + f, "w")
            for line in incontents:
                newline = line
                for r in regreplace[f]:
                    # replacements made on the replacement string:
                    # %REVISION% is replaced with the revision number
                    replacement = re.sub("%REVISION%", str(revision), r[1])
                    newline = re.sub(r[0], replacement, newline)
                    # %BUILD% is replaced with buildid as passed on the command line
                    if buildid != None:
                        replacement = re.sub("%BUILDID%", "-" + str(buildid), replacement)
                    else:
                        replacement = re.sub("%BUILDID%", "", replacement)
                    newline = re.sub(r[0], replacement, newline)
                outfile.write(newline)
            outfile.close()

        if source == True:
            print("Creating source tarball %s\n" % archivename)
            tf = tarfile.open(archivename, mode='w:bz2')
            tf.add(sourcefolder, os.path.basename(re.subn('/$', '', sourcefolder)[0]))
            tf.close()
            if binary == False:
                shutil.rmtree(workfolder)
                sys.exit(0)
    else:
        # figure version from sources. Need to take path to project file into account.
        versionfile = re.subn('[\w\.]+$', "version.h", proj)[0]
        ver = findversion(versionfile) + "-dev" + datetime.now().strftime('%Y%m%d%H%M%S')
    # append buildid if any.
    if buildid != None:
        ver += "-" + buildid

    # check project file
    if not os.path.exists(proj):
        print("ERROR: path to project file wrong.")
        sys.exit(1)

    # copy specified (--add) files to working folder
    for f in addfiles:
        shutil.copy(f, sourcefolder)
    buildstart = time.time()
    header = "Building %s %s" % (program, ver)
    print(header)
    print(len(header) * "=")

    # build it.
    if not qmake(qm, proj, platform, sourcefolder, static, cross) == 0:
        tempclean(workfolder, cleanup and not keeptemp)
        sys.exit(1)
    if not build(sourcefolder, platform, cross) == 0:
        tempclean(workfolder, cleanup and not keeptemp)
        sys.exit(1)
    buildtime = time.time() - buildstart
    progfiles = programfiles
    progfiles.append(progexe[platform])
    if platform == "win32":
        if useupx == True:
            if not upxfile(sourcefolder, platform) == 0:
                tempclean(workfolder, cleanup and not keeptemp)
                sys.exit(1)
        dllfiles = finddlls(sourcefolder + "/" + progexe[platform], \
                            [os.path.dirname(qm)], cross)
        if len(dllfiles) > 0:
            progfiles.extend(dllfiles)
        archive = zipball(progfiles, ver, sourcefolder, platform)
        # only when running native right now.
        if nsisscript != "" and makensis != "":
            nsisfileinject(sourcefolder + "/" + nsisscript, sourcefolder \
                           + "/" + nsisscript + ".tmp", dllfiles)
            runnsis(ver, makensis, nsisscript + ".tmp", sourcefolder)
    elif platform == "darwin":
        archive = macdeploy(ver, sourcefolder, platform)
    else:
        if platform in ['linux', 'linux2']:
            for p in progfiles:
                prog = sourcefolder + "/" + p
                output = subprocess.Popen(
                    ["file", prog], stdout=subprocess.PIPE)
                res = output.communicate()
                if re.findall("ELF 64-bit", res[0]):
                    ver += "-64bit"
                    break

        archive = tarball(progfiles, ver, sourcefolder)

    # remove temporary files
    tempclean(workfolder, cleanup)

    # display summary
    headline = "Build Summary for %s" % program
    print("\n%s\n%s" % (headline, "=" * len(headline)))
    if archivename != "":
        filestats(archivename)
    filestats(archive)
    duration = time.time() - startup
    durmins = (int)(duration / 60)
    dursecs = (int)(duration % 60)
    buildmins = (int)(buildtime / 60)
    buildsecs = (int)(buildtime % 60)
    print("Overall time %smin %ssec, building took %smin %ssec." % \
        (durmins, dursecs, buildmins, buildsecs))


if __name__ == "__main__":
    print("You cannot run this module directly!")
    print("Set required environment and call deploy().")