build: add bundle build

A bob-bundle collects all scm's neccessary to build a package in a
tar-file. Next to the tar-file a scmOverrides file is generated to be
able to rebuild the package only with the bundle.

Such a bundle can be used to build on a air-gapped system or to archive
all sources of a build.

ATM only bundling of git and url-scm's is supported.

contrib/bash-completion/bob

@@ -118,7 +118,7 @@ __bob_clean()
 
 __bob_cook()
 {
-   if [[ "$prev" = "--destination" ]] ; then
+   if [[ "$prev" = "--destination" || "$prev" == "--bundle" ]] ; then
       __bob_complete_dir "$cur"
    elif [[ "$prev" = "--download" ]] ; then
          __bob_complete_words "yes no deps forced forced-deps forced-fallback"
@@ -127,7 +127,7 @@ __bob_cook()
    elif [[ "$prev" = "--always-checkout" ]] ; then
       COMPREPLY=( )
    else
-      __bob_complete_path "--destination -j --jobs -k --keep-going -f --force -n --no-deps -p --with-provided --without-provided -A --no-audit --audit -b --build-only -B --checkout-only --normal --clean --incremental --always-checkout --resume -q --quiet -v --verbose --no-logfiles -D -c -e -E -M --upload --link-deps --no-link-deps --download --download-layer --shared --no-shared --install --no-install --sandbox --no-sandbox --clean-checkout --attic --no-attic"
+      __bob_complete_path "--destination -j --jobs -k --keep-going -f --force -n --no-deps -p --with-provided --without-provided -A --no-audit --audit -b --build-only -B --checkout-only --normal --clean --incremental --always-checkout --resume -q --quiet -v --verbose --no-logfiles -D -c -e -E -M --upload --link-deps --no-link-deps --download --download-layer --shared --no-shared --install --no-install --sandbox --no-sandbox --clean-checkout --attic --no-attic --bundle --bundle-exclude"
    fi
 }
 

doc/manpages/bob-build-dev.rst

@@ -35,6 +35,16 @@ Options
 
     This is the default unless the user changed it in ``default.yaml``.
 
+``--bundle BUNDLE``
+    Bundle all the sources needed to build the package. The bunlde is a tar-file
+    containing the sources and a overrides file. To use the bundle call bob
+    dev/build with ``-c`` pointing to the scmOverrides-file. In addition to this
+    the ``LOCAL_BUNDLE_BASE`` environment variable needs to be set to point to
+    the base-directoy where the bundle has been extracted.
+
+``--bundle-exclude RE``
+    Do not add packages matching  RE to the bundle.
+
 ``--clean``
     Do clean builds by clearing the build directory before executing the build
     commands. It will *not* clean all build results (e.g. like ``make clean``)

doc/manpages/bob-build.rst

@@ -24,6 +24,7 @@ Synopsis
               [--shared | --no-shared] [--install | --no-install]
               [--sandbox | --no-sandbox] [--clean-checkout]
               [--attic | --no-attic]
+              [--bundle BUNDLE] [--bundle-exclude BUNDLE_EXCLUDE]
               PACKAGE [PACKAGE ...]
 
 

doc/manpages/bob-dev.rst

@@ -24,6 +24,7 @@ Synopsis
             [--shared | --no-shared] [--install | --no-install]
             [--sandbox | --no-sandbox] [--clean-checkout]
             [--attic | --no-attic]
+            [--bundle BUNDLE] [--bundle-exclude BUNDLE_EXCLUDE]
             PACKAGE [PACKAGE ...]
 
 

pym/bob/builder.py

@@ -16,7 +16,7 @@
 from .tty import log, stepMessage, stepAction, stepExec, setProgress, ttyReinit, \
     SKIPPED, EXECUTED, INFO, WARNING, DEFAULT, \
     ALWAYS, IMPORTANT, NORMAL, INFO, DEBUG, TRACE
-from .utils import asHexStr, hashDirectory, removePath, emptyDirectory, \
+from .utils import asHexStr, hashDirectory, hashFile, removePath, emptyDirectory, \
     isWindows, INVALID_CHAR_TRANS, quoteCmdExe, getPlatformTag, canSymlink
 from .share import NullShare
 from shlex import quote
@@ -26,6 +26,7 @@
 import concurrent.futures
 import datetime
 import hashlib
+import fnmatch
 import io
 import locale
 import os
@@ -34,7 +35,9 @@
 import signal
 import stat
 import sys
+import tarfile
 import tempfile
+import yaml
 
 # Output verbosity:
 #    <= -2: package name
@@ -311,6 +314,76 @@ def __match(self, nestedScmPath):
                 return p
         return None
 
+class Bundler:
+    def __init__(self, name, excludes):
+        self.__name = name
+        self.__bundleFile = os.path.join(os.getcwd(), self.__name) + ".tar"
+        self.__excludes = excludes
+        self.__tempDir = tempfile.TemporaryDirectory()
+        self.__tempDirPath = os.path.join(self.__tempDir.name, self.__name)
+        self.__bundled = {}
+
+        if os.path.exists(self.__bundleFile):
+            raise BuildError(f"Bundle {self.__bundleFile} already exists!")
+        os.mkdir(self.__tempDirPath)
+
+    def add(self, path, bundle):
+        self.__bundled[path] = bundle
+
+    def bundle(self, step):
+        for e in self.__excludes:
+            if fnmatch.fnmatch(step.getPackage().getName(), e):
+                return False
+        return True
+
+    def getName(self):
+        return self.__name
+
+    def getTempDir (self):
+        return self.__tempDirPath
+
+    def finalize(self):
+        def _delFilter(f):
+            noDel = ["__*", "recipe", "digest*", "extract", "overridden", "dir",
+                     "stripComponents"]
+            for d in noDel:
+                if fnmatch.fnmatch(f, d):
+                    return False
+            return True
+
+        overrides = {
+            "scmOverrides" : []
+        }
+        with tarfile.open(self.__bundleFile, "w") as bundle_tar:
+            for path, bundles in self.__bundled.items():
+                for (scm, bundleFile) in bundles:
+                    bundlePath = os.path.join(self.__name,
+                                              os.path.relpath(bundleFile, self.__tempDirPath))
+                    properties = scm.getProperties(False)
+                    data = {
+                        "match": {
+                            "url": properties.get("url")
+                        },
+                        "del" : list(filter(_delFilter, [name for name, value in properties.items()
+                                                         if value is not None])),
+                        "set": {
+                            "url": "file://${LOCAL_BUNDLE_BASE}/" + bundlePath,
+                            "scm": "url",
+                        }
+                    }
+                    if properties.get("scm") != "url":
+                        d = hashFile(os.path.join(self.__tempDirPath, bundleFile),
+                                     hashlib.sha256).hex()
+                        data["set"].update({"digestSHA256" : d})
+                    overrides["scmOverrides"].append(data)
+                    bundle_tar.add(os.path.join(self.__tempDirPath, bundleFile),
+                              arcname=bundlePath)
+
+            overridesFile= os.path.join(self.__tempDirPath, self.__name + "_overrides.yaml")
+            with open(overridesFile, "w") as f:
+                yaml.dump(overrides, f, default_flow_style=False)
+            bundle_tar.add(overridesFile, arcname=os.path.join(self.__name, os.path.basename(overridesFile)))
+
 class LocalBuilder:
 
     RUN_TEMPLATE_POSIX = """#!/bin/bash
@@ -407,6 +480,7 @@ def __init__(self, verbose, force, skipDeps, buildOnly, preserveEnv,
         self.__installSharedPackages = False
         self.__executor = None
         self.__attic = True
+        self.__bundler = None
 
     def setExecutor(self, executor):
         self.__executor = executor
@@ -505,6 +579,10 @@ def setAuditMeta(self, keys):
     def setAtticEnable(self, enable):
         self.__attic = enable
 
+    def setBundle(self, dest, excludes):
+        if dest is not None:
+            self.__bundler = Bundler(dest, excludes)
+
     def setShareHandler(self, handler):
         self.__share = handler
 
@@ -618,6 +696,10 @@ def __workspaceLock(self, step):
             self.__workspaceLocks[path] = ret = asyncio.Lock()
         return ret
 
+    def bundle(self):
+        if self.__bundler:
+            self.__bundler.finalize()
+
     async def _generateAudit(self, step, depth, resultHash, buildId, executed=True):
         auditPath = os.path.join(os.path.dirname(step.getWorkspacePath()), "audit.json.gz")
         if os.path.lexists(auditPath): removePath(auditPath)
@@ -790,6 +872,35 @@ async def _runShell(self, step, scriptName, logger, cleanWorkspace=None,
                              help="You may resume at this point with '--resume' after fixing the error.",
                              returncode=ret)
 
+    async def __doBundle(self, checkoutStep, logger):
+        dest = os.path.join(self.__bundler.getTempDir(), checkoutStep.getPackage().getRecipe().getName())
+        os.makedirs(dest, exist_ok=True)
+        logFile = os.path.join(dest, ".bundle.log")
+        spec = StepSpec.fromStep(checkoutStep, None, self.__envWhiteList, logFile, False)
+
+        bundled = []
+        for scm in checkoutStep.getScmList():
+            invoker = Invoker(spec, self.__preserveEnv, self.__noLogFile,
+                self.__verbose >= INFO, self.__verbose >= NORMAL,
+                self.__verbose >= DEBUG, self.__bufferedStdIO,
+                executor=self.__executor, tmpdir=scm.getProperties(checkoutStep.JENKINS)['scm']=="git")
+
+            bundleFile = os.path.join(dest, scm.bundleName)
+            ret = await invoker.executeScmBundle(scm, bundleFile)
+            bundled.append((scm, os.path.join(checkoutStep.getPackage().getName(), bundleFile)))
+
+        if not self.__bufferedStdIO: ttyReinit() # work around MSYS2 messing up the console
+        if ret == -int(signal.SIGINT):
+            raise BuildError("User aborted while running bundling {}".format(self.__bundler.getName()));
+        elif ret != 0:
+            if self.__bufferedStdIO:
+                logger.setError(invoker.getStdio().strip())
+            raise BuildError("Bundling {} returned with {}"
+                                .format(checkoutStep.getPackage().getName(), ret),
+                             returncode=ret)
+
+        self.__bundler.add(checkoutStep.getPackage().getName(), bundled)
+
     def getStatistic(self):
         return self.__statistic
 
@@ -1284,6 +1395,11 @@ async def _cookCheckoutStep(self, checkoutStep, depth):
             assert predicted, "Non-predicted incorrect Build-Id found!"
             self.__handleChangedBuildId(checkoutStep, checkoutHash)
 
+        if self.__bundler and self.__bundler.bundle(checkoutStep):
+            with stepExec(checkoutStep, "BUNDLE",
+                "{} {}".format(checkoutStep.getPackage().getName(), overridesString)) as a:
+               await self.__doBundle (checkoutStep, a);
+
     async def _cookBuildStep(self, buildStep, depth, buildBuildId):
         # Add the execution path of the build step to the buildDigest to
         # detect changes between sandbox and non-sandbox builds. This is

pym/bob/cmds/build/build.py

@@ -211,6 +211,10 @@ def _downloadLayerArgument(arg):
         help="Move scm to attic if inline switch is not possible (default).")
     group.add_argument('--no-attic', action='store_false', default=None, dest='attic',
         help="Do not move to attic, instead fail the build.")
+    parser.add_argument('--bundle', metavar='BUNDLE', default=None,
+        help="Bundle all matching packages to BUNDLE")
+    parser.add_argument('--bundle-exclude', action='append', default=[],
+        help="Do not add matching packages to bundle.")
     args = parser.parse_args(argv)
 
     defines = processDefines(args.defines)
@@ -296,6 +300,9 @@ def _downloadLayerArgument(arg):
         packages = recipes.generatePackages(nameFormatter, args.sandbox)
         if develop: developPersister.prime(packages)
 
+        if args.bundle and args.build_mode == 'build-only':
+            parser.error("--bundle can't be used with --build-only")
+
         verbosity = cfg.get('verbosity', 0) + args.verbose - args.quiet
         setVerbosity(verbosity)
         builder = LocalBuilder(verbosity, args.force,
@@ -319,6 +326,7 @@ def _downloadLayerArgument(arg):
         builder.setShareHandler(getShare(recipes.getShareConfig()))
         builder.setShareMode(args.shared, args.install)
         builder.setAtticEnable(args.attic)
+        builder.setBundle(args.bundle, args.bundle_exclude)
         if args.resume: builder.loadBuildState()
 
         backlog = []
@@ -380,6 +388,8 @@ def _downloadLayerArgument(arg):
                     + " package" + ("s" if (stats.packagesBuilt != 1) else "") + " built, "
                 + str(stats.packagesDownloaded) + " downloaded.")
 
+        builder.bundle()
+
         # Copy build result if requested. It's ok to overwrite files that are
         # already at the destination. Warn if built packages overwrite
         # themselves, though.

pym/bob/invoker.py

@@ -81,9 +81,13 @@ class InvocationMode(Enum):
 
 class Invoker:
     def __init__(self, spec, preserveEnv, noLogFiles, showStdOut, showStdErr,
-                 trace, redirect, executor=None):
+                 trace, redirect, executor=None, tmpdir=False):
         self.__spec = spec
-        self.__cwd = spec.workspaceWorkspacePath
+        if not tmpdir:
+            self.__cwd = spec.workspaceWorkspacePath
+        else:
+            self.__tempDir = tempfile.TemporaryDirectory()
+            self.__cwd = self.__tempDir.name
         self.__preserveEnv = preserveEnv
         if preserveEnv:
             self.__env = os.environ.copy()
@@ -515,6 +519,33 @@ async def executeScmSwitch(self, scm, oldSpec):
 
         return ret
 
+    async def executeScmBundle(self, scm, bundleFile):
+        ret = 1
+        try:
+            self.__openLog("SCM Bundle")
+            try:
+                await scm.invoke(self, bundleFile=bundleFile)
+            except CmdFailedError as e:
+                self.error(scm.getSource(), "failed")
+                self.error(e.what)
+                raise
+            except Exception:
+                self.error(scm.getSource(), "failed")
+                raise
+
+            # everything went well
+            ret = 0
+
+        except OSError as e:
+            self.error("Something went wrong:", str(e))
+            ret = 1
+        except InvocationError as e:
+            ret = e.returncode
+        finally:
+            self.__closeLog(ret)
+
+        return ret
+
     async def runCommand(self, args, cwd=None, **kwargs):
         cwd = os.path.join(self.__cwd, cwd) if cwd else self.__cwd
         ret = await self.__runCommand(args, cwd, **kwargs)

pym/bob/scm/cvs.py

@@ -46,7 +46,9 @@ def getProperties(self, isJenkins, pretty=False):
         })
         return ret
 
-    async def invoke(self, invoker):
+    async def invoke(self, invoker, bundleFile=None):
+        if bundleFile is not None:
+            invoker.fail("Bundling of an cvs scm is not implemented!")
         # If given a ":ssh:" cvsroot, translate that to CVS_RSH using ssh, and ":ext:"
         # (some versions of CVS do that internally)
         m = re.match('^:ssh:(.*)', self.__cvsroot)

pym/bob/scm/git.py

@@ -6,7 +6,7 @@
 from ..errors import ParseError, BuildError
 from ..stringparser import isTrue, IfExpression
 from ..tty import WarnOnce, stepAction, INFO, TRACE, WARNING
-from ..utils import check_output, joinLines, run, removeUserFromUrl
+from ..utils import check_output, joinLines, run, removeUserFromUrl, hashFile
 from .scm import Scm, ScmAudit, ScmStatus, ScmTaint
 from shlex import quote
 from textwrap import dedent, indent
@@ -15,10 +15,12 @@
 import concurrent.futures
 import hashlib
 import locale
+import gzip
 import os, os.path
 import re
 import schema
 import subprocess
+import tarfile
 
 def normPath(p):
     return os.path.normcase(os.path.normpath(p))
@@ -163,7 +165,11 @@ def getProperties(self, isJenkins, pretty=False):
             properties.update({GitScm.REMOTE_PREFIX+key : val})
         return properties
 
-    async def invoke(self, invoker, switch=False):
+    @property
+    def bundleName(self):
+        return os.path.join(self.__dir, "-".join(filter(None, (self.__branch, self.__tag, self.__commit))) + ".tgz")
+
+    async def invoke(self, invoker, switch=False, bundleFile=None):
         alternatesFile = invoker.joinPath(self.__dir, ".git/objects/info/alternates")
 
         # make sure the git directory exists
@@ -246,6 +252,27 @@ async def invoke(self, invoker, switch=False):
             await invoker.checkCommand(["git", "repack", "-a"], cwd=self.__dir)
             os.unlink(alternatesFile)
 
+        if bundleFile:
+            def reset(tarinfo):
+                tarinfo.uid = tarinfo.gid = 0
+                tarinfo.uname = tarinfo.gname = "root"
+                tarinfo.mtime = 0
+                return tarinfo
+            files = []
+            for root, dirs, filenames in os.walk(invoker.joinPath(self.__dir)):
+                if ".git" in root: continue
+                for f in filenames:
+                    if f == ".bundle.log": continue
+                    files.append(os.path.join(root, f))
+            files.sort()
+            os.makedirs(os.path.dirname(bundleFile), exist_ok=True)
+            with open(bundleFile, 'wb') as outfile:
+                with gzip.GzipFile(fileobj=outfile, mode='wb', mtime=0) as zipfile:
+                    with tarfile.open(fileobj=zipfile, mode="w:") as bundle:
+                        for f in files:
+                            bundle.add(f, arcname=os.path.relpath(f, invoker.joinPath(self.__dir)),
+                                       recursive=False, filter=reset)
+
     async def __checkoutTagOnBranch(self, invoker, fetchCmd, switch):
         # Only do something if nothing is checked out yet or a forceful switch
         # is requested.