diff --git a/bin/check_release b/bin/check_release index 7b6e807..9f98f25 100755 --- a/bin/check_release +++ b/bin/check_release @@ -2,11 +2,9 @@ import ast import os import re -import subprocess +import subprocess # nosec import sys -from typing import cast -from typing import List -from typing import Optional +from typing import List, Optional, cast import requests @@ -15,10 +13,14 @@ def github_repo() -> str: repo = os.environ.get("GITHUB_REPOSITORY") if repo: return repo - return (subprocess.run( - ["git", "remote", "get-url", "upstream"], - check=True, - capture_output=True).stdout.decode("utf-8").strip().split(":")[1]) + return ( + subprocess.run( + ["git", "remote", "get-url", "upstream"], check=True, capture_output=True + ) + .stdout.decode("utf-8") + .strip() + .split(":")[1] + ) def release_github() -> Optional[str]: @@ -72,9 +74,12 @@ def release_milestone() -> str: sys.exit(1) return print_semver( - sorted(v for v in tuple( - parse_semver(cast(str, ms["title"])) for ms in milestones) - if v)[0]) + sorted( + v + for v in tuple(parse_semver(cast(str, ms["title"])) for ms in milestones) + if v + )[0] + ) def release_pr_milestone() -> Optional[str]: @@ -98,14 +103,18 @@ def release_local(path: str) -> tuple[Optional[str], str, bool]: with open(os.path.join(path, "BUILD.bazel"), "r") as fh: bzl = ast.parse(fh.read(), filename=path) for stmt in bzl.body: - if (isinstance(stmt, ast.Expr) - and isinstance(stmt.value, ast.Call) - and isinstance(stmt.value.func, ast.Name) - and stmt.value.func.id == "haskell_library"): + if ( + isinstance(stmt, ast.Expr) + and isinstance(stmt.value, ast.Call) + and isinstance(stmt.value.func, ast.Name) + and stmt.value.func.id == "haskell_library" + ): for arg in stmt.value.keywords: - if (arg.arg == "version" - and isinstance(arg.value, ast.Constant) - and isinstance(arg.value.s, str)): + if ( + arg.arg == "version" + and isinstance(arg.value, ast.Constant) + and isinstance(arg.value.s, str) + ): return arg.value.s, "BUILD.bazel", True if os.path.exists(os.path.join(path, "configure.ac")): @@ -147,24 +156,34 @@ def main(prog: str, args: List[str]) -> None: print(f"Local release: {local_release} ({local_origin})") if local_required and gh_release and gh_release != local_release: - print(f"\nFAIL: GitHub draft release {gh_release} does not match " - f"{local_origin} {local_release}") + print( + f"\nFAIL: GitHub draft release {gh_release} does not match " + f"{local_origin} {local_release}" + ) sys.exit(1) if local_required and ms_release != local_release: - print(f"\nFAIL: Next GitHub Milestone release {ms_release} does not " - f"match {local_origin} {local_release}") + print( + f"\nFAIL: Next GitHub Milestone release {ms_release} does not " + f"match {local_origin} {local_release}" + ) sys.exit(1) if local_required and pr_release and pr_release != local_release: - print(f"\nFAIL: PR milestone {pr_release} does not match " - f"{local_origin} {local_release}") + print( + f"\nFAIL: PR milestone {pr_release} does not match " + f"{local_origin} {local_release}" + ) sys.exit(1) if gh_release and gh_release != ms_release: - print(f"\nFAIL: GitHub draft release {gh_release} does not match " - f"next GitHub Milestone release {ms_release}") + print( + f"\nFAIL: GitHub draft release {gh_release} does not match " + f"next GitHub Milestone release {ms_release}" + ) sys.exit(1) if pr_release and pr_release != ms_release: - print(f"\nFAIL: PR milestone {pr_release} does not match " - f"next GitHub Milestone release {ms_release}") + print( + f"\nFAIL: PR milestone {pr_release} does not match " + f"next GitHub Milestone release {ms_release}" + ) sys.exit(1) print(f"\nPASS: Upcoming release version is {gh_release}") diff --git a/platform/appimage/build.sh b/platform/appimage/build.sh index c9c593d..c4f6e9a 100755 --- a/platform/appimage/build.sh +++ b/platform/appimage/build.sh @@ -7,6 +7,17 @@ # Fail out on error set -exuo pipefail +# https://stackoverflow.com/questions/72978485/git-submodule-update-failed-with-fatal-detected-dubious-ownership-in-reposit +git config --global --add safe.directory '*' + +# Ensure consistent file permissions +umask 022 + +# Support reproducible builds +if [ -z "${SOURCE_DATE_EPOCH:-}" ]; then + export SOURCE_DATE_EPOCH="$(git log -1 --format=%ct)" +fi + usage() { echo "$0 --src-dir SRC_DIR --project-name PROJECT_NAME [cmake args]" echo "Builds an app image in the CWD based off PROJECT_NAME installation at SRC_DIR" @@ -60,9 +71,6 @@ if [ -z "${PROJECT_NAME+x}" ]; then exit 1 fi -# https://stackoverflow.com/questions/72978485/git-submodule-update-failed-with-fatal-detected-dubious-ownership-in-reposit -git config --global --add safe.directory '*' - # Check if we can git describe git describe --tags --match 'v*' @@ -70,14 +78,24 @@ git describe --tags --match 'v*' readonly BUILD_DIR="$(realpath .)" readonly PROJECT_APP_DIR="$BUILD_DIR/$PROJECT_NAME.AppDir" +# Pin appimagetool version for reproducibility +readonly APPIMAGE_TOOL_VERSION="940" +readonly APPIMAGE_TOOL_URL="https://github.com/probonopd/go-appimage/releases/download/continuous/appimagetool-$APPIMAGE_TOOL_VERSION-x86_64.AppImage" + rm -f appimagetool-*.AppImage -wget "https://github.com/$(wget -q https://github.com/probonopd/go-appimage/releases/expanded_assets/continuous -O - | grep "appimagetool-.*-x86_64.AppImage" | head -n 1 | cut -d '"' -f 2)" +wget "$APPIMAGE_TOOL_URL" -O "appimagetool-$APPIMAGE_TOOL_VERSION-x86_64.AppImage" chmod +x appimagetool-*.AppImage +# Extract tool to a isolated directory to avoid polluting the workspace/AppImage +readonly TOOL_EXTRACT_DIR="$BUILD_DIR/tool-extract" +rm -rf "$TOOL_EXTRACT_DIR" +"./appimagetool-$APPIMAGE_TOOL_VERSION-x86_64.AppImage" --appimage-extract +mv squashfs-root "$TOOL_EXTRACT_DIR" + +# Patch the tool for musl compatibility. # https://github.com/probonopd/go-appimage/blob/fced8b8831039daa246ab355f4e2335074abc206/src/appimagetool/appdirtool.go#L400 # This line in the appimagetool breaks musl DNS lookups (looking for /EEE/resolv.conf). -./appimagetool-*.AppImage --appimage-extract -sed -i -e 's!/EEE!/etc!g' squashfs-root/usr/bin/appimagetool +sed -i -e 's!/EEE!/etc!g' "$TOOL_EXTRACT_DIR/usr/bin/appimagetool" export PKG_CONFIG_PATH=/opt/buildhome/lib/pkgconfig @@ -92,6 +110,7 @@ cmake "$SRC_DIR" \ -DCMAKE_INSTALL_PREFIX=/usr \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_EXE_LINKER_FLAGS="-Wl,--build-id=none" \ -B _build \ "$@" cmake --build _build @@ -99,6 +118,13 @@ cmake --install _build --prefix "$PROJECT_NAME.AppDir/usr" ccache --show-stats +# Normalize file permissions and timestamps for reproducibility +echo "Normalizing AppDir..." +find "$PROJECT_APP_DIR" -exec touch -h -d @"$SOURCE_DATE_EPOCH" {} + +find "$PROJECT_APP_DIR" -type d -exec chmod 0755 {} + +find "$PROJECT_APP_DIR" -type f -perm /0111 -exec chmod 0755 {} + +find "$PROJECT_APP_DIR" -type f ! -perm /0111 -exec chmod 0644 {} + + export QTDIR=/opt/buildhome/qt export LD_LIBRARY_PATH="/opt/buildhome/lib:/opt/buildhome/lib64:$QTDIR/lib" @@ -109,12 +135,19 @@ cp -r "$QTDIR/plugins/platforms/libqwayland-generic.so" "$PROJECT_APP_DIR/$QTDIR # Copy the tls plugins to the app dir, needed for https connections. cp -r "$QTDIR/plugins/tls/" "$PROJECT_APP_DIR/$QTDIR/plugins/" -squashfs-root/AppRun -s deploy "$PROJECT_APP_DIR"/usr/share/applications/*.desktop +"$TOOL_EXTRACT_DIR/AppRun" -s deploy "$PROJECT_APP_DIR"/usr/share/applications/*.desktop # print all links not contained inside the AppDir LD_LIBRARY_PATH='' find "$PROJECT_APP_DIR" -type f -exec ldd {} \; 2>&1 | grep '=>' | grep -v "$PROJECT_APP_DIR" -squashfs-root/AppRun "$PROJECT_APP_DIR" +# appimagetool (go-appimage) passes -fstime to mksquashfs, which conflicts with +# SOURCE_DATE_EPOCH environment variable. +( + unset SOURCE_DATE_EPOCH + "$TOOL_EXTRACT_DIR/AppRun" "$PROJECT_APP_DIR" +) -APPIMAGE_FILE="$PROJECT_NAME-$(git rev-parse --short HEAD | head -c7)-$ARCH.AppImage" +# Deterministic filename +readonly SHA="$(git rev-parse --short HEAD | head -c7)" +APPIMAGE_FILE="$PROJECT_NAME-$SHA-$ARCH.AppImage" sha256sum "$APPIMAGE_FILE" >"$APPIMAGE_FILE.sha256" diff --git a/tools/verify_all.py b/tools/verify_all.py new file mode 100755 index 0000000..ec7c0c3 --- /dev/null +++ b/tools/verify_all.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-or-later +# Copyright © 2026 The TokTok team +import argparse +import sys +from dataclasses import dataclass + +import verify_appimage +import verify_common +from lib import git + + +@dataclass +class Config: + tag: str + repo: str + + +def parse_args() -> Config: + parser = argparse.ArgumentParser( + description="Run all reproducibility verifications." + ) + parser.add_argument("--tag", help="Tag to verify", default=git.current_tag()) + parser.add_argument( + "--repo", help="Repository name", default=verify_common.get_default_repo() + ) + return Config(**vars(parser.parse_args())) + + +def main(config: Config) -> int: + tag = config.tag + repo = config.repo + + failed = [] + + print(f"--- Running verify_appimage for {repo} {tag} ---", file=sys.stderr) + appimage_config = verify_appimage.Config(tag=tag, repo=repo) + if verify_appimage.main(appimage_config) != 0: + failed.append("verify_appimage.py") + + if failed: + print(f"Verifications failed: {', '.join(failed)}", file=sys.stderr) + return 1 + + print(f"All verifications passed for {repo} {tag}!", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main(parse_args())) diff --git a/tools/verify_appimage.py b/tools/verify_appimage.py new file mode 100755 index 0000000..96fdd62 --- /dev/null +++ b/tools/verify_appimage.py @@ -0,0 +1,220 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-or-later +# Copyright © 2026 The TokTok team +import argparse +import os +import shutil +import subprocess # nosec +import sys +from dataclasses import dataclass + +import verify_common +from lib import git, github + + +@dataclass +class Config: + tag: str + repo: str + project_name: str | None = None + + +def parse_args() -> Config: + parser = argparse.ArgumentParser(description="Verify AppImage reproducibility.") + parser.add_argument("--tag", help="Tag to verify", default=git.current_tag()) + parser.add_argument( + "--repo", help="Repository name", default=verify_common.get_default_repo() + ) + parser.add_argument( + "--project-name", help="Project name (defaults to repo name)", default=None + ) + return Config(**vars(parser.parse_args())) + + +def get_sha256(path: str) -> str: + sha = subprocess.check_output(["sha256sum", path], text=True) # nosec + return sha.split()[0] + + +def main(config: Config) -> int: + repo_name = config.repo + tag = config.tag + + gh = github.GitHub() + + with verify_common.Workspace(repo_name, tag) as workspace: + # Detect project name from the workspace (the cloned repo) + project_name = config.project_name or verify_common.detect_project_name( + workspace.root + ) + + if not project_name: + print( + f"Error: Could not detect project name from {workspace.root}/CMakeLists.txt", + file=sys.stderr, + ) + print("Please provide it manually using --project-name", file=sys.stderr) + return 1 + + print(f"Detected project name: {project_name}", file=sys.stderr) + + assets = gh.release_assets(tag) + # Search for AppImage assets matching the detected project name. + appimage_asset = next( + ( + a + for a in assets + if a.name.lower().startswith(project_name.lower()) + and a.name.lower().endswith(".appimage") + and "x86_64" in a.name.lower() + ), + None, + ) + + if not appimage_asset: + print( + f"No x86_64 AppImage found for tag {tag} matching {project_name}", + file=sys.stderr, + ) + return 1 + + released_path = os.path.join(workspace.root, "released.AppImage") + print(f"Downloading {appimage_asset.name}...", file=sys.stderr) + with open(released_path, "wb") as f: + f.write(gh.download_asset(appimage_asset.id)) + + # 2. Build local + print("Building local AppImage via Docker...", file=sys.stderr) + workspace.run_docker( + "alpine-appimage", + [ + "third_party/ci-tools/platform/appimage/build.sh", + "--arch", + "x86_64", + "--src-dir", + "/qtox", + "--project-name", + project_name, + "--", + ], + env={ + "GITHUB_REPOSITORY": f"TokTok/{repo_name}", + "GITHUB_REF": f"refs/tags/{tag}", + }, + ) + print("Build completed successfully.", file=sys.stderr) + + # Find the produced AppImage in the workspace root + # Filename format: $PROJECT_NAME-$(git rev-parse --short HEAD | head -c7)-$ARCH.AppImage + sha = subprocess.check_output( # nosec + ["git", "-C", workspace.root, "rev-parse", "--short", "HEAD"], text=True + ).strip()[:7] + local_name = f"{project_name}-{sha}-x86_64.AppImage" + local_path = os.path.join(workspace.root, local_name) + + if not os.path.exists(local_path): + print(f"Local build failed to produce {local_path}", file=sys.stderr) + return 1 + + # 3. Compare + print("Comparing released and local AppImages...", file=sys.stderr) + + released_sha = get_sha256(released_path) + local_sha = get_sha256(local_path) + + print(f"Released SHA256: {released_sha}") + print(f"Local SHA256: {local_sha}") + + if released_sha != local_sha: + print("AppImages differ!", file=sys.stderr) + + # Deep investigation + print("\n--- Deep Investigation ---", file=sys.stderr) + released_dir = os.path.join(workspace.root, "released_ext") + local_dir = os.path.join(workspace.root, "local_ext") + + def extract_appimage(path: str, out: str) -> None: + """Extract SquashFS content from AppImage.""" + # AppImages extract themselves when called with --appimage-extract + # We make sure it's executable first. + os.chmod(path, 0o755) # nosec + try: + # In some environments (like Docker/restricted shells), + # we might need --appimage-extract-and-run or specific env vars. + # But usually this is the most direct way. + subprocess.run( # nosec + [path, "--appimage-extract"], + cwd=workspace.root, + check=True, + capture_output=True, + ) + target_squash = os.path.join(workspace.root, "squashfs-root") + if os.path.exists(target_squash): + shutil.move(target_squash, out) + else: + print( + f"Extraction of {path} failed to produce squashfs-root", + file=sys.stderr, + ) + except Exception as e: + print(f"Extraction of {path} failed: {e}", file=sys.stderr) + # Fallback to unsquashfs if the binary can't run + subprocess.run( # nosec + ["unsquashfs", "-d", out, "-f", path], + check=True, + capture_output=True, + ) + + try: + print("Extracting released AppImage...", file=sys.stderr) + extract_appimage(released_path, released_dir) + print("Extracting local AppImage...", file=sys.stderr) + extract_appimage(local_path, local_dir) + + print("Comparing extracted filesystem contents...", file=sys.stderr) + # Compare file structures and content hashes + diff_proc = subprocess.run( # nosec + ["diff", "-rq", released_dir, local_dir], + capture_output=True, + text=True, + ) + + if diff_proc.returncode == 0: + print( + "SUCCESS: Filesystem contents are bit-for-bit identical!", + file=sys.stderr, + ) + print( + "The difference lies solely in the AppImage packaging (SquashFS metadata or Runtime).", + file=sys.stderr, + ) + else: + print("DIFFERENCES FOUND in filesystem contents:", file=sys.stderr) + print(diff_proc.stdout) + + # Run diffoscope on the directories for a detailed breakdown + if shutil.which("diffoscope"): + print( + "\nRunning diffoscope on extracted directories...", + file=sys.stderr, + ) + # Limit output to avoid huge logs + subprocess.run( # nosec + ["diffoscope", "--text", "-", released_dir, local_dir] + ) + except Exception as e: + print(f"Deep investigation failed: {e}", file=sys.stderr) + + # Always run diffoscope on the full AppImage as a final reference + if shutil.which("diffoscope"): + print("\n--- Full AppImage Diff (Reference) ---", file=sys.stderr) + subprocess.run(["diffoscope", released_path, local_path]) # nosec + + return 1 + else: + print("AppImages are identical!", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main(parse_args())) diff --git a/tools/verify_common.py b/tools/verify_common.py new file mode 100644 index 0000000..a89eadf --- /dev/null +++ b/tools/verify_common.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-3.0-or-later +# Copyright © 2026 The TokTok team +import os +import shutil +import subprocess # nosec +import sys +import tempfile +from typing import Any + +from lib import git, github + + +class Workspace: + def __init__(self, repo_name: str, tag: str) -> None: + self.repo_name = repo_name + self.tag = tag + self.root: str = "" + self.temp_dir: tempfile.TemporaryDirectory[str] | None = None + self.ci_tools_path = git.root() + + def __enter__(self) -> "Workspace": + self.temp_dir = tempfile.TemporaryDirectory(prefix=f"verify-{self.repo_name}-") + self.root = self.temp_dir.name + print(f"Created temporary workspace: {self.root}", file=sys.stderr) + + # 1. Clone the target repository + repo_url = f"https://github.com/TokTok/{self.repo_name}.git" + print(f"Cloning {self.repo_name} at tag {self.tag}...", file=sys.stderr) + subprocess.run( # nosec + [ + "git", + "clone", + "--quiet", + "--depth", + "1", + "--branch", + self.tag, + repo_url, + self.root, + ], + check=True, + capture_output=True, + ) + + # 2. Setup third_party/ci-tools + # We want to use the current ci-tools code for verification. + # We must COPY rather than symlink, otherwise Docker cannot see it. + tp_dir = os.path.join(self.root, "third_party") + os.makedirs(tp_dir, exist_ok=True) + ci_tools_dst = os.path.join(tp_dir, "ci-tools") + print("Copying ci-tools into workspace...", file=sys.stderr) + + # We ignore the temp directories and git to keep it fast + shutil.copytree( + self.ci_tools_path, + ci_tools_dst, + ignore=shutil.ignore_patterns( + ".git", "third_party", "__pycache__", "*.AppImage", "*.flatpak" + ), + dirs_exist_ok=True, + ) + + # 3. Setup TokTok/dockerfiles + dockerfiles_dir = os.path.join(tp_dir, "dockerfiles") + print("Cloning TokTok/dockerfiles...", file=sys.stderr) + subprocess.run( # nosec + [ + "git", + "clone", + "--quiet", + "--depth", + "1", + "https://github.com/TokTok/dockerfiles.git", + dockerfiles_dir, + ], + check=True, + capture_output=True, + ) + + # 4. Copy docker-compose.yml to root + src_compose = os.path.join(dockerfiles_dir, "docker-compose.yml") + dst_compose = os.path.join(self.root, "docker-compose.yml") + shutil.copy2(src_compose, dst_compose) + + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: Any, + ) -> None: + if self.temp_dir: + print(f"Cleaning up workspace: {self.root}", file=sys.stderr) + self.temp_dir.cleanup() + + def run_docker( + self, service: str, command: list[str], env: dict[str, str] | None = None + ) -> None: + """Run a command via docker compose in the workspace.""" + cmd = ["docker", "compose", "run", "--rm"] + if env: + for k, v in env.items(): + cmd.extend(["-e", f"{k}={v}"]) + cmd.extend([service] + command) + + process = subprocess.run( # nosec + cmd, cwd=self.root, capture_output=True, text=True + ) + + if process.returncode != 0: + print(f"Error running docker command in {service}:", file=sys.stderr) + print(process.stdout, file=sys.stderr) + print(process.stderr, file=sys.stderr) + process.check_returncode() + + +def get_default_repo() -> str: + try: + return github.repository_name() + except Exception: + return "ci-tools" + + +def detect_project_name(root_dir: str) -> str | None: + """Detect the project name from CMakeLists.txt, mirroring CI logic.""" + cmake_path = os.path.join(root_dir, "CMakeLists.txt") + if not os.path.exists(cmake_path): + return None + + # Mirroring: pcregrep -M -o1 'project\(\s*(\S+)' CMakeLists.txt + import re + + with open(cmake_path, "r") as f: + content = f.read() + match = re.search(r"project\(\s*(\S+)", content, re.MULTILINE) + if match: + # Strip potential quotes or closing parenthesis + name = match.group(1).split(")")[0].strip("\"'") + return name + return None