From 56434ba7edd191d231a0e6409f66fb4a70ba34b7 Mon Sep 17 00:00:00 2001 From: agw Date: Tue, 1 Sep 2026 22:05:00 +0200 Subject: [PATCH] Bundle non-system libraries instead of version-pinning depends Bundling R alone was not enough: R links libicuuc, and Arch moves ICU regularly. The obvious fix - pinning 'icu>=78 <79' in depends - is worse than the disease on a rolling distro, because pacman would then refuse to upgrade ICU while JASP is installed, blocking the user's whole system upgrade until a rebuild lands. So bundle-libs.sh copies every non-system shared library into /opt/jasp-desktop/lib and rewrites rpaths, by denylist rather than allowlist. depends now carries no version constraints at all. Qt6 stays system-provided on purpose - Qt is binary compatible across Qt 6 minors, and bundling QtWebEngine would add ~500 MB. That also means the only rebuild triggers left are a new JASP release and a Qt major bump; rebuilding for anything else would cost users a 1.7 GB re-download. Also fixes check-upstream.py querying only repo=Extra, which silently returned nothing for icu (it lives in core). --- README.md | 35 +++++++++---- build/build-jasp.sh | 7 +++ build/bundle-libs.sh | 104 ++++++++++++++++++++++++++++++++++++++ packaging/PKGBUILD.in | 32 ++++++------ scripts/check-upstream.py | 41 ++++++++------- scripts/gen-pkgbuild.py | 10 +--- scripts/record-state.sh | 2 +- 7 files changed, 179 insertions(+), 52 deletions(-) create mode 100755 build/bundle-libs.sh diff --git a/README.md b/README.md index 0f8e46a..6cc2cd9 100644 --- a/README.md +++ b/README.md @@ -30,14 +30,28 @@ does (`.github/workflows/ubuntu.yml` sets `JASP_R_VERSION: 4.5.3` and an explici `R_HOME`). `Tools/CMake/R.cmake` reads `R_HOME` from the environment on Linux, so this is a supported configuration rather than a hack. -Consequences: +## What is bundled, and why nothing is version-pinned -1. `r` is **not** a dependency of the package. An Arch R upgrade cannot break an - installed JASP. -2. What the binary is still linked against is Qt6 and ICU, both of which break ABI - across releases. The generated PKGBUILD pins those - (`qt6-base>=6.11 <6.12`, `icu>=78 <79`), and `scripts/check-upstream.py` - triggers a rebuild when Arch moves either — or when JASP releases. +After staging, `build/bundle-libs.sh` walks every ELF object in the tree, copies +each non-system shared library it depends on into `/opt/jasp-desktop/lib`, and +rewrites the rpath to look there. It works by *denylist*: glibc, libstdc++, Qt6, +the GL/X11/Wayland/dbus stack, fontconfig and freetype stay system-provided; +everything else — ICU, jsoncpp, glpk, JAGS, readstat, librdata, BLAS/LAPACK, +pcre2, readline — is bundled. + +The point is that **`depends` contains no version constraints at all**. That is a +deliberate reversal of the obvious design. Pinning (`icu>=78 <79`) looks safer but +is actively harmful on a rolling distro: when Arch ships the next ICU, pacman +would refuse to upgrade it while JASP is installed, blocking the user's entire +system upgrade until a rebuild lands. Bundling means JASP never constrains the +system, and a `pacman -Syu` cannot break it. + +Qt6 is the deliberate exception and stays a system dependency. Qt guarantees +binary compatibility across Qt 6 minor releases, and bundling QtWebEngine would +add ~500 MB for very little gain. A Qt 7 transition will need a rebuild — which is +why `scripts/check-upstream.py` watches the Qt *major* and nothing else besides +the JASP version. Rebuilding for anything else would cost every user a ~1.7 GB +re-download for no benefit. `jags` and `librdata-git` are AUR-only, and JASP needs `freexl >= 2.0.99` while Arch ships 2.0.0 — so the build compiles those itself, matching what upstream CI does. @@ -47,7 +61,8 @@ ships 2.0.0 — so the build compiles those itself, matching what upstream CI do | Path | Purpose | | --- | --- | | `build/build-jasp.sh` | The actual build. Runs in an `archlinux:base-devel` container, emits the tarball plus `build-info.json`. | -| `scripts/check-upstream.py` | Compares upstream JASP / Arch Qt6 / Arch ICU against `state.json`; decides whether to build. | +| `build/bundle-libs.sh` | Copies non-system shared libraries into `/opt/jasp-desktop/lib` and rewrites rpaths, so the package needs no version-pinned deps. | +| `scripts/check-upstream.py` | Compares upstream JASP and the Arch Qt major against `state.json`; decides whether to build. | | `scripts/gen-pkgbuild.py` | Fills `packaging/PKGBUILD.in` from `build-info.json`; derives `pkgrel` from what is on the AUR. | | `scripts/publish-github-release.sh` | Creates the GitHub release and uploads the tarball. | | `scripts/push-aur.sh` | Generates `.SRCINFO` and pushes to `aur.archlinux.org`. | @@ -59,10 +74,10 @@ ships 2.0.0 — so the build compiles those itself, matching what upstream CI do ``` nightly (03:00) ─► check-upstream.py - │ new JASP tag, or Arch bumped qt6-base / icu? + │ new JASP tag, or a Qt major transition? ▼ build-jasp.sh (Arch container, hours) - │ builds R 4.5.3, JASP, then all R modules + │ builds R 4.5.3, JASP, all R modules, then bundles libs │ jasp-desktop--x86_64.tar.zst + build-info.json ▼ GitHub release ──► gen-pkgbuild.py ──► push to AUR diff --git a/build/build-jasp.sh b/build/build-jasp.sh index 1dc17fa..43076a7 100755 --- a/build/build-jasp.sh +++ b/build/build-jasp.sh @@ -196,11 +196,17 @@ find . -maxdepth 2 \( -name 'COPYING*' -o -name 'LICENSE*' \) -type f \ find "$STAGE_DIR" -name '*.o' -delete 2>/dev/null || true +# Make the tree independent of Arch's rolling libraries (ICU, jsoncpp, glpk, +# JAGS, readstat, librdata, BLAS/LAPACK ...). Qt6 stays a system dependency. +log "bundling non-system shared libraries" +bash /work/build/bundle-libs.sh "$STAGE_DIR" "$PREFIX" + # ------------------------------------------------------------------- metadata pkgver_of() { pacman -Q "$1" 2>/dev/null | awk '{print $2}'; } QT_FULL="$(pkgver_of qt6-base)" QT_MINOR="$(echo "$QT_FULL" | grep -oE '^[0-9]+\.[0-9]+')" +QT_MAJOR="$(echo "$QT_FULL" | grep -oE '^[0-9]+')" ICU_FULL="$(pkgver_of icu)" ICU_MAJOR="$(echo "$ICU_FULL" | grep -oE '^[0-9]+')" BOOST_FULL="$(pkgver_of boost-libs)" @@ -225,6 +231,7 @@ cat >"$OUT_DIR/build-info.json" </dev/null || true + ) + + for obj in "${scan[@]}"; do + [[ -f "$obj" ]] || continue + # ldd resolves against the build container, which is exactly the Arch + # userland we want to capture. + while read -r soname path; do + [[ -n "$path" && -f "$path" ]] || continue + is_system_lib "$soname" && continue + [[ -f "$LIBDIR/$soname" ]] && continue + # Never bundle something that already lives inside our own tree. + case "$path" in "$STAGE_DIR"*|"$PREFIX"*) continue ;; esac + cp -L "$path" "$LIBDIR/$soname" + echo ":: bundled $soname" + copied_any=1 + done < <(ldd "$obj" 2>/dev/null | awk '$2=="=>" && $3 ~ /^\// {print $1, $3}') + done +done +echo ":: bundling converged after $round round(s)" + +chmod u+w "$LIBDIR"/*.so* 2>/dev/null || true + +# An absolute rpath is safe here because the tarball always unpacks to $PREFIX; +# it avoids $ORIGIN arithmetic across bin/, R/lib/R/lib/ and the per-package +# libs/ directories, which all sit at different depths. +echo ":: setting rpath on ELF objects" +patched=0 +for obj in "${elf_files[@]}" "$LIBDIR"/*.so*; do + [[ -f "$obj" ]] || continue + if patchelf --set-rpath "${PREFIX}/lib" "$obj" 2>/dev/null; then + patched=$((patched + 1)) + fi +done +echo ":: rpath set on $patched objects" + +echo ":: bundled library count: $(find "$LIBDIR" -name '*.so*' | wc -l)" +du -sh "$LIBDIR" diff --git a/packaging/PKGBUILD.in b/packaging/PKGBUILD.in index 5251be8..2f86b4b 100644 --- a/packaging/PKGBUILD.in +++ b/packaging/PKGBUILD.in @@ -16,27 +16,27 @@ license=('AGPL-3.0-or-later') provides=("jasp-desktop=${pkgver}") conflicts=('jasp-desktop') -# JASP's R packages are pinned by upstream against a frozen CRAN snapshot that -# targets R @BUNDLED_R@, so this package ships its own R at -# /opt/jasp-desktop/R rather than using Arch's. That deliberately keeps `r` out -# of depends: an Arch R upgrade cannot break an installed JASP. +# This package ships its own R @BUNDLED_R@ plus the non-system libraries it and +# the JASP R modules link against (ICU, jsoncpp, glpk, JAGS, readstat, librdata, +# BLAS/LAPACK ...) under /opt/jasp-desktop/lib, with rpath pointing there. # -# Qt6 and ICU are still system libraries, and both break ABI across releases, so -# they are pinned to the versions this binary was linked against. +# That is deliberate, and it is why nothing below is version-pinned: on a rolling +# distro a pin like 'icu<79' would make pacman refuse to upgrade ICU while JASP +# is installed, blocking the user's entire system upgrade. Bundling means JASP +# never constrains the system. +# +# Qt6 is the exception and stays a system dependency: Qt guarantees binary +# compatibility across Qt 6 minor releases, and bundling QtWebEngine would add +# ~500 MB for no real gain. A Qt 7 transition will need a rebuild. depends=( - "qt6-base>=@QT_MINOR@" "qt6-base<@QT_NEXT_MINOR@" - 'qt6-declarative' 'qt6-svg' 'qt6-positioning' 'qt6-webchannel' + 'qt6-base' 'qt6-declarative' 'qt6-svg' 'qt6-positioning' 'qt6-webchannel' 'qt6-webengine' 'qt6-5compat' 'qt6-httpserver' - "icu>=@ICU_MAJOR@" "icu<@ICU_NEXT_MAJOR@" - 'boost-libs' 'glpk' 'jsoncpp' 'libarchive' 'openssl' 'curl' 'sqlite' 'zlib' - 'readstat' 'blas' 'lapack' 'pcre2' 'readline' 'bzip2' 'xz' - 'libpng' 'libtiff' 'libjpeg-turbo' 'cairo' 'pango' 'fontconfig' - 'jags' 'librdata-git' + 'fontconfig' 'freetype2' 'libxkbcommon' 'hicolor-icon-theme' ) -# The tarball is a complete /opt + /usr install tree; there is nothing to -# compile, and stripping a bundled R plus several hundred R modules is both slow -# and a good way to break them. +# The tarball is a complete /opt + /usr install tree. Stripping would be slow and +# is a good way to break a bundled R plus several hundred compiled R modules; +# the rpath set at build time must also survive untouched. options=('!strip' '!debug') source=("jasp-desktop-${pkgver}-${pkgrel}-x86_64.tar.zst::@DOWNLOAD_URL@") diff --git a/scripts/check-upstream.py b/scripts/check-upstream.py index cf6aab7..19b6208 100755 --- a/scripts/check-upstream.py +++ b/scripts/check-upstream.py @@ -1,10 +1,13 @@ #!/usr/bin/env python3 """Decide whether a rebuild is needed. -The build bundles its own R, so Arch's `r` is irrelevant here. What the binary -*is* still linked against is Qt6 and ICU, both of which break ABI across -releases — so a rebuild is triggered by an upstream JASP release or by Arch -moving either of those. +The build bundles its own R plus every non-system library it links against, so +Arch moving `r`, `icu`, `jsoncpp` and friends cannot affect an installed JASP and +must not trigger a rebuild — republishing costs every user a ~1.7 GB download. + +What is left is Qt6, which stays a system dependency. Qt guarantees binary +compatibility across Qt 6 minor releases, so only a *major* Qt transition +warrants a rebuild. Writes GITHUB_OUTPUT-style key=value pairs to stdout (and to $GITHUB_OUTPUT). """ @@ -16,7 +19,9 @@ import urllib.request STATE_FILE = "state.json" GH_LATEST = "https://api.github.com/repos/jasp-stats/jasp-desktop/releases/latest" -ARCH_PKG = "https://archlinux.org/packages/search/json/?name={}&repo=Extra" +# No repo filter: icu lives in core while qt6-base lives in extra, and pinning +# the wrong repo silently returns zero results and disables the trigger. +ARCH_PKG = "https://archlinux.org/packages/search/json/?name={}" def fetch_json(url: str) -> dict: @@ -27,9 +32,11 @@ def fetch_json(url: str) -> dict: def arch_version(pkg: str) -> str: results = fetch_json(ARCH_PKG.format(pkg)).get("results", []) - if not results: - raise RuntimeError(f"package {pkg} not found in Arch Extra") - return results[0]["pkgver"] + # Ignore staging/testing/kde-unstable: users are on the stable repos. + stable = [r for r in results if r["repo"] in ("core", "extra")] + if not stable: + raise RuntimeError(f"package {pkg} not found in Arch core/extra") + return stable[0]["pkgver"] def minor(version: str) -> str: @@ -41,8 +48,9 @@ def main() -> int: pinned_tag = os.environ.get("JASP_TAG", "").strip() jasp_tag = pinned_tag or fetch_json(GH_LATEST)["tag_name"] - qt_minor = minor(arch_version("qt6-base")) - icu_major = arch_version("icu").split(".")[0] + qt_version = arch_version("qt6-base") + qt_major = qt_version.split(".")[0] + qt_minor = minor(qt_version) try: state = json.loads(open(STATE_FILE).read()) @@ -54,27 +62,26 @@ def main() -> int: reasons.append("manually forced") if state.get("jasp_tag") != jasp_tag: reasons.append(f"JASP {state.get('jasp_tag', 'none')} -> {jasp_tag}") - if state.get("qt6_minor") != qt_minor: - reasons.append(f"Qt6 {state.get('qt6_minor', 'none')} -> {qt_minor}") - if state.get("icu_major") != icu_major: - reasons.append(f"ICU {state.get('icu_major', 'none')} -> {icu_major}") + if state.get("qt6_major") != qt_major: + reasons.append(f"Qt major {state.get('qt6_major', 'none')} -> {qt_major}") needs_build = bool(reasons) print(f"jasp_tag = {jasp_tag}") - print(f"qt6_minor = {qt_minor}") - print(f"icu_major = {icu_major}") + print(f"qt = {qt_version} (major {qt_major})") print(f"decision = {'BUILD' if needs_build else 'up to date'}") for reason in reasons: print(f" reason: {reason}") + if not needs_build: + print(" (Qt minor bumps are binary-compatible and everything else is bundled)") out = os.environ.get("GITHUB_OUTPUT") if out: with open(out, "a") as fh: fh.write(f"needs_build={'true' if needs_build else 'false'}\n") fh.write(f"jasp_tag={jasp_tag}\n") + fh.write(f"qt6_major={qt_major}\n") fh.write(f"qt6_minor={qt_minor}\n") - fh.write(f"icu_major={icu_major}\n") fh.write(f"reasons={'; '.join(reasons)}\n") return 0 diff --git a/scripts/gen-pkgbuild.py b/scripts/gen-pkgbuild.py index 63cb2c5..98786fe 100755 --- a/scripts/gen-pkgbuild.py +++ b/scripts/gen-pkgbuild.py @@ -64,17 +64,12 @@ def main() -> int: ) qt_minor = info["qt6_minor"] - icu_major = info["icu_major"] substitutions = { "@MAINTAINER@": args.maintainer, "@JASP_VERSION@": jasp_version, "@PKGREL@": str(pkgrel), "@BUNDLED_R@": info["bundled_r_version"], - "@QT_MINOR@": qt_minor, - "@QT_NEXT_MINOR@": next_minor(qt_minor), - "@ICU_MAJOR@": icu_major, - "@ICU_NEXT_MAJOR@": str(int(icu_major) + 1), "@DOWNLOAD_URL@": download_url, "@SHA256@": info["sha256"], } @@ -96,9 +91,8 @@ def main() -> int: json.dump({"tag": tag, "pkgrel": pkgrel, "download_url": download_url, **info}, fh, indent=2) print(f"wrote {args.out} ({args.pkgname} {jasp_version}-{pkgrel})") - print(f" R bundled at {info['bundled_r_version']} (not an Arch dependency)") - print(f" Qt pinned to >={qt_minor} <{next_minor(qt_minor)}") - print(f" ICU pinned to >={icu_major} <{int(icu_major) + 1}") + print(f" R bundled at {info['bundled_r_version']}; built against Qt {qt_minor}") + print(" depends are unpinned - everything volatile is bundled") return 0 diff --git a/scripts/record-state.sh b/scripts/record-state.sh index 7c752a4..19b6bcd 100755 --- a/scripts/record-state.sh +++ b/scripts/record-state.sh @@ -18,7 +18,7 @@ state = { "pkgrel": meta["pkgrel"], "bundled_r_version": meta["bundled_r_version"], "qt6_minor": meta["qt6_minor"], - "icu_major": meta["icu_major"], + "qt6_major": meta["qt6_major"], "sha256": meta["sha256"], "download_url": meta["download_url"], }