From 7dd99f0b7dfcb62218d35054878fb73cee91094f Mon Sep 17 00:00:00 2001 From: agw Date: Tue, 1 Sep 2026 23:30:00 +0200 Subject: [PATCH] Actually ship the R libraries the modules need The first full build succeeded and produced a package that would not have worked. Three separate causes, all silent: - R was configured --with-blas --with-lapack, so it linked Arch's external BLAS and never built libRblas.so/libRlapack.so. The JASP module bundles expect those inside R_HOME, the way a conventional R and the Flatpak ship them. Now --without-blas, with a hard check after install. - buildModuleBundlesLocally.R installs jaspModuleTools via install_github, which failed on anonymous GitHub rate limits. No bundles were produced at all. GITHUB_PAT is now required rather than merely recommended. - buildAllDefaultJaspModules.R calls the bundle builder through system2(), which ignores exit status, so all of the above exited 0. Added explicit guards: jaspModuleTools must exist, Modules/local must be non-empty, and the staged tree must contain the bundles and all three R libraries. Staging was also wrong: it copied ModuleBundleBuildDir/Modules, which is cloned module *sources* with zero compiled objects, while dropping LibraryR entirely - 267MB of R packages built against the bundled R. --- build/build-jasp.sh | 99 ++++++++++++++++-- scripts/gen-tarball-index.py | 192 +++++++++++++++++++++++++++++++++++ 2 files changed, 281 insertions(+), 10 deletions(-) create mode 100644 scripts/gen-tarball-index.py diff --git a/build/build-jasp.sh b/build/build-jasp.sh index 43076a7..74e1d8a 100755 --- a/build/build-jasp.sh +++ b/build/build-jasp.sh @@ -103,19 +103,38 @@ cd "R-${R_VERSION}" # config.site would otherwise pick, which keeps the rpath below predictable. # The install prefix equals the final install path, so the rpath stays valid # once the tarball is unpacked on the user's machine. +# +# --without-blas/--without-lapack is deliberate and load-bearing. With +# --with-blas, R links Arch's external BLAS and does NOT build libRblas.so or +# libRlapack.so. The JASP module bundles expect those files to exist inside +# R_HOME (that is how a conventionally built R, and the Flatpak, ship), so +# linking externally leaves the modules unable to resolve them. Reference BLAS +# is slower than OpenBLAS, but a working install beats a fast broken one. ./configure \ --prefix="${R_HOME_DIR}" \ --libdir="${R_HOME_DIR}/lib" \ --enable-R-shlib \ --enable-memory-profiling \ - --with-blas \ - --with-lapack \ + --without-blas \ + --without-lapack \ --with-x=no \ --with-tcltk=no \ LDFLAGS="-Wl,-rpath,${R_HOME_DIR}/lib/R/lib" make -j"$JOBS" make install +# Guard the above: a silently external-BLAS build produces a JASP whose modules +# fail at runtime, which is exactly the class of failure this package exists to +# avoid. Better to stop here than ship it. +for lib in libR.so libRblas.so libRlapack.so; do + if [[ ! -f "${R_HOME_DIR}/lib/R/lib/${lib}" ]]; then + echo "error: ${lib} missing from the bundled R — R linked an external BLAS?" >&2 + ls -la "${R_HOME_DIR}/lib/R/lib/" >&2 + exit 1 + fi +done +log "bundled R provides its own libR/libRblas/libRlapack" + export PATH="${R_HOME_DIR}/bin:${PATH}" # Ask R where it actually put itself rather than assuming lib vs lib64. # Deliberately NOT exported: R.cmake notes that a set R_HOME makes R print @@ -156,15 +175,46 @@ log "compiling JASP (-j$JOBS)" ninja -C "$BUILD_DIR" -j"$JOBS" # The R modules are the long pole: several hundred R packages compiled from -# source, installed into the bundled R's library. +# source. buildAllDefaultJaspModules.R shells out to buildModuleBundlesLocally.R +# via system2(), which does NOT check exit status — so every failure below is +# silent and the script still exits 0. The guards after it are what actually +# catch a broken module build. +# +# GITHUB_PAT is required, not optional: buildModuleBundlesLocally.R does +# install_github('jasp-stats/jaspModuleTools'), and anonymous GitHub API rate +# limits make that fail. When it fails, no bundles are produced at all and the +# build otherwise looks successful. +if [[ -z "${GITHUB_PAT:-}" ]]; then + echo "error: GITHUB_PAT is required — jaspModuleTools is installed from GitHub" >&2 + echo " without it the module build silently produces nothing" >&2 + exit 1 +fi +export GITHUB_PAT +export GITHUB_TOKEN="${GITHUB_PAT}" + log "building R modules — this is the slow part" mkdir -p "$BUILD_DIR"/Modules/{binary_pkgs,manifests,module_libs,Tools} cd "$SRC_DIR/Tools" -if [[ -z "${GITHUB_PAT:-}" ]]; then - echo "warning: GITHUB_PAT is unset — R module downloads from GitHub may hit anonymous rate limits" >&2 -fi Rscript buildAllDefaultJaspModules.R +MODULE_LIBRARY="$SRC_DIR/Tools/ModuleBundleBuildDir/LibraryR" +MODULE_BUNDLES="$SRC_DIR/Modules/local" + +if [[ ! -d "$MODULE_LIBRARY/jaspModuleTools" ]]; then + echo "error: jaspModuleTools was not installed — install_github failed" >&2 + echo " (GitHub rate limit, or GITHUB_PAT lacks read access)" >&2 + exit 1 +fi + +bundle_count="$(find "$MODULE_BUNDLES" -mindepth 1 -maxdepth 1 2>/dev/null | wc -l)" +if [[ "$bundle_count" -eq 0 ]]; then + echo "error: the module build produced no bundles in ${MODULE_BUNDLES}" >&2 + echo " system2() in buildAllDefaultJaspModules.R hides the real error;" >&2 + echo " re-run buildModuleBundlesLocally.R by hand to see it" >&2 + exit 1 +fi +log "module build produced ${bundle_count} bundles" + # -------------------------------------------------------------------- staging log "staging install tree" rm -rf "$STAGE_DIR" @@ -176,10 +226,25 @@ DESTDIR="$STAGE_DIR" ninja -C "$BUILD_DIR" install install -d "$STAGE_DIR$PREFIX" cp -a "$R_HOME_DIR" "$STAGE_DIR$PREFIX/" +# Everything the module build produced has to travel with the tarball, or JASP +# falls back to downloading bundles at runtime — which is the failure mode this +# package is meant to remove. +# +# Modules/local the compiled module bundles themselves +# LibraryR the R library they were built with (jaspModuleTools, devtools +# and ~95 dependencies), compiled against the bundled R 4.5.3 +# +# ModuleBundleBuildDir/Modules is deliberately NOT copied: it is the cloned +# module *sources* (no compiled objects at all), several hundred MB of git +# checkouts that would bloat the tarball without being usable at runtime. install -d "$STAGE_DIR$PREFIX/Modules" -if [[ -d "$SRC_DIR/Tools/ModuleBundleBuildDir/Modules" ]]; then - cp -a "$SRC_DIR/Tools/ModuleBundleBuildDir/Modules/." "$STAGE_DIR$PREFIX/Modules/" -fi + +log "staging module bundles from ${MODULE_BUNDLES}" +cp -a "$MODULE_BUNDLES/." "$STAGE_DIR$PREFIX/Modules/" + +log "staging module R library from ${MODULE_LIBRARY}" +install -d "$STAGE_DIR$PREFIX/ModuleLibrary" +cp -a "$MODULE_LIBRARY/." "$STAGE_DIR$PREFIX/ModuleLibrary/" install -Dm644 /work/packaging/org.jaspstats.JASP.desktop \ "$STAGE_DIR/usr/share/applications/org.jaspstats.JASP.desktop" @@ -197,10 +262,24 @@ find . -maxdepth 2 \( -name 'COPYING*' -o -name 'LICENSE*' \) -type f \ find "$STAGE_DIR" -name '*.o' -delete 2>/dev/null || true # Make the tree independent of Arch's rolling libraries (ICU, jsoncpp, glpk, -# JAGS, readstat, librdata, BLAS/LAPACK ...). Qt6 stays a system dependency. +# JAGS, readstat, librdata ...). Qt6 stays a system dependency; BLAS/LAPACK now +# come from the bundled R itself. log "bundling non-system shared libraries" bash /work/build/bundle-libs.sh "$STAGE_DIR" "$PREFIX" +# Final sanity check on what is about to be shipped. Each of these has already +# been shipped missing once. +log "verifying the staged tree" +staged_r="$STAGE_DIR$PREFIX/R/lib/R/lib" +for lib in libR.so libRblas.so libRlapack.so; do + [[ -f "$staged_r/$lib" ]] || { echo "error: staged tree lacks $lib" >&2; exit 1; } +done +staged_bundles="$(find "$STAGE_DIR$PREFIX/Modules" -mindepth 1 -maxdepth 1 2>/dev/null | wc -l)" +[[ "$staged_bundles" -gt 0 ]] || { echo "error: staged tree has no module bundles" >&2; exit 1; } +staged_so="$(find "$STAGE_DIR$PREFIX" -name '*.so' 2>/dev/null | wc -l)" +log "staged: ${staged_bundles} module bundles, ${staged_so} shared objects" +du -sh "$STAGE_DIR$PREFIX"/* 2>/dev/null | sort -rh | head -8 + # ------------------------------------------------------------------- metadata pkgver_of() { pacman -Q "$1" 2>/dev/null | awk '{print $2}'; } diff --git a/scripts/gen-tarball-index.py b/scripts/gen-tarball-index.py new file mode 100644 index 0000000..387cdff --- /dev/null +++ b/scripts/gen-tarball-index.py @@ -0,0 +1,192 @@ +#!/usr/bin/env python3 +"""Render the index page for https://asger.weirsoe.dk/tarballz/. + +Reads the sidecar .json files that publish-tarball.sh writes next to each +tarball (on stdin, one JSON object per line) and emits a standalone HTML page. + +The theme tokens are inlined rather than linked to the main site's stylesheet: +that file is content-hashed (_astro/_slug_..css) and its name changes on +every site rebuild, so a link would silently rot. Inlining costs a few hundred +bytes and cannot break. +""" + +import html +import json +import os +import sys +from datetime import datetime, timezone + +GITEA_URL = "https://gitea.weircon.dk/agw/jasp-desktop-bin" +AUR_URL = "https://aur.archlinux.org/packages/jasp-desktop-bin" + +PAGE = """ + + + + +Tarballs · Asger Weirsøe + + + + + + + +
+ Asger Weirsøe. + +
+ +
+

Tarballs

+

+ Prækompilerede JASP-binærer til Arch Linux. + JASP udgiver kun installere til Windows og macOS, så disse er bygget fra + kildekoden af en automatisk pipeline og udgivet på AUR som + jasp-desktop-bin. +

+ +
+

Du behøver normalt ikke hente herfra manuelt — brug AUR:

+ yay -S jasp-desktop-bin +
+ +

Tilgængelige builds

+ {releases} + +

+ Hver build indeholder sin egen R og de biblioteker den linker mod, så den ikke + bindes til bestemte versioner af noget på dit system. + Kun de nyeste {keep} builds gemmes. +

+
+ +
+

bygget automatisk · self-hosted på proxmox · byggeopskrift

+

© {year} Asger Weirsøe

+
+ + +""" + + +def human_size(n: int) -> str: + for unit in ("B", "KB", "MB", "GB"): + if n < 1024 or unit == "GB": + return f"{n:.1f} {unit}".replace(".0 ", " ") + n /= 1024 + return f"{n} B" + + +def main() -> int: + entries = [] + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + entries.append(json.loads(line)) + except json.JSONDecodeError: + print(f"warning: skipping unparsable sidecar line", file=sys.stderr) + + # Newest first. + entries.sort(key=lambda e: e.get("built", ""), reverse=True) + + if entries: + items = [] + for e in entries: + name = html.escape(e["tarball"]) + ver = html.escape(e.get("jasp_version", "?")) + built = e.get("built", "") + try: + built_h = datetime.fromisoformat(built).strftime("%d-%m-%Y") + except ValueError: + built_h = html.escape(built) + size = human_size(int(e.get("size_bytes", 0))) + sha = html.escape(e.get("sha256", "")) + r = html.escape(e.get("bundled_r_version", "")) + qt = html.escape(e.get("qt6_minor", "")) + items.append( + f'
  • \n' + f'
    \n' + f' JASP {ver}\n' + f' Hent · {size}\n' + f'
    \n' + f'
    {built_h} · x86_64 · bundlet R {r} · bygget mod Qt {qt}
    \n' + f'
    sha256 {sha}
    \n' + f'
    • ' + ) + releases = '" + else: + releases = '

    Ingen builds tilgængelige lige nu.

    ' + + sys.stdout.write( + PAGE.format( + releases=releases, + gitea=GITEA_URL, + aur=AUR_URL, + keep=len(entries) if entries else 3, + year=datetime.now(timezone.utc).year, + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main())