From 5601c0eb96dd8ddc97363633b106fc9a6980917e Mon Sep 17 00:00:00 2001 From: agw Date: Wed, 30 Sep 2026 19:55:00 +0200 Subject: [PATCH] Keep build caches across runs so a rebuild compiles only what changed Every build started from an empty container and recompiled R, JASP and 1000+ R packages, which is days of CPU on the runner. Now: - /srv/jasp-cache on the host holds ccache and the renv package cache, linked into the paths jaspModuleTools::compile() hard-codes. - RENV_PATHS_PREFIX is fixed: renv otherwise keys the cache on the archlinux image's build date, so any newer image started cold. - Cached packages with unresolved sonames after an Arch update are purged, so only those are rebuilt. - R packages compile without -g (the Stan/Eigen ones spend real time and GBs of memory on debug info), and retry passes skip modules that already built. Claude-Session: https://claude.ai/code/session_01GGe7kUhooMFuzY6GaDb2w9 --- .gitea/workflows/build.yml | 1 + README.md | 5 +++ build/build-jasp.sh | 83 ++++++++++++++++++++++++++++++++++---- 3 files changed, 81 insertions(+), 8 deletions(-) diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml index 82db269..ccce603 100644 --- a/.gitea/workflows/build.yml +++ b/.gitea/workflows/build.yml @@ -86,6 +86,7 @@ jobs: docker rm -f jasp-build 2>/dev/null || true docker run --rm --name jasp-build \ -v "${WORK_DIR}:/work" \ + -v /srv/jasp-cache:/cache \ -e JASP_TAG="${{ needs.check.outputs.jasp_tag }}" \ -e OUT_DIR=/work/out \ -e JOBS=5 \ diff --git a/README.md b/README.md index 902b898..e809391 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,11 @@ The build runs for hours and needs disk. On the Gitea runner host: 3h) must match, otherwise Gitea kills the job at 3h regardless of the runner. - Roughly 40–60 GB of free disk for the build tree, the bundled R and the staged install. +- `/srv/jasp-cache` on the runner host, mounted into the build as `/cache`. + It holds ccache and the compiled R package cache (renv), and is what keeps a + rebuild from recompiling 1000+ R packages from scratch. It is never wiped by + the workflow; entries whose system libraries have gone missing after an Arch + update are purged at the start of each build. Delete it to force a clean build. - **Swap.** `rstanarm` (required by `jaspMixedModels`) and `OpenMx` generate translation units where a single `cc1plus` wants 4–8 GB. Without swap an over-commit is an instant OOM kill, which surfaces only as a module that never diff --git a/build/build-jasp.sh b/build/build-jasp.sh index 856a259..612fd93 100755 --- a/build/build-jasp.sh +++ b/build/build-jasp.sh @@ -41,6 +41,23 @@ mkdir -p "$OUT_DIR" /build log() { echo ":: $* "; } +# ------------------------------------------------------------- build caches +# /cache is a directory on the runner host that outlives the container. Without +# it every build recompiles R, JASP and 1000+ R packages from nothing, which is +# days of CPU on the runner; with it a rebuild only compiles what changed. +# /cache/ccache compiler cache for R and JASP's own C++ +# /cache/renv-cache compiled R packages, shared by every module +# /cache/renv-root renv's download cache (source tarballs) +CACHE_DIR="${CACHE_DIR:-/cache}" +mkdir -p "$CACHE_DIR"/{ccache,renv-cache,renv-root} +export CCACHE_DIR="$CACHE_DIR/ccache" +export CCACHE_MAXSIZE="${CCACHE_MAXSIZE:-15G}" +# renv files its cache under the OS release, and the archlinux image stamps its +# build date there (linux-arch-20260809.0.570793), so every newer image would +# start from an empty cache. A rolling distro has no release to key on; staleness +# is handled by purge_stale_renv_cache below instead. +export RENV_PATHS_PREFIX=linux-arch + # ---------------------------------------------------------------- system deps log "installing Arch dependencies" pacman -Syu --noconfirm --needed \ @@ -121,7 +138,10 @@ cd "R-${R_VERSION}" --with-x=no \ --with-tcltk=no \ LDFLAGS="-Wl,-rpath,${R_HOME_DIR}/lib/R/lib" -make -j"$JOBS" +# ccache through PATH rather than CC="ccache gcc": configure would bake the +# latter into the shipped Makeconf, and every package a user installs into +# JASP's R would then need ccache. +PATH="/usr/lib/ccache/bin:$PATH" make -j"$JOBS" make install # Guard the above: a silently external-BLAS build produces a JASP whose modules @@ -268,6 +288,50 @@ PY # with less parallelism than the C++ build uses. R_JOBS="${R_JOBS:-2}" export MAKEFLAGS="-j${R_JOBS}" + +# R's default flags are -g -O2. Debug info in the Eigen/Stan-heavy packages +# (rstanarm, OpenMx) costs compile time and GBs of cc1plus memory, and nobody +# debugs them. User Makevars only affects this build, not the shipped Makeconf. +mkdir -p ~/.R +cat >~/.R/Makevars <<'EOF' +CFLAGS=-O2 -pipe +CXXFLAGS=-O2 -pipe +CXX11FLAGS=-O2 -pipe +CXX14FLAGS=-O2 -pipe +CXX17FLAGS=-O2 -pipe +CXX20FLAGS=-O2 -pipe +FFLAGS=-O2 -pipe +FCFLAGS=-O2 -pipe +EOF + +# jaspModuleTools::compile() hard-codes RENV_PATHS_CACHE and RENV_PATHS_ROOT to +# directories under its workdir, so the persistent cache is linked in there. +MODULE_WORKDIR="$SRC_DIR/Tools/ModuleBundleBuildDir/build_dir" +mkdir -p "$MODULE_WORKDIR" +ln -sfn "$CACHE_DIR/renv-cache" "$MODULE_WORKDIR/renv-cache" +ln -sfn "$CACHE_DIR/renv-root" "$MODULE_WORKDIR/renv-root" + +# A cached package is only reusable while the system libraries it linked +# against still exist. Arch rolls (an ICU or gsl soname bump), and renv's cache +# key knows nothing about that, so drop every entry with an unresolved library +# and let renv rebuild just those. Layout: .../////libs/*.so +purge_stale_renv_cache() { + local so entry purged=0 + while IFS= read -r so; do + if LD_LIBRARY_PATH="${JASP_R_HOME}/lib" ldd "$so" 2>/dev/null | grep -q 'not found'; then + entry="$(dirname "$(dirname "$(dirname "$so")")")" + [[ -d "$entry" ]] || continue + rm -rf "$entry" + purged=$((purged + 1)) + fi + done < <(find "$CACHE_DIR/renv-cache/" -path '*/libs/*.so' 2>/dev/null) + log "renv cache: purged ${purged} entries with missing system libraries" +} +purge_stale_renv_cache + +# ccache for R package compiles too: cheap insurance for when renv reinstalls a +# package whose compiler inputs did not actually change. +export PATH="/usr/lib/ccache/bin:$PATH" # R leaves 00LOCK- directories behind when an install is interrupted, and # refuses to touch the library afterwards ("failed to lock directory ... for # modifying"). A build that is killed mid-install - a timeout, an OOM, a stopped @@ -340,14 +404,17 @@ for module in "${ALL_MODULES[@]}"; do fi done -MODULE_PATHS=() -for module in "${ALL_MODULES[@]}"; do - [[ -d "ModuleBundleBuildDir/Modules/${module}" ]] && \ - MODULE_PATHS+=("ModuleBundleBuildDir/Modules/${module}") -done - for attempt in 1 2 3; do - log "module build pass ${attempt}/3" + # Hand over only modules without a bundle yet: the script rebuilds everything + # it is given, so a retry would otherwise redo every finished module. + MODULE_PATHS=() + for module in "${ALL_MODULES[@]}"; do + [[ -f "${MODULE_BUNDLES}/${module}.JASPModule" ]] && continue + [[ -d "ModuleBundleBuildDir/Modules/${module}" ]] && \ + MODULE_PATHS+=("ModuleBundleBuildDir/Modules/${module}") + done + [[ ${#MODULE_PATHS[@]} -eq 0 ]] && break + log "module build pass ${attempt}/3: ${#MODULE_PATHS[@]} modules left" ./buildModuleBundlesLocally.R "${MODULE_PATHS[@]}" || true built=0 for module in "${ALL_MODULES[@]}"; do