From e49be3dfa4375945a891cb56d177c343e23b3ed9 Mon Sep 17 00:00:00 2001 From: agw Date: Wed, 2 Sep 2026 22:15:00 +0200 Subject: [PATCH] Build only the modules JASP ships, not the developer helper's 22 Modules/modules-settings.json splits modules into 'common' - the seven bundled with the application - and 'extra', which JASP fetches on demand into ~/.local/share/JASP. buildAllDefaultJaspModules.R builds 22 regardless: it is a developer convenience for having everything locally, not a definition of what a release contains. I had been treating it as the latter. The cost of that mistake was not just size. Most of the blockers this build hit came from modules that are never shipped: gsl blocked jaspNetwork and jaspBFF, OpenMx blocked jaspSem - all 'extra'. Building the right list drops them entirely. Measured: the seven common modules come to ~2.3 GB against ~8.3 GB for all 22. Bundles are already zstd-compressed, so the tarball does not shrink them further; ~2.5-3 GB total is the realistic figure, against the Flatpak's 1.4 GB. The remaining gap is bundleAll=TRUE giving every module its own copy of its R dependencies. The guard now names the modules it expects and reports which are missing. --- build/build-jasp.sh | 61 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 49 insertions(+), 12 deletions(-) diff --git a/build/build-jasp.sh b/build/build-jasp.sh index 7b41022..a6c2079 100755 --- a/build/build-jasp.sh +++ b/build/build-jasp.sh @@ -52,7 +52,8 @@ pacman -Syu --noconfirm --needed \ boost boost-libs glpk jsoncpp libarchive openssl curl sqlite zlib \ readstat netcdf mpfr cairo pango fontconfig harfbuzz fribidi icu pcre2 \ readline bzip2 xz libpng libtiff libjpeg-turbo minizip \ - libxkbcommon libxkbcommon-x11 libxt wget unzip python + libxkbcommon libxkbcommon-x11 libxt wget unzip python \ + gsl libsodium gmp nlopt libxml2 fftw # Stock makepkg.conf leaves MAKEFLAGS at -j1, which makes the AUR dependencies # (JAGS especially) crawl. @@ -269,12 +270,46 @@ R_JOBS="${R_JOBS:-2}" export MAKEFLAGS="-j${R_JOBS}" log "compiling R packages with -j${R_JOBS} (OpenMx needs the headroom)" -EXPECTED_MODULES=22 +# Build only the modules JASP actually ships. Modules/modules-settings.json +# splits them into "common" (bundled with the application) and "extra" +# (fetched on demand at runtime, into ~/.local/share/JASP). +# +# buildAllDefaultJaspModules.R is a *developer* helper that builds 22 modules +# regardless of that split — its name says "all default modules", not "what is +# released". Building from it produces a ~9 GB tree, most of it modules the user +# would have downloaded on demand anyway, and drags in system dependencies that +# only the extras need (gsl for jaspNetwork, OpenMx for jaspSem). +mapfile -t COMMON_MODULES < <( + python3 -c " +import json +print('\n'.join(json.load(open('../Modules/modules-settings.json'))['common'])) +" +) +log "shipping the ${#COMMON_MODULES[@]} common modules: ${COMMON_MODULES[*]}" + +mkdir -p ModuleBundleBuildDir/Modules +for module in "${COMMON_MODULES[@]}"; do + if [[ ! -d "ModuleBundleBuildDir/Modules/${module}" ]]; then + log "cloning ${module}" + git clone --depth 1 "https://github.com/jasp-stats/${module}" \ + "ModuleBundleBuildDir/Modules/${module}" + fi +done + +EXPECTED_MODULES="${#COMMON_MODULES[@]}" +MODULE_PATHS=() +for module in "${COMMON_MODULES[@]}"; do + MODULE_PATHS+=("ModuleBundleBuildDir/Modules/${module}") +done + for attempt in 1 2 3; do log "module build pass ${attempt}/3" - Rscript buildAllDefaultJaspModules.R || true - built="$(find "$MODULE_BUNDLES" -mindepth 1 -maxdepth 1 2>/dev/null | wc -l)" - log "pass ${attempt} finished: ${built}/${EXPECTED_MODULES} bundles" + ./buildModuleBundlesLocally.R "${MODULE_PATHS[@]}" || true + built=0 + for module in "${COMMON_MODULES[@]}"; do + [[ -f "${MODULE_BUNDLES}/${module}.JASPModule" ]] && built=$((built + 1)) + done + log "pass ${attempt} finished: ${built}/${EXPECTED_MODULES} common modules" [[ "$built" -ge "$EXPECTED_MODULES" ]] && break done @@ -287,15 +322,17 @@ fi # Guard on the expected count, not merely "non-empty". An earlier run shipped a # package containing 2 of 22 modules because a non-empty check passed. -bundle_count="$(find "$MODULE_BUNDLES" -mindepth 1 -maxdepth 1 2>/dev/null | wc -l)" -if [[ "$bundle_count" -lt "$EXPECTED_MODULES" ]]; then - echo "error: only ${bundle_count}/${EXPECTED_MODULES} module bundles were built" >&2 - echo " system2() in buildAllDefaultJaspModules.R hides the real error;" >&2 +missing=() +for module in "${COMMON_MODULES[@]}"; do + [[ -f "${MODULE_BUNDLES}/${module}.JASPModule" ]] || missing+=("$module") +done +if [[ ${#missing[@]} -gt 0 ]]; then + echo "error: ${#missing[@]} of ${EXPECTED_MODULES} common modules were not built" >&2 + printf ' missing: %s\n' "${missing[@]}" >&2 echo " search the log above for 'failed to install'" >&2 - find "$MODULE_BUNDLES" -mindepth 1 -maxdepth 1 -printf ' have: %f\n' 2>/dev/null >&2 exit 1 fi -log "module build produced ${bundle_count} bundles" +log "all ${EXPECTED_MODULES} common modules built" # -------------------------------------------------------------------- staging log "staging install tree" @@ -356,7 +393,7 @@ staged_r="$STAGE_DIR$PREFIX/R/lib/R/lib" for lib in libR.so libRblas.so libRlapack.so; do [[ -f "$staged_r/$lib" ]] || { echo "error: staged tree lacks $lib" >&2; exit 1; } done -staged_bundles="$(find "$STAGE_DIR$PREFIX/Modules" -mindepth 1 -maxdepth 1 2>/dev/null | wc -l)" +staged_bundles="$(find "$STAGE_DIR$PREFIX/Modules" -maxdepth 1 -name '*.JASPModule' 2>/dev/null | wc -l)" [[ "$staged_bundles" -gt 0 ]] || { echo "error: staged tree has no module bundles" >&2; exit 1; } staged_so="$(find "$STAGE_DIR$PREFIX" -name '*.so' 2>/dev/null | wc -l)" log "staged: ${staged_bundles} module bundles, ${staged_so} shared objects"