Build only the modules JASP ships, not the developer helper's 22
Modules/modules-settings.json splits modules into 'common' - the seven bundled with the application - and 'extra', which JASP fetches on demand into ~/.local/share/JASP. buildAllDefaultJaspModules.R builds 22 regardless: it is a developer convenience for having everything locally, not a definition of what a release contains. I had been treating it as the latter. The cost of that mistake was not just size. Most of the blockers this build hit came from modules that are never shipped: gsl blocked jaspNetwork and jaspBFF, OpenMx blocked jaspSem - all 'extra'. Building the right list drops them entirely. Measured: the seven common modules come to ~2.3 GB against ~8.3 GB for all 22. Bundles are already zstd-compressed, so the tarball does not shrink them further; ~2.5-3 GB total is the realistic figure, against the Flatpak's 1.4 GB. The remaining gap is bundleAll=TRUE giving every module its own copy of its R dependencies. The guard now names the modules it expects and reports which are missing.
This commit is contained in:
+49
-12
@@ -52,7 +52,8 @@ pacman -Syu --noconfirm --needed \
|
||||
boost boost-libs glpk jsoncpp libarchive openssl curl sqlite zlib \
|
||||
readstat netcdf mpfr cairo pango fontconfig harfbuzz fribidi icu pcre2 \
|
||||
readline bzip2 xz libpng libtiff libjpeg-turbo minizip \
|
||||
libxkbcommon libxkbcommon-x11 libxt wget unzip python
|
||||
libxkbcommon libxkbcommon-x11 libxt wget unzip python \
|
||||
gsl libsodium gmp nlopt libxml2 fftw
|
||||
|
||||
# Stock makepkg.conf leaves MAKEFLAGS at -j1, which makes the AUR dependencies
|
||||
# (JAGS especially) crawl.
|
||||
@@ -269,12 +270,46 @@ R_JOBS="${R_JOBS:-2}"
|
||||
export MAKEFLAGS="-j${R_JOBS}"
|
||||
log "compiling R packages with -j${R_JOBS} (OpenMx needs the headroom)"
|
||||
|
||||
EXPECTED_MODULES=22
|
||||
# Build only the modules JASP actually ships. Modules/modules-settings.json
|
||||
# splits them into "common" (bundled with the application) and "extra"
|
||||
# (fetched on demand at runtime, into ~/.local/share/JASP).
|
||||
#
|
||||
# buildAllDefaultJaspModules.R is a *developer* helper that builds 22 modules
|
||||
# regardless of that split — its name says "all default modules", not "what is
|
||||
# released". Building from it produces a ~9 GB tree, most of it modules the user
|
||||
# would have downloaded on demand anyway, and drags in system dependencies that
|
||||
# only the extras need (gsl for jaspNetwork, OpenMx for jaspSem).
|
||||
mapfile -t COMMON_MODULES < <(
|
||||
python3 -c "
|
||||
import json
|
||||
print('\n'.join(json.load(open('../Modules/modules-settings.json'))['common']))
|
||||
"
|
||||
)
|
||||
log "shipping the ${#COMMON_MODULES[@]} common modules: ${COMMON_MODULES[*]}"
|
||||
|
||||
mkdir -p ModuleBundleBuildDir/Modules
|
||||
for module in "${COMMON_MODULES[@]}"; do
|
||||
if [[ ! -d "ModuleBundleBuildDir/Modules/${module}" ]]; then
|
||||
log "cloning ${module}"
|
||||
git clone --depth 1 "https://github.com/jasp-stats/${module}" \
|
||||
"ModuleBundleBuildDir/Modules/${module}"
|
||||
fi
|
||||
done
|
||||
|
||||
EXPECTED_MODULES="${#COMMON_MODULES[@]}"
|
||||
MODULE_PATHS=()
|
||||
for module in "${COMMON_MODULES[@]}"; do
|
||||
MODULE_PATHS+=("ModuleBundleBuildDir/Modules/${module}")
|
||||
done
|
||||
|
||||
for attempt in 1 2 3; do
|
||||
log "module build pass ${attempt}/3"
|
||||
Rscript buildAllDefaultJaspModules.R || true
|
||||
built="$(find "$MODULE_BUNDLES" -mindepth 1 -maxdepth 1 2>/dev/null | wc -l)"
|
||||
log "pass ${attempt} finished: ${built}/${EXPECTED_MODULES} bundles"
|
||||
./buildModuleBundlesLocally.R "${MODULE_PATHS[@]}" || true
|
||||
built=0
|
||||
for module in "${COMMON_MODULES[@]}"; do
|
||||
[[ -f "${MODULE_BUNDLES}/${module}.JASPModule" ]] && built=$((built + 1))
|
||||
done
|
||||
log "pass ${attempt} finished: ${built}/${EXPECTED_MODULES} common modules"
|
||||
[[ "$built" -ge "$EXPECTED_MODULES" ]] && break
|
||||
done
|
||||
|
||||
@@ -287,15 +322,17 @@ fi
|
||||
|
||||
# Guard on the expected count, not merely "non-empty". An earlier run shipped a
|
||||
# package containing 2 of 22 modules because a non-empty check passed.
|
||||
bundle_count="$(find "$MODULE_BUNDLES" -mindepth 1 -maxdepth 1 2>/dev/null | wc -l)"
|
||||
if [[ "$bundle_count" -lt "$EXPECTED_MODULES" ]]; then
|
||||
echo "error: only ${bundle_count}/${EXPECTED_MODULES} module bundles were built" >&2
|
||||
echo " system2() in buildAllDefaultJaspModules.R hides the real error;" >&2
|
||||
missing=()
|
||||
for module in "${COMMON_MODULES[@]}"; do
|
||||
[[ -f "${MODULE_BUNDLES}/${module}.JASPModule" ]] || missing+=("$module")
|
||||
done
|
||||
if [[ ${#missing[@]} -gt 0 ]]; then
|
||||
echo "error: ${#missing[@]} of ${EXPECTED_MODULES} common modules were not built" >&2
|
||||
printf ' missing: %s\n' "${missing[@]}" >&2
|
||||
echo " search the log above for 'failed to install'" >&2
|
||||
find "$MODULE_BUNDLES" -mindepth 1 -maxdepth 1 -printf ' have: %f\n' 2>/dev/null >&2
|
||||
exit 1
|
||||
fi
|
||||
log "module build produced ${bundle_count} bundles"
|
||||
log "all ${EXPECTED_MODULES} common modules built"
|
||||
|
||||
# -------------------------------------------------------------------- staging
|
||||
log "staging install tree"
|
||||
@@ -356,7 +393,7 @@ staged_r="$STAGE_DIR$PREFIX/R/lib/R/lib"
|
||||
for lib in libR.so libRblas.so libRlapack.so; do
|
||||
[[ -f "$staged_r/$lib" ]] || { echo "error: staged tree lacks $lib" >&2; exit 1; }
|
||||
done
|
||||
staged_bundles="$(find "$STAGE_DIR$PREFIX/Modules" -mindepth 1 -maxdepth 1 2>/dev/null | wc -l)"
|
||||
staged_bundles="$(find "$STAGE_DIR$PREFIX/Modules" -maxdepth 1 -name '*.JASPModule' 2>/dev/null | wc -l)"
|
||||
[[ "$staged_bundles" -gt 0 ]] || { echo "error: staged tree has no module bundles" >&2; exit 1; }
|
||||
staged_so="$(find "$STAGE_DIR$PREFIX" -name '*.so' 2>/dev/null | wc -l)"
|
||||
log "staged: ${staged_bundles} module bundles, ${staged_so} shared objects"
|
||||
|
||||
Reference in New Issue
Block a user