agw
8f5bd0953f
JASP 0.98.8 requires the Qt NetworkAuth component, so the build installs qt6-networkauth and the package depends on it at runtime. Without it CMake stopped at configure and the run failed every evening. The scheduled run now starts at 22:00 Danish time instead of 18:00, so the hours-long build no longer competes with evening use of the runner host. Claude-Session: https://claude.ai/code/session_01JmhJG6JnhXfzNiKv8f2vHu
652 lines
28 KiB
Bash
Executable File
652 lines
28 KiB
Bash
Executable File
#!/bin/bash
|
|
# Build JASP Desktop on Arch Linux and emit a relocatable binary tarball.
|
|
#
|
|
# Runs as root inside an `archlinux:base-devel` container. The resulting tarball
|
|
# is what the AUR `jasp-desktop-bin` package downloads and unpacks.
|
|
#
|
|
# JASP_TAG upstream tag to build, e.g. v0.98.1 (required)
|
|
# OUT_DIR where the tarball + metadata land (default /work/out)
|
|
# JOBS compile parallelism (default nproc)
|
|
# R_VERSION R to build and bundle (default 4.5.3)
|
|
# GITHUB_PAT token for R package downloads from GitHub (optional but
|
|
# strongly recommended — several JASP modules are installed from
|
|
# GitHub and anonymous API rate limits will fail the build)
|
|
#
|
|
# Why we build our own R instead of using Arch's:
|
|
#
|
|
# JASP pins its R package set with renv lockfiles against a frozen CRAN snapshot
|
|
# (Modules/Rcpp_RInside.lock points at the 2026-03-31 Posit snapshot for 0.98.1).
|
|
# That set targets R 4.5. Building against Arch's R 4.6 fails immediately —
|
|
# Rcpp 1.1.1 does not compile there (R_NamespaceRegistry is gone in R 4.6), and
|
|
# the pinned snapshot predates the Rcpp release that fixes it. Upstream's own CI
|
|
# installs a specific R and points R_HOME at it, which is what we do here.
|
|
#
|
|
# Bundling R also means the package does not depend on Arch's `r` at all, so an
|
|
# Arch R bump can no longer break an installed JASP.
|
|
|
|
set -euo pipefail
|
|
|
|
JASP_TAG="${JASP_TAG:?JASP_TAG is required, e.g. v0.98.1}"
|
|
OUT_DIR="${OUT_DIR:-/work/out}"
|
|
JOBS="${JOBS:-$(nproc)}"
|
|
R_VERSION="${R_VERSION:-4.5.3}"
|
|
|
|
SRC_DIR=/build/jasp-desktop
|
|
BUILD_DIR=/build/build
|
|
STAGE_DIR=/build/stage
|
|
PREFIX=/opt/jasp-desktop
|
|
R_HOME_DIR="${PREFIX}/R"
|
|
|
|
mkdir -p "$OUT_DIR" /build
|
|
|
|
# Take the GitHub token out of the environment until the R module build, the
|
|
# only step that needs it. JASP's Tools/CMake/Config.cmake bakes GITHUB_PAT into
|
|
# the application (Desktop/gui/githubpat.h, the default token for its update
|
|
# check) whenever GITHUB_PAT_DEF is empty, so a token visible at configure time
|
|
# ships inside bin/JASP of the public tarball. That happened once and leaked the
|
|
# build token. GITHUB_PAT_DEF is pinned empty for the same reason.
|
|
BUILD_GITHUB_PAT="${GITHUB_PAT:-}"
|
|
unset GITHUB_PAT GITHUB_TOKEN
|
|
export GITHUB_PAT_DEF=""
|
|
|
|
log() { echo ":: $* "; }
|
|
|
|
# ------------------------------------------------------------- build caches
|
|
# /cache is a directory on the runner host that outlives the container. Without
|
|
# it every build recompiles R, JASP and 1000+ R packages from nothing, which is
|
|
# days of CPU on the runner; with it a rebuild only compiles what changed.
|
|
# /cache/ccache compiler cache for R and JASP's own C++
|
|
# /cache/renv-cache compiled R packages, shared by every module
|
|
# /cache/renv-root renv's download cache (source tarballs)
|
|
CACHE_DIR="${CACHE_DIR:-/cache}"
|
|
mkdir -p "$CACHE_DIR"/{ccache,renv-cache,renv-root}
|
|
export CCACHE_DIR="$CACHE_DIR/ccache"
|
|
export CCACHE_MAXSIZE="${CCACHE_MAXSIZE:-15G}"
|
|
# renv files its cache under the OS release, and the archlinux image stamps its
|
|
# build date there (linux-arch-20260809.0.570793), so every newer image would
|
|
# start from an empty cache. A rolling distro has no release to key on; staleness
|
|
# is handled by purge_stale_renv_cache below instead.
|
|
export RENV_PATHS_PREFIX=linux-arch
|
|
|
|
# ---------------------------------------------------------------- system deps
|
|
log "installing Arch dependencies"
|
|
pacman -Syu --noconfirm --needed \
|
|
base-devel git cmake ninja gcc gcc-fortran make pkgconf patchelf \
|
|
autoconf automake libtool bison flex gettext blas lapack ccache \
|
|
qt6-base qt6-declarative qt6-svg qt6-positioning qt6-webchannel \
|
|
qt6-webengine qt6-5compat qt6-shadertools qt6-websockets qt6-tools \
|
|
qt6-httpserver qt6-networkauth vulkan-headers \
|
|
boost boost-libs glpk jsoncpp libarchive openssl curl sqlite zlib \
|
|
readstat netcdf mpfr cairo pango fontconfig harfbuzz fribidi icu pcre2 \
|
|
readline bzip2 xz libpng libtiff libjpeg-turbo minizip \
|
|
libxkbcommon libxkbcommon-x11 libxt wget unzip python \
|
|
gsl libsodium gmp nlopt libxml2 fftw
|
|
|
|
# Stock makepkg.conf leaves MAKEFLAGS at -j1, which makes the AUR dependencies
|
|
# (JAGS especially) crawl.
|
|
sed -i "s|^#\?MAKEFLAGS=.*|MAKEFLAGS=\"-j${JOBS}\"|" /etc/makepkg.conf
|
|
export MAKEFLAGS="-j${JOBS}"
|
|
|
|
# makepkg refuses to run as root.
|
|
if ! id builder &>/dev/null; then
|
|
useradd -m builder
|
|
echo "builder ALL=(ALL) NOPASSWD: ALL" >/etc/sudoers.d/builder
|
|
fi
|
|
|
|
# jags and librdata are AUR-only; build them the same way a user would.
|
|
build_aur() {
|
|
local pkg="$1"
|
|
log "building AUR package $pkg"
|
|
sudo -u builder bash -euc "
|
|
cd /home/builder
|
|
rm -rf '$pkg'
|
|
git clone --depth 1 'https://aur.archlinux.org/$pkg.git'
|
|
cd '$pkg'
|
|
makepkg -si --noconfirm --needed
|
|
"
|
|
}
|
|
build_aur jags
|
|
build_aur librdata-git
|
|
|
|
# JASP requires freexl >= 2.0.99; Arch ships 2.0.0, so use upstream's own fork
|
|
# exactly as the JASP CI does.
|
|
log "building jasp-stats/freexl"
|
|
rm -rf /build/freexl
|
|
git clone --depth 1 https://github.com/jasp-stats/freexl.git /build/freexl
|
|
cd /build/freexl
|
|
./configure --prefix=/usr
|
|
make -j"$JOBS"
|
|
make install
|
|
|
|
# ------------------------------------------------------- R that we ship along
|
|
log "building R ${R_VERSION} into ${R_HOME_DIR}"
|
|
rm -rf "/build/R-${R_VERSION}"
|
|
cd /build
|
|
wget -q "https://cran.r-project.org/src/base/R-4/R-${R_VERSION}.tar.gz"
|
|
tar -xzf "R-${R_VERSION}.tar.gz"
|
|
cd "R-${R_VERSION}"
|
|
|
|
# --enable-R-shlib is required: JASP embeds R through RInside/libR.
|
|
# --libdir is pinned so R lands in lib/R rather than the lib64/R that Arch's
|
|
# config.site would otherwise pick, which keeps the rpath below predictable.
|
|
# The install prefix equals the final install path, so the rpath stays valid
|
|
# once the tarball is unpacked on the user's machine.
|
|
#
|
|
# --without-blas/--without-lapack is deliberate and load-bearing. With
|
|
# --with-blas, R links Arch's external BLAS and does NOT build libRblas.so or
|
|
# libRlapack.so. The JASP module bundles expect those files to exist inside
|
|
# R_HOME (that is how a conventionally built R, and the Flatpak, ship), so
|
|
# linking externally leaves the modules unable to resolve them. Reference BLAS
|
|
# is slower than OpenBLAS, but a working install beats a fast broken one.
|
|
./configure \
|
|
--prefix="${R_HOME_DIR}" \
|
|
--libdir="${R_HOME_DIR}/lib" \
|
|
--enable-R-shlib \
|
|
--enable-memory-profiling \
|
|
--without-blas \
|
|
--without-lapack \
|
|
--with-x=no \
|
|
--with-tcltk=no \
|
|
LDFLAGS="-Wl,-rpath,${R_HOME_DIR}/lib/R/lib"
|
|
# ccache through PATH rather than CC="ccache gcc": configure would bake the
|
|
# latter into the shipped Makeconf, and every package a user installs into
|
|
# JASP's R would then need ccache.
|
|
PATH="/usr/lib/ccache/bin:$PATH" make -j"$JOBS"
|
|
make install
|
|
|
|
# Guard the above: a silently external-BLAS build produces a JASP whose modules
|
|
# fail at runtime, which is exactly the class of failure this package exists to
|
|
# avoid. Better to stop here than ship it.
|
|
for lib in libR.so libRblas.so libRlapack.so; do
|
|
if [[ ! -f "${R_HOME_DIR}/lib/R/lib/${lib}" ]]; then
|
|
echo "error: ${lib} missing from the bundled R — R linked an external BLAS?" >&2
|
|
ls -la "${R_HOME_DIR}/lib/R/lib/" >&2
|
|
exit 1
|
|
fi
|
|
done
|
|
log "bundled R provides its own libR/libRblas/libRlapack"
|
|
|
|
export PATH="${R_HOME_DIR}/bin:${PATH}"
|
|
# Ask R where it actually put itself rather than assuming lib vs lib64.
|
|
# Deliberately NOT exported: R.cmake notes that a set R_HOME makes R print
|
|
# "WARNING: ignoring environment value of R_HOME", which corrupts the output it
|
|
# parses. CUSTOM_R_PATH is the supported way to point JASP at a specific R.
|
|
JASP_R_HOME="$("${R_HOME_DIR}/bin/R" RHOME)"
|
|
log "bundled R_HOME is ${JASP_R_HOME}"
|
|
"${R_HOME_DIR}/bin/R" --version | sed -n 1p || true
|
|
|
|
# ------------------------------------------------------------------- checkout
|
|
log "cloning jasp-desktop $JASP_TAG"
|
|
rm -rf "$SRC_DIR"
|
|
git clone --depth 1 --branch "$JASP_TAG" --recurse-submodules --shallow-submodules \
|
|
https://github.com/jasp-stats/jasp-desktop.git "$SRC_DIR"
|
|
|
|
cd "$SRC_DIR"
|
|
JASP_COMMIT="$(git rev-parse HEAD)"
|
|
|
|
# ---------------------------------------------------------------------- build
|
|
log "configuring"
|
|
export CMAKE_C_COMPILER_LAUNCHER=ccache
|
|
export CMAKE_CXX_COMPILER_LAUNCHER=ccache
|
|
export CMAKE_Fortran_COMPILER_LAUNCHER=ccache
|
|
export CFLAGS="${CFLAGS:-} -fPIC"
|
|
export CXXFLAGS="${CXXFLAGS:-} -fPIC"
|
|
export FCFLAGS="${FCFLAGS:-} -fPIC"
|
|
export FFLAGS="${FFLAGS:-} -fPIC"
|
|
|
|
rm -rf "$BUILD_DIR"
|
|
cmake -S "$SRC_DIR" -B "$BUILD_DIR" -G Ninja \
|
|
-DCMAKE_BUILD_TYPE=Release \
|
|
-DCMAKE_INSTALL_PREFIX="$PREFIX" \
|
|
-DLINUX_LOCAL_BUILD=OFF \
|
|
-DCUSTOM_R_PATH="${JASP_R_HOME}" \
|
|
-Wno-dev
|
|
|
|
log "compiling JASP (-j$JOBS)"
|
|
ninja -C "$BUILD_DIR" -j"$JOBS"
|
|
|
|
# The R modules are the long pole: several hundred R packages compiled from
|
|
# source. build/build-modules.R reports a failed module and carries on rather
|
|
# than exiting non-zero, so the guards after it are what actually catch a
|
|
# broken module build.
|
|
#
|
|
# GITHUB_PAT is required, not optional: jaspModuleTools is installed with
|
|
# install_github('jasp-stats/jaspModuleTools'), and anonymous GitHub API rate
|
|
# limits make that fail. When it fails, no bundles are produced at all and the
|
|
# build otherwise looks successful.
|
|
GITHUB_PAT="${BUILD_GITHUB_PAT}"
|
|
if [[ -z "${GITHUB_PAT:-}" ]]; then
|
|
echo "error: GITHUB_PAT is required — jaspModuleTools is installed from GitHub" >&2
|
|
echo " without it the module build silently produces nothing" >&2
|
|
exit 1
|
|
fi
|
|
export GITHUB_PAT
|
|
export GITHUB_TOKEN="${GITHUB_PAT}"
|
|
|
|
log "building R modules — this is the slow part"
|
|
mkdir -p "$BUILD_DIR"/Modules/{binary_pkgs,manifests,module_libs,Tools}
|
|
cd "$SRC_DIR/Tools"
|
|
|
|
# jaspModuleTools is installed from GitHub with `remotes`, which devtools >= 2.5
|
|
# no longer pulls in ("The package \"remotes\" is required."). Pre-seed it into
|
|
# the library build-modules.R uses, since that does .libPaths(new=userLibrary)
|
|
# and would not see remotes anywhere else.
|
|
MODULE_LIBRARY="$SRC_DIR/Tools/ModuleBundleBuildDir/LibraryR"
|
|
MODULE_BUNDLES="$SRC_DIR/Modules/local"
|
|
|
|
log "pre-installing remotes into the module build library"
|
|
mkdir -p "$MODULE_LIBRARY"
|
|
Rscript -e "
|
|
lib <- '${MODULE_LIBRARY}'
|
|
.libPaths(new = lib)
|
|
install.packages('remotes', lib = lib, repos = 'https://cran.r-project.org')
|
|
if (!requireNamespace('remotes', lib.loc = lib, quietly = TRUE))
|
|
stop('failed to install remotes')
|
|
"
|
|
|
|
|
|
# renv restores packages concurrently and its ordering is not always
|
|
# dependency-correct: a module can be lazy-loaded while a package it needs is
|
|
# still being built, e.g. "forecast" failing with "there is no package called
|
|
# 'vctrs'" six log lines before vctrs finishes compiling. That aborts the whole
|
|
# sapply() loop, so a single race costs every remaining module.
|
|
#
|
|
# Retrying is the cheap fix: renv skips what is already installed, so each pass
|
|
# starts from more complete state and converges. Passes are capped so a genuine
|
|
# failure cannot loop forever.
|
|
# OpenMx instantiates enough Eigen templates that a single cc1plus can take
|
|
# several GB. At the top-level -j the compiler gets OOM-killed
|
|
# ("g++: fatal error: Killed signal terminated program cc1plus"), which is not
|
|
# a race — it stalls at the same module count on every retry. Compile R packages
|
|
# with less parallelism than the C++ build uses.
|
|
R_JOBS="${R_JOBS:-2}"
|
|
export MAKEFLAGS="-j${R_JOBS}"
|
|
|
|
# R's default flags are -g -O2. Debug info in the Eigen/Stan-heavy packages
|
|
# (rstanarm, OpenMx) costs compile time and GBs of cc1plus memory, and nobody
|
|
# debugs them. User Makevars only affects this build, not the shipped Makeconf.
|
|
mkdir -p ~/.R
|
|
cat >~/.R/Makevars <<'EOF'
|
|
CFLAGS=-O2 -pipe
|
|
CXXFLAGS=-O2 -pipe
|
|
CXX11FLAGS=-O2 -pipe
|
|
CXX14FLAGS=-O2 -pipe
|
|
CXX17FLAGS=-O2 -pipe
|
|
CXX20FLAGS=-O2 -pipe
|
|
FFLAGS=-O2 -pipe
|
|
FCFLAGS=-O2 -pipe
|
|
EOF
|
|
|
|
# jaspModuleTools::compile() hard-codes RENV_PATHS_CACHE and RENV_PATHS_ROOT to
|
|
# directories under its workdir, so the persistent cache is linked in there.
|
|
MODULE_WORKDIR="$SRC_DIR/Tools/ModuleBundleBuildDir/build_dir"
|
|
mkdir -p "$MODULE_WORKDIR"
|
|
ln -sfn "$CACHE_DIR/renv-cache" "$MODULE_WORKDIR/renv-cache"
|
|
ln -sfn "$CACHE_DIR/renv-root" "$MODULE_WORKDIR/renv-root"
|
|
|
|
# A cached package is only reusable while the system libraries it linked
|
|
# against still exist. Arch rolls (an ICU or gsl soname bump), and renv's cache
|
|
# key knows nothing about that, so drop every entry with an unresolved library
|
|
# and let renv rebuild just those. Layout: .../<pkg>/<ver>/<hash>/<pkg>/libs/*.so
|
|
purge_stale_renv_cache() {
|
|
local so entry purged=0
|
|
while IFS= read -r so; do
|
|
if LD_LIBRARY_PATH="${JASP_R_HOME}/lib" ldd "$so" 2>/dev/null | grep -q 'not found'; then
|
|
entry="$(dirname "$(dirname "$(dirname "$so")")")"
|
|
[[ -d "$entry" ]] || continue
|
|
rm -rf "$entry"
|
|
purged=$((purged + 1))
|
|
fi
|
|
done < <(find "$CACHE_DIR/renv-cache/" -path '*/libs/*.so' 2>/dev/null)
|
|
log "renv cache: purged ${purged} entries with missing system libraries"
|
|
}
|
|
purge_stale_renv_cache
|
|
|
|
# The R package V8 (jaspPower, via gt and juicyjuice) needs libv8, which Arch
|
|
# does not package; this makes its configure fetch upstream's static build.
|
|
export DOWNLOAD_STATIC_LIBV8=1
|
|
|
|
# ccache for R package compiles too: cheap insurance for when renv reinstalls a
|
|
# package whose compiler inputs did not actually change.
|
|
export PATH="/usr/lib/ccache/bin:$PATH"
|
|
# R leaves 00LOCK-<pkg> directories behind when an install is interrupted, and
|
|
# refuses to touch the library afterwards ("failed to lock directory ... for
|
|
# modifying"). A build that is killed mid-install - a timeout, an OOM, a stopped
|
|
# container - therefore poisons the library for every later run.
|
|
log "clearing stale R install locks"
|
|
rm -rf "${MODULE_LIBRARY}"/00LOCK-* 2>/dev/null || true
|
|
|
|
# The interrupted install can also leave the library without devtools, which
|
|
# the jaspModuleTools install relies on. Reinstall whatever is
|
|
# missing rather than assuming a clean slate.
|
|
Rscript -e "
|
|
lib <- '${MODULE_LIBRARY}'
|
|
dir.create(lib, recursive = TRUE, showWarnings = FALSE)
|
|
.libPaths(new = lib)
|
|
for (pkg in c('remotes', 'devtools')) {
|
|
if (!requireNamespace(pkg, lib.loc = lib, quietly = TRUE)) {
|
|
message('installing missing ', pkg)
|
|
install.packages(pkg, lib = lib, repos = 'https://cran.r-project.org')
|
|
}
|
|
if (!requireNamespace(pkg, lib.loc = lib, quietly = TRUE))
|
|
stop('failed to install ', pkg)
|
|
}
|
|
"
|
|
|
|
log "compiling R packages with -j${R_JOBS} (OpenMx needs the headroom)"
|
|
|
|
# Build only the modules JASP actually ships. Modules/modules-settings.json
|
|
# splits them into "common" (bundled with the application) and "extra"
|
|
# (fetched on demand at runtime, into ~/.local/share/JASP).
|
|
#
|
|
# buildAllDefaultJaspModules.R is a *developer* helper that builds 22 modules
|
|
# regardless of that split — its name says "all default modules", not "what is
|
|
# released". Building from it produces a ~9 GB tree, most of it modules the user
|
|
# would have downloaded on demand anyway, and drags in system dependencies that
|
|
# only the extras need (gsl for jaspNetwork, OpenMx for jaspSem).
|
|
# Build every module JASP ships a definition for, not just the "common" seven.
|
|
#
|
|
# The extras are normally downloaded at runtime — but those bundles are built
|
|
# against the Flatpak runtime, and on Arch they fail on sonames that do not
|
|
# exist here (the first one hit was libjpeg.so.62, which Arch replaced with
|
|
# libjpeg.so.8 long ago). A downloaded module is therefore never reliable in
|
|
# this package; anything the user might enable has to be built against the same
|
|
# Arch userland as the application itself.
|
|
#
|
|
# common modules are mandatory: without them the package is not JASP.
|
|
# extras are best-effort: a few need dependencies that are not packaged for
|
|
# Arch at all (jaspProcess wants V8), and losing one of those is better than
|
|
# losing the build.
|
|
mapfile -t COMMON_MODULES < <(
|
|
python3 -c "
|
|
import json
|
|
print('\n'.join(json.load(open('../Modules/modules-settings.json'))['common']))
|
|
"
|
|
)
|
|
mapfile -t EXTRA_MODULES < <(
|
|
python3 -c "
|
|
import json
|
|
s = json.load(open('../Modules/modules-settings.json'))
|
|
print('\n'.join(m for m in s['extra'] if m != 'jaspTestModule'))
|
|
"
|
|
)
|
|
ALL_MODULES=("${COMMON_MODULES[@]}" "${EXTRA_MODULES[@]}")
|
|
log "building ${#ALL_MODULES[@]} modules (${#COMMON_MODULES[@]} common + ${#EXTRA_MODULES[@]} extra)"
|
|
|
|
mkdir -p ModuleBundleBuildDir/Modules
|
|
for module in "${ALL_MODULES[@]}"; do
|
|
if [[ ! -d "ModuleBundleBuildDir/Modules/${module}" ]]; then
|
|
git clone --depth 1 "https://github.com/jasp-stats/${module}" \
|
|
"ModuleBundleBuildDir/Modules/${module}" 2>/dev/null || log "no repo for ${module}, skipping"
|
|
fi
|
|
done
|
|
|
|
# This has to run after the modules are cloned: it once ran before, found no
|
|
# lockfiles, patched nothing, and every module then failed on later 1.4.6.
|
|
#
|
|
# Some of the versions JASP pins in its module renv.lock files predate the
|
|
# toolchain on a current Arch and cannot compile. Bump only those, by name, with
|
|
# the reason recorded — never a blanket "use latest", which would silently
|
|
# change the analysis code users get.
|
|
#
|
|
# later 1.4.6 bundles a tinycthread whose `#define once_flag pthread_once_t`
|
|
# collides with the C11 once_flag that glibc >= 2.43 now declares under
|
|
# C23, so it fails with "conflicting types for pthread_once_t".
|
|
# Fixed upstream in later 1.4.7 (r-lib/later#257). The 2026-04-22 RSPM
|
|
# snapshot the lockfiles already reference carries 1.4.8.
|
|
log "patching module lockfile pins that cannot build on this toolchain"
|
|
python3 - <<'PY'
|
|
import json, pathlib
|
|
|
|
OVERRIDES = {"later": "1.4.8"}
|
|
|
|
root = pathlib.Path("ModuleBundleBuildDir/Modules")
|
|
patched = 0
|
|
for lock in root.glob("*/renv.lock"):
|
|
data = json.loads(lock.read_text())
|
|
packages = data.get("Packages", {})
|
|
changed = False
|
|
for name, want in OVERRIDES.items():
|
|
pkg = packages.get(name)
|
|
if pkg and pkg.get("Version") != want:
|
|
print(f" {lock.parent.name}: {name} {pkg['Version']} -> {want}")
|
|
pkg["Version"] = want
|
|
# The recorded hash no longer matches the bumped version.
|
|
pkg.pop("Hash", None)
|
|
changed = True
|
|
if changed:
|
|
lock.write_text(json.dumps(data, indent=2) + "\n")
|
|
patched += 1
|
|
print(f"patched {patched} lockfiles")
|
|
if not any(root.glob("*/renv.lock")):
|
|
raise SystemExit("error: no module lockfiles to patch - modules not cloned yet?")
|
|
PY
|
|
|
|
for attempt in 1 2 3; do
|
|
# Hand over only modules without a bundle yet: the script rebuilds everything
|
|
# it is given, so a retry would otherwise redo every finished module.
|
|
MODULE_PATHS=()
|
|
for module in "${ALL_MODULES[@]}"; do
|
|
[[ -f "${MODULE_BUNDLES}/${module}.JASPModule" ]] && continue
|
|
[[ -d "ModuleBundleBuildDir/Modules/${module}" ]] && \
|
|
MODULE_PATHS+=("ModuleBundleBuildDir/Modules/${module}")
|
|
done
|
|
[[ ${#MODULE_PATHS[@]} -eq 0 ]] && break
|
|
log "module build pass ${attempt}/3: ${#MODULE_PATHS[@]} modules left"
|
|
# One R process per module under a time limit. A module once spun at 100%
|
|
# CPU for 8 hours inside jaspModuleTools' bundling step, after it had
|
|
# installed fine, and stalled the whole build; the same module bundled in two
|
|
# minutes on the previous run. A timed-out module is simply retried in the
|
|
# next pass.
|
|
for path in "${MODULE_PATHS[@]}"; do
|
|
timeout --kill-after=60 "${MODULE_TIMEOUT:-5400}" \
|
|
Rscript /work/build/build-modules.R "$path" \
|
|
|| { rc=$?; log "$(basename "$path") did not finish (exit $rc$([[ $rc -eq 124 ]] && echo ', timed out')), retrying next pass"; }
|
|
done
|
|
built=0
|
|
for module in "${ALL_MODULES[@]}"; do
|
|
[[ -f "${MODULE_BUNDLES}/${module}.JASPModule" ]] && built=$((built + 1))
|
|
done
|
|
log "pass ${attempt} finished: ${built}/${#ALL_MODULES[@]} modules"
|
|
[[ "$built" -ge "${#ALL_MODULES[@]}" ]] && break
|
|
done
|
|
|
|
if [[ ! -d "$MODULE_LIBRARY/jaspModuleTools" ]]; then
|
|
echo "error: jaspModuleTools was not installed — install_github failed" >&2
|
|
echo " check the install_github output above: a missing 'remotes'," >&2
|
|
echo " a GitHub rate limit, or a PAT without read access all do this" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Guard on the expected count, not merely "non-empty". An earlier run shipped a
|
|
# package containing 2 of 22 modules because a non-empty check passed.
|
|
missing_common=()
|
|
for module in "${COMMON_MODULES[@]}"; do
|
|
[[ -f "${MODULE_BUNDLES}/${module}.JASPModule" ]] || missing_common+=("$module")
|
|
done
|
|
if [[ ${#missing_common[@]} -gt 0 ]]; then
|
|
echo "error: mandatory common modules were not built" >&2
|
|
printf ' missing: %s\n' "${missing_common[@]}" >&2
|
|
exit 1
|
|
fi
|
|
|
|
missing_extra=()
|
|
for module in "${EXTRA_MODULES[@]}"; do
|
|
[[ -f "${MODULE_BUNDLES}/${module}.JASPModule" ]] || missing_extra+=("$module")
|
|
done
|
|
built_total=$(find "$MODULE_BUNDLES" -maxdepth 1 -name '*.JASPModule' | wc -l)
|
|
log "built ${built_total} modules; all ${#COMMON_MODULES[@]} common present"
|
|
if [[ ${#missing_extra[@]} -gt 0 ]]; then
|
|
log "extras that did not build (users will fall back to downloading these):"
|
|
printf ' %s\n' "${missing_extra[@]}"
|
|
fi
|
|
|
|
# -------------------------------------------------------------------- staging
|
|
log "staging install tree"
|
|
rm -rf "$STAGE_DIR"
|
|
mkdir -p "$STAGE_DIR"
|
|
DESTDIR="$STAGE_DIR" ninja -C "$BUILD_DIR" install
|
|
|
|
# The bundled R was installed straight to $PREFIX/R on the build host; copy it
|
|
# into the staged tree so it ends up in the tarball.
|
|
install -d "$STAGE_DIR$PREFIX"
|
|
cp -a "$R_HOME_DIR" "$STAGE_DIR$PREFIX/"
|
|
|
|
# Everything the module build produced has to travel with the tarball, or JASP
|
|
# falls back to downloading bundles at runtime — which is the failure mode this
|
|
# package is meant to remove.
|
|
#
|
|
# Modules/local the compiled module bundles themselves
|
|
#
|
|
# ModuleBundleBuildDir/LibraryR is deliberately NOT shipped: it is the *build*
|
|
# toolchain (devtools, roxygen2, testthat, pkgdown, rcmdcheck, jaspModuleTools),
|
|
# not anything JASP loads at runtime. Each .JASPModule already carries its own
|
|
# dependency closure — jaspTTests alone unpacks to 139 content-addressed package
|
|
# directories — so the library would be 283 MB of pure duplication.
|
|
#
|
|
# ModuleBundleBuildDir/Modules is deliberately NOT copied: it is the cloned
|
|
# module *sources* (no compiled objects at all), several hundred MB of git
|
|
# checkouts that would bloat the tarball without being usable at runtime.
|
|
install -d "$STAGE_DIR$PREFIX/Modules"
|
|
|
|
# Deliberately do NOT copy Modules/local/*.JASPModule here. `ninja install`
|
|
# has already placed the same packages in Modules/binary_pkgs as a
|
|
# deduplicated, content-addressed store — 677 package directories, in which
|
|
# Rcpp, ggplot2 and the rest of the shared base appear once rather than once
|
|
# per module. The .JASPModule files are the *distribution* form used for
|
|
# on-demand downloads; shipping them alongside the installed form sends the
|
|
# same content twice and took the tarball from 1.86 GB to 4.3 GB.
|
|
|
|
# jaspModuleTools is installed from GitHub HEAD, which writes the manifest
|
|
# version as an array ([[0,96,5]]). JASP's InstalledModules::parseModuleInfo
|
|
# calls isMember() on it, which jsoncpp refuses for arrays; the exception is
|
|
# caught per manifest and logged only to the (off by default) log file, so
|
|
# every bundled module is silently dropped and JASP starts with none at all.
|
|
# A plain version string is the first form that parser accepts.
|
|
log "normalising module manifest versions"
|
|
python3 - "$STAGE_DIR$PREFIX/Modules/manifests" <<'PY'
|
|
import json, pathlib, sys
|
|
manifests = sorted(pathlib.Path(sys.argv[1]).glob("*_manifest.json"))
|
|
if not manifests:
|
|
sys.exit("error: no module manifests were staged")
|
|
for p in manifests:
|
|
d = json.loads(p.read_text())
|
|
v = d.get("version")
|
|
if isinstance(v, list):
|
|
parts = v[0] if v and isinstance(v[0], list) else v
|
|
d["version"] = ".".join(str(x) for x in parts)
|
|
p.write_text(json.dumps(d, indent=1) + "\n")
|
|
if not isinstance(d.get("version"), (str, dict)):
|
|
sys.exit(f"error: {p.name} has a version JASP cannot parse: {d.get('version')!r}")
|
|
print(f" {len(manifests)} manifests carry a version JASP can parse")
|
|
PY
|
|
|
|
|
|
install -Dm644 /work/packaging/org.jaspstats.JASP.desktop \
|
|
"$STAGE_DIR/usr/share/applications/org.jaspstats.JASP.desktop"
|
|
|
|
if [[ -d "$STAGE_DIR$PREFIX/share/icons/hicolor" ]]; then
|
|
install -d "$STAGE_DIR/usr/share/icons/hicolor"
|
|
cp -a "$STAGE_DIR$PREFIX/share/icons/hicolor/." "$STAGE_DIR/usr/share/icons/hicolor/"
|
|
fi
|
|
|
|
install -d "$STAGE_DIR/usr/share/licenses/jasp-desktop-bin"
|
|
cd "$SRC_DIR"
|
|
find . -maxdepth 2 \( -name 'COPYING*' -o -name 'LICENSE*' \) -type f \
|
|
-exec install -m644 {} "$STAGE_DIR/usr/share/licenses/jasp-desktop-bin/" \;
|
|
|
|
find "$STAGE_DIR" -name '*.o' -delete 2>/dev/null || true
|
|
|
|
# Make the tree independent of Arch's rolling libraries (ICU, jsoncpp, glpk,
|
|
# JAGS, readstat, librdata ...). Qt6 stays a system dependency; BLAS/LAPACK now
|
|
# come from the bundled R itself.
|
|
log "bundling non-system shared libraries"
|
|
bash /work/build/bundle-libs.sh "$STAGE_DIR" "$PREFIX"
|
|
|
|
# Final sanity check on what is about to be shipped. Each of these has already
|
|
# been shipped missing once.
|
|
log "verifying the staged tree"
|
|
# The token must never reach the package (see the top of this script). Check
|
|
# for its exact value and for anything shaped like a GitHub token.
|
|
if [[ -n "${BUILD_GITHUB_PAT}" ]] && grep -rlaF -- "${BUILD_GITHUB_PAT}" "$STAGE_DIR" >/dev/null 2>&1; then
|
|
echo "error: the build's GitHub token is embedded in the staged tree:" >&2
|
|
grep -rlaF -- "${BUILD_GITHUB_PAT}" "$STAGE_DIR" | sed "s|^$STAGE_DIR||" >&2
|
|
exit 1
|
|
fi
|
|
if grep -rlaE 'github_pat_[A-Za-z0-9_]{60,}|gh[pousr]_[A-Za-z0-9]{36}' "$STAGE_DIR/opt/jasp-desktop/bin" >/dev/null 2>&1; then
|
|
echo "error: something shaped like a GitHub token is in the JASP binaries" >&2
|
|
exit 1
|
|
fi
|
|
staged_r="$STAGE_DIR$PREFIX/R/lib/R/lib"
|
|
for lib in libR.so libRblas.so libRlapack.so; do
|
|
[[ -f "$staged_r/$lib" ]] || { echo "error: staged tree lacks $lib" >&2; exit 1; }
|
|
done
|
|
# Without the bundled R's lib dir on the rpath, libR.so falls through to the
|
|
# user's system R (Arch's `r` puts itself in ld.so.conf), whose libR lacks
|
|
# internals like utf8clen: grDevices fails to load and so does every module.
|
|
for bin in JASP JASPEngine; do
|
|
rp="$(patchelf --print-rpath "$STAGE_DIR$PREFIX/bin/$bin")"
|
|
[[ ":$rp:" == *":$PREFIX/R/lib/R/lib:"* ]] || {
|
|
echo "error: $bin rpath '$rp' lacks $PREFIX/R/lib/R/lib" >&2; exit 1; }
|
|
done
|
|
# Modules ship installed (module_libs/<module> over the binary_pkgs store), not
|
|
# as .JASPModule bundles, so check that every module that built got installed.
|
|
unstaged=()
|
|
for bundle in "$MODULE_BUNDLES"/*.JASPModule; do
|
|
module="$(basename "$bundle" .JASPModule)"
|
|
[[ -d "$STAGE_DIR$PREFIX/Modules/module_libs/$module" ]] || unstaged+=("$module")
|
|
done
|
|
if [[ ${#unstaged[@]} -gt 0 ]]; then
|
|
echo "error: built modules missing from the staged module_libs: ${unstaged[*]}" >&2
|
|
exit 1
|
|
fi
|
|
staged_modules="$(find "$STAGE_DIR$PREFIX/Modules/module_libs" -mindepth 1 -maxdepth 1 -type d | wc -l)"
|
|
staged_so="$(find "$STAGE_DIR$PREFIX" -name '*.so' 2>/dev/null | wc -l)"
|
|
log "staged: ${staged_modules} modules, ${staged_so} shared objects"
|
|
du -sh "$STAGE_DIR$PREFIX"/* 2>/dev/null | sort -rh | head -8
|
|
|
|
# ------------------------------------------------------------------- metadata
|
|
pkgver_of() { pacman -Q "$1" 2>/dev/null | awk '{print $2}'; }
|
|
|
|
QT_FULL="$(pkgver_of qt6-base)"
|
|
QT_MINOR="$(echo "$QT_FULL" | grep -oE '^[0-9]+\.[0-9]+')"
|
|
QT_MAJOR="$(echo "$QT_FULL" | grep -oE '^[0-9]+')"
|
|
ICU_FULL="$(pkgver_of icu)"
|
|
ICU_MAJOR="$(echo "$ICU_FULL" | grep -oE '^[0-9]+')"
|
|
BOOST_FULL="$(pkgver_of boost-libs)"
|
|
|
|
JASP_VER="${JASP_TAG#v}"
|
|
# The name carries a prefix of the content hash. The tarball host serves these
|
|
# as immutable for a year, so a name must never be reused for different bytes:
|
|
# a rebuild of the same JASP version would otherwise let a cache hand out the
|
|
# old tarball against the new PKGBUILD's sha256.
|
|
log "creating tarball"
|
|
cd "$STAGE_DIR"
|
|
tar --zstd -cf "$OUT_DIR/jasp-desktop.tar.zst.tmp" .
|
|
SHA256="$(sha256sum "$OUT_DIR/jasp-desktop.tar.zst.tmp" | awk '{print $1}')"
|
|
TARBALL="jasp-desktop-${JASP_VER}-${SHA256:0:12}-x86_64.tar.zst"
|
|
mv "$OUT_DIR/jasp-desktop.tar.zst.tmp" "$OUT_DIR/$TARBALL"
|
|
SIZE="$(stat -c%s "$OUT_DIR/$TARBALL")"
|
|
log "created $TARBALL"
|
|
|
|
cat >"$OUT_DIR/build-info.json" <<EOF
|
|
{
|
|
"jasp_version": "${JASP_VER}",
|
|
"jasp_tag": "${JASP_TAG}",
|
|
"jasp_commit": "${JASP_COMMIT}",
|
|
"tarball": "${TARBALL}",
|
|
"sha256": "${SHA256}",
|
|
"size_bytes": ${SIZE},
|
|
"bundled_r_version": "${R_VERSION}",
|
|
"qt6_version": "${QT_FULL}",
|
|
"qt6_minor": "${QT_MINOR}",
|
|
"qt6_major": "${QT_MAJOR}",
|
|
"icu_version": "${ICU_FULL}",
|
|
"icu_major": "${ICU_MAJOR}",
|
|
"boost_version": "${BOOST_FULL}"
|
|
}
|
|
EOF
|
|
|
|
log "done"
|
|
cat "$OUT_DIR/build-info.json"
|