From aa08dc108e2a50e8f621b5fe2e712f13059dd975 Mon Sep 17 00:00:00 2001 From: agw Date: Thu, 1 Oct 2026 17:05:00 +0200 Subject: [PATCH] Keep infrastructure details out of the repository The README, the publish script and the workflow named the tarball host, its address and filesystem paths on the web host and the runner. Hosts and paths now come only from repository variables, and the workflow shares the work tree and the build cache through the named Docker volumes jasp-build and jasp-cache instead of host paths. (The runner does not expand ${{ vars }} in container.volumes, so a variable could not have replaced the host path there.) Also drop the unused scripts/gen-tarball-index.py, and give each module build its own process under a time limit: a module once spun for eight hours in jaspModuleTools' bundling step and stalled the whole build. Claude-Session: https://claude.ai/code/session_01GGe7kUhooMFuzY6GaDb2w9 --- .gitea/workflows/build.yml | 21 ++-- .gitignore | 2 + README.md | 32 +++--- build/build-jasp.sh | 11 +- scripts/gen-tarball-index.py | 192 ----------------------------------- scripts/publish-tarball.sh | 14 +-- 6 files changed, 45 insertions(+), 227 deletions(-) delete mode 100644 scripts/gen-tarball-index.py diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml index aa7b153..697ede4 100644 --- a/.gitea/workflows/build.yml +++ b/.gitea/workflows/build.yml @@ -35,7 +35,7 @@ env: TARBALL_HOST: ${{ vars.TARBALL_HOST }} TARBALL_PATH: ${{ vars.TARBALL_PATH }} TARBALL_BASE_URL: ${{ vars.TARBALL_BASE_URL }} - WORK_DIR: /srv/jasp-build + WORK_DIR: /work jobs: check: @@ -62,15 +62,16 @@ jobs: # full build runs for hours. The act_runner timeout and Gitea's # ENDLESS_TASK_TIMEOUT must both allow for this (see README). timeout-minutes: 900 - # `docker run -v ${WORK_DIR}:/work` below talks to the runner host's docker, - # so it mounts the *host's* WORK_DIR. Without this bind the job container - # has its own private copy: the scripts it prepares never reach the build - # (which then ran a stale script left on the host) and the tarball the - # build writes is invisible to the publish steps. + # The build step's `docker run` talks to the runner's docker daemon, not to + # this job container, so the two only share files through a Docker volume. + # Without it the scripts prepared here never reach the build and the + # tarball the build writes is invisible to the publish steps. Named volumes + # rather than host paths: the runner does not expand ${{ vars }} in + # container.volumes, and a host path would put runner internals in the repo. container: image: docker.gitea.com/runner-images:ubuntu-latest volumes: - - /srv/jasp-build:/srv/jasp-build + - jasp-build:/work steps: - uses: actions/checkout@v4 @@ -81,7 +82,7 @@ jobs: - name: Prepare work directory run: | - # WORK_DIR is a bind mount now, so empty it rather than removing it. + # WORK_DIR is a volume mount point, so empty it rather than removing it. sudo find "${WORK_DIR}" -mindepth 1 -delete sudo mkdir -p "${WORK_DIR}/out" sudo cp -a build packaging scripts "${WORK_DIR}/" @@ -93,8 +94,8 @@ jobs: run: | docker rm -f jasp-build 2>/dev/null || true docker run --rm --name jasp-build \ - -v "${WORK_DIR}:/work" \ - -v /srv/jasp-cache:/cache \ + -v jasp-build:/work \ + -v jasp-cache:/cache \ -e JASP_TAG="${{ needs.check.outputs.jasp_tag }}" \ -e OUT_DIR=/work/out \ -e JOBS=5 \ diff --git a/.gitignore b/.gitignore index 031de7e..2121566 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ release-meta.json __pycache__/ *.pyc +PKGBUILD +build-info.json diff --git a/README.md b/README.md index 76e13f3..5f904e9 100644 --- a/README.md +++ b/README.md @@ -96,30 +96,27 @@ Set these in the Gitea repo before the first run. | Name | Example | Meaning | | --- | --- | --- | | `AUR_MAINTAINER` | `Asger Geel Weirsoe ` | Written into the PKGBUILD header. The guidelines suggest obfuscating the address against spam harvesting. | -| `TARBALL_HOST` | `192.168.10.148` | CT148, where tarballs are uploaded | -| `TARBALL_PATH` | `/srv/tarballz` | Directory on that host | +| `TARBALL_HOST` | `files.example.org` | SSH host the tarballs are uploaded to | +| `TARBALL_PATH` | `/path/to/tarballs` | Directory on that host that the web server publishes | | `TARBALL_BASE_URL` | `https://asger.weirsoe.dk/tarballz` | Public URL the PKGBUILD downloads from | -| `PUBLISH` | `true` | Master switch. Unless it is `true`, a run only builds and verifies, leaving the tarball in `/srv/jasp-build/out`: no upload, no `state.json` commit, no AUR push. | +| `PUBLISH` | `true` | Master switch. Unless it is `true`, a run only builds and verifies, leaving the tarball in the `jasp-build` volume under `out/`: no upload, no `state.json` commit, no AUR push. | **Secrets** (Settings → Actions → Secrets): | Name | Meaning | | --- | --- | | `GH_TOKEN` | *Optional.* A GitHub PAT, passed to the build as `GITHUB_PAT`. Not used for publishing — several JASP R modules install from GitHub and anonymous API rate limits can fail the build. | -| `TARBALL_SSH_KEY` | Private key authorised on CT148 for uploading tarballs | +| `TARBALL_SSH_KEY` | Private key authorised on `TARBALL_HOST` for uploading tarballs | | `AUR_SSH_PRIVATE_KEY` | Private SSH key registered with your AUR account. Copy the value already set on `agw/pi3-smart-workspace`. | | `STATE_PUSH_TOKEN` | Gitea token with write access to this repo, for committing `state.json`. (Gitea reserves the `GITEA_` secret prefix.) | ### Where the tarballs live -Tarballs are served from `https://asger.weirsoe.dk/tarballz/`, backed by -`/srv/tarballz` on CT148. - -They are **not** stored under `/var/www/asger`. The `asger-web` deploy replaces -that entire directory atomically — it unpacks into `.new`, swaps it in and -deletes the old one — so anything kept inside it would vanish on the next site -deploy. nginx exposes `/srv/tarballz` through a `location /tarballz/` alias -instead, with `gzip off` so makepkg gets a plain byte stream. +Tarballs are served from `https://asger.weirsoe.dk/tarballz/`. They are kept +outside the website's own web root: the site deploy replaces its web root +atomically, so anything stored inside it would vanish on the next deploy. The +web server exposes the tarball directory through a separate `/tarballz/` +location instead, with gzip off so makepkg gets a plain byte stream. Note that `asger.weirsoe.dk` 301-redirects to the IDN form `asger.xn--weirse-fya.dk`. makepkg follows redirects, so the readable hostname is @@ -135,17 +132,18 @@ pointing at a missing or truncated file. The build runs for hours and needs disk. On the Gitea runner host: -- `runner.timeout` in `/var/lib/gitea-worker/config.yaml` must exceed the build - time (set to `16h`). +- `runner.timeout` in the runner's `config.yaml` must exceed the build time + (set to `16h`). - Gitea's own job timeout (`[actions] ENDLESS_TASK_TIMEOUT` in `app.ini`, default 3h) must match, otherwise Gitea kills the job at 3h regardless of the runner. - Roughly 40–60 GB of free disk for the build tree, the bundled R and the staged install. -- `/srv/jasp-cache` on the runner host, mounted into the build as `/cache`. - It holds ccache and the compiled R package cache (renv), and is what keeps a +- The Docker volume `jasp-cache`, mounted into the build as `/cache`. It holds ccache and the compiled R package cache (renv), and is what keeps a rebuild from recompiling 1000+ R packages from scratch. It is never wiped by the workflow; entries whose system libraries have gone missing after an Arch - update are purged at the start of each build. Delete it to force a clean build. + update are purged at the start of each build. `docker volume rm jasp-cache` + forces a clean build. A second volume, `jasp-build`, holds the work tree and + output; the workflow empties it at the start of every run. - **Swap.** `rstanarm` (required by `jaspMixedModels`) and `OpenMx` generate translation units where a single `cc1plus` wants 4–8 GB. Without swap an over-commit is an instant OOM kill, which surfaces only as a module that never diff --git a/build/build-jasp.sh b/build/build-jasp.sh index b1215e2..aabc42d 100755 --- a/build/build-jasp.sh +++ b/build/build-jasp.sh @@ -422,7 +422,16 @@ for attempt in 1 2 3; do done [[ ${#MODULE_PATHS[@]} -eq 0 ]] && break log "module build pass ${attempt}/3: ${#MODULE_PATHS[@]} modules left" - Rscript /work/build/build-modules.R "${MODULE_PATHS[@]}" || true + # One R process per module under a time limit. A module once spun at 100% + # CPU for 8 hours inside jaspModuleTools' bundling step, after it had + # installed fine, and stalled the whole build; the same module bundled in two + # minutes on the previous run. A timed-out module is simply retried in the + # next pass. + for path in "${MODULE_PATHS[@]}"; do + timeout --kill-after=60 "${MODULE_TIMEOUT:-5400}" \ + Rscript /work/build/build-modules.R "$path" \ + || log "$(basename "$path") did not finish (exit $?), retrying next pass" + done built=0 for module in "${ALL_MODULES[@]}"; do [[ -f "${MODULE_BUNDLES}/${module}.JASPModule" ]] && built=$((built + 1)) diff --git a/scripts/gen-tarball-index.py b/scripts/gen-tarball-index.py deleted file mode 100644 index 387cdff..0000000 --- a/scripts/gen-tarball-index.py +++ /dev/null @@ -1,192 +0,0 @@ -#!/usr/bin/env python3 -"""Render the index page for https://asger.weirsoe.dk/tarballz/. - -Reads the sidecar .json files that publish-tarball.sh writes next to each -tarball (on stdin, one JSON object per line) and emits a standalone HTML page. - -The theme tokens are inlined rather than linked to the main site's stylesheet: -that file is content-hashed (_astro/_slug_..css) and its name changes on -every site rebuild, so a link would silently rot. Inlining costs a few hundred -bytes and cannot break. -""" - -import html -import json -import os -import sys -from datetime import datetime, timezone - -GITEA_URL = "https://gitea.weircon.dk/agw/jasp-desktop-bin" -AUR_URL = "https://aur.archlinux.org/packages/jasp-desktop-bin" - -PAGE = """ - - - - -Tarballs · Asger Weirsøe - - - - - - - -
- Asger Weirsøe. - -
- -
-

Tarballs

-

- Prækompilerede JASP-binærer til Arch Linux. - JASP udgiver kun installere til Windows og macOS, så disse er bygget fra - kildekoden af en automatisk pipeline og udgivet på AUR som - jasp-desktop-bin. -

- -
-

Du behøver normalt ikke hente herfra manuelt — brug AUR:

- yay -S jasp-desktop-bin -
- -

Tilgængelige builds

- {releases} - -

- Hver build indeholder sin egen R og de biblioteker den linker mod, så den ikke - bindes til bestemte versioner af noget på dit system. - Kun de nyeste {keep} builds gemmes. -

-
- -
-

bygget automatisk · self-hosted på proxmox · byggeopskrift

-

© {year} Asger Weirsøe

-
- - -""" - - -def human_size(n: int) -> str: - for unit in ("B", "KB", "MB", "GB"): - if n < 1024 or unit == "GB": - return f"{n:.1f} {unit}".replace(".0 ", " ") - n /= 1024 - return f"{n} B" - - -def main() -> int: - entries = [] - for line in sys.stdin: - line = line.strip() - if not line: - continue - try: - entries.append(json.loads(line)) - except json.JSONDecodeError: - print(f"warning: skipping unparsable sidecar line", file=sys.stderr) - - # Newest first. - entries.sort(key=lambda e: e.get("built", ""), reverse=True) - - if entries: - items = [] - for e in entries: - name = html.escape(e["tarball"]) - ver = html.escape(e.get("jasp_version", "?")) - built = e.get("built", "") - try: - built_h = datetime.fromisoformat(built).strftime("%d-%m-%Y") - except ValueError: - built_h = html.escape(built) - size = human_size(int(e.get("size_bytes", 0))) - sha = html.escape(e.get("sha256", "")) - r = html.escape(e.get("bundled_r_version", "")) - qt = html.escape(e.get("qt6_minor", "")) - items.append( - f'
  • \n' - f'
    \n' - f' JASP {ver}\n' - f' Hent · {size}\n' - f'
    \n' - f'
    {built_h} · x86_64 · bundlet R {r} · bygget mod Qt {qt}
    \n' - f'
    sha256 {sha}
    \n' - f'
    • ' - ) - releases = '
      \n' + "\n".join(items) + "\n
    " - else: - releases = '

    Ingen builds tilgængelige lige nu.

    ' - - sys.stdout.write( - PAGE.format( - releases=releases, - gitea=GITEA_URL, - aur=AUR_URL, - keep=len(entries) if entries else 3, - year=datetime.now(timezone.utc).year, - ) - ) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/publish-tarball.sh b/scripts/publish-tarball.sh index 5e0ad10..231a103 100755 --- a/scripts/publish-tarball.sh +++ b/scripts/publish-tarball.sh @@ -2,16 +2,16 @@ # Publish the built tarball to asger.weirsoe.dk, which is what the AUR PKGBUILD # downloads from. # -# TARBALL_SSH_KEY private key authorised on the web host (required) -# TARBALL_HOST web host, e.g. 192.168.10.148 (required) -# TARBALL_PATH directory on that host (required) +# TARBALL_SSH_KEY private key authorised on the web host (required) +# TARBALL_HOST web host to upload to over SSH (required) +# TARBALL_PATH directory on that host (required) # KEEP_RELEASES how many tarballs to retain (default 3) # $1 path to the tarball to upload # -# The files live in /srv/tarballz, deliberately NOT under /var/www/asger: the -# asger-web deploy replaces that directory atomically (build into .new, swap, -# delete .old), so anything stored inside it disappears on the next site deploy. -# nginx exposes /srv/tarballz via a `location /tarballz/` alias. +# TARBALL_PATH is deliberately outside the website's own web root: the site +# deploy replaces its web root atomically, so anything stored inside it would +# disappear on the next deploy. The web server publishes the directory through +# a separate /tarballz/ location. set -euo pipefail