diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml index aa7b153..697ede4 100644 --- a/.gitea/workflows/build.yml +++ b/.gitea/workflows/build.yml @@ -35,7 +35,7 @@ env: TARBALL_HOST: ${{ vars.TARBALL_HOST }} TARBALL_PATH: ${{ vars.TARBALL_PATH }} TARBALL_BASE_URL: ${{ vars.TARBALL_BASE_URL }} - WORK_DIR: /srv/jasp-build + WORK_DIR: /work jobs: check: @@ -62,15 +62,16 @@ jobs: # full build runs for hours. The act_runner timeout and Gitea's # ENDLESS_TASK_TIMEOUT must both allow for this (see README). timeout-minutes: 900 - # `docker run -v ${WORK_DIR}:/work` below talks to the runner host's docker, - # so it mounts the *host's* WORK_DIR. Without this bind the job container - # has its own private copy: the scripts it prepares never reach the build - # (which then ran a stale script left on the host) and the tarball the - # build writes is invisible to the publish steps. + # The build step's `docker run` talks to the runner's docker daemon, not to + # this job container, so the two only share files through a Docker volume. + # Without it the scripts prepared here never reach the build and the + # tarball the build writes is invisible to the publish steps. Named volumes + # rather than host paths: the runner does not expand ${{ vars }} in + # container.volumes, and a host path would put runner internals in the repo. container: image: docker.gitea.com/runner-images:ubuntu-latest volumes: - - /srv/jasp-build:/srv/jasp-build + - jasp-build:/work steps: - uses: actions/checkout@v4 @@ -81,7 +82,7 @@ jobs: - name: Prepare work directory run: | - # WORK_DIR is a bind mount now, so empty it rather than removing it. + # WORK_DIR is a volume mount point, so empty it rather than removing it. sudo find "${WORK_DIR}" -mindepth 1 -delete sudo mkdir -p "${WORK_DIR}/out" sudo cp -a build packaging scripts "${WORK_DIR}/" @@ -93,8 +94,8 @@ jobs: run: | docker rm -f jasp-build 2>/dev/null || true docker run --rm --name jasp-build \ - -v "${WORK_DIR}:/work" \ - -v /srv/jasp-cache:/cache \ + -v jasp-build:/work \ + -v jasp-cache:/cache \ -e JASP_TAG="${{ needs.check.outputs.jasp_tag }}" \ -e OUT_DIR=/work/out \ -e JOBS=5 \ diff --git a/.gitignore b/.gitignore index 031de7e..2121566 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ release-meta.json __pycache__/ *.pyc +PKGBUILD +build-info.json diff --git a/README.md b/README.md index 76e13f3..5f904e9 100644 --- a/README.md +++ b/README.md @@ -96,30 +96,27 @@ Set these in the Gitea repo before the first run. | Name | Example | Meaning | | --- | --- | --- | | `AUR_MAINTAINER` | `Asger Geel Weirsoe ` | Written into the PKGBUILD header. The guidelines suggest obfuscating the address against spam harvesting. | -| `TARBALL_HOST` | `192.168.10.148` | CT148, where tarballs are uploaded | -| `TARBALL_PATH` | `/srv/tarballz` | Directory on that host | +| `TARBALL_HOST` | `files.example.org` | SSH host the tarballs are uploaded to | +| `TARBALL_PATH` | `/path/to/tarballs` | Directory on that host that the web server publishes | | `TARBALL_BASE_URL` | `https://asger.weirsoe.dk/tarballz` | Public URL the PKGBUILD downloads from | -| `PUBLISH` | `true` | Master switch. Unless it is `true`, a run only builds and verifies, leaving the tarball in `/srv/jasp-build/out`: no upload, no `state.json` commit, no AUR push. | +| `PUBLISH` | `true` | Master switch. Unless it is `true`, a run only builds and verifies, leaving the tarball in the `jasp-build` volume under `out/`: no upload, no `state.json` commit, no AUR push. | **Secrets** (Settings → Actions → Secrets): | Name | Meaning | | --- | --- | | `GH_TOKEN` | *Optional.* A GitHub PAT, passed to the build as `GITHUB_PAT`. Not used for publishing — several JASP R modules install from GitHub and anonymous API rate limits can fail the build. | -| `TARBALL_SSH_KEY` | Private key authorised on CT148 for uploading tarballs | +| `TARBALL_SSH_KEY` | Private key authorised on `TARBALL_HOST` for uploading tarballs | | `AUR_SSH_PRIVATE_KEY` | Private SSH key registered with your AUR account. Copy the value already set on `agw/pi3-smart-workspace`. | | `STATE_PUSH_TOKEN` | Gitea token with write access to this repo, for committing `state.json`. (Gitea reserves the `GITEA_` secret prefix.) | ### Where the tarballs live -Tarballs are served from `https://asger.weirsoe.dk/tarballz/`, backed by -`/srv/tarballz` on CT148. - -They are **not** stored under `/var/www/asger`. The `asger-web` deploy replaces -that entire directory atomically — it unpacks into `.new`, swaps it in and -deletes the old one — so anything kept inside it would vanish on the next site -deploy. nginx exposes `/srv/tarballz` through a `location /tarballz/` alias -instead, with `gzip off` so makepkg gets a plain byte stream. +Tarballs are served from `https://asger.weirsoe.dk/tarballz/`. They are kept +outside the website's own web root: the site deploy replaces its web root +atomically, so anything stored inside it would vanish on the next deploy. The +web server exposes the tarball directory through a separate `/tarballz/` +location instead, with gzip off so makepkg gets a plain byte stream. Note that `asger.weirsoe.dk` 301-redirects to the IDN form `asger.xn--weirse-fya.dk`. makepkg follows redirects, so the readable hostname is @@ -135,17 +132,18 @@ pointing at a missing or truncated file. The build runs for hours and needs disk. On the Gitea runner host: -- `runner.timeout` in `/var/lib/gitea-worker/config.yaml` must exceed the build - time (set to `16h`). +- `runner.timeout` in the runner's `config.yaml` must exceed the build time + (set to `16h`). - Gitea's own job timeout (`[actions] ENDLESS_TASK_TIMEOUT` in `app.ini`, default 3h) must match, otherwise Gitea kills the job at 3h regardless of the runner. - Roughly 40–60 GB of free disk for the build tree, the bundled R and the staged install. -- `/srv/jasp-cache` on the runner host, mounted into the build as `/cache`. - It holds ccache and the compiled R package cache (renv), and is what keeps a +- The Docker volume `jasp-cache`, mounted into the build as `/cache`. It holds ccache and the compiled R package cache (renv), and is what keeps a rebuild from recompiling 1000+ R packages from scratch. It is never wiped by the workflow; entries whose system libraries have gone missing after an Arch - update are purged at the start of each build. Delete it to force a clean build. + update are purged at the start of each build. `docker volume rm jasp-cache` + forces a clean build. A second volume, `jasp-build`, holds the work tree and + output; the workflow empties it at the start of every run. - **Swap.** `rstanarm` (required by `jaspMixedModels`) and `OpenMx` generate translation units where a single `cc1plus` wants 4–8 GB. Without swap an over-commit is an instant OOM kill, which surfaces only as a module that never diff --git a/build/build-jasp.sh b/build/build-jasp.sh index b1215e2..aabc42d 100755 --- a/build/build-jasp.sh +++ b/build/build-jasp.sh @@ -422,7 +422,16 @@ for attempt in 1 2 3; do done [[ ${#MODULE_PATHS[@]} -eq 0 ]] && break log "module build pass ${attempt}/3: ${#MODULE_PATHS[@]} modules left" - Rscript /work/build/build-modules.R "${MODULE_PATHS[@]}" || true + # One R process per module under a time limit. A module once spun at 100% + # CPU for 8 hours inside jaspModuleTools' bundling step, after it had + # installed fine, and stalled the whole build; the same module bundled in two + # minutes on the previous run. A timed-out module is simply retried in the + # next pass. + for path in "${MODULE_PATHS[@]}"; do + timeout --kill-after=60 "${MODULE_TIMEOUT:-5400}" \ + Rscript /work/build/build-modules.R "$path" \ + || log "$(basename "$path") did not finish (exit $?), retrying next pass" + done built=0 for module in "${ALL_MODULES[@]}"; do [[ -f "${MODULE_BUNDLES}/${module}.JASPModule" ]] && built=$((built + 1)) diff --git a/scripts/gen-tarball-index.py b/scripts/gen-tarball-index.py deleted file mode 100644 index 387cdff..0000000 --- a/scripts/gen-tarball-index.py +++ /dev/null @@ -1,192 +0,0 @@ -#!/usr/bin/env python3 -"""Render the index page for https://asger.weirsoe.dk/tarballz/. - -Reads the sidecar .json files that publish-tarball.sh writes next to each -tarball (on stdin, one JSON object per line) and emits a standalone HTML page. - -The theme tokens are inlined rather than linked to the main site's stylesheet: -that file is content-hashed (_astro/_slug_..css) and its name changes on -every site rebuild, so a link would silently rot. Inlining costs a few hundred -bytes and cannot break. -""" - -import html -import json -import os -import sys -from datetime import datetime, timezone - -GITEA_URL = "https://gitea.weircon.dk/agw/jasp-desktop-bin" -AUR_URL = "https://aur.archlinux.org/packages/jasp-desktop-bin" - -PAGE = """ - - - - -Tarballs · Asger Weirsøe - - - - - - - -
- Asger Weirsøe. - -
- -
-

Tarballs

-

- Prækompilerede JASP-binærer til Arch Linux. - JASP udgiver kun installere til Windows og macOS, så disse er bygget fra - kildekoden af en automatisk pipeline og udgivet på AUR som - jasp-desktop-bin. -

- -
-

Du behøver normalt ikke hente herfra manuelt — brug AUR:

- yay -S jasp-desktop-bin -
- -

Tilgængelige builds

- {releases} - -

- Hver build indeholder sin egen R og de biblioteker den linker mod, så den ikke - bindes til bestemte versioner af noget på dit system. - Kun de nyeste {keep} builds gemmes. -

-
- -
-

bygget automatisk · self-hosted på proxmox · byggeopskrift

-

© {year} Asger Weirsøe

-
- - -""" - - -def human_size(n: int) -> str: - for unit in ("B", "KB", "MB", "GB"): - if n < 1024 or unit == "GB": - return f"{n:.1f} {unit}".replace(".0 ", " ") - n /= 1024 - return f"{n} B" - - -def main() -> int: - entries = [] - for line in sys.stdin: - line = line.strip() - if not line: - continue - try: - entries.append(json.loads(line)) - except json.JSONDecodeError: - print(f"warning: skipping unparsable sidecar line", file=sys.stderr) - - # Newest first. - entries.sort(key=lambda e: e.get("built", ""), reverse=True) - - if entries: - items = [] - for e in entries: - name = html.escape(e["tarball"]) - ver = html.escape(e.get("jasp_version", "?")) - built = e.get("built", "") - try: - built_h = datetime.fromisoformat(built).strftime("%d-%m-%Y") - except ValueError: - built_h = html.escape(built) - size = human_size(int(e.get("size_bytes", 0))) - sha = html.escape(e.get("sha256", "")) - r = html.escape(e.get("bundled_r_version", "")) - qt = html.escape(e.get("qt6_minor", "")) - items.append( - f'
  • \n' - f'
    \n' - f' JASP {ver}\n' - f' Hent · {size}\n' - f'
    \n' - f'
    {built_h} · x86_64 · bundlet R {r} · bygget mod Qt {qt}
    \n' - f'
    sha256 {sha}
    \n' - f'
    • ' - ) - releases = '" - else: - releases = '

    Ingen builds tilgængelige lige nu.

    ' - - sys.stdout.write( - PAGE.format( - releases=releases, - gitea=GITEA_URL, - aur=AUR_URL, - keep=len(entries) if entries else 3, - year=datetime.now(timezone.utc).year, - ) - ) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/publish-tarball.sh b/scripts/publish-tarball.sh index 5e0ad10..231a103 100755 --- a/scripts/publish-tarball.sh +++ b/scripts/publish-tarball.sh @@ -2,16 +2,16 @@ # Publish the built tarball to asger.weirsoe.dk, which is what the AUR PKGBUILD # downloads from. # -# TARBALL_SSH_KEY private key authorised on the web host (required) -# TARBALL_HOST web host, e.g. 192.168.10.148 (required) -# TARBALL_PATH directory on that host (required) +# TARBALL_SSH_KEY private key authorised on the web host (required) +# TARBALL_HOST web host to upload to over SSH (required) +# TARBALL_PATH directory on that host (required) # KEEP_RELEASES how many tarballs to retain (default 3) # $1 path to the tarball to upload # -# The files live in /srv/tarballz, deliberately NOT under /var/www/asger: the -# asger-web deploy replaces that directory atomically (build into .new, swap, -# delete .old), so anything stored inside it disappears on the next site deploy. -# nginx exposes /srv/tarballz via a `location /tarballz/` alias. +# TARBALL_PATH is deliberately outside the website's own web root: the site +# deploy replaces its web root atomically, so anything stored inside it would +# disappear on the next deploy. The web server publishes the directory through +# a separate /tarballz/ location. set -euo pipefail