From f3a6822744a15b9c3693a0d13fabb6fbdc64c30a Mon Sep 17 00:00:00 2001 From: agw Date: Tue, 1 Sep 2026 22:50:00 +0200 Subject: [PATCH] Self-host tarballs on asger.weirsoe.dk instead of GitHub Releases Serve from /srv/tarballz on CT148 via a 'location /tarballz/' alias, NOT from under /var/www/asger: the asger-web deploy swaps that whole directory atomically, so tarballs kept inside it would be deleted on the next site deploy. CT148's rootfs was 3G with 2.7G free - one tarball would have filled it - so it is grown to 15G; speedy's thin pool has room even though vgs reports the VG nearly full. Publishing uploads to a .part name and renames on arrival so a download can never see a truncated file, verifies sha256 on the host, then fetches the public URL and checks it against the sha256 baked into the PKGBUILD. Both checks run before the AUR push, so the package cannot go out pointing at a missing file. Old tarballs are pruned to the newest three. GH_TOKEN is kept but demoted to an optional GITHUB_PAT for R module downloads; it no longer has anything to do with publishing. --- .gitea/workflows/build.yml | 36 +++++++++++------ README.md | 37 ++++++++++++++---- packaging/PKGBUILD.in | 7 ++-- scripts/gen-pkgbuild.py | 9 ++--- scripts/publish-github-release.sh | 63 ------------------------------ scripts/publish-tarball.sh | 64 +++++++++++++++++++++++++++++++ 6 files changed, 123 insertions(+), 93 deletions(-) delete mode 100755 scripts/publish-github-release.sh create mode 100755 scripts/publish-tarball.sh diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml index 3fac416..c311805 100644 --- a/.gitea/workflows/build.yml +++ b/.gitea/workflows/build.yml @@ -24,9 +24,10 @@ concurrency: cancel-in-progress: false env: - GH_OWNER: ${{ vars.GH_OWNER }} - GH_REPO: ${{ vars.GH_REPO }} AUR_MAINTAINER: ${{ vars.AUR_MAINTAINER }} + TARBALL_HOST: ${{ vars.TARBALL_HOST }} + TARBALL_PATH: ${{ vars.TARBALL_PATH }} + TARBALL_BASE_URL: ${{ vars.TARBALL_BASE_URL }} WORK_DIR: /srv/jasp-build jobs: @@ -95,22 +96,33 @@ jobs: --build-info build-info.json \ --template packaging/PKGBUILD.in \ --out PKGBUILD \ - --gh-owner "${GH_OWNER}" \ - --gh-repo "${GH_REPO}" \ + --base-url "${TARBALL_BASE_URL}" \ --maintainer "${AUR_MAINTAINER}" cat PKGBUILD - - name: Publish tarball to GitHub Releases + # Must run before the AUR push: the PKGBUILD's sha256sum refers to this + # file, so publishing the package before the tarball exists would leave + # users with a 404. + - name: Publish tarball to asger.weirsoe.dk env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} + TARBALL_SSH_KEY: ${{ secrets.TARBALL_SSH_KEY }} run: | - TAG="$(python3 -c 'import json;print(json.load(open("release-meta.json"))["tag"])')" ASSET="$(python3 -c 'import json;print(json.load(open("release-meta.json"))["tarball"])')" - export RELEASE_NOTES="Automated build of JASP ${{ needs.check.outputs.jasp_tag }} for Arch Linux. - Consumed by the AUR package jasp-desktop-bin. - Trigger: ${{ needs.check.outputs.reasons }}" - TAG="$TAG" ASSET="${WORK_DIR}/out/${ASSET}" \ - bash scripts/publish-github-release.sh + bash scripts/publish-tarball.sh "${WORK_DIR}/out/${ASSET}" + + - name: Verify the published URL actually serves the tarball + run: | + URL="$(python3 -c 'import json;print(json.load(open("release-meta.json"))["download_url"])')" + WANT="$(python3 -c 'import json;print(json.load(open("release-meta.json"))["sha256"])')" + echo "checking $URL" + GOT="$(curl -fsSL --retry 3 "$URL" | sha256sum | awk '{print $1}')" + if [ "$GOT" != "$WANT" ]; then + echo "error: what the public URL serves does not match the PKGBUILD sha256" >&2 + echo " want $WANT" >&2 + echo " got $GOT" >&2 + exit 1 + fi + echo "public URL serves the expected bytes" # Only the PKGBUILD and metadata cross the job boundary — the tarball # itself already lives on GitHub Releases. diff --git a/README.md b/README.md index ec999ef..247bfba 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ ships 2.0.0 — so the build compiles those itself, matching what upstream CI do | `build/bundle-libs.sh` | Copies non-system shared libraries into `/opt/jasp-desktop/lib` and rewrites rpaths, so the package needs no version-pinned deps. | | `scripts/check-upstream.py` | Compares upstream JASP and the Arch Qt major against `state.json`; decides whether to build. | | `scripts/gen-pkgbuild.py` | Fills `packaging/PKGBUILD.in` from `build-info.json`; derives `pkgrel` from what is on the AUR. | -| `scripts/publish-github-release.sh` | Creates the GitHub release and uploads the tarball. | +| `scripts/publish-tarball.sh` | Uploads the tarball to `asger.weirsoe.dk`, verifies the checksum on arrival and prunes old releases. | | `scripts/push-aur.sh` | Generates `.SRCINFO` and pushes to `aur.archlinux.org`. | | `scripts/record-state.sh` | Commits `state.json` so the next nightly run knows what was built. | | `packaging/PKGBUILD.in` | Template for the AUR package. | @@ -80,7 +80,8 @@ nightly (03:00) ─► check-upstream.py │ builds R 4.5.3, JASP, all R modules, then bundles libs │ jasp-desktop--x86_64.tar.zst + build-info.json ▼ - GitHub release ──► gen-pkgbuild.py ──► push to AUR + upload to asger.weirsoe.dk ──► verify public URL + │ ──► push to AUR │ ▼ state.json committed back @@ -94,20 +95,40 @@ Set these in the Gitea repo before the first run. | Name | Example | Meaning | | --- | --- | --- | -| `GH_OWNER` | `asgerw` | GitHub account hosting the release repo | -| `GH_REPO` | `jasp-desktop-bin` | Release repo name | -| `AUR_MAINTAINER` | `Asger Geel Weirsoe ` | Written into the PKGBUILD header | +| `AUR_MAINTAINER` | `Asger Geel Weirsoe ` | Written into the PKGBUILD header. The guidelines suggest obfuscating the address against spam harvesting. | +| `TARBALL_HOST` | `192.168.10.148` | CT148, where tarballs are uploaded | +| `TARBALL_PATH` | `/srv/tarballz` | Directory on that host | +| `TARBALL_BASE_URL` | `https://asger.weirsoe.dk/tarballz` | Public URL the PKGBUILD downloads from | **Secrets** (Settings → Actions → Secrets): | Name | Meaning | | --- | --- | -| `GH_TOKEN` | GitHub PAT with `contents: write` on the release repo | +| `GH_TOKEN` | *Optional.* A GitHub PAT, passed to the build as `GITHUB_PAT`. Not used for publishing — several JASP R modules install from GitHub and anonymous API rate limits can fail the build. | +| `TARBALL_SSH_KEY` | Private key authorised on CT148 for uploading tarballs | | `AUR_SSH_PRIVATE_KEY` | Private SSH key registered with your AUR account. Copy the value already set on `agw/pi3-smart-workspace`. | | `STATE_PUSH_TOKEN` | Gitea token with write access to this repo, for committing `state.json`. (Gitea reserves the `GITEA_` secret prefix.) | -The GitHub repo `GH_OWNER/GH_REPO` must exist and be public — the AUR PKGBUILD -downloads the tarball from its releases, so it has to be reachable anonymously. +### Where the tarballs live + +Tarballs are served from `https://asger.weirsoe.dk/tarballz/`, backed by +`/srv/tarballz` on CT148. + +They are **not** stored under `/var/www/asger`. The `asger-web` deploy replaces +that entire directory atomically — it unpacks into `.new`, swaps it in and +deletes the old one — so anything kept inside it would vanish on the next site +deploy. nginx exposes `/srv/tarballz` through a `location /tarballz/` alias +instead, with `gzip off` so makepkg gets a plain byte stream. + +Note that `asger.weirsoe.dk` 301-redirects to the IDN form +`asger.xn--weirse-fya.dk`. makepkg follows redirects, so the readable hostname is +used in the PKGBUILD, but that redirect is load-bearing — if it is ever removed, +the `source=` URL must be repointed. + +The publish step verifies the uploaded copy's sha256 on the host, and a separate +step then fetches the *public* URL and checks it against the sha256 baked into +the PKGBUILD. Both run before the AUR push, so the package can never be published +pointing at a missing or truncated file. ## Runner requirements diff --git a/packaging/PKGBUILD.in b/packaging/PKGBUILD.in index a6852ce..5a0d7fb 100644 --- a/packaging/PKGBUILD.in +++ b/packaging/PKGBUILD.in @@ -5,12 +5,11 @@ # Upstream JASP publishes installers for Windows and macOS only; the official # Linux channel is Flathub. The binary this package installs is therefore NOT an # upstream release artifact. It is compiled from the tagged JASP sources -# (@JASP_TAG@, commit @JASP_COMMIT@) on Arch Linux by an automated -# pipeline, and -# published to GitHub Releases: +# (@JASP_TAG@, commit @JASP_COMMIT@) against an Arch Linux userland, in an +# archlinux:base-devel container, by an automated pipeline: # # build recipe: https://gitea.weircon.dk/agw/jasp-desktop-bin -# release: @DOWNLOAD_URL@ +# download: @DOWNLOAD_URL@ # # It bundles its own R @BUNDLED_R@ and the non-system libraries it links against # under /opt/jasp-desktop, so it does not constrain the versions of anything on diff --git a/scripts/gen-pkgbuild.py b/scripts/gen-pkgbuild.py index 255ad52..e5a18a3 100755 --- a/scripts/gen-pkgbuild.py +++ b/scripts/gen-pkgbuild.py @@ -40,8 +40,8 @@ def main() -> int: ap.add_argument("--build-info", required=True) ap.add_argument("--template", required=True) ap.add_argument("--out", required=True) - ap.add_argument("--gh-owner", required=True) - ap.add_argument("--gh-repo", default="jasp-desktop-bin") + ap.add_argument("--base-url", required=True, + help="public directory the tarball is served from") ap.add_argument("--maintainer", required=True) ap.add_argument("--pkgname", default="jasp-desktop-bin") args = ap.parse_args() @@ -58,10 +58,7 @@ def main() -> int: print(f"new version {jasp_version}: pkgrel 1") tag = f"v{jasp_version}-{pkgrel}" - download_url = ( - f"https://github.com/{args.gh_owner}/{args.gh_repo}" - f"/releases/download/{tag}/{info['tarball']}" - ) + download_url = f"{args.base_url.rstrip('/')}/{info['tarball']}" qt_minor = info["qt6_minor"] diff --git a/scripts/publish-github-release.sh b/scripts/publish-github-release.sh deleted file mode 100755 index 1a0eca7..0000000 --- a/scripts/publish-github-release.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -# Create (or reuse) a GitHub release and upload the JASP tarball to it. -# Uses the REST API directly so the workflow does not depend on `gh` being -# present in the runner image. -# -# GH_TOKEN PAT with `contents: write` on the target repo (required) -# GH_OWNER GitHub account owning the release repo (required) -# GH_REPO release repo name (default jasp-desktop-bin) -# TAG release tag, e.g. v0.98.1-1 (required) -# ASSET path to the tarball to upload (required) - -set -euo pipefail - -: "${GH_TOKEN:?GH_TOKEN is required}" -: "${GH_OWNER:?GH_OWNER is required}" -: "${TAG:?TAG is required}" -: "${ASSET:?ASSET is required}" -GH_REPO="${GH_REPO:-jasp-desktop-bin}" - -API="https://api.github.com/repos/${GH_OWNER}/${GH_REPO}" -UPLOADS="https://uploads.github.com/repos/${GH_OWNER}/${GH_REPO}" -AUTH=(-H "Authorization: Bearer ${GH_TOKEN}" -H "Accept: application/vnd.github+json") - -asset_name="$(basename "$ASSET")" - -echo ":: looking for existing release $TAG" -release_id="$(curl -sSf "${AUTH[@]}" "${API}/releases/tags/${TAG}" 2>/dev/null \ - | python3 -c 'import json,sys; print(json.load(sys.stdin).get("id",""))' 2>/dev/null || true)" - -if [[ -z "$release_id" ]]; then - echo ":: creating release $TAG" - release_id="$(curl -sSf -X POST "${AUTH[@]}" "${API}/releases" \ - -d "$(python3 -c ' -import json, os -print(json.dumps({ - "tag_name": os.environ["TAG"], - "name": "JASP " + os.environ["TAG"], - "body": os.environ.get("RELEASE_NOTES", ""), - "draft": False, - "prerelease": False, -}))')" | python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])')" -else - echo ":: reusing release $release_id" - # Replace a same-named asset so re-runs are idempotent. - old="$(curl -sSf "${AUTH[@]}" "${API}/releases/${release_id}/assets" \ - | python3 -c " -import json, sys -name = '${asset_name}' -print(next((a['id'] for a in json.load(sys.stdin) if a['name'] == name), '')) -")" - if [[ -n "$old" ]]; then - echo ":: deleting previous asset $old" - curl -sSf -X DELETE "${AUTH[@]}" "${API}/releases/assets/${old}" - fi -fi - -echo ":: uploading $asset_name ($(du -h "$ASSET" | cut -f1))" -curl -sSf -X POST "${AUTH[@]}" \ - -H "Content-Type: application/zstd" \ - --data-binary "@${ASSET}" \ - "${UPLOADS}/releases/${release_id}/assets?name=${asset_name}" >/dev/null - -echo ":: uploaded to https://github.com/${GH_OWNER}/${GH_REPO}/releases/tag/${TAG}" diff --git a/scripts/publish-tarball.sh b/scripts/publish-tarball.sh new file mode 100755 index 0000000..5e0ad10 --- /dev/null +++ b/scripts/publish-tarball.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Publish the built tarball to asger.weirsoe.dk, which is what the AUR PKGBUILD +# downloads from. +# +# TARBALL_SSH_KEY private key authorised on the web host (required) +# TARBALL_HOST web host, e.g. 192.168.10.148 (required) +# TARBALL_PATH directory on that host (required) +# KEEP_RELEASES how many tarballs to retain (default 3) +# $1 path to the tarball to upload +# +# The files live in /srv/tarballz, deliberately NOT under /var/www/asger: the +# asger-web deploy replaces that directory atomically (build into .new, swap, +# delete .old), so anything stored inside it disappears on the next site deploy. +# nginx exposes /srv/tarballz via a `location /tarballz/` alias. + +set -euo pipefail + +TARBALL="${1:?path to tarball is required}" +: "${TARBALL_SSH_KEY:?TARBALL_SSH_KEY is required}" +: "${TARBALL_HOST:?TARBALL_HOST is required}" +: "${TARBALL_PATH:?TARBALL_PATH is required}" +KEEP_RELEASES="${KEEP_RELEASES:-3}" + +name="$(basename "$TARBALL")" + +mkdir -p ~/.ssh +printf '%s\n' "$TARBALL_SSH_KEY" > ~/.ssh/tarball_key +chmod 600 ~/.ssh/tarball_key +ssh-keyscan -t ed25519 "$TARBALL_HOST" >> ~/.ssh/known_hosts 2>/dev/null || true +SSH=(ssh -i "$HOME/.ssh/tarball_key" -o StrictHostKeyChecking=accept-new) + +echo ":: uploading ${name} ($(du -h "$TARBALL" | cut -f1)) to ${TARBALL_HOST}:${TARBALL_PATH}" + +# Upload to a temporary name and rename on arrival, so a download that starts +# mid-transfer can never see a truncated file at the final URL. +"${SSH[@]}" "root@${TARBALL_HOST}" "mkdir -p '${TARBALL_PATH}'" +"${SSH[@]}" "root@${TARBALL_HOST}" "cat > '${TARBALL_PATH}/.${name}.part'" < "$TARBALL" +"${SSH[@]}" "root@${TARBALL_HOST}" "mv '${TARBALL_PATH}/.${name}.part' '${TARBALL_PATH}/${name}' && chmod 644 '${TARBALL_PATH}/${name}'" + +echo ":: verifying the uploaded copy matches" +local_sum="$(sha256sum "$TARBALL" | awk '{print $1}')" +remote_sum="$("${SSH[@]}" "root@${TARBALL_HOST}" "sha256sum '${TARBALL_PATH}/${name}'" | awk '{print $1}')" +if [[ "$local_sum" != "$remote_sum" ]]; then + echo "error: checksum mismatch after upload" >&2 + echo " local $local_sum" >&2 + echo " remote $remote_sum" >&2 + exit 1 +fi +echo ":: sha256 matches ($local_sum)" + +# Old releases are kept so that a user mid-install of the previous version is +# not left with a dead URL, but not forever - each one is well over a gigabyte. +echo ":: pruning to the newest ${KEEP_RELEASES} tarballs" +"${SSH[@]}" "root@${TARBALL_HOST}" " + cd '${TARBALL_PATH}' || exit 0 + ls -1t jasp-desktop-*.tar.zst 2>/dev/null | tail -n +\$((${KEEP_RELEASES} + 1)) | while read -r old; do + echo \" removing \$old\" + rm -f -- \"\$old\" + done + echo ' remaining:' + ls -1sh jasp-desktop-*.tar.zst 2>/dev/null | sed 's/^/ /' +" + +echo ":: published"