diff --git a/.gitea/workflows/build.yml b/.gitea/workflows/build.yml index 3fac416..c311805 100644 --- a/.gitea/workflows/build.yml +++ b/.gitea/workflows/build.yml @@ -24,9 +24,10 @@ concurrency: cancel-in-progress: false env: - GH_OWNER: ${{ vars.GH_OWNER }} - GH_REPO: ${{ vars.GH_REPO }} AUR_MAINTAINER: ${{ vars.AUR_MAINTAINER }} + TARBALL_HOST: ${{ vars.TARBALL_HOST }} + TARBALL_PATH: ${{ vars.TARBALL_PATH }} + TARBALL_BASE_URL: ${{ vars.TARBALL_BASE_URL }} WORK_DIR: /srv/jasp-build jobs: @@ -95,22 +96,33 @@ jobs: --build-info build-info.json \ --template packaging/PKGBUILD.in \ --out PKGBUILD \ - --gh-owner "${GH_OWNER}" \ - --gh-repo "${GH_REPO}" \ + --base-url "${TARBALL_BASE_URL}" \ --maintainer "${AUR_MAINTAINER}" cat PKGBUILD - - name: Publish tarball to GitHub Releases + # Must run before the AUR push: the PKGBUILD's sha256sum refers to this + # file, so publishing the package before the tarball exists would leave + # users with a 404. + - name: Publish tarball to asger.weirsoe.dk env: - GH_TOKEN: ${{ secrets.GH_TOKEN }} + TARBALL_SSH_KEY: ${{ secrets.TARBALL_SSH_KEY }} run: | - TAG="$(python3 -c 'import json;print(json.load(open("release-meta.json"))["tag"])')" ASSET="$(python3 -c 'import json;print(json.load(open("release-meta.json"))["tarball"])')" - export RELEASE_NOTES="Automated build of JASP ${{ needs.check.outputs.jasp_tag }} for Arch Linux. - Consumed by the AUR package jasp-desktop-bin. - Trigger: ${{ needs.check.outputs.reasons }}" - TAG="$TAG" ASSET="${WORK_DIR}/out/${ASSET}" \ - bash scripts/publish-github-release.sh + bash scripts/publish-tarball.sh "${WORK_DIR}/out/${ASSET}" + + - name: Verify the published URL actually serves the tarball + run: | + URL="$(python3 -c 'import json;print(json.load(open("release-meta.json"))["download_url"])')" + WANT="$(python3 -c 'import json;print(json.load(open("release-meta.json"))["sha256"])')" + echo "checking $URL" + GOT="$(curl -fsSL --retry 3 "$URL" | sha256sum | awk '{print $1}')" + if [ "$GOT" != "$WANT" ]; then + echo "error: what the public URL serves does not match the PKGBUILD sha256" >&2 + echo " want $WANT" >&2 + echo " got $GOT" >&2 + exit 1 + fi + echo "public URL serves the expected bytes" # Only the PKGBUILD and metadata cross the job boundary — the tarball # itself already lives on GitHub Releases. diff --git a/README.md b/README.md index ec999ef..247bfba 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ ships 2.0.0 — so the build compiles those itself, matching what upstream CI do | `build/bundle-libs.sh` | Copies non-system shared libraries into `/opt/jasp-desktop/lib` and rewrites rpaths, so the package needs no version-pinned deps. | | `scripts/check-upstream.py` | Compares upstream JASP and the Arch Qt major against `state.json`; decides whether to build. | | `scripts/gen-pkgbuild.py` | Fills `packaging/PKGBUILD.in` from `build-info.json`; derives `pkgrel` from what is on the AUR. | -| `scripts/publish-github-release.sh` | Creates the GitHub release and uploads the tarball. | +| `scripts/publish-tarball.sh` | Uploads the tarball to `asger.weirsoe.dk`, verifies the checksum on arrival and prunes old releases. | | `scripts/push-aur.sh` | Generates `.SRCINFO` and pushes to `aur.archlinux.org`. | | `scripts/record-state.sh` | Commits `state.json` so the next nightly run knows what was built. | | `packaging/PKGBUILD.in` | Template for the AUR package. | @@ -80,7 +80,8 @@ nightly (03:00) ─► check-upstream.py │ builds R 4.5.3, JASP, all R modules, then bundles libs │ jasp-desktop--x86_64.tar.zst + build-info.json ▼ - GitHub release ──► gen-pkgbuild.py ──► push to AUR + upload to asger.weirsoe.dk ──► verify public URL + │ ──► push to AUR │ ▼ state.json committed back @@ -94,20 +95,40 @@ Set these in the Gitea repo before the first run. | Name | Example | Meaning | | --- | --- | --- | -| `GH_OWNER` | `asgerw` | GitHub account hosting the release repo | -| `GH_REPO` | `jasp-desktop-bin` | Release repo name | -| `AUR_MAINTAINER` | `Asger Geel Weirsoe ` | Written into the PKGBUILD header | +| `AUR_MAINTAINER` | `Asger Geel Weirsoe ` | Written into the PKGBUILD header. The guidelines suggest obfuscating the address against spam harvesting. | +| `TARBALL_HOST` | `192.168.10.148` | CT148, where tarballs are uploaded | +| `TARBALL_PATH` | `/srv/tarballz` | Directory on that host | +| `TARBALL_BASE_URL` | `https://asger.weirsoe.dk/tarballz` | Public URL the PKGBUILD downloads from | **Secrets** (Settings → Actions → Secrets): | Name | Meaning | | --- | --- | -| `GH_TOKEN` | GitHub PAT with `contents: write` on the release repo | +| `GH_TOKEN` | *Optional.* A GitHub PAT, passed to the build as `GITHUB_PAT`. Not used for publishing — several JASP R modules install from GitHub and anonymous API rate limits can fail the build. | +| `TARBALL_SSH_KEY` | Private key authorised on CT148 for uploading tarballs | | `AUR_SSH_PRIVATE_KEY` | Private SSH key registered with your AUR account. Copy the value already set on `agw/pi3-smart-workspace`. | | `STATE_PUSH_TOKEN` | Gitea token with write access to this repo, for committing `state.json`. (Gitea reserves the `GITEA_` secret prefix.) | -The GitHub repo `GH_OWNER/GH_REPO` must exist and be public — the AUR PKGBUILD -downloads the tarball from its releases, so it has to be reachable anonymously. +### Where the tarballs live + +Tarballs are served from `https://asger.weirsoe.dk/tarballz/`, backed by +`/srv/tarballz` on CT148. + +They are **not** stored under `/var/www/asger`. The `asger-web` deploy replaces +that entire directory atomically — it unpacks into `.new`, swaps it in and +deletes the old one — so anything kept inside it would vanish on the next site +deploy. nginx exposes `/srv/tarballz` through a `location /tarballz/` alias +instead, with `gzip off` so makepkg gets a plain byte stream. + +Note that `asger.weirsoe.dk` 301-redirects to the IDN form +`asger.xn--weirse-fya.dk`. makepkg follows redirects, so the readable hostname is +used in the PKGBUILD, but that redirect is load-bearing — if it is ever removed, +the `source=` URL must be repointed. + +The publish step verifies the uploaded copy's sha256 on the host, and a separate +step then fetches the *public* URL and checks it against the sha256 baked into +the PKGBUILD. Both run before the AUR push, so the package can never be published +pointing at a missing or truncated file. ## Runner requirements diff --git a/packaging/PKGBUILD.in b/packaging/PKGBUILD.in index a6852ce..5a0d7fb 100644 --- a/packaging/PKGBUILD.in +++ b/packaging/PKGBUILD.in @@ -5,12 +5,11 @@ # Upstream JASP publishes installers for Windows and macOS only; the official # Linux channel is Flathub. The binary this package installs is therefore NOT an # upstream release artifact. It is compiled from the tagged JASP sources -# (@JASP_TAG@, commit @JASP_COMMIT@) on Arch Linux by an automated -# pipeline, and -# published to GitHub Releases: +# (@JASP_TAG@, commit @JASP_COMMIT@) against an Arch Linux userland, in an +# archlinux:base-devel container, by an automated pipeline: # # build recipe: https://gitea.weircon.dk/agw/jasp-desktop-bin -# release: @DOWNLOAD_URL@ +# download: @DOWNLOAD_URL@ # # It bundles its own R @BUNDLED_R@ and the non-system libraries it links against # under /opt/jasp-desktop, so it does not constrain the versions of anything on diff --git a/scripts/gen-pkgbuild.py b/scripts/gen-pkgbuild.py index 255ad52..e5a18a3 100755 --- a/scripts/gen-pkgbuild.py +++ b/scripts/gen-pkgbuild.py @@ -40,8 +40,8 @@ def main() -> int: ap.add_argument("--build-info", required=True) ap.add_argument("--template", required=True) ap.add_argument("--out", required=True) - ap.add_argument("--gh-owner", required=True) - ap.add_argument("--gh-repo", default="jasp-desktop-bin") + ap.add_argument("--base-url", required=True, + help="public directory the tarball is served from") ap.add_argument("--maintainer", required=True) ap.add_argument("--pkgname", default="jasp-desktop-bin") args = ap.parse_args() @@ -58,10 +58,7 @@ def main() -> int: print(f"new version {jasp_version}: pkgrel 1") tag = f"v{jasp_version}-{pkgrel}" - download_url = ( - f"https://github.com/{args.gh_owner}/{args.gh_repo}" - f"/releases/download/{tag}/{info['tarball']}" - ) + download_url = f"{args.base_url.rstrip('/')}/{info['tarball']}" qt_minor = info["qt6_minor"] diff --git a/scripts/publish-github-release.sh b/scripts/publish-github-release.sh deleted file mode 100755 index 1a0eca7..0000000 --- a/scripts/publish-github-release.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -# Create (or reuse) a GitHub release and upload the JASP tarball to it. -# Uses the REST API directly so the workflow does not depend on `gh` being -# present in the runner image. -# -# GH_TOKEN PAT with `contents: write` on the target repo (required) -# GH_OWNER GitHub account owning the release repo (required) -# GH_REPO release repo name (default jasp-desktop-bin) -# TAG release tag, e.g. v0.98.1-1 (required) -# ASSET path to the tarball to upload (required) - -set -euo pipefail - -: "${GH_TOKEN:?GH_TOKEN is required}" -: "${GH_OWNER:?GH_OWNER is required}" -: "${TAG:?TAG is required}" -: "${ASSET:?ASSET is required}" -GH_REPO="${GH_REPO:-jasp-desktop-bin}" - -API="https://api.github.com/repos/${GH_OWNER}/${GH_REPO}" -UPLOADS="https://uploads.github.com/repos/${GH_OWNER}/${GH_REPO}" -AUTH=(-H "Authorization: Bearer ${GH_TOKEN}" -H "Accept: application/vnd.github+json") - -asset_name="$(basename "$ASSET")" - -echo ":: looking for existing release $TAG" -release_id="$(curl -sSf "${AUTH[@]}" "${API}/releases/tags/${TAG}" 2>/dev/null \ - | python3 -c 'import json,sys; print(json.load(sys.stdin).get("id",""))' 2>/dev/null || true)" - -if [[ -z "$release_id" ]]; then - echo ":: creating release $TAG" - release_id="$(curl -sSf -X POST "${AUTH[@]}" "${API}/releases" \ - -d "$(python3 -c ' -import json, os -print(json.dumps({ - "tag_name": os.environ["TAG"], - "name": "JASP " + os.environ["TAG"], - "body": os.environ.get("RELEASE_NOTES", ""), - "draft": False, - "prerelease": False, -}))')" | python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])')" -else - echo ":: reusing release $release_id" - # Replace a same-named asset so re-runs are idempotent. - old="$(curl -sSf "${AUTH[@]}" "${API}/releases/${release_id}/assets" \ - | python3 -c " -import json, sys -name = '${asset_name}' -print(next((a['id'] for a in json.load(sys.stdin) if a['name'] == name), '')) -")" - if [[ -n "$old" ]]; then - echo ":: deleting previous asset $old" - curl -sSf -X DELETE "${AUTH[@]}" "${API}/releases/assets/${old}" - fi -fi - -echo ":: uploading $asset_name ($(du -h "$ASSET" | cut -f1))" -curl -sSf -X POST "${AUTH[@]}" \ - -H "Content-Type: application/zstd" \ - --data-binary "@${ASSET}" \ - "${UPLOADS}/releases/${release_id}/assets?name=${asset_name}" >/dev/null - -echo ":: uploaded to https://github.com/${GH_OWNER}/${GH_REPO}/releases/tag/${TAG}" diff --git a/scripts/publish-tarball.sh b/scripts/publish-tarball.sh new file mode 100755 index 0000000..5e0ad10 --- /dev/null +++ b/scripts/publish-tarball.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Publish the built tarball to asger.weirsoe.dk, which is what the AUR PKGBUILD +# downloads from. +# +# TARBALL_SSH_KEY private key authorised on the web host (required) +# TARBALL_HOST web host, e.g. 192.168.10.148 (required) +# TARBALL_PATH directory on that host (required) +# KEEP_RELEASES how many tarballs to retain (default 3) +# $1 path to the tarball to upload +# +# The files live in /srv/tarballz, deliberately NOT under /var/www/asger: the +# asger-web deploy replaces that directory atomically (build into .new, swap, +# delete .old), so anything stored inside it disappears on the next site deploy. +# nginx exposes /srv/tarballz via a `location /tarballz/` alias. + +set -euo pipefail + +TARBALL="${1:?path to tarball is required}" +: "${TARBALL_SSH_KEY:?TARBALL_SSH_KEY is required}" +: "${TARBALL_HOST:?TARBALL_HOST is required}" +: "${TARBALL_PATH:?TARBALL_PATH is required}" +KEEP_RELEASES="${KEEP_RELEASES:-3}" + +name="$(basename "$TARBALL")" + +mkdir -p ~/.ssh +printf '%s\n' "$TARBALL_SSH_KEY" > ~/.ssh/tarball_key +chmod 600 ~/.ssh/tarball_key +ssh-keyscan -t ed25519 "$TARBALL_HOST" >> ~/.ssh/known_hosts 2>/dev/null || true +SSH=(ssh -i "$HOME/.ssh/tarball_key" -o StrictHostKeyChecking=accept-new) + +echo ":: uploading ${name} ($(du -h "$TARBALL" | cut -f1)) to ${TARBALL_HOST}:${TARBALL_PATH}" + +# Upload to a temporary name and rename on arrival, so a download that starts +# mid-transfer can never see a truncated file at the final URL. +"${SSH[@]}" "root@${TARBALL_HOST}" "mkdir -p '${TARBALL_PATH}'" +"${SSH[@]}" "root@${TARBALL_HOST}" "cat > '${TARBALL_PATH}/.${name}.part'" < "$TARBALL" +"${SSH[@]}" "root@${TARBALL_HOST}" "mv '${TARBALL_PATH}/.${name}.part' '${TARBALL_PATH}/${name}' && chmod 644 '${TARBALL_PATH}/${name}'" + +echo ":: verifying the uploaded copy matches" +local_sum="$(sha256sum "$TARBALL" | awk '{print $1}')" +remote_sum="$("${SSH[@]}" "root@${TARBALL_HOST}" "sha256sum '${TARBALL_PATH}/${name}'" | awk '{print $1}')" +if [[ "$local_sum" != "$remote_sum" ]]; then + echo "error: checksum mismatch after upload" >&2 + echo " local $local_sum" >&2 + echo " remote $remote_sum" >&2 + exit 1 +fi +echo ":: sha256 matches ($local_sum)" + +# Old releases are kept so that a user mid-install of the previous version is +# not left with a dead URL, but not forever - each one is well over a gigabyte. +echo ":: pruning to the newest ${KEEP_RELEASES} tarballs" +"${SSH[@]}" "root@${TARBALL_HOST}" " + cd '${TARBALL_PATH}' || exit 0 + ls -1t jasp-desktop-*.tar.zst 2>/dev/null | tail -n +\$((${KEEP_RELEASES} + 1)) | while read -r old; do + echo \" removing \$old\" + rm -f -- \"\$old\" + done + echo ' remaining:' + ls -1sh jasp-desktop-*.tar.zst 2>/dev/null | sed 's/^/ /' +" + +echo ":: published"