From 986c212b82e8819934fcdcd747f387b882b589a7 Mon Sep 17 00:00:00 2001 From: Bryan Joshua Pedini Date: Sun, 26 Jul 2026 22:50:07 +0200 Subject: [PATCH] Replace per-package push scripts with a self-healing AUR reconciler The six nightly workflows each cloned a package repo and ran that repo's own push.sh. Two problems compounded into months of silent breakage. push.sh pushed to origin before the AUR. When the AUR push failed, origin already carried the new version, so every later run hit "same (old) version specified", bare-exit 0'd, and the workflow went green while the AUR rotted. reflex-appimage sat at 1.0.11 locally against 1.0.10 published; nethlink-appimage was never submitted at all. The AUR pushes were failing because .SRCINFO was empty -- 0 bytes in reflex-appimage, 1 byte in nethlink-appimage and pman-helper, against 500-2000 in the healthy ones. The AUR rejects an invalid .SRCINFO. Both PKGBUILDs verify clean under a real Arch makepkg, so this was the CI environment, and with no set -e the script committed and pushed the truncated file regardless. pman-helper failed independently: the workflows queried the Gitea endpoint /repos/{owner}/{repo}/repo/tags, which 404s -- the correct path is /tags. The empty result fell through to an interactive read that got EOF, and it committed pkgver="" plus the sha256 of a zero-byte download. The replacement is a reconciler rather than a version-bump script. A new upstream release, a manual push to origin, and a push that failed last night all take the same path, and the AUR reconcile runs on every pass -- so a failed push simply retries. Sync is decided by comparing tree content against the AUR, not by whether this run happened to find something new. pkgrel is now derived from the AUR rather than from local history, which is what makes it idempotent: a version differing from the published one resets it to 1, a packaging change increments it, and once published the next run sees no difference and does nothing. Editing a PKGBUILD no longer needs a manual bump. Checksums are computed directly instead of via makepkg -g / updpkgsums. Those generate every arch array in one pass into a single directory, so when two arches rename to the same target the second finds the first one's file and emits an identical, silently wrong sum -- verified against freetube-appimage. Downloads go to content-addressed cache paths instead. Multi-arch works for when it is needed; single-arch is the degenerate case of the same loop. Guards for each observed failure: a real Arch makepkg in an archlinux container as non-root, set -euo pipefail throughout, .SRCINFO validated against the PKGBUILD before any commit, and a failed version lookup exiting 1 so it can never again be mistaken for a quiet "no new version" night. Package-specific detail moves out of this repo entirely, into an .autoupdate manifest committed to each package repo. sha256 only; sha1sums and md5sums are removed. deskflow-bin and freetube-appimage are dropped from automation, having been deleted from the AUR. Covered by test.sh: 37 assertions against local bare repos and a local HTTP server, no network and never touching the real AUR. The retry, pkgrel idempotence and multi-arch collision tests were each verified by mutation. Co-Authored-By: Claude Opus 5 --- .gitea/workflows/deskflow-bin.yaml | 34 -- .gitea/workflows/nethlink-appimage.yaml | 34 -- .../open-video-downloader-appimage.yaml | 34 -- .../workflows/open-video-downloader-bin.yaml | 34 -- .gitea/workflows/pman-helper.yaml | 34 -- .gitea/workflows/reflex-appimage.yaml | 34 -- .gitea/workflows/update.yaml | 122 +++++ README.md | 132 +++++- autoupdate.sh | 266 +++++++++++ common.sh | 200 ++++++++ latest-version.sh | 81 ++++ sync-aur.sh | 79 ++++ test.sh | 441 ++++++++++++++++++ update-checksums.sh | 105 +++++ 14 files changed, 1425 insertions(+), 205 deletions(-) delete mode 100644 .gitea/workflows/deskflow-bin.yaml delete mode 100644 .gitea/workflows/nethlink-appimage.yaml delete mode 100644 .gitea/workflows/open-video-downloader-appimage.yaml delete mode 100644 .gitea/workflows/open-video-downloader-bin.yaml delete mode 100644 .gitea/workflows/pman-helper.yaml delete mode 100644 .gitea/workflows/reflex-appimage.yaml create mode 100644 .gitea/workflows/update.yaml create mode 100755 autoupdate.sh create mode 100755 common.sh create mode 100755 latest-version.sh create mode 100755 sync-aur.sh create mode 100755 test.sh create mode 100755 update-checksums.sh diff --git a/.gitea/workflows/deskflow-bin.yaml b/.gitea/workflows/deskflow-bin.yaml deleted file mode 100644 index a9a0952..0000000 --- a/.gitea/workflows/deskflow-bin.yaml +++ /dev/null @@ -1,34 +0,0 @@ ---- -name: Update deskflow-bin Package - -on: - schedule: - - cron: "0 0 * * *" - -defaults: - run: - shell: bash - -jobs: - update: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - run: | - apt-get --quiet update && apt-get --quiet --assume-yes install makepkg - git config --global user.name "${GIT_NAME}" - git config --global user.email "${GIT_EMAIL}" - git config --global credential.helper store - echo "https://${GIT_EMAIL//@/%40}:${GIT_TOKEN}@${GIT_HOST}" > ~/.git-credentials - git clone --depth 1 https://${GIT_HOST}/aur/deskflow-bin - cd deskflow-bin - source PKGBUILD - ./push.sh $(curl -L -H "Accept: application/vnd.github+json" \ - https://api.github.com/repos/${_srcmntr}/${_srcname}/releases/latest | jq -r ".tag_name" | sed 's|v||') - env: - GIT_NAME: ${{ vars.GIT_NAME }} - GIT_EMAIL: ${{ vars.GIT_EMAIL }} - GIT_HOST: ${{ vars.GIT_HOST }} - GIT_TOKEN: ${{ secrets.GIT_TOKEN }} diff --git a/.gitea/workflows/nethlink-appimage.yaml b/.gitea/workflows/nethlink-appimage.yaml deleted file mode 100644 index da197ba..0000000 --- a/.gitea/workflows/nethlink-appimage.yaml +++ /dev/null @@ -1,34 +0,0 @@ ---- -name: Update nethlink-appimage Package - -on: - schedule: - - cron: "0 0 * * *" - -defaults: - run: - shell: bash - -jobs: - update: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - run: | - apt-get --quiet update && apt-get --quiet --assume-yes install makepkg - git config --global user.name "${GIT_NAME}" - git config --global user.email "${GIT_EMAIL}" - git config --global credential.helper store - echo "https://${GIT_EMAIL//@/%40}:${GIT_TOKEN}@${GIT_HOST}" > ~/.git-credentials - git clone --depth 1 https://${GIT_HOST}/aur/nethlink-appimage - cd nethlink-appimage - source PKGBUILD - ./push.sh $(curl -L -H "Accept: application/vnd.github+json" \ - https://api.github.com/repos/${_srcmntr}/${_srcname}/releases/latest | jq -r ".tag_name" | sed 's|v||') - env: - GIT_NAME: ${{ vars.GIT_NAME }} - GIT_EMAIL: ${{ vars.GIT_EMAIL }} - GIT_HOST: ${{ vars.GIT_HOST }} - GIT_TOKEN: ${{ secrets.GIT_TOKEN }} diff --git a/.gitea/workflows/open-video-downloader-appimage.yaml b/.gitea/workflows/open-video-downloader-appimage.yaml deleted file mode 100644 index 16d5e90..0000000 --- a/.gitea/workflows/open-video-downloader-appimage.yaml +++ /dev/null @@ -1,34 +0,0 @@ ---- -name: Update open-video-downloader-appimage Package - -on: - schedule: - - cron: "0 0 * * *" - -defaults: - run: - shell: bash - -jobs: - update: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - run: | - apt-get --quiet update && apt-get --quiet --assume-yes install makepkg - git config --global user.name "${GIT_NAME}" - git config --global user.email "${GIT_EMAIL}" - git config --global credential.helper store - echo "https://${GIT_EMAIL//@/%40}:${GIT_TOKEN}@${GIT_HOST}" > ~/.git-credentials - git clone --depth 1 https://${GIT_HOST}/aur/open-video-downloader-appimage - cd open-video-downloader-appimage - source PKGBUILD - ./push.sh $(curl -L -H "Accept: application/vnd.github+json" \ - https://api.github.com/repos/${_srcmntr}/${_srcname}/releases/latest | jq -r ".tag_name" | sed 's|app-v||') - env: - GIT_NAME: ${{ vars.GIT_NAME }} - GIT_EMAIL: ${{ vars.GIT_EMAIL }} - GIT_HOST: ${{ vars.GIT_HOST }} - GIT_TOKEN: ${{ secrets.GIT_TOKEN }} diff --git a/.gitea/workflows/open-video-downloader-bin.yaml b/.gitea/workflows/open-video-downloader-bin.yaml deleted file mode 100644 index 7202c78..0000000 --- a/.gitea/workflows/open-video-downloader-bin.yaml +++ /dev/null @@ -1,34 +0,0 @@ ---- -name: Update open-video-downloader-bin Package - -on: - schedule: - - cron: "0 0 * * *" - -defaults: - run: - shell: bash - -jobs: - update: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - run: | - apt-get --quiet update && apt-get --quiet --assume-yes install makepkg - git config --global user.name "${GIT_NAME}" - git config --global user.email "${GIT_EMAIL}" - git config --global credential.helper store - echo "https://${GIT_EMAIL//@/%40}:${GIT_TOKEN}@${GIT_HOST}" > ~/.git-credentials - git clone --depth 1 https://${GIT_HOST}/aur/open-video-downloader-bin - cd open-video-downloader-bin - source PKGBUILD - ./push.sh $(curl -L -H "Accept: application/vnd.github+json" \ - https://api.github.com/repos/${_srcmntr}/${_srcname}/releases/latest | jq -r ".tag_name" | sed 's|app-v||') - env: - GIT_NAME: ${{ vars.GIT_NAME }} - GIT_EMAIL: ${{ vars.GIT_EMAIL }} - GIT_HOST: ${{ vars.GIT_HOST }} - GIT_TOKEN: ${{ secrets.GIT_TOKEN }} diff --git a/.gitea/workflows/pman-helper.yaml b/.gitea/workflows/pman-helper.yaml deleted file mode 100644 index 3d93889..0000000 --- a/.gitea/workflows/pman-helper.yaml +++ /dev/null @@ -1,34 +0,0 @@ ---- -name: Update pman-helper Package - -on: - schedule: - - cron: "0 0 * * *" - -defaults: - run: - shell: bash - -jobs: - update: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - run: | - apt-get --quiet update && apt-get --quiet --assume-yes install makepkg - git config --global user.name "${GIT_NAME}" - git config --global user.email "${GIT_EMAIL}" - git config --global credential.helper store - echo "https://${GIT_EMAIL//@/%40}:${GIT_TOKEN}@${GIT_HOST}" > ~/.git-credentials - git clone --depth 1 https://${GIT_HOST}/aur/pman-helper - cd pman-helper - source PKGBUILD - ./push.sh $(curl -L -H "Accept: application/json" \ - https://git.bjphoster.com/api/v1/repos/${_srcmntr}/${_srcname}/repo/tags | jq -r ".[].name" | sort -V | tail -n 1) - env: - GIT_NAME: ${{ vars.GIT_NAME }} - GIT_EMAIL: ${{ vars.GIT_EMAIL }} - GIT_HOST: ${{ vars.GIT_HOST }} - GIT_TOKEN: ${{ secrets.GIT_TOKEN }} diff --git a/.gitea/workflows/reflex-appimage.yaml b/.gitea/workflows/reflex-appimage.yaml deleted file mode 100644 index f18630f..0000000 --- a/.gitea/workflows/reflex-appimage.yaml +++ /dev/null @@ -1,34 +0,0 @@ ---- -name: Update reflex-appimage Package - -on: - schedule: - - cron: "0 0 * * *" - -defaults: - run: - shell: bash - -jobs: - update: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - run: | - apt-get --quiet update && apt-get --quiet --assume-yes install makepkg - git config --global user.name "${GIT_NAME}" - git config --global user.email "${GIT_EMAIL}" - git config --global credential.helper store - echo "https://${GIT_EMAIL//@/%40}:${GIT_TOKEN}@${GIT_HOST}" > ~/.git-credentials - git clone --depth 1 https://${GIT_HOST}/aur/reflex-appimage - cd reflex-appimage - source PKGBUILD - ./push.sh $(curl -L -H "Accept: application/vnd.github+json" \ - https://api.github.com/repos/${_srcmntr}/${_srcname}/releases/latest | jq -r ".tag_name" | sed 's|v||') - env: - GIT_NAME: ${{ vars.GIT_NAME }} - GIT_EMAIL: ${{ vars.GIT_EMAIL }} - GIT_HOST: ${{ vars.GIT_HOST }} - GIT_TOKEN: ${{ secrets.GIT_TOKEN }} diff --git a/.gitea/workflows/update.yaml b/.gitea/workflows/update.yaml new file mode 100644 index 0000000..dcf7601 --- /dev/null +++ b/.gitea/workflows/update.yaml @@ -0,0 +1,122 @@ +--- +name: Update AUR Packages + +on: + schedule: + - cron: "0 0 * * *" + workflow_dispatch: + inputs: + package: + description: "Single package to update (blank = all)" + required: false + type: string + force_rebuild: + description: "Bump pkgrel even with no detected change" + required: false + type: boolean + default: false + force_refresh: + description: "Re-download remote sources even without a version change" + required: false + type: boolean + default: false + no_git_push: + description: "Dry run: do everything except push" + required: false + type: boolean + default: false + +defaults: + run: + shell: bash + +jobs: + update: + runs-on: ubuntu-latest + # A real Arch makepkg is required. Ubuntu's build environment silently + # emits an empty .SRCINFO, which the AUR then rejects -- that is what broke + # reflex-appimage, nethlink-appimage and pman-helper. regen_srcinfo() in + # common.sh fails loudly if this ever regresses. + container: + image: archlinux:base-devel + + strategy: + # One broken package must not cancel the rest of the matrix. + fail-fast: false + matrix: + package: + - nethlink-appimage + - open-video-downloader-appimage + - open-video-downloader-bin + - pman-helper + - reflex-appimage + - tsparams + + steps: + - uses: actions/checkout@v4 + + - name: Install dependencies + run: | + pacman -Syu --noconfirm --needed \ + git openssh jq curl pacman-contrib namcap fuse2 python + + - name: Configure git and SSH + env: + GIT_NAME: ${{ vars.GIT_NAME }} + GIT_EMAIL: ${{ vars.GIT_EMAIL }} + GIT_HOST: ${{ vars.GIT_HOST }} + GIT_TOKEN: ${{ secrets.GIT_TOKEN }} + AUR_SSH_KEY: ${{ secrets.AUR_SSH_KEY }} + run: | + set -euo pipefail + git config --global user.name "${GIT_NAME}" + git config --global user.email "${GIT_EMAIL}" + git config --global credential.helper store + echo "https://${GIT_EMAIL//@/%40}:${GIT_TOKEN}@${GIT_HOST}" > ~/.git-credentials + chmod 600 ~/.git-credentials + + mkdir -p ~/.ssh && chmod 700 ~/.ssh + printf '%s\n' "${AUR_SSH_KEY}" > ~/.ssh/aur + chmod 600 ~/.ssh/aur + ssh-keyscan -t rsa,ecdsa,ed25519 aur.archlinux.org >> ~/.ssh/known_hosts 2>/dev/null + cat >> ~/.ssh/config <<'EOF' + Host aur.archlinux.org + User aur + IdentityFile ~/.ssh/aur + IdentitiesOnly yes + EOF + chmod 600 ~/.ssh/config + + # makepkg refuses to run as root, and CI containers are root by default. + - name: Create build user + run: | + set -euo pipefail + useradd --create-home --shell /bin/bash builder + chown -R builder:builder "${GITHUB_WORKSPACE}" + cp -r /root/.ssh /root/.gitconfig /root/.git-credentials /home/builder/ 2>/dev/null || true + chown -R builder:builder /home/builder + + - name: Update ${{ matrix.package }} + env: + GIT_HOST: ${{ vars.GIT_HOST }} + # Compared against both `true` and 'true' on purpose. A bare + # `${{ inputs.x && '1' || '' }}` is wrong if the runner hands the + # input over as the STRING "false", which is truthy -- that would + # silently enable the flag on every run. + FORCE_REBUILD: ${{ (inputs.force_rebuild == true || inputs.force_rebuild == 'true') && '1' || '' }} + FORCE_REFRESH: ${{ (inputs.force_refresh == true || inputs.force_refresh == 'true') && '1' || '' }} + NO_GIT_PUSH: ${{ (inputs.no_git_push == true || inputs.no_git_push == 'true') && '1' || '' }} + run: | + set -euo pipefail + # workflow_dispatch with a package name runs only that one. + want="${{ inputs.package }}" + if [ -n "${want}" ] && [ "${want}" != "${{ matrix.package }}" ]; then + echo "skipping ${{ matrix.package }} (dispatch asked for ${want})" + exit 0 + fi + runuser -u builder -- env \ + GIT_HOST="${GIT_HOST}" \ + FORCE_REBUILD="${FORCE_REBUILD}" \ + FORCE_REFRESH="${FORCE_REFRESH}" \ + NO_GIT_PUSH="${NO_GIT_PUSH}" \ + ./autoupdate.sh "${{ matrix.package }}" diff --git a/README.md b/README.md index 6f186f3..a8d2907 100644 --- a/README.md +++ b/README.md @@ -1 +1,131 @@ -# AUR Auto Update Workflows +# AUR Auto Update + +Nightly reconciler that keeps the `aur/*` package repos and their AUR +counterparts in sync. + +This is a **reconciler**, not a version-bump script. A new upstream release is +only one of the things that can make a package diverge from the AUR; a manual +push to origin is another, and a push that failed last night is a third. All +three take the same path, and the AUR reconcile runs on every pass. + +## Layout + +| file | role | +|---|---| +| `autoupdate.sh` | entry point; orchestrates one or more packages | +| `latest-version.sh` | resolves the latest upstream version (GitHub / Gitea) | +| `update-checksums.sh` | downloads sources, computes sha256, rewrites the arrays | +| `sync-aur.sh` | reconciles the repo against the AUR | +| `common.sh` | logging, guards, PKGBUILD parsing and array splicing | +| `test.sh` | hermetic test suite | +| `.gitea/workflows/update.yaml` | nightly matrix job | + +Nothing here is package-specific. Everything that differs between packages +lives in a `.autoupdate` manifest committed to the package repo itself. + +## Per-package manifest + +Each package repo carries a `.autoupdate` file: + +```sh +srchost="github" # github | gitea +srcmntr="Sunhaiy" # upstream owner/org +srcname="Reflex" # upstream repo +verstrip="v" # tag prefix to strip ("app-v" for open-video-downloader-*) +gui="true" # extract .desktop + icon from the AppImage +desktop="reflex.desktop" # name *inside* the AppImage +icon="reflex.png" +``` + +It rides along to the AUR on push, as `README.md` and `LICENSE` already do. +Harmless: the AUR reads only `PKGBUILD` and `.SRCINFO`. + +## What a run does + +1. Clone the package repo, read its manifest. +2. Fetch the AUR over **HTTPS** as the reference state. This happens before + anything else, because `pkgrel` is derived from it. Read-only, so no SSH key + is required for a dry run. +3. Resolve the upstream version. A *failed lookup* exits 1; "no new version" is + not a failure. +4. If the version moved, re-download and re-hash the remote sources. +5. Always re-hash local sources (`.desktop`, icons) and regenerate `.SRCINFO`. + Local hashing needs no network, so this is free, and it is what lets a + hand-edited repo self-correct. +6. Derive `pkgrel` (below). +7. Validate `.SRCINFO`; abort **before** committing if it is empty or disagrees + with the PKGBUILD. +8. Commit only if something actually changed. +9. Push to the AUR first, then origin, so a failed AUR push leaves origin clean. + +## pkgrel + +`X.Y.Z-N` — everything before the dash is upstream, `-N` is packaging-only +revisions. You should never need to touch it by hand. + +`pkgrel` is derived from **the AUR**, because that is what `-N` is numbered +against: + +- version differs from the published one → `pkgrel = 1` +- same version, packaging content differs → `pkgrel = AUR's + 1` +- same version, packaging identical → unchanged +- you already bumped it yourself → respected, not bumped again + +Using the AUR as the reference rather than local history is what makes this +idempotent: once the bump is published, the next night sees no difference and +does nothing. There is a test for exactly that (`pkgrel stable on re-run`). + +An edit to a local source such as an icon counts as a packaging change, because +its `sha256sums` entry is regenerated first. A README-only edit does not. + +## Environment + +| variable | effect | +|---|---| +| `NO_GIT_PUSH=1` | dry run: do everything except push | +| `FORCE_REBUILD=1` | bump `pkgrel` even with no detected change | +| `FORCE_REFRESH=1` | re-download remote sources without a version change | +| `AUR_FORCE=1` | force-push over diverged AUR history (see below) | +| `GIT_HOST` | git host holding the package repos | +| `GITHUB_TOKEN` | optional; avoids GitHub's 60/hour anonymous rate limit | + +A rejected AUR push usually means someone pushed there directly. The run fails +loudly rather than overwriting it; inspect, then re-run with `AUR_FORCE=1`. + +## Exit codes + +"No new upstream version" is the overwhelmingly common outcome and exits **0**. +The trap the previous implementation fell into was giving that same silent +`exit 0` to a lookup that had *broken* — which is how `pman-helper` shipped +`pkgver=""` unnoticed. A failed lookup now exits **1**. + +## Requirements + +Needs a genuine Arch `makepkg`. Ubuntu's build environment emits an empty +`.SRCINFO`, which the AUR rejects — this silently broke three packages. The CI +job therefore runs in an `archlinux:base-devel` container as a non-root user +(`makepkg` refuses to run as root), and `regen_srcinfo` in `common.sh` fails +loudly if the output is ever empty again. + +## Notes on checksums + +sha256 only; `sha1sums`/`md5sums` are actively removed. + +Checksums are computed directly rather than via `makepkg -g` / `updpkgsums`. +Those generate every arch array in one pass and download all arches into a +single directory, so when two arches rename to the same target +(`foo.AppImage::…-amd64`, `foo.AppImage::…-arm64`) the second finds the first +one's file already there and emits an identical, silently wrong checksum. +Overriding `CARCH` does not help. Downloads here go to content-addressed cache +paths instead, never to the `::` rename target. Multi-arch is supported for +when a package needs it; single-arch is the degenerate case of the same loop. + +## Tests + +```sh +./test.sh # all +./test.sh pkgrel # filter by name +``` + +Runs against local bare repos and a local HTTP server — no network, and it +never touches the real AUR. Requires a real `makepkg`. diff --git a/autoupdate.sh b/autoupdate.sh new file mode 100755 index 0000000..98902de --- /dev/null +++ b/autoupdate.sh @@ -0,0 +1,266 @@ +#!/usr/bin/env bash +# Nightly AUR package reconciler. +# +# usage: autoupdate.sh [...] +# +# env: +# GIT_HOST git host holding the package repos (default git.bjphoster.com) +# WORKDIR where to clone; a temp dir by default +# NO_GIT_PUSH=1 do everything except push (dry run) +# FORCE_REBUILD=1 bump pkgrel even with no detected change +# FORCE_REFRESH=1 re-download remote sources even without a version change +# AUR_FORCE=1 force-push over diverged AUR history +# +# This is a RECONCILER, not a version-bump script. A new upstream release is +# only one of the things that can make a package diverge from the AUR; a manual +# push to origin is another, and a failed push last night is a third. All three +# funnel through the same path, and the AUR reconcile at the end runs +# unconditionally. + +set -euo pipefail + +_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${_here}/common.sh" + +require_noninteractive + +GIT_HOST="${GIT_HOST:-git.bjphoster.com}" +# Where the package repos live, and how the upstream version is resolved. +# Both are overridable so the test suite can run hermetically against local +# bare repos; the defaults are the production values. +PKG_REPO_BASE="${PKG_REPO_BASE:-https://${GIT_HOST}/aur}" +LATEST_VERSION_CMD="${LATEST_VERSION_CMD:-${_here}/latest-version.sh}" + +[ "$#" -ge 1 ] || die "usage: autoupdate.sh [...]" + +WORKDIR="${WORKDIR:-$(mktemp -d)}" +mkdir -p "$WORKDIR" + +update_one() { + local pkg="$1" + local dir="${WORKDIR}/${pkg}" + + log "=== ${pkg} ===" + + if [ -d "$dir/.git" ]; then + git -C "$dir" fetch --quiet origin + git -C "$dir" reset --quiet --hard origin/HEAD + else + rm -rf "$dir" + git clone --quiet "${PKG_REPO_BASE}/${pkg}" "$dir" \ + || die "could not clone ${pkg} from ${PKG_REPO_BASE}" + fi + + cd "$dir" + [ -f PKGBUILD ] || die "${pkg} has no PKGBUILD" + [ -f .autoupdate ] || die "${pkg} has no .autoupdate manifest" + + # shellcheck disable=SC1091 + source ./.autoupdate + + # Per-run download cache, shared between the AppImage extraction below and + # update-checksums.sh so a 100 MB AppImage is fetched at most once. + export DL_CACHE="${WORKDIR}/.cache" + mkdir -p "$DL_CACHE" + + # --- AUR reference state ------------------------------------------------- + # Fetched up front: it is the reference for the pkgrel decision, not just a + # push target. Comparing against the AUR rather than against local history is + # what makes the pkgrel bump idempotent -- once pushed, the AUR matches and + # the next run finds nothing to do. + # Read-only HTTPS: establishing the reference state needs no SSH key, so + # dry runs work anywhere. Only sync-aur.sh's push is authenticated. + local aur_pkgbuild="" aur_pkgrel="" aur_pkgver="" aur_name + aur_name="$(pkgbuild_expand PKGBUILD 'printf %s "$pkgname"')" + git remote remove aur-ro >/dev/null 2>&1 || true + git remote add aur-ro "${AUR_HTTPS_BASE}/${aur_name}.git" + if git fetch --quiet aur-ro master 2>/dev/null; then + aur_pkgbuild="${WORKDIR}/${pkg}.aur.PKGBUILD" + if git show FETCH_HEAD:PKGBUILD > "$aur_pkgbuild" 2>/dev/null; then + aur_pkgrel="$(pkgbuild_get pkgrel "$aur_pkgbuild")" + aur_pkgver="$(pkgbuild_get pkgver "$aur_pkgbuild")" + info "AUR has ${aur_pkgver}-${aur_pkgrel}" + else + aur_pkgbuild="" + fi + else + info "not on the AUR yet" + fi + + # --- upstream version ---------------------------------------------------- + # A failure in here exits non-zero and is NOT mistaken for "no new version". + local oldver newver + oldver="$(pkgbuild_get pkgver)" + newver="$("$LATEST_VERSION_CMD" .)" + info "upstream ${newver}, local ${oldver:-}" + + local version_changed=0 + if [ "$newver" != "$oldver" ]; then + version_changed=1 + log "new upstream version ${oldver:-} -> ${newver}" + pkgbuild_set pkgver "$newver" + # pkgrel is not touched here; apply_pkgrel_rule derives it from the AUR. + fi + + # --- AppImage assets ----------------------------------------------------- + if [ "${gui:-false}" = "true" ] && [ "$version_changed" -eq 1 ]; then + extract_appimage_assets + fi + + # --- checksums + .SRCINFO ------------------------------------------------ + # Local sources are always re-hashed (free, no network) so a hand-edited + # .desktop or icon self-corrects. Remote sources are only re-fetched when the + # version moved, or on demand. + local cs_args=() + if [ "$version_changed" -eq 1 ] || [ -n "${FORCE_REFRESH:-}" ]; then + cs_args+=(--remote) + fi + "${_here}/update-checksums.sh" "${cs_args[@]}" . + + regen_srcinfo . + + # --- pkgrel --------------------------------------------------------------- + apply_pkgrel_rule + + # --- commit --------------------------------------------------------------- + git add -A + if git diff --cached --quiet; then + # Nothing changed. Deliberately do NOT exit here: the AUR may still be out + # of sync from a previous failed push, and an unguarded `git commit` with + # nothing staged would exit non-zero and, under `set -e`, abort before the + # reconcile. + log "no local changes for ${pkg}" + else + local ver rel + ver="$(pkgbuild_get pkgver)"; rel="$(pkgbuild_get pkgrel)" + git commit --quiet -m "Updated ${pkg} to ${ver}-${rel}" + log "committed ${pkg} ${ver}-${rel}" + fi + + # --- publish -------------------------------------------------------------- + # AUR first: if it rejects the push, origin is left unpoisoned and the whole + # thing is recomputed cleanly on the next run. + "${_here}/sync-aur.sh" . + + if [ -n "${NO_GIT_PUSH:-}" ]; then + log "NO_GIT_PUSH set: not pushing ${pkg} to origin" + else + git push --quiet origin HEAD + fi +} + +# Derive pkgrel so that fixing a package() bug and pushing needs no manual bump. +apply_pkgrel_rule() { + local cur_ver cur_rel + cur_ver="$(pkgbuild_get pkgver)" + cur_rel="$(pkgbuild_get pkgrel)" + + # Nothing on the AUR to compare against (new package): push as-is. + if [ -z "${aur_pkgbuild:-}" ]; then + return 0 + fi + + # pkgrel is derived purely from the AUR reference, because the AUR is what + # `-N` is numbered against. If the version we are about to publish differs + # from the published one, this is a new upstream version as far as the AUR is + # concerned and the packaging revision restarts at 1 -- whether the bump + # happened in this run or in an earlier one that failed to push. That second + # case is real: reflex-appimage sat at 1.0.11-4 locally against 1.0.10-4 on + # the AUR, because the old script only reset pkgrel under FORCE_REBUILD. + if [ "$cur_ver" != "${aur_pkgver:-}" ]; then + if [ "$cur_rel" != "1" ]; then + pkgbuild_set pkgrel "1" + log "new version relative to the AUR (${aur_pkgver} -> ${cur_ver}); pkgrel ${cur_rel} -> 1" + regen_srcinfo . + fi + return 0 + fi + + local base_rel="${aur_pkgrel:-0}" + [[ "$base_rel" =~ ^[0-9]+$ ]] || base_rel=0 + + if [ -n "${FORCE_REBUILD:-}" ]; then + pkgbuild_set pkgrel "$(( base_rel + 1 ))" + log "FORCE_REBUILD: pkgrel -> $(( base_rel + 1 ))" + regen_srcinfo . + return 0 + fi + + # Compare with the pkgrel line stripped, so the decision is about substantive + # packaging content. An edit to a local source lands here too, because its + # sha256sums entry was just regenerated. A README-only edit does not touch + # the PKGBUILD and so correctly does not bump. + if diff -q <(grep -v '^pkgrel=' PKGBUILD) \ + <(grep -v '^pkgrel=' "$aur_pkgbuild") >/dev/null 2>&1; then + return 0 # identical packaging; nothing to do + fi + + if [ "$cur_rel" = "$aur_pkgrel" ]; then + pkgbuild_set pkgrel "$(( base_rel + 1 ))" + log "packaging changed without a pkgrel bump; pkgrel -> $(( base_rel + 1 ))" + regen_srcinfo . + else + info "pkgrel already moved by hand (${aur_pkgrel} -> ${cur_rel}); leaving it" + fi +} + +# Pull the .desktop and icon out of the AppImage so they can be shipped as +# local sources. Unchanged in substance from the old push.sh. +extract_appimage_assets() { + local host_arr="source_$(uname -m)" entry loc img newdesktop newicon + local candidates=() + + # A multi-arch AppImage is extracted once, against the host arch's image. + if pkgbuild_has_array PKGBUILD "$host_arr"; then + mapfile -t candidates < <(pkgbuild_array PKGBUILD "$host_arr") + fi + if [ "${#candidates[@]}" -eq 0 ]; then + mapfile -t candidates < <(pkgbuild_array PKGBUILD source) + fi + + img="" + for entry in "${candidates[@]}"; do + loc="$(source_loc "$entry")" + if printf '%s' "${entry,,}" | grep -q "appimage" && is_remote "$entry"; then + img="$(download_cached "$loc")" + break + fi + done + [ -n "$img" ] || { warn "gui=true but no AppImage source found; skipping asset extraction"; return 0; } + + local pkgbase + pkgbase="$(pkgbuild_expand PKGBUILD 'printf %s "${_pkgname:-$pkgname}"')" + newdesktop="${pkgbase}.desktop" + newicon="${pkgbase}.png" + + local tmp; tmp="$(mktemp -d)" + cp "$img" "${tmp}/app.AppImage" + chmod +x "${tmp}/app.AppImage" + ( cd "$tmp" && ./app.AppImage --appimage-extract >/dev/null 2>&1 ) \ + || { warn "could not extract ${pkgbase} AppImage; keeping existing assets"; rm -rf "$tmp"; return 0; } + + if [ -f "${tmp}/squashfs-root/${desktop}" ]; then + cp "${tmp}/squashfs-root/${desktop}" "$newdesktop" + sed -i "/^X-AppImage-/d;/^$/d" "$newdesktop" + sed -i "s|^Exec=.*|Exec=${pkgbase} %U|" "$newdesktop" + info "extracted ${newdesktop}" + else + warn "${desktop} not found inside the AppImage" + fi + + if [ -f "${tmp}/squashfs-root/${icon}" ]; then + cp "${tmp}/squashfs-root/${icon}" "$newicon" + info "extracted ${newicon}" + else + warn "${icon} not found inside the AppImage" + fi + + rm -rf "$(readlink -f "${tmp}/squashfs-root" 2>/dev/null || true)" "$tmp" +} + +rc=0 +for pkg in "$@"; do + ( update_one "$pkg" ) || { rc=1; warn "${pkg} FAILED"; } +done +exit "$rc" diff --git a/common.sh b/common.sh new file mode 100755 index 0000000..579a356 --- /dev/null +++ b/common.sh @@ -0,0 +1,200 @@ +#!/usr/bin/env bash +# Shared helpers for the AUR autoupdater. +# Sourced by autoupdate.sh and the standalone scripts; never executed directly. + +set -euo pipefail + +# Endpoints. Overridable so the test suite can point at local bare repos +# instead of the real AUR; defaults are the production values. +AUR_SSH_BASE="${AUR_SSH_BASE:-ssh://aur@aur.archlinux.org}" +AUR_HTTPS_BASE="${AUR_HTTPS_BASE:-https://aur.archlinux.org}" + +log() { printf '==> %s\n' "$*"; } +info() { printf ' -> %s\n' "$*"; } +warn() { printf 'WARNING: %s\n' "$*" >&2; } +die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } + +# The old push.sh fell back to an interactive `read` when no version was passed. +# In CI that reads EOF and yields an empty version, which is how pman-helper +# committed pkgver="" and the sha256 of a zero-byte download. +require_noninteractive() { + [ -t 0 ] && die "refusing to prompt: this script must never depend on interactive input" + return 0 +} + +# A version we are willing to write into a PKGBUILD. +VERSION_RE='^[0-9][0-9a-zA-Z._+]*$' + +validate_version() { + local ver="${1-}" what="${2-version}" + [ -n "$ver" ] || die "${what} is empty" + [[ "$ver" =~ $VERSION_RE ]] || die "${what} '${ver}' does not look like a version" + return 0 +} + +# Read a scalar assignment out of a PKGBUILD without sourcing it. +# Handles pkgver="1.2.3", pkgver='1.2.3' and bare pkgver=1.2.3. +pkgbuild_get() { + local var="$1" file="${2:-PKGBUILD}" + sed -n -E "s/^${var}=[\"']?([^\"']*)[\"']?[[:space:]]*(#.*)?$/\1/p" "$file" | head -n1 +} + +# Rewrite a scalar assignment in place, preserving double-quoted style. +pkgbuild_set() { + local var="$1" val="$2" file="${3:-PKGBUILD}" + grep -qE "^${var}=" "$file" || die "no ${var}= assignment in ${file}" + sed -i -E "s|^${var}=.*|${var}=\"${val}\"|" "$file" +} + +# Expand a PKGBUILD's arrays/vars by sourcing it in a subshell and printing the +# result. Safe enough: these are our own package definitions. Used instead of +# grep/sed so that ${pkgver} / ${_srcname} substitutions actually resolve. +# +# Usage: pkgbuild_expand +pkgbuild_expand() { + local file="$1" snippet="$2" + ( + set +euo pipefail + # Neutralise anything that would run during a real build. + # shellcheck disable=SC1090 + source "$file" >/dev/null 2>&1 || true + eval "$snippet" + ) +} + +# Print the elements of an array variable, one per line, NUL-safe enough for +# source entries (which never contain newlines). +pkgbuild_array() { + local file="$1" name="$2" + pkgbuild_expand "$file" "printf '%s\n' \"\${${name}[@]}\" 2>/dev/null" +} + +# True if the named array exists in the PKGBUILD (even if empty). +pkgbuild_has_array() { + local file="$1" name="$2" + [ "$(pkgbuild_expand "$file" "declare -p ${name} >/dev/null 2>&1 && echo yes")" = "yes" ] +} + +# Replace `name=( ... )` in a PKGBUILD with the given values. +# +# This exists to replace the line-counting sed in the old freetube push.sh +# (`sednewline="n;"` accumulating `n;n;n;`), which silently corrupts the file +# whenever a source array's length changes. Matches `^name=(` through its +# closing `)`, so it handles both one-line and multi-line blocks, and appends +# the array if it is missing entirely. +replace_array() { + local file="$1" name="$2"; shift 2 + local block v + if [ "$#" -eq 1 ]; then + block="${name}=(\"$1\")" + else + block="${name}=(" + for v in "$@"; do block+=$'\n'" \"${v}\""; done + block+=$'\n'")" + fi + + if ! grep -qE "^${name}=\(" "$file"; then + printf '%s\n' "$block" >> "$file" + return 0 + fi + + awk -v name="$name" -v block="$block" ' + BEGIN { skip = 0 } + skip { + # Consume until the line that closes the array. + if ($0 ~ /\)[[:space:]]*$/) skip = 0 + next + } + $0 ~ "^" name "=\\(" { + print block + # A one-line `name=("x")` closes on the same line; a block does not. + if ($0 !~ /\)[[:space:]]*$/) skip = 1 + next + } + { print } + ' "$file" > "${file}.tmp" + + mv "${file}.tmp" "$file" +} + +# Delete a whole `name=( ... )` block (used to retire sha1sums/md5sums). +remove_array() { + local file="$1" name="$2" + grep -qE "^${name}=\(" "$file" || return 0 + awk -v name="$name" ' + BEGIN { skip = 0 } + skip { if ($0 ~ /\)[[:space:]]*$/) skip = 0; next } + $0 ~ "^" name "=\\(" { if ($0 !~ /\)[[:space:]]*$/) skip = 1; next } + { print } + ' "$file" > "${file}.tmp" + mv "${file}.tmp" "$file" +} + +# Split a source entry into its rename target and its URL/path. +# "foo.AppImage::https://x/y.AppImage" -> name=foo.AppImage loc=https://x/y.AppImage +# "https://x/y.tar.gz" -> name=y.tar.gz loc=https://x/y.tar.gz +# "reflex.desktop" -> name=reflex.desktop loc=reflex.desktop +source_name() { + local e="$1" + if [[ "$e" == *"::"* ]]; then printf '%s' "${e%%::*}"; else printf '%s' "${e##*/}"; fi +} +source_loc() { local e="$1"; printf '%s' "${e##*::}"; } + +is_remote() { [[ "$(source_loc "$1")" =~ ^(https?|ftp):// ]]; } + +# VCS sources (git+https://..., svn+..., etc) are not fixed artefacts and +# makepkg expects a literal SKIP for them rather than a digest. +is_vcs() { [[ "$(source_loc "$1")" =~ ^(git|svn|hg|bzr)(\+|:) ]]; } + +# Download a URL once per run and hand back the cached path. +# +# Downloads go to a content-addressed cache path, never to the source entry's +# `::` rename target. That matters for multi-arch: freetube's x86_64 and +# aarch64 entries both rename to freetube.AppImage, so writing to the rename +# target makes the second arch find the first arch's file and silently hash +# the wrong binary. This is the same collision that makes `makepkg -g` +# unusable for multi-arch PKGBUILDs. +download_cached() { + local url="$1" key path + [ -n "${DL_CACHE:-}" ] || die "download_cached: DL_CACHE is not set" + mkdir -p "$DL_CACHE" + key="$(printf '%s' "$url" | sha256sum | cut -c1-32)" + path="${DL_CACHE}/${key}" + + if [ ! -s "$path" ]; then + info "fetching ${url}" >&2 + curl --silent --show-error --location --fail --retry 3 --retry-delay 2 \ + --max-time 900 -o "${path}.part" "$url" \ + || die "download failed: ${url}" + mv "${path}.part" "$path" + fi + printf '%s' "$path" +} + +# Regenerate .SRCINFO and refuse to accept a broken one. +# +# This is the guard that would have prevented every observed breakage: an empty +# or inconsistent .SRCINFO is exactly what the AUR rejects, and the old script +# committed and pushed it anyway because it had no `set -e`. +regen_srcinfo() { + local dir="${1:-.}" + local out rc=0 + out="$(cd "$dir" && makepkg --printsrcinfo 2>&1)" || rc=$? + + if [ "$rc" -ne 0 ] || [ -z "${out//[[:space:]]/}" ]; then + printf '%s\n' "$out" >&2 + die "makepkg --printsrcinfo produced no usable output (exit ${rc}). + Is this running under a real Arch makepkg? The CI job must use the + archlinux container; Ubuntu's build environment silently yields an + empty .SRCINFO, which the AUR then rejects." + fi + + printf '%s\n' "$out" > "${dir}/.SRCINFO" + + local pb_ver sr_ver + pb_ver="$(pkgbuild_get pkgver "${dir}/PKGBUILD")" + sr_ver="$(sed -n -E 's/^[[:space:]]*pkgver = (.*)$/\1/p' "${dir}/.SRCINFO" | head -n1)" + [ -n "$sr_ver" ] || die ".SRCINFO has no pkgver" + [ "$pb_ver" = "$sr_ver" ] \ + || die ".SRCINFO pkgver '${sr_ver}' disagrees with PKGBUILD pkgver '${pb_ver}'" +} diff --git a/latest-version.sh b/latest-version.sh new file mode 100755 index 0000000..7fadec0 --- /dev/null +++ b/latest-version.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Resolve the latest upstream version for a package and print it on stdout. +# +# Reads the package's .autoupdate manifest (srchost/srcmntr/srcname/verstrip). +# +# usage: latest-version.sh [package-dir] +# +# Exit 0 with a version on stdout, or exit 1 with a message on stderr. +# +# There is deliberately NO "returned nothing so assume no update" path. The old +# workflow could not tell a broken query from a quiet night: pman-helper's +# lookup hit a 404, produced an empty string, and push.sh treated it as a +# version. "Query returned nothing" is an error, not a version. + +set -euo pipefail + +_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${_here}/common.sh" + +pkgdir="${1:-.}" +manifest="${pkgdir}/.autoupdate" +[ -f "$manifest" ] || die "no .autoupdate manifest in ${pkgdir}" + +# shellcheck disable=SC1090 +source "$manifest" + +: "${srchost:?manifest is missing srchost}" +: "${srcmntr:?manifest is missing srcmntr}" +: "${srcname:?manifest is missing srcname}" +verstrip="${verstrip:-}" + +GIT_HOST="${GIT_HOST:-git.bjphoster.com}" + +# curl that fails loudly on HTTP errors instead of returning an error page. +fetch() { + local url="$1"; shift + local out rc=0 + out="$(curl --silent --show-error --location --fail-with-body \ + --retry 3 --retry-delay 2 --max-time 60 "$@" "$url" 2>&1)" || rc=$? + if [ "$rc" -ne 0 ]; then + printf '%s\n' "$out" >&2 + die "request to ${url} failed (curl exit ${rc})" + fi + printf '%s' "$out" +} + +case "$srchost" in + github) + url="https://api.github.com/repos/${srcmntr}/${srcname}/releases/latest" + hdrs=(-H "Accept: application/vnd.github+json") + # Optional: avoids the 60/hour unauthenticated rate limit in CI. + [ -n "${GITHUB_TOKEN:-}" ] && hdrs+=(-H "Authorization: Bearer ${GITHUB_TOKEN}") + body="$(fetch "$url" "${hdrs[@]}")" + tag="$(printf '%s' "$body" | jq -r '.tag_name // empty')" + [ -n "$tag" ] || die "no tag_name in the latest release of ${srcmntr}/${srcname}" + ;; + + gitea) + # NB: the correct Gitea path is /repos/{owner}/{repo}/tags. The old + # workflows used /repos/{owner}/{repo}/repo/tags, which 404s. + url="https://${GIT_HOST}/api/v1/repos/${srcmntr}/${srcname}/tags" + body="$(fetch "$url" -H "Accept: application/json")" + printf '%s' "$body" | jq -e 'type == "array"' >/dev/null 2>&1 \ + || die "unexpected response shape from ${url}" + tag="$(printf '%s' "$body" | jq -r '.[].name' | sort -V | tail -n1)" + [ -n "$tag" ] || die "${srcmntr}/${srcname} has no tags" + ;; + + *) + die "unknown srchost '${srchost}' (expected 'github' or 'gitea')" + ;; +esac + +version="$tag" +if [ -n "$verstrip" ]; then + version="${version#"$verstrip"}" +fi + +validate_version "$version" "resolved version (from tag '${tag}')" +printf '%s\n' "$version" diff --git a/sync-aur.sh b/sync-aur.sh new file mode 100755 index 0000000..c62119d --- /dev/null +++ b/sync-aur.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Reconcile a package repo against its AUR counterpart. +# +# usage: sync-aur.sh [package-dir] +# +# This is the fix for the original bug. The old push.sh pushed to origin first +# and to the AUR second, so a failed AUR push left origin already carrying the +# new version; every later run then hit "same (old) version specified" and +# bare-exit 0'd, and the AUR silently rotted forever. +# +# Here the AUR is reconciled from STATE, not from events: this runs on every +# nightly pass regardless of whether a new upstream version was found, so a +# push that failed last night is simply retried tonight. + +set -euo pipefail + +_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${_here}/common.sh" + +pkgdir="${1:-.}" +cd "$pkgdir" + +pkgname="$(pkgbuild_get pkgname)" +# pkgname may be written as "${_pkgname}-appimage"; expand it properly. +case "$pkgname" in + *'$'*) pkgname="$(pkgbuild_expand PKGBUILD 'printf %s "$pkgname"')" ;; +esac +[ -n "$pkgname" ] || die "could not determine pkgname" + +aur_url="${AUR_SSH_BASE}/${pkgname}.git" +# Read over HTTPS so determining sync state needs no SSH key; only the push +# itself is authenticated. This keeps NO_GIT_PUSH dry runs working anywhere. +aur_ro_url="${AUR_HTTPS_BASE}/${pkgname}.git" + +# `git remote add` fails if the remote already exists, which it does whenever +# this runs twice in one working copy. +git remote remove aur >/dev/null 2>&1 || true +git remote add aur "$aur_url" +git remote remove aur-ro >/dev/null 2>&1 || true +git remote add aur-ro "$aur_ro_url" + +if git fetch --quiet aur-ro master 2>/dev/null; then + # Compare the CONTENT of the whole tree rather than commit hashes: origin + # accumulates commits the AUR never needs, and a re-derived commit has a + # different hash but an identical tree. No pathspec, because we push the + # whole tree -- restricting to PKGBUILD/.SRCINFO would report "in sync" + # while .desktop/.png/LICENSE actually differ. + if git diff --quiet FETCH_HEAD HEAD; then + log "AUR already in sync (${pkgname})" + exit 0 + fi + log "AUR differs from origin; pushing (${pkgname})" +else + log "no AUR repo for ${pkgname} yet; the first push will create it" +fi + +if [ -n "${NO_GIT_PUSH:-}" ]; then + log "NO_GIT_PUSH set: skipping push to ${aur_url}" + exit 0 +fi + +if git push aur HEAD:master; then + log "pushed ${pkgname} to the AUR" + exit 0 +fi + +# A rejected push here usually means the AUR has commits origin does not, +# i.e. someone pushed directly. Overwriting that silently would destroy it. +if [ -n "${AUR_FORCE:-}" ]; then + warn "AUR_FORCE set: force-pushing over the AUR history for ${pkgname}" + git push --force aur HEAD:master + exit 0 +fi + +die "push to the AUR was rejected for ${pkgname}. + The AUR history has probably diverged from origin (someone pushed there + directly). Inspect it, then re-run with AUR_FORCE=1 to overwrite. + This run leaves origin untouched and will be retried on the next pass." diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..ece4c66 --- /dev/null +++ b/test.sh @@ -0,0 +1,441 @@ +#!/usr/bin/env bash +# Hermetic tests for the autoupdater. +# +# usage: ./test.sh [name-filter] +# +# Runs against local bare repos standing in for origin and the AUR, with a +# stubbed version lookup, so nothing here touches the network or the real AUR. +# Needs a genuine Arch makepkg (same requirement as the CI job). + +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FILTER="${1:-}" + +PASS=0; FAIL=0; FAILED_NAMES=() + +pass() { printf ' \033[32mok\033[0m %s\n' "$1"; PASS=$((PASS+1)); } +fail() { printf ' \033[31mFAIL\033[0m %s\n %s\n' "$1" "${2-}"; FAIL=$((FAIL+1)); FAILED_NAMES+=("$1"); } + +assert_eq() { # expected actual label + if [ "$1" = "$2" ]; then pass "$3"; else fail "$3" "expected '$1', got '$2'"; fi +} +assert_ne() { + if [ "$1" != "$2" ]; then pass "$3"; else fail "$3" "expected something other than '$1'"; fi +} + +# ---------------------------------------------------------------- sandbox --- + +SANDBOX="" +setup() { + SANDBOX="$(mktemp -d)" + mkdir -p "$SANDBOX"/{origin,aur,work} + + cat > "$SANDBOX/fake-version.sh" <<'EOS' +#!/usr/bin/env bash +# Stands in for latest-version.sh. FAKE_VERSION_FAIL=1 simulates a broken +# upstream query; FAKE_VERSION="" simulates one that returns nothing. +if [ -n "${FAKE_VERSION_FAIL:-}" ]; then + echo "ERROR: simulated upstream lookup failure" >&2 + exit 1 +fi +printf '%s\n' "${FAKE_VERSION-1.0.0}" +EOS + chmod +x "$SANDBOX/fake-version.sh" + + # A local HTTP server so remote-source paths can be tested without network. + mkdir -p "$SANDBOX/www" + printf 'x86_64 binary payload\n' > "$SANDBOX/www/x86.bin" + printf 'aarch64 binary payload\n' > "$SANDBOX/www/arm.bin" + # -u: without it the "Serving HTTP on ... port N" line stays buffered and the + # port can never be read back. + ( cd "$SANDBOX/www" && exec python3 -u -m http.server 0 --bind 127.0.0.1 ) \ + > "$SANDBOX/www.log" 2>&1 & + WWW_PID=$! + for _ in $(seq 1 50); do + PORT="$(sed -n -E 's/.*http:\/\/127\.0\.0\.1:([0-9]+).*/\1/p' "$SANDBOX/www.log" | head -1)" + [ -n "$PORT" ] && break + sleep 0.1 + done + [ -n "${PORT:-}" ] || { echo "could not start test http server" >&2; exit 1; } + + export PKG_REPO_BASE="file://$SANDBOX/origin" + export AUR_HTTPS_BASE="file://$SANDBOX/aur" + export AUR_SSH_BASE="file://$SANDBOX/aur" + export LATEST_VERSION_CMD="$SANDBOX/fake-version.sh" + export GIT_AUTHOR_NAME=test GIT_AUTHOR_EMAIL=test@test + export GIT_COMMITTER_NAME=test GIT_COMMITTER_EMAIL=test@test +} + +teardown() { + [ -n "${WWW_PID:-}" ] && kill "$WWW_PID" 2>/dev/null + [ -n "$SANDBOX" ] && rm -rf "$SANDBOX" + return 0 +} + +# A multi-arch package whose arches download DIFFERENT files that both rename to +# the SAME target -- freetube-appimage's exact shape, and the case that makes +# `makepkg -g` emit identical sums for both arches. +make_pkg_colliding_remote() { + local pkg="$1" + local src="$SANDBOX/build/$pkg" + rm -rf "$src"; mkdir -p "$src" + + cat > "$src/PKGBUILD" < "$src/.autoupdate" <<'EOF' +srchost="gitea" +srcmntr="test" +srcname="test" +verstrip="" +gui="false" +EOF + + ( cd "$src" && "$HERE/update-checksums.sh" --remote . >/dev/null \ + && makepkg --printsrcinfo > .SRCINFO ) + + git init --quiet --bare --initial-branch=main "$SANDBOX/origin/$pkg" + git init --quiet --bare --initial-branch=master "$SANDBOX/aur/${pkg}.git" + ( cd "$src" && git init --quiet --initial-branch=main && git add -A \ + && git commit --quiet -m "initial" \ + && git push --quiet "file://$SANDBOX/origin/$pkg" main \ + && git push --quiet "file://$SANDBOX/aur/${pkg}.git" main:master ) +} + +# Create a package repo (origin) plus its AUR counterpart. +# make_pkg [multiarch] +make_pkg() { + local pkg="$1" multi="${2:-}" + local src="$SANDBOX/build/$pkg" + rm -rf "$src"; mkdir -p "$src" + + echo "payload for $pkg" > "$src/local.txt" + + if [ -n "$multi" ]; then + echo "x86 payload" > "$src/pay-x86_64.txt" + echo "arm payload" > "$src/pay-aarch64.txt" + cat > "$src/PKGBUILD" < "$src/PKGBUILD" < "$src/.autoupdate" <<'EOF' +srchost="gitea" +srcmntr="test" +srcname="test" +verstrip="" +gui="false" +EOF + echo "readme" > "$src/README.md" + + # Fix up the SKIP placeholders and generate a valid .SRCINFO. + ( cd "$src" && "$HERE/update-checksums.sh" . >/dev/null \ + && makepkg --printsrcinfo > .SRCINFO ) + + git init --quiet --bare --initial-branch=main "$SANDBOX/origin/$pkg" + git init --quiet --bare --initial-branch=master "$SANDBOX/aur/${pkg}.git" + + ( cd "$src" && git init --quiet --initial-branch=main && git add -A \ + && git commit --quiet -m "initial" \ + && git push --quiet "file://$SANDBOX/origin/$pkg" main \ + && git push --quiet "file://$SANDBOX/aur/${pkg}.git" main:master ) +} + +# Run the updater. Any extra env goes in front of the call by the caller. +run_update() { + local pkg="$1"; shift + WORKDIR="$SANDBOX/work/$(date +%s%N)" \ + "$HERE/autoupdate.sh" "$pkg" "$@" >"$SANDBOX/last.log" 2>&1 +} + +# Read a field from a bare repo's tip. +field_at() { # + git -C "$1" show "$2:PKGBUILD" 2>/dev/null \ + | sed -n -E "s/^$3=[\"']?([^\"']*)[\"']?$/\1/p" | head -n1 +} +aur_field() { field_at "$SANDBOX/aur/$1.git" master "$2"; } +origin_field() { field_at "$SANDBOX/origin/$1" main "$2"; } +aur_head() { git -C "$SANDBOX/aur/$1.git" rev-parse master 2>/dev/null; } + +should_run() { [ -z "$FILTER" ] || [[ "$1" == *"$FILTER"* ]]; } + +# ------------------------------------------------------------------ tests --- + +test_new_version() { + make_pkg p1 + FAKE_VERSION=2.0.0 run_update p1 + assert_eq "0" "$?" "new version: exits 0" + assert_eq "2.0.0" "$(aur_field p1 pkgver)" "new version: pkgver reaches the AUR" + assert_eq "1" "$(aur_field p1 pkgrel)" "new version: pkgrel resets to 1" +} + +test_noop_is_quiet_and_green() { + make_pkg p2 + FAKE_VERSION=1.0.0 run_update p2 + local rc=$? before after + before="$(aur_head p2)" + FAKE_VERSION=1.0.0 run_update p2 + after="$(aur_head p2)" + assert_eq "0" "$rc" "no-op: exits 0" + assert_eq "$before" "$after" "no-op: second run pushes nothing" + grep -q "AUR already in sync" "$SANDBOX/last.log" \ + && pass "no-op: reports already in sync" \ + || fail "no-op: reports already in sync" "$(tail -3 "$SANDBOX/last.log")" +} + +# The headline regression: origin ahead of the AUR, nothing new upstream. +# The old script aborted with "same (old) version" and exit 0, forever. +# Reproduces the exact reported failure: a previous run published to origin but +# its AUR push failed. Origin is therefore fully consistent -- correct sums, +# correct .SRCINFO, pkgrel already bumped -- so this run computes NO local +# changes at all. The only thing wrong is that the AUR is behind. The old +# script hit "same (old) version specified", bare-exit 0'd, and never retried. +# +# The pkgrel is pre-bumped deliberately: otherwise the pkgrel rule would itself +# produce a local change and the run would push for that reason instead, +# leaving the actual retry path untested. +test_failed_push_is_retried() { + make_pkg p3 + local clone="$SANDBOX/manual-p3" + git clone --quiet "file://$SANDBOX/origin/p3" "$clone" + ( cd "$clone" \ + && sed -i 's/^pkgdesc=.*/pkgdesc="changed out of band"/;s/^pkgrel=.*/pkgrel="2"/' PKGBUILD \ + && makepkg --printsrcinfo > .SRCINFO \ + && git commit --quiet -am "published to origin; AUR push failed" \ + && git push --quiet origin main ) + + assert_ne "changed out of band" "$(aur_field p3 pkgdesc)" "retry: AUR starts out of date" + + FAKE_VERSION=1.0.0 run_update p3 + local rc=$? + assert_eq "0" "$rc" "retry: exits 0" + + grep -q "no local changes" "$SANDBOX/last.log" \ + && pass "retry: run produced no local changes (exercises the reconcile path)" \ + || fail "retry: run produced no local changes" \ + "the run committed something, so this is not the retry path: $(tail -5 "$SANDBOX/last.log")" + + assert_eq "changed out of band" "$(aur_field p3 pkgdesc)" \ + "retry: reconciles to the AUR with nothing new locally or upstream" +} + +test_manual_edit_bumps_pkgrel() { + make_pkg p4 + local clone="$SANDBOX/manual-p4" + git clone --quiet "file://$SANDBOX/origin/p4" "$clone" + ( cd "$clone" && sed -i 's/^pkgdesc=.*/pkgdesc="fixed package() bug"/' PKGBUILD \ + && makepkg --printsrcinfo > .SRCINFO \ + && git commit --quiet -am "fix package()" && git push --quiet origin main ) + + FAKE_VERSION=1.0.0 run_update p4 + assert_eq "2" "$(aur_field p4 pkgrel)" "manual edit: pkgrel auto-bumped 1 -> 2" + + # The important one: re-running must not ratchet pkgrel upward. + FAKE_VERSION=1.0.0 run_update p4 + assert_eq "2" "$(aur_field p4 pkgrel)" "manual edit: pkgrel stable on re-run (idempotent)" + FAKE_VERSION=1.0.0 run_update p4 + assert_eq "2" "$(aur_field p4 pkgrel)" "manual edit: pkgrel still stable on third run" +} + +test_hand_bumped_pkgrel_respected() { + make_pkg p5 + local clone="$SANDBOX/manual-p5" + git clone --quiet "file://$SANDBOX/origin/p5" "$clone" + ( cd "$clone" && sed -i 's/^pkgdesc=.*/pkgdesc="edited"/;s/^pkgrel=.*/pkgrel="7"/' PKGBUILD \ + && makepkg --printsrcinfo > .SRCINFO \ + && git commit --quiet -am "edit + own pkgrel" && git push --quiet origin main ) + + FAKE_VERSION=1.0.0 run_update p5 + assert_eq "7" "$(aur_field p5 pkgrel)" "hand-bumped pkgrel is left alone" +} + +test_readme_only_edit() { + make_pkg p6 + local clone="$SANDBOX/manual-p6" + git clone --quiet "file://$SANDBOX/origin/p6" "$clone" + ( cd "$clone" && echo "more docs" >> README.md \ + && git commit --quiet -am "docs" && git push --quiet origin main ) + + FAKE_VERSION=1.0.0 run_update p6 + local rc=$? # must be captured before any other command clobbers $? + assert_eq "0" "$rc" "README-only edit: still exits 0" + assert_eq "1" "$(aur_field p6 pkgrel)" "README-only edit: no pkgrel bump" + assert_eq "more docs" "$(git -C "$SANDBOX/aur/p6.git" show master:README.md | tail -1)" \ + "README-only edit: still reconciled to the AUR" +} + +test_local_source_edit_bumps_pkgrel() { + make_pkg p7 + local clone="$SANDBOX/manual-p7" + git clone --quiet "file://$SANDBOX/origin/p7" "$clone" + ( cd "$clone" && echo "changed payload" > local.txt \ + && git commit --quiet -am "edit local source" && git push --quiet origin main ) + + FAKE_VERSION=1.0.0 run_update p7 + assert_eq "2" "$(aur_field p7 pkgrel)" "local source edit: re-hashed and pkgrel bumped" +} + +test_force_rebuild() { + make_pkg p8 + FAKE_VERSION=1.0.0 run_update p8 + FORCE_REBUILD=1 FAKE_VERSION=1.0.0 run_update p8 + assert_eq "2" "$(aur_field p8 pkgrel)" "FORCE_REBUILD: pkgrel +1 with no other change" +} + +test_lookup_failure_is_loud() { + make_pkg p9 + local before; before="$(aur_head p9)" + FAKE_VERSION_FAIL=1 run_update p9 + assert_eq "1" "$?" "broken upstream lookup: exits 1, not mistaken for 'no new version'" + assert_eq "$before" "$(aur_head p9)" "broken upstream lookup: nothing pushed" +} + +test_empty_version_aborts_before_commit() { + make_pkg p10 + local before; before="$(aur_head p10)" + FAKE_VERSION="" run_update p10 + assert_eq "1" "$?" "empty version: exits 1" + assert_eq "$before" "$(aur_head p10)" "empty version: aborts before committing" + assert_eq "1.0.0" "$(origin_field p10 pkgver)" "empty version: origin left unpoisoned" +} + +# reflex-appimage's real state: local ahead of the AUR carrying a stale pkgrel +# that the old script never reset. +test_stale_pkgrel_repaired() { + make_pkg p11 + local clone="$SANDBOX/manual-p11" + git clone --quiet "file://$SANDBOX/origin/p11" "$clone" + ( cd "$clone" && sed -i 's/^pkgver=.*/pkgver="1.5.0"/;s/^pkgrel=.*/pkgrel="4"/' PKGBUILD \ + && makepkg --printsrcinfo > .SRCINFO \ + && git commit --quiet -am "version bumped without resetting pkgrel" \ + && git push --quiet origin main ) + + FAKE_VERSION=1.5.0 run_update p11 + assert_eq "1.5.0" "$(aur_field p11 pkgver)" "stale pkgrel: version published" + assert_eq "1" "$(aur_field p11 pkgrel)" "stale pkgrel: reset to 1 against the AUR" +} + +test_multiarch_distinct_sums() { + make_pkg p12 multi + FAKE_VERSION=1.0.0 run_update p12 + + local x a + x="$(git -C "$SANDBOX/aur/p12.git" show master:PKGBUILD \ + | sed -n '/^sha256sums_x86_64=/,/)/p' | grep -oE '[0-9a-f]{64}')" + a="$(git -C "$SANDBOX/aur/p12.git" show master:PKGBUILD \ + | sed -n '/^sha256sums_aarch64=/,/)/p' | grep -oE '[0-9a-f]{64}')" + + assert_eq "$(sha256sum "$SANDBOX/build/p12/pay-x86_64.txt" | awk '{print $1}')" "$x" \ + "multi-arch: x86_64 sum correct" + assert_eq "$(sha256sum "$SANDBOX/build/p12/pay-aarch64.txt" | awk '{print $1}')" "$a" \ + "multi-arch: aarch64 sum correct" + assert_ne "$x" "$a" "multi-arch: arches get distinct sums (the makepkg -g failure mode)" +} + +# The regression that rules out `makepkg -g`: two arches downloading different +# files that rename to the same target must not end up sharing a checksum. +test_multiarch_colliding_downloads() { + make_pkg_colliding_remote p15 + FAKE_VERSION=1.0.0 FORCE_REFRESH=1 run_update p15 + + local pb x a + pb="$(git -C "$SANDBOX/aur/p15.git" show master:PKGBUILD)" + x="$(printf '%s' "$pb" | sed -n '/^sha256sums_x86_64=/,/)/p' | grep -oE '[0-9a-f]{64}')" + a="$(printf '%s' "$pb" | sed -n '/^sha256sums_aarch64=/,/)/p' | grep -oE '[0-9a-f]{64}')" + + assert_eq "$(sha256sum "$SANDBOX/www/x86.bin" | awk '{print $1}')" "$x" \ + "colliding downloads: x86_64 hashed its own file" + assert_eq "$(sha256sum "$SANDBOX/www/arm.bin" | awk '{print $1}')" "$a" \ + "colliding downloads: aarch64 hashed its own file, not x86_64's" + assert_ne "$x" "$a" "colliding downloads: sums differ despite the shared :: target" +} + +test_srcinfo_guard() { + make_pkg p13 + local clone="$SANDBOX/manual-p13" + git clone --quiet "file://$SANDBOX/origin/p13" "$clone" + # pkgver that makepkg rejects -> .SRCINFO cannot be generated. + ( cd "$clone" && sed -i 's/^pkgdesc=.*/pkgdesc="x"/' PKGBUILD \ + && git commit --quiet -am "touch" && git push --quiet origin main ) + local before; before="$(aur_head p13)" + FAKE_VERSION="not a version!" run_update p13 + assert_eq "1" "$?" ".SRCINFO guard: invalid version rejected" + assert_eq "$before" "$(aur_head p13)" ".SRCINFO guard: nothing reached the AUR" +} + +test_no_git_push_dry_run() { + make_pkg p14 + local before_aur before_origin + before_aur="$(aur_head p14)" + before_origin="$(git -C "$SANDBOX/origin/p14" rev-parse main)" + NO_GIT_PUSH=1 FAKE_VERSION=3.0.0 run_update p14 + assert_eq "0" "$?" "NO_GIT_PUSH: exits 0" + assert_eq "$before_aur" "$(aur_head p14)" "NO_GIT_PUSH: AUR untouched" + assert_eq "$before_origin" "$(git -C "$SANDBOX/origin/p14" rev-parse main)" \ + "NO_GIT_PUSH: origin untouched" +} + +# ------------------------------------------------------------------- main --- + +trap teardown EXIT +setup + +for t in $(declare -F | awk '{print $3}' | grep '^test_' | sort); do + should_run "$t" || continue + printf '\n\033[1m%s\033[0m\n' "$t" + "$t" +done + +printf '\n────────────────────────\n' +printf 'passed: %d failed: %d\n' "$PASS" "$FAIL" +if [ "$FAIL" -gt 0 ]; then + printf 'failing: %s\n' "${FAILED_NAMES[*]}" + exit 1 +fi diff --git a/update-checksums.sh b/update-checksums.sh new file mode 100755 index 0000000..8d9599c --- /dev/null +++ b/update-checksums.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Recompute sha256sums for a PKGBUILD, single- or multi-arch. +# +# usage: update-checksums.sh [--remote] [package-dir] +# +# --remote also re-download and re-hash remote sources. +# Without it only local repo files (.desktop, icons) are +# re-hashed, which needs no network and is what lets a +# hand-edited repo self-correct on every nightly run. +# +# sha256 only: sha1sums/md5sums are retired and actively removed. +# +# Why this does not shell out to `makepkg -g` / `updpkgsums`: those generate +# every arch array in one pass and download all arches into a single directory. +# When two arches rename to the same target (`foo.AppImage::url-amd64`, +# `foo.AppImage::url-arm64`) the second arch finds the first arch's file +# already present, skips the download, and emits an sha256sums_aarch64 +# identical to sha256sums_x86_64 -- silently wrong. Overriding CARCH does not +# help; makepkg.conf resets it, and --config still downloads every arch. + +set -euo pipefail + +_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${_here}/common.sh" + +refresh_remote=0 +if [ "${1-}" = "--remote" ]; then refresh_remote=1; shift; fi + +pkgdir="${1:-.}" +pkgbuild="${pkgdir}/PKGBUILD" +[ -f "$pkgbuild" ] || die "no PKGBUILD in ${pkgdir}" + +export DL_CACHE="${DL_CACHE:-$(mktemp -d)}" + +mapfile -t arches < <(pkgbuild_array "$pkgbuild" arch) + +# Build the work list: every source array present, paired with its sums array. +# Single-arch is just the degenerate case of the same loop -- no special path. +pairs=() +if pkgbuild_has_array "$pkgbuild" source; then + pairs+=("source:sha256sums") +fi +for a in "${arches[@]}"; do + [ "$a" = "any" ] && continue + if pkgbuild_has_array "$pkgbuild" "source_${a}"; then + pairs+=("source_${a}:sha256sums_${a}") + fi +done + +[ "${#pairs[@]}" -gt 0 ] || die "PKGBUILD declares no source arrays" + +for pair in "${pairs[@]}"; do + src_arr="${pair%%:*}" + sum_arr="${pair##*:}" + + mapfile -t entries < <(pkgbuild_array "$pkgbuild" "$src_arr") + [ "${#entries[@]}" -gt 0 ] || continue + + mapfile -t existing < <(pkgbuild_array "$pkgbuild" "$sum_arr") + + # If the existing sums don't line up with the sources we cannot reuse any of + # them by index, so everything must be recomputed. + reuse=1 + if [ "${#existing[@]}" -ne "${#entries[@]}" ]; then reuse=0; fi + + log "${src_arr} -> ${sum_arr} (${#entries[@]} source(s))" + + sums=() + for i in "${!entries[@]}"; do + entry="${entries[$i]}" + loc="$(source_loc "$entry")" + + if is_vcs "$entry"; then + sum="SKIP" + info "$(source_name "$entry") SKIP (vcs)" + elif is_remote "$entry"; then + if [ "$refresh_remote" -eq 1 ] || [ "$reuse" -eq 0 ] \ + || [ -z "${existing[$i]:-}" ] || [ "${existing[$i]}" = "SKIP" ]; then + path="$(download_cached "$loc")" + sum="$(sha256sum "$path" | awk '{print $1}')" + info "$(source_name "$entry") ${sum}" + else + sum="${existing[$i]}" + info "$(source_name "$entry") ${sum} (kept)" + fi + else + # A file that lives in the package repo itself. + [ -f "${pkgdir}/${loc}" ] || die "local source '${loc}' not found in ${pkgdir}" + sum="$(sha256sum "${pkgdir}/${loc}" | awk '{print $1}')" + info "${loc} ${sum} (local)" + fi + sums+=("$sum") + done + + replace_array "$pkgbuild" "$sum_arr" "${sums[@]}" +done + +# Retire the legacy digests wherever they still appear. +for a in "" "${arches[@]}"; do + suffix="${a:+_$a}" + for algo in sha1sums md5sums sha224sums sha384sums sha512sums b2sums; do + remove_array "$pkgbuild" "${algo}${suffix}" + done +done