diff --git a/.gitea/workflows/deskflow-bin.yaml b/.gitea/workflows/deskflow-bin.yaml deleted file mode 100644 index a9a0952..0000000 --- a/.gitea/workflows/deskflow-bin.yaml +++ /dev/null @@ -1,34 +0,0 @@ ---- -name: Update deskflow-bin Package - -on: - schedule: - - cron: "0 0 * * *" - -defaults: - run: - shell: bash - -jobs: - update: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - run: | - apt-get --quiet update && apt-get --quiet --assume-yes install makepkg - git config --global user.name "${GIT_NAME}" - git config --global user.email "${GIT_EMAIL}" - git config --global credential.helper store - echo "https://${GIT_EMAIL//@/%40}:${GIT_TOKEN}@${GIT_HOST}" > ~/.git-credentials - git clone --depth 1 https://${GIT_HOST}/aur/deskflow-bin - cd deskflow-bin - source PKGBUILD - ./push.sh $(curl -L -H "Accept: application/vnd.github+json" \ - https://api.github.com/repos/${_srcmntr}/${_srcname}/releases/latest | jq -r ".tag_name" | sed 's|v||') - env: - GIT_NAME: ${{ vars.GIT_NAME }} - GIT_EMAIL: ${{ vars.GIT_EMAIL }} - GIT_HOST: ${{ vars.GIT_HOST }} - GIT_TOKEN: ${{ secrets.GIT_TOKEN }} diff --git a/.gitea/workflows/nethlink-appimage.yaml b/.gitea/workflows/nethlink-appimage.yaml deleted file mode 100644 index da197ba..0000000 --- a/.gitea/workflows/nethlink-appimage.yaml +++ /dev/null @@ -1,34 +0,0 @@ ---- -name: Update nethlink-appimage Package - -on: - schedule: - - cron: "0 0 * * *" - -defaults: - run: - shell: bash - -jobs: - update: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - run: | - apt-get --quiet update && apt-get --quiet --assume-yes install makepkg - git config --global user.name "${GIT_NAME}" - git config --global user.email "${GIT_EMAIL}" - git config --global credential.helper store - echo "https://${GIT_EMAIL//@/%40}:${GIT_TOKEN}@${GIT_HOST}" > ~/.git-credentials - git clone --depth 1 https://${GIT_HOST}/aur/nethlink-appimage - cd nethlink-appimage - source PKGBUILD - ./push.sh $(curl -L -H "Accept: application/vnd.github+json" \ - https://api.github.com/repos/${_srcmntr}/${_srcname}/releases/latest | jq -r ".tag_name" | sed 's|v||') - env: - GIT_NAME: ${{ vars.GIT_NAME }} - GIT_EMAIL: ${{ vars.GIT_EMAIL }} - GIT_HOST: ${{ vars.GIT_HOST }} - GIT_TOKEN: ${{ secrets.GIT_TOKEN }} diff --git a/.gitea/workflows/open-video-downloader-appimage.yaml b/.gitea/workflows/open-video-downloader-appimage.yaml deleted file mode 100644 index 16d5e90..0000000 --- a/.gitea/workflows/open-video-downloader-appimage.yaml +++ /dev/null @@ -1,34 +0,0 @@ ---- -name: Update open-video-downloader-appimage Package - -on: - schedule: - - cron: "0 0 * * *" - -defaults: - run: - shell: bash - -jobs: - update: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - run: | - apt-get --quiet update && apt-get --quiet --assume-yes install makepkg - git config --global user.name "${GIT_NAME}" - git config --global user.email "${GIT_EMAIL}" - git config --global credential.helper store - echo "https://${GIT_EMAIL//@/%40}:${GIT_TOKEN}@${GIT_HOST}" > ~/.git-credentials - git clone --depth 1 https://${GIT_HOST}/aur/open-video-downloader-appimage - cd open-video-downloader-appimage - source PKGBUILD - ./push.sh $(curl -L -H "Accept: application/vnd.github+json" \ - https://api.github.com/repos/${_srcmntr}/${_srcname}/releases/latest | jq -r ".tag_name" | sed 's|app-v||') - env: - GIT_NAME: ${{ vars.GIT_NAME }} - GIT_EMAIL: ${{ vars.GIT_EMAIL }} - GIT_HOST: ${{ vars.GIT_HOST }} - GIT_TOKEN: ${{ secrets.GIT_TOKEN }} diff --git a/.gitea/workflows/open-video-downloader-bin.yaml b/.gitea/workflows/open-video-downloader-bin.yaml deleted file mode 100644 index 7202c78..0000000 --- a/.gitea/workflows/open-video-downloader-bin.yaml +++ /dev/null @@ -1,34 +0,0 @@ ---- -name: Update open-video-downloader-bin Package - -on: - schedule: - - cron: "0 0 * * *" - -defaults: - run: - shell: bash - -jobs: - update: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - run: | - apt-get --quiet update && apt-get --quiet --assume-yes install makepkg - git config --global user.name "${GIT_NAME}" - git config --global user.email "${GIT_EMAIL}" - git config --global credential.helper store - echo "https://${GIT_EMAIL//@/%40}:${GIT_TOKEN}@${GIT_HOST}" > ~/.git-credentials - git clone --depth 1 https://${GIT_HOST}/aur/open-video-downloader-bin - cd open-video-downloader-bin - source PKGBUILD - ./push.sh $(curl -L -H "Accept: application/vnd.github+json" \ - https://api.github.com/repos/${_srcmntr}/${_srcname}/releases/latest | jq -r ".tag_name" | sed 's|app-v||') - env: - GIT_NAME: ${{ vars.GIT_NAME }} - GIT_EMAIL: ${{ vars.GIT_EMAIL }} - GIT_HOST: ${{ vars.GIT_HOST }} - GIT_TOKEN: ${{ secrets.GIT_TOKEN }} diff --git a/.gitea/workflows/pman-helper.yaml b/.gitea/workflows/pman-helper.yaml deleted file mode 100644 index 3d93889..0000000 --- a/.gitea/workflows/pman-helper.yaml +++ /dev/null @@ -1,34 +0,0 @@ ---- -name: Update pman-helper Package - -on: - schedule: - - cron: "0 0 * * *" - -defaults: - run: - shell: bash - -jobs: - update: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - run: | - apt-get --quiet update && apt-get --quiet --assume-yes install makepkg - git config --global user.name "${GIT_NAME}" - git config --global user.email "${GIT_EMAIL}" - git config --global credential.helper store - echo "https://${GIT_EMAIL//@/%40}:${GIT_TOKEN}@${GIT_HOST}" > ~/.git-credentials - git clone --depth 1 https://${GIT_HOST}/aur/pman-helper - cd pman-helper - source PKGBUILD - ./push.sh $(curl -L -H "Accept: application/json" \ - https://git.bjphoster.com/api/v1/repos/${_srcmntr}/${_srcname}/repo/tags | jq -r ".[].name" | sort -V | tail -n 1) - env: - GIT_NAME: ${{ vars.GIT_NAME }} - GIT_EMAIL: ${{ vars.GIT_EMAIL }} - GIT_HOST: ${{ vars.GIT_HOST }} - GIT_TOKEN: ${{ secrets.GIT_TOKEN }} diff --git a/.gitea/workflows/reflex-appimage.yaml b/.gitea/workflows/reflex-appimage.yaml deleted file mode 100644 index f18630f..0000000 --- a/.gitea/workflows/reflex-appimage.yaml +++ /dev/null @@ -1,34 +0,0 @@ ---- -name: Update reflex-appimage Package - -on: - schedule: - - cron: "0 0 * * *" - -defaults: - run: - shell: bash - -jobs: - update: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - run: | - apt-get --quiet update && apt-get --quiet --assume-yes install makepkg - git config --global user.name "${GIT_NAME}" - git config --global user.email "${GIT_EMAIL}" - git config --global credential.helper store - echo "https://${GIT_EMAIL//@/%40}:${GIT_TOKEN}@${GIT_HOST}" > ~/.git-credentials - git clone --depth 1 https://${GIT_HOST}/aur/reflex-appimage - cd reflex-appimage - source PKGBUILD - ./push.sh $(curl -L -H "Accept: application/vnd.github+json" \ - https://api.github.com/repos/${_srcmntr}/${_srcname}/releases/latest | jq -r ".tag_name" | sed 's|v||') - env: - GIT_NAME: ${{ vars.GIT_NAME }} - GIT_EMAIL: ${{ vars.GIT_EMAIL }} - GIT_HOST: ${{ vars.GIT_HOST }} - GIT_TOKEN: ${{ secrets.GIT_TOKEN }} diff --git a/.gitea/workflows/update.yaml b/.gitea/workflows/update.yaml new file mode 100644 index 0000000..dcf7601 --- /dev/null +++ b/.gitea/workflows/update.yaml @@ -0,0 +1,122 @@ +--- +name: Update AUR Packages + +on: + schedule: + - cron: "0 0 * * *" + workflow_dispatch: + inputs: + package: + description: "Single package to update (blank = all)" + required: false + type: string + force_rebuild: + description: "Bump pkgrel even with no detected change" + required: false + type: boolean + default: false + force_refresh: + description: "Re-download remote sources even without a version change" + required: false + type: boolean + default: false + no_git_push: + description: "Dry run: do everything except push" + required: false + type: boolean + default: false + +defaults: + run: + shell: bash + +jobs: + update: + runs-on: ubuntu-latest + # A real Arch makepkg is required. Ubuntu's build environment silently + # emits an empty .SRCINFO, which the AUR then rejects -- that is what broke + # reflex-appimage, nethlink-appimage and pman-helper. regen_srcinfo() in + # common.sh fails loudly if this ever regresses. + container: + image: archlinux:base-devel + + strategy: + # One broken package must not cancel the rest of the matrix. + fail-fast: false + matrix: + package: + - nethlink-appimage + - open-video-downloader-appimage + - open-video-downloader-bin + - pman-helper + - reflex-appimage + - tsparams + + steps: + - uses: actions/checkout@v4 + + - name: Install dependencies + run: | + pacman -Syu --noconfirm --needed \ + git openssh jq curl pacman-contrib namcap fuse2 python + + - name: Configure git and SSH + env: + GIT_NAME: ${{ vars.GIT_NAME }} + GIT_EMAIL: ${{ vars.GIT_EMAIL }} + GIT_HOST: ${{ vars.GIT_HOST }} + GIT_TOKEN: ${{ secrets.GIT_TOKEN }} + AUR_SSH_KEY: ${{ secrets.AUR_SSH_KEY }} + run: | + set -euo pipefail + git config --global user.name "${GIT_NAME}" + git config --global user.email "${GIT_EMAIL}" + git config --global credential.helper store + echo "https://${GIT_EMAIL//@/%40}:${GIT_TOKEN}@${GIT_HOST}" > ~/.git-credentials + chmod 600 ~/.git-credentials + + mkdir -p ~/.ssh && chmod 700 ~/.ssh + printf '%s\n' "${AUR_SSH_KEY}" > ~/.ssh/aur + chmod 600 ~/.ssh/aur + ssh-keyscan -t rsa,ecdsa,ed25519 aur.archlinux.org >> ~/.ssh/known_hosts 2>/dev/null + cat >> ~/.ssh/config <<'EOF' + Host aur.archlinux.org + User aur + IdentityFile ~/.ssh/aur + IdentitiesOnly yes + EOF + chmod 600 ~/.ssh/config + + # makepkg refuses to run as root, and CI containers are root by default. + - name: Create build user + run: | + set -euo pipefail + useradd --create-home --shell /bin/bash builder + chown -R builder:builder "${GITHUB_WORKSPACE}" + cp -r /root/.ssh /root/.gitconfig /root/.git-credentials /home/builder/ 2>/dev/null || true + chown -R builder:builder /home/builder + + - name: Update ${{ matrix.package }} + env: + GIT_HOST: ${{ vars.GIT_HOST }} + # Compared against both `true` and 'true' on purpose. A bare + # `${{ inputs.x && '1' || '' }}` is wrong if the runner hands the + # input over as the STRING "false", which is truthy -- that would + # silently enable the flag on every run. + FORCE_REBUILD: ${{ (inputs.force_rebuild == true || inputs.force_rebuild == 'true') && '1' || '' }} + FORCE_REFRESH: ${{ (inputs.force_refresh == true || inputs.force_refresh == 'true') && '1' || '' }} + NO_GIT_PUSH: ${{ (inputs.no_git_push == true || inputs.no_git_push == 'true') && '1' || '' }} + run: | + set -euo pipefail + # workflow_dispatch with a package name runs only that one. + want="${{ inputs.package }}" + if [ -n "${want}" ] && [ "${want}" != "${{ matrix.package }}" ]; then + echo "skipping ${{ matrix.package }} (dispatch asked for ${want})" + exit 0 + fi + runuser -u builder -- env \ + GIT_HOST="${GIT_HOST}" \ + FORCE_REBUILD="${FORCE_REBUILD}" \ + FORCE_REFRESH="${FORCE_REFRESH}" \ + NO_GIT_PUSH="${NO_GIT_PUSH}" \ + ./autoupdate.sh "${{ matrix.package }}" diff --git a/README.md b/README.md index 6f186f3..a8d2907 100644 --- a/README.md +++ b/README.md @@ -1 +1,131 @@ -# AUR Auto Update Workflows +# AUR Auto Update + +Nightly reconciler that keeps the `aur/*` package repos and their AUR +counterparts in sync. + +This is a **reconciler**, not a version-bump script. A new upstream release is +only one of the things that can make a package diverge from the AUR; a manual +push to origin is another, and a push that failed last night is a third. All +three take the same path, and the AUR reconcile runs on every pass. + +## Layout + +| file | role | +|---|---| +| `autoupdate.sh` | entry point; orchestrates one or more packages | +| `latest-version.sh` | resolves the latest upstream version (GitHub / Gitea) | +| `update-checksums.sh` | downloads sources, computes sha256, rewrites the arrays | +| `sync-aur.sh` | reconciles the repo against the AUR | +| `common.sh` | logging, guards, PKGBUILD parsing and array splicing | +| `test.sh` | hermetic test suite | +| `.gitea/workflows/update.yaml` | nightly matrix job | + +Nothing here is package-specific. Everything that differs between packages +lives in a `.autoupdate` manifest committed to the package repo itself. + +## Per-package manifest + +Each package repo carries a `.autoupdate` file: + +```sh +srchost="github" # github | gitea +srcmntr="Sunhaiy" # upstream owner/org +srcname="Reflex" # upstream repo +verstrip="v" # tag prefix to strip ("app-v" for open-video-downloader-*) +gui="true" # extract .desktop + icon from the AppImage +desktop="reflex.desktop" # name *inside* the AppImage +icon="reflex.png" +``` + +It rides along to the AUR on push, as `README.md` and `LICENSE` already do. +Harmless: the AUR reads only `PKGBUILD` and `.SRCINFO`. + +## What a run does + +1. Clone the package repo, read its manifest. +2. Fetch the AUR over **HTTPS** as the reference state. This happens before + anything else, because `pkgrel` is derived from it. Read-only, so no SSH key + is required for a dry run. +3. Resolve the upstream version. A *failed lookup* exits 1; "no new version" is + not a failure. +4. If the version moved, re-download and re-hash the remote sources. +5. Always re-hash local sources (`.desktop`, icons) and regenerate `.SRCINFO`. + Local hashing needs no network, so this is free, and it is what lets a + hand-edited repo self-correct. +6. Derive `pkgrel` (below). +7. Validate `.SRCINFO`; abort **before** committing if it is empty or disagrees + with the PKGBUILD. +8. Commit only if something actually changed. +9. Push to the AUR first, then origin, so a failed AUR push leaves origin clean. + +## pkgrel + +`X.Y.Z-N` — everything before the dash is upstream, `-N` is packaging-only +revisions. You should never need to touch it by hand. + +`pkgrel` is derived from **the AUR**, because that is what `-N` is numbered +against: + +- version differs from the published one → `pkgrel = 1` +- same version, packaging content differs → `pkgrel = AUR's + 1` +- same version, packaging identical → unchanged +- you already bumped it yourself → respected, not bumped again + +Using the AUR as the reference rather than local history is what makes this +idempotent: once the bump is published, the next night sees no difference and +does nothing. There is a test for exactly that (`pkgrel stable on re-run`). + +An edit to a local source such as an icon counts as a packaging change, because +its `sha256sums` entry is regenerated first. A README-only edit does not. + +## Environment + +| variable | effect | +|---|---| +| `NO_GIT_PUSH=1` | dry run: do everything except push | +| `FORCE_REBUILD=1` | bump `pkgrel` even with no detected change | +| `FORCE_REFRESH=1` | re-download remote sources without a version change | +| `AUR_FORCE=1` | force-push over diverged AUR history (see below) | +| `GIT_HOST` | git host holding the package repos | +| `GITHUB_TOKEN` | optional; avoids GitHub's 60/hour anonymous rate limit | + +A rejected AUR push usually means someone pushed there directly. The run fails +loudly rather than overwriting it; inspect, then re-run with `AUR_FORCE=1`. + +## Exit codes + +"No new upstream version" is the overwhelmingly common outcome and exits **0**. +The trap the previous implementation fell into was giving that same silent +`exit 0` to a lookup that had *broken* — which is how `pman-helper` shipped +`pkgver=""` unnoticed. A failed lookup now exits **1**. + +## Requirements + +Needs a genuine Arch `makepkg`. Ubuntu's build environment emits an empty +`.SRCINFO`, which the AUR rejects — this silently broke three packages. The CI +job therefore runs in an `archlinux:base-devel` container as a non-root user +(`makepkg` refuses to run as root), and `regen_srcinfo` in `common.sh` fails +loudly if the output is ever empty again. + +## Notes on checksums + +sha256 only; `sha1sums`/`md5sums` are actively removed. + +Checksums are computed directly rather than via `makepkg -g` / `updpkgsums`. +Those generate every arch array in one pass and download all arches into a +single directory, so when two arches rename to the same target +(`foo.AppImage::…-amd64`, `foo.AppImage::…-arm64`) the second finds the first +one's file already there and emits an identical, silently wrong checksum. +Overriding `CARCH` does not help. Downloads here go to content-addressed cache +paths instead, never to the `::` rename target. Multi-arch is supported for +when a package needs it; single-arch is the degenerate case of the same loop. + +## Tests + +```sh +./test.sh # all +./test.sh pkgrel # filter by name +``` + +Runs against local bare repos and a local HTTP server — no network, and it +never touches the real AUR. Requires a real `makepkg`. diff --git a/autoupdate.sh b/autoupdate.sh new file mode 100755 index 0000000..98902de --- /dev/null +++ b/autoupdate.sh @@ -0,0 +1,266 @@ +#!/usr/bin/env bash +# Nightly AUR package reconciler. +# +# usage: autoupdate.sh [...] +# +# env: +# GIT_HOST git host holding the package repos (default git.bjphoster.com) +# WORKDIR where to clone; a temp dir by default +# NO_GIT_PUSH=1 do everything except push (dry run) +# FORCE_REBUILD=1 bump pkgrel even with no detected change +# FORCE_REFRESH=1 re-download remote sources even without a version change +# AUR_FORCE=1 force-push over diverged AUR history +# +# This is a RECONCILER, not a version-bump script. A new upstream release is +# only one of the things that can make a package diverge from the AUR; a manual +# push to origin is another, and a failed push last night is a third. All three +# funnel through the same path, and the AUR reconcile at the end runs +# unconditionally. + +set -euo pipefail + +_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${_here}/common.sh" + +require_noninteractive + +GIT_HOST="${GIT_HOST:-git.bjphoster.com}" +# Where the package repos live, and how the upstream version is resolved. +# Both are overridable so the test suite can run hermetically against local +# bare repos; the defaults are the production values. +PKG_REPO_BASE="${PKG_REPO_BASE:-https://${GIT_HOST}/aur}" +LATEST_VERSION_CMD="${LATEST_VERSION_CMD:-${_here}/latest-version.sh}" + +[ "$#" -ge 1 ] || die "usage: autoupdate.sh [...]" + +WORKDIR="${WORKDIR:-$(mktemp -d)}" +mkdir -p "$WORKDIR" + +update_one() { + local pkg="$1" + local dir="${WORKDIR}/${pkg}" + + log "=== ${pkg} ===" + + if [ -d "$dir/.git" ]; then + git -C "$dir" fetch --quiet origin + git -C "$dir" reset --quiet --hard origin/HEAD + else + rm -rf "$dir" + git clone --quiet "${PKG_REPO_BASE}/${pkg}" "$dir" \ + || die "could not clone ${pkg} from ${PKG_REPO_BASE}" + fi + + cd "$dir" + [ -f PKGBUILD ] || die "${pkg} has no PKGBUILD" + [ -f .autoupdate ] || die "${pkg} has no .autoupdate manifest" + + # shellcheck disable=SC1091 + source ./.autoupdate + + # Per-run download cache, shared between the AppImage extraction below and + # update-checksums.sh so a 100 MB AppImage is fetched at most once. + export DL_CACHE="${WORKDIR}/.cache" + mkdir -p "$DL_CACHE" + + # --- AUR reference state ------------------------------------------------- + # Fetched up front: it is the reference for the pkgrel decision, not just a + # push target. Comparing against the AUR rather than against local history is + # what makes the pkgrel bump idempotent -- once pushed, the AUR matches and + # the next run finds nothing to do. + # Read-only HTTPS: establishing the reference state needs no SSH key, so + # dry runs work anywhere. Only sync-aur.sh's push is authenticated. + local aur_pkgbuild="" aur_pkgrel="" aur_pkgver="" aur_name + aur_name="$(pkgbuild_expand PKGBUILD 'printf %s "$pkgname"')" + git remote remove aur-ro >/dev/null 2>&1 || true + git remote add aur-ro "${AUR_HTTPS_BASE}/${aur_name}.git" + if git fetch --quiet aur-ro master 2>/dev/null; then + aur_pkgbuild="${WORKDIR}/${pkg}.aur.PKGBUILD" + if git show FETCH_HEAD:PKGBUILD > "$aur_pkgbuild" 2>/dev/null; then + aur_pkgrel="$(pkgbuild_get pkgrel "$aur_pkgbuild")" + aur_pkgver="$(pkgbuild_get pkgver "$aur_pkgbuild")" + info "AUR has ${aur_pkgver}-${aur_pkgrel}" + else + aur_pkgbuild="" + fi + else + info "not on the AUR yet" + fi + + # --- upstream version ---------------------------------------------------- + # A failure in here exits non-zero and is NOT mistaken for "no new version". + local oldver newver + oldver="$(pkgbuild_get pkgver)" + newver="$("$LATEST_VERSION_CMD" .)" + info "upstream ${newver}, local ${oldver:-}" + + local version_changed=0 + if [ "$newver" != "$oldver" ]; then + version_changed=1 + log "new upstream version ${oldver:-} -> ${newver}" + pkgbuild_set pkgver "$newver" + # pkgrel is not touched here; apply_pkgrel_rule derives it from the AUR. + fi + + # --- AppImage assets ----------------------------------------------------- + if [ "${gui:-false}" = "true" ] && [ "$version_changed" -eq 1 ]; then + extract_appimage_assets + fi + + # --- checksums + .SRCINFO ------------------------------------------------ + # Local sources are always re-hashed (free, no network) so a hand-edited + # .desktop or icon self-corrects. Remote sources are only re-fetched when the + # version moved, or on demand. + local cs_args=() + if [ "$version_changed" -eq 1 ] || [ -n "${FORCE_REFRESH:-}" ]; then + cs_args+=(--remote) + fi + "${_here}/update-checksums.sh" "${cs_args[@]}" . + + regen_srcinfo . + + # --- pkgrel --------------------------------------------------------------- + apply_pkgrel_rule + + # --- commit --------------------------------------------------------------- + git add -A + if git diff --cached --quiet; then + # Nothing changed. Deliberately do NOT exit here: the AUR may still be out + # of sync from a previous failed push, and an unguarded `git commit` with + # nothing staged would exit non-zero and, under `set -e`, abort before the + # reconcile. + log "no local changes for ${pkg}" + else + local ver rel + ver="$(pkgbuild_get pkgver)"; rel="$(pkgbuild_get pkgrel)" + git commit --quiet -m "Updated ${pkg} to ${ver}-${rel}" + log "committed ${pkg} ${ver}-${rel}" + fi + + # --- publish -------------------------------------------------------------- + # AUR first: if it rejects the push, origin is left unpoisoned and the whole + # thing is recomputed cleanly on the next run. + "${_here}/sync-aur.sh" . + + if [ -n "${NO_GIT_PUSH:-}" ]; then + log "NO_GIT_PUSH set: not pushing ${pkg} to origin" + else + git push --quiet origin HEAD + fi +} + +# Derive pkgrel so that fixing a package() bug and pushing needs no manual bump. +apply_pkgrel_rule() { + local cur_ver cur_rel + cur_ver="$(pkgbuild_get pkgver)" + cur_rel="$(pkgbuild_get pkgrel)" + + # Nothing on the AUR to compare against (new package): push as-is. + if [ -z "${aur_pkgbuild:-}" ]; then + return 0 + fi + + # pkgrel is derived purely from the AUR reference, because the AUR is what + # `-N` is numbered against. If the version we are about to publish differs + # from the published one, this is a new upstream version as far as the AUR is + # concerned and the packaging revision restarts at 1 -- whether the bump + # happened in this run or in an earlier one that failed to push. That second + # case is real: reflex-appimage sat at 1.0.11-4 locally against 1.0.10-4 on + # the AUR, because the old script only reset pkgrel under FORCE_REBUILD. + if [ "$cur_ver" != "${aur_pkgver:-}" ]; then + if [ "$cur_rel" != "1" ]; then + pkgbuild_set pkgrel "1" + log "new version relative to the AUR (${aur_pkgver} -> ${cur_ver}); pkgrel ${cur_rel} -> 1" + regen_srcinfo . + fi + return 0 + fi + + local base_rel="${aur_pkgrel:-0}" + [[ "$base_rel" =~ ^[0-9]+$ ]] || base_rel=0 + + if [ -n "${FORCE_REBUILD:-}" ]; then + pkgbuild_set pkgrel "$(( base_rel + 1 ))" + log "FORCE_REBUILD: pkgrel -> $(( base_rel + 1 ))" + regen_srcinfo . + return 0 + fi + + # Compare with the pkgrel line stripped, so the decision is about substantive + # packaging content. An edit to a local source lands here too, because its + # sha256sums entry was just regenerated. A README-only edit does not touch + # the PKGBUILD and so correctly does not bump. + if diff -q <(grep -v '^pkgrel=' PKGBUILD) \ + <(grep -v '^pkgrel=' "$aur_pkgbuild") >/dev/null 2>&1; then + return 0 # identical packaging; nothing to do + fi + + if [ "$cur_rel" = "$aur_pkgrel" ]; then + pkgbuild_set pkgrel "$(( base_rel + 1 ))" + log "packaging changed without a pkgrel bump; pkgrel -> $(( base_rel + 1 ))" + regen_srcinfo . + else + info "pkgrel already moved by hand (${aur_pkgrel} -> ${cur_rel}); leaving it" + fi +} + +# Pull the .desktop and icon out of the AppImage so they can be shipped as +# local sources. Unchanged in substance from the old push.sh. +extract_appimage_assets() { + local host_arr="source_$(uname -m)" entry loc img newdesktop newicon + local candidates=() + + # A multi-arch AppImage is extracted once, against the host arch's image. + if pkgbuild_has_array PKGBUILD "$host_arr"; then + mapfile -t candidates < <(pkgbuild_array PKGBUILD "$host_arr") + fi + if [ "${#candidates[@]}" -eq 0 ]; then + mapfile -t candidates < <(pkgbuild_array PKGBUILD source) + fi + + img="" + for entry in "${candidates[@]}"; do + loc="$(source_loc "$entry")" + if printf '%s' "${entry,,}" | grep -q "appimage" && is_remote "$entry"; then + img="$(download_cached "$loc")" + break + fi + done + [ -n "$img" ] || { warn "gui=true but no AppImage source found; skipping asset extraction"; return 0; } + + local pkgbase + pkgbase="$(pkgbuild_expand PKGBUILD 'printf %s "${_pkgname:-$pkgname}"')" + newdesktop="${pkgbase}.desktop" + newicon="${pkgbase}.png" + + local tmp; tmp="$(mktemp -d)" + cp "$img" "${tmp}/app.AppImage" + chmod +x "${tmp}/app.AppImage" + ( cd "$tmp" && ./app.AppImage --appimage-extract >/dev/null 2>&1 ) \ + || { warn "could not extract ${pkgbase} AppImage; keeping existing assets"; rm -rf "$tmp"; return 0; } + + if [ -f "${tmp}/squashfs-root/${desktop}" ]; then + cp "${tmp}/squashfs-root/${desktop}" "$newdesktop" + sed -i "/^X-AppImage-/d;/^$/d" "$newdesktop" + sed -i "s|^Exec=.*|Exec=${pkgbase} %U|" "$newdesktop" + info "extracted ${newdesktop}" + else + warn "${desktop} not found inside the AppImage" + fi + + if [ -f "${tmp}/squashfs-root/${icon}" ]; then + cp "${tmp}/squashfs-root/${icon}" "$newicon" + info "extracted ${newicon}" + else + warn "${icon} not found inside the AppImage" + fi + + rm -rf "$(readlink -f "${tmp}/squashfs-root" 2>/dev/null || true)" "$tmp" +} + +rc=0 +for pkg in "$@"; do + ( update_one "$pkg" ) || { rc=1; warn "${pkg} FAILED"; } +done +exit "$rc" diff --git a/common.sh b/common.sh new file mode 100755 index 0000000..579a356 --- /dev/null +++ b/common.sh @@ -0,0 +1,200 @@ +#!/usr/bin/env bash +# Shared helpers for the AUR autoupdater. +# Sourced by autoupdate.sh and the standalone scripts; never executed directly. + +set -euo pipefail + +# Endpoints. Overridable so the test suite can point at local bare repos +# instead of the real AUR; defaults are the production values. +AUR_SSH_BASE="${AUR_SSH_BASE:-ssh://aur@aur.archlinux.org}" +AUR_HTTPS_BASE="${AUR_HTTPS_BASE:-https://aur.archlinux.org}" + +log() { printf '==> %s\n' "$*"; } +info() { printf ' -> %s\n' "$*"; } +warn() { printf 'WARNING: %s\n' "$*" >&2; } +die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } + +# The old push.sh fell back to an interactive `read` when no version was passed. +# In CI that reads EOF and yields an empty version, which is how pman-helper +# committed pkgver="" and the sha256 of a zero-byte download. +require_noninteractive() { + [ -t 0 ] && die "refusing to prompt: this script must never depend on interactive input" + return 0 +} + +# A version we are willing to write into a PKGBUILD. +VERSION_RE='^[0-9][0-9a-zA-Z._+]*$' + +validate_version() { + local ver="${1-}" what="${2-version}" + [ -n "$ver" ] || die "${what} is empty" + [[ "$ver" =~ $VERSION_RE ]] || die "${what} '${ver}' does not look like a version" + return 0 +} + +# Read a scalar assignment out of a PKGBUILD without sourcing it. +# Handles pkgver="1.2.3", pkgver='1.2.3' and bare pkgver=1.2.3. +pkgbuild_get() { + local var="$1" file="${2:-PKGBUILD}" + sed -n -E "s/^${var}=[\"']?([^\"']*)[\"']?[[:space:]]*(#.*)?$/\1/p" "$file" | head -n1 +} + +# Rewrite a scalar assignment in place, preserving double-quoted style. +pkgbuild_set() { + local var="$1" val="$2" file="${3:-PKGBUILD}" + grep -qE "^${var}=" "$file" || die "no ${var}= assignment in ${file}" + sed -i -E "s|^${var}=.*|${var}=\"${val}\"|" "$file" +} + +# Expand a PKGBUILD's arrays/vars by sourcing it in a subshell and printing the +# result. Safe enough: these are our own package definitions. Used instead of +# grep/sed so that ${pkgver} / ${_srcname} substitutions actually resolve. +# +# Usage: pkgbuild_expand +pkgbuild_expand() { + local file="$1" snippet="$2" + ( + set +euo pipefail + # Neutralise anything that would run during a real build. + # shellcheck disable=SC1090 + source "$file" >/dev/null 2>&1 || true + eval "$snippet" + ) +} + +# Print the elements of an array variable, one per line, NUL-safe enough for +# source entries (which never contain newlines). +pkgbuild_array() { + local file="$1" name="$2" + pkgbuild_expand "$file" "printf '%s\n' \"\${${name}[@]}\" 2>/dev/null" +} + +# True if the named array exists in the PKGBUILD (even if empty). +pkgbuild_has_array() { + local file="$1" name="$2" + [ "$(pkgbuild_expand "$file" "declare -p ${name} >/dev/null 2>&1 && echo yes")" = "yes" ] +} + +# Replace `name=( ... )` in a PKGBUILD with the given values. +# +# This exists to replace the line-counting sed in the old freetube push.sh +# (`sednewline="n;"` accumulating `n;n;n;`), which silently corrupts the file +# whenever a source array's length changes. Matches `^name=(` through its +# closing `)`, so it handles both one-line and multi-line blocks, and appends +# the array if it is missing entirely. +replace_array() { + local file="$1" name="$2"; shift 2 + local block v + if [ "$#" -eq 1 ]; then + block="${name}=(\"$1\")" + else + block="${name}=(" + for v in "$@"; do block+=$'\n'" \"${v}\""; done + block+=$'\n'")" + fi + + if ! grep -qE "^${name}=\(" "$file"; then + printf '%s\n' "$block" >> "$file" + return 0 + fi + + awk -v name="$name" -v block="$block" ' + BEGIN { skip = 0 } + skip { + # Consume until the line that closes the array. + if ($0 ~ /\)[[:space:]]*$/) skip = 0 + next + } + $0 ~ "^" name "=\\(" { + print block + # A one-line `name=("x")` closes on the same line; a block does not. + if ($0 !~ /\)[[:space:]]*$/) skip = 1 + next + } + { print } + ' "$file" > "${file}.tmp" + + mv "${file}.tmp" "$file" +} + +# Delete a whole `name=( ... )` block (used to retire sha1sums/md5sums). +remove_array() { + local file="$1" name="$2" + grep -qE "^${name}=\(" "$file" || return 0 + awk -v name="$name" ' + BEGIN { skip = 0 } + skip { if ($0 ~ /\)[[:space:]]*$/) skip = 0; next } + $0 ~ "^" name "=\\(" { if ($0 !~ /\)[[:space:]]*$/) skip = 1; next } + { print } + ' "$file" > "${file}.tmp" + mv "${file}.tmp" "$file" +} + +# Split a source entry into its rename target and its URL/path. +# "foo.AppImage::https://x/y.AppImage" -> name=foo.AppImage loc=https://x/y.AppImage +# "https://x/y.tar.gz" -> name=y.tar.gz loc=https://x/y.tar.gz +# "reflex.desktop" -> name=reflex.desktop loc=reflex.desktop +source_name() { + local e="$1" + if [[ "$e" == *"::"* ]]; then printf '%s' "${e%%::*}"; else printf '%s' "${e##*/}"; fi +} +source_loc() { local e="$1"; printf '%s' "${e##*::}"; } + +is_remote() { [[ "$(source_loc "$1")" =~ ^(https?|ftp):// ]]; } + +# VCS sources (git+https://..., svn+..., etc) are not fixed artefacts and +# makepkg expects a literal SKIP for them rather than a digest. +is_vcs() { [[ "$(source_loc "$1")" =~ ^(git|svn|hg|bzr)(\+|:) ]]; } + +# Download a URL once per run and hand back the cached path. +# +# Downloads go to a content-addressed cache path, never to the source entry's +# `::` rename target. That matters for multi-arch: freetube's x86_64 and +# aarch64 entries both rename to freetube.AppImage, so writing to the rename +# target makes the second arch find the first arch's file and silently hash +# the wrong binary. This is the same collision that makes `makepkg -g` +# unusable for multi-arch PKGBUILDs. +download_cached() { + local url="$1" key path + [ -n "${DL_CACHE:-}" ] || die "download_cached: DL_CACHE is not set" + mkdir -p "$DL_CACHE" + key="$(printf '%s' "$url" | sha256sum | cut -c1-32)" + path="${DL_CACHE}/${key}" + + if [ ! -s "$path" ]; then + info "fetching ${url}" >&2 + curl --silent --show-error --location --fail --retry 3 --retry-delay 2 \ + --max-time 900 -o "${path}.part" "$url" \ + || die "download failed: ${url}" + mv "${path}.part" "$path" + fi + printf '%s' "$path" +} + +# Regenerate .SRCINFO and refuse to accept a broken one. +# +# This is the guard that would have prevented every observed breakage: an empty +# or inconsistent .SRCINFO is exactly what the AUR rejects, and the old script +# committed and pushed it anyway because it had no `set -e`. +regen_srcinfo() { + local dir="${1:-.}" + local out rc=0 + out="$(cd "$dir" && makepkg --printsrcinfo 2>&1)" || rc=$? + + if [ "$rc" -ne 0 ] || [ -z "${out//[[:space:]]/}" ]; then + printf '%s\n' "$out" >&2 + die "makepkg --printsrcinfo produced no usable output (exit ${rc}). + Is this running under a real Arch makepkg? The CI job must use the + archlinux container; Ubuntu's build environment silently yields an + empty .SRCINFO, which the AUR then rejects." + fi + + printf '%s\n' "$out" > "${dir}/.SRCINFO" + + local pb_ver sr_ver + pb_ver="$(pkgbuild_get pkgver "${dir}/PKGBUILD")" + sr_ver="$(sed -n -E 's/^[[:space:]]*pkgver = (.*)$/\1/p' "${dir}/.SRCINFO" | head -n1)" + [ -n "$sr_ver" ] || die ".SRCINFO has no pkgver" + [ "$pb_ver" = "$sr_ver" ] \ + || die ".SRCINFO pkgver '${sr_ver}' disagrees with PKGBUILD pkgver '${pb_ver}'" +} diff --git a/latest-version.sh b/latest-version.sh new file mode 100755 index 0000000..7fadec0 --- /dev/null +++ b/latest-version.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Resolve the latest upstream version for a package and print it on stdout. +# +# Reads the package's .autoupdate manifest (srchost/srcmntr/srcname/verstrip). +# +# usage: latest-version.sh [package-dir] +# +# Exit 0 with a version on stdout, or exit 1 with a message on stderr. +# +# There is deliberately NO "returned nothing so assume no update" path. The old +# workflow could not tell a broken query from a quiet night: pman-helper's +# lookup hit a 404, produced an empty string, and push.sh treated it as a +# version. "Query returned nothing" is an error, not a version. + +set -euo pipefail + +_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${_here}/common.sh" + +pkgdir="${1:-.}" +manifest="${pkgdir}/.autoupdate" +[ -f "$manifest" ] || die "no .autoupdate manifest in ${pkgdir}" + +# shellcheck disable=SC1090 +source "$manifest" + +: "${srchost:?manifest is missing srchost}" +: "${srcmntr:?manifest is missing srcmntr}" +: "${srcname:?manifest is missing srcname}" +verstrip="${verstrip:-}" + +GIT_HOST="${GIT_HOST:-git.bjphoster.com}" + +# curl that fails loudly on HTTP errors instead of returning an error page. +fetch() { + local url="$1"; shift + local out rc=0 + out="$(curl --silent --show-error --location --fail-with-body \ + --retry 3 --retry-delay 2 --max-time 60 "$@" "$url" 2>&1)" || rc=$? + if [ "$rc" -ne 0 ]; then + printf '%s\n' "$out" >&2 + die "request to ${url} failed (curl exit ${rc})" + fi + printf '%s' "$out" +} + +case "$srchost" in + github) + url="https://api.github.com/repos/${srcmntr}/${srcname}/releases/latest" + hdrs=(-H "Accept: application/vnd.github+json") + # Optional: avoids the 60/hour unauthenticated rate limit in CI. + [ -n "${GITHUB_TOKEN:-}" ] && hdrs+=(-H "Authorization: Bearer ${GITHUB_TOKEN}") + body="$(fetch "$url" "${hdrs[@]}")" + tag="$(printf '%s' "$body" | jq -r '.tag_name // empty')" + [ -n "$tag" ] || die "no tag_name in the latest release of ${srcmntr}/${srcname}" + ;; + + gitea) + # NB: the correct Gitea path is /repos/{owner}/{repo}/tags. The old + # workflows used /repos/{owner}/{repo}/repo/tags, which 404s. + url="https://${GIT_HOST}/api/v1/repos/${srcmntr}/${srcname}/tags" + body="$(fetch "$url" -H "Accept: application/json")" + printf '%s' "$body" | jq -e 'type == "array"' >/dev/null 2>&1 \ + || die "unexpected response shape from ${url}" + tag="$(printf '%s' "$body" | jq -r '.[].name' | sort -V | tail -n1)" + [ -n "$tag" ] || die "${srcmntr}/${srcname} has no tags" + ;; + + *) + die "unknown srchost '${srchost}' (expected 'github' or 'gitea')" + ;; +esac + +version="$tag" +if [ -n "$verstrip" ]; then + version="${version#"$verstrip"}" +fi + +validate_version "$version" "resolved version (from tag '${tag}')" +printf '%s\n' "$version" diff --git a/sync-aur.sh b/sync-aur.sh new file mode 100755 index 0000000..c62119d --- /dev/null +++ b/sync-aur.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Reconcile a package repo against its AUR counterpart. +# +# usage: sync-aur.sh [package-dir] +# +# This is the fix for the original bug. The old push.sh pushed to origin first +# and to the AUR second, so a failed AUR push left origin already carrying the +# new version; every later run then hit "same (old) version specified" and +# bare-exit 0'd, and the AUR silently rotted forever. +# +# Here the AUR is reconciled from STATE, not from events: this runs on every +# nightly pass regardless of whether a new upstream version was found, so a +# push that failed last night is simply retried tonight. + +set -euo pipefail + +_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${_here}/common.sh" + +pkgdir="${1:-.}" +cd "$pkgdir" + +pkgname="$(pkgbuild_get pkgname)" +# pkgname may be written as "${_pkgname}-appimage"; expand it properly. +case "$pkgname" in + *'$'*) pkgname="$(pkgbuild_expand PKGBUILD 'printf %s "$pkgname"')" ;; +esac +[ -n "$pkgname" ] || die "could not determine pkgname" + +aur_url="${AUR_SSH_BASE}/${pkgname}.git" +# Read over HTTPS so determining sync state needs no SSH key; only the push +# itself is authenticated. This keeps NO_GIT_PUSH dry runs working anywhere. +aur_ro_url="${AUR_HTTPS_BASE}/${pkgname}.git" + +# `git remote add` fails if the remote already exists, which it does whenever +# this runs twice in one working copy. +git remote remove aur >/dev/null 2>&1 || true +git remote add aur "$aur_url" +git remote remove aur-ro >/dev/null 2>&1 || true +git remote add aur-ro "$aur_ro_url" + +if git fetch --quiet aur-ro master 2>/dev/null; then + # Compare the CONTENT of the whole tree rather than commit hashes: origin + # accumulates commits the AUR never needs, and a re-derived commit has a + # different hash but an identical tree. No pathspec, because we push the + # whole tree -- restricting to PKGBUILD/.SRCINFO would report "in sync" + # while .desktop/.png/LICENSE actually differ. + if git diff --quiet FETCH_HEAD HEAD; then + log "AUR already in sync (${pkgname})" + exit 0 + fi + log "AUR differs from origin; pushing (${pkgname})" +else + log "no AUR repo for ${pkgname} yet; the first push will create it" +fi + +if [ -n "${NO_GIT_PUSH:-}" ]; then + log "NO_GIT_PUSH set: skipping push to ${aur_url}" + exit 0 +fi + +if git push aur HEAD:master; then + log "pushed ${pkgname} to the AUR" + exit 0 +fi + +# A rejected push here usually means the AUR has commits origin does not, +# i.e. someone pushed directly. Overwriting that silently would destroy it. +if [ -n "${AUR_FORCE:-}" ]; then + warn "AUR_FORCE set: force-pushing over the AUR history for ${pkgname}" + git push --force aur HEAD:master + exit 0 +fi + +die "push to the AUR was rejected for ${pkgname}. + The AUR history has probably diverged from origin (someone pushed there + directly). Inspect it, then re-run with AUR_FORCE=1 to overwrite. + This run leaves origin untouched and will be retried on the next pass." diff --git a/test.sh b/test.sh new file mode 100755 index 0000000..ece4c66 --- /dev/null +++ b/test.sh @@ -0,0 +1,441 @@ +#!/usr/bin/env bash +# Hermetic tests for the autoupdater. +# +# usage: ./test.sh [name-filter] +# +# Runs against local bare repos standing in for origin and the AUR, with a +# stubbed version lookup, so nothing here touches the network or the real AUR. +# Needs a genuine Arch makepkg (same requirement as the CI job). + +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +FILTER="${1:-}" + +PASS=0; FAIL=0; FAILED_NAMES=() + +pass() { printf ' \033[32mok\033[0m %s\n' "$1"; PASS=$((PASS+1)); } +fail() { printf ' \033[31mFAIL\033[0m %s\n %s\n' "$1" "${2-}"; FAIL=$((FAIL+1)); FAILED_NAMES+=("$1"); } + +assert_eq() { # expected actual label + if [ "$1" = "$2" ]; then pass "$3"; else fail "$3" "expected '$1', got '$2'"; fi +} +assert_ne() { + if [ "$1" != "$2" ]; then pass "$3"; else fail "$3" "expected something other than '$1'"; fi +} + +# ---------------------------------------------------------------- sandbox --- + +SANDBOX="" +setup() { + SANDBOX="$(mktemp -d)" + mkdir -p "$SANDBOX"/{origin,aur,work} + + cat > "$SANDBOX/fake-version.sh" <<'EOS' +#!/usr/bin/env bash +# Stands in for latest-version.sh. FAKE_VERSION_FAIL=1 simulates a broken +# upstream query; FAKE_VERSION="" simulates one that returns nothing. +if [ -n "${FAKE_VERSION_FAIL:-}" ]; then + echo "ERROR: simulated upstream lookup failure" >&2 + exit 1 +fi +printf '%s\n' "${FAKE_VERSION-1.0.0}" +EOS + chmod +x "$SANDBOX/fake-version.sh" + + # A local HTTP server so remote-source paths can be tested without network. + mkdir -p "$SANDBOX/www" + printf 'x86_64 binary payload\n' > "$SANDBOX/www/x86.bin" + printf 'aarch64 binary payload\n' > "$SANDBOX/www/arm.bin" + # -u: without it the "Serving HTTP on ... port N" line stays buffered and the + # port can never be read back. + ( cd "$SANDBOX/www" && exec python3 -u -m http.server 0 --bind 127.0.0.1 ) \ + > "$SANDBOX/www.log" 2>&1 & + WWW_PID=$! + for _ in $(seq 1 50); do + PORT="$(sed -n -E 's/.*http:\/\/127\.0\.0\.1:([0-9]+).*/\1/p' "$SANDBOX/www.log" | head -1)" + [ -n "$PORT" ] && break + sleep 0.1 + done + [ -n "${PORT:-}" ] || { echo "could not start test http server" >&2; exit 1; } + + export PKG_REPO_BASE="file://$SANDBOX/origin" + export AUR_HTTPS_BASE="file://$SANDBOX/aur" + export AUR_SSH_BASE="file://$SANDBOX/aur" + export LATEST_VERSION_CMD="$SANDBOX/fake-version.sh" + export GIT_AUTHOR_NAME=test GIT_AUTHOR_EMAIL=test@test + export GIT_COMMITTER_NAME=test GIT_COMMITTER_EMAIL=test@test +} + +teardown() { + [ -n "${WWW_PID:-}" ] && kill "$WWW_PID" 2>/dev/null + [ -n "$SANDBOX" ] && rm -rf "$SANDBOX" + return 0 +} + +# A multi-arch package whose arches download DIFFERENT files that both rename to +# the SAME target -- freetube-appimage's exact shape, and the case that makes +# `makepkg -g` emit identical sums for both arches. +make_pkg_colliding_remote() { + local pkg="$1" + local src="$SANDBOX/build/$pkg" + rm -rf "$src"; mkdir -p "$src" + + cat > "$src/PKGBUILD" < "$src/.autoupdate" <<'EOF' +srchost="gitea" +srcmntr="test" +srcname="test" +verstrip="" +gui="false" +EOF + + ( cd "$src" && "$HERE/update-checksums.sh" --remote . >/dev/null \ + && makepkg --printsrcinfo > .SRCINFO ) + + git init --quiet --bare --initial-branch=main "$SANDBOX/origin/$pkg" + git init --quiet --bare --initial-branch=master "$SANDBOX/aur/${pkg}.git" + ( cd "$src" && git init --quiet --initial-branch=main && git add -A \ + && git commit --quiet -m "initial" \ + && git push --quiet "file://$SANDBOX/origin/$pkg" main \ + && git push --quiet "file://$SANDBOX/aur/${pkg}.git" main:master ) +} + +# Create a package repo (origin) plus its AUR counterpart. +# make_pkg [multiarch] +make_pkg() { + local pkg="$1" multi="${2:-}" + local src="$SANDBOX/build/$pkg" + rm -rf "$src"; mkdir -p "$src" + + echo "payload for $pkg" > "$src/local.txt" + + if [ -n "$multi" ]; then + echo "x86 payload" > "$src/pay-x86_64.txt" + echo "arm payload" > "$src/pay-aarch64.txt" + cat > "$src/PKGBUILD" < "$src/PKGBUILD" < "$src/.autoupdate" <<'EOF' +srchost="gitea" +srcmntr="test" +srcname="test" +verstrip="" +gui="false" +EOF + echo "readme" > "$src/README.md" + + # Fix up the SKIP placeholders and generate a valid .SRCINFO. + ( cd "$src" && "$HERE/update-checksums.sh" . >/dev/null \ + && makepkg --printsrcinfo > .SRCINFO ) + + git init --quiet --bare --initial-branch=main "$SANDBOX/origin/$pkg" + git init --quiet --bare --initial-branch=master "$SANDBOX/aur/${pkg}.git" + + ( cd "$src" && git init --quiet --initial-branch=main && git add -A \ + && git commit --quiet -m "initial" \ + && git push --quiet "file://$SANDBOX/origin/$pkg" main \ + && git push --quiet "file://$SANDBOX/aur/${pkg}.git" main:master ) +} + +# Run the updater. Any extra env goes in front of the call by the caller. +run_update() { + local pkg="$1"; shift + WORKDIR="$SANDBOX/work/$(date +%s%N)" \ + "$HERE/autoupdate.sh" "$pkg" "$@" >"$SANDBOX/last.log" 2>&1 +} + +# Read a field from a bare repo's tip. +field_at() { # + git -C "$1" show "$2:PKGBUILD" 2>/dev/null \ + | sed -n -E "s/^$3=[\"']?([^\"']*)[\"']?$/\1/p" | head -n1 +} +aur_field() { field_at "$SANDBOX/aur/$1.git" master "$2"; } +origin_field() { field_at "$SANDBOX/origin/$1" main "$2"; } +aur_head() { git -C "$SANDBOX/aur/$1.git" rev-parse master 2>/dev/null; } + +should_run() { [ -z "$FILTER" ] || [[ "$1" == *"$FILTER"* ]]; } + +# ------------------------------------------------------------------ tests --- + +test_new_version() { + make_pkg p1 + FAKE_VERSION=2.0.0 run_update p1 + assert_eq "0" "$?" "new version: exits 0" + assert_eq "2.0.0" "$(aur_field p1 pkgver)" "new version: pkgver reaches the AUR" + assert_eq "1" "$(aur_field p1 pkgrel)" "new version: pkgrel resets to 1" +} + +test_noop_is_quiet_and_green() { + make_pkg p2 + FAKE_VERSION=1.0.0 run_update p2 + local rc=$? before after + before="$(aur_head p2)" + FAKE_VERSION=1.0.0 run_update p2 + after="$(aur_head p2)" + assert_eq "0" "$rc" "no-op: exits 0" + assert_eq "$before" "$after" "no-op: second run pushes nothing" + grep -q "AUR already in sync" "$SANDBOX/last.log" \ + && pass "no-op: reports already in sync" \ + || fail "no-op: reports already in sync" "$(tail -3 "$SANDBOX/last.log")" +} + +# The headline regression: origin ahead of the AUR, nothing new upstream. +# The old script aborted with "same (old) version" and exit 0, forever. +# Reproduces the exact reported failure: a previous run published to origin but +# its AUR push failed. Origin is therefore fully consistent -- correct sums, +# correct .SRCINFO, pkgrel already bumped -- so this run computes NO local +# changes at all. The only thing wrong is that the AUR is behind. The old +# script hit "same (old) version specified", bare-exit 0'd, and never retried. +# +# The pkgrel is pre-bumped deliberately: otherwise the pkgrel rule would itself +# produce a local change and the run would push for that reason instead, +# leaving the actual retry path untested. +test_failed_push_is_retried() { + make_pkg p3 + local clone="$SANDBOX/manual-p3" + git clone --quiet "file://$SANDBOX/origin/p3" "$clone" + ( cd "$clone" \ + && sed -i 's/^pkgdesc=.*/pkgdesc="changed out of band"/;s/^pkgrel=.*/pkgrel="2"/' PKGBUILD \ + && makepkg --printsrcinfo > .SRCINFO \ + && git commit --quiet -am "published to origin; AUR push failed" \ + && git push --quiet origin main ) + + assert_ne "changed out of band" "$(aur_field p3 pkgdesc)" "retry: AUR starts out of date" + + FAKE_VERSION=1.0.0 run_update p3 + local rc=$? + assert_eq "0" "$rc" "retry: exits 0" + + grep -q "no local changes" "$SANDBOX/last.log" \ + && pass "retry: run produced no local changes (exercises the reconcile path)" \ + || fail "retry: run produced no local changes" \ + "the run committed something, so this is not the retry path: $(tail -5 "$SANDBOX/last.log")" + + assert_eq "changed out of band" "$(aur_field p3 pkgdesc)" \ + "retry: reconciles to the AUR with nothing new locally or upstream" +} + +test_manual_edit_bumps_pkgrel() { + make_pkg p4 + local clone="$SANDBOX/manual-p4" + git clone --quiet "file://$SANDBOX/origin/p4" "$clone" + ( cd "$clone" && sed -i 's/^pkgdesc=.*/pkgdesc="fixed package() bug"/' PKGBUILD \ + && makepkg --printsrcinfo > .SRCINFO \ + && git commit --quiet -am "fix package()" && git push --quiet origin main ) + + FAKE_VERSION=1.0.0 run_update p4 + assert_eq "2" "$(aur_field p4 pkgrel)" "manual edit: pkgrel auto-bumped 1 -> 2" + + # The important one: re-running must not ratchet pkgrel upward. + FAKE_VERSION=1.0.0 run_update p4 + assert_eq "2" "$(aur_field p4 pkgrel)" "manual edit: pkgrel stable on re-run (idempotent)" + FAKE_VERSION=1.0.0 run_update p4 + assert_eq "2" "$(aur_field p4 pkgrel)" "manual edit: pkgrel still stable on third run" +} + +test_hand_bumped_pkgrel_respected() { + make_pkg p5 + local clone="$SANDBOX/manual-p5" + git clone --quiet "file://$SANDBOX/origin/p5" "$clone" + ( cd "$clone" && sed -i 's/^pkgdesc=.*/pkgdesc="edited"/;s/^pkgrel=.*/pkgrel="7"/' PKGBUILD \ + && makepkg --printsrcinfo > .SRCINFO \ + && git commit --quiet -am "edit + own pkgrel" && git push --quiet origin main ) + + FAKE_VERSION=1.0.0 run_update p5 + assert_eq "7" "$(aur_field p5 pkgrel)" "hand-bumped pkgrel is left alone" +} + +test_readme_only_edit() { + make_pkg p6 + local clone="$SANDBOX/manual-p6" + git clone --quiet "file://$SANDBOX/origin/p6" "$clone" + ( cd "$clone" && echo "more docs" >> README.md \ + && git commit --quiet -am "docs" && git push --quiet origin main ) + + FAKE_VERSION=1.0.0 run_update p6 + local rc=$? # must be captured before any other command clobbers $? + assert_eq "0" "$rc" "README-only edit: still exits 0" + assert_eq "1" "$(aur_field p6 pkgrel)" "README-only edit: no pkgrel bump" + assert_eq "more docs" "$(git -C "$SANDBOX/aur/p6.git" show master:README.md | tail -1)" \ + "README-only edit: still reconciled to the AUR" +} + +test_local_source_edit_bumps_pkgrel() { + make_pkg p7 + local clone="$SANDBOX/manual-p7" + git clone --quiet "file://$SANDBOX/origin/p7" "$clone" + ( cd "$clone" && echo "changed payload" > local.txt \ + && git commit --quiet -am "edit local source" && git push --quiet origin main ) + + FAKE_VERSION=1.0.0 run_update p7 + assert_eq "2" "$(aur_field p7 pkgrel)" "local source edit: re-hashed and pkgrel bumped" +} + +test_force_rebuild() { + make_pkg p8 + FAKE_VERSION=1.0.0 run_update p8 + FORCE_REBUILD=1 FAKE_VERSION=1.0.0 run_update p8 + assert_eq "2" "$(aur_field p8 pkgrel)" "FORCE_REBUILD: pkgrel +1 with no other change" +} + +test_lookup_failure_is_loud() { + make_pkg p9 + local before; before="$(aur_head p9)" + FAKE_VERSION_FAIL=1 run_update p9 + assert_eq "1" "$?" "broken upstream lookup: exits 1, not mistaken for 'no new version'" + assert_eq "$before" "$(aur_head p9)" "broken upstream lookup: nothing pushed" +} + +test_empty_version_aborts_before_commit() { + make_pkg p10 + local before; before="$(aur_head p10)" + FAKE_VERSION="" run_update p10 + assert_eq "1" "$?" "empty version: exits 1" + assert_eq "$before" "$(aur_head p10)" "empty version: aborts before committing" + assert_eq "1.0.0" "$(origin_field p10 pkgver)" "empty version: origin left unpoisoned" +} + +# reflex-appimage's real state: local ahead of the AUR carrying a stale pkgrel +# that the old script never reset. +test_stale_pkgrel_repaired() { + make_pkg p11 + local clone="$SANDBOX/manual-p11" + git clone --quiet "file://$SANDBOX/origin/p11" "$clone" + ( cd "$clone" && sed -i 's/^pkgver=.*/pkgver="1.5.0"/;s/^pkgrel=.*/pkgrel="4"/' PKGBUILD \ + && makepkg --printsrcinfo > .SRCINFO \ + && git commit --quiet -am "version bumped without resetting pkgrel" \ + && git push --quiet origin main ) + + FAKE_VERSION=1.5.0 run_update p11 + assert_eq "1.5.0" "$(aur_field p11 pkgver)" "stale pkgrel: version published" + assert_eq "1" "$(aur_field p11 pkgrel)" "stale pkgrel: reset to 1 against the AUR" +} + +test_multiarch_distinct_sums() { + make_pkg p12 multi + FAKE_VERSION=1.0.0 run_update p12 + + local x a + x="$(git -C "$SANDBOX/aur/p12.git" show master:PKGBUILD \ + | sed -n '/^sha256sums_x86_64=/,/)/p' | grep -oE '[0-9a-f]{64}')" + a="$(git -C "$SANDBOX/aur/p12.git" show master:PKGBUILD \ + | sed -n '/^sha256sums_aarch64=/,/)/p' | grep -oE '[0-9a-f]{64}')" + + assert_eq "$(sha256sum "$SANDBOX/build/p12/pay-x86_64.txt" | awk '{print $1}')" "$x" \ + "multi-arch: x86_64 sum correct" + assert_eq "$(sha256sum "$SANDBOX/build/p12/pay-aarch64.txt" | awk '{print $1}')" "$a" \ + "multi-arch: aarch64 sum correct" + assert_ne "$x" "$a" "multi-arch: arches get distinct sums (the makepkg -g failure mode)" +} + +# The regression that rules out `makepkg -g`: two arches downloading different +# files that rename to the same target must not end up sharing a checksum. +test_multiarch_colliding_downloads() { + make_pkg_colliding_remote p15 + FAKE_VERSION=1.0.0 FORCE_REFRESH=1 run_update p15 + + local pb x a + pb="$(git -C "$SANDBOX/aur/p15.git" show master:PKGBUILD)" + x="$(printf '%s' "$pb" | sed -n '/^sha256sums_x86_64=/,/)/p' | grep -oE '[0-9a-f]{64}')" + a="$(printf '%s' "$pb" | sed -n '/^sha256sums_aarch64=/,/)/p' | grep -oE '[0-9a-f]{64}')" + + assert_eq "$(sha256sum "$SANDBOX/www/x86.bin" | awk '{print $1}')" "$x" \ + "colliding downloads: x86_64 hashed its own file" + assert_eq "$(sha256sum "$SANDBOX/www/arm.bin" | awk '{print $1}')" "$a" \ + "colliding downloads: aarch64 hashed its own file, not x86_64's" + assert_ne "$x" "$a" "colliding downloads: sums differ despite the shared :: target" +} + +test_srcinfo_guard() { + make_pkg p13 + local clone="$SANDBOX/manual-p13" + git clone --quiet "file://$SANDBOX/origin/p13" "$clone" + # pkgver that makepkg rejects -> .SRCINFO cannot be generated. + ( cd "$clone" && sed -i 's/^pkgdesc=.*/pkgdesc="x"/' PKGBUILD \ + && git commit --quiet -am "touch" && git push --quiet origin main ) + local before; before="$(aur_head p13)" + FAKE_VERSION="not a version!" run_update p13 + assert_eq "1" "$?" ".SRCINFO guard: invalid version rejected" + assert_eq "$before" "$(aur_head p13)" ".SRCINFO guard: nothing reached the AUR" +} + +test_no_git_push_dry_run() { + make_pkg p14 + local before_aur before_origin + before_aur="$(aur_head p14)" + before_origin="$(git -C "$SANDBOX/origin/p14" rev-parse main)" + NO_GIT_PUSH=1 FAKE_VERSION=3.0.0 run_update p14 + assert_eq "0" "$?" "NO_GIT_PUSH: exits 0" + assert_eq "$before_aur" "$(aur_head p14)" "NO_GIT_PUSH: AUR untouched" + assert_eq "$before_origin" "$(git -C "$SANDBOX/origin/p14" rev-parse main)" \ + "NO_GIT_PUSH: origin untouched" +} + +# ------------------------------------------------------------------- main --- + +trap teardown EXIT +setup + +for t in $(declare -F | awk '{print $3}' | grep '^test_' | sort); do + should_run "$t" || continue + printf '\n\033[1m%s\033[0m\n' "$t" + "$t" +done + +printf '\n────────────────────────\n' +printf 'passed: %d failed: %d\n' "$PASS" "$FAIL" +if [ "$FAIL" -gt 0 ]; then + printf 'failing: %s\n' "${FAILED_NAMES[*]}" + exit 1 +fi diff --git a/update-checksums.sh b/update-checksums.sh new file mode 100755 index 0000000..8d9599c --- /dev/null +++ b/update-checksums.sh @@ -0,0 +1,105 @@ +#!/usr/bin/env bash +# Recompute sha256sums for a PKGBUILD, single- or multi-arch. +# +# usage: update-checksums.sh [--remote] [package-dir] +# +# --remote also re-download and re-hash remote sources. +# Without it only local repo files (.desktop, icons) are +# re-hashed, which needs no network and is what lets a +# hand-edited repo self-correct on every nightly run. +# +# sha256 only: sha1sums/md5sums are retired and actively removed. +# +# Why this does not shell out to `makepkg -g` / `updpkgsums`: those generate +# every arch array in one pass and download all arches into a single directory. +# When two arches rename to the same target (`foo.AppImage::url-amd64`, +# `foo.AppImage::url-arm64`) the second arch finds the first arch's file +# already present, skips the download, and emits an sha256sums_aarch64 +# identical to sha256sums_x86_64 -- silently wrong. Overriding CARCH does not +# help; makepkg.conf resets it, and --config still downloads every arch. + +set -euo pipefail + +_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${_here}/common.sh" + +refresh_remote=0 +if [ "${1-}" = "--remote" ]; then refresh_remote=1; shift; fi + +pkgdir="${1:-.}" +pkgbuild="${pkgdir}/PKGBUILD" +[ -f "$pkgbuild" ] || die "no PKGBUILD in ${pkgdir}" + +export DL_CACHE="${DL_CACHE:-$(mktemp -d)}" + +mapfile -t arches < <(pkgbuild_array "$pkgbuild" arch) + +# Build the work list: every source array present, paired with its sums array. +# Single-arch is just the degenerate case of the same loop -- no special path. +pairs=() +if pkgbuild_has_array "$pkgbuild" source; then + pairs+=("source:sha256sums") +fi +for a in "${arches[@]}"; do + [ "$a" = "any" ] && continue + if pkgbuild_has_array "$pkgbuild" "source_${a}"; then + pairs+=("source_${a}:sha256sums_${a}") + fi +done + +[ "${#pairs[@]}" -gt 0 ] || die "PKGBUILD declares no source arrays" + +for pair in "${pairs[@]}"; do + src_arr="${pair%%:*}" + sum_arr="${pair##*:}" + + mapfile -t entries < <(pkgbuild_array "$pkgbuild" "$src_arr") + [ "${#entries[@]}" -gt 0 ] || continue + + mapfile -t existing < <(pkgbuild_array "$pkgbuild" "$sum_arr") + + # If the existing sums don't line up with the sources we cannot reuse any of + # them by index, so everything must be recomputed. + reuse=1 + if [ "${#existing[@]}" -ne "${#entries[@]}" ]; then reuse=0; fi + + log "${src_arr} -> ${sum_arr} (${#entries[@]} source(s))" + + sums=() + for i in "${!entries[@]}"; do + entry="${entries[$i]}" + loc="$(source_loc "$entry")" + + if is_vcs "$entry"; then + sum="SKIP" + info "$(source_name "$entry") SKIP (vcs)" + elif is_remote "$entry"; then + if [ "$refresh_remote" -eq 1 ] || [ "$reuse" -eq 0 ] \ + || [ -z "${existing[$i]:-}" ] || [ "${existing[$i]}" = "SKIP" ]; then + path="$(download_cached "$loc")" + sum="$(sha256sum "$path" | awk '{print $1}')" + info "$(source_name "$entry") ${sum}" + else + sum="${existing[$i]}" + info "$(source_name "$entry") ${sum} (kept)" + fi + else + # A file that lives in the package repo itself. + [ -f "${pkgdir}/${loc}" ] || die "local source '${loc}' not found in ${pkgdir}" + sum="$(sha256sum "${pkgdir}/${loc}" | awk '{print $1}')" + info "${loc} ${sum} (local)" + fi + sums+=("$sum") + done + + replace_array "$pkgbuild" "$sum_arr" "${sums[@]}" +done + +# Retire the legacy digests wherever they still appear. +for a in "" "${arches[@]}"; do + suffix="${a:+_$a}" + for algo in sha1sums md5sums sha224sums sha384sums sha512sums b2sums; do + remove_array "$pkgbuild" "${algo}${suffix}" + done +done