diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c4bbed9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +# downloaded tarballs +*.tar.gz +# built packages +*.deb +*.buildinfo +*.changes diff --git a/README.md b/README.md index e605adf..34af56e 100644 --- a/README.md +++ b/README.md @@ -1 +1,206 @@ -# DEB Auto Update Workflows +# DEB Auto Update + +Nightly reconciler that keeps the `deb/*` package repos and the `deb` org's +Gitea package registry in sync with their upstream releases. + +It mirrors [`aur/.autoupdate`](https://git.bjphoster.com/aur/.autoupdate): a new +upstream tag, a `debian/` edited by hand and pushed to origin, and an upload +that failed last night all take the same path. On every pass the version in +`debian/changelog` is compared with what the registry holds, and the registry +is brought in step. + +## Layout + +| file | role | +|---|---| +| `autoupdate.sh` | entry point: bump, build, upload, push, one package per process | +| `latest-version.sh` | resolves the latest upstream version (verbatim from `aur/.autoupdate`) | +| `common.sh` | logging, guards, version compare, tarball fetch, registry API | +| `dockerfile` | the Debian toolchain image the nightly job runs in | +| `.gitea/workflows/update.yaml` | nightly matrix job | +| `.gitea/workflows/build-image.yaml` | monthly image rebuild | + +Nothing here is package-specific. Everything that differs between packages +lives in the package repo itself. + +## Package repo requirements + +`deb/` carries: + +- `debian/` (at the top of the repo): `changelog` whose top version is + `-`, `control`, `rules`, `source/format`, and so + on. It is dropped into the unpacked upstream tarball at build time, replacing + any `debian/` upstream might ship. +- `.autoupdate`, the same manifest the AUR packages use: + +```sh +srchost="gitea" # gitea | github +srcmntr="source" # upstream owner/org +srcname="skeptical-updater" # upstream repo +verstrip="" # tag prefix to strip (tags are bare semver) +gui="false" # ignored here; AUR AppImage extraction only +``` + +Start a new package at a placeholder such as `0.0.1-1`: the first nightly run +bumps it to the newest tag. + +## What a run does + +1. Clone `https://git.bjphoster.com/deb/` and read `.autoupdate`. +2. Resolve the newest upstream tag (`latest-version.sh`: Gitea tags API, + `sort -V`). A *failed lookup* exits 1; "no new version" is not a failure. +3. If the tag is newer than the changelog's upstream version, prepend a stanza + in exactly the format `dch --newversion X-1 --distribution stable` writes: + + ``` + (X-1) stable; urgency=medium + + * Updated to X + + -- <> Sat, 10 Oct 2026 00:00:00 +0000 + + ``` + + Nothing below it changes, and `dpkg-parsechangelog` must read back exactly + `X-1` and parse every stanza without a warning before anything is + committed. The commit is `Updated to X-1`. +4. Ask the registry whether it already holds that version. +5. If it does not, download `https://git.bjphoster.com/source//archive/.tar.gz`, + place it as `_.orig.tar.gz` next to the unpacked tree, build with + `dpkg-buildpackage -us -uc -b` as the unprivileged `builder` user, run + `lintian` (advisory, never fatal) and upload every built `.deb` to the + `stable` distribution, `main` component. +6. Push to origin, last. The registry is the distribution channel and goes + first, as the AUR does in `aur/.autoupdate`: a failed build or upload leaves + origin untouched, and the next night recomputes everything. + +The image is Debian bookworm: a package built against the older dpkg and +debhelper installs on bookworm, trixie and Ubuntu alike. + +## Versions + +`X-N`: everything before the dash is upstream, `-N` is the Debian revision. + +- newer upstream tag: `X-1` +- `force_rebuild` with no new tag: `X-(N+1)`, changelog `* Rebuilt X` +- an older tag (a tag deleted upstream) never downgrades the package +- a hand edit that adds a changelog stanza and is pushed to origin is published + on the next run, because the registry lacks that version +- an epoch (`1:X-N`) is carried over unchanged + +## Environment and inputs + +| workflow input | variable | effect | +|---|---|---| +| `package` | | run one matrix entry only (blank = all) | +| `force_rebuild` | `FORCE_REBUILD=1` | bump the Debian revision even with no new upstream version; replaces a registry copy of that exact version if one exists | +| `force_refresh` | `FORCE_REFRESH=1` | re-download the source and rebuild even when nothing changed (a check; uploads only what is missing) | +| `no_git_push` | `NO_GIT_PUSH=1` | dry run: build, but neither upload nor push to origin | +| | `NO_UPLOAD=1` / `--no-upload` | never contact the registry: no lookup, no upload; builds only when this run bumped the version | +| | `DEB_DISTRIBUTION` | registry distribution and changelog distribution (default `stable`) | +| | `DEB_COMPONENT` | registry component (default `main`) | +| | `GIT_HOST` | git host and registry (default `git.bjphoster.com`) | +| | `GIT_NAME`, `GIT_EMAIL` | changelog and commit identity (default: `git config`) | +| | `GIT_TOKEN` | registry token (default: the git credential helper, as in CI) | +| | `WORKDIR` | where to clone and build (default: a temp dir) | +| | `AUTOUPDATE_TEST_VERSION` | local test mode: use this as the upstream version | +| | `AUTOUPDATE_TEST_TARBALL` | local test mode: build from this local tarball | +| | `PKG_REPO_BASE` | clone package repos from here (default `https://$GIT_HOST/deb`) | + +Either `AUTOUPDATE_TEST_*` variable forces `NO_GIT_PUSH=1` and `NO_UPLOAD=1`, +so a test run can never publish anything. + +## Setup + +Required on the `deb` org (Settings -> Actions -> Secrets / Variables), or on +the `.autoupdate` repo itself: + +| kind | name | value | +|---|---|---| +| secret | `GIT_TOKEN` | access token with **write:repository** (clone and push `deb/*`) and **write:package** (registry upload, image push, image pull) | +| variable | `GIT_NAME` | display name for commits and the changelog, e.g. `Bryan Joshua Pedini` | +| variable | `GIT_EMAIL` | email for commits and the changelog | +| variable | `REGISTRY_USER` | your Gitea **username** for the container registry login (not `GIT_NAME`, which has spaces) | +| variable | `GIT_HOST` | optional; defaults to `git.bjphoster.com` | + +Also: + +- enable **Actions** and **Packages** on `deb/.autoupdate` (repo Settings -> + Advanced Settings), and make sure a runner with the `ubuntu-latest` label + serves the `deb` org (an instance-wide runner does); +- the `deb` org's registry (`/api/packages/deb/...`) needs no switch of its + own: it answers whenever packages are enabled on the instance. Enabling + Packages on the package repos only adds the Packages tab and lets you link + the registry entry to the repo; +- build the image once by hand (Actions -> Build Autoupdater Image -> Run + workflow) before the first nightly run, since `update.yaml` pulls it; +- list every package in the `matrix.package` of `.gitea/workflows/update.yaml`. + A package repo that is not listed there is never processed. + +The token is read back by `autoupdate.sh` through `git credential fill`, from the +`~/.git-credentials` the workflow writes, and handed to `curl` on stdin; it is +never on a command line or in a log. + +## Running by hand + +Actions -> Update Debian Packages -> Run workflow. Inputs: `package` (blank for +the whole matrix), `force_rebuild`, `force_refresh`, `no_git_push`. + +## Installing packages + +Debian, Ubuntu and derivatives: + +```sh +sudo install -d -m 0755 /etc/apt/keyrings +sudo curl -fsSL https://git.bjphoster.com/api/packages/deb/debian/repository.key \ + -o /etc/apt/keyrings/git.bjphoster.com-deb.asc +echo "deb [signed-by=/etc/apt/keyrings/git.bjphoster.com-deb.asc] https://git.bjphoster.com/api/packages/deb/debian stable main" \ + | sudo tee /etc/apt/sources.list.d/git.bjphoster.com-deb.list +sudo apt update +sudo apt install skeptical-updater +``` + +The bare source line is +`deb https://git.bjphoster.com/api/packages/deb/debian stable main`; Gitea signs +the repository's `Release` files with +`https://git.bjphoster.com/api/packages/deb/debian/repository.key`. + +## Container image + +`dockerfile` builds the environment the nightly job runs in, published to the +Gitea registry as `git.bjphoster.com/deb/autoupdate:latest`: +`debian:bookworm-slim` with `dpkg-dev`, `debhelper`, `build-essential`, +`fakeroot`, `devscripts`, `lintian`, `git`, `curl`, `jq`, `ca-certificates`, +`nodejs` (actions/checkout is a JavaScript action) and a `builder` user. + +`.gitea/workflows/build-image.yaml` rebuilds and overwrites `:latest` monthly, +on any push touching the `dockerfile`, and on demand. It needs the same +`REGISTRY_USER` and `GIT_TOKEN`. + +## Testing locally + +A dry run against a throwaway copy of the real package repo, with a forced +upstream version and a local tarball built exactly like Gitea's archive, and no +network at all. The test variables imply `NO_GIT_PUSH=1` and `NO_UPLOAD=1`. + +```sh +docker build -t git.bjphoster.com/deb/autoupdate:latest . + +t="$(mktemp -d)" +git clone --quiet --bare https://git.bjphoster.com/deb/skeptical-updater "$t/origin/skeptical-updater" +git -C ~/projects/sources/skeptical-updater archive --format=tar \ + --prefix=skeptical-updater/ HEAD | gzip -n > "$t/src.tar.gz" + +docker run --rm --network none -v "$PWD:/src:ro" -v "$t:/t:ro" \ + git.bjphoster.com/deb/autoupdate:latest \ + bash -c 'cp -r /src /w && cp -r /t /tt && chown -R builder:builder /w /tt && cd /w && + runuser -u builder -- env GIT_NAME="Your Name" GIT_EMAIL=you@example.com \ + PKG_REPO_BASE=file:///tt/origin WORKDIR=/tt/work \ + AUTOUPDATE_TEST_VERSION=0.1.0 AUTOUPDATE_TEST_TARBALL=/tt/src.tar.gz \ + ./autoupdate.sh skeptical-updater && + runuser -u builder -- git -C /tt/work/skeptical-updater show --stat HEAD' +``` + +The built package lands in `/tt/work/.debbuild/skeptical-updater/` inside the +container. Lint the scripts with +`docker run --rm -v "$PWD:/mnt:ro" -w /mnt koalaman/shellcheck:stable -x *.sh`. diff --git a/autoupdate.sh b/autoupdate.sh new file mode 100755 index 0000000..69ebcf8 --- /dev/null +++ b/autoupdate.sh @@ -0,0 +1,312 @@ +#!/usr/bin/env bash +# Nightly Debian package reconciler. +# +# usage: autoupdate.sh [--no-upload] [...] +# +# env: +# GIT_HOST git host holding the package repos and the package +# registry (default git.bjphoster.com) +# WORKDIR where to clone and build; a temp dir by default +# NO_GIT_PUSH=1 dry run: build, but neither upload nor push to origin +# NO_UPLOAD=1 never contact the package registry (same as --no-upload) +# FORCE_REBUILD=1 bump the Debian revision even with no new upstream version +# FORCE_REFRESH=1 re-download the source and rebuild even when nothing changed +# DEB_DISTRIBUTION registry distribution, also written into the changelog +# (default stable) +# DEB_COMPONENT registry component (default main) +# GIT_NAME, GIT_EMAIL changelog and commit identity (default: git config) +# GIT_TOKEN registry token (default: the git credential helper) +# +# local test mode -- setting either of these implies NO_GIT_PUSH=1 and +# NO_UPLOAD=1, so a test can never publish anything: +# AUTOUPDATE_TEST_VERSION use this as the upstream version (no lookup) +# AUTOUPDATE_TEST_TARBALL build from this local tarball (no download) +# plus PKG_REPO_BASE=file:///some/dir to clone a throwaway copy of the repo. +# +# Like aur/.autoupdate this is a RECONCILER: the registry is the distribution +# channel, and it is compared against the package repo on every pass. A new +# upstream release, a hand-edited debian/ pushed to origin, and an upload that +# failed last night all end in the same place: whatever version the changelog +# says, the registry gets it. Distribution channel first, origin second, so a +# failed build or upload leaves origin untouched and tonight's work is simply +# recomputed tomorrow. + +set -euo pipefail + +_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${_here}/common.sh" + +require_noninteractive +require_tools dpkg-parsechangelog dpkg-buildpackage dpkg-deb + +while [ "$#" -gt 0 ]; do + case "$1" in + --no-upload) export NO_UPLOAD=1; shift ;; + --) shift; break ;; + -*) die "unknown option '$1' (usage: autoupdate.sh [--no-upload] [...])" ;; + *) break ;; + esac +done +[ "$#" -ge 1 ] || die "usage: autoupdate.sh [--no-upload] [...]" + +if [ -n "${AUTOUPDATE_TEST_VERSION:-}${AUTOUPDATE_TEST_TARBALL:-}" ]; then + export NO_GIT_PUSH=1 NO_UPLOAD=1 +fi + +GIT_HOST="${GIT_HOST:-git.bjphoster.com}" +# The org: where the package repos live and whose registry receives the debs. +REGISTRY_OWNER="${REGISTRY_OWNER:-deb}" +DEB_DISTRIBUTION="${DEB_DISTRIBUTION:-stable}" +DEB_COMPONENT="${DEB_COMPONENT:-main}" +# Overridable so a test can work on a throwaway clone; defaults are production. +PKG_REPO_BASE="${PKG_REPO_BASE:-https://${GIT_HOST}/${REGISTRY_OWNER}}" +LATEST_VERSION_CMD="${LATEST_VERSION_CMD:-${_here}/latest-version.sh}" + +WORKDIR="${WORKDIR:-$(mktemp -d)}" +mkdir -p "$WORKDIR" +REGISTRY_BODY="${WORKDIR}/.registry-response" + +# Prepend a changelog stanza in exactly the format dch writes: +# +# (X-N) ; urgency=medium +# +# * +# +# -- Name Sat, 10 Oct 2026 00:00:00 +0000 +# +# Written directly rather than through dch: no devscripts at run time, no +# distribution-name warnings, and the bytes are fully predictable. Nothing +# below the new stanza changes. +changelog_bump() { # + local version="$1" note="$2" srcpkg stamp got warnings + resolve_identity + srcpkg="$(dpkg-parsechangelog -l debian/changelog -S Source)" \ + || die "cannot read the source name from debian/changelog" + stamp="$(LC_ALL=C date -u -R)" + { + printf '%s (%s) %s; urgency=medium\n\n' "$srcpkg" "$version" "$DEB_DISTRIBUTION" + printf ' * %s\n\n' "$note" + printf ' -- %s <%s> %s\n\n' "$MAINT_NAME" "$MAINT_EMAIL" "$stamp" + cat debian/changelog + } > debian/changelog.new + cat debian/changelog.new > debian/changelog # keep the inode and mode + rm -f debian/changelog.new + + # Refuse to go on unless dpkg reads back exactly what was written, and the + # whole file (every stanza) still parses without a single warning. + got="$(dpkg-parsechangelog -l debian/changelog -S Version)" \ + || die "dpkg-parsechangelog cannot parse debian/changelog after the bump" + [ "$got" = "$version" ] || die "dpkg-parsechangelog reads ${got}, expected ${version}" + warnings="$(dpkg-parsechangelog -l debian/changelog --all 2>&1 >/dev/null)" || true + if [ -n "$warnings" ]; then + printf '%s\n' "$warnings" >&2 + die "debian/changelog does not parse cleanly after the bump" + fi +} + +# Binary package names debian/control declares (what the registry is keyed on). +control_names() { + sed -n -E 's/^Package:[[:space:]]*([^[:space:]]+)[[:space:]]*$/\1/p' debian/control | sort -u +} + +# dpkg-buildpackage -b from the upstream tarball, laid out the way dpkg-source +# expects: _.orig.tar.gz next to the unpacked tree, with the +# packaging repo's debian/ dropped into it. Fills the global BUILT array. +build_deb() { # + local tarball="$1" srcpkg="$2" upver="$3" bdir tree + bdir="${WORKDIR}/.debbuild/${pkg}" + rm -rf "$bdir" + mkdir -p "$bdir" + + cp "$tarball" "${bdir}/${srcpkg}_${upver}.orig.tar.gz" + tar -xzf "${bdir}/${srcpkg}_${upver}.orig.tar.gz" -C "$bdir" + mapfile -t tree < <(find "$bdir" -mindepth 1 -maxdepth 1 -type d) + [ "${#tree[@]}" -eq 1 ] || die "expected one top-level directory in the tarball, found ${#tree[@]}" + + # The packaging repo's debian/ always wins over anything upstream ships. + rm -rf "${tree[0]}/debian" + cp -a debian "${tree[0]}/debian" + + log "dpkg-buildpackage -us -uc -b" + ( cd "${tree[0]}" && dpkg-buildpackage -us -uc -b ) + + mapfile -t BUILT < <(find "$bdir" -maxdepth 1 -type f -name '*.deb' | sort) + [ "${#BUILT[@]}" -gt 0 ] || die "dpkg-buildpackage produced no packages" + local f + for f in "${BUILT[@]}"; do info "built $(basename "$f")"; done + + # Advisory only, like namcap is for the AUR: never blocks a release. + if command -v lintian >/dev/null 2>&1; then + log "lintian" + lintian "$bdir"/*.changes || warn "lintian reported problems (non-fatal)" + else + warn "lintian is not installed; skipping the lint" + fi +} + +publish_deb() { # + local replace="$1" pool f name version arch + pool="https://${GIT_HOST}/api/packages/${REGISTRY_OWNER}/debian/pool/${DEB_DISTRIBUTION}/${DEB_COMPONENT}" + for f in "${BUILT[@]}"; do + if [ "$replace" -eq 1 ]; then + name="$(dpkg-deb -f "$f" Package)" + version="$(dpkg-deb -f "$f" Version)" + arch="$(dpkg-deb -f "$f" Architecture)" + registry_delete "${pool}/${name}/${version}/${arch}" + fi + registry_upload "${pool}/upload" "$f" + done +} + +update_one() { + pkg="$1" + local dir="${WORKDIR}/${pkg}" + + log "=== ${pkg} ===" + + clone_or_reset "${PKG_REPO_BASE}/${pkg}" "$dir" + cd "$dir" + [ -f debian/changelog ] || die "${pkg} has no debian/changelog" + [ -f debian/control ] || die "${pkg} has no debian/control" + [ -f .autoupdate ] || die "${pkg} has no .autoupdate manifest" + + # shellcheck disable=SC1091 + source ./.autoupdate + : "${srchost:?manifest is missing srchost}" + : "${srcmntr:?manifest is missing srcmntr}" + : "${srcname:?manifest is missing srcname}" + + # --- packaged version ---------------------------------------------------- + # [epoch:]upstream-revision; the revision must be a plain integer so it can + # be reset to 1 or incremented. + local full epoch="" oldver oldrev srcpkg + full="$(dpkg-parsechangelog -l debian/changelog -S Version)" \ + || die "cannot parse debian/changelog" + srcpkg="$(dpkg-parsechangelog -l debian/changelog -S Source)" + if [[ "$full" == *:* ]]; then epoch="${full%%:*}:"; full="${full#*:}"; fi + [[ "$full" == *-* ]] || die "debian/changelog version '${full}' has no Debian revision (native package?)" + oldver="${full%-*}" + oldrev="${full##*-}" + validate_version "$oldver" "upstream part of the debian/changelog version" + [[ "$oldrev" =~ ^[0-9]+$ ]] \ + || die "Debian revision '${oldrev}' is not a plain integer; refusing to rewrite it" + + # --- upstream version ---------------------------------------------------- + # A failed lookup exits non-zero and is NOT mistaken for "no new version". + local newver + if [ -n "${AUTOUPDATE_TEST_VERSION:-}" ]; then + newver="$AUTOUPDATE_TEST_VERSION" + info "test mode: upstream version forced to ${newver}" + else + newver="$("$LATEST_VERSION_CMD" .)" || die "upstream version lookup failed for ${pkg}" + fi + validate_version "$newver" "upstream version for ${pkg}" + info "upstream ${newver}, packaged ${epoch}${oldver}-${oldrev}" + + # --- bump ------------------------------------------------------------------ + local ver="$oldver" rev="$oldrev" bumped=0 + if version_gt "$newver" "$oldver"; then + ver="$newver"; rev=1 + log "new upstream version ${oldver} -> ${ver}" + changelog_bump "${epoch}${ver}-${rev}" "Updated to ${ver}" + bumped=1 + else + if [ "$newver" != "$oldver" ]; then + warn "upstream ${newver} is older than the packaged ${oldver}; not downgrading" + fi + if [ -n "${FORCE_REBUILD:-}" ]; then + rev=$(( oldrev + 1 )) + log "FORCE_REBUILD: Debian revision ${oldrev} -> ${rev}" + changelog_bump "${epoch}${ver}-${rev}" "Rebuilt ${ver}" + bumped=1 + fi + fi + local version="${epoch}${ver}-${rev}" + + git add -- debian/changelog + if git diff --cached --quiet; then + log "no local changes for ${pkg}" + else + git_commit "Updated ${pkg} to ${ver}-${rev}" + log "committed ${pkg} ${version}" + fi + + # --- distribution channel ------------------------------------------------- + # Reference state first: which version does the registry hold? + local state="skipped" names=() name + if [ -z "${NO_UPLOAD:-}" ]; then + local list + list="$(control_names)" || die "cannot list the packages in debian/control" + [ -n "$list" ] || die "debian/control declares no binary packages" + mapfile -t names <<<"$list" + state="present" + for name in "${names[@]}"; do + if registry_has "$REGISTRY_OWNER" debian "$name" "$version"; then + info "registry has ${name} ${version}" + else + info "registry lacks ${name} ${version}" + state="absent" + fi + done + fi + + local build=0 upload=0 replace=0 + case "$state" in + absent) build=1; upload=1 ;; + present) + if [ -n "${FORCE_REBUILD:-}" ]; then build=1; upload=1; replace=1 + elif [ -n "${FORCE_REFRESH:-}" ]; then build=1 + else log "registry already has ${pkg} ${version}; nothing to build" + fi ;; + skipped) + if [ "$bumped" -eq 1 ] || [ -n "${FORCE_REFRESH:-}" ]; then build=1 + else log "NO_UPLOAD set and nothing changed; nothing to build" + fi ;; + esac + + if [ "$build" -eq 1 ]; then + local tag="${verstrip:-}${ver}" topdir="" url tarball="${WORKDIR}/.sources/${pkg}-${ver}.tar.gz" + case "$srchost" in + gitea) url="https://${GIT_HOST}/${srcmntr}/${srcname}/archive/${tag}.tar.gz" + topdir="$srcname" ;; + github) url="https://github.com/${srcmntr}/${srcname}/archive/refs/tags/${tag}.tar.gz" ;; + *) die "unknown srchost '${srchost}' (expected 'github' or 'gitea')" ;; + esac + fetch_tarball "$url" "$tarball" "$topdir" + build_deb "$tarball" "$srcpkg" "$ver" + fi + + if [ "$upload" -eq 1 ]; then + if [ -n "${NO_GIT_PUSH:-}" ]; then + log "NO_GIT_PUSH set: not uploading ${pkg} ${version} to ${REGISTRY_OWNER}'s registry" + else + publish_deb "$replace" + fi + fi + + # --- origin --------------------------------------------------------------- + # Only after the registry is in step, so a failed build or upload never + # leaves origin claiming a release that was not published. + if [ -n "${NO_GIT_PUSH:-}" ]; then + log "NO_GIT_PUSH set: not pushing ${pkg} to origin" + else + git push --quiet origin HEAD + fi +} + +# One package per process, deliberately -- see aur/.autoupdate: a subshell in +# an && / || list disables `set -e` for its whole body, so `( update_one ) ||` +# would sail past a failed build straight into `git push origin`. Re-exec'ing +# gives each package a fresh shell whose `set -e` is intact. The CI matrix +# passes exactly one package, so the loop only affects local runs. +if [ -n "${AUTOUPDATE_SINGLE:-}" ]; then + update_one "$1" + exit 0 +fi + +rc=0 +for pkg in "$@"; do + AUTOUPDATE_SINGLE=1 "$0" "$pkg" || { rc=1; warn "${pkg} FAILED"; } +done +exit "$rc" diff --git a/common.sh b/common.sh new file mode 100755 index 0000000..2f5aa4b --- /dev/null +++ b/common.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +# Shared helpers for the packaging autoupdaters (rpm/.autoupdate, deb/.autoupdate). +# Sourced by autoupdate.sh and latest-version.sh; never executed directly. +# +# latest-version.sh is a verbatim copy of aur/.autoupdate's, so the names it +# relies on (die, validate_version) keep the exact behaviour they have there. + +set -euo pipefail + +log() { printf '==> %s\n' "$*"; } +info() { printf ' -> %s\n' "$*"; } +warn() { printf 'WARNING: %s\n' "$*" >&2; } +die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } + +# Fail up front on a missing tool rather than midway through, where an absent +# binary tends to be read as "condition false" and silently takes the wrong +# branch (aur/.autoupdate learned this from diff going missing). +# +# usage: require_tools [extra-tool ...] +require_tools() { + local t missing=() + for t in git curl jq sha256sum awk sed grep tar gzip "$@"; do + command -v "$t" >/dev/null 2>&1 || missing+=("$t") + done + [ "${#missing[@]}" -eq 0 ] || die "missing required tool(s): ${missing[*]}" + return 0 +} + +require_noninteractive() { + [ -t 0 ] && die "refusing to prompt: this script must never depend on interactive input" + return 0 +} + +# A version we are willing to write into a spec or a changelog. +VERSION_RE='^[0-9][0-9a-zA-Z._+]*$' + +validate_version() { + local ver="${1-}" what="${2-version}" + [ -n "$ver" ] || die "${what} is empty" + [[ "$ver" =~ $VERSION_RE ]] || die "${what} '${ver}' does not look like a version" + return 0 +} + +# True if $1 is strictly newer than $2. Same ordering as latest-version.sh +# (sort -V), so "newest tag" and "newer than packaged" can never disagree. +version_gt() { + [ "$1" != "$2" ] && [ "$(printf '%s\n%s\n' "$1" "$2" | sort -V | tail -n1)" = "$1" ] +} + +# Clone the package repo, or bring an existing clone back to origin's state. +# usage: clone_or_reset +clone_or_reset() { + local url="$1" dir="$2" + if [ -d "$dir/.git" ]; then + git -C "$dir" fetch --quiet origin + git -C "$dir" reset --quiet --hard origin/HEAD + else + rm -rf "$dir" + git clone --quiet "$url" "$dir" || die "could not clone ${url}" + fi +} + +# Identity for the changelog entry and the bump commit: GIT_NAME/GIT_EMAIL +# (the same variables the workflow uses), else the git config the workflow +# wrote. Never guessed. Sets MAINT_NAME and MAINT_EMAIL in the calling shell +# rather than printing them, because a failing $(...) inside a larger string +# does not trip `set -e`. +resolve_identity() { + MAINT_NAME="${GIT_NAME:-}" + MAINT_EMAIL="${GIT_EMAIL:-}" + [ -n "$MAINT_NAME" ] || MAINT_NAME="$(git config user.name || true)" + [ -n "$MAINT_EMAIL" ] || MAINT_EMAIL="$(git config user.email || true)" + [ -n "$MAINT_NAME" ] || die "no maintainer name: set GIT_NAME or git config user.name" + [ -n "$MAINT_EMAIL" ] || die "no maintainer email: set GIT_EMAIL or git config user.email" + return 0 +} + +git_commit() { + resolve_identity + git -c user.name="$MAINT_NAME" -c user.email="$MAINT_EMAIL" commit --quiet -m "$1" +} + +# Download the upstream release tarball (or copy the local test tarball) to +# , and check that every entry sits under / when one is given. +# usage: fetch_tarball [topdir] +fetch_tarball() { + local url="$1" out="$2" topdir="${3:-}" + mkdir -p "$(dirname "$out")" + if [ -n "${AUTOUPDATE_TEST_TARBALL:-}" ]; then + [ -f "$AUTOUPDATE_TEST_TARBALL" ] || die "AUTOUPDATE_TEST_TARBALL '${AUTOUPDATE_TEST_TARBALL}' not found" + info "test mode: using ${AUTOUPDATE_TEST_TARBALL} instead of ${url}" + cp "$AUTOUPDATE_TEST_TARBALL" "$out" + else + info "fetching ${url}" + curl --silent --show-error --location --fail --retry 3 --retry-delay 2 \ + --max-time 900 -o "${out}.part" "$url" \ + || die "download failed: ${url}" + mv "${out}.part" "$out" + fi + gzip -t "$out" 2>/dev/null || die "${out} is not a gzip tarball" + if [ -n "$topdir" ]; then + # awk reads the whole listing, so tar never dies of SIGPIPE under pipefail. + tar -tzf "$out" | awk -F/ -v d="$topdir" '$1 != d { bad = 1 } END { exit bad }' \ + || die "tarball entries are not all under ${topdir}/" + fi + info "sha256 $(sha256sum "$out" | awk '{print $1}')" +} + +# --- Gitea package registry --------------------------------------------------- +# +# Basic auth with the same credential the workflow stores for git +# (https://:@): GIT_TOKEN from the environment +# if set, otherwise `git credential fill`. Gitea ignores the username when the +# password is a token. The credential is handed to curl as a config file on +# stdin, so it never appears on a command line, in `ps`, or in a log. + +registry_curl_config() { + local user="${GIT_EMAIL:-}" pass="${GIT_TOKEN:-}" cred + if [ -z "$pass" ]; then + cred="$(printf 'protocol=https\nhost=%s\n\n' "$GIT_HOST" \ + | GIT_TERMINAL_PROMPT=0 GIT_ASKPASS=true git credential fill 2>/dev/null)" \ + || die "no registry credentials: set GIT_TOKEN, or store a token for https://${GIT_HOST} in a git credential helper" + user="$(printf '%s\n' "$cred" | sed -n 's/^username=//p')" + pass="$(printf '%s\n' "$cred" | sed -n 's/^password=//p')" + fi + [ -n "$pass" ] || die "no registry token for https://${GIT_HOST}" + printf 'user = "%s:%s"\n' "${user:-autoupdate}" "$pass" +} + +# registry_request [curl args...] +# Prints the HTTP status; the response body lands in $REGISTRY_BODY. +registry_request() { + local method="$1" url="$2"; shift 2 + local cfg code + cfg="$(registry_curl_config)" || exit 1 + code="$(printf '%s' "$cfg" | curl --silent --show-error --max-time 600 \ + --config - --request "$method" --output "$REGISTRY_BODY" \ + --write-out '%{http_code}' "$@" "$url")" \ + || die "${method} ${url} failed" + printf '%s' "$code" +} + +# True if the registry already holds at ; false on 404; dies on +# anything else, so a broken query is never mistaken for "not published yet". +# usage: registry_has +registry_has() { + local url="https://${GIT_HOST}/api/v1/packages/$1/$2/$3/$4" code + code="$(registry_request GET "$url" --retry 3 --retry-delay 2)" + case "$code" in + 200) return 0 ;; + 404) return 1 ;; + *) cat "$REGISTRY_BODY" >&2 2>/dev/null || true + die "registry lookup ${url} failed${code:+ (HTTP ${code})}" ;; + esac +} + +# Upload with PUT; Gitea answers 201 Created. +# usage: registry_upload +registry_upload() { + local url="$1" file="$2" code + code="$(registry_request PUT "$url" --upload-file "$file")" + case "$code" in + 201) log "uploaded $(basename "$file")" ;; + 409) die "the registry already has $(basename "$file") (HTTP 409); refusing to overwrite without FORCE_REBUILD" ;; + *) cat "$REGISTRY_BODY" >&2 2>/dev/null || true + die "upload of $(basename "$file") to ${url} answered HTTP ${code}" ;; + esac +} + +# Delete one published file (FORCE_REBUILD replacing an existing build). +# usage: registry_delete +registry_delete() { + local url="$1" code + code="$(registry_request DELETE "$url")" + case "$code" in + 204|200) info "deleted ${url}" ;; + 404) info "nothing to delete at ${url}" ;; + *) cat "$REGISTRY_BODY" >&2 2>/dev/null || true + die "DELETE ${url} answered HTTP ${code}" ;; + esac +} diff --git a/latest-version.sh b/latest-version.sh new file mode 100755 index 0000000..7fadec0 --- /dev/null +++ b/latest-version.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Resolve the latest upstream version for a package and print it on stdout. +# +# Reads the package's .autoupdate manifest (srchost/srcmntr/srcname/verstrip). +# +# usage: latest-version.sh [package-dir] +# +# Exit 0 with a version on stdout, or exit 1 with a message on stderr. +# +# There is deliberately NO "returned nothing so assume no update" path. The old +# workflow could not tell a broken query from a quiet night: pman-helper's +# lookup hit a 404, produced an empty string, and push.sh treated it as a +# version. "Query returned nothing" is an error, not a version. + +set -euo pipefail + +_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${_here}/common.sh" + +pkgdir="${1:-.}" +manifest="${pkgdir}/.autoupdate" +[ -f "$manifest" ] || die "no .autoupdate manifest in ${pkgdir}" + +# shellcheck disable=SC1090 +source "$manifest" + +: "${srchost:?manifest is missing srchost}" +: "${srcmntr:?manifest is missing srcmntr}" +: "${srcname:?manifest is missing srcname}" +verstrip="${verstrip:-}" + +GIT_HOST="${GIT_HOST:-git.bjphoster.com}" + +# curl that fails loudly on HTTP errors instead of returning an error page. +fetch() { + local url="$1"; shift + local out rc=0 + out="$(curl --silent --show-error --location --fail-with-body \ + --retry 3 --retry-delay 2 --max-time 60 "$@" "$url" 2>&1)" || rc=$? + if [ "$rc" -ne 0 ]; then + printf '%s\n' "$out" >&2 + die "request to ${url} failed (curl exit ${rc})" + fi + printf '%s' "$out" +} + +case "$srchost" in + github) + url="https://api.github.com/repos/${srcmntr}/${srcname}/releases/latest" + hdrs=(-H "Accept: application/vnd.github+json") + # Optional: avoids the 60/hour unauthenticated rate limit in CI. + [ -n "${GITHUB_TOKEN:-}" ] && hdrs+=(-H "Authorization: Bearer ${GITHUB_TOKEN}") + body="$(fetch "$url" "${hdrs[@]}")" + tag="$(printf '%s' "$body" | jq -r '.tag_name // empty')" + [ -n "$tag" ] || die "no tag_name in the latest release of ${srcmntr}/${srcname}" + ;; + + gitea) + # NB: the correct Gitea path is /repos/{owner}/{repo}/tags. The old + # workflows used /repos/{owner}/{repo}/repo/tags, which 404s. + url="https://${GIT_HOST}/api/v1/repos/${srcmntr}/${srcname}/tags" + body="$(fetch "$url" -H "Accept: application/json")" + printf '%s' "$body" | jq -e 'type == "array"' >/dev/null 2>&1 \ + || die "unexpected response shape from ${url}" + tag="$(printf '%s' "$body" | jq -r '.[].name' | sort -V | tail -n1)" + [ -n "$tag" ] || die "${srcmntr}/${srcname} has no tags" + ;; + + *) + die "unknown srchost '${srchost}' (expected 'github' or 'gitea')" + ;; +esac + +version="$tag" +if [ -n "$verstrip" ]; then + version="${version#"$verstrip"}" +fi + +validate_version "$version" "resolved version (from tag '${tag}')" +printf '%s\n' "$version"