From f539442cfc1d5bc738b565a9a2fa670827190d65 Mon Sep 17 00:00:00 2001 From: Bryan Joshua Pedini Date: Sat, 10 Oct 2026 11:09:04 +0200 Subject: [PATCH] feat: add the nightly rpm reconciler --- .gitignore | 4 + README.md | 208 ++++++++++++++++++++++++++++++- autoupdate.sh | 302 ++++++++++++++++++++++++++++++++++++++++++++++ common.sh | 181 +++++++++++++++++++++++++++ latest-version.sh | 81 +++++++++++++ 5 files changed, 775 insertions(+), 1 deletion(-) create mode 100644 .gitignore create mode 100755 autoupdate.sh create mode 100755 common.sh create mode 100755 latest-version.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..06fa06f --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +# downloaded tarballs +*.tar.gz +# built packages +*.rpm diff --git a/README.md b/README.md index 0dff0a6..80f7628 100644 --- a/README.md +++ b/README.md @@ -1 +1,207 @@ -# RPM Auto Update Workflows +# RPM Auto Update + +Nightly reconciler that keeps the `rpm/*` package repos and the `rpm` org's +Gitea package registry in sync with their upstream releases. + +It mirrors [`aur/.autoupdate`](https://git.bjphoster.com/aur/.autoupdate): a new +upstream tag, a spec edited by hand and pushed to origin, and an upload that +failed last night all take the same path. On every pass the version-release in +the spec is compared with what the registry holds, and the registry is brought +in step. + +## Layout + +| file | role | +|---|---| +| `autoupdate.sh` | entry point: bump, build, upload, push, one package per process | +| `latest-version.sh` | resolves the latest upstream version (verbatim from `aur/.autoupdate`) | +| `common.sh` | logging, guards, version compare, tarball fetch, registry API | +| `dockerfile` | the Fedora toolchain image the nightly job runs in | +| `.gitea/workflows/update.yaml` | nightly matrix job | +| `.gitea/workflows/build-image.yaml` | monthly image rebuild | + +Nothing here is package-specific. Everything that differs between packages +lives in the package repo itself. + +## Package repo requirements + +`rpm/` carries: + +- `.spec` with literal `Version:` and `Release: N%{?dist}` lines and a + `%changelog` section. `Source0` should be the Gitea archive of the tag, + `https://git.bjphoster.com/source//archive/%{version}.tar.gz`, + unpacked with `%autosetup -n ` (the archive's top-level directory is + `/`). +- `.autoupdate`, the same manifest the AUR packages use: + +```sh +srchost="gitea" # gitea | github +srcmntr="source" # upstream owner/org +srcname="skeptical-updater" # upstream repo +verstrip="" # tag prefix to strip (tags are bare semver) +gui="false" # ignored here; AUR AppImage extraction only +``` + +Start a new package at a placeholder such as `0.0.1`: the first nightly run +bumps it to the newest tag. + +## What a run does + +1. Clone `https://git.bjphoster.com/rpm/` and read `.autoupdate`. +2. Resolve the newest upstream tag (`latest-version.sh`: Gitea tags API, + `sort -V`). A *failed lookup* exits 1; "no new version" is not a failure. +3. If the tag is newer than `Version:`, rewrite the spec: `Version:` set to the + tag, `Release:` back to `1%{?dist}`, and a `%changelog` entry prepended: + + ``` + * Sat Oct 10 2026 <> - X-1 + - Updated to X + ``` + + The rest of the file stays byte-identical, and `rpmspec` must read back + exactly `X-1` before anything is committed. The commit is + `Updated to X-1`. +4. Ask the registry whether it already holds that version-release. +5. If it does not, download `https://git.bjphoster.com/source//archive/.tar.gz`, + build with `rpmbuild -bb --define 'dist %{nil}'` as the unprivileged + `builder` user, run `rpmlint` (advisory, never fatal) and upload every built + `.rpm` to the registry. +6. Push to origin, last. The registry is the distribution channel and goes + first, as the AUR does in `aur/.autoupdate`: a failed build or upload leaves + origin untouched, and the next night recomputes everything. + +Building with `dist` undefined makes the noarch package `-X-1.noarch.rpm` +rather than `.fc43.noarch.rpm`, so the one file serves every RPM distribution +that points at the registry. + +## Versions + +`X-N`: everything before the dash is upstream, `-N` is the packaging revision +(`Release:`). + +- newer upstream tag: `X-1` +- `force_rebuild` with no new tag: `X-(N+1)`, changelog `- Rebuilt X` +- an older tag (a tag deleted upstream) never downgrades the package +- a hand edit that bumps `Release:` and is pushed to origin is published on the + next run, because the registry lacks that version-release + +## Environment and inputs + +| workflow input | variable | effect | +|---|---|---| +| `package` | | run one matrix entry only (blank = all) | +| `force_rebuild` | `FORCE_REBUILD=1` | bump `Release` even with no new upstream version; replaces a registry copy of that exact version-release if one exists | +| `force_refresh` | `FORCE_REFRESH=1` | re-download the source and rebuild even when nothing changed (a check; uploads only what is missing) | +| `no_git_push` | `NO_GIT_PUSH=1` | dry run: build, but neither upload nor push to origin | +| | `NO_UPLOAD=1` / `--no-upload` | never contact the registry: no lookup, no upload; builds only when this run bumped the version | +| | `GIT_HOST` | git host and registry (default `git.bjphoster.com`) | +| | `GIT_NAME`, `GIT_EMAIL` | `%changelog` and commit identity (default: `git config`) | +| | `GIT_TOKEN` | registry token (default: the git credential helper, as in CI) | +| | `WORKDIR` | where to clone and build (default: a temp dir) | +| | `AUTOUPDATE_TEST_VERSION` | local test mode: use this as the upstream version | +| | `AUTOUPDATE_TEST_TARBALL` | local test mode: build from this local tarball | +| | `PKG_REPO_BASE` | clone package repos from here (default `https://$GIT_HOST/rpm`) | + +Either `AUTOUPDATE_TEST_*` variable forces `NO_GIT_PUSH=1` and `NO_UPLOAD=1`, +so a test run can never publish anything. + +## Setup + +Required on the `rpm` org (Settings -> Actions -> Secrets / Variables), or on +the `.autoupdate` repo itself: + +| kind | name | value | +|---|---|---| +| secret | `GIT_TOKEN` | access token with **write:repository** (clone and push `rpm/*`) and **write:package** (registry upload, image push, image pull) | +| variable | `GIT_NAME` | display name for commits and `%changelog`, e.g. `Bryan Joshua Pedini` | +| variable | `GIT_EMAIL` | email for commits and `%changelog` | +| variable | `REGISTRY_USER` | your Gitea **username** for the container registry login (not `GIT_NAME`, which has spaces) | +| variable | `GIT_HOST` | optional; defaults to `git.bjphoster.com` | + +Also: + +- enable **Actions** and **Packages** on `rpm/.autoupdate` (repo Settings -> + Advanced Settings), and make sure a runner with the `ubuntu-latest` label + serves the `rpm` org (an instance-wide runner does); +- the `rpm` org's registry (`/api/packages/rpm/...`) needs no switch of its + own: it answers whenever packages are enabled on the instance. Enabling + Packages on the package repos only adds the Packages tab and lets you link + the registry entry to the repo; +- build the image once by hand (Actions -> Build Autoupdater Image -> Run + workflow) before the first nightly run, since `update.yaml` pulls it; +- list every package in the `matrix.package` of `.gitea/workflows/update.yaml`. + A package repo that is not listed there is never processed. + +The token is read back by `autoupdate.sh` through `git credential fill`, from the +`~/.git-credentials` the workflow writes, and handed to `curl` on stdin; it is +never on a command line or in a log. + +## Running by hand + +Actions -> Update RPM Packages -> Run workflow. Inputs: `package` (blank for +the whole matrix), `force_rebuild`, `force_refresh`, `no_git_push`. + +## Installing packages + +Fedora, RHEL and derivatives: + +```sh +# dnf4 (EL 8/9/10; needs dnf-plugins-core) +sudo dnf config-manager --add-repo https://git.bjphoster.com/api/packages/rpm/rpm.repo +# dnf5 (Fedora 41+) +sudo dnf config-manager addrepo --from-repofile=https://git.bjphoster.com/api/packages/rpm/rpm.repo + +sudo dnf install skeptical-updater +``` + +openSUSE: + +```sh +sudo zypper addrepo https://git.bjphoster.com/api/packages/rpm/rpm.repo +sudo zypper refresh # accept the repository key when asked +sudo zypper install skeptical-updater +``` + +The `.repo` file enables `gpgcheck` against +`https://git.bjphoster.com/api/packages/rpm/rpm/repository.key`; Gitea signs +each package with that key on upload, so nothing is signed here. + +## Container image + +`dockerfile` builds the environment the nightly job runs in, published to the +Gitea registry as `git.bjphoster.com/rpm/autoupdate:latest`: +`fedora:latest` with `rpm-build`, `rpmdevtools`, `rpmlint`, `git`, `curl`, +`jq`, `util-linux` (`runuser`), `nodejs` (actions/checkout is a JavaScript +action) and a `builder` user. + +`.gitea/workflows/build-image.yaml` rebuilds and overwrites `:latest` monthly, +on any push touching the `dockerfile`, and on demand. It needs the same +`REGISTRY_USER` and `GIT_TOKEN`. + +## Testing locally + +A dry run against a throwaway copy of the real package repo, with a forced +upstream version and a local tarball built exactly like Gitea's archive, and no +network at all. The test variables imply `NO_GIT_PUSH=1` and `NO_UPLOAD=1`. + +```sh +docker build -t git.bjphoster.com/rpm/autoupdate:latest . + +t="$(mktemp -d)" +git clone --quiet --bare https://git.bjphoster.com/rpm/skeptical-updater "$t/origin/skeptical-updater" +git -C ~/projects/sources/skeptical-updater archive --format=tar \ + --prefix=skeptical-updater/ HEAD | gzip -n > "$t/src.tar.gz" + +docker run --rm --network none -v "$PWD:/src:ro" -v "$t:/t:ro" \ + git.bjphoster.com/rpm/autoupdate:latest \ + bash -c 'cp -r /src /w && cp -r /t /tt && chown -R builder:builder /w /tt && cd /w && + runuser -u builder -- env GIT_NAME="Your Name" GIT_EMAIL=you@example.com \ + PKG_REPO_BASE=file:///tt/origin WORKDIR=/tt/work \ + AUTOUPDATE_TEST_VERSION=0.1.0 AUTOUPDATE_TEST_TARBALL=/tt/src.tar.gz \ + ./autoupdate.sh skeptical-updater && + runuser -u builder -- git -C /tt/work/skeptical-updater show --stat HEAD' +``` + +The built package lands in `/tt/work/.rpmbuild/skeptical-updater/RPMS/noarch/` +inside the container. Lint the scripts with +`docker run --rm -v "$PWD:/mnt:ro" -w /mnt koalaman/shellcheck:stable -x *.sh`. diff --git a/autoupdate.sh b/autoupdate.sh new file mode 100755 index 0000000..51a906f --- /dev/null +++ b/autoupdate.sh @@ -0,0 +1,302 @@ +#!/usr/bin/env bash +# Nightly RPM package reconciler. +# +# usage: autoupdate.sh [--no-upload] [...] +# +# env: +# GIT_HOST git host holding the package repos and the package +# registry (default git.bjphoster.com) +# WORKDIR where to clone and build; a temp dir by default +# NO_GIT_PUSH=1 dry run: build, but neither upload nor push to origin +# NO_UPLOAD=1 never contact the package registry (same as --no-upload) +# FORCE_REBUILD=1 bump Release even with no new upstream version +# FORCE_REFRESH=1 re-download the source and rebuild even when nothing changed +# GIT_NAME, GIT_EMAIL %changelog and commit identity (default: git config) +# GIT_TOKEN registry token (default: the git credential helper) +# +# local test mode -- setting either of these implies NO_GIT_PUSH=1 and +# NO_UPLOAD=1, so a test can never publish anything: +# AUTOUPDATE_TEST_VERSION use this as the upstream version (no lookup) +# AUTOUPDATE_TEST_TARBALL build from this local tarball (no download) +# plus PKG_REPO_BASE=file:///some/dir to clone a throwaway copy of the repo. +# +# Like aur/.autoupdate this is a RECONCILER: the registry is the distribution +# channel, and it is compared against the package repo on every pass. A new +# upstream release, a hand-edited spec pushed to origin, and an upload that +# failed last night all end in the same place: whatever version-release the +# spec says, the registry gets it. Distribution channel first, origin second, +# so a failed build or upload leaves origin untouched and tonight's work is +# simply recomputed tomorrow. + +set -euo pipefail + +_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${_here}/common.sh" + +require_noninteractive +require_tools rpmbuild rpmspec spectool rpm + +while [ "$#" -gt 0 ]; do + case "$1" in + --no-upload) export NO_UPLOAD=1; shift ;; + --) shift; break ;; + -*) die "unknown option '$1' (usage: autoupdate.sh [--no-upload] [...])" ;; + *) break ;; + esac +done +[ "$#" -ge 1 ] || die "usage: autoupdate.sh [--no-upload] [...]" + +if [ -n "${AUTOUPDATE_TEST_VERSION:-}${AUTOUPDATE_TEST_TARBALL:-}" ]; then + export NO_GIT_PUSH=1 NO_UPLOAD=1 +fi + +GIT_HOST="${GIT_HOST:-git.bjphoster.com}" +# The org: where the package repos live and whose registry receives the RPMs. +REGISTRY_OWNER="${REGISTRY_OWNER:-rpm}" +# Overridable so a test can work on a throwaway clone; defaults are production. +PKG_REPO_BASE="${PKG_REPO_BASE:-https://${GIT_HOST}/${REGISTRY_OWNER}}" +LATEST_VERSION_CMD="${LATEST_VERSION_CMD:-${_here}/latest-version.sh}" + +WORKDIR="${WORKDIR:-$(mktemp -d)}" +mkdir -p "$WORKDIR" +REGISTRY_BODY="${WORKDIR}/.registry-response" + +# Spec tags are read and written as literal `Tag: value` lines, never +# through rpmspec, so the rewrite can keep the rest of the file byte-identical. +spec_get() { # + sed -n -E "s/^$1:[[:space:]]*([^[:space:]]+)[[:space:]]*$/\1/p" "$2" | head -n1 +} + +# Set Version and Release (N%{?dist}) and prepend a %changelog entry: +# +# * Sat Oct 10 2026 Name - X-N +# - +# +# Nothing else in the file changes. The column alignment after `Version:` and +# `Release:` is preserved; a blank line separates the new entry from the old. +spec_bump() { # + local spec="$1" ver="$2" rel="$3" note="$4" day hdr rc=0 + resolve_identity + day="$(LC_ALL=C date -u '+%a %b %d %Y')" + hdr="* ${day} ${MAINT_NAME} <${MAINT_EMAIL}> - ${ver}-${rel}" + awk -v ver="$ver" -v rel="${rel}%{?dist}" -v hdr="$hdr" -v note="- ${note}" ' + !v && /^Version:[[:space:]]/ { match($0, /^Version:[[:space:]]+/); print substr($0, 1, RLENGTH) ver; v = 1; next } + !r && /^Release:[[:space:]]/ { match($0, /^Release:[[:space:]]+/); print substr($0, 1, RLENGTH) rel; r = 1; next } + !c && /^%changelog[[:space:]]*$/ { print; print hdr; print note; c = 1; sep = 1; next } + sep { if ($0 != "") print ""; sep = 0 } + { print } + END { if (!v || !r || !c) exit 3 } + ' "$spec" > "${spec}.new" || rc=$? + if [ "$rc" -ne 0 ]; then + rm -f "${spec}.new" + die "${spec} needs literal Version:, Release: and %changelog lines to be bumped" + fi + cat "${spec}.new" > "$spec" # keep the inode and mode of the tracked file + rm -f "${spec}.new" + + # Refuse to go on unless rpm itself reads back exactly what was written. + local got + got="$(rpmspec -q --srpm --define 'dist %{nil}' --qf '%{version}-%{release}\n' "$spec")" \ + || die "rpmspec cannot parse ${spec} after the bump" + [ "$got" = "${ver}-${rel}" ] || die "rpmspec reads ${got} from ${spec}, expected ${ver}-${rel}" +} + +# Binary package names the spec builds (what the registry is keyed on). +spec_names() { # + rpmspec -q --builtrpms --define 'dist %{nil}' --qf '%{name}\n' "$1" | sort -u +} + +# rpmbuild -bb from the upstream tarball, with dist undefined so the result is +# a distro-agnostic `X-N.noarch.rpm` that installs on Fedora, EL and openSUSE. +# Fills the global BUILT array with the produced packages. +build_rpm() { # + local spec="$1" tarball="$2" top src0 + top="${WORKDIR}/.rpmbuild/${pkg}" + rm -rf "$top" + mkdir -p "$top"/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS} + + # Anything else the repo carries (patches, extra SourceN files). + git ls-files -z | { grep -zv -x -F "$spec" || true; } | xargs -0 -r cp --parents -t "${top}/SOURCES" + + # rpmbuild looks Source0 up by its basename (or its #/rename fragment). + src0="$(spectool -l -s 0 "$spec" | sed -n -E 's/^Source0?:[[:space:]]*//p' | head -n1)" + [ -n "$src0" ] || die "${spec} declares no Source0" + [ "$src0" = "$tarball_url" ] || [[ "$src0" == "${tarball_url}#/"* ]] \ + || warn "Source0 is ${src0}, building from ${tarball_url}" + cp "$tarball" "${top}/SOURCES/${src0##*/}" + + log "rpmbuild -bb ${spec}" + rpmbuild -bb --define "_topdir ${top}" --define 'dist %{nil}' "$spec" + + mapfile -t BUILT < <(find "${top}/RPMS" -type f -name '*.rpm' | sort) + [ "${#BUILT[@]}" -gt 0 ] || die "rpmbuild produced no packages" + local f + for f in "${BUILT[@]}"; do info "built $(basename "$f")"; done + + # Advisory only, like namcap is for the AUR: never blocks a release. + if command -v rpmlint >/dev/null 2>&1; then + log "rpmlint" + rpmlint "$spec" "${BUILT[@]}" || warn "rpmlint reported problems (non-fatal)" + else + warn "rpmlint is not installed; skipping the lint" + fi +} + +publish_rpm() { # + local replace="$1" f name verrel arch + for f in "${BUILT[@]}"; do + if [ "$replace" -eq 1 ]; then + name="$(rpm -qp --qf '%{name}' "$f")" + verrel="$(rpm -qp --qf '%{version}-%{release}' "$f")" + arch="$(rpm -qp --qf '%{arch}' "$f")" + registry_delete "https://${GIT_HOST}/api/packages/${REGISTRY_OWNER}/rpm/package/${name}/${verrel}/${arch}" + fi + registry_upload "https://${GIT_HOST}/api/packages/${REGISTRY_OWNER}/rpm/upload" "$f" + done +} + +update_one() { + pkg="$1" + local dir="${WORKDIR}/${pkg}" + local spec="${pkg}.spec" + + log "=== ${pkg} ===" + + clone_or_reset "${PKG_REPO_BASE}/${pkg}" "$dir" + cd "$dir" + [ -f "$spec" ] || die "${pkg} has no ${spec}" + [ -f .autoupdate ] || die "${pkg} has no .autoupdate manifest" + + # shellcheck disable=SC1091 + source ./.autoupdate + : "${srchost:?manifest is missing srchost}" + : "${srcmntr:?manifest is missing srcmntr}" + : "${srcname:?manifest is missing srcname}" + + # --- packaged version ---------------------------------------------------- + local oldver oldrel + oldver="$(spec_get Version "$spec")" + oldrel="$(spec_get Release "$spec")" + validate_version "$oldver" "Version in ${spec}" + oldrel="${oldrel%"%{?dist}"}" + [[ "$oldrel" =~ ^[0-9]+$ ]] \ + || die "Release in ${spec} is not of the form N%{?dist}; refusing to rewrite it" + + # --- upstream version ---------------------------------------------------- + # A failed lookup exits non-zero and is NOT mistaken for "no new version". + local newver + if [ -n "${AUTOUPDATE_TEST_VERSION:-}" ]; then + newver="$AUTOUPDATE_TEST_VERSION" + info "test mode: upstream version forced to ${newver}" + else + newver="$("$LATEST_VERSION_CMD" .)" || die "upstream version lookup failed for ${pkg}" + fi + validate_version "$newver" "upstream version for ${pkg}" + info "upstream ${newver}, packaged ${oldver}-${oldrel}" + + # --- bump ------------------------------------------------------------------ + local ver="$oldver" rel="$oldrel" bumped=0 + if version_gt "$newver" "$oldver"; then + ver="$newver"; rel=1 + log "new upstream version ${oldver} -> ${ver}" + spec_bump "$spec" "$ver" "$rel" "Updated to ${ver}" + bumped=1 + else + if [ "$newver" != "$oldver" ]; then + warn "upstream ${newver} is older than the packaged ${oldver}; not downgrading" + fi + if [ -n "${FORCE_REBUILD:-}" ]; then + rel=$(( oldrel + 1 )) + log "FORCE_REBUILD: Release ${oldrel} -> ${rel}" + spec_bump "$spec" "$ver" "$rel" "Rebuilt ${ver}" + bumped=1 + fi + fi + + git add -- "$spec" + if git diff --cached --quiet; then + log "no local changes for ${pkg}" + else + git_commit "Updated ${pkg} to ${ver}-${rel}" + log "committed ${pkg} ${ver}-${rel}" + fi + + # --- distribution channel ------------------------------------------------- + # Reference state first: which version-release does the registry hold? + local state="skipped" names=() name + if [ -z "${NO_UPLOAD:-}" ]; then + local list + list="$(spec_names "$spec")" || die "rpmspec cannot list the packages ${spec} builds" + [ -n "$list" ] || die "${spec} builds no binary packages" + mapfile -t names <<<"$list" + state="present" + for name in "${names[@]}"; do + if registry_has "$REGISTRY_OWNER" rpm "$name" "${ver}-${rel}"; then + info "registry has ${name} ${ver}-${rel}" + else + info "registry lacks ${name} ${ver}-${rel}" + state="absent" + fi + done + fi + + local build=0 upload=0 replace=0 + case "$state" in + absent) build=1; upload=1 ;; + present) + if [ -n "${FORCE_REBUILD:-}" ]; then build=1; upload=1; replace=1 + elif [ -n "${FORCE_REFRESH:-}" ]; then build=1 + else log "registry already has ${pkg} ${ver}-${rel}; nothing to build" + fi ;; + skipped) + if [ "$bumped" -eq 1 ] || [ -n "${FORCE_REFRESH:-}" ]; then build=1 + else log "NO_UPLOAD set and nothing changed; nothing to build" + fi ;; + esac + + if [ "$build" -eq 1 ]; then + local tag="${verstrip:-}${ver}" topdir="" tarball="${WORKDIR}/.sources/${pkg}-${ver}.tar.gz" + case "$srchost" in + gitea) tarball_url="https://${GIT_HOST}/${srcmntr}/${srcname}/archive/${tag}.tar.gz" + topdir="$srcname" ;; + github) tarball_url="https://github.com/${srcmntr}/${srcname}/archive/refs/tags/${tag}.tar.gz" ;; + *) die "unknown srchost '${srchost}' (expected 'github' or 'gitea')" ;; + esac + fetch_tarball "$tarball_url" "$tarball" "$topdir" + build_rpm "$spec" "$tarball" + fi + + if [ "$upload" -eq 1 ]; then + if [ -n "${NO_GIT_PUSH:-}" ]; then + log "NO_GIT_PUSH set: not uploading ${pkg} ${ver}-${rel} to ${REGISTRY_OWNER}'s registry" + else + publish_rpm "$replace" + fi + fi + + # --- origin --------------------------------------------------------------- + # Only after the registry is in step, so a failed build or upload never + # leaves origin claiming a release that was not published. + if [ -n "${NO_GIT_PUSH:-}" ]; then + log "NO_GIT_PUSH set: not pushing ${pkg} to origin" + else + git push --quiet origin HEAD + fi +} + +# One package per process, deliberately -- see aur/.autoupdate: a subshell in +# an && / || list disables `set -e` for its whole body, so `( update_one ) ||` +# would sail past a failed build straight into `git push origin`. Re-exec'ing +# gives each package a fresh shell whose `set -e` is intact. The CI matrix +# passes exactly one package, so the loop only affects local runs. +if [ -n "${AUTOUPDATE_SINGLE:-}" ]; then + update_one "$1" + exit 0 +fi + +rc=0 +for pkg in "$@"; do + AUTOUPDATE_SINGLE=1 "$0" "$pkg" || { rc=1; warn "${pkg} FAILED"; } +done +exit "$rc" diff --git a/common.sh b/common.sh new file mode 100755 index 0000000..2f5aa4b --- /dev/null +++ b/common.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +# Shared helpers for the packaging autoupdaters (rpm/.autoupdate, deb/.autoupdate). +# Sourced by autoupdate.sh and latest-version.sh; never executed directly. +# +# latest-version.sh is a verbatim copy of aur/.autoupdate's, so the names it +# relies on (die, validate_version) keep the exact behaviour they have there. + +set -euo pipefail + +log() { printf '==> %s\n' "$*"; } +info() { printf ' -> %s\n' "$*"; } +warn() { printf 'WARNING: %s\n' "$*" >&2; } +die() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } + +# Fail up front on a missing tool rather than midway through, where an absent +# binary tends to be read as "condition false" and silently takes the wrong +# branch (aur/.autoupdate learned this from diff going missing). +# +# usage: require_tools [extra-tool ...] +require_tools() { + local t missing=() + for t in git curl jq sha256sum awk sed grep tar gzip "$@"; do + command -v "$t" >/dev/null 2>&1 || missing+=("$t") + done + [ "${#missing[@]}" -eq 0 ] || die "missing required tool(s): ${missing[*]}" + return 0 +} + +require_noninteractive() { + [ -t 0 ] && die "refusing to prompt: this script must never depend on interactive input" + return 0 +} + +# A version we are willing to write into a spec or a changelog. +VERSION_RE='^[0-9][0-9a-zA-Z._+]*$' + +validate_version() { + local ver="${1-}" what="${2-version}" + [ -n "$ver" ] || die "${what} is empty" + [[ "$ver" =~ $VERSION_RE ]] || die "${what} '${ver}' does not look like a version" + return 0 +} + +# True if $1 is strictly newer than $2. Same ordering as latest-version.sh +# (sort -V), so "newest tag" and "newer than packaged" can never disagree. +version_gt() { + [ "$1" != "$2" ] && [ "$(printf '%s\n%s\n' "$1" "$2" | sort -V | tail -n1)" = "$1" ] +} + +# Clone the package repo, or bring an existing clone back to origin's state. +# usage: clone_or_reset +clone_or_reset() { + local url="$1" dir="$2" + if [ -d "$dir/.git" ]; then + git -C "$dir" fetch --quiet origin + git -C "$dir" reset --quiet --hard origin/HEAD + else + rm -rf "$dir" + git clone --quiet "$url" "$dir" || die "could not clone ${url}" + fi +} + +# Identity for the changelog entry and the bump commit: GIT_NAME/GIT_EMAIL +# (the same variables the workflow uses), else the git config the workflow +# wrote. Never guessed. Sets MAINT_NAME and MAINT_EMAIL in the calling shell +# rather than printing them, because a failing $(...) inside a larger string +# does not trip `set -e`. +resolve_identity() { + MAINT_NAME="${GIT_NAME:-}" + MAINT_EMAIL="${GIT_EMAIL:-}" + [ -n "$MAINT_NAME" ] || MAINT_NAME="$(git config user.name || true)" + [ -n "$MAINT_EMAIL" ] || MAINT_EMAIL="$(git config user.email || true)" + [ -n "$MAINT_NAME" ] || die "no maintainer name: set GIT_NAME or git config user.name" + [ -n "$MAINT_EMAIL" ] || die "no maintainer email: set GIT_EMAIL or git config user.email" + return 0 +} + +git_commit() { + resolve_identity + git -c user.name="$MAINT_NAME" -c user.email="$MAINT_EMAIL" commit --quiet -m "$1" +} + +# Download the upstream release tarball (or copy the local test tarball) to +# , and check that every entry sits under / when one is given. +# usage: fetch_tarball [topdir] +fetch_tarball() { + local url="$1" out="$2" topdir="${3:-}" + mkdir -p "$(dirname "$out")" + if [ -n "${AUTOUPDATE_TEST_TARBALL:-}" ]; then + [ -f "$AUTOUPDATE_TEST_TARBALL" ] || die "AUTOUPDATE_TEST_TARBALL '${AUTOUPDATE_TEST_TARBALL}' not found" + info "test mode: using ${AUTOUPDATE_TEST_TARBALL} instead of ${url}" + cp "$AUTOUPDATE_TEST_TARBALL" "$out" + else + info "fetching ${url}" + curl --silent --show-error --location --fail --retry 3 --retry-delay 2 \ + --max-time 900 -o "${out}.part" "$url" \ + || die "download failed: ${url}" + mv "${out}.part" "$out" + fi + gzip -t "$out" 2>/dev/null || die "${out} is not a gzip tarball" + if [ -n "$topdir" ]; then + # awk reads the whole listing, so tar never dies of SIGPIPE under pipefail. + tar -tzf "$out" | awk -F/ -v d="$topdir" '$1 != d { bad = 1 } END { exit bad }' \ + || die "tarball entries are not all under ${topdir}/" + fi + info "sha256 $(sha256sum "$out" | awk '{print $1}')" +} + +# --- Gitea package registry --------------------------------------------------- +# +# Basic auth with the same credential the workflow stores for git +# (https://:@): GIT_TOKEN from the environment +# if set, otherwise `git credential fill`. Gitea ignores the username when the +# password is a token. The credential is handed to curl as a config file on +# stdin, so it never appears on a command line, in `ps`, or in a log. + +registry_curl_config() { + local user="${GIT_EMAIL:-}" pass="${GIT_TOKEN:-}" cred + if [ -z "$pass" ]; then + cred="$(printf 'protocol=https\nhost=%s\n\n' "$GIT_HOST" \ + | GIT_TERMINAL_PROMPT=0 GIT_ASKPASS=true git credential fill 2>/dev/null)" \ + || die "no registry credentials: set GIT_TOKEN, or store a token for https://${GIT_HOST} in a git credential helper" + user="$(printf '%s\n' "$cred" | sed -n 's/^username=//p')" + pass="$(printf '%s\n' "$cred" | sed -n 's/^password=//p')" + fi + [ -n "$pass" ] || die "no registry token for https://${GIT_HOST}" + printf 'user = "%s:%s"\n' "${user:-autoupdate}" "$pass" +} + +# registry_request [curl args...] +# Prints the HTTP status; the response body lands in $REGISTRY_BODY. +registry_request() { + local method="$1" url="$2"; shift 2 + local cfg code + cfg="$(registry_curl_config)" || exit 1 + code="$(printf '%s' "$cfg" | curl --silent --show-error --max-time 600 \ + --config - --request "$method" --output "$REGISTRY_BODY" \ + --write-out '%{http_code}' "$@" "$url")" \ + || die "${method} ${url} failed" + printf '%s' "$code" +} + +# True if the registry already holds at ; false on 404; dies on +# anything else, so a broken query is never mistaken for "not published yet". +# usage: registry_has +registry_has() { + local url="https://${GIT_HOST}/api/v1/packages/$1/$2/$3/$4" code + code="$(registry_request GET "$url" --retry 3 --retry-delay 2)" + case "$code" in + 200) return 0 ;; + 404) return 1 ;; + *) cat "$REGISTRY_BODY" >&2 2>/dev/null || true + die "registry lookup ${url} failed${code:+ (HTTP ${code})}" ;; + esac +} + +# Upload with PUT; Gitea answers 201 Created. +# usage: registry_upload +registry_upload() { + local url="$1" file="$2" code + code="$(registry_request PUT "$url" --upload-file "$file")" + case "$code" in + 201) log "uploaded $(basename "$file")" ;; + 409) die "the registry already has $(basename "$file") (HTTP 409); refusing to overwrite without FORCE_REBUILD" ;; + *) cat "$REGISTRY_BODY" >&2 2>/dev/null || true + die "upload of $(basename "$file") to ${url} answered HTTP ${code}" ;; + esac +} + +# Delete one published file (FORCE_REBUILD replacing an existing build). +# usage: registry_delete +registry_delete() { + local url="$1" code + code="$(registry_request DELETE "$url")" + case "$code" in + 204|200) info "deleted ${url}" ;; + 404) info "nothing to delete at ${url}" ;; + *) cat "$REGISTRY_BODY" >&2 2>/dev/null || true + die "DELETE ${url} answered HTTP ${code}" ;; + esac +} diff --git a/latest-version.sh b/latest-version.sh new file mode 100755 index 0000000..7fadec0 --- /dev/null +++ b/latest-version.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Resolve the latest upstream version for a package and print it on stdout. +# +# Reads the package's .autoupdate manifest (srchost/srcmntr/srcname/verstrip). +# +# usage: latest-version.sh [package-dir] +# +# Exit 0 with a version on stdout, or exit 1 with a message on stderr. +# +# There is deliberately NO "returned nothing so assume no update" path. The old +# workflow could not tell a broken query from a quiet night: pman-helper's +# lookup hit a 404, produced an empty string, and push.sh treated it as a +# version. "Query returned nothing" is an error, not a version. + +set -euo pipefail + +_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=common.sh +source "${_here}/common.sh" + +pkgdir="${1:-.}" +manifest="${pkgdir}/.autoupdate" +[ -f "$manifest" ] || die "no .autoupdate manifest in ${pkgdir}" + +# shellcheck disable=SC1090 +source "$manifest" + +: "${srchost:?manifest is missing srchost}" +: "${srcmntr:?manifest is missing srcmntr}" +: "${srcname:?manifest is missing srcname}" +verstrip="${verstrip:-}" + +GIT_HOST="${GIT_HOST:-git.bjphoster.com}" + +# curl that fails loudly on HTTP errors instead of returning an error page. +fetch() { + local url="$1"; shift + local out rc=0 + out="$(curl --silent --show-error --location --fail-with-body \ + --retry 3 --retry-delay 2 --max-time 60 "$@" "$url" 2>&1)" || rc=$? + if [ "$rc" -ne 0 ]; then + printf '%s\n' "$out" >&2 + die "request to ${url} failed (curl exit ${rc})" + fi + printf '%s' "$out" +} + +case "$srchost" in + github) + url="https://api.github.com/repos/${srcmntr}/${srcname}/releases/latest" + hdrs=(-H "Accept: application/vnd.github+json") + # Optional: avoids the 60/hour unauthenticated rate limit in CI. + [ -n "${GITHUB_TOKEN:-}" ] && hdrs+=(-H "Authorization: Bearer ${GITHUB_TOKEN}") + body="$(fetch "$url" "${hdrs[@]}")" + tag="$(printf '%s' "$body" | jq -r '.tag_name // empty')" + [ -n "$tag" ] || die "no tag_name in the latest release of ${srcmntr}/${srcname}" + ;; + + gitea) + # NB: the correct Gitea path is /repos/{owner}/{repo}/tags. The old + # workflows used /repos/{owner}/{repo}/repo/tags, which 404s. + url="https://${GIT_HOST}/api/v1/repos/${srcmntr}/${srcname}/tags" + body="$(fetch "$url" -H "Accept: application/json")" + printf '%s' "$body" | jq -e 'type == "array"' >/dev/null 2>&1 \ + || die "unexpected response shape from ${url}" + tag="$(printf '%s' "$body" | jq -r '.[].name' | sort -V | tail -n1)" + [ -n "$tag" ] || die "${srcmntr}/${srcname} has no tags" + ;; + + *) + die "unknown srchost '${srchost}' (expected 'github' or 'gitea')" + ;; +esac + +version="$tag" +if [ -n "$verstrip" ]; then + version="${version#"$verstrip"}" +fi + +validate_version "$version" "resolved version (from tag '${tag}')" +printf '%s\n' "$version"