feat: add the nightly rpm reconciler
This commit is contained in:
bryanpedini committed
1 parent
32c83444d2
commit
f539442cfc
5 files changed
+775
-1
No files matched your search
Executable
+302
@@ -0,0 +1,302 @@
|
||||
#!/usr/bin/env bash
|
||||
# Nightly RPM package reconciler.
|
||||
#
|
||||
# usage: autoupdate.sh [--no-upload] <package> [...]
|
||||
#
|
||||
# env:
|
||||
# GIT_HOST git host holding the package repos and the package
|
||||
# registry (default git.bjphoster.com)
|
||||
# WORKDIR where to clone and build; a temp dir by default
|
||||
# NO_GIT_PUSH=1 dry run: build, but neither upload nor push to origin
|
||||
# NO_UPLOAD=1 never contact the package registry (same as --no-upload)
|
||||
# FORCE_REBUILD=1 bump Release even with no new upstream version
|
||||
# FORCE_REFRESH=1 re-download the source and rebuild even when nothing changed
|
||||
# GIT_NAME, GIT_EMAIL %changelog and commit identity (default: git config)
|
||||
# GIT_TOKEN registry token (default: the git credential helper)
|
||||
#
|
||||
# local test mode -- setting either of these implies NO_GIT_PUSH=1 and
|
||||
# NO_UPLOAD=1, so a test can never publish anything:
|
||||
# AUTOUPDATE_TEST_VERSION use this as the upstream version (no lookup)
|
||||
# AUTOUPDATE_TEST_TARBALL build from this local tarball (no download)
|
||||
# plus PKG_REPO_BASE=file:///some/dir to clone a throwaway copy of the repo.
|
||||
#
|
||||
# Like aur/.autoupdate this is a RECONCILER: the registry is the distribution
|
||||
# channel, and it is compared against the package repo on every pass. A new
|
||||
# upstream release, a hand-edited spec pushed to origin, and an upload that
|
||||
# failed last night all end in the same place: whatever version-release the
|
||||
# spec says, the registry gets it. Distribution channel first, origin second,
|
||||
# so a failed build or upload leaves origin untouched and tonight's work is
|
||||
# simply recomputed tomorrow.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck source=common.sh
|
||||
source "${_here}/common.sh"
|
||||
|
||||
require_noninteractive
|
||||
require_tools rpmbuild rpmspec spectool rpm
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--no-upload) export NO_UPLOAD=1; shift ;;
|
||||
--) shift; break ;;
|
||||
-*) die "unknown option '$1' (usage: autoupdate.sh [--no-upload] <package> [...])" ;;
|
||||
*) break ;;
|
||||
esac
|
||||
done
|
||||
[ "$#" -ge 1 ] || die "usage: autoupdate.sh [--no-upload] <package> [...]"
|
||||
|
||||
if [ -n "${AUTOUPDATE_TEST_VERSION:-}${AUTOUPDATE_TEST_TARBALL:-}" ]; then
|
||||
export NO_GIT_PUSH=1 NO_UPLOAD=1
|
||||
fi
|
||||
|
||||
GIT_HOST="${GIT_HOST:-git.bjphoster.com}"
|
||||
# The org: where the package repos live and whose registry receives the RPMs.
|
||||
REGISTRY_OWNER="${REGISTRY_OWNER:-rpm}"
|
||||
# Overridable so a test can work on a throwaway clone; defaults are production.
|
||||
PKG_REPO_BASE="${PKG_REPO_BASE:-https://${GIT_HOST}/${REGISTRY_OWNER}}"
|
||||
LATEST_VERSION_CMD="${LATEST_VERSION_CMD:-${_here}/latest-version.sh}"
|
||||
|
||||
WORKDIR="${WORKDIR:-$(mktemp -d)}"
|
||||
mkdir -p "$WORKDIR"
|
||||
REGISTRY_BODY="${WORKDIR}/.registry-response"
|
||||
|
||||
# Spec tags are read and written as literal `Tag: value` lines, never
|
||||
# through rpmspec, so the rewrite can keep the rest of the file byte-identical.
|
||||
spec_get() { # <tag> <spec>
|
||||
sed -n -E "s/^$1:[[:space:]]*([^[:space:]]+)[[:space:]]*$/\1/p" "$2" | head -n1
|
||||
}
|
||||
|
||||
# Set Version and Release (N%{?dist}) and prepend a %changelog entry:
|
||||
#
|
||||
# * Sat Oct 10 2026 Name <email> - X-N
|
||||
# - <note>
|
||||
#
|
||||
# Nothing else in the file changes. The column alignment after `Version:` and
|
||||
# `Release:` is preserved; a blank line separates the new entry from the old.
|
||||
spec_bump() { # <spec> <version> <release> <note>
|
||||
local spec="$1" ver="$2" rel="$3" note="$4" day hdr rc=0
|
||||
resolve_identity
|
||||
day="$(LC_ALL=C date -u '+%a %b %d %Y')"
|
||||
hdr="* ${day} ${MAINT_NAME} <${MAINT_EMAIL}> - ${ver}-${rel}"
|
||||
awk -v ver="$ver" -v rel="${rel}%{?dist}" -v hdr="$hdr" -v note="- ${note}" '
|
||||
!v && /^Version:[[:space:]]/ { match($0, /^Version:[[:space:]]+/); print substr($0, 1, RLENGTH) ver; v = 1; next }
|
||||
!r && /^Release:[[:space:]]/ { match($0, /^Release:[[:space:]]+/); print substr($0, 1, RLENGTH) rel; r = 1; next }
|
||||
!c && /^%changelog[[:space:]]*$/ { print; print hdr; print note; c = 1; sep = 1; next }
|
||||
sep { if ($0 != "") print ""; sep = 0 }
|
||||
{ print }
|
||||
END { if (!v || !r || !c) exit 3 }
|
||||
' "$spec" > "${spec}.new" || rc=$?
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
rm -f "${spec}.new"
|
||||
die "${spec} needs literal Version:, Release: and %changelog lines to be bumped"
|
||||
fi
|
||||
cat "${spec}.new" > "$spec" # keep the inode and mode of the tracked file
|
||||
rm -f "${spec}.new"
|
||||
|
||||
# Refuse to go on unless rpm itself reads back exactly what was written.
|
||||
local got
|
||||
got="$(rpmspec -q --srpm --define 'dist %{nil}' --qf '%{version}-%{release}\n' "$spec")" \
|
||||
|| die "rpmspec cannot parse ${spec} after the bump"
|
||||
[ "$got" = "${ver}-${rel}" ] || die "rpmspec reads ${got} from ${spec}, expected ${ver}-${rel}"
|
||||
}
|
||||
|
||||
# Binary package names the spec builds (what the registry is keyed on).
|
||||
spec_names() { # <spec>
|
||||
rpmspec -q --builtrpms --define 'dist %{nil}' --qf '%{name}\n' "$1" | sort -u
|
||||
}
|
||||
|
||||
# rpmbuild -bb from the upstream tarball, with dist undefined so the result is
|
||||
# a distro-agnostic `X-N.noarch.rpm` that installs on Fedora, EL and openSUSE.
|
||||
# Fills the global BUILT array with the produced packages.
|
||||
build_rpm() { # <spec> <tarball>
|
||||
local spec="$1" tarball="$2" top src0
|
||||
top="${WORKDIR}/.rpmbuild/${pkg}"
|
||||
rm -rf "$top"
|
||||
mkdir -p "$top"/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS}
|
||||
|
||||
# Anything else the repo carries (patches, extra SourceN files).
|
||||
git ls-files -z | { grep -zv -x -F "$spec" || true; } | xargs -0 -r cp --parents -t "${top}/SOURCES"
|
||||
|
||||
# rpmbuild looks Source0 up by its basename (or its #/rename fragment).
|
||||
src0="$(spectool -l -s 0 "$spec" | sed -n -E 's/^Source0?:[[:space:]]*//p' | head -n1)"
|
||||
[ -n "$src0" ] || die "${spec} declares no Source0"
|
||||
[ "$src0" = "$tarball_url" ] || [[ "$src0" == "${tarball_url}#/"* ]] \
|
||||
|| warn "Source0 is ${src0}, building from ${tarball_url}"
|
||||
cp "$tarball" "${top}/SOURCES/${src0##*/}"
|
||||
|
||||
log "rpmbuild -bb ${spec}"
|
||||
rpmbuild -bb --define "_topdir ${top}" --define 'dist %{nil}' "$spec"
|
||||
|
||||
mapfile -t BUILT < <(find "${top}/RPMS" -type f -name '*.rpm' | sort)
|
||||
[ "${#BUILT[@]}" -gt 0 ] || die "rpmbuild produced no packages"
|
||||
local f
|
||||
for f in "${BUILT[@]}"; do info "built $(basename "$f")"; done
|
||||
|
||||
# Advisory only, like namcap is for the AUR: never blocks a release.
|
||||
if command -v rpmlint >/dev/null 2>&1; then
|
||||
log "rpmlint"
|
||||
rpmlint "$spec" "${BUILT[@]}" || warn "rpmlint reported problems (non-fatal)"
|
||||
else
|
||||
warn "rpmlint is not installed; skipping the lint"
|
||||
fi
|
||||
}
|
||||
|
||||
publish_rpm() { # <replace:0|1>
|
||||
local replace="$1" f name verrel arch
|
||||
for f in "${BUILT[@]}"; do
|
||||
if [ "$replace" -eq 1 ]; then
|
||||
name="$(rpm -qp --qf '%{name}' "$f")"
|
||||
verrel="$(rpm -qp --qf '%{version}-%{release}' "$f")"
|
||||
arch="$(rpm -qp --qf '%{arch}' "$f")"
|
||||
registry_delete "https://${GIT_HOST}/api/packages/${REGISTRY_OWNER}/rpm/package/${name}/${verrel}/${arch}"
|
||||
fi
|
||||
registry_upload "https://${GIT_HOST}/api/packages/${REGISTRY_OWNER}/rpm/upload" "$f"
|
||||
done
|
||||
}
|
||||
|
||||
update_one() {
|
||||
pkg="$1"
|
||||
local dir="${WORKDIR}/${pkg}"
|
||||
local spec="${pkg}.spec"
|
||||
|
||||
log "=== ${pkg} ==="
|
||||
|
||||
clone_or_reset "${PKG_REPO_BASE}/${pkg}" "$dir"
|
||||
cd "$dir"
|
||||
[ -f "$spec" ] || die "${pkg} has no ${spec}"
|
||||
[ -f .autoupdate ] || die "${pkg} has no .autoupdate manifest"
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
source ./.autoupdate
|
||||
: "${srchost:?manifest is missing srchost}"
|
||||
: "${srcmntr:?manifest is missing srcmntr}"
|
||||
: "${srcname:?manifest is missing srcname}"
|
||||
|
||||
# --- packaged version ----------------------------------------------------
|
||||
local oldver oldrel
|
||||
oldver="$(spec_get Version "$spec")"
|
||||
oldrel="$(spec_get Release "$spec")"
|
||||
validate_version "$oldver" "Version in ${spec}"
|
||||
oldrel="${oldrel%"%{?dist}"}"
|
||||
[[ "$oldrel" =~ ^[0-9]+$ ]] \
|
||||
|| die "Release in ${spec} is not of the form N%{?dist}; refusing to rewrite it"
|
||||
|
||||
# --- upstream version ----------------------------------------------------
|
||||
# A failed lookup exits non-zero and is NOT mistaken for "no new version".
|
||||
local newver
|
||||
if [ -n "${AUTOUPDATE_TEST_VERSION:-}" ]; then
|
||||
newver="$AUTOUPDATE_TEST_VERSION"
|
||||
info "test mode: upstream version forced to ${newver}"
|
||||
else
|
||||
newver="$("$LATEST_VERSION_CMD" .)" || die "upstream version lookup failed for ${pkg}"
|
||||
fi
|
||||
validate_version "$newver" "upstream version for ${pkg}"
|
||||
info "upstream ${newver}, packaged ${oldver}-${oldrel}"
|
||||
|
||||
# --- bump ------------------------------------------------------------------
|
||||
local ver="$oldver" rel="$oldrel" bumped=0
|
||||
if version_gt "$newver" "$oldver"; then
|
||||
ver="$newver"; rel=1
|
||||
log "new upstream version ${oldver} -> ${ver}"
|
||||
spec_bump "$spec" "$ver" "$rel" "Updated to ${ver}"
|
||||
bumped=1
|
||||
else
|
||||
if [ "$newver" != "$oldver" ]; then
|
||||
warn "upstream ${newver} is older than the packaged ${oldver}; not downgrading"
|
||||
fi
|
||||
if [ -n "${FORCE_REBUILD:-}" ]; then
|
||||
rel=$(( oldrel + 1 ))
|
||||
log "FORCE_REBUILD: Release ${oldrel} -> ${rel}"
|
||||
spec_bump "$spec" "$ver" "$rel" "Rebuilt ${ver}"
|
||||
bumped=1
|
||||
fi
|
||||
fi
|
||||
|
||||
git add -- "$spec"
|
||||
if git diff --cached --quiet; then
|
||||
log "no local changes for ${pkg}"
|
||||
else
|
||||
git_commit "Updated ${pkg} to ${ver}-${rel}"
|
||||
log "committed ${pkg} ${ver}-${rel}"
|
||||
fi
|
||||
|
||||
# --- distribution channel -------------------------------------------------
|
||||
# Reference state first: which version-release does the registry hold?
|
||||
local state="skipped" names=() name
|
||||
if [ -z "${NO_UPLOAD:-}" ]; then
|
||||
local list
|
||||
list="$(spec_names "$spec")" || die "rpmspec cannot list the packages ${spec} builds"
|
||||
[ -n "$list" ] || die "${spec} builds no binary packages"
|
||||
mapfile -t names <<<"$list"
|
||||
state="present"
|
||||
for name in "${names[@]}"; do
|
||||
if registry_has "$REGISTRY_OWNER" rpm "$name" "${ver}-${rel}"; then
|
||||
info "registry has ${name} ${ver}-${rel}"
|
||||
else
|
||||
info "registry lacks ${name} ${ver}-${rel}"
|
||||
state="absent"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
local build=0 upload=0 replace=0
|
||||
case "$state" in
|
||||
absent) build=1; upload=1 ;;
|
||||
present)
|
||||
if [ -n "${FORCE_REBUILD:-}" ]; then build=1; upload=1; replace=1
|
||||
elif [ -n "${FORCE_REFRESH:-}" ]; then build=1
|
||||
else log "registry already has ${pkg} ${ver}-${rel}; nothing to build"
|
||||
fi ;;
|
||||
skipped)
|
||||
if [ "$bumped" -eq 1 ] || [ -n "${FORCE_REFRESH:-}" ]; then build=1
|
||||
else log "NO_UPLOAD set and nothing changed; nothing to build"
|
||||
fi ;;
|
||||
esac
|
||||
|
||||
if [ "$build" -eq 1 ]; then
|
||||
local tag="${verstrip:-}${ver}" topdir="" tarball="${WORKDIR}/.sources/${pkg}-${ver}.tar.gz"
|
||||
case "$srchost" in
|
||||
gitea) tarball_url="https://${GIT_HOST}/${srcmntr}/${srcname}/archive/${tag}.tar.gz"
|
||||
topdir="$srcname" ;;
|
||||
github) tarball_url="https://github.com/${srcmntr}/${srcname}/archive/refs/tags/${tag}.tar.gz" ;;
|
||||
*) die "unknown srchost '${srchost}' (expected 'github' or 'gitea')" ;;
|
||||
esac
|
||||
fetch_tarball "$tarball_url" "$tarball" "$topdir"
|
||||
build_rpm "$spec" "$tarball"
|
||||
fi
|
||||
|
||||
if [ "$upload" -eq 1 ]; then
|
||||
if [ -n "${NO_GIT_PUSH:-}" ]; then
|
||||
log "NO_GIT_PUSH set: not uploading ${pkg} ${ver}-${rel} to ${REGISTRY_OWNER}'s registry"
|
||||
else
|
||||
publish_rpm "$replace"
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- origin ---------------------------------------------------------------
|
||||
# Only after the registry is in step, so a failed build or upload never
|
||||
# leaves origin claiming a release that was not published.
|
||||
if [ -n "${NO_GIT_PUSH:-}" ]; then
|
||||
log "NO_GIT_PUSH set: not pushing ${pkg} to origin"
|
||||
else
|
||||
git push --quiet origin HEAD
|
||||
fi
|
||||
}
|
||||
|
||||
# One package per process, deliberately -- see aur/.autoupdate: a subshell in
|
||||
# an && / || list disables `set -e` for its whole body, so `( update_one ) ||`
|
||||
# would sail past a failed build straight into `git push origin`. Re-exec'ing
|
||||
# gives each package a fresh shell whose `set -e` is intact. The CI matrix
|
||||
# passes exactly one package, so the loop only affects local runs.
|
||||
if [ -n "${AUTOUPDATE_SINGLE:-}" ]; then
|
||||
update_one "$1"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
rc=0
|
||||
for pkg in "$@"; do
|
||||
AUTOUPDATE_SINGLE=1 "$0" "$pkg" || { rc=1; warn "${pkg} FAILED"; }
|
||||
done
|
||||
exit "$rc"
|
||||
Reference in new issue
Block a user