feat: add the nightly deb reconciler
This commit is contained in:
bryanpedini committed
1 parent
4f730b914e
commit
f340028aae
5 files changed
+786
-1
No files matched your search
Executable
+312
@@ -0,0 +1,312 @@
|
||||
#!/usr/bin/env bash
|
||||
# Nightly Debian package reconciler.
|
||||
#
|
||||
# usage: autoupdate.sh [--no-upload] <package> [...]
|
||||
#
|
||||
# env:
|
||||
# GIT_HOST git host holding the package repos and the package
|
||||
# registry (default git.bjphoster.com)
|
||||
# WORKDIR where to clone and build; a temp dir by default
|
||||
# NO_GIT_PUSH=1 dry run: build, but neither upload nor push to origin
|
||||
# NO_UPLOAD=1 never contact the package registry (same as --no-upload)
|
||||
# FORCE_REBUILD=1 bump the Debian revision even with no new upstream version
|
||||
# FORCE_REFRESH=1 re-download the source and rebuild even when nothing changed
|
||||
# DEB_DISTRIBUTION registry distribution, also written into the changelog
|
||||
# (default stable)
|
||||
# DEB_COMPONENT registry component (default main)
|
||||
# GIT_NAME, GIT_EMAIL changelog and commit identity (default: git config)
|
||||
# GIT_TOKEN registry token (default: the git credential helper)
|
||||
#
|
||||
# local test mode -- setting either of these implies NO_GIT_PUSH=1 and
|
||||
# NO_UPLOAD=1, so a test can never publish anything:
|
||||
# AUTOUPDATE_TEST_VERSION use this as the upstream version (no lookup)
|
||||
# AUTOUPDATE_TEST_TARBALL build from this local tarball (no download)
|
||||
# plus PKG_REPO_BASE=file:///some/dir to clone a throwaway copy of the repo.
|
||||
#
|
||||
# Like aur/.autoupdate this is a RECONCILER: the registry is the distribution
|
||||
# channel, and it is compared against the package repo on every pass. A new
|
||||
# upstream release, a hand-edited debian/ pushed to origin, and an upload that
|
||||
# failed last night all end in the same place: whatever version the changelog
|
||||
# says, the registry gets it. Distribution channel first, origin second, so a
|
||||
# failed build or upload leaves origin untouched and tonight's work is simply
|
||||
# recomputed tomorrow.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
_here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
# shellcheck source=common.sh
|
||||
source "${_here}/common.sh"
|
||||
|
||||
require_noninteractive
|
||||
require_tools dpkg-parsechangelog dpkg-buildpackage dpkg-deb
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--no-upload) export NO_UPLOAD=1; shift ;;
|
||||
--) shift; break ;;
|
||||
-*) die "unknown option '$1' (usage: autoupdate.sh [--no-upload] <package> [...])" ;;
|
||||
*) break ;;
|
||||
esac
|
||||
done
|
||||
[ "$#" -ge 1 ] || die "usage: autoupdate.sh [--no-upload] <package> [...]"
|
||||
|
||||
if [ -n "${AUTOUPDATE_TEST_VERSION:-}${AUTOUPDATE_TEST_TARBALL:-}" ]; then
|
||||
export NO_GIT_PUSH=1 NO_UPLOAD=1
|
||||
fi
|
||||
|
||||
GIT_HOST="${GIT_HOST:-git.bjphoster.com}"
|
||||
# The org: where the package repos live and whose registry receives the debs.
|
||||
REGISTRY_OWNER="${REGISTRY_OWNER:-deb}"
|
||||
DEB_DISTRIBUTION="${DEB_DISTRIBUTION:-stable}"
|
||||
DEB_COMPONENT="${DEB_COMPONENT:-main}"
|
||||
# Overridable so a test can work on a throwaway clone; defaults are production.
|
||||
PKG_REPO_BASE="${PKG_REPO_BASE:-https://${GIT_HOST}/${REGISTRY_OWNER}}"
|
||||
LATEST_VERSION_CMD="${LATEST_VERSION_CMD:-${_here}/latest-version.sh}"
|
||||
|
||||
WORKDIR="${WORKDIR:-$(mktemp -d)}"
|
||||
mkdir -p "$WORKDIR"
|
||||
REGISTRY_BODY="${WORKDIR}/.registry-response"
|
||||
|
||||
# Prepend a changelog stanza in exactly the format dch writes:
|
||||
#
|
||||
# <source> (X-N) <distribution>; urgency=medium
|
||||
#
|
||||
# * <note>
|
||||
#
|
||||
# -- Name <email> Sat, 10 Oct 2026 00:00:00 +0000
|
||||
#
|
||||
# Written directly rather than through dch: no devscripts at run time, no
|
||||
# distribution-name warnings, and the bytes are fully predictable. Nothing
|
||||
# below the new stanza changes.
|
||||
changelog_bump() { # <full-version> <note>
|
||||
local version="$1" note="$2" srcpkg stamp got warnings
|
||||
resolve_identity
|
||||
srcpkg="$(dpkg-parsechangelog -l debian/changelog -S Source)" \
|
||||
|| die "cannot read the source name from debian/changelog"
|
||||
stamp="$(LC_ALL=C date -u -R)"
|
||||
{
|
||||
printf '%s (%s) %s; urgency=medium\n\n' "$srcpkg" "$version" "$DEB_DISTRIBUTION"
|
||||
printf ' * %s\n\n' "$note"
|
||||
printf ' -- %s <%s> %s\n\n' "$MAINT_NAME" "$MAINT_EMAIL" "$stamp"
|
||||
cat debian/changelog
|
||||
} > debian/changelog.new
|
||||
cat debian/changelog.new > debian/changelog # keep the inode and mode
|
||||
rm -f debian/changelog.new
|
||||
|
||||
# Refuse to go on unless dpkg reads back exactly what was written, and the
|
||||
# whole file (every stanza) still parses without a single warning.
|
||||
got="$(dpkg-parsechangelog -l debian/changelog -S Version)" \
|
||||
|| die "dpkg-parsechangelog cannot parse debian/changelog after the bump"
|
||||
[ "$got" = "$version" ] || die "dpkg-parsechangelog reads ${got}, expected ${version}"
|
||||
warnings="$(dpkg-parsechangelog -l debian/changelog --all 2>&1 >/dev/null)" || true
|
||||
if [ -n "$warnings" ]; then
|
||||
printf '%s\n' "$warnings" >&2
|
||||
die "debian/changelog does not parse cleanly after the bump"
|
||||
fi
|
||||
}
|
||||
|
||||
# Binary package names debian/control declares (what the registry is keyed on).
|
||||
control_names() {
|
||||
sed -n -E 's/^Package:[[:space:]]*([^[:space:]]+)[[:space:]]*$/\1/p' debian/control | sort -u
|
||||
}
|
||||
|
||||
# dpkg-buildpackage -b from the upstream tarball, laid out the way dpkg-source
|
||||
# expects: <source>_<upstream>.orig.tar.gz next to the unpacked tree, with the
|
||||
# packaging repo's debian/ dropped into it. Fills the global BUILT array.
|
||||
build_deb() { # <tarball> <source> <upstream-version>
|
||||
local tarball="$1" srcpkg="$2" upver="$3" bdir tree
|
||||
bdir="${WORKDIR}/.debbuild/${pkg}"
|
||||
rm -rf "$bdir"
|
||||
mkdir -p "$bdir"
|
||||
|
||||
cp "$tarball" "${bdir}/${srcpkg}_${upver}.orig.tar.gz"
|
||||
tar -xzf "${bdir}/${srcpkg}_${upver}.orig.tar.gz" -C "$bdir"
|
||||
mapfile -t tree < <(find "$bdir" -mindepth 1 -maxdepth 1 -type d)
|
||||
[ "${#tree[@]}" -eq 1 ] || die "expected one top-level directory in the tarball, found ${#tree[@]}"
|
||||
|
||||
# The packaging repo's debian/ always wins over anything upstream ships.
|
||||
rm -rf "${tree[0]}/debian"
|
||||
cp -a debian "${tree[0]}/debian"
|
||||
|
||||
log "dpkg-buildpackage -us -uc -b"
|
||||
( cd "${tree[0]}" && dpkg-buildpackage -us -uc -b )
|
||||
|
||||
mapfile -t BUILT < <(find "$bdir" -maxdepth 1 -type f -name '*.deb' | sort)
|
||||
[ "${#BUILT[@]}" -gt 0 ] || die "dpkg-buildpackage produced no packages"
|
||||
local f
|
||||
for f in "${BUILT[@]}"; do info "built $(basename "$f")"; done
|
||||
|
||||
# Advisory only, like namcap is for the AUR: never blocks a release.
|
||||
if command -v lintian >/dev/null 2>&1; then
|
||||
log "lintian"
|
||||
lintian "$bdir"/*.changes || warn "lintian reported problems (non-fatal)"
|
||||
else
|
||||
warn "lintian is not installed; skipping the lint"
|
||||
fi
|
||||
}
|
||||
|
||||
publish_deb() { # <replace:0|1>
|
||||
local replace="$1" pool f name version arch
|
||||
pool="https://${GIT_HOST}/api/packages/${REGISTRY_OWNER}/debian/pool/${DEB_DISTRIBUTION}/${DEB_COMPONENT}"
|
||||
for f in "${BUILT[@]}"; do
|
||||
if [ "$replace" -eq 1 ]; then
|
||||
name="$(dpkg-deb -f "$f" Package)"
|
||||
version="$(dpkg-deb -f "$f" Version)"
|
||||
arch="$(dpkg-deb -f "$f" Architecture)"
|
||||
registry_delete "${pool}/${name}/${version}/${arch}"
|
||||
fi
|
||||
registry_upload "${pool}/upload" "$f"
|
||||
done
|
||||
}
|
||||
|
||||
update_one() {
|
||||
pkg="$1"
|
||||
local dir="${WORKDIR}/${pkg}"
|
||||
|
||||
log "=== ${pkg} ==="
|
||||
|
||||
clone_or_reset "${PKG_REPO_BASE}/${pkg}" "$dir"
|
||||
cd "$dir"
|
||||
[ -f debian/changelog ] || die "${pkg} has no debian/changelog"
|
||||
[ -f debian/control ] || die "${pkg} has no debian/control"
|
||||
[ -f .autoupdate ] || die "${pkg} has no .autoupdate manifest"
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
source ./.autoupdate
|
||||
: "${srchost:?manifest is missing srchost}"
|
||||
: "${srcmntr:?manifest is missing srcmntr}"
|
||||
: "${srcname:?manifest is missing srcname}"
|
||||
|
||||
# --- packaged version ----------------------------------------------------
|
||||
# [epoch:]upstream-revision; the revision must be a plain integer so it can
|
||||
# be reset to 1 or incremented.
|
||||
local full epoch="" oldver oldrev srcpkg
|
||||
full="$(dpkg-parsechangelog -l debian/changelog -S Version)" \
|
||||
|| die "cannot parse debian/changelog"
|
||||
srcpkg="$(dpkg-parsechangelog -l debian/changelog -S Source)"
|
||||
if [[ "$full" == *:* ]]; then epoch="${full%%:*}:"; full="${full#*:}"; fi
|
||||
[[ "$full" == *-* ]] || die "debian/changelog version '${full}' has no Debian revision (native package?)"
|
||||
oldver="${full%-*}"
|
||||
oldrev="${full##*-}"
|
||||
validate_version "$oldver" "upstream part of the debian/changelog version"
|
||||
[[ "$oldrev" =~ ^[0-9]+$ ]] \
|
||||
|| die "Debian revision '${oldrev}' is not a plain integer; refusing to rewrite it"
|
||||
|
||||
# --- upstream version ----------------------------------------------------
|
||||
# A failed lookup exits non-zero and is NOT mistaken for "no new version".
|
||||
local newver
|
||||
if [ -n "${AUTOUPDATE_TEST_VERSION:-}" ]; then
|
||||
newver="$AUTOUPDATE_TEST_VERSION"
|
||||
info "test mode: upstream version forced to ${newver}"
|
||||
else
|
||||
newver="$("$LATEST_VERSION_CMD" .)" || die "upstream version lookup failed for ${pkg}"
|
||||
fi
|
||||
validate_version "$newver" "upstream version for ${pkg}"
|
||||
info "upstream ${newver}, packaged ${epoch}${oldver}-${oldrev}"
|
||||
|
||||
# --- bump ------------------------------------------------------------------
|
||||
local ver="$oldver" rev="$oldrev" bumped=0
|
||||
if version_gt "$newver" "$oldver"; then
|
||||
ver="$newver"; rev=1
|
||||
log "new upstream version ${oldver} -> ${ver}"
|
||||
changelog_bump "${epoch}${ver}-${rev}" "Updated to ${ver}"
|
||||
bumped=1
|
||||
else
|
||||
if [ "$newver" != "$oldver" ]; then
|
||||
warn "upstream ${newver} is older than the packaged ${oldver}; not downgrading"
|
||||
fi
|
||||
if [ -n "${FORCE_REBUILD:-}" ]; then
|
||||
rev=$(( oldrev + 1 ))
|
||||
log "FORCE_REBUILD: Debian revision ${oldrev} -> ${rev}"
|
||||
changelog_bump "${epoch}${ver}-${rev}" "Rebuilt ${ver}"
|
||||
bumped=1
|
||||
fi
|
||||
fi
|
||||
local version="${epoch}${ver}-${rev}"
|
||||
|
||||
git add -- debian/changelog
|
||||
if git diff --cached --quiet; then
|
||||
log "no local changes for ${pkg}"
|
||||
else
|
||||
git_commit "Updated ${pkg} to ${ver}-${rev}"
|
||||
log "committed ${pkg} ${version}"
|
||||
fi
|
||||
|
||||
# --- distribution channel -------------------------------------------------
|
||||
# Reference state first: which version does the registry hold?
|
||||
local state="skipped" names=() name
|
||||
if [ -z "${NO_UPLOAD:-}" ]; then
|
||||
local list
|
||||
list="$(control_names)" || die "cannot list the packages in debian/control"
|
||||
[ -n "$list" ] || die "debian/control declares no binary packages"
|
||||
mapfile -t names <<<"$list"
|
||||
state="present"
|
||||
for name in "${names[@]}"; do
|
||||
if registry_has "$REGISTRY_OWNER" debian "$name" "$version"; then
|
||||
info "registry has ${name} ${version}"
|
||||
else
|
||||
info "registry lacks ${name} ${version}"
|
||||
state="absent"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
local build=0 upload=0 replace=0
|
||||
case "$state" in
|
||||
absent) build=1; upload=1 ;;
|
||||
present)
|
||||
if [ -n "${FORCE_REBUILD:-}" ]; then build=1; upload=1; replace=1
|
||||
elif [ -n "${FORCE_REFRESH:-}" ]; then build=1
|
||||
else log "registry already has ${pkg} ${version}; nothing to build"
|
||||
fi ;;
|
||||
skipped)
|
||||
if [ "$bumped" -eq 1 ] || [ -n "${FORCE_REFRESH:-}" ]; then build=1
|
||||
else log "NO_UPLOAD set and nothing changed; nothing to build"
|
||||
fi ;;
|
||||
esac
|
||||
|
||||
if [ "$build" -eq 1 ]; then
|
||||
local tag="${verstrip:-}${ver}" topdir="" url tarball="${WORKDIR}/.sources/${pkg}-${ver}.tar.gz"
|
||||
case "$srchost" in
|
||||
gitea) url="https://${GIT_HOST}/${srcmntr}/${srcname}/archive/${tag}.tar.gz"
|
||||
topdir="$srcname" ;;
|
||||
github) url="https://github.com/${srcmntr}/${srcname}/archive/refs/tags/${tag}.tar.gz" ;;
|
||||
*) die "unknown srchost '${srchost}' (expected 'github' or 'gitea')" ;;
|
||||
esac
|
||||
fetch_tarball "$url" "$tarball" "$topdir"
|
||||
build_deb "$tarball" "$srcpkg" "$ver"
|
||||
fi
|
||||
|
||||
if [ "$upload" -eq 1 ]; then
|
||||
if [ -n "${NO_GIT_PUSH:-}" ]; then
|
||||
log "NO_GIT_PUSH set: not uploading ${pkg} ${version} to ${REGISTRY_OWNER}'s registry"
|
||||
else
|
||||
publish_deb "$replace"
|
||||
fi
|
||||
fi
|
||||
|
||||
# --- origin ---------------------------------------------------------------
|
||||
# Only after the registry is in step, so a failed build or upload never
|
||||
# leaves origin claiming a release that was not published.
|
||||
if [ -n "${NO_GIT_PUSH:-}" ]; then
|
||||
log "NO_GIT_PUSH set: not pushing ${pkg} to origin"
|
||||
else
|
||||
git push --quiet origin HEAD
|
||||
fi
|
||||
}
|
||||
|
||||
# One package per process, deliberately -- see aur/.autoupdate: a subshell in
|
||||
# an && / || list disables `set -e` for its whole body, so `( update_one ) ||`
|
||||
# would sail past a failed build straight into `git push origin`. Re-exec'ing
|
||||
# gives each package a fresh shell whose `set -e` is intact. The CI matrix
|
||||
# passes exactly one package, so the loop only affects local runs.
|
||||
if [ -n "${AUTOUPDATE_SINGLE:-}" ]; then
|
||||
update_one "$1"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
rc=0
|
||||
for pkg in "$@"; do
|
||||
AUTOUPDATE_SINGLE=1 "$0" "$pkg" || { rc=1; warn "${pkg} FAILED"; }
|
||||
done
|
||||
exit "$rc"
|
||||
Reference in new issue
Block a user