l.kirchner
d8565939ac
CI / Shell-Lint (bash -n, source-check, Validierungs-Tests) (pull_request) Successful in 3s
255 lines
9.6 KiB
Bash
Executable File
255 lines
9.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Daily cleaner for the instance-wide Gitea Actions runner (ct/runner.sh).
|
|
# Installed by install/runner-install.sh as /usr/local/sbin/ci-runner-cleanup,
|
|
# run by ci-runner-cleanup.timer once a day.
|
|
#
|
|
# A cancelled job never reaches its own teardown step, so what it started
|
|
# stays behind: a test database container, and a copy of the tree from
|
|
# counter-check in act-runner's private /tmp (l.kirchner/patchmgr, D64).
|
|
# This removes exactly that, and only what a project has claimed as its own.
|
|
# No `docker prune`: the runner works for every repository of the instance,
|
|
# and their containers and volumes are not ours to judge.
|
|
#
|
|
# A project claims its leftovers with one file in /etc/ci-runner-cleanup.d/:
|
|
#
|
|
# PROJECT=patchmgr
|
|
# CONTAINER_NAME_PATTERN='^patchmgr-testdb-'
|
|
# TMP_PATTERN='patchmgr-counter-check-*'
|
|
#
|
|
# The file is read, not run: three keys, a value either bare or in single
|
|
# quotes, comments and blank lines; anything else and the file is skipped.
|
|
#
|
|
# Ownership:
|
|
# - A project name is lower-case letters and digits only, no hyphen, so
|
|
# that "<project>-" is a prefix no other project's names can start with.
|
|
# It is handed out by whoever administers the runner, once per instance:
|
|
# the files here are root's, and two files naming the same project are
|
|
# both refused.
|
|
# - A container must carry the label org.luki-net.project=$PROJECT (set by
|
|
# the project itself) and its name must match CONTAINER_NAME_PATTERN:
|
|
# ^$PROJECT- and then only letters, digits and _.- -- a literal prefix,
|
|
# with no quantifier that could make the hyphen optional.
|
|
# - A tmp entry must match TMP_PATTERN: $PROJECT- and then only letters,
|
|
# digits and _.-*? -- a glob that names no directory.
|
|
# A container goes with its anonymous volumes (`rm -f -v`); named volumes
|
|
# are left alone.
|
|
#
|
|
# Known limit, accepted: label and name prefix say which project a leftover
|
|
# belongs to, not which repository. A fork of a registered project runs the
|
|
# same scripts unchanged on this runner and sets the same label and prefix,
|
|
# so its leftovers older than MAX_AGE_HOURS are removed as well. Accepted
|
|
# for patchmgr on 11.10.2026: the repository is private, has no forks, and
|
|
# only its owner and bots contribute. When it opens up for forks, its
|
|
# deploy/testdb.sh gets a repository label as well, and this script filters
|
|
# on that one too; tmp entries carry no label, so the prefix counter-check
|
|
# gives its directory has to name the repository then as well.
|
|
#
|
|
# Age: act_runner v0.2.13 ends a job after 3 h (runner.timeout, measured on
|
|
# runner-01..03 on 08.10.2026: no config file, LoadDefault sets 3 h), and
|
|
# each runner runs one job at a time. MAX_AGE_HOURS defaults to 4 and cannot
|
|
# be set to 3 or lower. What is older than that was not created or started
|
|
# by any job still inside its time limit:
|
|
# - a container is as old as the later of its creation and its last start,
|
|
# so one a job started again counts from that start;
|
|
# - a tmp entry is as old as its ctime. Not its mtime: cp -a, rsync -a and
|
|
# tar -x carry an old mtime over to a fresh copy, and nothing sets a
|
|
# ctime back.
|
|
# A project registers only containers its jobs create themselves with
|
|
# `docker run` and never start again, and only tmp entries its jobs create
|
|
# fresh for each run (mktemp) and never use again: a job that starts an
|
|
# existing container or works in an existing directory could do so between
|
|
# this script's look at its age and the removal, or without touching the
|
|
# directory's ctime at all, and nothing here can close that.
|
|
#
|
|
# DRY_RUN: empty or 0 removes; any other value only lists.
|
|
# Exit 1 when the cleanup of any project failed; the others still run.
|
|
set -euo pipefail
|
|
export LC_ALL=C
|
|
|
|
JOB_TIMEOUT_HOURS=3 # act_runner v0.2.13 runner.timeout, see above
|
|
MAX_AGE_HOURS=${MAX_AGE_HOURS:-4}
|
|
DRY_RUN=${DRY_RUN:-}
|
|
CONF_DIR=${CONF_DIR:-/etc/ci-runner-cleanup.d}
|
|
TMP_ROOT=${TMP_ROOT:-/tmp}
|
|
|
|
# At most three digits and no leading zero. Without an upper bound a large
|
|
# enough number (4611686018427387904, for one) overflows the seconds below
|
|
# to anything, 0 included; 999 h is far below that and longer than any age
|
|
# worth waiting for. A leading zero reads as octal.
|
|
if ! [[ "$MAX_AGE_HOURS" =~ ^[1-9][0-9]{0,2}$ ]] || [ "$MAX_AGE_HOURS" -le "$JOB_TIMEOUT_HOURS" ]; then
|
|
echo "MAX_AGE_HOURS must be a whole number from $((JOB_TIMEOUT_HOURS + 1)) to 999, above the job timeout of ${JOB_TIMEOUT_HOURS} h, not '$MAX_AGE_HOURS'" >&2
|
|
exit 1
|
|
fi
|
|
dry=0
|
|
case "$DRY_RUN" in
|
|
'' | 0) ;;
|
|
*) dry=1 ;;
|
|
esac
|
|
|
|
now=$(date +%s)
|
|
limit=$((MAX_AGE_HOURS * 3600))
|
|
|
|
# read_conf reads one project's file into PROJECT, CONTAINER_NAME_PATTERN and
|
|
# TMP_PATTERN, or says what is wrong with it and fails.
|
|
read_conf() {
|
|
local conf=$1 line key val
|
|
PROJECT='' CONTAINER_NAME_PATTERN='' TMP_PATTERN=''
|
|
while IFS= read -r line || [ -n "$line" ]; do
|
|
[[ "$line" =~ ^[[:space:]]*(#.*)?$ ]] && continue
|
|
if ! [[ "$line" =~ ^([A-Z_]+)=(.*)$ ]]; then
|
|
echo "$conf: not KEY=value: $line" >&2
|
|
return 1
|
|
fi
|
|
key=${BASH_REMATCH[1]} val=${BASH_REMATCH[2]}
|
|
if [[ "$val" =~ ^\'([^\']*)\'$ ]]; then
|
|
val=${BASH_REMATCH[1]}
|
|
elif ! [[ "$val" =~ ^[A-Za-z0-9_^*.-]*$ ]]; then
|
|
echo "$conf: $key: a value with anything but letters, digits and _^*.- goes in single quotes" >&2
|
|
return 1
|
|
fi
|
|
case "$key" in
|
|
PROJECT | CONTAINER_NAME_PATTERN | TMP_PATTERN) printf -v "$key" '%s' "$val" ;;
|
|
*)
|
|
echo "$conf: unknown key $key" >&2
|
|
return 1
|
|
;;
|
|
esac
|
|
done <"$conf"
|
|
}
|
|
|
|
# clean_project removes one project's leftovers and fails if anything it
|
|
# tried did not work. Run with errexit on, in a subshell of its own.
|
|
clean_project() {
|
|
local conf=$1 failed=0 n=0
|
|
if ! read_conf "$conf"; then
|
|
echo "$conf: skipped" >&2
|
|
return 1
|
|
fi
|
|
if ! [[ "$PROJECT" =~ ^[a-z0-9]+$ ]]; then
|
|
echo "$conf: PROJECT '$PROJECT' is not a project name (a-z, 0-9, no hyphen) -- skipped" >&2
|
|
return 1
|
|
fi
|
|
if [ -n "$CONTAINER_NAME_PATTERN" ]; then
|
|
if ! [[ "$CONTAINER_NAME_PATTERN" =~ ^\^$PROJECT-[A-Za-z0-9_.-]*$ ]]; then
|
|
echo "$conf: CONTAINER_NAME_PATTERN must be ^$PROJECT- followed only by letters, digits and _.- -- skipped" >&2
|
|
failed=1
|
|
else
|
|
local ids id fields name created started t_created t_started age
|
|
if ! ids=$(docker ps -aq --filter "label=org.luki-net.project=$PROJECT"); then
|
|
echo "$conf: docker ps failed" >&2
|
|
failed=1
|
|
ids=''
|
|
fi
|
|
for id in $ids; do
|
|
if ! fields=$(docker inspect -f '{{.Name}} {{.Created}} {{.State.StartedAt}}' "$id"); then
|
|
echo "$conf: docker inspect $id failed" >&2
|
|
failed=1
|
|
continue
|
|
fi
|
|
read -r name created started <<<"$fields"
|
|
name=${name#/}
|
|
[[ "$name" =~ $CONTAINER_NAME_PATTERN ]] || continue
|
|
if ! t_created=$(date -d "$created" +%s) || ! t_started=$(date -d "$started" +%s); then
|
|
echo "$conf: $name: cannot read its times '$created' '$started'" >&2
|
|
failed=1
|
|
continue
|
|
fi
|
|
# A container never started reports 0001-01-01 here.
|
|
[ "$t_started" -gt "$t_created" ] && t_created=$t_started
|
|
age=$((now - t_created))
|
|
[ "$age" -gt "$limit" ] || continue
|
|
echo "container $name ($((age / 3600)) h)"
|
|
[ "$dry" -eq 0 ] || continue
|
|
if docker rm -f -v "$id" >/dev/null; then
|
|
n=$((n + 1))
|
|
else
|
|
echo "$conf: docker rm $name failed" >&2
|
|
failed=1
|
|
fi
|
|
done
|
|
fi
|
|
fi
|
|
if [ -n "$TMP_PATTERN" ]; then
|
|
if ! [[ "$TMP_PATTERN" =~ ^$PROJECT-[A-Za-z0-9_.*?-]*$ ]]; then
|
|
echo "$conf: TMP_PATTERN must be $PROJECT- followed only by letters, digits and _.-*? -- skipped" >&2
|
|
failed=1
|
|
else
|
|
local dir list entry ctime age
|
|
local -a entries
|
|
list=$(mktemp)
|
|
# act-runner runs with PrivateTmp: its /tmp is under the host's.
|
|
for dir in "$TMP_ROOT"/systemd-private-*-act-runner.service-*/tmp; do
|
|
[ -d "$dir" ] || continue
|
|
if ! find "$dir" -mindepth 1 -maxdepth 1 -name "$TMP_PATTERN" -print0 >"$list"; then
|
|
echo "$conf: find in $dir failed" >&2
|
|
failed=1
|
|
continue
|
|
fi
|
|
mapfile -d '' entries <"$list"
|
|
for entry in "${entries[@]}"; do
|
|
if ! ctime=$(stat -c %Z -- "$entry"); then
|
|
echo "$conf: stat $entry failed" >&2
|
|
failed=1
|
|
continue
|
|
fi
|
|
age=$((now - ctime))
|
|
[ "$age" -gt "$limit" ] || continue
|
|
echo "tmp $entry ($((age / 3600)) h)"
|
|
[ "$dry" -eq 0 ] || continue
|
|
if rm -rf -- "$entry"; then
|
|
n=$((n + 1))
|
|
else
|
|
echo "$conf: rm $entry failed" >&2
|
|
failed=1
|
|
fi
|
|
done
|
|
done
|
|
rm -f -- "$list"
|
|
fi
|
|
fi
|
|
if [ "$dry" -eq 0 ]; then
|
|
echo "removed for $PROJECT: $n"
|
|
else
|
|
echo "removed for $PROJECT: none (DRY_RUN, the lines above are what would go)"
|
|
fi
|
|
return "$failed"
|
|
}
|
|
|
|
# project_of says which project a file claims, or nothing if it cannot be
|
|
# read; what is wrong with such a file is said by its own run below.
|
|
project_of() { (read_conf "$1" >/dev/null 2>&1 && printf '%s' "$PROJECT"); }
|
|
|
|
# A project named in two files is claimed twice, and neither file can be
|
|
# told from a copy that should not be there: both are refused.
|
|
status=0
|
|
shopt -s nullglob
|
|
declare -A claims=()
|
|
for conf in "$CONF_DIR"/*.conf; do
|
|
p=$(project_of "$conf") || continue
|
|
[ -z "$p" ] || claims[$p]=$((${claims[$p]:-0} + 1))
|
|
done
|
|
|
|
# One project at a time, each in a subshell of its own, with errexit on
|
|
# inside it: a failure one project's cleanup did not foresee still fails that
|
|
# project, and only that one. The run says so in its exit code.
|
|
for conf in "$CONF_DIR"/*.conf; do
|
|
p=$(project_of "$conf") || p=''
|
|
if [ -n "$p" ] && [ "${claims[$p]}" -gt 1 ]; then
|
|
echo "$conf: PROJECT $p is claimed by ${claims[$p]} files -- skipped" >&2
|
|
status=1
|
|
continue
|
|
fi
|
|
set +e
|
|
(
|
|
set -e
|
|
clean_project "$conf"
|
|
)
|
|
rc=$?
|
|
set -e
|
|
if [ "$rc" -ne 0 ]; then
|
|
echo "$conf: cleanup failed" >&2
|
|
status=1
|
|
fi
|
|
done
|
|
exit "$status"
|