Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3c77d34b7e | ||
|
|
8c83fb372b |
@@ -41,8 +41,6 @@ Two files per app, both sourcing the shared libs via `curl`:
|
|||||||
- **`ct/<app>.sh`** runs on the PVE host: prompts → unprivileged LXC → pushes a config env file into the container → bootstraps the installer. Apps that need Docker (authentik, runner) enable `nesting+keyctl` automatically. The standard prompts include an **SSH root login choice** (`SSH_ROOT_LOGIN`, default yes for homelab convenience; `no` keeps the Debian key-only default) — applied inside the container as an sshd drop-in by `configure_ssh_root_login`.
|
- **`ct/<app>.sh`** runs on the PVE host: prompts → unprivileged LXC → pushes a config env file into the container → bootstraps the installer. Apps that need Docker (authentik, runner) enable `nesting+keyctl` automatically. The standard prompts include an **SSH root login choice** (`SSH_ROOT_LOGIN`, default yes for homelab convenience; `no` keeps the Debian key-only default) — applied inside the container as an sshd drop-in by `configure_ssh_root_login`.
|
||||||
- **`install/<app>-install.sh`** runs inside the LXC: packages, unprivileged app user, secrets generated on-host (never printed), systemd units, a `/root/<app>.credentials` notes file — then shreds the bootstrap env. Idempotent where it matters: re-runs skip what exists. `setup_base_apt` also fixes the bare-template **locale situation**: `C.UTF-8` is exported up front (glibc built-in, covers the first apt run without perl warnings), then `en_US.UTF-8` is generated and set as the system default.
|
- **`install/<app>-install.sh`** runs inside the LXC: packages, unprivileged app user, secrets generated on-host (never printed), systemd units, a `/root/<app>.credentials` notes file — then shreds the bootstrap env. Idempotent where it matters: re-runs skip what exists. `setup_base_apt` also fixes the bare-template **locale situation**: `C.UTF-8` is exported up front (glibc built-in, covers the first apt run without perl warnings), then `en_US.UTF-8` is generated and set as the system default.
|
||||||
|
|
||||||
**DB installers** carry one extra rule: a PostgreSQL cluster/database freezes its encoding at initdb / `CREATE DATABASE` time and it can never be changed afterwards. A C (non-UTF-8) locale yields a `SQL_ASCII` cluster — psycopg3 then hands text back as bytes and SQLAlchemy crashes. So the pattern (helpers `ensure_utf8_locale_active` + `assert_db_encoding_utf8` in `lib/install.func`) is: make a UTF-8 locale **active** before the server package runs initdb, create the database **explicitly** with `TEMPLATE template0 ENCODING 'UTF8' LC_COLLATE/LC_CTYPE 'en_US.UTF-8'` (never inherit the cluster default), and **verify** `pg_encoding_to_char` returns `UTF8` before finishing — a wrong encoding aborts the install (it's DB damage, see wiki → Lessons). Maintenance examples use `su - postgres -c …`, not `sudo` — these minimal LXCs have no sudo.
|
|
||||||
|
|
||||||
Shared libs: [`lib/build.func`](lib/build.func) (host-side: prompts, LXC create, bootstrap) and [`lib/install.func`](lib/install.func) (in-container: apt, users, systemd, http-wait).
|
Shared libs: [`lib/build.func`](lib/build.func) (host-side: prompts, LXC create, bootstrap) and [`lib/install.func`](lib/install.func) (in-container: apt, users, systemd, http-wait).
|
||||||
|
|
||||||
## Security conventions
|
## Security conventions
|
||||||
|
|||||||
@@ -47,12 +47,6 @@ else
|
|||||||
msg_warn "PGDG repo already present, skipping"
|
msg_warn "PGDG repo already present, skipping"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# The server package runs initdb for the `main` cluster on install — its
|
|
||||||
# encoding is frozen there. Make a UTF-8 locale active for THIS process first
|
|
||||||
# so the cluster is never created as SQL_ASCII (issue #8: a pre-fix LXC was
|
|
||||||
# provisioned under LANG=C and ended up SQL_ASCII).
|
|
||||||
ensure_utf8_locale_active en_US.UTF-8
|
|
||||||
|
|
||||||
msg_info "Installing PostgreSQL $PG_MAJOR + pgvector..."
|
msg_info "Installing PostgreSQL $PG_MAJOR + pgvector..."
|
||||||
apt-get install -y -qq "postgresql-$PG_MAJOR" "postgresql-$PG_MAJOR-pgvector" >/dev/null
|
apt-get install -y -qq "postgresql-$PG_MAJOR" "postgresql-$PG_MAJOR-pgvector" >/dev/null
|
||||||
msg_ok "PostgreSQL $(psql --version | awk '{print $3}') installed"
|
msg_ok "PostgreSQL $(psql --version | awk '{print $3}') installed"
|
||||||
@@ -101,13 +95,6 @@ systemctl restart postgresql
|
|||||||
# ── role + database + extension (idempotent, password kept on re-run) ─────────
|
# ── role + database + extension (idempotent, password kept on re-run) ─────────
|
||||||
run_psql() { runuser -u postgres -- psql -v ON_ERROR_STOP=1 -qAt "$@"; }
|
run_psql() { runuser -u postgres -- psql -v ON_ERROR_STOP=1 -qAt "$@"; }
|
||||||
|
|
||||||
# Re-run safety (issue #8): if the database already exists, verify its
|
|
||||||
# encoding BEFORE touching roles/passwords. An old SQL_ASCII database must
|
|
||||||
# abort the run with NO side effects — not after rotating credentials.
|
|
||||||
if [[ "$(run_psql -c "SELECT 1 FROM pg_database WHERE datname='$DB_NAME'")" == "1" ]]; then
|
|
||||||
assert_db_encoding_utf8 "$DB_NAME"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ "$(run_psql -c "SELECT 1 FROM pg_roles WHERE rolname='$DB_USER'")" != "1" ]]; then
|
if [[ "$(run_psql -c "SELECT 1 FROM pg_roles WHERE rolname='$DB_USER'")" != "1" ]]; then
|
||||||
msg_info "Creating role $DB_USER + database $DB_NAME..."
|
msg_info "Creating role $DB_USER + database $DB_NAME..."
|
||||||
DB_PASS="$(openssl rand -base64 32 | tr -d '/+=' | head -c 32)"
|
DB_PASS="$(openssl rand -base64 32 | tr -d '/+=' | head -c 32)"
|
||||||
@@ -127,18 +114,12 @@ else
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ "$(run_psql -c "SELECT 1 FROM pg_database WHERE datname='$DB_NAME'")" != "1" ]]; then
|
if [[ "$(run_psql -c "SELECT 1 FROM pg_database WHERE datname='$DB_NAME'")" != "1" ]]; then
|
||||||
# Explicit encoding/collation from template0 — never inherit the cluster
|
run_psql -c "CREATE DATABASE $DB_NAME OWNER $DB_USER"
|
||||||
# default, which may be SQL_ASCII if initdb ran under a broken locale
|
|
||||||
# (issue #8). template0 is required to override LC_COLLATE/LC_CTYPE.
|
|
||||||
run_psql -c "CREATE DATABASE $DB_NAME OWNER $DB_USER ENCODING 'UTF8' LC_COLLATE 'en_US.UTF-8' LC_CTYPE 'en_US.UTF-8' TEMPLATE template0"
|
|
||||||
# Only the owner may connect — no PUBLIC access.
|
# Only the owner may connect — no PUBLIC access.
|
||||||
run_psql -c "REVOKE CONNECT ON DATABASE $DB_NAME FROM PUBLIC"
|
run_psql -c "REVOKE CONNECT ON DATABASE $DB_NAME FROM PUBLIC"
|
||||||
# Verify what we just created (the pre-existing case was already checked
|
msg_ok "Database $DB_NAME created (owner $DB_USER, PUBLIC revoked)"
|
||||||
# before the role block, issue #8).
|
|
||||||
assert_db_encoding_utf8 "$DB_NAME"
|
|
||||||
msg_ok "Database $DB_NAME created (UTF8, owner $DB_USER, PUBLIC revoked)"
|
|
||||||
else
|
else
|
||||||
msg_warn "Database $DB_NAME already exists, skipping creation"
|
msg_warn "Database $DB_NAME already exists, skipping"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# pgvector: CREATE EXTENSION needs superuser; installed now (per ADR-0002:
|
# pgvector: CREATE EXTENSION needs superuser; installed now (per ADR-0002:
|
||||||
@@ -160,12 +141,9 @@ Password: $DB_PASS
|
|||||||
DSN for /etc/nexus/env on the nexus LXC (NEXUS_DATABASE_URL):
|
DSN for /etc/nexus/env on the nexus LXC (NEXUS_DATABASE_URL):
|
||||||
postgresql+psycopg://$DB_USER:$DB_PASS@$IP_SELF:$DB_PORT/$DB_NAME
|
postgresql+psycopg://$DB_USER:$DB_PASS@$IP_SELF:$DB_PORT/$DB_NAME
|
||||||
|
|
||||||
Encoding: UTF8 (LC_COLLATE/LC_CTYPE en_US.UTF-8) — verified at install time.
|
|
||||||
|
|
||||||
Access policy (pg_hba): only $NEXUS_APP_IP/32 may connect; all other
|
Access policy (pg_hba): only $NEXUS_APP_IP/32 may connect; all other
|
||||||
hosts are rejected. Local socket stays peer-auth for maintenance (these
|
hosts are rejected. Local socket stays peer-auth for maintenance:
|
||||||
minimal LXCs have no sudo — use su, not sudo):
|
pct exec <CTID> -- runuser -u postgres -- psql -d $DB_NAME
|
||||||
pct exec <CTID> -- su - postgres -c "psql -d $DB_NAME"
|
|
||||||
EOF
|
EOF
|
||||||
chmod 600 "$CRED_FILE"
|
chmod 600 "$CRED_FILE"
|
||||||
msg_ok "Credentials written to $CRED_FILE (chmod 600)"
|
msg_ok "Credentials written to $CRED_FILE (chmod 600)"
|
||||||
|
|||||||
@@ -63,8 +63,34 @@ valid_token_word "$RUNNER_LABELS" || { msg_err "RUNNER_LABELS enthält unzuläss
|
|||||||
# der unprivilegierte User darf dort nicht schreiben.
|
# der unprivilegierte User darf dort nicht schreiben.
|
||||||
run_user() { runuser -u "$APP_USER" -- env -C "$RUNNER_DIR" HOME="$APP_HOME" "$@"; }
|
run_user() { runuser -u "$APP_USER" -- env -C "$RUNNER_DIR" HOME="$APP_HOME" "$@"; }
|
||||||
|
|
||||||
# ── Pakete: git/rsync, Node (checkout-Action), Docker (Test-Container) ────────
|
# ── Pakete: git/rsync, Node (checkout-Action), Docker (Test-Container),
|
||||||
setup_base_apt git rsync ca-certificates curl
|
# Build-Werkzeuge (Compiler-Sprachen in CI) ─────────────────────────────────
|
||||||
|
#
|
||||||
|
# make/gcc/protobuf: Der Runner faehrt Host-Mode, also gibt es kein Image, das
|
||||||
|
# Werkzeuge mitbringt — was hier nicht liegt, hat kein Job. Konkret gemessen an
|
||||||
|
# l.kirchner/patchmgr (Go):
|
||||||
|
# make "make: command not found" in jedem Gate-Job
|
||||||
|
# gcc ohne gefundenen C-Compiler bleibt cgo aus (go env
|
||||||
|
# meldet dann CGO_ENABLED=0), und "go test -race" ist
|
||||||
|
# nicht baubar
|
||||||
|
# protobuf-compiler protoc fuer die Codegenerierung
|
||||||
|
# libprotobuf-dev liefert /usr/include/google/protobuf/*.proto; ohne die
|
||||||
|
# Includes scheitert protoc trotz vorhandenem Binary
|
||||||
|
#
|
||||||
|
# ACHTUNG protoc: Das bindet jedes Repo der Instanz an die protoc-Fassung der
|
||||||
|
# Distribution (Debian 12: 3.21.x). Anders als make/gcc ist protoc ein
|
||||||
|
# Codegenerator — ein Distro-Upgrade aendert erzeugten Code fuer alle Nutzer
|
||||||
|
# gleichzeitig. Wer eine eigene Fassung braucht, pinnt sie im Projekt
|
||||||
|
# (Release-Tarball, buf, oder Docker — der Runner hat Docker) statt sie hier
|
||||||
|
# zu heben.
|
||||||
|
#
|
||||||
|
# Bewusst NICHT installiert: sudo. Ein Workflow soll auf diesem Runner nichts
|
||||||
|
# nachinstallieren koennen — was gebraucht wird, steht hier.
|
||||||
|
#
|
||||||
|
# Go selbst gehoert nicht hierher: Projekte holen es ueber actions/setup-go,
|
||||||
|
# weil CI-Matrizen mehrere Fassungen fahren.
|
||||||
|
setup_base_apt git rsync ca-certificates curl \
|
||||||
|
make gcc protobuf-compiler libprotobuf-dev
|
||||||
|
|
||||||
NODE_HAVE="$(command -v node >/dev/null 2>&1 && node -v | sed -E 's/^v([0-9]+).*/\1/' || echo 0)"
|
NODE_HAVE="$(command -v node >/dev/null 2>&1 && node -v | sed -E 's/^v([0-9]+).*/\1/' || echo 0)"
|
||||||
if [[ "$NODE_HAVE" != "$NODE_MAJOR" ]]; then
|
if [[ "$NODE_HAVE" != "$NODE_MAJOR" ]]; then
|
||||||
|
|||||||
@@ -54,60 +54,6 @@ apt_cleanup() {
|
|||||||
apt-get autoclean -qq >/dev/null || true
|
apt-get autoclean -qq >/dev/null || true
|
||||||
}
|
}
|
||||||
|
|
||||||
# ── database installers: UTF-8 locale before initdb ──────────────────────────
|
|
||||||
# Pattern for every DB installer. A PostgreSQL cluster/database freezes its
|
|
||||||
# encoding at initdb / CREATE DATABASE time and it cannot be changed later —
|
|
||||||
# a C (non-UTF-8) locale yields a SQL_ASCII cluster. psycopg3 then returns
|
|
||||||
# text as bytes and SQLAlchemy crashes on server-version detection; the app
|
|
||||||
# reports "db: unreachable". So: GENERATE the UTF-8 locale AND make it active
|
|
||||||
# for THIS process before the server package runs its automatic initdb, then
|
|
||||||
# fail loudly if it is not actually available (generating alone is not enough
|
|
||||||
# — the locale must be active when initdb runs).
|
|
||||||
# Generates+activates a UTF-8 locale (default en_US.UTF-8); the argument
|
|
||||||
# honours other UTF-8 locales consistently (match, locale.gen line and the
|
|
||||||
# exported value all derive from it). Matching normalises case and dashes so
|
|
||||||
# the canonical `en_US.UTF-8` matches `locale -a`'s `en_US.utf8`.
|
|
||||||
ensure_utf8_locale_active() {
|
|
||||||
local loc="${1:-en_US.UTF-8}"
|
|
||||||
local norm; norm="$(printf '%s' "$loc" | tr 'A-Z' 'a-z' | tr -d '-')"
|
|
||||||
_locale_present() { locale -a 2>/dev/null | tr 'A-Z' 'a-z' | tr -d '-' | grep -qx "$norm"; }
|
|
||||||
msg_info "Ensuring $loc is generated and active (DB encoding is frozen at initdb)..."
|
|
||||||
if ! _locale_present; then
|
|
||||||
# Uncomment the matching `# <loc> UTF-8` line, then generate.
|
|
||||||
sed -i "s/^# *${loc} UTF-8/${loc} UTF-8/" /etc/locale.gen
|
|
||||||
locale-gen >/dev/null
|
|
||||||
fi
|
|
||||||
if ! _locale_present; then
|
|
||||||
msg_err "Locale $loc not available after locale-gen — refusing to continue (initdb would create a SQL_ASCII cluster)"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
# Activate for the current process so any automatic initdb during the
|
|
||||||
# server package install inherits a UTF-8 locale, not the bare-template C.
|
|
||||||
export LANG="$loc" LC_ALL="$loc"
|
|
||||||
msg_ok "Locale active for initdb: LANG=$LANG"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Post-install guard: a database MUST be UTF8. Encoding is irreversible, so a
|
|
||||||
# wrong value is database damage — abort with a clear, actionable message
|
|
||||||
# instead of shipping a broken cluster. Uses argv-clean `runuser ... psql`
|
|
||||||
# with a quoted :'db' literal binding (robust regardless of caller); the
|
|
||||||
# credentials/README docs use `su - postgres -c` for hand maintenance (these
|
|
||||||
# minimal LXCs have no sudo).
|
|
||||||
assert_db_encoding_utf8() {
|
|
||||||
local db="$1" enc
|
|
||||||
enc="$(runuser -u postgres -- psql -X -qAt -v db="$db" \
|
|
||||||
-c "SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE datname = :'db'")"
|
|
||||||
if [[ "$enc" != "UTF8" ]]; then
|
|
||||||
msg_err "Database '$db' has encoding '${enc:-<not found>}', expected UTF8."
|
|
||||||
msg_err "Encoding is frozen at creation time — this is DB damage, not cosmetic."
|
|
||||||
msg_err "Fix: regenerate the locale (locale-gen en_US.UTF-8) and recreate the DB"
|
|
||||||
msg_err " with: CREATE DATABASE $db ... TEMPLATE template0 ENCODING 'UTF8'"
|
|
||||||
msg_err " LC_COLLATE 'en_US.UTF-8' LC_CTYPE 'en_US.UTF-8';"
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
msg_ok "Encoding check: database '$db' is UTF8"
|
|
||||||
}
|
|
||||||
|
|
||||||
# ── ssh ──────────────────────────────────────────────────────────────────────
|
# ── ssh ──────────────────────────────────────────────────────────────────────
|
||||||
# SSH-Root-Login gemäß Host-Prompt (prompt_lxc_config setzt SSH_ROOT_LOGIN,
|
# SSH-Root-Login gemäß Host-Prompt (prompt_lxc_config setzt SSH_ROOT_LOGIN,
|
||||||
# bootstrap_install_script reicht es als Env durch; Default: yes).
|
# bootstrap_install_script reicht es als Env durch; Default: yes).
|
||||||
|
|||||||
Reference in New Issue
Block a user