From bd4293f53efa3428566612d123b6b76d15f86d98 Mon Sep 17 00:00:00 2001 From: claude-bot Date: Sat, 13 Jun 2026 14:30:40 +0200 Subject: [PATCH] fix(nexus-db): enforce UTF8 encoding at install time (issue #8) A PostgreSQL cluster/database freezes its encoding at initdb / CREATE DATABASE time; a C (non-UTF-8) locale yields a SQL_ASCII cluster, which makes psycopg3 return bytes and crashes SQLAlchemy. Harden the installer and add a reusable pattern for future DB installers: - ensure_utf8_locale_active: generate AND activate en_US.UTF-8 for the install process before the server package runs initdb; abort if the locale is not actually available - create the database explicitly with TEMPLATE template0 ENCODING 'UTF8' LC_COLLATE/LC_CTYPE 'en_US.UTF-8' instead of inheriting the cluster default - assert_db_encoding_utf8: post-install guard, abort with an actionable message if pg_encoding_to_char is not UTF8 (catches old SQL_ASCII DBs on re-run too) - credentials/README docs use su - postgres -c (minimal LXCs have no sudo) --- README.md | 2 ++ install/nexus-db-install.sh | 27 ++++++++++++++++++----- lib/install.func | 44 +++++++++++++++++++++++++++++++++++++ 3 files changed, 68 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 917d226..13aaf9a 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,8 @@ Two files per app, both sourcing the shared libs via `curl`: - **`ct/.sh`** runs on the PVE host: prompts → unprivileged LXC → pushes a config env file into the container → bootstraps the installer. Apps that need Docker (authentik, runner) enable `nesting+keyctl` automatically. The standard prompts include an **SSH root login choice** (`SSH_ROOT_LOGIN`, default yes for homelab convenience; `no` keeps the Debian key-only default) — applied inside the container as an sshd drop-in by `configure_ssh_root_login`. - **`install/-install.sh`** runs inside the LXC: packages, unprivileged app user, secrets generated on-host (never printed), systemd units, a `/root/.credentials` notes file — then shreds the bootstrap env. Idempotent where it matters: re-runs skip what exists. `setup_base_apt` also fixes the bare-template **locale situation**: `C.UTF-8` is exported up front (glibc built-in, covers the first apt run without perl warnings), then `en_US.UTF-8` is generated and set as the system default. +**DB installers** carry one extra rule: a PostgreSQL cluster/database freezes its encoding at initdb / `CREATE DATABASE` time and it can never be changed afterwards. A C (non-UTF-8) locale yields a `SQL_ASCII` cluster — psycopg3 then hands text back as bytes and SQLAlchemy crashes. So the pattern (helpers `ensure_utf8_locale_active` + `assert_db_encoding_utf8` in `lib/install.func`) is: make a UTF-8 locale **active** before the server package runs initdb, create the database **explicitly** with `TEMPLATE template0 ENCODING 'UTF8' LC_COLLATE/LC_CTYPE 'en_US.UTF-8'` (never inherit the cluster default), and **verify** `pg_encoding_to_char` returns `UTF8` before finishing — a wrong encoding aborts the install (it's DB damage, see wiki → Lessons). Maintenance examples use `su - postgres -c …`, not `sudo` — these minimal LXCs have no sudo. + Shared libs: [`lib/build.func`](lib/build.func) (host-side: prompts, LXC create, bootstrap) and [`lib/install.func`](lib/install.func) (in-container: apt, users, systemd, http-wait). ## Security conventions diff --git a/install/nexus-db-install.sh b/install/nexus-db-install.sh index 9bf3f77..1f57a44 100755 --- a/install/nexus-db-install.sh +++ b/install/nexus-db-install.sh @@ -47,6 +47,12 @@ else msg_warn "PGDG repo already present, skipping" fi +# The server package runs initdb for the `main` cluster on install — its +# encoding is frozen there. Make a UTF-8 locale active for THIS process first +# so the cluster is never created as SQL_ASCII (issue #8: a pre-fix LXC was +# provisioned under LANG=C and ended up SQL_ASCII). +ensure_utf8_locale_active en_US.UTF-8 + msg_info "Installing PostgreSQL $PG_MAJOR + pgvector..." apt-get install -y -qq "postgresql-$PG_MAJOR" "postgresql-$PG_MAJOR-pgvector" >/dev/null msg_ok "PostgreSQL $(psql --version | awk '{print $3}') installed" @@ -114,14 +120,22 @@ else fi if [[ "$(run_psql -c "SELECT 1 FROM pg_database WHERE datname='$DB_NAME'")" != "1" ]]; then - run_psql -c "CREATE DATABASE $DB_NAME OWNER $DB_USER" + # Explicit encoding/collation from template0 — never inherit the cluster + # default, which may be SQL_ASCII if initdb ran under a broken locale + # (issue #8). template0 is required to override LC_COLLATE/LC_CTYPE. + run_psql -c "CREATE DATABASE $DB_NAME OWNER $DB_USER ENCODING 'UTF8' LC_COLLATE 'en_US.UTF-8' LC_CTYPE 'en_US.UTF-8' TEMPLATE template0" # Only the owner may connect — no PUBLIC access. run_psql -c "REVOKE CONNECT ON DATABASE $DB_NAME FROM PUBLIC" - msg_ok "Database $DB_NAME created (owner $DB_USER, PUBLIC revoked)" + msg_ok "Database $DB_NAME created (UTF8, owner $DB_USER, PUBLIC revoked)" else - msg_warn "Database $DB_NAME already exists, skipping" + msg_warn "Database $DB_NAME already exists, skipping creation" fi +# Encoding guard (issue #8): catch both a freshly mis-created DB and a +# pre-existing SQL_ASCII database from an old provisioning. Abort before the +# app ever connects — a wrong encoding is DB damage, not a warning. +assert_db_encoding_utf8 "$DB_NAME" + # pgvector: CREATE EXTENSION needs superuser; installed now (per ADR-0002: # "Extension ab Tag 1 installiert, ungenutzt bis Phase 2"). run_psql -d "$DB_NAME" -c "CREATE EXTENSION IF NOT EXISTS vector" >/dev/null @@ -141,9 +155,12 @@ Password: $DB_PASS DSN for /etc/nexus/env on the nexus LXC (NEXUS_DATABASE_URL): postgresql+psycopg://$DB_USER:$DB_PASS@$IP_SELF:$DB_PORT/$DB_NAME +Encoding: UTF8 (LC_COLLATE/LC_CTYPE en_US.UTF-8) — verified at install time. + Access policy (pg_hba): only $NEXUS_APP_IP/32 may connect; all other -hosts are rejected. Local socket stays peer-auth for maintenance: - pct exec -- runuser -u postgres -- psql -d $DB_NAME +hosts are rejected. Local socket stays peer-auth for maintenance (these +minimal LXCs have no sudo — use su, not sudo): + pct exec -- su - postgres -c "psql -d $DB_NAME" EOF chmod 600 "$CRED_FILE" msg_ok "Credentials written to $CRED_FILE (chmod 600)" diff --git a/lib/install.func b/lib/install.func index 2bc3a5c..d1bf1a1 100644 --- a/lib/install.func +++ b/lib/install.func @@ -54,6 +54,50 @@ apt_cleanup() { apt-get autoclean -qq >/dev/null || true } +# ── database installers: UTF-8 locale before initdb ────────────────────────── +# Pattern for every DB installer. A PostgreSQL cluster/database freezes its +# encoding at initdb / CREATE DATABASE time and it cannot be changed later — +# a C (non-UTF-8) locale yields a SQL_ASCII cluster. psycopg3 then returns +# text as bytes and SQLAlchemy crashes on server-version detection; the app +# reports "db: unreachable". So: GENERATE the UTF-8 locale AND make it active +# for THIS process before the server package runs its automatic initdb, then +# fail loudly if it is not actually available (generating alone is not enough +# — the locale must be active when initdb runs). +ensure_utf8_locale_active() { + local loc="${1:-en_US.UTF-8}" + msg_info "Ensuring $loc is generated and active (DB encoding is frozen at initdb)..." + if ! locale -a 2>/dev/null | tr 'A-Z' 'a-z' | grep -q '^en_us\.utf-\?8$'; then + sed -i 's/^# *en_US\.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen + locale-gen >/dev/null + fi + if ! locale -a 2>/dev/null | tr 'A-Z' 'a-z' | grep -q '^en_us\.utf-\?8$'; then + msg_err "Locale $loc not available after locale-gen — refusing to continue (initdb would create a SQL_ASCII cluster)" + return 1 + fi + # Activate for the current process so any automatic initdb during the + # server package install inherits a UTF-8 locale, not the bare-template C. + export LANG="$loc" LC_ALL="$loc" + msg_ok "Locale active for initdb: LANG=$LANG" +} + +# Post-install guard: a database MUST be UTF8. Encoding is irreversible, so a +# wrong value is database damage — abort with a clear, actionable message +# instead of shipping a broken cluster. Reads via `su - postgres -c` (minimal +# LXCs have no sudo). +assert_db_encoding_utf8() { + local db="$1" enc + enc="$(su - postgres -c "psql -X -qAt -c \"SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE datname='$db'\"")" + if [[ "$enc" != "UTF8" ]]; then + msg_err "Database '$db' has encoding '${enc:-}', expected UTF8." + msg_err "Encoding is frozen at creation time — this is DB damage, not cosmetic." + msg_err "Fix: regenerate the locale (locale-gen en_US.UTF-8) and recreate the DB" + msg_err " with: CREATE DATABASE $db ... TEMPLATE template0 ENCODING 'UTF8'" + msg_err " LC_COLLATE 'en_US.UTF-8' LC_CTYPE 'en_US.UTF-8';" + return 1 + fi + msg_ok "Encoding check: database '$db' is UTF8" +} + # ── ssh ────────────────────────────────────────────────────────────────────── # SSH-Root-Login gemäß Host-Prompt (prompt_lxc_config setzt SSH_ROOT_LOGIN, # bootstrap_install_script reicht es als Env durch; Default: yes).