diff --git a/nginx/Makefile b/nginx/Makefile index 4ab7f23..d43ffa6 100644 --- a/nginx/Makefile +++ b/nginx/Makefile @@ -1,2 +1,8 @@ PARENT_DIR := .. include $(PARENT_DIR)/Makefile + +.PHONY: test + +test: uninstall clean install + @echo "Running Nginx tests..." + curl -sSfL -I http://localhost/ diff --git a/nginx/README.md b/nginx/README.md index db9de21..3c8e209 100644 --- a/nginx/README.md +++ b/nginx/README.md @@ -52,3 +52,9 @@ Finally, remove the quadlets, their configuration and their data. ```sh sudo make uninstall clean ``` + +## Integration tests + +```sh +sudo make test +``` diff --git a/postgresql/Makefile b/postgresql/Makefile index 4ab7f23..c4744e1 100644 --- a/postgresql/Makefile +++ b/postgresql/Makefile @@ -1,2 +1,39 @@ PARENT_DIR := .. include $(PARENT_DIR)/Makefile + +.PHONY: test test-set-pgmajor + +PG_MAJOR_START ?= 14 +PG_MAJOR_LAST ?= 18 +test-set-pgmajor: + sed -i 's/^PG_MAJOR=.*/PG_MAJOR=$(PG_MAJOR_START)/' config/config.env + +test: uninstall clean test-set-pgmajor install + @echo "Running PostgreSQL integration tests..."; \ + set -Eeuo pipefail; \ + sleep 2; \ + echo "Creating a test database and a witness table..."; \ + podman exec postgresql-server su postgres -c "createdb test"; \ + podman exec postgresql-server su postgres -c "psql -U postgres -d test -c \"CREATE TABLE witness (id SERIAL PRIMARY KEY, version VARCHAR); INSERT INTO witness (version) SELECT version();\""; \ + podman exec postgresql-server su postgres -c "psql -U postgres -d test -c \"SELECT * FROM witness;\""; \ + for (( ver=$(PG_MAJOR_START); ver<$(PG_MAJOR_LAST); ver++ )); do \ + echo "Running a backup..."; \ + nextver=$$(($$ver + 1)); \ + systemctl start postgresql-backup.service; \ + systemctl stop postgresql.target; \ + sleep 1; \ + rm -rf /var/lib/quadlets/postgresql/{$$ver,$$nextver,data,latest,.initialized}; \ + echo "Restoring the backup to PostgreSQL $$ver..."; \ + systemctl start postgresql.target; \ + sleep 2; \ + podman exec postgresql-server su postgres -c "psql -U postgres -d test -c \"SELECT * FROM witness;\""; \ + echo "Testing upgrade from PostgreSQL $$ver to $$nextver..."; \ + systemctl stop postgresql.target; \ + sed -i "s/^PG_MAJOR=.*/PG_MAJOR=$$nextver/" /etc/quadlets/postgresql/config.env; \ + systemctl start postgresql.target; \ + sleep 2; \ + echo "Inserting line into the witness table..."; \ + podman exec postgresql-server su postgres -c "psql -U postgres -d test -c \"INSERT INTO witness (version) SELECT version();\""; \ + done; \ + podman exec postgresql-server su postgres -c "psql -U postgres -d test -c \"SELECT * FROM witness;\""; \ + echo "PostgreSQL upgrade tests completed." diff --git a/postgresql/README.md b/postgresql/README.md index 173d01c..09ea108 100644 --- a/postgresql/README.md +++ b/postgresql/README.md @@ -96,3 +96,9 @@ Finally, remove the quadlets, their configuration and their data. ```sh sudo make uninstall clean ``` + +## Integration tests + +```sh +sudo make test +``` diff --git a/postgresql/config/config.env b/postgresql/config/config.env index 24278f6..7cd2fe1 100644 --- a/postgresql/config/config.env +++ b/postgresql/config/config.env @@ -4,5 +4,5 @@ POSTGRES_DB=postgres POSTGRES_HOST_AUTH_METHOD=scram-sha-256 POSTGRES_INITDB_ARGS=--auth-host=scram-sha-256 PGPORT=5432 -PG_MAJOR=17 +PG_MAJOR=14 POSTGRES_BACKUP_RETENTION=7 diff --git a/postgresql/config/init.sh b/postgresql/config/init.sh index 204db76..5e62160 100755 --- a/postgresql/config/init.sh +++ b/postgresql/config/init.sh @@ -2,6 +2,12 @@ set -Eeuo pipefail +echo "PostgreSQL binaries version:" +postgres --version +echo +echo "PostgreSQL data directory: $PGDATA" +echo + last_backup="" for f in /var/lib/postgresql/backup/*/backup_manifest; do # If there are no backups, the glob pattern above won't match any files diff --git a/postgresql/config/upgrade.sh b/postgresql/config/upgrade.sh new file mode 100755 index 0000000..d1d3d8f --- /dev/null +++ b/postgresql/config/upgrade.sh @@ -0,0 +1,149 @@ +#!/bin/bash + +set -Eeuo pipefail + +# Find the latest PostgreSQL data directory +SOURCE_PGDATA="" +last_version="" +for version_file in /var/lib/postgresql/*/docker/PG_VERSION; do + if [ -f "$version_file" ]; then + version_dir=$(dirname "$version_file") + version_major=$(cat "$version_file") + if [ -z "$last_version" ] || [ "$version_major" -gt "$last_version" ]; then + last_version="$version_major" + SOURCE_PGDATA="$version_dir" + fi + fi +done +if [ -z "$SOURCE_PGDATA" ] || [ ! -d "$SOURCE_PGDATA" ]; then + echo "No PostgreSQL data directory found." + exit 1 +fi +echo "Using PostgreSQL data directory: $SOURCE_PGDATA" + +# Upgrade destination +TARGET_MAJOR_VERSION=${PGTARGET%%.*} +TARGET_PATH="/usr/local/bin/" +TARGET_PGDATA="/var/lib/postgresql/${TARGET_MAJOR_VERSION}/docker" + +# Upgrade source +SOURCE_MAJOR_VERSION=$(cat "${SOURCE_PGDATA}/PG_VERSION") +SOURCE_PATH="/usr/local-pg${SOURCE_MAJOR_VERSION}/bin" + +# Reuse functions from the official entrypoint script +source /usr/local/bin/postgres-docker-entrypoint.sh + +# Because they may have been over by the sourced script, reset all flags +set -Eeuo pipefail + +# if first arg looks like a flag, assume we want to run postgres server +if [ "$#" -eq 0 ] || [ "${1:0:1}" = '-' ]; then + set -- postgres "$@" +fi + +# Setup environment variables +docker_setup_env + +## +## Sanity checks +## + +# No need to upgrade if same major version +if [ "${SOURCE_MAJOR_VERSION}" == "${TARGET_MAJOR_VERSION}" ]; then + echo "PostgreSQL data files version matches target version. No upgrade required." + exit 0 +fi +# No automatic upgrade support for PostgreSQL versions less than 14 +if [ "${SOURCE_MAJOR_VERSION}" -lt 14 ]; then + echo "PosgreSQL <14 is no longer supported for automatic upgrade. Please perform a manual upgrade." + exit 1 +fi +# No downgrade support +if [ "${SOURCE_MAJOR_VERSION}" -gt "${TARGET_MAJOR_VERSION}" ]; then + echo "Downgrades are not supported. Aborting." + exit 1 +fi +# Check for concurrent upgrade processes +if [ -f "${SOURCE_PGDATA}/upgrade_in_progress.lock" ]; then + echo "Another upgrade process seems to be running (upgrade_in_progress.lock file found). Aborting." + exit 2 +fi + +# On PG v18, we have to check that data checksums, be it positive or negative, is set on the initdb args +# even when the user already provided initdb args, because otherwise Postgres v18 assumes you want checksums +# we now do this on every version to avoid one more conditional +# it also adds support for people who used Postgres with checksums before v18 +if [[ -z "${POSTGRES_INITDB_ARGS:-}" || "${POSTGRES_INITDB_ARGS:-}" != *"data-checksums"* ]]; then + DATA_CHECKSUMS_ENABLED=$(echo 'SHOW DATA_CHECKSUMS' | "${SOURCE_PATH}/postgres" --single "${@:2}" -D "${SOURCE_PGDATA}" "${POSTGRES_DB}" | grep 'data_checksums = "' | cut -d '"' -f 2) + + if [ "$DATA_CHECKSUMS_ENABLED" == "on" ]; then + DATA_CHECKSUMS_PARAMETER="--data-checksums" + elif [ "$TARGET_MAJOR_VERSION" -eq 18 ]; then + # Postgres v18 enables data checksums by default and is the only version with this opt-out parameter + DATA_CHECKSUMS_PARAMETER="--no-data-checksums" + fi + POSTGRES_INITDB_ARGS="${POSTGRES_INITDB_ARGS:-} ${DATA_CHECKSUMS_PARAMETER:-}" +fi + +# Flags the data directory as being in the middle of an upgrade +mkdir -p "${TARGET_PGDATA}" +touch "${SOURCE_PGDATA}/upgrade_in_progress.lock" + +# Now PGDATA points to the target data directory +export PGDATA="${TARGET_PGDATA}" + +# Initialize target data directory +docker_verify_minimum_env +docker_init_database_dir + +# Change into the PostgreSQL database directory, to avoid a pg_upgrade error about write permissions +cd "${PGDATA}" + +# Perform the upgrade +echo "Upgrading PostgreSQL from version ${SOURCE_MAJOR_VERSION} to ${TARGET_MAJOR_VERSION}..." +"${TARGET_PATH}/pg_upgrade" --username="${POSTGRES_USER}" --link \ + --old-datadir "${SOURCE_PGDATA}" --new-datadir "${TARGET_PGDATA}" \ + --old-bindir "${SOURCE_PATH}" --new-bindir "${TARGET_PATH}" \ + --socketdir="/var/run/postgresql" \ + --old-options "${run_options[*]}" --new-options "${run_options[*]}" + +# Re-use the pg_hba.conf and pg_ident.conf from the old data directory +cp -f "${SOURCE_PGDATA}/pg_hba.conf" "${SOURCE_PGDATA}/pg_ident.conf" "${TARGET_PGDATA}" + +# Set PGPASSWORD in case password authentication is used +if [ -z "${PGPASSWORD:-}" ] && [ -n "${POSTGRES_PASSWORD:-}" ]; then + export PGPASSWORD="${POSTGRES_PASSWORD}" +fi + +# Start a temporary PostgreSQL server +docker_temp_server_start "$@" + +if [ -n "${POSTGRES_UPDATE_SCRIPT:-}" ] && [ -f "${POSTGRES_UPDATE_SCRIPT}" ]; then + echo "Running update script: ${POSTGRES_UPDATE_SCRIPT}" + psql --username="${POSTGRES_USER}" -f "${POSTGRES_UPDATE_SCRIPT}" +fi + +echo "Updating query planner stats" +declare -a database_list=( $(echo 'SELECT datname FROM pg_catalog.pg_database WHERE datistemplate IS FALSE' | psql --username="${POSTGRES_USER}" -1t --csv "${POSTGRES_DB}") ) +for database in "${database_list[@]}"; do + echo "VACUUM (ANALYZE, VERBOSE, INDEX_CLEANUP FALSE)" | psql --username="${POSTGRES_USER}" -t --csv "${database}" +done + +if [[ "${PGAUTO_REINDEX:-}" != "no" ]]; then + echo "Reindexing the databases" + + if [[ "$TARGET_MAJOR_VERSION" -le 15 ]]; then + reindexdb --all --username="${POSTGRES_USER}" + else + reindexdb --all --concurrently --username="${POSTGRES_USER}" + fi + echo "End of reindexing the databases" +fi + +# Stop the temporary PostgreSQL server +unset PGPASSWORD +docker_temp_server_stop + +# Clean up lock files +rm -f "${SOURCE_PGDATA}/upgrade_in_progress.lock" +echo "PostgreSQL upgrade from version ${SOURCE_MAJOR_VERSION} to ${TARGET_MAJOR_VERSION} completed successfully." diff --git a/postgresql/postgresql-init.container b/postgresql/postgresql-init.container index 4852b57..18bf6cb 100644 --- a/postgresql/postgresql-init.container +++ b/postgresql/postgresql-init.container @@ -26,7 +26,7 @@ Environment=PGDATA=/var/lib/postgresql/${PG_MAJOR}/docker # Those environment variables will be injected by podman into the container EnvironmentFile=/etc/quadlets/postgresql/config.env -# Use the official entrypoint script to initialize the database +# Use our entrypoint script to initialize the database Entrypoint=/usr/local/bin/init.sh # Volume mounts diff --git a/postgresql/postgresql-upgrade.container b/postgresql/postgresql-upgrade.container index d1f2036..9ac0a94 100644 --- a/postgresql/postgresql-upgrade.container +++ b/postgresql/postgresql-upgrade.container @@ -22,15 +22,16 @@ Image=docker.io/pgautoupgrade/pgautoupgrade:${PG_MAJOR}-alpine # Network configuration Network=host -# PostgreSQL storage is specific to major version -Environment=PGDATA=/var/lib/postgresql/${PG_MAJOR}/docker -Environment=PGAUTO_ONESHOT=yes - # Those environment variables will be injected by podman into the container EnvironmentFile=/etc/quadlets/postgresql/config.env +# Use our entrypoint script to upgrade the database +Entrypoint=/usr/local/bin/upgrade.sh +User=postgres + # Volume mounts Volume=/var/lib/quadlets/postgresql:/var/lib/postgresql:z +Volume=/etc/quadlets/postgresql/upgrade.sh:/usr/local/bin/upgrade.sh:z,ro [Service] Restart=no