From 515e39d420a2d1f0c2c380d170ff8f81cf2282f0 Mon Sep 17 00:00:00 2001 From: adonis Date: Tue, 11 Aug 2026 21:34:02 +0330 Subject: [PATCH] feat(deploy): sync schema to replication subscribers before migrating Logical replication carries rows but not DDL, so a migration that adds a column stalls every subscriber with "missing replicated columns" until the column exists there too. _prisma_migrations is replicated, so migrate deploy on a subscriber reports nothing pending while its tables lag behind. sync-replica-schema.sh applies pending migrations to each subscriber first, tracking them locally in _replica_ddl_applied, then migrates the publisher. --check reports subscription state and columns missing on any replica. Co-authored-by: Cursor --- .gitignore | 3 + deploy/DEPLOY.md | 26 ++++++ deploy/replicas.env.example | 6 ++ deploy/sync-replica-schema.sh | 168 ++++++++++++++++++++++++++++++++++ package.json | 4 +- 5 files changed, 206 insertions(+), 1 deletion(-) create mode 100644 deploy/replicas.env.example create mode 100644 deploy/sync-replica-schema.sh diff --git a/.gitignore b/.gitignore index 6bbb333..eb4a55d 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,9 @@ coverage .env.local .env.*.local +# replica DSNs contain live DB passwords +deploy/replicas.env + # SMS import target files may contain live DB passwords prisma/khanegi-sms-targets.json prisma/*-sms-targets.json diff --git a/deploy/DEPLOY.md b/deploy/DEPLOY.md index 7722ac9..a791785 100644 --- a/deploy/DEPLOY.md +++ b/deploy/DEPLOY.md @@ -105,6 +105,32 @@ sudo certbot --nginx -d adm.montzar.ir 2. `https://adm.montzar.ir/get_user?domain=montpar.ir` (or any known domain) 3. Confirm uploads/logo URLs use `PUBLIC_API_URL` +## 8) Migrations with logical replication (failover replicas) + +`superadmin_pub` streams rows to the failover hosts (`adm.montech.ir`, `adm.montest.ir`). +Logical replication does **not** carry DDL: a migration that adds a column on the +publisher makes every subscriber fail with `missing replicated columns` and the +subscription stalls until that column exists there too. Because `_prisma_migrations` +is replicated, `prisma migrate deploy` on a subscriber wrongly reports +"No pending migrations". + +So never run `migrate deploy` alone. On the publisher host: + +```bash +cp deploy/replicas.env.example deploy/replicas.env # once; add real DSNs +bash deploy/sync-replica-schema.sh --init # once, when schemas already match +``` + +For every release: + +```bash +npm run replicas:sync # DDL to each replica first, then migrate deploy here +npm run replicas:check # subscription state + columns missing on any replica +``` + +Keep migrations schema-only. Data backfills would run once per replica and then +again through replication, so apply those on the publisher by hand. + ## Local development note Prisma provider is **PostgreSQL** only. Local options: diff --git a/deploy/replicas.env.example b/deploy/replicas.env.example new file mode 100644 index 0000000..ceecd28 --- /dev/null +++ b/deploy/replicas.env.example @@ -0,0 +1,6 @@ +# Copy to deploy/replicas.env on the publisher host (gitignored — holds passwords). +# +# Every logical-replication subscriber of superadmin_pub, space/comma/newline separated. +# The user must be able to run ALTER TABLE (table owner or superuser). +REPLICA_DSNS="postgresql://USER:PASSWORD@SLAVE_1_IP:5433/superadmin +postgresql://USER:PASSWORD@SLAVE_2_IP:5432/superadmin" diff --git a/deploy/sync-replica-schema.sh b/deploy/sync-replica-schema.sh new file mode 100644 index 0000000..91f0e50 --- /dev/null +++ b/deploy/sync-replica-schema.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +# Keep logical-replication subscribers in schema sync with the publisher. +# +# Logical replication ships rows, not DDL. When a migration adds a column on the +# publisher, every subscriber keeps crashing with +# "logical replication target relation ... is missing replicated columns" +# and the whole subscription stalls until the column exists there too. +# +# _prisma_migrations is itself replicated, so `prisma migrate deploy` on a +# subscriber reports "No pending migrations" even when its tables lag behind. +# Applied migrations are therefore tracked per subscriber in _replica_ddl_applied. +# +# Usage: +# bash deploy/sync-replica-schema.sh --init # baseline: mark current migrations as applied +# bash deploy/sync-replica-schema.sh --dry-run # list what would run +# bash deploy/sync-replica-schema.sh # apply to replicas, then migrate publisher +# bash deploy/sync-replica-schema.sh --check # replication + schema drift report +set -euo pipefail + +API_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +MIGRATIONS_DIR="$API_DIR/prisma/migrations" +CONFIG_FILE="${REPLICAS_ENV_FILE:-$API_DIR/deploy/replicas.env}" +MODE="${1:-apply}" + +case "$MODE" in + apply|--init|--dry-run|--check) ;; + *) + echo "Usage: $0 [--init|--dry-run|--check]" >&2 + exit 1 + ;; +esac + +command -v psql >/dev/null || { echo "psql not found" >&2; exit 1; } + +if [[ -f "$CONFIG_FILE" ]]; then + # shellcheck disable=SC1090 + set -a && source "$CONFIG_FILE" && set +a +fi + +if [[ -z "${REPLICA_DSNS:-}" ]]; then + echo "REPLICA_DSNS is empty. Create $CONFIG_FILE from deploy/replicas.env.example" >&2 + exit 1 +fi + +if [[ -z "${DATABASE_URL:-}" && -f "$API_DIR/.env" ]]; then + # shellcheck disable=SC1091 + set -a && source "$API_DIR/.env" && set +a +fi + +read -r -a REPLICAS <<< "$(tr ',\n' ' ' <<< "$REPLICA_DSNS")" + +# psql would otherwise swallow the loop input it is called from +run_psql() { + psql "$@" &2; return 1; } + + local bundle + bundle="$(mktemp)" + cat "$sql_file" > "$bundle" + printf "\nINSERT INTO _replica_ddl_applied (migration_name) VALUES ('%s') ON CONFLICT DO NOTHING;\n" \ + "$name" >> "$bundle" + + run_psql "$dsn" -v ON_ERROR_STOP=1 -q --single-transaction -f "$bundle" + rm -f "$bundle" +} + +column_fingerprint() { + run_psql "$1" -v ON_ERROR_STOP=1 -At -c " + SELECT table_name || '.' || column_name + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name NOT IN ('_replica_ddl_applied')" | LC_ALL=C sort +} + +if [[ "$MODE" == "--check" ]]; then + for dsn in "${REPLICAS[@]}"; do + echo "== $(label "$dsn")" + run_psql "$dsn" -v ON_ERROR_STOP=1 -At -c " + SELECT subname || ' received=' || COALESCE(received_lsn::text, '-') + || ' latest=' || COALESCE(latest_end_lsn::text, '-') + || ' last_msg=' || COALESCE(last_msg_receipt_time::text, '-') + FROM pg_stat_subscription" || true + + if [[ -n "${DATABASE_URL:-}" ]]; then + drift="$(comm -23 <(column_fingerprint "$DATABASE_URL") <(column_fingerprint "$dsn") || true)" + if [[ -n "$drift" ]]; then + echo " ! columns present on publisher but missing here:" + sed 's/^/ /' <<< "$drift" + else + echo " columns match publisher" + fi + fi + done + exit 0 +fi + +for dsn in "${REPLICAS[@]}"; do + echo "== $(label "$dsn")" + ensure_marker_table "$dsn" + + if [[ "$MODE" == "--init" ]]; then + while read -r name; do + run_psql "$dsn" -v ON_ERROR_STOP=1 -q -c \ + "INSERT INTO _replica_ddl_applied (migration_name) VALUES ('$name') ON CONFLICT DO NOTHING" + done < <(migration_names) + echo " baseline recorded (no SQL executed)" + continue + fi + + pending="$(pending_names "$dsn")" + if [[ -z "$pending" ]]; then + echo " up to date" + continue + fi + + while read -r name; do + if [[ "$MODE" == "--dry-run" ]]; then + echo " would apply $name" + else + echo " applying $name" + apply_migration "$dsn" "$name" + fi + done <<< "$pending" +done + +if [[ "$MODE" == "--dry-run" || "$MODE" == "--init" ]]; then + exit 0 +fi + +echo "== publisher" +cd "$API_DIR" +npx prisma migrate deploy + +echo +echo "Replicas apply DDL first, so the publisher can stream the new columns right away." +echo "Verify with: bash deploy/sync-replica-schema.sh --check" diff --git a/package.json b/package.json index 74b067b..ee3431b 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,9 @@ "prisma:bulk-khanegi-modular-brand": "ts-node --transpile-only prisma/bulk-khanegi-modular-brand.ts", "prisma:repair-khanegi-ui-config": "ts-node --transpile-only prisma/repair-khanegi-ui-config.ts", "prisma:copy-sms-between-tenants": "ts-node --transpile-only prisma/copy-sms-between-tenants.ts", - "prisma:list-tenants": "ts-node --transpile-only prisma/list-tenants.ts" + "prisma:list-tenants": "ts-node --transpile-only prisma/list-tenants.ts", + "replicas:sync": "bash deploy/sync-replica-schema.sh", + "replicas:check": "bash deploy/sync-replica-schema.sh --check" }, "prisma": { "seed": "ts-node --transpile-only prisma/seed.ts"