From 16d562caa5be5cfd52db75b70fc30e38a790f748 Mon Sep 17 00:00:00 2001 From: adonis Date: Tue, 18 Aug 2026 12:32:30 +0330 Subject: [PATCH] feat(replicas): auto-heal slave data drift and no-store public settings Add sync-replica-data repair on deploy, a publisher-side replica watch (60s + after tenant/settings writes), REFRESH PUBLICATION after migrate, and Cache-Control no-store on program-settings and get_version. Co-authored-by: Cursor --- deploy/DEPLOY.md | 64 ++++- deploy/replicas.env.example | 5 +- deploy/sync-replica-data.sh | 247 ++++++++++++++++++ deploy/sync-replica-schema.sh | 24 ++ package.json | 5 +- src/app.module.ts | 2 + src/get-version/get-version.controller.ts | 3 +- .../api-public-program-settings.controller.ts | 2 +- .../program-settings.service.ts | 6 + .../public-program-settings.controller.ts | 2 +- src/replica-watch/replica-watch.module.ts | 9 + src/replica-watch/replica-watch.service.ts | 93 +++++++ src/tenants/tenants.service.ts | 5 + 13 files changed, 456 insertions(+), 11 deletions(-) create mode 100644 deploy/sync-replica-data.sh create mode 100644 src/replica-watch/replica-watch.module.ts create mode 100644 src/replica-watch/replica-watch.service.ts diff --git a/deploy/DEPLOY.md b/deploy/DEPLOY.md index 3b0d823..d3f28a7 100644 --- a/deploy/DEPLOY.md +++ b/deploy/DEPLOY.md @@ -122,15 +122,69 @@ cp deploy/replicas.env.example deploy/replicas.env # once; add real DSNs bash deploy/sync-replica-schema.sh --init # once, when schemas already match ``` -For every release: +For every release on **publisher** (`/var/www/superadmin/api`): ```bash -npm run replicas:sync # DDL to each replica first, then migrate deploy here -npm run replicas:check # subscription state + columns missing on any replica +git pull +npm run replicas:sync # 1) DDL on replicas 2) migrate deploy 3) repair data drift if any +npx prisma generate +npm run build +pm2 restart superadmin-api ``` -Keep migrations schema-only. Data backfills would run once per replica and then -again through replication, so apply those on the publisher by hand. +Check only: + +```bash +npm run replicas:check # schema + subscription + row counts (settings, tenants, …) +``` + +Force data realign from publisher (rare — e.g. after fixing subscriber permissions): + +```bash +npm run replicas:repair-data +``` + +`replicas:sync` runs `sync-replica-data.sh --repair-if-needed` automatically when +`settings` / `setting_templates` / `tenants` counts differ from publisher. + +On the **publisher API** (`superadmin-api`), a replica watch also runs: ~8s after +boot, then every 60s, and immediately after tenant/settings writes. Drift is +healed without an operator command. Logical replication still streams live +INSERT/UPDATE/DELETE; the watch only fills **missed history** (WAL is not +rewound). + +Each replica API user (`montime` by default) needs `SELECT` on all tables — the +repair script re-applies grants. `REPLICA_DSNS` must connect as a role that can +`TRUNCATE` + `ALTER SUBSCRIPTION` on the subscriber. The watch is idle unless +`deploy/replicas.env` exists on that host. + +Keep migrations schema-only. After each migrate, subscribers get +`ALTER SUBSCRIPTION … REFRESH PUBLICATION WITH (copy_data = true)` so **new +tables** join replication (Postgres does not add them by itself). + +## 9) Data changes **without** deploy (import settings, edit tenant, …) + +**Normal path:** publisher writes → `superadmin_pub` WAL → slaves apply in +**seconds**. No `replicas:sync` for that. + +**Why LSN-healthy slaves can still have empty `settings`:** logical replication +is a **stream**, not a mirror. `received_lsn = latest_end_lsn` means “caught up +with current WAL”, not “every historical row exists”. Rows inserted while apply +was broken, or while the table was **not yet in the subscription**, are never +replayed. Postgres will not go back. + +That gap is healed **automatically on the publisher API** (watch every 60s + +right after import/edit). You do **not** run `replicas:repair-data` by hand +once this build is running on `/var/www/superadmin/api`. + +| Action | What actually happens | +|--------|------------------------| +| INSERT/UPDATE/DELETE while subscription is ready | Live WAL — slaves update immediately | +| New table after `FOR ALL TABLES` publication | Not replicated until `REFRESH PUBLICATION` (deploy `replicas:sync` does this) | +| Missed history (this tenant’s empty settings) | Watch compares counts and recopies from publisher, then live WAL resumes | + +`npm run replicas:check` / `replicas:repair-data` remain for operators; they are +not required for day-to-day imports after the watch is up. ## Local development note diff --git a/deploy/replicas.env.example b/deploy/replicas.env.example index 5c60be0..03c25cc 100644 --- a/deploy/replicas.env.example +++ b/deploy/replicas.env.example @@ -3,6 +3,9 @@ # One DSN per line (or comma). Do not put spaces between URLs. # Prefer URI form. Password must not contain @ : / # ? — or URL-encode it. # Keyword form (host=...) is OK only because the script splits on newlines, not spaces. -# The user must be able to run ALTER TABLE (table owner or superuser). +# The user must be able to run ALTER TABLE + ALTER SUBSCRIPTION + TRUNCATE on each replica. REPLICA_DSNS="postgresql://USER:PASSWORD@SLAVE_1_IP:5433/superadmin postgresql://USER:PASSWORD@SLAVE_2_IP:5432/superadmin" + +# Read-only app user on replicas (program-settings API). Grants re-applied after data repair. +# REPLICA_APP_DB_USER=montime diff --git a/deploy/sync-replica-data.sh b/deploy/sync-replica-data.sh new file mode 100644 index 0000000..9083de1 --- /dev/null +++ b/deploy/sync-replica-data.sh @@ -0,0 +1,247 @@ +#!/usr/bin/env bash +# Compare + repair row data on logical-replication subscribers vs publisher. +# +# replicas:sync only ships DDL. If a subscriber missed initial copy (permissions, +# stalled subscription, import before replication caught up), rows stay missing +# until repaired. This script makes publisher the source of truth for core tables. +# +# Usage (on publisher, same deploy/replicas.env as sync-replica-schema.sh): +# bash deploy/sync-replica-data.sh --check +# bash deploy/sync-replica-data.sh --repair +# bash deploy/sync-replica-data.sh --repair-if-needed +set -euo pipefail + +API_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CONFIG_FILE="${REPLICAS_ENV_FILE:-$API_DIR/deploy/replicas.env}" +MODE="${1:---repair-if-needed}" + +# Tables that must match publisher for public API (order: parents before children). +SYNC_TABLES=(setting_templates tenants settings) +APP_DB_USER="${REPLICA_APP_DB_USER:-montime}" + +case "$MODE" in + --check|--repair|--repair-if-needed) ;; + *) + echo "Usage: $0 [--check|--repair|--repair-if-needed]" >&2 + exit 1 + ;; +esac + +command -v psql >/dev/null || { echo "psql not found" >&2; exit 1; } +command -v pg_dump >/dev/null || { echo "pg_dump not found" >&2; exit 1; } + +if [[ -f "$CONFIG_FILE" ]]; then + # shellcheck disable=SC1090 + set -a && source "$CONFIG_FILE" && set +a +fi + +if [[ -z "${REPLICA_DSNS:-}" ]]; then + echo "REPLICA_DSNS is empty. Create $CONFIG_FILE from deploy/replicas.env.example" >&2 + exit 1 +fi + +if [[ -z "${DATABASE_URL:-}" && -f "$API_DIR/.env" ]]; then + # shellcheck disable=SC1091 + set -a && source "$API_DIR/.env" && set +a +fi + +if [[ -z "${DATABASE_URL:-}" ]]; then + echo "DATABASE_URL is empty (publisher)." >&2 + exit 1 +fi + +mapfile -t REPLICAS < <( + printf '%s\n' "$REPLICA_DSNS" | tr ',' '\n' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' | grep -v '^$' +) + +run_psql() { + psql "$@" /dev/null || true +} + +copy_table_from_publisher() { + local replica_dsn="$1" table="$2" + local pub_dsn + pub_dsn="$(libpq_dsn "$DATABASE_URL")" + pg_dump "$pub_dsn" \ + -t "public.\"$table\"" \ + --data-only \ + --no-owner \ + --no-privileges \ + --disable-triggers \ + | run_psql "$(libpq_dsn "$replica_dsn")" -v ON_ERROR_STOP=1 -q +} + +repair_replica() { + local replica_dsn="$1" + local lbl drifted=() + local table pub replica + + lbl="$(label "$replica_dsn")" + echo "== repair $lbl" + + mapfile -t subs < <(subscription_names "$replica_dsn") + if ((${#subs[@]} == 0)); then + echo " ! no subscription on replica — skip data repair (create subscription first)" >&2 + return 1 + fi + + for table in "${SYNC_TABLES[@]}"; do + pub="$(table_count "$DATABASE_URL" "$table")" + replica="$(table_count "$replica_dsn" "$table")" + if [[ "$pub" != "$replica" ]]; then + drifted+=("$table") + echo " drift $table publisher=$pub replica=$replica" + fi + done + + if ((${#drifted[@]} == 0)); then + echo " data already in sync" + grant_app_user "$replica_dsn" + return 0 + fi + + # TRUNCATE tenants CASCADE wipes settings — recopy the whole core set. + drifted=("${SYNC_TABLES[@]}") + + for sub in "${subs[@]}"; do + echo " disabling subscription $sub" + run_psql "$(libpq_dsn "$replica_dsn")" -v ON_ERROR_STOP=1 -q \ + -c "ALTER SUBSCRIPTION \"${sub}\" DISABLE" + done + + for table in "${drifted[@]}"; do + echo " truncating + copying $table" + run_psql "$(libpq_dsn "$replica_dsn")" -v ON_ERROR_STOP=1 -q \ + -c "TRUNCATE TABLE public.\"$table\" CASCADE" + copy_table_from_publisher "$replica_dsn" "$table" + pub="$(table_count "$DATABASE_URL" "$table")" + replica="$(table_count "$replica_dsn" "$table")" + if [[ "$pub" != "$replica" ]]; then + echo " ! repair failed for $table (publisher=$pub replica=$replica)" >&2 + for sub in "${subs[@]}"; do + run_psql "$(libpq_dsn "$replica_dsn")" -v ON_ERROR_STOP=1 -q \ + -c "ALTER SUBSCRIPTION \"${sub}\" ENABLE" || true + done + return 1 + fi + echo " ok $table=$replica" + done + + grant_app_user "$replica_dsn" + + for sub in "${subs[@]}"; do + echo " enabling subscription $sub" + run_psql "$(libpq_dsn "$replica_dsn")" -v ON_ERROR_STOP=1 -q \ + -c "ALTER SUBSCRIPTION \"${sub}\" ENABLE" + run_psql "$(libpq_dsn "$replica_dsn")" -v ON_ERROR_STOP=1 -q \ + -c "ALTER SUBSCRIPTION \"${sub}\" REFRESH PUBLICATION" || true + done + + echo " repair done" +} + +check_replicas() { + local needs_repair=0 + local dsn lbl table pub replica + + for dsn in "${REPLICAS[@]}"; do + lbl="$(label "$dsn")" + echo "== $lbl" + run_psql "$(libpq_dsn "$dsn")" -v ON_ERROR_STOP=1 -At -c \ + "SELECT subname || ' received=' || COALESCE(received_lsn::text, '-') + || ' latest=' || COALESCE(latest_end_lsn::text, '-') + FROM pg_stat_subscription" 2>/dev/null || echo " (no pg_stat_subscription)" + + for table in "${SYNC_TABLES[@]}"; do + pub="$(table_count "$DATABASE_URL" "$table")" + replica="$(table_count "$dsn" "$table")" + if [[ "$pub" == "$replica" ]]; then + echo " $table OK ($pub)" + else + echo " $table DRIFT publisher=$pub replica=$replica" + needs_repair=1 + fi + done + done + + return "$needs_repair" +} + +LOCK_FILE="${REPLICA_DATA_LOCK:-/tmp/superadmin-replica-data.lock}" +if [[ "$MODE" != "--check" ]] && command -v flock >/dev/null; then + exec 9>"$LOCK_FILE" + if ! flock -n 9; then + echo "replica data sync already running — skip" + exit 0 + fi +fi + +if [[ "$MODE" == "--check" ]]; then + check_replicas + exit $? +fi + +if [[ "$MODE" == "--repair-if-needed" ]]; then + if check_replicas; then + echo + echo "All replicas in sync." + exit 0 + fi + echo + echo "Drift detected — repairing..." +fi + +failed=0 +for dsn in "${REPLICAS[@]}"; do + repair_replica "$dsn" || failed=1 +done + +echo +if ((failed == 0)); then + echo "Replica data sync complete." + exit 0 +fi + +echo "Some replicas failed data repair." >&2 +exit 1 diff --git a/deploy/sync-replica-schema.sh b/deploy/sync-replica-schema.sh index 3386540..73b6558 100644 --- a/deploy/sync-replica-schema.sh +++ b/deploy/sync-replica-schema.sh @@ -181,6 +181,30 @@ echo "== publisher" cd "$API_DIR" npx prisma migrate deploy +refresh_subscriptions() { + local dsn="$1" + mapfile -t subs < <( + run_psql "$(libpq_dsn "$dsn")" -v ON_ERROR_STOP=1 -At -c \ + "SELECT subname FROM pg_subscription ORDER BY subname" + ) + if ((${#subs[@]} == 0)); then + echo " ! no subscription — skip REFRESH PUBLICATION" + return 0 + fi + for sub in "${subs[@]}"; do + echo " ALTER SUBSCRIPTION $sub REFRESH PUBLICATION (copy_data)" + # New tables added to FOR ALL TABLES are NOT replicated until this runs. + run_psql "$(libpq_dsn "$dsn")" -v ON_ERROR_STOP=1 -q \ + -c "ALTER SUBSCRIPTION \"${sub}\" REFRESH PUBLICATION WITH (copy_data = true)" + done +} + +echo "== refresh subscriptions (pick up new tables)" +for dsn in "${REPLICAS[@]}"; do + echo "== $(label "$dsn")" + refresh_subscriptions "$dsn" +done + echo echo "Replicas apply DDL first, so the publisher can stream the new columns right away." echo "Verify with: bash deploy/sync-replica-schema.sh --check" diff --git a/package.json b/package.json index 1c476fc..e18e24a 100644 --- a/package.json +++ b/package.json @@ -35,8 +35,9 @@ "prisma:repair-khanegi-ui-config": "ts-node --transpile-only prisma/repair-khanegi-ui-config.ts", "prisma:copy-sms-between-tenants": "ts-node --transpile-only prisma/copy-sms-between-tenants.ts", "prisma:list-tenants": "ts-node --transpile-only prisma/list-tenants.ts", - "replicas:sync": "bash deploy/sync-replica-schema.sh", - "replicas:check": "bash deploy/sync-replica-schema.sh --check", + "replicas:sync": "bash deploy/sync-replica-schema.sh && bash deploy/sync-replica-data.sh --repair-if-needed", + "replicas:check": "bash deploy/sync-replica-schema.sh --check && bash deploy/sync-replica-data.sh --check", + "replicas:repair-data": "bash deploy/sync-replica-data.sh --repair", "settings:seed-templates": "echo 'Templates seed on first GET /api/setting-templates or POST /api/setting-templates/seed'" }, "prisma": { diff --git a/src/app.module.ts b/src/app.module.ts index 3ea413e..2e4e60d 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -16,10 +16,12 @@ import { SupportModule } from './support/support.module'; import { NotificationsModule } from './notifications/notifications.module'; import { AdminsModule } from './admins/admins.module'; import { HealthController } from './health.controller'; +import { ReplicaWatchModule } from './replica-watch/replica-watch.module'; @Module({ imports: [ ConfigModule.forRoot({ isGlobal: true }), + ReplicaWatchModule, PrismaModule, NotificationsModule, AuthModule, diff --git a/src/get-version/get-version.controller.ts b/src/get-version/get-version.controller.ts index c2d65f0..ed93e61 100644 --- a/src/get-version/get-version.controller.ts +++ b/src/get-version/get-version.controller.ts @@ -1,4 +1,4 @@ -import { Controller, Get, Headers, Query } from '@nestjs/common'; +import { Controller, Get, Header, Headers, Query } from '@nestjs/common'; import { GetVersionService } from './get-version.service'; @Controller() @@ -6,6 +6,7 @@ export class GetVersionController { constructor(private readonly getVersionService: GetVersionService) {} @Get('get_version') + @Header('Cache-Control', 'no-store') getVersion( @Headers('host') host?: string, @Headers('x-forwarded-host') forwardedHost?: string, diff --git a/src/program-settings/api-public-program-settings.controller.ts b/src/program-settings/api-public-program-settings.controller.ts index 620a76c..a6a137a 100644 --- a/src/program-settings/api-public-program-settings.controller.ts +++ b/src/program-settings/api-public-program-settings.controller.ts @@ -6,7 +6,7 @@ export class ApiPublicProgramSettingsController { constructor(private readonly programSettings: ProgramSettingsService) {} @Get() - @Header('Cache-Control', 'public, max-age=30') + @Header('Cache-Control', 'no-store') getSettings( @Headers('host') host?: string, @Headers('x-forwarded-host') forwardedHost?: string, diff --git a/src/program-settings/program-settings.service.ts b/src/program-settings/program-settings.service.ts index 74dad8c..4ea4070 100644 --- a/src/program-settings/program-settings.service.ts +++ b/src/program-settings/program-settings.service.ts @@ -14,6 +14,7 @@ import { join } from 'path'; import { Prisma, ProductType, Setting, SettingTemplate } from '@prisma/client'; import { PrismaService } from '../prisma/prisma.service'; import { AuditService } from '../audit/audit.service'; +import { ReplicaWatchService } from '../replica-watch/replica-watch.service'; import { isSecretOrServerOnlySettingKey } from './secret-setting-keys'; import { isRuntimeProgramSettingKey } from './runtime-setting-keys'; import { DEPLOY_SETTING_CATALOG_KEYS } from './deploy-setting-catalog-keys'; @@ -57,6 +58,7 @@ export class ProgramSettingsService implements OnModuleInit { constructor( private readonly prisma: PrismaService, private readonly audit: AuditService, + private readonly replicaWatch: ReplicaWatchService, ) {} async onModuleInit() { @@ -173,6 +175,7 @@ export class ProgramSettingsService implements OnModuleInit { }); const items = await this.listTenantRows(tenantId); + this.replicaWatch.kick(); return { tenant: this.tenantMeta(tenant), items, @@ -320,6 +323,7 @@ export class ProgramSettingsService implements OnModuleInit { }); const items = await this.listTenantRows(tenantId); + this.replicaWatch.kick(); return { tenant: this.tenantMeta(tenant), apiBase, @@ -390,6 +394,7 @@ export class ProgramSettingsService implements OnModuleInit { after: { keys: incoming.map((r) => r.key), tenants: tenants.length } as Prisma.InputJsonValue, }); + this.replicaWatch.kick(); return { keys: incoming.map((r) => r.key), tenantsUpdated: tenants.length, @@ -416,6 +421,7 @@ export class ProgramSettingsService implements OnModuleInit { after: result as Prisma.InputJsonValue, }); + this.replicaWatch.kick(); return { key, ...result }; } diff --git a/src/program-settings/public-program-settings.controller.ts b/src/program-settings/public-program-settings.controller.ts index 46c019e..8475dda 100644 --- a/src/program-settings/public-program-settings.controller.ts +++ b/src/program-settings/public-program-settings.controller.ts @@ -6,7 +6,7 @@ export class PublicProgramSettingsController { constructor(private readonly programSettings: ProgramSettingsService) {} @Get('get_settings') - @Header('Cache-Control', 'public, max-age=30') + @Header('Cache-Control', 'no-store') getSettings( @Headers('host') host?: string, @Headers('x-forwarded-host') forwardedHost?: string, diff --git a/src/replica-watch/replica-watch.module.ts b/src/replica-watch/replica-watch.module.ts new file mode 100644 index 0000000..3cbd284 --- /dev/null +++ b/src/replica-watch/replica-watch.module.ts @@ -0,0 +1,9 @@ +import { Global, Module } from '@nestjs/common'; +import { ReplicaWatchService } from './replica-watch.service'; + +@Global() +@Module({ + providers: [ReplicaWatchService], + exports: [ReplicaWatchService], +}) +export class ReplicaWatchModule {} diff --git a/src/replica-watch/replica-watch.service.ts b/src/replica-watch/replica-watch.service.ts new file mode 100644 index 0000000..e8cd061 --- /dev/null +++ b/src/replica-watch/replica-watch.service.ts @@ -0,0 +1,93 @@ +import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; +import { spawn } from 'child_process'; +import { existsSync } from 'fs'; +import { join } from 'path'; + +/** + * Logical replication streams WAL, it does not re-copy missed history. + * If a replica drifted (new table never REFRESH'd, apply error, …) this + * process heals it on the publisher — no operator command required. + */ +@Injectable() +export class ReplicaWatchService implements OnModuleInit, OnModuleDestroy { + private readonly logger = new Logger(ReplicaWatchService.name); + private timer?: ReturnType; + private running = false; + private pending = false; + + onModuleInit() { + if (!this.enabled()) { + this.logger.log('replica watch idle (no deploy/replicas.env on this host)'); + return; + } + this.logger.log('replica watch on — drift auto-repairs every 60s'); + this.timer = setInterval(() => this.kick(), 60_000); + setTimeout(() => this.kick(), 8_000); + } + + onModuleDestroy() { + if (this.timer) clearInterval(this.timer); + } + + kick() { + if (!this.enabled()) return; + if (this.running) { + this.pending = true; + return; + } + void this.run(); + } + + private enabled() { + return existsSync(this.envFile()) && existsSync(this.scriptPath()); + } + + private envFile() { + return join(process.cwd(), 'deploy/replicas.env'); + } + + private scriptPath() { + return join(process.cwd(), 'deploy/sync-replica-data.sh'); + } + + private run(): Promise { + this.running = true; + return new Promise((resolve) => { + const child = spawn('bash', [this.scriptPath(), '--repair-if-needed'], { + cwd: process.cwd(), + env: process.env, + }); + let out = ''; + child.stdout?.on('data', (buf: Buffer) => { + out += buf.toString(); + }); + child.stderr?.on('data', (buf: Buffer) => { + out += buf.toString(); + }); + child.on('error', (err) => { + this.logger.warn(`replica watch failed to start: ${err.message}`); + this.finish(); + resolve(); + }); + child.on('close', (code) => { + const text = out.trim(); + const healed = /Drift detected|repair done|Replica data sync complete/i.test(text); + if (code !== 0) { + this.logger.warn(`replica watch exit ${code}\n${text.slice(-2000)}`); + } else if (healed) { + this.logger.log(text.slice(-2000) || 'replica data repaired'); + } + this.finish(); + resolve(); + }); + }); + } + + private finish() { + this.running = false; + if (this.pending) { + this.pending = false; + void this.run(); + } + } +} diff --git a/src/tenants/tenants.service.ts b/src/tenants/tenants.service.ts index db2a608..2bd6030 100644 --- a/src/tenants/tenants.service.ts +++ b/src/tenants/tenants.service.ts @@ -8,6 +8,7 @@ import { PrismaService } from '../prisma/prisma.service'; import { AuditService } from '../audit/audit.service'; import { AdminNotifyService } from '../notifications/admin-notify.service'; import { ProgramSettingsService } from '../program-settings/program-settings.service'; +import { ReplicaWatchService } from '../replica-watch/replica-watch.service'; import { CreateTenantDto } from './dto/create-tenant.dto'; import { UpdateTenantDto } from './dto/update-tenant.dto'; @@ -18,6 +19,7 @@ export class TenantsService { private readonly audit: AuditService, private readonly notify: AdminNotifyService, private readonly programSettings: ProgramSettingsService, + private readonly replicaWatch: ReplicaWatchService, ) {} async findAll(product?: ProductType) { @@ -124,6 +126,7 @@ export class TenantsService { `سوپرادمین: نسخه جدید «${tenant.nameFa}» (${tenant.domain}) ایجاد شد.`, ); + this.replicaWatch.kick(); return this.sanitize(tenant); } catch (error) { this.handleUnique(error); @@ -245,6 +248,7 @@ export class TenantsService { : `سوپرادمین: نسخه «${tenant.nameFa}» به‌روزرسانی شد.`, ); + this.replicaWatch.kick(); return this.sanitize(tenant); } catch (error) { this.handleUnique(error); @@ -273,6 +277,7 @@ export class TenantsService { `سوپرادمین: نسخه «${before.nameFa}» (${before.domain}) حذف شد.`, ); + this.replicaWatch.kick(); return { ok: true }; }