feat(replicas): auto-heal slave data drift and no-store public settings

Add sync-replica-data repair on deploy, a publisher-side replica watch
(60s + after tenant/settings writes), REFRESH PUBLICATION after migrate,
and Cache-Control no-store on program-settings and get_version.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-08-18 12:32:30 +03:30
co-authored by Cursor
parent 668bbcc1db
commit 16d562caa5
13 changed files with 456 additions and 11 deletions
@@ -0,0 +1,9 @@
import { Global, Module } from '@nestjs/common';
import { ReplicaWatchService } from './replica-watch.service';
@Global()
@Module({
providers: [ReplicaWatchService],
exports: [ReplicaWatchService],
})
export class ReplicaWatchModule {}
@@ -0,0 +1,93 @@
import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
import { spawn } from 'child_process';
import { existsSync } from 'fs';
import { join } from 'path';
/**
* Logical replication streams WAL, it does not re-copy missed history.
* If a replica drifted (new table never REFRESH'd, apply error, …) this
* process heals it on the publisher — no operator command required.
*/
@Injectable()
export class ReplicaWatchService implements OnModuleInit, OnModuleDestroy {
private readonly logger = new Logger(ReplicaWatchService.name);
private timer?: ReturnType<typeof setInterval>;
private running = false;
private pending = false;
onModuleInit() {
if (!this.enabled()) {
this.logger.log('replica watch idle (no deploy/replicas.env on this host)');
return;
}
this.logger.log('replica watch on — drift auto-repairs every 60s');
this.timer = setInterval(() => this.kick(), 60_000);
setTimeout(() => this.kick(), 8_000);
}
onModuleDestroy() {
if (this.timer) clearInterval(this.timer);
}
kick() {
if (!this.enabled()) return;
if (this.running) {
this.pending = true;
return;
}
void this.run();
}
private enabled() {
return existsSync(this.envFile()) && existsSync(this.scriptPath());
}
private envFile() {
return join(process.cwd(), 'deploy/replicas.env');
}
private scriptPath() {
return join(process.cwd(), 'deploy/sync-replica-data.sh');
}
private run(): Promise<void> {
this.running = true;
return new Promise((resolve) => {
const child = spawn('bash', [this.scriptPath(), '--repair-if-needed'], {
cwd: process.cwd(),
env: process.env,
});
let out = '';
child.stdout?.on('data', (buf: Buffer) => {
out += buf.toString();
});
child.stderr?.on('data', (buf: Buffer) => {
out += buf.toString();
});
child.on('error', (err) => {
this.logger.warn(`replica watch failed to start: ${err.message}`);
this.finish();
resolve();
});
child.on('close', (code) => {
const text = out.trim();
const healed = /Drift detected|repair done|Replica data sync complete/i.test(text);
if (code !== 0) {
this.logger.warn(`replica watch exit ${code}\n${text.slice(-2000)}`);
} else if (healed) {
this.logger.log(text.slice(-2000) || 'replica data repaired');
}
this.finish();
resolve();
});
});
}
private finish() {
this.running = false;
if (this.pending) {
this.pending = false;
void this.run();
}
}
}