feat(replicas): auto-heal slave data drift and no-store public settings
Add sync-replica-data repair on deploy, a publisher-side replica watch (60s + after tenant/settings writes), REFRESH PUBLICATION after migrate, and Cache-Control no-store on program-settings and get_version. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -0,0 +1,9 @@
|
||||
import { Global, Module } from '@nestjs/common';
|
||||
import { ReplicaWatchService } from './replica-watch.service';
|
||||
|
||||
@Global()
|
||||
@Module({
|
||||
providers: [ReplicaWatchService],
|
||||
exports: [ReplicaWatchService],
|
||||
})
|
||||
export class ReplicaWatchModule {}
|
||||
@@ -0,0 +1,93 @@
|
||||
import { Injectable, Logger, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
|
||||
import { spawn } from 'child_process';
|
||||
import { existsSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
|
||||
/**
|
||||
* Logical replication streams WAL, it does not re-copy missed history.
|
||||
* If a replica drifted (new table never REFRESH'd, apply error, …) this
|
||||
* process heals it on the publisher — no operator command required.
|
||||
*/
|
||||
@Injectable()
|
||||
export class ReplicaWatchService implements OnModuleInit, OnModuleDestroy {
|
||||
private readonly logger = new Logger(ReplicaWatchService.name);
|
||||
private timer?: ReturnType<typeof setInterval>;
|
||||
private running = false;
|
||||
private pending = false;
|
||||
|
||||
onModuleInit() {
|
||||
if (!this.enabled()) {
|
||||
this.logger.log('replica watch idle (no deploy/replicas.env on this host)');
|
||||
return;
|
||||
}
|
||||
this.logger.log('replica watch on — drift auto-repairs every 60s');
|
||||
this.timer = setInterval(() => this.kick(), 60_000);
|
||||
setTimeout(() => this.kick(), 8_000);
|
||||
}
|
||||
|
||||
onModuleDestroy() {
|
||||
if (this.timer) clearInterval(this.timer);
|
||||
}
|
||||
|
||||
kick() {
|
||||
if (!this.enabled()) return;
|
||||
if (this.running) {
|
||||
this.pending = true;
|
||||
return;
|
||||
}
|
||||
void this.run();
|
||||
}
|
||||
|
||||
private enabled() {
|
||||
return existsSync(this.envFile()) && existsSync(this.scriptPath());
|
||||
}
|
||||
|
||||
private envFile() {
|
||||
return join(process.cwd(), 'deploy/replicas.env');
|
||||
}
|
||||
|
||||
private scriptPath() {
|
||||
return join(process.cwd(), 'deploy/sync-replica-data.sh');
|
||||
}
|
||||
|
||||
private run(): Promise<void> {
|
||||
this.running = true;
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn('bash', [this.scriptPath(), '--repair-if-needed'], {
|
||||
cwd: process.cwd(),
|
||||
env: process.env,
|
||||
});
|
||||
let out = '';
|
||||
child.stdout?.on('data', (buf: Buffer) => {
|
||||
out += buf.toString();
|
||||
});
|
||||
child.stderr?.on('data', (buf: Buffer) => {
|
||||
out += buf.toString();
|
||||
});
|
||||
child.on('error', (err) => {
|
||||
this.logger.warn(`replica watch failed to start: ${err.message}`);
|
||||
this.finish();
|
||||
resolve();
|
||||
});
|
||||
child.on('close', (code) => {
|
||||
const text = out.trim();
|
||||
const healed = /Drift detected|repair done|Replica data sync complete/i.test(text);
|
||||
if (code !== 0) {
|
||||
this.logger.warn(`replica watch exit ${code}\n${text.slice(-2000)}`);
|
||||
} else if (healed) {
|
||||
this.logger.log(text.slice(-2000) || 'replica data repaired');
|
||||
}
|
||||
this.finish();
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private finish() {
|
||||
this.running = false;
|
||||
if (this.pending) {
|
||||
this.pending = false;
|
||||
void this.run();
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user