feat(import): henza → gold-bank data import tool

apps/api/scripts/legacy-import/ reads the legacy MySQL (read only) and
merges into the target Postgres. Every rule is from
docs/DATA-MIGRATION-MAP.md, and each step cites its section. It covers
cities, roles and permissions, branches, admin users, sales experts,
workshops, customers, brands, categories, products, variants, product
images, orders, the ledger, favourites, custom orders and a
reconciliation report.

Key rules:
- Debt is rebuilt from legacy's own formula (updateRemain): delivered
  orders minus confirmed type 1-4 documents. Type 9 «سفارش» documents
  mirror orders and are skipped, or every order's debt would count twice.
  A balance the records cannot explain is carried as one ADJUSTMENT entry.
  In the 2026-10-04 dump that is only customer #125 (owner ruling).
- Merge, not import: rows already in the target are matched by mobile or
  product code, and only their empty fields are filled. Never written:
  accounting* columns, ProductVariant.imageId, Accounting* tables. No
  ProductImage is ever deleted.
- Roles map by title, not number. Legacy 86 is «حسابدار» but target 86
  is «کارشناس فروش». An expert sharing a mobile with an admin never
  demotes the admin.
- Soft-deleted variants and products that live order lines still point
  at (473 of 610 lines) come over inactive, so order history stays whole.
- Owner rulings 2026-10-07: birthdays filtered (541 of 915), typed city
  kept plus exact-match cityId, the 7 legacy rejections get the sentinel
  reason, OTP-only accounts get OTP_ONLY_PASSWORD_HASH.
- Tehran wall-clock timestamps are converted with ICU rules, including
  Iran's pre-2022 DST. Jalali receipt dates are converted with the
  jalaali-js arithmetic (truncating division; verified on 8 known dates).

Safety: --target is required and never read from .env. Nothing is
written without --yes, and a target not named *import*/*staging* also
needs --confirm-db. The id map lives in its own schema, legacy_import,
so a re-run updates instead of duplicating. Writes bypass the API
services, so no SMS is sent and nothing is queued to accounting.

Tested against goldbank_import_test, a clone of the dev DB, in three
runs:
- 1,503/1,503 created customers match legacy gold and rial balances
  (±0.0001 g / ±1 rial)
- accounting data is byte-identical before and after
- 0 imported variants have an imageId
- runs 2 and 3 created zero duplicate rows
- 6,196 image files copied, 3 missing in the legacy storage

New devDependency: mysql2 ^3.24.5.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
s.esmaeili
2026-10-07 19:35:18 +03:30
co-authored by Claude Opus 5.5
parent 7019cad932
commit be5ca829d2
10 changed files with 2035 additions and 0 deletions
@@ -0,0 +1,327 @@
/**
* Shared state for one import run: both connections, the id map and the report.
*/
import { copyFileSync, existsSync, mkdirSync, statSync } from 'node:fs';
import { join } from 'node:path';
import { PrismaClient } from '@prisma/client';
import { createPool, type Pool, type RowDataPacket } from 'mysql2/promise';
export interface Options {
sourceUrl: string;
targetUrl: string;
/** henza `storage/` folder holding the image files (optional). */
imagesDir?: string;
/** Target UPLOAD_DIR the files are copied into (required with imagesDir). */
uploadDir?: string;
/** Run only these steps (by name); all when unset. */
only?: Set<string>;
reportPath: string;
}
/* ───────────── report ───────────── */
interface Section {
counts: Map<string, number>;
notes: string[];
}
/**
* What the run did, step by step. Counts say how many rows went which way;
* notes are the things a person must look at. Rendered to markdown at the end.
*/
export class Report {
readonly startedAt = new Date();
private readonly sections = new Map<string, Section>();
private section(step: string): Section {
let s = this.sections.get(step);
if (!s) {
s = { counts: new Map(), notes: [] };
this.sections.set(step, s);
}
return s;
}
count(step: string, key: string, n = 1) {
const s = this.section(step);
s.counts.set(key, (s.counts.get(key) ?? 0) + n);
}
note(step: string, message: string) {
this.section(step).notes.push(message);
}
get(step: string, key: string): number {
return this.sections.get(step)?.counts.get(key) ?? 0;
}
render(header: string[]): string {
const out: string[] = ['# henza → gold-bank import report', '', ...header, ''];
for (const [step, s] of this.sections) {
out.push(`## ${step}`, '');
if (s.counts.size) {
out.push('| | |', '|---|---:|');
for (const [k, v] of s.counts) out.push(`| ${k} | ${v} |`);
out.push('');
}
for (const n of s.notes) out.push(`- ${n}`);
if (s.notes.length) out.push('');
}
return out.join('\n');
}
}
/* ───────────── id map ───────────── */
/**
* Legacy id -> target id, per entity, persisted in the TARGET database under
* its own schema `legacy_import` — outside the `public` schema Prisma manages,
* so it is no app-schema change and drops cleanly after cutover
* (`DROP SCHEMA legacy_import CASCADE`).
*
* It is what makes a re-run safe (a mapped row is updated, never re-created)
* and what later redirects old links such as /product/123 (GO-LIVE-PLAN §7).
* `how` records whether the row was created by the import or matched to a row
* that already existed in the target (merge, §10.5).
*/
export type How = 'created' | 'matched' | 'merged';
export class IdMap {
private readonly cache = new Map<string, Map<string, { target: string; how: How }>>();
constructor(private readonly db: PrismaClient) {}
static async ensureSchema(db: PrismaClient) {
await db.$executeRawUnsafe(`CREATE SCHEMA IF NOT EXISTS legacy_import`);
await db.$executeRawUnsafe(`
CREATE TABLE IF NOT EXISTS legacy_import.id_map (
entity text NOT NULL,
legacy_id text NOT NULL,
target_id text NOT NULL,
how text NOT NULL,
mapped_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (entity, legacy_id)
)`);
}
async load(entity: string) {
const rows = await this.db.$queryRawUnsafe<{ legacy_id: string; target_id: string; how: How }[]>(
`SELECT legacy_id, target_id, how FROM legacy_import.id_map WHERE entity = $1`,
entity,
);
this.cache.set(entity, new Map(rows.map((r) => [r.legacy_id, { target: r.target_id, how: r.how }])));
}
get(entity: string, legacyId: unknown): string | undefined {
if (legacyId === null || legacyId === undefined) return undefined;
return this.cache.get(entity)?.get(String(legacyId))?.target;
}
how(entity: string, legacyId: unknown): How | undefined {
return this.cache.get(entity)?.get(String(legacyId))?.how;
}
async set(entity: string, legacyId: unknown, targetId: string, how: How) {
const key = String(legacyId);
await this.db.$executeRawUnsafe(
`INSERT INTO legacy_import.id_map (entity, legacy_id, target_id, how)
VALUES ($1, $2, $3, $4)
ON CONFLICT (entity, legacy_id) DO UPDATE SET target_id = EXCLUDED.target_id, how = EXCLUDED.how`,
entity, key, targetId, how,
);
let m = this.cache.get(entity);
if (!m) {
m = new Map();
this.cache.set(entity, m);
}
m.set(key, { target: targetId, how });
}
async drop(entity: string, legacyId: unknown) {
await this.db.$executeRawUnsafe(
`DELETE FROM legacy_import.id_map WHERE entity = $1 AND legacy_id = $2`,
entity, String(legacyId),
);
this.cache.get(entity)?.delete(String(legacyId));
}
}
/* ───────────── upsert discipline ───────────── */
/**
* The one way a step writes a row (§10.5: this is a MERGE into a live target):
*
* - already mapped -> `update(id, how)`; the step decides what it may touch
* - matches a target row -> mapped as 'matched', then `update(id, 'matched')`
* - otherwise -> `create()`, mapped as 'created'
*
* A mapping whose target row has since been deleted is dropped and redone, so a
* re-run heals instead of failing on a dangling id.
*/
export async function upsert(
ctx: Ctx,
entity: string,
legacyId: unknown,
ops: {
exists: (id: string) => Promise<boolean>;
find: () => Promise<{ id: string } | null>;
create: () => Promise<{ id: string }>;
update: (id: string, how: How) => Promise<void>;
},
): Promise<{ id: string; how: How; isNew: boolean }> {
const mapped = ctx.map.get(entity, legacyId);
if (mapped) {
if (await ops.exists(mapped)) {
const how = ctx.map.how(entity, legacyId) ?? 'created';
await ops.update(mapped, how);
return { id: mapped, how, isNew: false };
}
await ctx.map.drop(entity, legacyId);
}
const found = await ops.find();
if (found) {
await ctx.map.set(entity, legacyId, found.id, 'matched');
await ops.update(found.id, 'matched');
return { id: found.id, how: 'matched', isNew: false };
}
const created = await ops.create();
await ctx.map.set(entity, legacyId, created.id, 'created');
return { id: created.id, how: 'created', isNew: true };
}
/**
* For a row that already existed in the target (`matched`): only the fields
* the target left empty are filled. The target's own values win — the owner
* confirmed its data is real (§10.5).
*/
export function fillEmpty<T extends Record<string, unknown>>(current: Record<string, unknown>, patch: T): Partial<T> {
const out: Partial<T> = {};
for (const [k, v] of Object.entries(patch)) {
const cur = current[k];
if ((cur === null || cur === undefined || cur === '') && v !== null && v !== undefined) {
(out as Record<string, unknown>)[k] = v;
}
}
return out;
}
/* ───────────── context ───────────── */
export interface Ctx {
src: Pool;
db: PrismaClient;
map: IdMap;
report: Report;
opt: Options;
/** A read-only query against the legacy MySQL. */
q<T = Record<string, unknown>>(sql: string, params?: unknown[]): Promise<T[]>;
/** Public URL for a legacy `image` row, copying its file when a source dir is given. */
imageUrl(imageId: unknown, step: string): Promise<string | null>;
/**
* An `UploadedFile` row for a legacy image — what avatars point at
* (Customer/AdminUser.avatarFileId are FKs, not URLs). Mapped as 'file'.
*/
uploadedFile(imageId: unknown, purpose: string, step: string): Promise<string | null>;
}
const MIME: Record<string, string> = {
png: 'image/png', jpg: 'image/jpeg', jpeg: 'image/jpeg', gif: 'image/gif',
webp: 'image/webp', svg: 'image/svg+xml', pdf: 'application/pdf',
};
export function openSource(url: string): Pool {
return createPool({
uri: url,
connectionLimit: 4,
charset: 'utf8mb4',
// Timestamps come back as the wall-clock text legacy stored, and are
// converted from Tehran time explicitly (normalize.tehran). The session
// time zone is left at the server's own, which is what Laravel and the
// Navicat dump both used.
dateStrings: true,
supportBigNumbers: true,
bigNumberStrings: true,
});
}
export function makeContext(opt: Options, src: Pool, db: PrismaClient): Ctx {
const map = new IdMap(db);
const report = new Report();
const imageRows = new Map<string, string | null>();
const ctx: Ctx = {
src,
db,
map,
report,
opt,
async q<T>(sql: string, params: unknown[] = []) {
const [rows] = await src.query<RowDataPacket[]>(sql, params);
return rows as unknown as T[];
},
async imageUrl(imageId: unknown, step: string) {
if (imageId === null || imageId === undefined || imageId === '') return null;
const key = String(imageId);
if (!imageRows.has(key)) {
const [row] = await ctx.q<{ image: string | null }>('SELECT image FROM image WHERE id = ?', [key]);
imageRows.set(key, row?.image ? String(row.image).trim() : null);
}
const file = imageRows.get(key);
if (!file) {
report.count(step, 'image reference with no `image` row');
return null;
}
copyImage(ctx, file, step);
// Same shape the admin upload writes (admin-uploads.service.ts:61).
return `/uploads/${file}`;
},
async uploadedFile(imageId: unknown, purpose: string, step: string) {
if (imageId === null || imageId === undefined || imageId === '') return null;
// Resolve first even when mapped: resolving copies the file (see productImages).
const url = await ctx.imageUrl(imageId, step);
const mapped = map.get('file', imageId);
if (mapped) return mapped;
if (!url) return null;
const [row] = await ctx.q<{ original_name: string | null; ext: string | null }>(
'SELECT original_name, ext FROM image WHERE id = ?', [String(imageId)],
);
const file = url.slice('/uploads/'.length);
const ext = (row?.ext ?? file.split('.').pop() ?? '').toLowerCase().trim();
const src = ctx.opt.imagesDir ? join(ctx.opt.imagesDir, file) : null;
const created = await db.uploadedFile.create({
data: {
url,
originalName: (row?.original_name ?? '').trim() || file,
mimeType: MIME[ext] ?? 'application/octet-stream',
sizeBytes: src && existsSync(src) ? statSync(src).size : 0,
purpose,
},
});
await map.set('file', imageId, created.id, 'created');
return created.id;
},
};
return ctx;
}
const copied = new Set<string>();
function copyImage(ctx: Ctx, file: string, step: string) {
const { imagesDir, uploadDir } = ctx.opt;
if (!imagesDir || !uploadDir || copied.has(file)) return;
copied.add(file);
const from = join(imagesDir, file);
const to = join(uploadDir, file);
if (existsSync(to)) {
ctx.report.count('images', 'already in upload dir');
return;
}
if (!existsSync(from) || !statSync(from).isFile()) {
ctx.report.count('images', 'missing in henza storage');
ctx.report.note('images', `missing file ${file} (referenced from ${step})`);
return;
}
mkdirSync(uploadDir, { recursive: true });
copyFileSync(from, to);
ctx.report.count('images', 'copied');
}
@@ -0,0 +1,204 @@
/**
* Value conversion from the henza MySQL schema to the Prisma one.
*
* Every helper here is pure, so the rules can be read and tested in isolation.
* Each rule is the one recorded in docs/DATA-MIGRATION-MAP.md.
*/
import { Prisma } from '@prisma/client';
const PERSIAN_DIGITS = '۰۱۲۳۴۵۶۷۸۹';
const ARABIC_DIGITS = '٠١٢٣٤٥٦٧٨٩';
/** Persian/Arabic-Indic digits -> ASCII. Legacy did the same (Utility::faTOen). */
export function latinDigits(s: string): string {
return s.replace(/[۰-۹٠-٩]/g, (d) => {
const i = PERSIAN_DIGITS.indexOf(d);
return String(i >= 0 ? i : ARABIC_DIGITS.indexOf(d));
});
}
/**
* Persian text as stored: trimmed, single spaces, and Arabic ي/ك folded to
* Persian ی/ک. Empty -> null, so '' never reaches a nullable target column.
*/
export function text(v: unknown): string | null {
if (v === null || v === undefined) return null;
const s = String(v)
.replace(/ي/g, 'ی')
.replace(/ك/g, 'ک')
.replace(/\s+/g, ' ')
.trim();
return s === '' ? null : s;
}
/**
* Comparison key for matching names across systems. Also drops the zero-width
* non-joiner, which legacy typed inconsistently ("مدیر اصلی" vs "مدیراصلی").
* The ي/ك fold alone raised exact city matches from 32 to 72 (§10.4).
*/
export function matchKey(v: unknown): string | null {
const t = text(v);
return t === null ? null : t.replace(/‌/g, '').replace(/\s/g, '').toLowerCase();
}
/** Mobile as the login key: ASCII digits only, or null. */
export function mobile(v: unknown): string | null {
if (v === null || v === undefined) return null;
const d = latinDigits(String(v)).replace(/\D/g, '');
return d === '' ? null : d;
}
function num(v: unknown): number {
if (v === null || v === undefined || v === '') return 0;
const n = Number(v);
return Number.isFinite(n) ? n : 0;
}
/**
* Legacy stores money and weight as `double` (§4). Going through toFixed at the
* target scale drops the binary noise (12.3 is really 12.30000000000000071…)
* exactly as Decimal(14,4) would, so a reconciliation can compare like with like.
*/
export const grams = (v: unknown) => new Prisma.Decimal(num(v).toFixed(4));
export const rial = (v: unknown) => new Prisma.Decimal(num(v).toFixed(2));
export const percent = (v: unknown) => new Prisma.Decimal(num(v).toFixed(3));
export const isZero = (v: unknown) => Math.abs(num(v)) < 0.00005;
export const toNumber = num;
/** Integer or null — for tinyint flags and ids that may arrive as strings. */
export function int(v: unknown): number | null {
if (v === null || v === undefined || v === '') return null;
const n = Number(v);
return Number.isInteger(n) ? n : null;
}
/* ───────────── time ───────────── */
/**
* The Laravel app ran with timezone 'Asia/Tehran' (henza_server config/app.php:70),
* so every timestamp is Tehran wall-clock time. Read as a UTC instant it would
* be 3:30 (4:30 under the pre-2022 summer time) off, moving evening documents
* to the next day on a statement. The ICU zone rules know Iran's historical
* DST, so the offset is taken from them per instant rather than hard-coded.
*/
const tehranFormat = new Intl.DateTimeFormat('en-US', {
timeZone: 'Asia/Tehran',
hourCycle: 'h23',
year: 'numeric',
month: '2-digit',
day: '2-digit',
hour: '2-digit',
minute: '2-digit',
second: '2-digit',
});
function tehranOffsetMs(utcMs: number): number {
const p: Record<string, string> = {};
for (const part of tehranFormat.formatToParts(new Date(utcMs))) p[part.type] = part.value;
const asUtc = Date.UTC(+p.year, +p.month - 1, +p.day, +p.hour, +p.minute, +p.second);
return asUtc - utcMs;
}
const WALL = /^(\d{4})-(\d{2})-(\d{2})(?:[ T](\d{2}):(\d{2}):(\d{2}))?/;
/** 'YYYY-MM-DD HH:MM:SS' in Tehran wall-clock time -> the UTC instant. */
export function tehran(v: unknown): Date | null {
if (v === null || v === undefined || v === '') return null;
const m = WALL.exec(String(v));
if (!m || m[1] === '0000') return null;
const wall = Date.UTC(+m[1], +m[2] - 1, +m[3], +(m[4] ?? 0), +(m[5] ?? 0), +(m[6] ?? 0));
// Two passes: the offset at the guessed instant can differ across a DST edge.
let utc = wall - tehranOffsetMs(wall);
utc = wall - tehranOffsetMs(utc);
return new Date(utc);
}
/**
* Birthdays (owner ruling 2026-10-07, §10.7): of 915 set values only 541 are
* real. Dropped: the sign-up default (birth date == sign-up date, 190 rows) and
* the impossible ones (a time of day, or born after 2008; 184 rows). A
* birthday is a calendar date, so it is kept as UTC midnight of that date.
*/
export function birthday(birth: unknown, signup: unknown): Date | null {
const b = WALL.exec(String(birth ?? ''));
if (!b) return null;
if ((b[4] ?? '00') !== '00' || (b[5] ?? '00') !== '00' || (b[6] ?? '00') !== '00') return null;
if (+b[1] > 2008 || +b[1] < 1900) return null;
const s = WALL.exec(String(signup ?? ''));
if (s && s[1] === b[1] && s[2] === b[2] && s[3] === b[3]) return null;
return new Date(Date.UTC(+b[1], +b[2] - 1, +b[3]));
}
/* ───────────── Jalali dates ───────────── */
/**
* `cashes.recive_date` is a varchar holding a Jalali date ("1403/05/12"; 88 of
* 89 values start with 14). This is the standard jalaali-js arithmetic
* (Borkowski), inlined so the tool needs no extra dependency.
*
* Its integer division TRUNCATES toward zero. Math.floor is not a substitute:
* g2d's (gm - 8) / 6 is negative for March, and flooring it shifted every
* converted date by a year.
*/
const div = (a: number, b: number) => Math.trunc(a / b);
const mod = (a: number, b: number) => a - Math.trunc(a / b) * b;
function jalCal(jy: number) {
const breaks = [-61, 9, 38, 199, 426, 686, 756, 818, 1111, 1181, 1210, 1635, 2060, 2097, 2192, 2262, 2324, 2394, 2456, 3178];
const gy = jy + 621;
let leapJ = -14;
let jp = breaks[0];
let jump = 0;
for (let i = 1; i < breaks.length; i++) {
const jm = breaks[i];
jump = jm - jp;
if (jy < jm) break;
leapJ += div(jump, 33) * 8 + div(mod(jump, 33), 4);
jp = jm;
}
const n = jy - jp;
leapJ += div(n, 33) * 8 + div(mod(n, 33) + 3, 4);
if (mod(jump, 33) === 4 && jump - n === 4) leapJ += 1;
const leapG = div(gy, 4) - div((div(gy, 100) + 1) * 3, 4) - 150;
return { gy, march: 20 + leapJ - leapG };
}
function g2d(gy: number, gm: number, gd: number) {
const d =
div((gy + div(gm - 8, 6) + 100100) * 1461, 4) +
div(153 * mod(gm + 9, 12) + 2, 5) +
gd - 34840408;
return d - div(div(gy + 100100 + div(gm - 8, 6), 100) * 3, 4) + 752;
}
function d2g(jdn: number) {
let j = 4 * jdn + 139361631;
j = j + div(div(4 * jdn + 183187720, 146097) * 3, 4) * 4 - 3908;
const i = div(mod(j, 1461), 4) * 5 + 308;
const gd = div(mod(i, 153), 5) + 1;
const gm = mod(div(i, 153), 12) + 1;
const gy = div(j, 1461) - 100100 + div(8 - gm, 6);
return { gy, gm, gd };
}
/** Jalali y/m/d -> Gregorian y/m/d. */
export function jalaliToGregorian(jy: number, jm: number, jd: number) {
const r = jalCal(jy);
const jdn = g2d(r.gy, 3, r.march) + (jm - 1) * 31 - div(jm, 7) * (jm - 7) + jd - 1;
return d2g(jdn);
}
/**
* A free-text Jalali date ("1403/5/12", "۱۴۰۳-۰۵-۱۲") -> UTC midnight of the
* same day, or null when it does not parse. Callers keep the raw text too.
*/
export function jalaliDate(v: unknown): Date | null {
const s = latinDigits(String(v ?? '')).trim();
const m = /^(1[34]\d{2})\D(\d{1,2})\D(\d{1,2})$/.exec(s);
if (!m) return null;
const jm = +m[2];
const jd = +m[3];
if (jm < 1 || jm > 12 || jd < 1 || jd > 31) return null;
const g = jalaliToGregorian(+m[1], jm, jd);
return new Date(Date.UTC(g.gy, g.gm - 1, g.gd));
}