From c629729a9af178e86f536b5775b48b951908ef11 Mon Sep 17 00:00:00 2001 From: Mahdi Date: Sat, 3 Oct 2026 17:42:11 +0330 Subject: [PATCH] Allow large activity Excel imports (40k+ rows) with bigger chunks and caches. Co-authored-by: Cursor --- .../activities/activities-excel.service.ts | 47 +++++++++++++++---- apps/api/src/common/excel-import-session.ts | 34 +++++++++++++- apps/api/src/common/excel.ts | 12 +---- apps/web/src/components/ExcelImportFlow.vue | 25 +++++++++- 4 files changed, 96 insertions(+), 22 deletions(-) diff --git a/apps/api/src/activities/activities-excel.service.ts b/apps/api/src/activities/activities-excel.service.ts index 9c8d9de..71c78d1 100644 --- a/apps/api/src/activities/activities-excel.service.ts +++ b/apps/api/src/activities/activities-excel.service.ts @@ -18,6 +18,8 @@ import { } from '../common/excel'; import { chunkImportMeta, + destroyExcelImportSession, + getExcelImportSessionCaches, resolveExcelImportChunk, } from '../common/excel-import-session'; @@ -184,22 +186,49 @@ export class ActivitiesExcelService { return result; } - const MAX_ROWS = 25000; + const MAX_ROWS = 100_000; if (ctx.total > MAX_ROWS) { + destroyExcelImportSession(ctx.sessionId); result.errors.push({ row: 1, - message: `فایل بیش از ${MAX_ROWS} ردیف دارد.`, + message: `فایل بیش از ${MAX_ROWS} ردیف دارد. فایل را تقسیم کنید.`, }); - return result; + return { ...result, hasMore: false, total: ctx.total, processed: 0 }; } - const userCache = new Map(); + const caches = getExcelImportSessionCaches(ctx.sessionId, actor.organizationId); const resolveUser = async (name?: string) => { const key = (name ?? '').trim().toLowerCase(); if (!key) return null; - if (userCache.has(key)) return userCache.get(key) ?? null; + if (caches.user.has(key)) return caches.user.get(key) ?? null; const id = await this.findUserId(actor.organizationId, name); - userCache.set(key, id); + caches.user.set(key, id); + return id; + }; + const resolvePerson = async (row: Record) => { + const mobile = normalizePhone(row.personMobile); + const name = (row.personName ?? '').trim().toLowerCase(); + const key = mobile ? `m:${mobile}` : name ? `n:${name}` : ''; + if (!key) return null; + if (caches.person.has(key)) return caches.person.get(key) ?? null; + const id = await this.findPersonId(actor.organizationId, row); + caches.person.set(key, id); + return id; + }; + const resolveCompany = async (name?: string) => { + const key = (name ?? '').trim().toLowerCase(); + if (!key) return null; + if (caches.company.has(key)) return caches.company.get(key) ?? null; + const id = await this.findCompanyId(actor.organizationId, name); + caches.company.set(key, id); + return id; + }; + const resolveDeal = async (title?: string) => { + const key = (title ?? '').trim().toLowerCase(); + if (!key) return null; + if (caches.deal.has(key)) return caches.deal.get(key) ?? null; + const id = await this.findDealId(actor.organizationId, title); + caches.deal.set(key, id); return id; }; @@ -223,9 +252,9 @@ export class ActivitiesExcelService { const status = parseStatus(row.status); try { - const personId = await this.findPersonId(actor.organizationId, row); - const companyId = await this.findCompanyId(actor.organizationId, row.companyName); - const dealId = await this.findDealId(actor.organizationId, row.dealTitle); + const personId = await resolvePerson(row); + const companyId = await resolveCompany(row.companyName); + const dealId = await resolveDeal(row.dealTitle); const assigneeNames = String(row.assignee ?? '') .split(/[,،]/) .map((s) => s.trim()) diff --git a/apps/api/src/common/excel-import-session.ts b/apps/api/src/common/excel-import-session.ts index 70ab7c1..9083a5d 100644 --- a/apps/api/src/common/excel-import-session.ts +++ b/apps/api/src/common/excel-import-session.ts @@ -1,16 +1,24 @@ import { randomBytes } from 'node:crypto'; import { BadRequestException } from '@nestjs/common'; -export const EXCEL_IMPORT_CHUNK_SIZE = 80; +export const EXCEL_IMPORT_CHUNK_SIZE = 150; /** بعد از این تعداد ردیف، کلاینت حتماً چندمرحله‌ای می‌رود (پیش‌نمایش هم نشان می‌دهد) */ export const EXCEL_IMPORT_LARGE_THRESHOLD = 80; const SESSION_TTL_MS = 45 * 60 * 1000; const MAX_SESSIONS = 40; +export type ExcelImportLookupCaches = { + user: Map; + person: Map; + company: Map; + deal: Map; +}; + type ExcelImportSession = { organizationId: string; rows: Array>; expiresAt: number; + caches: ExcelImportLookupCaches; }; const sessions = new Map(); @@ -40,6 +48,12 @@ export function createExcelImportSession( organizationId, rows, expiresAt: Date.now() + SESSION_TTL_MS, + caches: { + user: new Map(), + person: new Map(), + company: new Map(), + deal: new Map(), + }, }); return id; } @@ -64,6 +78,24 @@ export function destroyExcelImportSession(sessionId: string | undefined) { sessions.delete(sessionId); } +export function getExcelImportSessionCaches( + sessionId: string, + organizationId: string, +): ExcelImportLookupCaches { + purgeExpired(); + const session = sessions.get(sessionId); + if (!session || session.organizationId !== organizationId) { + return { + user: new Map(), + person: new Map(), + company: new Map(), + deal: new Map(), + }; + } + session.expiresAt = Date.now() + SESSION_TTL_MS; + return session.caches; +} + export function parseChunkParams(raw: { offset?: string | number; limit?: string | number; diff --git a/apps/api/src/common/excel.ts b/apps/api/src/common/excel.ts index ae8132a..98216aa 100644 --- a/apps/api/src/common/excel.ts +++ b/apps/api/src/common/excel.ts @@ -319,16 +319,8 @@ export async function previewExcel( const missingRequired = required.some((key) => !mapped.has(key)); const unrecognized = columns.some((col) => !suggested[col.header]); - let rowCount = 0; - sheet.eachRow((row, rowNumber) => { - if (rowNumber === 1) return; - let any = false; - row.eachCell((cell) => { - if (any) return; - if (cellStr(cell.value)) any = true; - }); - if (any) rowCount += 1; - }); + // برای فایل‌های خیلی بزرگ (ده‌ها هزار ردیف) پیمایش کامل کند است؛ rowCount شیت کافی است + const rowCount = Math.max(0, (sheet.actualRowCount || sheet.rowCount || 1) - 1); return { columns, diff --git a/apps/web/src/components/ExcelImportFlow.vue b/apps/web/src/components/ExcelImportFlow.vue index 7daaf49..0f35103 100644 --- a/apps/web/src/components/ExcelImportFlow.vue +++ b/apps/web/src/components/ExcelImportFlow.vue @@ -103,9 +103,11 @@ async function postImportChunk(opts: { } if (opts.sessionId) form.append('sessionId', opts.sessionId); if (opts.offset != null) form.append('offset', String(opts.offset)); + form.append('limit', '250'); const { data } = await api.post(props.importPath, form, { headers: { 'Content-Type': undefined }, - timeout: 3 * 60 * 1000, + // خواندن فایل‌های خیلی بزرگ در مرحلهٔ اول ممکن است طول بکشد + timeout: opts.file ? 10 * 60 * 1000 : 3 * 60 * 1000, }); return data; } @@ -118,8 +120,9 @@ async function runChunkedImport(file: File, map?: Record) { let sessionId: string | undefined; let offset = 0; let guard = 0; + const maxChunks = 2000; - while (guard < 500) { + while (guard < maxChunks) { guard += 1; const part = await postImportChunk({ file: sessionId ? undefined : file, @@ -135,11 +138,28 @@ async function runChunkedImport(file: File, map?: Record) { progressLabel.value = `ورود چندمرحله‌ای: ${processed.toLocaleString('fa-IR')} از ${total.toLocaleString('fa-IR')} ردیف`; } + // خطای سراسری (مثلاً سقف ردیف) — ادامه نده + if ( + part.errors?.some((e) => e.row === 1 && /بیش از .* ردیف/.test(e.message)) + ) { + break; + } + if (!part.hasMore || !part.sessionId) break; sessionId = part.sessionId; offset = part.nextOffset ?? processed; } + if (guard >= maxChunks && acc.hasMore) { + acc.errors = [ + ...acc.errors, + { + row: 1, + message: 'ورود به‌خاطر تعداد مراحل زیاد متوقف شد. از جایی که مانده ادامه دهید یا فایل را تقسیم کنید.', + }, + ]; + } + return acc; } @@ -164,6 +184,7 @@ async function onFile(ev: Event) { form.append('file', file); const { data } = await api.post(props.previewPath, form, { headers: { 'Content-Type': undefined }, + timeout: 5 * 60 * 1000, }); if (!data.needsMapping) { try {