From ac57b8ac33e3beca6a0ee33ffd2ec3d1ffde2335 Mon Sep 17 00:00:00 2001 From: Mahdi Date: Sat, 3 Oct 2026 18:14:17 +0330 Subject: [PATCH] Parse large Excel imports asynchronously to avoid gateway 502 timeouts. Co-authored-by: Cursor --- .../activities/activities-excel.service.ts | 64 +++---- .../src/activities/activities.controller.ts | 8 + apps/api/src/common/excel-import-session.ts | 181 ++++++++++++++++-- apps/web/src/components/ExcelImportFlow.vue | 64 +++++-- packages/shared/src/index.ts | 13 ++ 5 files changed, 270 insertions(+), 60 deletions(-) diff --git a/apps/api/src/activities/activities-excel.service.ts b/apps/api/src/activities/activities-excel.service.ts index ee1a475..33661bf 100644 --- a/apps/api/src/activities/activities-excel.service.ts +++ b/apps/api/src/activities/activities-excel.service.ts @@ -17,11 +17,13 @@ import { type ExcelPreviewResult, } from '../common/excel'; import { + beginExcelImportPrepare, chunkImportMeta, - createExcelImportSession, destroyExcelImportSession, + getExcelImportPrepareStatus, getExcelImportSessionCaches, resolveExcelImportChunk, + type ExcelImportSessionStatus, } from '../common/excel-import-session'; const ACTIVITY_COLUMNS = [ @@ -161,49 +163,45 @@ export class ActivitiesExcelService { return previewExcel(file, ACTIVITY_ALIASES, excelFieldsFromColumns(ACTIVITY_COLUMNS, ['title'])); } - /** فقط خواندن و جلسه؛ بدون درج — برای فایل‌های خیلی بزرگ قبل از تکه‌تکه */ - async prepare( + /** + * آپلود فوری + پارس در پس‌زمینه (جلوگیری از ۵۰۲ nginx روی فایل‌های بزرگ). + * وضعیت را با prepareStatus پول کنید. + */ + prepare( actor: Actor, file: Express.Multer.File, mappingRaw?: string, - ): Promise { - const result: ImportContactsResult = { created: 0, skipped: 0, errors: [] }; - let rows: Record[]; - try { - rows = await readMappedRows( - file, - ACTIVITY_ALIASES, - parseExcelMapping(mappingRaw), - ); - } catch (e) { - if (e instanceof BadRequestException) throw e; - result.errors.push({ - row: 1, - message: e instanceof Error ? e.message : 'خواندن فایل اکسل ناموفق بود', - }); - return result; + ): ImportContactsResult { + if (!file?.buffer?.length) { + throw new BadRequestException('فایل ارسال نشده'); } - - const MAX_ROWS = 100_000; - if (rows.length > MAX_ROWS) { - result.errors.push({ - row: 1, - message: `فایل بیش از ${MAX_ROWS} ردیف دارد. فایل را تقسیم کنید.`, - }); - return { ...result, total: rows.length, processed: 0, hasMore: false }; - } - - const sessionId = createExcelImportSession(actor.organizationId, rows); + const mapping = parseExcelMapping(mappingRaw); + const sessionId = beginExcelImportPrepare( + actor.organizationId, + file.buffer, + (f) => readMappedRows(f, ACTIVITY_ALIASES, mapping), + ); return { - ...result, + created: 0, + skipped: 0, + errors: [], sessionId, - total: rows.length, + status: 'parsing', + total: 0, + parsed: 0, processed: 0, - hasMore: rows.length > 0, + hasMore: true, nextOffset: 0, }; } + prepareStatus( + actor: Actor, + sessionId: string, + ): ExcelImportSessionStatus { + return getExcelImportPrepareStatus(sessionId, actor.organizationId); + } + async import( actor: Actor, file: Express.Multer.File | undefined, diff --git a/apps/api/src/activities/activities.controller.ts b/apps/api/src/activities/activities.controller.ts index 8a39e4b..cbf2a89 100644 --- a/apps/api/src/activities/activities.controller.ts +++ b/apps/api/src/activities/activities.controller.ts @@ -113,6 +113,14 @@ export class ActivitiesController { return this.excelService.prepare(user, file, mapping); } + @Get('import/prepare/:sessionId') + importPrepareStatus( + @CurrentUser() user: Actor, + @Param('sessionId') sessionId: string, + ) { + return this.excelService.prepareStatus(user, sessionId); + } + @Post('import') @UseInterceptors(excelUpload) import( diff --git a/apps/api/src/common/excel-import-session.ts b/apps/api/src/common/excel-import-session.ts index 9083a5d..371b53e 100644 --- a/apps/api/src/common/excel-import-session.ts +++ b/apps/api/src/common/excel-import-session.ts @@ -1,4 +1,7 @@ import { randomBytes } from 'node:crypto'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; import { BadRequestException } from '@nestjs/common'; export const EXCEL_IMPORT_CHUNK_SIZE = 150; @@ -6,6 +9,7 @@ export const EXCEL_IMPORT_CHUNK_SIZE = 150; export const EXCEL_IMPORT_LARGE_THRESHOLD = 80; const SESSION_TTL_MS = 45 * 60 * 1000; const MAX_SESSIONS = 40; +const MAX_ASYNC_ROWS = 100_000; export type ExcelImportLookupCaches = { user: Map; @@ -14,19 +18,46 @@ export type ExcelImportLookupCaches = { deal: Map; }; +export type ExcelImportSessionStatus = { + sessionId: string; + status: 'parsing' | 'ready' | 'error'; + total: number; + parsed: number; + error?: string; + hasMore?: boolean; + nextOffset?: number; +}; + type ExcelImportSession = { organizationId: string; rows: Array>; expiresAt: number; caches: ExcelImportLookupCaches; + status: 'parsing' | 'ready' | 'error'; + error?: string; + total: number; + parsed: number; }; const sessions = new Map(); +const TMP_DIR = path.join(os.tmpdir(), 'mento-excel-import'); + +function emptyCaches(): ExcelImportLookupCaches { + return { + user: new Map(), + person: new Map(), + company: new Map(), + deal: new Map(), + }; +} function purgeExpired() { const now = Date.now(); for (const [id, session] of sessions) { - if (session.expiresAt <= now) sessions.delete(id); + if (session.expiresAt <= now) { + sessions.delete(id); + cleanupSessionFiles(id); + } } if (sessions.size <= MAX_SESSIONS) return; const ordered = [...sessions.entries()].sort( @@ -35,9 +66,24 @@ function purgeExpired() { while (sessions.size > MAX_SESSIONS && ordered.length) { const [id] = ordered.shift()!; sessions.delete(id); + cleanupSessionFiles(id); } } +function cleanupSessionFiles(sessionId: string) { + for (const ext of ['.xlsx', '.jsonl']) { + try { + fs.unlinkSync(path.join(TMP_DIR, `${sessionId}${ext}`)); + } catch { + /* ignore */ + } + } +} + +function touch(session: ExcelImportSession) { + session.expiresAt = Date.now() + SESSION_TTL_MS; +} + export function createExcelImportSession( organizationId: string, rows: Array>, @@ -48,16 +94,115 @@ export function createExcelImportSession( organizationId, rows, expiresAt: Date.now() + SESSION_TTL_MS, - caches: { - user: new Map(), - person: new Map(), - company: new Map(), - deal: new Map(), - }, + caches: emptyCaches(), + status: 'ready', + total: rows.length, + parsed: rows.length, }); return id; } +/** + * آپلود را فوری قبول می‌کند و پارس را در پس‌زمینه انجام می‌دهد + * تا پروکسی nginx با ۵۰۲ قطع نکند. + */ +export function beginExcelImportPrepare( + organizationId: string, + buffer: Buffer, + loadRows: (file: Express.Multer.File) => Promise>>, +): string { + purgeExpired(); + fs.mkdirSync(TMP_DIR, { recursive: true }); + const id = randomBytes(16).toString('hex'); + const xlsxPath = path.join(TMP_DIR, `${id}.xlsx`); + fs.writeFileSync(xlsxPath, buffer); + + sessions.set(id, { + organizationId, + rows: [], + expiresAt: Date.now() + SESSION_TTL_MS, + caches: emptyCaches(), + status: 'parsing', + total: 0, + parsed: 0, + }); + + const fakeFile = { + fieldname: 'file', + originalname: 'import.xlsx', + encoding: '7bit', + mimetype: + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + size: buffer.length, + buffer, + destination: '', + filename: '', + path: xlsxPath, + } as Express.Multer.File; + + void (async () => { + const session = sessions.get(id); + if (!session) return; + try { + const rows = await loadRows(fakeFile); + const current = sessions.get(id); + if (!current) return; + if (rows.length > MAX_ASYNC_ROWS) { + current.status = 'error'; + current.error = `فایل بیش از ${MAX_ASYNC_ROWS} ردیف دارد. فایل را تقسیم کنید.`; + current.total = rows.length; + current.parsed = rows.length; + touch(current); + cleanupSessionFiles(id); + return; + } + current.rows = rows; + current.total = rows.length; + current.parsed = rows.length; + current.status = 'ready'; + touch(current); + try { + fs.unlinkSync(xlsxPath); + } catch { + /* ignore */ + } + } catch (e) { + const current = sessions.get(id); + if (!current) return; + current.status = 'error'; + current.error = + e instanceof Error ? e.message : 'خواندن فایل اکسل ناموفق بود'; + touch(current); + cleanupSessionFiles(id); + } + })(); + + return id; +} + +export function getExcelImportPrepareStatus( + sessionId: string, + organizationId: string, +): ExcelImportSessionStatus { + purgeExpired(); + const session = sessions.get(sessionId); + if (!session || session.organizationId !== organizationId) { + throw new BadRequestException( + 'جلسه ورود اکسل منقضی یا نامعتبر است. فایل را دوباره انتخاب کنید.', + ); + } + touch(session); + return { + sessionId, + status: session.status, + total: session.total, + parsed: session.parsed, + error: session.error, + hasMore: session.status === 'ready' ? session.total > 0 : true, + nextOffset: 0, + }; +} + export function getExcelImportSession( sessionId: string, organizationId: string, @@ -69,13 +214,24 @@ export function getExcelImportSession( 'جلسه ورود اکسل منقضی یا نامعتبر است. فایل را دوباره انتخاب کنید.', ); } - session.expiresAt = Date.now() + SESSION_TTL_MS; + if (session.status === 'parsing') { + throw new BadRequestException( + 'فایل هنوز در حال آماده‌سازی است. چند لحظه صبر کنید.', + ); + } + if (session.status === 'error') { + throw new BadRequestException( + session.error || 'آماده‌سازی فایل اکسل ناموفق بود.', + ); + } + touch(session); return session.rows; } export function destroyExcelImportSession(sessionId: string | undefined) { if (!sessionId) return; sessions.delete(sessionId); + cleanupSessionFiles(sessionId); } export function getExcelImportSessionCaches( @@ -85,14 +241,9 @@ export function getExcelImportSessionCaches( purgeExpired(); const session = sessions.get(sessionId); if (!session || session.organizationId !== organizationId) { - return { - user: new Map(), - person: new Map(), - company: new Map(), - deal: new Map(), - }; + return emptyCaches(); } - session.expiresAt = Date.now() + SESSION_TTL_MS; + touch(session); return session.caches; } diff --git a/apps/web/src/components/ExcelImportFlow.vue b/apps/web/src/components/ExcelImportFlow.vue index b0e4d74..4b662d0 100644 --- a/apps/web/src/components/ExcelImportFlow.vue +++ b/apps/web/src/components/ExcelImportFlow.vue @@ -2,7 +2,11 @@ import { computed, reactive, ref } from 'vue'; import axios from 'axios'; import { api } from '@/lib/api'; -import type { ExcelPreviewResult, ImportContactsResult } from '@mento/shared'; +import type { + ExcelImportPrepareStatus, + ExcelPreviewResult, + ImportContactsResult, +} from '@mento/shared'; import { UiBusyOverlay, UiButton, UiModal, UiSelect } from '@/components/ui'; const props = defineProps<{ @@ -131,6 +135,10 @@ function updateImportProgress(processed: number, total: number) { progressLabel.value = `ورود فایل: ${processed.toLocaleString('fa-IR')} از ${total.toLocaleString('fa-IR')} ردیف`; } +function sleep(ms: number) { + return new Promise((r) => setTimeout(r, ms)); +} + async function postPrepare(file: File, map?: Record) { if (!props.preparePath) { throw new Error('preparePath تنظیم نشده'); @@ -140,16 +148,43 @@ async function postPrepare(file: File, map?: Record) { if (map && Object.values(map).some(Boolean)) { form.append('mapping', JSON.stringify(map)); } + // فقط آپلود — پارس روی سرور ناهمگام است؛ تایم‌اوت کوتاه کافی است const { data } = await api.post(props.preparePath, form, { headers: { 'Content-Type': undefined }, - timeout: 10 * 60 * 1000, + timeout: 3 * 60 * 1000, }); return data; } +async function waitPrepareReady(sessionId: string) { + if (!props.preparePath) throw new Error('preparePath تنظیم نشده'); + const statusUrl = `${props.preparePath}/${sessionId}`; + const started = Date.now(); + const maxMs = 15 * 60 * 1000; + while (Date.now() - started < maxMs) { + const { data } = await api.get(statusUrl, { + timeout: 30_000, + }); + if (data.status === 'ready') { + progressLabel.value = `فایل آماده شد — ${data.total.toLocaleString('fa-IR')} ردیف`; + return data; + } + if (data.status === 'error') { + throw new Error(data.error || 'آماده‌سازی فایل اکسل ناموفق بود'); + } + const parsed = data.parsed || 0; + progressPct.value = null; + progressLabel.value = parsed + ? `در حال خواندن فایل روی سرور… ${parsed.toLocaleString('fa-IR')} ردیف` + : 'در حال خواندن فایل روی سرور… لطفاً صبر کنید'; + await sleep(1200); + } + throw new Error('آماده‌سازی فایل بیش از حد طول کشید. دوباره تلاش کنید.'); +} + async function runChunkedImport(file: File, map?: Record) { phase.value = 'import'; - progressLabel.value = 'در حال آماده‌سازی و خواندن فایل…'; + progressLabel.value = 'در حال ارسال فایل…'; progressPct.value = 0; let acc: ImportContactsResult = { created: 0, skipped: 0, errors: [] }; @@ -160,19 +195,24 @@ async function runChunkedImport(file: File, map?: Record) { if (props.preparePath) { const prep = await postPrepare(file, map); - acc = mergeImportResults(acc, prep); - if ( - prep.errors?.some((e) => e.row === 1 && /بیش از .* ردیف|خواندن فایل/.test(e.message)) - ) { - return acc; - } if (!prep.sessionId) { - if (!prep.total) return acc; throw new Error(prep.errors[0]?.message || 'آماده‌سازی فایل ناموفق بود'); } sessionId = prep.sessionId; - updateImportProgress(0, prep.total ?? 0); - if (!prep.hasMore && (prep.total ?? 0) === 0) return acc; + progressLabel.value = 'در حال خواندن فایل روی سرور…'; + const ready = await waitPrepareReady(sessionId); + if (!ready.total) { + return { + ...acc, + total: 0, + processed: 0, + hasMore: false, + errors: ready.error + ? [{ row: 1, message: ready.error }] + : [{ row: 1, message: 'هیچ ردیف داده‌ای در فایل نیست' }], + }; + } + updateImportProgress(0, ready.total); } while (guard < maxChunks) { diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 6343253..6d109bf 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -1260,6 +1260,19 @@ export interface ImportContactsResult { /** مرحله بعدی وجود دارد */ hasMore?: boolean; nextOffset?: number; + /** وضعیت آماده‌سازی ناهمگام فایل بزرگ */ + status?: "parsing" | "ready" | "error"; + parsed?: number; +} + +export interface ExcelImportPrepareStatus { + sessionId: string; + status: "parsing" | "ready" | "error"; + total: number; + parsed: number; + error?: string; + hasMore?: boolean; + nextOffset?: number; } export interface ExcelFieldOption {