Fixing the error when importing a large Excel file

apps/api/src/common/excel-import-session.ts

@@ -0,0 +1,142 @@
+import { randomBytes } from 'node:crypto';
+import { BadRequestException } from '@nestjs/common';
+
+export const EXCEL_IMPORT_CHUNK_SIZE = 80;
+/** بعد از این تعداد ردیف، کلاینت حتماً چندمرحله‌ای می‌رود (پیش‌نمایش هم نشان می‌دهد) */
+export const EXCEL_IMPORT_LARGE_THRESHOLD = 80;
+const SESSION_TTL_MS = 45 * 60 * 1000;
+const MAX_SESSIONS = 40;
+
+type ExcelImportSession = {
+  organizationId: string;
+  rows: Array<Record<string, string>>;
+  expiresAt: number;
+};
+
+const sessions = new Map<string, ExcelImportSession>();
+
+function purgeExpired() {
+  const now = Date.now();
+  for (const [id, session] of sessions) {
+    if (session.expiresAt <= now) sessions.delete(id);
+  }
+  if (sessions.size <= MAX_SESSIONS) return;
+  const ordered = [...sessions.entries()].sort(
+    (a, b) => a[1].expiresAt - b[1].expiresAt,
+  );
+  while (sessions.size > MAX_SESSIONS && ordered.length) {
+    const [id] = ordered.shift()!;
+    sessions.delete(id);
+  }
+}
+
+export function createExcelImportSession(
+  organizationId: string,
+  rows: Array<Record<string, string>>,
+): string {
+  purgeExpired();
+  const id = randomBytes(16).toString('hex');
+  sessions.set(id, {
+    organizationId,
+    rows,
+    expiresAt: Date.now() + SESSION_TTL_MS,
+  });
+  return id;
+}
+
+export function getExcelImportSession(
+  sessionId: string,
+  organizationId: string,
+): Array<Record<string, string>> {
+  purgeExpired();
+  const session = sessions.get(sessionId);
+  if (!session || session.organizationId !== organizationId) {
+    throw new BadRequestException(
+      'جلسه ورود اکسل منقضی یا نامعتبر است. فایل را دوباره انتخاب کنید.',
+    );
+  }
+  session.expiresAt = Date.now() + SESSION_TTL_MS;
+  return session.rows;
+}
+
+export function destroyExcelImportSession(sessionId: string | undefined) {
+  if (!sessionId) return;
+  sessions.delete(sessionId);
+}
+
+export function parseChunkParams(raw: {
+  offset?: string | number;
+  limit?: string | number;
+}): { offset: number; limit: number } {
+  const offset = Math.max(0, Number(raw.offset ?? 0) || 0);
+  const limit = Math.min(
+    250,
+    Math.max(1, Number(raw.limit ?? EXCEL_IMPORT_CHUNK_SIZE) || EXCEL_IMPORT_CHUNK_SIZE),
+  );
+  return { offset, limit };
+}
+
+export type ExcelImportChunkContext = {
+  sessionId: string;
+  allRows: Array<Record<string, string>>;
+  slice: Array<Record<string, string>>;
+  offset: number;
+  nextOffset: number;
+  hasMore: boolean;
+  total: number;
+};
+
+/**
+ * فایل را یک‌بار می‌خواند و در حافظه نگه می‌دارد؛ درخواست‌های بعدی فقط با sessionId + offset.
+ */
+export async function resolveExcelImportChunk(opts: {
+  organizationId: string;
+  file?: Express.Multer.File | null;
+  sessionId?: string;
+  offset?: string | number;
+  limit?: string | number;
+  loadRows: () => Promise<Array<Record<string, string>>>;
+}): Promise<ExcelImportChunkContext> {
+  const { offset, limit } = parseChunkParams({
+    offset: opts.offset,
+    limit: opts.limit,
+  });
+
+  let sessionId = (opts.sessionId ?? '').trim();
+  let allRows: Array<Record<string, string>>;
+
+  if (sessionId) {
+    allRows = getExcelImportSession(sessionId, opts.organizationId);
+  } else {
+    if (!opts.file?.buffer?.length) {
+      throw new BadRequestException('فایل ارسال نشده');
+    }
+    allRows = await opts.loadRows();
+    sessionId = createExcelImportSession(opts.organizationId, allRows);
+  }
+
+  const slice = allRows.slice(offset, offset + limit);
+  const nextOffset = offset + slice.length;
+  const hasMore = nextOffset < allRows.length;
+  if (!hasMore) destroyExcelImportSession(sessionId);
+
+  return {
+    sessionId,
+    allRows,
+    slice,
+    offset,
+    nextOffset,
+    hasMore,
+    total: allRows.length,
+  };
+}
+
+export function chunkImportMeta(ctx: ExcelImportChunkContext) {
+  return {
+    sessionId: ctx.hasMore ? ctx.sessionId : undefined,
+    total: ctx.total,
+    processed: ctx.nextOffset,
+    hasMore: ctx.hasMore,
+    nextOffset: ctx.hasMore ? ctx.nextOffset : undefined,
+  };
+}

apps/api/src/common/excel.ts

@@ -1,5 +1,6 @@
 import { BadRequestException } from '@nestjs/common';
 import ExcelJS from 'exceljs';
+import { EXCEL_IMPORT_LARGE_THRESHOLD } from './excel-import-session';
 
 export function cellStr(value: ExcelJS.CellValue | undefined): string {
   try {
@@ -229,6 +230,8 @@ export type ExcelPreviewResult = {
   fields: ExcelFieldOption[];
   suggested: Record<string, string>;
   needsMapping: boolean;
+  rowCount: number;
+  largeFile?: boolean;
 };
 
 export function excelFieldsFromColumns(
@@ -315,11 +318,25 @@ export async function previewExcel(
   const mapped = new Set(Object.values(suggested));
   const missingRequired = required.some((key) => !mapped.has(key));
   const unrecognized = columns.some((col) => !suggested[col.header]);
+
+  let rowCount = 0;
+  sheet.eachRow((row, rowNumber) => {
+    if (rowNumber === 1) return;
+    let any = false;
+    row.eachCell((cell) => {
+      if (any) return;
+      if (cellStr(cell.value)) any = true;
+    });
+    if (any) rowCount += 1;
+  });
+
   return {
     columns,
     fields,
     suggested,
     needsMapping: unrecognized || missingRequired || !mapped.size,
+    rowCount,
+    largeFile: rowCount >= EXCEL_IMPORT_LARGE_THRESHOLD,
   };
 }