Allow large activity Excel imports (40k+ rows) with bigger chunks and caches.

Co-authored-by: Cursor <cursoragent@cursor.com>

apps/api/src/activities/activities-excel.service.ts

@@ -18,6 +18,8 @@ import {
 } from '../common/excel';
 import {
   chunkImportMeta,
+  destroyExcelImportSession,
+  getExcelImportSessionCaches,
   resolveExcelImportChunk,
 } from '../common/excel-import-session';
 
@@ -184,22 +186,49 @@ export class ActivitiesExcelService {
       return result;
     }
 
-    const MAX_ROWS = 25000;
+    const MAX_ROWS = 100_000;
     if (ctx.total > MAX_ROWS) {
+      destroyExcelImportSession(ctx.sessionId);
       result.errors.push({
         row: 1,
-        message: `فایل بیش از ${MAX_ROWS} ردیف دارد.`,
+        message: `فایل بیش از ${MAX_ROWS} ردیف دارد. فایل را تقسیم کنید.`,
       });
-      return result;
+      return { ...result, hasMore: false, total: ctx.total, processed: 0 };
     }
 
-    const userCache = new Map<string, string | null>();
+    const caches = getExcelImportSessionCaches(ctx.sessionId, actor.organizationId);
     const resolveUser = async (name?: string) => {
       const key = (name ?? '').trim().toLowerCase();
       if (!key) return null;
-      if (userCache.has(key)) return userCache.get(key) ?? null;
+      if (caches.user.has(key)) return caches.user.get(key) ?? null;
       const id = await this.findUserId(actor.organizationId, name);
-      userCache.set(key, id);
+      caches.user.set(key, id);
+      return id;
+    };
+    const resolvePerson = async (row: Record<string, string>) => {
+      const mobile = normalizePhone(row.personMobile);
+      const name = (row.personName ?? '').trim().toLowerCase();
+      const key = mobile ? `m:${mobile}` : name ? `n:${name}` : '';
+      if (!key) return null;
+      if (caches.person.has(key)) return caches.person.get(key) ?? null;
+      const id = await this.findPersonId(actor.organizationId, row);
+      caches.person.set(key, id);
+      return id;
+    };
+    const resolveCompany = async (name?: string) => {
+      const key = (name ?? '').trim().toLowerCase();
+      if (!key) return null;
+      if (caches.company.has(key)) return caches.company.get(key) ?? null;
+      const id = await this.findCompanyId(actor.organizationId, name);
+      caches.company.set(key, id);
+      return id;
+    };
+    const resolveDeal = async (title?: string) => {
+      const key = (title ?? '').trim().toLowerCase();
+      if (!key) return null;
+      if (caches.deal.has(key)) return caches.deal.get(key) ?? null;
+      const id = await this.findDealId(actor.organizationId, title);
+      caches.deal.set(key, id);
       return id;
     };
 
@@ -223,9 +252,9 @@ export class ActivitiesExcelService {
       const status = parseStatus(row.status);
 
       try {
-        const personId = await this.findPersonId(actor.organizationId, row);
+        const personId = await resolvePerson(row);
-        const companyId = await this.findCompanyId(actor.organizationId, row.companyName);
+        const companyId = await resolveCompany(row.companyName);
-        const dealId = await this.findDealId(actor.organizationId, row.dealTitle);
+        const dealId = await resolveDeal(row.dealTitle);
         const assigneeNames = String(row.assignee ?? '')
           .split(/[,،]/)
           .map((s) => s.trim())

apps/api/src/common/excel-import-session.ts

@@ -1,16 +1,24 @@
 import { randomBytes } from 'node:crypto';
 import { BadRequestException } from '@nestjs/common';
 
-export const EXCEL_IMPORT_CHUNK_SIZE = 80;
+export const EXCEL_IMPORT_CHUNK_SIZE = 150;
 /** بعد از این تعداد ردیف، کلاینت حتماً چندمرحله‌ای می‌رود (پیش‌نمایش هم نشان می‌دهد) */
 export const EXCEL_IMPORT_LARGE_THRESHOLD = 80;
 const SESSION_TTL_MS = 45 * 60 * 1000;
 const MAX_SESSIONS = 40;
 
+export type ExcelImportLookupCaches = {
+  user: Map<string, string | null>;
+  person: Map<string, string | null>;
+  company: Map<string, string | null>;
+  deal: Map<string, string | null>;
+};
+
 type ExcelImportSession = {
   organizationId: string;
   rows: Array<Record<string, string>>;
   expiresAt: number;
+  caches: ExcelImportLookupCaches;
 };
 
 const sessions = new Map<string, ExcelImportSession>();
@@ -40,6 +48,12 @@ export function createExcelImportSession(
     organizationId,
     rows,
     expiresAt: Date.now() + SESSION_TTL_MS,
+    caches: {
+      user: new Map(),
+      person: new Map(),
+      company: new Map(),
+      deal: new Map(),
+    },
   });
   return id;
 }
@@ -64,6 +78,24 @@ export function destroyExcelImportSession(sessionId: string | undefined) {
   sessions.delete(sessionId);
 }
 
+export function getExcelImportSessionCaches(
+  sessionId: string,
+  organizationId: string,
+): ExcelImportLookupCaches {
+  purgeExpired();
+  const session = sessions.get(sessionId);
+  if (!session || session.organizationId !== organizationId) {
+    return {
+      user: new Map(),
+      person: new Map(),
+      company: new Map(),
+      deal: new Map(),
+    };
+  }
+  session.expiresAt = Date.now() + SESSION_TTL_MS;
+  return session.caches;
+}
+
 export function parseChunkParams(raw: {
   offset?: string | number;
   limit?: string | number;

apps/api/src/common/excel.ts

@@ -319,16 +319,8 @@ export async function previewExcel(
   const missingRequired = required.some((key) => !mapped.has(key));
   const unrecognized = columns.some((col) => !suggested[col.header]);
 
-  let rowCount = 0;
+  // برای فایل‌های خیلی بزرگ (ده‌ها هزار ردیف) پیمایش کامل کند است؛ rowCount شیت کافی است
-  sheet.eachRow((row, rowNumber) => {
+  const rowCount = Math.max(0, (sheet.actualRowCount || sheet.rowCount || 1) - 1);
-    if (rowNumber === 1) return;
-    let any = false;
-    row.eachCell((cell) => {
-      if (any) return;
-      if (cellStr(cell.value)) any = true;
-    });
-    if (any) rowCount += 1;
-  });
 
   return {
     columns,

apps/web/src/components/ExcelImportFlow.vue

@@ -103,9 +103,11 @@ async function postImportChunk(opts: {
   }
   if (opts.sessionId) form.append('sessionId', opts.sessionId);
   if (opts.offset != null) form.append('offset', String(opts.offset));
+  form.append('limit', '250');
   const { data } = await api.post<ImportContactsResult>(props.importPath, form, {
     headers: { 'Content-Type': undefined },
-    timeout: 3 * 60 * 1000,
+    // خواندن فایل‌های خیلی بزرگ در مرحلهٔ اول ممکن است طول بکشد
+    timeout: opts.file ? 10 * 60 * 1000 : 3 * 60 * 1000,
   });
   return data;
 }
@@ -118,8 +120,9 @@ async function runChunkedImport(file: File, map?: Record<string, string>) {
   let sessionId: string | undefined;
   let offset = 0;
   let guard = 0;
+  const maxChunks = 2000;
 
-  while (guard < 500) {
+  while (guard < maxChunks) {
     guard += 1;
     const part = await postImportChunk({
       file: sessionId ? undefined : file,
@@ -135,11 +138,28 @@ async function runChunkedImport(file: File, map?: Record<string, string>) {
       progressLabel.value = `ورود چندمرحله‌ای: ${processed.toLocaleString('fa-IR')} از ${total.toLocaleString('fa-IR')} ردیف`;
     }
 
+    // خطای سراسری (مثلاً سقف ردیف) — ادامه نده
+    if (
+      part.errors?.some((e) => e.row === 1 && /بیش از .* ردیف/.test(e.message))
+    ) {
+      break;
+    }
+
     if (!part.hasMore || !part.sessionId) break;
     sessionId = part.sessionId;
     offset = part.nextOffset ?? processed;
   }
 
+  if (guard >= maxChunks && acc.hasMore) {
+    acc.errors = [
+      ...acc.errors,
+      {
+        row: 1,
+        message: 'ورود به‌خاطر تعداد مراحل زیاد متوقف شد. از جایی که مانده ادامه دهید یا فایل را تقسیم کنید.',
+      },
+    ];
+  }
+
   return acc;
 }
 
@@ -164,6 +184,7 @@ async function onFile(ev: Event) {
     form.append('file', file);
     const { data } = await api.post<ExcelPreviewResult>(props.previewPath, form, {
       headers: { 'Content-Type': undefined },
+      timeout: 5 * 60 * 1000,
     });
     if (!data.needsMapping) {
       try {