From 2e315c9cfd798a380399cc87c251c9c611cd3296 Mon Sep 17 00:00:00 2001 From: Mahdi Date: Sat, 3 Oct 2026 15:01:01 +0330 Subject: [PATCH] Fixing the error when importing a large Excel file --- .../activities/activities-excel.service.ts | 76 ++++++++-- .../src/activities/activities.controller.ts | 39 +++-- apps/api/src/common/excel-import-session.ts | 142 ++++++++++++++++++ apps/api/src/common/excel.ts | 17 +++ apps/api/src/contacts/companies.controller.ts | 11 +- .../src/contacts/contacts-excel.service.ts | 52 +++++-- apps/api/src/contacts/people.controller.ts | 11 +- apps/api/src/pipelines/deals-excel.service.ts | 25 ++- apps/api/src/pipelines/deals.controller.ts | 11 +- .../src/products/products-excel.service.ts | 25 ++- apps/api/src/products/products.controller.ts | 11 +- .../project-boards.controller.ts | 11 +- .../project-cards-excel.service.ts | 25 ++- apps/web/src/components/ExcelImportFlow.vue | 120 ++++++++++++--- packages/shared/src/index.ts | 13 ++ 15 files changed, 503 insertions(+), 86 deletions(-) create mode 100644 apps/api/src/common/excel-import-session.ts diff --git a/apps/api/src/activities/activities-excel.service.ts b/apps/api/src/activities/activities-excel.service.ts index 9440d17..9c8d9de 100644 --- a/apps/api/src/activities/activities-excel.service.ts +++ b/apps/api/src/activities/activities-excel.service.ts @@ -1,4 +1,4 @@ -import { Injectable } from '@nestjs/common'; +import { BadRequestException, Injectable } from '@nestjs/common'; import type { ActivityType, ImportContactsResult } from '@mento/shared'; import { PrismaService } from '../prisma/prisma.service'; import type { Actor } from '../common/record-scope'; @@ -16,6 +16,10 @@ import { writeXlsx, type ExcelPreviewResult, } from '../common/excel'; +import { + chunkImportMeta, + resolveExcelImportChunk, +} from '../common/excel-import-session'; const ACTIVITY_COLUMNS = [ { header: '_type', key: '_type', width: 12 }, @@ -156,15 +160,52 @@ export class ActivitiesExcelService { async import( actor: Actor, - file: Express.Multer.File, + file: Express.Multer.File | undefined, mappingRaw?: string, + chunk?: { sessionId?: string; offset?: string | number; limit?: string | number }, ): Promise { - const rows = await readMappedRows(file, ACTIVITY_ALIASES, parseExcelMapping(mappingRaw)); const result: ImportContactsResult = { created: 0, skipped: 0, errors: [] }; + let ctx; + try { + ctx = await resolveExcelImportChunk({ + organizationId: actor.organizationId, + file, + sessionId: chunk?.sessionId, + offset: chunk?.offset, + limit: chunk?.limit, + loadRows: () => + readMappedRows(file!, ACTIVITY_ALIASES, parseExcelMapping(mappingRaw)), + }); + } catch (e) { + if (e instanceof BadRequestException) throw e; + const message = + e instanceof Error ? e.message : 'خواندن فایل اکسل ناموفق بود'; + result.errors.push({ row: 1, message }); + return result; + } - for (let i = 0; i < rows.length; i++) { - const row = rows[i]!; - const rowNum = i + 2; + const MAX_ROWS = 25000; + if (ctx.total > MAX_ROWS) { + result.errors.push({ + row: 1, + message: `فایل بیش از ${MAX_ROWS} ردیف دارد.`, + }); + return result; + } + + const userCache = new Map(); + const resolveUser = async (name?: string) => { + const key = (name ?? '').trim().toLowerCase(); + if (!key) return null; + if (userCache.has(key)) return userCache.get(key) ?? null; + const id = await this.findUserId(actor.organizationId, name); + userCache.set(key, id); + return id; + }; + + for (let i = 0; i < ctx.slice.length; i++) { + const row = ctx.slice[i]!; + const rowNum = ctx.offset + i + 2; if ((row._type || '').trim() && row._type.trim().toLowerCase() !== 'activity') { continue; } @@ -191,13 +232,13 @@ export class ActivitiesExcelService { .filter(Boolean); const assigneeIds: string[] = []; for (const name of assigneeNames) { - const uid = await this.findUserId(actor.organizationId, name); + const uid = await resolveUser(name); if (uid && !assigneeIds.includes(uid)) assigneeIds.push(uid); } if (!assigneeIds.length) assigneeIds.push(actor.userId); const assigneeId = assigneeIds[0]!; const creatorId = - (await this.findUserId(actor.organizationId, row.creator)) ?? actor.userId; + (await resolveUser(row.creator)) ?? actor.userId; const duplicate = dueAt ? await this.prisma.activity.findFirst({ @@ -215,7 +256,7 @@ export class ActivitiesExcelService { continue; } - await this.prisma.activity.create({ + const created = await this.prisma.activity.create({ data: { organizationId: actor.organizationId, title, @@ -230,10 +271,23 @@ export class ActivitiesExcelService { companyId, dealId, assigneeId, - assignees: { connect: assigneeIds.map((id) => ({ id })) }, createdById: creatorId, }, + select: { id: true }, }); + // M2M مسئولان — اگر مایگریشن روی سرور نباشد، ایمپورت نباید بترکد + if (assigneeIds.length) { + try { + await this.prisma.activity.update({ + where: { id: created.id }, + data: { + assignees: { set: assigneeIds.map((id) => ({ id })) }, + }, + }); + } catch { + /* assigneeId به‌تنهایی کافی است */ + } + } result.created += 1; } catch (e) { result.errors.push({ @@ -243,7 +297,7 @@ export class ActivitiesExcelService { } } - return result; + return { ...result, ...chunkImportMeta(ctx) }; } private async findPersonId(organizationId: string, row: Record) { diff --git a/apps/api/src/activities/activities.controller.ts b/apps/api/src/activities/activities.controller.ts index 7ccea97..7b507ae 100644 --- a/apps/api/src/activities/activities.controller.ts +++ b/apps/api/src/activities/activities.controller.ts @@ -1,4 +1,5 @@ import { + BadRequestException, Body, Controller, Delete, @@ -28,6 +29,11 @@ import { } from './dto'; import { brand } from '@mento/shared'; +const excelUpload = FileInterceptor('file', { + storage: memoryStorage(), + limits: { fileSize: 15_000_000 }, +}); + @Controller('activities') @UseGuards(JwtAuthGuard) export class ActivitiesController { @@ -86,29 +92,32 @@ export class ActivitiesController { } @Post('import/preview') - @UseInterceptors( - FileInterceptor('file', { - storage: memoryStorage(), - limits: { fileSize: 15_000_000 }, - }), - ) - importPreview(@UploadedFile() file: Express.Multer.File) { + @UseInterceptors(excelUpload) + importPreview(@UploadedFile() file?: Express.Multer.File) { + if (!file?.buffer?.length) { + throw new BadRequestException('فایل ارسال نشده'); + } return this.excelService.preview(file); } @Post('import') - @UseInterceptors( - FileInterceptor('file', { - storage: memoryStorage(), - limits: { fileSize: 15_000_000 }, - }), - ) + @UseInterceptors(excelUpload) import( @CurrentUser() user: Actor, - @UploadedFile() file: Express.Multer.File, + @UploadedFile() file?: Express.Multer.File, @Body('mapping') mapping?: string, + @Body('sessionId') sessionId?: string, + @Body('offset') offset?: string, + @Body('limit') limit?: string, ) { - return this.excelService.import(user, file, mapping); + if (!file?.buffer?.length && !sessionId) { + throw new BadRequestException('فایل ارسال نشده'); + } + return this.excelService.import(user, file, mapping, { + sessionId, + offset, + limit, + }); } @Get(':id') diff --git a/apps/api/src/common/excel-import-session.ts b/apps/api/src/common/excel-import-session.ts new file mode 100644 index 0000000..70ab7c1 --- /dev/null +++ b/apps/api/src/common/excel-import-session.ts @@ -0,0 +1,142 @@ +import { randomBytes } from 'node:crypto'; +import { BadRequestException } from '@nestjs/common'; + +export const EXCEL_IMPORT_CHUNK_SIZE = 80; +/** بعد از این تعداد ردیف، کلاینت حتماً چندمرحله‌ای می‌رود (پیش‌نمایش هم نشان می‌دهد) */ +export const EXCEL_IMPORT_LARGE_THRESHOLD = 80; +const SESSION_TTL_MS = 45 * 60 * 1000; +const MAX_SESSIONS = 40; + +type ExcelImportSession = { + organizationId: string; + rows: Array>; + expiresAt: number; +}; + +const sessions = new Map(); + +function purgeExpired() { + const now = Date.now(); + for (const [id, session] of sessions) { + if (session.expiresAt <= now) sessions.delete(id); + } + if (sessions.size <= MAX_SESSIONS) return; + const ordered = [...sessions.entries()].sort( + (a, b) => a[1].expiresAt - b[1].expiresAt, + ); + while (sessions.size > MAX_SESSIONS && ordered.length) { + const [id] = ordered.shift()!; + sessions.delete(id); + } +} + +export function createExcelImportSession( + organizationId: string, + rows: Array>, +): string { + purgeExpired(); + const id = randomBytes(16).toString('hex'); + sessions.set(id, { + organizationId, + rows, + expiresAt: Date.now() + SESSION_TTL_MS, + }); + return id; +} + +export function getExcelImportSession( + sessionId: string, + organizationId: string, +): Array> { + purgeExpired(); + const session = sessions.get(sessionId); + if (!session || session.organizationId !== organizationId) { + throw new BadRequestException( + 'جلسه ورود اکسل منقضی یا نامعتبر است. فایل را دوباره انتخاب کنید.', + ); + } + session.expiresAt = Date.now() + SESSION_TTL_MS; + return session.rows; +} + +export function destroyExcelImportSession(sessionId: string | undefined) { + if (!sessionId) return; + sessions.delete(sessionId); +} + +export function parseChunkParams(raw: { + offset?: string | number; + limit?: string | number; +}): { offset: number; limit: number } { + const offset = Math.max(0, Number(raw.offset ?? 0) || 0); + const limit = Math.min( + 250, + Math.max(1, Number(raw.limit ?? EXCEL_IMPORT_CHUNK_SIZE) || EXCEL_IMPORT_CHUNK_SIZE), + ); + return { offset, limit }; +} + +export type ExcelImportChunkContext = { + sessionId: string; + allRows: Array>; + slice: Array>; + offset: number; + nextOffset: number; + hasMore: boolean; + total: number; +}; + +/** + * فایل را یک‌بار می‌خواند و در حافظه نگه می‌دارد؛ درخواست‌های بعدی فقط با sessionId + offset. + */ +export async function resolveExcelImportChunk(opts: { + organizationId: string; + file?: Express.Multer.File | null; + sessionId?: string; + offset?: string | number; + limit?: string | number; + loadRows: () => Promise>>; +}): Promise { + const { offset, limit } = parseChunkParams({ + offset: opts.offset, + limit: opts.limit, + }); + + let sessionId = (opts.sessionId ?? '').trim(); + let allRows: Array>; + + if (sessionId) { + allRows = getExcelImportSession(sessionId, opts.organizationId); + } else { + if (!opts.file?.buffer?.length) { + throw new BadRequestException('فایل ارسال نشده'); + } + allRows = await opts.loadRows(); + sessionId = createExcelImportSession(opts.organizationId, allRows); + } + + const slice = allRows.slice(offset, offset + limit); + const nextOffset = offset + slice.length; + const hasMore = nextOffset < allRows.length; + if (!hasMore) destroyExcelImportSession(sessionId); + + return { + sessionId, + allRows, + slice, + offset, + nextOffset, + hasMore, + total: allRows.length, + }; +} + +export function chunkImportMeta(ctx: ExcelImportChunkContext) { + return { + sessionId: ctx.hasMore ? ctx.sessionId : undefined, + total: ctx.total, + processed: ctx.nextOffset, + hasMore: ctx.hasMore, + nextOffset: ctx.hasMore ? ctx.nextOffset : undefined, + }; +} diff --git a/apps/api/src/common/excel.ts b/apps/api/src/common/excel.ts index 5a1885e..ae8132a 100644 --- a/apps/api/src/common/excel.ts +++ b/apps/api/src/common/excel.ts @@ -1,5 +1,6 @@ import { BadRequestException } from '@nestjs/common'; import ExcelJS from 'exceljs'; +import { EXCEL_IMPORT_LARGE_THRESHOLD } from './excel-import-session'; export function cellStr(value: ExcelJS.CellValue | undefined): string { try { @@ -229,6 +230,8 @@ export type ExcelPreviewResult = { fields: ExcelFieldOption[]; suggested: Record; needsMapping: boolean; + rowCount: number; + largeFile?: boolean; }; export function excelFieldsFromColumns( @@ -315,11 +318,25 @@ export async function previewExcel( const mapped = new Set(Object.values(suggested)); const missingRequired = required.some((key) => !mapped.has(key)); const unrecognized = columns.some((col) => !suggested[col.header]); + + let rowCount = 0; + sheet.eachRow((row, rowNumber) => { + if (rowNumber === 1) return; + let any = false; + row.eachCell((cell) => { + if (any) return; + if (cellStr(cell.value)) any = true; + }); + if (any) rowCount += 1; + }); + return { columns, fields, suggested, needsMapping: unrecognized || missingRequired || !mapped.size, + rowCount, + largeFile: rowCount >= EXCEL_IMPORT_LARGE_THRESHOLD, }; } diff --git a/apps/api/src/contacts/companies.controller.ts b/apps/api/src/contacts/companies.controller.ts index eb16911..6741eb9 100644 --- a/apps/api/src/contacts/companies.controller.ts +++ b/apps/api/src/contacts/companies.controller.ts @@ -78,11 +78,18 @@ export class CompaniesController { @CurrentUser() user: Actor, @UploadedFile() file?: Express.Multer.File, @Body('mapping') mapping?: string, + @Body('sessionId') sessionId?: string, + @Body('offset') offset?: string, + @Body('limit') limit?: string, ) { - if (!file?.buffer?.length) { + if (!file?.buffer?.length && !sessionId) { throw new BadRequestException('فایل ارسال نشده'); } - return this.excelService.importCompanies(user, file, mapping); + return this.excelService.importCompanies(user, file, mapping, { + sessionId, + offset, + limit, + }); } @Get(':id') diff --git a/apps/api/src/contacts/contacts-excel.service.ts b/apps/api/src/contacts/contacts-excel.service.ts index e957fe1..cbd2260 100644 --- a/apps/api/src/contacts/contacts-excel.service.ts +++ b/apps/api/src/contacts/contacts-excel.service.ts @@ -16,9 +16,19 @@ import { writeXlsx, type ExcelPreviewResult, } from '../common/excel'; +import { + chunkImportMeta, + resolveExcelImportChunk, +} from '../common/excel-import-session'; import { CompaniesService } from './companies.service'; import { PeopleService } from './people.service'; +type ImportChunkOpts = { + sessionId?: string; + offset?: string | number; + limit?: string | number; +}; + function parseMood(raw?: string): PersonMood | undefined { const value = raw?.trim(); if (!value) return undefined; @@ -363,16 +373,25 @@ export class ContactsExcelService { async importPeople( actor: Actor, - file: Express.Multer.File, + file: Express.Multer.File | undefined, mappingRaw?: string, + chunk?: ImportChunkOpts, ): Promise { - const rows = await readMappedRows(file, PERSON_ALIASES, parseExcelMapping(mappingRaw)); + const ctx = await resolveExcelImportChunk({ + organizationId: actor.organizationId, + file, + sessionId: chunk?.sessionId, + offset: chunk?.offset, + limit: chunk?.limit, + loadRows: () => + readMappedRows(file!, PERSON_ALIASES, parseExcelMapping(mappingRaw)), + }); const result: ImportContactsResult = { created: 0, skipped: 0, errors: [] }; const ownerCache = new Map(); - for (let i = 0; i < rows.length; i++) { - const row = rows[i]!; - const rowNum = i + 2; + for (let i = 0; i < ctx.slice.length; i++) { + const row = ctx.slice[i]!; + const rowNum = ctx.offset + i + 2; if ((row._type || '').trim() && row._type.trim().toLowerCase() !== 'contact') { continue; } @@ -476,20 +495,29 @@ export class ContactsExcelService { } } - return result; + return { ...result, ...chunkImportMeta(ctx) }; } async importCompanies( actor: Actor, - file: Express.Multer.File, + file: Express.Multer.File | undefined, mappingRaw?: string, + chunk?: ImportChunkOpts, ): Promise { - const rows = await readMappedRows(file, COMPANY_ALIASES, parseExcelMapping(mappingRaw)); + const ctx = await resolveExcelImportChunk({ + organizationId: actor.organizationId, + file, + sessionId: chunk?.sessionId, + offset: chunk?.offset, + limit: chunk?.limit, + loadRows: () => + readMappedRows(file!, COMPANY_ALIASES, parseExcelMapping(mappingRaw)), + }); const result: ImportContactsResult = { created: 0, skipped: 0, errors: [] }; - for (let i = 0; i < rows.length; i++) { - const row = rows[i]!; - const rowNum = i + 2; + for (let i = 0; i < ctx.slice.length; i++) { + const row = ctx.slice[i]!; + const rowNum = ctx.offset + i + 2; const name = row.name?.trim() ?? ''; if (!name) { result.errors.push({ row: rowNum, message: 'نام شرکت الزامی است' }); @@ -518,7 +546,7 @@ export class ContactsExcelService { } } - return result; + return { ...result, ...chunkImportMeta(ctx) }; } private async findOrCreateCompanyId(actor: Actor, name: string): Promise { diff --git a/apps/api/src/contacts/people.controller.ts b/apps/api/src/contacts/people.controller.ts index 46af7a2..5937514 100644 --- a/apps/api/src/contacts/people.controller.ts +++ b/apps/api/src/contacts/people.controller.ts @@ -82,11 +82,18 @@ export class PeopleController { @CurrentUser() user: Actor, @UploadedFile() file?: Express.Multer.File, @Body('mapping') mapping?: string, + @Body('sessionId') sessionId?: string, + @Body('offset') offset?: string, + @Body('limit') limit?: string, ) { - if (!file?.buffer?.length) { + if (!file?.buffer?.length && !sessionId) { throw new BadRequestException('فایل ارسال نشده'); } - return this.excelService.importPeople(user, file, mapping); + return this.excelService.importPeople(user, file, mapping, { + sessionId, + offset, + limit, + }); } @Get(':id') diff --git a/apps/api/src/pipelines/deals-excel.service.ts b/apps/api/src/pipelines/deals-excel.service.ts index 41bb106..5b98f7c 100644 --- a/apps/api/src/pipelines/deals-excel.service.ts +++ b/apps/api/src/pipelines/deals-excel.service.ts @@ -17,6 +17,10 @@ import { writeXlsx, type ExcelPreviewResult, } from '../common/excel'; +import { + chunkImportMeta, + resolveExcelImportChunk, +} from '../common/excel-import-session'; import { PipelinesService } from './pipelines.service'; const DEAL_COLUMNS = [ @@ -154,16 +158,25 @@ export class DealsExcelService { async import( actor: Actor, - file: Express.Multer.File, + file: Express.Multer.File | undefined, mappingRaw?: string, + chunk?: { sessionId?: string; offset?: string | number; limit?: string | number }, ): Promise { - const rows = await readMappedRows(file, DEAL_ALIASES, parseExcelMapping(mappingRaw)); + const ctx = await resolveExcelImportChunk({ + organizationId: actor.organizationId, + file, + sessionId: chunk?.sessionId, + offset: chunk?.offset, + limit: chunk?.limit, + loadRows: () => + readMappedRows(file!, DEAL_ALIASES, parseExcelMapping(mappingRaw)), + }); const result: ImportContactsResult = { created: 0, skipped: 0, errors: [] }; const affected = new Map(); - for (let i = 0; i < rows.length; i++) { - const row = rows[i]!; - const rowNum = i + 2; + for (let i = 0; i < ctx.slice.length; i++) { + const row = ctx.slice[i]!; + const rowNum = ctx.offset + i + 2; if ((row._type || '').trim() && row._type.trim().toLowerCase() !== 'deal') { continue; } @@ -235,7 +248,7 @@ export class DealsExcelService { result.boardId = entries[0][0]; result.boards = entries.map(([, n]) => n); } - return result; + return { ...result, ...chunkImportMeta(ctx) }; } private async resolvePipelineAndStage( diff --git a/apps/api/src/pipelines/deals.controller.ts b/apps/api/src/pipelines/deals.controller.ts index 76a3170..7c10d5d 100644 --- a/apps/api/src/pipelines/deals.controller.ts +++ b/apps/api/src/pipelines/deals.controller.ts @@ -70,10 +70,17 @@ export class DealsController { ) import( @CurrentUser() user: Actor, - @UploadedFile() file: Express.Multer.File, + @UploadedFile() file: Express.Multer.File | undefined, @Body('mapping') mapping?: string, + @Body('sessionId') sessionId?: string, + @Body('offset') offset?: string, + @Body('limit') limit?: string, ) { - return this.excelService.import(user, file, mapping); + return this.excelService.import(user, file, mapping, { + sessionId, + offset, + limit, + }); } @Get(':id') diff --git a/apps/api/src/products/products-excel.service.ts b/apps/api/src/products/products-excel.service.ts index ee19ff0..a55e3a7 100644 --- a/apps/api/src/products/products-excel.service.ts +++ b/apps/api/src/products/products-excel.service.ts @@ -10,6 +10,10 @@ import { writeXlsx, type ExcelPreviewResult, } from '../common/excel'; +import { + chunkImportMeta, + resolveExcelImportChunk, +} from '../common/excel-import-session'; import { ProductsService } from './products.service'; const PRODUCT_COLUMNS = [ @@ -116,15 +120,24 @@ export class ProductsExcelService { async import( organizationId: string, - file: Express.Multer.File, + file: Express.Multer.File | undefined, mappingRaw?: string, + chunk?: { sessionId?: string; offset?: string | number; limit?: string | number }, ): Promise { - const rows = await readMappedRows(file, PRODUCT_ALIASES, parseExcelMapping(mappingRaw)); + const ctx = await resolveExcelImportChunk({ + organizationId, + file, + sessionId: chunk?.sessionId, + offset: chunk?.offset, + limit: chunk?.limit, + loadRows: () => + readMappedRows(file!, PRODUCT_ALIASES, parseExcelMapping(mappingRaw)), + }); const result: ImportContactsResult = { created: 0, skipped: 0, errors: [] }; - for (let i = 0; i < rows.length; i++) { - const row = rows[i]!; - const rowNum = i + 2; + for (let i = 0; i < ctx.slice.length; i++) { + const row = ctx.slice[i]!; + const rowNum = ctx.offset + i + 2; if ((row._type || '').trim() && row._type.trim().toLowerCase() !== 'product') { continue; } @@ -176,6 +189,6 @@ export class ProductsExcelService { } } - return result; + return { ...result, ...chunkImportMeta(ctx) }; } } diff --git a/apps/api/src/products/products.controller.ts b/apps/api/src/products/products.controller.ts index 3f7caf8..cf46dc7 100644 --- a/apps/api/src/products/products.controller.ts +++ b/apps/api/src/products/products.controller.ts @@ -84,10 +84,17 @@ export class ProductsController { ) import( @CurrentUser() user: Actor, - @UploadedFile() file: Express.Multer.File, + @UploadedFile() file: Express.Multer.File | undefined, @Body('mapping') mapping?: string, + @Body('sessionId') sessionId?: string, + @Body('offset') offset?: string, + @Body('limit') limit?: string, ) { - return this.excelService.import(user.organizationId, file, mapping); + return this.excelService.import(user.organizationId, file, mapping, { + sessionId, + offset, + limit, + }); } @Get(':id') diff --git a/apps/api/src/project-boards/project-boards.controller.ts b/apps/api/src/project-boards/project-boards.controller.ts index 97d7b34..120275b 100644 --- a/apps/api/src/project-boards/project-boards.controller.ts +++ b/apps/api/src/project-boards/project-boards.controller.ts @@ -148,10 +148,17 @@ export class ProjectBoardsController { ) importCards( @CurrentUser() user: Actor, - @UploadedFile() file: Express.Multer.File, + @UploadedFile() file: Express.Multer.File | undefined, @Body('mapping') mapping?: string, + @Body('sessionId') sessionId?: string, + @Body('offset') offset?: string, + @Body('limit') limit?: string, ) { - return this.excelService.import(user, file, mapping); + return this.excelService.import(user, file, mapping, { + sessionId, + offset, + limit, + }); } @Get('project-cards/:id') diff --git a/apps/api/src/project-boards/project-cards-excel.service.ts b/apps/api/src/project-boards/project-cards-excel.service.ts index 1dd465b..fb832ae 100644 --- a/apps/api/src/project-boards/project-cards-excel.service.ts +++ b/apps/api/src/project-boards/project-cards-excel.service.ts @@ -15,6 +15,10 @@ import { writeXlsx, type ExcelPreviewResult, } from '../common/excel'; +import { + chunkImportMeta, + resolveExcelImportChunk, +} from '../common/excel-import-session'; import { ProjectBoardsService } from './project-boards.service'; const CASE_COLUMNS = [ @@ -115,16 +119,25 @@ export class ProjectCardsExcelService { async import( actor: Actor, - file: Express.Multer.File, + file: Express.Multer.File | undefined, mappingRaw?: string, + chunk?: { sessionId?: string; offset?: string | number; limit?: string | number }, ): Promise { - const rows = await readMappedRows(file, CASE_ALIASES, parseExcelMapping(mappingRaw)); + const ctx = await resolveExcelImportChunk({ + organizationId: actor.organizationId, + file, + sessionId: chunk?.sessionId, + offset: chunk?.offset, + limit: chunk?.limit, + loadRows: () => + readMappedRows(file!, CASE_ALIASES, parseExcelMapping(mappingRaw)), + }); const result: ImportContactsResult = { created: 0, skipped: 0, errors: [] }; const affected = new Map(); - for (let i = 0; i < rows.length; i++) { - const row = rows[i]!; - const rowNum = i + 2; + for (let i = 0; i < ctx.slice.length; i++) { + const row = ctx.slice[i]!; + const rowNum = ctx.offset + i + 2; if ((row._type || '').trim() && row._type.trim().toLowerCase() !== 'case') { continue; } @@ -185,7 +198,7 @@ export class ProjectCardsExcelService { result.boards = entries.map(([, n]) => n); } - return result; + return { ...result, ...chunkImportMeta(ctx) }; } private async resolveBoardAndStage( diff --git a/apps/web/src/components/ExcelImportFlow.vue b/apps/web/src/components/ExcelImportFlow.vue index e351ad5..7daaf49 100644 --- a/apps/web/src/components/ExcelImportFlow.vue +++ b/apps/web/src/components/ExcelImportFlow.vue @@ -24,6 +24,7 @@ const preview = ref(null); const mapping = reactive>({}); const submitting = ref(false); const phase = ref<'idle' | 'preview' | 'import'>('idle'); +const progressLabel = ref(''); const requiredKeys = computed( () => preview.value?.fields.filter((f) => f.required).map((f) => f.key) ?? [], @@ -34,9 +35,12 @@ const mappedFields = computed(() => new Set(Object.values(mapping).filter(Boolea const canImport = computed(() => requiredKeys.value.every((key) => mappedFields.value.has(key))); const busyOpen = computed(() => submitting.value && phase.value !== 'idle'); -const busyMessage = computed(() => - phase.value === 'import' ? 'در حال ورود اطلاعات از اکسل…' : 'در حال خواندن و بررسی فایل اکسل…', -); +const busyMessage = computed(() => { + if (phase.value === 'import') { + return progressLabel.value || 'در حال ورود اطلاعات از اکسل…'; + } + return 'در حال خواندن و بررسی فایل اکسل…'; +}); function pick() { if (props.disabled || submitting.value) return; @@ -48,6 +52,7 @@ function reset() { preview.value = null; submitting.value = false; phase.value = 'idle'; + progressLabel.value = ''; for (const key of Object.keys(mapping)) delete mapping[key]; } @@ -66,21 +71,76 @@ function setColumnField(header: string, field: string) { mapping[header] = field; } -async function postImport(file: File, map?: Record) { - phase.value = 'import'; +function mergeImportResults( + acc: ImportContactsResult, + part: ImportContactsResult, +): ImportContactsResult { + const errors = [...acc.errors, ...(part.errors ?? [])]; + return { + created: acc.created + (part.created ?? 0), + skipped: acc.skipped + (part.skipped ?? 0), + errors: errors.slice(0, 200), + boardId: part.boardId ?? acc.boardId, + boards: part.boards?.length ? part.boards : acc.boards, + sessionId: part.sessionId, + total: part.total ?? acc.total, + processed: part.processed ?? acc.processed, + hasMore: part.hasMore, + nextOffset: part.nextOffset, + }; +} + +async function postImportChunk(opts: { + file?: File; + map?: Record; + sessionId?: string; + offset?: number; +}) { const form = new FormData(); - form.append('file', file); - if (map && Object.values(map).some(Boolean)) { - form.append('mapping', JSON.stringify(map)); + if (opts.file) form.append('file', opts.file); + if (opts.map && Object.values(opts.map).some(Boolean)) { + form.append('mapping', JSON.stringify(opts.map)); } - const { data } = await api.post>( - props.importPath, - form, - { - headers: { 'Content-Type': undefined }, - }, - ); - return data as ImportContactsResult; + if (opts.sessionId) form.append('sessionId', opts.sessionId); + if (opts.offset != null) form.append('offset', String(opts.offset)); + const { data } = await api.post(props.importPath, form, { + headers: { 'Content-Type': undefined }, + timeout: 3 * 60 * 1000, + }); + return data; +} + +async function runChunkedImport(file: File, map?: Record) { + phase.value = 'import'; + progressLabel.value = 'در حال ورود اطلاعات از اکسل…'; + + let acc: ImportContactsResult = { created: 0, skipped: 0, errors: [] }; + let sessionId: string | undefined; + let offset = 0; + let guard = 0; + + while (guard < 500) { + guard += 1; + const part = await postImportChunk({ + file: sessionId ? undefined : file, + map: sessionId ? undefined : map, + sessionId, + offset: sessionId ? offset : undefined, + }); + acc = mergeImportResults(acc, part); + + const total = part.total ?? 0; + const processed = part.processed ?? 0; + if (total > 0) { + progressLabel.value = `ورود چندمرحله‌ای: ${processed.toLocaleString('fa-IR')} از ${total.toLocaleString('fa-IR')} ردیف`; + } + + if (!part.hasMore || !part.sessionId) break; + sessionId = part.sessionId; + offset = part.nextOffset ?? processed; + } + + return acc; } function apiMessage(e: unknown, fallback: string) { @@ -106,9 +166,15 @@ async function onFile(ev: Event) { headers: { 'Content-Type': undefined }, }); if (!data.needsMapping) { - emit('done', await postImport(file)); - reset(); - emit('settled'); + try { + emit('done', await runChunkedImport(file)); + reset(); + emit('settled'); + } catch (impErr) { + emit('fail', apiMessage(impErr, 'ایمپورت ناموفق بود')); + reset(); + emit('settled'); + } return; } pending.value = file; @@ -133,13 +199,14 @@ async function confirmMapping() { submitting.value = true; phase.value = 'import'; try { - emit('done', await postImport(pending.value, { ...mapping })); + emit('done', await runChunkedImport(pending.value, { ...mapping })); reset(); emit('settled'); } catch (e) { emit('fail', apiMessage(e, 'ایمپورت ناموفق بود')); phase.value = 'idle'; submitting.value = false; + progressLabel.value = ''; emit('settled'); } } @@ -165,6 +232,9 @@ async function confirmMapping() {

ستون‌های این فایل با قالب سیستم یکی نیست. مشخص کنید هر ستون فایل در کدام فیلد ریخته شود.

+

+ فایل بزرگ است ({{ (preview.rowCount ?? 0).toLocaleString('fa-IR') }} ردیف) — ورود به‌صورت چندمرحله‌ای انجام می‌شود. +

@@ -209,6 +279,16 @@ async function confirmMapping() { line-height: 1.7; } +.large-hint { + margin: -0.35rem 0 1rem; + padding: 0.55rem 0.75rem; + border-radius: 10px; + background: color-mix(in srgb, var(--accent, #2563eb) 12%, transparent); + color: var(--fg, inherit); + font-size: 0.88rem; + line-height: 1.6; +} + .rows { display: grid; gap: 0.75rem; diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 46648ba..6343253 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -1251,6 +1251,15 @@ export interface ImportContactsResult { errors: Array<{ row: number; message: string }>; boardId?: string; boards?: string[]; + /** جلسه چندمرحله‌ای برای فایل‌های بزرگ */ + sessionId?: string; + /** تعداد کل ردیف‌های داده‌ای فایل */ + total?: number; + /** تا این آفست پردازش شده */ + processed?: number; + /** مرحله بعدی وجود دارد */ + hasMore?: boolean; + nextOffset?: number; } export interface ExcelFieldOption { @@ -1270,6 +1279,10 @@ export interface ExcelPreviewResult { fields: ExcelFieldOption[]; suggested: Record; needsMapping: boolean; + /** تعداد ردیف‌های غیرخالی (بدون هدر) */ + rowCount: number; + /** پیشنهاد ایمپورت چندمرحله‌ای */ + largeFile?: boolean; } export interface ProductDto {