Fixing the error when importing a large Excel file

This commit is contained in:
2026-10-03 15:01:01 +03:30
parent f8313c0ea5
commit 2e315c9cfd
15 changed files with 503 additions and 86 deletions
@@ -1,4 +1,4 @@
import { Injectable } from '@nestjs/common'; import { BadRequestException, Injectable } from '@nestjs/common';
import type { ActivityType, ImportContactsResult } from '@mento/shared'; import type { ActivityType, ImportContactsResult } from '@mento/shared';
import { PrismaService } from '../prisma/prisma.service'; import { PrismaService } from '../prisma/prisma.service';
import type { Actor } from '../common/record-scope'; import type { Actor } from '../common/record-scope';
@@ -16,6 +16,10 @@ import {
writeXlsx, writeXlsx,
type ExcelPreviewResult, type ExcelPreviewResult,
} from '../common/excel'; } from '../common/excel';
import {
chunkImportMeta,
resolveExcelImportChunk,
} from '../common/excel-import-session';
const ACTIVITY_COLUMNS = [ const ACTIVITY_COLUMNS = [
{ header: '_type', key: '_type', width: 12 }, { header: '_type', key: '_type', width: 12 },
@@ -156,15 +160,52 @@ export class ActivitiesExcelService {
async import( async import(
actor: Actor, actor: Actor,
file: Express.Multer.File, file: Express.Multer.File | undefined,
mappingRaw?: string, mappingRaw?: string,
chunk?: { sessionId?: string; offset?: string | number; limit?: string | number },
): Promise<ImportContactsResult> { ): Promise<ImportContactsResult> {
const rows = await readMappedRows(file, ACTIVITY_ALIASES, parseExcelMapping(mappingRaw));
const result: ImportContactsResult = { created: 0, skipped: 0, errors: [] }; const result: ImportContactsResult = { created: 0, skipped: 0, errors: [] };
let ctx;
try {
ctx = await resolveExcelImportChunk({
organizationId: actor.organizationId,
file,
sessionId: chunk?.sessionId,
offset: chunk?.offset,
limit: chunk?.limit,
loadRows: () =>
readMappedRows(file!, ACTIVITY_ALIASES, parseExcelMapping(mappingRaw)),
});
} catch (e) {
if (e instanceof BadRequestException) throw e;
const message =
e instanceof Error ? e.message : 'خواندن فایل اکسل ناموفق بود';
result.errors.push({ row: 1, message });
return result;
}
for (let i = 0; i < rows.length; i++) { const MAX_ROWS = 25000;
const row = rows[i]!; if (ctx.total > MAX_ROWS) {
const rowNum = i + 2; result.errors.push({
row: 1,
message: `فایل بیش از ${MAX_ROWS} ردیف دارد.`,
});
return result;
}
const userCache = new Map<string, string | null>();
const resolveUser = async (name?: string) => {
const key = (name ?? '').trim().toLowerCase();
if (!key) return null;
if (userCache.has(key)) return userCache.get(key) ?? null;
const id = await this.findUserId(actor.organizationId, name);
userCache.set(key, id);
return id;
};
for (let i = 0; i < ctx.slice.length; i++) {
const row = ctx.slice[i]!;
const rowNum = ctx.offset + i + 2;
if ((row._type || '').trim() && row._type.trim().toLowerCase() !== 'activity') { if ((row._type || '').trim() && row._type.trim().toLowerCase() !== 'activity') {
continue; continue;
} }
@@ -191,13 +232,13 @@ export class ActivitiesExcelService {
.filter(Boolean); .filter(Boolean);
const assigneeIds: string[] = []; const assigneeIds: string[] = [];
for (const name of assigneeNames) { for (const name of assigneeNames) {
const uid = await this.findUserId(actor.organizationId, name); const uid = await resolveUser(name);
if (uid && !assigneeIds.includes(uid)) assigneeIds.push(uid); if (uid && !assigneeIds.includes(uid)) assigneeIds.push(uid);
} }
if (!assigneeIds.length) assigneeIds.push(actor.userId); if (!assigneeIds.length) assigneeIds.push(actor.userId);
const assigneeId = assigneeIds[0]!; const assigneeId = assigneeIds[0]!;
const creatorId = const creatorId =
(await this.findUserId(actor.organizationId, row.creator)) ?? actor.userId; (await resolveUser(row.creator)) ?? actor.userId;
const duplicate = dueAt const duplicate = dueAt
? await this.prisma.activity.findFirst({ ? await this.prisma.activity.findFirst({
@@ -215,7 +256,7 @@ export class ActivitiesExcelService {
continue; continue;
} }
await this.prisma.activity.create({ const created = await this.prisma.activity.create({
data: { data: {
organizationId: actor.organizationId, organizationId: actor.organizationId,
title, title,
@@ -230,10 +271,23 @@ export class ActivitiesExcelService {
companyId, companyId,
dealId, dealId,
assigneeId, assigneeId,
assignees: { connect: assigneeIds.map((id) => ({ id })) },
createdById: creatorId, createdById: creatorId,
}, },
select: { id: true },
}); });
// M2M مسئولان — اگر مایگریشن روی سرور نباشد، ایمپورت نباید بترکد
if (assigneeIds.length) {
try {
await this.prisma.activity.update({
where: { id: created.id },
data: {
assignees: { set: assigneeIds.map((id) => ({ id })) },
},
});
} catch {
/* assigneeId به‌تنهایی کافی است */
}
}
result.created += 1; result.created += 1;
} catch (e) { } catch (e) {
result.errors.push({ result.errors.push({
@@ -243,7 +297,7 @@ export class ActivitiesExcelService {
} }
} }
return result; return { ...result, ...chunkImportMeta(ctx) };
} }
private async findPersonId(organizationId: string, row: Record<string, string>) { private async findPersonId(organizationId: string, row: Record<string, string>) {
@@ -1,4 +1,5 @@
import { import {
BadRequestException,
Body, Body,
Controller, Controller,
Delete, Delete,
@@ -28,6 +29,11 @@ import {
} from './dto'; } from './dto';
import { brand } from '@mento/shared'; import { brand } from '@mento/shared';
const excelUpload = FileInterceptor('file', {
storage: memoryStorage(),
limits: { fileSize: 15_000_000 },
});
@Controller('activities') @Controller('activities')
@UseGuards(JwtAuthGuard) @UseGuards(JwtAuthGuard)
export class ActivitiesController { export class ActivitiesController {
@@ -86,29 +92,32 @@ export class ActivitiesController {
} }
@Post('import/preview') @Post('import/preview')
@UseInterceptors( @UseInterceptors(excelUpload)
FileInterceptor('file', { importPreview(@UploadedFile() file?: Express.Multer.File) {
storage: memoryStorage(), if (!file?.buffer?.length) {
limits: { fileSize: 15_000_000 }, throw new BadRequestException('فایل ارسال نشده');
}), }
)
importPreview(@UploadedFile() file: Express.Multer.File) {
return this.excelService.preview(file); return this.excelService.preview(file);
} }
@Post('import') @Post('import')
@UseInterceptors( @UseInterceptors(excelUpload)
FileInterceptor('file', {
storage: memoryStorage(),
limits: { fileSize: 15_000_000 },
}),
)
import( import(
@CurrentUser() user: Actor, @CurrentUser() user: Actor,
@UploadedFile() file: Express.Multer.File, @UploadedFile() file?: Express.Multer.File,
@Body('mapping') mapping?: string, @Body('mapping') mapping?: string,
@Body('sessionId') sessionId?: string,
@Body('offset') offset?: string,
@Body('limit') limit?: string,
) { ) {
return this.excelService.import(user, file, mapping); if (!file?.buffer?.length && !sessionId) {
throw new BadRequestException('فایل ارسال نشده');
}
return this.excelService.import(user, file, mapping, {
sessionId,
offset,
limit,
});
} }
@Get(':id') @Get(':id')
+142
View File
@@ -0,0 +1,142 @@
import { randomBytes } from 'node:crypto';
import { BadRequestException } from '@nestjs/common';
export const EXCEL_IMPORT_CHUNK_SIZE = 80;
/** بعد از این تعداد ردیف، کلاینت حتماً چندمرحله‌ای می‌رود (پیش‌نمایش هم نشان می‌دهد) */
export const EXCEL_IMPORT_LARGE_THRESHOLD = 80;
const SESSION_TTL_MS = 45 * 60 * 1000;
const MAX_SESSIONS = 40;
type ExcelImportSession = {
organizationId: string;
rows: Array<Record<string, string>>;
expiresAt: number;
};
const sessions = new Map<string, ExcelImportSession>();
function purgeExpired() {
const now = Date.now();
for (const [id, session] of sessions) {
if (session.expiresAt <= now) sessions.delete(id);
}
if (sessions.size <= MAX_SESSIONS) return;
const ordered = [...sessions.entries()].sort(
(a, b) => a[1].expiresAt - b[1].expiresAt,
);
while (sessions.size > MAX_SESSIONS && ordered.length) {
const [id] = ordered.shift()!;
sessions.delete(id);
}
}
export function createExcelImportSession(
organizationId: string,
rows: Array<Record<string, string>>,
): string {
purgeExpired();
const id = randomBytes(16).toString('hex');
sessions.set(id, {
organizationId,
rows,
expiresAt: Date.now() + SESSION_TTL_MS,
});
return id;
}
export function getExcelImportSession(
sessionId: string,
organizationId: string,
): Array<Record<string, string>> {
purgeExpired();
const session = sessions.get(sessionId);
if (!session || session.organizationId !== organizationId) {
throw new BadRequestException(
'جلسه ورود اکسل منقضی یا نامعتبر است. فایل را دوباره انتخاب کنید.',
);
}
session.expiresAt = Date.now() + SESSION_TTL_MS;
return session.rows;
}
export function destroyExcelImportSession(sessionId: string | undefined) {
if (!sessionId) return;
sessions.delete(sessionId);
}
export function parseChunkParams(raw: {
offset?: string | number;
limit?: string | number;
}): { offset: number; limit: number } {
const offset = Math.max(0, Number(raw.offset ?? 0) || 0);
const limit = Math.min(
250,
Math.max(1, Number(raw.limit ?? EXCEL_IMPORT_CHUNK_SIZE) || EXCEL_IMPORT_CHUNK_SIZE),
);
return { offset, limit };
}
export type ExcelImportChunkContext = {
sessionId: string;
allRows: Array<Record<string, string>>;
slice: Array<Record<string, string>>;
offset: number;
nextOffset: number;
hasMore: boolean;
total: number;
};
/**
* فایل را یک‌بار می‌خواند و در حافظه نگه می‌دارد؛ درخواست‌های بعدی فقط با sessionId + offset.
*/
export async function resolveExcelImportChunk(opts: {
organizationId: string;
file?: Express.Multer.File | null;
sessionId?: string;
offset?: string | number;
limit?: string | number;
loadRows: () => Promise<Array<Record<string, string>>>;
}): Promise<ExcelImportChunkContext> {
const { offset, limit } = parseChunkParams({
offset: opts.offset,
limit: opts.limit,
});
let sessionId = (opts.sessionId ?? '').trim();
let allRows: Array<Record<string, string>>;
if (sessionId) {
allRows = getExcelImportSession(sessionId, opts.organizationId);
} else {
if (!opts.file?.buffer?.length) {
throw new BadRequestException('فایل ارسال نشده');
}
allRows = await opts.loadRows();
sessionId = createExcelImportSession(opts.organizationId, allRows);
}
const slice = allRows.slice(offset, offset + limit);
const nextOffset = offset + slice.length;
const hasMore = nextOffset < allRows.length;
if (!hasMore) destroyExcelImportSession(sessionId);
return {
sessionId,
allRows,
slice,
offset,
nextOffset,
hasMore,
total: allRows.length,
};
}
export function chunkImportMeta(ctx: ExcelImportChunkContext) {
return {
sessionId: ctx.hasMore ? ctx.sessionId : undefined,
total: ctx.total,
processed: ctx.nextOffset,
hasMore: ctx.hasMore,
nextOffset: ctx.hasMore ? ctx.nextOffset : undefined,
};
}
+17
View File
@@ -1,5 +1,6 @@
import { BadRequestException } from '@nestjs/common'; import { BadRequestException } from '@nestjs/common';
import ExcelJS from 'exceljs'; import ExcelJS from 'exceljs';
import { EXCEL_IMPORT_LARGE_THRESHOLD } from './excel-import-session';
export function cellStr(value: ExcelJS.CellValue | undefined): string { export function cellStr(value: ExcelJS.CellValue | undefined): string {
try { try {
@@ -229,6 +230,8 @@ export type ExcelPreviewResult = {
fields: ExcelFieldOption[]; fields: ExcelFieldOption[];
suggested: Record<string, string>; suggested: Record<string, string>;
needsMapping: boolean; needsMapping: boolean;
rowCount: number;
largeFile?: boolean;
}; };
export function excelFieldsFromColumns( export function excelFieldsFromColumns(
@@ -315,11 +318,25 @@ export async function previewExcel(
const mapped = new Set(Object.values(suggested)); const mapped = new Set(Object.values(suggested));
const missingRequired = required.some((key) => !mapped.has(key)); const missingRequired = required.some((key) => !mapped.has(key));
const unrecognized = columns.some((col) => !suggested[col.header]); const unrecognized = columns.some((col) => !suggested[col.header]);
let rowCount = 0;
sheet.eachRow((row, rowNumber) => {
if (rowNumber === 1) return;
let any = false;
row.eachCell((cell) => {
if (any) return;
if (cellStr(cell.value)) any = true;
});
if (any) rowCount += 1;
});
return { return {
columns, columns,
fields, fields,
suggested, suggested,
needsMapping: unrecognized || missingRequired || !mapped.size, needsMapping: unrecognized || missingRequired || !mapped.size,
rowCount,
largeFile: rowCount >= EXCEL_IMPORT_LARGE_THRESHOLD,
}; };
} }
@@ -78,11 +78,18 @@ export class CompaniesController {
@CurrentUser() user: Actor, @CurrentUser() user: Actor,
@UploadedFile() file?: Express.Multer.File, @UploadedFile() file?: Express.Multer.File,
@Body('mapping') mapping?: string, @Body('mapping') mapping?: string,
@Body('sessionId') sessionId?: string,
@Body('offset') offset?: string,
@Body('limit') limit?: string,
) { ) {
if (!file?.buffer?.length) { if (!file?.buffer?.length && !sessionId) {
throw new BadRequestException('فایل ارسال نشده'); throw new BadRequestException('فایل ارسال نشده');
} }
return this.excelService.importCompanies(user, file, mapping); return this.excelService.importCompanies(user, file, mapping, {
sessionId,
offset,
limit,
});
} }
@Get(':id') @Get(':id')
+40 -12
View File
@@ -16,9 +16,19 @@ import {
writeXlsx, writeXlsx,
type ExcelPreviewResult, type ExcelPreviewResult,
} from '../common/excel'; } from '../common/excel';
import {
chunkImportMeta,
resolveExcelImportChunk,
} from '../common/excel-import-session';
import { CompaniesService } from './companies.service'; import { CompaniesService } from './companies.service';
import { PeopleService } from './people.service'; import { PeopleService } from './people.service';
type ImportChunkOpts = {
sessionId?: string;
offset?: string | number;
limit?: string | number;
};
function parseMood(raw?: string): PersonMood | undefined { function parseMood(raw?: string): PersonMood | undefined {
const value = raw?.trim(); const value = raw?.trim();
if (!value) return undefined; if (!value) return undefined;
@@ -363,16 +373,25 @@ export class ContactsExcelService {
async importPeople( async importPeople(
actor: Actor, actor: Actor,
file: Express.Multer.File, file: Express.Multer.File | undefined,
mappingRaw?: string, mappingRaw?: string,
chunk?: ImportChunkOpts,
): Promise<ImportContactsResult> { ): Promise<ImportContactsResult> {
const rows = await readMappedRows(file, PERSON_ALIASES, parseExcelMapping(mappingRaw)); const ctx = await resolveExcelImportChunk({
organizationId: actor.organizationId,
file,
sessionId: chunk?.sessionId,
offset: chunk?.offset,
limit: chunk?.limit,
loadRows: () =>
readMappedRows(file!, PERSON_ALIASES, parseExcelMapping(mappingRaw)),
});
const result: ImportContactsResult = { created: 0, skipped: 0, errors: [] }; const result: ImportContactsResult = { created: 0, skipped: 0, errors: [] };
const ownerCache = new Map<string, string | null>(); const ownerCache = new Map<string, string | null>();
for (let i = 0; i < rows.length; i++) { for (let i = 0; i < ctx.slice.length; i++) {
const row = rows[i]!; const row = ctx.slice[i]!;
const rowNum = i + 2; const rowNum = ctx.offset + i + 2;
if ((row._type || '').trim() && row._type.trim().toLowerCase() !== 'contact') { if ((row._type || '').trim() && row._type.trim().toLowerCase() !== 'contact') {
continue; continue;
} }
@@ -476,20 +495,29 @@ export class ContactsExcelService {
} }
} }
return result; return { ...result, ...chunkImportMeta(ctx) };
} }
async importCompanies( async importCompanies(
actor: Actor, actor: Actor,
file: Express.Multer.File, file: Express.Multer.File | undefined,
mappingRaw?: string, mappingRaw?: string,
chunk?: ImportChunkOpts,
): Promise<ImportContactsResult> { ): Promise<ImportContactsResult> {
const rows = await readMappedRows(file, COMPANY_ALIASES, parseExcelMapping(mappingRaw)); const ctx = await resolveExcelImportChunk({
organizationId: actor.organizationId,
file,
sessionId: chunk?.sessionId,
offset: chunk?.offset,
limit: chunk?.limit,
loadRows: () =>
readMappedRows(file!, COMPANY_ALIASES, parseExcelMapping(mappingRaw)),
});
const result: ImportContactsResult = { created: 0, skipped: 0, errors: [] }; const result: ImportContactsResult = { created: 0, skipped: 0, errors: [] };
for (let i = 0; i < rows.length; i++) { for (let i = 0; i < ctx.slice.length; i++) {
const row = rows[i]!; const row = ctx.slice[i]!;
const rowNum = i + 2; const rowNum = ctx.offset + i + 2;
const name = row.name?.trim() ?? ''; const name = row.name?.trim() ?? '';
if (!name) { if (!name) {
result.errors.push({ row: rowNum, message: 'نام شرکت الزامی است' }); result.errors.push({ row: rowNum, message: 'نام شرکت الزامی است' });
@@ -518,7 +546,7 @@ export class ContactsExcelService {
} }
} }
return result; return { ...result, ...chunkImportMeta(ctx) };
} }
private async findOrCreateCompanyId(actor: Actor, name: string): Promise<string> { private async findOrCreateCompanyId(actor: Actor, name: string): Promise<string> {
+9 -2
View File
@@ -82,11 +82,18 @@ export class PeopleController {
@CurrentUser() user: Actor, @CurrentUser() user: Actor,
@UploadedFile() file?: Express.Multer.File, @UploadedFile() file?: Express.Multer.File,
@Body('mapping') mapping?: string, @Body('mapping') mapping?: string,
@Body('sessionId') sessionId?: string,
@Body('offset') offset?: string,
@Body('limit') limit?: string,
) { ) {
if (!file?.buffer?.length) { if (!file?.buffer?.length && !sessionId) {
throw new BadRequestException('فایل ارسال نشده'); throw new BadRequestException('فایل ارسال نشده');
} }
return this.excelService.importPeople(user, file, mapping); return this.excelService.importPeople(user, file, mapping, {
sessionId,
offset,
limit,
});
} }
@Get(':id') @Get(':id')
+19 -6
View File
@@ -17,6 +17,10 @@ import {
writeXlsx, writeXlsx,
type ExcelPreviewResult, type ExcelPreviewResult,
} from '../common/excel'; } from '../common/excel';
import {
chunkImportMeta,
resolveExcelImportChunk,
} from '../common/excel-import-session';
import { PipelinesService } from './pipelines.service'; import { PipelinesService } from './pipelines.service';
const DEAL_COLUMNS = [ const DEAL_COLUMNS = [
@@ -154,16 +158,25 @@ export class DealsExcelService {
async import( async import(
actor: Actor, actor: Actor,
file: Express.Multer.File, file: Express.Multer.File | undefined,
mappingRaw?: string, mappingRaw?: string,
chunk?: { sessionId?: string; offset?: string | number; limit?: string | number },
): Promise<ImportContactsResult> { ): Promise<ImportContactsResult> {
const rows = await readMappedRows(file, DEAL_ALIASES, parseExcelMapping(mappingRaw)); const ctx = await resolveExcelImportChunk({
organizationId: actor.organizationId,
file,
sessionId: chunk?.sessionId,
offset: chunk?.offset,
limit: chunk?.limit,
loadRows: () =>
readMappedRows(file!, DEAL_ALIASES, parseExcelMapping(mappingRaw)),
});
const result: ImportContactsResult = { created: 0, skipped: 0, errors: [] }; const result: ImportContactsResult = { created: 0, skipped: 0, errors: [] };
const affected = new Map<string, string>(); const affected = new Map<string, string>();
for (let i = 0; i < rows.length; i++) { for (let i = 0; i < ctx.slice.length; i++) {
const row = rows[i]!; const row = ctx.slice[i]!;
const rowNum = i + 2; const rowNum = ctx.offset + i + 2;
if ((row._type || '').trim() && row._type.trim().toLowerCase() !== 'deal') { if ((row._type || '').trim() && row._type.trim().toLowerCase() !== 'deal') {
continue; continue;
} }
@@ -235,7 +248,7 @@ export class DealsExcelService {
result.boardId = entries[0][0]; result.boardId = entries[0][0];
result.boards = entries.map(([, n]) => n); result.boards = entries.map(([, n]) => n);
} }
return result; return { ...result, ...chunkImportMeta(ctx) };
} }
private async resolvePipelineAndStage( private async resolvePipelineAndStage(
+9 -2
View File
@@ -70,10 +70,17 @@ export class DealsController {
) )
import( import(
@CurrentUser() user: Actor, @CurrentUser() user: Actor,
@UploadedFile() file: Express.Multer.File, @UploadedFile() file: Express.Multer.File | undefined,
@Body('mapping') mapping?: string, @Body('mapping') mapping?: string,
@Body('sessionId') sessionId?: string,
@Body('offset') offset?: string,
@Body('limit') limit?: string,
) { ) {
return this.excelService.import(user, file, mapping); return this.excelService.import(user, file, mapping, {
sessionId,
offset,
limit,
});
} }
@Get(':id') @Get(':id')
@@ -10,6 +10,10 @@ import {
writeXlsx, writeXlsx,
type ExcelPreviewResult, type ExcelPreviewResult,
} from '../common/excel'; } from '../common/excel';
import {
chunkImportMeta,
resolveExcelImportChunk,
} from '../common/excel-import-session';
import { ProductsService } from './products.service'; import { ProductsService } from './products.service';
const PRODUCT_COLUMNS = [ const PRODUCT_COLUMNS = [
@@ -116,15 +120,24 @@ export class ProductsExcelService {
async import( async import(
organizationId: string, organizationId: string,
file: Express.Multer.File, file: Express.Multer.File | undefined,
mappingRaw?: string, mappingRaw?: string,
chunk?: { sessionId?: string; offset?: string | number; limit?: string | number },
): Promise<ImportContactsResult> { ): Promise<ImportContactsResult> {
const rows = await readMappedRows(file, PRODUCT_ALIASES, parseExcelMapping(mappingRaw)); const ctx = await resolveExcelImportChunk({
organizationId,
file,
sessionId: chunk?.sessionId,
offset: chunk?.offset,
limit: chunk?.limit,
loadRows: () =>
readMappedRows(file!, PRODUCT_ALIASES, parseExcelMapping(mappingRaw)),
});
const result: ImportContactsResult = { created: 0, skipped: 0, errors: [] }; const result: ImportContactsResult = { created: 0, skipped: 0, errors: [] };
for (let i = 0; i < rows.length; i++) { for (let i = 0; i < ctx.slice.length; i++) {
const row = rows[i]!; const row = ctx.slice[i]!;
const rowNum = i + 2; const rowNum = ctx.offset + i + 2;
if ((row._type || '').trim() && row._type.trim().toLowerCase() !== 'product') { if ((row._type || '').trim() && row._type.trim().toLowerCase() !== 'product') {
continue; continue;
} }
@@ -176,6 +189,6 @@ export class ProductsExcelService {
} }
} }
return result; return { ...result, ...chunkImportMeta(ctx) };
} }
} }
+9 -2
View File
@@ -84,10 +84,17 @@ export class ProductsController {
) )
import( import(
@CurrentUser() user: Actor, @CurrentUser() user: Actor,
@UploadedFile() file: Express.Multer.File, @UploadedFile() file: Express.Multer.File | undefined,
@Body('mapping') mapping?: string, @Body('mapping') mapping?: string,
@Body('sessionId') sessionId?: string,
@Body('offset') offset?: string,
@Body('limit') limit?: string,
) { ) {
return this.excelService.import(user.organizationId, file, mapping); return this.excelService.import(user.organizationId, file, mapping, {
sessionId,
offset,
limit,
});
} }
@Get(':id') @Get(':id')
@@ -148,10 +148,17 @@ export class ProjectBoardsController {
) )
importCards( importCards(
@CurrentUser() user: Actor, @CurrentUser() user: Actor,
@UploadedFile() file: Express.Multer.File, @UploadedFile() file: Express.Multer.File | undefined,
@Body('mapping') mapping?: string, @Body('mapping') mapping?: string,
@Body('sessionId') sessionId?: string,
@Body('offset') offset?: string,
@Body('limit') limit?: string,
) { ) {
return this.excelService.import(user, file, mapping); return this.excelService.import(user, file, mapping, {
sessionId,
offset,
limit,
});
} }
@Get('project-cards/:id') @Get('project-cards/:id')
@@ -15,6 +15,10 @@ import {
writeXlsx, writeXlsx,
type ExcelPreviewResult, type ExcelPreviewResult,
} from '../common/excel'; } from '../common/excel';
import {
chunkImportMeta,
resolveExcelImportChunk,
} from '../common/excel-import-session';
import { ProjectBoardsService } from './project-boards.service'; import { ProjectBoardsService } from './project-boards.service';
const CASE_COLUMNS = [ const CASE_COLUMNS = [
@@ -115,16 +119,25 @@ export class ProjectCardsExcelService {
async import( async import(
actor: Actor, actor: Actor,
file: Express.Multer.File, file: Express.Multer.File | undefined,
mappingRaw?: string, mappingRaw?: string,
chunk?: { sessionId?: string; offset?: string | number; limit?: string | number },
): Promise<ImportContactsResult> { ): Promise<ImportContactsResult> {
const rows = await readMappedRows(file, CASE_ALIASES, parseExcelMapping(mappingRaw)); const ctx = await resolveExcelImportChunk({
organizationId: actor.organizationId,
file,
sessionId: chunk?.sessionId,
offset: chunk?.offset,
limit: chunk?.limit,
loadRows: () =>
readMappedRows(file!, CASE_ALIASES, parseExcelMapping(mappingRaw)),
});
const result: ImportContactsResult = { created: 0, skipped: 0, errors: [] }; const result: ImportContactsResult = { created: 0, skipped: 0, errors: [] };
const affected = new Map<string, string>(); const affected = new Map<string, string>();
for (let i = 0; i < rows.length; i++) { for (let i = 0; i < ctx.slice.length; i++) {
const row = rows[i]!; const row = ctx.slice[i]!;
const rowNum = i + 2; const rowNum = ctx.offset + i + 2;
if ((row._type || '').trim() && row._type.trim().toLowerCase() !== 'case') { if ((row._type || '').trim() && row._type.trim().toLowerCase() !== 'case') {
continue; continue;
} }
@@ -185,7 +198,7 @@ export class ProjectCardsExcelService {
result.boards = entries.map(([, n]) => n); result.boards = entries.map(([, n]) => n);
} }
return result; return { ...result, ...chunkImportMeta(ctx) };
} }
private async resolveBoardAndStage( private async resolveBoardAndStage(
+100 -20
View File
@@ -24,6 +24,7 @@ const preview = ref<ExcelPreviewResult | null>(null);
const mapping = reactive<Record<string, string>>({}); const mapping = reactive<Record<string, string>>({});
const submitting = ref(false); const submitting = ref(false);
const phase = ref<'idle' | 'preview' | 'import'>('idle'); const phase = ref<'idle' | 'preview' | 'import'>('idle');
const progressLabel = ref('');
const requiredKeys = computed( const requiredKeys = computed(
() => preview.value?.fields.filter((f) => f.required).map((f) => f.key) ?? [], () => preview.value?.fields.filter((f) => f.required).map((f) => f.key) ?? [],
@@ -34,9 +35,12 @@ const mappedFields = computed(() => new Set(Object.values(mapping).filter(Boolea
const canImport = computed(() => requiredKeys.value.every((key) => mappedFields.value.has(key))); const canImport = computed(() => requiredKeys.value.every((key) => mappedFields.value.has(key)));
const busyOpen = computed(() => submitting.value && phase.value !== 'idle'); const busyOpen = computed(() => submitting.value && phase.value !== 'idle');
const busyMessage = computed(() => const busyMessage = computed(() => {
phase.value === 'import' ? 'در حال ورود اطلاعات از اکسل…' : 'در حال خواندن و بررسی فایل اکسل…', if (phase.value === 'import') {
); return progressLabel.value || 'در حال ورود اطلاعات از اکسل…';
}
return 'در حال خواندن و بررسی فایل اکسل…';
});
function pick() { function pick() {
if (props.disabled || submitting.value) return; if (props.disabled || submitting.value) return;
@@ -48,6 +52,7 @@ function reset() {
preview.value = null; preview.value = null;
submitting.value = false; submitting.value = false;
phase.value = 'idle'; phase.value = 'idle';
progressLabel.value = '';
for (const key of Object.keys(mapping)) delete mapping[key]; for (const key of Object.keys(mapping)) delete mapping[key];
} }
@@ -66,21 +71,76 @@ function setColumnField(header: string, field: string) {
mapping[header] = field; mapping[header] = field;
} }
async function postImport(file: File, map?: Record<string, string>) { function mergeImportResults(
phase.value = 'import'; acc: ImportContactsResult,
part: ImportContactsResult,
): ImportContactsResult {
const errors = [...acc.errors, ...(part.errors ?? [])];
return {
created: acc.created + (part.created ?? 0),
skipped: acc.skipped + (part.skipped ?? 0),
errors: errors.slice(0, 200),
boardId: part.boardId ?? acc.boardId,
boards: part.boards?.length ? part.boards : acc.boards,
sessionId: part.sessionId,
total: part.total ?? acc.total,
processed: part.processed ?? acc.processed,
hasMore: part.hasMore,
nextOffset: part.nextOffset,
};
}
async function postImportChunk(opts: {
file?: File;
map?: Record<string, string>;
sessionId?: string;
offset?: number;
}) {
const form = new FormData(); const form = new FormData();
form.append('file', file); if (opts.file) form.append('file', opts.file);
if (map && Object.values(map).some(Boolean)) { if (opts.map && Object.values(opts.map).some(Boolean)) {
form.append('mapping', JSON.stringify(map)); form.append('mapping', JSON.stringify(opts.map));
} }
const { data } = await api.post<ImportContactsResult | Record<string, unknown>>( if (opts.sessionId) form.append('sessionId', opts.sessionId);
props.importPath, if (opts.offset != null) form.append('offset', String(opts.offset));
form, const { data } = await api.post<ImportContactsResult>(props.importPath, form, {
{ headers: { 'Content-Type': undefined },
headers: { 'Content-Type': undefined }, timeout: 3 * 60 * 1000,
}, });
); return data;
return data as ImportContactsResult; }
async function runChunkedImport(file: File, map?: Record<string, string>) {
phase.value = 'import';
progressLabel.value = 'در حال ورود اطلاعات از اکسل…';
let acc: ImportContactsResult = { created: 0, skipped: 0, errors: [] };
let sessionId: string | undefined;
let offset = 0;
let guard = 0;
while (guard < 500) {
guard += 1;
const part = await postImportChunk({
file: sessionId ? undefined : file,
map: sessionId ? undefined : map,
sessionId,
offset: sessionId ? offset : undefined,
});
acc = mergeImportResults(acc, part);
const total = part.total ?? 0;
const processed = part.processed ?? 0;
if (total > 0) {
progressLabel.value = `ورود چندمرحله‌ای: ${processed.toLocaleString('fa-IR')} از ${total.toLocaleString('fa-IR')} ردیف`;
}
if (!part.hasMore || !part.sessionId) break;
sessionId = part.sessionId;
offset = part.nextOffset ?? processed;
}
return acc;
} }
function apiMessage(e: unknown, fallback: string) { function apiMessage(e: unknown, fallback: string) {
@@ -106,9 +166,15 @@ async function onFile(ev: Event) {
headers: { 'Content-Type': undefined }, headers: { 'Content-Type': undefined },
}); });
if (!data.needsMapping) { if (!data.needsMapping) {
emit('done', await postImport(file)); try {
reset(); emit('done', await runChunkedImport(file));
emit('settled'); reset();
emit('settled');
} catch (impErr) {
emit('fail', apiMessage(impErr, 'ایمپورت ناموفق بود'));
reset();
emit('settled');
}
return; return;
} }
pending.value = file; pending.value = file;
@@ -133,13 +199,14 @@ async function confirmMapping() {
submitting.value = true; submitting.value = true;
phase.value = 'import'; phase.value = 'import';
try { try {
emit('done', await postImport(pending.value, { ...mapping })); emit('done', await runChunkedImport(pending.value, { ...mapping }));
reset(); reset();
emit('settled'); emit('settled');
} catch (e) { } catch (e) {
emit('fail', apiMessage(e, 'ایمپورت ناموفق بود')); emit('fail', apiMessage(e, 'ایمپورت ناموفق بود'));
phase.value = 'idle'; phase.value = 'idle';
submitting.value = false; submitting.value = false;
progressLabel.value = '';
emit('settled'); emit('settled');
} }
} }
@@ -165,6 +232,9 @@ async function confirmMapping() {
<p class="lead"> <p class="lead">
ستون‌های این فایل با قالب سیستم یکی نیست. مشخص کنید هر ستون فایل در کدام فیلد ریخته شود. ستون‌های این فایل با قالب سیستم یکی نیست. مشخص کنید هر ستون فایل در کدام فیلد ریخته شود.
</p> </p>
<p v-if="preview?.largeFile" class="large-hint">
فایل بزرگ است ({{ (preview.rowCount ?? 0).toLocaleString('fa-IR') }} ردیف) — ورود به‌صورت چندمرحله‌ای انجام می‌شود.
</p>
<div class="rows"> <div class="rows">
<div v-for="col in preview?.columns ?? []" :key="col.header" class="row"> <div v-for="col in preview?.columns ?? []" :key="col.header" class="row">
<div class="source"> <div class="source">
@@ -209,6 +279,16 @@ async function confirmMapping() {
line-height: 1.7; line-height: 1.7;
} }
.large-hint {
margin: -0.35rem 0 1rem;
padding: 0.55rem 0.75rem;
border-radius: 10px;
background: color-mix(in srgb, var(--accent, #2563eb) 12%, transparent);
color: var(--fg, inherit);
font-size: 0.88rem;
line-height: 1.6;
}
.rows { .rows {
display: grid; display: grid;
gap: 0.75rem; gap: 0.75rem;
+13
View File
@@ -1251,6 +1251,15 @@ export interface ImportContactsResult {
errors: Array<{ row: number; message: string }>; errors: Array<{ row: number; message: string }>;
boardId?: string; boardId?: string;
boards?: string[]; boards?: string[];
/** جلسه چندمرحله‌ای برای فایل‌های بزرگ */
sessionId?: string;
/** تعداد کل ردیف‌های داده‌ای فایل */
total?: number;
/** تا این آفست پردازش شده */
processed?: number;
/** مرحله بعدی وجود دارد */
hasMore?: boolean;
nextOffset?: number;
} }
export interface ExcelFieldOption { export interface ExcelFieldOption {
@@ -1270,6 +1279,10 @@ export interface ExcelPreviewResult {
fields: ExcelFieldOption[]; fields: ExcelFieldOption[];
suggested: Record<string, string>; suggested: Record<string, string>;
needsMapping: boolean; needsMapping: boolean;
/** تعداد ردیف‌های غیرخالی (بدون هدر) */
rowCount: number;
/** پیشنهاد ایمپورت چندمرحله‌ای */
largeFile?: boolean;
} }
export interface ProductDto { export interface ProductDto {