diff --git a/apps/client/src/components/icons/confluence-icon.tsx b/apps/client/src/components/icons/confluence-icon.tsx new file mode 100644 index 00000000..499f18da --- /dev/null +++ b/apps/client/src/components/icons/confluence-icon.tsx @@ -0,0 +1,20 @@ +import { rem } from "@mantine/core"; + +interface Props { + size?: number | string; +} + +export function ConfluenceIcon({ size }: Props) { + return ( + + + + ); +} diff --git a/apps/client/src/features/file-task/services/file-task-service.ts b/apps/client/src/features/file-task/services/file-task-service.ts new file mode 100644 index 00000000..ffccbaae --- /dev/null +++ b/apps/client/src/features/file-task/services/file-task-service.ts @@ -0,0 +1,14 @@ +import api from "@/lib/api-client"; +import { IFileTask } from "@/features/file-task/types/file-task.types.ts"; + +export async function getFileTaskById(fileTaskId: string): Promise { + const req = await api.post("/file-tasks/info", { + fileTaskId: fileTaskId, + }); + return req.data; +} + +export async function getFileTasks(): Promise { + const req = await api.post("/file-tasks"); + return req.data; +} diff --git a/apps/client/src/features/file-task/types/file-task.types.ts b/apps/client/src/features/file-task/types/file-task.types.ts new file mode 100644 index 00000000..917e1757 --- /dev/null +++ b/apps/client/src/features/file-task/types/file-task.types.ts @@ -0,0 +1,17 @@ +export interface IFileTask { + id: string; + type: "import" | "export"; + source: string; + status: string; + fileName: string; + filePath: string; + fileSize: number; + fileExt: string; + errorMessage: string | null; + creatorId: string; + spaceId: string; + workspaceId: string; + createdAt: string; + updatedAt: string; + deletedAt: string | null; +} \ No newline at end of file diff --git a/apps/client/src/features/page/components/page-import-modal.tsx b/apps/client/src/features/page/components/page-import-modal.tsx index f07fd8a9..edf105cf 100644 --- a/apps/client/src/features/page/components/page-import-modal.tsx +++ b/apps/client/src/features/page/components/page-import-modal.tsx @@ -1,18 +1,36 @@ -import { Modal, Button, SimpleGrid, FileButton } from "@mantine/core"; import { + Modal, + Button, + SimpleGrid, + FileButton, + Group, + Text, + Tooltip, +} from "@mantine/core"; +import { + IconBrandNotion, IconCheck, IconFileCode, + IconFileTypeZip, IconMarkdown, IconX, } from "@tabler/icons-react"; -import { importPage } from "@/features/page/services/page-service.ts"; +import { + importPage, + importZip, +} from "@/features/page/services/page-service.ts"; import { notifications } from "@mantine/notifications"; import { treeDataAtom } from "@/features/page/tree/atoms/tree-data-atom.ts"; import { useAtom } from "jotai"; import { buildTree } from "@/features/page/tree/utils"; import { IPage } from "@/features/page/types/page.types.ts"; -import React from "react"; +import React, { useEffect, useState } from "react"; import { useTranslation } from "react-i18next"; +import { ConfluenceIcon } from "@/components/icons/confluence-icon.tsx"; +import { getFileImportSizeLimit, isCloud } from "@/lib/config.ts"; +import { formatBytes } from "@/lib"; +import { workspaceAtom } from "@/features/user/atoms/current-user-atom.ts"; +import { getFileTaskById } from "@/features/file-task/services/file-task-service.ts"; interface PageImportModalProps { spaceId: string; @@ -59,6 +77,113 @@ interface ImportFormatSelection { function ImportFormatSelection({ spaceId, onClose }: ImportFormatSelection) { const { t } = useTranslation(); const [treeData, setTreeData] = useAtom(treeDataAtom); + const [workspace] = useAtom(workspaceAtom); + const [fileTaskId, setFileTaskId] = useState(null); + + const canUseConfluence = isCloud() || workspace?.hasLicenseKey; + + const handleZipUpload = async (selectedFile: File, source: string) => { + if (!selectedFile) { + return; + } + + onClose(); + + try { + const importTask = await importZip(selectedFile, spaceId, source); + notifications.show({ + id: "import", + title: t("Importing pages"), + message: t( + "Page import is in progress. Refresh this tab after a while.", + ), + loading: true, + withCloseButton: false, + autoClose: false, + }); + + setFileTaskId(importTask.id); + console.log("taskId set", importTask.id); + } catch (err) { + console.log("Failed to import page", err); + notifications.update({ + id: "import", + color: "red", + title: t("Failed to import pages"), + message: t("Unable to import pages. Please try again."), + icon: , + loading: false, + withCloseButton: true, + autoClose: 5000, + }); + } + }; + + useEffect(() => { + if (!fileTaskId) return; + + const intervalId = setInterval(async () => { + try { + const fileTask = await getFileTaskById(fileTaskId); + const status = fileTask.status; + + if (status === "success") { + notifications.update({ + id: "import", + color: "teal", + title: t("Import complete"), + message: t("Your pages were successfully imported."), + icon: , + loading: false, + withCloseButton: true, + autoClose: 5000, + }); + clearInterval(intervalId); + setFileTaskId(null); + } + + if (status === "failed") { + notifications.update({ + id: "import", + color: "red", + title: t("Page import failed"), + message: t( + "Something went wrong while importing pages: {{reason}}.", + { + reason: fileTask.errorMessage, + }, + ), + icon: , + loading: false, + withCloseButton: true, + autoClose: 5000, + }); + clearInterval(intervalId); + setFileTaskId(null); + console.error(fileTask.errorMessage); + } + } catch (err) { + notifications.update({ + id: "import", + color: "red", + title: t("Import failed"), + message: t( + "Something went wrong while importing pages: {{reason}}.", + { + reason: err.response?.data.message, + }, + ), + icon: , + loading: false, + withCloseButton: true, + autoClose: 5000, + }); + clearInterval(intervalId); + setFileTaskId(null); + console.error("Failed to fetch import status", err); + } + }, 3000); + }, [fileTaskId]); const handleFileUpload = async (selectedFiles: File[]) => { if (!selectedFiles) { @@ -120,6 +245,7 @@ function ImportFormatSelection({ spaceId, onClose }: ImportFormatSelection) { } }; + // @ts-ignore return ( <> @@ -148,7 +274,76 @@ function ImportFormatSelection({ spaceId, onClose }: ImportFormatSelection) { )} + + handleZipUpload(file, "notion")} + accept="application/zip" + > + {(props) => ( + + )} + + handleZipUpload(file, "confluence")} + accept="application/zip" + > + {(props) => ( + + + + )} + + + +
+ + Import zip file + + + {t( + `Upload zip file containing Markdown and HTML files. Max: {{sizeLimit}}`, + { + sizeLimit: formatBytes(getFileImportSizeLimit()), + }, + )} + + handleZipUpload(file, "generic")} + accept="application/zip" + > + {(props) => ( + + + + )} + +
+ ); } diff --git a/apps/client/src/features/page/services/page-service.ts b/apps/client/src/features/page/services/page-service.ts index 5e69a34a..f058750f 100644 --- a/apps/client/src/features/page/services/page-service.ts +++ b/apps/client/src/features/page/services/page-service.ts @@ -7,9 +7,10 @@ import { IPage, IPageInput, SidebarPagesParams, -} from "@/features/page/types/page.types"; +} from '@/features/page/types/page.types'; import { IAttachment, IPagination } from "@/lib/types.ts"; import { saveAs } from "file-saver"; +import { IFileTask } from '@/features/file-task/types/file-task.types.ts'; export async function createPage(data: Partial): Promise { const req = await api.post("/pages/create", data); @@ -92,6 +93,25 @@ export async function importPage(file: File, spaceId: string) { return req.data; } +export async function importZip( + file: File, + spaceId: string, + source?: string, +): Promise { + const formData = new FormData(); + formData.append("spaceId", spaceId); + formData.append("source", source); + formData.append("file", file); + + const req = await api.post("/pages/import-zip", formData, { + headers: { + "Content-Type": "multipart/form-data", + }, + }); + + return req.data; +} + export async function uploadFile( file: File, pageId: string, diff --git a/apps/client/src/lib/config.ts b/apps/client/src/lib/config.ts index 2f621b91..717bf9ff 100644 --- a/apps/client/src/lib/config.ts +++ b/apps/client/src/lib/config.ts @@ -70,6 +70,11 @@ export function getFileUploadSizeLimit() { return bytes(limit); } +export function getFileImportSizeLimit() { + const limit = getConfigValue("FILE_IMPORT_SIZE_LIMIT", "200mb"); + return bytes(limit); +} + export function getDrawioUrl() { return getConfigValue("DRAWIO_URL", "https://embed.diagrams.net"); } diff --git a/apps/client/vite.config.ts b/apps/client/vite.config.ts index a6efc4bc..cc8a01fd 100644 --- a/apps/client/vite.config.ts +++ b/apps/client/vite.config.ts @@ -8,6 +8,7 @@ export default defineConfig(({ mode }) => { const { APP_URL, FILE_UPLOAD_SIZE_LIMIT, + FILE_IMPORT_SIZE_LIMIT, DRAWIO_URL, CLOUD, SUBDOMAIN_HOST, @@ -20,6 +21,7 @@ export default defineConfig(({ mode }) => { "process.env": { APP_URL, FILE_UPLOAD_SIZE_LIMIT, + FILE_IMPORT_SIZE_LIMIT, DRAWIO_URL, CLOUD, SUBDOMAIN_HOST, diff --git a/apps/server/src/database/migrations/20250521T154949-file_tasks.ts b/apps/server/src/database/migrations/20250521T154949-file_tasks.ts index 61a1d5f7..523ae86b 100644 --- a/apps/server/src/database/migrations/20250521T154949-file_tasks.ts +++ b/apps/server/src/database/migrations/20250521T154949-file_tasks.ts @@ -6,22 +6,17 @@ export async function up(db: Kysely): Promise { .addColumn('id', 'uuid', (col) => col.primaryKey().defaultTo(sql`gen_uuid_v7()`), ) - //type: import or export + // type (import, export) .addColumn('type', 'varchar', (col) => col) - // source - generic, notion, confluence - // type or provider? + // source (generic, notion, confluence) .addColumn('source', 'varchar', (col) => col) - // status (enum: PENDING|PROCESSING|SUCCESS|FAILED), + // status (pending|processing|success|failed), .addColumn('status', 'varchar', (col) => col) - // file name - // file path - // file size - .addColumn('file_name', 'varchar', (col) => col.notNull()) .addColumn('file_path', 'varchar', (col) => col.notNull()) .addColumn('file_size', 'int8', (col) => col) .addColumn('file_ext', 'varchar', (col) => col) - + .addColumn('error_message', 'varchar', (col) => col) .addColumn('creator_id', 'uuid', (col) => col.references('users.id')) .addColumn('space_id', 'uuid', (col) => col.references('spaces.id').onDelete('cascade'), @@ -35,7 +30,6 @@ export async function up(db: Kysely): Promise { .addColumn('updated_at', 'timestamptz', (col) => col.notNull().defaultTo(sql`now()`), ) - .addColumn('completed_at', 'timestamptz', (col) => col) .addColumn('deleted_at', 'timestamptz', (col) => col) .execute(); } diff --git a/apps/server/src/database/types/db.d.ts b/apps/server/src/database/types/db.d.ts index 69f352f8..4545ebc4 100644 --- a/apps/server/src/database/types/db.d.ts +++ b/apps/server/src/database/types/db.d.ts @@ -123,10 +123,10 @@ export interface Comments { } export interface FileTasks { - completedAt: Timestamp | null; createdAt: Generated; creatorId: string | null; deletedAt: Timestamp | null; + errorMessage: string | null; fileExt: string | null; fileName: string; filePath: string; diff --git a/apps/server/src/integrations/environment/environment.service.ts b/apps/server/src/integrations/environment/environment.service.ts index ac26b4fb..d6336993 100644 --- a/apps/server/src/integrations/environment/environment.service.ts +++ b/apps/server/src/integrations/environment/environment.service.ts @@ -67,6 +67,10 @@ export class EnvironmentService { return this.configService.get('FILE_UPLOAD_SIZE_LIMIT', '50mb'); } + getFileImportSizeLimit(): string { + return this.configService.get('FILE_IMPORT_SIZE_LIMIT', '200mb'); + } + getAwsS3AccessKeyId(): string { return this.configService.get('AWS_S3_ACCESS_KEY_ID'); } diff --git a/apps/server/src/integrations/import/dto/file-task-dto.ts b/apps/server/src/integrations/import/dto/file-task-dto.ts new file mode 100644 index 00000000..3d100413 --- /dev/null +++ b/apps/server/src/integrations/import/dto/file-task-dto.ts @@ -0,0 +1,7 @@ +import { IsNotEmpty, IsUUID } from 'class-validator'; + +export class FileTaskIdDto { + @IsNotEmpty() + @IsUUID() + fileTaskId: string; +} diff --git a/apps/server/src/integrations/import/file-task.controller.ts b/apps/server/src/integrations/import/file-task.controller.ts new file mode 100644 index 00000000..305779b4 --- /dev/null +++ b/apps/server/src/integrations/import/file-task.controller.ts @@ -0,0 +1,79 @@ +import { + Body, + Controller, + ForbiddenException, + HttpCode, + HttpStatus, + NotFoundException, + Post, + UseGuards, +} from '@nestjs/common'; +import SpaceAbilityFactory from '../../core/casl/abilities/space-ability.factory'; +import { JwtAuthGuard } from '../../common/guards/jwt-auth.guard'; +import { User } from '@docmost/db/types/entity.types'; +import { + SpaceCaslAction, + SpaceCaslSubject, +} from '../../core/casl/interfaces/space-ability.type'; +import { InjectKysely } from 'nestjs-kysely'; +import { KyselyDB } from '@docmost/db/types/kysely.types'; +import { AuthUser } from '../../common/decorators/auth-user.decorator'; +import { FileTaskIdDto } from './dto/file-task-dto'; +import { SpaceMemberRepo } from '@docmost/db/repos/space/space-member.repo'; + +@Controller('file-tasks') +export class FileTaskController { + constructor( + private readonly spaceMemberRepo: SpaceMemberRepo, + private readonly spaceAbility: SpaceAbilityFactory, + @InjectKysely() private readonly db: KyselyDB, + ) {} + + @UseGuards(JwtAuthGuard) + @HttpCode(HttpStatus.OK) + @Post() + async getFileTasks(@AuthUser() user: User) { + const userSpaceIds = await this.spaceMemberRepo.getUserSpaceIds(user.id); + + if (!userSpaceIds || userSpaceIds.length === 0) { + return []; + } + + const fileTasks = await this.db + .selectFrom('fileTasks') + .selectAll() + .where('spaceId', 'in', userSpaceIds) + .execute(); + + if (!fileTasks) { + throw new NotFoundException('File task not found'); + } + + return fileTasks; + } + + @UseGuards(JwtAuthGuard) + @HttpCode(HttpStatus.OK) + @Post('info') + async getFileTask(@Body() dto: FileTaskIdDto, @AuthUser() user: User) { + const fileTask = await this.db + .selectFrom('fileTasks') + .selectAll() + .where('id', '=', dto.fileTaskId) + .executeTakeFirst(); + + if (!fileTask || !fileTask.spaceId) { + throw new NotFoundException('File task not found'); + } + + const ability = await this.spaceAbility.createForUser( + user, + fileTask.spaceId, + ); + if (ability.cannot(SpaceCaslAction.Read, SpaceCaslSubject.Page)) { + throw new ForbiddenException(); + } + + return fileTask; + } +} diff --git a/apps/server/src/integrations/import/import.controller.ts b/apps/server/src/integrations/import/import.controller.ts index abf68962..1adb82eb 100644 --- a/apps/server/src/integrations/import/import.controller.ts +++ b/apps/server/src/integrations/import/import.controller.ts @@ -23,6 +23,7 @@ import * as bytes from 'bytes'; import * as path from 'path'; import { ImportService } from './services/import.service'; import { AuthWorkspace } from '../../common/decorators/auth-workspace.decorator'; +import { EnvironmentService } from '../environment/environment.service'; @Controller() export class ImportController { @@ -31,6 +32,7 @@ export class ImportController { constructor( private readonly importService: ImportService, private readonly spaceAbility: SpaceAbilityFactory, + private readonly environmentService: EnvironmentService, ) {} @UseInterceptors(FileInterceptor) @@ -44,18 +46,18 @@ export class ImportController { ) { const validFileExtensions = ['.md', '.html']; - const maxFileSize = bytes('100mb'); + const maxFileSize = bytes('10mb'); let file = null; try { file = await req.file({ - limits: { fileSize: maxFileSize, fields: 3, files: 1 }, + limits: { fileSize: maxFileSize, fields: 4, files: 1 }, }); } catch (err: any) { this.logger.error(err.message); if (err?.statusCode === 413) { throw new BadRequestException( - `File too large. Exceeds the 100mb import limit`, + `File too large. Exceeds the 10mb import limit`, ); } } @@ -73,7 +75,7 @@ export class ImportController { const spaceId = file.fields?.spaceId?.value; if (!spaceId) { - throw new BadRequestException('spaceId or format not found'); + throw new BadRequestException('spaceId is required'); } const ability = await this.spaceAbility.createForUser(user, spaceId); @@ -87,7 +89,6 @@ export class ImportController { @UseInterceptors(FileInterceptor) @UseGuards(JwtAuthGuard) @HttpCode(HttpStatus.OK) - // temporary naming @Post('pages/import-zip') async importZip( @Req() req: any, @@ -96,7 +97,7 @@ export class ImportController { ) { const validFileExtensions = ['.zip']; - const maxFileSize = bytes('100mb'); + const maxFileSize = bytes(this.environmentService.getFileImportSizeLimit()); let file = null; try { @@ -107,7 +108,7 @@ export class ImportController { this.logger.error(err.message); if (err?.statusCode === 413) { throw new BadRequestException( - `File too large. Exceeds the 100mb import limit`, + `File too large. Exceeds the ${this.environmentService.getFileImportSizeLimit()} import limit`, ); } } @@ -119,14 +120,21 @@ export class ImportController { if ( !validFileExtensions.includes(path.extname(file.filename).toLowerCase()) ) { - throw new BadRequestException('Invalid import file type.'); + throw new BadRequestException('Invalid import file extension.'); } const spaceId = file.fields?.spaceId?.value; const source = file.fields?.source?.value; + const validZipSources = ['generic', 'notion', 'confluence']; + if (!validZipSources.includes(source)) { + throw new BadRequestException( + 'Invalid import source. Import source must either be generic, notion or confluence.', + ); + } + if (!spaceId) { - throw new BadRequestException('spaceId or format not found'); + throw new BadRequestException('spaceId is required'); } const ability = await this.spaceAbility.createForUser(user, spaceId); @@ -134,6 +142,12 @@ export class ImportController { throw new ForbiddenException(); } - return this.importService.importZip(file, source, user.id, spaceId, workspace.id); + return this.importService.importZip( + file, + source, + user.id, + spaceId, + workspace.id, + ); } } diff --git a/apps/server/src/integrations/import/import.module.ts b/apps/server/src/integrations/import/import.module.ts index 83022af3..1b5611f0 100644 --- a/apps/server/src/integrations/import/import.module.ts +++ b/apps/server/src/integrations/import/import.module.ts @@ -5,6 +5,7 @@ import { StorageModule } from '../storage/storage.module'; import { FileTaskService } from './services/file-task.service'; import { FileTaskProcessor } from './processors/file-task.processor'; import { ImportAttachmentService } from './services/import-attachment.service'; +import { FileTaskController } from './file-task.controller'; @Module({ providers: [ @@ -14,7 +15,7 @@ import { ImportAttachmentService } from './services/import-attachment.service'; ImportAttachmentService, ], exports: [ImportService, ImportAttachmentService], - controllers: [ImportController], + controllers: [ImportController, FileTaskController], imports: [StorageModule], }) export class ImportModule {} diff --git a/apps/server/src/integrations/import/processors/file-task.processor.ts b/apps/server/src/integrations/import/processors/file-task.processor.ts index 31fefe98..2985fba0 100644 --- a/apps/server/src/integrations/import/processors/file-task.processor.ts +++ b/apps/server/src/integrations/import/processors/file-task.processor.ts @@ -3,11 +3,17 @@ import { OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq'; import { Job } from 'bullmq'; import { QueueJob, QueueName } from 'src/integrations/queue/constants'; import { FileTaskService } from '../services/file-task.service'; +import { FileTaskStatus } from '../utils/file.utils'; +import { StorageService } from '../../storage/storage.service'; @Processor(QueueName.FILE_TASK_QUEUE) export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy { private readonly logger = new Logger(FileTaskProcessor.name); - constructor(private readonly fileTaskService: FileTaskService) { + + constructor( + private readonly fileTaskService: FileTaskService, + private readonly storageService: StorageService, + ) { super(); } @@ -18,10 +24,11 @@ export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy { await this.fileTaskService.processZIpImport(job.data.fileTaskId); break; case QueueJob.EXPORT_TASK: - console.log('export task', job.data.fileTaskId); + // TODO: export task + break; } } catch (err) { - console.error(err); + this.logger.error('File task failed', err); throw err; } } @@ -32,15 +39,45 @@ export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy { } @OnWorkerEvent('failed') - onError(job: Job) { + async onFailed(job: Job) { this.logger.error( `Error processing ${job.name} job. Reason: ${job.failedReason}`, ); + + const MAX_JOB_ATTEMPTS = 3; + const fileTaskId = job.data.fileTaskId; + + if (job.attemptsMade >= MAX_JOB_ATTEMPTS) { + this.logger.error(`Max import attempts reached for Task ${fileTaskId}.`); + await this.fileTaskService.updateTaskStatus( + fileTaskId, + FileTaskStatus.Failed, + job.failedReason, + ); + + try { + const fileTask = await this.fileTaskService.getFileTask(fileTaskId); + if (fileTask) { + await this.storageService.delete(fileTask.filePath); + } + } catch (err) { + this.logger.error(err); + } + } + } + + @OnWorkerEvent('stalled') + async onStalled(job: Job) { + this.logger.error( + `Stalled processing ${job.name} job. Reason: ${job.failedReason}`, + ); } @OnWorkerEvent('completed') onCompleted(job: Job) { - this.logger.debug(`Completed ${job.name} job`); + this.logger.log( + `Completed ${job.name} job for File task ID ${job.data.fileTaskId}`, + ); } async onModuleDestroy(): Promise { diff --git a/apps/server/src/integrations/import/services/file-task.service.ts b/apps/server/src/integrations/import/services/file-task.service.ts index a6136d7d..62bcd130 100644 --- a/apps/server/src/integrations/import/services/file-task.service.ts +++ b/apps/server/src/integrations/import/services/file-task.service.ts @@ -5,7 +5,7 @@ import { InjectKysely } from 'nestjs-kysely'; import { KyselyDB } from '@docmost/db/types/kysely.types'; import { extractZip, - FileImportType, + FileImportSource, FileTaskStatus, } from '../utils/file.utils'; import { StorageService } from '../../storage/storage.service'; @@ -40,7 +40,6 @@ export class FileTaskService { private readonly backlinkRepo: BacklinkRepo, @InjectKysely() private readonly db: KyselyDB, private readonly importAttachmentService: ImportAttachmentService, - // private readonly confluenceTaskService: ConfluenceImportService, private moduleRef: ModuleRef, ) {} @@ -72,15 +71,23 @@ export class FileTaskService { unsafeCleanup: true, }); - const fileStream = await this.storageService.readStream(fileTask.filePath); - await pipeline(fileStream, createWriteStream(tmpZipPath)); + try { + const fileStream = await this.storageService.readStream( + fileTask.filePath, + ); + await pipeline(fileStream, createWriteStream(tmpZipPath)); + await extractZip(tmpZipPath, tmpExtractDir); + } catch (err) { + await cleanupTmpFile(); + await cleanupTmpDir(); - await extractZip(tmpZipPath, tmpExtractDir); + throw err; + } try { if ( - fileTask.source === FileImportType.Generic || - fileTask.source === FileImportType.Notion + fileTask.source === FileImportSource.Generic || + fileTask.source === FileImportSource.Notion ) { await this.processGenericImport({ extractDir: tmpExtractDir, @@ -88,7 +95,7 @@ export class FileTaskService { }); } - if (fileTask.source === FileImportType.Confluence) { + if (fileTask.source === FileImportSource.Confluence) { let ConfluenceModule: any; try { // eslint-disable-next-line @typescript-eslint/no-require-imports @@ -109,13 +116,21 @@ export class FileTaskService { fileTask, }); } - await this.updateTaskStatus(fileTaskId, FileTaskStatus.Success); - } catch (error) { - await this.updateTaskStatus(fileTaskId, FileTaskStatus.Failed); - this.logger.error(error); - } finally { + try { + await this.updateTaskStatus(fileTaskId, FileTaskStatus.Success, null); + // delete stored file on success + await this.storageService.delete(fileTask.filePath); + } catch (err) { + this.logger.error( + `Failed to delete import file from storage. Task ID: ${fileTaskId}`, + err, + ); + } + } catch (err) { await cleanupTmpFile(); await cleanupTmpDir(); + + throw err; } } @@ -279,11 +294,27 @@ export class FileTaskService { }); } - async updateTaskStatus(fileTaskId: string, status: FileTaskStatus) { - await this.db - .updateTable('fileTasks') - .set({ status: status }) + async getFileTask(fileTaskId: string) { + return this.db + .selectFrom('fileTasks') + .selectAll() .where('id', '=', fileTaskId) - .execute(); + .executeTakeFirst(); + } + + async updateTaskStatus( + fileTaskId: string, + status: FileTaskStatus, + errorMessage?: string, + ) { + try { + await this.db + .updateTable('fileTasks') + .set({ status: status, errorMessage, updatedAt: new Date() }) + .where('id', '=', fileTaskId) + .execute(); + } catch (err) { + this.logger.error(err); + } } } diff --git a/apps/server/src/integrations/import/services/import.service.ts b/apps/server/src/integrations/import/services/import.service.ts index 9612004f..a3da4918 100644 --- a/apps/server/src/integrations/import/services/import.service.ts +++ b/apps/server/src/integrations/import/services/import.service.ts @@ -10,7 +10,7 @@ import { } from '../../../collaboration/collaboration.util'; import { InjectKysely } from 'nestjs-kysely'; import { KyselyDB } from '@docmost/db/types/kysely.types'; -import { generateSlugId } from '../../../common/helpers'; +import { generateSlugId, sanitizeFileName } from '../../../common/helpers'; import { generateJitteredKeyBetween } from 'fractional-indexing-jittered'; import { TiptapTransformer } from '@hocuspocus/transformer'; import * as Y from 'yjs'; @@ -20,15 +20,11 @@ import { FileTaskType, getFileTaskFolderPath, } from '../utils/file.utils'; -import { v7, v7 as uuid7 } from 'uuid'; +import { v7 as uuid7 } from 'uuid'; import { StorageService } from '../../storage/storage.service'; import { InjectQueue } from '@nestjs/bullmq'; import { Queue } from 'bullmq'; import { QueueJob, QueueName } from '../../queue/constants'; -import { Node as PMNode } from '@tiptap/pm/model'; -import { EditorState, Transaction } from '@tiptap/pm/state'; -import { getSchema } from '@tiptap/core'; -import { FileTask } from '@docmost/db/types/entity.types'; @Injectable() export class ImportService { @@ -204,13 +200,15 @@ export class ImportService { const file = await filePromise; const fileBuffer = await file.toBuffer(); const fileExtension = path.extname(file.filename).toLowerCase(); - const fileName = sanitize( - path.basename(file.filename, fileExtension).slice(0, 255), + const fileName = sanitizeFileName( + path.basename(file.filename, fileExtension), ); const fileSize = fileBuffer.length; + const fileNameWithExt = fileName + fileExtension; + const fileTaskId = uuid7(); - const filePath = `${getFileTaskFolderPath(FileTaskType.Import, workspaceId)}/${fileTaskId}/${fileName}`; + const filePath = `${getFileTaskFolderPath(FileTaskType.Import, workspaceId)}/${fileTaskId}/${fileNameWithExt}`; // upload file await this.storageService.upload(filePath, fileBuffer); @@ -222,7 +220,7 @@ export class ImportService { type: FileTaskType.Import, source: source, status: FileTaskStatus.Processing, - fileName: fileName, + fileName: fileNameWithExt, filePath: filePath, fileSize: fileSize, fileExt: 'zip', @@ -231,7 +229,7 @@ export class ImportService { workspaceId: workspaceId, }) .returningAll() - .execute(); + .executeTakeFirst(); await this.fileTaskQueue.add(QueueJob.IMPORT_TASK, { fileTaskId: fileTaskId, @@ -239,89 +237,4 @@ export class ImportService { return fileTask; } - - async markdownOrHtmlToProsemirror( - fileContent: string, - fileExtension: string, - ): Promise { - let prosemirrorState = ''; - if (fileExtension === '.md') { - prosemirrorState = await this.processMarkdown(fileContent); - } else if (fileExtension.endsWith('.html')) { - prosemirrorState = await this.processHTML(fileContent); - } - return prosemirrorState; - } - - async convertInternalLinksToMentionsPM( - doc: PMNode, - currentFilePath: string, - filePathToPageMetaMap: Map< - string, - { id: string; title: string; slugId: string } - >, - ): Promise { - const schema = getSchema(tiptapExtensions); - const state = EditorState.create({ doc, schema }); - let tr: Transaction = state.tr; - - const normalizePath = (p: string) => p.replace(/\\/g, '/'); - - // Collect replacements from the original doc. - const replacements: Array<{ - from: number; - to: number; - mentionNode: PMNode; - }> = []; - - doc.descendants((node, pos) => { - if (!node.isText || !node.marks?.length) return; - - // Look for the link mark - const linkMark = node.marks.find( - (mark) => mark.type.name === 'link' && mark.attrs?.href, - ); - if (!linkMark) return; - - // Compute the range for the entire text node. - const from = pos; - const to = pos + node.nodeSize; - - // Resolve the path and get page meta. - const resolvedPath = normalizePath( - path.join(path.dirname(currentFilePath), linkMark.attrs.href), - ); - const pageMeta = filePathToPageMetaMap.get(resolvedPath); - if (!pageMeta) return; - - // Create the mention node with all required attributes. - const mentionNode = schema.nodes.mention.create({ - id: v7(), - entityType: 'page', - entityId: pageMeta.id, - label: node.text || pageMeta.title, - slugId: pageMeta.slugId, - creatorId: 'not available', // This is required per your schema. - }); - - replacements.push({ from, to, mentionNode }); - }); - - // Apply replacements in reverse order. - for (let i = replacements.length - 1; i >= 0; i--) { - const { from, to, mentionNode } = replacements[i]; - try { - tr = tr.replaceWith(from, to, mentionNode); - } catch (err) { - console.error('❌ Failed to insert mention:', err); - } - } - if (tr.docChanged) { - console.log('doc changed'); - console.log(JSON.stringify(state.apply(tr).doc.toJSON())); - } - - // Return the updated document if any change was made. - return tr.docChanged ? state.apply(tr).doc : doc; - } } diff --git a/apps/server/src/integrations/import/utils/file.utils.ts b/apps/server/src/integrations/import/utils/file.utils.ts index 4105042b..b3d39cda 100644 --- a/apps/server/src/integrations/import/utils/file.utils.ts +++ b/apps/server/src/integrations/import/utils/file.utils.ts @@ -7,7 +7,7 @@ export enum FileTaskType { Export = 'export', } -export enum FileImportType { +export enum FileImportSource { Generic = 'generic', Notion = 'notion', Confluence = 'confluence', diff --git a/apps/server/src/integrations/queue/queue.module.ts b/apps/server/src/integrations/queue/queue.module.ts index c8f7f79c..d8d3c244 100644 --- a/apps/server/src/integrations/queue/queue.module.ts +++ b/apps/server/src/integrations/queue/queue.module.ts @@ -51,6 +51,10 @@ import { BacklinksProcessor } from './processors/backlinks.processor'; }), BullModule.registerQueue({ name: QueueName.FILE_TASK_QUEUE, + defaultJobOptions: { + removeOnComplete: true, + removeOnFail: true, + }, }), ], exports: [BullModule],