bug fixes and UI

apps/server/src/database/migrations/20250521T154949-file_tasks.ts

@@ -6,22 +6,17 @@ export async function up(db: Kysely<any>): Promise<void> {
     .addColumn('id', 'uuid', (col) =>
       col.primaryKey().defaultTo(sql`gen_uuid_v7()`),
     )
-    //type: import or export
+    // type (import, export)
     .addColumn('type', 'varchar', (col) => col)
-    // source - generic, notion, confluence
-    // type or provider?
+    // source (generic, notion, confluence)
     .addColumn('source', 'varchar', (col) => col)
-    // status (enum: PENDING|PROCESSING|SUCCESS|FAILED),
+    // status (pending|processing|success|failed),
     .addColumn('status', 'varchar', (col) => col)
-    // file name
-    // file path
-    // file size
-
     .addColumn('file_name', 'varchar', (col) => col.notNull())
     .addColumn('file_path', 'varchar', (col) => col.notNull())
     .addColumn('file_size', 'int8', (col) => col)
     .addColumn('file_ext', 'varchar', (col) => col)
-
+    .addColumn('error_message', 'varchar', (col) => col)
     .addColumn('creator_id', 'uuid', (col) => col.references('users.id'))
     .addColumn('space_id', 'uuid', (col) =>
       col.references('spaces.id').onDelete('cascade'),
@@ -35,7 +30,6 @@ export async function up(db: Kysely<any>): Promise<void> {
     .addColumn('updated_at', 'timestamptz', (col) =>
       col.notNull().defaultTo(sql`now()`),
     )
-    .addColumn('completed_at', 'timestamptz', (col) => col)
     .addColumn('deleted_at', 'timestamptz', (col) => col)
     .execute();
 }

apps/server/src/database/types/db.d.ts

@@ -123,10 +123,10 @@ export interface Comments {
 }
 
 export interface FileTasks {
-  completedAt: Timestamp | null;
   createdAt: Generated<Timestamp>;
   creatorId: string | null;
   deletedAt: Timestamp | null;
+  errorMessage: string | null;
   fileExt: string | null;
   fileName: string;
   filePath: string;

apps/server/src/integrations/environment/environment.service.ts

@@ -67,6 +67,10 @@ export class EnvironmentService {
     return this.configService.get<string>('FILE_UPLOAD_SIZE_LIMIT', '50mb');
   }
 
+  getFileImportSizeLimit(): string {
+    return this.configService.get<string>('FILE_IMPORT_SIZE_LIMIT', '200mb');
+  }
+
   getAwsS3AccessKeyId(): string {
     return this.configService.get<string>('AWS_S3_ACCESS_KEY_ID');
   }

apps/server/src/integrations/import/dto/file-task-dto.ts

@@ -0,0 +1,7 @@
+import { IsNotEmpty, IsUUID } from 'class-validator';
+
+export class FileTaskIdDto {
+  @IsNotEmpty()
+  @IsUUID()
+  fileTaskId: string;
+}

apps/server/src/integrations/import/file-task.controller.ts

@@ -0,0 +1,79 @@
+import {
+  Body,
+  Controller,
+  ForbiddenException,
+  HttpCode,
+  HttpStatus,
+  NotFoundException,
+  Post,
+  UseGuards,
+} from '@nestjs/common';
+import SpaceAbilityFactory from '../../core/casl/abilities/space-ability.factory';
+import { JwtAuthGuard } from '../../common/guards/jwt-auth.guard';
+import { User } from '@docmost/db/types/entity.types';
+import {
+  SpaceCaslAction,
+  SpaceCaslSubject,
+} from '../../core/casl/interfaces/space-ability.type';
+import { InjectKysely } from 'nestjs-kysely';
+import { KyselyDB } from '@docmost/db/types/kysely.types';
+import { AuthUser } from '../../common/decorators/auth-user.decorator';
+import { FileTaskIdDto } from './dto/file-task-dto';
+import { SpaceMemberRepo } from '@docmost/db/repos/space/space-member.repo';
+
+@Controller('file-tasks')
+export class FileTaskController {
+  constructor(
+    private readonly spaceMemberRepo: SpaceMemberRepo,
+    private readonly spaceAbility: SpaceAbilityFactory,
+    @InjectKysely() private readonly db: KyselyDB,
+  ) {}
+
+  @UseGuards(JwtAuthGuard)
+  @HttpCode(HttpStatus.OK)
+  @Post()
+  async getFileTasks(@AuthUser() user: User) {
+    const userSpaceIds = await this.spaceMemberRepo.getUserSpaceIds(user.id);
+
+    if (!userSpaceIds || userSpaceIds.length === 0) {
+      return [];
+    }
+
+    const fileTasks = await this.db
+      .selectFrom('fileTasks')
+      .selectAll()
+      .where('spaceId', 'in', userSpaceIds)
+      .execute();
+
+    if (!fileTasks) {
+      throw new NotFoundException('File task not found');
+    }
+
+    return fileTasks;
+  }
+
+  @UseGuards(JwtAuthGuard)
+  @HttpCode(HttpStatus.OK)
+  @Post('info')
+  async getFileTask(@Body() dto: FileTaskIdDto, @AuthUser() user: User) {
+    const fileTask = await this.db
+      .selectFrom('fileTasks')
+      .selectAll()
+      .where('id', '=', dto.fileTaskId)
+      .executeTakeFirst();
+
+    if (!fileTask || !fileTask.spaceId) {
+      throw new NotFoundException('File task not found');
+    }
+
+    const ability = await this.spaceAbility.createForUser(
+      user,
+      fileTask.spaceId,
+    );
+    if (ability.cannot(SpaceCaslAction.Read, SpaceCaslSubject.Page)) {
+      throw new ForbiddenException();
+    }
+
+    return fileTask;
+  }
+}

apps/server/src/integrations/import/import.controller.ts

@@ -23,6 +23,7 @@ import * as bytes from 'bytes';
 import * as path from 'path';
 import { ImportService } from './services/import.service';
 import { AuthWorkspace } from '../../common/decorators/auth-workspace.decorator';
+import { EnvironmentService } from '../environment/environment.service';
 
 @Controller()
 export class ImportController {
@@ -31,6 +32,7 @@ export class ImportController {
   constructor(
     private readonly importService: ImportService,
     private readonly spaceAbility: SpaceAbilityFactory,
+    private readonly environmentService: EnvironmentService,
   ) {}
 
   @UseInterceptors(FileInterceptor)
@@ -44,18 +46,18 @@ export class ImportController {
   ) {
     const validFileExtensions = ['.md', '.html'];
 
-    const maxFileSize = bytes('100mb');
+    const maxFileSize = bytes('10mb');
 
     let file = null;
     try {
       file = await req.file({
-        limits: { fileSize: maxFileSize, fields: 3, files: 1 },
+        limits: { fileSize: maxFileSize, fields: 4, files: 1 },
       });
     } catch (err: any) {
       this.logger.error(err.message);
       if (err?.statusCode === 413) {
         throw new BadRequestException(
-          `File too large. Exceeds the 100mb import limit`,
+          `File too large. Exceeds the 10mb import limit`,
         );
       }
     }
@@ -73,7 +75,7 @@ export class ImportController {
     const spaceId = file.fields?.spaceId?.value;
 
     if (!spaceId) {
-      throw new BadRequestException('spaceId or format not found');
+      throw new BadRequestException('spaceId is required');
     }
 
     const ability = await this.spaceAbility.createForUser(user, spaceId);
@@ -87,7 +89,6 @@ export class ImportController {
   @UseInterceptors(FileInterceptor)
   @UseGuards(JwtAuthGuard)
   @HttpCode(HttpStatus.OK)
-  // temporary naming
   @Post('pages/import-zip')
   async importZip(
     @Req() req: any,
@@ -96,7 +97,7 @@ export class ImportController {
   ) {
     const validFileExtensions = ['.zip'];
 
-    const maxFileSize = bytes('100mb');
+    const maxFileSize = bytes(this.environmentService.getFileImportSizeLimit());
 
     let file = null;
     try {
@@ -107,7 +108,7 @@ export class ImportController {
       this.logger.error(err.message);
       if (err?.statusCode === 413) {
         throw new BadRequestException(
-          `File too large. Exceeds the 100mb import limit`,
+          `File too large. Exceeds the ${this.environmentService.getFileImportSizeLimit()} import limit`,
         );
       }
     }
@@ -119,14 +120,21 @@ export class ImportController {
     if (
       !validFileExtensions.includes(path.extname(file.filename).toLowerCase())
     ) {
-      throw new BadRequestException('Invalid import file type.');
+      throw new BadRequestException('Invalid import file extension.');
     }
 
     const spaceId = file.fields?.spaceId?.value;
     const source = file.fields?.source?.value;
 
+    const validZipSources = ['generic', 'notion', 'confluence'];
+    if (!validZipSources.includes(source)) {
+      throw new BadRequestException(
+        'Invalid import source. Import source must either be generic, notion or confluence.',
+      );
+    }
+
     if (!spaceId) {
-      throw new BadRequestException('spaceId or format not found');
+      throw new BadRequestException('spaceId is required');
     }
 
     const ability = await this.spaceAbility.createForUser(user, spaceId);
@@ -134,6 +142,12 @@ export class ImportController {
       throw new ForbiddenException();
     }
 
-    return this.importService.importZip(file, source, user.id, spaceId, workspace.id);
+    return this.importService.importZip(
+      file,
+      source,
+      user.id,
+      spaceId,
+      workspace.id,
+    );
   }
 }

apps/server/src/integrations/import/import.module.ts

@@ -5,6 +5,7 @@ import { StorageModule } from '../storage/storage.module';
 import { FileTaskService } from './services/file-task.service';
 import { FileTaskProcessor } from './processors/file-task.processor';
 import { ImportAttachmentService } from './services/import-attachment.service';
+import { FileTaskController } from './file-task.controller';
 
 @Module({
   providers: [
@@ -14,7 +15,7 @@ import { ImportAttachmentService } from './services/import-attachment.service';
     ImportAttachmentService,
   ],
   exports: [ImportService, ImportAttachmentService],
-  controllers: [ImportController],
+  controllers: [ImportController, FileTaskController],
   imports: [StorageModule],
 })
 export class ImportModule {}

apps/server/src/integrations/import/processors/file-task.processor.ts

@@ -3,11 +3,17 @@ import { OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq';
 import { Job } from 'bullmq';
 import { QueueJob, QueueName } from 'src/integrations/queue/constants';
 import { FileTaskService } from '../services/file-task.service';
+import { FileTaskStatus } from '../utils/file.utils';
+import { StorageService } from '../../storage/storage.service';
 
 @Processor(QueueName.FILE_TASK_QUEUE)
 export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy {
   private readonly logger = new Logger(FileTaskProcessor.name);
-  constructor(private readonly fileTaskService: FileTaskService) {
+
+  constructor(
+    private readonly fileTaskService: FileTaskService,
+    private readonly storageService: StorageService,
+  ) {
     super();
   }
 
@@ -18,10 +24,11 @@ export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy {
           await this.fileTaskService.processZIpImport(job.data.fileTaskId);
           break;
         case QueueJob.EXPORT_TASK:
-          console.log('export task', job.data.fileTaskId);
+          // TODO: export task
+          break;
       }
     } catch (err) {
-      console.error(err);
+      this.logger.error('File task failed', err);
       throw err;
     }
   }
@@ -32,15 +39,45 @@ export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy {
   }
 
   @OnWorkerEvent('failed')
-  onError(job: Job) {
+  async onFailed(job: Job) {
     this.logger.error(
       `Error processing ${job.name} job. Reason: ${job.failedReason}`,
     );
+
+    const MAX_JOB_ATTEMPTS = 3;
+    const fileTaskId = job.data.fileTaskId;
+
+    if (job.attemptsMade >= MAX_JOB_ATTEMPTS) {
+      this.logger.error(`Max import attempts reached for Task ${fileTaskId}.`);
+      await this.fileTaskService.updateTaskStatus(
+        fileTaskId,
+        FileTaskStatus.Failed,
+        job.failedReason,
+      );
+
+      try {
+        const fileTask = await this.fileTaskService.getFileTask(fileTaskId);
+        if (fileTask) {
+          await this.storageService.delete(fileTask.filePath);
+        }
+      } catch (err) {
+        this.logger.error(err);
+      }
+    }
+  }
+
+  @OnWorkerEvent('stalled')
+  async onStalled(job: Job) {
+    this.logger.error(
+      `Stalled processing ${job.name} job. Reason: ${job.failedReason}`,
+    );
   }
 
   @OnWorkerEvent('completed')
   onCompleted(job: Job) {
-    this.logger.debug(`Completed ${job.name} job`);
+    this.logger.log(
+      `Completed ${job.name} job for File task ID ${job.data.fileTaskId}`,
+    );
   }
 
   async onModuleDestroy(): Promise<void> {

apps/server/src/integrations/import/services/file-task.service.ts

@@ -5,7 +5,7 @@ import { InjectKysely } from 'nestjs-kysely';
 import { KyselyDB } from '@docmost/db/types/kysely.types';
 import {
   extractZip,
-  FileImportType,
+  FileImportSource,
   FileTaskStatus,
 } from '../utils/file.utils';
 import { StorageService } from '../../storage/storage.service';
@@ -40,7 +40,6 @@ export class FileTaskService {
     private readonly backlinkRepo: BacklinkRepo,
     @InjectKysely() private readonly db: KyselyDB,
     private readonly importAttachmentService: ImportAttachmentService,
-    // private readonly confluenceTaskService: ConfluenceImportService,
     private moduleRef: ModuleRef,
   ) {}
 
@@ -72,15 +71,23 @@ export class FileTaskService {
       unsafeCleanup: true,
     });
 
-    const fileStream = await this.storageService.readStream(fileTask.filePath);
-    await pipeline(fileStream, createWriteStream(tmpZipPath));
+    try {
+      const fileStream = await this.storageService.readStream(
+        fileTask.filePath,
+      );
+      await pipeline(fileStream, createWriteStream(tmpZipPath));
+      await extractZip(tmpZipPath, tmpExtractDir);
+    } catch (err) {
+      await cleanupTmpFile();
+      await cleanupTmpDir();
 
-    await extractZip(tmpZipPath, tmpExtractDir);
+      throw err;
+    }
 
     try {
       if (
-        fileTask.source === FileImportType.Generic ||
-        fileTask.source === FileImportType.Notion
+        fileTask.source === FileImportSource.Generic ||
+        fileTask.source === FileImportSource.Notion
       ) {
         await this.processGenericImport({
           extractDir: tmpExtractDir,
@@ -88,7 +95,7 @@ export class FileTaskService {
         });
       }
 
-      if (fileTask.source === FileImportType.Confluence) {
+      if (fileTask.source === FileImportSource.Confluence) {
         let ConfluenceModule: any;
         try {
           // eslint-disable-next-line @typescript-eslint/no-require-imports
@@ -109,13 +116,21 @@ export class FileTaskService {
           fileTask,
         });
       }
-      await this.updateTaskStatus(fileTaskId, FileTaskStatus.Success);
-    } catch (error) {
-      await this.updateTaskStatus(fileTaskId, FileTaskStatus.Failed);
-      this.logger.error(error);
-    } finally {
+      try {
+        await this.updateTaskStatus(fileTaskId, FileTaskStatus.Success, null);
+        // delete stored file on success
+        await this.storageService.delete(fileTask.filePath);
+      } catch (err) {
+        this.logger.error(
+          `Failed to delete import file from storage. Task ID: ${fileTaskId}`,
+          err,
+        );
+      }
+    } catch (err) {
       await cleanupTmpFile();
       await cleanupTmpDir();
+
+      throw err;
     }
   }
 
@@ -279,11 +294,27 @@ export class FileTaskService {
     });
   }
 
-  async updateTaskStatus(fileTaskId: string, status: FileTaskStatus) {
-    await this.db
-      .updateTable('fileTasks')
-      .set({ status: status })
+  async getFileTask(fileTaskId: string) {
+    return this.db
+      .selectFrom('fileTasks')
+      .selectAll()
       .where('id', '=', fileTaskId)
-      .execute();
+      .executeTakeFirst();
+  }
+
+  async updateTaskStatus(
+    fileTaskId: string,
+    status: FileTaskStatus,
+    errorMessage?: string,
+  ) {
+    try {
+      await this.db
+        .updateTable('fileTasks')
+        .set({ status: status, errorMessage, updatedAt: new Date() })
+        .where('id', '=', fileTaskId)
+        .execute();
+    } catch (err) {
+      this.logger.error(err);
+    }
   }
 }

apps/server/src/integrations/import/services/import.service.ts

@@ -10,7 +10,7 @@ import {
 } from '../../../collaboration/collaboration.util';
 import { InjectKysely } from 'nestjs-kysely';
 import { KyselyDB } from '@docmost/db/types/kysely.types';
-import { generateSlugId } from '../../../common/helpers';
+import { generateSlugId, sanitizeFileName } from '../../../common/helpers';
 import { generateJitteredKeyBetween } from 'fractional-indexing-jittered';
 import { TiptapTransformer } from '@hocuspocus/transformer';
 import * as Y from 'yjs';
@@ -20,15 +20,11 @@ import {
   FileTaskType,
   getFileTaskFolderPath,
 } from '../utils/file.utils';
-import { v7, v7 as uuid7 } from 'uuid';
+import { v7 as uuid7 } from 'uuid';
 import { StorageService } from '../../storage/storage.service';
 import { InjectQueue } from '@nestjs/bullmq';
 import { Queue } from 'bullmq';
 import { QueueJob, QueueName } from '../../queue/constants';
-import { Node as PMNode } from '@tiptap/pm/model';
-import { EditorState, Transaction } from '@tiptap/pm/state';
-import { getSchema } from '@tiptap/core';
-import { FileTask } from '@docmost/db/types/entity.types';
 
 @Injectable()
 export class ImportService {
@@ -204,13 +200,15 @@ export class ImportService {
     const file = await filePromise;
     const fileBuffer = await file.toBuffer();
     const fileExtension = path.extname(file.filename).toLowerCase();
-    const fileName = sanitize(
-      path.basename(file.filename, fileExtension).slice(0, 255),
+    const fileName = sanitizeFileName(
+      path.basename(file.filename, fileExtension),
     );
     const fileSize = fileBuffer.length;
 
+    const fileNameWithExt = fileName + fileExtension;
+
     const fileTaskId = uuid7();
-    const filePath = `${getFileTaskFolderPath(FileTaskType.Import, workspaceId)}/${fileTaskId}/${fileName}`;
+    const filePath = `${getFileTaskFolderPath(FileTaskType.Import, workspaceId)}/${fileTaskId}/${fileNameWithExt}`;
 
     // upload file
     await this.storageService.upload(filePath, fileBuffer);
@@ -222,7 +220,7 @@ export class ImportService {
         type: FileTaskType.Import,
         source: source,
         status: FileTaskStatus.Processing,
-        fileName: fileName,
+        fileName: fileNameWithExt,
         filePath: filePath,
         fileSize: fileSize,
         fileExt: 'zip',
@@ -231,7 +229,7 @@ export class ImportService {
         workspaceId: workspaceId,
       })
       .returningAll()
-      .execute();
+      .executeTakeFirst();
 
     await this.fileTaskQueue.add(QueueJob.IMPORT_TASK, {
       fileTaskId: fileTaskId,
@@ -239,89 +237,4 @@ export class ImportService {
 
     return fileTask;
   }
-
-  async markdownOrHtmlToProsemirror(
-    fileContent: string,
-    fileExtension: string,
-  ): Promise<any> {
-    let prosemirrorState = '';
-    if (fileExtension === '.md') {
-      prosemirrorState = await this.processMarkdown(fileContent);
-    } else if (fileExtension.endsWith('.html')) {
-      prosemirrorState = await this.processHTML(fileContent);
-    }
-    return prosemirrorState;
-  }
-
-  async convertInternalLinksToMentionsPM(
-    doc: PMNode,
-    currentFilePath: string,
-    filePathToPageMetaMap: Map<
-      string,
-      { id: string; title: string; slugId: string }
-    >,
-  ): Promise<PMNode> {
-    const schema = getSchema(tiptapExtensions);
-    const state = EditorState.create({ doc, schema });
-    let tr: Transaction = state.tr;
-
-    const normalizePath = (p: string) => p.replace(/\\/g, '/');
-
-    // Collect replacements from the original doc.
-    const replacements: Array<{
-      from: number;
-      to: number;
-      mentionNode: PMNode;
-    }> = [];
-
-    doc.descendants((node, pos) => {
-      if (!node.isText || !node.marks?.length) return;
-
-      // Look for the link mark
-      const linkMark = node.marks.find(
-        (mark) => mark.type.name === 'link' && mark.attrs?.href,
-      );
-      if (!linkMark) return;
-
-      // Compute the range for the entire text node.
-      const from = pos;
-      const to = pos + node.nodeSize;
-
-      // Resolve the path and get page meta.
-      const resolvedPath = normalizePath(
-        path.join(path.dirname(currentFilePath), linkMark.attrs.href),
-      );
-      const pageMeta = filePathToPageMetaMap.get(resolvedPath);
-      if (!pageMeta) return;
-
-      // Create the mention node with all required attributes.
-      const mentionNode = schema.nodes.mention.create({
-        id: v7(),
-        entityType: 'page',
-        entityId: pageMeta.id,
-        label: node.text || pageMeta.title,
-        slugId: pageMeta.slugId,
-        creatorId: 'not available', // This is required per your schema.
-      });
-
-      replacements.push({ from, to, mentionNode });
-    });
-
-    // Apply replacements in reverse order.
-    for (let i = replacements.length - 1; i >= 0; i--) {
-      const { from, to, mentionNode } = replacements[i];
-      try {
-        tr = tr.replaceWith(from, to, mentionNode);
-      } catch (err) {
-        console.error('❌ Failed to insert mention:', err);
-      }
-    }
-    if (tr.docChanged) {
-      console.log('doc changed');
-      console.log(JSON.stringify(state.apply(tr).doc.toJSON()));
-    }
-
-    // Return the updated document if any change was made.
-    return tr.docChanged ? state.apply(tr).doc : doc;
-  }
 }

apps/server/src/integrations/import/utils/file.utils.ts

@@ -7,7 +7,7 @@ export enum FileTaskType {
   Export = 'export',
 }
 
-export enum FileImportType {
+export enum FileImportSource {
   Generic = 'generic',
   Notion = 'notion',
   Confluence = 'confluence',

apps/server/src/integrations/queue/queue.module.ts

@@ -51,6 +51,10 @@ import { BacklinksProcessor } from './processors/backlinks.processor';
     }),
     BullModule.registerQueue({
       name: QueueName.FILE_TASK_QUEUE,
+      defaultJobOptions: {
+        removeOnComplete: true,
+        removeOnFail: true,
+      },
     }),
   ],
   exports: [BullModule],