feat(EE): full-text search in attachments (#1502)

* feat(EE): fulltext search in attachments

* feat: global search
- search filters
- attachments search ui
- and more

* fix import

* fix import

* rename migration

* add GIN index

* fix table name

* sanitize

apps/server/src/common/helpers/utils.ts

@@ -87,3 +87,12 @@ export function extractBearerTokenFromHeader(
   const [type, token] = request.headers.authorization?.split(' ') ?? [];
   return type === 'Bearer' ? token : undefined;
 }
+
+export function hasLicenseOrEE(opts: {
+  licenseKey: string;
+  plan: string;
+  isCloud: boolean;
+}): boolean {
+  const { licenseKey, plan, isCloud } = opts;
+  return Boolean(licenseKey) || (isCloud && plan === 'business');
+}

apps/server/src/core/attachment/processors/attachment.processor.ts

@@ -3,12 +3,15 @@ import { OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq';
 import { Job } from 'bullmq';
 import { AttachmentService } from '../services/attachment.service';
 import { QueueJob, QueueName } from 'src/integrations/queue/constants';
-import { Space } from '@docmost/db/types/entity.types';
+import { ModuleRef } from '@nestjs/core';
 
 @Processor(QueueName.ATTACHMENT_QUEUE)
 export class AttachmentProcessor extends WorkerHost implements OnModuleDestroy {
   private readonly logger = new Logger(AttachmentProcessor.name);
-  constructor(private readonly attachmentService: AttachmentService) {
+  constructor(
+    private readonly attachmentService: AttachmentService,
+    private moduleRef: ModuleRef,
+  ) {
     super();
   }
 
@@ -25,6 +28,33 @@ export class AttachmentProcessor extends WorkerHost implements OnModuleDestroy {
           job.data.pageId,
         );
       }
+      if (
+        job.name === QueueJob.ATTACHMENT_INDEX_CONTENT ||
+        job.name === QueueJob.ATTACHMENT_INDEXING
+      ) {
+        let AttachmentEeModule: any;
+        try {
+          // eslint-disable-next-line @typescript-eslint/no-require-imports
+          AttachmentEeModule = require('./../../../ee/attachments-ee/attachment-ee.service');
+        } catch (err) {
+          this.logger.error(
+            'Attachment enterprise module requested but EE module not bundled in this build',
+          );
+          return;
+        }
+        const attachmentEeService = this.moduleRef.get(
+          AttachmentEeModule.AttachmentEeService,
+          { strict: false },
+        );
+
+        if (job.name === QueueJob.ATTACHMENT_INDEX_CONTENT) {
+          await attachmentEeService.indexAttachment(job.data.attachmentId);
+        } else if (job.name === QueueJob.ATTACHMENT_INDEXING) {
+          await attachmentEeService.indexAttachments(
+            job.data.workspaceId,
+          );
+        }
+      }
     } catch (err) {
       throw err;
     }

apps/server/src/core/attachment/services/attachment.service.ts

@@ -22,6 +22,9 @@ import { executeTx } from '@docmost/db/utils';
 import { UserRepo } from '@docmost/db/repos/user/user.repo';
 import { WorkspaceRepo } from '@docmost/db/repos/workspace/workspace.repo';
 import { SpaceRepo } from '@docmost/db/repos/space/space.repo';
+import { InjectQueue } from '@nestjs/bullmq';
+import { QueueJob, QueueName } from '../../../integrations/queue/constants';
+import { Queue } from 'bullmq';
 
 @Injectable()
 export class AttachmentService {
@@ -33,6 +36,7 @@ export class AttachmentService {
     private readonly workspaceRepo: WorkspaceRepo,
     private readonly spaceRepo: SpaceRepo,
     @InjectKysely() private readonly db: KyselyDB,
+    @InjectQueue(QueueName.ATTACHMENT_QUEUE) private attachmentQueue: Queue,
   ) {}
 
   async uploadFile(opts: {
@@ -99,6 +103,23 @@ export class AttachmentService {
           pageId,
         });
       }
+
+      // Only index PDFs and DOCX files
+      if (['.pdf', '.docx'].includes(attachment.fileExt.toLowerCase())) {
+        await this.attachmentQueue.add(
+          QueueJob.ATTACHMENT_INDEX_CONTENT,
+          {
+            attachmentId: attachmentId,
+          },
+          {
+            attempts: 2,
+            backoff: {
+              type: 'exponential',
+              delay: 10000,
+            },
+          },
+        );
+      }
     } catch (err) {
       // delete uploaded file on error
       this.logger.error(err);
@@ -367,4 +388,5 @@ export class AttachmentService {
       throw err;
     }
   }
+
 }

apps/server/src/core/search/dto/search.dto.ts

@@ -5,15 +5,13 @@ import {
   IsOptional,
   IsString,
 } from 'class-validator';
-import { PartialType } from '@nestjs/mapped-types';
-import { CreateWorkspaceDto } from '../../workspace/dto/create-workspace.dto';
 
 export class SearchDTO {
   @IsNotEmpty()
   @IsString()
   query: string;
 
-  @IsNotEmpty()
+  @IsOptional()
   @IsString()
   spaceId: string;
 

apps/server/src/core/share/share.controller.ts

@@ -31,6 +31,7 @@ import { Public } from '../../common/decorators/public.decorator';
 import { ShareRepo } from '@docmost/db/repos/share/share.repo';
 import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
 import { EnvironmentService } from '../../integrations/environment/environment.service';
+import { hasLicenseOrEE } from '../../common/helpers';
 
 @UseGuards(JwtAuthGuard)
 @Controller('shares')
@@ -65,9 +66,11 @@ export class ShareController {
 
     return {
       ...(await this.shareService.getSharedPage(dto, workspace.id)),
-      hasLicenseKey:
-        Boolean(workspace.licenseKey) ||
-        (this.environmentService.isCloud() && workspace.plan === 'business'),
+      hasLicenseKey: hasLicenseOrEE({
+        licenseKey: workspace.licenseKey,
+        isCloud: this.environmentService.isCloud(),
+        plan: workspace.plan,
+      }),
     };
   }
 
@@ -175,9 +178,11 @@ export class ShareController {
   ) {
     return {
       ...(await this.shareService.getShareTree(dto.shareId, workspace.id)),
-      hasLicenseKey:
-        Boolean(workspace.licenseKey) ||
-        (this.environmentService.isCloud() && workspace.plan === 'business'),
+      hasLicenseKey: hasLicenseOrEE({
+        licenseKey: workspace.licenseKey,
+        isCloud: this.environmentService.isCloud(),
+        plan: workspace.plan,
+      }),
     };
   }
 }

apps/server/src/database/migrations/20250901T184612-attachments-search.ts

@@ -0,0 +1,29 @@
+import { type Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .alterTable('attachments')
+    .addColumn('text_content', 'text', (col) => col)
+    .addColumn('tsv', sql`tsvector`, (col) => col)
+    .execute();
+
+  await db.schema
+    .createIndex('attachments_tsv_idx')
+    .on('attachments')
+    .using('GIN')
+    .column('tsv')
+    .execute();
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .alterTable('attachments')
+    .dropIndex('attachments_tsv_idx')
+    .execute();
+
+  await db.schema
+    .alterTable('attachments')
+    .dropColumn('text_content')
+    .dropColumn('tsv')
+    .execute();
+}

apps/server/src/database/repos/attachment/attachment.repo.ts

@@ -12,6 +12,23 @@ import {
 export class AttachmentRepo {
   constructor(@InjectKysely() private readonly db: KyselyDB) {}
 
+  private baseFields: Array<keyof Attachment> = [
+    'id',
+    'fileName',
+    'filePath',
+    'fileSize',
+    'fileExt',
+    'mimeType',
+    'type',
+    'creatorId',
+    'pageId',
+    'spaceId',
+    'workspaceId',
+    'createdAt',
+    'updatedAt',
+    'deletedAt',
+  ];
+
   async findById(
     attachmentId: string,
     opts?: {
@@ -22,7 +39,7 @@ export class AttachmentRepo {
 
     return db
       .selectFrom('attachments')
-      .selectAll()
+      .select(this.baseFields)
       .where('id', '=', attachmentId)
       .executeTakeFirst();
   }
@@ -36,7 +53,7 @@ export class AttachmentRepo {
     return db
       .insertInto('attachments')
       .values(insertableAttachment)
-      .returningAll()
+      .returning(this.baseFields)
       .executeTakeFirst();
   }
 
@@ -50,7 +67,7 @@ export class AttachmentRepo {
 
     return db
       .selectFrom('attachments')
-      .selectAll()
+      .select(this.baseFields)
       .where('spaceId', '=', spaceId)
       .execute();
   }
@@ -64,6 +81,7 @@ export class AttachmentRepo {
       .updateTable('attachments')
       .set(updatableAttachment)
       .where('pageId', 'in', pageIds)
+      .returning(this.baseFields)
       .executeTakeFirst();
   }
 
@@ -75,7 +93,7 @@ export class AttachmentRepo {
       .updateTable('attachments')
       .set(updatableAttachment)
       .where('id', '=', attachmentId)
-      .returningAll()
+      .returning(this.baseFields)
       .executeTakeFirst();
   }
 

apps/server/src/database/types/db.d.ts

@@ -37,6 +37,8 @@ export interface Attachments {
   mimeType: string | null;
   pageId: string | null;
   spaceId: string | null;
+  textContent: string | null;
+  tsv: string | null;
   type: string | null;
   updatedAt: Generated<Timestamp>;
   workspaceId: string;

apps/server/src/integrations/queue/constants/queue.constants.ts

@@ -9,6 +9,8 @@ export enum QueueName {
 export enum QueueJob {
   SEND_EMAIL = 'send-email',
   DELETE_SPACE_ATTACHMENTS = 'delete-space-attachments',
+  ATTACHMENT_INDEX_CONTENT = 'attachment-index-content',
+  ATTACHMENT_INDEXING = 'attachment-indexing',
   DELETE_PAGE_ATTACHMENTS = 'delete-page-attachments',
   PAGE_CONTENT_UPDATE = 'page-content-update',