Merge branch 'main' into ai-vector

apps/server/src/integrations/environment/environment.service.ts

@@ -214,6 +214,26 @@ export class EnvironmentService {
     return this.configService.get<string>('POSTHOG_KEY');
   }
 
+  getSearchDriver(): string {
+    return this.configService
+      .get<string>('SEARCH_DRIVER', 'database')
+      .toLowerCase();
+  }
+
+  getTypesenseUrl(): string {
+    return this.configService
+      .get<string>('TYPESENSE_URL', 'http://localhost:8108')
+      .toLowerCase();
+  }
+
+  getTypesenseApiKey(): string {
+    return this.configService.get<string>('TYPESENSE_API_KEY');
+  }
+
+  getTypesenseLocale(): string {
+    return this.configService.get<string>('TYPESENSE_LOCALE', 'en').toLowerCase();
+  }
+
   getOpenAiApiKey(): string {
     return this.configService.get<string>('OPENAI_API_KEY');
   }

apps/server/src/integrations/environment/environment.validation.ts

@@ -3,12 +3,14 @@ import {
   IsNotEmpty,
   IsNotIn,
   IsOptional,
+  IsString,
   IsUrl,
   MinLength,
   ValidateIf,
   validateSync,
 } from 'class-validator';
 import { plainToInstance } from 'class-transformer';
+import { IsISO6391 } from '../../common/validator/is-iso6391';
 
 export class EnvironmentVariables {
   @IsNotEmpty()
@@ -68,6 +70,37 @@ export class EnvironmentVariables {
   )
   @ValidateIf((obj) => obj.CLOUD === 'true'.toLowerCase())
   SUBDOMAIN_HOST: string;
+
+  @IsOptional()
+  @IsIn(['database', 'typesense'])
+  @IsString()
+  SEARCH_DRIVER: string;
+
+  @IsOptional()
+  @IsUrl(
+    {
+      protocols: ['http', 'https'],
+      require_tld: false,
+      allow_underscores: true,
+    },
+    {
+      message:
+        'TYPESENSE_URL must be a valid typesense url e.g http://localhost:8108',
+    },
+  )
+  @ValidateIf((obj) => obj.SEARCH_DRIVER === 'typesense')
+  TYPESENSE_URL: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.SEARCH_DRIVER === 'typesense')
+  @IsString()
+  TYPESENSE_API_KEY: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.SEARCH_DRIVER === 'typesense')
+  @IsISO6391()
+  @IsString()
+  TYPESENSE_LOCALE: string;
 }
 
 export function validate(config: Record<string, any>) {

apps/server/src/integrations/import/services/file-import-task.service.ts

@@ -32,6 +32,8 @@ import { ImportAttachmentService } from './import-attachment.service';
 import { ModuleRef } from '@nestjs/core';
 import { PageService } from '../../../core/page/services/page.service';
 import { ImportPageNode } from '../dto/file-task-dto';
+import { EventEmitter2 } from '@nestjs/event-emitter';
+import { EventName } from '../../../common/events/event.contants';
 
 @Injectable()
 export class FileImportTaskService {
@@ -45,6 +47,7 @@ export class FileImportTaskService {
     @InjectKysely() private readonly db: KyselyDB,
     private readonly importAttachmentService: ImportAttachmentService,
     private moduleRef: ModuleRef,
+    private eventEmitter: EventEmitter2,
   ) {}
 
   async processZIpImport(fileTaskId: string): Promise<void> {
@@ -396,6 +399,12 @@ export class FileImportTaskService {
           }
         }
 
+        if (validPageIds.size > 0) {
+          this.eventEmitter.emit(EventName.PAGE_CREATED, {
+            pageIds: Array.from(validPageIds),
+          });
+        }
+
         this.logger.log(
           `Successfully imported ${totalPagesProcessed} pages with ${filteredBacklinks.length} backlinks`,
         );

apps/server/src/integrations/queue/constants/queue.constants.ts

@@ -4,6 +4,7 @@ export enum QueueName {
   GENERAL_QUEUE = '{general-queue}',
   BILLING_QUEUE = '{billing-queue}',
   FILE_TASK_QUEUE = '{file-task-queue}',
+  SEARCH_QUEUE = '{search-queue}',
   AI_QUEUE = '{ai-queue}',
 }
 
@@ -27,6 +28,23 @@ export enum QueueJob {
   IMPORT_TASK = 'import-task',
   EXPORT_TASK = 'export-task',
 
+  SEARCH_INDEX_PAGE = 'search-index-page',
+  SEARCH_INDEX_PAGES = 'search-index-pages',
+  SEARCH_INDEX_COMMENT = 'search-index-comment',
+  SEARCH_INDEX_COMMENTS = 'search-index-comments',
+  SEARCH_INDEX_ATTACHMENT = 'search-index-attachment',
+  SEARCH_INDEX_ATTACHMENTS = 'search-index-attachments',
+  SEARCH_REMOVE_PAGE = 'search-remove-page',
+  SEARCH_REMOVE_ASSET = 'search-remove-attachment',
+  SEARCH_REMOVE_FACE = 'search-remove-comment',
+  TYPESENSE_FLUSH = 'typesense-flush',
+
+  PAGE_CREATED = 'page-created',
+  PAGE_UPDATED = 'page-updated',
+  PAGE_SOFT_DELETED = 'page-soft-deleted',
+  PAGE_RESTORED = 'page-restored',
+  PAGE_DELETED = 'page-deleted',
+
   GENERATE_PAGE_EMBEDDINGS = 'generate-page-embeddings',
   DELETE_PAGE_EMBEDDINGS = 'delete-page-embeddings',
 }

apps/server/src/integrations/queue/queue.module.ts

@@ -57,6 +57,14 @@ import { BacklinksProcessor } from './processors/backlinks.processor';
         attempts: 1,
       },
     }),
+    BullModule.registerQueue({
+      name: QueueName.SEARCH_QUEUE,
+      defaultJobOptions: {
+        removeOnComplete: true,
+        removeOnFail: true,
+        attempts: 2,
+      },
+    }),
     BullModule.registerQueue({
       name: QueueName.AI_QUEUE,
       defaultJobOptions: {

apps/server/src/integrations/redis/redis-config.service.ts

@@ -0,0 +1,26 @@
+import { Injectable } from '@nestjs/common';
+import {
+  RedisModuleOptions,
+  RedisOptionsFactory,
+} from '@nestjs-labs/nestjs-ioredis';
+import { createRetryStrategy, parseRedisUrl } from '../../common/helpers';
+import { EnvironmentService } from '../environment/environment.service';
+
+@Injectable()
+export class RedisConfigService implements RedisOptionsFactory {
+  constructor(private readonly environmentService: EnvironmentService) {}
+  createRedisOptions(): RedisModuleOptions {
+    const redisConfig = parseRedisUrl(this.environmentService.getRedisUrl());
+    return {
+      readyLog: true,
+      config: {
+        host: redisConfig.host,
+        port: redisConfig.port,
+        password: redisConfig.password,
+        db: redisConfig.db,
+        family: redisConfig.family,
+        retryStrategy: createRetryStrategy(),
+      },
+    };
+  }
+}