Merge branch 'main' into ai-vector

apps/server/package.json

@@ -38,6 +38,7 @@
     "@fastify/multipart": "^9.0.3",
     "@fastify/static": "^8.2.0",
     "@langchain/textsplitters": "^0.1.0",
+    "@nestjs-labs/nestjs-ioredis": "^11.0.4",
     "@nestjs/bullmq": "^11.0.2",
     "@nestjs/common": "^11.1.3",
     "@nestjs/config": "^4.0.2",
@@ -56,14 +57,15 @@
     "@react-email/render": "1.0.2",
     "@socket.io/redis-adapter": "^8.3.0",
     "bcrypt": "^5.1.1",
-    "bullmq": "^5.53.2",
+    "bullmq": "^5.61.0",
     "cache-manager": "^6.4.3",
     "cheerio": "^1.1.0",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.1",
     "cookie": "^1.0.2",
     "fs-extra": "^11.3.0",
-    "happy-dom": "^15.11.6",
+    "happy-dom": "^18.0.1",
+    "ioredis": "^5.4.1",
     "jsonwebtoken": "^9.0.2",
     "kysely": "^0.28.2",
     "kysely-migration-cli": "^0.4.2",
@@ -92,6 +94,7 @@
     "socket.io": "^4.8.1",
     "stripe": "^17.5.0",
     "tmp-promise": "^3.0.3",
+    "typesense": "^2.1.0",
     "ws": "^8.18.2",
     "yauzl": "^3.2.0"
   },

apps/server/src/app.module.ts

@@ -16,6 +16,8 @@ import { ExportModule } from './integrations/export/export.module';
 import { ImportModule } from './integrations/import/import.module';
 import { SecurityModule } from './integrations/security/security.module';
 import { TelemetryModule } from './integrations/telemetry/telemetry.module';
+import { RedisModule } from '@nestjs-labs/nestjs-ioredis';
+import { RedisConfigService } from './integrations/redis/redis-config.service';
 
 const enterpriseModules = [];
 try {
@@ -36,6 +38,9 @@ try {
     CoreModule,
     DatabaseModule,
     EnvironmentModule,
+    RedisModule.forRootAsync({
+      useClass: RedisConfigService,
+    }),
     CollaborationModule,
     WsModule,
     QueueModule,

apps/server/src/collaboration/collaboration.util.ts

@@ -35,11 +35,10 @@ import {
   Subpages,
 } from '@docmost/editor-ext';
 import { generateText, getSchema, JSONContent } from '@tiptap/core';
-import { generateHTML } from '../common/helpers/prosemirror/html';
+import { generateHTML, generateJSON } from '../common/helpers/prosemirror/html';
 // @tiptap/html library works best for generating prosemirror json state but not HTML
 // see: https://github.com/ueberdosis/tiptap/issues/5352
 // see:https://github.com/ueberdosis/tiptap/issues/4089
-import { generateJSON } from '@tiptap/html';
 import { Node } from '@tiptap/pm/model';
 
 export const tiptapExtensions = [

apps/server/src/common/events/event.contants.ts

@@ -1,3 +1,8 @@
 export enum EventName {
   COLLAB_PAGE_UPDATED = 'collab.page.updated',
-}
+  PAGE_CREATED = 'page.created',
+  PAGE_UPDATED = 'page.updated',
+  PAGE_DELETED = 'page.deleted',
+  PAGE_SOFT_DELETED = 'page.soft_deleted',
+  PAGE_RESTORED = 'page.restored',
+}

apps/server/src/common/helpers/prosemirror/html/generateHTML.ts

@@ -1,21 +1,29 @@
-import { Extensions, getSchema, JSONContent } from '@tiptap/core';
-import { DOMSerializer, Node } from '@tiptap/pm/model';
-import { Window } from 'happy-dom';
+import { type Extensions, type JSONContent, getSchema } from '@tiptap/core';
+import { Node } from '@tiptap/pm/model';
+import { getHTMLFromFragment } from './getHTMLFromFragment';
 
+/**
+ * This function generates HTML from a ProseMirror JSON content object.
+ *
+ * @remarks **Important**: This function requires `happy-dom` to be installed in your project.
+ * @param doc - The ProseMirror JSON content object.
+ * @param extensions - The Tiptap extensions used to build the schema.
+ * @returns The generated HTML string.
+ * @example
+ * ```js
+ * const html = generateHTML(doc, extensions)
+ * console.log(html)
+ * ```
+ */
 export function generateHTML(doc: JSONContent, extensions: Extensions): string {
+  if (typeof window !== 'undefined') {
+    throw new Error(
+      'generateHTML can only be used in a Node environment\nIf you want to use this in a browser environment, use the `@tiptap/html` import instead.',
+    );
+  }
+
   const schema = getSchema(extensions);
   const contentNode = Node.fromJSON(schema, doc);
 
-  const window = new Window();
-
-  const fragment = DOMSerializer.fromSchema(schema).serializeFragment(
-    contentNode.content,
-    {
-      document: window.document as unknown as Document,
-    },
-  );
-
-  const serializer = new window.XMLSerializer();
-  // @ts-ignore
-  return serializer.serializeToString(fragment as unknown as Node);
+  return getHTMLFromFragment(contentNode, schema);
 }

apps/server/src/common/helpers/prosemirror/html/generateJSON.ts

@@ -1,21 +1,55 @@
-import { Extensions, getSchema } from '@tiptap/core';
-import { DOMParser, ParseOptions } from '@tiptap/pm/model';
+import type { Extensions } from '@tiptap/core';
+import { getSchema } from '@tiptap/core';
+import { type ParseOptions, DOMParser as PMDOMParser } from '@tiptap/pm/model';
 import { Window } from 'happy-dom';
 
-// this function does not work as intended
-// it has issues with closing tags
+/**
+ * Generates a JSON object from the given HTML string and converts it into a Prosemirror node with content.
+ * @remarks **Important**: This function requires `happy-dom` to be installed in your project.
+ * @param {string} html - The HTML string to be converted into a Prosemirror node.
+ * @param {Extensions} extensions - The extensions to be used for generating the schema.
+ * @param {ParseOptions} options - The options to be supplied to the parser.
+ * @returns {Promise<Record<string, any>>} - A promise with the generated JSON object.
+ * @example
+ * const html = '<p>Hello, world!</p>'
+ * const extensions = [...]
+ * const json = generateJSON(html, extensions)
+ * console.log(json) // { type: 'doc', content: [{ type: 'paragraph', content: [{ type: 'text', text: 'Hello, world!' }] }] }
+ */
 export function generateJSON(
   html: string,
   extensions: Extensions,
   options?: ParseOptions,
 ): Record<string, any> {
-  const schema = getSchema(extensions);
+  if (typeof window !== 'undefined') {
+    throw new Error(
+      'generateJSON can only be used in a Node environment\nIf you want to use this in a browser environment, use the `@tiptap/html` import instead.',
+    );
+  }
 
-  const window = new Window();
-  const document = window.document;
-  document.body.innerHTML = html;
+  const localWindow = new Window();
+  const localDOMParser = new localWindow.DOMParser();
+  let result: Record<string, any>;
 
-  return DOMParser.fromSchema(schema)
-    .parse(document as never, options)
-    .toJSON();
+  try {
+    const schema = getSchema(extensions);
+    let doc: ReturnType<typeof localDOMParser.parseFromString> | null = null;
+
+    const htmlString = `<!DOCTYPE html><html><body>${html}</body></html>`;
+    doc = localDOMParser.parseFromString(htmlString, 'text/html');
+
+    if (!doc) {
+      throw new Error('Failed to parse HTML string');
+    }
+
+    result = PMDOMParser.fromSchema(schema)
+      .parse(doc.body as unknown as Node, options)
+      .toJSON();
+  } finally {
+    // clean up happy-dom to avoid memory leaks
+    localWindow.happyDOM.abort();
+    localWindow.happyDOM.close();
+  }
+
+  return result;
 }

apps/server/src/common/helpers/prosemirror/html/getHTMLFromFragment.ts

@@ -0,0 +1,54 @@
+import type { Node, Schema } from '@tiptap/pm/model';
+import { DOMSerializer } from '@tiptap/pm/model';
+import { Window } from 'happy-dom';
+
+/**
+ * Returns the HTML string representation of a given document node.
+ *
+ * @remarks **Important**: This function requires `happy-dom` to be installed in your project.
+ * @param doc - The document node to serialize.
+ * @param schema - The Prosemirror schema to use for serialization.
+ * @returns A promise containing the HTML string representation of the document fragment.
+ *
+ * @example
+ * ```typescript
+ * const html = getHTMLFromFragment(doc, schema)
+ * ```
+ */
+export function getHTMLFromFragment(
+  doc: Node,
+  schema: Schema,
+  options?: { document?: Document },
+): string {
+  if (options?.document) {
+    const wrap = options.document.createElement('div');
+
+    DOMSerializer.fromSchema(schema).serializeFragment(
+      doc.content,
+      { document: options.document },
+      wrap,
+    );
+    return wrap.innerHTML;
+  }
+
+  const localWindow = new Window();
+  let result: string;
+
+  try {
+    const fragment = DOMSerializer.fromSchema(schema).serializeFragment(
+      doc.content,
+      {
+        document: localWindow.document as unknown as Document,
+      },
+    );
+
+    const serializer = new localWindow.XMLSerializer();
+    result = serializer.serializeToString(fragment as any);
+  } finally {
+    // clean up happy-dom to avoid memory leaks
+    localWindow.happyDOM.abort();
+    localWindow.happyDOM.close();
+  }
+
+  return result;
+}

apps/server/src/common/validator/is-iso6391.ts

@@ -0,0 +1,34 @@
+// MIT - https://github.com/typestack/class-validator/pull/2626
+import isISO6391Validator from 'validator/lib/isISO6391';
+import { buildMessage, ValidateBy, ValidationOptions } from 'class-validator';
+
+export const IS_ISO6391 = 'isISO6391';
+
+/**
+ * Check if the string is a valid [ISO 639-1](https://en.wikipedia.org/wiki/ISO_639-1) officially assigned language code.
+ */
+export function isISO6391(value: unknown): boolean {
+  return typeof value === 'string' && isISO6391Validator(value);
+}
+
+/**
+ * Check if the string is a valid [ISO 639-1](https://en.wikipedia.org/wiki/ISO_639-1) officially assigned language code.
+ */
+export function IsISO6391(
+  validationOptions?: ValidationOptions,
+): PropertyDecorator {
+  return ValidateBy(
+    {
+      name: IS_ISO6391,
+      validator: {
+        validate: (value, args): boolean => isISO6391(value),
+        defaultMessage: buildMessage(
+          (eachPrefix) =>
+            eachPrefix + '$property must be a valid ISO 639-1 language code',
+          validationOptions,
+        ),
+      },
+    },
+    validationOptions,
+  );
+}

apps/server/src/core/auth/dto/jwt-payload.ts

@@ -4,6 +4,7 @@ export enum JwtType {
   EXCHANGE = 'exchange',
   ATTACHMENT = 'attachment',
   MFA_TOKEN = 'mfa_token',
+  API_KEY = 'api_key',
 }
 export type JwtPayload = {
   sub: string;
@@ -36,3 +37,10 @@ export interface JwtMfaTokenPayload {
   workspaceId: string;
   type: 'mfa_token';
 }
+
+export type JwtApiKeyPayload = {
+  sub: string;
+  workspaceId: string;
+  apiKeyId: string;
+  type: 'api_key';
+};

apps/server/src/core/auth/services/token.service.ts

@@ -6,6 +6,7 @@ import {
 import { JwtService } from '@nestjs/jwt';
 import { EnvironmentService } from '../../../integrations/environment/environment.service';
 import {
+  JwtApiKeyPayload,
   JwtAttachmentPayload,
   JwtCollabPayload,
   JwtExchangePayload,
@@ -77,10 +78,7 @@ export class TokenService {
     return this.jwtService.sign(payload, { expiresIn: '1h' });
   }
 
-  async generateMfaToken(
-    user: User,
-    workspaceId: string,
-  ): Promise<string> {
+  async generateMfaToken(user: User, workspaceId: string): Promise<string> {
     if (user.deactivatedAt || user.deletedAt) {
       throw new ForbiddenException();
     }
@@ -93,6 +91,27 @@ export class TokenService {
     return this.jwtService.sign(payload, { expiresIn: '5m' });
   }
 
+  async generateApiToken(opts: {
+    apiKeyId: string;
+    user: User;
+    workspaceId: string;
+    expiresIn?: string | number;
+  }): Promise<string> {
+    const { apiKeyId, user, workspaceId, expiresIn } = opts;
+    if (user.deactivatedAt || user.deletedAt) {
+      throw new ForbiddenException();
+    }
+
+    const payload: JwtApiKeyPayload = {
+      sub: user.id,
+      apiKeyId: apiKeyId,
+      workspaceId,
+      type: JwtType.API_KEY,
+    };
+
+    return this.jwtService.sign(payload, expiresIn ? { expiresIn } : {});
+  }
+
   async verifyJwt(token: string, tokenType: string) {
     const payload = await this.jwtService.verifyAsync(token, {
       secret: this.environmentService.getAppSecret(),

apps/server/src/core/auth/strategies/jwt.strategy.ts

@@ -2,11 +2,12 @@ import { Injectable, Logger, UnauthorizedException } from '@nestjs/common';
 import { PassportStrategy } from '@nestjs/passport';
 import { Strategy } from 'passport-jwt';
 import { EnvironmentService } from '../../../integrations/environment/environment.service';
-import { JwtPayload, JwtType } from '../dto/jwt-payload';
+import { JwtApiKeyPayload, JwtPayload, JwtType } from '../dto/jwt-payload';
 import { WorkspaceRepo } from '@docmost/db/repos/workspace/workspace.repo';
 import { UserRepo } from '@docmost/db/repos/user/user.repo';
 import { FastifyRequest } from 'fastify';
 import { extractBearerTokenFromHeader } from '../../../common/helpers';
+import { ModuleRef } from '@nestjs/core';
 
 @Injectable()
 export class JwtStrategy extends PassportStrategy(Strategy, 'jwt') {
@@ -16,6 +17,7 @@ export class JwtStrategy extends PassportStrategy(Strategy, 'jwt') {
     private userRepo: UserRepo,
     private workspaceRepo: WorkspaceRepo,
     private readonly environmentService: EnvironmentService,
+    private moduleRef: ModuleRef,
   ) {
     super({
       jwtFromRequest: (req: FastifyRequest) => {
@@ -27,8 +29,8 @@ export class JwtStrategy extends PassportStrategy(Strategy, 'jwt') {
     });
   }
 
-  async validate(req: any, payload: JwtPayload) {
-    if (!payload.workspaceId || payload.type !== JwtType.ACCESS) {
+  async validate(req: any, payload: JwtPayload | JwtApiKeyPayload) {
+    if (!payload.workspaceId) {
       throw new UnauthorizedException();
     }
 
@@ -36,6 +38,14 @@ export class JwtStrategy extends PassportStrategy(Strategy, 'jwt') {
       throw new UnauthorizedException('Workspace does not match');
     }
 
+    if (payload.type === JwtType.API_KEY) {
+      return this.validateApiKey(req, payload as JwtApiKeyPayload);
+    }
+
+    if (payload.type !== JwtType.ACCESS) {
+      throw new UnauthorizedException();
+    }
+
     const workspace = await this.workspaceRepo.findById(payload.workspaceId);
 
     if (!workspace) {
@@ -49,4 +59,30 @@ export class JwtStrategy extends PassportStrategy(Strategy, 'jwt') {
 
     return { user, workspace };
   }
+
+  private async validateApiKey(req: any, payload: JwtApiKeyPayload) {
+    let ApiKeyModule: any;
+    let isApiKeyModuleReady = false;
+
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-require-imports
+      ApiKeyModule = require('./../../../ee/api-key/api-key.service');
+      isApiKeyModuleReady = true;
+    } catch (err) {
+      this.logger.debug(
+        'API Key module requested but enterprise module not bundled in this build',
+      );
+      isApiKeyModuleReady = false;
+    }
+
+    if (isApiKeyModuleReady) {
+      const ApiKeyService = this.moduleRef.get(ApiKeyModule.ApiKeyService, {
+        strict: false,
+      });
+
+      return ApiKeyService.validateApiKey(payload);
+    }
+
+    throw new UnauthorizedException('Enterprise API Key module missing');
+  }
 }

apps/server/src/core/casl/abilities/workspace-ability.factory.ts

@@ -40,6 +40,7 @@ function buildWorkspaceOwnerAbility() {
   can(WorkspaceCaslAction.Manage, WorkspaceCaslSubject.Group);
   can(WorkspaceCaslAction.Manage, WorkspaceCaslSubject.Member);
   can(WorkspaceCaslAction.Manage, WorkspaceCaslSubject.Attachment);
+  can(WorkspaceCaslAction.Manage, WorkspaceCaslSubject.API);
 
   return build();
 }
@@ -55,6 +56,7 @@ function buildWorkspaceAdminAbility() {
   can(WorkspaceCaslAction.Manage, WorkspaceCaslSubject.Group);
   can(WorkspaceCaslAction.Manage, WorkspaceCaslSubject.Member);
   can(WorkspaceCaslAction.Manage, WorkspaceCaslSubject.Attachment);
+  can(WorkspaceCaslAction.Manage, WorkspaceCaslSubject.API);
 
   return build();
 }
@@ -68,6 +70,7 @@ function buildWorkspaceMemberAbility() {
   can(WorkspaceCaslAction.Read, WorkspaceCaslSubject.Space);
   can(WorkspaceCaslAction.Read, WorkspaceCaslSubject.Group);
   can(WorkspaceCaslAction.Manage, WorkspaceCaslSubject.Attachment);
+  can(WorkspaceCaslAction.Create, WorkspaceCaslSubject.API);
 
   return build();
 }

apps/server/src/core/casl/interfaces/workspace-ability.type.ts

@@ -11,6 +11,7 @@ export enum WorkspaceCaslSubject {
   Space = 'space',
   Group = 'group',
   Attachment = 'attachment',
+  API = 'api_key',
 }
 
 export type IWorkspaceAbility =
@@ -18,4 +19,5 @@ export type IWorkspaceAbility =
   | [WorkspaceCaslAction, WorkspaceCaslSubject.Member]
   | [WorkspaceCaslAction, WorkspaceCaslSubject.Space]
   | [WorkspaceCaslAction, WorkspaceCaslSubject.Group]
-  | [WorkspaceCaslAction, WorkspaceCaslSubject.Attachment];
+  | [WorkspaceCaslAction, WorkspaceCaslSubject.Attachment]
+  | [WorkspaceCaslAction, WorkspaceCaslSubject.API];

apps/server/src/core/page/page.module.ts

@@ -9,6 +9,6 @@ import { StorageModule } from '../../integrations/storage/storage.module';
   controllers: [PageController],
   providers: [PageService, PageHistoryService, TrashCleanupService],
   exports: [PageService, PageHistoryService],
-  imports: [StorageModule]
+  imports: [StorageModule],
 })
 export class PageModule {}

apps/server/src/core/page/services/page.service.ts

@@ -38,6 +38,8 @@ import { StorageService } from '../../../integrations/storage/storage.service';
 import { InjectQueue } from '@nestjs/bullmq';
 import { Queue } from 'bullmq';
 import { QueueJob, QueueName } from '../../../integrations/queue/constants';
+import { EventName } from '../../../common/events/event.contants';
+import { EventEmitter2 } from '@nestjs/event-emitter';
 
 @Injectable()
 export class PageService {
@@ -49,6 +51,7 @@ export class PageService {
     @InjectKysely() private readonly db: KyselyDB,
     private readonly storageService: StorageService,
     @InjectQueue(QueueName.ATTACHMENT_QUEUE) private attachmentQueue: Queue,
+    private eventEmitter: EventEmitter2,
   ) {}
 
   async findById(
@@ -231,21 +234,28 @@ export class PageService {
         );
       }
 
-      // update spaceId in shares
       if (pageIds.length > 0) {
+        // update spaceId in shares
         await trx
           .updateTable('shares')
           .set({ spaceId: spaceId })
           .where('pageId', 'in', pageIds)
           .execute();
-      }
 
-      // Update attachments
-      await this.attachmentRepo.updateAttachmentsByPageId(
-        { spaceId },
-        pageIds,
-        trx,
-      );
+        // Update comments
+        await trx
+          .updateTable('comments')
+          .set({ spaceId: spaceId })
+          .where('pageId', 'in', pageIds)
+          .execute();
+
+        // Update attachments
+        await this.attachmentRepo.updateAttachmentsByPageId(
+          { spaceId },
+          pageIds,
+          trx,
+        );
+      }
     });
   }
 
@@ -380,6 +390,11 @@ export class PageService {
 
     await this.db.insertInto('pages').values(insertablePages).execute();
 
+    const insertedPageIds = insertablePages.map((page) => page.id);
+    this.eventEmitter.emit(EventName.PAGE_CREATED, {
+      pageIds: insertedPageIds,
+    });
+
     //TODO: best to handle this in a queue
     const attachmentsIds = Array.from(attachmentMap.keys());
     if (attachmentsIds.length > 0) {
@@ -606,6 +621,9 @@ export class PageService {
 
     if (pageIds.length > 0) {
       await this.db.deleteFrom('pages').where('id', 'in', pageIds).execute();
+      this.eventEmitter.emit(EventName.PAGE_DELETED, {
+        pageIds: pageIds,
+      });
     }
   }
 

apps/server/src/core/search/dto/search-response.dto.ts

@@ -1,3 +1,5 @@
+import { Space } from '@docmost/db/types/entity.types';
+
 export class SearchResponseDto {
   id: string;
   title: string;
@@ -8,4 +10,5 @@ export class SearchResponseDto {
   highlight: string;
   createdAt: Date;
   updatedAt: Date;
+  space: Partial<Space>;
 }

apps/server/src/core/search/search.controller.ts

@@ -5,6 +5,7 @@ import {
   ForbiddenException,
   HttpCode,
   HttpStatus,
+  Logger,
   Post,
   UseGuards,
 } from '@nestjs/common';
@@ -24,13 +25,19 @@ import {
 } from '../casl/interfaces/space-ability.type';
 import { AuthUser } from '../../common/decorators/auth-user.decorator';
 import { Public } from 'src/common/decorators/public.decorator';
+import { EnvironmentService } from '../../integrations/environment/environment.service';
+import { ModuleRef } from '@nestjs/core';
 
 @UseGuards(JwtAuthGuard)
 @Controller('search')
 export class SearchController {
+  private readonly logger = new Logger(SearchController.name);
+
   constructor(
     private readonly searchService: SearchService,
     private readonly spaceAbility: SpaceAbilityFactory,
+    private readonly environmentService: EnvironmentService,
+    private moduleRef: ModuleRef,
   ) {}
 
   @HttpCode(HttpStatus.OK)
@@ -53,7 +60,14 @@ export class SearchController {
       }
     }
 
-    return this.searchService.searchPage(searchDto.query, searchDto, {
+    if (this.environmentService.getSearchDriver() === 'typesense') {
+      return this.searchTypesense(searchDto, {
+        userId: user.id,
+        workspaceId: workspace.id,
+      });
+    }
+
+    return this.searchService.searchPage(searchDto, {
       userId: user.id,
       workspaceId: workspace.id,
     });
@@ -81,8 +95,47 @@ export class SearchController {
       throw new BadRequestException('shareId is required');
     }
 
-    return this.searchService.searchPage(searchDto.query, searchDto, {
+    if (this.environmentService.getSearchDriver() === 'typesense') {
+      return this.searchTypesense(searchDto, {
+        workspaceId: workspace.id,
+      });
+    }
+
+    return this.searchService.searchPage(searchDto, {
       workspaceId: workspace.id,
     });
   }
+
+  async searchTypesense(
+    searchParams: SearchDTO,
+    opts: {
+      userId?: string;
+      workspaceId: string;
+    },
+  ) {
+    const { userId, workspaceId } = opts;
+    let TypesenseModule: any;
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-require-imports
+      TypesenseModule = require('./../../ee/typesense/services/page-search.service');
+
+      const PageSearchService = this.moduleRef.get(
+        TypesenseModule.PageSearchService,
+        {
+          strict: false,
+        },
+      );
+
+      return PageSearchService.searchPage(searchParams, {
+        userId: userId,
+        workspaceId,
+      });
+    } catch (err) {
+      this.logger.debug(
+        'Typesense module requested but enterprise module not bundled in this build',
+      );
+    }
+
+    throw new BadRequestException('Enterprise Typesense search module missing');
+  }
 }

apps/server/src/core/search/search.service.ts

@@ -21,13 +21,14 @@ export class SearchService {
   ) {}
 
   async searchPage(
-    query: string,
     searchParams: SearchDTO,
     opts: {
       userId?: string;
       workspaceId: string;
     },
   ): Promise<SearchResponseDto[]> {
+    const { query } = searchParams;
+
     if (query.length < 1) {
       return;
     }

apps/server/src/core/workspace/dto/update-workspace.dto.ts

@@ -18,4 +18,8 @@ export class UpdateWorkspaceDto extends PartialType(CreateWorkspaceDto) {
   @IsOptional()
   @IsBoolean()
   enforceMfa: boolean;
+
+  @IsOptional()
+  @IsBoolean()
+  restrictApiToAdmins: boolean;
 }

apps/server/src/core/workspace/services/workspace.service.ts

@@ -303,6 +303,15 @@ export class WorkspaceService {
       }
     }
 
+    if (typeof updateWorkspaceDto.restrictApiToAdmins !== 'undefined') {
+      await this.workspaceRepo.updateApiSettings(
+        workspaceId,
+        'restrictToAdmins',
+        updateWorkspaceDto.restrictApiToAdmins,
+      );
+      delete updateWorkspaceDto.restrictApiToAdmins;
+    }
+
     await this.workspaceRepo.updateWorkspace(updateWorkspaceDto, workspaceId);
 
     const workspace = await this.workspaceRepo.findById(workspaceId, {

apps/server/src/database/database.module.ts

@@ -25,6 +25,7 @@ import { MigrationService } from '@docmost/db/services/migration.service';
 import { UserTokenRepo } from './repos/user-token/user-token.repo';
 import { BacklinkRepo } from '@docmost/db/repos/backlink/backlink.repo';
 import { ShareRepo } from '@docmost/db/repos/share/share.repo';
+import { PageListener } from '@docmost/db/listeners/page.listener';
 
 // https://github.com/brianc/node-postgres/issues/811
 types.setTypeParser(types.builtins.INT8, (val) => Number(val));
@@ -75,7 +76,8 @@ types.setTypeParser(types.builtins.INT8, (val) => Number(val));
     AttachmentRepo,
     UserTokenRepo,
     BacklinkRepo,
-    ShareRepo
+    ShareRepo,
+    PageListener,
   ],
   exports: [
     WorkspaceRepo,
@@ -90,7 +92,7 @@ types.setTypeParser(types.builtins.INT8, (val) => Number(val));
     AttachmentRepo,
     UserTokenRepo,
     BacklinkRepo,
-    ShareRepo
+    ShareRepo,
   ],
 })
 export class DatabaseModule

apps/server/src/database/listeners/page.listener.ts

@@ -0,0 +1,49 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { OnEvent } from '@nestjs/event-emitter';
+import { EventName } from '../../common/events/event.contants';
+import { InjectQueue } from '@nestjs/bullmq';
+import { QueueJob, QueueName } from '../../integrations/queue/constants';
+import { Queue } from 'bullmq';
+
+export class PageEvent {
+  pageIds: string[];
+}
+
+@Injectable()
+export class PageListener {
+  private readonly logger = new Logger(PageListener.name);
+
+  constructor(
+    @InjectQueue(QueueName.SEARCH_QUEUE) private searchQueue: Queue,
+  ) {}
+
+  @OnEvent(EventName.PAGE_CREATED)
+  async handlePageCreated(event: PageEvent) {
+    const { pageIds } = event;
+    await this.searchQueue.add(QueueJob.PAGE_CREATED, { pageIds });
+  }
+
+  @OnEvent(EventName.PAGE_UPDATED)
+  async handlePageUpdated(event: PageEvent) {
+    const { pageIds } = event;
+    await this.searchQueue.add(QueueJob.PAGE_UPDATED, { pageIds });
+  }
+
+  @OnEvent(EventName.PAGE_DELETED)
+  async handlePageDeleted(event: PageEvent) {
+    const { pageIds } = event;
+    await this.searchQueue.add(QueueJob.PAGE_DELETED, { pageIds });
+  }
+
+  @OnEvent(EventName.PAGE_SOFT_DELETED)
+  async handlePageSoftDeleted(event: PageEvent) {
+    const { pageIds } = event;
+    await this.searchQueue.add(QueueJob.PAGE_SOFT_DELETED, { pageIds });
+  }
+
+  @OnEvent(EventName.PAGE_RESTORED)
+  async handlePageRestored(event: PageEvent) {
+    const { pageIds } = event;
+    await this.searchQueue.add(QueueJob.PAGE_RESTORED, { pageIds });
+  }
+}

apps/server/src/database/migrations/20250912T101500-api-keys.ts

@@ -0,0 +1,30 @@
+import { Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .createTable('api_keys')
+    .addColumn('id', 'uuid', (col) =>
+      col.primaryKey().defaultTo(sql`gen_uuid_v7()`),
+    )
+    .addColumn('name', 'text', (col) => col)
+    .addColumn('creator_id', 'uuid', (col) =>
+      col.notNull().references('users.id').onDelete('cascade'),
+    )
+    .addColumn('workspace_id', 'uuid', (col) =>
+      col.notNull().references('workspaces.id').onDelete('cascade'),
+    )
+    .addColumn('expires_at', 'timestamptz')
+    .addColumn('last_used_at', 'timestamptz')
+    .addColumn('created_at', 'timestamptz', (col) =>
+      col.notNull().defaultTo(sql`now()`),
+    )
+    .addColumn('updated_at', 'timestamptz', (col) =>
+      col.notNull().defaultTo(sql`now()`),
+    )
+    .addColumn('deleted_at', 'timestamptz', (col) => col)
+    .execute();
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await db.schema.dropTable('api_keys').execute();
+}

apps/server/src/database/pagination/pagination-options.ts

@@ -1,4 +1,5 @@
 import {
+  IsBoolean,
   IsNumber,
   IsOptional,
   IsPositive,
@@ -23,4 +24,8 @@ export class PaginationOptions {
   @IsOptional()
   @IsString()
   query: string;
+
+  @IsOptional()
+  @IsBoolean()
+  adminView: boolean;
 }

apps/server/src/database/repos/page/page.repo.ts

@@ -14,32 +14,17 @@ import { ExpressionBuilder, sql } from 'kysely';
 import { DB } from '@docmost/db/types/db';
 import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres';
 import { SpaceMemberRepo } from '@docmost/db/repos/space/space-member.repo';
+import { EventEmitter2 } from '@nestjs/event-emitter';
+import { EventName } from '../../../common/events/event.contants';
 
 @Injectable()
 export class PageRepo {
   constructor(
     @InjectKysely() private readonly db: KyselyDB,
     private spaceMemberRepo: SpaceMemberRepo,
+    private eventEmitter: EventEmitter2,
   ) {}
 
-  withHasChildren(eb: ExpressionBuilder<DB, 'pages'>) {
-    return eb
-      .selectFrom('pages as child')
-      .select((eb) =>
-        eb
-          .case()
-          .when(eb.fn.countAll(), '>', 0)
-          .then(true)
-          .else(false)
-          .end()
-          .as('count'),
-      )
-      .whereRef('child.parentPageId', '=', 'pages.id')
-      .where('child.deletedAt', 'is', null)
-      .limit(1)
-      .as('hasChildren');
-  }
-
   private baseFields: Array<keyof Page> = [
     'id',
     'slugId',
@@ -63,7 +48,7 @@ export class PageRepo {
     pageId: string,
     opts?: {
       includeContent?: boolean;
-      includeText?: boolean;
+      includeTextContent?: boolean;
       includeYdoc?: boolean;
       includeSpace?: boolean;
       includeCreator?: boolean;
@@ -80,8 +65,8 @@ export class PageRepo {
       .selectFrom('pages')
       .select(this.baseFields)
       .$if(opts?.includeContent, (qb) => qb.select('content'))
-      .$if(opts?.includeText, (qb) => qb.select('textContent'))
       .$if(opts?.includeYdoc, (qb) => qb.select('ydoc'))
+      .$if(opts?.includeTextContent, (qb) => qb.select('textContent'))
       .$if(opts?.includeHasChildren, (qb) =>
         qb.select((eb) => this.withHasChildren(eb)),
       );
@@ -128,7 +113,7 @@ export class PageRepo {
     pageIds: string[],
     trx?: KyselyTransaction,
   ) {
-    return dbOrTx(this.db, trx)
+    const result = await dbOrTx(this.db, trx)
       .updateTable('pages')
       .set({ ...updatePageData, updatedAt: new Date() })
       .where(
@@ -137,6 +122,12 @@ export class PageRepo {
         pageIds,
       )
       .executeTakeFirst();
+
+    this.eventEmitter.emit(EventName.PAGE_UPDATED, {
+      pageIds: pageIds,
+    });
+
+    return result;
   }
 
   async insertPage(
@@ -144,11 +135,17 @@ export class PageRepo {
     trx?: KyselyTransaction,
   ): Promise<Page> {
     const db = dbOrTx(this.db, trx);
-    return db
+    const result = await db
       .insertInto('pages')
       .values(insertablePage)
       .returning(this.baseFields)
       .executeTakeFirst();
+
+    this.eventEmitter.emit(EventName.PAGE_CREATED, {
+      pageIds: [result.id],
+    });
+
+    return result;
   }
 
   async deletePage(pageId: string): Promise<void> {
@@ -198,6 +195,9 @@ export class PageRepo {
 
         await trx.deleteFrom('shares').where('pageId', 'in', pageIds).execute();
       });
+      this.eventEmitter.emit(EventName.PAGE_SOFT_DELETED, {
+        pageIds: pageIds,
+      });
     }
   }
 
@@ -261,6 +261,9 @@ export class PageRepo {
         .where('id', '=', pageId)
         .execute();
     }
+    this.eventEmitter.emit(EventName.PAGE_RESTORED, {
+      pageIds: pageIds,
+    });
   }
 
   async getRecentPagesInSpace(spaceId: string, pagination: PaginationOptions) {
@@ -381,6 +384,24 @@ export class PageRepo {
     ).as('contributors');
   }
 
+  withHasChildren(eb: ExpressionBuilder<DB, 'pages'>) {
+    return eb
+      .selectFrom('pages as child')
+      .select((eb) =>
+        eb
+          .case()
+          .when(eb.fn.countAll(), '>', 0)
+          .then(true)
+          .else(false)
+          .end()
+          .as('count'),
+      )
+      .whereRef('child.parentPageId', '=', 'pages.id')
+      .where('child.deletedAt', 'is', null)
+      .limit(1)
+      .as('hasChildren');
+  }
+
   async getPageAndDescendants(
     parentPageId: string,
     opts: { includeContent: boolean },

apps/server/src/database/repos/workspace/workspace.repo.ts

@@ -157,4 +157,22 @@ export class WorkspaceRepo {
 
     return activeUsers.length;
   }
+
+  async updateApiSettings(
+    workspaceId: string,
+    prefKey: string,
+    prefValue: string | boolean,
+  ) {
+    return this.db
+      .updateTable('workspaces')
+      .set({
+        settings: sql`COALESCE(settings, '{}'::jsonb)
+                || jsonb_build_object('api', COALESCE(settings->'api', '{}'::jsonb) 
+                || jsonb_build_object('${sql.raw(prefKey)}', ${sql.lit(prefValue)}))`,
+        updatedAt: new Date(),
+      })
+      .where('id', '=', workspaceId)
+      .returning(this.baseFields)
+      .executeTakeFirst();
+  }
 }

apps/server/src/database/types/db.d.ts

@@ -3,13 +3,18 @@
  * Please do not edit it manually.
  */
 
-import type { ColumnType } from "kysely";
+import type { ColumnType } from 'kysely';
 
-export type Generated<T> = T extends ColumnType<infer S, infer I, infer U>
-  ? ColumnType<S, I | undefined, U>
-  : ColumnType<T, T | undefined, T>;
+export type Generated<T> =
+  T extends ColumnType<infer S, infer I, infer U>
+    ? ColumnType<S, I | undefined, U>
+    : ColumnType<T, T | undefined, T>;
 
-export type Int8 = ColumnType<string, bigint | number | string, bigint | number | string>;
+export type Int8 = ColumnType<
+  string,
+  bigint | number | string,
+  bigint | number | string
+>;
 
 export type Json = JsonValue;
 
@@ -25,6 +30,18 @@ export type JsonValue = JsonArray | JsonObject | JsonPrimitive;
 
 export type Timestamp = ColumnType<Date, Date | string, Date | string>;
 
+export interface ApiKeys {
+  createdAt: Generated<Timestamp>;
+  deletedAt: Timestamp | null;
+  expiresAt: Timestamp | null;
+  id: Generated<string>;
+  lastUsedAt: Timestamp | null;
+  name: string | null;
+  updatedAt: Generated<Timestamp>;
+  creatorId: string;
+  workspaceId: string;
+}
+
 export interface Attachments {
   createdAt: Generated<Timestamp>;
   creatorId: string;
@@ -344,6 +361,7 @@ export interface Workspaces {
 }
 
 export interface DB {
+  apiKeys: ApiKeys;
   attachments: Attachments;
   authAccounts: AuthAccounts;
   authProviders: AuthProviders;

apps/server/src/database/types/db.interface.ts

@@ -1,4 +1,5 @@
 import {
+  ApiKeys,
   Attachments,
   AuthAccounts,
   AuthProviders,
@@ -42,4 +43,5 @@ export interface DbInterface {
   userTokens: UserTokens;
   workspaceInvitations: WorkspaceInvitations;
   workspaces: Workspaces;
+  apiKeys: ApiKeys;
 }

apps/server/src/database/types/entity.types.ts

@@ -19,6 +19,7 @@ import {
   Shares,
   FileTasks,
   UserMfa as _UserMFA,
+  ApiKeys,
 } from './db';
 import { Embeddings } from '@docmost/db/types/embeddings.types';
 
@@ -121,6 +122,11 @@ export type UserMFA = Selectable<_UserMFA>;
 export type InsertableUserMFA = Insertable<_UserMFA>;
 export type UpdatableUserMFA = Updateable<Omit<_UserMFA, 'id'>>;
 
+// Api Keys
+export type ApiKey = Selectable<ApiKeys>;
+export type InsertableApiKey = Insertable<ApiKeys>;
+export type UpdatableApiKey = Updateable<Omit<ApiKeys, 'id'>>;
+
 // Page Embedding
 export type Embedding = Selectable<Embeddings>;
 export type InsertableEmbedding = Insertable<Embeddings>;

apps/server/src/integrations/environment/environment.service.ts

@@ -214,6 +214,26 @@ export class EnvironmentService {
     return this.configService.get<string>('POSTHOG_KEY');
   }
 
+  getSearchDriver(): string {
+    return this.configService
+      .get<string>('SEARCH_DRIVER', 'database')
+      .toLowerCase();
+  }
+
+  getTypesenseUrl(): string {
+    return this.configService
+      .get<string>('TYPESENSE_URL', 'http://localhost:8108')
+      .toLowerCase();
+  }
+
+  getTypesenseApiKey(): string {
+    return this.configService.get<string>('TYPESENSE_API_KEY');
+  }
+
+  getTypesenseLocale(): string {
+    return this.configService.get<string>('TYPESENSE_LOCALE', 'en').toLowerCase();
+  }
+
   getOpenAiApiKey(): string {
     return this.configService.get<string>('OPENAI_API_KEY');
   }

apps/server/src/integrations/environment/environment.validation.ts

@@ -3,12 +3,14 @@ import {
   IsNotEmpty,
   IsNotIn,
   IsOptional,
+  IsString,
   IsUrl,
   MinLength,
   ValidateIf,
   validateSync,
 } from 'class-validator';
 import { plainToInstance } from 'class-transformer';
+import { IsISO6391 } from '../../common/validator/is-iso6391';
 
 export class EnvironmentVariables {
   @IsNotEmpty()
@@ -68,6 +70,37 @@ export class EnvironmentVariables {
   )
   @ValidateIf((obj) => obj.CLOUD === 'true'.toLowerCase())
   SUBDOMAIN_HOST: string;
+
+  @IsOptional()
+  @IsIn(['database', 'typesense'])
+  @IsString()
+  SEARCH_DRIVER: string;
+
+  @IsOptional()
+  @IsUrl(
+    {
+      protocols: ['http', 'https'],
+      require_tld: false,
+      allow_underscores: true,
+    },
+    {
+      message:
+        'TYPESENSE_URL must be a valid typesense url e.g http://localhost:8108',
+    },
+  )
+  @ValidateIf((obj) => obj.SEARCH_DRIVER === 'typesense')
+  TYPESENSE_URL: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.SEARCH_DRIVER === 'typesense')
+  @IsString()
+  TYPESENSE_API_KEY: string;
+
+  @IsOptional()
+  @ValidateIf((obj) => obj.SEARCH_DRIVER === 'typesense')
+  @IsISO6391()
+  @IsString()
+  TYPESENSE_LOCALE: string;
 }
 
 export function validate(config: Record<string, any>) {

apps/server/src/integrations/import/services/file-import-task.service.ts

@@ -32,6 +32,8 @@ import { ImportAttachmentService } from './import-attachment.service';
 import { ModuleRef } from '@nestjs/core';
 import { PageService } from '../../../core/page/services/page.service';
 import { ImportPageNode } from '../dto/file-task-dto';
+import { EventEmitter2 } from '@nestjs/event-emitter';
+import { EventName } from '../../../common/events/event.contants';
 
 @Injectable()
 export class FileImportTaskService {
@@ -45,6 +47,7 @@ export class FileImportTaskService {
     @InjectKysely() private readonly db: KyselyDB,
     private readonly importAttachmentService: ImportAttachmentService,
     private moduleRef: ModuleRef,
+    private eventEmitter: EventEmitter2,
   ) {}
 
   async processZIpImport(fileTaskId: string): Promise<void> {
@@ -396,6 +399,12 @@ export class FileImportTaskService {
           }
         }
 
+        if (validPageIds.size > 0) {
+          this.eventEmitter.emit(EventName.PAGE_CREATED, {
+            pageIds: Array.from(validPageIds),
+          });
+        }
+
         this.logger.log(
           `Successfully imported ${totalPagesProcessed} pages with ${filteredBacklinks.length} backlinks`,
         );

apps/server/src/integrations/queue/constants/queue.constants.ts

@@ -4,6 +4,7 @@ export enum QueueName {
   GENERAL_QUEUE = '{general-queue}',
   BILLING_QUEUE = '{billing-queue}',
   FILE_TASK_QUEUE = '{file-task-queue}',
+  SEARCH_QUEUE = '{search-queue}',
   AI_QUEUE = '{ai-queue}',
 }
 
@@ -27,6 +28,23 @@ export enum QueueJob {
   IMPORT_TASK = 'import-task',
   EXPORT_TASK = 'export-task',
 
+  SEARCH_INDEX_PAGE = 'search-index-page',
+  SEARCH_INDEX_PAGES = 'search-index-pages',
+  SEARCH_INDEX_COMMENT = 'search-index-comment',
+  SEARCH_INDEX_COMMENTS = 'search-index-comments',
+  SEARCH_INDEX_ATTACHMENT = 'search-index-attachment',
+  SEARCH_INDEX_ATTACHMENTS = 'search-index-attachments',
+  SEARCH_REMOVE_PAGE = 'search-remove-page',
+  SEARCH_REMOVE_ASSET = 'search-remove-attachment',
+  SEARCH_REMOVE_FACE = 'search-remove-comment',
+  TYPESENSE_FLUSH = 'typesense-flush',
+
+  PAGE_CREATED = 'page-created',
+  PAGE_UPDATED = 'page-updated',
+  PAGE_SOFT_DELETED = 'page-soft-deleted',
+  PAGE_RESTORED = 'page-restored',
+  PAGE_DELETED = 'page-deleted',
+
   GENERATE_PAGE_EMBEDDINGS = 'generate-page-embeddings',
   DELETE_PAGE_EMBEDDINGS = 'delete-page-embeddings',
 }

apps/server/src/integrations/queue/queue.module.ts

@@ -57,6 +57,14 @@ import { BacklinksProcessor } from './processors/backlinks.processor';
         attempts: 1,
       },
     }),
+    BullModule.registerQueue({
+      name: QueueName.SEARCH_QUEUE,
+      defaultJobOptions: {
+        removeOnComplete: true,
+        removeOnFail: true,
+        attempts: 2,
+      },
+    }),
     BullModule.registerQueue({
       name: QueueName.AI_QUEUE,
       defaultJobOptions: {

apps/server/src/integrations/redis/redis-config.service.ts

@@ -0,0 +1,26 @@
+import { Injectable } from '@nestjs/common';
+import {
+  RedisModuleOptions,
+  RedisOptionsFactory,
+} from '@nestjs-labs/nestjs-ioredis';
+import { createRetryStrategy, parseRedisUrl } from '../../common/helpers';
+import { EnvironmentService } from '../environment/environment.service';
+
+@Injectable()
+export class RedisConfigService implements RedisOptionsFactory {
+  constructor(private readonly environmentService: EnvironmentService) {}
+  createRedisOptions(): RedisModuleOptions {
+    const redisConfig = parseRedisUrl(this.environmentService.getRedisUrl());
+    return {
+      readyLog: true,
+      config: {
+        host: redisConfig.host,
+        port: redisConfig.port,
+        password: redisConfig.password,
+        db: redisConfig.db,
+        family: redisConfig.family,
+        retryStrategy: createRetryStrategy(),
+      },
+    };
+  }
+}