From bf8cf6254f570a1a6b33b5a743af915481908b29 Mon Sep 17 00:00:00 2001 From: Philip Okugbe <16838612+Philipinho@users.noreply.github.com> Date: Tue, 7 Oct 2025 17:34:32 +0100 Subject: [PATCH] feat: Typesense search driver (EE) (#1664) * feat: typesense driver (EE) - WIP * feat: typesense driver (EE) - WIP * feat: typesense * sync * fix --- apps/server/package.json | 5 +- apps/server/src/app.module.ts | 5 + .../src/common/events/event.contants.ts | 7 +- .../server/src/common/validator/is-iso6391.ts | 34 +++++++ apps/server/src/core/page/page.module.ts | 2 +- .../src/core/page/services/page.service.ts | 11 +++ .../core/search/dto/search-response.dto.ts | 3 + .../src/core/search/search.controller.ts | 57 ++++++++++- apps/server/src/core/search/search.service.ts | 3 +- apps/server/src/database/database.module.ts | 6 +- .../src/database/listeners/page.listener.ts | 49 ++++++++++ .../src/database/repos/page/page.repo.ts | 63 ++++++++---- apps/server/src/ee | 2 +- .../environment/environment.service.ts | 20 ++++ .../environment/environment.validation.ts | 33 +++++++ .../services/file-import-task.service.ts | 9 ++ .../queue/constants/queue.constants.ts | 18 ++++ .../src/integrations/queue/queue.module.ts | 8 ++ .../redis/redis-config.service.ts | 26 +++++ pnpm-lock.yaml | 98 ++++++++++++++----- 20 files changed, 406 insertions(+), 53 deletions(-) create mode 100644 apps/server/src/common/validator/is-iso6391.ts create mode 100644 apps/server/src/database/listeners/page.listener.ts create mode 100644 apps/server/src/integrations/redis/redis-config.service.ts diff --git a/apps/server/package.json b/apps/server/package.json index 0c5ea90e..38be6184 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -37,6 +37,7 @@ "@fastify/cookie": "^11.0.2", "@fastify/multipart": "^9.0.3", "@fastify/static": "^8.2.0", + "@nestjs-labs/nestjs-ioredis": "^11.0.4", "@nestjs/bullmq": "^11.0.2", "@nestjs/common": "^11.1.3", "@nestjs/config": "^4.0.2", @@ -55,7 +56,7 @@ "@react-email/render": "1.0.2", "@socket.io/redis-adapter": "^8.3.0", "bcrypt": "^5.1.1", - "bullmq": "^5.53.2", + "bullmq": "^5.61.0", "cache-manager": "^6.4.3", "cheerio": "^1.1.0", "class-transformer": "^0.5.1", @@ -63,6 +64,7 @@ "cookie": "^1.0.2", "fs-extra": "^11.3.0", "happy-dom": "^18.0.1", + "ioredis": "^5.4.1", "jsonwebtoken": "^9.0.2", "kysely": "^0.28.2", "kysely-migration-cli": "^0.4.2", @@ -89,6 +91,7 @@ "socket.io": "^4.8.1", "stripe": "^17.5.0", "tmp-promise": "^3.0.3", + "typesense": "^2.1.0", "ws": "^8.18.2", "yauzl": "^3.2.0" }, diff --git a/apps/server/src/app.module.ts b/apps/server/src/app.module.ts index 052faed2..56691444 100644 --- a/apps/server/src/app.module.ts +++ b/apps/server/src/app.module.ts @@ -16,6 +16,8 @@ import { ExportModule } from './integrations/export/export.module'; import { ImportModule } from './integrations/import/import.module'; import { SecurityModule } from './integrations/security/security.module'; import { TelemetryModule } from './integrations/telemetry/telemetry.module'; +import { RedisModule } from '@nestjs-labs/nestjs-ioredis'; +import { RedisConfigService } from './integrations/redis/redis-config.service'; const enterpriseModules = []; try { @@ -36,6 +38,9 @@ try { CoreModule, DatabaseModule, EnvironmentModule, + RedisModule.forRootAsync({ + useClass: RedisConfigService, + }), CollaborationModule, WsModule, QueueModule, diff --git a/apps/server/src/common/events/event.contants.ts b/apps/server/src/common/events/event.contants.ts index 23149288..7adeb043 100644 --- a/apps/server/src/common/events/event.contants.ts +++ b/apps/server/src/common/events/event.contants.ts @@ -1,3 +1,8 @@ export enum EventName { COLLAB_PAGE_UPDATED = 'collab.page.updated', -} \ No newline at end of file + PAGE_CREATED = 'page.created', + PAGE_UPDATED = 'page.updated', + PAGE_DELETED = 'page.deleted', + PAGE_SOFT_DELETED = 'page.soft_deleted', + PAGE_RESTORED = 'page.restored', +} diff --git a/apps/server/src/common/validator/is-iso6391.ts b/apps/server/src/common/validator/is-iso6391.ts new file mode 100644 index 00000000..888157f0 --- /dev/null +++ b/apps/server/src/common/validator/is-iso6391.ts @@ -0,0 +1,34 @@ +// MIT - https://github.com/typestack/class-validator/pull/2626 +import isISO6391Validator from 'validator/lib/isISO6391'; +import { buildMessage, ValidateBy, ValidationOptions } from 'class-validator'; + +export const IS_ISO6391 = 'isISO6391'; + +/** + * Check if the string is a valid [ISO 639-1](https://en.wikipedia.org/wiki/ISO_639-1) officially assigned language code. + */ +export function isISO6391(value: unknown): boolean { + return typeof value === 'string' && isISO6391Validator(value); +} + +/** + * Check if the string is a valid [ISO 639-1](https://en.wikipedia.org/wiki/ISO_639-1) officially assigned language code. + */ +export function IsISO6391( + validationOptions?: ValidationOptions, +): PropertyDecorator { + return ValidateBy( + { + name: IS_ISO6391, + validator: { + validate: (value, args): boolean => isISO6391(value), + defaultMessage: buildMessage( + (eachPrefix) => + eachPrefix + '$property must be a valid ISO 639-1 language code', + validationOptions, + ), + }, + }, + validationOptions, + ); +} diff --git a/apps/server/src/core/page/page.module.ts b/apps/server/src/core/page/page.module.ts index 42693e3d..9dfba84a 100644 --- a/apps/server/src/core/page/page.module.ts +++ b/apps/server/src/core/page/page.module.ts @@ -9,6 +9,6 @@ import { StorageModule } from '../../integrations/storage/storage.module'; controllers: [PageController], providers: [PageService, PageHistoryService, TrashCleanupService], exports: [PageService, PageHistoryService], - imports: [StorageModule] + imports: [StorageModule], }) export class PageModule {} diff --git a/apps/server/src/core/page/services/page.service.ts b/apps/server/src/core/page/services/page.service.ts index a538eedf..3ffa2042 100644 --- a/apps/server/src/core/page/services/page.service.ts +++ b/apps/server/src/core/page/services/page.service.ts @@ -38,6 +38,8 @@ import { StorageService } from '../../../integrations/storage/storage.service'; import { InjectQueue } from '@nestjs/bullmq'; import { Queue } from 'bullmq'; import { QueueJob, QueueName } from '../../../integrations/queue/constants'; +import { EventName } from '../../../common/events/event.contants'; +import { EventEmitter2 } from '@nestjs/event-emitter'; @Injectable() export class PageService { @@ -49,6 +51,7 @@ export class PageService { @InjectKysely() private readonly db: KyselyDB, private readonly storageService: StorageService, @InjectQueue(QueueName.ATTACHMENT_QUEUE) private attachmentQueue: Queue, + private eventEmitter: EventEmitter2, ) {} async findById( @@ -380,6 +383,11 @@ export class PageService { await this.db.insertInto('pages').values(insertablePages).execute(); + const insertedPageIds = insertablePages.map((page) => page.id); + this.eventEmitter.emit(EventName.PAGE_CREATED, { + pageIds: insertedPageIds, + }); + //TODO: best to handle this in a queue const attachmentsIds = Array.from(attachmentMap.keys()); if (attachmentsIds.length > 0) { @@ -606,6 +614,9 @@ export class PageService { if (pageIds.length > 0) { await this.db.deleteFrom('pages').where('id', 'in', pageIds).execute(); + this.eventEmitter.emit(EventName.PAGE_DELETED, { + pageIds: pageIds, + }); } } diff --git a/apps/server/src/core/search/dto/search-response.dto.ts b/apps/server/src/core/search/dto/search-response.dto.ts index bf8db9d1..8f5b343d 100644 --- a/apps/server/src/core/search/dto/search-response.dto.ts +++ b/apps/server/src/core/search/dto/search-response.dto.ts @@ -1,3 +1,5 @@ +import { Space } from '@docmost/db/types/entity.types'; + export class SearchResponseDto { id: string; title: string; @@ -8,4 +10,5 @@ export class SearchResponseDto { highlight: string; createdAt: Date; updatedAt: Date; + space: Partial; } diff --git a/apps/server/src/core/search/search.controller.ts b/apps/server/src/core/search/search.controller.ts index 35083faf..c968c344 100644 --- a/apps/server/src/core/search/search.controller.ts +++ b/apps/server/src/core/search/search.controller.ts @@ -5,6 +5,7 @@ import { ForbiddenException, HttpCode, HttpStatus, + Logger, Post, UseGuards, } from '@nestjs/common'; @@ -24,13 +25,19 @@ import { } from '../casl/interfaces/space-ability.type'; import { AuthUser } from '../../common/decorators/auth-user.decorator'; import { Public } from 'src/common/decorators/public.decorator'; +import { EnvironmentService } from '../../integrations/environment/environment.service'; +import { ModuleRef } from '@nestjs/core'; @UseGuards(JwtAuthGuard) @Controller('search') export class SearchController { + private readonly logger = new Logger(SearchController.name); + constructor( private readonly searchService: SearchService, private readonly spaceAbility: SpaceAbilityFactory, + private readonly environmentService: EnvironmentService, + private moduleRef: ModuleRef, ) {} @HttpCode(HttpStatus.OK) @@ -53,7 +60,14 @@ export class SearchController { } } - return this.searchService.searchPage(searchDto.query, searchDto, { + if (this.environmentService.getSearchDriver() === 'typesense') { + return this.searchTypesense(searchDto, { + userId: user.id, + workspaceId: workspace.id, + }); + } + + return this.searchService.searchPage(searchDto, { userId: user.id, workspaceId: workspace.id, }); @@ -81,8 +95,47 @@ export class SearchController { throw new BadRequestException('shareId is required'); } - return this.searchService.searchPage(searchDto.query, searchDto, { + if (this.environmentService.getSearchDriver() === 'typesense') { + return this.searchTypesense(searchDto, { + workspaceId: workspace.id, + }); + } + + return this.searchService.searchPage(searchDto, { workspaceId: workspace.id, }); } + + async searchTypesense( + searchParams: SearchDTO, + opts: { + userId?: string; + workspaceId: string; + }, + ) { + const { userId, workspaceId } = opts; + let TypesenseModule: any; + try { + // eslint-disable-next-line @typescript-eslint/no-require-imports + TypesenseModule = require('./../../ee/typesense/services/page-search.service'); + + const PageSearchService = this.moduleRef.get( + TypesenseModule.PageSearchService, + { + strict: false, + }, + ); + + return PageSearchService.searchPage(searchParams, { + userId: userId, + workspaceId, + }); + } catch (err) { + this.logger.debug( + 'Typesense module requested but enterprise module not bundled in this build', + ); + } + + throw new BadRequestException('Enterprise Typesense search module missing'); + } } diff --git a/apps/server/src/core/search/search.service.ts b/apps/server/src/core/search/search.service.ts index 60432ce8..0f8dbb90 100644 --- a/apps/server/src/core/search/search.service.ts +++ b/apps/server/src/core/search/search.service.ts @@ -21,13 +21,14 @@ export class SearchService { ) {} async searchPage( - query: string, searchParams: SearchDTO, opts: { userId?: string; workspaceId: string; }, ): Promise { + const { query } = searchParams; + if (query.length < 1) { return; } diff --git a/apps/server/src/database/database.module.ts b/apps/server/src/database/database.module.ts index 68c35dd3..bd331ada 100644 --- a/apps/server/src/database/database.module.ts +++ b/apps/server/src/database/database.module.ts @@ -25,6 +25,7 @@ import { MigrationService } from '@docmost/db/services/migration.service'; import { UserTokenRepo } from './repos/user-token/user-token.repo'; import { BacklinkRepo } from '@docmost/db/repos/backlink/backlink.repo'; import { ShareRepo } from '@docmost/db/repos/share/share.repo'; +import { PageListener } from '@docmost/db/listeners/page.listener'; // https://github.com/brianc/node-postgres/issues/811 types.setTypeParser(types.builtins.INT8, (val) => Number(val)); @@ -75,7 +76,8 @@ types.setTypeParser(types.builtins.INT8, (val) => Number(val)); AttachmentRepo, UserTokenRepo, BacklinkRepo, - ShareRepo + ShareRepo, + PageListener, ], exports: [ WorkspaceRepo, @@ -90,7 +92,7 @@ types.setTypeParser(types.builtins.INT8, (val) => Number(val)); AttachmentRepo, UserTokenRepo, BacklinkRepo, - ShareRepo + ShareRepo, ], }) export class DatabaseModule diff --git a/apps/server/src/database/listeners/page.listener.ts b/apps/server/src/database/listeners/page.listener.ts new file mode 100644 index 00000000..7e2d97e2 --- /dev/null +++ b/apps/server/src/database/listeners/page.listener.ts @@ -0,0 +1,49 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { OnEvent } from '@nestjs/event-emitter'; +import { EventName } from '../../common/events/event.contants'; +import { InjectQueue } from '@nestjs/bullmq'; +import { QueueJob, QueueName } from '../../integrations/queue/constants'; +import { Queue } from 'bullmq'; + +export class PageEvent { + pageIds: string[]; +} + +@Injectable() +export class PageListener { + private readonly logger = new Logger(PageListener.name); + + constructor( + @InjectQueue(QueueName.SEARCH_QUEUE) private searchQueue: Queue, + ) {} + + @OnEvent(EventName.PAGE_CREATED) + async handlePageCreated(event: PageEvent) { + const { pageIds } = event; + await this.searchQueue.add(QueueJob.PAGE_CREATED, { pageIds }); + } + + @OnEvent(EventName.PAGE_UPDATED) + async handlePageUpdated(event: PageEvent) { + const { pageIds } = event; + await this.searchQueue.add(QueueJob.PAGE_UPDATED, { pageIds }); + } + + @OnEvent(EventName.PAGE_DELETED) + async handlePageDeleted(event: PageEvent) { + const { pageIds } = event; + await this.searchQueue.add(QueueJob.PAGE_DELETED, { pageIds }); + } + + @OnEvent(EventName.PAGE_SOFT_DELETED) + async handlePageSoftDeleted(event: PageEvent) { + const { pageIds } = event; + await this.searchQueue.add(QueueJob.PAGE_SOFT_DELETED, { pageIds }); + } + + @OnEvent(EventName.PAGE_RESTORED) + async handlePageRestored(event: PageEvent) { + const { pageIds } = event; + await this.searchQueue.add(QueueJob.PAGE_RESTORED, { pageIds }); + } +} diff --git a/apps/server/src/database/repos/page/page.repo.ts b/apps/server/src/database/repos/page/page.repo.ts index e577cc43..ca46ddc9 100644 --- a/apps/server/src/database/repos/page/page.repo.ts +++ b/apps/server/src/database/repos/page/page.repo.ts @@ -14,32 +14,17 @@ import { ExpressionBuilder, sql } from 'kysely'; import { DB } from '@docmost/db/types/db'; import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres'; import { SpaceMemberRepo } from '@docmost/db/repos/space/space-member.repo'; +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { EventName } from '../../../common/events/event.contants'; @Injectable() export class PageRepo { constructor( @InjectKysely() private readonly db: KyselyDB, private spaceMemberRepo: SpaceMemberRepo, + private eventEmitter: EventEmitter2, ) {} - withHasChildren(eb: ExpressionBuilder) { - return eb - .selectFrom('pages as child') - .select((eb) => - eb - .case() - .when(eb.fn.countAll(), '>', 0) - .then(true) - .else(false) - .end() - .as('count'), - ) - .whereRef('child.parentPageId', '=', 'pages.id') - .where('child.deletedAt', 'is', null) - .limit(1) - .as('hasChildren'); - } - private baseFields: Array = [ 'id', 'slugId', @@ -63,6 +48,7 @@ export class PageRepo { pageId: string, opts?: { includeContent?: boolean; + includeTextContent?: boolean; includeYdoc?: boolean; includeSpace?: boolean; includeCreator?: boolean; @@ -80,6 +66,7 @@ export class PageRepo { .select(this.baseFields) .$if(opts?.includeContent, (qb) => qb.select('content')) .$if(opts?.includeYdoc, (qb) => qb.select('ydoc')) + .$if(opts?.includeTextContent, (qb) => qb.select('textContent')) .$if(opts?.includeHasChildren, (qb) => qb.select((eb) => this.withHasChildren(eb)), ); @@ -126,7 +113,7 @@ export class PageRepo { pageIds: string[], trx?: KyselyTransaction, ) { - return dbOrTx(this.db, trx) + const result = await dbOrTx(this.db, trx) .updateTable('pages') .set({ ...updatePageData, updatedAt: new Date() }) .where( @@ -135,6 +122,12 @@ export class PageRepo { pageIds, ) .executeTakeFirst(); + + this.eventEmitter.emit(EventName.PAGE_UPDATED, { + pageIds: pageIds, + }); + + return result; } async insertPage( @@ -142,11 +135,17 @@ export class PageRepo { trx?: KyselyTransaction, ): Promise { const db = dbOrTx(this.db, trx); - return db + const result = await db .insertInto('pages') .values(insertablePage) .returning(this.baseFields) .executeTakeFirst(); + + this.eventEmitter.emit(EventName.PAGE_CREATED, { + pageIds: [result.id], + }); + + return result; } async deletePage(pageId: string): Promise { @@ -196,6 +195,9 @@ export class PageRepo { await trx.deleteFrom('shares').where('pageId', 'in', pageIds).execute(); }); + this.eventEmitter.emit(EventName.PAGE_SOFT_DELETED, { + pageIds: pageIds, + }); } } @@ -259,6 +261,9 @@ export class PageRepo { .where('id', '=', pageId) .execute(); } + this.eventEmitter.emit(EventName.PAGE_RESTORED, { + pageIds: pageIds, + }); } async getRecentPagesInSpace(spaceId: string, pagination: PaginationOptions) { @@ -379,6 +384,24 @@ export class PageRepo { ).as('contributors'); } + withHasChildren(eb: ExpressionBuilder) { + return eb + .selectFrom('pages as child') + .select((eb) => + eb + .case() + .when(eb.fn.countAll(), '>', 0) + .then(true) + .else(false) + .end() + .as('count'), + ) + .whereRef('child.parentPageId', '=', 'pages.id') + .where('child.deletedAt', 'is', null) + .limit(1) + .as('hasChildren'); + } + async getPageAndDescendants( parentPageId: string, opts: { includeContent: boolean }, diff --git a/apps/server/src/ee b/apps/server/src/ee index d2ead431..a4a19f71 160000 --- a/apps/server/src/ee +++ b/apps/server/src/ee @@ -1 +1 @@ -Subproject commit d2ead431819025e735e8b8e63d6d898d76c417e6 +Subproject commit a4a19f71e15e3770e6a4af24d54c198aba65254a diff --git a/apps/server/src/integrations/environment/environment.service.ts b/apps/server/src/integrations/environment/environment.service.ts index 3ce728ea..e41a5ec3 100644 --- a/apps/server/src/integrations/environment/environment.service.ts +++ b/apps/server/src/integrations/environment/environment.service.ts @@ -213,4 +213,24 @@ export class EnvironmentService { getPostHogKey(): string { return this.configService.get('POSTHOG_KEY'); } + + getSearchDriver(): string { + return this.configService + .get('SEARCH_DRIVER', 'database') + .toLowerCase(); + } + + getTypesenseUrl(): string { + return this.configService + .get('TYPESENSE_URL', 'http://localhost:8108') + .toLowerCase(); + } + + getTypesenseApiKey(): string { + return this.configService.get('TYPESENSE_API_KEY'); + } + + getTypesenseLocale(): string { + return this.configService.get('TYPESENSE_LOCALE', 'en').toLowerCase(); + } } diff --git a/apps/server/src/integrations/environment/environment.validation.ts b/apps/server/src/integrations/environment/environment.validation.ts index a2aeb6dd..d59558f8 100644 --- a/apps/server/src/integrations/environment/environment.validation.ts +++ b/apps/server/src/integrations/environment/environment.validation.ts @@ -3,12 +3,14 @@ import { IsNotEmpty, IsNotIn, IsOptional, + IsString, IsUrl, MinLength, ValidateIf, validateSync, } from 'class-validator'; import { plainToInstance } from 'class-transformer'; +import { IsISO6391 } from '../../common/validator/is-iso6391'; export class EnvironmentVariables { @IsNotEmpty() @@ -68,6 +70,37 @@ export class EnvironmentVariables { ) @ValidateIf((obj) => obj.CLOUD === 'true'.toLowerCase()) SUBDOMAIN_HOST: string; + + @IsOptional() + @IsIn(['database', 'typesense']) + @IsString() + SEARCH_DRIVER: string; + + @IsOptional() + @IsUrl( + { + protocols: ['http', 'https'], + require_tld: false, + allow_underscores: true, + }, + { + message: + 'TYPESENSE_URL must be a valid typesense url e.g http://localhost:8108', + }, + ) + @ValidateIf((obj) => obj.SEARCH_DRIVER === 'typesense') + TYPESENSE_URL: string; + + @IsOptional() + @ValidateIf((obj) => obj.SEARCH_DRIVER === 'typesense') + @IsString() + TYPESENSE_API_KEY: string; + + @IsOptional() + @ValidateIf((obj) => obj.SEARCH_DRIVER === 'typesense') + @IsISO6391() + @IsString() + TYPESENSE_LOCALE: string; } export function validate(config: Record) { diff --git a/apps/server/src/integrations/import/services/file-import-task.service.ts b/apps/server/src/integrations/import/services/file-import-task.service.ts index f7d93ec0..6337f9e1 100644 --- a/apps/server/src/integrations/import/services/file-import-task.service.ts +++ b/apps/server/src/integrations/import/services/file-import-task.service.ts @@ -32,6 +32,8 @@ import { ImportAttachmentService } from './import-attachment.service'; import { ModuleRef } from '@nestjs/core'; import { PageService } from '../../../core/page/services/page.service'; import { ImportPageNode } from '../dto/file-task-dto'; +import { EventEmitter2 } from '@nestjs/event-emitter'; +import { EventName } from '../../../common/events/event.contants'; @Injectable() export class FileImportTaskService { @@ -45,6 +47,7 @@ export class FileImportTaskService { @InjectKysely() private readonly db: KyselyDB, private readonly importAttachmentService: ImportAttachmentService, private moduleRef: ModuleRef, + private eventEmitter: EventEmitter2, ) {} async processZIpImport(fileTaskId: string): Promise { @@ -396,6 +399,12 @@ export class FileImportTaskService { } } + if (validPageIds.size > 0) { + this.eventEmitter.emit(EventName.PAGE_CREATED, { + pageIds: Array.from(validPageIds), + }); + } + this.logger.log( `Successfully imported ${totalPagesProcessed} pages with ${filteredBacklinks.length} backlinks`, ); diff --git a/apps/server/src/integrations/queue/constants/queue.constants.ts b/apps/server/src/integrations/queue/constants/queue.constants.ts index 4a1b1d1c..122d2a76 100644 --- a/apps/server/src/integrations/queue/constants/queue.constants.ts +++ b/apps/server/src/integrations/queue/constants/queue.constants.ts @@ -4,6 +4,7 @@ export enum QueueName { GENERAL_QUEUE = '{general-queue}', BILLING_QUEUE = '{billing-queue}', FILE_TASK_QUEUE = '{file-task-queue}', + SEARCH_QUEUE = '{search-queue}', } export enum QueueJob { @@ -25,4 +26,21 @@ export enum QueueJob { IMPORT_TASK = 'import-task', EXPORT_TASK = 'export-task', + + SEARCH_INDEX_PAGE = 'search-index-page', + SEARCH_INDEX_PAGES = 'search-index-pages', + SEARCH_INDEX_COMMENT = 'search-index-comment', + SEARCH_INDEX_COMMENTS = 'search-index-comments', + SEARCH_INDEX_ATTACHMENT = 'search-index-attachment', + SEARCH_INDEX_ATTACHMENTS = 'search-index-attachments', + SEARCH_REMOVE_PAGE = 'search-remove-page', + SEARCH_REMOVE_ASSET = 'search-remove-attachment', + SEARCH_REMOVE_FACE = 'search-remove-comment', + TYPESENSE_FLUSH = 'typesense-flush', + + PAGE_CREATED = 'page-created', + PAGE_UPDATED = 'page-updated', + PAGE_SOFT_DELETED = 'page-soft-deleted', + PAGE_RESTORED = 'page-restored', + PAGE_DELETED = 'page-deleted', } diff --git a/apps/server/src/integrations/queue/queue.module.ts b/apps/server/src/integrations/queue/queue.module.ts index 81aa0a5f..32d009ad 100644 --- a/apps/server/src/integrations/queue/queue.module.ts +++ b/apps/server/src/integrations/queue/queue.module.ts @@ -57,6 +57,14 @@ import { BacklinksProcessor } from './processors/backlinks.processor'; attempts: 1, }, }), + BullModule.registerQueue({ + name: QueueName.SEARCH_QUEUE, + defaultJobOptions: { + removeOnComplete: true, + removeOnFail: true, + attempts: 2, + }, + }), ], exports: [BullModule], providers: [BacklinksProcessor], diff --git a/apps/server/src/integrations/redis/redis-config.service.ts b/apps/server/src/integrations/redis/redis-config.service.ts new file mode 100644 index 00000000..719f89f1 --- /dev/null +++ b/apps/server/src/integrations/redis/redis-config.service.ts @@ -0,0 +1,26 @@ +import { Injectable } from '@nestjs/common'; +import { + RedisModuleOptions, + RedisOptionsFactory, +} from '@nestjs-labs/nestjs-ioredis'; +import { createRetryStrategy, parseRedisUrl } from '../../common/helpers'; +import { EnvironmentService } from '../environment/environment.service'; + +@Injectable() +export class RedisConfigService implements RedisOptionsFactory { + constructor(private readonly environmentService: EnvironmentService) {} + createRedisOptions(): RedisModuleOptions { + const redisConfig = parseRedisUrl(this.environmentService.getRedisUrl()); + return { + readyLog: true, + config: { + host: redisConfig.host, + port: redisConfig.port, + password: redisConfig.password, + db: redisConfig.db, + family: redisConfig.family, + retryStrategy: createRetryStrategy(), + }, + }; + } +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index aad3a5ed..b9ac61ba 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -447,9 +447,12 @@ importers: '@fastify/static': specifier: ^8.2.0 version: 8.2.0 + '@nestjs-labs/nestjs-ioredis': + specifier: ^11.0.4 + version: 11.0.4(@nestjs/common@11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.3)(ioredis@5.4.1) '@nestjs/bullmq': specifier: ^11.0.2 - version: 11.0.2(@nestjs/common@11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.3)(bullmq@5.53.2) + version: 11.0.2(@nestjs/common@11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.3)(bullmq@5.61.0) '@nestjs/common': specifier: ^11.1.3 version: 11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2) @@ -502,8 +505,8 @@ importers: specifier: ^5.1.1 version: 5.1.1 bullmq: - specifier: ^5.53.2 - version: 5.53.2 + specifier: ^5.61.0 + version: 5.61.0 cache-manager: specifier: ^6.4.3 version: 6.4.3 @@ -523,8 +526,11 @@ importers: specifier: ^11.3.0 version: 11.3.0 happy-dom: - specifier: ^15.11.6 - version: 15.11.7 + specifier: ^18.0.1 + version: 18.0.1 + ioredis: + specifier: ^5.4.1 + version: 5.4.1 jsonwebtoken: specifier: ^9.0.2 version: 9.0.2 @@ -603,6 +609,9 @@ importers: tmp-promise: specifier: ^3.0.3 version: 3.0.3 + typesense: + specifier: ^2.1.0 + version: 2.1.0(@babel/runtime@7.25.6) ws: specifier: ^8.18.2 version: 8.18.2 @@ -2820,6 +2829,14 @@ packages: '@napi-rs/wasm-runtime@0.2.4': resolution: {integrity: sha512-9zESzOO5aDByvhIAsOy9TbpZ0Ur2AJbUI7UT73kcUTS2mxAMHOBaa1st/jAymNoCtvrit99kkzT1FZuXVcgfIQ==} + '@nestjs-labs/nestjs-ioredis@11.0.4': + resolution: {integrity: sha512-4jPNOrxDiwNMIN5OLmsMWhA782kxv/ZBxkySX9l8n6sr55acHX/BciaFsOXVa/ILsm+Y7893y98/6WNhmEoiNQ==} + engines: {node: '>=16'} + peerDependencies: + '@nestjs/common': ^10.0.0 || ^11.0.0 + '@nestjs/core': ^10.0.0 || ^11.0.0 + ioredis: ^5.0.0 + '@nestjs/bull-shared@11.0.2': resolution: {integrity: sha512-dFlttJvBqIFD6M8JVFbkrR4Feb39OTAJPJpFVILU50NOJCM4qziRw3dSNG84Q3v+7/M6xUGMFdZRRGvBBKxoSA==} peerDependencies: @@ -4547,6 +4564,9 @@ packages: '@types/ms@2.1.0': resolution: {integrity: sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==} + '@types/node@20.19.19': + resolution: {integrity: sha512-pb1Uqj5WJP7wrcbLU7Ru4QtA0+3kAXrkutGiD26wUKzSMgNNaPARTUDQmElUXp64kh3cWdou3Q0C7qwwxqSFmg==} + '@types/node@22.10.0': resolution: {integrity: sha512-XC70cRZVElFHfIUB40FgZOBbgJYFKKMa5nb9lxcwYstFG/Mi+/Y0bGS+rs6Dmhmkpq4pnNiLiuZAbc02YCOnmA==} @@ -4637,6 +4657,9 @@ packages: '@types/validator@13.12.0': resolution: {integrity: sha512-nH45Lk7oPIJ1RVOF6JgFI6Dy0QpHEzq4QecZhvguxYPDwT8c93prCMqAtiIttm39voZ+DDR+qkNnMpJmMBRqag==} + '@types/whatwg-mimetype@3.0.2': + resolution: {integrity: sha512-c2AKvDT8ToxLIOUlN51gTiHXflsfIFisS4pO7pDPoKouJCESkhZnEy623gwP9laCy5lnLDAw1vAzu2vM2YLOrA==} + '@types/ws@8.5.14': resolution: {integrity: sha512-bd/YFLW+URhBzMXurx7lWByOu+xzU9+kb3RboOteXYDfW+tr+JZa99OyNmPINEGB/ahzKrEuc8rcv4gnpJmxTw==} @@ -5217,8 +5240,8 @@ packages: builtins@5.0.1: resolution: {integrity: sha512-qwVpFEHNfhYJIzNRBvd2C1kyo6jz3ZSMPyyuR47OPdiKWlbYnZNyDWuyR175qDnAJLiCo5fBBqPb3RiXgWlkOQ==} - bullmq@5.53.2: - resolution: {integrity: sha512-xHgxrP/yNJHD7VCw1h+eRBh+2TCPBCM39uC9gCyksYc6ufcJP+HTZ/A2lzB2x7qMFWrvsX7tM40AT2BmdkYL/Q==} + bullmq@5.61.0: + resolution: {integrity: sha512-khaTjc1JnzaYFl4FrUtsSsqugAW/urRrcZ9Q0ZE+REAw8W+gkHFqxbGlutOu6q7j7n91wibVaaNlOUMdiEvoSQ==} busboy@1.6.0: resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==} @@ -6537,9 +6560,9 @@ packages: hachure-fill@0.5.2: resolution: {integrity: sha512-3GKBOn+m2LX9iq+JC1064cSFprJY4jL1jCXTcpnfER5HYE2l/4EfWSGzkPa/ZDBmYI0ZOEj5VHV/eKnPGkHuOg==} - happy-dom@15.11.7: - resolution: {integrity: sha512-KyrFvnl+J9US63TEzwoiJOQzZBJY7KgBushJA8X61DMbNsH+2ONkDuLDnCnwUiPTF42tLoEmrPyoqbenVA5zrg==} - engines: {node: '>=18.0.0'} + happy-dom@18.0.1: + resolution: {integrity: sha512-qn+rKOW7KWpVTtgIUi6RVmTBZJSe2k0Db0vh1f7CWrWclkkc7/Q+FrOfkZIb2eiErLyqu5AXEzE7XthO9JVxRA==} + engines: {node: '>=20.0.0'} has-bigints@1.0.2: resolution: {integrity: sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==} @@ -7442,10 +7465,6 @@ packages: resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==} engines: {node: '>=10'} - luxon@3.5.0: - resolution: {integrity: sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==} - engines: {node: '>=12'} - luxon@3.6.1: resolution: {integrity: sha512-tJLxrKJhO2ukZ5z0gyjY1zPh3Rh88Ej9P7jNrZiHMUXHae1yvI2imgOZtL1TO8TW6biMMKfTtAOoEJANgtWBMQ==} engines: {node: '>=12'} @@ -9457,6 +9476,12 @@ packages: engines: {node: '>=14.17'} hasBin: true + typesense@2.1.0: + resolution: {integrity: sha512-a/IRTL+dRXlpRDU4UodyGj8hl5xBz3nKihVRd/KfSFAfFPGcpdX6lxIgwdXy3O6VLNNiEsN8YwIsPHQPVT0vNw==} + engines: {node: '>=18'} + peerDependencies: + '@babel/runtime': ^7.23.2 + uc.micro@2.1.0: resolution: {integrity: sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==} @@ -9487,6 +9512,9 @@ packages: undici-types@6.20.0: resolution: {integrity: sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==} + undici-types@6.21.0: + resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + undici@7.10.0: resolution: {integrity: sha512-u5otvFBOBZvmdjWLVW+5DAc9Nkq8f24g0O9oY7qw2JVIF1VocIFoyz9JFkuVOS2j41AufeO0xnlweJ2RLT8nGw==} engines: {node: '>=20.18.1'} @@ -12870,18 +12898,25 @@ snapshots: '@emnapi/runtime': 1.2.0 '@tybys/wasm-util': 0.9.0 + '@nestjs-labs/nestjs-ioredis@11.0.4(@nestjs/common@11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.3)(ioredis@5.4.1)': + dependencies: + '@nestjs/common': 11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2) + '@nestjs/core': 11.1.3(@nestjs/common@11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/websockets@11.1.3)(reflect-metadata@0.2.2)(rxjs@7.8.2) + ioredis: 5.4.1 + tslib: 2.8.1 + '@nestjs/bull-shared@11.0.2(@nestjs/common@11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.3)': dependencies: '@nestjs/common': 11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2) '@nestjs/core': 11.1.3(@nestjs/common@11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/websockets@11.1.3)(reflect-metadata@0.2.2)(rxjs@7.8.2) tslib: 2.8.1 - '@nestjs/bullmq@11.0.2(@nestjs/common@11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.3)(bullmq@5.53.2)': + '@nestjs/bullmq@11.0.2(@nestjs/common@11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.3)(bullmq@5.61.0)': dependencies: '@nestjs/bull-shared': 11.0.2(@nestjs/common@11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.3) '@nestjs/common': 11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2) '@nestjs/core': 11.1.3(@nestjs/common@11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/websockets@11.1.3)(reflect-metadata@0.2.2)(rxjs@7.8.2) - bullmq: 5.53.2 + bullmq: 5.61.0 tslib: 2.8.1 '@nestjs/cli@11.0.4(@swc/core@1.5.25(@swc/helpers@0.5.5))(@types/node@22.13.4)': @@ -14666,6 +14701,10 @@ snapshots: '@types/ms@2.1.0': {} + '@types/node@20.19.19': + dependencies: + undici-types: 6.21.0 + '@types/node@22.10.0': dependencies: undici-types: 6.20.0 @@ -14778,6 +14817,8 @@ snapshots: '@types/validator@13.12.0': {} + '@types/whatwg-mimetype@3.0.2': {} + '@types/ws@8.5.14': dependencies: '@types/node': 22.13.4 @@ -15541,7 +15582,7 @@ snapshots: dependencies: semver: 7.7.2 - bullmq@5.53.2: + bullmq@5.61.0: dependencies: cron-parser: 4.9.0 ioredis: 5.4.1 @@ -15549,7 +15590,7 @@ snapshots: node-abort-controller: 3.1.1 semver: 7.7.2 tslib: 2.8.1 - uuid: 9.0.1 + uuid: 11.1.0 transitivePeerDependencies: - supports-color @@ -15873,7 +15914,7 @@ snapshots: cron-parser@4.9.0: dependencies: - luxon: 3.5.0 + luxon: 3.6.1 cron@4.3.0: dependencies: @@ -17085,10 +17126,10 @@ snapshots: hachure-fill@0.5.2: {} - happy-dom@15.11.7: + happy-dom@18.0.1: dependencies: - entities: 4.5.0 - webidl-conversions: 7.0.0 + '@types/node': 20.19.19 + '@types/whatwg-mimetype': 3.0.2 whatwg-mimetype: 3.0.0 has-bigints@1.0.2: {} @@ -18176,8 +18217,6 @@ snapshots: dependencies: yallist: 4.0.0 - luxon@3.5.0: {} - luxon@3.6.1: {} magic-string@0.30.17: @@ -20495,6 +20534,15 @@ snapshots: typescript@5.7.3: {} + typesense@2.1.0(@babel/runtime@7.25.6): + dependencies: + '@babel/runtime': 7.25.6 + axios: 1.9.0 + loglevel: 1.9.1 + tslib: 2.8.1 + transitivePeerDependencies: + - debug + uc.micro@2.1.0: {} ufo@1.6.1: {} @@ -20520,6 +20568,8 @@ snapshots: undici-types@6.20.0: {} + undici-types@6.21.0: {} + undici@7.10.0: {} unicode-canonical-property-names-ecmascript@2.0.0: {}