mirror of
https://github.com/docmost/docmost.git
synced 2025-11-10 06:52:07 +10:00
feat: Typesense search driver (EE) (#1664)
* feat: typesense driver (EE) - WIP * feat: typesense driver (EE) - WIP * feat: typesense * sync * fix
This commit is contained in:
@ -37,6 +37,7 @@
|
||||
"@fastify/cookie": "^11.0.2",
|
||||
"@fastify/multipart": "^9.0.3",
|
||||
"@fastify/static": "^8.2.0",
|
||||
"@nestjs-labs/nestjs-ioredis": "^11.0.4",
|
||||
"@nestjs/bullmq": "^11.0.2",
|
||||
"@nestjs/common": "^11.1.3",
|
||||
"@nestjs/config": "^4.0.2",
|
||||
@ -55,7 +56,7 @@
|
||||
"@react-email/render": "1.0.2",
|
||||
"@socket.io/redis-adapter": "^8.3.0",
|
||||
"bcrypt": "^5.1.1",
|
||||
"bullmq": "^5.53.2",
|
||||
"bullmq": "^5.61.0",
|
||||
"cache-manager": "^6.4.3",
|
||||
"cheerio": "^1.1.0",
|
||||
"class-transformer": "^0.5.1",
|
||||
@ -63,6 +64,7 @@
|
||||
"cookie": "^1.0.2",
|
||||
"fs-extra": "^11.3.0",
|
||||
"happy-dom": "^18.0.1",
|
||||
"ioredis": "^5.4.1",
|
||||
"jsonwebtoken": "^9.0.2",
|
||||
"kysely": "^0.28.2",
|
||||
"kysely-migration-cli": "^0.4.2",
|
||||
@ -89,6 +91,7 @@
|
||||
"socket.io": "^4.8.1",
|
||||
"stripe": "^17.5.0",
|
||||
"tmp-promise": "^3.0.3",
|
||||
"typesense": "^2.1.0",
|
||||
"ws": "^8.18.2",
|
||||
"yauzl": "^3.2.0"
|
||||
},
|
||||
|
||||
@ -16,6 +16,8 @@ import { ExportModule } from './integrations/export/export.module';
|
||||
import { ImportModule } from './integrations/import/import.module';
|
||||
import { SecurityModule } from './integrations/security/security.module';
|
||||
import { TelemetryModule } from './integrations/telemetry/telemetry.module';
|
||||
import { RedisModule } from '@nestjs-labs/nestjs-ioredis';
|
||||
import { RedisConfigService } from './integrations/redis/redis-config.service';
|
||||
|
||||
const enterpriseModules = [];
|
||||
try {
|
||||
@ -36,6 +38,9 @@ try {
|
||||
CoreModule,
|
||||
DatabaseModule,
|
||||
EnvironmentModule,
|
||||
RedisModule.forRootAsync({
|
||||
useClass: RedisConfigService,
|
||||
}),
|
||||
CollaborationModule,
|
||||
WsModule,
|
||||
QueueModule,
|
||||
|
||||
@ -1,3 +1,8 @@
|
||||
export enum EventName {
|
||||
COLLAB_PAGE_UPDATED = 'collab.page.updated',
|
||||
PAGE_CREATED = 'page.created',
|
||||
PAGE_UPDATED = 'page.updated',
|
||||
PAGE_DELETED = 'page.deleted',
|
||||
PAGE_SOFT_DELETED = 'page.soft_deleted',
|
||||
PAGE_RESTORED = 'page.restored',
|
||||
}
|
||||
34
apps/server/src/common/validator/is-iso6391.ts
Normal file
34
apps/server/src/common/validator/is-iso6391.ts
Normal file
@ -0,0 +1,34 @@
|
||||
// MIT - https://github.com/typestack/class-validator/pull/2626
|
||||
import isISO6391Validator from 'validator/lib/isISO6391';
|
||||
import { buildMessage, ValidateBy, ValidationOptions } from 'class-validator';
|
||||
|
||||
export const IS_ISO6391 = 'isISO6391';
|
||||
|
||||
/**
|
||||
* Check if the string is a valid [ISO 639-1](https://en.wikipedia.org/wiki/ISO_639-1) officially assigned language code.
|
||||
*/
|
||||
export function isISO6391(value: unknown): boolean {
|
||||
return typeof value === 'string' && isISO6391Validator(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the string is a valid [ISO 639-1](https://en.wikipedia.org/wiki/ISO_639-1) officially assigned language code.
|
||||
*/
|
||||
export function IsISO6391(
|
||||
validationOptions?: ValidationOptions,
|
||||
): PropertyDecorator {
|
||||
return ValidateBy(
|
||||
{
|
||||
name: IS_ISO6391,
|
||||
validator: {
|
||||
validate: (value, args): boolean => isISO6391(value),
|
||||
defaultMessage: buildMessage(
|
||||
(eachPrefix) =>
|
||||
eachPrefix + '$property must be a valid ISO 639-1 language code',
|
||||
validationOptions,
|
||||
),
|
||||
},
|
||||
},
|
||||
validationOptions,
|
||||
);
|
||||
}
|
||||
@ -9,6 +9,6 @@ import { StorageModule } from '../../integrations/storage/storage.module';
|
||||
controllers: [PageController],
|
||||
providers: [PageService, PageHistoryService, TrashCleanupService],
|
||||
exports: [PageService, PageHistoryService],
|
||||
imports: [StorageModule]
|
||||
imports: [StorageModule],
|
||||
})
|
||||
export class PageModule {}
|
||||
|
||||
@ -38,6 +38,8 @@ import { StorageService } from '../../../integrations/storage/storage.service';
|
||||
import { InjectQueue } from '@nestjs/bullmq';
|
||||
import { Queue } from 'bullmq';
|
||||
import { QueueJob, QueueName } from '../../../integrations/queue/constants';
|
||||
import { EventName } from '../../../common/events/event.contants';
|
||||
import { EventEmitter2 } from '@nestjs/event-emitter';
|
||||
|
||||
@Injectable()
|
||||
export class PageService {
|
||||
@ -49,6 +51,7 @@ export class PageService {
|
||||
@InjectKysely() private readonly db: KyselyDB,
|
||||
private readonly storageService: StorageService,
|
||||
@InjectQueue(QueueName.ATTACHMENT_QUEUE) private attachmentQueue: Queue,
|
||||
private eventEmitter: EventEmitter2,
|
||||
) {}
|
||||
|
||||
async findById(
|
||||
@ -380,6 +383,11 @@ export class PageService {
|
||||
|
||||
await this.db.insertInto('pages').values(insertablePages).execute();
|
||||
|
||||
const insertedPageIds = insertablePages.map((page) => page.id);
|
||||
this.eventEmitter.emit(EventName.PAGE_CREATED, {
|
||||
pageIds: insertedPageIds,
|
||||
});
|
||||
|
||||
//TODO: best to handle this in a queue
|
||||
const attachmentsIds = Array.from(attachmentMap.keys());
|
||||
if (attachmentsIds.length > 0) {
|
||||
@ -606,6 +614,9 @@ export class PageService {
|
||||
|
||||
if (pageIds.length > 0) {
|
||||
await this.db.deleteFrom('pages').where('id', 'in', pageIds).execute();
|
||||
this.eventEmitter.emit(EventName.PAGE_DELETED, {
|
||||
pageIds: pageIds,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
import { Space } from '@docmost/db/types/entity.types';
|
||||
|
||||
export class SearchResponseDto {
|
||||
id: string;
|
||||
title: string;
|
||||
@ -8,4 +10,5 @@ export class SearchResponseDto {
|
||||
highlight: string;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
space: Partial<Space>;
|
||||
}
|
||||
|
||||
@ -5,6 +5,7 @@ import {
|
||||
ForbiddenException,
|
||||
HttpCode,
|
||||
HttpStatus,
|
||||
Logger,
|
||||
Post,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
@ -24,13 +25,19 @@ import {
|
||||
} from '../casl/interfaces/space-ability.type';
|
||||
import { AuthUser } from '../../common/decorators/auth-user.decorator';
|
||||
import { Public } from 'src/common/decorators/public.decorator';
|
||||
import { EnvironmentService } from '../../integrations/environment/environment.service';
|
||||
import { ModuleRef } from '@nestjs/core';
|
||||
|
||||
@UseGuards(JwtAuthGuard)
|
||||
@Controller('search')
|
||||
export class SearchController {
|
||||
private readonly logger = new Logger(SearchController.name);
|
||||
|
||||
constructor(
|
||||
private readonly searchService: SearchService,
|
||||
private readonly spaceAbility: SpaceAbilityFactory,
|
||||
private readonly environmentService: EnvironmentService,
|
||||
private moduleRef: ModuleRef,
|
||||
) {}
|
||||
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@ -53,7 +60,14 @@ export class SearchController {
|
||||
}
|
||||
}
|
||||
|
||||
return this.searchService.searchPage(searchDto.query, searchDto, {
|
||||
if (this.environmentService.getSearchDriver() === 'typesense') {
|
||||
return this.searchTypesense(searchDto, {
|
||||
userId: user.id,
|
||||
workspaceId: workspace.id,
|
||||
});
|
||||
}
|
||||
|
||||
return this.searchService.searchPage(searchDto, {
|
||||
userId: user.id,
|
||||
workspaceId: workspace.id,
|
||||
});
|
||||
@ -81,8 +95,47 @@ export class SearchController {
|
||||
throw new BadRequestException('shareId is required');
|
||||
}
|
||||
|
||||
return this.searchService.searchPage(searchDto.query, searchDto, {
|
||||
if (this.environmentService.getSearchDriver() === 'typesense') {
|
||||
return this.searchTypesense(searchDto, {
|
||||
workspaceId: workspace.id,
|
||||
});
|
||||
}
|
||||
|
||||
return this.searchService.searchPage(searchDto, {
|
||||
workspaceId: workspace.id,
|
||||
});
|
||||
}
|
||||
|
||||
async searchTypesense(
|
||||
searchParams: SearchDTO,
|
||||
opts: {
|
||||
userId?: string;
|
||||
workspaceId: string;
|
||||
},
|
||||
) {
|
||||
const { userId, workspaceId } = opts;
|
||||
let TypesenseModule: any;
|
||||
try {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
TypesenseModule = require('./../../ee/typesense/services/page-search.service');
|
||||
|
||||
const PageSearchService = this.moduleRef.get(
|
||||
TypesenseModule.PageSearchService,
|
||||
{
|
||||
strict: false,
|
||||
},
|
||||
);
|
||||
|
||||
return PageSearchService.searchPage(searchParams, {
|
||||
userId: userId,
|
||||
workspaceId,
|
||||
});
|
||||
} catch (err) {
|
||||
this.logger.debug(
|
||||
'Typesense module requested but enterprise module not bundled in this build',
|
||||
);
|
||||
}
|
||||
|
||||
throw new BadRequestException('Enterprise Typesense search module missing');
|
||||
}
|
||||
}
|
||||
|
||||
@ -21,13 +21,14 @@ export class SearchService {
|
||||
) {}
|
||||
|
||||
async searchPage(
|
||||
query: string,
|
||||
searchParams: SearchDTO,
|
||||
opts: {
|
||||
userId?: string;
|
||||
workspaceId: string;
|
||||
},
|
||||
): Promise<SearchResponseDto[]> {
|
||||
const { query } = searchParams;
|
||||
|
||||
if (query.length < 1) {
|
||||
return;
|
||||
}
|
||||
|
||||
@ -25,6 +25,7 @@ import { MigrationService } from '@docmost/db/services/migration.service';
|
||||
import { UserTokenRepo } from './repos/user-token/user-token.repo';
|
||||
import { BacklinkRepo } from '@docmost/db/repos/backlink/backlink.repo';
|
||||
import { ShareRepo } from '@docmost/db/repos/share/share.repo';
|
||||
import { PageListener } from '@docmost/db/listeners/page.listener';
|
||||
|
||||
// https://github.com/brianc/node-postgres/issues/811
|
||||
types.setTypeParser(types.builtins.INT8, (val) => Number(val));
|
||||
@ -75,7 +76,8 @@ types.setTypeParser(types.builtins.INT8, (val) => Number(val));
|
||||
AttachmentRepo,
|
||||
UserTokenRepo,
|
||||
BacklinkRepo,
|
||||
ShareRepo
|
||||
ShareRepo,
|
||||
PageListener,
|
||||
],
|
||||
exports: [
|
||||
WorkspaceRepo,
|
||||
@ -90,7 +92,7 @@ types.setTypeParser(types.builtins.INT8, (val) => Number(val));
|
||||
AttachmentRepo,
|
||||
UserTokenRepo,
|
||||
BacklinkRepo,
|
||||
ShareRepo
|
||||
ShareRepo,
|
||||
],
|
||||
})
|
||||
export class DatabaseModule
|
||||
|
||||
49
apps/server/src/database/listeners/page.listener.ts
Normal file
49
apps/server/src/database/listeners/page.listener.ts
Normal file
@ -0,0 +1,49 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { OnEvent } from '@nestjs/event-emitter';
|
||||
import { EventName } from '../../common/events/event.contants';
|
||||
import { InjectQueue } from '@nestjs/bullmq';
|
||||
import { QueueJob, QueueName } from '../../integrations/queue/constants';
|
||||
import { Queue } from 'bullmq';
|
||||
|
||||
export class PageEvent {
|
||||
pageIds: string[];
|
||||
}
|
||||
|
||||
@Injectable()
|
||||
export class PageListener {
|
||||
private readonly logger = new Logger(PageListener.name);
|
||||
|
||||
constructor(
|
||||
@InjectQueue(QueueName.SEARCH_QUEUE) private searchQueue: Queue,
|
||||
) {}
|
||||
|
||||
@OnEvent(EventName.PAGE_CREATED)
|
||||
async handlePageCreated(event: PageEvent) {
|
||||
const { pageIds } = event;
|
||||
await this.searchQueue.add(QueueJob.PAGE_CREATED, { pageIds });
|
||||
}
|
||||
|
||||
@OnEvent(EventName.PAGE_UPDATED)
|
||||
async handlePageUpdated(event: PageEvent) {
|
||||
const { pageIds } = event;
|
||||
await this.searchQueue.add(QueueJob.PAGE_UPDATED, { pageIds });
|
||||
}
|
||||
|
||||
@OnEvent(EventName.PAGE_DELETED)
|
||||
async handlePageDeleted(event: PageEvent) {
|
||||
const { pageIds } = event;
|
||||
await this.searchQueue.add(QueueJob.PAGE_DELETED, { pageIds });
|
||||
}
|
||||
|
||||
@OnEvent(EventName.PAGE_SOFT_DELETED)
|
||||
async handlePageSoftDeleted(event: PageEvent) {
|
||||
const { pageIds } = event;
|
||||
await this.searchQueue.add(QueueJob.PAGE_SOFT_DELETED, { pageIds });
|
||||
}
|
||||
|
||||
@OnEvent(EventName.PAGE_RESTORED)
|
||||
async handlePageRestored(event: PageEvent) {
|
||||
const { pageIds } = event;
|
||||
await this.searchQueue.add(QueueJob.PAGE_RESTORED, { pageIds });
|
||||
}
|
||||
}
|
||||
@ -14,32 +14,17 @@ import { ExpressionBuilder, sql } from 'kysely';
|
||||
import { DB } from '@docmost/db/types/db';
|
||||
import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres';
|
||||
import { SpaceMemberRepo } from '@docmost/db/repos/space/space-member.repo';
|
||||
import { EventEmitter2 } from '@nestjs/event-emitter';
|
||||
import { EventName } from '../../../common/events/event.contants';
|
||||
|
||||
@Injectable()
|
||||
export class PageRepo {
|
||||
constructor(
|
||||
@InjectKysely() private readonly db: KyselyDB,
|
||||
private spaceMemberRepo: SpaceMemberRepo,
|
||||
private eventEmitter: EventEmitter2,
|
||||
) {}
|
||||
|
||||
withHasChildren(eb: ExpressionBuilder<DB, 'pages'>) {
|
||||
return eb
|
||||
.selectFrom('pages as child')
|
||||
.select((eb) =>
|
||||
eb
|
||||
.case()
|
||||
.when(eb.fn.countAll(), '>', 0)
|
||||
.then(true)
|
||||
.else(false)
|
||||
.end()
|
||||
.as('count'),
|
||||
)
|
||||
.whereRef('child.parentPageId', '=', 'pages.id')
|
||||
.where('child.deletedAt', 'is', null)
|
||||
.limit(1)
|
||||
.as('hasChildren');
|
||||
}
|
||||
|
||||
private baseFields: Array<keyof Page> = [
|
||||
'id',
|
||||
'slugId',
|
||||
@ -63,6 +48,7 @@ export class PageRepo {
|
||||
pageId: string,
|
||||
opts?: {
|
||||
includeContent?: boolean;
|
||||
includeTextContent?: boolean;
|
||||
includeYdoc?: boolean;
|
||||
includeSpace?: boolean;
|
||||
includeCreator?: boolean;
|
||||
@ -80,6 +66,7 @@ export class PageRepo {
|
||||
.select(this.baseFields)
|
||||
.$if(opts?.includeContent, (qb) => qb.select('content'))
|
||||
.$if(opts?.includeYdoc, (qb) => qb.select('ydoc'))
|
||||
.$if(opts?.includeTextContent, (qb) => qb.select('textContent'))
|
||||
.$if(opts?.includeHasChildren, (qb) =>
|
||||
qb.select((eb) => this.withHasChildren(eb)),
|
||||
);
|
||||
@ -126,7 +113,7 @@ export class PageRepo {
|
||||
pageIds: string[],
|
||||
trx?: KyselyTransaction,
|
||||
) {
|
||||
return dbOrTx(this.db, trx)
|
||||
const result = await dbOrTx(this.db, trx)
|
||||
.updateTable('pages')
|
||||
.set({ ...updatePageData, updatedAt: new Date() })
|
||||
.where(
|
||||
@ -135,6 +122,12 @@ export class PageRepo {
|
||||
pageIds,
|
||||
)
|
||||
.executeTakeFirst();
|
||||
|
||||
this.eventEmitter.emit(EventName.PAGE_UPDATED, {
|
||||
pageIds: pageIds,
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async insertPage(
|
||||
@ -142,11 +135,17 @@ export class PageRepo {
|
||||
trx?: KyselyTransaction,
|
||||
): Promise<Page> {
|
||||
const db = dbOrTx(this.db, trx);
|
||||
return db
|
||||
const result = await db
|
||||
.insertInto('pages')
|
||||
.values(insertablePage)
|
||||
.returning(this.baseFields)
|
||||
.executeTakeFirst();
|
||||
|
||||
this.eventEmitter.emit(EventName.PAGE_CREATED, {
|
||||
pageIds: [result.id],
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
async deletePage(pageId: string): Promise<void> {
|
||||
@ -196,6 +195,9 @@ export class PageRepo {
|
||||
|
||||
await trx.deleteFrom('shares').where('pageId', 'in', pageIds).execute();
|
||||
});
|
||||
this.eventEmitter.emit(EventName.PAGE_SOFT_DELETED, {
|
||||
pageIds: pageIds,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@ -259,6 +261,9 @@ export class PageRepo {
|
||||
.where('id', '=', pageId)
|
||||
.execute();
|
||||
}
|
||||
this.eventEmitter.emit(EventName.PAGE_RESTORED, {
|
||||
pageIds: pageIds,
|
||||
});
|
||||
}
|
||||
|
||||
async getRecentPagesInSpace(spaceId: string, pagination: PaginationOptions) {
|
||||
@ -379,6 +384,24 @@ export class PageRepo {
|
||||
).as('contributors');
|
||||
}
|
||||
|
||||
withHasChildren(eb: ExpressionBuilder<DB, 'pages'>) {
|
||||
return eb
|
||||
.selectFrom('pages as child')
|
||||
.select((eb) =>
|
||||
eb
|
||||
.case()
|
||||
.when(eb.fn.countAll(), '>', 0)
|
||||
.then(true)
|
||||
.else(false)
|
||||
.end()
|
||||
.as('count'),
|
||||
)
|
||||
.whereRef('child.parentPageId', '=', 'pages.id')
|
||||
.where('child.deletedAt', 'is', null)
|
||||
.limit(1)
|
||||
.as('hasChildren');
|
||||
}
|
||||
|
||||
async getPageAndDescendants(
|
||||
parentPageId: string,
|
||||
opts: { includeContent: boolean },
|
||||
|
||||
Submodule apps/server/src/ee updated: d2ead43181...a4a19f71e1
@ -213,4 +213,24 @@ export class EnvironmentService {
|
||||
getPostHogKey(): string {
|
||||
return this.configService.get<string>('POSTHOG_KEY');
|
||||
}
|
||||
|
||||
getSearchDriver(): string {
|
||||
return this.configService
|
||||
.get<string>('SEARCH_DRIVER', 'database')
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
getTypesenseUrl(): string {
|
||||
return this.configService
|
||||
.get<string>('TYPESENSE_URL', 'http://localhost:8108')
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
getTypesenseApiKey(): string {
|
||||
return this.configService.get<string>('TYPESENSE_API_KEY');
|
||||
}
|
||||
|
||||
getTypesenseLocale(): string {
|
||||
return this.configService.get<string>('TYPESENSE_LOCALE', 'en').toLowerCase();
|
||||
}
|
||||
}
|
||||
|
||||
@ -3,12 +3,14 @@ import {
|
||||
IsNotEmpty,
|
||||
IsNotIn,
|
||||
IsOptional,
|
||||
IsString,
|
||||
IsUrl,
|
||||
MinLength,
|
||||
ValidateIf,
|
||||
validateSync,
|
||||
} from 'class-validator';
|
||||
import { plainToInstance } from 'class-transformer';
|
||||
import { IsISO6391 } from '../../common/validator/is-iso6391';
|
||||
|
||||
export class EnvironmentVariables {
|
||||
@IsNotEmpty()
|
||||
@ -68,6 +70,37 @@ export class EnvironmentVariables {
|
||||
)
|
||||
@ValidateIf((obj) => obj.CLOUD === 'true'.toLowerCase())
|
||||
SUBDOMAIN_HOST: string;
|
||||
|
||||
@IsOptional()
|
||||
@IsIn(['database', 'typesense'])
|
||||
@IsString()
|
||||
SEARCH_DRIVER: string;
|
||||
|
||||
@IsOptional()
|
||||
@IsUrl(
|
||||
{
|
||||
protocols: ['http', 'https'],
|
||||
require_tld: false,
|
||||
allow_underscores: true,
|
||||
},
|
||||
{
|
||||
message:
|
||||
'TYPESENSE_URL must be a valid typesense url e.g http://localhost:8108',
|
||||
},
|
||||
)
|
||||
@ValidateIf((obj) => obj.SEARCH_DRIVER === 'typesense')
|
||||
TYPESENSE_URL: string;
|
||||
|
||||
@IsOptional()
|
||||
@ValidateIf((obj) => obj.SEARCH_DRIVER === 'typesense')
|
||||
@IsString()
|
||||
TYPESENSE_API_KEY: string;
|
||||
|
||||
@IsOptional()
|
||||
@ValidateIf((obj) => obj.SEARCH_DRIVER === 'typesense')
|
||||
@IsISO6391()
|
||||
@IsString()
|
||||
TYPESENSE_LOCALE: string;
|
||||
}
|
||||
|
||||
export function validate(config: Record<string, any>) {
|
||||
|
||||
@ -32,6 +32,8 @@ import { ImportAttachmentService } from './import-attachment.service';
|
||||
import { ModuleRef } from '@nestjs/core';
|
||||
import { PageService } from '../../../core/page/services/page.service';
|
||||
import { ImportPageNode } from '../dto/file-task-dto';
|
||||
import { EventEmitter2 } from '@nestjs/event-emitter';
|
||||
import { EventName } from '../../../common/events/event.contants';
|
||||
|
||||
@Injectable()
|
||||
export class FileImportTaskService {
|
||||
@ -45,6 +47,7 @@ export class FileImportTaskService {
|
||||
@InjectKysely() private readonly db: KyselyDB,
|
||||
private readonly importAttachmentService: ImportAttachmentService,
|
||||
private moduleRef: ModuleRef,
|
||||
private eventEmitter: EventEmitter2,
|
||||
) {}
|
||||
|
||||
async processZIpImport(fileTaskId: string): Promise<void> {
|
||||
@ -396,6 +399,12 @@ export class FileImportTaskService {
|
||||
}
|
||||
}
|
||||
|
||||
if (validPageIds.size > 0) {
|
||||
this.eventEmitter.emit(EventName.PAGE_CREATED, {
|
||||
pageIds: Array.from(validPageIds),
|
||||
});
|
||||
}
|
||||
|
||||
this.logger.log(
|
||||
`Successfully imported ${totalPagesProcessed} pages with ${filteredBacklinks.length} backlinks`,
|
||||
);
|
||||
|
||||
@ -4,6 +4,7 @@ export enum QueueName {
|
||||
GENERAL_QUEUE = '{general-queue}',
|
||||
BILLING_QUEUE = '{billing-queue}',
|
||||
FILE_TASK_QUEUE = '{file-task-queue}',
|
||||
SEARCH_QUEUE = '{search-queue}',
|
||||
}
|
||||
|
||||
export enum QueueJob {
|
||||
@ -25,4 +26,21 @@ export enum QueueJob {
|
||||
|
||||
IMPORT_TASK = 'import-task',
|
||||
EXPORT_TASK = 'export-task',
|
||||
|
||||
SEARCH_INDEX_PAGE = 'search-index-page',
|
||||
SEARCH_INDEX_PAGES = 'search-index-pages',
|
||||
SEARCH_INDEX_COMMENT = 'search-index-comment',
|
||||
SEARCH_INDEX_COMMENTS = 'search-index-comments',
|
||||
SEARCH_INDEX_ATTACHMENT = 'search-index-attachment',
|
||||
SEARCH_INDEX_ATTACHMENTS = 'search-index-attachments',
|
||||
SEARCH_REMOVE_PAGE = 'search-remove-page',
|
||||
SEARCH_REMOVE_ASSET = 'search-remove-attachment',
|
||||
SEARCH_REMOVE_FACE = 'search-remove-comment',
|
||||
TYPESENSE_FLUSH = 'typesense-flush',
|
||||
|
||||
PAGE_CREATED = 'page-created',
|
||||
PAGE_UPDATED = 'page-updated',
|
||||
PAGE_SOFT_DELETED = 'page-soft-deleted',
|
||||
PAGE_RESTORED = 'page-restored',
|
||||
PAGE_DELETED = 'page-deleted',
|
||||
}
|
||||
|
||||
@ -57,6 +57,14 @@ import { BacklinksProcessor } from './processors/backlinks.processor';
|
||||
attempts: 1,
|
||||
},
|
||||
}),
|
||||
BullModule.registerQueue({
|
||||
name: QueueName.SEARCH_QUEUE,
|
||||
defaultJobOptions: {
|
||||
removeOnComplete: true,
|
||||
removeOnFail: true,
|
||||
attempts: 2,
|
||||
},
|
||||
}),
|
||||
],
|
||||
exports: [BullModule],
|
||||
providers: [BacklinksProcessor],
|
||||
|
||||
26
apps/server/src/integrations/redis/redis-config.service.ts
Normal file
26
apps/server/src/integrations/redis/redis-config.service.ts
Normal file
@ -0,0 +1,26 @@
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import {
|
||||
RedisModuleOptions,
|
||||
RedisOptionsFactory,
|
||||
} from '@nestjs-labs/nestjs-ioredis';
|
||||
import { createRetryStrategy, parseRedisUrl } from '../../common/helpers';
|
||||
import { EnvironmentService } from '../environment/environment.service';
|
||||
|
||||
@Injectable()
|
||||
export class RedisConfigService implements RedisOptionsFactory {
|
||||
constructor(private readonly environmentService: EnvironmentService) {}
|
||||
createRedisOptions(): RedisModuleOptions {
|
||||
const redisConfig = parseRedisUrl(this.environmentService.getRedisUrl());
|
||||
return {
|
||||
readyLog: true,
|
||||
config: {
|
||||
host: redisConfig.host,
|
||||
port: redisConfig.port,
|
||||
password: redisConfig.password,
|
||||
db: redisConfig.db,
|
||||
family: redisConfig.family,
|
||||
retryStrategy: createRetryStrategy(),
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
98
pnpm-lock.yaml
generated
98
pnpm-lock.yaml
generated
@ -447,9 +447,12 @@ importers:
|
||||
'@fastify/static':
|
||||
specifier: ^8.2.0
|
||||
version: 8.2.0
|
||||
'@nestjs-labs/nestjs-ioredis':
|
||||
specifier: ^11.0.4
|
||||
version: 11.0.4(@nestjs/common@11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.3)(ioredis@5.4.1)
|
||||
'@nestjs/bullmq':
|
||||
specifier: ^11.0.2
|
||||
version: 11.0.2(@nestjs/common@11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.3)(bullmq@5.53.2)
|
||||
version: 11.0.2(@nestjs/common@11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.3)(bullmq@5.61.0)
|
||||
'@nestjs/common':
|
||||
specifier: ^11.1.3
|
||||
version: 11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2)
|
||||
@ -502,8 +505,8 @@ importers:
|
||||
specifier: ^5.1.1
|
||||
version: 5.1.1
|
||||
bullmq:
|
||||
specifier: ^5.53.2
|
||||
version: 5.53.2
|
||||
specifier: ^5.61.0
|
||||
version: 5.61.0
|
||||
cache-manager:
|
||||
specifier: ^6.4.3
|
||||
version: 6.4.3
|
||||
@ -523,8 +526,11 @@ importers:
|
||||
specifier: ^11.3.0
|
||||
version: 11.3.0
|
||||
happy-dom:
|
||||
specifier: ^15.11.6
|
||||
version: 15.11.7
|
||||
specifier: ^18.0.1
|
||||
version: 18.0.1
|
||||
ioredis:
|
||||
specifier: ^5.4.1
|
||||
version: 5.4.1
|
||||
jsonwebtoken:
|
||||
specifier: ^9.0.2
|
||||
version: 9.0.2
|
||||
@ -603,6 +609,9 @@ importers:
|
||||
tmp-promise:
|
||||
specifier: ^3.0.3
|
||||
version: 3.0.3
|
||||
typesense:
|
||||
specifier: ^2.1.0
|
||||
version: 2.1.0(@babel/runtime@7.25.6)
|
||||
ws:
|
||||
specifier: ^8.18.2
|
||||
version: 8.18.2
|
||||
@ -2820,6 +2829,14 @@ packages:
|
||||
'@napi-rs/wasm-runtime@0.2.4':
|
||||
resolution: {integrity: sha512-9zESzOO5aDByvhIAsOy9TbpZ0Ur2AJbUI7UT73kcUTS2mxAMHOBaa1st/jAymNoCtvrit99kkzT1FZuXVcgfIQ==}
|
||||
|
||||
'@nestjs-labs/nestjs-ioredis@11.0.4':
|
||||
resolution: {integrity: sha512-4jPNOrxDiwNMIN5OLmsMWhA782kxv/ZBxkySX9l8n6sr55acHX/BciaFsOXVa/ILsm+Y7893y98/6WNhmEoiNQ==}
|
||||
engines: {node: '>=16'}
|
||||
peerDependencies:
|
||||
'@nestjs/common': ^10.0.0 || ^11.0.0
|
||||
'@nestjs/core': ^10.0.0 || ^11.0.0
|
||||
ioredis: ^5.0.0
|
||||
|
||||
'@nestjs/bull-shared@11.0.2':
|
||||
resolution: {integrity: sha512-dFlttJvBqIFD6M8JVFbkrR4Feb39OTAJPJpFVILU50NOJCM4qziRw3dSNG84Q3v+7/M6xUGMFdZRRGvBBKxoSA==}
|
||||
peerDependencies:
|
||||
@ -4547,6 +4564,9 @@ packages:
|
||||
'@types/ms@2.1.0':
|
||||
resolution: {integrity: sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==}
|
||||
|
||||
'@types/node@20.19.19':
|
||||
resolution: {integrity: sha512-pb1Uqj5WJP7wrcbLU7Ru4QtA0+3kAXrkutGiD26wUKzSMgNNaPARTUDQmElUXp64kh3cWdou3Q0C7qwwxqSFmg==}
|
||||
|
||||
'@types/node@22.10.0':
|
||||
resolution: {integrity: sha512-XC70cRZVElFHfIUB40FgZOBbgJYFKKMa5nb9lxcwYstFG/Mi+/Y0bGS+rs6Dmhmkpq4pnNiLiuZAbc02YCOnmA==}
|
||||
|
||||
@ -4637,6 +4657,9 @@ packages:
|
||||
'@types/validator@13.12.0':
|
||||
resolution: {integrity: sha512-nH45Lk7oPIJ1RVOF6JgFI6Dy0QpHEzq4QecZhvguxYPDwT8c93prCMqAtiIttm39voZ+DDR+qkNnMpJmMBRqag==}
|
||||
|
||||
'@types/whatwg-mimetype@3.0.2':
|
||||
resolution: {integrity: sha512-c2AKvDT8ToxLIOUlN51gTiHXflsfIFisS4pO7pDPoKouJCESkhZnEy623gwP9laCy5lnLDAw1vAzu2vM2YLOrA==}
|
||||
|
||||
'@types/ws@8.5.14':
|
||||
resolution: {integrity: sha512-bd/YFLW+URhBzMXurx7lWByOu+xzU9+kb3RboOteXYDfW+tr+JZa99OyNmPINEGB/ahzKrEuc8rcv4gnpJmxTw==}
|
||||
|
||||
@ -5217,8 +5240,8 @@ packages:
|
||||
builtins@5.0.1:
|
||||
resolution: {integrity: sha512-qwVpFEHNfhYJIzNRBvd2C1kyo6jz3ZSMPyyuR47OPdiKWlbYnZNyDWuyR175qDnAJLiCo5fBBqPb3RiXgWlkOQ==}
|
||||
|
||||
bullmq@5.53.2:
|
||||
resolution: {integrity: sha512-xHgxrP/yNJHD7VCw1h+eRBh+2TCPBCM39uC9gCyksYc6ufcJP+HTZ/A2lzB2x7qMFWrvsX7tM40AT2BmdkYL/Q==}
|
||||
bullmq@5.61.0:
|
||||
resolution: {integrity: sha512-khaTjc1JnzaYFl4FrUtsSsqugAW/urRrcZ9Q0ZE+REAw8W+gkHFqxbGlutOu6q7j7n91wibVaaNlOUMdiEvoSQ==}
|
||||
|
||||
busboy@1.6.0:
|
||||
resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==}
|
||||
@ -6537,9 +6560,9 @@ packages:
|
||||
hachure-fill@0.5.2:
|
||||
resolution: {integrity: sha512-3GKBOn+m2LX9iq+JC1064cSFprJY4jL1jCXTcpnfER5HYE2l/4EfWSGzkPa/ZDBmYI0ZOEj5VHV/eKnPGkHuOg==}
|
||||
|
||||
happy-dom@15.11.7:
|
||||
resolution: {integrity: sha512-KyrFvnl+J9US63TEzwoiJOQzZBJY7KgBushJA8X61DMbNsH+2ONkDuLDnCnwUiPTF42tLoEmrPyoqbenVA5zrg==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
happy-dom@18.0.1:
|
||||
resolution: {integrity: sha512-qn+rKOW7KWpVTtgIUi6RVmTBZJSe2k0Db0vh1f7CWrWclkkc7/Q+FrOfkZIb2eiErLyqu5AXEzE7XthO9JVxRA==}
|
||||
engines: {node: '>=20.0.0'}
|
||||
|
||||
has-bigints@1.0.2:
|
||||
resolution: {integrity: sha512-tSvCKtBr9lkF0Ex0aQiP9N+OpV4zi2r/Nee5VkRDbaqv35RLYMzbwQfFSZZH0kR+Rd6302UJZ2p/bJCEoR3VoQ==}
|
||||
@ -7442,10 +7465,6 @@ packages:
|
||||
resolution: {integrity: sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==}
|
||||
engines: {node: '>=10'}
|
||||
|
||||
luxon@3.5.0:
|
||||
resolution: {integrity: sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==}
|
||||
engines: {node: '>=12'}
|
||||
|
||||
luxon@3.6.1:
|
||||
resolution: {integrity: sha512-tJLxrKJhO2ukZ5z0gyjY1zPh3Rh88Ej9P7jNrZiHMUXHae1yvI2imgOZtL1TO8TW6biMMKfTtAOoEJANgtWBMQ==}
|
||||
engines: {node: '>=12'}
|
||||
@ -9457,6 +9476,12 @@ packages:
|
||||
engines: {node: '>=14.17'}
|
||||
hasBin: true
|
||||
|
||||
typesense@2.1.0:
|
||||
resolution: {integrity: sha512-a/IRTL+dRXlpRDU4UodyGj8hl5xBz3nKihVRd/KfSFAfFPGcpdX6lxIgwdXy3O6VLNNiEsN8YwIsPHQPVT0vNw==}
|
||||
engines: {node: '>=18'}
|
||||
peerDependencies:
|
||||
'@babel/runtime': ^7.23.2
|
||||
|
||||
uc.micro@2.1.0:
|
||||
resolution: {integrity: sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==}
|
||||
|
||||
@ -9487,6 +9512,9 @@ packages:
|
||||
undici-types@6.20.0:
|
||||
resolution: {integrity: sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==}
|
||||
|
||||
undici-types@6.21.0:
|
||||
resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==}
|
||||
|
||||
undici@7.10.0:
|
||||
resolution: {integrity: sha512-u5otvFBOBZvmdjWLVW+5DAc9Nkq8f24g0O9oY7qw2JVIF1VocIFoyz9JFkuVOS2j41AufeO0xnlweJ2RLT8nGw==}
|
||||
engines: {node: '>=20.18.1'}
|
||||
@ -12870,18 +12898,25 @@ snapshots:
|
||||
'@emnapi/runtime': 1.2.0
|
||||
'@tybys/wasm-util': 0.9.0
|
||||
|
||||
'@nestjs-labs/nestjs-ioredis@11.0.4(@nestjs/common@11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.3)(ioredis@5.4.1)':
|
||||
dependencies:
|
||||
'@nestjs/common': 11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2)
|
||||
'@nestjs/core': 11.1.3(@nestjs/common@11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/websockets@11.1.3)(reflect-metadata@0.2.2)(rxjs@7.8.2)
|
||||
ioredis: 5.4.1
|
||||
tslib: 2.8.1
|
||||
|
||||
'@nestjs/bull-shared@11.0.2(@nestjs/common@11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.3)':
|
||||
dependencies:
|
||||
'@nestjs/common': 11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2)
|
||||
'@nestjs/core': 11.1.3(@nestjs/common@11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/websockets@11.1.3)(reflect-metadata@0.2.2)(rxjs@7.8.2)
|
||||
tslib: 2.8.1
|
||||
|
||||
'@nestjs/bullmq@11.0.2(@nestjs/common@11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.3)(bullmq@5.53.2)':
|
||||
'@nestjs/bullmq@11.0.2(@nestjs/common@11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.3)(bullmq@5.61.0)':
|
||||
dependencies:
|
||||
'@nestjs/bull-shared': 11.0.2(@nestjs/common@11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.3)
|
||||
'@nestjs/common': 11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2)
|
||||
'@nestjs/core': 11.1.3(@nestjs/common@11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/websockets@11.1.3)(reflect-metadata@0.2.2)(rxjs@7.8.2)
|
||||
bullmq: 5.53.2
|
||||
bullmq: 5.61.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@nestjs/cli@11.0.4(@swc/core@1.5.25(@swc/helpers@0.5.5))(@types/node@22.13.4)':
|
||||
@ -14666,6 +14701,10 @@ snapshots:
|
||||
|
||||
'@types/ms@2.1.0': {}
|
||||
|
||||
'@types/node@20.19.19':
|
||||
dependencies:
|
||||
undici-types: 6.21.0
|
||||
|
||||
'@types/node@22.10.0':
|
||||
dependencies:
|
||||
undici-types: 6.20.0
|
||||
@ -14778,6 +14817,8 @@ snapshots:
|
||||
|
||||
'@types/validator@13.12.0': {}
|
||||
|
||||
'@types/whatwg-mimetype@3.0.2': {}
|
||||
|
||||
'@types/ws@8.5.14':
|
||||
dependencies:
|
||||
'@types/node': 22.13.4
|
||||
@ -15541,7 +15582,7 @@ snapshots:
|
||||
dependencies:
|
||||
semver: 7.7.2
|
||||
|
||||
bullmq@5.53.2:
|
||||
bullmq@5.61.0:
|
||||
dependencies:
|
||||
cron-parser: 4.9.0
|
||||
ioredis: 5.4.1
|
||||
@ -15549,7 +15590,7 @@ snapshots:
|
||||
node-abort-controller: 3.1.1
|
||||
semver: 7.7.2
|
||||
tslib: 2.8.1
|
||||
uuid: 9.0.1
|
||||
uuid: 11.1.0
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
@ -15873,7 +15914,7 @@ snapshots:
|
||||
|
||||
cron-parser@4.9.0:
|
||||
dependencies:
|
||||
luxon: 3.5.0
|
||||
luxon: 3.6.1
|
||||
|
||||
cron@4.3.0:
|
||||
dependencies:
|
||||
@ -17085,10 +17126,10 @@ snapshots:
|
||||
|
||||
hachure-fill@0.5.2: {}
|
||||
|
||||
happy-dom@15.11.7:
|
||||
happy-dom@18.0.1:
|
||||
dependencies:
|
||||
entities: 4.5.0
|
||||
webidl-conversions: 7.0.0
|
||||
'@types/node': 20.19.19
|
||||
'@types/whatwg-mimetype': 3.0.2
|
||||
whatwg-mimetype: 3.0.0
|
||||
|
||||
has-bigints@1.0.2: {}
|
||||
@ -18176,8 +18217,6 @@ snapshots:
|
||||
dependencies:
|
||||
yallist: 4.0.0
|
||||
|
||||
luxon@3.5.0: {}
|
||||
|
||||
luxon@3.6.1: {}
|
||||
|
||||
magic-string@0.30.17:
|
||||
@ -20495,6 +20534,15 @@ snapshots:
|
||||
|
||||
typescript@5.7.3: {}
|
||||
|
||||
typesense@2.1.0(@babel/runtime@7.25.6):
|
||||
dependencies:
|
||||
'@babel/runtime': 7.25.6
|
||||
axios: 1.9.0
|
||||
loglevel: 1.9.1
|
||||
tslib: 2.8.1
|
||||
transitivePeerDependencies:
|
||||
- debug
|
||||
|
||||
uc.micro@2.1.0: {}
|
||||
|
||||
ufo@1.6.1: {}
|
||||
@ -20520,6 +20568,8 @@ snapshots:
|
||||
|
||||
undici-types@6.20.0: {}
|
||||
|
||||
undici-types@6.21.0: {}
|
||||
|
||||
undici@7.10.0: {}
|
||||
|
||||
unicode-canonical-property-names-ecmascript@2.0.0: {}
|
||||
|
||||
Reference in New Issue
Block a user