feat: search

apps/server/package.json

@@ -48,6 +48,7 @@
     "fs-extra": "^11.1.1",
     "mime-types": "^2.1.35",
     "pg": "^8.11.3",
+    "pg-tsquery": "^8.4.1",
     "reflect-metadata": "^0.1.13",
     "rxjs": "^7.8.1",
     "sanitize-filename-ts": "^1.0.2",

apps/server/src/collaboration/collaboration.util.ts

@@ -0,0 +1,44 @@
+import { StarterKit } from '@tiptap/starter-kit';
+import { TextAlign } from '@tiptap/extension-text-align';
+import { TaskList } from '@tiptap/extension-task-list';
+import { TaskItem } from '@tiptap/extension-task-item';
+import { Underline } from '@tiptap/extension-underline';
+import { Link } from '@tiptap/extension-link';
+import { Superscript } from '@tiptap/extension-superscript';
+import SubScript from '@tiptap/extension-subscript';
+import { Highlight } from '@tiptap/extension-highlight';
+import { Typography } from '@tiptap/extension-typography';
+import { TextStyle } from '@tiptap/extension-text-style';
+import { Color } from '@tiptap/extension-color';
+import { TrailingNode, Comment } from '@docmost/editor-ext';
+import { generateHTML, generateJSON } from '@tiptap/html';
+import { generateText, JSONContent } from '@tiptap/core';
+
+export const tiptapExtensions = [
+  StarterKit,
+  Comment,
+  TextAlign,
+  TaskList,
+  TaskItem,
+  Underline,
+  Link,
+  Superscript,
+  SubScript,
+  Highlight,
+  Typography,
+  TrailingNode,
+  TextStyle,
+  Color,
+];
+
+export function jsonToHtml(tiptapJson: JSONContent) {
+  return generateHTML(tiptapJson, tiptapExtensions);
+}
+
+export function htmlToJson(html: string) {
+  return generateJSON(html, tiptapExtensions);
+}
+
+export function jsonToText(tiptapJson: JSONContent) {
+  return generateText(tiptapJson, tiptapExtensions);
+}

apps/server/src/collaboration/extensions/persistence.extension.ts

@@ -7,19 +7,8 @@ import * as Y from 'yjs';
 import { PageService } from '../../core/page/services/page.service';
 import { Injectable } from '@nestjs/common';
 import { TiptapTransformer } from '@hocuspocus/transformer';
-import { StarterKit } from '@tiptap/starter-kit';
-import { TextAlign } from '@tiptap/extension-text-align';
-import { TaskList } from '@tiptap/extension-task-list';
-import { TaskItem } from '@tiptap/extension-task-item';
-import { Underline } from '@tiptap/extension-underline';
-import { Link } from '@tiptap/extension-link';
-import { Superscript } from '@tiptap/extension-superscript';
-import SubScript from '@tiptap/extension-subscript';
-import { Highlight } from '@tiptap/extension-highlight';
-import { Typography } from '@tiptap/extension-typography';
-import { TextStyle } from '@tiptap/extension-text-style';
-import { Color } from '@tiptap/extension-color';
-import { TrailingNode, Comment } from '@docmost/editor-ext';
+import { jsonToHtml, jsonToText, tiptapExtensions } from '../collaboration.util';
+import { generateText } from '@tiptap/core'
 
 @Injectable()
 export class PersistenceExtension implements Extension {
@@ -55,22 +44,12 @@ export class PersistenceExtension implements Extension {
     if (page.content) {
       console.log('converting json to ydoc');
 
-      const ydoc = TiptapTransformer.toYdoc(page.content, 'default', [
-        StarterKit,
-        Comment,
-        TextAlign,
-        TaskList,
-        TaskItem,
-        Underline,
-        Link,
-        Superscript,
-        SubScript,
-        Highlight,
-        Typography,
-        TrailingNode,
-        TextStyle,
-        Color,
-      ]);
+      const ydoc = TiptapTransformer.toYdoc(
+        page.content,
+        'default',
+        tiptapExtensions,
+      );
+
       Y.encodeStateAsUpdate(ydoc);
       return ydoc;
     }
@@ -87,8 +66,16 @@ export class PersistenceExtension implements Extension {
     const tiptapJson = TiptapTransformer.fromYdoc(document, 'default');
     const ydocState = Buffer.from(Y.encodeStateAsUpdate(document));
 
+    const textContent = jsonToText(tiptapJson);
+    console.log(jsonToText(tiptapJson));
+
     try {
-      await this.pageService.updateState(pageId, tiptapJson, ydocState);
+      await this.pageService.updateState(
+        pageId,
+        tiptapJson,
+        textContent,
+        ydocState,
+      );
     } catch (err) {
       console.error(`Failed to update page ${documentName}`);
     }

apps/server/src/core/core.module.ts

@@ -7,6 +7,7 @@ import { StorageModule } from './storage/storage.module';
 import { AttachmentModule } from './attachment/attachment.module';
 import { EnvironmentModule } from '../environment/environment.module';
 import { CommentModule } from './comment/comment.module';
+import { SearchModule } from './search/search.module';
 
 @Module({
   imports: [
@@ -19,6 +20,7 @@ import { CommentModule } from './comment/comment.module';
     }),
     AttachmentModule,
     CommentModule,
+    SearchModule,
   ],
 })
 export class CoreModule {}

apps/server/src/core/page/entities/page.entity.ts

@@ -22,21 +22,34 @@ export class Page {
   @Column({ length: 500, nullable: true })
   title: string;
 
+  @Column({ nullable: true })
+  icon: string;
+
   @Column({ type: 'jsonb', nullable: true })
   content: string;
 
   @Column({ type: 'text', nullable: true })
   html: string;
 
+  @Column({ type: 'text', nullable: true })
+  textContent: string;
+
+  @Column({
+    type: 'tsvector',
+    generatedType: 'STORED',
+    asExpression:
+      "setweight(to_tsvector('english', coalesce(pages.title, '')), 'A') || setweight(to_tsvector('english', coalesce(pages.\"textContent\", '')), 'B')",
+    select: false,
+    nullable: true,
+  })
+  tsv: string;
+
   @Column({ type: 'bytea', nullable: true })
   ydoc: any;
 
   @Column({ nullable: true })
   slug: string;
 
-  @Column({ nullable: true })
-  icon: string;
-
   @Column({ nullable: true })
   coverPhoto: string;
 

apps/server/src/core/page/page.module.ts

@@ -26,6 +26,11 @@ import { PageHistoryRepository } from './repositories/page-history.repository';
     PageRepository,
     PageHistoryRepository,
   ],
-  exports: [PageService, PageOrderingService, PageHistoryService],
+  exports: [
+    PageService,
+    PageOrderingService,
+    PageHistoryService,
+    PageRepository,
+  ],
 })
 export class PageModule {}

apps/server/src/core/page/services/page.service.ts

@@ -108,11 +108,13 @@ export class PageService {
   async updateState(
     pageId: string,
     content: any,
+    textContent: string,
     ydoc: any,
     userId?: string, // TODO: fix this
   ): Promise<void> {
     await this.pageRepository.update(pageId, {
       content: content,
+      textContent: textContent,
       ydoc: ydoc,
       ...(userId && { lastUpdatedById: userId }),
     });

apps/server/src/core/search/dto/search-response.dto.ts

@@ -0,0 +1,9 @@
+export class SearchResponseDto {
+  id: string;
+  title: string;
+  icon: string;
+  parentPageId: string;
+  creatorId: string;
+  rank: string;
+  highlight: string;
+}

apps/server/src/core/search/dto/search.dto.ts

@@ -0,0 +1,18 @@
+import { IsNumber, IsOptional, IsString } from 'class-validator';
+
+export class SearchDTO {
+  @IsString()
+  query: string;
+
+  @IsOptional()
+  @IsString()
+  creatorId?: string;
+
+  @IsOptional()
+  @IsNumber()
+  limit?: number;
+
+  @IsOptional()
+  @IsNumber()
+  offset?: number;
+}

apps/server/src/core/search/search.controller.spec.ts

@@ -0,0 +1,18 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { SearchController } from './search.controller';
+
+describe('SearchController', () => {
+  let controller: SearchController;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      controllers: [SearchController],
+    }).compile();
+
+    controller = module.get<SearchController>(SearchController);
+  });
+
+  it('should be defined', () => {
+    expect(controller).toBeDefined();
+  });
+});

apps/server/src/core/search/search.controller.ts

@@ -0,0 +1,44 @@
+import {
+  Body,
+  Controller,
+  HttpCode,
+  HttpStatus,
+  Post,
+  Query,
+  UseGuards,
+} from '@nestjs/common';
+import { JwtUser } from '../../decorators/jwt-user.decorator';
+import { WorkspaceService } from '../workspace/services/workspace.service';
+import { JwtGuard } from '../auth/guards/JwtGuard';
+import { SearchService } from './search.service';
+import { SearchDTO } from './dto/search.dto';
+
+@UseGuards(JwtGuard)
+@Controller('search')
+export class SearchController {
+  constructor(
+    private readonly searchService: SearchService,
+    private readonly workspaceService: WorkspaceService,
+  ) {}
+
+  @HttpCode(HttpStatus.OK)
+  @Post()
+  async pageSearch(
+    @Query('type') type: string,
+    @Body() searchDto: SearchDTO,
+    @JwtUser() jwtUser,
+  ) {
+    const workspaceId = (
+      await this.workspaceService.getUserCurrentWorkspace(jwtUser.id)
+    ).id;
+
+    if (!type || type === 'page') {
+      return this.searchService.searchPage(
+        searchDto.query,
+        searchDto,
+        workspaceId,
+      );
+    }
+    return;
+  }
+}

apps/server/src/core/search/search.module.ts

@@ -0,0 +1,13 @@
+import { Module } from '@nestjs/common';
+import { SearchController } from './search.controller';
+import { SearchService } from './search.service';
+import { AuthModule } from '../auth/auth.module';
+import { WorkspaceModule } from '../workspace/workspace.module';
+import { PageModule } from '../page/page.module';
+
+@Module({
+  imports: [AuthModule, WorkspaceModule, PageModule],
+  controllers: [SearchController],
+  providers: [SearchService],
+})
+export class SearchModule {}

apps/server/src/core/search/search.service.spec.ts

@@ -0,0 +1,18 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { SearchService } from './search.service';
+
+describe('SearchService', () => {
+  let service: SearchService;
+
+  beforeEach(async () => {
+    const module: TestingModule = await Test.createTestingModule({
+      providers: [SearchService],
+    }).compile();
+
+    service = module.get<SearchService>(SearchService);
+  });
+
+  it('should be defined', () => {
+    expect(service).toBeDefined();
+  });
+});

apps/server/src/core/search/search.service.ts

@@ -0,0 +1,72 @@
+import { Injectable } from '@nestjs/common';
+import { PageRepository } from '../page/repositories/page.repository';
+import { SearchDTO } from './dto/search.dto';
+import { SearchResponseDto } from './dto/search-response.dto';
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const tsquery = require('pg-tsquery')();
+
+@Injectable()
+export class SearchService {
+  constructor(private pageRepository: PageRepository) {}
+
+  async searchPage(
+    query: string,
+    searchParams: SearchDTO,
+    workspaceId: string,
+  ): Promise<SearchResponseDto[]> {
+    if (query.length < 1) {
+      return;
+    }
+    const searchQuery = tsquery(query.trim() + '*');
+
+    const selectColumns = [
+      'page.id as id',
+      'page.title as title',
+      'page.icon as icon',
+      'page.parentPageId as "parentPageId"',
+      'page.creatorId as "creatorId"',
+      'page.createdAt as "createdAt"',
+      'page.updatedAt as "updatedAt"',
+    ];
+
+    const searchQueryBuilder = await this.pageRepository
+      .createQueryBuilder('page')
+      .select(selectColumns);
+
+    searchQueryBuilder.andWhere('page.workspaceId = :workspaceId', {
+      workspaceId,
+    });
+
+    searchQueryBuilder
+      .addSelect('ts_rank(page.tsv, to_tsquery(:searchQuery))', 'rank')
+      .addSelect(
+        `ts_headline('english', page.textContent, to_tsquery(:searchQuery), 'MinWords=9, MaxWords=10, MaxFragments=10')`,
+        'highlight',
+      )
+      .andWhere('page.tsv @@ to_tsquery(:searchQuery)', { searchQuery })
+      .orderBy('rank', 'DESC');
+
+    if (searchParams?.creatorId) {
+      searchQueryBuilder.andWhere('page.creatorId = :creatorId', {
+        creatorId: searchParams.creatorId,
+      });
+    }
+
+    searchQueryBuilder
+      .take(searchParams.limit || 20)
+      .offset(searchParams.offset || 0);
+
+    const results = await searchQueryBuilder.getRawMany();
+
+    const searchResults = results.map((result) => {
+      if (result.highlight) {
+        result.highlight = result.highlight
+          .replace(/\r\n|\r|\n/g, ' ')
+          .replace(/\s+/g, ' ');
+      }
+      return result;
+    });
+
+    return searchResults;
+  }
+}