Merge branch 'main' into ai-vector

apps/server/package.json

@@ -67,6 +67,8 @@
     "jsonwebtoken": "^9.0.2",
     "kysely": "^0.28.2",
     "kysely-migration-cli": "^0.4.2",
+    "ldapts": "^7.4.0",
+    "mammoth": "^1.10.0",
     "mime-types": "^2.1.35",
     "nanoid": "3.3.11",
     "nestjs-kysely": "^1.2.0",
@@ -77,6 +79,7 @@
     "p-limit": "^6.2.0",
     "passport-google-oauth20": "^2.0.0",
     "passport-jwt": "^4.0.1",
+    "pdfjs-dist": "^5.4.54",
     "pg": "^8.16.0",
     "pg-tsquery": "^8.4.2",
     "pgvector": "^0.2.1",

apps/server/src/collaboration/collaboration.util.ts

@@ -32,6 +32,7 @@ import {
   Excalidraw,
   Embed,
   Mention,
+  Subpages,
 } from '@docmost/editor-ext';
 import { generateText, getSchema, JSONContent } from '@tiptap/core';
 import { generateHTML } from '../common/helpers/prosemirror/html';
@@ -79,6 +80,7 @@ export const tiptapExtensions = [
   Excalidraw,
   Embed,
   Mention,
+  Subpages,
 ] as any;
 
 export function jsonToHtml(tiptapJson: any) {

apps/server/src/common/helpers/utils.ts

@@ -87,3 +87,12 @@ export function extractBearerTokenFromHeader(
   const [type, token] = request.headers.authorization?.split(' ') ?? [];
   return type === 'Bearer' ? token : undefined;
 }
+
+export function hasLicenseOrEE(opts: {
+  licenseKey: string;
+  plan: string;
+  isCloud: boolean;
+}): boolean {
+  const { licenseKey, plan, isCloud } = opts;
+  return Boolean(licenseKey) || (isCloud && plan === 'business');
+}

apps/server/src/core/attachment/processors/attachment.processor.ts

@@ -3,12 +3,15 @@ import { OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq';
 import { Job } from 'bullmq';
 import { AttachmentService } from '../services/attachment.service';
 import { QueueJob, QueueName } from 'src/integrations/queue/constants';
-import { Space } from '@docmost/db/types/entity.types';
+import { ModuleRef } from '@nestjs/core';
 
 @Processor(QueueName.ATTACHMENT_QUEUE)
 export class AttachmentProcessor extends WorkerHost implements OnModuleDestroy {
   private readonly logger = new Logger(AttachmentProcessor.name);
-  constructor(private readonly attachmentService: AttachmentService) {
+  constructor(
+    private readonly attachmentService: AttachmentService,
+    private moduleRef: ModuleRef,
+  ) {
     super();
   }
 
@@ -25,6 +28,33 @@ export class AttachmentProcessor extends WorkerHost implements OnModuleDestroy {
           job.data.pageId,
         );
       }
+      if (
+        job.name === QueueJob.ATTACHMENT_INDEX_CONTENT ||
+        job.name === QueueJob.ATTACHMENT_INDEXING
+      ) {
+        let AttachmentEeModule: any;
+        try {
+          // eslint-disable-next-line @typescript-eslint/no-require-imports
+          AttachmentEeModule = require('./../../../ee/attachments-ee/attachment-ee.service');
+        } catch (err) {
+          this.logger.error(
+            'Attachment enterprise module requested but EE module not bundled in this build',
+          );
+          return;
+        }
+        const attachmentEeService = this.moduleRef.get(
+          AttachmentEeModule.AttachmentEeService,
+          { strict: false },
+        );
+
+        if (job.name === QueueJob.ATTACHMENT_INDEX_CONTENT) {
+          await attachmentEeService.indexAttachment(job.data.attachmentId);
+        } else if (job.name === QueueJob.ATTACHMENT_INDEXING) {
+          await attachmentEeService.indexAttachments(
+            job.data.workspaceId,
+          );
+        }
+      }
     } catch (err) {
       throw err;
     }

apps/server/src/core/attachment/services/attachment.service.ts

@@ -22,6 +22,9 @@ import { executeTx } from '@docmost/db/utils';
 import { UserRepo } from '@docmost/db/repos/user/user.repo';
 import { WorkspaceRepo } from '@docmost/db/repos/workspace/workspace.repo';
 import { SpaceRepo } from '@docmost/db/repos/space/space.repo';
+import { InjectQueue } from '@nestjs/bullmq';
+import { QueueJob, QueueName } from '../../../integrations/queue/constants';
+import { Queue } from 'bullmq';
 
 @Injectable()
 export class AttachmentService {
@@ -33,6 +36,7 @@ export class AttachmentService {
     private readonly workspaceRepo: WorkspaceRepo,
     private readonly spaceRepo: SpaceRepo,
     @InjectKysely() private readonly db: KyselyDB,
+    @InjectQueue(QueueName.ATTACHMENT_QUEUE) private attachmentQueue: Queue,
   ) {}
 
   async uploadFile(opts: {
@@ -99,6 +103,23 @@ export class AttachmentService {
           pageId,
         });
       }
+
+      // Only index PDFs and DOCX files
+      if (['.pdf', '.docx'].includes(attachment.fileExt.toLowerCase())) {
+        await this.attachmentQueue.add(
+          QueueJob.ATTACHMENT_INDEX_CONTENT,
+          {
+            attachmentId: attachmentId,
+          },
+          {
+            attempts: 2,
+            backoff: {
+              type: 'exponential',
+              delay: 10000,
+            },
+          },
+        );
+      }
     } catch (err) {
       // delete uploaded file on error
       this.logger.error(err);
@@ -367,4 +388,5 @@ export class AttachmentService {
       throw err;
     }
   }
+
 }

apps/server/src/core/auth/services/auth.service.ts

@@ -106,6 +106,7 @@ export class AuthService {
     await this.userRepo.updateUser(
       {
         password: newPasswordHash,
+        hasGeneratedPassword: false,
       },
       userId,
       workspaceId,
@@ -186,6 +187,7 @@ export class AuthService {
       await this.userRepo.updateUser(
         {
           password: newPasswordHash,
+          hasGeneratedPassword: false,
         },
         user.id,
         workspace.id,

apps/server/src/core/page/dto/sidebar-page.dto.ts

@@ -1,7 +1,11 @@
-import { IsOptional, IsString } from 'class-validator';
+import { IsOptional, IsString, IsUUID } from 'class-validator';
 import { SpaceIdDto } from './page.dto';
 
-export class SidebarPageDto extends SpaceIdDto {
+export class SidebarPageDto {
+  @IsOptional()
+  @IsUUID()
+  spaceId: string;
+
   @IsOptional()
   @IsString()
   pageId: string;

apps/server/src/core/page/page.controller.ts

@@ -254,21 +254,28 @@ export class PageController {
     @Body() pagination: PaginationOptions,
     @AuthUser() user: User,
   ) {
-    const ability = await this.spaceAbility.createForUser(user, dto.spaceId);
+    if (!dto.spaceId && !dto.pageId) {
+      throw new BadRequestException(
+        'Either spaceId or pageId must be provided',
+      );
+    }
+    let spaceId = dto.spaceId;
+
+    if (dto.pageId) {
+      const page = await this.pageRepo.findById(dto.pageId);
+      if (!page) {
+        throw new ForbiddenException();
+      }
+
+      spaceId = page.spaceId;
+    }
+
+    const ability = await this.spaceAbility.createForUser(user, spaceId);
     if (ability.cannot(SpaceCaslAction.Read, SpaceCaslSubject.Page)) {
       throw new ForbiddenException();
     }
 
-    let pageId = null;
-    if (dto.pageId) {
-      const page = await this.pageRepo.findById(dto.pageId);
-      if (page.spaceId !== dto.spaceId) {
-        throw new ForbiddenException();
-      }
-      pageId = page.id;
-    }
-
-    return this.pageService.getSidebarPages(dto.spaceId, pagination, pageId);
+    return this.pageService.getSidebarPages(spaceId, pagination, dto.pageId);
   }
 
   @HttpCode(HttpStatus.OK)

apps/server/src/core/search/dto/search.dto.ts

@@ -5,15 +5,13 @@ import {
   IsOptional,
   IsString,
 } from 'class-validator';
-import { PartialType } from '@nestjs/mapped-types';
-import { CreateWorkspaceDto } from '../../workspace/dto/create-workspace.dto';
 
 export class SearchDTO {
   @IsNotEmpty()
   @IsString()
   query: string;
 
-  @IsNotEmpty()
+  @IsOptional()
   @IsString()
   spaceId: string;
 

apps/server/src/core/share/share.controller.ts

@@ -31,6 +31,7 @@ import { Public } from '../../common/decorators/public.decorator';
 import { ShareRepo } from '@docmost/db/repos/share/share.repo';
 import { PaginationOptions } from '@docmost/db/pagination/pagination-options';
 import { EnvironmentService } from '../../integrations/environment/environment.service';
+import { hasLicenseOrEE } from '../../common/helpers';
 
 @UseGuards(JwtAuthGuard)
 @Controller('shares')
@@ -65,9 +66,11 @@ export class ShareController {
 
     return {
       ...(await this.shareService.getSharedPage(dto, workspace.id)),
-      hasLicenseKey:
-        Boolean(workspace.licenseKey) ||
-        (this.environmentService.isCloud() && workspace.plan === 'business'),
+      hasLicenseKey: hasLicenseOrEE({
+        licenseKey: workspace.licenseKey,
+        isCloud: this.environmentService.isCloud(),
+        plan: workspace.plan,
+      }),
     };
   }
 
@@ -175,9 +178,11 @@ export class ShareController {
   ) {
     return {
       ...(await this.shareService.getShareTree(dto.shareId, workspace.id)),
-      hasLicenseKey:
-        Boolean(workspace.licenseKey) ||
-        (this.environmentService.isCloud() && workspace.plan === 'business'),
+      hasLicenseKey: hasLicenseOrEE({
+        licenseKey: workspace.licenseKey,
+        isCloud: this.environmentService.isCloud(),
+        plan: workspace.plan,
+      }),
     };
   }
 }

apps/server/src/database/migrations/20250831T191600-add-group-sync-to-auth-providers.ts

@@ -0,0 +1,15 @@
+import { type Kysely } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .alterTable('auth_providers')
+    .addColumn('group_sync', 'boolean', (col) => col.defaultTo(false).notNull())
+    .execute();
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .alterTable('auth_providers')
+    .dropColumn('group_sync')
+    .execute();
+}

apps/server/src/database/migrations/20250831T202306-ldap-auth.ts

@@ -0,0 +1,68 @@
+import { type Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  // switch type to text column since you can't add value to PG types in a transaction
+  await db.schema
+    .alterTable('auth_providers')
+    .alterColumn('type', (col) => col.setDataType('text'))
+    .execute();
+
+  await db.schema.dropType('auth_provider_type').ifExists().execute();
+
+  await db.schema
+    .alterTable('users')
+    .addColumn('has_generated_password', 'boolean', (col) =>
+      col.notNull().defaultTo(false).ifNotExists(),
+    )
+    .execute();
+
+  await db.schema
+    .alterTable('auth_providers')
+    .addColumn('ldap_url', 'varchar', (col) => col)
+    .addColumn('ldap_bind_dn', 'varchar', (col) => col)
+    .addColumn('ldap_bind_password', 'varchar', (col) => col)
+    .addColumn('ldap_base_dn', 'varchar', (col) => col)
+    .addColumn('ldap_user_search_filter', 'varchar', (col) => col)
+    .addColumn('ldap_user_attributes', 'jsonb', (col) =>
+      col.defaultTo(sql`'{}'::jsonb`),
+    )
+    .addColumn('ldap_tls_enabled', 'boolean', (col) => col.defaultTo(false))
+    .addColumn('ldap_tls_ca_cert', 'text', (col) => col)
+    .addColumn('ldap_config', 'jsonb', (col) => col.defaultTo(sql`'{}'::jsonb`))
+    .addColumn('settings', 'jsonb', (col) => col.defaultTo(sql`'{}'::jsonb`))
+    .execute();
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .alterTable('users')
+    .dropColumn('has_generated_password')
+    .execute();
+
+  await db.schema
+    .alterTable('auth_providers')
+    .dropColumn('ldap_url')
+    .dropColumn('ldap_bind_dn')
+    .dropColumn('ldap_bind_password')
+    .dropColumn('ldap_base_dn')
+    .dropColumn('ldap_user_search_filter')
+    .dropColumn('ldap_user_attributes')
+    .dropColumn('ldap_tls_enabled')
+    .dropColumn('ldap_tls_ca_cert')
+    .dropColumn('ldap_config')
+    .dropColumn('settings')
+    .execute();
+
+  await db.schema
+    .createType('auth_provider_type')
+    .asEnum(['saml', 'oidc', 'google'])
+    .execute();
+
+  await db.deleteFrom('auth_providers').where('type', '=', 'ldap').execute();
+
+  await sql`
+    ALTER TABLE auth_providers 
+    ALTER COLUMN type TYPE auth_provider_type 
+    USING type::auth_provider_type
+  `.execute(db);
+}

apps/server/src/database/migrations/20250901T184612-attachments-search.ts

@@ -0,0 +1,29 @@
+import { type Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .alterTable('attachments')
+    .addColumn('text_content', 'text', (col) => col)
+    .addColumn('tsv', sql`tsvector`, (col) => col)
+    .execute();
+
+  await db.schema
+    .createIndex('attachments_tsv_idx')
+    .on('attachments')
+    .using('GIN')
+    .column('tsv')
+    .execute();
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .alterTable('attachments')
+    .dropIndex('attachments_tsv_idx')
+    .execute();
+
+  await db.schema
+    .alterTable('attachments')
+    .dropColumn('text_content')
+    .dropColumn('tsv')
+    .execute();
+}

apps/server/src/database/repos/attachment/attachment.repo.ts

@@ -12,6 +12,23 @@ import {
 export class AttachmentRepo {
   constructor(@InjectKysely() private readonly db: KyselyDB) {}
 
+  private baseFields: Array<keyof Attachment> = [
+    'id',
+    'fileName',
+    'filePath',
+    'fileSize',
+    'fileExt',
+    'mimeType',
+    'type',
+    'creatorId',
+    'pageId',
+    'spaceId',
+    'workspaceId',
+    'createdAt',
+    'updatedAt',
+    'deletedAt',
+  ];
+
   async findById(
     attachmentId: string,
     opts?: {
@@ -22,7 +39,7 @@ export class AttachmentRepo {
 
     return db
       .selectFrom('attachments')
-      .selectAll()
+      .select(this.baseFields)
       .where('id', '=', attachmentId)
       .executeTakeFirst();
   }
@@ -36,7 +53,7 @@ export class AttachmentRepo {
     return db
       .insertInto('attachments')
       .values(insertableAttachment)
-      .returningAll()
+      .returning(this.baseFields)
       .executeTakeFirst();
   }
 
@@ -50,7 +67,7 @@ export class AttachmentRepo {
 
     return db
       .selectFrom('attachments')
-      .selectAll()
+      .select(this.baseFields)
       .where('spaceId', '=', spaceId)
       .execute();
   }
@@ -64,6 +81,7 @@ export class AttachmentRepo {
       .updateTable('attachments')
       .set(updatableAttachment)
       .where('pageId', 'in', pageIds)
+      .returning(this.baseFields)
       .executeTakeFirst();
   }
 
@@ -75,7 +93,7 @@ export class AttachmentRepo {
       .updateTable('attachments')
       .set(updatableAttachment)
       .where('id', '=', attachmentId)
-      .returningAll()
+      .returning(this.baseFields)
       .executeTakeFirst();
   }
 

apps/server/src/database/repos/page/page.repo.ts

@@ -401,6 +401,7 @@ export class PageRepo {
           ])
           .$if(opts?.includeContent, (qb) => qb.select('content'))
           .where('id', '=', parentPageId)
+          .where('deletedAt', 'is', null)
           .unionAll((exp) =>
             exp
               .selectFrom('pages as p')
@@ -415,7 +416,8 @@ export class PageRepo {
                 'p.workspaceId',
               ])
               .$if(opts?.includeContent, (qb) => qb.select('p.content'))
-              .innerJoin('page_hierarchy as ph', 'p.parentPageId', 'ph.id'),
+              .innerJoin('page_hierarchy as ph', 'p.parentPageId', 'ph.id')
+              .where('p.deletedAt', 'is', null),
           ),
       )
       .selectFrom('page_hierarchy')

apps/server/src/database/repos/user/user.repo.ts

@@ -34,6 +34,7 @@ export class UserRepo {
     'createdAt',
     'updatedAt',
     'deletedAt',
+    'hasGeneratedPassword',
   ];
 
   async findById(

apps/server/src/database/types/db.d.ts

@@ -5,8 +5,6 @@
 
 import type { ColumnType } from "kysely";
 
-export type AuthProviderType = "google" | "oidc" | "saml";
-
 export type Generated<T> = T extends ColumnType<infer S, infer I, infer U>
   ? ColumnType<S, I | undefined, U>
   : ColumnType<T, T | undefined, T>;
@@ -39,6 +37,8 @@ export interface Attachments {
   mimeType: string | null;
   pageId: string | null;
   spaceId: string | null;
+  textContent: string | null;
+  tsv: string | null;
   type: string | null;
   updatedAt: Generated<Timestamp>;
   workspaceId: string;
@@ -62,13 +62,24 @@ export interface AuthProviders {
   deletedAt: Timestamp | null;
   id: Generated<string>;
   isEnabled: Generated<boolean>;
+  groupSync: Generated<boolean>;
+  ldapBaseDn: string | null;
+  ldapBindDn: string | null;
+  ldapBindPassword: string | null;
+  ldapTlsCaCert: string | null;
+  ldapTlsEnabled: Generated<boolean | null>;
+  ldapUrl: string | null;
+  ldapUserAttributes: Json | null;
+  ldapUserSearchFilter: string | null;
+  ldapConfig: Json | null;
+  settings: Json | null;
   name: string;
   oidcClientId: string | null;
   oidcClientSecret: string | null;
   oidcIssuer: string | null;
   samlCertificate: string | null;
   samlUrl: string | null;
-  type: AuthProviderType;
+  type: string;
   updatedAt: Generated<Timestamp>;
   workspaceId: string;
 }
@@ -275,6 +286,7 @@ export interface Users {
   lastActiveAt: Timestamp | null;
   lastLoginAt: Timestamp | null;
   locale: string | null;
+  hasGeneratedPassword: Generated<boolean | null>;
   name: string | null;
   password: string | null;
   role: string | null;

apps/server/src/integrations/export/export.controller.ts

@@ -46,7 +46,7 @@ export class ExportController {
       includeContent: true,
     });
 
-    if (!page) {
+    if (!page || page.deletedAt) {
       throw new NotFoundException('Page not found');
     }
 

apps/server/src/integrations/queue/constants/queue.constants.ts

@@ -10,6 +10,8 @@ export enum QueueName {
 export enum QueueJob {
   SEND_EMAIL = 'send-email',
   DELETE_SPACE_ATTACHMENTS = 'delete-space-attachments',
+  ATTACHMENT_INDEX_CONTENT = 'attachment-index-content',
+  ATTACHMENT_INDEXING = 'attachment-indexing',
   DELETE_PAGE_ATTACHMENTS = 'delete-page-attachments',
   PAGE_CONTENT_UPDATE = 'page-content-update',