Merge branch 'main' into perm-x

apps/server/src/integrations/export/export.controller.ts

@@ -55,7 +55,7 @@ export class ExportController {
       throw new ForbiddenException();
     }
 
-    const zipFileBuffer = await this.exportService.exportPages(
+    const zipFileStream = await this.exportService.exportPages(
       dto.pageId,
       dto.format,
       dto.includeAttachments,
@@ -71,7 +71,7 @@ export class ExportController {
         'attachment; filename="' + encodeURIComponent(fileName) + '"',
     });
 
-    res.send(zipFileBuffer);
+    res.send(zipFileStream);
   }
 
   @UseGuards(JwtAuthGuard)
@@ -102,6 +102,6 @@ export class ExportController {
         '"',
     });
 
-    res.send(exportFile.fileBuffer);
+    res.send(exportFile.fileStream);
   }
 }

apps/server/src/integrations/export/export.service.ts

@@ -5,7 +5,6 @@ import {
   NotFoundException,
 } from '@nestjs/common';
 import { jsonToHtml, jsonToNode } from '../../collaboration/collaboration.util';
-import { turndown } from './turndown-utils';
 import { ExportFormat } from './dto/export-dto';
 import { Page } from '@docmost/db/types/entity.types';
 import { InjectKysely } from 'nestjs-kysely';
@@ -32,6 +31,7 @@ import {
   getAttachmentIds,
   getProsemirrorContent,
 } from '../../common/helpers/prosemirror/utils';
+import { htmlToMarkdown } from '@docmost/editor-ext';
 
 @Injectable()
 export class ExportService {
@@ -85,7 +85,7 @@ export class ExportService {
         /<colgroup[^>]*>[\s\S]*?<\/colgroup>/gim,
         '',
       );
-      return turndown(newPageHtml);
+      return htmlToMarkdown(newPageHtml);
     }
 
     return;
@@ -227,7 +227,7 @@ export class ExportService {
 
     const fileName = `${space.name}-space-export.zip`;
     return {
-      fileBuffer: zipFile,
+      fileStream: zipFile,
       fileName,
     };
   }

apps/server/src/integrations/export/turndown-utils.ts

@@ -1,165 +0,0 @@
-import * as TurndownService from '@joplin/turndown';
-import * as TurndownPluginGfm from '@joplin/turndown-plugin-gfm';
-import * as path from 'path';
-
-export function turndown(html: string): string {
-  const turndownService = new TurndownService({
-    headingStyle: 'atx',
-    codeBlockStyle: 'fenced',
-    hr: '---',
-    bulletListMarker: '-',
-  });
-  const tables = TurndownPluginGfm.tables;
-  const strikethrough = TurndownPluginGfm.strikethrough;
-  const highlightedCodeBlock = TurndownPluginGfm.highlightedCodeBlock;
-
-  turndownService.use([
-    tables,
-    strikethrough,
-    highlightedCodeBlock,
-    taskList,
-    callout,
-    preserveDetail,
-    listParagraph,
-    mathInline,
-    mathBlock,
-    iframeEmbed,
-    video,
-  ]);
-  return turndownService.turndown(html).replaceAll('<br>', ' ');
-}
-
-function listParagraph(turndownService: TurndownService) {
-  turndownService.addRule('paragraph', {
-    filter: ['p'],
-    replacement: (content: any, node: HTMLInputElement) => {
-      if (node.parentElement?.nodeName === 'LI') {
-        return content;
-      }
-
-      return `\n\n${content}\n\n`;
-    },
-  });
-}
-
-function callout(turndownService: TurndownService) {
-  turndownService.addRule('callout', {
-    filter: function (node: HTMLInputElement) {
-      return (
-        node.nodeName === 'DIV' && node.getAttribute('data-type') === 'callout'
-      );
-    },
-    replacement: function (content: any, node: HTMLInputElement) {
-      const calloutType = node.getAttribute('data-callout-type');
-      return `\n\n:::${calloutType}\n${content.trim()}\n:::\n\n`;
-    },
-  });
-}
-
-function taskList(turndownService: TurndownService) {
-  turndownService.addRule('taskListItem', {
-    filter: function (node: HTMLInputElement) {
-      return (
-        node.getAttribute('data-type') === 'taskItem' &&
-        node.parentNode.nodeName === 'UL'
-      );
-    },
-    replacement: function (content: any, node: HTMLInputElement) {
-      const checkbox = node.querySelector(
-        'input[type="checkbox"]',
-      ) as HTMLInputElement;
-      const isChecked = checkbox.checked;
-      
-      // Process content like regular list items
-      content = content
-        .replace(/^\n+/, '') // remove leading newlines
-        .replace(/\n+$/, '\n') // replace trailing newlines with just a single one
-        .replace(/\n/gm, '\n  '); // indent nested content with 2 spaces
-      
-      // Create the checkbox prefix
-      const prefix = `- ${isChecked ? '[x]' : '[ ]'} `;
-      
-      return prefix + content + (node.nextSibling && !/\n$/.test(content) ? '\n' : '');
-    },
-  });
-}
-
-function preserveDetail(turndownService: TurndownService) {
-  turndownService.addRule('preserveDetail', {
-    filter: function (node: HTMLInputElement) {
-      return node.nodeName === 'DETAILS';
-    },
-    replacement: function (content: any, node: HTMLInputElement) {
-      const summary = node.querySelector(':scope > summary');
-      let detailSummary = '';
-
-      if (summary) {
-        detailSummary = `<summary>${turndownService.turndown(summary.innerHTML)}</summary>`;
-      }
-
-      const detailsContent = Array.from(node.childNodes)
-        .filter((child) => child.nodeName !== 'SUMMARY')
-        .map((child) =>
-          child.nodeType === 1
-            ? turndownService.turndown((child as HTMLElement).outerHTML)
-            : child.textContent,
-        )
-        .join('');
-
-      return `\n<details>\n${detailSummary}\n\n${detailsContent}\n\n</details>\n`;
-    },
-  });
-}
-
-function mathInline(turndownService: TurndownService) {
-  turndownService.addRule('mathInline', {
-    filter: function (node: HTMLInputElement) {
-      return (
-        node.nodeName === 'SPAN' &&
-        node.getAttribute('data-type') === 'mathInline'
-      );
-    },
-    replacement: function (content: any, node: HTMLInputElement) {
-      return `$${content}$`;
-    },
-  });
-}
-
-function mathBlock(turndownService: TurndownService) {
-  turndownService.addRule('mathBlock', {
-    filter: function (node: HTMLInputElement) {
-      return (
-        node.nodeName === 'DIV' &&
-        node.getAttribute('data-type') === 'mathBlock'
-      );
-    },
-    replacement: function (content: any, node: HTMLInputElement) {
-      return `\n$$\n${content}\n$$\n`;
-    },
-  });
-}
-
-function iframeEmbed(turndownService: TurndownService) {
-  turndownService.addRule('iframeEmbed', {
-    filter: function (node: HTMLInputElement) {
-      return node.nodeName === 'IFRAME';
-    },
-    replacement: function (content: any, node: HTMLInputElement) {
-      const src = node.getAttribute('src');
-      return '[' + src + '](' + src + ')';
-    },
-  });
-}
-
-function video(turndownService: TurndownService) {
-  turndownService.addRule('video', {
-    filter: function (node: HTMLInputElement) {
-      return node.tagName === 'VIDEO';
-    },
-    replacement: function (content: any, node: HTMLInputElement) {
-      const src = node.getAttribute('src') || '';
-      const name = path.basename(src);
-      return '[' + name + '](' + src + ')';
-    },
-  });
-}

apps/server/src/integrations/export/utils.ts

@@ -1,4 +1,5 @@
 import { jsonToNode } from 'src/collaboration/collaboration.util';
+import { Logger } from '@nestjs/common';
 import { ExportFormat } from './dto/export-dto';
 import { Node } from '@tiptap/pm/model';
 import { validate as isValidUUID } from 'uuid';
@@ -88,7 +89,7 @@ export function replaceInternalLinks(
             // if link and text are same, use page title
             if (markLink === node.text) {
               //@ts-expect-error
-              node.text = getInternalLinkPageName(relativePath);
+              node.text = getInternalLinkPageName(relativePath, currentPagePath);
             }
           }
         }
@@ -99,10 +100,19 @@ export function replaceInternalLinks(
   return doc.toJSON();
 }
 
-export function getInternalLinkPageName(path: string): string {
-  return decodeURIComponent(
-    path?.split('/').pop().split('.').slice(0, -1).join('.'),
-  );
+export function getInternalLinkPageName(path: string, currentFilePath?: string): string {
+  const name = path?.split('/').pop().split('.').slice(0, -1).join('.');
+  try {
+    return decodeURIComponent(name);
+  } catch (err) {
+    if (currentFilePath) {
+      Logger.warn(
+        `URI malformed in page ${currentFilePath}: ${name}. Falling back to raw name.`,
+        'ExportUtils',
+      );
+    }
+    return name;
+  }
 }
 
 export function extractPageSlugId(input: string): string {

apps/server/src/integrations/import/services/import.service.ts

@@ -10,7 +10,11 @@ import {
 } from '../../../collaboration/collaboration.util';
 import { InjectKysely } from 'nestjs-kysely';
 import { KyselyDB } from '@docmost/db/types/kysely.types';
-import { generateSlugId, sanitizeFileName } from '../../../common/helpers';
+import {
+  generateSlugId,
+  sanitizeFileName,
+  createByteCountingStream,
+} from '../../../common/helpers';
 import { generateJitteredKeyBetween } from 'fractional-indexing-jittered';
 import { TiptapTransformer } from '@hocuspocus/transformer';
 import * as Y from 'yjs';
@@ -173,15 +177,24 @@ export class ImportService {
     };
   }
 
-  async getNewPagePosition(spaceId: string): Promise<string> {
-    const lastPage = await this.db
+  async getNewPagePosition(
+    spaceId: string,
+    parentPageId?: string,
+  ): Promise<string> {
+    let query = this.db
       .selectFrom('pages')
       .select(['id', 'position'])
       .where('spaceId', '=', spaceId)
       .orderBy('position', (ob) => ob.collate('C').desc())
-      .limit(1)
-      .where('parentPageId', 'is', null)
-      .executeTakeFirst();
+      .limit(1);
+
+    if (parentPageId) {
+      query = query.where('parentPageId', '=', parentPageId);
+    } else {
+      query = query.where('parentPageId', 'is', null);
+    }
+
+    const lastPage = await query.executeTakeFirst();
 
     if (lastPage) {
       return generateJitteredKeyBetween(lastPage.position, null);
@@ -198,20 +211,21 @@ export class ImportService {
     workspaceId: string,
   ) {
     const file = await filePromise;
-    const fileBuffer = await file.toBuffer();
     const fileExtension = path.extname(file.filename).toLowerCase();
     const fileName = sanitizeFileName(
       path.basename(file.filename, fileExtension),
     );
-    const fileSize = fileBuffer.length;
-
     const fileNameWithExt = fileName + fileExtension;
 
     const fileTaskId = uuid7();
     const filePath = `${getFileTaskFolderPath(FileTaskType.Import, workspaceId)}/${fileTaskId}/${fileNameWithExt}`;
 
     // upload file
-    await this.storageService.upload(filePath, fileBuffer);
+    const { stream, getBytesRead } = createByteCountingStream(file.file);
+
+    await this.storageService.upload(filePath, stream);
+
+    const fileSize = getBytesRead();
 
     const fileTask = await this.db
       .insertInto('fileTasks')

apps/server/src/integrations/import/utils/import-formatter.ts

@@ -1,4 +1,5 @@
 import { getEmbedUrlAndProvider } from '@docmost/editor-ext';
+import { Logger } from '@nestjs/common';
 import * as path from 'path';
 import { v7 } from 'uuid';
 import { InsertableBacklink } from '@docmost/db/types/entity.types';
@@ -280,8 +281,18 @@ export async function rewriteInternalLinksToMentionHtml(
     const $a = $(el);
     const raw = $a.attr('href')!;
     if (raw.startsWith('http') || raw.startsWith('/api/')) return;
+    let decodedRaw = raw;
+    try {
+      decodedRaw = decodeURIComponent(raw);
+    } catch (err) {
+      Logger.warn(
+        `URI malformed in page ${currentFilePath}: ${raw}. Falling back to raw path.`,
+        'ImportFormatter',
+      );
+    }
+
     const resolved = normalize(
-      path.join(path.dirname(currentFilePath), decodeURIComponent(raw)),
+      path.join(path.dirname(currentFilePath), decodedRaw),
     );
     const meta = filePathToPageMetaMap.get(resolved);
     if (!meta) return;

apps/server/src/integrations/import/utils/import.utils.ts

@@ -1,3 +1,4 @@
+import { Logger } from '@nestjs/common';
 import { promises as fs } from 'fs';
 import * as path from 'path';
 
@@ -30,8 +31,13 @@ export function resolveRelativeAttachmentPath(
   pageDir: string,
   attachmentCandidates: Map<string, string>,
 ): string | null {
-  const mainRel = decodeURIComponent(raw.replace(/^\.?\/+/, ''));
-  const fallback = path.normalize(path.join(pageDir, mainRel));
+  let mainRel = raw.replace(/^\.?\/+/, '');
+  try {
+    mainRel = decodeURIComponent(mainRel);
+  } catch (err) {
+    Logger.warn(`URI malformed for attachment path: ${mainRel}. Falling back to raw path.`, 'ImportUtils');
+  }
+  const fallback = path.normalize(path.join(pageDir, mainRel)).split(path.sep).join('/');
 
   if (attachmentCandidates.has(mainRel)) {
     return mainRel;

apps/server/src/integrations/storage/drivers/local.driver.ts

@@ -20,9 +20,15 @@ export class LocalDriver implements StorageDriver {
     return join(this.config.storagePath, filePath);
   }
 
-  async upload(filePath: string, file: Buffer): Promise<void> {
+  async upload(filePath: string, file: Buffer | Readable): Promise<void> {
     try {
-      await fs.outputFile(this._fullPath(filePath), file);
+      const fullPath = this._fullPath(filePath);
+      if (file instanceof Buffer) {
+        await fs.outputFile(fullPath, file);
+      } else {
+        await fs.mkdir(dirname(fullPath), { recursive: true });
+        await pipeline(file, createWriteStream(fullPath));
+      }
     } catch (err) {
       throw new Error(`Failed to upload file: ${(err as Error).message}`);
     }
@@ -42,7 +48,7 @@ export class LocalDriver implements StorageDriver {
     try {
       const fromFullPath = this._fullPath(fromFilePath);
       const toFullPath = this._fullPath(toFilePath);
-      
+
       if (await this.exists(fromFilePath)) {
         await fs.copy(fromFullPath, toFullPath);
       }

apps/server/src/integrations/storage/drivers/s3.driver.ts

@@ -23,19 +23,21 @@ export class S3Driver implements StorageDriver {
     this.s3Client = new S3Client(config as any);
   }
 
-  async upload(filePath: string, file: Buffer): Promise<void> {
+  async upload(filePath: string, file: Buffer | Readable): Promise<void> {
     try {
       const contentType = getMimeType(filePath);
 
-      const command = new PutObjectCommand({
-        Bucket: this.config.bucket,
-        Key: filePath,
-        Body: file,
-        ContentType: contentType,
-        // ACL: "public-read",
+      const upload = new Upload({
+        client: this.s3Client,
+        params: {
+          Bucket: this.config.bucket,
+          Key: filePath,
+          Body: file,
+          ContentType: contentType,
+        },
       });
 
-      await this.s3Client.send(command);
+      await upload.done();
     } catch (err) {
       throw new Error(`Failed to upload file: ${(err as Error).message}`);
     }

apps/server/src/integrations/storage/interfaces/storage-driver.interface.ts

@@ -1,7 +1,7 @@
 import { Readable } from 'stream';
 
 export interface StorageDriver {
-  upload(filePath: string, file: Buffer): Promise<void>;
+  upload(filePath: string, file: Buffer | Readable): Promise<void>;
 
   uploadStream(filePath: string, file: Readable, options?: { recreateClient?: boolean }): Promise<void>;
 

apps/server/src/integrations/storage/storage.service.ts

@@ -8,9 +8,9 @@ export class StorageService {
   private readonly logger = new Logger(StorageService.name);
   constructor(
     @Inject(STORAGE_DRIVER_TOKEN) private storageDriver: StorageDriver,
-  ) {}
+  ) { }
 
-  async upload(filePath: string, fileContent: Buffer | any) {
+  async upload(filePath: string, fileContent: Buffer | Readable) {
     await this.storageDriver.upload(filePath, fileContent);
     this.logger.debug(`File uploaded successfully. Path: ${filePath}`);
   }