* Make page import handling better

apps/client/src/features/page/components/page-import-modal.tsx

@@ -84,14 +84,14 @@ function ImportFormatSelection({ spaceId, onClose }: ImportFormatSelection) {
       }
     }
 
-    const newTreeNodes = buildTree(pages);
+    if (pages?.length > 0 && pageCount > 0) {
-    const fullTree = treeData.concat(newTreeNodes);
+      const newTreeNodes = buildTree(pages);
+      const fullTree = treeData.concat(newTreeNodes);
 
-    if (newTreeNodes?.length && fullTree?.length > 0) {
+      if (newTreeNodes?.length && fullTree?.length > 0) {
-      setTreeData(fullTree);
+        setTreeData(fullTree);
-    }
+      }
 
-    if (pageCount > 0) {
       const pageCountText = pageCount === 1 ? "1 page" : `${pageCount} pages`;
 
       notifications.update({

apps/server/src/collaboration/collaboration.util.ts

@@ -28,7 +28,9 @@ import {
   TrailingNode,
 } from '@docmost/editor-ext';
 import { generateText, JSONContent } from '@tiptap/core';
-import { generateHTML, generateJSON } from '../common/helpers/prosemirror/html';
+import { generateHTML } from '../common/helpers/prosemirror/html';
+// default tiptap library works best generating prosemirror json state
+import { generateJSON } from '@tiptap/html';
 
 export const tiptapExtensions = [
   StarterKit,

apps/server/src/integrations/import/import.controller.ts

@@ -20,6 +20,7 @@ import {
 } from '../../core/casl/interfaces/space-ability.type';
 import { FileInterceptor } from '../../common/interceptors/file.interceptor';
 import * as bytes from 'bytes';
+import * as path from 'path';
 import { MAX_FILE_SIZE } from '../../core/attachment/attachment.constants';
 import { ImportService } from './import.service';
 import { AuthWorkspace } from '../../common/decorators/auth-workspace.decorator';
@@ -42,6 +43,8 @@ export class ImportController {
     @AuthUser() user: User,
     @AuthWorkspace() workspace: Workspace,
   ) {
+    const validFileExtensions = ['.md', '.html'];
+
     const maxFileSize = bytes(MAX_FILE_SIZE);
 
     let file = null;
@@ -62,6 +65,12 @@ export class ImportController {
       throw new BadRequestException('Failed to upload file');
     }
 
+    if (
+      !validFileExtensions.includes(path.extname(file.filename).toLowerCase())
+    ) {
+      throw new BadRequestException('Invalid import file type.');
+    }
+
     const spaceId = file.fields?.spaceId?.value;
 
     if (!spaceId) {

apps/server/src/integrations/import/import.service.ts

@@ -4,12 +4,11 @@ import { MultipartFile } from '@fastify/multipart';
 import { sanitize } from 'sanitize-filename-ts';
 import * as path from 'path';
 import { htmlToJson } from '../../collaboration/collaboration.util';
-import { marked } from 'marked';
 import { InjectKysely } from 'nestjs-kysely';
 import { KyselyDB } from '@docmost/db/types/kysely.types';
 import { generateSlugId } from '../../common/helpers';
 import { generateJitteredKeyBetween } from 'fractional-indexing-jittered';
-import { transformHTML } from './utils/html.utils';
+import { markdownToHtml } from './utils/marked.utils';
 
 @Injectable()
 export class ImportService {
@@ -36,16 +35,23 @@ export class ImportService {
     let prosemirrorState = null;
     let createdPage = null;
 
-    if (fileExtension.endsWith('.md') && fileMimeType === 'text/markdown') {
+    try {
-      prosemirrorState = await this.processMarkdown(fileContent);
+      if (fileExtension.endsWith('.md') && fileMimeType === 'text/markdown') {
-    }
+        prosemirrorState = await this.processMarkdown(fileContent);
-
+      } else if (
-    if (fileExtension.endsWith('.html') && fileMimeType === 'text/html') {
+        fileExtension.endsWith('.html') &&
-      prosemirrorState = await this.processHTML(fileContent);
+        fileMimeType === 'text/html'
+      ) {
+        prosemirrorState = await this.processHTML(fileContent);
+      }
+    } catch (err) {
+      const message = 'Error processing file content';
+      this.logger.error(message, err);
+      throw new BadRequestException(message);
     }
 
     if (!prosemirrorState) {
-      const message = 'Unsupported file format or mime type';
+      const message = 'Failed to create ProseMirror state';
       this.logger.error(message);
       throw new BadRequestException(message);
     }
@@ -69,8 +75,12 @@ export class ImportService {
           workspaceId: workspaceId,
           lastUpdatedById: userId,
         });
+
+        this.logger.debug(
+          `Successfully imported "${title}${fileExtension}. ID: ${createdPage.id} - SlugId: ${createdPage.slugId}"`,
+        );
       } catch (err) {
-        const message = 'Failed to create page';
+        const message = 'Failed to create imported page';
         this.logger.error(message, err);
         throw new BadRequestException(message);
       }
@@ -80,14 +90,20 @@ export class ImportService {
   }
 
   async processMarkdown(markdownInput: string): Promise<any> {
-    // turn markdown to html
+    try {
-    const html = await marked.parse(markdownInput);
+      const html = await markdownToHtml(markdownInput);
-    return await this.processHTML(html);
+      return this.processHTML(html);
+    } catch (err) {
+      throw err;
+    }
   }
 
   async processHTML(htmlInput: string): Promise<any> {
-    // turn html to prosemirror state
+    try {
-    return htmlToJson(transformHTML(htmlInput));
+      return htmlToJson(htmlInput);
+    } catch (err) {
+      throw err;
+    }
   }
 
   extractTitleAndRemoveHeading(prosemirrorState: any) {

apps/server/src/integrations/import/utils/html.utils.ts

@@ -1,80 +0,0 @@
-import { Window, DOMParser } from 'happy-dom';
-
-function transformTaskList(html: string): string {
-  const window = new Window();
-  const doc = new DOMParser(window).parseFromString(html, 'text/html');
-
-  const ulElements = doc.querySelectorAll('ul');
-  ulElements.forEach((ul) => {
-    let isTaskList = false;
-
-    const liElements = ul.querySelectorAll('li');
-    liElements.forEach((li) => {
-      const checkbox = li.querySelector('input[type="checkbox"]');
-
-      if (checkbox) {
-        isTaskList = true;
-        // Add taskItem data type
-        li.setAttribute('data-type', 'taskItem');
-        // Set data-checked attribute based on the checkbox state
-        // @ts-ignore
-        li.setAttribute('data-checked', checkbox.checked ? 'true' : 'false');
-        // Remove the checkbox from the li
-        checkbox.remove();
-
-        // Move the content of <p> out of the <p> and remove <p>
-        const pElements = li.querySelectorAll('p');
-        pElements.forEach((p) => {
-          // Append the content of the <p> element to its parent (the <li> element)
-          while (p.firstChild) {
-            li.appendChild(p.firstChild);
-          }
-          // Remove the now empty <p> element
-          p.remove();
-        });
-      }
-    });
-
-    // If any <li> contains a checkbox, mark the <ul> as a task list
-    if (isTaskList) {
-      ul.setAttribute('data-type', 'taskList');
-    }
-  });
-
-  return doc.body.innerHTML;
-}
-
-function transformCallouts(html: string): string {
-  const window = new Window();
-  const doc = new DOMParser(window).parseFromString(html, 'text/html');
-
-  const calloutRegex = /:::(\w+)\s*([\s\S]*?)\s*:::/g;
-
-  const createCalloutDiv = (type: string, content: string): HTMLElement => {
-    const div = doc.createElement('div');
-    div.setAttribute('data-type', 'callout');
-    div.setAttribute('data-callout-type', type);
-    const p = doc.createElement('p');
-    p.textContent = content.trim();
-    div.appendChild(p);
-    return div as unknown as HTMLElement;
-  };
-
-  const pElements = doc.querySelectorAll('p');
-
-  pElements.forEach((p) => {
-    if (calloutRegex.test(p.innerHTML) && !p.closest('ul, ol')) {
-      calloutRegex.lastIndex = 0;
-      const [, type, content] = calloutRegex.exec(p.innerHTML) || [];
-      const calloutDiv = createCalloutDiv(type, content);
-      // @ts-ignore
-      p.replaceWith(calloutDiv);
-    }
-  });
-
-  return doc.body.innerHTML;
-}
-
-export function transformHTML(html: string): string {
-  return transformTaskList(transformCallouts(html));
-}

apps/server/src/integrations/import/utils/marked.utils.ts

@@ -0,0 +1,36 @@
+import { marked } from 'marked';
+
+marked.use({
+  renderer: {
+    // @ts-ignore
+    list(body: string, isOrdered: boolean, start: number) {
+      if (isOrdered) {
+        const startAttr = start !== 1 ? ` start="${start}"` : '';
+        return `<ol ${startAttr}>\n${body}</ol>\n`;
+      }
+
+      const dataType = body.includes(`<input`) ? ' data-type="taskList"' : '';
+      return `<ul${dataType}>\n${body}</ul>\n`;
+    },
+    // @ts-ignore
+    listitem({ text, raw, task: isTask, checked: isChecked }): string {
+      if (!isTask) {
+        return `<li>${text}</li>\n`;
+      }
+      const checkedAttr = isChecked
+        ? 'data-checked="true"'
+        : 'data-checked="false"';
+      return `<li data-type="taskItem" ${checkedAttr}>${text}</li>\n`;
+    },
+  },
+});
+
+export async function markdownToHtml(markdownInput: string): Promise<string> {
+  const YAML_FONT_MATTER_REGEX = /^\s*---[\s\S]*?---\s*/;
+
+  const markdown = markdownInput
+    .replace(YAML_FONT_MATTER_REGEX, '')
+    .trimStart();
+
+  return marked.parse(markdown);
+}

package.json

@@ -53,6 +53,7 @@
     "@tiptap/extension-typography": "^2.5.4",
     "@tiptap/extension-underline": "^2.5.4",
     "@tiptap/extension-youtube": "^2.5.4",
+    "@tiptap/html": "^2.5.4",
     "@tiptap/pm": "^2.5.4",
     "@tiptap/react": "^2.5.4",
     "@tiptap/starter-kit": "^2.5.4",

pnpm-lock.yaml

@@ -119,6 +119,9 @@ importers:
       '@tiptap/extension-youtube':
         specifier: ^2.5.4
         version: 2.5.4(@tiptap/core@2.5.4(@tiptap/pm@2.5.4))
+      '@tiptap/html':
+        specifier: ^2.5.4
+        version: 2.5.4(@tiptap/core@2.5.4(@tiptap/pm@2.5.4))(@tiptap/pm@2.5.4)
       '@tiptap/pm':
         specifier: ^2.5.4
         version: 2.5.4
@@ -3780,6 +3783,12 @@ packages:
     peerDependencies:
       '@tiptap/core': ^2.5.4
 
+  '@tiptap/html@2.5.4':
+    resolution: {integrity: sha512-Fcvsa7kkO+Id7WBFimDN5zdHksVGVnyHnffaN/PaAgbKmzP53BC38Pd0XuHS+KL6btqQIFE2GlqNYnyIos7i+g==}
+    peerDependencies:
+      '@tiptap/core': ^2.5.4
+      '@tiptap/pm': ^2.5.4
+
   '@tiptap/pm@2.5.4':
     resolution: {integrity: sha512-oFIsuniptdUXn93x4aM2sVN3hYKo9Fj55zAkYrWhwxFYUYcPxd5ibra2we+wRK5TaiPu098wpC+yMSTZ/KKMpA==}
 
@@ -4757,6 +4766,10 @@ packages:
   css-to-mat@1.1.1:
     resolution: {integrity: sha512-kvpxFYZb27jRd2vium35G7q5XZ2WJ9rWjDUMNT36M3Hc41qCrLXFM5iEKMGXcrPsKfXEN+8l/riB4QzwwwiEyQ==}
 
+  css-what@6.1.0:
+    resolution: {integrity: sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==}
+    engines: {node: '>= 6'}
+
   cssesc@3.0.0:
     resolution: {integrity: sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==}
     engines: {node: '>=4'}
@@ -6178,8 +6191,8 @@ packages:
   makeerror@1.0.12:
     resolution: {integrity: sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg==}
 
-  markdown-it@14.0.0:
+  markdown-it@14.1.0:
-    resolution: {integrity: sha512-seFjF0FIcPt4P9U39Bq1JYblX0KZCjDLFFQPHpL5AzHpqPEKtosxmdq/LTVZnjfH7tjt9BxStm+wXcDBNuYmzw==}
+    resolution: {integrity: sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==}
     hasBin: true
 
   marked@13.0.2:
@@ -7790,8 +7803,8 @@ packages:
     engines: {node: '>=14.17'}
     hasBin: true
 
-  uc.micro@2.0.0:
+  uc.micro@2.1.0:
-    resolution: {integrity: sha512-DffL94LsNOccVn4hyfRe5rdKa273swqeA5DJpMOeFmEn1wCDc7nAbbB0gXlgBCL7TNzeTv6G7XVWzan7iJtfig==}
+    resolution: {integrity: sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==}
 
   uid2@1.0.0:
     resolution: {integrity: sha512-+I6aJUv63YAcY9n4mQreLUt0d4lvwkkopDNmpomkAUz0fAkEMV9pRWxN0EjhW1YfRhcuyHg2v3mwddCDW1+LFQ==}
@@ -8160,6 +8173,10 @@ packages:
     resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==}
     engines: {node: '>=10'}
 
+  zeed-dom@0.10.11:
+    resolution: {integrity: sha512-7ukbu6aQKx34OQ7PfUIxOuAhk2MvyZY/t4/IJsVzy76zuMzfhE74+Dbyp8SHiUJPTPkF0FflP1KVrGJ7gk9IHw==}
+    engines: {node: '>=14.13.1'}
+
   zod@3.23.8:
     resolution: {integrity: sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==}
 
@@ -12059,6 +12076,12 @@ snapshots:
     dependencies:
       '@tiptap/core': 2.5.4(@tiptap/pm@2.5.4)
 
+  '@tiptap/html@2.5.4(@tiptap/core@2.5.4(@tiptap/pm@2.5.4))(@tiptap/pm@2.5.4)':
+    dependencies:
+      '@tiptap/core': 2.5.4(@tiptap/pm@2.5.4)
+      '@tiptap/pm': 2.5.4
+      zeed-dom: 0.10.11
+
   '@tiptap/pm@2.5.4':
     dependencies:
       prosemirror-changeset: 2.2.1
@@ -13272,6 +13295,8 @@ snapshots:
       '@daybrush/utils': 1.13.0
       '@scena/matrix': 1.1.1
 
+  css-what@6.1.0: {}
+
   cssesc@3.0.0: {}
 
   cssstyle@3.0.0:
@@ -14899,7 +14924,7 @@ snapshots:
 
   linkify-it@5.0.0:
     dependencies:
-      uc.micro: 2.0.0
+      uc.micro: 2.1.0
 
   linkifyjs@4.1.3: {}
 
@@ -14994,14 +15019,14 @@ snapshots:
     dependencies:
       tmpl: 1.0.5
 
-  markdown-it@14.0.0:
+  markdown-it@14.1.0:
     dependencies:
       argparse: 2.0.1
       entities: 4.5.0
       linkify-it: 5.0.0
       mdurl: 2.0.0
       punycode.js: 2.3.1
-      uc.micro: 2.0.0
+      uc.micro: 2.1.0
 
   marked@13.0.2: {}
 
@@ -15690,7 +15715,7 @@ snapshots:
 
   prosemirror-markdown@1.13.0:
     dependencies:
-      markdown-it: 14.0.0
+      markdown-it: 14.1.0
       prosemirror-model: 1.22.1
 
   prosemirror-menu@1.2.4:
@@ -16778,7 +16803,7 @@ snapshots:
 
   typescript@5.5.2: {}
 
-  uc.micro@2.0.0: {}
+  uc.micro@2.1.0: {}
 
   uid2@1.0.0: {}
 
@@ -17101,4 +17126,8 @@ snapshots:
 
   yocto-queue@0.1.0: {}
 
+  zeed-dom@0.10.11:
+    dependencies:
+      css-what: 6.1.0
+
   zod@3.23.8: {}