feat: bulk page imports (#1219)

* refactor imports - WIP

* Add readstream

* WIP

* fix attachmentId render

* fix attachmentId render

* turndown video tag

* feat: add stream upload support and improve file handling

- Add stream upload functionality to storage drivers\n- Improve ZIP file extraction with better encoding handling\n- Fix attachment ID rendering issues\n- Add AWS S3 upload stream support\n- Update dependencies for better compatibility

* WIP

* notion formatter

* move embed parser to editor-ext package

* import embeds

* utility files

* cleanup

* Switch from happy-dom to cheerio
* Refine code

* WIP

* bug fixes and UI

* sync

* WIP

* sync

* keep import modal mounted

* Show modal during upload

* WIP

* WIP

packages/editor-ext/src/index.ts

@@ -16,4 +16,5 @@ export * from "./lib/drawio";
 export * from "./lib/excalidraw";
 export * from "./lib/embed";
 export * from "./lib/mention";
-export * from "./lib/markdown";
+export * from "./lib/markdown";
+export * from "./lib/embed-provider";

packages/editor-ext/src/lib/embed-provider.ts

@@ -0,0 +1,134 @@
+export interface IEmbedProvider {
+  id: string;
+  name: string;
+  regex: RegExp;
+  getEmbedUrl: (match: RegExpMatchArray, url?: string) => string;
+}
+
+export const embedProviders: IEmbedProvider[] = [
+  {
+    id: "loom",
+    name: "Loom",
+    regex: /^https?:\/\/(?:www\.)?loom\.com\/(?:share|embed)\/([\da-zA-Z]+)\/?/,
+    getEmbedUrl: (match, url) => {
+      if (url.includes("/embed/")) {
+        return url;
+      }
+      return `https://loom.com/embed/${match[1]}`;
+    },
+  },
+  {
+    id: "airtable",
+    name: "Airtable",
+    regex: /^https:\/\/(www.)?airtable.com\/([a-zA-Z0-9]{2,})\/.*/,
+    getEmbedUrl: (match, url: string) => {
+      const path = url.split("airtable.com/");
+      if (url.includes("/embed/")) {
+        return url;
+      }
+      return `https://airtable.com/embed/${path[1]}`;
+    },
+  },
+  {
+    id: "figma",
+    name: "Figma",
+    regex:
+      /^https:\/\/[\w\.-]+\.?figma.com\/(file|proto|board|design|slides|deck)\/([0-9a-zA-Z]{22,128})/,
+    getEmbedUrl: (match, url: string) => {
+      return `https://www.figma.com/embed?url=${url}&embed_host=docmost`;
+    },
+  },
+  {
+    id: "typeform",
+    name: "Typeform",
+    regex: /^(https?:)?(\/\/)?[\w\.]+\.typeform\.com\/to\/.+/,
+    getEmbedUrl: (match, url: string) => {
+      return url;
+    },
+  },
+  {
+    id: "miro",
+    name: "Miro",
+    regex: /^https:\/\/(www\.)?miro\.com\/app\/board\/([\w-]+=)/,
+    getEmbedUrl: (match, url) => {
+      if (url.includes("/live-embed/")) {
+        return url;
+      }
+      return `https://miro.com/app/live-embed/${match[2]}?embedMode=view_only_without_ui&autoplay=true&embedSource=docmost`;
+    },
+  },
+  {
+    id: "youtube",
+    name: "YouTube",
+    regex:
+      /^((?:https?:)?\/\/)?((?:www|m|music)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w\-]+\?v=|embed\/|v\/)?)([\w\-]+)(\S+)?$/,
+    getEmbedUrl: (match, url) => {
+      if (url.includes("/embed/")) {
+        return url;
+      }
+      return `https://www.youtube-nocookie.com/embed/${match[5]}`;
+    },
+  },
+  {
+    id: "vimeo",
+    name: "Vimeo",
+    regex:
+      /^(https:)?\/\/(?:www\.|player\.)?vimeo.com\/(?:channels\/(?:\w+\/)?|groups\/([^/]*)\/videos\/|album\/(\d+)\/video\/|video\/|)(\d+)/,
+    getEmbedUrl: (match) => {
+      return `https://player.vimeo.com/video/${match[4]}`;
+    },
+  },
+  {
+    id: "framer",
+    name: "Framer",
+    regex: /^https:\/\/(www\.)?framer\.com\/embed\/([\w-]+)/,
+    getEmbedUrl: (match, url: string) => {
+      return url;
+    },
+  },
+  {
+    id: "gdrive",
+    name: "Google Drive",
+    regex:
+      /^((?:https?:)?\/\/)?((?:www|m)\.)?(drive\.google\.com)\/file\/d\/([a-zA-Z0-9_-]+)\/.*$/,
+    getEmbedUrl: (match) => {
+      return `https://drive.google.com/file/d/${match[4]}/preview`;
+    },
+  },
+  {
+    id: "gsheets",
+    name: "Google Sheets",
+    regex:
+      /^((?:https?:)?\/\/)?((?:www|m)\.)?(docs\.google\.com)\/spreadsheets\/d\/e\/([a-zA-Z0-9_-]+)\/.*$/,
+    getEmbedUrl: (match, url: string) => {
+      return url;
+    },
+  },
+];
+
+export function getEmbedProviderById(id: string) {
+  return embedProviders.find(
+    (provider) => provider.id.toLowerCase() === id.toLowerCase(),
+  );
+}
+
+export interface IEmbedResult {
+  embedUrl: string;
+  provider: string;
+}
+
+export function getEmbedUrlAndProvider(url: string): IEmbedResult {
+  for (const provider of embedProviders) {
+    const match = url.match(provider.regex);
+    if (match) {
+      return {
+        embedUrl: provider.getEmbedUrl(match, url),
+        provider: provider.name.toLowerCase(),
+      };
+    }
+  }
+  return {
+    embedUrl: url,
+    provider: "iframe",
+  };
+}

packages/editor-ext/src/lib/image/image.ts

@@ -80,7 +80,7 @@ export const TiptapImage = Image.extend<ImageOptions>({
         default: undefined,
         parseHTML: (element) => element.getAttribute("data-attachment-id"),
         renderHTML: (attributes: ImageAttributes) => ({
-          "data-attachment-id": attributes.align,
+          "data-attachment-id": attributes.attachmentId,
         }),
       },
       size: {

packages/editor-ext/src/lib/trailing-node.ts

@@ -51,8 +51,13 @@ export const TrailingNode = Extension.create<TrailingNodeExtensionOptions>({
         },
         state: {
           init: (_, state) => {
-            const lastNode = state.tr.doc.lastChild
-            return !nodeEqualsType({ node: lastNode, types: disabledNodes })
+            try {
+              const lastNode = state.tr.doc.lastChild
+              return !nodeEqualsType({ node: lastNode, types: disabledNodes })
+            } catch (err){
+              console.log(err)
+            }
+            return true;
           },
           apply: (tr, value) => {
             if (!tr.docChanged) {

packages/editor-ext/src/lib/video/video.ts

@@ -57,7 +57,7 @@ export const TiptapVideo = Node.create<VideoOptions>({
         default: undefined,
         parseHTML: (element) => element.getAttribute("data-attachment-id"),
         renderHTML: (attributes: VideoAttributes) => ({
-          "data-attachment-id": attributes.align,
+          "data-attachment-id": attributes.attachmentId,
         }),
       },
       width: {
@@ -84,6 +84,14 @@ export const TiptapVideo = Node.create<VideoOptions>({
     };
   },
 
+  parseHTML() {
+    return [
+      {
+        tag: 'video',
+      },
+    ]
+  },
+
   renderHTML({ HTMLAttributes }) {
     return [
       "video",