feat: bulk page imports (#1219)

* refactor imports - WIP

* Add readstream

* WIP

* fix attachmentId render

* fix attachmentId render

* turndown video tag

* feat: add stream upload support and improve file handling

- Add stream upload functionality to storage drivers\n- Improve ZIP file extraction with better encoding handling\n- Fix attachment ID rendering issues\n- Add AWS S3 upload stream support\n- Update dependencies for better compatibility

* WIP

* notion formatter

* move embed parser to editor-ext package

* import embeds

* utility files

* cleanup

* Switch from happy-dom to cheerio
* Refine code

* WIP

* bug fixes and UI

* sync

* WIP

* sync

* keep import modal mounted

* Show modal during upload

* WIP

* WIP
This commit is contained in:
Philip Okugbe
2025-06-09 04:29:27 +01:00
committed by GitHub
parent ce1503af85
commit 6d024fc3de
45 changed files with 2362 additions and 149 deletions

View File

@ -0,0 +1,20 @@
import { rem } from "@mantine/core";
interface Props {
size?: number | string;
}
export function ConfluenceIcon({ size }: Props) {
return (
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
strokeWidth="2"
style={{ width: rem(size), height: rem(size) }}
>
<path d="M.87 18.257c-.248.382-.53.875-.763 1.245a.764.764 0 0 0 .255 1.04l4.965 3.054a.764.764 0 0 0 1.058-.26c.199-.332.454-.763.733-1.221 1.967-3.247 3.945-2.853 7.508-1.146l4.957 2.337a.764.764 0 0 0 1.028-.382l2.364-5.346a.764.764 0 0 0-.382-1 599.851 599.851 0 0 1-4.965-2.361C10.911 10.97 5.224 11.185.87 18.257zM23.131 5.743c.249-.405.531-.875.764-1.25a.764.764 0 0 0-.256-1.034L18.675.404a.764.764 0 0 0-1.058.26c-.195.335-.451.763-.734 1.225-1.966 3.246-3.945 2.85-7.508 1.146L4.437.694a.764.764 0 0 0-1.027.382L1.046 6.422a.764.764 0 0 0 .382 1c1.039.49 3.105 1.467 4.965 2.361 6.698 3.246 12.392 3.029 16.738-4.04z" />
</svg>
);
}

View File

@ -15,13 +15,10 @@ import {
import { IconEdit } from "@tabler/icons-react";
import { z } from "zod";
import { useForm, zodResolver } from "@mantine/form";
import {
getEmbedProviderById,
getEmbedUrlAndProvider,
} from "@/features/editor/components/embed/providers.ts";
import { notifications } from "@mantine/notifications";
import { useTranslation } from "react-i18next";
import i18n from "i18next";
import { getEmbedProviderById, getEmbedUrlAndProvider } from '@docmost/editor-ext';
const schema = z.object({
url: z
@ -101,7 +98,7 @@ export default function EmbedView(props: NodeViewProps) {
<Text component="span" size="lg" c="dimmed">
{t("Embed {{provider}}", {
provider: getEmbedProviderById(provider).name,
provider: getEmbedProviderById(provider)?.name,
})}
</Text>
</div>

View File

@ -1,129 +0,0 @@
export interface IEmbedProvider {
id: string;
name: string;
regex: RegExp;
getEmbedUrl: (match: RegExpMatchArray, url?: string) => string;
}
export const embedProviders: IEmbedProvider[] = [
{
id: 'loom',
name: 'Loom',
regex: /^https?:\/\/(?:www\.)?loom\.com\/(?:share|embed)\/([\da-zA-Z]+)\/?/,
getEmbedUrl: (match, url) => {
if(url.includes("/embed/")){
return url;
}
return `https://loom.com/embed/${match[1]}`;
}
},
{
id: 'airtable',
name: 'Airtable',
regex: /^https:\/\/(www.)?airtable.com\/([a-zA-Z0-9]{2,})\/.*/,
getEmbedUrl: (match, url: string) => {
const path = url.split('airtable.com/');
if(url.includes("/embed/")){
return url;
}
return `https://airtable.com/embed/${path[1]}`;
}
},
{
id: 'figma',
name: 'Figma',
regex: /^https:\/\/[\w\.-]+\.?figma.com\/(file|proto|board|design|slides|deck)\/([0-9a-zA-Z]{22,128})/,
getEmbedUrl: (match, url: string) => {
return `https://www.figma.com/embed?url=${url}&embed_host=docmost`;
}
},
{
'id': 'typeform',
name: 'Typeform',
regex: /^(https?:)?(\/\/)?[\w\.]+\.typeform\.com\/to\/.+/,
getEmbedUrl: (match, url: string) => {
return url;
}
},
{
id: 'miro',
name: 'Miro',
regex: /^https:\/\/(www\.)?miro\.com\/app\/board\/([\w-]+=)/,
getEmbedUrl: (match, url) => {
if(url.includes("/live-embed/")){
return url;
}
return `https://miro.com/app/live-embed/${match[2]}?embedMode=view_only_without_ui&autoplay=true&embedSource=docmost`;
}
},
{
id: 'youtube',
name: 'YouTube',
regex: /^((?:https?:)?\/\/)?((?:www|m|music)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w\-]+\?v=|embed\/|v\/)?)([\w\-]+)(\S+)?$/,
getEmbedUrl: (match, url) => {
if (url.includes("/embed/")){
return url;
}
return `https://www.youtube-nocookie.com/embed/${match[5]}`;
}
},
{
id: 'vimeo',
name: 'Vimeo',
regex: /^(https:)?\/\/(?:www\.|player\.)?vimeo.com\/(?:channels\/(?:\w+\/)?|groups\/([^/]*)\/videos\/|album\/(\d+)\/video\/|video\/|)(\d+)/,
getEmbedUrl: (match) => {
return `https://player.vimeo.com/video/${match[4]}`;
}
},
{
id: 'framer',
name: 'Framer',
regex: /^https:\/\/(www\.)?framer\.com\/embed\/([\w-]+)/,
getEmbedUrl: (match, url: string) => {
return url;
}
},
{
id: 'gdrive',
name: 'Google Drive',
regex: /^((?:https?:)?\/\/)?((?:www|m)\.)?(drive\.google\.com)\/file\/d\/([a-zA-Z0-9_-]+)\/.*$/,
getEmbedUrl: (match) => {
return `https://drive.google.com/file/d/${match[4]}/preview`;
}
},
{
id: 'gsheets',
name: 'Google Sheets',
regex: /^((?:https?:)?\/\/)?((?:www|m)\.)?(docs\.google\.com)\/spreadsheets\/d\/e\/([a-zA-Z0-9_-]+)\/.*$/,
getEmbedUrl: (match, url: string) => {
return url
}
},
];
export function getEmbedProviderById(id: string) {
return embedProviders.find(provider => provider.id.toLowerCase() === id.toLowerCase());
}
export interface IEmbedResult {
embedUrl: string;
provider: string;
}
export function getEmbedUrlAndProvider(url: string): IEmbedResult {
for (const provider of embedProviders) {
const match = url.match(provider.regex);
if (match) {
return {
embedUrl: provider.getEmbedUrl(match, url),
provider: provider.name.toLowerCase()
};
}
}
return {
embedUrl: url,
provider: 'iframe',
};
}

View File

@ -0,0 +1,14 @@
import api from "@/lib/api-client";
import { IFileTask } from "@/features/file-task/types/file-task.types.ts";
export async function getFileTaskById(fileTaskId: string): Promise<IFileTask> {
const req = await api.post<IFileTask>("/file-tasks/info", {
fileTaskId: fileTaskId,
});
return req.data;
}
export async function getFileTasks(): Promise<IFileTask[]> {
const req = await api.post<IFileTask[]>("/file-tasks");
return req.data;
}

View File

@ -0,0 +1,17 @@
export interface IFileTask {
id: string;
type: "import" | "export";
source: string;
status: string;
fileName: string;
filePath: string;
fileSize: number;
fileExt: string;
errorMessage: string | null;
creatorId: string;
spaceId: string;
workspaceId: string;
createdAt: string;
updatedAt: string;
deletedAt: string | null;
}

View File

@ -1,18 +1,38 @@
import { Modal, Button, SimpleGrid, FileButton } from "@mantine/core";
import {
Modal,
Button,
SimpleGrid,
FileButton,
Group,
Text,
Tooltip,
} from "@mantine/core";
import {
IconBrandNotion,
IconCheck,
IconFileCode,
IconFileTypeZip,
IconMarkdown,
IconX,
} from "@tabler/icons-react";
import { importPage } from "@/features/page/services/page-service.ts";
import {
importPage,
importZip,
} from "@/features/page/services/page-service.ts";
import { notifications } from "@mantine/notifications";
import { treeDataAtom } from "@/features/page/tree/atoms/tree-data-atom.ts";
import { useAtom } from "jotai";
import { buildTree } from "@/features/page/tree/utils";
import { IPage } from "@/features/page/types/page.types.ts";
import React from "react";
import React, { useEffect, useState } from "react";
import { useTranslation } from "react-i18next";
import { ConfluenceIcon } from "@/components/icons/confluence-icon.tsx";
import { getFileImportSizeLimit, isCloud } from "@/lib/config.ts";
import { formatBytes } from "@/lib";
import { workspaceAtom } from "@/features/user/atoms/current-user-atom.ts";
import { getFileTaskById } from "@/features/file-task/services/file-task-service.ts";
import { queryClient } from "@/main.tsx";
import { useQueryEmit } from "@/features/websocket/use-query-emit.ts";
interface PageImportModalProps {
spaceId: string;
@ -36,6 +56,7 @@ export default function PageImportModal({
yOffset="10vh"
xOffset={0}
mah={400}
keepMounted={true}
>
<Modal.Overlay />
<Modal.Content style={{ overflow: "hidden" }}>
@ -59,6 +80,133 @@ interface ImportFormatSelection {
function ImportFormatSelection({ spaceId, onClose }: ImportFormatSelection) {
const { t } = useTranslation();
const [treeData, setTreeData] = useAtom(treeDataAtom);
const [workspace] = useAtom(workspaceAtom);
const [fileTaskId, setFileTaskId] = useState<string | null>(null);
const emit = useQueryEmit();
const canUseConfluence = isCloud() || workspace?.hasLicenseKey;
const handleZipUpload = async (selectedFile: File, source: string) => {
if (!selectedFile) {
return;
}
try {
onClose();
notifications.show({
id: "import",
title: t("Uploading import file"),
message: t("Please don't close this tab."),
loading: true,
withCloseButton: false,
autoClose: false,
});
const importTask = await importZip(selectedFile, spaceId, source);
notifications.update({
id: "import",
title: t("Importing pages"),
message: t(
"Page import is in progress. You can check back later if this takes longer.",
),
loading: true,
withCloseButton: true,
autoClose: false,
});
setFileTaskId(importTask.id);
} catch (err) {
console.log("Failed to upload import file", err);
notifications.update({
id: "import",
color: "red",
title: t("Failed to upload import file"),
message: err?.response.data.message,
icon: <IconX size={18} />,
loading: false,
withCloseButton: true,
autoClose: false,
});
}
};
useEffect(() => {
if (!fileTaskId) return;
const intervalId = setInterval(async () => {
try {
const fileTask = await getFileTaskById(fileTaskId);
const status = fileTask.status;
if (status === "success") {
notifications.update({
id: "import",
color: "teal",
title: t("Import complete"),
message: t("Your pages were successfully imported."),
icon: <IconCheck size={18} />,
loading: false,
withCloseButton: true,
autoClose: false,
});
clearInterval(intervalId);
setFileTaskId(null);
await queryClient.refetchQueries({
queryKey: ["root-sidebar-pages", fileTask.spaceId],
});
setTimeout(() => {
emit({
operation: "refetchRootTreeNodeEvent",
spaceId: spaceId,
});
}, 50);
}
if (status === "failed") {
notifications.update({
id: "import",
color: "red",
title: t("Page import failed"),
message: t(
"Something went wrong while importing pages: {{reason}}.",
{
reason: fileTask.errorMessage,
},
),
icon: <IconX size={18} />,
loading: false,
withCloseButton: true,
autoClose: false,
});
clearInterval(intervalId);
setFileTaskId(null);
console.error(fileTask.errorMessage);
}
} catch (err) {
notifications.update({
id: "import",
color: "red",
title: t("Import failed"),
message: t(
"Something went wrong while importing pages: {{reason}}.",
{
reason: err.response?.data.message,
},
),
icon: <IconX size={18} />,
loading: false,
withCloseButton: true,
autoClose: false,
});
clearInterval(intervalId);
setFileTaskId(null);
console.error("Failed to fetch import status", err);
}
}, 3000);
}, [fileTaskId]);
const handleFileUpload = async (selectedFiles: File[]) => {
if (!selectedFiles) {
@ -120,6 +268,7 @@ function ImportFormatSelection({ spaceId, onClose }: ImportFormatSelection) {
}
};
// @ts-ignore
return (
<>
<SimpleGrid cols={2}>
@ -148,7 +297,76 @@ function ImportFormatSelection({ spaceId, onClose }: ImportFormatSelection) {
</Button>
)}
</FileButton>
<FileButton
onChange={(file) => handleZipUpload(file, "notion")}
accept="application/zip"
>
{(props) => (
<Button
justify="start"
variant="default"
leftSection={<IconBrandNotion size={18} />}
{...props}
>
Notion
</Button>
)}
</FileButton>
<FileButton
onChange={(file) => handleZipUpload(file, "confluence")}
accept="application/zip"
>
{(props) => (
<Tooltip
label="Available in enterprise edition"
disabled={canUseConfluence}
>
<Button
disabled={!canUseConfluence}
justify="start"
variant="default"
leftSection={<ConfluenceIcon size={18} />}
{...props}
>
Confluence
</Button>
</Tooltip>
)}
</FileButton>
</SimpleGrid>
<Group justify="center" gap="xl" mih={150}>
<div>
<Text ta="center" size="lg" inline>
Import zip file
</Text>
<Text ta="center" size="sm" c="dimmed" inline py="sm">
{t(
`Upload zip file containing Markdown and HTML files. Max: {{sizeLimit}}`,
{
sizeLimit: formatBytes(getFileImportSizeLimit()),
},
)}
</Text>
<FileButton
onChange={(file) => handleZipUpload(file, "generic")}
accept="application/zip"
>
{(props) => (
<Group justify="center">
<Button
justify="center"
leftSection={<IconFileTypeZip size={18} />}
{...props}
>
{t("Upload file")}
</Button>
</Group>
)}
</FileButton>
</div>
</Group>
</>
);
}

View File

@ -7,10 +7,11 @@ import {
IPage,
IPageInput,
SidebarPagesParams,
} from "@/features/page/types/page.types";
} from '@/features/page/types/page.types';
import { IAttachment, IPagination } from "@/lib/types.ts";
import { saveAs } from "file-saver";
import { InfiniteData } from "@tanstack/react-query";
import { IFileTask } from '@/features/file-task/types/file-task.types.ts';
export async function createPage(data: Partial<IPage>): Promise<IPage> {
const req = await api.post<IPage>("/pages/create", data);
@ -119,6 +120,25 @@ export async function importPage(file: File, spaceId: string) {
return req.data;
}
export async function importZip(
file: File,
spaceId: string,
source?: string,
): Promise<IFileTask> {
const formData = new FormData();
formData.append("spaceId", spaceId);
formData.append("source", source);
formData.append("file", file);
const req = await api.post<any>("/pages/import-zip", formData, {
headers: {
"Content-Type": "multipart/form-data",
},
});
return req.data;
}
export async function uploadFile(
file: File,
pageId: string,

View File

@ -24,7 +24,10 @@ import {
IconPointFilled,
IconTrash,
} from "@tabler/icons-react";
import { appendNodeChildrenAtom, treeDataAtom } from "@/features/page/tree/atoms/tree-data-atom.ts";
import {
appendNodeChildrenAtom,
treeDataAtom,
} from "@/features/page/tree/atoms/tree-data-atom.ts";
import clsx from "clsx";
import EmojiPicker from "@/components/ui/emoji-picker.tsx";
import { useTreeMutation } from "@/features/page/tree/hooks/use-tree-mutation.ts";
@ -32,6 +35,7 @@ import {
appendNodeChildren,
buildTree,
buildTreeWithChildren,
mergeRootTrees,
updateTreeNodeIcon,
} from "@/features/page/tree/utils/utils.ts";
import { SpaceTreeNode } from "@/features/page/tree/types.ts";
@ -104,17 +108,17 @@ export default function SpaceTree({ spaceId, readOnly }: SpaceTreeProps) {
const allItems = pagesData.pages.flatMap((page) => page.items);
const treeData = buildTree(allItems);
if (data.length < 1 || data?.[0].spaceId !== spaceId) {
//Thoughts
// don't reset if there is data in state
// we only expect to call this once on initial load
// even if we decide to refetch, it should only update
// and append root pages instead of resetting the entire tree
// which looses async loaded children too
setData(treeData);
setIsDataLoaded(true);
setOpenTreeNodes({});
}
setData((prev) => {
// fresh space; full reset
if (prev.length === 0 || prev[0]?.spaceId !== spaceId) {
setIsDataLoaded(true);
setOpenTreeNodes({});
return treeData;
}
// same space; append only missing roots
return mergeRootTrees(prev, treeData);
});
}
}, [pagesData, hasNextPage]);
@ -297,17 +301,19 @@ function Node({ node, style, dragHandle, tree }: NodeRendererProps<any>) {
const handleEmojiSelect = (emoji: { native: string }) => {
handleUpdateNodeIcon(node.id, emoji.native);
updatePageMutation.mutateAsync({ pageId: node.id, icon: emoji.native }).then((data) => {
setTimeout(() => {
emit({
operation: "updateOne",
spaceId: node.data.spaceId,
entity: ["pages"],
id: node.id,
payload: { icon: emoji.native, parentPageId: data.parentPageId},
});
}, 50);
});
updatePageMutation
.mutateAsync({ pageId: node.id, icon: emoji.native })
.then((data) => {
setTimeout(() => {
emit({
operation: "updateOne",
spaceId: node.data.spaceId,
entity: ["pages"],
id: node.id,
payload: { icon: emoji.native, parentPageId: data.parentPageId },
});
}, 50);
});
};
const handleRemoveEmoji = () => {
@ -570,7 +576,7 @@ interface PageArrowProps {
function PageArrow({ node, onExpandTree }: PageArrowProps) {
useEffect(() => {
if(node.isOpen){
if (node.isOpen) {
onExpandTree();
}
}, []);

View File

@ -121,7 +121,6 @@ export const deleteTreeNode = (
.filter((node) => node !== null);
};
export function buildTreeWithChildren(items: SpaceTreeNode[]): SpaceTreeNode[] {
const nodeMap = {};
let result: SpaceTreeNode[] = [];
@ -167,10 +166,12 @@ export function appendNodeChildren(
// Preserve deeper children if they exist and remove node if deleted
return treeItems.map((node) => {
if (node.id === nodeId) {
const newIds = new Set(children.map(c => c.id));
const newIds = new Set(children.map((c) => c.id));
const existingMap = new Map(
(node.children ?? []).filter(c => newIds.has(c.id)).map(c => [c.id, c])
(node.children ?? [])
.filter((c) => newIds.has(c.id))
.map((c) => [c.id, c]),
);
const merged = children.map((newChild) => {
@ -196,3 +197,21 @@ export function appendNodeChildren(
return node;
});
}
/**
* Merge root nodes; keep existing ones intact, append new ones,
*/
export function mergeRootTrees(
prevRoots: SpaceTreeNode[],
incomingRoots: SpaceTreeNode[],
): SpaceTreeNode[] {
const seen = new Set(prevRoots.map((r) => r.id));
// add new roots that were not present before
const merged = [...prevRoots];
incomingRoots.forEach((node) => {
if (!seen.has(node.id)) merged.push(node);
});
return sortPositionKeys(merged);
}

View File

@ -47,15 +47,28 @@ export type MoveTreeNodeEvent = {
parentId: string;
index: number;
position: string;
}
};
};
export type DeleteTreeNodeEvent = {
operation: "deleteTreeNode";
spaceId: string;
payload: {
node: SpaceTreeNode
}
node: SpaceTreeNode;
};
};
export type WebSocketEvent = InvalidateEvent | InvalidateCommentsEvent | UpdateEvent | DeleteEvent | AddTreeNodeEvent | MoveTreeNodeEvent | DeleteTreeNodeEvent;
export type RefetchRootTreeNodeEvent = {
operation: "refetchRootTreeNodeEvent";
spaceId: string;
};
export type WebSocketEvent =
| InvalidateEvent
| InvalidateCommentsEvent
| UpdateEvent
| DeleteEvent
| AddTreeNodeEvent
| MoveTreeNodeEvent
| DeleteTreeNodeEvent
| RefetchRootTreeNodeEvent;

View File

@ -5,8 +5,14 @@ import { InfiniteData, useQueryClient } from "@tanstack/react-query";
import { WebSocketEvent } from "@/features/websocket/types";
import { IPage } from "../page/types/page.types";
import { IPagination } from "@/lib/types";
import { invalidateOnCreatePage, invalidateOnDeletePage, invalidateOnMovePage, invalidateOnUpdatePage } from "../page/queries/page-query";
import {
invalidateOnCreatePage,
invalidateOnDeletePage,
invalidateOnMovePage,
invalidateOnUpdatePage,
} from "../page/queries/page-query";
import { RQ_KEY } from "../comment/queries/comment-query";
import { queryClient } from "@/main.tsx";
export const useQuerySubscription = () => {
const queryClient = useQueryClient();
@ -37,8 +43,7 @@ export const useQuerySubscription = () => {
invalidateOnMovePage();
break;
case "deleteTreeNode":
const pageId = data.payload.node.id;
invalidateOnDeletePage(pageId);
invalidateOnDeletePage(data.payload.node.id);
break;
case "updateOne":
entity = data.entity[0];
@ -50,17 +55,23 @@ export const useQuerySubscription = () => {
}
// only update if data was already in cache
if(queryClient.getQueryData([...data.entity, queryKeyId])){
if (queryClient.getQueryData([...data.entity, queryKeyId])) {
queryClient.setQueryData([...data.entity, queryKeyId], {
...queryClient.getQueryData([...data.entity, queryKeyId]),
...data.payload,
});
}
if (entity === "pages") {
invalidateOnUpdatePage(data.spaceId, data.payload.parentPageId, data.id, data.payload.title, data.payload.icon);
invalidateOnUpdatePage(
data.spaceId,
data.payload.parentPageId,
data.id,
data.payload.title,
data.payload.icon,
);
}
/*
queryClient.setQueriesData(
{ queryKey: [data.entity, data.id] },
@ -72,8 +83,19 @@ export const useQuerySubscription = () => {
: update(oldData as Record<string, unknown>);
},
);
*/
*/
break;
case "refetchRootTreeNodeEvent": {
const spaceId = data.spaceId;
queryClient.refetchQueries({
queryKey: ["root-sidebar-pages", spaceId],
});
queryClient.invalidateQueries({
queryKey: ["recent-changes", spaceId],
});
break;
}
}
});
}, [queryClient, socket]);

View File

@ -70,6 +70,11 @@ export function getFileUploadSizeLimit() {
return bytes(limit);
}
export function getFileImportSizeLimit() {
const limit = getConfigValue("FILE_IMPORT_SIZE_LIMIT", "200mb");
return bytes(limit);
}
export function getDrawioUrl() {
return getConfigValue("DRAWIO_URL", "https://embed.diagrams.net");
}

View File

@ -8,6 +8,7 @@ export default defineConfig(({ mode }) => {
const {
APP_URL,
FILE_UPLOAD_SIZE_LIMIT,
FILE_IMPORT_SIZE_LIMIT,
DRAWIO_URL,
CLOUD,
SUBDOMAIN_HOST,
@ -20,6 +21,7 @@ export default defineConfig(({ mode }) => {
"process.env": {
APP_URL,
FILE_UPLOAD_SIZE_LIMIT,
FILE_IMPORT_SIZE_LIMIT,
DRAWIO_URL,
CLOUD,
SUBDOMAIN_HOST,