Ryan/docmost
1
0
Fork 0
mirror of https://github.com/docmost/docmost.git synced 2025-11-19 09:41:11 +10:00
Code Issues Packages Projects Releases Wiki Activity

feat: bulk page imports (#1219)

Browse Source 
* refactor imports - WIP

* Add readstream

* WIP

* fix attachmentId render

* fix attachmentId render

* turndown video tag

* feat: add stream upload support and improve file handling

- Add stream upload functionality to storage drivers\n- Improve ZIP file extraction with better encoding handling\n- Fix attachment ID rendering issues\n- Add AWS S3 upload stream support\n- Update dependencies for better compatibility

* WIP

* notion formatter

* move embed parser to editor-ext package

* import embeds

* utility files

* cleanup

* Switch from happy-dom to cheerio
* Refine code

* WIP

* bug fixes and UI

* sync

* WIP

* sync

* keep import modal mounted

* Show modal during upload

* WIP

* WIP
This commit is contained in:
Philip Okugbe
2025-06-09 04:29:27 +01:00
committed by GitHub
parent ce1503af85
commit 6d024fc3de
45 changed files with 2362 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

224
apps/client/src/features/page/components/page-import-modal.tsx
View File

@ -1,18 +1,38 @@
import { Modal, Button, SimpleGrid, FileButton } from "@mantine/core";
import {
Modal,
Button,
SimpleGrid,
FileButton,
Group,
Text,
Tooltip,
} from "@mantine/core";
import {
IconBrandNotion,
IconCheck,
IconFileCode,
IconFileTypeZip,
IconMarkdown,
IconX,
} from "@tabler/icons-react";
import { importPage } from "@/features/page/services/page-service.ts";
import {
importPage,
importZip,
} from "@/features/page/services/page-service.ts";
import { notifications } from "@mantine/notifications";
import { treeDataAtom } from "@/features/page/tree/atoms/tree-data-atom.ts";
import { useAtom } from "jotai";
import { buildTree } from "@/features/page/tree/utils";
import { IPage } from "@/features/page/types/page.types.ts";
import React from "react";
import React, { useEffect, useState } from "react";
import { useTranslation } from "react-i18next";
import { ConfluenceIcon } from "@/components/icons/confluence-icon.tsx";
import { getFileImportSizeLimit, isCloud } from "@/lib/config.ts";
import { formatBytes } from "@/lib";
import { workspaceAtom } from "@/features/user/atoms/current-user-atom.ts";
import { getFileTaskById } from "@/features/file-task/services/file-task-service.ts";
import { queryClient } from "@/main.tsx";
import { useQueryEmit } from "@/features/websocket/use-query-emit.ts";
interface PageImportModalProps {
spaceId: string;
@ -36,6 +56,7 @@ export default function PageImportModal({
yOffset="10vh"
xOffset={0}
mah={400}
keepMounted={true}
>
<Modal.Overlay />
<Modal.Content style={{ overflow: "hidden" }}>
@ -59,6 +80,133 @@ interface ImportFormatSelection {
function ImportFormatSelection({ spaceId, onClose }: ImportFormatSelection) {
const { t } = useTranslation();
const [treeData, setTreeData] = useAtom(treeDataAtom);
const [workspace] = useAtom(workspaceAtom);
const [fileTaskId, setFileTaskId] = useState<string | null>(null);
const emit = useQueryEmit();
const canUseConfluence = isCloud() || workspace?.hasLicenseKey;
const handleZipUpload = async (selectedFile: File, source: string) => {
if (!selectedFile) {
return;
}
try {
onClose();
notifications.show({
id: "import",
title: t("Uploading import file"),
message: t("Please don't close this tab."),
loading: true,
withCloseButton: false,
autoClose: false,
});
const importTask = await importZip(selectedFile, spaceId, source);
notifications.update({
id: "import",
title: t("Importing pages"),
message: t(
"Page import is in progress. You can check back later if this takes longer.",
),
loading: true,
withCloseButton: true,
autoClose: false,
});
setFileTaskId(importTask.id);
} catch (err) {
console.log("Failed to upload import file", err);
notifications.update({
id: "import",
color: "red",
title: t("Failed to upload import file"),
message: err?.response.data.message,
icon: <IconX size={18} />,
loading: false,
withCloseButton: true,
autoClose: false,
});
}
};
useEffect(() => {
if (!fileTaskId) return;
const intervalId = setInterval(async () => {
try {
const fileTask = await getFileTaskById(fileTaskId);
const status = fileTask.status;
if (status === "success") {
notifications.update({
id: "import",
color: "teal",
title: t("Import complete"),
message: t("Your pages were successfully imported."),
icon: <IconCheck size={18} />,
loading: false,
withCloseButton: true,
autoClose: false,
});
clearInterval(intervalId);
setFileTaskId(null);
await queryClient.refetchQueries({
queryKey: ["root-sidebar-pages", fileTask.spaceId],
});
setTimeout(() => {
emit({
operation: "refetchRootTreeNodeEvent",
spaceId: spaceId,
});
}, 50);
}
if (status === "failed") {
notifications.update({
id: "import",
color: "red",
title: t("Page import failed"),
message: t(
"Something went wrong while importing pages: {{reason}}.",
{
reason: fileTask.errorMessage,
},
),
icon: <IconX size={18} />,
loading: false,
withCloseButton: true,
autoClose: false,
});
clearInterval(intervalId);
setFileTaskId(null);
console.error(fileTask.errorMessage);
}
} catch (err) {
notifications.update({
id: "import",
color: "red",
title: t("Import failed"),
message: t(
"Something went wrong while importing pages: {{reason}}.",
{
reason: err.response?.data.message,
},
),
icon: <IconX size={18} />,
loading: false,
withCloseButton: true,
autoClose: false,
});
clearInterval(intervalId);
setFileTaskId(null);
console.error("Failed to fetch import status", err);
}
}, 3000);
}, [fileTaskId]);
const handleFileUpload = async (selectedFiles: File[]) => {
if (!selectedFiles) {
@ -120,6 +268,7 @@ function ImportFormatSelection({ spaceId, onClose }: ImportFormatSelection) {
}
};
// @ts-ignore
return (
<>
<SimpleGrid cols={2}>
@ -148,7 +297,76 @@ function ImportFormatSelection({ spaceId, onClose }: ImportFormatSelection) {
</Button>
)}
</FileButton>
<FileButton
onChange={(file) => handleZipUpload(file, "notion")}
accept="application/zip"
>
{(props) => (
<Button
justify="start"
variant="default"
leftSection={<IconBrandNotion size={18} />}
{...props}
>
Notion
</Button>
)}
</FileButton>
<FileButton
onChange={(file) => handleZipUpload(file, "confluence")}
accept="application/zip"
>
{(props) => (
<Tooltip
label="Available in enterprise edition"
disabled={canUseConfluence}
>
<Button
disabled={!canUseConfluence}
justify="start"
variant="default"
leftSection={<ConfluenceIcon size={18} />}
{...props}
>
Confluence
</Button>
</Tooltip>
)}
</FileButton>
</SimpleGrid>
<Group justify="center" gap="xl" mih={150}>
<div>
<Text ta="center" size="lg" inline>
Import zip file
</Text>
<Text ta="center" size="sm" c="dimmed" inline py="sm">
{t(
`Upload zip file containing Markdown and HTML files. Max: {{sizeLimit}}`,
{
sizeLimit: formatBytes(getFileImportSizeLimit()),
},
)}
</Text>
<FileButton
onChange={(file) => handleZipUpload(file, "generic")}
accept="application/zip"
>
{(props) => (
<Group justify="center">
<Button
justify="center"
leftSection={<IconFileTypeZip size={18} />}
{...props}
>
{t("Upload file")}
</Button>
</Group>
)}
</FileButton>
</div>
</Group>
</>
);
}

22
apps/client/src/features/page/services/page-service.ts
View File

@ -7,10 +7,11 @@ import {
IPage,
IPageInput,
SidebarPagesParams,
} from "@/features/page/types/page.types";
} from '@/features/page/types/page.types';
import { IAttachment, IPagination } from "@/lib/types.ts";
import { saveAs } from "file-saver";
import { InfiniteData } from "@tanstack/react-query";
import { IFileTask } from '@/features/file-task/types/file-task.types.ts';
export async function createPage(data: Partial<IPage>): Promise<IPage> {
const req = await api.post<IPage>("/pages/create", data);
@ -119,6 +120,25 @@ export async function importPage(file: File, spaceId: string) {
return req.data;
}
export async function importZip(
file: File,
spaceId: string,
source?: string,
): Promise<IFileTask> {
const formData = new FormData();
formData.append("spaceId", spaceId);
formData.append("source", source);
formData.append("file", file);
const req = await api.post<any>("/pages/import-zip", formData, {
headers: {
"Content-Type": "multipart/form-data",
},
});
return req.data;
}
export async function uploadFile(
file: File,
pageId: string,

54
apps/client/src/features/page/tree/components/space-tree.tsx
View File

@ -24,7 +24,10 @@ import {
IconPointFilled,
IconTrash,
} from "@tabler/icons-react";
import { appendNodeChildrenAtom, treeDataAtom } from "@/features/page/tree/atoms/tree-data-atom.ts";
import {
appendNodeChildrenAtom,
treeDataAtom,
} from "@/features/page/tree/atoms/tree-data-atom.ts";
import clsx from "clsx";
import EmojiPicker from "@/components/ui/emoji-picker.tsx";
import { useTreeMutation } from "@/features/page/tree/hooks/use-tree-mutation.ts";
@ -32,6 +35,7 @@ import {
appendNodeChildren,
buildTree,
buildTreeWithChildren,
mergeRootTrees,
updateTreeNodeIcon,
} from "@/features/page/tree/utils/utils.ts";
import { SpaceTreeNode } from "@/features/page/tree/types.ts";
@ -104,17 +108,17 @@ export default function SpaceTree({ spaceId, readOnly }: SpaceTreeProps) {
const allItems = pagesData.pages.flatMap((page) => page.items);
const treeData = buildTree(allItems);
if (data.length < 1 || data?.[0].spaceId !== spaceId) {
//Thoughts
// don't reset if there is data in state
// we only expect to call this once on initial load
// even if we decide to refetch, it should only update
// and append root pages instead of resetting the entire tree
// which looses async loaded children too
setData(treeData);
setIsDataLoaded(true);
setOpenTreeNodes({});
}
setData((prev) => {
// fresh space; full reset
if (prev.length === 0 || prev[0]?.spaceId !== spaceId) {
setIsDataLoaded(true);
setOpenTreeNodes({});
return treeData;
}
// same space; append only missing roots
return mergeRootTrees(prev, treeData);
});
}
}, [pagesData, hasNextPage]);
@ -297,17 +301,19 @@ function Node({ node, style, dragHandle, tree }: NodeRendererProps<any>) {
const handleEmojiSelect = (emoji: { native: string }) => {
handleUpdateNodeIcon(node.id, emoji.native);
updatePageMutation.mutateAsync({ pageId: node.id, icon: emoji.native }).then((data) => {
setTimeout(() => {
emit({
operation: "updateOne",
spaceId: node.data.spaceId,
entity: ["pages"],
id: node.id,
payload: { icon: emoji.native, parentPageId: data.parentPageId},
});
}, 50);
});
updatePageMutation
.mutateAsync({ pageId: node.id, icon: emoji.native })
.then((data) => {
setTimeout(() => {
emit({
operation: "updateOne",
spaceId: node.data.spaceId,
entity: ["pages"],
id: node.id,
payload: { icon: emoji.native, parentPageId: data.parentPageId },
});
}, 50);
});
};
const handleRemoveEmoji = () => {
@ -570,7 +576,7 @@ interface PageArrowProps {
function PageArrow({ node, onExpandTree }: PageArrowProps) {
useEffect(() => {
if(node.isOpen){
if (node.isOpen) {
onExpandTree();
}
}, []);

25
apps/client/src/features/page/tree/utils/utils.ts
View File

@ -121,7 +121,6 @@ export const deleteTreeNode = (
.filter((node) => node !== null);
};
export function buildTreeWithChildren(items: SpaceTreeNode[]): SpaceTreeNode[] {
const nodeMap = {};
let result: SpaceTreeNode[] = [];
@ -167,10 +166,12 @@ export function appendNodeChildren(
// Preserve deeper children if they exist and remove node if deleted
return treeItems.map((node) => {
if (node.id === nodeId) {
const newIds = new Set(children.map(c => c.id));
const newIds = new Set(children.map((c) => c.id));
const existingMap = new Map(
(node.children ?? []).filter(c => newIds.has(c.id)).map(c => [c.id, c])
(node.children ?? [])
.filter((c) => newIds.has(c.id))
.map((c) => [c.id, c]),
);
const merged = children.map((newChild) => {
@ -196,3 +197,21 @@ export function appendNodeChildren(
return node;
});
}
/**
* Merge root nodes; keep existing ones intact, append new ones,
*/
export function mergeRootTrees(
prevRoots: SpaceTreeNode[],
incomingRoots: SpaceTreeNode[],
): SpaceTreeNode[] {
const seen = new Set(prevRoots.map((r) => r.id));
// add new roots that were not present before
const merged = [...prevRoots];
incomingRoots.forEach((node) => {
if (!seen.has(node.id)) merged.push(node);
});
return sortPositionKeys(merged);
}