mirror of
https://github.com/docmost/docmost.git
synced 2025-11-10 06:52:07 +10:00
feat: bulk page imports (#1219)
* refactor imports - WIP * Add readstream * WIP * fix attachmentId render * fix attachmentId render * turndown video tag * feat: add stream upload support and improve file handling - Add stream upload functionality to storage drivers\n- Improve ZIP file extraction with better encoding handling\n- Fix attachment ID rendering issues\n- Add AWS S3 upload stream support\n- Update dependencies for better compatibility * WIP * notion formatter * move embed parser to editor-ext package * import embeds * utility files * cleanup * Switch from happy-dom to cheerio * Refine code * WIP * bug fixes and UI * sync * WIP * sync * keep import modal mounted * Show modal during upload * WIP * WIP
This commit is contained in:
Philip Okugbe
GitHub
20
apps/client/src/components/icons/confluence-icon.tsx
Normal file
20
apps/client/src/components/icons/confluence-icon.tsx
Normal file
@ -0,0 +1,20 @@
|
||||
import { rem } from "@mantine/core";
|
||||
|
||||
interface Props {
|
||||
size?: number | string;
|
||||
}
|
||||
|
||||
export function ConfluenceIcon({ size }: Props) {
|
||||
return (
|
||||
<svg
|
||||
xmlns="http://www.w3.org/2000/svg"
|
||||
viewBox="0 0 24 24"
|
||||
fill="none"
|
||||
stroke="currentColor"
|
||||
strokeWidth="2"
|
||||
style={{ width: rem(size), height: rem(size) }}
|
||||
>
|
||||
<path d="M.87 18.257c-.248.382-.53.875-.763 1.245a.764.764 0 0 0 .255 1.04l4.965 3.054a.764.764 0 0 0 1.058-.26c.199-.332.454-.763.733-1.221 1.967-3.247 3.945-2.853 7.508-1.146l4.957 2.337a.764.764 0 0 0 1.028-.382l2.364-5.346a.764.764 0 0 0-.382-1 599.851 599.851 0 0 1-4.965-2.361C10.911 10.97 5.224 11.185.87 18.257zM23.131 5.743c.249-.405.531-.875.764-1.25a.764.764 0 0 0-.256-1.034L18.675.404a.764.764 0 0 0-1.058.26c-.195.335-.451.763-.734 1.225-1.966 3.246-3.945 2.85-7.508 1.146L4.437.694a.764.764 0 0 0-1.027.382L1.046 6.422a.764.764 0 0 0 .382 1c1.039.49 3.105 1.467 4.965 2.361 6.698 3.246 12.392 3.029 16.738-4.04z" />
|
||||
</svg>
|
||||
);
|
||||
}
|
||||
@ -15,13 +15,10 @@ import {
|
||||
import { IconEdit } from "@tabler/icons-react";
|
||||
import { z } from "zod";
|
||||
import { useForm, zodResolver } from "@mantine/form";
|
||||
import {
|
||||
getEmbedProviderById,
|
||||
getEmbedUrlAndProvider,
|
||||
} from "@/features/editor/components/embed/providers.ts";
|
||||
import { notifications } from "@mantine/notifications";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import i18n from "i18next";
|
||||
import { getEmbedProviderById, getEmbedUrlAndProvider } from '@docmost/editor-ext';
|
||||
|
||||
const schema = z.object({
|
||||
url: z
|
||||
@ -101,7 +98,7 @@ export default function EmbedView(props: NodeViewProps) {
|
||||
|
||||
<Text component="span" size="lg" c="dimmed">
|
||||
{t("Embed {{provider}}", {
|
||||
provider: getEmbedProviderById(provider).name,
|
||||
provider: getEmbedProviderById(provider)?.name,
|
||||
})}
|
||||
</Text>
|
||||
</div>
|
||||
|
||||
@ -0,0 +1,14 @@
|
||||
import api from "@/lib/api-client";
|
||||
import { IFileTask } from "@/features/file-task/types/file-task.types.ts";
|
||||
|
||||
export async function getFileTaskById(fileTaskId: string): Promise<IFileTask> {
|
||||
const req = await api.post<IFileTask>("/file-tasks/info", {
|
||||
fileTaskId: fileTaskId,
|
||||
});
|
||||
return req.data;
|
||||
}
|
||||
|
||||
export async function getFileTasks(): Promise<IFileTask[]> {
|
||||
const req = await api.post<IFileTask[]>("/file-tasks");
|
||||
return req.data;
|
||||
}
|
||||
17
apps/client/src/features/file-task/types/file-task.types.ts
Normal file
17
apps/client/src/features/file-task/types/file-task.types.ts
Normal file
@ -0,0 +1,17 @@
|
||||
export interface IFileTask {
|
||||
id: string;
|
||||
type: "import" | "export";
|
||||
source: string;
|
||||
status: string;
|
||||
fileName: string;
|
||||
filePath: string;
|
||||
fileSize: number;
|
||||
fileExt: string;
|
||||
errorMessage: string | null;
|
||||
creatorId: string;
|
||||
spaceId: string;
|
||||
workspaceId: string;
|
||||
createdAt: string;
|
||||
updatedAt: string;
|
||||
deletedAt: string | null;
|
||||
}
|
||||
@ -1,18 +1,38 @@
|
||||
import { Modal, Button, SimpleGrid, FileButton } from "@mantine/core";
|
||||
import {
|
||||
Modal,
|
||||
Button,
|
||||
SimpleGrid,
|
||||
FileButton,
|
||||
Group,
|
||||
Text,
|
||||
Tooltip,
|
||||
} from "@mantine/core";
|
||||
import {
|
||||
IconBrandNotion,
|
||||
IconCheck,
|
||||
IconFileCode,
|
||||
IconFileTypeZip,
|
||||
IconMarkdown,
|
||||
IconX,
|
||||
} from "@tabler/icons-react";
|
||||
import { importPage } from "@/features/page/services/page-service.ts";
|
||||
import {
|
||||
importPage,
|
||||
importZip,
|
||||
} from "@/features/page/services/page-service.ts";
|
||||
import { notifications } from "@mantine/notifications";
|
||||
import { treeDataAtom } from "@/features/page/tree/atoms/tree-data-atom.ts";
|
||||
import { useAtom } from "jotai";
|
||||
import { buildTree } from "@/features/page/tree/utils";
|
||||
import { IPage } from "@/features/page/types/page.types.ts";
|
||||
import React from "react";
|
||||
import React, { useEffect, useState } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { ConfluenceIcon } from "@/components/icons/confluence-icon.tsx";
|
||||
import { getFileImportSizeLimit, isCloud } from "@/lib/config.ts";
|
||||
import { formatBytes } from "@/lib";
|
||||
import { workspaceAtom } from "@/features/user/atoms/current-user-atom.ts";
|
||||
import { getFileTaskById } from "@/features/file-task/services/file-task-service.ts";
|
||||
import { queryClient } from "@/main.tsx";
|
||||
import { useQueryEmit } from "@/features/websocket/use-query-emit.ts";
|
||||
|
||||
interface PageImportModalProps {
|
||||
spaceId: string;
|
||||
@ -36,6 +56,7 @@ export default function PageImportModal({
|
||||
yOffset="10vh"
|
||||
xOffset={0}
|
||||
mah={400}
|
||||
keepMounted={true}
|
||||
>
|
||||
<Modal.Overlay />
|
||||
<Modal.Content style={{ overflow: "hidden" }}>
|
||||
@ -59,6 +80,133 @@ interface ImportFormatSelection {
|
||||
function ImportFormatSelection({ spaceId, onClose }: ImportFormatSelection) {
|
||||
const { t } = useTranslation();
|
||||
const [treeData, setTreeData] = useAtom(treeDataAtom);
|
||||
const [workspace] = useAtom(workspaceAtom);
|
||||
const [fileTaskId, setFileTaskId] = useState<string | null>(null);
|
||||
const emit = useQueryEmit();
|
||||
|
||||
const canUseConfluence = isCloud() || workspace?.hasLicenseKey;
|
||||
|
||||
const handleZipUpload = async (selectedFile: File, source: string) => {
|
||||
if (!selectedFile) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
onClose();
|
||||
|
||||
notifications.show({
|
||||
id: "import",
|
||||
title: t("Uploading import file"),
|
||||
message: t("Please don't close this tab."),
|
||||
loading: true,
|
||||
withCloseButton: false,
|
||||
autoClose: false,
|
||||
});
|
||||
|
||||
const importTask = await importZip(selectedFile, spaceId, source);
|
||||
notifications.update({
|
||||
id: "import",
|
||||
title: t("Importing pages"),
|
||||
message: t(
|
||||
"Page import is in progress. You can check back later if this takes longer.",
|
||||
),
|
||||
loading: true,
|
||||
withCloseButton: true,
|
||||
autoClose: false,
|
||||
});
|
||||
|
||||
setFileTaskId(importTask.id);
|
||||
} catch (err) {
|
||||
console.log("Failed to upload import file", err);
|
||||
notifications.update({
|
||||
id: "import",
|
||||
color: "red",
|
||||
title: t("Failed to upload import file"),
|
||||
message: err?.response.data.message,
|
||||
icon: <IconX size={18} />,
|
||||
loading: false,
|
||||
withCloseButton: true,
|
||||
autoClose: false,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => {
|
||||
if (!fileTaskId) return;
|
||||
|
||||
const intervalId = setInterval(async () => {
|
||||
try {
|
||||
const fileTask = await getFileTaskById(fileTaskId);
|
||||
const status = fileTask.status;
|
||||
|
||||
if (status === "success") {
|
||||
notifications.update({
|
||||
id: "import",
|
||||
color: "teal",
|
||||
title: t("Import complete"),
|
||||
message: t("Your pages were successfully imported."),
|
||||
icon: <IconCheck size={18} />,
|
||||
loading: false,
|
||||
withCloseButton: true,
|
||||
autoClose: false,
|
||||
});
|
||||
clearInterval(intervalId);
|
||||
setFileTaskId(null);
|
||||
|
||||
await queryClient.refetchQueries({
|
||||
queryKey: ["root-sidebar-pages", fileTask.spaceId],
|
||||
});
|
||||
|
||||
setTimeout(() => {
|
||||
emit({
|
||||
operation: "refetchRootTreeNodeEvent",
|
||||
spaceId: spaceId,
|
||||
});
|
||||
}, 50);
|
||||
}
|
||||
|
||||
if (status === "failed") {
|
||||
notifications.update({
|
||||
id: "import",
|
||||
color: "red",
|
||||
title: t("Page import failed"),
|
||||
message: t(
|
||||
"Something went wrong while importing pages: {{reason}}.",
|
||||
{
|
||||
reason: fileTask.errorMessage,
|
||||
},
|
||||
),
|
||||
icon: <IconX size={18} />,
|
||||
loading: false,
|
||||
withCloseButton: true,
|
||||
autoClose: false,
|
||||
});
|
||||
clearInterval(intervalId);
|
||||
setFileTaskId(null);
|
||||
console.error(fileTask.errorMessage);
|
||||
}
|
||||
} catch (err) {
|
||||
notifications.update({
|
||||
id: "import",
|
||||
color: "red",
|
||||
title: t("Import failed"),
|
||||
message: t(
|
||||
"Something went wrong while importing pages: {{reason}}.",
|
||||
{
|
||||
reason: err.response?.data.message,
|
||||
},
|
||||
),
|
||||
icon: <IconX size={18} />,
|
||||
loading: false,
|
||||
withCloseButton: true,
|
||||
autoClose: false,
|
||||
});
|
||||
clearInterval(intervalId);
|
||||
setFileTaskId(null);
|
||||
console.error("Failed to fetch import status", err);
|
||||
}
|
||||
}, 3000);
|
||||
}, [fileTaskId]);
|
||||
|
||||
const handleFileUpload = async (selectedFiles: File[]) => {
|
||||
if (!selectedFiles) {
|
||||
@ -120,6 +268,7 @@ function ImportFormatSelection({ spaceId, onClose }: ImportFormatSelection) {
|
||||
}
|
||||
};
|
||||
|
||||
// @ts-ignore
|
||||
return (
|
||||
<>
|
||||
<SimpleGrid cols={2}>
|
||||
@ -148,7 +297,76 @@ function ImportFormatSelection({ spaceId, onClose }: ImportFormatSelection) {
|
||||
</Button>
|
||||
)}
|
||||
</FileButton>
|
||||
|
||||
<FileButton
|
||||
onChange={(file) => handleZipUpload(file, "notion")}
|
||||
accept="application/zip"
|
||||
>
|
||||
{(props) => (
|
||||
<Button
|
||||
justify="start"
|
||||
variant="default"
|
||||
leftSection={<IconBrandNotion size={18} />}
|
||||
{...props}
|
||||
>
|
||||
Notion
|
||||
</Button>
|
||||
)}
|
||||
</FileButton>
|
||||
<FileButton
|
||||
onChange={(file) => handleZipUpload(file, "confluence")}
|
||||
accept="application/zip"
|
||||
>
|
||||
{(props) => (
|
||||
<Tooltip
|
||||
label="Available in enterprise edition"
|
||||
disabled={canUseConfluence}
|
||||
>
|
||||
<Button
|
||||
disabled={!canUseConfluence}
|
||||
justify="start"
|
||||
variant="default"
|
||||
leftSection={<ConfluenceIcon size={18} />}
|
||||
{...props}
|
||||
>
|
||||
Confluence
|
||||
</Button>
|
||||
</Tooltip>
|
||||
)}
|
||||
</FileButton>
|
||||
</SimpleGrid>
|
||||
|
||||
<Group justify="center" gap="xl" mih={150}>
|
||||
<div>
|
||||
<Text ta="center" size="lg" inline>
|
||||
Import zip file
|
||||
</Text>
|
||||
<Text ta="center" size="sm" c="dimmed" inline py="sm">
|
||||
{t(
|
||||
`Upload zip file containing Markdown and HTML files. Max: {{sizeLimit}}`,
|
||||
{
|
||||
sizeLimit: formatBytes(getFileImportSizeLimit()),
|
||||
},
|
||||
)}
|
||||
</Text>
|
||||
<FileButton
|
||||
onChange={(file) => handleZipUpload(file, "generic")}
|
||||
accept="application/zip"
|
||||
>
|
||||
{(props) => (
|
||||
<Group justify="center">
|
||||
<Button
|
||||
justify="center"
|
||||
leftSection={<IconFileTypeZip size={18} />}
|
||||
{...props}
|
||||
>
|
||||
{t("Upload file")}
|
||||
</Button>
|
||||
</Group>
|
||||
)}
|
||||
</FileButton>
|
||||
</div>
|
||||
</Group>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
@ -7,10 +7,11 @@ import {
|
||||
IPage,
|
||||
IPageInput,
|
||||
SidebarPagesParams,
|
||||
} from "@/features/page/types/page.types";
|
||||
} from '@/features/page/types/page.types';
|
||||
import { IAttachment, IPagination } from "@/lib/types.ts";
|
||||
import { saveAs } from "file-saver";
|
||||
import { InfiniteData } from "@tanstack/react-query";
|
||||
import { IFileTask } from '@/features/file-task/types/file-task.types.ts';
|
||||
|
||||
export async function createPage(data: Partial<IPage>): Promise<IPage> {
|
||||
const req = await api.post<IPage>("/pages/create", data);
|
||||
@ -119,6 +120,25 @@ export async function importPage(file: File, spaceId: string) {
|
||||
return req.data;
|
||||
}
|
||||
|
||||
export async function importZip(
|
||||
file: File,
|
||||
spaceId: string,
|
||||
source?: string,
|
||||
): Promise<IFileTask> {
|
||||
const formData = new FormData();
|
||||
formData.append("spaceId", spaceId);
|
||||
formData.append("source", source);
|
||||
formData.append("file", file);
|
||||
|
||||
const req = await api.post<any>("/pages/import-zip", formData, {
|
||||
headers: {
|
||||
"Content-Type": "multipart/form-data",
|
||||
},
|
||||
});
|
||||
|
||||
return req.data;
|
||||
}
|
||||
|
||||
export async function uploadFile(
|
||||
file: File,
|
||||
pageId: string,
|
||||
|
||||
@ -24,7 +24,10 @@ import {
|
||||
IconPointFilled,
|
||||
IconTrash,
|
||||
} from "@tabler/icons-react";
|
||||
import { appendNodeChildrenAtom, treeDataAtom } from "@/features/page/tree/atoms/tree-data-atom.ts";
|
||||
import {
|
||||
appendNodeChildrenAtom,
|
||||
treeDataAtom,
|
||||
} from "@/features/page/tree/atoms/tree-data-atom.ts";
|
||||
import clsx from "clsx";
|
||||
import EmojiPicker from "@/components/ui/emoji-picker.tsx";
|
||||
import { useTreeMutation } from "@/features/page/tree/hooks/use-tree-mutation.ts";
|
||||
@ -32,6 +35,7 @@ import {
|
||||
appendNodeChildren,
|
||||
buildTree,
|
||||
buildTreeWithChildren,
|
||||
mergeRootTrees,
|
||||
updateTreeNodeIcon,
|
||||
} from "@/features/page/tree/utils/utils.ts";
|
||||
import { SpaceTreeNode } from "@/features/page/tree/types.ts";
|
||||
@ -104,17 +108,17 @@ export default function SpaceTree({ spaceId, readOnly }: SpaceTreeProps) {
|
||||
const allItems = pagesData.pages.flatMap((page) => page.items);
|
||||
const treeData = buildTree(allItems);
|
||||
|
||||
if (data.length < 1 || data?.[0].spaceId !== spaceId) {
|
||||
//Thoughts
|
||||
// don't reset if there is data in state
|
||||
// we only expect to call this once on initial load
|
||||
// even if we decide to refetch, it should only update
|
||||
// and append root pages instead of resetting the entire tree
|
||||
// which looses async loaded children too
|
||||
setData(treeData);
|
||||
setIsDataLoaded(true);
|
||||
setOpenTreeNodes({});
|
||||
}
|
||||
setData((prev) => {
|
||||
// fresh space; full reset
|
||||
if (prev.length === 0 || prev[0]?.spaceId !== spaceId) {
|
||||
setIsDataLoaded(true);
|
||||
setOpenTreeNodes({});
|
||||
return treeData;
|
||||
}
|
||||
|
||||
// same space; append only missing roots
|
||||
return mergeRootTrees(prev, treeData);
|
||||
});
|
||||
}
|
||||
}, [pagesData, hasNextPage]);
|
||||
|
||||
@ -297,17 +301,19 @@ function Node({ node, style, dragHandle, tree }: NodeRendererProps<any>) {
|
||||
|
||||
const handleEmojiSelect = (emoji: { native: string }) => {
|
||||
handleUpdateNodeIcon(node.id, emoji.native);
|
||||
updatePageMutation.mutateAsync({ pageId: node.id, icon: emoji.native }).then((data) => {
|
||||
setTimeout(() => {
|
||||
emit({
|
||||
operation: "updateOne",
|
||||
spaceId: node.data.spaceId,
|
||||
entity: ["pages"],
|
||||
id: node.id,
|
||||
payload: { icon: emoji.native, parentPageId: data.parentPageId},
|
||||
});
|
||||
}, 50);
|
||||
});
|
||||
updatePageMutation
|
||||
.mutateAsync({ pageId: node.id, icon: emoji.native })
|
||||
.then((data) => {
|
||||
setTimeout(() => {
|
||||
emit({
|
||||
operation: "updateOne",
|
||||
spaceId: node.data.spaceId,
|
||||
entity: ["pages"],
|
||||
id: node.id,
|
||||
payload: { icon: emoji.native, parentPageId: data.parentPageId },
|
||||
});
|
||||
}, 50);
|
||||
});
|
||||
};
|
||||
|
||||
const handleRemoveEmoji = () => {
|
||||
@ -570,7 +576,7 @@ interface PageArrowProps {
|
||||
|
||||
function PageArrow({ node, onExpandTree }: PageArrowProps) {
|
||||
useEffect(() => {
|
||||
if(node.isOpen){
|
||||
if (node.isOpen) {
|
||||
onExpandTree();
|
||||
}
|
||||
}, []);
|
||||
|
||||
@ -121,7 +121,6 @@ export const deleteTreeNode = (
|
||||
.filter((node) => node !== null);
|
||||
};
|
||||
|
||||
|
||||
export function buildTreeWithChildren(items: SpaceTreeNode[]): SpaceTreeNode[] {
|
||||
const nodeMap = {};
|
||||
let result: SpaceTreeNode[] = [];
|
||||
@ -167,10 +166,12 @@ export function appendNodeChildren(
|
||||
// Preserve deeper children if they exist and remove node if deleted
|
||||
return treeItems.map((node) => {
|
||||
if (node.id === nodeId) {
|
||||
const newIds = new Set(children.map(c => c.id));
|
||||
const newIds = new Set(children.map((c) => c.id));
|
||||
|
||||
const existingMap = new Map(
|
||||
(node.children ?? []).filter(c => newIds.has(c.id)).map(c => [c.id, c])
|
||||
(node.children ?? [])
|
||||
.filter((c) => newIds.has(c.id))
|
||||
.map((c) => [c.id, c]),
|
||||
);
|
||||
|
||||
const merged = children.map((newChild) => {
|
||||
@ -196,3 +197,21 @@ export function appendNodeChildren(
|
||||
return node;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge root nodes; keep existing ones intact, append new ones,
|
||||
*/
|
||||
export function mergeRootTrees(
|
||||
prevRoots: SpaceTreeNode[],
|
||||
incomingRoots: SpaceTreeNode[],
|
||||
): SpaceTreeNode[] {
|
||||
const seen = new Set(prevRoots.map((r) => r.id));
|
||||
|
||||
// add new roots that were not present before
|
||||
const merged = [...prevRoots];
|
||||
incomingRoots.forEach((node) => {
|
||||
if (!seen.has(node.id)) merged.push(node);
|
||||
});
|
||||
|
||||
return sortPositionKeys(merged);
|
||||
}
|
||||
|
||||
@ -47,15 +47,28 @@ export type MoveTreeNodeEvent = {
|
||||
parentId: string;
|
||||
index: number;
|
||||
position: string;
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
export type DeleteTreeNodeEvent = {
|
||||
operation: "deleteTreeNode";
|
||||
spaceId: string;
|
||||
payload: {
|
||||
node: SpaceTreeNode
|
||||
}
|
||||
node: SpaceTreeNode;
|
||||
};
|
||||
};
|
||||
|
||||
export type WebSocketEvent = InvalidateEvent | InvalidateCommentsEvent | UpdateEvent | DeleteEvent | AddTreeNodeEvent | MoveTreeNodeEvent | DeleteTreeNodeEvent;
|
||||
export type RefetchRootTreeNodeEvent = {
|
||||
operation: "refetchRootTreeNodeEvent";
|
||||
spaceId: string;
|
||||
};
|
||||
|
||||
export type WebSocketEvent =
|
||||
| InvalidateEvent
|
||||
| InvalidateCommentsEvent
|
||||
| UpdateEvent
|
||||
| DeleteEvent
|
||||
| AddTreeNodeEvent
|
||||
| MoveTreeNodeEvent
|
||||
| DeleteTreeNodeEvent
|
||||
| RefetchRootTreeNodeEvent;
|
||||
|
||||
@ -5,8 +5,14 @@ import { InfiniteData, useQueryClient } from "@tanstack/react-query";
|
||||
import { WebSocketEvent } from "@/features/websocket/types";
|
||||
import { IPage } from "../page/types/page.types";
|
||||
import { IPagination } from "@/lib/types";
|
||||
import { invalidateOnCreatePage, invalidateOnDeletePage, invalidateOnMovePage, invalidateOnUpdatePage } from "../page/queries/page-query";
|
||||
import {
|
||||
invalidateOnCreatePage,
|
||||
invalidateOnDeletePage,
|
||||
invalidateOnMovePage,
|
||||
invalidateOnUpdatePage,
|
||||
} from "../page/queries/page-query";
|
||||
import { RQ_KEY } from "../comment/queries/comment-query";
|
||||
import { queryClient } from "@/main.tsx";
|
||||
|
||||
export const useQuerySubscription = () => {
|
||||
const queryClient = useQueryClient();
|
||||
@ -37,8 +43,7 @@ export const useQuerySubscription = () => {
|
||||
invalidateOnMovePage();
|
||||
break;
|
||||
case "deleteTreeNode":
|
||||
const pageId = data.payload.node.id;
|
||||
invalidateOnDeletePage(pageId);
|
||||
invalidateOnDeletePage(data.payload.node.id);
|
||||
break;
|
||||
case "updateOne":
|
||||
entity = data.entity[0];
|
||||
@ -50,17 +55,23 @@ export const useQuerySubscription = () => {
|
||||
}
|
||||
|
||||
// only update if data was already in cache
|
||||
if(queryClient.getQueryData([...data.entity, queryKeyId])){
|
||||
if (queryClient.getQueryData([...data.entity, queryKeyId])) {
|
||||
queryClient.setQueryData([...data.entity, queryKeyId], {
|
||||
...queryClient.getQueryData([...data.entity, queryKeyId]),
|
||||
...data.payload,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
if (entity === "pages") {
|
||||
invalidateOnUpdatePage(data.spaceId, data.payload.parentPageId, data.id, data.payload.title, data.payload.icon);
|
||||
invalidateOnUpdatePage(
|
||||
data.spaceId,
|
||||
data.payload.parentPageId,
|
||||
data.id,
|
||||
data.payload.title,
|
||||
data.payload.icon,
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
queryClient.setQueriesData(
|
||||
{ queryKey: [data.entity, data.id] },
|
||||
@ -72,8 +83,19 @@ export const useQuerySubscription = () => {
|
||||
: update(oldData as Record<string, unknown>);
|
||||
},
|
||||
);
|
||||
*/
|
||||
*/
|
||||
break;
|
||||
case "refetchRootTreeNodeEvent": {
|
||||
const spaceId = data.spaceId;
|
||||
queryClient.refetchQueries({
|
||||
queryKey: ["root-sidebar-pages", spaceId],
|
||||
});
|
||||
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: ["recent-changes", spaceId],
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
}, [queryClient, socket]);
|
||||
|
||||
@ -70,6 +70,11 @@ export function getFileUploadSizeLimit() {
|
||||
return bytes(limit);
|
||||
}
|
||||
|
||||
export function getFileImportSizeLimit() {
|
||||
const limit = getConfigValue("FILE_IMPORT_SIZE_LIMIT", "200mb");
|
||||
return bytes(limit);
|
||||
}
|
||||
|
||||
export function getDrawioUrl() {
|
||||
return getConfigValue("DRAWIO_URL", "https://embed.diagrams.net");
|
||||
}
|
||||
|
||||
@ -8,6 +8,7 @@ export default defineConfig(({ mode }) => {
|
||||
const {
|
||||
APP_URL,
|
||||
FILE_UPLOAD_SIZE_LIMIT,
|
||||
FILE_IMPORT_SIZE_LIMIT,
|
||||
DRAWIO_URL,
|
||||
CLOUD,
|
||||
SUBDOMAIN_HOST,
|
||||
@ -20,6 +21,7 @@ export default defineConfig(({ mode }) => {
|
||||
"process.env": {
|
||||
APP_URL,
|
||||
FILE_UPLOAD_SIZE_LIMIT,
|
||||
FILE_IMPORT_SIZE_LIMIT,
|
||||
DRAWIO_URL,
|
||||
CLOUD,
|
||||
SUBDOMAIN_HOST,
|
||||
|
||||
@ -31,6 +31,7 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "3.701.0",
|
||||
"@aws-sdk/lib-storage": "3.701.0",
|
||||
"@aws-sdk/s3-request-presigner": "3.701.0",
|
||||
"@casl/ability": "^6.7.3",
|
||||
"@fastify/cookie": "^11.0.2",
|
||||
@ -56,6 +57,7 @@
|
||||
"bcrypt": "^5.1.1",
|
||||
"bullmq": "^5.41.3",
|
||||
"cache-manager": "^6.4.0",
|
||||
"cheerio": "^1.0.0",
|
||||
"class-transformer": "^0.5.1",
|
||||
"class-validator": "^0.14.1",
|
||||
"cookie": "^1.0.2",
|
||||
@ -80,7 +82,9 @@
|
||||
"sanitize-filename-ts": "^1.0.2",
|
||||
"socket.io": "^4.8.1",
|
||||
"stripe": "^17.5.0",
|
||||
"ws": "^8.18.0"
|
||||
"tmp-promise": "^3.0.3",
|
||||
"ws": "^8.18.0",
|
||||
"yauzl": "^3.2.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@eslint/js": "^9.20.0",
|
||||
@ -99,6 +103,7 @@
|
||||
"@types/pg": "^8.11.11",
|
||||
"@types/supertest": "^6.0.2",
|
||||
"@types/ws": "^8.5.14",
|
||||
"@types/yauzl": "^2.10.3",
|
||||
"eslint": "^9.20.1",
|
||||
"eslint-config-prettier": "^10.0.1",
|
||||
"globals": "^15.15.0",
|
||||
|
||||
@ -130,7 +130,7 @@ export class PersistenceExtension implements Extension {
|
||||
);
|
||||
this.contributors.delete(documentName);
|
||||
} catch (err) {
|
||||
this.logger.debug('Contributors error:' + err?.['message']);
|
||||
//this.logger.debug('Contributors error:' + err?.['message']);
|
||||
}
|
||||
|
||||
await this.pageRepo.updatePage(
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import * as path from 'path';
|
||||
import * as bcrypt from 'bcrypt';
|
||||
import { sanitize } from 'sanitize-filename-ts';
|
||||
|
||||
export const envPath = path.resolve(process.cwd(), '..', '..', '.env');
|
||||
|
||||
@ -62,3 +63,8 @@ export function extractDateFromUuid7(uuid7: string) {
|
||||
|
||||
return new Date(timestamp);
|
||||
}
|
||||
|
||||
export function sanitizeFileName(fileName: string): string {
|
||||
const sanitizedFilename = sanitize(fileName).replace(/ /g, '_');
|
||||
return sanitizedFilename.slice(0, 255);
|
||||
}
|
||||
|
||||
@ -0,0 +1,39 @@
|
||||
import { Kysely, sql } from 'kysely';
|
||||
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await db.schema
|
||||
.createTable('file_tasks')
|
||||
.addColumn('id', 'uuid', (col) =>
|
||||
col.primaryKey().defaultTo(sql`gen_uuid_v7()`),
|
||||
)
|
||||
// type (import, export)
|
||||
.addColumn('type', 'varchar', (col) => col)
|
||||
// source (generic, notion, confluence)
|
||||
.addColumn('source', 'varchar', (col) => col)
|
||||
// status (pending|processing|success|failed),
|
||||
.addColumn('status', 'varchar', (col) => col)
|
||||
.addColumn('file_name', 'varchar', (col) => col.notNull())
|
||||
.addColumn('file_path', 'varchar', (col) => col.notNull())
|
||||
.addColumn('file_size', 'int8', (col) => col)
|
||||
.addColumn('file_ext', 'varchar', (col) => col)
|
||||
.addColumn('error_message', 'varchar', (col) => col)
|
||||
.addColumn('creator_id', 'uuid', (col) => col.references('users.id'))
|
||||
.addColumn('space_id', 'uuid', (col) =>
|
||||
col.references('spaces.id').onDelete('cascade'),
|
||||
)
|
||||
.addColumn('workspace_id', 'uuid', (col) =>
|
||||
col.references('workspaces.id').onDelete('cascade').notNull(),
|
||||
)
|
||||
.addColumn('created_at', 'timestamptz', (col) =>
|
||||
col.notNull().defaultTo(sql`now()`),
|
||||
)
|
||||
.addColumn('updated_at', 'timestamptz', (col) =>
|
||||
col.notNull().defaultTo(sql`now()`),
|
||||
)
|
||||
.addColumn('deleted_at', 'timestamptz', (col) => col)
|
||||
.execute();
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await db.schema.dropTable('file_tasks').execute();
|
||||
}
|
||||
19
apps/server/src/database/types/db.d.ts
vendored
19
apps/server/src/database/types/db.d.ts
vendored
@ -122,6 +122,24 @@ export interface Comments {
|
||||
workspaceId: string;
|
||||
}
|
||||
|
||||
export interface FileTasks {
|
||||
createdAt: Generated<Timestamp>;
|
||||
creatorId: string | null;
|
||||
deletedAt: Timestamp | null;
|
||||
errorMessage: string | null;
|
||||
fileExt: string | null;
|
||||
fileName: string;
|
||||
filePath: string;
|
||||
fileSize: Int8 | null;
|
||||
id: Generated<string>;
|
||||
source: string | null;
|
||||
spaceId: string | null;
|
||||
status: string | null;
|
||||
type: string | null;
|
||||
updatedAt: Generated<Timestamp>;
|
||||
workspaceId: string;
|
||||
}
|
||||
|
||||
export interface Groups {
|
||||
createdAt: Generated<Timestamp>;
|
||||
creatorId: string | null;
|
||||
@ -298,6 +316,7 @@ export interface DB {
|
||||
backlinks: Backlinks;
|
||||
billing: Billing;
|
||||
comments: Comments;
|
||||
fileTasks: FileTasks;
|
||||
groups: Groups;
|
||||
groupUsers: GroupUsers;
|
||||
pageHistory: PageHistory;
|
||||
|
||||
@ -17,6 +17,7 @@ import {
|
||||
AuthProviders,
|
||||
AuthAccounts,
|
||||
Shares,
|
||||
FileTasks,
|
||||
} from './db';
|
||||
|
||||
// Workspace
|
||||
@ -107,3 +108,8 @@ export type UpdatableAuthAccount = Updateable<Omit<AuthAccounts, 'id'>>;
|
||||
export type Share = Selectable<Shares>;
|
||||
export type InsertableShare = Insertable<Shares>;
|
||||
export type UpdatableShare = Updateable<Omit<Shares, 'id'>>;
|
||||
|
||||
// File Task
|
||||
export type FileTask = Selectable<FileTasks>;
|
||||
export type InsertableFileTask = Insertable<FileTasks>;
|
||||
export type UpdatableFileTask = Updateable<Omit<FileTasks, 'id'>>;
|
||||
|
||||
Submodule apps/server/src/ee updated: b312008b4b...70eb45eaec
@ -67,6 +67,10 @@ export class EnvironmentService {
|
||||
return this.configService.get<string>('FILE_UPLOAD_SIZE_LIMIT', '50mb');
|
||||
}
|
||||
|
||||
getFileImportSizeLimit(): string {
|
||||
return this.configService.get<string>('FILE_IMPORT_SIZE_LIMIT', '200mb');
|
||||
}
|
||||
|
||||
getAwsS3AccessKeyId(): string {
|
||||
return this.configService.get<string>('AWS_S3_ACCESS_KEY_ID');
|
||||
}
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
import * as TurndownService from '@joplin/turndown';
|
||||
import * as TurndownPluginGfm from '@joplin/turndown-plugin-gfm';
|
||||
import * as path from 'path';
|
||||
|
||||
export function turndown(html: string): string {
|
||||
const turndownService = new TurndownService({
|
||||
@ -23,6 +24,7 @@ export function turndown(html: string): string {
|
||||
mathInline,
|
||||
mathBlock,
|
||||
iframeEmbed,
|
||||
video,
|
||||
]);
|
||||
return turndownService.turndown(html).replaceAll('<br>', ' ');
|
||||
}
|
||||
@ -87,8 +89,12 @@ function preserveDetail(turndownService: TurndownService) {
|
||||
}
|
||||
|
||||
const detailsContent = Array.from(node.childNodes)
|
||||
.filter(child => child.nodeName !== 'SUMMARY')
|
||||
.map(child => (child.nodeType === 1 ? turndownService.turndown((child as HTMLElement).outerHTML) : child.textContent))
|
||||
.filter((child) => child.nodeName !== 'SUMMARY')
|
||||
.map((child) =>
|
||||
child.nodeType === 1
|
||||
? turndownService.turndown((child as HTMLElement).outerHTML)
|
||||
: child.textContent,
|
||||
)
|
||||
.join('');
|
||||
|
||||
return `\n<details>\n${detailSummary}\n\n${detailsContent}\n\n</details>\n`;
|
||||
@ -135,3 +141,16 @@ function iframeEmbed(turndownService: TurndownService) {
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function video(turndownService: TurndownService) {
|
||||
turndownService.addRule('video', {
|
||||
filter: function (node: HTMLInputElement) {
|
||||
return node.tagName === 'VIDEO';
|
||||
},
|
||||
replacement: function (content: any, node: HTMLInputElement) {
|
||||
const src = node.getAttribute('src') || '';
|
||||
const name = path.basename(src);
|
||||
return '[' + name + '](' + src + ')';
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
18
apps/server/src/integrations/import/dto/file-task-dto.ts
Normal file
18
apps/server/src/integrations/import/dto/file-task-dto.ts
Normal file
@ -0,0 +1,18 @@
|
||||
import { IsNotEmpty, IsUUID } from 'class-validator';
|
||||
|
||||
export class FileTaskIdDto {
|
||||
@IsNotEmpty()
|
||||
@IsUUID()
|
||||
fileTaskId: string;
|
||||
}
|
||||
|
||||
export type ImportPageNode = {
|
||||
id: string;
|
||||
slugId: string;
|
||||
name: string;
|
||||
content: string;
|
||||
position?: string | null;
|
||||
parentPageId: string | null;
|
||||
fileExtension: string;
|
||||
filePath: string;
|
||||
};
|
||||
79
apps/server/src/integrations/import/file-task.controller.ts
Normal file
79
apps/server/src/integrations/import/file-task.controller.ts
Normal file
@ -0,0 +1,79 @@
|
||||
import {
|
||||
Body,
|
||||
Controller,
|
||||
ForbiddenException,
|
||||
HttpCode,
|
||||
HttpStatus,
|
||||
NotFoundException,
|
||||
Post,
|
||||
UseGuards,
|
||||
} from '@nestjs/common';
|
||||
import SpaceAbilityFactory from '../../core/casl/abilities/space-ability.factory';
|
||||
import { JwtAuthGuard } from '../../common/guards/jwt-auth.guard';
|
||||
import { User } from '@docmost/db/types/entity.types';
|
||||
import {
|
||||
SpaceCaslAction,
|
||||
SpaceCaslSubject,
|
||||
} from '../../core/casl/interfaces/space-ability.type';
|
||||
import { InjectKysely } from 'nestjs-kysely';
|
||||
import { KyselyDB } from '@docmost/db/types/kysely.types';
|
||||
import { AuthUser } from '../../common/decorators/auth-user.decorator';
|
||||
import { FileTaskIdDto } from './dto/file-task-dto';
|
||||
import { SpaceMemberRepo } from '@docmost/db/repos/space/space-member.repo';
|
||||
|
||||
@Controller('file-tasks')
|
||||
export class FileTaskController {
|
||||
constructor(
|
||||
private readonly spaceMemberRepo: SpaceMemberRepo,
|
||||
private readonly spaceAbility: SpaceAbilityFactory,
|
||||
@InjectKysely() private readonly db: KyselyDB,
|
||||
) {}
|
||||
|
||||
@UseGuards(JwtAuthGuard)
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@Post()
|
||||
async getFileTasks(@AuthUser() user: User) {
|
||||
const userSpaceIds = await this.spaceMemberRepo.getUserSpaceIds(user.id);
|
||||
|
||||
if (!userSpaceIds || userSpaceIds.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const fileTasks = await this.db
|
||||
.selectFrom('fileTasks')
|
||||
.selectAll()
|
||||
.where('spaceId', 'in', userSpaceIds)
|
||||
.execute();
|
||||
|
||||
if (!fileTasks) {
|
||||
throw new NotFoundException('File task not found');
|
||||
}
|
||||
|
||||
return fileTasks;
|
||||
}
|
||||
|
||||
@UseGuards(JwtAuthGuard)
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@Post('info')
|
||||
async getFileTask(@Body() dto: FileTaskIdDto, @AuthUser() user: User) {
|
||||
const fileTask = await this.db
|
||||
.selectFrom('fileTasks')
|
||||
.selectAll()
|
||||
.where('id', '=', dto.fileTaskId)
|
||||
.executeTakeFirst();
|
||||
|
||||
if (!fileTask || !fileTask.spaceId) {
|
||||
throw new NotFoundException('File task not found');
|
||||
}
|
||||
|
||||
const ability = await this.spaceAbility.createForUser(
|
||||
user,
|
||||
fileTask.spaceId,
|
||||
);
|
||||
if (ability.cannot(SpaceCaslAction.Read, SpaceCaslSubject.Page)) {
|
||||
throw new ForbiddenException();
|
||||
}
|
||||
|
||||
return fileTask;
|
||||
}
|
||||
}
|
||||
@ -21,8 +21,9 @@ import {
|
||||
import { FileInterceptor } from '../../common/interceptors/file.interceptor';
|
||||
import * as bytes from 'bytes';
|
||||
import * as path from 'path';
|
||||
import { ImportService } from './import.service';
|
||||
import { ImportService } from './services/import.service';
|
||||
import { AuthWorkspace } from '../../common/decorators/auth-workspace.decorator';
|
||||
import { EnvironmentService } from '../environment/environment.service';
|
||||
|
||||
@Controller()
|
||||
export class ImportController {
|
||||
@ -31,6 +32,7 @@ export class ImportController {
|
||||
constructor(
|
||||
private readonly importService: ImportService,
|
||||
private readonly spaceAbility: SpaceAbilityFactory,
|
||||
private readonly environmentService: EnvironmentService,
|
||||
) {}
|
||||
|
||||
@UseInterceptors(FileInterceptor)
|
||||
@ -44,18 +46,18 @@ export class ImportController {
|
||||
) {
|
||||
const validFileExtensions = ['.md', '.html'];
|
||||
|
||||
const maxFileSize = bytes('100mb');
|
||||
const maxFileSize = bytes('10mb');
|
||||
|
||||
let file = null;
|
||||
try {
|
||||
file = await req.file({
|
||||
limits: { fileSize: maxFileSize, fields: 3, files: 1 },
|
||||
limits: { fileSize: maxFileSize, fields: 4, files: 1 },
|
||||
});
|
||||
} catch (err: any) {
|
||||
this.logger.error(err.message);
|
||||
if (err?.statusCode === 413) {
|
||||
throw new BadRequestException(
|
||||
`File too large. Exceeds the 100mb import limit`,
|
||||
`File too large. Exceeds the 10mb import limit`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -73,7 +75,7 @@ export class ImportController {
|
||||
const spaceId = file.fields?.spaceId?.value;
|
||||
|
||||
if (!spaceId) {
|
||||
throw new BadRequestException('spaceId or format not found');
|
||||
throw new BadRequestException('spaceId is required');
|
||||
}
|
||||
|
||||
const ability = await this.spaceAbility.createForUser(user, spaceId);
|
||||
@ -83,4 +85,69 @@ export class ImportController {
|
||||
|
||||
return this.importService.importPage(file, user.id, spaceId, workspace.id);
|
||||
}
|
||||
|
||||
@UseInterceptors(FileInterceptor)
|
||||
@UseGuards(JwtAuthGuard)
|
||||
@HttpCode(HttpStatus.OK)
|
||||
@Post('pages/import-zip')
|
||||
async importZip(
|
||||
@Req() req: any,
|
||||
@AuthUser() user: User,
|
||||
@AuthWorkspace() workspace: Workspace,
|
||||
) {
|
||||
const validFileExtensions = ['.zip'];
|
||||
|
||||
const maxFileSize = bytes(this.environmentService.getFileImportSizeLimit());
|
||||
|
||||
let file = null;
|
||||
try {
|
||||
file = await req.file({
|
||||
limits: { fileSize: maxFileSize, fields: 3, files: 1 },
|
||||
});
|
||||
} catch (err: any) {
|
||||
this.logger.error(err.message);
|
||||
if (err?.statusCode === 413) {
|
||||
throw new BadRequestException(
|
||||
`File too large. Exceeds the ${this.environmentService.getFileImportSizeLimit()} import limit`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (!file) {
|
||||
throw new BadRequestException('Failed to upload file');
|
||||
}
|
||||
|
||||
if (
|
||||
!validFileExtensions.includes(path.extname(file.filename).toLowerCase())
|
||||
) {
|
||||
throw new BadRequestException('Invalid import file extension.');
|
||||
}
|
||||
|
||||
const spaceId = file.fields?.spaceId?.value;
|
||||
const source = file.fields?.source?.value;
|
||||
|
||||
const validZipSources = ['generic', 'notion', 'confluence'];
|
||||
if (!validZipSources.includes(source)) {
|
||||
throw new BadRequestException(
|
||||
'Invalid import source. Import source must either be generic, notion or confluence.',
|
||||
);
|
||||
}
|
||||
|
||||
if (!spaceId) {
|
||||
throw new BadRequestException('spaceId is required');
|
||||
}
|
||||
|
||||
const ability = await this.spaceAbility.createForUser(user, spaceId);
|
||||
if (ability.cannot(SpaceCaslAction.Edit, SpaceCaslSubject.Page)) {
|
||||
throw new ForbiddenException();
|
||||
}
|
||||
|
||||
return this.importService.importZip(
|
||||
file,
|
||||
source,
|
||||
user.id,
|
||||
spaceId,
|
||||
workspace.id,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,9 +1,22 @@
|
||||
import { Module } from '@nestjs/common';
|
||||
import { ImportService } from './import.service';
|
||||
import { ImportService } from './services/import.service';
|
||||
import { ImportController } from './import.controller';
|
||||
import { StorageModule } from '../storage/storage.module';
|
||||
import { FileTaskService } from './services/file-task.service';
|
||||
import { FileTaskProcessor } from './processors/file-task.processor';
|
||||
import { ImportAttachmentService } from './services/import-attachment.service';
|
||||
import { FileTaskController } from './file-task.controller';
|
||||
import { PageModule } from '../../core/page/page.module';
|
||||
|
||||
@Module({
|
||||
providers: [ImportService],
|
||||
controllers: [ImportController],
|
||||
providers: [
|
||||
ImportService,
|
||||
FileTaskService,
|
||||
FileTaskProcessor,
|
||||
ImportAttachmentService,
|
||||
],
|
||||
exports: [ImportService, ImportAttachmentService],
|
||||
controllers: [ImportController, FileTaskController],
|
||||
imports: [StorageModule, PageModule],
|
||||
})
|
||||
export class ImportModule {}
|
||||
|
||||
@ -0,0 +1,76 @@
|
||||
import { Logger, OnModuleDestroy } from '@nestjs/common';
|
||||
import { OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq';
|
||||
import { Job } from 'bullmq';
|
||||
import { QueueJob, QueueName } from 'src/integrations/queue/constants';
|
||||
import { FileTaskService } from '../services/file-task.service';
|
||||
import { FileTaskStatus } from '../utils/file.utils';
|
||||
import { StorageService } from '../../storage/storage.service';
|
||||
|
||||
@Processor(QueueName.FILE_TASK_QUEUE)
|
||||
export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy {
|
||||
private readonly logger = new Logger(FileTaskProcessor.name);
|
||||
|
||||
constructor(
|
||||
private readonly fileTaskService: FileTaskService,
|
||||
private readonly storageService: StorageService,
|
||||
) {
|
||||
super();
|
||||
}
|
||||
|
||||
async process(job: Job<any, void>): Promise<void> {
|
||||
try {
|
||||
switch (job.name) {
|
||||
case QueueJob.IMPORT_TASK:
|
||||
await this.fileTaskService.processZIpImport(job.data.fileTaskId);
|
||||
break;
|
||||
case QueueJob.EXPORT_TASK:
|
||||
// TODO: export task
|
||||
break;
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.error('File task failed', err);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
@OnWorkerEvent('active')
|
||||
onActive(job: Job) {
|
||||
this.logger.debug(`Processing ${job.name} job`);
|
||||
}
|
||||
|
||||
@OnWorkerEvent('failed')
|
||||
async onFailed(job: Job) {
|
||||
this.logger.error(
|
||||
`Error processing ${job.name} job. Reason: ${job.failedReason}`,
|
||||
);
|
||||
|
||||
try {
|
||||
const fileTaskId = job.data.fileTaskId;
|
||||
await this.fileTaskService.updateTaskStatus(
|
||||
fileTaskId,
|
||||
FileTaskStatus.Failed,
|
||||
job.failedReason,
|
||||
);
|
||||
|
||||
const fileTask = await this.fileTaskService.getFileTask(fileTaskId);
|
||||
if (fileTask) {
|
||||
await this.storageService.delete(fileTask.filePath);
|
||||
}
|
||||
} catch (err) {
|
||||
this.logger.error(err);
|
||||
}
|
||||
}
|
||||
|
||||
@OnWorkerEvent('completed')
|
||||
onCompleted(job: Job) {
|
||||
this.logger.log(
|
||||
`Completed ${job.name} job for File task ID ${job.data.fileTaskId}`,
|
||||
);
|
||||
}
|
||||
|
||||
async onModuleDestroy(): Promise<void> {
|
||||
if (this.worker) {
|
||||
await this.worker.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,346 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import * as path from 'path';
|
||||
import { jsonToText } from '../../../collaboration/collaboration.util';
|
||||
import { InjectKysely } from 'nestjs-kysely';
|
||||
import { KyselyDB } from '@docmost/db/types/kysely.types';
|
||||
import {
|
||||
extractZip,
|
||||
FileImportSource,
|
||||
FileTaskStatus,
|
||||
} from '../utils/file.utils';
|
||||
import { StorageService } from '../../storage/storage.service';
|
||||
import * as tmp from 'tmp-promise';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import { createWriteStream } from 'node:fs';
|
||||
import { ImportService } from './import.service';
|
||||
import { promises as fs } from 'fs';
|
||||
import { generateSlugId } from '../../../common/helpers';
|
||||
import { v7 } from 'uuid';
|
||||
import { generateJitteredKeyBetween } from 'fractional-indexing-jittered';
|
||||
import { FileTask, InsertablePage } from '@docmost/db/types/entity.types';
|
||||
import { markdownToHtml } from '@docmost/editor-ext';
|
||||
import { getProsemirrorContent } from '../../../common/helpers/prosemirror/utils';
|
||||
import { formatImportHtml } from '../utils/import-formatter';
|
||||
import {
|
||||
buildAttachmentCandidates,
|
||||
collectMarkdownAndHtmlFiles,
|
||||
} from '../utils/import.utils';
|
||||
import { executeTx } from '@docmost/db/utils';
|
||||
import { BacklinkRepo } from '@docmost/db/repos/backlink/backlink.repo';
|
||||
import { ImportAttachmentService } from './import-attachment.service';
|
||||
import { ModuleRef } from '@nestjs/core';
|
||||
import { PageService } from '../../../core/page/services/page.service';
|
||||
import { ImportPageNode } from '../dto/file-task-dto';
|
||||
|
||||
@Injectable()
|
||||
export class FileTaskService {
|
||||
private readonly logger = new Logger(FileTaskService.name);
|
||||
|
||||
constructor(
|
||||
private readonly storageService: StorageService,
|
||||
private readonly importService: ImportService,
|
||||
private readonly pageService: PageService,
|
||||
private readonly backlinkRepo: BacklinkRepo,
|
||||
@InjectKysely() private readonly db: KyselyDB,
|
||||
private readonly importAttachmentService: ImportAttachmentService,
|
||||
private moduleRef: ModuleRef,
|
||||
) {}
|
||||
|
||||
async processZIpImport(fileTaskId: string): Promise<void> {
|
||||
const fileTask = await this.db
|
||||
.selectFrom('fileTasks')
|
||||
.selectAll()
|
||||
.where('id', '=', fileTaskId)
|
||||
.executeTakeFirst();
|
||||
|
||||
if (!fileTask) {
|
||||
this.logger.log(`Import file task with ID ${fileTaskId} not found`);
|
||||
return;
|
||||
}
|
||||
|
||||
if (fileTask.status === FileTaskStatus.Failed) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (fileTask.status === FileTaskStatus.Success) {
|
||||
this.logger.log('Imported task already processed.');
|
||||
return;
|
||||
}
|
||||
|
||||
const { path: tmpZipPath, cleanup: cleanupTmpFile } = await tmp.file({
|
||||
prefix: 'docmost-import',
|
||||
postfix: '.zip',
|
||||
discardDescriptor: true,
|
||||
});
|
||||
|
||||
const { path: tmpExtractDir, cleanup: cleanupTmpDir } = await tmp.dir({
|
||||
prefix: 'docmost-extract-',
|
||||
unsafeCleanup: true,
|
||||
});
|
||||
|
||||
try {
|
||||
const fileStream = await this.storageService.readStream(
|
||||
fileTask.filePath,
|
||||
);
|
||||
await pipeline(fileStream, createWriteStream(tmpZipPath));
|
||||
await extractZip(tmpZipPath, tmpExtractDir);
|
||||
} catch (err) {
|
||||
await cleanupTmpFile();
|
||||
await cleanupTmpDir();
|
||||
|
||||
throw err;
|
||||
}
|
||||
|
||||
try {
|
||||
if (
|
||||
fileTask.source === FileImportSource.Generic ||
|
||||
fileTask.source === FileImportSource.Notion
|
||||
) {
|
||||
await this.processGenericImport({
|
||||
extractDir: tmpExtractDir,
|
||||
fileTask,
|
||||
});
|
||||
}
|
||||
|
||||
if (fileTask.source === FileImportSource.Confluence) {
|
||||
let ConfluenceModule: any;
|
||||
try {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
ConfluenceModule = require('./../../../ee/confluence-import/confluence-import.service');
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
'Confluence import requested but EE module not bundled in this build',
|
||||
);
|
||||
return;
|
||||
}
|
||||
const confluenceImportService = this.moduleRef.get(
|
||||
ConfluenceModule.ConfluenceImportService,
|
||||
{ strict: false },
|
||||
);
|
||||
|
||||
await confluenceImportService.processConfluenceImport({
|
||||
extractDir: tmpExtractDir,
|
||||
fileTask,
|
||||
});
|
||||
}
|
||||
try {
|
||||
await this.updateTaskStatus(fileTaskId, FileTaskStatus.Success, null);
|
||||
await cleanupTmpFile();
|
||||
await cleanupTmpDir();
|
||||
// delete stored file on success
|
||||
await this.storageService.delete(fileTask.filePath);
|
||||
} catch (err) {
|
||||
this.logger.error(
|
||||
`Failed to delete import file from storage. Task ID: ${fileTaskId}`,
|
||||
err,
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
await cleanupTmpFile();
|
||||
await cleanupTmpDir();
|
||||
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async processGenericImport(opts: {
|
||||
extractDir: string;
|
||||
fileTask: FileTask;
|
||||
}): Promise<void> {
|
||||
const { extractDir, fileTask } = opts;
|
||||
const allFiles = await collectMarkdownAndHtmlFiles(extractDir);
|
||||
const attachmentCandidates = await buildAttachmentCandidates(extractDir);
|
||||
|
||||
const pagesMap = new Map<string, ImportPageNode>();
|
||||
|
||||
for (const absPath of allFiles) {
|
||||
const relPath = path
|
||||
.relative(extractDir, absPath)
|
||||
.split(path.sep)
|
||||
.join('/'); // normalize to forward-slashes
|
||||
const ext = path.extname(relPath).toLowerCase();
|
||||
let content = await fs.readFile(absPath, 'utf-8');
|
||||
|
||||
if (ext.toLowerCase() === '.md') {
|
||||
content = await markdownToHtml(content);
|
||||
}
|
||||
|
||||
pagesMap.set(relPath, {
|
||||
id: v7(),
|
||||
slugId: generateSlugId(),
|
||||
name: path.basename(relPath, ext),
|
||||
content,
|
||||
parentPageId: null,
|
||||
fileExtension: ext,
|
||||
filePath: relPath,
|
||||
});
|
||||
}
|
||||
|
||||
// parent/child linking
|
||||
pagesMap.forEach((page, filePath) => {
|
||||
const segments = filePath.split('/');
|
||||
segments.pop();
|
||||
let parentPage = null;
|
||||
while (segments.length) {
|
||||
const tryMd = segments.join('/') + '.md';
|
||||
const tryHtml = segments.join('/') + '.html';
|
||||
if (pagesMap.has(tryMd)) {
|
||||
parentPage = pagesMap.get(tryMd)!;
|
||||
break;
|
||||
}
|
||||
if (pagesMap.has(tryHtml)) {
|
||||
parentPage = pagesMap.get(tryHtml)!;
|
||||
break;
|
||||
}
|
||||
segments.pop();
|
||||
}
|
||||
if (parentPage) page.parentPageId = parentPage.id;
|
||||
});
|
||||
|
||||
// generate position keys
|
||||
const siblingsMap = new Map<string | null, ImportPageNode[]>();
|
||||
|
||||
pagesMap.forEach((page) => {
|
||||
const group = siblingsMap.get(page.parentPageId) ?? [];
|
||||
group.push(page);
|
||||
siblingsMap.set(page.parentPageId, group);
|
||||
});
|
||||
|
||||
// get root pages
|
||||
const rootSibs = siblingsMap.get(null);
|
||||
|
||||
if (rootSibs?.length) {
|
||||
rootSibs.sort((a, b) => a.name.localeCompare(b.name));
|
||||
|
||||
// get first position key from the server
|
||||
const nextPosition = await this.pageService.nextPagePosition(
|
||||
fileTask.spaceId,
|
||||
);
|
||||
|
||||
let prevPos: string | null = null;
|
||||
rootSibs.forEach((page, idx) => {
|
||||
if (idx === 0) {
|
||||
page.position = nextPosition;
|
||||
} else {
|
||||
page.position = generateJitteredKeyBetween(prevPos, null);
|
||||
}
|
||||
prevPos = page.position;
|
||||
});
|
||||
}
|
||||
|
||||
// non-root buckets (children & deeper levels)
|
||||
siblingsMap.forEach((sibs, parentId) => {
|
||||
if (parentId === null) return; // root already done
|
||||
|
||||
sibs.sort((a, b) => a.name.localeCompare(b.name));
|
||||
|
||||
let prevPos: string | null = null;
|
||||
for (const page of sibs) {
|
||||
page.position = generateJitteredKeyBetween(prevPos, null);
|
||||
prevPos = page.position;
|
||||
}
|
||||
});
|
||||
|
||||
// internal page links
|
||||
const filePathToPageMetaMap = new Map<
|
||||
string,
|
||||
{ id: string; title: string; slugId: string }
|
||||
>();
|
||||
pagesMap.forEach((page) => {
|
||||
filePathToPageMetaMap.set(page.filePath, {
|
||||
id: page.id,
|
||||
title: page.name,
|
||||
slugId: page.slugId,
|
||||
});
|
||||
});
|
||||
|
||||
const pageResults = await Promise.all(
|
||||
Array.from(pagesMap.values()).map(async (page) => {
|
||||
const htmlContent =
|
||||
await this.importAttachmentService.processAttachments({
|
||||
html: page.content,
|
||||
pageRelativePath: page.filePath,
|
||||
extractDir,
|
||||
pageId: page.id,
|
||||
fileTask,
|
||||
attachmentCandidates,
|
||||
});
|
||||
|
||||
const { html, backlinks } = await formatImportHtml({
|
||||
html: htmlContent,
|
||||
currentFilePath: page.filePath,
|
||||
filePathToPageMetaMap: filePathToPageMetaMap,
|
||||
creatorId: fileTask.creatorId,
|
||||
sourcePageId: page.id,
|
||||
workspaceId: fileTask.workspaceId,
|
||||
});
|
||||
|
||||
const pmState = getProsemirrorContent(
|
||||
await this.importService.processHTML(html),
|
||||
);
|
||||
|
||||
const { title, prosemirrorJson } =
|
||||
this.importService.extractTitleAndRemoveHeading(pmState);
|
||||
|
||||
const insertablePage: InsertablePage = {
|
||||
id: page.id,
|
||||
slugId: page.slugId,
|
||||
title: title || page.name,
|
||||
content: prosemirrorJson,
|
||||
textContent: jsonToText(prosemirrorJson),
|
||||
ydoc: await this.importService.createYdoc(prosemirrorJson),
|
||||
position: page.position!,
|
||||
spaceId: fileTask.spaceId,
|
||||
workspaceId: fileTask.workspaceId,
|
||||
creatorId: fileTask.creatorId,
|
||||
lastUpdatedById: fileTask.creatorId,
|
||||
parentPageId: page.parentPageId,
|
||||
};
|
||||
|
||||
return { insertablePage, backlinks };
|
||||
}),
|
||||
);
|
||||
|
||||
const insertablePages = pageResults.map((r) => r.insertablePage);
|
||||
const insertableBacklinks = pageResults.flatMap((r) => r.backlinks);
|
||||
|
||||
if (insertablePages.length < 1) return;
|
||||
const validPageIds = new Set(insertablePages.map((row) => row.id));
|
||||
const filteredBacklinks = insertableBacklinks.filter(
|
||||
({ sourcePageId, targetPageId }) =>
|
||||
validPageIds.has(sourcePageId) && validPageIds.has(targetPageId),
|
||||
);
|
||||
|
||||
await executeTx(this.db, async (trx) => {
|
||||
await trx.insertInto('pages').values(insertablePages).execute();
|
||||
|
||||
if (filteredBacklinks.length > 0) {
|
||||
await this.backlinkRepo.insertBacklink(filteredBacklinks, trx);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async getFileTask(fileTaskId: string) {
|
||||
return this.db
|
||||
.selectFrom('fileTasks')
|
||||
.selectAll()
|
||||
.where('id', '=', fileTaskId)
|
||||
.executeTakeFirst();
|
||||
}
|
||||
|
||||
async updateTaskStatus(
|
||||
fileTaskId: string,
|
||||
status: FileTaskStatus,
|
||||
errorMessage?: string,
|
||||
) {
|
||||
try {
|
||||
await this.db
|
||||
.updateTable('fileTasks')
|
||||
.set({ status: status, errorMessage, updatedAt: new Date() })
|
||||
.where('id', '=', fileTaskId)
|
||||
.execute();
|
||||
} catch (err) {
|
||||
this.logger.error(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,303 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import * as path from 'path';
|
||||
import { InjectKysely } from 'nestjs-kysely';
|
||||
import { KyselyDB } from '@docmost/db/types/kysely.types';
|
||||
import { cleanUrlString } from '../utils/file.utils';
|
||||
import { StorageService } from '../../storage/storage.service';
|
||||
import { createReadStream } from 'node:fs';
|
||||
import { promises as fs } from 'fs';
|
||||
import { getMimeType, sanitizeFileName } from '../../../common/helpers';
|
||||
import { v7 } from 'uuid';
|
||||
import { FileTask } from '@docmost/db/types/entity.types';
|
||||
import { getAttachmentFolderPath } from '../../../core/attachment/attachment.utils';
|
||||
import { AttachmentType } from '../../../core/attachment/attachment.constants';
|
||||
import { unwrapFromParagraph } from '../utils/import-formatter';
|
||||
import { resolveRelativeAttachmentPath } from '../utils/import.utils';
|
||||
import { load } from 'cheerio';
|
||||
|
||||
@Injectable()
|
||||
export class ImportAttachmentService {
|
||||
private readonly logger = new Logger(ImportAttachmentService.name);
|
||||
|
||||
constructor(
|
||||
private readonly storageService: StorageService,
|
||||
@InjectKysely() private readonly db: KyselyDB,
|
||||
) {}
|
||||
|
||||
async processAttachments(opts: {
|
||||
html: string;
|
||||
pageRelativePath: string;
|
||||
extractDir: string;
|
||||
pageId: string;
|
||||
fileTask: FileTask;
|
||||
attachmentCandidates: Map<string, string>;
|
||||
}): Promise<string> {
|
||||
const {
|
||||
html,
|
||||
pageRelativePath,
|
||||
extractDir,
|
||||
pageId,
|
||||
fileTask,
|
||||
attachmentCandidates,
|
||||
} = opts;
|
||||
|
||||
const attachmentTasks: Promise<void>[] = [];
|
||||
|
||||
/**
|
||||
* Cache keyed by the *relative* path that appears in the HTML.
|
||||
* Ensures we upload (and DB-insert) each attachment at most once,
|
||||
* even if it’s referenced multiple times on the page.
|
||||
*/
|
||||
const processed = new Map<
|
||||
string,
|
||||
{
|
||||
attachmentId: string;
|
||||
storageFilePath: string;
|
||||
apiFilePath: string;
|
||||
fileNameWithExt: string;
|
||||
abs: string;
|
||||
}
|
||||
>();
|
||||
|
||||
const uploadOnce = (relPath: string) => {
|
||||
const abs = attachmentCandidates.get(relPath)!;
|
||||
const attachmentId = v7();
|
||||
const ext = path.extname(abs);
|
||||
|
||||
const fileNameWithExt =
|
||||
sanitizeFileName(path.basename(abs, ext)) + ext.toLowerCase();
|
||||
|
||||
const storageFilePath = `${getAttachmentFolderPath(
|
||||
AttachmentType.File,
|
||||
fileTask.workspaceId,
|
||||
)}/${attachmentId}/${fileNameWithExt}`;
|
||||
|
||||
const apiFilePath = `/api/files/${attachmentId}/${fileNameWithExt}`;
|
||||
|
||||
attachmentTasks.push(
|
||||
(async () => {
|
||||
const fileStream = createReadStream(abs);
|
||||
await this.storageService.uploadStream(storageFilePath, fileStream);
|
||||
const stat = await fs.stat(abs);
|
||||
|
||||
await this.db
|
||||
.insertInto('attachments')
|
||||
.values({
|
||||
id: attachmentId,
|
||||
filePath: storageFilePath,
|
||||
fileName: fileNameWithExt,
|
||||
fileSize: stat.size,
|
||||
mimeType: getMimeType(fileNameWithExt),
|
||||
type: 'file',
|
||||
fileExt: ext,
|
||||
creatorId: fileTask.creatorId,
|
||||
workspaceId: fileTask.workspaceId,
|
||||
pageId,
|
||||
spaceId: fileTask.spaceId,
|
||||
})
|
||||
.execute();
|
||||
})(),
|
||||
);
|
||||
|
||||
return {
|
||||
attachmentId,
|
||||
storageFilePath,
|
||||
apiFilePath,
|
||||
fileNameWithExt,
|
||||
abs,
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* – Returns cached data if we’ve already processed this path.
|
||||
* – Otherwise calls `uploadOnce`, stores the result, and returns it.
|
||||
*/
|
||||
const processFile = (relPath: string) => {
|
||||
const cached = processed.get(relPath);
|
||||
if (cached) return cached;
|
||||
|
||||
const fresh = uploadOnce(relPath);
|
||||
processed.set(relPath, fresh);
|
||||
return fresh;
|
||||
};
|
||||
|
||||
const pageDir = path.dirname(pageRelativePath);
|
||||
const $ = load(html);
|
||||
|
||||
// image
|
||||
for (const imgEl of $('img').toArray()) {
|
||||
const $img = $(imgEl);
|
||||
const src = cleanUrlString($img.attr('src') ?? '')!;
|
||||
if (!src || src.startsWith('http')) continue;
|
||||
|
||||
const relPath = resolveRelativeAttachmentPath(
|
||||
src,
|
||||
pageDir,
|
||||
attachmentCandidates,
|
||||
);
|
||||
if (!relPath) continue;
|
||||
|
||||
const { attachmentId, apiFilePath, abs } = processFile(relPath);
|
||||
const stat = await fs.stat(abs);
|
||||
|
||||
const width = $img.attr('width') ?? '100%';
|
||||
const align = $img.attr('data-align') ?? 'center';
|
||||
|
||||
$img
|
||||
.attr('src', apiFilePath)
|
||||
.attr('data-attachment-id', attachmentId)
|
||||
.attr('data-size', stat.size.toString())
|
||||
.attr('width', width)
|
||||
.attr('data-align', align);
|
||||
|
||||
unwrapFromParagraph($, $img);
|
||||
}
|
||||
|
||||
// video
|
||||
for (const vidEl of $('video').toArray()) {
|
||||
const $vid = $(vidEl);
|
||||
const src = cleanUrlString($vid.attr('src') ?? '')!;
|
||||
if (!src || src.startsWith('http')) continue;
|
||||
|
||||
const relPath = resolveRelativeAttachmentPath(
|
||||
src,
|
||||
pageDir,
|
||||
attachmentCandidates,
|
||||
);
|
||||
if (!relPath) continue;
|
||||
|
||||
const { attachmentId, apiFilePath, abs } = processFile(relPath);
|
||||
const stat = await fs.stat(abs);
|
||||
|
||||
const width = $vid.attr('width') ?? '100%';
|
||||
const align = $vid.attr('data-align') ?? 'center';
|
||||
|
||||
$vid
|
||||
.attr('src', apiFilePath)
|
||||
.attr('data-attachment-id', attachmentId)
|
||||
.attr('data-size', stat.size.toString())
|
||||
.attr('width', width)
|
||||
.attr('data-align', align);
|
||||
|
||||
unwrapFromParagraph($, $vid);
|
||||
}
|
||||
|
||||
// <div data-type="attachment">
|
||||
for (const el of $('div[data-type="attachment"]').toArray()) {
|
||||
const $oldDiv = $(el);
|
||||
const rawUrl = cleanUrlString($oldDiv.attr('data-attachment-url') ?? '')!;
|
||||
if (!rawUrl || rawUrl.startsWith('http')) continue;
|
||||
|
||||
const relPath = resolveRelativeAttachmentPath(
|
||||
rawUrl,
|
||||
pageDir,
|
||||
attachmentCandidates,
|
||||
);
|
||||
if (!relPath) continue;
|
||||
|
||||
const { attachmentId, apiFilePath, abs } = processFile(relPath);
|
||||
const stat = await fs.stat(abs);
|
||||
const fileName = path.basename(abs);
|
||||
const mime = getMimeType(abs);
|
||||
|
||||
const $newDiv = $('<div>')
|
||||
.attr('data-type', 'attachment')
|
||||
.attr('data-attachment-url', apiFilePath)
|
||||
.attr('data-attachment-name', fileName)
|
||||
.attr('data-attachment-mime', mime)
|
||||
.attr('data-attachment-size', stat.size.toString())
|
||||
.attr('data-attachment-id', attachmentId);
|
||||
|
||||
$oldDiv.replaceWith($newDiv);
|
||||
unwrapFromParagraph($, $newDiv);
|
||||
}
|
||||
|
||||
// rewrite other attachments via <a>
|
||||
for (const aEl of $('a').toArray()) {
|
||||
const $a = $(aEl);
|
||||
const href = cleanUrlString($a.attr('href') ?? '')!;
|
||||
if (!href || href.startsWith('http')) continue;
|
||||
|
||||
const relPath = resolveRelativeAttachmentPath(
|
||||
href,
|
||||
pageDir,
|
||||
attachmentCandidates,
|
||||
);
|
||||
if (!relPath) continue;
|
||||
|
||||
const { attachmentId, apiFilePath, abs } = processFile(relPath);
|
||||
const stat = await fs.stat(abs);
|
||||
const ext = path.extname(relPath).toLowerCase();
|
||||
|
||||
if (ext === '.mp4') {
|
||||
const $video = $('<video>')
|
||||
.attr('src', apiFilePath)
|
||||
.attr('data-attachment-id', attachmentId)
|
||||
.attr('data-size', stat.size.toString())
|
||||
.attr('width', '100%')
|
||||
.attr('data-align', 'center');
|
||||
$a.replaceWith($video);
|
||||
unwrapFromParagraph($, $video);
|
||||
} else {
|
||||
const confAliasName = $a.attr('data-linked-resource-default-alias');
|
||||
let attachmentName = path.basename(abs);
|
||||
if (confAliasName) attachmentName = confAliasName;
|
||||
|
||||
const $div = $('<div>')
|
||||
.attr('data-type', 'attachment')
|
||||
.attr('data-attachment-url', apiFilePath)
|
||||
.attr('data-attachment-name', attachmentName)
|
||||
.attr('data-attachment-mime', getMimeType(abs))
|
||||
.attr('data-attachment-size', stat.size.toString())
|
||||
.attr('data-attachment-id', attachmentId);
|
||||
|
||||
$a.replaceWith($div);
|
||||
unwrapFromParagraph($, $div);
|
||||
}
|
||||
}
|
||||
|
||||
// excalidraw and drawio
|
||||
for (const type of ['excalidraw', 'drawio'] as const) {
|
||||
for (const el of $(`div[data-type="${type}"]`).toArray()) {
|
||||
const $oldDiv = $(el);
|
||||
const rawSrc = cleanUrlString($oldDiv.attr('data-src') ?? '')!;
|
||||
if (!rawSrc || rawSrc.startsWith('http')) continue;
|
||||
|
||||
const relPath = resolveRelativeAttachmentPath(
|
||||
rawSrc,
|
||||
pageDir,
|
||||
attachmentCandidates,
|
||||
);
|
||||
if (!relPath) continue;
|
||||
|
||||
const { attachmentId, apiFilePath, abs } = processFile(relPath);
|
||||
const stat = await fs.stat(abs);
|
||||
const fileName = path.basename(abs);
|
||||
|
||||
const width = $oldDiv.attr('data-width') || '100%';
|
||||
const align = $oldDiv.attr('data-align') || 'center';
|
||||
|
||||
const $newDiv = $('<div>')
|
||||
.attr('data-type', type)
|
||||
.attr('data-src', apiFilePath)
|
||||
.attr('data-title', fileName)
|
||||
.attr('data-width', width)
|
||||
.attr('data-size', stat.size.toString())
|
||||
.attr('data-align', align)
|
||||
.attr('data-attachment-id', attachmentId);
|
||||
|
||||
$oldDiv.replaceWith($newDiv);
|
||||
unwrapFromParagraph($, $newDiv);
|
||||
}
|
||||
}
|
||||
|
||||
// wait for all uploads & DB inserts
|
||||
try {
|
||||
await Promise.all(attachmentTasks);
|
||||
} catch (err) {
|
||||
this.logger.log('Import attachment upload error', err);
|
||||
}
|
||||
|
||||
return $.root().html() || '';
|
||||
}
|
||||
}
|
||||
@ -4,16 +4,27 @@ import { MultipartFile } from '@fastify/multipart';
|
||||
import { sanitize } from 'sanitize-filename-ts';
|
||||
import * as path from 'path';
|
||||
import {
|
||||
htmlToJson, jsonToText,
|
||||
htmlToJson,
|
||||
jsonToText,
|
||||
tiptapExtensions,
|
||||
} from '../../collaboration/collaboration.util';
|
||||
} from '../../../collaboration/collaboration.util';
|
||||
import { InjectKysely } from 'nestjs-kysely';
|
||||
import { KyselyDB } from '@docmost/db/types/kysely.types';
|
||||
import { generateSlugId } from '../../common/helpers';
|
||||
import { generateSlugId, sanitizeFileName } from '../../../common/helpers';
|
||||
import { generateJitteredKeyBetween } from 'fractional-indexing-jittered';
|
||||
import { TiptapTransformer } from '@hocuspocus/transformer';
|
||||
import * as Y from 'yjs';
|
||||
import { markdownToHtml } from "@docmost/editor-ext";
|
||||
import { markdownToHtml } from '@docmost/editor-ext';
|
||||
import {
|
||||
FileTaskStatus,
|
||||
FileTaskType,
|
||||
getFileTaskFolderPath,
|
||||
} from '../utils/file.utils';
|
||||
import { v7 as uuid7 } from 'uuid';
|
||||
import { StorageService } from '../../storage/storage.service';
|
||||
import { InjectQueue } from '@nestjs/bullmq';
|
||||
import { Queue } from 'bullmq';
|
||||
import { QueueJob, QueueName } from '../../queue/constants';
|
||||
|
||||
@Injectable()
|
||||
export class ImportService {
|
||||
@ -21,7 +32,10 @@ export class ImportService {
|
||||
|
||||
constructor(
|
||||
private readonly pageRepo: PageRepo,
|
||||
private readonly storageService: StorageService,
|
||||
@InjectKysely() private readonly db: KyselyDB,
|
||||
@InjectQueue(QueueName.FILE_TASK_QUEUE)
|
||||
private readonly fileTaskQueue: Queue,
|
||||
) {}
|
||||
|
||||
async importPage(
|
||||
@ -113,7 +127,7 @@ export class ImportService {
|
||||
|
||||
async createYdoc(prosemirrorJson: any): Promise<Buffer | null> {
|
||||
if (prosemirrorJson) {
|
||||
this.logger.debug(`Converting prosemirror json state to ydoc`);
|
||||
// this.logger.debug(`Converting prosemirror json state to ydoc`);
|
||||
|
||||
const ydoc = TiptapTransformer.toYdoc(
|
||||
prosemirrorJson,
|
||||
@ -129,20 +143,34 @@ export class ImportService {
|
||||
}
|
||||
|
||||
extractTitleAndRemoveHeading(prosemirrorState: any) {
|
||||
let title = null;
|
||||
let title: string | null = null;
|
||||
|
||||
const content = prosemirrorState.content ?? [];
|
||||
|
||||
if (
|
||||
prosemirrorState?.content?.length > 0 &&
|
||||
prosemirrorState.content[0].type === 'heading' &&
|
||||
prosemirrorState.content[0].attrs?.level === 1
|
||||
content.length > 0 &&
|
||||
content[0].type === 'heading' &&
|
||||
content[0].attrs?.level === 1
|
||||
) {
|
||||
title = prosemirrorState.content[0].content[0].text;
|
||||
|
||||
// remove h1 header node from state
|
||||
prosemirrorState.content.shift();
|
||||
title = content[0].content?.[0]?.text ?? null;
|
||||
content.shift();
|
||||
}
|
||||
|
||||
return { title, prosemirrorJson: prosemirrorState };
|
||||
// ensure at least one paragraph
|
||||
if (content.length === 0) {
|
||||
content.push({
|
||||
type: 'paragraph',
|
||||
content: [],
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
title,
|
||||
prosemirrorJson: {
|
||||
...prosemirrorState,
|
||||
content,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async getNewPagePosition(spaceId: string): Promise<string> {
|
||||
@ -161,4 +189,52 @@ export class ImportService {
|
||||
return generateJitteredKeyBetween(null, null);
|
||||
}
|
||||
}
|
||||
|
||||
async importZip(
|
||||
filePromise: Promise<MultipartFile>,
|
||||
source: string,
|
||||
userId: string,
|
||||
spaceId: string,
|
||||
workspaceId: string,
|
||||
) {
|
||||
const file = await filePromise;
|
||||
const fileBuffer = await file.toBuffer();
|
||||
const fileExtension = path.extname(file.filename).toLowerCase();
|
||||
const fileName = sanitizeFileName(
|
||||
path.basename(file.filename, fileExtension),
|
||||
);
|
||||
const fileSize = fileBuffer.length;
|
||||
|
||||
const fileNameWithExt = fileName + fileExtension;
|
||||
|
||||
const fileTaskId = uuid7();
|
||||
const filePath = `${getFileTaskFolderPath(FileTaskType.Import, workspaceId)}/${fileTaskId}/${fileNameWithExt}`;
|
||||
|
||||
// upload file
|
||||
await this.storageService.upload(filePath, fileBuffer);
|
||||
|
||||
const fileTask = await this.db
|
||||
.insertInto('fileTasks')
|
||||
.values({
|
||||
id: fileTaskId,
|
||||
type: FileTaskType.Import,
|
||||
source: source,
|
||||
status: FileTaskStatus.Processing,
|
||||
fileName: fileNameWithExt,
|
||||
filePath: filePath,
|
||||
fileSize: fileSize,
|
||||
fileExt: 'zip',
|
||||
creatorId: userId,
|
||||
spaceId: spaceId,
|
||||
workspaceId: workspaceId,
|
||||
})
|
||||
.returningAll()
|
||||
.executeTakeFirst();
|
||||
|
||||
await this.fileTaskQueue.add(QueueJob.IMPORT_TASK, {
|
||||
fileTaskId: fileTaskId,
|
||||
});
|
||||
|
||||
return fileTask;
|
||||
}
|
||||
}
|
||||
187
apps/server/src/integrations/import/utils/file.utils.ts
Normal file
187
apps/server/src/integrations/import/utils/file.utils.ts
Normal file
@ -0,0 +1,187 @@
|
||||
import * as yauzl from 'yauzl';
|
||||
import * as path from 'path';
|
||||
import * as fs from 'node:fs';
|
||||
|
||||
export enum FileTaskType {
|
||||
Import = 'import',
|
||||
Export = 'export',
|
||||
}
|
||||
|
||||
export enum FileImportSource {
|
||||
Generic = 'generic',
|
||||
Notion = 'notion',
|
||||
Confluence = 'confluence',
|
||||
}
|
||||
|
||||
export enum FileTaskStatus {
|
||||
Processing = 'processing',
|
||||
Success = 'success',
|
||||
Failed = 'failed',
|
||||
}
|
||||
|
||||
export function getFileTaskFolderPath(
|
||||
type: FileTaskType,
|
||||
workspaceId: string,
|
||||
): string {
|
||||
switch (type) {
|
||||
case FileTaskType.Import:
|
||||
return `${workspaceId}/imports`;
|
||||
case FileTaskType.Export:
|
||||
return `${workspaceId}/exports`;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts a ZIP archive.
|
||||
*/
|
||||
export async function extractZip(
|
||||
source: string,
|
||||
target: string,
|
||||
): Promise<void> {
|
||||
return extractZipInternal(source, target, true);
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal helper to extract a ZIP, with optional single-nested-ZIP handling.
|
||||
* @param source Path to the ZIP file
|
||||
* @param target Directory to extract into
|
||||
* @param allowNested Whether to check and unwrap one level of nested ZIP
|
||||
*/
|
||||
function extractZipInternal(
|
||||
source: string,
|
||||
target: string,
|
||||
allowNested: boolean,
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
yauzl.open(
|
||||
source,
|
||||
{ lazyEntries: true, decodeStrings: false, autoClose: true },
|
||||
(err, zipfile) => {
|
||||
if (err) return reject(err);
|
||||
|
||||
// Handle one level of nested ZIP if allowed
|
||||
if (allowNested && zipfile.entryCount === 1) {
|
||||
zipfile.readEntry();
|
||||
zipfile.once('entry', (entry) => {
|
||||
const name = entry.fileName.toString('utf8').replace(/^\/+/, '');
|
||||
const isZip =
|
||||
!/\/$/.test(entry.fileName) &&
|
||||
name.toLowerCase().endsWith('.zip');
|
||||
if (isZip) {
|
||||
// temporary name to avoid overwriting file
|
||||
const nestedPath = source.endsWith('.zip')
|
||||
? source.slice(0, -4) + '.inner.zip'
|
||||
: source + '.inner.zip';
|
||||
|
||||
zipfile.openReadStream(entry, (openErr, rs) => {
|
||||
if (openErr) return reject(openErr);
|
||||
const ws = fs.createWriteStream(nestedPath);
|
||||
rs.on('error', reject);
|
||||
ws.on('error', reject);
|
||||
ws.on('finish', () => {
|
||||
zipfile.close();
|
||||
extractZipInternal(nestedPath, target, false)
|
||||
.then(() => {
|
||||
fs.unlinkSync(nestedPath);
|
||||
resolve();
|
||||
})
|
||||
.catch(reject);
|
||||
});
|
||||
rs.pipe(ws);
|
||||
});
|
||||
} else {
|
||||
zipfile.close();
|
||||
extractZipInternal(source, target, false).then(resolve, reject);
|
||||
}
|
||||
});
|
||||
zipfile.once('error', reject);
|
||||
return;
|
||||
}
|
||||
|
||||
// Normal extraction
|
||||
zipfile.readEntry();
|
||||
zipfile.on('entry', (entry) => {
|
||||
const name = entry.fileName.toString('utf8');
|
||||
const safe = name.replace(/^\/+/, '');
|
||||
if (safe.startsWith('__MACOSX/')) {
|
||||
zipfile.readEntry();
|
||||
return;
|
||||
}
|
||||
|
||||
const fullPath = path.join(target, safe);
|
||||
|
||||
// Handle directories
|
||||
if (/\/$/.test(name)) {
|
||||
try {
|
||||
fs.mkdirSync(fullPath, { recursive: true });
|
||||
} catch (mkdirErr: any) {
|
||||
if (mkdirErr.code === 'ENAMETOOLONG') {
|
||||
console.warn(`Skipping directory (path too long): ${fullPath}`);
|
||||
zipfile.readEntry();
|
||||
return;
|
||||
}
|
||||
return reject(mkdirErr);
|
||||
}
|
||||
zipfile.readEntry();
|
||||
return;
|
||||
}
|
||||
|
||||
// Handle files
|
||||
try {
|
||||
fs.mkdirSync(path.dirname(fullPath), { recursive: true });
|
||||
} catch (mkdirErr: any) {
|
||||
if (mkdirErr.code === 'ENAMETOOLONG') {
|
||||
console.warn(
|
||||
`Skipping file directory creation (path too long): ${fullPath}`,
|
||||
);
|
||||
zipfile.readEntry();
|
||||
return;
|
||||
}
|
||||
return reject(mkdirErr);
|
||||
}
|
||||
|
||||
zipfile.openReadStream(entry, (openErr, rs) => {
|
||||
if (openErr) return reject(openErr);
|
||||
|
||||
let ws: fs.WriteStream;
|
||||
try {
|
||||
ws = fs.createWriteStream(fullPath);
|
||||
} catch (openWsErr: any) {
|
||||
if (openWsErr.code === 'ENAMETOOLONG') {
|
||||
console.warn(
|
||||
`Skipping file write (path too long): ${fullPath}`,
|
||||
);
|
||||
zipfile.readEntry();
|
||||
return;
|
||||
}
|
||||
return reject(openWsErr);
|
||||
}
|
||||
|
||||
rs.on('error', (err) => reject(err));
|
||||
ws.on('error', (err) => {
|
||||
if ((err as any).code === 'ENAMETOOLONG') {
|
||||
console.warn(
|
||||
`Skipping file write on stream (path too long): ${fullPath}`,
|
||||
);
|
||||
zipfile.readEntry();
|
||||
} else {
|
||||
reject(err);
|
||||
}
|
||||
});
|
||||
ws.on('finish', () => zipfile.readEntry());
|
||||
rs.pipe(ws);
|
||||
});
|
||||
});
|
||||
|
||||
zipfile.on('end', () => resolve());
|
||||
zipfile.on('error', (err) => reject(err));
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
export function cleanUrlString(url: string): string {
|
||||
if (!url) return null;
|
||||
const [mainUrl] = url.split('?', 1);
|
||||
return mainUrl;
|
||||
}
|
||||
254
apps/server/src/integrations/import/utils/import-formatter.ts
Normal file
254
apps/server/src/integrations/import/utils/import-formatter.ts
Normal file
@ -0,0 +1,254 @@
|
||||
import { getEmbedUrlAndProvider } from '@docmost/editor-ext';
|
||||
import * as path from 'path';
|
||||
import { v7 } from 'uuid';
|
||||
import { InsertableBacklink } from '@docmost/db/types/entity.types';
|
||||
import { Cheerio, CheerioAPI, load } from 'cheerio';
|
||||
|
||||
export async function formatImportHtml(opts: {
|
||||
html: string;
|
||||
currentFilePath: string;
|
||||
filePathToPageMetaMap: Map<
|
||||
string,
|
||||
{ id: string; title: string; slugId: string }
|
||||
>;
|
||||
creatorId: string;
|
||||
sourcePageId: string;
|
||||
workspaceId: string;
|
||||
pageDir?: string;
|
||||
attachmentCandidates?: string[];
|
||||
}): Promise<{ html: string; backlinks: InsertableBacklink[] }> {
|
||||
const {
|
||||
html,
|
||||
currentFilePath,
|
||||
filePathToPageMetaMap,
|
||||
creatorId,
|
||||
sourcePageId,
|
||||
workspaceId,
|
||||
} = opts;
|
||||
const $: CheerioAPI = load(html);
|
||||
const $root: Cheerio<any> = $.root();
|
||||
|
||||
notionFormatter($, $root);
|
||||
defaultHtmlFormatter($, $root);
|
||||
|
||||
const backlinks = await rewriteInternalLinksToMentionHtml(
|
||||
$,
|
||||
$root,
|
||||
currentFilePath,
|
||||
filePathToPageMetaMap,
|
||||
creatorId,
|
||||
sourcePageId,
|
||||
workspaceId,
|
||||
);
|
||||
|
||||
return {
|
||||
html: $root.html() || '',
|
||||
backlinks,
|
||||
};
|
||||
}
|
||||
|
||||
export function defaultHtmlFormatter($: CheerioAPI, $root: Cheerio<any>) {
|
||||
$root.find('a[href]').each((_, el) => {
|
||||
const $el = $(el);
|
||||
const url = $el.attr('href')!;
|
||||
const { provider } = getEmbedUrlAndProvider(url);
|
||||
if (provider === 'iframe') return;
|
||||
|
||||
const embed = `<div data-type=\"embed\" data-src=\"${url}\" data-provider=\"${provider}\" data-align=\"center\" data-width=\"640\" data-height=\"480\"></div>`;
|
||||
$el.replaceWith(embed);
|
||||
});
|
||||
|
||||
$root.find('iframe[src]').each((_, el) => {
|
||||
const $el = $(el);
|
||||
const url = $el.attr('src')!;
|
||||
const { provider } = getEmbedUrlAndProvider(url);
|
||||
|
||||
const embed = `<div data-type=\"embed\" data-src=\"${url}\" data-provider=\"${provider}\" data-align=\"center\" data-width=\"640\" data-height=\"480\"></div>`;
|
||||
$el.replaceWith(embed);
|
||||
});
|
||||
}
|
||||
|
||||
export function notionFormatter($: CheerioAPI, $root: Cheerio<any>) {
|
||||
// remove empty description paragraphs
|
||||
$root.find('p.page-description').each((_, el) => {
|
||||
if (!$(el).text().trim()) $(el).remove();
|
||||
});
|
||||
|
||||
// block math → mathBlock
|
||||
$root.find('figure.equation').each((_: any, fig: any) => {
|
||||
const $fig = $(fig);
|
||||
const tex = $fig
|
||||
.find('annotation[encoding="application/x-tex"]')
|
||||
.text()
|
||||
.trim();
|
||||
const $math = $('<div>')
|
||||
.attr('data-type', 'mathBlock')
|
||||
.attr('data-katex', 'true')
|
||||
.text(tex);
|
||||
$fig.replaceWith($math);
|
||||
});
|
||||
|
||||
// inline math → mathInline
|
||||
$root.find('span.notion-text-equation-token').each((_, tok) => {
|
||||
const $tok = $(tok);
|
||||
const $prev = $tok.prev('style');
|
||||
if ($prev.length) $prev.remove();
|
||||
const tex = $tok
|
||||
.find('annotation[encoding="application/x-tex"]')
|
||||
.text()
|
||||
.trim();
|
||||
const $inline = $('<span>')
|
||||
.attr('data-type', 'mathInline')
|
||||
.attr('data-katex', 'true')
|
||||
.text(tex);
|
||||
$tok.replaceWith($inline);
|
||||
});
|
||||
|
||||
// callouts
|
||||
$root
|
||||
.find('figure.callout')
|
||||
.get()
|
||||
.reverse()
|
||||
.forEach((fig) => {
|
||||
const $fig = $(fig);
|
||||
const $content = $fig.find('div').eq(1);
|
||||
if (!$content.length) return;
|
||||
const $wrapper = $('<div>')
|
||||
.attr('data-type', 'callout')
|
||||
.attr('data-callout-type', 'info');
|
||||
// @ts-ignore
|
||||
$content.children().each((_, child) => $wrapper.append(child));
|
||||
$fig.replaceWith($wrapper);
|
||||
});
|
||||
|
||||
// to-do lists
|
||||
$root.find('ul.to-do-list').each((_, list) => {
|
||||
const $old = $(list);
|
||||
const $new = $('<ul>').attr('data-type', 'taskList');
|
||||
$old.find('li').each((_, li) => {
|
||||
const $li = $(li);
|
||||
const isChecked = $li.find('.checkbox.checkbox-on').length > 0;
|
||||
const text =
|
||||
$li
|
||||
.find('span.to-do-children-unchecked, span.to-do-children-checked')
|
||||
.first()
|
||||
.text()
|
||||
.trim() || '';
|
||||
const $taskItem = $('<li>')
|
||||
.attr('data-type', 'taskItem')
|
||||
.attr('data-checked', String(isChecked));
|
||||
const $label = $('<label>');
|
||||
const $input = $('<input>').attr('type', 'checkbox');
|
||||
if (isChecked) $input.attr('checked', '');
|
||||
$label.append($input, $('<span>'));
|
||||
const $container = $('<div>').append($('<p>').text(text));
|
||||
$taskItem.append($label, $container);
|
||||
$new.append($taskItem);
|
||||
});
|
||||
$old.replaceWith($new);
|
||||
});
|
||||
|
||||
// toggle blocks
|
||||
$root
|
||||
.find('ul.toggle details')
|
||||
.get()
|
||||
.reverse()
|
||||
.forEach((det) => {
|
||||
const $det = $(det);
|
||||
const $li = $det.closest('li');
|
||||
if ($li.length) {
|
||||
$li.before($det);
|
||||
if (!$li.children().length) $li.remove();
|
||||
}
|
||||
const $ul = $det.closest('ul.toggle');
|
||||
if ($ul.length) {
|
||||
$ul.before($det);
|
||||
if (!$ul.children().length) $ul.remove();
|
||||
}
|
||||
});
|
||||
|
||||
// bookmarks
|
||||
$root
|
||||
.find('figure')
|
||||
.filter((_, fig) => $(fig).find('a.bookmark.source').length > 0)
|
||||
.get()
|
||||
.reverse()
|
||||
.forEach((fig) => {
|
||||
const $fig = $(fig);
|
||||
const $link = $fig.find('a.bookmark.source').first();
|
||||
if (!$link.length) return;
|
||||
|
||||
const href = $link.attr('href')!;
|
||||
const title = $link.find('.bookmark-title').text().trim() || href;
|
||||
|
||||
const $newAnchor = $('<a>')
|
||||
.addClass('bookmark source')
|
||||
.attr('href', href)
|
||||
.append($('<div>').addClass('bookmark-info').text(title));
|
||||
|
||||
$fig.replaceWith($newAnchor);
|
||||
});
|
||||
|
||||
// remove toc
|
||||
$root.find('nav.table_of_contents').remove();
|
||||
}
|
||||
|
||||
export function unwrapFromParagraph($: CheerioAPI, $node: Cheerio<any>) {
|
||||
// find the nearest <p> or <a> ancestor
|
||||
let $wrapper = $node.closest('p, a');
|
||||
|
||||
while ($wrapper.length) {
|
||||
// if the wrapper has only our node inside, replace it entirely
|
||||
if ($wrapper.contents().length === 1) {
|
||||
$wrapper.replaceWith($node);
|
||||
} else {
|
||||
// otherwise just move the node to before the wrapper
|
||||
$wrapper.before($node);
|
||||
}
|
||||
// look again for any new wrapper around $node
|
||||
$wrapper = $node.closest('p, a');
|
||||
}
|
||||
}
|
||||
|
||||
export async function rewriteInternalLinksToMentionHtml(
|
||||
$: CheerioAPI,
|
||||
$root: Cheerio<any>,
|
||||
currentFilePath: string,
|
||||
filePathToPageMetaMap: Map<
|
||||
string,
|
||||
{ id: string; title: string; slugId: string }
|
||||
>,
|
||||
creatorId: string,
|
||||
sourcePageId: string,
|
||||
workspaceId: string,
|
||||
): Promise<InsertableBacklink[]> {
|
||||
const normalize = (p: string) => p.replace(/\\/g, '/');
|
||||
const backlinks: InsertableBacklink[] = [];
|
||||
|
||||
$root.find('a[href]').each((_, el) => {
|
||||
const $a = $(el);
|
||||
const raw = $a.attr('href')!;
|
||||
if (raw.startsWith('http') || raw.startsWith('/api/')) return;
|
||||
const resolved = normalize(
|
||||
path.join(path.dirname(currentFilePath), decodeURIComponent(raw)),
|
||||
);
|
||||
const meta = filePathToPageMetaMap.get(resolved);
|
||||
if (!meta) return;
|
||||
const mentionId = v7();
|
||||
const $mention = $('<span>')
|
||||
.attr({
|
||||
'data-type': 'mention',
|
||||
'data-id': mentionId,
|
||||
'data-entity-type': 'page',
|
||||
'data-entity-id': meta.id,
|
||||
'data-label': meta.title,
|
||||
'data-slug-id': meta.slugId,
|
||||
'data-creator-id': creatorId,
|
||||
})
|
||||
.text(meta.title);
|
||||
$a.replaceWith($mention);
|
||||
backlinks.push({ sourcePageId, targetPageId: meta.id, workspaceId });
|
||||
});
|
||||
|
||||
return backlinks;
|
||||
}
|
||||
66
apps/server/src/integrations/import/utils/import.utils.ts
Normal file
66
apps/server/src/integrations/import/utils/import.utils.ts
Normal file
@ -0,0 +1,66 @@
|
||||
import { promises as fs } from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
export async function buildAttachmentCandidates(
|
||||
extractDir: string,
|
||||
): Promise<Map<string, string>> {
|
||||
const map = new Map<string, string>();
|
||||
async function walk(dir: string) {
|
||||
for (const ent of await fs.readdir(dir, { withFileTypes: true })) {
|
||||
const abs = path.join(dir, ent.name);
|
||||
if (ent.isDirectory()) {
|
||||
await walk(abs);
|
||||
} else {
|
||||
if (['.md', '.html'].includes(path.extname(ent.name).toLowerCase())) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const rel = path.relative(extractDir, abs).split(path.sep).join('/');
|
||||
map.set(rel, abs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await walk(extractDir);
|
||||
return map;
|
||||
}
|
||||
|
||||
export function resolveRelativeAttachmentPath(
|
||||
raw: string,
|
||||
pageDir: string,
|
||||
attachmentCandidates: Map<string, string>,
|
||||
): string | null {
|
||||
const mainRel = decodeURIComponent(raw.replace(/^\.?\/+/, ''));
|
||||
const fallback = path.normalize(path.join(pageDir, mainRel));
|
||||
|
||||
if (attachmentCandidates.has(mainRel)) {
|
||||
return mainRel;
|
||||
}
|
||||
if (attachmentCandidates.has(fallback)) {
|
||||
return fallback;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export async function collectMarkdownAndHtmlFiles(
|
||||
dir: string,
|
||||
): Promise<string[]> {
|
||||
const results: string[] = [];
|
||||
|
||||
async function walk(current: string) {
|
||||
const entries = await fs.readdir(current, { withFileTypes: true });
|
||||
for (const ent of entries) {
|
||||
const fullPath = path.join(current, ent.name);
|
||||
if (ent.isDirectory()) {
|
||||
await walk(fullPath);
|
||||
} else if (
|
||||
['.md', '.html'].includes(path.extname(ent.name).toLowerCase())
|
||||
) {
|
||||
results.push(fullPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await walk(dir);
|
||||
return results;
|
||||
}
|
||||
@ -3,6 +3,7 @@ export enum QueueName {
|
||||
ATTACHMENT_QUEUE = '{attachment-queue}',
|
||||
GENERAL_QUEUE = '{general-queue}',
|
||||
BILLING_QUEUE = '{billing-queue}',
|
||||
FILE_TASK_QUEUE = '{file-task-queue}',
|
||||
}
|
||||
|
||||
export enum QueueJob {
|
||||
@ -19,4 +20,7 @@ export enum QueueJob {
|
||||
TRIAL_ENDED = 'trial-ended',
|
||||
WELCOME_EMAIL = 'welcome-email',
|
||||
FIRST_PAYMENT_EMAIL = 'first-payment-email',
|
||||
|
||||
IMPORT_TASK = 'import-task',
|
||||
EXPORT_TASK = 'export-task',
|
||||
}
|
||||
|
||||
@ -49,6 +49,14 @@ import { BacklinksProcessor } from './processors/backlinks.processor';
|
||||
BullModule.registerQueue({
|
||||
name: QueueName.BILLING_QUEUE,
|
||||
}),
|
||||
BullModule.registerQueue({
|
||||
name: QueueName.FILE_TASK_QUEUE,
|
||||
defaultJobOptions: {
|
||||
removeOnComplete: true,
|
||||
removeOnFail: true,
|
||||
attempts: 1,
|
||||
},
|
||||
}),
|
||||
],
|
||||
exports: [BullModule],
|
||||
providers: [BacklinksProcessor],
|
||||
|
||||
@ -3,8 +3,11 @@ import {
|
||||
LocalStorageConfig,
|
||||
StorageOption,
|
||||
} from '../interfaces';
|
||||
import { join } from 'path';
|
||||
import { join, dirname } from 'path';
|
||||
import * as fs from 'fs-extra';
|
||||
import { Readable } from 'stream';
|
||||
import { createReadStream, createWriteStream } from 'node:fs';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
|
||||
export class LocalDriver implements StorageDriver {
|
||||
private readonly config: LocalStorageConfig;
|
||||
@ -25,6 +28,16 @@ export class LocalDriver implements StorageDriver {
|
||||
}
|
||||
}
|
||||
|
||||
async uploadStream(filePath: string, file: Readable): Promise<void> {
|
||||
try {
|
||||
const fullPath = this._fullPath(filePath);
|
||||
await fs.mkdir(dirname(fullPath), { recursive: true });
|
||||
await pipeline(file, createWriteStream(fullPath));
|
||||
} catch (err) {
|
||||
throw new Error(`Failed to upload file: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async copy(fromFilePath: string, toFilePath: string): Promise<void> {
|
||||
try {
|
||||
if (await this.exists(fromFilePath)) {
|
||||
@ -43,6 +56,14 @@ export class LocalDriver implements StorageDriver {
|
||||
}
|
||||
}
|
||||
|
||||
async readStream(filePath: string): Promise<Readable> {
|
||||
try {
|
||||
return createReadStream(this._fullPath(filePath));
|
||||
} catch (err) {
|
||||
throw new Error(`Failed to read file: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async exists(filePath: string): Promise<boolean> {
|
||||
try {
|
||||
return await fs.pathExists(this._fullPath(filePath));
|
||||
|
||||
@ -12,6 +12,7 @@ import { streamToBuffer } from '../storage.utils';
|
||||
import { Readable } from 'stream';
|
||||
import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
|
||||
import { getMimeType } from '../../../common/helpers';
|
||||
import { Upload } from '@aws-sdk/lib-storage';
|
||||
|
||||
export class S3Driver implements StorageDriver {
|
||||
private readonly s3Client: S3Client;
|
||||
@ -40,6 +41,26 @@ export class S3Driver implements StorageDriver {
|
||||
}
|
||||
}
|
||||
|
||||
async uploadStream(filePath: string, file: Readable): Promise<void> {
|
||||
try {
|
||||
const contentType = getMimeType(filePath);
|
||||
|
||||
const upload = new Upload({
|
||||
client: this.s3Client,
|
||||
params: {
|
||||
Bucket: this.config.bucket,
|
||||
Key: filePath,
|
||||
Body: file,
|
||||
ContentType: contentType,
|
||||
},
|
||||
});
|
||||
|
||||
await upload.done();
|
||||
} catch (err) {
|
||||
throw new Error(`Failed to upload file: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async copy(fromFilePath: string, toFilePath: string): Promise<void> {
|
||||
try {
|
||||
if (await this.exists(fromFilePath)) {
|
||||
@ -71,6 +92,21 @@ export class S3Driver implements StorageDriver {
|
||||
}
|
||||
}
|
||||
|
||||
async readStream(filePath: string): Promise<Readable> {
|
||||
try {
|
||||
const command = new GetObjectCommand({
|
||||
Bucket: this.config.bucket,
|
||||
Key: filePath,
|
||||
});
|
||||
|
||||
const response = await this.s3Client.send(command);
|
||||
|
||||
return response.Body as Readable;
|
||||
} catch (err) {
|
||||
throw new Error(`Failed to read file from S3: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async exists(filePath: string): Promise<boolean> {
|
||||
try {
|
||||
const command = new HeadObjectCommand({
|
||||
|
||||
@ -1,10 +1,16 @@
|
||||
import { Readable } from 'stream';
|
||||
|
||||
export interface StorageDriver {
|
||||
upload(filePath: string, file: Buffer): Promise<void>;
|
||||
|
||||
uploadStream(filePath: string, file: Readable): Promise<void>;
|
||||
|
||||
copy(fromFilePath: string, toFilePath: string): Promise<void>;
|
||||
|
||||
read(filePath: string): Promise<Buffer>;
|
||||
|
||||
readStream(filePath: string): Promise<Readable>;
|
||||
|
||||
exists(filePath: string): Promise<boolean>;
|
||||
|
||||
getUrl(filePath: string): string;
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import { Inject, Injectable, Logger } from '@nestjs/common';
|
||||
import { STORAGE_DRIVER_TOKEN } from './constants/storage.constants';
|
||||
import { StorageDriver } from './interfaces';
|
||||
import { Readable } from 'stream';
|
||||
|
||||
@Injectable()
|
||||
export class StorageService {
|
||||
@ -14,6 +15,11 @@ export class StorageService {
|
||||
this.logger.debug(`File uploaded successfully. Path: ${filePath}`);
|
||||
}
|
||||
|
||||
async uploadStream(filePath: string, fileContent: Readable) {
|
||||
await this.storageDriver.uploadStream(filePath, fileContent);
|
||||
this.logger.debug(`File uploaded successfully. Path: ${filePath}`);
|
||||
}
|
||||
|
||||
async copy(fromFilePath: string, toFilePath: string) {
|
||||
await this.storageDriver.copy(fromFilePath, toFilePath);
|
||||
this.logger.debug(`File copied successfully. Path: ${toFilePath}`);
|
||||
@ -23,6 +29,10 @@ export class StorageService {
|
||||
return this.storageDriver.read(filePath);
|
||||
}
|
||||
|
||||
async readStream(filePath: string): Promise<Readable> {
|
||||
return this.storageDriver.readStream(filePath);
|
||||
}
|
||||
|
||||
async exists(filePath: string): Promise<boolean> {
|
||||
return this.storageDriver.exists(filePath);
|
||||
}
|
||||
|
||||
@ -88,7 +88,7 @@ async function bootstrap() {
|
||||
const logger = new Logger('NestApplication');
|
||||
|
||||
process.on('unhandledRejection', (reason, promise) => {
|
||||
logger.error(`UnhandledRejection: ${promise}, reason: ${reason}`);
|
||||
logger.error(`UnhandledRejection, reason: ${reason}`, promise);
|
||||
});
|
||||
|
||||
process.on('uncaughtException', (error) => {
|
||||
|
||||
@ -16,4 +16,5 @@ export * from "./lib/drawio";
|
||||
export * from "./lib/excalidraw";
|
||||
export * from "./lib/embed";
|
||||
export * from "./lib/mention";
|
||||
export * from "./lib/markdown";
|
||||
export * from "./lib/markdown";
|
||||
export * from "./lib/embed-provider";
|
||||
|
||||
@ -7,102 +7,109 @@ export interface IEmbedProvider {
|
||||
|
||||
export const embedProviders: IEmbedProvider[] = [
|
||||
{
|
||||
id: 'loom',
|
||||
name: 'Loom',
|
||||
id: "loom",
|
||||
name: "Loom",
|
||||
regex: /^https?:\/\/(?:www\.)?loom\.com\/(?:share|embed)\/([\da-zA-Z]+)\/?/,
|
||||
getEmbedUrl: (match, url) => {
|
||||
if(url.includes("/embed/")){
|
||||
if (url.includes("/embed/")) {
|
||||
return url;
|
||||
}
|
||||
return `https://loom.com/embed/${match[1]}`;
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'airtable',
|
||||
name: 'Airtable',
|
||||
id: "airtable",
|
||||
name: "Airtable",
|
||||
regex: /^https:\/\/(www.)?airtable.com\/([a-zA-Z0-9]{2,})\/.*/,
|
||||
getEmbedUrl: (match, url: string) => {
|
||||
const path = url.split('airtable.com/');
|
||||
if(url.includes("/embed/")){
|
||||
const path = url.split("airtable.com/");
|
||||
if (url.includes("/embed/")) {
|
||||
return url;
|
||||
}
|
||||
return `https://airtable.com/embed/${path[1]}`;
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'figma',
|
||||
name: 'Figma',
|
||||
regex: /^https:\/\/[\w\.-]+\.?figma.com\/(file|proto|board|design|slides|deck)\/([0-9a-zA-Z]{22,128})/,
|
||||
id: "figma",
|
||||
name: "Figma",
|
||||
regex:
|
||||
/^https:\/\/[\w\.-]+\.?figma.com\/(file|proto|board|design|slides|deck)\/([0-9a-zA-Z]{22,128})/,
|
||||
getEmbedUrl: (match, url: string) => {
|
||||
return `https://www.figma.com/embed?url=${url}&embed_host=docmost`;
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
'id': 'typeform',
|
||||
name: 'Typeform',
|
||||
id: "typeform",
|
||||
name: "Typeform",
|
||||
regex: /^(https?:)?(\/\/)?[\w\.]+\.typeform\.com\/to\/.+/,
|
||||
getEmbedUrl: (match, url: string) => {
|
||||
return url;
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'miro',
|
||||
name: 'Miro',
|
||||
id: "miro",
|
||||
name: "Miro",
|
||||
regex: /^https:\/\/(www\.)?miro\.com\/app\/board\/([\w-]+=)/,
|
||||
getEmbedUrl: (match, url) => {
|
||||
if(url.includes("/live-embed/")){
|
||||
if (url.includes("/live-embed/")) {
|
||||
return url;
|
||||
}
|
||||
return `https://miro.com/app/live-embed/${match[2]}?embedMode=view_only_without_ui&autoplay=true&embedSource=docmost`;
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'youtube',
|
||||
name: 'YouTube',
|
||||
regex: /^((?:https?:)?\/\/)?((?:www|m|music)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w\-]+\?v=|embed\/|v\/)?)([\w\-]+)(\S+)?$/,
|
||||
id: "youtube",
|
||||
name: "YouTube",
|
||||
regex:
|
||||
/^((?:https?:)?\/\/)?((?:www|m|music)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w\-]+\?v=|embed\/|v\/)?)([\w\-]+)(\S+)?$/,
|
||||
getEmbedUrl: (match, url) => {
|
||||
if (url.includes("/embed/")){
|
||||
if (url.includes("/embed/")) {
|
||||
return url;
|
||||
}
|
||||
return `https://www.youtube-nocookie.com/embed/${match[5]}`;
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'vimeo',
|
||||
name: 'Vimeo',
|
||||
regex: /^(https:)?\/\/(?:www\.|player\.)?vimeo.com\/(?:channels\/(?:\w+\/)?|groups\/([^/]*)\/videos\/|album\/(\d+)\/video\/|video\/|)(\d+)/,
|
||||
id: "vimeo",
|
||||
name: "Vimeo",
|
||||
regex:
|
||||
/^(https:)?\/\/(?:www\.|player\.)?vimeo.com\/(?:channels\/(?:\w+\/)?|groups\/([^/]*)\/videos\/|album\/(\d+)\/video\/|video\/|)(\d+)/,
|
||||
getEmbedUrl: (match) => {
|
||||
return `https://player.vimeo.com/video/${match[4]}`;
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'framer',
|
||||
name: 'Framer',
|
||||
id: "framer",
|
||||
name: "Framer",
|
||||
regex: /^https:\/\/(www\.)?framer\.com\/embed\/([\w-]+)/,
|
||||
getEmbedUrl: (match, url: string) => {
|
||||
return url;
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'gdrive',
|
||||
name: 'Google Drive',
|
||||
regex: /^((?:https?:)?\/\/)?((?:www|m)\.)?(drive\.google\.com)\/file\/d\/([a-zA-Z0-9_-]+)\/.*$/,
|
||||
id: "gdrive",
|
||||
name: "Google Drive",
|
||||
regex:
|
||||
/^((?:https?:)?\/\/)?((?:www|m)\.)?(drive\.google\.com)\/file\/d\/([a-zA-Z0-9_-]+)\/.*$/,
|
||||
getEmbedUrl: (match) => {
|
||||
return `https://drive.google.com/file/d/${match[4]}/preview`;
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
id: 'gsheets',
|
||||
name: 'Google Sheets',
|
||||
regex: /^((?:https?:)?\/\/)?((?:www|m)\.)?(docs\.google\.com)\/spreadsheets\/d\/e\/([a-zA-Z0-9_-]+)\/.*$/,
|
||||
id: "gsheets",
|
||||
name: "Google Sheets",
|
||||
regex:
|
||||
/^((?:https?:)?\/\/)?((?:www|m)\.)?(docs\.google\.com)\/spreadsheets\/d\/e\/([a-zA-Z0-9_-]+)\/.*$/,
|
||||
getEmbedUrl: (match, url: string) => {
|
||||
return url
|
||||
}
|
||||
return url;
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
export function getEmbedProviderById(id: string) {
|
||||
return embedProviders.find(provider => provider.id.toLowerCase() === id.toLowerCase());
|
||||
return embedProviders.find(
|
||||
(provider) => provider.id.toLowerCase() === id.toLowerCase(),
|
||||
);
|
||||
}
|
||||
|
||||
export interface IEmbedResult {
|
||||
@ -116,14 +123,12 @@ export function getEmbedUrlAndProvider(url: string): IEmbedResult {
|
||||
if (match) {
|
||||
return {
|
||||
embedUrl: provider.getEmbedUrl(match, url),
|
||||
provider: provider.name.toLowerCase()
|
||||
provider: provider.name.toLowerCase(),
|
||||
};
|
||||
}
|
||||
}
|
||||
return {
|
||||
embedUrl: url,
|
||||
provider: 'iframe',
|
||||
provider: "iframe",
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@ -80,7 +80,7 @@ export const TiptapImage = Image.extend<ImageOptions>({
|
||||
default: undefined,
|
||||
parseHTML: (element) => element.getAttribute("data-attachment-id"),
|
||||
renderHTML: (attributes: ImageAttributes) => ({
|
||||
"data-attachment-id": attributes.align,
|
||||
"data-attachment-id": attributes.attachmentId,
|
||||
}),
|
||||
},
|
||||
size: {
|
||||
|
||||
@ -51,8 +51,13 @@ export const TrailingNode = Extension.create<TrailingNodeExtensionOptions>({
|
||||
},
|
||||
state: {
|
||||
init: (_, state) => {
|
||||
const lastNode = state.tr.doc.lastChild
|
||||
return !nodeEqualsType({ node: lastNode, types: disabledNodes })
|
||||
try {
|
||||
const lastNode = state.tr.doc.lastChild
|
||||
return !nodeEqualsType({ node: lastNode, types: disabledNodes })
|
||||
} catch (err){
|
||||
console.log(err)
|
||||
}
|
||||
return true;
|
||||
},
|
||||
apply: (tr, value) => {
|
||||
if (!tr.docChanged) {
|
||||
|
||||
@ -57,7 +57,7 @@ export const TiptapVideo = Node.create<VideoOptions>({
|
||||
default: undefined,
|
||||
parseHTML: (element) => element.getAttribute("data-attachment-id"),
|
||||
renderHTML: (attributes: VideoAttributes) => ({
|
||||
"data-attachment-id": attributes.align,
|
||||
"data-attachment-id": attributes.attachmentId,
|
||||
}),
|
||||
},
|
||||
width: {
|
||||
@ -84,6 +84,14 @@ export const TiptapVideo = Node.create<VideoOptions>({
|
||||
};
|
||||
},
|
||||
|
||||
parseHTML() {
|
||||
return [
|
||||
{
|
||||
tag: 'video',
|
||||
},
|
||||
]
|
||||
},
|
||||
|
||||
renderHTML({ HTMLAttributes }) {
|
||||
return [
|
||||
"video",
|
||||
|
||||
215
pnpm-lock.yaml
generated
215
pnpm-lock.yaml
generated
@ -405,6 +405,9 @@ importers:
|
||||
'@aws-sdk/client-s3':
|
||||
specifier: 3.701.0
|
||||
version: 3.701.0
|
||||
'@aws-sdk/lib-storage':
|
||||
specifier: 3.701.0
|
||||
version: 3.701.0(@aws-sdk/client-s3@3.701.0)
|
||||
'@aws-sdk/s3-request-presigner':
|
||||
specifier: 3.701.0
|
||||
version: 3.701.0
|
||||
@ -480,6 +483,9 @@ importers:
|
||||
cache-manager:
|
||||
specifier: ^6.4.0
|
||||
version: 6.4.0
|
||||
cheerio:
|
||||
specifier: ^1.0.0
|
||||
version: 1.0.0
|
||||
class-transformer:
|
||||
specifier: ^0.5.1
|
||||
version: 0.5.1
|
||||
@ -494,7 +500,7 @@ importers:
|
||||
version: 11.3.0
|
||||
happy-dom:
|
||||
specifier: ^15.11.6
|
||||
version: 15.11.6
|
||||
version: 15.11.7
|
||||
jsonwebtoken:
|
||||
specifier: ^9.0.2
|
||||
version: 9.0.2
|
||||
@ -552,9 +558,15 @@ importers:
|
||||
stripe:
|
||||
specifier: ^17.5.0
|
||||
version: 17.5.0
|
||||
tmp-promise:
|
||||
specifier: ^3.0.3
|
||||
version: 3.0.3
|
||||
ws:
|
||||
specifier: ^8.18.0
|
||||
version: 8.18.0
|
||||
yauzl:
|
||||
specifier: ^3.2.0
|
||||
version: 3.2.0
|
||||
devDependencies:
|
||||
'@eslint/js':
|
||||
specifier: ^9.20.0
|
||||
@ -604,6 +616,9 @@ importers:
|
||||
'@types/ws':
|
||||
specifier: ^8.5.14
|
||||
version: 8.5.14
|
||||
'@types/yauzl':
|
||||
specifier: ^2.10.3
|
||||
version: 2.10.3
|
||||
eslint:
|
||||
specifier: ^9.20.1
|
||||
version: 9.20.1(jiti@1.21.0)
|
||||
@ -769,6 +784,12 @@ packages:
|
||||
peerDependencies:
|
||||
'@aws-sdk/client-sts': ^3.696.0
|
||||
|
||||
'@aws-sdk/lib-storage@3.701.0':
|
||||
resolution: {integrity: sha512-eAbJ/3OgyFp1NnFdQfkZ7PuKCjrhbSQWf0EVTMhlg4aE5piCZ1We38NI1dQ58yr53rGc2gBkbYr8+/9CehpEvA==}
|
||||
engines: {node: '>=16.0.0'}
|
||||
peerDependencies:
|
||||
'@aws-sdk/client-s3': ^3.701.0
|
||||
|
||||
'@aws-sdk/middleware-bucket-endpoint@3.696.0':
|
||||
resolution: {integrity: sha512-V07jishKHUS5heRNGFpCWCSTjRJyQLynS/ncUeE8ZYtG66StOOQWftTwDfFOSoXlIqrXgb4oT9atryzXq7Z4LQ==}
|
||||
engines: {node: '>=16.0.0'}
|
||||
@ -3302,8 +3323,8 @@ packages:
|
||||
resolution: {integrity: sha512-bNwBYYmN8Eh9RyjS1p2gW6MIhSO2rl7X9QeLM8iTdcGRP+eDiIWDt66c9IysCc22gefKszZv+ubV9qZc7hdESg==}
|
||||
engines: {node: '>=16.0.0'}
|
||||
|
||||
'@smithy/types@4.1.0':
|
||||
resolution: {integrity: sha512-enhjdwp4D7CXmwLtD6zbcDMbo6/T6WtuuKCY49Xxc6OMOmUWlBEBDREsxxgV2LIdeQPW756+f97GzcgAwp3iLw==}
|
||||
'@smithy/types@4.3.0':
|
||||
resolution: {integrity: sha512-+1iaIQHthDh9yaLhRzaoQxRk+l9xlk+JjMFxGRhNLz+m9vKOkjNeU8QuB4w3xvzHyVR/BVlp/4AXDHjoRIkfgQ==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
'@smithy/url-parser@3.0.11':
|
||||
@ -4093,6 +4114,9 @@ packages:
|
||||
'@types/yargs@17.0.32':
|
||||
resolution: {integrity: sha512-xQ67Yc/laOG5uMfX/093MRlGGCIBzZMarVa+gfNKJxWAIgykYpVGkBdbqEzGDDfCrVUj6Hiff4mTZ5BA6TmAog==}
|
||||
|
||||
'@types/yauzl@2.10.3':
|
||||
resolution: {integrity: sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==}
|
||||
|
||||
'@typescript-eslint/eslint-plugin@8.17.0':
|
||||
resolution: {integrity: sha512-HU1KAdW3Tt8zQkdvNoIijfWDMvdSweFYm4hWh+KwhPstv+sCmWb89hCIP8msFm9N1R/ooh9honpSuvqKWlYy3w==}
|
||||
engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0}
|
||||
@ -4577,6 +4601,9 @@ packages:
|
||||
bluebird@3.7.2:
|
||||
resolution: {integrity: sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==}
|
||||
|
||||
boolbase@1.0.0:
|
||||
resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==}
|
||||
|
||||
bowser@2.11.0:
|
||||
resolution: {integrity: sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==}
|
||||
|
||||
@ -4610,12 +4637,18 @@ packages:
|
||||
bser@2.1.1:
|
||||
resolution: {integrity: sha512-gQxTNE/GAfIIrmHLUE3oJyp5FO6HRBfhjnw4/wMmA63ZGDJnWBmgY/lyQBpnDUkGmAhbSe39tx2d/iTOAfglwQ==}
|
||||
|
||||
buffer-crc32@0.2.13:
|
||||
resolution: {integrity: sha512-VO9Ht/+p3SN7SKWqcrgEzjGbRSJYTx+Q1pTQC0wrWqHx0vpJraQ6GtHx8tvcg1rlK1byhU5gccxgOgj7B0TDkQ==}
|
||||
|
||||
buffer-equal-constant-time@1.0.1:
|
||||
resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==}
|
||||
|
||||
buffer-from@1.1.2:
|
||||
resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==}
|
||||
|
||||
buffer@5.6.0:
|
||||
resolution: {integrity: sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw==}
|
||||
|
||||
buffer@5.7.1:
|
||||
resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==}
|
||||
|
||||
@ -4681,6 +4714,13 @@ packages:
|
||||
resolution: {integrity: sha512-drVkSqfwA+TvuEhFipiR1OC9boEGZL5RrWvVsOthdcvQNXyCCuKkEiTOTXZ7qxSf/GLwq4GvzfrQD/Wz325hgw==}
|
||||
engines: {node: '>=16'}
|
||||
|
||||
cheerio-select@2.1.0:
|
||||
resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==}
|
||||
|
||||
cheerio@1.0.0:
|
||||
resolution: {integrity: sha512-quS9HgjQpdaXOvsZz82Oz7uxtXiy6UIsIQcpBj7HRw2M63Skasm9qlDocAM7jNuaxdhpPU7c4kJN+gA5MCu4ww==}
|
||||
engines: {node: '>=18.17'}
|
||||
|
||||
chevrotain-allstar@0.3.1:
|
||||
resolution: {integrity: sha512-b7g+y9A0v4mxCW1qUhf3BSVPg+/NvGErk/dOkrDaHA0nQIQGAtrOjlX//9OQtRlSCy+x9rfB5N8yC71lH1nvMw==}
|
||||
peerDependencies:
|
||||
@ -4928,6 +4968,9 @@ packages:
|
||||
resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==}
|
||||
engines: {node: '>= 8'}
|
||||
|
||||
css-select@5.1.0:
|
||||
resolution: {integrity: sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==}
|
||||
|
||||
css-what@6.1.0:
|
||||
resolution: {integrity: sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==}
|
||||
engines: {node: '>= 6'}
|
||||
@ -5322,6 +5365,9 @@ packages:
|
||||
emoji-regex@9.2.2:
|
||||
resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==}
|
||||
|
||||
encoding-sniffer@0.2.0:
|
||||
resolution: {integrity: sha512-ju7Wq1kg04I3HtiYIOrUrdfdDvkyO9s5XM8QAj/bN61Yo/Vb4vgJxy5vi4Yxk01gWHbrofpPtpxM8bKger9jhg==}
|
||||
|
||||
end-of-stream@1.4.4:
|
||||
resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==}
|
||||
|
||||
@ -5826,8 +5872,8 @@ packages:
|
||||
hachure-fill@0.5.2:
|
||||
resolution: {integrity: sha512-3GKBOn+m2LX9iq+JC1064cSFprJY4jL1jCXTcpnfER5HYE2l/4EfWSGzkPa/ZDBmYI0ZOEj5VHV/eKnPGkHuOg==}
|
||||
|
||||
happy-dom@15.11.6:
|
||||
resolution: {integrity: sha512-elX7iUTu+5+3b2+NGQc0L3eWyq9jKhuJJ4GpOMxxT/c2pg9O3L5H3ty2VECX0XXZgRmmRqXyOK8brA2hDI6LsQ==}
|
||||
happy-dom@15.11.7:
|
||||
resolution: {integrity: sha512-KyrFvnl+J9US63TEzwoiJOQzZBJY7KgBushJA8X61DMbNsH+2ONkDuLDnCnwUiPTF42tLoEmrPyoqbenVA5zrg==}
|
||||
engines: {node: '>=18.0.0'}
|
||||
|
||||
has-bigints@1.0.2:
|
||||
@ -5905,6 +5951,9 @@ packages:
|
||||
htmlparser2@8.0.2:
|
||||
resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==}
|
||||
|
||||
htmlparser2@9.1.0:
|
||||
resolution: {integrity: sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ==}
|
||||
|
||||
http-errors@2.0.0:
|
||||
resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==}
|
||||
engines: {node: '>= 0.8'}
|
||||
@ -6978,6 +7027,9 @@ packages:
|
||||
resolution: {integrity: sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==}
|
||||
deprecated: This package is no longer supported.
|
||||
|
||||
nth-check@2.1.1:
|
||||
resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==}
|
||||
|
||||
nwsapi@2.2.16:
|
||||
resolution: {integrity: sha512-F1I/bimDpj3ncaNDhfyMWuFqmQDBwDB0Fogc2qpL3BWvkQteFD/8BzWuIRl83rq0DXfm8SGt/HFhLXZyljTXcQ==}
|
||||
|
||||
@ -7124,6 +7176,12 @@ packages:
|
||||
resolution: {integrity: sha512-3YHlOa/JgH6Mnpr05jP9eDG254US9ek25LyIxZlDItp2iJtwyaXQb57lBYLdT3MowkUFYEV2XXNAYIPlESvJlA==}
|
||||
engines: {node: '>= 0.10'}
|
||||
|
||||
parse5-htmlparser2-tree-adapter@7.1.0:
|
||||
resolution: {integrity: sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==}
|
||||
|
||||
parse5-parser-stream@7.1.2:
|
||||
resolution: {integrity: sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==}
|
||||
|
||||
parse5@7.1.2:
|
||||
resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==}
|
||||
|
||||
@ -7196,6 +7254,9 @@ packages:
|
||||
resolution: {integrity: sha512-nri2TO5JE3/mRryik9LlHFT53cgHfRK0Lt0BAZQXku/AW3E6XLt2GaY8siWi7dvW/m1z0ecn+J+bpDa9ZN3IsQ==}
|
||||
engines: {node: '>=18'}
|
||||
|
||||
pend@1.2.0:
|
||||
resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==}
|
||||
|
||||
pg-cloudflare@1.1.1:
|
||||
resolution: {integrity: sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==}
|
||||
|
||||
@ -8057,6 +8118,9 @@ packages:
|
||||
resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==}
|
||||
engines: {node: '>= 0.8'}
|
||||
|
||||
stream-browserify@3.0.0:
|
||||
resolution: {integrity: sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA==}
|
||||
|
||||
streamsearch@1.1.0:
|
||||
resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==}
|
||||
engines: {node: '>=10.0.0'}
|
||||
@ -8248,6 +8312,9 @@ packages:
|
||||
resolution: {integrity: sha512-QNtgIqSUb9o2CoUjX9T5TwaIvUUJFU1+12PJkgt42DFV2yf9J6549yTF2uGloQsJ/JOC8X+gIB81ind97hRiIQ==}
|
||||
hasBin: true
|
||||
|
||||
tmp-promise@3.0.3:
|
||||
resolution: {integrity: sha512-RwM7MoPojPxsOBYnyd2hy0bxtIlVrihNs9pj5SUvY8Zz1sQcQG2tG1hSr8PDxfgEB8RNKDhqbIlroIarSNDNsQ==}
|
||||
|
||||
tmp@0.0.33:
|
||||
resolution: {integrity: sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==}
|
||||
engines: {node: '>=0.6.0'}
|
||||
@ -8487,6 +8554,10 @@ packages:
|
||||
undici-types@6.20.0:
|
||||
resolution: {integrity: sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==}
|
||||
|
||||
undici@6.21.3:
|
||||
resolution: {integrity: sha512-gBLkYIlEnSp8pFbT64yFgGE6UIB9tAkhukC23PmMDCe5Nd+cRqKxSjw5y54MK2AZMgZfJWMaNE4nYUHgi1XEOw==}
|
||||
engines: {node: '>=18.17'}
|
||||
|
||||
unicode-canonical-property-names-ecmascript@2.0.0:
|
||||
resolution: {integrity: sha512-yY5PpDlfVIU5+y/BSCxAJRBIS1Zc2dDG3Ujq+sR0U+JjUevW2JhocOF+soROYDSaAezOzOKuyyixhD6mBknSmQ==}
|
||||
engines: {node: '>=4'}
|
||||
@ -8904,6 +8975,10 @@ packages:
|
||||
resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==}
|
||||
engines: {node: '>=12'}
|
||||
|
||||
yauzl@3.2.0:
|
||||
resolution: {integrity: sha512-Ow9nuGZE+qp1u4JIPvg+uCiUr7xGQWdff7JQSk5VGYTAZMDe2q8lxJ10ygv10qmSj031Ty/6FNJpLO4o1Sgc+w==}
|
||||
engines: {node: '>=12'}
|
||||
|
||||
yjs@13.6.20:
|
||||
resolution: {integrity: sha512-Z2YZI+SYqK7XdWlloI3lhMiKnCdFCVC4PchpdO+mCYwtiTwncjUbnRK9R1JmkNfdmHyDXuWN3ibJAt0wsqTbLQ==}
|
||||
engines: {node: '>=16.0.0', npm: '>=8.0.0'}
|
||||
@ -9334,6 +9409,17 @@ snapshots:
|
||||
'@smithy/types': 3.7.2
|
||||
tslib: 2.8.1
|
||||
|
||||
'@aws-sdk/lib-storage@3.701.0(@aws-sdk/client-s3@3.701.0)':
|
||||
dependencies:
|
||||
'@aws-sdk/client-s3': 3.701.0
|
||||
'@smithy/abort-controller': 3.1.9
|
||||
'@smithy/middleware-endpoint': 3.2.8
|
||||
'@smithy/smithy-client': 3.7.0
|
||||
buffer: 5.6.0
|
||||
events: 3.3.0
|
||||
stream-browserify: 3.0.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@aws-sdk/middleware-bucket-endpoint@3.696.0':
|
||||
dependencies:
|
||||
'@aws-sdk/types': 3.696.0
|
||||
@ -9471,7 +9557,7 @@ snapshots:
|
||||
|
||||
'@aws-sdk/types@3.734.0':
|
||||
dependencies:
|
||||
'@smithy/types': 4.1.0
|
||||
'@smithy/types': 4.3.0
|
||||
tslib: 2.8.1
|
||||
|
||||
'@aws-sdk/util-arn-parser@3.693.0':
|
||||
@ -9564,7 +9650,7 @@ snapshots:
|
||||
'@babel/traverse': 7.25.9
|
||||
'@babel/types': 7.24.6
|
||||
convert-source-map: 2.0.0
|
||||
debug: 4.3.7
|
||||
debug: 4.4.0
|
||||
gensync: 1.0.0-beta.2
|
||||
json5: 2.2.3
|
||||
semver: 6.3.1
|
||||
@ -9690,7 +9776,7 @@ snapshots:
|
||||
'@babel/core': 7.26.0
|
||||
'@babel/helper-compilation-targets': 7.25.9
|
||||
'@babel/helper-plugin-utils': 7.25.9
|
||||
debug: 4.3.7
|
||||
debug: 4.4.0
|
||||
lodash.debounce: 4.0.8
|
||||
resolve: 1.22.8
|
||||
transitivePeerDependencies:
|
||||
@ -10615,7 +10701,7 @@ snapshots:
|
||||
'@babel/parser': 7.26.2
|
||||
'@babel/template': 7.25.9
|
||||
'@babel/types': 7.26.0
|
||||
debug: 4.3.7
|
||||
debug: 4.4.0
|
||||
globals: 11.12.0
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
@ -10627,7 +10713,7 @@ snapshots:
|
||||
'@babel/parser': 7.27.0
|
||||
'@babel/template': 7.27.0
|
||||
'@babel/types': 7.27.0
|
||||
debug: 4.3.7
|
||||
debug: 4.4.0
|
||||
globals: 11.12.0
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
@ -12383,7 +12469,7 @@ snapshots:
|
||||
dependencies:
|
||||
tslib: 2.8.1
|
||||
|
||||
'@smithy/types@4.1.0':
|
||||
'@smithy/types@4.3.0':
|
||||
dependencies:
|
||||
tslib: 2.8.1
|
||||
|
||||
@ -13261,6 +13347,10 @@ snapshots:
|
||||
dependencies:
|
||||
'@types/yargs-parser': 21.0.3
|
||||
|
||||
'@types/yauzl@2.10.3':
|
||||
dependencies:
|
||||
'@types/node': 22.13.4
|
||||
|
||||
'@typescript-eslint/eslint-plugin@8.17.0(@typescript-eslint/parser@8.17.0(eslint@9.15.0(jiti@1.21.0))(typescript@5.7.2))(eslint@9.15.0(jiti@1.21.0))(typescript@5.7.2)':
|
||||
dependencies:
|
||||
'@eslint-community/regexpp': 4.12.1
|
||||
@ -13335,7 +13425,7 @@ snapshots:
|
||||
dependencies:
|
||||
'@typescript-eslint/typescript-estree': 8.17.0(typescript@5.7.2)
|
||||
'@typescript-eslint/utils': 8.17.0(eslint@9.15.0(jiti@1.21.0))(typescript@5.7.2)
|
||||
debug: 4.3.7
|
||||
debug: 4.4.0
|
||||
eslint: 9.15.0(jiti@1.21.0)
|
||||
ts-api-utils: 1.3.0(typescript@5.7.2)
|
||||
optionalDependencies:
|
||||
@ -13347,7 +13437,7 @@ snapshots:
|
||||
dependencies:
|
||||
'@typescript-eslint/typescript-estree': 8.24.1(typescript@5.7.3)
|
||||
'@typescript-eslint/utils': 8.24.1(eslint@9.20.1(jiti@1.21.0))(typescript@5.7.3)
|
||||
debug: 4.3.7
|
||||
debug: 4.4.0
|
||||
eslint: 9.20.1(jiti@1.21.0)
|
||||
ts-api-utils: 2.0.1(typescript@5.7.3)
|
||||
typescript: 5.7.3
|
||||
@ -13362,7 +13452,7 @@ snapshots:
|
||||
dependencies:
|
||||
'@typescript-eslint/types': 8.17.0
|
||||
'@typescript-eslint/visitor-keys': 8.17.0
|
||||
debug: 4.3.7
|
||||
debug: 4.4.0
|
||||
fast-glob: 3.3.2
|
||||
is-glob: 4.0.3
|
||||
minimatch: 9.0.4
|
||||
@ -13571,13 +13661,13 @@ snapshots:
|
||||
|
||||
agent-base@6.0.2:
|
||||
dependencies:
|
||||
debug: 4.3.7
|
||||
debug: 4.4.0
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
agent-base@7.1.1:
|
||||
dependencies:
|
||||
debug: 4.3.7
|
||||
debug: 4.4.0
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
@ -13914,6 +14004,8 @@ snapshots:
|
||||
|
||||
bluebird@3.7.2: {}
|
||||
|
||||
boolbase@1.0.0: {}
|
||||
|
||||
bowser@2.11.0: {}
|
||||
|
||||
boxen@5.1.2:
|
||||
@ -13959,10 +14051,17 @@ snapshots:
|
||||
dependencies:
|
||||
node-int64: 0.4.0
|
||||
|
||||
buffer-crc32@0.2.13: {}
|
||||
|
||||
buffer-equal-constant-time@1.0.1: {}
|
||||
|
||||
buffer-from@1.1.2: {}
|
||||
|
||||
buffer@5.6.0:
|
||||
dependencies:
|
||||
base64-js: 1.5.1
|
||||
ieee754: 1.2.1
|
||||
|
||||
buffer@5.7.1:
|
||||
dependencies:
|
||||
base64-js: 1.5.1
|
||||
@ -14034,6 +14133,29 @@ snapshots:
|
||||
|
||||
check-disk-space@3.4.0: {}
|
||||
|
||||
cheerio-select@2.1.0:
|
||||
dependencies:
|
||||
boolbase: 1.0.0
|
||||
css-select: 5.1.0
|
||||
css-what: 6.1.0
|
||||
domelementtype: 2.3.0
|
||||
domhandler: 5.0.3
|
||||
domutils: 3.1.0
|
||||
|
||||
cheerio@1.0.0:
|
||||
dependencies:
|
||||
cheerio-select: 2.1.0
|
||||
dom-serializer: 2.0.0
|
||||
domhandler: 5.0.3
|
||||
domutils: 3.1.0
|
||||
encoding-sniffer: 0.2.0
|
||||
htmlparser2: 9.1.0
|
||||
parse5: 7.1.2
|
||||
parse5-htmlparser2-tree-adapter: 7.1.0
|
||||
parse5-parser-stream: 7.1.2
|
||||
undici: 6.21.3
|
||||
whatwg-mimetype: 4.0.0
|
||||
|
||||
chevrotain-allstar@0.3.1(chevrotain@11.0.3):
|
||||
dependencies:
|
||||
chevrotain: 11.0.3
|
||||
@ -14287,6 +14409,14 @@ snapshots:
|
||||
shebang-command: 2.0.0
|
||||
which: 2.0.2
|
||||
|
||||
css-select@5.1.0:
|
||||
dependencies:
|
||||
boolbase: 1.0.0
|
||||
css-what: 6.1.0
|
||||
domhandler: 5.0.3
|
||||
domutils: 3.1.0
|
||||
nth-check: 2.1.1
|
||||
|
||||
css-what@6.1.0: {}
|
||||
|
||||
cssesc@3.0.0: {}
|
||||
@ -14675,6 +14805,11 @@ snapshots:
|
||||
|
||||
emoji-regex@9.2.2: {}
|
||||
|
||||
encoding-sniffer@0.2.0:
|
||||
dependencies:
|
||||
iconv-lite: 0.6.3
|
||||
whatwg-encoding: 3.1.1
|
||||
|
||||
end-of-stream@1.4.4:
|
||||
dependencies:
|
||||
once: 1.4.0
|
||||
@ -15406,7 +15541,7 @@ snapshots:
|
||||
|
||||
hachure-fill@0.5.2: {}
|
||||
|
||||
happy-dom@15.11.6:
|
||||
happy-dom@15.11.7:
|
||||
dependencies:
|
||||
entities: 4.5.0
|
||||
webidl-conversions: 7.0.0
|
||||
@ -15479,6 +15614,13 @@ snapshots:
|
||||
domutils: 3.1.0
|
||||
entities: 4.5.0
|
||||
|
||||
htmlparser2@9.1.0:
|
||||
dependencies:
|
||||
domelementtype: 2.3.0
|
||||
domhandler: 5.0.3
|
||||
domutils: 3.1.0
|
||||
entities: 4.5.0
|
||||
|
||||
http-errors@2.0.0:
|
||||
dependencies:
|
||||
depd: 2.0.0
|
||||
@ -15490,21 +15632,21 @@ snapshots:
|
||||
http-proxy-agent@7.0.2:
|
||||
dependencies:
|
||||
agent-base: 7.1.1
|
||||
debug: 4.3.7
|
||||
debug: 4.4.0
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
https-proxy-agent@5.0.1:
|
||||
dependencies:
|
||||
agent-base: 6.0.2
|
||||
debug: 4.3.7
|
||||
debug: 4.4.0
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
https-proxy-agent@7.0.5:
|
||||
dependencies:
|
||||
agent-base: 7.1.1
|
||||
debug: 4.3.7
|
||||
debug: 4.4.0
|
||||
transitivePeerDependencies:
|
||||
- supports-color
|
||||
|
||||
@ -15767,7 +15909,7 @@ snapshots:
|
||||
|
||||
istanbul-lib-source-maps@4.0.1:
|
||||
dependencies:
|
||||
debug: 4.3.7
|
||||
debug: 4.4.0
|
||||
istanbul-lib-coverage: 3.2.2
|
||||
source-map: 0.6.1
|
||||
transitivePeerDependencies:
|
||||
@ -16732,6 +16874,10 @@ snapshots:
|
||||
gauge: 3.0.2
|
||||
set-blocking: 2.0.0
|
||||
|
||||
nth-check@2.1.1:
|
||||
dependencies:
|
||||
boolbase: 1.0.0
|
||||
|
||||
nwsapi@2.2.16: {}
|
||||
|
||||
nx@20.4.5(@swc/core@1.5.25(@swc/helpers@0.5.5)):
|
||||
@ -16928,6 +17074,15 @@ snapshots:
|
||||
parse-node-version@1.0.1:
|
||||
optional: true
|
||||
|
||||
parse5-htmlparser2-tree-adapter@7.1.0:
|
||||
dependencies:
|
||||
domhandler: 5.0.3
|
||||
parse5: 7.1.2
|
||||
|
||||
parse5-parser-stream@7.1.2:
|
||||
dependencies:
|
||||
parse5: 7.1.2
|
||||
|
||||
parse5@7.1.2:
|
||||
dependencies:
|
||||
entities: 4.5.0
|
||||
@ -16994,6 +17149,8 @@ snapshots:
|
||||
|
||||
peek-readable@7.0.0: {}
|
||||
|
||||
pend@1.2.0: {}
|
||||
|
||||
pg-cloudflare@1.1.1:
|
||||
optional: true
|
||||
|
||||
@ -17948,6 +18105,11 @@ snapshots:
|
||||
|
||||
statuses@2.0.1: {}
|
||||
|
||||
stream-browserify@3.0.0:
|
||||
dependencies:
|
||||
inherits: 2.0.4
|
||||
readable-stream: 3.6.2
|
||||
|
||||
streamsearch@1.1.0: {}
|
||||
|
||||
string-length@4.0.2:
|
||||
@ -18163,6 +18325,10 @@ snapshots:
|
||||
dependencies:
|
||||
tldts-core: 6.1.72
|
||||
|
||||
tmp-promise@3.0.3:
|
||||
dependencies:
|
||||
tmp: 0.2.1
|
||||
|
||||
tmp@0.0.33:
|
||||
dependencies:
|
||||
os-tmpdir: 1.0.2
|
||||
@ -18404,6 +18570,8 @@ snapshots:
|
||||
|
||||
undici-types@6.20.0: {}
|
||||
|
||||
undici@6.21.3: {}
|
||||
|
||||
unicode-canonical-property-names-ecmascript@2.0.0: {}
|
||||
|
||||
unicode-match-property-ecmascript@2.0.0:
|
||||
@ -18759,6 +18927,11 @@ snapshots:
|
||||
y18n: 5.0.8
|
||||
yargs-parser: 21.1.1
|
||||
|
||||
yauzl@3.2.0:
|
||||
dependencies:
|
||||
buffer-crc32: 0.2.13
|
||||
pend: 1.2.0
|
||||
|
||||
yjs@13.6.20:
|
||||
dependencies:
|
||||
lib0: 0.2.98
|
||||
|
||||
Reference in New Issue
Block a user