Compare commits

..

3 Commits

Author SHA1 Message Date
247876d83f generic iframe embed 2025-06-08 21:34:19 -07:00
6d024fc3de feat: bulk page imports (#1219)
* refactor imports - WIP

* Add readstream

* WIP

* fix attachmentId render

* fix attachmentId render

* turndown video tag

* feat: add stream upload support and improve file handling

- Add stream upload functionality to storage drivers\n- Improve ZIP file extraction with better encoding handling\n- Fix attachment ID rendering issues\n- Add AWS S3 upload stream support\n- Update dependencies for better compatibility

* WIP

* notion formatter

* move embed parser to editor-ext package

* import embeds

* utility files

* cleanup

* Switch from happy-dom to cheerio
* Refine code

* WIP

* bug fixes and UI

* sync

* WIP

* sync

* keep import modal mounted

* Show modal during upload

* WIP

* WIP
2025-06-09 04:29:27 +01:00
ce1503af85 fix: sidebar list when changing workspace (#1150)
* init

* navigate in overview if current page is in deleted node

* fix: implement pagination in sidebar-pages queries

* fix: appendNodeChildren()

Preserve deeper children if they exist and remove node if deleted
2025-06-08 03:27:09 +01:00
43 changed files with 2278 additions and 1399 deletions

View File

@ -0,0 +1,20 @@
import { rem } from "@mantine/core";
interface Props {
size?: number | string;
}
export function ConfluenceIcon({ size }: Props) {
return (
<svg
xmlns="http://www.w3.org/2000/svg"
viewBox="0 0 24 24"
fill="none"
stroke="currentColor"
strokeWidth="2"
style={{ width: rem(size), height: rem(size) }}
>
<path d="M.87 18.257c-.248.382-.53.875-.763 1.245a.764.764 0 0 0 .255 1.04l4.965 3.054a.764.764 0 0 0 1.058-.26c.199-.332.454-.763.733-1.221 1.967-3.247 3.945-2.853 7.508-1.146l4.957 2.337a.764.764 0 0 0 1.028-.382l2.364-5.346a.764.764 0 0 0-.382-1 599.851 599.851 0 0 1-4.965-2.361C10.911 10.97 5.224 11.185.87 18.257zM23.131 5.743c.249-.405.531-.875.764-1.25a.764.764 0 0 0-.256-1.034L18.675.404a.764.764 0 0 0-1.058.26c-.195.335-.451.763-.734 1.225-1.966 3.246-3.945 2.85-7.508 1.146L4.437.694a.764.764 0 0 0-1.027.382L1.046 6.422a.764.764 0 0 0 .382 1c1.039.49 3.105 1.467 4.965 2.361 6.698 3.246 12.392 3.029 16.738-4.04z" />
</svg>
);
}

View File

@ -15,13 +15,13 @@ import {
import { IconEdit } from "@tabler/icons-react";
import { z } from "zod";
import { useForm, zodResolver } from "@mantine/form";
import {
getEmbedProviderById,
getEmbedUrlAndProvider,
} from "@/features/editor/components/embed/providers.ts";
import { notifications } from "@mantine/notifications";
import { useTranslation } from "react-i18next";
import i18n from "i18next";
import {
getEmbedProviderById,
getEmbedUrlAndProvider,
} from "@docmost/editor-ext";
const schema = z.object({
url: z
@ -52,6 +52,10 @@ export default function EmbedView(props: NodeViewProps) {
async function onSubmit(data: { url: string }) {
if (provider) {
const embedProvider = getEmbedProviderById(provider);
if (embedProvider.id === "iframe") {
updateAttributes({ src: data.url });
return;
}
if (embedProvider.regex.test(data.url)) {
updateAttributes({ src: data.url });
} else {
@ -101,7 +105,7 @@ export default function EmbedView(props: NodeViewProps) {
<Text component="span" size="lg" c="dimmed">
{t("Embed {{provider}}", {
provider: getEmbedProviderById(provider).name,
provider: getEmbedProviderById(provider)?.name,
})}
</Text>
</div>

View File

@ -17,8 +17,8 @@ import {
IconTable,
IconTypography,
IconMenu4,
IconCalendar,
} from "@tabler/icons-react";
IconCalendar, IconAppWindow,
} from '@tabler/icons-react';
import {
CommandProps,
SlashMenuGroupedItemsType,
@ -357,6 +357,20 @@ const CommandGroups: SlashMenuGroupedItemsType = {
.run();
},
},
{
title: "Iframe embed",
description: "Embed any Iframe",
searchTerms: ["iframe"],
icon: IconAppWindow,
command: ({ editor, range }: CommandProps) => {
editor
.chain()
.focus()
.deleteRange(range)
.setEmbed({ provider: "iframe" })
.run();
},
},
{
title: "Airtable",
description: "Embed Airtable",

View File

@ -103,7 +103,7 @@ export function TitleEditor({
spaceId: page.spaceId,
entity: ["pages"],
id: page.id,
payload: { title: page.title, slugId: page.slugId },
payload: { title: page.title, slugId: page.slugId, parentPageId: page.parentPageId, icon: page.icon },
};
if (page.title !== titleEditor.getText()) return;

View File

@ -0,0 +1,14 @@
import api from "@/lib/api-client";
import { IFileTask } from "@/features/file-task/types/file-task.types.ts";
export async function getFileTaskById(fileTaskId: string): Promise<IFileTask> {
const req = await api.post<IFileTask>("/file-tasks/info", {
fileTaskId: fileTaskId,
});
return req.data;
}
export async function getFileTasks(): Promise<IFileTask[]> {
const req = await api.post<IFileTask[]>("/file-tasks");
return req.data;
}

View File

@ -0,0 +1,17 @@
export interface IFileTask {
id: string;
type: "import" | "export";
source: string;
status: string;
fileName: string;
filePath: string;
fileSize: number;
fileExt: string;
errorMessage: string | null;
creatorId: string;
spaceId: string;
workspaceId: string;
createdAt: string;
updatedAt: string;
deletedAt: string | null;
}

View File

@ -1,18 +1,38 @@
import { Modal, Button, SimpleGrid, FileButton } from "@mantine/core";
import {
Modal,
Button,
SimpleGrid,
FileButton,
Group,
Text,
Tooltip,
} from "@mantine/core";
import {
IconBrandNotion,
IconCheck,
IconFileCode,
IconFileTypeZip,
IconMarkdown,
IconX,
} from "@tabler/icons-react";
import { importPage } from "@/features/page/services/page-service.ts";
import {
importPage,
importZip,
} from "@/features/page/services/page-service.ts";
import { notifications } from "@mantine/notifications";
import { treeDataAtom } from "@/features/page/tree/atoms/tree-data-atom.ts";
import { useAtom } from "jotai";
import { buildTree } from "@/features/page/tree/utils";
import { IPage } from "@/features/page/types/page.types.ts";
import React from "react";
import React, { useEffect, useState } from "react";
import { useTranslation } from "react-i18next";
import { ConfluenceIcon } from "@/components/icons/confluence-icon.tsx";
import { getFileImportSizeLimit, isCloud } from "@/lib/config.ts";
import { formatBytes } from "@/lib";
import { workspaceAtom } from "@/features/user/atoms/current-user-atom.ts";
import { getFileTaskById } from "@/features/file-task/services/file-task-service.ts";
import { queryClient } from "@/main.tsx";
import { useQueryEmit } from "@/features/websocket/use-query-emit.ts";
interface PageImportModalProps {
spaceId: string;
@ -36,6 +56,7 @@ export default function PageImportModal({
yOffset="10vh"
xOffset={0}
mah={400}
keepMounted={true}
>
<Modal.Overlay />
<Modal.Content style={{ overflow: "hidden" }}>
@ -59,6 +80,133 @@ interface ImportFormatSelection {
function ImportFormatSelection({ spaceId, onClose }: ImportFormatSelection) {
const { t } = useTranslation();
const [treeData, setTreeData] = useAtom(treeDataAtom);
const [workspace] = useAtom(workspaceAtom);
const [fileTaskId, setFileTaskId] = useState<string | null>(null);
const emit = useQueryEmit();
const canUseConfluence = isCloud() || workspace?.hasLicenseKey;
const handleZipUpload = async (selectedFile: File, source: string) => {
if (!selectedFile) {
return;
}
try {
onClose();
notifications.show({
id: "import",
title: t("Uploading import file"),
message: t("Please don't close this tab."),
loading: true,
withCloseButton: false,
autoClose: false,
});
const importTask = await importZip(selectedFile, spaceId, source);
notifications.update({
id: "import",
title: t("Importing pages"),
message: t(
"Page import is in progress. You can check back later if this takes longer.",
),
loading: true,
withCloseButton: true,
autoClose: false,
});
setFileTaskId(importTask.id);
} catch (err) {
console.log("Failed to upload import file", err);
notifications.update({
id: "import",
color: "red",
title: t("Failed to upload import file"),
message: err?.response.data.message,
icon: <IconX size={18} />,
loading: false,
withCloseButton: true,
autoClose: false,
});
}
};
useEffect(() => {
if (!fileTaskId) return;
const intervalId = setInterval(async () => {
try {
const fileTask = await getFileTaskById(fileTaskId);
const status = fileTask.status;
if (status === "success") {
notifications.update({
id: "import",
color: "teal",
title: t("Import complete"),
message: t("Your pages were successfully imported."),
icon: <IconCheck size={18} />,
loading: false,
withCloseButton: true,
autoClose: false,
});
clearInterval(intervalId);
setFileTaskId(null);
await queryClient.refetchQueries({
queryKey: ["root-sidebar-pages", fileTask.spaceId],
});
setTimeout(() => {
emit({
operation: "refetchRootTreeNodeEvent",
spaceId: spaceId,
});
}, 50);
}
if (status === "failed") {
notifications.update({
id: "import",
color: "red",
title: t("Page import failed"),
message: t(
"Something went wrong while importing pages: {{reason}}.",
{
reason: fileTask.errorMessage,
},
),
icon: <IconX size={18} />,
loading: false,
withCloseButton: true,
autoClose: false,
});
clearInterval(intervalId);
setFileTaskId(null);
console.error(fileTask.errorMessage);
}
} catch (err) {
notifications.update({
id: "import",
color: "red",
title: t("Import failed"),
message: t(
"Something went wrong while importing pages: {{reason}}.",
{
reason: err.response?.data.message,
},
),
icon: <IconX size={18} />,
loading: false,
withCloseButton: true,
autoClose: false,
});
clearInterval(intervalId);
setFileTaskId(null);
console.error("Failed to fetch import status", err);
}
}, 3000);
}, [fileTaskId]);
const handleFileUpload = async (selectedFiles: File[]) => {
if (!selectedFiles) {
@ -120,6 +268,7 @@ function ImportFormatSelection({ spaceId, onClose }: ImportFormatSelection) {
}
};
// @ts-ignore
return (
<>
<SimpleGrid cols={2}>
@ -148,7 +297,76 @@ function ImportFormatSelection({ spaceId, onClose }: ImportFormatSelection) {
</Button>
)}
</FileButton>
<FileButton
onChange={(file) => handleZipUpload(file, "notion")}
accept="application/zip"
>
{(props) => (
<Button
justify="start"
variant="default"
leftSection={<IconBrandNotion size={18} />}
{...props}
>
Notion
</Button>
)}
</FileButton>
<FileButton
onChange={(file) => handleZipUpload(file, "confluence")}
accept="application/zip"
>
{(props) => (
<Tooltip
label="Available in enterprise edition"
disabled={canUseConfluence}
>
<Button
disabled={!canUseConfluence}
justify="start"
variant="default"
leftSection={<ConfluenceIcon size={18} />}
{...props}
>
Confluence
</Button>
</Tooltip>
)}
</FileButton>
</SimpleGrid>
<Group justify="center" gap="xl" mih={150}>
<div>
<Text ta="center" size="lg" inline>
Import zip file
</Text>
<Text ta="center" size="sm" c="dimmed" inline py="sm">
{t(
`Upload zip file containing Markdown and HTML files. Max: {{sizeLimit}}`,
{
sizeLimit: formatBytes(getFileImportSizeLimit()),
},
)}
</Text>
<FileButton
onChange={(file) => handleZipUpload(file, "generic")}
accept="application/zip"
>
{(props) => (
<Group justify="center">
<Button
justify="center"
leftSection={<IconFileTypeZip size={18} />}
{...props}
>
{t("Upload file")}
</Button>
</Group>
)}
</FileButton>
</div>
</Group>
</>
);
}

View File

@ -1,5 +1,8 @@
import {
InfiniteData,
QueryKey,
useInfiniteQuery,
UseInfiniteQueryResult,
useMutation,
useQuery,
useQueryClient,
@ -14,6 +17,7 @@ import {
movePage,
getPageBreadcrumbs,
getRecentChanges,
getAllSidebarPages,
} from "@/features/page/services/page-service";
import {
IMovePage,
@ -56,7 +60,9 @@ export function useCreatePageMutation() {
const { t } = useTranslation();
return useMutation<IPage, Error, Partial<IPageInput>>({
mutationFn: (data) => createPage(data),
onSuccess: (data) => {},
onSuccess: (data) => {
invalidateOnCreatePage(data);
},
onError: (error) => {
notifications.show({ message: t("Failed to create page"), color: "red" });
},
@ -80,6 +86,8 @@ export function updatePageData(data: IPage) {
if (pageById) {
queryClient.setQueryData(["pages", data.id], { ...pageById, ...data });
}
invalidateOnUpdatePage(data.spaceId, data.parentPageId, data.id, data.title, data.icon);
}
export function useUpdateTitlePageMutation() {
@ -93,6 +101,8 @@ export function useUpdatePageMutation() {
mutationFn: (data) => updatePage(data),
onSuccess: (data) => {
updatePage(data);
invalidateOnUpdatePage(data.spaceId, data.parentPageId, data.id, data.title, data.icon);
},
});
}
@ -101,8 +111,9 @@ export function useDeletePageMutation() {
const { t } = useTranslation();
return useMutation({
mutationFn: (pageId: string) => deletePage(pageId),
onSuccess: () => {
onSuccess: (data, pageId) => {
notifications.show({ message: t("Page deleted successfully") });
invalidateOnDeletePage(pageId);
},
onError: (error) => {
notifications.show({ message: t("Failed to delete page"), color: "red" });
@ -113,15 +124,21 @@ export function useDeletePageMutation() {
export function useMovePageMutation() {
return useMutation<void, Error, IMovePage>({
mutationFn: (data) => movePage(data),
onSuccess: () => {
invalidateOnMovePage();
},
});
}
export function useGetSidebarPagesQuery(
data: SidebarPagesParams,
): UseQueryResult<IPagination<IPage>, Error> {
return useQuery({
export function useGetSidebarPagesQuery(data: SidebarPagesParams|null): UseInfiniteQueryResult<InfiniteData<IPagination<IPage>, unknown>> {
return useInfiniteQuery({
queryKey: ["sidebar-pages", data],
queryFn: () => getSidebarPages(data),
queryFn: ({ pageParam }) => getSidebarPages({ ...data, page: pageParam }),
initialPageParam: 1,
getPreviousPageParam: (firstPage) =>
firstPage.meta.hasPrevPage ? firstPage.meta.page - 1 : undefined,
getNextPageParam: (lastPage) =>
lastPage.meta.hasNextPage ? lastPage.meta.page + 1 : undefined,
});
}
@ -149,14 +166,16 @@ export function usePageBreadcrumbsQuery(
});
}
export async function fetchAncestorChildren(params: SidebarPagesParams) {
export async function fetchAllAncestorChildren(params: SidebarPagesParams) {
// not using a hook here, so we can call it inside a useEffect hook
const response = await queryClient.fetchQuery({
queryKey: ["sidebar-pages", params],
queryFn: () => getSidebarPages(params),
queryFn: () => getAllSidebarPages(params),
staleTime: 30 * 60 * 1000,
});
return buildTree(response.items);
const allItems = response.pages.flatMap((page) => page.items);
return buildTree(allItems);
}
export function useRecentChangesQuery(
@ -168,3 +187,157 @@ export function useRecentChangesQuery(
refetchOnMount: true,
});
}
export function invalidateOnCreatePage(data: Partial<IPage>) {
const newPage: Partial<IPage> = {
creatorId: data.creatorId,
hasChildren: data.hasChildren,
icon: data.icon,
id: data.id,
parentPageId: data.parentPageId,
position: data.position,
slugId: data.slugId,
spaceId: data.spaceId,
title: data.title,
};
let queryKey: QueryKey = null;
if (data.parentPageId===null) {
queryKey = ['root-sidebar-pages', data.spaceId];
}else{
queryKey = ['sidebar-pages', {pageId: data.parentPageId, spaceId: data.spaceId}]
}
//update all sidebar pages
queryClient.setQueryData<InfiniteData<IPagination<Partial<IPage>>>>(queryKey, (old) => {
if (!old) return old;
return {
...old,
pages: old.pages.map((page,index) => {
if (index === old.pages.length - 1) {
return {
...page,
items: [...page.items, newPage],
};
}
return page;
}),
};
});
//update sidebar haschildren
if (data.parentPageId!==null){
//update sub sidebar pages haschildern
const subSideBarMatches = queryClient.getQueriesData({
queryKey: ['sidebar-pages'],
exact: false,
});
subSideBarMatches.forEach(([key, d]) => {
queryClient.setQueryData<InfiniteData<IPagination<IPage>>>(key, (old) => {
if (!old) return old;
return {
...old,
pages: old.pages.map((page) => ({
...page,
items: page.items.map((sidebarPage: IPage) =>
sidebarPage.id === data.parentPageId ? { ...sidebarPage, hasChildren: true } : sidebarPage
)
})),
};
});
});
//update root sidebar pages haschildern
const rootSideBarMatches = queryClient.getQueriesData({
queryKey: ['root-sidebar-pages', data.spaceId],
exact: false,
});
rootSideBarMatches.forEach(([key, d]) => {
queryClient.setQueryData<InfiniteData<IPagination<IPage>>>(key, (old) => {
if (!old) return old;
return {
...old,
pages: old.pages.map((page) => ({
...page,
items: page.items.map((sidebarPage: IPage) =>
sidebarPage.id === data.parentPageId ? { ...sidebarPage, hasChildren: true } : sidebarPage
)
})),
};
});
});
}
//update recent changes
queryClient.invalidateQueries({
queryKey: ["recent-changes", data.spaceId],
});
}
export function invalidateOnUpdatePage(spaceId: string, parentPageId: string, id: string, title: string, icon: string) {
let queryKey: QueryKey = null;
if(parentPageId===null){
queryKey = ['root-sidebar-pages', spaceId];
}else{
queryKey = ['sidebar-pages', {pageId: parentPageId, spaceId: spaceId}]
}
//update all sidebar pages
queryClient.setQueryData<InfiniteData<IPagination<IPage>>>(queryKey, (old) => {
if (!old) return old;
return {
...old,
pages: old.pages.map((page) => ({
...page,
items: page.items.map((sidebarPage: IPage) =>
sidebarPage.id === id ? { ...sidebarPage, title: title, icon: icon } : sidebarPage
)
})),
};
});
//update recent changes
queryClient.invalidateQueries({
queryKey: ["recent-changes", spaceId],
});
}
export function invalidateOnMovePage() {
//for move invalidate all sidebars for now (how to do???)
//invalidate all root sidebar pages
queryClient.invalidateQueries({
queryKey: ["root-sidebar-pages"],
});
//invalidate all sub sidebar pages
queryClient.invalidateQueries({
queryKey: ['sidebar-pages'],
});
// ---
}
export function invalidateOnDeletePage(pageId: string) {
//update all sidebar pages
const allSideBarMatches = queryClient.getQueriesData({
predicate: (query) =>
query.queryKey[0] === 'root-sidebar-pages' || query.queryKey[0] === 'sidebar-pages',
});
allSideBarMatches.forEach(([key, d]) => {
queryClient.setQueryData<InfiniteData<IPagination<IPage>>>(key, (old) => {
if (!old) return old;
return {
...old,
pages: old.pages.map((page) => ({
...page,
items: page.items.filter((sidebarPage: IPage) => sidebarPage.id !== pageId),
})),
};
});
});
//update recent changes
queryClient.invalidateQueries({
queryKey: ["recent-changes"],
});
}

View File

@ -7,9 +7,11 @@ import {
IPage,
IPageInput,
SidebarPagesParams,
} from "@/features/page/types/page.types";
} from '@/features/page/types/page.types';
import { IAttachment, IPagination } from "@/lib/types.ts";
import { saveAs } from "file-saver";
import { InfiniteData } from "@tanstack/react-query";
import { IFileTask } from '@/features/file-task/types/file-task.types.ts';
export async function createPage(data: Partial<IPage>): Promise<IPage> {
const req = await api.post<IPage>("/pages/create", data);
@ -52,6 +54,32 @@ export async function getSidebarPages(
return req.data;
}
export async function getAllSidebarPages(
params: SidebarPagesParams,
): Promise<InfiniteData<IPagination<IPage>, unknown>> {
let page = 1;
let hasNextPage = false;
const pages: IPagination<IPage>[] = [];
const pageParams: number[] = [];
do {
const req = await api.post("/pages/sidebar-pages", { ...params, page: page });
const data: IPagination<IPage> = req.data;
pages.push(data);
pageParams.push(page);
hasNextPage = data.meta.hasNextPage;
page += 1;
} while (hasNextPage);
return {
pageParams,
pages,
};
}
export async function getPageBreadcrumbs(
pageId: string,
): Promise<Partial<IPage[]>> {
@ -92,6 +120,25 @@ export async function importPage(file: File, spaceId: string) {
return req.data;
}
export async function importZip(
file: File,
spaceId: string,
source?: string,
): Promise<IFileTask> {
const formData = new FormData();
formData.append("spaceId", spaceId);
formData.append("source", source);
formData.append("file", file);
const req = await api.post<any>("/pages/import-zip", formData, {
headers: {
"Content-Type": "multipart/form-data",
},
});
return req.data;
}
export async function uploadFile(
file: File,
pageId: string,

View File

@ -1,4 +1,19 @@
import { atom } from "jotai";
import { SpaceTreeNode } from "@/features/page/tree/types";
import { appendNodeChildren } from "../utils";
export const treeDataAtom = atom<SpaceTreeNode[]>([]);
// Atom
export const appendNodeChildrenAtom = atom(
null,
(
get,
set,
{ parentId, children }: { parentId: string; children: SpaceTreeNode[] }
) => {
const currentTree = get(treeDataAtom);
const updatedTree = appendNodeChildren(currentTree, parentId, children);
set(treeDataAtom, updatedTree);
}
);

View File

@ -2,7 +2,7 @@ import { NodeApi, NodeRendererProps, Tree, TreeApi } from "react-arborist";
import { atom, useAtom } from "jotai";
import { treeApiAtom } from "@/features/page/tree/atoms/tree-api-atom.ts";
import {
fetchAncestorChildren,
fetchAllAncestorChildren,
useGetRootSidebarPagesQuery,
usePageQuery,
useUpdatePageMutation,
@ -24,7 +24,10 @@ import {
IconPointFilled,
IconTrash,
} from "@tabler/icons-react";
import { treeDataAtom } from "@/features/page/tree/atoms/tree-data-atom.ts";
import {
appendNodeChildrenAtom,
treeDataAtom,
} from "@/features/page/tree/atoms/tree-data-atom.ts";
import clsx from "clsx";
import EmojiPicker from "@/components/ui/emoji-picker.tsx";
import { useTreeMutation } from "@/features/page/tree/hooks/use-tree-mutation.ts";
@ -32,6 +35,7 @@ import {
appendNodeChildren,
buildTree,
buildTreeWithChildren,
mergeRootTrees,
updateTreeNodeIcon,
} from "@/features/page/tree/utils/utils.ts";
import { SpaceTreeNode } from "@/features/page/tree/types.ts";
@ -104,17 +108,17 @@ export default function SpaceTree({ spaceId, readOnly }: SpaceTreeProps) {
const allItems = pagesData.pages.flatMap((page) => page.items);
const treeData = buildTree(allItems);
if (data.length < 1 || data?.[0].spaceId !== spaceId) {
//Thoughts
// don't reset if there is data in state
// we only expect to call this once on initial load
// even if we decide to refetch, it should only update
// and append root pages instead of resetting the entire tree
// which looses async loaded children too
setData(treeData);
setIsDataLoaded(true);
setOpenTreeNodes({});
}
setData((prev) => {
// fresh space; full reset
if (prev.length === 0 || prev[0]?.spaceId !== spaceId) {
setIsDataLoaded(true);
setOpenTreeNodes({});
return treeData;
}
// same space; append only missing roots
return mergeRootTrees(prev, treeData);
});
}
}, [pagesData, hasNextPage]);
@ -140,7 +144,7 @@ export default function SpaceTree({ spaceId, readOnly }: SpaceTreeProps) {
if (ancestor.id === currentPage.id) {
return;
}
const children = await fetchAncestorChildren({
const children = await fetchAllAncestorChildren({
pageId: ancestor.id,
spaceId: ancestor.spaceId,
});
@ -237,6 +241,7 @@ function Node({ node, style, dragHandle, tree }: NodeRendererProps<any>) {
const { t } = useTranslation();
const updatePageMutation = useUpdatePageMutation();
const [treeData, setTreeData] = useAtom(treeDataAtom);
const [, appendChildren] = useAtom(appendNodeChildrenAtom);
const emit = useQueryEmit();
const { spaceSlug } = useParams();
const timerRef = useRef(null);
@ -262,9 +267,10 @@ function Node({ node, style, dragHandle, tree }: NodeRendererProps<any>) {
async function handleLoadChildren(node: NodeApi<SpaceTreeNode>) {
if (!node.data.hasChildren) return;
if (node.data.children && node.data.children.length > 0) {
return;
}
// in conflict with use-query-subscription.ts => case "addTreeNode","moveTreeNode" etc with websocket
// if (node.data.children && node.data.children.length > 0) {
// return;
// }
try {
const params: SidebarPagesParams = {
@ -272,21 +278,12 @@ function Node({ node, style, dragHandle, tree }: NodeRendererProps<any>) {
spaceId: node.data.spaceId,
};
const newChildren = await queryClient.fetchQuery({
queryKey: ["sidebar-pages", params],
queryFn: () => getSidebarPages(params),
staleTime: 10 * 60 * 1000,
const childrenTree = await fetchAllAncestorChildren(params);
appendChildren({
parentId: node.data.id,
children: childrenTree,
});
const childrenTree = buildTree(newChildren.items);
const updatedTreeData = appendNodeChildren(
treeData,
node.data.id,
childrenTree,
);
setTreeData(updatedTreeData);
} catch (error) {
console.error("Failed to fetch children:", error);
}
@ -304,17 +301,19 @@ function Node({ node, style, dragHandle, tree }: NodeRendererProps<any>) {
const handleEmojiSelect = (emoji: { native: string }) => {
handleUpdateNodeIcon(node.id, emoji.native);
updatePageMutation.mutateAsync({ pageId: node.id, icon: emoji.native });
setTimeout(() => {
emit({
operation: "updateOne",
spaceId: node.data.spaceId,
entity: ["pages"],
id: node.id,
payload: { icon: emoji.native },
updatePageMutation
.mutateAsync({ pageId: node.id, icon: emoji.native })
.then((data) => {
setTimeout(() => {
emit({
operation: "updateOne",
spaceId: node.data.spaceId,
entity: ["pages"],
id: node.id,
payload: { icon: emoji.native, parentPageId: data.parentPageId },
});
}, 50);
});
}, 50);
};
const handleRemoveEmoji = () => {
@ -576,6 +575,12 @@ interface PageArrowProps {
}
function PageArrow({ node, onExpandTree }: PageArrowProps) {
useEffect(() => {
if (node.isOpen) {
onExpandTree();
}
}, []);
return (
<ActionIcon
size={20}

View File

@ -93,7 +93,7 @@ export function useTreeMutation<T>(spaceId: string) {
return data;
};
const onMove: MoveHandler<T> = (args: {
const onMove: MoveHandler<T> = async (args: {
dragIds: string[];
dragNodes: NodeApi<T>[];
parentId: string | null;
@ -176,7 +176,7 @@ export function useTreeMutation<T>(spaceId: string) {
};
try {
movePageMutation.mutateAsync(payload);
await movePageMutation.mutateAsync(payload);
setTimeout(() => {
emit({
@ -206,6 +206,23 @@ export function useTreeMutation<T>(spaceId: string) {
}
};
const isPageInNode = (
node: { data: SpaceTreeNode; children?: any[] },
pageSlug: string
): boolean => {
if (node.data.slugId === pageSlug) {
return true;
}
for (const item of node.children) {
if (item.data.slugId === pageSlug) {
return true;
} else {
return isPageInNode(item, pageSlug);
}
}
return false;
};
const onDelete: DeleteHandler<T> = async (args: { ids: string[] }) => {
try {
await deletePageMutation.mutateAsync(args.ids[0]);
@ -218,8 +235,7 @@ export function useTreeMutation<T>(spaceId: string) {
tree.drop({ id: args.ids[0] });
setData(tree.data);
// navigate only if the current url is same as the deleted page
if (pageSlug && node.data.slugId === pageSlug.split("-")[1]) {
if (pageSlug && isPageInNode(node, pageSlug.split("-")[1])) {
navigate(getSpaceUrl(spaceSlug));
}

View File

@ -121,7 +121,6 @@ export const deleteTreeNode = (
.filter((node) => node !== null);
};
export function buildTreeWithChildren(items: SpaceTreeNode[]): SpaceTreeNode[] {
const nodeMap = {};
let result: SpaceTreeNode[] = [];
@ -164,16 +163,55 @@ export function appendNodeChildren(
nodeId: string,
children: SpaceTreeNode[],
) {
return treeItems.map((nodeItem) => {
if (nodeItem.id === nodeId) {
return { ...nodeItem, children };
}
if (nodeItem.children) {
// Preserve deeper children if they exist and remove node if deleted
return treeItems.map((node) => {
if (node.id === nodeId) {
const newIds = new Set(children.map((c) => c.id));
const existingMap = new Map(
(node.children ?? [])
.filter((c) => newIds.has(c.id))
.map((c) => [c.id, c]),
);
const merged = children.map((newChild) => {
const existing = existingMap.get(newChild.id);
return existing && existing.children
? { ...newChild, children: existing.children }
: newChild;
});
return {
...nodeItem,
children: appendNodeChildren(nodeItem.children, nodeId, children),
...node,
children: merged,
};
}
return nodeItem;
if (node.children) {
return {
...node,
children: appendNodeChildren(node.children, nodeId, children),
};
}
return node;
});
}
/**
* Merge root nodes; keep existing ones intact, append new ones,
*/
export function mergeRootTrees(
prevRoots: SpaceTreeNode[],
incomingRoots: SpaceTreeNode[],
): SpaceTreeNode[] {
const seen = new Set(prevRoots.map((r) => r.id));
// add new roots that were not present before
const merged = [...prevRoots];
incomingRoots.forEach((node) => {
if (!seen.has(node.id)) merged.push(node);
});
return sortPositionKeys(merged);
}

View File

@ -1,4 +1,5 @@
import { SpaceTreeNode } from "@/features/page/tree/types.ts";
import { IPage } from "@/features/page/types/page.types";
export type InvalidateEvent = {
operation: "invalidate";
@ -17,7 +18,7 @@ export type UpdateEvent = {
spaceId: string;
entity: Array<string>;
id: string;
payload: Partial<any>;
payload: Partial<IPage>;
};
export type DeleteEvent = {
@ -25,7 +26,7 @@ export type DeleteEvent = {
spaceId: string;
entity: Array<string>;
id: string;
payload?: Partial<any>;
payload?: Partial<IPage>;
};
export type AddTreeNodeEvent = {
@ -46,15 +47,28 @@ export type MoveTreeNodeEvent = {
parentId: string;
index: number;
position: string;
}
};
};
export type DeleteTreeNodeEvent = {
operation: "deleteTreeNode";
spaceId: string;
payload: {
node: SpaceTreeNode
}
node: SpaceTreeNode;
};
};
export type WebSocketEvent = InvalidateEvent | InvalidateCommentsEvent | UpdateEvent | DeleteEvent | AddTreeNodeEvent | MoveTreeNodeEvent | DeleteTreeNodeEvent;
export type RefetchRootTreeNodeEvent = {
operation: "refetchRootTreeNodeEvent";
spaceId: string;
};
export type WebSocketEvent =
| InvalidateEvent
| InvalidateCommentsEvent
| UpdateEvent
| DeleteEvent
| AddTreeNodeEvent
| MoveTreeNodeEvent
| DeleteTreeNodeEvent
| RefetchRootTreeNodeEvent;

View File

@ -1,9 +1,18 @@
import React from "react";
import { socketAtom } from "@/features/websocket/atoms/socket-atom.ts";
import { useAtom } from "jotai";
import { useQueryClient } from "@tanstack/react-query";
import { InfiniteData, useQueryClient } from "@tanstack/react-query";
import { WebSocketEvent } from "@/features/websocket/types";
import { IPage } from "../page/types/page.types";
import { IPagination } from "@/lib/types";
import {
invalidateOnCreatePage,
invalidateOnDeletePage,
invalidateOnMovePage,
invalidateOnUpdatePage,
} from "../page/queries/page-query";
import { RQ_KEY } from "../comment/queries/comment-query";
import { queryClient } from "@/main.tsx";
export const useQuerySubscription = () => {
const queryClient = useQueryClient();
@ -27,6 +36,15 @@ export const useQuerySubscription = () => {
queryKey: RQ_KEY(data.pageId),
});
break;
case "addTreeNode":
invalidateOnCreatePage(data.payload.data);
break;
case "moveTreeNode":
invalidateOnMovePage();
break;
case "deleteTreeNode":
invalidateOnDeletePage(data.payload.node.id);
break;
case "updateOne":
entity = data.entity[0];
if (entity === "pages") {
@ -37,13 +55,23 @@ export const useQuerySubscription = () => {
}
// only update if data was already in cache
if(queryClient.getQueryData([...data.entity, queryKeyId])){
if (queryClient.getQueryData([...data.entity, queryKeyId])) {
queryClient.setQueryData([...data.entity, queryKeyId], {
...queryClient.getQueryData([...data.entity, queryKeyId]),
...data.payload,
});
}
if (entity === "pages") {
invalidateOnUpdatePage(
data.spaceId,
data.payload.parentPageId,
data.id,
data.payload.title,
data.payload.icon,
);
}
/*
queryClient.setQueriesData(
{ queryKey: [data.entity, data.id] },
@ -57,6 +85,17 @@ export const useQuerySubscription = () => {
);
*/
break;
case "refetchRootTreeNodeEvent": {
const spaceId = data.spaceId;
queryClient.refetchQueries({
queryKey: ["root-sidebar-pages", spaceId],
});
queryClient.invalidateQueries({
queryKey: ["recent-changes", spaceId],
});
break;
}
}
});
}, [queryClient, socket]);

View File

@ -70,6 +70,11 @@ export function getFileUploadSizeLimit() {
return bytes(limit);
}
export function getFileImportSizeLimit() {
const limit = getConfigValue("FILE_IMPORT_SIZE_LIMIT", "200mb");
return bytes(limit);
}
export function getDrawioUrl() {
return getConfigValue("DRAWIO_URL", "https://embed.diagrams.net");
}

View File

@ -8,6 +8,7 @@ export default defineConfig(({ mode }) => {
const {
APP_URL,
FILE_UPLOAD_SIZE_LIMIT,
FILE_IMPORT_SIZE_LIMIT,
DRAWIO_URL,
CLOUD,
SUBDOMAIN_HOST,
@ -20,6 +21,7 @@ export default defineConfig(({ mode }) => {
"process.env": {
APP_URL,
FILE_UPLOAD_SIZE_LIMIT,
FILE_IMPORT_SIZE_LIMIT,
DRAWIO_URL,
CLOUD,
SUBDOMAIN_HOST,

View File

@ -31,7 +31,7 @@
},
"dependencies": {
"@aws-sdk/client-s3": "3.701.0",
"@aws-sdk/lib-storage": "^3.701.0",
"@aws-sdk/lib-storage": "3.701.0",
"@aws-sdk/s3-request-presigner": "3.701.0",
"@casl/ability": "^6.7.3",
"@fastify/cookie": "^11.0.2",
@ -57,6 +57,7 @@
"bcrypt": "^5.1.1",
"bullmq": "^5.41.3",
"cache-manager": "^6.4.0",
"cheerio": "^1.0.0",
"class-transformer": "^0.5.1",
"class-validator": "^0.14.1",
"cookie": "^1.0.2",

View File

@ -130,7 +130,7 @@ export class PersistenceExtension implements Extension {
);
this.contributors.delete(documentName);
} catch (err) {
this.logger.debug('Contributors error:' + err?.['message']);
//this.logger.debug('Contributors error:' + err?.['message']);
}
await this.pageRepo.updatePage(

View File

@ -6,22 +6,17 @@ export async function up(db: Kysely<any>): Promise<void> {
.addColumn('id', 'uuid', (col) =>
col.primaryKey().defaultTo(sql`gen_uuid_v7()`),
)
//type: import or export
// type (import, export)
.addColumn('type', 'varchar', (col) => col)
// source - generic, notion, confluence
// type or provider?
// source (generic, notion, confluence)
.addColumn('source', 'varchar', (col) => col)
// status (enum: PENDING|PROCESSING|SUCCESS|FAILED),
// status (pending|processing|success|failed),
.addColumn('status', 'varchar', (col) => col)
// file name
// file path
// file size
.addColumn('file_name', 'varchar', (col) => col.notNull())
.addColumn('file_path', 'varchar', (col) => col.notNull())
.addColumn('file_size', 'int8', (col) => col)
.addColumn('file_ext', 'varchar', (col) => col)
.addColumn('error_message', 'varchar', (col) => col)
.addColumn('creator_id', 'uuid', (col) => col.references('users.id'))
.addColumn('space_id', 'uuid', (col) =>
col.references('spaces.id').onDelete('cascade'),
@ -35,7 +30,6 @@ export async function up(db: Kysely<any>): Promise<void> {
.addColumn('updated_at', 'timestamptz', (col) =>
col.notNull().defaultTo(sql`now()`),
)
.addColumn('completed_at', 'timestamptz', (col) => col)
.addColumn('deleted_at', 'timestamptz', (col) => col)
.execute();
}

View File

@ -123,10 +123,10 @@ export interface Comments {
}
export interface FileTasks {
completedAt: Timestamp | null;
createdAt: Generated<Timestamp>;
creatorId: string | null;
deletedAt: Timestamp | null;
errorMessage: string | null;
fileExt: string | null;
fileName: string;
filePath: string;

View File

@ -67,6 +67,10 @@ export class EnvironmentService {
return this.configService.get<string>('FILE_UPLOAD_SIZE_LIMIT', '50mb');
}
getFileImportSizeLimit(): string {
return this.configService.get<string>('FILE_IMPORT_SIZE_LIMIT', '200mb');
}
getAwsS3AccessKeyId(): string {
return this.configService.get<string>('AWS_S3_ACCESS_KEY_ID');
}

View File

@ -0,0 +1,18 @@
import { IsNotEmpty, IsUUID } from 'class-validator';
export class FileTaskIdDto {
@IsNotEmpty()
@IsUUID()
fileTaskId: string;
}
export type ImportPageNode = {
id: string;
slugId: string;
name: string;
content: string;
position?: string | null;
parentPageId: string | null;
fileExtension: string;
filePath: string;
};

View File

@ -0,0 +1,79 @@
import {
Body,
Controller,
ForbiddenException,
HttpCode,
HttpStatus,
NotFoundException,
Post,
UseGuards,
} from '@nestjs/common';
import SpaceAbilityFactory from '../../core/casl/abilities/space-ability.factory';
import { JwtAuthGuard } from '../../common/guards/jwt-auth.guard';
import { User } from '@docmost/db/types/entity.types';
import {
SpaceCaslAction,
SpaceCaslSubject,
} from '../../core/casl/interfaces/space-ability.type';
import { InjectKysely } from 'nestjs-kysely';
import { KyselyDB } from '@docmost/db/types/kysely.types';
import { AuthUser } from '../../common/decorators/auth-user.decorator';
import { FileTaskIdDto } from './dto/file-task-dto';
import { SpaceMemberRepo } from '@docmost/db/repos/space/space-member.repo';
@Controller('file-tasks')
export class FileTaskController {
constructor(
private readonly spaceMemberRepo: SpaceMemberRepo,
private readonly spaceAbility: SpaceAbilityFactory,
@InjectKysely() private readonly db: KyselyDB,
) {}
@UseGuards(JwtAuthGuard)
@HttpCode(HttpStatus.OK)
@Post()
async getFileTasks(@AuthUser() user: User) {
const userSpaceIds = await this.spaceMemberRepo.getUserSpaceIds(user.id);
if (!userSpaceIds || userSpaceIds.length === 0) {
return [];
}
const fileTasks = await this.db
.selectFrom('fileTasks')
.selectAll()
.where('spaceId', 'in', userSpaceIds)
.execute();
if (!fileTasks) {
throw new NotFoundException('File task not found');
}
return fileTasks;
}
@UseGuards(JwtAuthGuard)
@HttpCode(HttpStatus.OK)
@Post('info')
async getFileTask(@Body() dto: FileTaskIdDto, @AuthUser() user: User) {
const fileTask = await this.db
.selectFrom('fileTasks')
.selectAll()
.where('id', '=', dto.fileTaskId)
.executeTakeFirst();
if (!fileTask || !fileTask.spaceId) {
throw new NotFoundException('File task not found');
}
const ability = await this.spaceAbility.createForUser(
user,
fileTask.spaceId,
);
if (ability.cannot(SpaceCaslAction.Read, SpaceCaslSubject.Page)) {
throw new ForbiddenException();
}
return fileTask;
}
}

View File

@ -1,623 +0,0 @@
import { Injectable, Logger } from '@nestjs/common';
import * as path from 'path';
import { jsonToText } from '../../collaboration/collaboration.util';
import { InjectKysely } from 'nestjs-kysely';
import { KyselyDB } from '@docmost/db/types/kysely.types';
import { cleanUrlString, extractZip, FileTaskStatus } from './file.utils';
import { StorageService } from '../storage/storage.service';
import * as tmp from 'tmp-promise';
import { pipeline } from 'node:stream/promises';
import { createReadStream, createWriteStream } from 'node:fs';
import { ImportService } from './import.service';
import { promises as fs } from 'fs';
import {
generateSlugId,
getMimeType,
sanitizeFileName,
} from '../../common/helpers';
import { v7 } from 'uuid';
import { generateJitteredKeyBetween } from 'fractional-indexing-jittered';
import { FileTask, InsertablePage } from '@docmost/db/types/entity.types';
import {
DOMParser,
Node as HDNode,
Element as HDElement,
Window,
} from 'happy-dom';
import { markdownToHtml } from '@docmost/editor-ext';
import { getAttachmentFolderPath } from '../../core/attachment/attachment.utils';
import { AttachmentType } from '../../core/attachment/attachment.constants';
import { getProsemirrorContent } from '../../common/helpers/prosemirror/utils';
import { not } from 'rxjs/internal/util/not';
import { notionFormatter } from './import-formatter';
@Injectable()
export class FileTaskService {
private readonly logger = new Logger(FileTaskService.name);
constructor(
private readonly storageService: StorageService,
private readonly importService: ImportService,
@InjectKysely() private readonly db: KyselyDB,
) {}
async processZIpImport(fileTaskId: string): Promise<void> {
const fileTask = await this.db
.selectFrom('fileTasks')
.selectAll()
.where('id', '=', fileTaskId)
.executeTakeFirst();
if (!fileTask) {
this.logger.log(`File task with ID ${fileTaskId} not found`);
return;
}
const { path: tmpZipPath, cleanup: cleanupTmpFile } = await tmp.file({
prefix: 'docmost-import',
postfix: '.zip',
discardDescriptor: true,
});
const { path: tmpExtractDir, cleanup: cleanupTmpDir } = await tmp.dir({
prefix: 'docmost-extract-',
unsafeCleanup: true,
});
const fileStream = await this.storageService.readStream(fileTask.filePath);
await pipeline(fileStream, createWriteStream(tmpZipPath));
await extractZip(tmpZipPath, tmpExtractDir);
console.log('extract here');
// TODO: internal link mentions, backlinks, attachments
try {
await this.updateTaskStatus(fileTaskId, FileTaskStatus.Processing);
// if type == generic
await this.processGenericImport({ extractDir: tmpExtractDir, fileTask });
await this.updateTaskStatus(fileTaskId, FileTaskStatus.Success);
} catch (error) {
await this.updateTaskStatus(fileTaskId, FileTaskStatus.Failed);
console.error(error);
} finally {
await cleanupTmpFile();
await cleanupTmpDir();
}
}
async processGenericImport(opts: {
extractDir: string;
fileTask: FileTask;
}): Promise<void> {
const { extractDir, fileTask } = opts;
const allFiles = await this.collectMarkdownAndHtmlFiles(extractDir);
const attachmentCandidates =
await this.buildAttachmentCandidates(extractDir);
console.log('attachment count: ', attachmentCandidates.size);
const pagesMap = new Map<
string,
{
id: string;
slugId: string;
name: string;
content: string;
position?: string | null;
parentPageId: string | null;
fileExtension: string;
filePath: string;
}
>();
for (const absPath of allFiles) {
const relPath = path
.relative(extractDir, absPath)
.split(path.sep)
.join('/'); // normalize to forward-slashes
const ext = path.extname(relPath).toLowerCase();
let content = await fs.readFile(absPath, 'utf-8');
console.log('relative path: ', relPath, ' abs path: ', absPath);
if (ext.toLowerCase() === '.html' || ext.toLowerCase() === '.md') {
// we want to process all inputs as markr
if (ext === '.md') {
content = await markdownToHtml(content);
}
//content = this.stripAllStyles(content)
content = await this.rewriteLocalFilesInHtml({
html: content,
pageRelativePath: relPath,
extractDir,
pageId: v7(),
fileTask,
attachmentCandidates,
});
}
pagesMap.set(relPath, {
id: v7(),
slugId: generateSlugId(),
name: path.basename(relPath, ext),
content,
parentPageId: null,
fileExtension: ext,
filePath: relPath,
});
}
// parent/child linking
pagesMap.forEach((page, filePath) => {
const segments = filePath.split('/');
segments.pop();
let parentPage = null;
while (segments.length) {
const tryMd = segments.join('/') + '.md';
const tryHtml = segments.join('/') + '.html';
if (pagesMap.has(tryMd)) {
parentPage = pagesMap.get(tryMd)!;
break;
}
if (pagesMap.has(tryHtml)) {
parentPage = pagesMap.get(tryHtml)!;
break;
}
segments.pop();
}
if (parentPage) page.parentPageId = parentPage.id;
});
// generate position keys
const siblingsMap = new Map<string | null, typeof Array.prototype>();
pagesMap.forEach((page) => {
const sibs = siblingsMap.get(page.parentPageId) || [];
sibs.push(page);
siblingsMap.set(page.parentPageId, sibs);
});
siblingsMap.forEach((sibs) => {
sibs.sort((a, b) => a.name.localeCompare(b.name));
let prevPos: string | null = null;
for (const page of sibs) {
page.position = generateJitteredKeyBetween(prevPos, null);
prevPos = page.position;
}
});
const filePathToPageMetaMap = new Map<
string,
{ id: string; title: string; slugId: string }
>();
pagesMap.forEach((page) => {
filePathToPageMetaMap.set(page.filePath, {
id: page.id,
title: page.name,
slugId: page.slugId,
});
});
const insertablePages: InsertablePage[] = await Promise.all(
Array.from(pagesMap.values()).map(async (page) => {
const htmlContent = await this.rewriteInternalLinksToMentionHtml(
page.content,
page.filePath,
filePathToPageMetaMap,
fileTask.creatorId,
);
const pmState = getProsemirrorContent(
await this.importService.processHTML(notionFormatter(htmlContent)),
);
const { title, prosemirrorJson } =
this.importService.extractTitleAndRemoveHeading(pmState);
return {
id: page.id,
slugId: page.slugId,
title: title || page.name,
content: prosemirrorJson,
textContent: jsonToText(prosemirrorJson),
ydoc: await this.importService.createYdoc(prosemirrorJson),
position: page.position!,
spaceId: fileTask.spaceId,
workspaceId: fileTask.workspaceId,
creatorId: fileTask.creatorId,
lastUpdatedById: fileTask.creatorId,
parentPageId: page.parentPageId,
};
}),
);
try {
await this.db.insertInto('pages').values(insertablePages).execute();
//todo: avoid duplicates
// log success
// backlinks mapping
// handle svg diagram nodes
} catch (e) {
console.error(e);
}
}
async rewriteLocalFilesInHtml(opts: {
html: string;
pageRelativePath: string;
extractDir: string;
pageId: string;
fileTask: FileTask;
attachmentCandidates: Map<string, string>;
}): Promise<string> {
const {
html,
pageRelativePath,
extractDir,
pageId,
fileTask,
attachmentCandidates,
} = opts;
const window = new Window();
const doc = window.document;
doc.body.innerHTML = html;
const tasks: Promise<void>[] = [];
const processFile = (relPath: string) => {
const abs = attachmentCandidates.get(relPath)!;
const attachmentId = v7();
const ext = path.extname(abs);
const fileNameWithExt =
sanitizeFileName(path.basename(abs, ext)) + ext.toLowerCase();
const storageFilePath = `${getAttachmentFolderPath(AttachmentType.File, fileTask.workspaceId)}/${attachmentId}/${fileNameWithExt}`;
const apiFilePath = `/api/files/${attachmentId}/${fileNameWithExt}`;
tasks.push(
(async () => {
const fileStream = createReadStream(abs);
await this.storageService.uploadStream(storageFilePath, fileStream);
const stat = await fs.stat(abs);
const uploaded = await this.db
.insertInto('attachments')
.values({
id: attachmentId,
filePath: storageFilePath,
fileName: fileNameWithExt,
fileSize: stat.size,
mimeType: getMimeType(fileNameWithExt),
type: 'file',
fileExt: ext,
creatorId: fileTask.creatorId,
workspaceId: fileTask.workspaceId,
pageId,
spaceId: fileTask.spaceId,
})
.returningAll()
.execute();
console.log(uploaded);
})(),
);
return {
attachmentId,
storageFilePath,
apiFilePath,
fileNameWithExt,
abs,
};
};
const pageDir = path.dirname(pageRelativePath);
for (const img of Array.from(doc.getElementsByTagName('img'))) {
const src = cleanUrlString(img.getAttribute('src')) ?? '';
if (!src || src.startsWith('http')) continue;
const relPath = this.resolveRelativeAttachmentPath(
src,
pageDir,
attachmentCandidates,
);
if (!relPath) continue;
const { attachmentId, apiFilePath, abs } = processFile(relPath);
const stat = await fs.stat(abs);
const width = img.getAttribute('width') || '100%';
const align = img.getAttribute('data-align') || 'center';
img.setAttribute('src', apiFilePath);
img.setAttribute('data-attachment-id', attachmentId);
img.setAttribute('data-size', stat.size.toString());
img.setAttribute('width', width);
img.setAttribute('data-align', align);
this.unwrapFromParagraph(img);
}
// rewrite <video>
for (const vid of Array.from(doc.getElementsByTagName('video'))) {
const src = cleanUrlString(vid.getAttribute('src')) ?? '';
if (!src || src.startsWith('http')) continue;
const relPath = this.resolveRelativeAttachmentPath(
src,
pageDir,
attachmentCandidates,
);
if (!relPath) continue;
const { attachmentId, apiFilePath, abs } = processFile(relPath);
const stat = await fs.stat(abs);
const width = vid.getAttribute('width') || '100%';
const align = vid.getAttribute('data-align') || 'center';
vid.setAttribute('src', apiFilePath);
vid.setAttribute('data-attachment-id', attachmentId);
vid.setAttribute('data-size', stat.size.toString());
vid.setAttribute('width', width);
vid.setAttribute('data-align', align);
// @ts-ignore
this.unwrapFromParagraph(vid);
}
// rewrite other attachments via <a>
for (const a of Array.from(doc.getElementsByTagName('a'))) {
const href = cleanUrlString(a.getAttribute('href')) ?? '';
if (!href || href.startsWith('http')) continue;
const relPath = this.resolveRelativeAttachmentPath(
href,
pageDir,
attachmentCandidates,
);
if (!relPath) continue;
const { attachmentId, apiFilePath, abs } = processFile(relPath);
const stat = await fs.stat(abs);
const ext = path.extname(relPath).toLowerCase();
if (ext === '.mp4') {
const video = doc.createElement('video');
video.setAttribute('src', apiFilePath);
video.setAttribute('data-attachment-id', attachmentId);
video.setAttribute('data-size', stat.size.toString());
video.setAttribute('width', '100%');
video.setAttribute('data-align', 'center');
a.replaceWith(video);
// @ts-ignore
this.unwrapFromParagraph(video);
} else {
const div = doc.createElement('div') as HDElement;
div.setAttribute('data-type', 'attachment');
div.setAttribute('data-attachment-url', apiFilePath);
div.setAttribute('data-attachment-name', path.basename(abs));
div.setAttribute('data-attachment-mime', getMimeType(abs));
div.setAttribute('data-attachment-size', stat.size.toString());
div.setAttribute('data-attachment-id', attachmentId);
a.replaceWith(div);
this.unwrapFromParagraph(div);
}
}
const attachmentDivs = Array.from(
doc.querySelectorAll('div[data-type="attachment"]'),
);
for (const oldDiv of attachmentDivs) {
const rawUrl =
cleanUrlString(oldDiv.getAttribute('data-attachment-url')) ?? '';
if (!rawUrl || rawUrl.startsWith('http')) continue;
const relPath = this.resolveRelativeAttachmentPath(
rawUrl,
pageDir,
attachmentCandidates,
);
if (!relPath) continue;
const { attachmentId, apiFilePath, abs } = processFile(relPath);
const stat = await fs.stat(abs);
const fileName = path.basename(abs);
const mime = getMimeType(abs);
const div = doc.createElement('div') as HDElement;
div.setAttribute('data-type', 'attachment');
div.setAttribute('data-attachment-url', apiFilePath);
div.setAttribute('data-attachment-name', fileName);
div.setAttribute('data-attachment-mime', mime);
div.setAttribute('data-attachment-size', stat.size.toString());
div.setAttribute('data-attachment-id', attachmentId);
oldDiv.replaceWith(div);
this.unwrapFromParagraph(div);
}
for (const type of ['excalidraw', 'drawio'] as const) {
const selector = `div[data-type="${type}"]`;
const oldDivs = Array.from(doc.querySelectorAll(selector));
for (const oldDiv of oldDivs) {
const rawSrc = cleanUrlString(oldDiv.getAttribute('data-src')) ?? '';
if (!rawSrc || rawSrc.startsWith('http')) continue;
const relPath = this.resolveRelativeAttachmentPath(
rawSrc,
pageDir,
attachmentCandidates,
);
if (!relPath) continue;
const { attachmentId, apiFilePath, abs } = processFile(relPath);
const stat = await fs.stat(abs);
const fileName = path.basename(abs);
const width = oldDiv.getAttribute('data-width') || '100%';
const align = oldDiv.getAttribute('data-align') || 'center';
const newDiv = doc.createElement('div') as HDElement;
newDiv.setAttribute('data-type', type);
newDiv.setAttribute('data-src', apiFilePath);
newDiv.setAttribute('data-title', fileName);
newDiv.setAttribute('data-width', width);
newDiv.setAttribute('data-size', stat.size.toString());
newDiv.setAttribute('data-align', align);
newDiv.setAttribute('data-attachment-id', attachmentId);
oldDiv.replaceWith(newDiv);
this.unwrapFromParagraph(newDiv);
}
}
// wait for all uploads & DB inserts
await Promise.all(tasks);
return doc.documentElement.outerHTML;
}
async rewriteInternalLinksToMentionHtml(
html: string,
currentFilePath: string,
filePathToPageMetaMap: Map<
string,
{ id: string; title: string; slugId: string }
>,
creatorId: string,
): Promise<string> {
const window = new Window();
const doc = window.document;
doc.body.innerHTML = html;
// normalize helper
const normalize = (p: string) => p.replace(/\\/g, '/');
for (const a of Array.from(doc.getElementsByTagName('a'))) {
const rawHref = a.getAttribute('href');
if (!rawHref) continue;
// skip absolute/external URLs
if (rawHref.startsWith('http') || rawHref.startsWith('/api/')) {
continue;
}
const decodedRef = decodeURIComponent(rawHref);
const parentDir = path.dirname(currentFilePath);
const joined = path.join(parentDir, decodedRef);
const resolved = normalize(joined);
const pageMeta = filePathToPageMetaMap.get(resolved);
if (!pageMeta) {
// not an internal link we know about
continue;
}
const mentionEl = doc.createElement('span') as HDElement;
mentionEl.setAttribute('data-type', 'mention');
mentionEl.setAttribute('data-id', v7());
mentionEl.setAttribute('data-entity-type', 'page');
mentionEl.setAttribute('data-entity-id', pageMeta.id);
mentionEl.setAttribute('data-label', pageMeta.title);
mentionEl.setAttribute('data-slug-id', pageMeta.slugId);
mentionEl.setAttribute('data-creator-id', creatorId);
mentionEl.textContent = pageMeta.title;
a.replaceWith(mentionEl);
}
return doc.body.innerHTML;
}
unwrapFromParagraph(node: HDElement) {
let wrapper = node.closest('p, a') as HDElement | null;
while (wrapper) {
if (wrapper.childNodes.length === 1) {
// e.g. <p><node/></p> or <a><node/></a> → <node/>
wrapper.replaceWith(node);
} else {
wrapper.parentNode!.insertBefore(node, wrapper);
}
wrapper = node.closest('p, a') as HDElement | null;
}
}
async buildAttachmentCandidates(
extractDir: string,
): Promise<Map<string, string>> {
const map = new Map<string, string>();
async function walk(dir: string) {
for (const ent of await fs.readdir(dir, { withFileTypes: true })) {
const abs = path.join(dir, ent.name);
if (ent.isDirectory()) {
await walk(abs);
} else {
if (['.md', '.html'].includes(path.extname(ent.name).toLowerCase())) {
continue;
}
const rel = path.relative(extractDir, abs).split(path.sep).join('/');
map.set(rel, abs);
}
}
}
await walk(extractDir);
return map;
}
async collectMarkdownAndHtmlFiles(dir: string): Promise<string[]> {
const results: string[] = [];
async function walk(current: string) {
const entries = await fs.readdir(current, { withFileTypes: true });
for (const ent of entries) {
const fullPath = path.join(current, ent.name);
if (ent.isDirectory()) {
await walk(fullPath);
} else if (
['.md', '.html'].includes(path.extname(ent.name).toLowerCase())
) {
results.push(fullPath);
}
}
}
await walk(dir);
return results;
}
resolveRelativeAttachmentPath(
raw: string,
pageDir: string,
attachmentCandidates: Map<string, string>,
): string | null {
const mainRel = decodeURIComponent(raw.replace(/^\.?\/+/, ''));
const fallback = path.normalize(path.join(pageDir, mainRel));
if (attachmentCandidates.has(mainRel)) {
return mainRel;
}
if (attachmentCandidates.has(fallback)) {
return fallback;
}
return null;
}
async updateTaskStatus(fileTaskId: string, status: FileTaskStatus) {
await this.db
.updateTable('fileTasks')
.set({ status: status })
.where('id', '=', fileTaskId)
.execute();
}
}

View File

@ -1,94 +0,0 @@
import * as yauzl from 'yauzl';
import * as path from 'path';
import * as fs from 'node:fs';
export enum FileTaskType {
Import = 'import',
Export = 'export',
}
export enum FileImportType {
Generic = 'generic',
Notion = 'notion',
Confluence = 'confluence',
}
export enum FileTaskStatus {
Pending = 'pending',
Processing = 'processing',
Success = 'success',
Failed = 'failed',
}
export function getFileTaskFolderPath(
type: FileTaskType,
workspaceId: string,
): string {
switch (type) {
case FileTaskType.Import:
return `${workspaceId}/imports`;
case FileTaskType.Export:
return `${workspaceId}/exports`;
}
}
export function extractZip(source: string, target: string) {
//https://github.com/Surfer-Org
return new Promise((resolve, reject) => {
yauzl.open(
source,
{ lazyEntries: true, decodeStrings: false, autoClose: true },
(err, zipfile) => {
if (err) return reject(err);
zipfile.readEntry();
zipfile.on('entry', (entry) => {
const name = entry.fileName.toString('utf8'); // or 'cp437' if you need the original DOS charset
const safeName = name.replace(/^\/+/, ''); // strip any leading slashes
const fullPath = path.join(target, safeName);
const directory = path.dirname(fullPath);
// <-- skip all macOS metadata
if (safeName.startsWith('__MACOSX/')) {
return zipfile.readEntry();
}
if (/\/$/.test(entry.fileName)) {
// Directory entry
try {
fs.mkdirSync(fullPath, { recursive: true });
zipfile.readEntry();
} catch (err) {
reject(err);
}
} else {
// File entry
try {
fs.mkdirSync(directory, { recursive: true });
zipfile.openReadStream(entry, (err, readStream) => {
if (err) return reject(err);
const writeStream = fs.createWriteStream(fullPath);
readStream.on('end', () => {
writeStream.end();
zipfile.readEntry();
});
readStream.pipe(writeStream);
});
} catch (err) {
reject(err);
}
}
});
zipfile.on('end', resolve);
zipfile.on('error', reject);
},
);
});
}
export function cleanUrlString(url: string): string {
const [maybePath] = url.split('?', 1);
return maybePath;
}

View File

@ -1,127 +0,0 @@
import { Window } from 'happy-dom';
export function notionFormatter(html: string): string {
const window = new Window();
const doc = window.document;
doc.body.innerHTML = html;
// Block math
for (const fig of Array.from(doc.querySelectorAll('figure.equation'))) {
// get TeX source from the MathML <annotation>
const annotation = fig.querySelector(
'annotation[encoding="application/x-tex"]',
);
const tex = annotation?.textContent?.trim() ?? '';
const mathBlock = doc.createElement('div');
mathBlock.setAttribute('data-type', 'mathBlock');
mathBlock.setAttribute('data-katex', 'true');
mathBlock.textContent = tex;
fig.replaceWith(mathBlock);
}
// Inline math
for (const token of Array.from(
doc.querySelectorAll('span.notion-text-equation-token'),
)) {
// remove the preceding <style> if its that KaTeX import
const prev = token.previousElementSibling;
if (prev?.tagName === 'STYLE') prev.remove();
const annotation = token.querySelector(
'annotation[encoding="application/x-tex"]',
);
const tex = annotation?.textContent?.trim() ?? '';
const mathInline = doc.createElement('span');
mathInline.setAttribute('data-type', 'mathInline');
mathInline.setAttribute('data-katex', 'true');
mathInline.textContent = tex;
token.replaceWith(mathInline);
}
// Callouts
const figs = Array.from(doc.querySelectorAll('figure.callout')).reverse();
for (const fig of figs) {
// find the content <div> (always the 2nd child in a Notion callout)
const contentDiv = fig.querySelector(
'div:nth-of-type(2)',
) as unknown as HTMLElement | null;
if (!contentDiv) continue;
// pull out every block inside (tables, p, nested callouts, lists…)
const blocks = Array.from(contentDiv.childNodes);
const wrapper = fig.ownerDocument.createElement('div');
wrapper.setAttribute('data-type', 'callout');
wrapper.setAttribute('data-callout-type', 'info');
// move each real node into the wrapper (preserves nested structure)
// @ts-ignore
wrapper.append(...blocks);
fig.replaceWith(wrapper);
}
// Todolist
const todoLists = Array.from(doc.querySelectorAll('ul.to-do-list'));
for (const oldList of todoLists) {
const newList = doc.createElement('ul');
newList.setAttribute('data-type', 'taskList');
// for each old <li>, create a <li data-type="taskItem" data-checked="…">
for (const li of oldList.querySelectorAll('li')) {
const isChecked = li.querySelector('.checkbox.checkbox-on') != null;
const textSpan = li.querySelector(
'span.to-do-children-unchecked, span.to-do-children-checked',
);
const text = textSpan?.textContent?.trim() ?? '';
// <li data-type="taskItem" data-checked="true|false">
const taskItem = doc.createElement('li');
taskItem.setAttribute('data-type', 'taskItem');
taskItem.setAttribute('data-checked', String(isChecked));
// <label><input type="checkbox" [checked]><span></span></label>
const label = doc.createElement('label');
const input = doc.createElement('input');
input.type = 'checkbox';
if (isChecked) input.checked = true;
const spacer = doc.createElement('span');
label.append(input, spacer);
const container = doc.createElement('div');
const p = doc.createElement('p');
p.textContent = text;
container.appendChild(p);
taskItem.append(label, container);
newList.appendChild(taskItem);
}
oldList.replaceWith(newList);
}
// Fix toggle blocks
const detailsList = Array.from(
doc.querySelectorAll('ul.toggle details'),
).reverse();
// unwrap from ul and li tags
for (const details of detailsList) {
const li = details.closest('li');
if (li) {
li.parentNode!.insertBefore(details, li);
if (li.childNodes.length === 0) li.remove();
}
const ul = details.closest('ul.toggle');
if (ul) {
ul.parentNode!.insertBefore(details, ul);
if (ul.childNodes.length === 0) ul.remove();
}
}
return doc.body.innerHTML;
}

View File

@ -21,8 +21,9 @@ import {
import { FileInterceptor } from '../../common/interceptors/file.interceptor';
import * as bytes from 'bytes';
import * as path from 'path';
import { ImportService } from './import.service';
import { ImportService } from './services/import.service';
import { AuthWorkspace } from '../../common/decorators/auth-workspace.decorator';
import { EnvironmentService } from '../environment/environment.service';
@Controller()
export class ImportController {
@ -31,6 +32,7 @@ export class ImportController {
constructor(
private readonly importService: ImportService,
private readonly spaceAbility: SpaceAbilityFactory,
private readonly environmentService: EnvironmentService,
) {}
@UseInterceptors(FileInterceptor)
@ -44,18 +46,18 @@ export class ImportController {
) {
const validFileExtensions = ['.md', '.html'];
const maxFileSize = bytes('100mb');
const maxFileSize = bytes('10mb');
let file = null;
try {
file = await req.file({
limits: { fileSize: maxFileSize, fields: 3, files: 1 },
limits: { fileSize: maxFileSize, fields: 4, files: 1 },
});
} catch (err: any) {
this.logger.error(err.message);
if (err?.statusCode === 413) {
throw new BadRequestException(
`File too large. Exceeds the 100mb import limit`,
`File too large. Exceeds the 10mb import limit`,
);
}
}
@ -73,7 +75,7 @@ export class ImportController {
const spaceId = file.fields?.spaceId?.value;
if (!spaceId) {
throw new BadRequestException('spaceId or format not found');
throw new BadRequestException('spaceId is required');
}
const ability = await this.spaceAbility.createForUser(user, spaceId);
@ -87,7 +89,6 @@ export class ImportController {
@UseInterceptors(FileInterceptor)
@UseGuards(JwtAuthGuard)
@HttpCode(HttpStatus.OK)
// temporary naming
@Post('pages/import-zip')
async importZip(
@Req() req: any,
@ -96,7 +97,7 @@ export class ImportController {
) {
const validFileExtensions = ['.zip'];
const maxFileSize = bytes('100mb');
const maxFileSize = bytes(this.environmentService.getFileImportSizeLimit());
let file = null;
try {
@ -107,7 +108,7 @@ export class ImportController {
this.logger.error(err.message);
if (err?.statusCode === 413) {
throw new BadRequestException(
`File too large. Exceeds the 100mb import limit`,
`File too large. Exceeds the ${this.environmentService.getFileImportSizeLimit()} import limit`,
);
}
}
@ -119,14 +120,21 @@ export class ImportController {
if (
!validFileExtensions.includes(path.extname(file.filename).toLowerCase())
) {
throw new BadRequestException('Invalid import file type.');
throw new BadRequestException('Invalid import file extension.');
}
const spaceId = file.fields?.spaceId?.value;
const source = file.fields?.source?.value;
const validZipSources = ['generic', 'notion', 'confluence'];
if (!validZipSources.includes(source)) {
throw new BadRequestException(
'Invalid import source. Import source must either be generic, notion or confluence.',
);
}
if (!spaceId) {
throw new BadRequestException('spaceId or format not found');
throw new BadRequestException('spaceId is required');
}
const ability = await this.spaceAbility.createForUser(user, spaceId);
@ -134,6 +142,12 @@ export class ImportController {
throw new ForbiddenException();
}
return this.importService.importZip(file, source, user.id, spaceId, workspace.id);
return this.importService.importZip(
file,
source,
user.id,
spaceId,
workspace.id,
);
}
}

View File

@ -1,13 +1,22 @@
import { Module } from '@nestjs/common';
import { ImportService } from './import.service';
import { ImportService } from './services/import.service';
import { ImportController } from './import.controller';
import { StorageModule } from '../storage/storage.module';
import { FileTaskService } from './file-task.service';
import { FileTaskService } from './services/file-task.service';
import { FileTaskProcessor } from './processors/file-task.processor';
import { ImportAttachmentService } from './services/import-attachment.service';
import { FileTaskController } from './file-task.controller';
import { PageModule } from '../../core/page/page.module';
@Module({
providers: [ImportService, FileTaskService, FileTaskProcessor],
controllers: [ImportController],
imports: [StorageModule],
providers: [
ImportService,
FileTaskService,
FileTaskProcessor,
ImportAttachmentService,
],
exports: [ImportService, ImportAttachmentService],
controllers: [ImportController, FileTaskController],
imports: [StorageModule, PageModule],
})
export class ImportModule {}

View File

@ -2,12 +2,18 @@ import { Logger, OnModuleDestroy } from '@nestjs/common';
import { OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq';
import { Job } from 'bullmq';
import { QueueJob, QueueName } from 'src/integrations/queue/constants';
import { FileTaskService } from '../file-task.service';
import { FileTaskService } from '../services/file-task.service';
import { FileTaskStatus } from '../utils/file.utils';
import { StorageService } from '../../storage/storage.service';
@Processor(QueueName.FILE_TASK_QUEUE)
export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy {
private readonly logger = new Logger(FileTaskProcessor.name);
constructor(private readonly fileTaskService: FileTaskService) {
constructor(
private readonly fileTaskService: FileTaskService,
private readonly storageService: StorageService,
) {
super();
}
@ -15,14 +21,14 @@ export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy {
try {
switch (job.name) {
case QueueJob.IMPORT_TASK:
console.log('import task', job.data.fileTaskId);
await this.fileTaskService.processZIpImport(job.data.fileTaskId);
break;
case QueueJob.EXPORT_TASK:
console.log('export task', job.data.fileTaskId);
// TODO: export task
break;
}
} catch (err) {
console.error(err);
this.logger.error('File task failed', err);
throw err;
}
}
@ -33,15 +39,33 @@ export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy {
}
@OnWorkerEvent('failed')
onError(job: Job) {
async onFailed(job: Job) {
this.logger.error(
`Error processing ${job.name} job. Reason: ${job.failedReason}`,
);
try {
const fileTaskId = job.data.fileTaskId;
await this.fileTaskService.updateTaskStatus(
fileTaskId,
FileTaskStatus.Failed,
job.failedReason,
);
const fileTask = await this.fileTaskService.getFileTask(fileTaskId);
if (fileTask) {
await this.storageService.delete(fileTask.filePath);
}
} catch (err) {
this.logger.error(err);
}
}
@OnWorkerEvent('completed')
onCompleted(job: Job) {
this.logger.debug(`Completed ${job.name} job`);
this.logger.log(
`Completed ${job.name} job for File task ID ${job.data.fileTaskId}`,
);
}
async onModuleDestroy(): Promise<void> {

View File

@ -0,0 +1,346 @@
import { Injectable, Logger } from '@nestjs/common';
import * as path from 'path';
import { jsonToText } from '../../../collaboration/collaboration.util';
import { InjectKysely } from 'nestjs-kysely';
import { KyselyDB } from '@docmost/db/types/kysely.types';
import {
extractZip,
FileImportSource,
FileTaskStatus,
} from '../utils/file.utils';
import { StorageService } from '../../storage/storage.service';
import * as tmp from 'tmp-promise';
import { pipeline } from 'node:stream/promises';
import { createWriteStream } from 'node:fs';
import { ImportService } from './import.service';
import { promises as fs } from 'fs';
import { generateSlugId } from '../../../common/helpers';
import { v7 } from 'uuid';
import { generateJitteredKeyBetween } from 'fractional-indexing-jittered';
import { FileTask, InsertablePage } from '@docmost/db/types/entity.types';
import { markdownToHtml } from '@docmost/editor-ext';
import { getProsemirrorContent } from '../../../common/helpers/prosemirror/utils';
import { formatImportHtml } from '../utils/import-formatter';
import {
buildAttachmentCandidates,
collectMarkdownAndHtmlFiles,
} from '../utils/import.utils';
import { executeTx } from '@docmost/db/utils';
import { BacklinkRepo } from '@docmost/db/repos/backlink/backlink.repo';
import { ImportAttachmentService } from './import-attachment.service';
import { ModuleRef } from '@nestjs/core';
import { PageService } from '../../../core/page/services/page.service';
import { ImportPageNode } from '../dto/file-task-dto';
@Injectable()
export class FileTaskService {
private readonly logger = new Logger(FileTaskService.name);
constructor(
private readonly storageService: StorageService,
private readonly importService: ImportService,
private readonly pageService: PageService,
private readonly backlinkRepo: BacklinkRepo,
@InjectKysely() private readonly db: KyselyDB,
private readonly importAttachmentService: ImportAttachmentService,
private moduleRef: ModuleRef,
) {}
async processZIpImport(fileTaskId: string): Promise<void> {
const fileTask = await this.db
.selectFrom('fileTasks')
.selectAll()
.where('id', '=', fileTaskId)
.executeTakeFirst();
if (!fileTask) {
this.logger.log(`Import file task with ID ${fileTaskId} not found`);
return;
}
if (fileTask.status === FileTaskStatus.Failed) {
return;
}
if (fileTask.status === FileTaskStatus.Success) {
this.logger.log('Imported task already processed.');
return;
}
const { path: tmpZipPath, cleanup: cleanupTmpFile } = await tmp.file({
prefix: 'docmost-import',
postfix: '.zip',
discardDescriptor: true,
});
const { path: tmpExtractDir, cleanup: cleanupTmpDir } = await tmp.dir({
prefix: 'docmost-extract-',
unsafeCleanup: true,
});
try {
const fileStream = await this.storageService.readStream(
fileTask.filePath,
);
await pipeline(fileStream, createWriteStream(tmpZipPath));
await extractZip(tmpZipPath, tmpExtractDir);
} catch (err) {
await cleanupTmpFile();
await cleanupTmpDir();
throw err;
}
try {
if (
fileTask.source === FileImportSource.Generic ||
fileTask.source === FileImportSource.Notion
) {
await this.processGenericImport({
extractDir: tmpExtractDir,
fileTask,
});
}
if (fileTask.source === FileImportSource.Confluence) {
let ConfluenceModule: any;
try {
// eslint-disable-next-line @typescript-eslint/no-require-imports
ConfluenceModule = require('./../../../ee/confluence-import/confluence-import.service');
} catch (err) {
this.logger.error(
'Confluence import requested but EE module not bundled in this build',
);
return;
}
const confluenceImportService = this.moduleRef.get(
ConfluenceModule.ConfluenceImportService,
{ strict: false },
);
await confluenceImportService.processConfluenceImport({
extractDir: tmpExtractDir,
fileTask,
});
}
try {
await this.updateTaskStatus(fileTaskId, FileTaskStatus.Success, null);
await cleanupTmpFile();
await cleanupTmpDir();
// delete stored file on success
await this.storageService.delete(fileTask.filePath);
} catch (err) {
this.logger.error(
`Failed to delete import file from storage. Task ID: ${fileTaskId}`,
err,
);
}
} catch (err) {
await cleanupTmpFile();
await cleanupTmpDir();
throw err;
}
}
async processGenericImport(opts: {
extractDir: string;
fileTask: FileTask;
}): Promise<void> {
const { extractDir, fileTask } = opts;
const allFiles = await collectMarkdownAndHtmlFiles(extractDir);
const attachmentCandidates = await buildAttachmentCandidates(extractDir);
const pagesMap = new Map<string, ImportPageNode>();
for (const absPath of allFiles) {
const relPath = path
.relative(extractDir, absPath)
.split(path.sep)
.join('/'); // normalize to forward-slashes
const ext = path.extname(relPath).toLowerCase();
let content = await fs.readFile(absPath, 'utf-8');
if (ext.toLowerCase() === '.md') {
content = await markdownToHtml(content);
}
pagesMap.set(relPath, {
id: v7(),
slugId: generateSlugId(),
name: path.basename(relPath, ext),
content,
parentPageId: null,
fileExtension: ext,
filePath: relPath,
});
}
// parent/child linking
pagesMap.forEach((page, filePath) => {
const segments = filePath.split('/');
segments.pop();
let parentPage = null;
while (segments.length) {
const tryMd = segments.join('/') + '.md';
const tryHtml = segments.join('/') + '.html';
if (pagesMap.has(tryMd)) {
parentPage = pagesMap.get(tryMd)!;
break;
}
if (pagesMap.has(tryHtml)) {
parentPage = pagesMap.get(tryHtml)!;
break;
}
segments.pop();
}
if (parentPage) page.parentPageId = parentPage.id;
});
// generate position keys
const siblingsMap = new Map<string | null, ImportPageNode[]>();
pagesMap.forEach((page) => {
const group = siblingsMap.get(page.parentPageId) ?? [];
group.push(page);
siblingsMap.set(page.parentPageId, group);
});
// get root pages
const rootSibs = siblingsMap.get(null);
if (rootSibs?.length) {
rootSibs.sort((a, b) => a.name.localeCompare(b.name));
// get first position key from the server
const nextPosition = await this.pageService.nextPagePosition(
fileTask.spaceId,
);
let prevPos: string | null = null;
rootSibs.forEach((page, idx) => {
if (idx === 0) {
page.position = nextPosition;
} else {
page.position = generateJitteredKeyBetween(prevPos, null);
}
prevPos = page.position;
});
}
// non-root buckets (children & deeper levels)
siblingsMap.forEach((sibs, parentId) => {
if (parentId === null) return; // root already done
sibs.sort((a, b) => a.name.localeCompare(b.name));
let prevPos: string | null = null;
for (const page of sibs) {
page.position = generateJitteredKeyBetween(prevPos, null);
prevPos = page.position;
}
});
// internal page links
const filePathToPageMetaMap = new Map<
string,
{ id: string; title: string; slugId: string }
>();
pagesMap.forEach((page) => {
filePathToPageMetaMap.set(page.filePath, {
id: page.id,
title: page.name,
slugId: page.slugId,
});
});
const pageResults = await Promise.all(
Array.from(pagesMap.values()).map(async (page) => {
const htmlContent =
await this.importAttachmentService.processAttachments({
html: page.content,
pageRelativePath: page.filePath,
extractDir,
pageId: page.id,
fileTask,
attachmentCandidates,
});
const { html, backlinks } = await formatImportHtml({
html: htmlContent,
currentFilePath: page.filePath,
filePathToPageMetaMap: filePathToPageMetaMap,
creatorId: fileTask.creatorId,
sourcePageId: page.id,
workspaceId: fileTask.workspaceId,
});
const pmState = getProsemirrorContent(
await this.importService.processHTML(html),
);
const { title, prosemirrorJson } =
this.importService.extractTitleAndRemoveHeading(pmState);
const insertablePage: InsertablePage = {
id: page.id,
slugId: page.slugId,
title: title || page.name,
content: prosemirrorJson,
textContent: jsonToText(prosemirrorJson),
ydoc: await this.importService.createYdoc(prosemirrorJson),
position: page.position!,
spaceId: fileTask.spaceId,
workspaceId: fileTask.workspaceId,
creatorId: fileTask.creatorId,
lastUpdatedById: fileTask.creatorId,
parentPageId: page.parentPageId,
};
return { insertablePage, backlinks };
}),
);
const insertablePages = pageResults.map((r) => r.insertablePage);
const insertableBacklinks = pageResults.flatMap((r) => r.backlinks);
if (insertablePages.length < 1) return;
const validPageIds = new Set(insertablePages.map((row) => row.id));
const filteredBacklinks = insertableBacklinks.filter(
({ sourcePageId, targetPageId }) =>
validPageIds.has(sourcePageId) && validPageIds.has(targetPageId),
);
await executeTx(this.db, async (trx) => {
await trx.insertInto('pages').values(insertablePages).execute();
if (filteredBacklinks.length > 0) {
await this.backlinkRepo.insertBacklink(filteredBacklinks, trx);
}
});
}
async getFileTask(fileTaskId: string) {
return this.db
.selectFrom('fileTasks')
.selectAll()
.where('id', '=', fileTaskId)
.executeTakeFirst();
}
async updateTaskStatus(
fileTaskId: string,
status: FileTaskStatus,
errorMessage?: string,
) {
try {
await this.db
.updateTable('fileTasks')
.set({ status: status, errorMessage, updatedAt: new Date() })
.where('id', '=', fileTaskId)
.execute();
} catch (err) {
this.logger.error(err);
}
}
}

View File

@ -0,0 +1,303 @@
import { Injectable, Logger } from '@nestjs/common';
import * as path from 'path';
import { InjectKysely } from 'nestjs-kysely';
import { KyselyDB } from '@docmost/db/types/kysely.types';
import { cleanUrlString } from '../utils/file.utils';
import { StorageService } from '../../storage/storage.service';
import { createReadStream } from 'node:fs';
import { promises as fs } from 'fs';
import { getMimeType, sanitizeFileName } from '../../../common/helpers';
import { v7 } from 'uuid';
import { FileTask } from '@docmost/db/types/entity.types';
import { getAttachmentFolderPath } from '../../../core/attachment/attachment.utils';
import { AttachmentType } from '../../../core/attachment/attachment.constants';
import { unwrapFromParagraph } from '../utils/import-formatter';
import { resolveRelativeAttachmentPath } from '../utils/import.utils';
import { load } from 'cheerio';
@Injectable()
export class ImportAttachmentService {
private readonly logger = new Logger(ImportAttachmentService.name);
constructor(
private readonly storageService: StorageService,
@InjectKysely() private readonly db: KyselyDB,
) {}
async processAttachments(opts: {
html: string;
pageRelativePath: string;
extractDir: string;
pageId: string;
fileTask: FileTask;
attachmentCandidates: Map<string, string>;
}): Promise<string> {
const {
html,
pageRelativePath,
extractDir,
pageId,
fileTask,
attachmentCandidates,
} = opts;
const attachmentTasks: Promise<void>[] = [];
/**
* Cache keyed by the *relative* path that appears in the HTML.
* Ensures we upload (and DB-insert) each attachment at most once,
* even if its referenced multiple times on the page.
*/
const processed = new Map<
string,
{
attachmentId: string;
storageFilePath: string;
apiFilePath: string;
fileNameWithExt: string;
abs: string;
}
>();
const uploadOnce = (relPath: string) => {
const abs = attachmentCandidates.get(relPath)!;
const attachmentId = v7();
const ext = path.extname(abs);
const fileNameWithExt =
sanitizeFileName(path.basename(abs, ext)) + ext.toLowerCase();
const storageFilePath = `${getAttachmentFolderPath(
AttachmentType.File,
fileTask.workspaceId,
)}/${attachmentId}/${fileNameWithExt}`;
const apiFilePath = `/api/files/${attachmentId}/${fileNameWithExt}`;
attachmentTasks.push(
(async () => {
const fileStream = createReadStream(abs);
await this.storageService.uploadStream(storageFilePath, fileStream);
const stat = await fs.stat(abs);
await this.db
.insertInto('attachments')
.values({
id: attachmentId,
filePath: storageFilePath,
fileName: fileNameWithExt,
fileSize: stat.size,
mimeType: getMimeType(fileNameWithExt),
type: 'file',
fileExt: ext,
creatorId: fileTask.creatorId,
workspaceId: fileTask.workspaceId,
pageId,
spaceId: fileTask.spaceId,
})
.execute();
})(),
);
return {
attachmentId,
storageFilePath,
apiFilePath,
fileNameWithExt,
abs,
};
};
/**
* Returns cached data if weve already processed this path.
* Otherwise calls `uploadOnce`, stores the result, and returns it.
*/
const processFile = (relPath: string) => {
const cached = processed.get(relPath);
if (cached) return cached;
const fresh = uploadOnce(relPath);
processed.set(relPath, fresh);
return fresh;
};
const pageDir = path.dirname(pageRelativePath);
const $ = load(html);
// image
for (const imgEl of $('img').toArray()) {
const $img = $(imgEl);
const src = cleanUrlString($img.attr('src') ?? '')!;
if (!src || src.startsWith('http')) continue;
const relPath = resolveRelativeAttachmentPath(
src,
pageDir,
attachmentCandidates,
);
if (!relPath) continue;
const { attachmentId, apiFilePath, abs } = processFile(relPath);
const stat = await fs.stat(abs);
const width = $img.attr('width') ?? '100%';
const align = $img.attr('data-align') ?? 'center';
$img
.attr('src', apiFilePath)
.attr('data-attachment-id', attachmentId)
.attr('data-size', stat.size.toString())
.attr('width', width)
.attr('data-align', align);
unwrapFromParagraph($, $img);
}
// video
for (const vidEl of $('video').toArray()) {
const $vid = $(vidEl);
const src = cleanUrlString($vid.attr('src') ?? '')!;
if (!src || src.startsWith('http')) continue;
const relPath = resolveRelativeAttachmentPath(
src,
pageDir,
attachmentCandidates,
);
if (!relPath) continue;
const { attachmentId, apiFilePath, abs } = processFile(relPath);
const stat = await fs.stat(abs);
const width = $vid.attr('width') ?? '100%';
const align = $vid.attr('data-align') ?? 'center';
$vid
.attr('src', apiFilePath)
.attr('data-attachment-id', attachmentId)
.attr('data-size', stat.size.toString())
.attr('width', width)
.attr('data-align', align);
unwrapFromParagraph($, $vid);
}
// <div data-type="attachment">
for (const el of $('div[data-type="attachment"]').toArray()) {
const $oldDiv = $(el);
const rawUrl = cleanUrlString($oldDiv.attr('data-attachment-url') ?? '')!;
if (!rawUrl || rawUrl.startsWith('http')) continue;
const relPath = resolveRelativeAttachmentPath(
rawUrl,
pageDir,
attachmentCandidates,
);
if (!relPath) continue;
const { attachmentId, apiFilePath, abs } = processFile(relPath);
const stat = await fs.stat(abs);
const fileName = path.basename(abs);
const mime = getMimeType(abs);
const $newDiv = $('<div>')
.attr('data-type', 'attachment')
.attr('data-attachment-url', apiFilePath)
.attr('data-attachment-name', fileName)
.attr('data-attachment-mime', mime)
.attr('data-attachment-size', stat.size.toString())
.attr('data-attachment-id', attachmentId);
$oldDiv.replaceWith($newDiv);
unwrapFromParagraph($, $newDiv);
}
// rewrite other attachments via <a>
for (const aEl of $('a').toArray()) {
const $a = $(aEl);
const href = cleanUrlString($a.attr('href') ?? '')!;
if (!href || href.startsWith('http')) continue;
const relPath = resolveRelativeAttachmentPath(
href,
pageDir,
attachmentCandidates,
);
if (!relPath) continue;
const { attachmentId, apiFilePath, abs } = processFile(relPath);
const stat = await fs.stat(abs);
const ext = path.extname(relPath).toLowerCase();
if (ext === '.mp4') {
const $video = $('<video>')
.attr('src', apiFilePath)
.attr('data-attachment-id', attachmentId)
.attr('data-size', stat.size.toString())
.attr('width', '100%')
.attr('data-align', 'center');
$a.replaceWith($video);
unwrapFromParagraph($, $video);
} else {
const confAliasName = $a.attr('data-linked-resource-default-alias');
let attachmentName = path.basename(abs);
if (confAliasName) attachmentName = confAliasName;
const $div = $('<div>')
.attr('data-type', 'attachment')
.attr('data-attachment-url', apiFilePath)
.attr('data-attachment-name', attachmentName)
.attr('data-attachment-mime', getMimeType(abs))
.attr('data-attachment-size', stat.size.toString())
.attr('data-attachment-id', attachmentId);
$a.replaceWith($div);
unwrapFromParagraph($, $div);
}
}
// excalidraw and drawio
for (const type of ['excalidraw', 'drawio'] as const) {
for (const el of $(`div[data-type="${type}"]`).toArray()) {
const $oldDiv = $(el);
const rawSrc = cleanUrlString($oldDiv.attr('data-src') ?? '')!;
if (!rawSrc || rawSrc.startsWith('http')) continue;
const relPath = resolveRelativeAttachmentPath(
rawSrc,
pageDir,
attachmentCandidates,
);
if (!relPath) continue;
const { attachmentId, apiFilePath, abs } = processFile(relPath);
const stat = await fs.stat(abs);
const fileName = path.basename(abs);
const width = $oldDiv.attr('data-width') || '100%';
const align = $oldDiv.attr('data-align') || 'center';
const $newDiv = $('<div>')
.attr('data-type', type)
.attr('data-src', apiFilePath)
.attr('data-title', fileName)
.attr('data-width', width)
.attr('data-size', stat.size.toString())
.attr('data-align', align)
.attr('data-attachment-id', attachmentId);
$oldDiv.replaceWith($newDiv);
unwrapFromParagraph($, $newDiv);
}
}
// wait for all uploads & DB inserts
try {
await Promise.all(attachmentTasks);
} catch (err) {
this.logger.log('Import attachment upload error', err);
}
return $.root().html() || '';
}
}

View File

@ -7,10 +7,10 @@ import {
htmlToJson,
jsonToText,
tiptapExtensions,
} from '../../collaboration/collaboration.util';
} from '../../../collaboration/collaboration.util';
import { InjectKysely } from 'nestjs-kysely';
import { KyselyDB } from '@docmost/db/types/kysely.types';
import { generateSlugId } from '../../common/helpers';
import { generateSlugId, sanitizeFileName } from '../../../common/helpers';
import { generateJitteredKeyBetween } from 'fractional-indexing-jittered';
import { TiptapTransformer } from '@hocuspocus/transformer';
import * as Y from 'yjs';
@ -19,15 +19,12 @@ import {
FileTaskStatus,
FileTaskType,
getFileTaskFolderPath,
} from './file.utils';
import { v7, v7 as uuid7 } from 'uuid';
import { StorageService } from '../storage/storage.service';
} from '../utils/file.utils';
import { v7 as uuid7 } from 'uuid';
import { StorageService } from '../../storage/storage.service';
import { InjectQueue } from '@nestjs/bullmq';
import { Queue } from 'bullmq';
import { QueueJob, QueueName } from '../queue/constants';
import { Node as PMNode } from '@tiptap/pm/model';
import { EditorState, Transaction } from '@tiptap/pm/state';
import { getSchema } from '@tiptap/core';
import { QueueJob, QueueName } from '../../queue/constants';
@Injectable()
export class ImportService {
@ -199,134 +196,45 @@ export class ImportService {
userId: string,
spaceId: string,
workspaceId: string,
): Promise<void> {
) {
const file = await filePromise;
const fileBuffer = await file.toBuffer();
const fileExtension = path.extname(file.filename).toLowerCase();
const fileName = sanitize(
path.basename(file.filename, fileExtension).slice(0, 255),
const fileName = sanitizeFileName(
path.basename(file.filename, fileExtension),
);
const fileSize = fileBuffer.length;
const fileNameWithExt = fileName + fileExtension;
const fileTaskId = uuid7();
const filePath = `${getFileTaskFolderPath(FileTaskType.Import, workspaceId)}/${fileTaskId}/${fileName}`;
const filePath = `${getFileTaskFolderPath(FileTaskType.Import, workspaceId)}/${fileTaskId}/${fileNameWithExt}`;
// upload file
await this.storageService.upload(filePath, fileBuffer);
// store in fileTasks table
await this.db
const fileTask = await this.db
.insertInto('fileTasks')
.values({
id: fileTaskId,
type: FileTaskType.Import,
source: source,
status: FileTaskStatus.Pending,
fileName: fileName,
status: FileTaskStatus.Processing,
fileName: fileNameWithExt,
filePath: filePath,
fileSize: 0,
fileSize: fileSize,
fileExt: 'zip',
creatorId: userId,
spaceId: spaceId,
workspaceId: workspaceId,
})
.execute();
.returningAll()
.executeTakeFirst();
// what to send to queue
// pass the task ID
await this.fileTaskQueue.add(QueueJob.IMPORT_TASK, {
fileTaskId: fileTaskId,
});
// return tasks info
// when the processor picks it up
// we change the status to processing
// if it gets processed successfully,
// we change the status to success
// else failed
}
async markdownOrHtmlToProsemirror(
fileContent: string,
fileExtension: string,
): Promise<any> {
let prosemirrorState = '';
if (fileExtension === '.md') {
prosemirrorState = await this.processMarkdown(fileContent);
} else if (fileExtension.endsWith('.html')) {
prosemirrorState = await this.processHTML(fileContent);
}
return prosemirrorState;
}
async convertInternalLinksToMentionsPM(
doc: PMNode,
currentFilePath: string,
filePathToPageMetaMap: Map<
string,
{ id: string; title: string; slugId: string }
>,
): Promise<PMNode> {
const schema = getSchema(tiptapExtensions);
const state = EditorState.create({ doc, schema });
let tr: Transaction = state.tr;
const normalizePath = (p: string) => p.replace(/\\/g, '/');
// Collect replacements from the original doc.
const replacements: Array<{
from: number;
to: number;
mentionNode: PMNode;
}> = [];
doc.descendants((node, pos) => {
if (!node.isText || !node.marks?.length) return;
// Look for the link mark
const linkMark = node.marks.find(
(mark) => mark.type.name === 'link' && mark.attrs?.href,
);
if (!linkMark) return;
// Compute the range for the entire text node.
const from = pos;
const to = pos + node.nodeSize;
// Resolve the path and get page meta.
const resolvedPath = normalizePath(
path.join(path.dirname(currentFilePath), linkMark.attrs.href),
);
const pageMeta = filePathToPageMetaMap.get(resolvedPath);
if (!pageMeta) return;
// Create the mention node with all required attributes.
const mentionNode = schema.nodes.mention.create({
id: v7(),
entityType: 'page',
entityId: pageMeta.id,
label: node.text || pageMeta.title,
slugId: pageMeta.slugId,
creatorId: 'not available', // This is required per your schema.
});
replacements.push({ from, to, mentionNode });
});
// Apply replacements in reverse order.
for (let i = replacements.length - 1; i >= 0; i--) {
const { from, to, mentionNode } = replacements[i];
try {
tr = tr.replaceWith(from, to, mentionNode);
} catch (err) {
console.error('❌ Failed to insert mention:', err);
}
}
if (tr.docChanged) {
console.log('doc changed');
console.log(JSON.stringify(state.apply(tr).doc.toJSON()));
}
// Return the updated document if any change was made.
return tr.docChanged ? state.apply(tr).doc : doc;
return fileTask;
}
}

View File

@ -0,0 +1,187 @@
import * as yauzl from 'yauzl';
import * as path from 'path';
import * as fs from 'node:fs';
export enum FileTaskType {
Import = 'import',
Export = 'export',
}
export enum FileImportSource {
Generic = 'generic',
Notion = 'notion',
Confluence = 'confluence',
}
export enum FileTaskStatus {
Processing = 'processing',
Success = 'success',
Failed = 'failed',
}
export function getFileTaskFolderPath(
type: FileTaskType,
workspaceId: string,
): string {
switch (type) {
case FileTaskType.Import:
return `${workspaceId}/imports`;
case FileTaskType.Export:
return `${workspaceId}/exports`;
}
}
/**
* Extracts a ZIP archive.
*/
export async function extractZip(
source: string,
target: string,
): Promise<void> {
return extractZipInternal(source, target, true);
}
/**
* Internal helper to extract a ZIP, with optional single-nested-ZIP handling.
* @param source Path to the ZIP file
* @param target Directory to extract into
* @param allowNested Whether to check and unwrap one level of nested ZIP
*/
function extractZipInternal(
source: string,
target: string,
allowNested: boolean,
): Promise<void> {
return new Promise((resolve, reject) => {
yauzl.open(
source,
{ lazyEntries: true, decodeStrings: false, autoClose: true },
(err, zipfile) => {
if (err) return reject(err);
// Handle one level of nested ZIP if allowed
if (allowNested && zipfile.entryCount === 1) {
zipfile.readEntry();
zipfile.once('entry', (entry) => {
const name = entry.fileName.toString('utf8').replace(/^\/+/, '');
const isZip =
!/\/$/.test(entry.fileName) &&
name.toLowerCase().endsWith('.zip');
if (isZip) {
// temporary name to avoid overwriting file
const nestedPath = source.endsWith('.zip')
? source.slice(0, -4) + '.inner.zip'
: source + '.inner.zip';
zipfile.openReadStream(entry, (openErr, rs) => {
if (openErr) return reject(openErr);
const ws = fs.createWriteStream(nestedPath);
rs.on('error', reject);
ws.on('error', reject);
ws.on('finish', () => {
zipfile.close();
extractZipInternal(nestedPath, target, false)
.then(() => {
fs.unlinkSync(nestedPath);
resolve();
})
.catch(reject);
});
rs.pipe(ws);
});
} else {
zipfile.close();
extractZipInternal(source, target, false).then(resolve, reject);
}
});
zipfile.once('error', reject);
return;
}
// Normal extraction
zipfile.readEntry();
zipfile.on('entry', (entry) => {
const name = entry.fileName.toString('utf8');
const safe = name.replace(/^\/+/, '');
if (safe.startsWith('__MACOSX/')) {
zipfile.readEntry();
return;
}
const fullPath = path.join(target, safe);
// Handle directories
if (/\/$/.test(name)) {
try {
fs.mkdirSync(fullPath, { recursive: true });
} catch (mkdirErr: any) {
if (mkdirErr.code === 'ENAMETOOLONG') {
console.warn(`Skipping directory (path too long): ${fullPath}`);
zipfile.readEntry();
return;
}
return reject(mkdirErr);
}
zipfile.readEntry();
return;
}
// Handle files
try {
fs.mkdirSync(path.dirname(fullPath), { recursive: true });
} catch (mkdirErr: any) {
if (mkdirErr.code === 'ENAMETOOLONG') {
console.warn(
`Skipping file directory creation (path too long): ${fullPath}`,
);
zipfile.readEntry();
return;
}
return reject(mkdirErr);
}
zipfile.openReadStream(entry, (openErr, rs) => {
if (openErr) return reject(openErr);
let ws: fs.WriteStream;
try {
ws = fs.createWriteStream(fullPath);
} catch (openWsErr: any) {
if (openWsErr.code === 'ENAMETOOLONG') {
console.warn(
`Skipping file write (path too long): ${fullPath}`,
);
zipfile.readEntry();
return;
}
return reject(openWsErr);
}
rs.on('error', (err) => reject(err));
ws.on('error', (err) => {
if ((err as any).code === 'ENAMETOOLONG') {
console.warn(
`Skipping file write on stream (path too long): ${fullPath}`,
);
zipfile.readEntry();
} else {
reject(err);
}
});
ws.on('finish', () => zipfile.readEntry());
rs.pipe(ws);
});
});
zipfile.on('end', () => resolve());
zipfile.on('error', (err) => reject(err));
},
);
});
}
export function cleanUrlString(url: string): string {
if (!url) return null;
const [mainUrl] = url.split('?', 1);
return mainUrl;
}

View File

@ -0,0 +1,254 @@
import { getEmbedUrlAndProvider } from '@docmost/editor-ext';
import * as path from 'path';
import { v7 } from 'uuid';
import { InsertableBacklink } from '@docmost/db/types/entity.types';
import { Cheerio, CheerioAPI, load } from 'cheerio';
export async function formatImportHtml(opts: {
html: string;
currentFilePath: string;
filePathToPageMetaMap: Map<
string,
{ id: string; title: string; slugId: string }
>;
creatorId: string;
sourcePageId: string;
workspaceId: string;
pageDir?: string;
attachmentCandidates?: string[];
}): Promise<{ html: string; backlinks: InsertableBacklink[] }> {
const {
html,
currentFilePath,
filePathToPageMetaMap,
creatorId,
sourcePageId,
workspaceId,
} = opts;
const $: CheerioAPI = load(html);
const $root: Cheerio<any> = $.root();
notionFormatter($, $root);
defaultHtmlFormatter($, $root);
const backlinks = await rewriteInternalLinksToMentionHtml(
$,
$root,
currentFilePath,
filePathToPageMetaMap,
creatorId,
sourcePageId,
workspaceId,
);
return {
html: $root.html() || '',
backlinks,
};
}
export function defaultHtmlFormatter($: CheerioAPI, $root: Cheerio<any>) {
$root.find('a[href]').each((_, el) => {
const $el = $(el);
const url = $el.attr('href')!;
const { provider } = getEmbedUrlAndProvider(url);
if (provider === 'iframe') return;
const embed = `<div data-type=\"embed\" data-src=\"${url}\" data-provider=\"${provider}\" data-align=\"center\" data-width=\"640\" data-height=\"480\"></div>`;
$el.replaceWith(embed);
});
$root.find('iframe[src]').each((_, el) => {
const $el = $(el);
const url = $el.attr('src')!;
const { provider } = getEmbedUrlAndProvider(url);
const embed = `<div data-type=\"embed\" data-src=\"${url}\" data-provider=\"${provider}\" data-align=\"center\" data-width=\"640\" data-height=\"480\"></div>`;
$el.replaceWith(embed);
});
}
export function notionFormatter($: CheerioAPI, $root: Cheerio<any>) {
// remove empty description paragraphs
$root.find('p.page-description').each((_, el) => {
if (!$(el).text().trim()) $(el).remove();
});
// block math → mathBlock
$root.find('figure.equation').each((_: any, fig: any) => {
const $fig = $(fig);
const tex = $fig
.find('annotation[encoding="application/x-tex"]')
.text()
.trim();
const $math = $('<div>')
.attr('data-type', 'mathBlock')
.attr('data-katex', 'true')
.text(tex);
$fig.replaceWith($math);
});
// inline math → mathInline
$root.find('span.notion-text-equation-token').each((_, tok) => {
const $tok = $(tok);
const $prev = $tok.prev('style');
if ($prev.length) $prev.remove();
const tex = $tok
.find('annotation[encoding="application/x-tex"]')
.text()
.trim();
const $inline = $('<span>')
.attr('data-type', 'mathInline')
.attr('data-katex', 'true')
.text(tex);
$tok.replaceWith($inline);
});
// callouts
$root
.find('figure.callout')
.get()
.reverse()
.forEach((fig) => {
const $fig = $(fig);
const $content = $fig.find('div').eq(1);
if (!$content.length) return;
const $wrapper = $('<div>')
.attr('data-type', 'callout')
.attr('data-callout-type', 'info');
// @ts-ignore
$content.children().each((_, child) => $wrapper.append(child));
$fig.replaceWith($wrapper);
});
// to-do lists
$root.find('ul.to-do-list').each((_, list) => {
const $old = $(list);
const $new = $('<ul>').attr('data-type', 'taskList');
$old.find('li').each((_, li) => {
const $li = $(li);
const isChecked = $li.find('.checkbox.checkbox-on').length > 0;
const text =
$li
.find('span.to-do-children-unchecked, span.to-do-children-checked')
.first()
.text()
.trim() || '';
const $taskItem = $('<li>')
.attr('data-type', 'taskItem')
.attr('data-checked', String(isChecked));
const $label = $('<label>');
const $input = $('<input>').attr('type', 'checkbox');
if (isChecked) $input.attr('checked', '');
$label.append($input, $('<span>'));
const $container = $('<div>').append($('<p>').text(text));
$taskItem.append($label, $container);
$new.append($taskItem);
});
$old.replaceWith($new);
});
// toggle blocks
$root
.find('ul.toggle details')
.get()
.reverse()
.forEach((det) => {
const $det = $(det);
const $li = $det.closest('li');
if ($li.length) {
$li.before($det);
if (!$li.children().length) $li.remove();
}
const $ul = $det.closest('ul.toggle');
if ($ul.length) {
$ul.before($det);
if (!$ul.children().length) $ul.remove();
}
});
// bookmarks
$root
.find('figure')
.filter((_, fig) => $(fig).find('a.bookmark.source').length > 0)
.get()
.reverse()
.forEach((fig) => {
const $fig = $(fig);
const $link = $fig.find('a.bookmark.source').first();
if (!$link.length) return;
const href = $link.attr('href')!;
const title = $link.find('.bookmark-title').text().trim() || href;
const $newAnchor = $('<a>')
.addClass('bookmark source')
.attr('href', href)
.append($('<div>').addClass('bookmark-info').text(title));
$fig.replaceWith($newAnchor);
});
// remove toc
$root.find('nav.table_of_contents').remove();
}
export function unwrapFromParagraph($: CheerioAPI, $node: Cheerio<any>) {
// find the nearest <p> or <a> ancestor
let $wrapper = $node.closest('p, a');
while ($wrapper.length) {
// if the wrapper has only our node inside, replace it entirely
if ($wrapper.contents().length === 1) {
$wrapper.replaceWith($node);
} else {
// otherwise just move the node to before the wrapper
$wrapper.before($node);
}
// look again for any new wrapper around $node
$wrapper = $node.closest('p, a');
}
}
export async function rewriteInternalLinksToMentionHtml(
$: CheerioAPI,
$root: Cheerio<any>,
currentFilePath: string,
filePathToPageMetaMap: Map<
string,
{ id: string; title: string; slugId: string }
>,
creatorId: string,
sourcePageId: string,
workspaceId: string,
): Promise<InsertableBacklink[]> {
const normalize = (p: string) => p.replace(/\\/g, '/');
const backlinks: InsertableBacklink[] = [];
$root.find('a[href]').each((_, el) => {
const $a = $(el);
const raw = $a.attr('href')!;
if (raw.startsWith('http') || raw.startsWith('/api/')) return;
const resolved = normalize(
path.join(path.dirname(currentFilePath), decodeURIComponent(raw)),
);
const meta = filePathToPageMetaMap.get(resolved);
if (!meta) return;
const mentionId = v7();
const $mention = $('<span>')
.attr({
'data-type': 'mention',
'data-id': mentionId,
'data-entity-type': 'page',
'data-entity-id': meta.id,
'data-label': meta.title,
'data-slug-id': meta.slugId,
'data-creator-id': creatorId,
})
.text(meta.title);
$a.replaceWith($mention);
backlinks.push({ sourcePageId, targetPageId: meta.id, workspaceId });
});
return backlinks;
}

View File

@ -0,0 +1,66 @@
import { promises as fs } from 'fs';
import * as path from 'path';
export async function buildAttachmentCandidates(
extractDir: string,
): Promise<Map<string, string>> {
const map = new Map<string, string>();
async function walk(dir: string) {
for (const ent of await fs.readdir(dir, { withFileTypes: true })) {
const abs = path.join(dir, ent.name);
if (ent.isDirectory()) {
await walk(abs);
} else {
if (['.md', '.html'].includes(path.extname(ent.name).toLowerCase())) {
continue;
}
const rel = path.relative(extractDir, abs).split(path.sep).join('/');
map.set(rel, abs);
}
}
}
await walk(extractDir);
return map;
}
export function resolveRelativeAttachmentPath(
raw: string,
pageDir: string,
attachmentCandidates: Map<string, string>,
): string | null {
const mainRel = decodeURIComponent(raw.replace(/^\.?\/+/, ''));
const fallback = path.normalize(path.join(pageDir, mainRel));
if (attachmentCandidates.has(mainRel)) {
return mainRel;
}
if (attachmentCandidates.has(fallback)) {
return fallback;
}
return null;
}
export async function collectMarkdownAndHtmlFiles(
dir: string,
): Promise<string[]> {
const results: string[] = [];
async function walk(current: string) {
const entries = await fs.readdir(current, { withFileTypes: true });
for (const ent of entries) {
const fullPath = path.join(current, ent.name);
if (ent.isDirectory()) {
await walk(fullPath);
} else if (
['.md', '.html'].includes(path.extname(ent.name).toLowerCase())
) {
results.push(fullPath);
}
}
}
await walk(dir);
return results;
}

View File

@ -51,6 +51,11 @@ import { BacklinksProcessor } from './processors/backlinks.processor';
}),
BullModule.registerQueue({
name: QueueName.FILE_TASK_QUEUE,
defaultJobOptions: {
removeOnComplete: true,
removeOnFail: true,
attempts: 1,
},
}),
],
exports: [BullModule],

View File

@ -88,7 +88,7 @@ async function bootstrap() {
const logger = new Logger('NestApplication');
process.on('unhandledRejection', (reason, promise) => {
logger.error(`UnhandledRejection: ${promise}, reason: ${reason}`);
logger.error(`UnhandledRejection, reason: ${reason}`, promise);
});
process.on('uncaughtException', (error) => {

View File

@ -16,4 +16,5 @@ export * from "./lib/drawio";
export * from "./lib/excalidraw";
export * from "./lib/embed";
export * from "./lib/mention";
export * from "./lib/markdown";
export * from "./lib/markdown";
export * from "./lib/embed-provider";

View File

@ -7,102 +7,117 @@ export interface IEmbedProvider {
export const embedProviders: IEmbedProvider[] = [
{
id: 'loom',
name: 'Loom',
id: "loom",
name: "Loom",
regex: /^https?:\/\/(?:www\.)?loom\.com\/(?:share|embed)\/([\da-zA-Z]+)\/?/,
getEmbedUrl: (match, url) => {
if(url.includes("/embed/")){
if (url.includes("/embed/")) {
return url;
}
return `https://loom.com/embed/${match[1]}`;
}
},
},
{
id: 'airtable',
name: 'Airtable',
id: "airtable",
name: "Airtable",
regex: /^https:\/\/(www.)?airtable.com\/([a-zA-Z0-9]{2,})\/.*/,
getEmbedUrl: (match, url: string) => {
const path = url.split('airtable.com/');
if(url.includes("/embed/")){
const path = url.split("airtable.com/");
if (url.includes("/embed/")) {
return url;
}
return `https://airtable.com/embed/${path[1]}`;
}
},
},
{
id: 'figma',
name: 'Figma',
regex: /^https:\/\/[\w\.-]+\.?figma.com\/(file|proto|board|design|slides|deck)\/([0-9a-zA-Z]{22,128})/,
id: "figma",
name: "Figma",
regex:
/^https:\/\/[\w\.-]+\.?figma.com\/(file|proto|board|design|slides|deck)\/([0-9a-zA-Z]{22,128})/,
getEmbedUrl: (match, url: string) => {
return `https://www.figma.com/embed?url=${url}&embed_host=docmost`;
}
},
},
{
'id': 'typeform',
name: 'Typeform',
id: "typeform",
name: "Typeform",
regex: /^(https?:)?(\/\/)?[\w\.]+\.typeform\.com\/to\/.+/,
getEmbedUrl: (match, url: string) => {
return url;
}
},
},
{
id: 'miro',
name: 'Miro',
id: "miro",
name: "Miro",
regex: /^https:\/\/(www\.)?miro\.com\/app\/board\/([\w-]+=)/,
getEmbedUrl: (match, url) => {
if(url.includes("/live-embed/")){
if (url.includes("/live-embed/")) {
return url;
}
return `https://miro.com/app/live-embed/${match[2]}?embedMode=view_only_without_ui&autoplay=true&embedSource=docmost`;
}
},
},
{
id: 'youtube',
name: 'YouTube',
regex: /^((?:https?:)?\/\/)?((?:www|m|music)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w\-]+\?v=|embed\/|v\/)?)([\w\-]+)(\S+)?$/,
id: "youtube",
name: "YouTube",
regex:
/^((?:https?:)?\/\/)?((?:www|m|music)\.)?((?:youtube\.com|youtu.be))(\/(?:[\w\-]+\?v=|embed\/|v\/)?)([\w\-]+)(\S+)?$/,
getEmbedUrl: (match, url) => {
if (url.includes("/embed/")){
if (url.includes("/embed/")) {
return url;
}
return `https://www.youtube-nocookie.com/embed/${match[5]}`;
}
},
},
{
id: 'vimeo',
name: 'Vimeo',
regex: /^(https:)?\/\/(?:www\.|player\.)?vimeo.com\/(?:channels\/(?:\w+\/)?|groups\/([^/]*)\/videos\/|album\/(\d+)\/video\/|video\/|)(\d+)/,
id: "vimeo",
name: "Vimeo",
regex:
/^(https:)?\/\/(?:www\.|player\.)?vimeo.com\/(?:channels\/(?:\w+\/)?|groups\/([^/]*)\/videos\/|album\/(\d+)\/video\/|video\/|)(\d+)/,
getEmbedUrl: (match) => {
return `https://player.vimeo.com/video/${match[4]}`;
}
},
},
{
id: 'framer',
name: 'Framer',
id: "framer",
name: "Framer",
regex: /^https:\/\/(www\.)?framer\.com\/embed\/([\w-]+)/,
getEmbedUrl: (match, url: string) => {
return url;
}
},
},
{
id: 'gdrive',
name: 'Google Drive',
regex: /^((?:https?:)?\/\/)?((?:www|m)\.)?(drive\.google\.com)\/file\/d\/([a-zA-Z0-9_-]+)\/.*$/,
id: "gdrive",
name: "Google Drive",
regex:
/^((?:https?:)?\/\/)?((?:www|m)\.)?(drive\.google\.com)\/file\/d\/([a-zA-Z0-9_-]+)\/.*$/,
getEmbedUrl: (match) => {
return `https://drive.google.com/file/d/${match[4]}/preview`;
}
},
},
{
id: 'gsheets',
name: 'Google Sheets',
regex: /^((?:https?:)?\/\/)?((?:www|m)\.)?(docs\.google\.com)\/spreadsheets\/d\/e\/([a-zA-Z0-9_-]+)\/.*$/,
id: "gsheets",
name: "Google Sheets",
regex:
/^((?:https?:)?\/\/)?((?:www|m)\.)?(docs\.google\.com)\/spreadsheets\/d\/e\/([a-zA-Z0-9_-]+)\/.*$/,
getEmbedUrl: (match, url: string) => {
return url
}
return url;
},
},
{
id: "iframe",
name: "Iframe",
regex: /any-iframe/,
getEmbedUrl: (match, url) => {
return url;
},
},
];
export function getEmbedProviderById(id: string) {
return embedProviders.find(provider => provider.id.toLowerCase() === id.toLowerCase());
return embedProviders.find(
(provider) => provider.id.toLowerCase() === id.toLowerCase(),
);
}
export interface IEmbedResult {
@ -116,14 +131,12 @@ export function getEmbedUrlAndProvider(url: string): IEmbedResult {
if (match) {
return {
embedUrl: provider.getEmbedUrl(match, url),
provider: provider.name.toLowerCase()
provider: provider.name.toLowerCase(),
};
}
}
return {
embedUrl: url,
provider: 'iframe',
provider: "iframe",
};
}

View File

@ -84,6 +84,14 @@ export const TiptapVideo = Node.create<VideoOptions>({
};
},
parseHTML() {
return [
{
tag: 'video',
},
]
},
renderHTML({ HTMLAttributes }) {
return [
"video",

396
pnpm-lock.yaml generated
View File

@ -406,8 +406,8 @@ importers:
specifier: 3.701.0
version: 3.701.0
'@aws-sdk/lib-storage':
specifier: ^3.701.0
version: 3.817.0(@aws-sdk/client-s3@3.701.0)
specifier: 3.701.0
version: 3.701.0(@aws-sdk/client-s3@3.701.0)
'@aws-sdk/s3-request-presigner':
specifier: 3.701.0
version: 3.701.0
@ -483,6 +483,9 @@ importers:
cache-manager:
specifier: ^6.4.0
version: 6.4.0
cheerio:
specifier: ^1.0.0
version: 1.0.0
class-transformer:
specifier: ^0.5.1
version: 0.5.1
@ -497,7 +500,7 @@ importers:
version: 11.3.0
happy-dom:
specifier: ^15.11.6
version: 15.11.6
version: 15.11.7
jsonwebtoken:
specifier: ^9.0.2
version: 9.0.2
@ -781,11 +784,11 @@ packages:
peerDependencies:
'@aws-sdk/client-sts': ^3.696.0
'@aws-sdk/lib-storage@3.817.0':
resolution: {integrity: sha512-2zOO8+2EmiS049PjLSNdqmmZMQj7fzE1hZJ70A94vO+KNaVhVZYuMOOiOmwMw6ePkTCcFwK40vZIIXwEQQ1v1g==}
engines: {node: '>=18.0.0'}
'@aws-sdk/lib-storage@3.701.0':
resolution: {integrity: sha512-eAbJ/3OgyFp1NnFdQfkZ7PuKCjrhbSQWf0EVTMhlg4aE5piCZ1We38NI1dQ58yr53rGc2gBkbYr8+/9CehpEvA==}
engines: {node: '>=16.0.0'}
peerDependencies:
'@aws-sdk/client-s3': ^3.817.0
'@aws-sdk/client-s3': ^3.701.0
'@aws-sdk/middleware-bucket-endpoint@3.696.0':
resolution: {integrity: sha512-V07jishKHUS5heRNGFpCWCSTjRJyQLynS/ncUeE8ZYtG66StOOQWftTwDfFOSoXlIqrXgb4oT9atryzXq7Z4LQ==}
@ -3191,10 +3194,6 @@ packages:
resolution: {integrity: sha512-yiW0WI30zj8ZKoSYNx90no7ugVn3khlyH/z5W8qtKBtVE6awRALbhSG+2SAHA1r6bO/6M9utxYKVZ3PCJ1rWxw==}
engines: {node: '>=16.0.0'}
'@smithy/abort-controller@4.0.3':
resolution: {integrity: sha512-AqXFf6DXnuRBXy4SoK/n1mfgHaKaq36bmkphmD1KO0nHq6xK/g9KHSW4HEsPQUBCGdIEfuJifGHwxFXPIFay9Q==}
engines: {node: '>=18.0.0'}
'@smithy/chunked-blob-reader-native@3.0.1':
resolution: {integrity: sha512-VEYtPvh5rs/xlyqpm5NRnfYLZn+q0SRPELbvBV+C/G7IQ+ouTuo+NKKa3ShG5OaFR8NYVMXls9hPYLTvIKKDrQ==}
@ -3209,10 +3208,6 @@ packages:
resolution: {integrity: sha512-8olpW6mKCa0v+ibCjoCzgZHQx1SQmZuW/WkrdZo73wiTprTH6qhmskT60QLFdT9DRa5mXxjz89kQPZ7ZSsoqqg==}
engines: {node: '>=16.0.0'}
'@smithy/core@3.4.0':
resolution: {integrity: sha512-dDYISQo7k0Ml/rXlFIjkTmTcQze/LxhtIRAEmZ6HJ/EI0inVxVEVnrUXJ7jPx6ZP0GHUhFm40iQcCgS5apXIXA==}
engines: {node: '>=18.0.0'}
'@smithy/credential-provider-imds@3.2.8':
resolution: {integrity: sha512-ZCY2yD0BY+K9iMXkkbnjo+08T2h8/34oHd0Jmh6BZUSZwaaGlGCyBT/3wnS7u7Xl33/EEfN4B6nQr3Gx5bYxgw==}
engines: {node: '>=16.0.0'}
@ -3239,10 +3234,6 @@ packages:
'@smithy/fetch-http-handler@4.1.3':
resolution: {integrity: sha512-6SxNltSncI8s689nvnzZQc/dPXcpHQ34KUj6gR/HBroytKOd/isMG3gJF/zBE1TBmTT18TXyzhg3O3SOOqGEhA==}
'@smithy/fetch-http-handler@5.0.3':
resolution: {integrity: sha512-yBZwavI31roqTndNI7ONHqesfH01JmjJK6L3uUpZAhyAmr86LN5QiPzfyZGIxQmed8VEK2NRSQT3/JX5V1njfQ==}
engines: {node: '>=18.0.0'}
'@smithy/hash-blob-browser@3.1.10':
resolution: {integrity: sha512-elwslXOoNunmfS0fh55jHggyhccobFkexLYC1ZeZ1xP2BTSrcIBaHV2b4xUQOdctrSNOpMqOZH1r2XzWTEhyfA==}
@ -3265,10 +3256,6 @@ packages:
resolution: {integrity: sha512-+Fsu6Q6C4RSJiy81Y8eApjEB5gVtM+oFKTffg+jSuwtvomJJrhUJBu2zS8wjXSgH/g1MKEWrzyChTBe6clb5FQ==}
engines: {node: '>=16.0.0'}
'@smithy/is-array-buffer@4.0.0':
resolution: {integrity: sha512-saYhF8ZZNoJDTvJBEWgeBccCg+yvp1CX+ed12yORU3NilJScfc6gfch2oVb4QgxZrGUx3/ZJlb+c/dJbyupxlw==}
engines: {node: '>=18.0.0'}
'@smithy/md5-js@3.0.11':
resolution: {integrity: sha512-3NM0L3i2Zm4bbgG6Ymi9NBcxXhryi3uE8fIfHJZIOfZVxOkGdjdgjR9A06SFIZCfnEIWKXZdm6Yq5/aPXFFhsQ==}
@ -3280,10 +3267,6 @@ packages:
resolution: {integrity: sha512-OEJZKVUEhMOqMs3ktrTWp7UvvluMJEvD5XgQwRePSbDg1VvBaL8pX8mwPltFn6wk1GySbcVwwyldL8S+iqnrEQ==}
engines: {node: '>=16.0.0'}
'@smithy/middleware-endpoint@4.1.7':
resolution: {integrity: sha512-KDzM7Iajo6K7eIWNNtukykRT4eWwlHjCEsULZUaSfi/SRSBK8BPRqG5FsVfp58lUxcvre8GT8AIPIqndA0ERKw==}
engines: {node: '>=18.0.0'}
'@smithy/middleware-retry@3.0.34':
resolution: {integrity: sha512-yVRr/AAtPZlUvwEkrq7S3x7Z8/xCd97m2hLDaqdz6ucP2RKHsBjEqaUA2ebNv2SsZoPEi+ZD0dZbOB1u37tGCA==}
engines: {node: '>=16.0.0'}
@ -3292,66 +3275,34 @@ packages:
resolution: {integrity: sha512-KzPAeySp/fOoQA82TpnwItvX8BBURecpx6ZMu75EZDkAcnPtO6vf7q4aH5QHs/F1s3/snQaSFbbUMcFFZ086Mw==}
engines: {node: '>=16.0.0'}
'@smithy/middleware-serde@4.0.6':
resolution: {integrity: sha512-YECyl7uNII+jCr/9qEmCu8xYL79cU0fqjo0qxpcVIU18dAPHam/iYwcknAu4Jiyw1uN+sAx7/SMf/Kmef/Jjsg==}
engines: {node: '>=18.0.0'}
'@smithy/middleware-stack@3.0.11':
resolution: {integrity: sha512-1HGo9a6/ikgOMrTrWL/WiN9N8GSVYpuRQO5kjstAq4CvV59bjqnh7TbdXGQ4vxLD3xlSjfBjq5t1SOELePsLnA==}
engines: {node: '>=16.0.0'}
'@smithy/middleware-stack@4.0.3':
resolution: {integrity: sha512-baeV7t4jQfQtFxBADFmnhmqBmqR38dNU5cvEgHcMK/Kp3D3bEI0CouoX2Sr/rGuntR+Eg0IjXdxnGGTc6SbIkw==}
engines: {node: '>=18.0.0'}
'@smithy/node-config-provider@3.1.12':
resolution: {integrity: sha512-O9LVEu5J/u/FuNlZs+L7Ikn3lz7VB9hb0GtPT9MQeiBmtK8RSY3ULmsZgXhe6VAlgTw0YO+paQx4p8xdbs43vQ==}
engines: {node: '>=16.0.0'}
'@smithy/node-config-provider@4.1.2':
resolution: {integrity: sha512-SUvNup8iU1v7fmM8XPk+27m36udmGCfSz+VZP5Gb0aJ3Ne0X28K/25gnsrg3X1rWlhcnhzNUUysKW/Ied46ivQ==}
engines: {node: '>=18.0.0'}
'@smithy/node-http-handler@3.3.3':
resolution: {integrity: sha512-BrpZOaZ4RCbcJ2igiSNG16S+kgAc65l/2hmxWdmhyoGWHTLlzQzr06PXavJp9OBlPEG/sHlqdxjWmjzV66+BSQ==}
engines: {node: '>=16.0.0'}
'@smithy/node-http-handler@4.0.5':
resolution: {integrity: sha512-T7QglZC1vS7SPT44/1qSIAQEx5bFKb3LfO6zw/o4Xzt1eC5HNoH1TkS4lMYA9cWFbacUhx4hRl/blLun4EOCkg==}
engines: {node: '>=18.0.0'}
'@smithy/property-provider@3.1.11':
resolution: {integrity: sha512-I/+TMc4XTQ3QAjXfOcUWbSS073oOEAxgx4aZy8jHaf8JQnRkq2SZWw8+PfDtBvLUjcGMdxl+YwtzWe6i5uhL/A==}
engines: {node: '>=16.0.0'}
'@smithy/property-provider@4.0.3':
resolution: {integrity: sha512-Wcn17QNdawJZcZZPBuMuzyBENVi1AXl4TdE0jvzo4vWX2x5df/oMlmr/9M5XAAC6+yae4kWZlOYIsNsgDrMU9A==}
engines: {node: '>=18.0.0'}
'@smithy/protocol-http@4.1.8':
resolution: {integrity: sha512-hmgIAVyxw1LySOwkgMIUN0kjN8TG9Nc85LJeEmEE/cNEe2rkHDUWhnJf2gxcSRFLWsyqWsrZGw40ROjUogg+Iw==}
engines: {node: '>=16.0.0'}
'@smithy/protocol-http@5.1.1':
resolution: {integrity: sha512-Vsay2mzq05DwNi9jK01yCFtfvu9HimmgC7a4HTs7lhX12Sx8aWsH0mfz6q/02yspSp+lOB+Q2HJwi4IV2GKz7A==}
engines: {node: '>=18.0.0'}
'@smithy/querystring-builder@3.0.11':
resolution: {integrity: sha512-u+5HV/9uJaeLj5XTb6+IEF/dokWWkEqJ0XiaRRogyREmKGUgZnNecLucADLdauWFKUNbQfulHFEZEdjwEBjXRg==}
engines: {node: '>=16.0.0'}
'@smithy/querystring-builder@4.0.3':
resolution: {integrity: sha512-UUzIWMVfPmDZcOutk2/r1vURZqavvQW0OHvgsyNV0cKupChvqg+/NKPRMaMEe+i8tP96IthMFeZOZWpV+E4RAw==}
engines: {node: '>=18.0.0'}
'@smithy/querystring-parser@3.0.11':
resolution: {integrity: sha512-Je3kFvCsFMnso1ilPwA7GtlbPaTixa3WwC+K21kmMZHsBEOZYQaqxcMqeFFoU7/slFjKDIpiiPydvdJm8Q/MCw==}
engines: {node: '>=16.0.0'}
'@smithy/querystring-parser@4.0.3':
resolution: {integrity: sha512-K5M4ZJQpFCblOJ5Oyw7diICpFg1qhhR47m2/5Ef1PhGE19RaIZf50tjYFrxa6usqcuXyTiFPGo4d1geZdH4YcQ==}
engines: {node: '>=18.0.0'}
'@smithy/service-error-classification@3.0.11':
resolution: {integrity: sha512-QnYDPkyewrJzCyaeI2Rmp7pDwbUETe+hU8ADkXmgNusO1bgHBH7ovXJiYmba8t0fNfJx75fE8dlM6SEmZxheog==}
engines: {node: '>=16.0.0'}
@ -3360,10 +3311,6 @@ packages:
resolution: {integrity: sha512-1xKSGI+U9KKdbG2qDvIR9dGrw3CNx+baqJfyr0igKEpjbHL5stsqAesYBzHChYHlelWtb87VnLWlhvfCz13H8Q==}
engines: {node: '>=16.0.0'}
'@smithy/shared-ini-file-loader@4.0.3':
resolution: {integrity: sha512-vHwlrqhZGIoLwaH8vvIjpHnloShqdJ7SUPNM2EQtEox+yEDFTVQ7E+DLZ+6OhnYEgFUwPByJyz6UZaOu2tny6A==}
engines: {node: '>=18.0.0'}
'@smithy/signature-v4@4.2.4':
resolution: {integrity: sha512-5JWeMQYg81TgU4cG+OexAWdvDTs5JDdbEZx+Qr1iPbvo91QFGzjy0IkXAKaXUHqmKUJgSHK0ZxnCkgZpzkeNTA==}
engines: {node: '>=16.0.0'}
@ -3372,10 +3319,6 @@ packages:
resolution: {integrity: sha512-9wYrjAZFlqWhgVo3C4y/9kpc68jgiSsKUnsFPzr/MSiRL93+QRDafGTfhhKAb2wsr69Ru87WTiqSfQusSmWipA==}
engines: {node: '>=16.0.0'}
'@smithy/smithy-client@4.3.0':
resolution: {integrity: sha512-DNsRA38pN6tYHUjebmwD9e4KcgqTLldYQb2gC6K+oxXYdCTxPn6wV9+FvOa6wrU2FQEnGJoi+3GULzOTKck/tg==}
engines: {node: '>=18.0.0'}
'@smithy/types@3.7.2':
resolution: {integrity: sha512-bNwBYYmN8Eh9RyjS1p2gW6MIhSO2rl7X9QeLM8iTdcGRP+eDiIWDt66c9IysCc22gefKszZv+ubV9qZc7hdESg==}
engines: {node: '>=16.0.0'}
@ -3387,25 +3330,13 @@ packages:
'@smithy/url-parser@3.0.11':
resolution: {integrity: sha512-TmlqXkSk8ZPhfc+SQutjmFr5FjC0av3GZP4B/10caK1SbRwe/v+Wzu/R6xEKxoNqL+8nY18s1byiy6HqPG37Aw==}
'@smithy/url-parser@4.0.3':
resolution: {integrity: sha512-n5/DnosDu/tweOqUUNtUbu7eRIR4J/Wz9nL7V5kFYQQVb8VYdj7a4G5NJHCw6o21ul7CvZoJkOpdTnsQDLT0tQ==}
engines: {node: '>=18.0.0'}
'@smithy/util-base64@3.0.0':
resolution: {integrity: sha512-Kxvoh5Qtt0CDsfajiZOCpJxgtPHXOKwmM+Zy4waD43UoEMA+qPxxa98aE/7ZhdnBFZFXMOiBR5xbcaMhLtznQQ==}
engines: {node: '>=16.0.0'}
'@smithy/util-base64@4.0.0':
resolution: {integrity: sha512-CvHfCmO2mchox9kjrtzoHkWHxjHZzaFojLc8quxXY7WAAMAg43nuxwv95tATVgQFNDwd4M9S1qFzj40Ul41Kmg==}
engines: {node: '>=18.0.0'}
'@smithy/util-body-length-browser@3.0.0':
resolution: {integrity: sha512-cbjJs2A1mLYmqmyVl80uoLTJhAcfzMOyPgjwAYusWKMdLeNtzmMz9YxNl3/jRLoxSS3wkqkf0jwNdtXWtyEBaQ==}
'@smithy/util-body-length-browser@4.0.0':
resolution: {integrity: sha512-sNi3DL0/k64/LO3A256M+m3CDdG6V7WKWHdAiBBMUN8S3hK3aMPhwnPik2A/a2ONN+9doY9UxaLfgqsIRg69QA==}
engines: {node: '>=18.0.0'}
'@smithy/util-body-length-node@3.0.0':
resolution: {integrity: sha512-Tj7pZ4bUloNUP6PzwhN7K386tmSmEET9QtQg0TgdNOnxhZvCssHji+oZTUIuzxECRfG8rdm2PMw2WCFs6eIYkA==}
engines: {node: '>=16.0.0'}
@ -3418,10 +3349,6 @@ packages:
resolution: {integrity: sha512-aEOHCgq5RWFbP+UDPvPot26EJHjOC+bRgse5A8V3FSShqd5E5UN4qc7zkwsvJPPAVsf73QwYcHN1/gt/rtLwQA==}
engines: {node: '>=16.0.0'}
'@smithy/util-buffer-from@4.0.0':
resolution: {integrity: sha512-9TOQ7781sZvddgO8nxueKi3+yGvkY35kotA0Y6BWRajAv8jjmigQ1sBwz0UX47pQMYXJPahSKEKYFgt+rXdcug==}
engines: {node: '>=18.0.0'}
'@smithy/util-config-provider@3.0.0':
resolution: {integrity: sha512-pbjk4s0fwq3Di/ANL+rCvJMKM5bzAQdE5S/6RL5NXgMExFAi6UgQMPOm5yPaIWPpr+EOXKXRonJ3FoxKf4mCJQ==}
engines: {node: '>=16.0.0'}
@ -3442,18 +3369,10 @@ packages:
resolution: {integrity: sha512-eFndh1WEK5YMUYvy3lPlVmYY/fZcQE1D8oSf41Id2vCeIkKJXPcYDCZD+4+xViI6b1XSd7tE+s5AmXzz5ilabQ==}
engines: {node: '>=16.0.0'}
'@smithy/util-hex-encoding@4.0.0':
resolution: {integrity: sha512-Yk5mLhHtfIgW2W2WQZWSg5kuMZCVbvhFmC7rV4IO2QqnZdbEFPmQnCcGMAX2z/8Qj3B9hYYNjZOhWym+RwhePw==}
engines: {node: '>=18.0.0'}
'@smithy/util-middleware@3.0.11':
resolution: {integrity: sha512-dWpyc1e1R6VoXrwLoLDd57U1z6CwNSdkM69Ie4+6uYh2GC7Vg51Qtan7ITzczuVpqezdDTKJGJB95fFvvjU/ow==}
engines: {node: '>=16.0.0'}
'@smithy/util-middleware@4.0.3':
resolution: {integrity: sha512-iIsC6qZXxkD7V3BzTw3b1uK8RVC1M8WvwNxK1PKrH9FnxntCd30CSunXjL/8iJBE8Z0J14r2P69njwIpRG4FBQ==}
engines: {node: '>=18.0.0'}
'@smithy/util-retry@3.0.11':
resolution: {integrity: sha512-hJUC6W7A3DQgaee3Hp9ZFcOxVDZzmBIRBPlUAk8/fSOEl7pE/aX7Dci0JycNOnm9Mfr0KV2XjIlUOcGWXQUdVQ==}
engines: {node: '>=16.0.0'}
@ -3462,18 +3381,10 @@ packages:
resolution: {integrity: sha512-SGhGBG/KupieJvJSZp/rfHHka8BFgj56eek9px4pp7lZbOF+fRiVr4U7A3y3zJD8uGhxq32C5D96HxsTC9BckQ==}
engines: {node: '>=16.0.0'}
'@smithy/util-stream@4.2.1':
resolution: {integrity: sha512-W3IR0x5DY6iVtjj5p902oNhD+Bz7vs5S+p6tppbPa509rV9BdeXZjGuRSCtVEad9FA0Mba+tNUtUmtnSI1nwUw==}
engines: {node: '>=18.0.0'}
'@smithy/util-uri-escape@3.0.0':
resolution: {integrity: sha512-LqR7qYLgZTD7nWLBecUi4aqolw8Mhza9ArpNEQ881MJJIU2sE5iHCK6TdyqqzcDLy0OPe10IY4T8ctVdtynubg==}
engines: {node: '>=16.0.0'}
'@smithy/util-uri-escape@4.0.0':
resolution: {integrity: sha512-77yfbCbQMtgtTylO9itEAdpPXSog3ZxMe09AEhm0dU0NLTalV70ghDZFR+Nfi1C60jnJoh/Re4090/DuZh2Omg==}
engines: {node: '>=18.0.0'}
'@smithy/util-utf8@2.3.0':
resolution: {integrity: sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==}
engines: {node: '>=14.0.0'}
@ -3482,10 +3393,6 @@ packages:
resolution: {integrity: sha512-rUeT12bxFnplYDe815GXbq/oixEGHfRFFtcTF3YdDi/JaENIM6aSYYLJydG83UNzLXeRI5K8abYd/8Sp/QM0kA==}
engines: {node: '>=16.0.0'}
'@smithy/util-utf8@4.0.0':
resolution: {integrity: sha512-b+zebfKCfRdgNJDknHCob3O7FpeYQN6ZG6YLExMcasDHsCXlsXCEuiPZeLnJLpwa5dvPetGlnGCiMHuLwGvFow==}
engines: {node: '>=18.0.0'}
'@smithy/util-waiter@3.2.0':
resolution: {integrity: sha512-PpjSboaDUE6yl+1qlg3Si57++e84oXdWGbuFUSAciXsVfEZJJJupR2Nb0QuXHiunt2vGR+1PTizOMvnUPaG2Qg==}
engines: {node: '>=16.0.0'}
@ -4694,6 +4601,9 @@ packages:
bluebird@3.7.2:
resolution: {integrity: sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==}
boolbase@1.0.0:
resolution: {integrity: sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==}
bowser@2.11.0:
resolution: {integrity: sha512-AlcaJBi/pqqJBIQ8U9Mcpc9i8Aqxn88Skv5d+xBX006BY5u8N3mGLHa5Lgppa7L/HfwgwLgZ6NYs+Ag6uUmJRA==}
@ -4804,6 +4714,13 @@ packages:
resolution: {integrity: sha512-drVkSqfwA+TvuEhFipiR1OC9boEGZL5RrWvVsOthdcvQNXyCCuKkEiTOTXZ7qxSf/GLwq4GvzfrQD/Wz325hgw==}
engines: {node: '>=16'}
cheerio-select@2.1.0:
resolution: {integrity: sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==}
cheerio@1.0.0:
resolution: {integrity: sha512-quS9HgjQpdaXOvsZz82Oz7uxtXiy6UIsIQcpBj7HRw2M63Skasm9qlDocAM7jNuaxdhpPU7c4kJN+gA5MCu4ww==}
engines: {node: '>=18.17'}
chevrotain-allstar@0.3.1:
resolution: {integrity: sha512-b7g+y9A0v4mxCW1qUhf3BSVPg+/NvGErk/dOkrDaHA0nQIQGAtrOjlX//9OQtRlSCy+x9rfB5N8yC71lH1nvMw==}
peerDependencies:
@ -5051,6 +4968,9 @@ packages:
resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==}
engines: {node: '>= 8'}
css-select@5.1.0:
resolution: {integrity: sha512-nwoRF1rvRRnnCqqY7updORDsuqKzqYJ28+oSMaJMMgOauh3fvwHqMS7EZpIPqK8GL+g9mKxF1vP/ZjSeNjEVHg==}
css-what@6.1.0:
resolution: {integrity: sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==}
engines: {node: '>= 6'}
@ -5445,6 +5365,9 @@ packages:
emoji-regex@9.2.2:
resolution: {integrity: sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==}
encoding-sniffer@0.2.0:
resolution: {integrity: sha512-ju7Wq1kg04I3HtiYIOrUrdfdDvkyO9s5XM8QAj/bN61Yo/Vb4vgJxy5vi4Yxk01gWHbrofpPtpxM8bKger9jhg==}
end-of-stream@1.4.4:
resolution: {integrity: sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==}
@ -5949,8 +5872,8 @@ packages:
hachure-fill@0.5.2:
resolution: {integrity: sha512-3GKBOn+m2LX9iq+JC1064cSFprJY4jL1jCXTcpnfER5HYE2l/4EfWSGzkPa/ZDBmYI0ZOEj5VHV/eKnPGkHuOg==}
happy-dom@15.11.6:
resolution: {integrity: sha512-elX7iUTu+5+3b2+NGQc0L3eWyq9jKhuJJ4GpOMxxT/c2pg9O3L5H3ty2VECX0XXZgRmmRqXyOK8brA2hDI6LsQ==}
happy-dom@15.11.7:
resolution: {integrity: sha512-KyrFvnl+J9US63TEzwoiJOQzZBJY7KgBushJA8X61DMbNsH+2ONkDuLDnCnwUiPTF42tLoEmrPyoqbenVA5zrg==}
engines: {node: '>=18.0.0'}
has-bigints@1.0.2:
@ -6028,6 +5951,9 @@ packages:
htmlparser2@8.0.2:
resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==}
htmlparser2@9.1.0:
resolution: {integrity: sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ==}
http-errors@2.0.0:
resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==}
engines: {node: '>= 0.8'}
@ -7101,6 +7027,9 @@ packages:
resolution: {integrity: sha512-AqZtDUWOMKs1G/8lwylVjrdYgqA4d9nu8hc+0gzRxlDb1I10+FHBGMXs6aiQHFdCUUlqH99MUMuLfzWDNDtfxw==}
deprecated: This package is no longer supported.
nth-check@2.1.1:
resolution: {integrity: sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==}
nwsapi@2.2.16:
resolution: {integrity: sha512-F1I/bimDpj3ncaNDhfyMWuFqmQDBwDB0Fogc2qpL3BWvkQteFD/8BzWuIRl83rq0DXfm8SGt/HFhLXZyljTXcQ==}
@ -7247,6 +7176,12 @@ packages:
resolution: {integrity: sha512-3YHlOa/JgH6Mnpr05jP9eDG254US9ek25LyIxZlDItp2iJtwyaXQb57lBYLdT3MowkUFYEV2XXNAYIPlESvJlA==}
engines: {node: '>= 0.10'}
parse5-htmlparser2-tree-adapter@7.1.0:
resolution: {integrity: sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==}
parse5-parser-stream@7.1.2:
resolution: {integrity: sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==}
parse5@7.1.2:
resolution: {integrity: sha512-Czj1WaSVpaoj0wbhMzLmWD69anp2WH7FXMB9n1Sy8/ZFF9jolSQVMu1Ij5WIyGmcBmhk7EOndpO4mIpihVqAXw==}
@ -8619,6 +8554,10 @@ packages:
undici-types@6.20.0:
resolution: {integrity: sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==}
undici@6.21.3:
resolution: {integrity: sha512-gBLkYIlEnSp8pFbT64yFgGE6UIB9tAkhukC23PmMDCe5Nd+cRqKxSjw5y54MK2AZMgZfJWMaNE4nYUHgi1XEOw==}
engines: {node: '>=18.17'}
unicode-canonical-property-names-ecmascript@2.0.0:
resolution: {integrity: sha512-yY5PpDlfVIU5+y/BSCxAJRBIS1Zc2dDG3Ujq+sR0U+JjUevW2JhocOF+soROYDSaAezOzOKuyyixhD6mBknSmQ==}
engines: {node: '>=4'}
@ -9470,12 +9409,12 @@ snapshots:
'@smithy/types': 3.7.2
tslib: 2.8.1
'@aws-sdk/lib-storage@3.817.0(@aws-sdk/client-s3@3.701.0)':
'@aws-sdk/lib-storage@3.701.0(@aws-sdk/client-s3@3.701.0)':
dependencies:
'@aws-sdk/client-s3': 3.701.0
'@smithy/abort-controller': 4.0.3
'@smithy/middleware-endpoint': 4.1.7
'@smithy/smithy-client': 4.3.0
'@smithy/abort-controller': 3.1.9
'@smithy/middleware-endpoint': 3.2.8
'@smithy/smithy-client': 3.7.0
buffer: 5.6.0
events: 3.3.0
stream-browserify: 3.0.0
@ -9711,7 +9650,7 @@ snapshots:
'@babel/traverse': 7.25.9
'@babel/types': 7.24.6
convert-source-map: 2.0.0
debug: 4.3.7
debug: 4.4.0
gensync: 1.0.0-beta.2
json5: 2.2.3
semver: 6.3.1
@ -9837,7 +9776,7 @@ snapshots:
'@babel/core': 7.26.0
'@babel/helper-compilation-targets': 7.25.9
'@babel/helper-plugin-utils': 7.25.9
debug: 4.3.7
debug: 4.4.0
lodash.debounce: 4.0.8
resolve: 1.22.8
transitivePeerDependencies:
@ -10762,7 +10701,7 @@ snapshots:
'@babel/parser': 7.26.2
'@babel/template': 7.25.9
'@babel/types': 7.26.0
debug: 4.3.7
debug: 4.4.0
globals: 11.12.0
transitivePeerDependencies:
- supports-color
@ -10774,7 +10713,7 @@ snapshots:
'@babel/parser': 7.27.0
'@babel/template': 7.27.0
'@babel/types': 7.27.0
debug: 4.3.7
debug: 4.4.0
globals: 11.12.0
transitivePeerDependencies:
- supports-color
@ -12308,11 +12247,6 @@ snapshots:
'@smithy/types': 3.7.2
tslib: 2.8.1
'@smithy/abort-controller@4.0.3':
dependencies:
'@smithy/types': 4.3.0
tslib: 2.8.1
'@smithy/chunked-blob-reader-native@3.0.1':
dependencies:
'@smithy/util-base64': 3.0.0
@ -12341,17 +12275,6 @@ snapshots:
'@smithy/util-utf8': 3.0.0
tslib: 2.8.1
'@smithy/core@3.4.0':
dependencies:
'@smithy/middleware-serde': 4.0.6
'@smithy/protocol-http': 5.1.1
'@smithy/types': 4.3.0
'@smithy/util-body-length-browser': 4.0.0
'@smithy/util-middleware': 4.0.3
'@smithy/util-stream': 4.2.1
'@smithy/util-utf8': 4.0.0
tslib: 2.8.1
'@smithy/credential-provider-imds@3.2.8':
dependencies:
'@smithy/node-config-provider': 3.1.12
@ -12398,14 +12321,6 @@ snapshots:
'@smithy/util-base64': 3.0.0
tslib: 2.8.1
'@smithy/fetch-http-handler@5.0.3':
dependencies:
'@smithy/protocol-http': 5.1.1
'@smithy/querystring-builder': 4.0.3
'@smithy/types': 4.3.0
'@smithy/util-base64': 4.0.0
tslib: 2.8.1
'@smithy/hash-blob-browser@3.1.10':
dependencies:
'@smithy/chunked-blob-reader': 4.0.0
@ -12439,10 +12354,6 @@ snapshots:
dependencies:
tslib: 2.8.1
'@smithy/is-array-buffer@4.0.0':
dependencies:
tslib: 2.8.1
'@smithy/md5-js@3.0.11':
dependencies:
'@smithy/types': 3.7.2
@ -12466,17 +12377,6 @@ snapshots:
'@smithy/util-middleware': 3.0.11
tslib: 2.8.1
'@smithy/middleware-endpoint@4.1.7':
dependencies:
'@smithy/core': 3.4.0
'@smithy/middleware-serde': 4.0.6
'@smithy/node-config-provider': 4.1.2
'@smithy/shared-ini-file-loader': 4.0.3
'@smithy/types': 4.3.0
'@smithy/url-parser': 4.0.3
'@smithy/util-middleware': 4.0.3
tslib: 2.8.1
'@smithy/middleware-retry@3.0.34':
dependencies:
'@smithy/node-config-provider': 3.1.12
@ -12494,22 +12394,11 @@ snapshots:
'@smithy/types': 3.7.2
tslib: 2.8.1
'@smithy/middleware-serde@4.0.6':
dependencies:
'@smithy/protocol-http': 5.1.1
'@smithy/types': 4.3.0
tslib: 2.8.1
'@smithy/middleware-stack@3.0.11':
dependencies:
'@smithy/types': 3.7.2
tslib: 2.8.1
'@smithy/middleware-stack@4.0.3':
dependencies:
'@smithy/types': 4.3.0
tslib: 2.8.1
'@smithy/node-config-provider@3.1.12':
dependencies:
'@smithy/property-provider': 3.1.11
@ -12517,13 +12406,6 @@ snapshots:
'@smithy/types': 3.7.2
tslib: 2.8.1
'@smithy/node-config-provider@4.1.2':
dependencies:
'@smithy/property-provider': 4.0.3
'@smithy/shared-ini-file-loader': 4.0.3
'@smithy/types': 4.3.0
tslib: 2.8.1
'@smithy/node-http-handler@3.3.3':
dependencies:
'@smithy/abort-controller': 3.1.9
@ -12532,56 +12414,27 @@ snapshots:
'@smithy/types': 3.7.2
tslib: 2.8.1
'@smithy/node-http-handler@4.0.5':
dependencies:
'@smithy/abort-controller': 4.0.3
'@smithy/protocol-http': 5.1.1
'@smithy/querystring-builder': 4.0.3
'@smithy/types': 4.3.0
tslib: 2.8.1
'@smithy/property-provider@3.1.11':
dependencies:
'@smithy/types': 3.7.2
tslib: 2.8.1
'@smithy/property-provider@4.0.3':
dependencies:
'@smithy/types': 4.3.0
tslib: 2.8.1
'@smithy/protocol-http@4.1.8':
dependencies:
'@smithy/types': 3.7.2
tslib: 2.8.1
'@smithy/protocol-http@5.1.1':
dependencies:
'@smithy/types': 4.3.0
tslib: 2.8.1
'@smithy/querystring-builder@3.0.11':
dependencies:
'@smithy/types': 3.7.2
'@smithy/util-uri-escape': 3.0.0
tslib: 2.8.1
'@smithy/querystring-builder@4.0.3':
dependencies:
'@smithy/types': 4.3.0
'@smithy/util-uri-escape': 4.0.0
tslib: 2.8.1
'@smithy/querystring-parser@3.0.11':
dependencies:
'@smithy/types': 3.7.2
tslib: 2.8.1
'@smithy/querystring-parser@4.0.3':
dependencies:
'@smithy/types': 4.3.0
tslib: 2.8.1
'@smithy/service-error-classification@3.0.11':
dependencies:
'@smithy/types': 3.7.2
@ -12591,11 +12444,6 @@ snapshots:
'@smithy/types': 3.7.2
tslib: 2.8.1
'@smithy/shared-ini-file-loader@4.0.3':
dependencies:
'@smithy/types': 4.3.0
tslib: 2.8.1
'@smithy/signature-v4@4.2.4':
dependencies:
'@smithy/is-array-buffer': 3.0.0
@ -12617,16 +12465,6 @@ snapshots:
'@smithy/util-stream': 3.3.4
tslib: 2.8.1
'@smithy/smithy-client@4.3.0':
dependencies:
'@smithy/core': 3.4.0
'@smithy/middleware-endpoint': 4.1.7
'@smithy/middleware-stack': 4.0.3
'@smithy/protocol-http': 5.1.1
'@smithy/types': 4.3.0
'@smithy/util-stream': 4.2.1
tslib: 2.8.1
'@smithy/types@3.7.2':
dependencies:
tslib: 2.8.1
@ -12641,32 +12479,16 @@ snapshots:
'@smithy/types': 3.7.2
tslib: 2.8.1
'@smithy/url-parser@4.0.3':
dependencies:
'@smithy/querystring-parser': 4.0.3
'@smithy/types': 4.3.0
tslib: 2.8.1
'@smithy/util-base64@3.0.0':
dependencies:
'@smithy/util-buffer-from': 3.0.0
'@smithy/util-utf8': 3.0.0
tslib: 2.8.1
'@smithy/util-base64@4.0.0':
dependencies:
'@smithy/util-buffer-from': 4.0.0
'@smithy/util-utf8': 4.0.0
tslib: 2.8.1
'@smithy/util-body-length-browser@3.0.0':
dependencies:
tslib: 2.8.1
'@smithy/util-body-length-browser@4.0.0':
dependencies:
tslib: 2.8.1
'@smithy/util-body-length-node@3.0.0':
dependencies:
tslib: 2.8.1
@ -12681,11 +12503,6 @@ snapshots:
'@smithy/is-array-buffer': 3.0.0
tslib: 2.8.1
'@smithy/util-buffer-from@4.0.0':
dependencies:
'@smithy/is-array-buffer': 4.0.0
tslib: 2.8.1
'@smithy/util-config-provider@3.0.0':
dependencies:
tslib: 2.8.1
@ -12718,20 +12535,11 @@ snapshots:
dependencies:
tslib: 2.8.1
'@smithy/util-hex-encoding@4.0.0':
dependencies:
tslib: 2.8.1
'@smithy/util-middleware@3.0.11':
dependencies:
'@smithy/types': 3.7.2
tslib: 2.8.1
'@smithy/util-middleware@4.0.3':
dependencies:
'@smithy/types': 4.3.0
tslib: 2.8.1
'@smithy/util-retry@3.0.11':
dependencies:
'@smithy/service-error-classification': 3.0.11
@ -12749,25 +12557,10 @@ snapshots:
'@smithy/util-utf8': 3.0.0
tslib: 2.8.1
'@smithy/util-stream@4.2.1':
dependencies:
'@smithy/fetch-http-handler': 5.0.3
'@smithy/node-http-handler': 4.0.5
'@smithy/types': 4.3.0
'@smithy/util-base64': 4.0.0
'@smithy/util-buffer-from': 4.0.0
'@smithy/util-hex-encoding': 4.0.0
'@smithy/util-utf8': 4.0.0
tslib: 2.8.1
'@smithy/util-uri-escape@3.0.0':
dependencies:
tslib: 2.8.1
'@smithy/util-uri-escape@4.0.0':
dependencies:
tslib: 2.8.1
'@smithy/util-utf8@2.3.0':
dependencies:
'@smithy/util-buffer-from': 2.2.0
@ -12778,11 +12571,6 @@ snapshots:
'@smithy/util-buffer-from': 3.0.0
tslib: 2.8.1
'@smithy/util-utf8@4.0.0':
dependencies:
'@smithy/util-buffer-from': 4.0.0
tslib: 2.8.1
'@smithy/util-waiter@3.2.0':
dependencies:
'@smithy/abort-controller': 3.1.9
@ -13637,7 +13425,7 @@ snapshots:
dependencies:
'@typescript-eslint/typescript-estree': 8.17.0(typescript@5.7.2)
'@typescript-eslint/utils': 8.17.0(eslint@9.15.0(jiti@1.21.0))(typescript@5.7.2)
debug: 4.3.7
debug: 4.4.0
eslint: 9.15.0(jiti@1.21.0)
ts-api-utils: 1.3.0(typescript@5.7.2)
optionalDependencies:
@ -13649,7 +13437,7 @@ snapshots:
dependencies:
'@typescript-eslint/typescript-estree': 8.24.1(typescript@5.7.3)
'@typescript-eslint/utils': 8.24.1(eslint@9.20.1(jiti@1.21.0))(typescript@5.7.3)
debug: 4.3.7
debug: 4.4.0
eslint: 9.20.1(jiti@1.21.0)
ts-api-utils: 2.0.1(typescript@5.7.3)
typescript: 5.7.3
@ -13664,7 +13452,7 @@ snapshots:
dependencies:
'@typescript-eslint/types': 8.17.0
'@typescript-eslint/visitor-keys': 8.17.0
debug: 4.3.7
debug: 4.4.0
fast-glob: 3.3.2
is-glob: 4.0.3
minimatch: 9.0.4
@ -13873,13 +13661,13 @@ snapshots:
agent-base@6.0.2:
dependencies:
debug: 4.3.7
debug: 4.4.0
transitivePeerDependencies:
- supports-color
agent-base@7.1.1:
dependencies:
debug: 4.3.7
debug: 4.4.0
transitivePeerDependencies:
- supports-color
@ -14216,6 +14004,8 @@ snapshots:
bluebird@3.7.2: {}
boolbase@1.0.0: {}
bowser@2.11.0: {}
boxen@5.1.2:
@ -14343,6 +14133,29 @@ snapshots:
check-disk-space@3.4.0: {}
cheerio-select@2.1.0:
dependencies:
boolbase: 1.0.0
css-select: 5.1.0
css-what: 6.1.0
domelementtype: 2.3.0
domhandler: 5.0.3
domutils: 3.1.0
cheerio@1.0.0:
dependencies:
cheerio-select: 2.1.0
dom-serializer: 2.0.0
domhandler: 5.0.3
domutils: 3.1.0
encoding-sniffer: 0.2.0
htmlparser2: 9.1.0
parse5: 7.1.2
parse5-htmlparser2-tree-adapter: 7.1.0
parse5-parser-stream: 7.1.2
undici: 6.21.3
whatwg-mimetype: 4.0.0
chevrotain-allstar@0.3.1(chevrotain@11.0.3):
dependencies:
chevrotain: 11.0.3
@ -14596,6 +14409,14 @@ snapshots:
shebang-command: 2.0.0
which: 2.0.2
css-select@5.1.0:
dependencies:
boolbase: 1.0.0
css-what: 6.1.0
domhandler: 5.0.3
domutils: 3.1.0
nth-check: 2.1.1
css-what@6.1.0: {}
cssesc@3.0.0: {}
@ -14984,6 +14805,11 @@ snapshots:
emoji-regex@9.2.2: {}
encoding-sniffer@0.2.0:
dependencies:
iconv-lite: 0.6.3
whatwg-encoding: 3.1.1
end-of-stream@1.4.4:
dependencies:
once: 1.4.0
@ -15715,7 +15541,7 @@ snapshots:
hachure-fill@0.5.2: {}
happy-dom@15.11.6:
happy-dom@15.11.7:
dependencies:
entities: 4.5.0
webidl-conversions: 7.0.0
@ -15788,6 +15614,13 @@ snapshots:
domutils: 3.1.0
entities: 4.5.0
htmlparser2@9.1.0:
dependencies:
domelementtype: 2.3.0
domhandler: 5.0.3
domutils: 3.1.0
entities: 4.5.0
http-errors@2.0.0:
dependencies:
depd: 2.0.0
@ -15799,21 +15632,21 @@ snapshots:
http-proxy-agent@7.0.2:
dependencies:
agent-base: 7.1.1
debug: 4.3.7
debug: 4.4.0
transitivePeerDependencies:
- supports-color
https-proxy-agent@5.0.1:
dependencies:
agent-base: 6.0.2
debug: 4.3.7
debug: 4.4.0
transitivePeerDependencies:
- supports-color
https-proxy-agent@7.0.5:
dependencies:
agent-base: 7.1.1
debug: 4.3.7
debug: 4.4.0
transitivePeerDependencies:
- supports-color
@ -16076,7 +15909,7 @@ snapshots:
istanbul-lib-source-maps@4.0.1:
dependencies:
debug: 4.3.7
debug: 4.4.0
istanbul-lib-coverage: 3.2.2
source-map: 0.6.1
transitivePeerDependencies:
@ -17041,6 +16874,10 @@ snapshots:
gauge: 3.0.2
set-blocking: 2.0.0
nth-check@2.1.1:
dependencies:
boolbase: 1.0.0
nwsapi@2.2.16: {}
nx@20.4.5(@swc/core@1.5.25(@swc/helpers@0.5.5)):
@ -17237,6 +17074,15 @@ snapshots:
parse-node-version@1.0.1:
optional: true
parse5-htmlparser2-tree-adapter@7.1.0:
dependencies:
domhandler: 5.0.3
parse5: 7.1.2
parse5-parser-stream@7.1.2:
dependencies:
parse5: 7.1.2
parse5@7.1.2:
dependencies:
entities: 4.5.0
@ -18724,6 +18570,8 @@ snapshots:
undici-types@6.20.0: {}
undici@6.21.3: {}
unicode-canonical-property-names-ecmascript@2.0.0: {}
unicode-match-property-ecmascript@2.0.0: