feat: space export (#506)

* wip

* Space export
* option to export pages with children
* include attachments in exports
* unified export UI

* cleanup

* fix: change export icon

* add export button to space settings

* cleanups

* export name

apps/server/src/collaboration/collaboration.util.ts

@@ -1,15 +1,15 @@
-import {StarterKit} from '@tiptap/starter-kit';
-import {TextAlign} from '@tiptap/extension-text-align';
-import {TaskList} from '@tiptap/extension-task-list';
-import {TaskItem} from '@tiptap/extension-task-item';
-import {Underline} from '@tiptap/extension-underline';
-import {Superscript} from '@tiptap/extension-superscript';
+import { StarterKit } from '@tiptap/starter-kit';
+import { TextAlign } from '@tiptap/extension-text-align';
+import { TaskList } from '@tiptap/extension-task-list';
+import { TaskItem } from '@tiptap/extension-task-item';
+import { Underline } from '@tiptap/extension-underline';
+import { Superscript } from '@tiptap/extension-superscript';
 import SubScript from '@tiptap/extension-subscript';
-import {Highlight} from '@tiptap/extension-highlight';
-import {Typography} from '@tiptap/extension-typography';
-import {TextStyle} from '@tiptap/extension-text-style';
-import {Color} from '@tiptap/extension-color';
-import {Youtube} from '@tiptap/extension-youtube';
+import { Highlight } from '@tiptap/extension-highlight';
+import { Typography } from '@tiptap/extension-typography';
+import { TextStyle } from '@tiptap/extension-text-style';
+import { Color } from '@tiptap/extension-color';
+import { Youtube } from '@tiptap/extension-youtube';
 import Table from '@tiptap/extension-table';
 import TableHeader from '@tiptap/extension-table-header';
 import {
@@ -30,14 +30,15 @@ import {
   Attachment,
   Drawio,
   Excalidraw,
-  Embed
+  Embed,
 } from '@docmost/editor-ext';
-import {generateText, JSONContent} from '@tiptap/core';
-import {generateHTML} from '../common/helpers/prosemirror/html';
+import { generateText, getSchema, JSONContent } from '@tiptap/core';
+import { generateHTML } from '../common/helpers/prosemirror/html';
 // @tiptap/html library works best for generating prosemirror json state but not HTML
 // see: https://github.com/ueberdosis/tiptap/issues/5352
 // see:https://github.com/ueberdosis/tiptap/issues/4089
-import {generateJSON} from '@tiptap/html';
+import { generateJSON } from '@tiptap/html';
+import { Node } from '@tiptap/pm/model';
 
 export const tiptapExtensions = [
   StarterKit.configure({
@@ -73,7 +74,7 @@ export const tiptapExtensions = [
   CustomCodeBlock,
   Drawio,
   Excalidraw,
-  Embed
+  Embed,
 ] as any;
 
 export function jsonToHtml(tiptapJson: any) {
@@ -88,6 +89,10 @@ export function jsonToText(tiptapJson: JSONContent) {
   return generateText(tiptapJson, tiptapExtensions);
 }
 
+export function jsonToNode(tiptapJson: JSONContent) {
+  return Node.fromJSON(getSchema(tiptapExtensions), tiptapJson);
+}
+
 export function getPageId(documentName: string) {
   return documentName.split('.')[1];
 }

apps/server/src/database/repos/page/page.repo.ts

@@ -160,4 +160,30 @@ export class PageRepo {
         .whereRef('spaces.id', '=', 'pages.spaceId'),
     ).as('space');
   }
+
+  async getPageAndDescendants(parentPageId: string) {
+    return this.db
+      .withRecursive('page_hierarchy', (db) =>
+        db
+          .selectFrom('pages')
+          .select(['id', 'slugId', 'title', 'icon', 'content', 'parentPageId'])
+          .where('id', '=', parentPageId)
+          .unionAll((exp) =>
+            exp
+              .selectFrom('pages as p')
+              .select([
+                'p.id',
+                'p.slugId',
+                'p.title',
+                'p.icon',
+                'p.content',
+                'p.parentPageId',
+              ])
+              .innerJoin('page_hierarchy as ph', 'p.parentPageId', 'ph.id'),
+          ),
+      )
+      .selectFrom('page_hierarchy')
+      .selectAll()
+      .execute();
+  }
 }

apps/server/src/integrations/export/dto/export-dto.ts

@@ -22,5 +22,19 @@ export class ExportPageDto {
 
   @IsOptional()
   @IsBoolean()
-  includeFiles?: boolean;
+  includeChildren?: boolean;
 }
+
+export class ExportSpaceDto {
+  @IsString()
+  @IsNotEmpty()
+  spaceId: string;
+
+  @IsString()
+  @IsIn(['html', 'markdown'])
+  format: ExportFormat;
+
+  @IsOptional()
+  @IsBoolean()
+  includeAttachments?: boolean;
+}

apps/server/src/integrations/export/export.controller.ts

@@ -10,7 +10,7 @@ import {
   UseGuards,
 } from '@nestjs/common';
 import { ExportService } from './export.service';
-import { ExportPageDto } from './dto/export-dto';
+import { ExportPageDto, ExportSpaceDto } from './dto/export-dto';
 import { AuthUser } from '../../common/decorators/auth-user.decorator';
 import { User } from '@docmost/db/types/entity.types';
 import SpaceAbilityFactory from '../../core/casl/abilities/space-ability.factory';
@@ -54,10 +54,28 @@ export class ImportController {
       throw new ForbiddenException();
     }
 
-    const rawContent = await this.exportService.exportPage(dto.format, page);
-
     const fileExt = getExportExtension(dto.format);
-    const fileName = sanitize(page.title || 'Untitled') + fileExt;
+    const fileName = sanitize(page.title || 'untitled') + fileExt;
+
+    if (dto.includeChildren) {
+      const zipFileBuffer = await this.exportService.exportPageWithChildren(
+        dto.pageId,
+        dto.format,
+      );
+
+      const newName = fileName + '.zip';
+
+      res.headers({
+        'Content-Type': 'application/zip',
+        'Content-Disposition':
+          'attachment; filename="' + encodeURIComponent(newName) + '"',
+      });
+
+      res.send(zipFileBuffer);
+      return;
+    }
+
+    const rawContent = await this.exportService.exportPage(dto.format, page);
 
     res.headers({
       'Content-Type': getMimeType(fileExt),
@@ -67,4 +85,34 @@ export class ImportController {
 
     res.send(rawContent);
   }
+
+  @UseGuards(JwtAuthGuard)
+  @HttpCode(HttpStatus.OK)
+  @Post('spaces/export')
+  async exportSpace(
+    @Body() dto: ExportSpaceDto,
+    @AuthUser() user: User,
+    @Res() res: FastifyReply,
+  ) {
+    const ability = await this.spaceAbility.createForUser(user, dto.spaceId);
+    if (ability.cannot(SpaceCaslAction.Manage, SpaceCaslSubject.Page)) {
+      throw new ForbiddenException();
+    }
+
+    const exportFile = await this.exportService.exportSpace(
+      dto.spaceId,
+      dto.format,
+      dto.includeAttachments,
+    );
+
+    res.headers({
+      'Content-Type': 'application/zip',
+      'Content-Disposition':
+        'attachment; filename="' +
+        encodeURIComponent(sanitize(exportFile.fileName)) +
+        '"',
+    });
+
+    res.send(exportFile.fileBuffer);
+  }
 }

apps/server/src/integrations/export/export.module.ts

@@ -1,8 +1,10 @@
 import { Module } from '@nestjs/common';
 import { ExportService } from './export.service';
 import { ImportController } from './export.controller';
+import { StorageModule } from '../storage/storage.module';
 
 @Module({
+  imports: [StorageModule],
   providers: [ExportService],
   controllers: [ImportController],
 })

apps/server/src/integrations/export/export.service.ts

@@ -1,19 +1,48 @@
-import { Injectable } from '@nestjs/common';
+import {
+  BadRequestException,
+  Injectable,
+  Logger,
+  NotFoundException,
+} from '@nestjs/common';
 import { jsonToHtml } from '../../collaboration/collaboration.util';
 import { turndown } from './turndown-utils';
 import { ExportFormat } from './dto/export-dto';
 import { Page } from '@docmost/db/types/entity.types';
+import { InjectKysely } from 'nestjs-kysely';
+import { KyselyDB } from '@docmost/db/types/kysely.types';
+import * as JSZip from 'jszip';
+import { StorageService } from '../storage/storage.service';
+import {
+  buildTree,
+  computeLocalPath,
+  getAttachmentIds,
+  getExportExtension,
+  getPageTitle,
+  getProsemirrorContent,
+  PageExportTree,
+  replaceInternalLinks,
+  updateAttachmentUrls,
+} from './utils';
+import { PageRepo } from '@docmost/db/repos/page/page.repo';
 
 @Injectable()
 export class ExportService {
+  private readonly logger = new Logger(ExportService.name);
+
+  constructor(
+    private readonly pageRepo: PageRepo,
+    @InjectKysely() private readonly db: KyselyDB,
+    private readonly storageService: StorageService,
+  ) {}
+
   async exportPage(format: string, page: Page) {
     const titleNode = {
       type: 'heading',
       attrs: { level: 1 },
-      content: [{ type: 'text', text: page.title }],
+      content: [{ type: 'text', text: getPageTitle(page.title) }],
     };
 
-    let prosemirrorJson: any = page.content || { type: 'doc', content: [] };
+    let prosemirrorJson: any = getProsemirrorContent(page.content);
 
     if (page.title) {
       prosemirrorJson.content.unshift(titleNode);
@@ -22,7 +51,13 @@ export class ExportService {
     const pageHtml = jsonToHtml(prosemirrorJson);
 
     if (format === ExportFormat.HTML) {
-      return `<!DOCTYPE html><html><head><title>${page.title}</title></head><body>${pageHtml}</body></html>`;
+      return `<!DOCTYPE html>
+      <html>
+        <head>
+         <title>${getPageTitle(page.title)}</title>
+        </head>
+        <body>${pageHtml}</body>
+      </html>`;
     }
 
     if (format === ExportFormat.Markdown) {
@@ -31,4 +66,156 @@ export class ExportService {
 
     return;
   }
+
+  async exportPageWithChildren(pageId: string, format: string) {
+    const pages = await this.pageRepo.getPageAndDescendants(pageId);
+
+    if (!pages || pages.length === 0) {
+      throw new BadRequestException('No pages to export');
+    }
+
+    const parentPageIndex = pages.findIndex((obj) => obj.id === pageId);
+    // set to null to make export of pages with parentId work
+    pages[parentPageIndex].parentPageId = null;
+
+    const tree = buildTree(pages as Page[]);
+
+    const zip = new JSZip();
+    await this.zipPages(tree, format, zip);
+
+    const zipFile = zip.generateNodeStream({
+      type: 'nodebuffer',
+      streamFiles: true,
+      compression: 'DEFLATE',
+    });
+
+    return zipFile;
+  }
+
+  async exportSpace(
+    spaceId: string,
+    format: string,
+    includeAttachments: boolean,
+  ) {
+    const space = await this.db
+      .selectFrom('spaces')
+      .selectAll()
+      .where('id', '=', spaceId)
+      .executeTakeFirst();
+
+    if (!space) {
+      throw new NotFoundException('Space not found');
+    }
+
+    const pages = await this.db
+      .selectFrom('pages')
+      .select([
+        'pages.id',
+        'pages.slugId',
+        'pages.title',
+        'pages.content',
+        'pages.parentPageId',
+      ])
+      .where('spaceId', '=', spaceId)
+      .execute();
+
+    const tree = buildTree(pages as Page[]);
+
+    const zip = new JSZip();
+
+    await this.zipPages(tree, format, zip, includeAttachments);
+
+    const zipFile = zip.generateNodeStream({
+      type: 'nodebuffer',
+      streamFiles: true,
+      compression: 'DEFLATE',
+    });
+
+    const fileName = `${space.name}-space-export${getExportExtension(format)}.zip`;
+    return {
+      fileBuffer: zipFile,
+      fileName,
+    };
+  }
+
+  async zipPages(
+    tree: PageExportTree,
+    format: string,
+    zip: JSZip,
+    includeAttachments = true,
+  ): Promise<void> {
+    const slugIdToPath: Record<string, string> = {};
+
+    computeLocalPath(tree, format, null, '', slugIdToPath);
+
+    const stack: { folder: JSZip; parentPageId: string }[] = [
+      { folder: zip, parentPageId: null },
+    ];
+
+    while (stack.length > 0) {
+      const { folder, parentPageId } = stack.pop();
+      const children = tree[parentPageId] || [];
+
+      for (const page of children) {
+        const childPages = tree[page.id] || [];
+
+        const prosemirrorJson = getProsemirrorContent(page.content);
+
+        const currentPagePath = slugIdToPath[page.slugId];
+
+        let updatedJsonContent = replaceInternalLinks(
+          prosemirrorJson,
+          slugIdToPath,
+          currentPagePath,
+        );
+
+        if (includeAttachments) {
+          await this.zipAttachments(updatedJsonContent, page.spaceId, folder);
+          updatedJsonContent = updateAttachmentUrls(updatedJsonContent);
+        }
+
+        const pageTitle = getPageTitle(page.title);
+        const pageExportContent = await this.exportPage(format, {
+          ...page,
+          content: updatedJsonContent,
+        });
+
+        folder.file(
+          `${pageTitle}${getExportExtension(format)}`,
+          pageExportContent,
+        );
+        if (childPages.length > 0) {
+          const pageFolder = folder.folder(pageTitle);
+          stack.push({ folder: pageFolder, parentPageId: page.id });
+        }
+      }
+    }
+  }
+
+  async zipAttachments(prosemirrorJson: any, spaceId: string, zip: JSZip) {
+    const attachmentIds = getAttachmentIds(prosemirrorJson);
+
+    if (attachmentIds.length > 0) {
+      const attachments = await this.db
+        .selectFrom('attachments')
+        .selectAll()
+        .where('id', 'in', attachmentIds)
+        .where('spaceId', '=', spaceId)
+        .execute();
+
+      await Promise.all(
+        attachments.map(async (attachment) => {
+          try {
+            const fileBuffer = await this.storageService.read(
+              attachment.filePath,
+            );
+            const filePath = `/files/${attachment.id}/${attachment.fileName}`;
+            zip.file(filePath, fileBuffer);
+          } catch (err) {
+            this.logger.debug(`Attachment export error ${attachment.id}`, err);
+          }
+        }),
+      );
+    }
+  }
 }

apps/server/src/integrations/export/utils.ts

@@ -1,4 +1,11 @@
+import { jsonToNode } from 'src/collaboration/collaboration.util';
 import { ExportFormat } from './dto/export-dto';
+import { Node } from '@tiptap/pm/model';
+import { validate as isValidUUID } from 'uuid';
+import * as path from 'path';
+import { Page } from '@docmost/db/types/entity.types';
+
+export type PageExportTree = Record<string, Page[]>;
 
 export function getExportExtension(format: string) {
   if (format === ExportFormat.HTML) {
@@ -10,3 +17,171 @@ export function getExportExtension(format: string) {
   }
   return;
 }
+
+export function getPageTitle(title: string) {
+  return title ? title : 'untitled';
+}
+
+export function getProsemirrorContent(content: any) {
+  return (
+    content ?? {
+      type: 'doc',
+      content: [{ type: 'paragraph', attrs: { textAlign: 'left' } }],
+    }
+  );
+}
+
+export function getAttachmentIds(prosemirrorJson: any) {
+  const doc = jsonToNode(prosemirrorJson);
+  const attachmentIds = [];
+
+  doc?.descendants((node: Node) => {
+    if (isAttachmentNode(node.type.name)) {
+      if (node.attrs.attachmentId && isValidUUID(node.attrs.attachmentId)) {
+        if (!attachmentIds.includes(node.attrs.attachmentId)) {
+          attachmentIds.push(node.attrs.attachmentId);
+        }
+      }
+    }
+  });
+
+  return attachmentIds;
+}
+
+export function isAttachmentNode(nodeType: string) {
+  const attachmentNodeTypes = [
+    'attachment',
+    'image',
+    'video',
+    'excalidraw',
+    'drawio',
+  ];
+  return attachmentNodeTypes.includes(nodeType);
+}
+
+export function updateAttachmentUrls(prosemirrorJson: any) {
+  const doc = jsonToNode(prosemirrorJson);
+
+  doc?.descendants((node: Node) => {
+    if (isAttachmentNode(node.type.name)) {
+      if (node.attrs.src && node.attrs.src.startsWith('/files')) {
+        //@ts-expect-error
+        node.attrs.src = node.attrs.src.replace('/files', 'files');
+      } else if (node.attrs.url && node.attrs.url.startsWith('/files')) {
+        //@ts-expect-error
+        node.attrs.url = node.attrs.url.replace('/files', 'files');
+      }
+    }
+  });
+
+  return doc.toJSON();
+}
+
+export function replaceInternalLinks(
+  prosemirrorJson: any,
+  slugIdToPath: Record<string, string>,
+  currentPagePath: string,
+) {
+  const doc = jsonToNode(prosemirrorJson);
+  const internalLinkRegex =
+    /^(https?:\/\/)?([^\/]+)?(\/s\/([^\/]+)\/)?p\/([a-zA-Z0-9-]+)\/?$/;
+
+  doc.descendants((node: Node) => {
+    for (const mark of node.marks) {
+      if (mark.type.name === 'link' && mark.attrs.href) {
+        const match = mark.attrs.href.match(internalLinkRegex);
+        if (match) {
+          const markLink = mark.attrs.href;
+
+          const slugId = extractPageSlugId(match[5]);
+          const localPath = slugIdToPath[slugId];
+
+          if (!localPath) {
+            continue;
+          }
+
+          const relativePath = computeRelativePath(currentPagePath, localPath);
+
+          //@ts-expect-error
+          mark.attrs.href = relativePath;
+          //@ts-expect-error
+          mark.attrs.target = '_self';
+          if (node.isText) {
+            // if link and text are same, use page title
+            if (markLink === node.text) {
+              //@ts-expect-error
+              node.text = getInternalLinkPageName(relativePath);
+            }
+          }
+        }
+      }
+    }
+  });
+
+  return doc.toJSON();
+}
+
+export function getInternalLinkPageName(path: string): string {
+  return decodeURIComponent(
+    path?.split('/').pop().split('.').slice(0, -1).join('.'),
+  );
+}
+
+export function extractPageSlugId(input: string): string {
+  if (!input) {
+    return undefined;
+  }
+  const parts = input.split('-');
+  return parts.length > 1 ? parts[parts.length - 1] : input;
+}
+
+export function buildTree(pages: Page[]): PageExportTree {
+  const tree: PageExportTree = {};
+  const titleCount: Record<string, Record<string, number>> = {};
+
+  for (const page of pages) {
+    const parentPageId = page.parentPageId;
+
+    if (!titleCount[parentPageId]) {
+      titleCount[parentPageId] = {};
+    }
+
+    let title = getPageTitle(page.title);
+
+    if (titleCount[parentPageId][title]) {
+      title = `${title} (${titleCount[parentPageId][title]})`;
+      titleCount[parentPageId][getPageTitle(page.title)] += 1;
+    } else {
+      titleCount[parentPageId][title] = 1;
+    }
+
+    page.title = title;
+    if (!tree[parentPageId]) {
+      tree[parentPageId] = [];
+    }
+    tree[parentPageId].push(page);
+  }
+  return tree;
+}
+
+export function computeLocalPath(
+  tree: PageExportTree,
+  format: string,
+  parentPageId: string | null,
+  currentPath: string,
+  slugIdToPath: Record<string, string>,
+) {
+  const children = tree[parentPageId] || [];
+
+  for (const page of children) {
+    const title = encodeURIComponent(getPageTitle(page.title));
+    const localPath = `${currentPath}${title}`;
+    slugIdToPath[page.slugId] = `${localPath}${getExportExtension(format)}`;
+
+    computeLocalPath(tree, format, page.id, `${localPath}/`, slugIdToPath);
+  }
+}
+
+function computeRelativePath(from: string, to: string) {
+  return path.relative(path.dirname(from), to);
+}