From 47c54174b3e62c51b1e1036a07205d6da0b6010f Mon Sep 17 00:00:00 2001 From: Philipinho <16838612+Philipinho@users.noreply.github.com> Date: Thu, 11 Sep 2025 00:50:15 +0100 Subject: [PATCH 1/7] sync --- apps/server/src/ee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/server/src/ee b/apps/server/src/ee index e71f70c2..d90ce7a2 160000 --- a/apps/server/src/ee +++ b/apps/server/src/ee @@ -1 +1 @@ -Subproject commit e71f70c29950efc02f9fe795b7d4b4c9d1060db4 +Subproject commit d90ce7a20f20575c4afa74b0373a18e9555ed0fe From 7ada3cb1f9abde14d8f952b4ee3076ac4b9d803c Mon Sep 17 00:00:00 2001 From: Philip Okugbe <16838612+Philipinho@users.noreply.github.com> Date: Sat, 13 Sep 2025 03:14:59 +0100 Subject: [PATCH 2/7] fix: page import task (#1551) * fix import * - fix notion importer - support notion page icon import - fix horizontal rule css - rename service file * sync * 3 mins delay --- .../src/features/editor/styles/core.css | 3 +- apps/server/src/ee | 2 +- .../src/integrations/import/import.module.ts | 4 +- .../import/processors/file-task.processor.ts | 4 +- ...service.ts => file-import-task.service.ts} | 7 +- .../services/import-attachment.service.ts | 408 ++++++++++-------- .../import/utils/import-formatter.ts | 27 +- 7 files changed, 253 insertions(+), 202 deletions(-) rename apps/server/src/integrations/import/services/{file-task.service.ts => file-import-task.service.ts} (98%) diff --git a/apps/client/src/features/editor/styles/core.css b/apps/client/src/features/editor/styles/core.css index 051921de..f08f5aa9 100644 --- a/apps/client/src/features/editor/styles/core.css +++ b/apps/client/src/features/editor/styles/core.css @@ -94,8 +94,7 @@ hr { border: none; - border-top: 2px solid #ced4da; - margin: 2rem 0; + border-top: 1px solid #ced4da; &:hover { cursor: pointer; diff --git a/apps/server/src/ee b/apps/server/src/ee index d90ce7a2..d03a6a3f 160000 --- a/apps/server/src/ee +++ b/apps/server/src/ee @@ -1 +1 @@ -Subproject commit d90ce7a20f20575c4afa74b0373a18e9555ed0fe +Subproject commit d03a6a3f2de77df4447b56135e1600243bd67173 diff --git a/apps/server/src/integrations/import/import.module.ts b/apps/server/src/integrations/import/import.module.ts index 40a49023..8fffde54 100644 --- a/apps/server/src/integrations/import/import.module.ts +++ b/apps/server/src/integrations/import/import.module.ts @@ -2,7 +2,7 @@ import { Module } from '@nestjs/common'; import { ImportService } from './services/import.service'; import { ImportController } from './import.controller'; import { StorageModule } from '../storage/storage.module'; -import { FileTaskService } from './services/file-task.service'; +import { FileImportTaskService } from './services/file-import-task.service'; import { FileTaskProcessor } from './processors/file-task.processor'; import { ImportAttachmentService } from './services/import-attachment.service'; import { FileTaskController } from './file-task.controller'; @@ -11,7 +11,7 @@ import { PageModule } from '../../core/page/page.module'; @Module({ providers: [ ImportService, - FileTaskService, + FileImportTaskService, FileTaskProcessor, ImportAttachmentService, ], diff --git a/apps/server/src/integrations/import/processors/file-task.processor.ts b/apps/server/src/integrations/import/processors/file-task.processor.ts index 9431ccec..38ef8dec 100644 --- a/apps/server/src/integrations/import/processors/file-task.processor.ts +++ b/apps/server/src/integrations/import/processors/file-task.processor.ts @@ -2,7 +2,7 @@ import { Logger, OnModuleDestroy } from '@nestjs/common'; import { OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq'; import { Job } from 'bullmq'; import { QueueJob, QueueName } from 'src/integrations/queue/constants'; -import { FileTaskService } from '../services/file-task.service'; +import { FileImportTaskService } from '../services/file-import-task.service'; import { FileTaskStatus } from '../utils/file.utils'; import { StorageService } from '../../storage/storage.service'; @@ -11,7 +11,7 @@ export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy { private readonly logger = new Logger(FileTaskProcessor.name); constructor( - private readonly fileTaskService: FileTaskService, + private readonly fileTaskService: FileImportTaskService, private readonly storageService: StorageService, ) { super(); diff --git a/apps/server/src/integrations/import/services/file-task.service.ts b/apps/server/src/integrations/import/services/file-import-task.service.ts similarity index 98% rename from apps/server/src/integrations/import/services/file-task.service.ts rename to apps/server/src/integrations/import/services/file-import-task.service.ts index f054017d..30338568 100644 --- a/apps/server/src/integrations/import/services/file-task.service.ts +++ b/apps/server/src/integrations/import/services/file-import-task.service.ts @@ -33,8 +33,8 @@ import { PageService } from '../../../core/page/services/page.service'; import { ImportPageNode } from '../dto/file-task-dto'; @Injectable() -export class FileTaskService { - private readonly logger = new Logger(FileTaskService.name); +export class FileImportTaskService { + private readonly logger = new Logger(FileImportTaskService.name); constructor( private readonly storageService: StorageService, @@ -266,7 +266,7 @@ export class FileTaskService { attachmentCandidates, }); - const { html, backlinks } = await formatImportHtml({ + const { html, backlinks, pageIcon } = await formatImportHtml({ html: htmlContent, currentFilePath: page.filePath, filePathToPageMetaMap: filePathToPageMetaMap, @@ -286,6 +286,7 @@ export class FileTaskService { id: page.id, slugId: page.slugId, title: title || page.name, + icon: pageIcon || null, content: prosemirrorJson, textContent: jsonToText(prosemirrorJson), ydoc: await this.importService.createYdoc(prosemirrorJson), diff --git a/apps/server/src/integrations/import/services/import-attachment.service.ts b/apps/server/src/integrations/import/services/import-attachment.service.ts index 660239eb..92780395 100644 --- a/apps/server/src/integrations/import/services/import-attachment.service.ts +++ b/apps/server/src/integrations/import/services/import-attachment.service.ts @@ -35,7 +35,7 @@ interface DrawioPair { @Injectable() export class ImportAttachmentService { private readonly logger = new Logger(ImportAttachmentService.name); - private readonly CONCURRENT_UPLOADS = 3; + private readonly CONCURRENT_UPLOADS = 5; private readonly MAX_RETRIES = 2; private readonly RETRY_DELAY = 2000; @@ -53,6 +53,7 @@ export class ImportAttachmentService { fileTask: FileTask; attachmentCandidates: Map; pageAttachments?: AttachmentInfo[]; + isConfluenceImport?: boolean; }): Promise { const { html, @@ -62,6 +63,7 @@ export class ImportAttachmentService { fileTask, attachmentCandidates, pageAttachments = [], + isConfluenceImport, } = opts; const attachmentTasks: (() => Promise)[] = []; @@ -90,7 +92,10 @@ export class ImportAttachmentService { >(); // Analyze attachments to identify Draw.io pairs - const { drawioPairs, skipFiles } = this.analyzeAttachments(pageAttachments); + const { drawioPairs, skipFiles } = this.analyzeAttachments( + pageAttachments, + isConfluenceImport, + ); // Map to store processed Draw.io SVGs const drawioSvgMap = new Map< @@ -235,202 +240,197 @@ export class ImportAttachmentService { const pageDir = path.dirname(pageRelativePath); const $ = load(html); - // Cache for resolved paths to avoid repeated lookups - const resolvedPathCache = new Map(); + // image + for (const imgEl of $('img').toArray()) { + const $img = $(imgEl); + const src = cleanUrlString($img.attr('src') ?? '')!; + if (!src || src.startsWith('http')) continue; - const getCachedResolvedPath = (rawPath: string): string | null => { - if (resolvedPathCache.has(rawPath)) { - return resolvedPathCache.get(rawPath)!; - } - const resolved = resolveRelativeAttachmentPath( - rawPath, + const relPath = resolveRelativeAttachmentPath( + src, pageDir, attachmentCandidates, ); - resolvedPathCache.set(rawPath, resolved); - return resolved; - }; + if (!relPath) continue; - // Cache for file stats to avoid repeated file system calls - const statCache = new Map(); + // Check if this image is part of a Draw.io pair + const drawioSvg = drawioSvgMap.get(relPath); + if (drawioSvg) { + const $drawio = $('
') + .attr('data-type', 'drawio') + .attr('data-src', drawioSvg.apiFilePath) + .attr('data-title', 'diagram') + .attr('data-width', '100%') + .attr('data-align', 'center') + .attr('data-attachment-id', drawioSvg.attachmentId); - const getCachedStat = async (absPath: string) => { - if (statCache.has(absPath)) { - return statCache.get(absPath); + $img.replaceWith($drawio); + unwrapFromParagraph($, $drawio); + continue; } - const stat = await fs.stat(absPath); - statCache.set(absPath, stat); - return stat; - }; - // Single DOM traversal for all attachment elements - const selector = - 'img, video, div[data-type="attachment"], a, div[data-type="excalidraw"], div[data-type="drawio"]'; - const elements = $(selector).toArray(); + const { attachmentId, apiFilePath } = processFile(relPath); - for (const element of elements) { - const $el = $(element); - const tagName = element.tagName.toLowerCase(); + const width = $img.attr('width') ?? '100%'; + const align = $img.attr('data-align') ?? 'center'; - // Process based on element type - if (tagName === 'img') { - const src = cleanUrlString($el.attr('src') ?? ''); - if (!src || src.startsWith('http')) continue; + $img + .attr('src', apiFilePath) + .attr('data-attachment-id', attachmentId) + .attr('width', width) + .attr('data-align', align); - const relPath = getCachedResolvedPath(src); - if (!relPath) continue; + unwrapFromParagraph($, $img); + } - // Check if this image is part of a Draw.io pair - const drawioSvg = drawioSvgMap.get(relPath); - if (drawioSvg) { - const $drawio = $('
') - .attr('data-type', 'drawio') - .attr('data-src', drawioSvg.apiFilePath) - .attr('data-title', 'diagram') - .attr('data-width', '100%') - .attr('data-align', 'center') - .attr('data-attachment-id', drawioSvg.attachmentId); + // video + for (const vidEl of $('video').toArray()) { + const $vid = $(vidEl); + const src = cleanUrlString($vid.attr('src') ?? '')!; + if (!src || src.startsWith('http')) continue; - $el.replaceWith($drawio); - unwrapFromParagraph($, $drawio); - continue; - } + const relPath = resolveRelativeAttachmentPath( + src, + pageDir, + attachmentCandidates, + ); + if (!relPath) continue; - const { attachmentId, apiFilePath, abs } = processFile(relPath); - const stat = await getCachedStat(abs); + const { attachmentId, apiFilePath } = processFile(relPath); - $el + const width = $vid.attr('width') ?? '100%'; + const align = $vid.attr('data-align') ?? 'center'; + + $vid + .attr('src', apiFilePath) + .attr('data-attachment-id', attachmentId) + .attr('width', width) + .attr('data-align', align); + + unwrapFromParagraph($, $vid); + } + + //
+ for (const el of $('div[data-type="attachment"]').toArray()) { + const $oldDiv = $(el); + const rawUrl = cleanUrlString($oldDiv.attr('data-attachment-url') ?? '')!; + if (!rawUrl || rawUrl.startsWith('http')) continue; + + const relPath = resolveRelativeAttachmentPath( + rawUrl, + pageDir, + attachmentCandidates, + ); + if (!relPath) continue; + + const { attachmentId, apiFilePath, abs } = processFile(relPath); + const fileName = path.basename(abs); + const mime = getMimeType(abs); + + const $newDiv = $('
') + .attr('data-type', 'attachment') + .attr('data-attachment-url', apiFilePath) + .attr('data-attachment-name', fileName) + .attr('data-attachment-mime', mime) + .attr('data-attachment-id', attachmentId); + + $oldDiv.replaceWith($newDiv); + unwrapFromParagraph($, $newDiv); + } + + // rewrite other attachments via + for (const aEl of $('a').toArray()) { + const $a = $(aEl); + const href = cleanUrlString($a.attr('href') ?? '')!; + if (!href || href.startsWith('http')) continue; + + const relPath = resolveRelativeAttachmentPath( + href, + pageDir, + attachmentCandidates, + ); + if (!relPath) continue; + + // Check if this is a Draw.io file + const drawioSvg = drawioSvgMap.get(relPath); + if (drawioSvg) { + const $drawio = $('
') + .attr('data-type', 'drawio') + .attr('data-src', drawioSvg.apiFilePath) + .attr('data-title', 'diagram') + .attr('data-width', '100%') + .attr('data-align', 'center') + .attr('data-attachment-id', drawioSvg.attachmentId); + + $a.replaceWith($drawio); + unwrapFromParagraph($, $drawio); + continue; + } + + // Skip files that should be ignored + if (skipFiles.has(relPath)) { + $a.remove(); + continue; + } + + const { attachmentId, apiFilePath, abs } = processFile(relPath); + const ext = path.extname(relPath).toLowerCase(); + + if (ext === '.mp4') { + const $video = $('