mirror of
https://github.com/Shadowfita/docmost.git
synced 2025-11-12 15:52:32 +10:00
* Make page import handling better
This commit is contained in:
Philipinho
@ -84,14 +84,14 @@ function ImportFormatSelection({ spaceId, onClose }: ImportFormatSelection) {
|
||||
}
|
||||
}
|
||||
|
||||
const newTreeNodes = buildTree(pages);
|
||||
const fullTree = treeData.concat(newTreeNodes);
|
||||
if (pages?.length > 0 && pageCount > 0) {
|
||||
const newTreeNodes = buildTree(pages);
|
||||
const fullTree = treeData.concat(newTreeNodes);
|
||||
|
||||
if (newTreeNodes?.length && fullTree?.length > 0) {
|
||||
setTreeData(fullTree);
|
||||
}
|
||||
if (newTreeNodes?.length && fullTree?.length > 0) {
|
||||
setTreeData(fullTree);
|
||||
}
|
||||
|
||||
if (pageCount > 0) {
|
||||
const pageCountText = pageCount === 1 ? "1 page" : `${pageCount} pages`;
|
||||
|
||||
notifications.update({
|
||||
|
||||
@ -28,7 +28,9 @@ import {
|
||||
TrailingNode,
|
||||
} from '@docmost/editor-ext';
|
||||
import { generateText, JSONContent } from '@tiptap/core';
|
||||
import { generateHTML, generateJSON } from '../common/helpers/prosemirror/html';
|
||||
import { generateHTML } from '../common/helpers/prosemirror/html';
|
||||
// default tiptap library works best generating prosemirror json state
|
||||
import { generateJSON } from '@tiptap/html';
|
||||
|
||||
export const tiptapExtensions = [
|
||||
StarterKit,
|
||||
|
||||
@ -20,6 +20,7 @@ import {
|
||||
} from '../../core/casl/interfaces/space-ability.type';
|
||||
import { FileInterceptor } from '../../common/interceptors/file.interceptor';
|
||||
import * as bytes from 'bytes';
|
||||
import * as path from 'path';
|
||||
import { MAX_FILE_SIZE } from '../../core/attachment/attachment.constants';
|
||||
import { ImportService } from './import.service';
|
||||
import { AuthWorkspace } from '../../common/decorators/auth-workspace.decorator';
|
||||
@ -42,6 +43,8 @@ export class ImportController {
|
||||
@AuthUser() user: User,
|
||||
@AuthWorkspace() workspace: Workspace,
|
||||
) {
|
||||
const validFileExtensions = ['.md', '.html'];
|
||||
|
||||
const maxFileSize = bytes(MAX_FILE_SIZE);
|
||||
|
||||
let file = null;
|
||||
@ -62,6 +65,12 @@ export class ImportController {
|
||||
throw new BadRequestException('Failed to upload file');
|
||||
}
|
||||
|
||||
if (
|
||||
!validFileExtensions.includes(path.extname(file.filename).toLowerCase())
|
||||
) {
|
||||
throw new BadRequestException('Invalid import file type.');
|
||||
}
|
||||
|
||||
const spaceId = file.fields?.spaceId?.value;
|
||||
|
||||
if (!spaceId) {
|
||||
|
||||
@ -4,12 +4,11 @@ import { MultipartFile } from '@fastify/multipart';
|
||||
import { sanitize } from 'sanitize-filename-ts';
|
||||
import * as path from 'path';
|
||||
import { htmlToJson } from '../../collaboration/collaboration.util';
|
||||
import { marked } from 'marked';
|
||||
import { InjectKysely } from 'nestjs-kysely';
|
||||
import { KyselyDB } from '@docmost/db/types/kysely.types';
|
||||
import { generateSlugId } from '../../common/helpers';
|
||||
import { generateJitteredKeyBetween } from 'fractional-indexing-jittered';
|
||||
import { transformHTML } from './utils/html.utils';
|
||||
import { markdownToHtml } from './utils/marked.utils';
|
||||
|
||||
@Injectable()
|
||||
export class ImportService {
|
||||
@ -36,16 +35,23 @@ export class ImportService {
|
||||
let prosemirrorState = null;
|
||||
let createdPage = null;
|
||||
|
||||
if (fileExtension.endsWith('.md') && fileMimeType === 'text/markdown') {
|
||||
prosemirrorState = await this.processMarkdown(fileContent);
|
||||
}
|
||||
|
||||
if (fileExtension.endsWith('.html') && fileMimeType === 'text/html') {
|
||||
prosemirrorState = await this.processHTML(fileContent);
|
||||
try {
|
||||
if (fileExtension.endsWith('.md') && fileMimeType === 'text/markdown') {
|
||||
prosemirrorState = await this.processMarkdown(fileContent);
|
||||
} else if (
|
||||
fileExtension.endsWith('.html') &&
|
||||
fileMimeType === 'text/html'
|
||||
) {
|
||||
prosemirrorState = await this.processHTML(fileContent);
|
||||
}
|
||||
} catch (err) {
|
||||
const message = 'Error processing file content';
|
||||
this.logger.error(message, err);
|
||||
throw new BadRequestException(message);
|
||||
}
|
||||
|
||||
if (!prosemirrorState) {
|
||||
const message = 'Unsupported file format or mime type';
|
||||
const message = 'Failed to create ProseMirror state';
|
||||
this.logger.error(message);
|
||||
throw new BadRequestException(message);
|
||||
}
|
||||
@ -69,8 +75,12 @@ export class ImportService {
|
||||
workspaceId: workspaceId,
|
||||
lastUpdatedById: userId,
|
||||
});
|
||||
|
||||
this.logger.debug(
|
||||
`Successfully imported "${title}${fileExtension}. ID: ${createdPage.id} - SlugId: ${createdPage.slugId}"`,
|
||||
);
|
||||
} catch (err) {
|
||||
const message = 'Failed to create page';
|
||||
const message = 'Failed to create imported page';
|
||||
this.logger.error(message, err);
|
||||
throw new BadRequestException(message);
|
||||
}
|
||||
@ -80,14 +90,20 @@ export class ImportService {
|
||||
}
|
||||
|
||||
async processMarkdown(markdownInput: string): Promise<any> {
|
||||
// turn markdown to html
|
||||
const html = await marked.parse(markdownInput);
|
||||
return await this.processHTML(html);
|
||||
try {
|
||||
const html = await markdownToHtml(markdownInput);
|
||||
return this.processHTML(html);
|
||||
} catch (err) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async processHTML(htmlInput: string): Promise<any> {
|
||||
// turn html to prosemirror state
|
||||
return htmlToJson(transformHTML(htmlInput));
|
||||
try {
|
||||
return htmlToJson(htmlInput);
|
||||
} catch (err) {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
extractTitleAndRemoveHeading(prosemirrorState: any) {
|
||||
|
||||
@ -1,80 +0,0 @@
|
||||
import { Window, DOMParser } from 'happy-dom';
|
||||
|
||||
function transformTaskList(html: string): string {
|
||||
const window = new Window();
|
||||
const doc = new DOMParser(window).parseFromString(html, 'text/html');
|
||||
|
||||
const ulElements = doc.querySelectorAll('ul');
|
||||
ulElements.forEach((ul) => {
|
||||
let isTaskList = false;
|
||||
|
||||
const liElements = ul.querySelectorAll('li');
|
||||
liElements.forEach((li) => {
|
||||
const checkbox = li.querySelector('input[type="checkbox"]');
|
||||
|
||||
if (checkbox) {
|
||||
isTaskList = true;
|
||||
// Add taskItem data type
|
||||
li.setAttribute('data-type', 'taskItem');
|
||||
// Set data-checked attribute based on the checkbox state
|
||||
// @ts-ignore
|
||||
li.setAttribute('data-checked', checkbox.checked ? 'true' : 'false');
|
||||
// Remove the checkbox from the li
|
||||
checkbox.remove();
|
||||
|
||||
// Move the content of <p> out of the <p> and remove <p>
|
||||
const pElements = li.querySelectorAll('p');
|
||||
pElements.forEach((p) => {
|
||||
// Append the content of the <p> element to its parent (the <li> element)
|
||||
while (p.firstChild) {
|
||||
li.appendChild(p.firstChild);
|
||||
}
|
||||
// Remove the now empty <p> element
|
||||
p.remove();
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// If any <li> contains a checkbox, mark the <ul> as a task list
|
||||
if (isTaskList) {
|
||||
ul.setAttribute('data-type', 'taskList');
|
||||
}
|
||||
});
|
||||
|
||||
return doc.body.innerHTML;
|
||||
}
|
||||
|
||||
function transformCallouts(html: string): string {
|
||||
const window = new Window();
|
||||
const doc = new DOMParser(window).parseFromString(html, 'text/html');
|
||||
|
||||
const calloutRegex = /:::(\w+)\s*([\s\S]*?)\s*:::/g;
|
||||
|
||||
const createCalloutDiv = (type: string, content: string): HTMLElement => {
|
||||
const div = doc.createElement('div');
|
||||
div.setAttribute('data-type', 'callout');
|
||||
div.setAttribute('data-callout-type', type);
|
||||
const p = doc.createElement('p');
|
||||
p.textContent = content.trim();
|
||||
div.appendChild(p);
|
||||
return div as unknown as HTMLElement;
|
||||
};
|
||||
|
||||
const pElements = doc.querySelectorAll('p');
|
||||
|
||||
pElements.forEach((p) => {
|
||||
if (calloutRegex.test(p.innerHTML) && !p.closest('ul, ol')) {
|
||||
calloutRegex.lastIndex = 0;
|
||||
const [, type, content] = calloutRegex.exec(p.innerHTML) || [];
|
||||
const calloutDiv = createCalloutDiv(type, content);
|
||||
// @ts-ignore
|
||||
p.replaceWith(calloutDiv);
|
||||
}
|
||||
});
|
||||
|
||||
return doc.body.innerHTML;
|
||||
}
|
||||
|
||||
export function transformHTML(html: string): string {
|
||||
return transformTaskList(transformCallouts(html));
|
||||
}
|
||||
36
apps/server/src/integrations/import/utils/marked.utils.ts
Normal file
36
apps/server/src/integrations/import/utils/marked.utils.ts
Normal file
@ -0,0 +1,36 @@
|
||||
import { marked } from 'marked';
|
||||
|
||||
marked.use({
|
||||
renderer: {
|
||||
// @ts-ignore
|
||||
list(body: string, isOrdered: boolean, start: number) {
|
||||
if (isOrdered) {
|
||||
const startAttr = start !== 1 ? ` start="${start}"` : '';
|
||||
return `<ol ${startAttr}>\n${body}</ol>\n`;
|
||||
}
|
||||
|
||||
const dataType = body.includes(`<input`) ? ' data-type="taskList"' : '';
|
||||
return `<ul${dataType}>\n${body}</ul>\n`;
|
||||
},
|
||||
// @ts-ignore
|
||||
listitem({ text, raw, task: isTask, checked: isChecked }): string {
|
||||
if (!isTask) {
|
||||
return `<li>${text}</li>\n`;
|
||||
}
|
||||
const checkedAttr = isChecked
|
||||
? 'data-checked="true"'
|
||||
: 'data-checked="false"';
|
||||
return `<li data-type="taskItem" ${checkedAttr}>${text}</li>\n`;
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
export async function markdownToHtml(markdownInput: string): Promise<string> {
|
||||
const YAML_FONT_MATTER_REGEX = /^\s*---[\s\S]*?---\s*/;
|
||||
|
||||
const markdown = markdownInput
|
||||
.replace(YAML_FONT_MATTER_REGEX, '')
|
||||
.trimStart();
|
||||
|
||||
return marked.parse(markdown);
|
||||
}
|
||||
@ -53,6 +53,7 @@
|
||||
"@tiptap/extension-typography": "^2.5.4",
|
||||
"@tiptap/extension-underline": "^2.5.4",
|
||||
"@tiptap/extension-youtube": "^2.5.4",
|
||||
"@tiptap/html": "^2.5.4",
|
||||
"@tiptap/pm": "^2.5.4",
|
||||
"@tiptap/react": "^2.5.4",
|
||||
"@tiptap/starter-kit": "^2.5.4",
|
||||
|
||||
47
pnpm-lock.yaml
generated
47
pnpm-lock.yaml
generated
@ -119,6 +119,9 @@ importers:
|
||||
'@tiptap/extension-youtube':
|
||||
specifier: ^2.5.4
|
||||
version: 2.5.4(@tiptap/core@2.5.4(@tiptap/pm@2.5.4))
|
||||
'@tiptap/html':
|
||||
specifier: ^2.5.4
|
||||
version: 2.5.4(@tiptap/core@2.5.4(@tiptap/pm@2.5.4))(@tiptap/pm@2.5.4)
|
||||
'@tiptap/pm':
|
||||
specifier: ^2.5.4
|
||||
version: 2.5.4
|
||||
@ -3780,6 +3783,12 @@ packages:
|
||||
peerDependencies:
|
||||
'@tiptap/core': ^2.5.4
|
||||
|
||||
'@tiptap/html@2.5.4':
|
||||
resolution: {integrity: sha512-Fcvsa7kkO+Id7WBFimDN5zdHksVGVnyHnffaN/PaAgbKmzP53BC38Pd0XuHS+KL6btqQIFE2GlqNYnyIos7i+g==}
|
||||
peerDependencies:
|
||||
'@tiptap/core': ^2.5.4
|
||||
'@tiptap/pm': ^2.5.4
|
||||
|
||||
'@tiptap/pm@2.5.4':
|
||||
resolution: {integrity: sha512-oFIsuniptdUXn93x4aM2sVN3hYKo9Fj55zAkYrWhwxFYUYcPxd5ibra2we+wRK5TaiPu098wpC+yMSTZ/KKMpA==}
|
||||
|
||||
@ -4757,6 +4766,10 @@ packages:
|
||||
css-to-mat@1.1.1:
|
||||
resolution: {integrity: sha512-kvpxFYZb27jRd2vium35G7q5XZ2WJ9rWjDUMNT36M3Hc41qCrLXFM5iEKMGXcrPsKfXEN+8l/riB4QzwwwiEyQ==}
|
||||
|
||||
css-what@6.1.0:
|
||||
resolution: {integrity: sha512-HTUrgRJ7r4dsZKU6GjmpfRK1O76h97Z8MfS1G0FozR+oF2kG6Vfe8JE6zwrkbxigziPHinCJ+gCPjA9EaBDtRw==}
|
||||
engines: {node: '>= 6'}
|
||||
|
||||
cssesc@3.0.0:
|
||||
resolution: {integrity: sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==}
|
||||
engines: {node: '>=4'}
|
||||
@ -6178,8 +6191,8 @@ packages:
|
||||
makeerror@1.0.12:
|
||||
resolution: {integrity: sha512-JmqCvUhmt43madlpFzG4BQzG2Z3m6tvQDNKdClZnO3VbIudJYmxsT0FNJMeiB2+JTSlTQTSbU8QdesVmwJcmLg==}
|
||||
|
||||
markdown-it@14.0.0:
|
||||
resolution: {integrity: sha512-seFjF0FIcPt4P9U39Bq1JYblX0KZCjDLFFQPHpL5AzHpqPEKtosxmdq/LTVZnjfH7tjt9BxStm+wXcDBNuYmzw==}
|
||||
markdown-it@14.1.0:
|
||||
resolution: {integrity: sha512-a54IwgWPaeBCAAsv13YgmALOF1elABB08FxO9i+r4VFk5Vl4pKokRPeX8u5TCgSsPi6ec1otfLjdOpVcgbpshg==}
|
||||
hasBin: true
|
||||
|
||||
marked@13.0.2:
|
||||
@ -7790,8 +7803,8 @@ packages:
|
||||
engines: {node: '>=14.17'}
|
||||
hasBin: true
|
||||
|
||||
uc.micro@2.0.0:
|
||||
resolution: {integrity: sha512-DffL94LsNOccVn4hyfRe5rdKa273swqeA5DJpMOeFmEn1wCDc7nAbbB0gXlgBCL7TNzeTv6G7XVWzan7iJtfig==}
|
||||
uc.micro@2.1.0:
|
||||
resolution: {integrity: sha512-ARDJmphmdvUk6Glw7y9DQ2bFkKBHwQHLi2lsaH6PPmz/Ka9sFOBsBluozhDltWmnv9u/cF6Rt87znRTPV+yp/A==}
|
||||
|
||||
uid2@1.0.0:
|
||||
resolution: {integrity: sha512-+I6aJUv63YAcY9n4mQreLUt0d4lvwkkopDNmpomkAUz0fAkEMV9pRWxN0EjhW1YfRhcuyHg2v3mwddCDW1+LFQ==}
|
||||
@ -8160,6 +8173,10 @@ packages:
|
||||
resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==}
|
||||
engines: {node: '>=10'}
|
||||
|
||||
zeed-dom@0.10.11:
|
||||
resolution: {integrity: sha512-7ukbu6aQKx34OQ7PfUIxOuAhk2MvyZY/t4/IJsVzy76zuMzfhE74+Dbyp8SHiUJPTPkF0FflP1KVrGJ7gk9IHw==}
|
||||
engines: {node: '>=14.13.1'}
|
||||
|
||||
zod@3.23.8:
|
||||
resolution: {integrity: sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==}
|
||||
|
||||
@ -12059,6 +12076,12 @@ snapshots:
|
||||
dependencies:
|
||||
'@tiptap/core': 2.5.4(@tiptap/pm@2.5.4)
|
||||
|
||||
'@tiptap/html@2.5.4(@tiptap/core@2.5.4(@tiptap/pm@2.5.4))(@tiptap/pm@2.5.4)':
|
||||
dependencies:
|
||||
'@tiptap/core': 2.5.4(@tiptap/pm@2.5.4)
|
||||
'@tiptap/pm': 2.5.4
|
||||
zeed-dom: 0.10.11
|
||||
|
||||
'@tiptap/pm@2.5.4':
|
||||
dependencies:
|
||||
prosemirror-changeset: 2.2.1
|
||||
@ -13272,6 +13295,8 @@ snapshots:
|
||||
'@daybrush/utils': 1.13.0
|
||||
'@scena/matrix': 1.1.1
|
||||
|
||||
css-what@6.1.0: {}
|
||||
|
||||
cssesc@3.0.0: {}
|
||||
|
||||
cssstyle@3.0.0:
|
||||
@ -14899,7 +14924,7 @@ snapshots:
|
||||
|
||||
linkify-it@5.0.0:
|
||||
dependencies:
|
||||
uc.micro: 2.0.0
|
||||
uc.micro: 2.1.0
|
||||
|
||||
linkifyjs@4.1.3: {}
|
||||
|
||||
@ -14994,14 +15019,14 @@ snapshots:
|
||||
dependencies:
|
||||
tmpl: 1.0.5
|
||||
|
||||
markdown-it@14.0.0:
|
||||
markdown-it@14.1.0:
|
||||
dependencies:
|
||||
argparse: 2.0.1
|
||||
entities: 4.5.0
|
||||
linkify-it: 5.0.0
|
||||
mdurl: 2.0.0
|
||||
punycode.js: 2.3.1
|
||||
uc.micro: 2.0.0
|
||||
uc.micro: 2.1.0
|
||||
|
||||
marked@13.0.2: {}
|
||||
|
||||
@ -15690,7 +15715,7 @@ snapshots:
|
||||
|
||||
prosemirror-markdown@1.13.0:
|
||||
dependencies:
|
||||
markdown-it: 14.0.0
|
||||
markdown-it: 14.1.0
|
||||
prosemirror-model: 1.22.1
|
||||
|
||||
prosemirror-menu@1.2.4:
|
||||
@ -16778,7 +16803,7 @@ snapshots:
|
||||
|
||||
typescript@5.5.2: {}
|
||||
|
||||
uc.micro@2.0.0: {}
|
||||
uc.micro@2.1.0: {}
|
||||
|
||||
uid2@1.0.0: {}
|
||||
|
||||
@ -17101,4 +17126,8 @@ snapshots:
|
||||
|
||||
yocto-queue@0.1.0: {}
|
||||
|
||||
zeed-dom@0.10.11:
|
||||
dependencies:
|
||||
css-what: 6.1.0
|
||||
|
||||
zod@3.23.8: {}
|
||||
|
||||
Reference in New Issue
Block a user