Compare commits

..

3 Commits

15 changed files with 5 additions and 363 deletions

View File

@ -29,7 +29,6 @@
"clsx": "^2.1.1", "clsx": "^2.1.1",
"emoji-mart": "^5.6.0", "emoji-mart": "^5.6.0",
"file-saver": "^2.0.5", "file-saver": "^2.0.5",
"highlightjs-sap-abap": "^0.3.0",
"i18next": "^23.14.0", "i18next": "^23.14.0",
"i18next-http-backend": "^2.6.1", "i18next-http-backend": "^2.6.1",
"jotai": "^2.12.1", "jotai": "^2.12.1",

View File

@ -58,7 +58,6 @@ import ExcalidrawView from "@/features/editor/components/excalidraw/excalidraw-v
import EmbedView from "@/features/editor/components/embed/embed-view.tsx"; import EmbedView from "@/features/editor/components/embed/embed-view.tsx";
import plaintext from "highlight.js/lib/languages/plaintext"; import plaintext from "highlight.js/lib/languages/plaintext";
import powershell from "highlight.js/lib/languages/powershell"; import powershell from "highlight.js/lib/languages/powershell";
import abap from "highlightjs-sap-abap";
import elixir from "highlight.js/lib/languages/elixir"; import elixir from "highlight.js/lib/languages/elixir";
import erlang from "highlight.js/lib/languages/erlang"; import erlang from "highlight.js/lib/languages/erlang";
import dockerfile from "highlight.js/lib/languages/dockerfile"; import dockerfile from "highlight.js/lib/languages/dockerfile";
@ -77,7 +76,7 @@ import { CharacterCount } from "@tiptap/extension-character-count";
const lowlight = createLowlight(common); const lowlight = createLowlight(common);
lowlight.register("mermaid", plaintext); lowlight.register("mermaid", plaintext);
lowlight.register("powershell", powershell); lowlight.register("powershell", powershell);
lowlight.register("abap", abap); lowlight.register("powershell", powershell);
lowlight.register("erlang", erlang); lowlight.register("erlang", erlang);
lowlight.register("elixir", elixir); lowlight.register("elixir", elixir);
lowlight.register("dockerfile", dockerfile); lowlight.register("dockerfile", dockerfile);

View File

@ -1,45 +0,0 @@
import { Kysely, sql } from 'kysely';
export async function up(db: Kysely<any>): Promise<void> {
await db.schema
.createTable('file_tasks')
.addColumn('id', 'uuid', (col) =>
col.primaryKey().defaultTo(sql`gen_uuid_v7()`),
)
//type: import or export
.addColumn('type', 'varchar', (col) => col)
// source - generic, notion, confluence
// type or provider?
.addColumn('source', 'varchar', (col) => col)
// status (enum: PENDING|PROCESSING|SUCCESS|FAILED),
.addColumn('status', 'varchar', (col) => col)
// file name
// file path
// file size
.addColumn('file_name', 'varchar', (col) => col.notNull())
.addColumn('file_path', 'varchar', (col) => col.notNull())
.addColumn('file_size', 'int8', (col) => col)
.addColumn('file_ext', 'varchar', (col) => col)
.addColumn('creator_id', 'uuid', (col) => col.references('users.id'))
.addColumn('space_id', 'uuid', (col) =>
col.references('spaces.id').onDelete('cascade'),
)
.addColumn('workspace_id', 'uuid', (col) =>
col.references('workspaces.id').onDelete('cascade').notNull(),
)
.addColumn('created_at', 'timestamptz', (col) =>
col.notNull().defaultTo(sql`now()`),
)
.addColumn('updated_at', 'timestamptz', (col) =>
col.notNull().defaultTo(sql`now()`),
)
.addColumn('completed_at', 'timestamptz', (col) => col)
.addColumn('deleted_at', 'timestamptz', (col) => col)
.execute();
}
export async function down(db: Kysely<any>): Promise<void> {
await db.schema.dropTable('file_tasks').execute();
}

View File

@ -122,24 +122,6 @@ export interface Comments {
workspaceId: string; workspaceId: string;
} }
export interface FileTasks {
completedAt: Timestamp | null;
createdAt: Generated<Timestamp>;
creatorId: string | null;
deletedAt: Timestamp | null;
fileExt: string | null;
fileName: string;
filePath: string;
fileSize: Int8 | null;
id: Generated<string>;
source: string | null;
spaceId: string | null;
status: string | null;
type: string | null;
updatedAt: Generated<Timestamp>;
workspaceId: string;
}
export interface Groups { export interface Groups {
createdAt: Generated<Timestamp>; createdAt: Generated<Timestamp>;
creatorId: string | null; creatorId: string | null;
@ -316,7 +298,6 @@ export interface DB {
backlinks: Backlinks; backlinks: Backlinks;
billing: Billing; billing: Billing;
comments: Comments; comments: Comments;
fileTasks: FileTasks;
groups: Groups; groups: Groups;
groupUsers: GroupUsers; groupUsers: GroupUsers;
pageHistory: PageHistory; pageHistory: PageHistory;

View File

@ -17,7 +17,6 @@ import {
AuthProviders, AuthProviders,
AuthAccounts, AuthAccounts,
Shares, Shares,
FileTasks,
} from './db'; } from './db';
// Workspace // Workspace
@ -108,8 +107,3 @@ export type UpdatableAuthAccount = Updateable<Omit<AuthAccounts, 'id'>>;
export type Share = Selectable<Shares>; export type Share = Selectable<Shares>;
export type InsertableShare = Insertable<Shares>; export type InsertableShare = Insertable<Shares>;
export type UpdatableShare = Updateable<Omit<Shares, 'id'>>; export type UpdatableShare = Updateable<Omit<Shares, 'id'>>;
// File Task
export type FileTask = Selectable<FileTasks>;
export type InsertableFileTask = Insertable<FileTasks>;
export type UpdatableFileTask = Updateable<Omit<FileTasks, 'id'>>;

View File

@ -1,68 +0,0 @@
import { BadRequestException, Injectable, Logger } from '@nestjs/common';
import { PageRepo } from '@docmost/db/repos/page/page.repo';
import { MultipartFile } from '@fastify/multipart';
import { sanitize } from 'sanitize-filename-ts';
import * as path from 'path';
import {
htmlToJson,
jsonToText,
tiptapExtensions,
} from '../../collaboration/collaboration.util';
import { InjectKysely } from 'nestjs-kysely';
import { KyselyDB } from '@docmost/db/types/kysely.types';
import { generateSlugId } from '../../common/helpers';
import { generateJitteredKeyBetween } from 'fractional-indexing-jittered';
import { TiptapTransformer } from '@hocuspocus/transformer';
import * as Y from 'yjs';
import { markdownToHtml } from '@docmost/editor-ext';
import {
FileTaskStatus,
FileTaskType,
getFileTaskFolderPath,
} from './file.utils';
import { v7 as uuid7 } from 'uuid';
import { StorageService } from '../storage/storage.service';
import { InjectQueue } from '@nestjs/bullmq';
import { Queue } from 'bullmq';
import { QueueJob, QueueName } from '../queue/constants';
@Injectable()
export class FileTaskService {
private readonly logger = new Logger(FileTaskService.name);
constructor(
private readonly storageService: StorageService,
@InjectKysely() private readonly db: KyselyDB,
) {}
async processZIpImport(fileTaskId: string): Promise<void> {
console.log(`Processing zip import: ${fileTaskId}`);
const fileTask = await this.db
.selectFrom('fileTasks')
.selectAll()
.where('id', '=', fileTaskId)
.executeTakeFirst();
if (!fileTask) {
this.logger.log(`File task with ID ${fileTaskId} not found`);
return;
}
// update status to processing
await this.db
.updateTable('fileTasks')
.set({ status: FileTaskStatus.Processing })
.execute();
// it did, what next?
const file = await this.storageService.read(fileTask.filePath);
}
// receive the file
async processGenericImport(fileTaskId: string): Promise<void> {
}
}

View File

@ -1,29 +0,0 @@
export enum FileTaskType {
Import = 'import',
Export = 'export',
}
export enum FileImportType {
Generic = 'generic',
Notion = 'notion',
Confluence = 'confluence',
}
export enum FileTaskStatus {
Pending = 'pending',
Processing = 'processing',
Success = 'success',
Failed = 'failed',
}
export function getFileTaskFolderPath(
type: FileTaskType,
workspaceId: string,
): string {
switch (type) {
case FileTaskType.Import:
return `${workspaceId}/imports`;
case FileTaskType.Export:
return `${workspaceId}/exports`;
}
}

View File

@ -83,57 +83,4 @@ export class ImportController {
return this.importService.importPage(file, user.id, spaceId, workspace.id); return this.importService.importPage(file, user.id, spaceId, workspace.id);
} }
@UseInterceptors(FileInterceptor)
@UseGuards(JwtAuthGuard)
@HttpCode(HttpStatus.OK)
// temporary naming
@Post('pages/import-zip')
async importZip(
@Req() req: any,
@AuthUser() user: User,
@AuthWorkspace() workspace: Workspace,
) {
const validFileExtensions = ['.zip'];
const maxFileSize = bytes('100mb');
let file = null;
try {
file = await req.file({
limits: { fileSize: maxFileSize, fields: 3, files: 1 },
});
} catch (err: any) {
this.logger.error(err.message);
if (err?.statusCode === 413) {
throw new BadRequestException(
`File too large. Exceeds the 100mb import limit`,
);
}
}
if (!file) {
throw new BadRequestException('Failed to upload file');
}
if (
!validFileExtensions.includes(path.extname(file.filename).toLowerCase())
) {
throw new BadRequestException('Invalid import file type.');
}
const spaceId = file.fields?.spaceId?.value;
const source = file.fields?.source?.value;
if (!spaceId) {
throw new BadRequestException('spaceId or format not found');
}
const ability = await this.spaceAbility.createForUser(user, spaceId);
if (ability.cannot(SpaceCaslAction.Edit, SpaceCaslSubject.Page)) {
throw new ForbiddenException();
}
return this.importService.importZip(file, source, user.id, spaceId, workspace.id);
}
} }

View File

@ -1,13 +1,9 @@
import { Module } from '@nestjs/common'; import { Module } from '@nestjs/common';
import { ImportService } from './import.service'; import { ImportService } from './import.service';
import { ImportController } from './import.controller'; import { ImportController } from './import.controller';
import { StorageModule } from '../storage/storage.module';
import { FileTaskService } from './file-task.service';
import { FileTaskProcessor } from './processors/file-task.processor';
@Module({ @Module({
providers: [ImportService, FileTaskService, FileTaskProcessor], providers: [ImportService],
controllers: [ImportController], controllers: [ImportController],
imports: [StorageModule],
}) })
export class ImportModule {} export class ImportModule {}

View File

@ -4,8 +4,7 @@ import { MultipartFile } from '@fastify/multipart';
import { sanitize } from 'sanitize-filename-ts'; import { sanitize } from 'sanitize-filename-ts';
import * as path from 'path'; import * as path from 'path';
import { import {
htmlToJson, htmlToJson, jsonToText,
jsonToText,
tiptapExtensions, tiptapExtensions,
} from '../../collaboration/collaboration.util'; } from '../../collaboration/collaboration.util';
import { InjectKysely } from 'nestjs-kysely'; import { InjectKysely } from 'nestjs-kysely';
@ -14,17 +13,7 @@ import { generateSlugId } from '../../common/helpers';
import { generateJitteredKeyBetween } from 'fractional-indexing-jittered'; import { generateJitteredKeyBetween } from 'fractional-indexing-jittered';
import { TiptapTransformer } from '@hocuspocus/transformer'; import { TiptapTransformer } from '@hocuspocus/transformer';
import * as Y from 'yjs'; import * as Y from 'yjs';
import { markdownToHtml } from '@docmost/editor-ext'; import { markdownToHtml } from "@docmost/editor-ext";
import {
FileTaskStatus,
FileTaskType,
getFileTaskFolderPath,
} from './file.utils';
import { v7 as uuid7 } from 'uuid';
import { StorageService } from '../storage/storage.service';
import { InjectQueue } from '@nestjs/bullmq';
import { Queue } from 'bullmq';
import { QueueJob, QueueName } from '../queue/constants';
@Injectable() @Injectable()
export class ImportService { export class ImportService {
@ -32,10 +21,7 @@ export class ImportService {
constructor( constructor(
private readonly pageRepo: PageRepo, private readonly pageRepo: PageRepo,
private readonly storageService: StorageService,
@InjectKysely() private readonly db: KyselyDB, @InjectKysely() private readonly db: KyselyDB,
@InjectQueue(QueueName.FILE_TASK_QUEUE)
private readonly fileTaskQueue: Queue,
) {} ) {}
async importPage( async importPage(
@ -175,56 +161,4 @@ export class ImportService {
return generateJitteredKeyBetween(null, null); return generateJitteredKeyBetween(null, null);
} }
} }
async importZip(
filePromise: Promise<MultipartFile>,
source: string,
userId: string,
spaceId: string,
workspaceId: string,
): Promise<void> {
const file = await filePromise;
const fileBuffer = await file.toBuffer();
const fileExtension = path.extname(file.filename).toLowerCase();
const fileName = sanitize(
path.basename(file.filename, fileExtension).slice(0, 255),
);
const fileTaskId = uuid7();
const filePath = `${getFileTaskFolderPath(FileTaskType.Import, workspaceId)}/${fileTaskId}/${fileName}`;
// upload file
await this.storageService.upload(filePath, fileBuffer);
// store in fileTasks table
await this.db
.insertInto('fileTasks')
.values({
id: fileTaskId,
type: FileTaskType.Import,
source: source,
status: FileTaskStatus.Pending,
fileName: fileName,
filePath: filePath,
fileSize: 0,
fileExt: 'zip',
creatorId: userId,
spaceId: spaceId,
workspaceId: workspaceId,
})
.execute();
// what to send to queue
// pass the task ID
await this.fileTaskQueue.add(QueueJob.IMPORT_TASK, {
fileTaskId: fileTaskId,
});
// return tasks info
// when the processor picks it up
// we change the status to processing
// if it gets processed successfully,
// we change the status to success
// else failed
}
} }

View File

@ -1,51 +0,0 @@
import { Logger, OnModuleDestroy } from '@nestjs/common';
import { OnWorkerEvent, Processor, WorkerHost } from '@nestjs/bullmq';
import { Job } from 'bullmq';
import { QueueJob, QueueName } from 'src/integrations/queue/constants';
import { FileTaskService } from '../file-task.service';
@Processor(QueueName.FILE_TASK_QUEUE)
export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy {
private readonly logger = new Logger(FileTaskProcessor.name);
constructor(private readonly fileTaskService: FileTaskService) {
super();
}
async process(job: Job<any, void>): Promise<void> {
try {
switch (job.name) {
case QueueJob.IMPORT_TASK:
console.log('import task', job.data.fileTaskId);
await this.fileTaskService.processZIpImport(job.data.fileTaskId);
break;
case QueueJob.EXPORT_TASK:
console.log('export task', job.data.fileTaskId);
}
} catch (err) {
throw err;
}
}
@OnWorkerEvent('active')
onActive(job: Job) {
this.logger.debug(`Processing ${job.name} job`);
}
@OnWorkerEvent('failed')
onError(job: Job) {
this.logger.error(
`Error processing ${job.name} job. Reason: ${job.failedReason}`,
);
}
@OnWorkerEvent('completed')
onCompleted(job: Job) {
this.logger.debug(`Completed ${job.name} job`);
}
async onModuleDestroy(): Promise<void> {
if (this.worker) {
await this.worker.close();
}
}
}

View File

@ -3,7 +3,6 @@ export enum QueueName {
ATTACHMENT_QUEUE = '{attachment-queue}', ATTACHMENT_QUEUE = '{attachment-queue}',
GENERAL_QUEUE = '{general-queue}', GENERAL_QUEUE = '{general-queue}',
BILLING_QUEUE = '{billing-queue}', BILLING_QUEUE = '{billing-queue}',
FILE_TASK_QUEUE = '{file-task-queue}',
} }
export enum QueueJob { export enum QueueJob {
@ -20,7 +19,4 @@ export enum QueueJob {
TRIAL_ENDED = 'trial-ended', TRIAL_ENDED = 'trial-ended',
WELCOME_EMAIL = 'welcome-email', WELCOME_EMAIL = 'welcome-email',
FIRST_PAYMENT_EMAIL = 'first-payment-email', FIRST_PAYMENT_EMAIL = 'first-payment-email',
IMPORT_TASK = 'import-task',
EXPORT_TASK = 'export-task',
} }

View File

@ -49,9 +49,6 @@ import { BacklinksProcessor } from './processors/backlinks.processor';
BullModule.registerQueue({ BullModule.registerQueue({
name: QueueName.BILLING_QUEUE, name: QueueName.BILLING_QUEUE,
}), }),
BullModule.registerQueue({
name: QueueName.FILE_TASK_QUEUE,
}),
], ],
exports: [BullModule], exports: [BullModule],
providers: [BacklinksProcessor], providers: [BacklinksProcessor],

8
pnpm-lock.yaml generated
View File

@ -257,9 +257,6 @@ importers:
file-saver: file-saver:
specifier: ^2.0.5 specifier: ^2.0.5
version: 2.0.5 version: 2.0.5
highlightjs-sap-abap:
specifier: ^0.3.0
version: 0.3.0
i18next: i18next:
specifier: ^23.14.0 specifier: ^23.14.0
version: 23.14.0 version: 23.14.0
@ -5875,9 +5872,6 @@ packages:
resolution: {integrity: sha512-SYVnVFswQER+zu1laSya563s+F8VDGt7o35d4utbamowvUNLLMovFqwCLSocpZTz3MgaSRA1IbqRWZv97dtErQ==} resolution: {integrity: sha512-SYVnVFswQER+zu1laSya563s+F8VDGt7o35d4utbamowvUNLLMovFqwCLSocpZTz3MgaSRA1IbqRWZv97dtErQ==}
engines: {node: '>=12.0.0'} engines: {node: '>=12.0.0'}
highlightjs-sap-abap@0.3.0:
resolution: {integrity: sha512-nSiUvEOCycjtFA3pHaTowrbAAk5+lciBHyoVkDsd6FTRBtW9sT2dt42o2jAKbXjZVUidtacdk+j0Y2xnd233Mw==}
hoist-non-react-statics@3.3.2: hoist-non-react-statics@3.3.2:
resolution: {integrity: sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==} resolution: {integrity: sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==}
@ -15442,8 +15436,6 @@ snapshots:
highlight.js@11.10.0: {} highlight.js@11.10.0: {}
highlightjs-sap-abap@0.3.0: {}
hoist-non-react-statics@3.3.2: hoist-non-react-statics@3.3.2:
dependencies: dependencies:
react-is: 16.13.1 react-is: 16.13.1