make file upload size limit configurable (#386)

This commit is contained in:
Philip Okugbe
2024-10-10 21:28:28 +01:00
committed by GitHub
parent e333eee08b
commit 384f11f2b7
14 changed files with 428 additions and 400 deletions

View File

@ -1,6 +1,8 @@
import { handleAttachmentUpload } from "@docmost/editor-ext";
import { uploadFile } from "@/features/page/services/page-service.ts";
import { notifications } from "@mantine/notifications";
import {getFileUploadSizeLimit} from "@/lib/config.ts";
import {formatBytes} from "@/lib";
export const uploadAttachmentAction = handleAttachmentUpload({
onUpload: async (file: File, pageId: string): Promise<any> => {
@ -18,10 +20,10 @@ export const uploadAttachmentAction = handleAttachmentUpload({
if (file.type.includes("image/") || file.type.includes("video/")) {
return false;
}
if (file.size / 1024 / 1024 > 50) {
if (file.size > getFileUploadSizeLimit()) {
notifications.show({
color: "red",
message: `File exceeds the 50 MB attachment limit`,
message: `File exceeds the ${formatBytes(getFileUploadSizeLimit())} attachment limit`,
});
return false;
}

View File

@ -1,6 +1,8 @@
import { handleImageUpload } from "@docmost/editor-ext";
import { uploadFile } from "@/features/page/services/page-service.ts";
import { notifications } from "@mantine/notifications";
import {getFileUploadSizeLimit} from "@/lib/config.ts";
import { formatBytes } from "@/lib";
export const uploadImageAction = handleImageUpload({
onUpload: async (file: File, pageId: string): Promise<any> => {
@ -18,10 +20,10 @@ export const uploadImageAction = handleImageUpload({
if (!file.type.includes("image/")) {
return false;
}
if (file.size / 1024 / 1024 > 50) {
if (file.size > getFileUploadSizeLimit()) {
notifications.show({
color: "red",
message: `File exceeds the 50 MB attachment limit`,
message: `File exceeds the ${formatBytes(getFileUploadSizeLimit())} attachment limit`,
});
return false;
}

View File

@ -1,6 +1,8 @@
import { handleVideoUpload } from "@docmost/editor-ext";
import { uploadFile } from "@/features/page/services/page-service.ts";
import { notifications } from "@mantine/notifications";
import {getFileUploadSizeLimit} from "@/lib/config.ts";
import {formatBytes} from "@/lib";
export const uploadVideoAction = handleVideoUpload({
onUpload: async (file: File, pageId: string): Promise<any> => {
@ -19,13 +21,14 @@ export const uploadVideoAction = handleVideoUpload({
return false;
}
if (file.size / 1024 / 1024 > 50) {
if (file.size > getFileUploadSizeLimit()) {
notifications.show({
color: "red",
message: `File exceeds the 50 MB attachment limit`,
message: `File exceeds the ${formatBytes(getFileUploadSizeLimit())} attachment limit`,
});
return false;
}
return true;
},
});

View File

@ -1,51 +1,58 @@
import bytes from "bytes";
declare global {
interface Window {
CONFIG?: Record<string, string>;
}
interface Window {
CONFIG?: Record<string, string>;
}
}
export function getAppUrl(): string {
//let appUrl = window.CONFIG?.APP_URL || process.env.APP_URL;
//let appUrl = window.CONFIG?.APP_URL || process.env.APP_URL;
// if (import.meta.env.DEV) {
// return appUrl || "http://localhost:3000";
//}
// if (import.meta.env.DEV) {
// return appUrl || "http://localhost:3000";
//}
return `${window.location.protocol}//${window.location.host}`;
return `${window.location.protocol}//${window.location.host}`;
}
export function getBackendUrl(): string {
return getAppUrl() + '/api';
return getAppUrl() + '/api';
}
export function getCollaborationUrl(): string {
const COLLAB_PATH = '/collab';
const COLLAB_PATH = '/collab';
let url = getAppUrl();
if (import.meta.env.DEV) {
url = process.env.APP_URL;
}
let url = getAppUrl();
if (import.meta.env.DEV) {
url = process.env.APP_URL;
}
const wsProtocol = url.startsWith('https') ? 'wss' : 'ws';
return `${wsProtocol}://${url.split('://')[1]}${COLLAB_PATH}`;
const wsProtocol = url.startsWith('https') ? 'wss' : 'ws';
return `${wsProtocol}://${url.split('://')[1]}${COLLAB_PATH}`;
}
export function getAvatarUrl(avatarUrl: string) {
if (!avatarUrl) {
return null;
}
if (!avatarUrl) {
return null;
}
if (avatarUrl?.startsWith('http')) {
return avatarUrl;
}
if (avatarUrl?.startsWith('http')) {
return avatarUrl;
}
return getBackendUrl() + '/attachments/img/avatar/' + avatarUrl;
return getBackendUrl() + '/attachments/img/avatar/' + avatarUrl;
}
export function getSpaceUrl(spaceSlug: string) {
return '/s/' + spaceSlug;
return '/s/' + spaceSlug;
}
export function getFileUrl(src: string) {
return src?.startsWith('/files/') ? getBackendUrl() + src : src;
return src?.startsWith('/files/') ? getBackendUrl() + src : src;
}
export function getFileUploadSizeLimit() {
const limit = window.CONFIG?.FILE_UPLOAD_SIZE_LIMIT || process?.env.FILE_UPLOAD_SIZE_LIMIT || '50mb';
return bytes(limit);
}

View File

@ -5,12 +5,13 @@ import * as path from "path";
export const envPath = path.resolve(process.cwd(), "..", "..");
export default defineConfig(({ mode }) => {
const { APP_URL } = loadEnv(mode, envPath, "");
const { APP_URL, FILE_UPLOAD_SIZE_LIMIT } = loadEnv(mode, envPath, "");
return {
define: {
"process.env": {
APP_URL,
FILE_UPLOAD_SIZE_LIMIT
},
'APP_VERSION': JSON.stringify(process.env.npm_package_version),
},

View File

@ -51,7 +51,6 @@
"@socket.io/redis-adapter": "^8.3.0",
"bcrypt": "^5.1.1",
"bullmq": "^5.12.12",
"bytes": "^3.1.2",
"class-transformer": "^0.5.1",
"class-validator": "^0.14.1",
"fix-esm": "^1.0.1",
@ -81,7 +80,6 @@
"@nestjs/schematics": "^10.1.4",
"@nestjs/testing": "^10.4.1",
"@types/bcrypt": "^5.0.2",
"@types/bytes": "^3.1.4",
"@types/debounce": "^1.2.4",
"@types/fs-extra": "^11.0.4",
"@types/jest": "^29.5.12",

View File

@ -16,4 +16,3 @@ export const inlineFileExtensions = [
'.mp4',
'.mov',
];
export const MAX_FILE_SIZE = '50MB';

View File

@ -1,308 +1,310 @@
import {
BadRequestException,
Controller,
ForbiddenException,
Get,
HttpCode,
HttpStatus,
Logger,
NotFoundException,
Param,
Post,
Req,
Res,
UseGuards,
UseInterceptors,
BadRequestException,
Controller,
ForbiddenException,
Get,
HttpCode,
HttpStatus,
Logger,
NotFoundException,
Param,
Post,
Req,
Res,
UseGuards,
UseInterceptors,
} from '@nestjs/common';
import { AttachmentService } from './services/attachment.service';
import { FastifyReply } from 'fastify';
import { FileInterceptor } from '../../common/interceptors/file.interceptor';
import {AttachmentService} from './services/attachment.service';
import {FastifyReply} from 'fastify';
import {FileInterceptor} from '../../common/interceptors/file.interceptor';
import * as bytes from 'bytes';
import { AuthUser } from '../../common/decorators/auth-user.decorator';
import { AuthWorkspace } from '../../common/decorators/auth-workspace.decorator';
import { JwtAuthGuard } from '../../common/guards/jwt-auth.guard';
import { User, Workspace } from '@docmost/db/types/entity.types';
import { StorageService } from '../../integrations/storage/storage.service';
import {AuthUser} from '../../common/decorators/auth-user.decorator';
import {AuthWorkspace} from '../../common/decorators/auth-workspace.decorator';
import {JwtAuthGuard} from '../../common/guards/jwt-auth.guard';
import {User, Workspace} from '@docmost/db/types/entity.types';
import {StorageService} from '../../integrations/storage/storage.service';
import {
getAttachmentFolderPath,
validAttachmentTypes,
getAttachmentFolderPath,
validAttachmentTypes,
} from './attachment.utils';
import { getMimeType } from '../../common/helpers';
import {getMimeType} from '../../common/helpers';
import {
AttachmentType,
inlineFileExtensions,
MAX_AVATAR_SIZE,
MAX_FILE_SIZE,
AttachmentType,
inlineFileExtensions,
MAX_AVATAR_SIZE,
} from './attachment.constants';
import {
SpaceCaslAction,
SpaceCaslSubject,
SpaceCaslAction,
SpaceCaslSubject,
} from '../casl/interfaces/space-ability.type';
import SpaceAbilityFactory from '../casl/abilities/space-ability.factory';
import {
WorkspaceCaslAction,
WorkspaceCaslSubject,
WorkspaceCaslAction,
WorkspaceCaslSubject,
} from '../casl/interfaces/workspace-ability.type';
import WorkspaceAbilityFactory from '../casl/abilities/workspace-ability.factory';
import { PageRepo } from '@docmost/db/repos/page/page.repo';
import { AttachmentRepo } from '@docmost/db/repos/attachment/attachment.repo';
import { validate as isValidUUID } from 'uuid';
import {PageRepo} from '@docmost/db/repos/page/page.repo';
import {AttachmentRepo} from '@docmost/db/repos/attachment/attachment.repo';
import {validate as isValidUUID} from 'uuid';
import {EnvironmentService} from "../../integrations/environment/environment.service";
@Controller()
export class AttachmentController {
private readonly logger = new Logger(AttachmentController.name);
private readonly logger = new Logger(AttachmentController.name);
constructor(
private readonly attachmentService: AttachmentService,
private readonly storageService: StorageService,
private readonly workspaceAbility: WorkspaceAbilityFactory,
private readonly spaceAbility: SpaceAbilityFactory,
private readonly pageRepo: PageRepo,
private readonly attachmentRepo: AttachmentRepo,
) {}
@UseGuards(JwtAuthGuard)
@HttpCode(HttpStatus.OK)
@Post('files/upload')
@UseInterceptors(FileInterceptor)
async uploadFile(
@Req() req: any,
@Res() res: FastifyReply,
@AuthUser() user: User,
@AuthWorkspace() workspace: Workspace,
) {
const maxFileSize = bytes(MAX_FILE_SIZE);
let file = null;
try {
file = await req.file({
limits: { fileSize: maxFileSize, fields: 3, files: 1 },
});
} catch (err: any) {
this.logger.error(err.message);
if (err?.statusCode === 413) {
throw new BadRequestException(
`File too large. Exceeds the ${MAX_FILE_SIZE} limit`,
);
}
}
if (!file) {
throw new BadRequestException('Failed to upload file');
}
const pageId = file.fields?.pageId?.value;
if (!pageId) {
throw new BadRequestException('PageId is required');
}
const page = await this.pageRepo.findById(pageId);
if (!page) {
throw new NotFoundException('Page not found');
}
const spaceAbility = await this.spaceAbility.createForUser(
user,
page.spaceId,
);
if (spaceAbility.cannot(SpaceCaslAction.Manage, SpaceCaslSubject.Page)) {
throw new ForbiddenException();
}
const spaceId = page.spaceId;
const attachmentId = file.fields?.attachmentId?.value;
if (attachmentId && !isValidUUID(attachmentId)) {
throw new BadRequestException('Invalid attachment id');
}
try {
const fileResponse = await this.attachmentService.uploadFile({
filePromise: file,
pageId: pageId,
spaceId: spaceId,
userId: user.id,
workspaceId: workspace.id,
attachmentId: attachmentId,
});
return res.send(fileResponse);
} catch (err: any) {
if (err?.statusCode === 413) {
const errMessage = `File too large. Exceeds the ${MAX_FILE_SIZE} limit`;
this.logger.error(errMessage);
throw new BadRequestException(errMessage);
}
this.logger.error(err);
throw new BadRequestException('Error processing file upload.');
}
}
@UseGuards(JwtAuthGuard)
@Get('/files/:fileId/:fileName')
async getFile(
@Res() res: FastifyReply,
@AuthUser() user: User,
@AuthWorkspace() workspace: Workspace,
@Param('fileId') fileId: string,
@Param('fileName') fileName?: string,
) {
if (!isValidUUID(fileId)) {
throw new NotFoundException('Invalid file id');
}
const attachment = await this.attachmentRepo.findById(fileId);
if (
!attachment ||
attachment.workspaceId !== workspace.id ||
!attachment.pageId ||
!attachment.spaceId
constructor(
private readonly attachmentService: AttachmentService,
private readonly storageService: StorageService,
private readonly workspaceAbility: WorkspaceAbilityFactory,
private readonly spaceAbility: SpaceAbilityFactory,
private readonly pageRepo: PageRepo,
private readonly attachmentRepo: AttachmentRepo,
private readonly environmentService: EnvironmentService,
) {
throw new NotFoundException();
}
const spaceAbility = await this.spaceAbility.createForUser(
user,
attachment.spaceId,
);
if (spaceAbility.cannot(SpaceCaslAction.Read, SpaceCaslSubject.Page)) {
throw new ForbiddenException();
}
try {
const fileStream = await this.storageService.read(attachment.filePath);
res.headers({
'Content-Type': attachment.mimeType,
'Cache-Control': 'public, max-age=3600',
});
if (!inlineFileExtensions.includes(attachment.fileExt)) {
res.header(
'Content-Disposition',
`attachment; filename="${encodeURIComponent(attachment.fileName)}"`,
);
}
return res.send(fileStream);
} catch (err) {
this.logger.error(err);
throw new NotFoundException('File not found');
}
}
@UseGuards(JwtAuthGuard)
@HttpCode(HttpStatus.OK)
@Post('attachments/upload-image')
@UseInterceptors(FileInterceptor)
async uploadAvatarOrLogo(
@Req() req: any,
@Res() res: FastifyReply,
@AuthUser() user: User,
@AuthWorkspace() workspace: Workspace,
) {
const maxFileSize = bytes(MAX_AVATAR_SIZE);
let file = null;
try {
file = await req.file({
limits: { fileSize: maxFileSize, fields: 3, files: 1 },
});
} catch (err: any) {
if (err?.statusCode === 413) {
throw new BadRequestException(
`File too large. Exceeds the ${MAX_AVATAR_SIZE} limit`,
);
}
}
if (!file) {
throw new BadRequestException('Invalid file upload');
}
const attachmentType = file.fields?.type?.value;
const spaceId = file.fields?.spaceId?.value;
if (!attachmentType) {
throw new BadRequestException('attachment type is required');
}
if (
!validAttachmentTypes.includes(attachmentType) ||
attachmentType === AttachmentType.File
@UseGuards(JwtAuthGuard)
@HttpCode(HttpStatus.OK)
@Post('files/upload')
@UseInterceptors(FileInterceptor)
async uploadFile(
@Req() req: any,
@Res() res: FastifyReply,
@AuthUser() user: User,
@AuthWorkspace() workspace: Workspace,
) {
throw new BadRequestException('Invalid image attachment type');
const maxFileSize = bytes(this.environmentService.getFileUploadSizeLimit());
let file = null;
try {
file = await req.file({
limits: {fileSize: maxFileSize, fields: 3, files: 1},
});
} catch (err: any) {
this.logger.error(err.message);
if (err?.statusCode === 413) {
throw new BadRequestException(
`File too large. Exceeds the ${this.environmentService.getFileUploadSizeLimit()} limit`,
);
}
}
if (!file) {
throw new BadRequestException('Failed to upload file');
}
const pageId = file.fields?.pageId?.value;
if (!pageId) {
throw new BadRequestException('PageId is required');
}
const page = await this.pageRepo.findById(pageId);
if (!page) {
throw new NotFoundException('Page not found');
}
const spaceAbility = await this.spaceAbility.createForUser(
user,
page.spaceId,
);
if (spaceAbility.cannot(SpaceCaslAction.Manage, SpaceCaslSubject.Page)) {
throw new ForbiddenException();
}
const spaceId = page.spaceId;
const attachmentId = file.fields?.attachmentId?.value;
if (attachmentId && !isValidUUID(attachmentId)) {
throw new BadRequestException('Invalid attachment id');
}
try {
const fileResponse = await this.attachmentService.uploadFile({
filePromise: file,
pageId: pageId,
spaceId: spaceId,
userId: user.id,
workspaceId: workspace.id,
attachmentId: attachmentId,
});
return res.send(fileResponse);
} catch (err: any) {
if (err?.statusCode === 413) {
const errMessage = `File too large. Exceeds the ${this.environmentService.getFileUploadSizeLimit()} limit`;
this.logger.error(errMessage);
throw new BadRequestException(errMessage);
}
this.logger.error(err);
throw new BadRequestException('Error processing file upload.');
}
}
if (attachmentType === AttachmentType.WorkspaceLogo) {
const ability = this.workspaceAbility.createForUser(user, workspace);
if (
ability.cannot(
WorkspaceCaslAction.Manage,
WorkspaceCaslSubject.Settings,
)
) {
throw new ForbiddenException();
}
}
if (attachmentType === AttachmentType.SpaceLogo) {
if (!spaceId) {
throw new BadRequestException('spaceId is required');
}
const spaceAbility = await this.spaceAbility.createForUser(user, spaceId);
if (
spaceAbility.cannot(SpaceCaslAction.Manage, SpaceCaslSubject.Settings)
) {
throw new ForbiddenException();
}
}
try {
const fileResponse = await this.attachmentService.uploadImage(
file,
attachmentType,
user.id,
workspace.id,
spaceId,
);
return res.send(fileResponse);
} catch (err: any) {
this.logger.error(err);
throw new BadRequestException('Error processing file upload.');
}
}
@Get('attachments/img/:attachmentType/:fileName')
async getLogoOrAvatar(
@Res() res: FastifyReply,
@AuthWorkspace() workspace: Workspace,
@Param('attachmentType') attachmentType: AttachmentType,
@Param('fileName') fileName?: string,
) {
if (
!validAttachmentTypes.includes(attachmentType) ||
attachmentType === AttachmentType.File
@UseGuards(JwtAuthGuard)
@Get('/files/:fileId/:fileName')
async getFile(
@Res() res: FastifyReply,
@AuthUser() user: User,
@AuthWorkspace() workspace: Workspace,
@Param('fileId') fileId: string,
@Param('fileName') fileName?: string,
) {
throw new BadRequestException('Invalid image attachment type');
if (!isValidUUID(fileId)) {
throw new NotFoundException('Invalid file id');
}
const attachment = await this.attachmentRepo.findById(fileId);
if (
!attachment ||
attachment.workspaceId !== workspace.id ||
!attachment.pageId ||
!attachment.spaceId
) {
throw new NotFoundException();
}
const spaceAbility = await this.spaceAbility.createForUser(
user,
attachment.spaceId,
);
if (spaceAbility.cannot(SpaceCaslAction.Read, SpaceCaslSubject.Page)) {
throw new ForbiddenException();
}
try {
const fileStream = await this.storageService.read(attachment.filePath);
res.headers({
'Content-Type': attachment.mimeType,
'Cache-Control': 'public, max-age=3600',
});
if (!inlineFileExtensions.includes(attachment.fileExt)) {
res.header(
'Content-Disposition',
`attachment; filename="${encodeURIComponent(attachment.fileName)}"`,
);
}
return res.send(fileStream);
} catch (err) {
this.logger.error(err);
throw new NotFoundException('File not found');
}
}
const filePath = `${getAttachmentFolderPath(attachmentType, workspace.id)}/${fileName}`;
@UseGuards(JwtAuthGuard)
@HttpCode(HttpStatus.OK)
@Post('attachments/upload-image')
@UseInterceptors(FileInterceptor)
async uploadAvatarOrLogo(
@Req() req: any,
@Res() res: FastifyReply,
@AuthUser() user: User,
@AuthWorkspace() workspace: Workspace,
) {
const maxFileSize = bytes(MAX_AVATAR_SIZE);
try {
const fileStream = await this.storageService.read(filePath);
res.headers({
'Content-Type': getMimeType(filePath),
'Cache-Control': 'public, max-age=86400',
});
return res.send(fileStream);
} catch (err) {
this.logger.error(err);
throw new NotFoundException('File not found');
let file = null;
try {
file = await req.file({
limits: {fileSize: maxFileSize, fields: 3, files: 1},
});
} catch (err: any) {
if (err?.statusCode === 413) {
throw new BadRequestException(
`File too large. Exceeds the ${MAX_AVATAR_SIZE} limit`,
);
}
}
if (!file) {
throw new BadRequestException('Invalid file upload');
}
const attachmentType = file.fields?.type?.value;
const spaceId = file.fields?.spaceId?.value;
if (!attachmentType) {
throw new BadRequestException('attachment type is required');
}
if (
!validAttachmentTypes.includes(attachmentType) ||
attachmentType === AttachmentType.File
) {
throw new BadRequestException('Invalid image attachment type');
}
if (attachmentType === AttachmentType.WorkspaceLogo) {
const ability = this.workspaceAbility.createForUser(user, workspace);
if (
ability.cannot(
WorkspaceCaslAction.Manage,
WorkspaceCaslSubject.Settings,
)
) {
throw new ForbiddenException();
}
}
if (attachmentType === AttachmentType.SpaceLogo) {
if (!spaceId) {
throw new BadRequestException('spaceId is required');
}
const spaceAbility = await this.spaceAbility.createForUser(user, spaceId);
if (
spaceAbility.cannot(SpaceCaslAction.Manage, SpaceCaslSubject.Settings)
) {
throw new ForbiddenException();
}
}
try {
const fileResponse = await this.attachmentService.uploadImage(
file,
attachmentType,
user.id,
workspace.id,
spaceId,
);
return res.send(fileResponse);
} catch (err: any) {
this.logger.error(err);
throw new BadRequestException('Error processing file upload.');
}
}
@Get('attachments/img/:attachmentType/:fileName')
async getLogoOrAvatar(
@Res() res: FastifyReply,
@AuthWorkspace() workspace: Workspace,
@Param('attachmentType') attachmentType: AttachmentType,
@Param('fileName') fileName?: string,
) {
if (
!validAttachmentTypes.includes(attachmentType) ||
attachmentType === AttachmentType.File
) {
throw new BadRequestException('Invalid image attachment type');
}
const filePath = `${getAttachmentFolderPath(attachmentType, workspace.id)}/${fileName}`;
try {
const fileStream = await this.storageService.read(filePath);
res.headers({
'Content-Type': getMimeType(filePath),
'Cache-Control': 'public, max-age=86400',
});
return res.send(fileStream);
} catch (err) {
this.logger.error(err);
throw new NotFoundException('File not found');
}
}
}
}

View File

@ -43,6 +43,11 @@ export class EnvironmentService {
return this.configService.get<string>('STORAGE_DRIVER', 'local');
}
getFileUploadSizeLimit(): string {
return this.configService.get<string>('FILE_UPLOAD_SIZE_LIMIT', '50mb');
}
getAwsS3AccessKeyId(): string {
return this.configService.get<string>('AWS_S3_ACCESS_KEY_ID');
}

View File

@ -21,7 +21,6 @@ import {
import { FileInterceptor } from '../../common/interceptors/file.interceptor';
import * as bytes from 'bytes';
import * as path from 'path';
import { MAX_FILE_SIZE } from '../../core/attachment/attachment.constants';
import { ImportService } from './import.service';
import { AuthWorkspace } from '../../common/decorators/auth-workspace.decorator';
@ -45,7 +44,7 @@ export class ImportController {
) {
const validFileExtensions = ['.md', '.html'];
const maxFileSize = bytes(MAX_FILE_SIZE);
const maxFileSize = bytes('100mb');
let file = null;
try {
@ -56,7 +55,7 @@ export class ImportController {
this.logger.error(err.message);
if (err?.statusCode === 413) {
throw new BadRequestException(
`File too large. Exceeds the ${MAX_FILE_SIZE} limit`,
`File too large. Exceeds the 100mb import limit`,
);
}
}

View File

@ -33,6 +33,7 @@ export class StaticModule implements OnModuleInit {
ENV: this.environmentService.getNodeEnv(),
APP_URL: this.environmentService.getAppUrl(),
IS_CLOUD: this.environmentService.isCloud(),
FILE_UPLOAD_SIZE_LIMIT: this.environmentService.getFileUploadSizeLimit()
};
const windowScriptContent = `<script>window.CONFIG=${JSON.stringify(configString)};</script>`;