From 7f38d3bffe246dcca7fdcfa8727af6c4cc483a8f Mon Sep 17 00:00:00 2001 From: Philipinho <16838612+Philipinho@users.noreply.github.com> Date: Sat, 23 Sep 2023 13:54:11 +0100 Subject: [PATCH] storage module --- server/.env.example | 23 ++++ server/package.json | 6 + .../src/core/attachment/attachment.module.ts | 7 ++ .../attachment/attachment.service.spec.ts | 18 +++ .../src/core/attachment/attachment.service.ts | 4 + server/src/core/core.module.ts | 14 ++- .../storage/constants/storage.constants.ts | 2 + server/src/core/storage/drivers/index.ts | 2 + .../src/core/storage/drivers/local.driver.ts | 71 +++++++++++ server/src/core/storage/drivers/s3.driver.ts | 115 ++++++++++++++++++ server/src/core/storage/interfaces/index.ts | 2 + .../interfaces/storage-driver.interface.ts | 19 +++ .../storage/interfaces/storage.interface.ts | 33 +++++ .../storage/providers/storage.provider.ts | 66 ++++++++++ server/src/core/storage/storage.module.ts | 24 ++++ .../src/core/storage/storage.service.spec.ts | 18 +++ server/src/core/storage/storage.service.ts | 34 ++++++ server/src/core/storage/storage.utils.ts | 10 ++ server/src/environment/environment.service.ts | 36 ++++++ 19 files changed, 503 insertions(+), 1 deletion(-) create mode 100644 server/.env.example create mode 100644 server/src/core/attachment/attachment.module.ts create mode 100644 server/src/core/attachment/attachment.service.spec.ts create mode 100644 server/src/core/attachment/attachment.service.ts create mode 100644 server/src/core/storage/constants/storage.constants.ts create mode 100644 server/src/core/storage/drivers/index.ts create mode 100644 server/src/core/storage/drivers/local.driver.ts create mode 100644 server/src/core/storage/drivers/s3.driver.ts create mode 100644 server/src/core/storage/interfaces/index.ts create mode 100644 server/src/core/storage/interfaces/storage-driver.interface.ts create mode 100644 server/src/core/storage/interfaces/storage.interface.ts create mode 100644 server/src/core/storage/providers/storage.provider.ts create mode 100644 server/src/core/storage/storage.module.ts create mode 100644 server/src/core/storage/storage.service.spec.ts create mode 100644 server/src/core/storage/storage.service.ts create mode 100644 server/src/core/storage/storage.utils.ts diff --git a/server/.env.example b/server/.env.example new file mode 100644 index 00000000..ed0b93f1 --- /dev/null +++ b/server/.env.example @@ -0,0 +1,23 @@ +PORT=3001 +DEBUG_MODE=true +NODE_ENV=production + +JWT_SECRET_KEY=ba8642edbed7f6c450e46875e8c835c7e417031abe1f7b03f3e56bb7481706d8 +JWT_TOKEN_EXPIRES_IN=30d + +DATABASE_URL="postgresql://postgres:password@localhost:5432/dc?schema=public" + +# local | s3 +STORAGE_DRIVER=local + +# local config +LOCAL_STORAGE_PATH=/storage + +# S3 Config +AWS_S3_ACCESS_KEY_ID= +AWS_S3_SECRET_ACCESS_KEY= +AWS_S3_REGION= +AWS_S3_BUCKET= +AWS_S3_ENDPOINT= +AWS_S3_URL= +AWS_S3_USE_PATH_STYLE_ENDPOINT=false diff --git a/server/package.json b/server/package.json index ec1f3e4a..2bcc8ab2 100644 --- a/server/package.json +++ b/server/package.json @@ -27,6 +27,8 @@ "migration:show": "npm run typeorm migration:show" }, "dependencies": { + "@aws-sdk/client-s3": "^3.417.0", + "@aws-sdk/s3-request-presigner": "^3.418.0", "@hocuspocus/server": "^2.5.0", "@hocuspocus/transformer": "^2.5.0", "@nestjs/common": "^10.0.0", @@ -42,6 +44,8 @@ "class-transformer": "^0.5.1", "class-validator": "^0.14.0", "fastify": "^4.22.2", + "fs-extra": "^11.1.1", + "mime-types": "^2.1.35", "pg": "^8.11.3", "reflect-metadata": "^0.1.13", "rxjs": "^7.8.1", @@ -55,7 +59,9 @@ "@nestjs/testing": "^10.0.0", "@types/bcrypt": "^5.0.0", "@types/express": "^4.17.17", + "@types/fs-extra": "^11.0.2", "@types/jest": "^29.5.2", + "@types/mime-types": "^2.1.1", "@types/node": "^20.3.1", "@types/supertest": "^2.0.12", "@types/uuid": "^9.0.3", diff --git a/server/src/core/attachment/attachment.module.ts b/server/src/core/attachment/attachment.module.ts new file mode 100644 index 00000000..856f9062 --- /dev/null +++ b/server/src/core/attachment/attachment.module.ts @@ -0,0 +1,7 @@ +import { Module } from '@nestjs/common'; +import { AttachmentService } from './attachment.service'; + +@Module({ + providers: [AttachmentService], +}) +export class AttachmentModule {} diff --git a/server/src/core/attachment/attachment.service.spec.ts b/server/src/core/attachment/attachment.service.spec.ts new file mode 100644 index 00000000..5b4a892b --- /dev/null +++ b/server/src/core/attachment/attachment.service.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { AttachmentService } from './attachment.service'; + +describe('AttachmentService', () => { + let service: AttachmentService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [AttachmentService], + }).compile(); + + service = module.get(AttachmentService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/server/src/core/attachment/attachment.service.ts b/server/src/core/attachment/attachment.service.ts new file mode 100644 index 00000000..7834c7b4 --- /dev/null +++ b/server/src/core/attachment/attachment.service.ts @@ -0,0 +1,4 @@ +import { Injectable } from '@nestjs/common'; + +@Injectable() +export class AttachmentService {} diff --git a/server/src/core/core.module.ts b/server/src/core/core.module.ts index 2389bbf1..e823c818 100644 --- a/server/src/core/core.module.ts +++ b/server/src/core/core.module.ts @@ -3,8 +3,20 @@ import { UserModule } from './user/user.module'; import { AuthModule } from './auth/auth.module'; import { WorkspaceModule } from './workspace/workspace.module'; import { PageModule } from './page/page.module'; +import { StorageModule } from './storage/storage.module'; +import { AttachmentModule } from './attachment/attachment.module'; +import { EnvironmentModule } from '../environment/environment.module'; @Module({ - imports: [UserModule, AuthModule, WorkspaceModule, PageModule], + imports: [ + UserModule, + AuthModule, + WorkspaceModule, + PageModule, + StorageModule.forRootAsync({ + imports: [EnvironmentModule], + }), + AttachmentModule, + ], }) export class CoreModule {} diff --git a/server/src/core/storage/constants/storage.constants.ts b/server/src/core/storage/constants/storage.constants.ts new file mode 100644 index 00000000..ae11bee4 --- /dev/null +++ b/server/src/core/storage/constants/storage.constants.ts @@ -0,0 +1,2 @@ +export const STORAGE_DRIVER_TOKEN = 'STORAGE_DRIVER_TOKEN'; +export const STORAGE_CONFIG_TOKEN = 'STORAGE_CONFIG_TOKEN'; diff --git a/server/src/core/storage/drivers/index.ts b/server/src/core/storage/drivers/index.ts new file mode 100644 index 00000000..02ab4b30 --- /dev/null +++ b/server/src/core/storage/drivers/index.ts @@ -0,0 +1,2 @@ +export { LocalDriver } from './local.driver'; +export { S3Driver } from './s3.driver'; diff --git a/server/src/core/storage/drivers/local.driver.ts b/server/src/core/storage/drivers/local.driver.ts new file mode 100644 index 00000000..2a16f625 --- /dev/null +++ b/server/src/core/storage/drivers/local.driver.ts @@ -0,0 +1,71 @@ +import { + StorageDriver, + LocalStorageConfig, + StorageOption, +} from '../interfaces'; +import { join } from 'path'; +import * as fs from 'fs-extra'; + +export class LocalDriver implements StorageDriver { + private readonly config: LocalStorageConfig; + + constructor(config: LocalStorageConfig) { + this.config = config; + } + + private _fullPath(filePath: string): string { + return join(this.config.storagePath, filePath); + } + + async upload(filePath: string, file: Buffer): Promise { + try { + await fs.outputFile(this._fullPath(filePath), file); + } catch (error) { + throw new Error(`Failed to upload file: ${error.message}`); + } + } + + async read(filePath: string): Promise { + try { + return await fs.readFile(this._fullPath(filePath)); + } catch (error) { + throw new Error(`Failed to read file: ${error.message}`); + } + } + + async exists(filePath: string): Promise { + try { + return await fs.pathExists(this._fullPath(filePath)); + } catch (error) { + throw new Error(`Failed to check file existence: ${error.message}`); + } + } + + async getSignedUrl(filePath: string, expireIn: number): Promise { + throw new Error('Signed URLs are not supported for local storage.'); + } + + getUrl(filePath: string): string { + return this._fullPath(filePath); + } + + async delete(filePath: string): Promise { + try { + await fs.remove(this._fullPath(filePath)); + } catch (error) { + throw new Error(`Failed to delete file: ${error.message}`); + } + } + + getDriver(): typeof fs { + return fs; + } + + getDriverName(): string { + return StorageOption.LOCAL; + } + + getConfig(): Record { + return this.config; + } +} diff --git a/server/src/core/storage/drivers/s3.driver.ts b/server/src/core/storage/drivers/s3.driver.ts new file mode 100644 index 00000000..06a28784 --- /dev/null +++ b/server/src/core/storage/drivers/s3.driver.ts @@ -0,0 +1,115 @@ +import { S3StorageConfig, StorageDriver, StorageOption } from '../interfaces'; +import { + DeleteObjectCommand, + GetObjectCommand, + HeadObjectCommand, + NoSuchKey, + PutObjectCommand, + S3Client, +} from '@aws-sdk/client-s3'; +import { streamToBuffer } from '../storage.utils'; +import { Readable } from 'stream'; +import * as mime from 'mime-types'; +import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; + +export class S3Driver implements StorageDriver { + private readonly s3Client: S3Client; + private readonly config: S3StorageConfig; + + constructor(config: S3StorageConfig) { + this.config = config; + this.s3Client = new S3Client(config); + } + + async upload(filePath: string, file: Buffer): Promise { + try { + const contentType = + mime.contentType(filePath) || 'application/octet-stream'; + + const command = new PutObjectCommand({ + Bucket: this.config.bucket, + Key: filePath, + Body: file, + ContentType: contentType, + // ACL: "public-read", + }); + + await this.s3Client.send(command); + // we can get the path from location + + console.log(`File uploaded successfully: ${filePath}`); + } catch (error) { + throw new Error(`Failed to upload file: ${error.message}`); + } + } + + async read(filePath: string): Promise { + try { + const command = new GetObjectCommand({ + Bucket: this.config.bucket, + Key: filePath, + }); + + const response = await this.s3Client.send(command); + + return streamToBuffer(response.Body as Readable); + } catch (error) { + throw new Error(`Failed to read file from S3: ${error.message}`); + } + } + + async exists(filePath: string): Promise { + try { + const command = new HeadObjectCommand({ + Bucket: this.config.bucket, + Key: filePath, + }); + + await this.s3Client.send(command); + return true; + } catch (err) { + if (err instanceof NoSuchKey) { + return false; + } + throw err; + } + } + getUrl(filePath: string): string { + return `${this.config.endpoint}/${this.config.bucket}/${filePath}`; + } + + async getSignedUrl(filePath: string, expiresIn: number): Promise { + const command = new GetObjectCommand({ + Bucket: this.config.bucket, + Key: filePath, + }); + return await getSignedUrl(this.s3Client, command, { expiresIn }); + } + + async delete(filePath: string): Promise { + try { + const command = new DeleteObjectCommand({ + Bucket: this.config.bucket, + Key: filePath, + }); + + await this.s3Client.send(command); + } catch (err) { + throw new Error( + `Error deleting file ${filePath} from S3. ${err.message}`, + ); + } + } + + getDriver(): S3Client { + return this.s3Client; + } + + getDriverName(): string { + return StorageOption.S3; + } + + getConfig(): Record { + return this.config; + } +} diff --git a/server/src/core/storage/interfaces/index.ts b/server/src/core/storage/interfaces/index.ts new file mode 100644 index 00000000..c50d62bf --- /dev/null +++ b/server/src/core/storage/interfaces/index.ts @@ -0,0 +1,2 @@ +export * from './storage-driver.interface'; +export * from './storage.interface'; diff --git a/server/src/core/storage/interfaces/storage-driver.interface.ts b/server/src/core/storage/interfaces/storage-driver.interface.ts new file mode 100644 index 00000000..419587f4 --- /dev/null +++ b/server/src/core/storage/interfaces/storage-driver.interface.ts @@ -0,0 +1,19 @@ +export interface StorageDriver { + upload(filePath: string, file: Buffer): Promise; + + read(filePath: string): Promise; + + exists(filePath: string): Promise; + + getUrl(filePath: string): string; + + getSignedUrl(filePath: string, expireIn: number): Promise; + + delete(filePath: string): Promise; + + getDriver(): any; + + getDriverName(): string; + + getConfig(): Record; +} diff --git a/server/src/core/storage/interfaces/storage.interface.ts b/server/src/core/storage/interfaces/storage.interface.ts new file mode 100644 index 00000000..48c68491 --- /dev/null +++ b/server/src/core/storage/interfaces/storage.interface.ts @@ -0,0 +1,33 @@ +import { S3ClientConfig } from '@aws-sdk/client-s3'; + +export enum StorageOption { + LOCAL = 'local', + S3 = 's3', +} + +export type StorageConfig = + | { driver: StorageOption.LOCAL; config: LocalStorageConfig } + | { driver: StorageOption.S3; config: S3StorageConfig }; + +export interface LocalStorageConfig { + storagePath: string; +} + +export interface S3StorageConfig + extends Omit { + endpoint: string; // Enforce endpoint + bucket: string; // Enforce bucket + baseUrl?: string; // Optional CDN URL for assets +} + +export interface StorageOptions { + disk: StorageConfig; +} + +export interface StorageOptionsFactory { + createStorageOptions(): Promise | StorageConfig; +} + +export interface StorageModuleOptions { + imports?: any[]; +} diff --git a/server/src/core/storage/providers/storage.provider.ts b/server/src/core/storage/providers/storage.provider.ts new file mode 100644 index 00000000..9bd33269 --- /dev/null +++ b/server/src/core/storage/providers/storage.provider.ts @@ -0,0 +1,66 @@ +import { + STORAGE_CONFIG_TOKEN, + STORAGE_DRIVER_TOKEN, +} from '../constants/storage.constants'; +import { EnvironmentService } from '../../../environment/environment.service'; +import { + LocalStorageConfig, + S3StorageConfig, + StorageConfig, + StorageDriver, + StorageOption, +} from '../interfaces'; +import { LocalDriver, S3Driver } from '../drivers'; + +function createStorageDriver(disk: StorageConfig): StorageDriver { + switch (disk.driver) { + case StorageOption.LOCAL: + return new LocalDriver(disk.config as LocalStorageConfig); + case StorageOption.S3: + return new S3Driver(disk.config as S3StorageConfig); + default: + throw new Error(`Unknown storage driver`); + } +} + +export const storageDriverConfigProvider = { + provide: STORAGE_CONFIG_TOKEN, + useFactory: async (environmentService: EnvironmentService) => { + const driver = environmentService.getStorageDriver(); + + if (driver === StorageOption.LOCAL) { + return { + driver, + config: { + storagePath: + process.cwd() + '/' + environmentService.getLocalStoragePath(), + }, + }; + } + + if (driver === StorageOption.S3) { + return { + driver, + config: { + region: environmentService.getAwsS3Region(), + endpoint: environmentService.getAwsS3Endpoint(), + bucket: environmentService.getAwsS3Bucket(), + credentials: { + accessKeyId: environmentService.getAwsS3AccessKeyId(), + secretAccessKey: environmentService.getAwsS3SecretAccessKey(), + }, + }, + }; + } + + throw new Error(`Unknown storage driver: ${driver}`); + }, + + inject: [EnvironmentService], +}; + +export const storageDriverProvider = { + provide: STORAGE_DRIVER_TOKEN, + useFactory: (config) => createStorageDriver(config), + inject: [STORAGE_CONFIG_TOKEN], +}; diff --git a/server/src/core/storage/storage.module.ts b/server/src/core/storage/storage.module.ts new file mode 100644 index 00000000..fe9404ca --- /dev/null +++ b/server/src/core/storage/storage.module.ts @@ -0,0 +1,24 @@ +import { DynamicModule, Global, Module } from '@nestjs/common'; +import { StorageModuleOptions } from './interfaces'; +import { StorageService } from './storage.service'; +import { + storageDriverConfigProvider, + storageDriverProvider, +} from './providers/storage.provider'; + +@Global() +@Module({}) +export class StorageModule { + static forRootAsync(options: StorageModuleOptions): DynamicModule { + return { + module: StorageModule, + imports: options.imports || [], + providers: [ + storageDriverConfigProvider, + storageDriverProvider, + StorageService, + ], + exports: [StorageService], + }; + } +} diff --git a/server/src/core/storage/storage.service.spec.ts b/server/src/core/storage/storage.service.spec.ts new file mode 100644 index 00000000..0b277788 --- /dev/null +++ b/server/src/core/storage/storage.service.spec.ts @@ -0,0 +1,18 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { StorageService } from './storage.service'; + +describe('StorageService', () => { + let service: StorageService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [StorageService], + }).compile(); + + service = module.get(StorageService); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); +}); diff --git a/server/src/core/storage/storage.service.ts b/server/src/core/storage/storage.service.ts new file mode 100644 index 00000000..a477d370 --- /dev/null +++ b/server/src/core/storage/storage.service.ts @@ -0,0 +1,34 @@ +import { Inject, Injectable } from '@nestjs/common'; +import { STORAGE_DRIVER_TOKEN } from './constants/storage.constants'; +import { StorageDriver } from './interfaces'; + +@Injectable() +export class StorageService { + constructor( + @Inject(STORAGE_DRIVER_TOKEN) private storageDriver: StorageDriver, + ) {} + + async upload(filePath: string, fileContent: Buffer | any) { + await this.storageDriver.upload(filePath, fileContent); + } + + async read(filePath: string): Promise { + return this.storageDriver.read(filePath); + } + + async exists(filePath: string): Promise { + return this.storageDriver.exists(filePath); + } + + async signedUrl(path: string, expireIn: number): Promise { + return this.storageDriver.getSignedUrl(path, expireIn); + } + + url(filePath: string): string { + return this.storageDriver.getUrl(filePath); + } + + async delete(filePath: string): Promise { + await this.storageDriver.delete(filePath); + } +} diff --git a/server/src/core/storage/storage.utils.ts b/server/src/core/storage/storage.utils.ts new file mode 100644 index 00000000..96bf351b --- /dev/null +++ b/server/src/core/storage/storage.utils.ts @@ -0,0 +1,10 @@ +import { Readable } from 'stream'; + +export function streamToBuffer(readableStream: Readable): Promise { + return new Promise((resolve, reject) => { + const chunks: Uint8Array[] = []; + readableStream.on('data', (chunk) => chunks.push(chunk)); + readableStream.on('end', () => resolve(Buffer.concat(chunks))); + readableStream.on('error', reject); + }); +} diff --git a/server/src/environment/environment.service.ts b/server/src/environment/environment.service.ts index 66245077..2c28ce80 100644 --- a/server/src/environment/environment.service.ts +++ b/server/src/environment/environment.service.ts @@ -23,4 +23,40 @@ export class EnvironmentService { getJwtTokenExpiresIn(): string { return this.configService.get('JWT_TOKEN_EXPIRES_IN'); } + + getStorageDriver(): string { + return this.configService.get('STORAGE_DRIVER'); + } + + getLocalStoragePath(): string { + return this.configService.get('LOCAL_STORAGE_PATH'); + } + + getAwsS3AccessKeyId(): string { + return this.configService.get('AWS_S3_ACCESS_KEY_ID'); + } + + getAwsS3SecretAccessKey(): string { + return this.configService.get('AWS_S3_SECRET_ACCESS_KEY'); + } + + getAwsS3Region(): string { + return this.configService.get('AWS_S3_REGION'); + } + + getAwsS3Bucket(): string { + return this.configService.get('AWS_S3_BUCKET'); + } + + getAwsS3Endpoint(): string { + return this.configService.get('AWS_S3_ENDPOINT'); + } + + getAwsS3Url(): string { + return this.configService.get('AWS_S3_URL'); + } + + getAwsS3UsePathStyleEndpoint(): boolean { + return this.configService.get('AWS_S3_USE_PATH_STYLE_ENDPOINT'); + } }