* integrate websocket redis adapter
* use APP_SECRET for jwt signing
* auto migrate database on startup in production
* add updatedAt to update db operations
* create enterprise ee package directory
* fix comment editor focus
* other fixes
This commit is contained in:
Philipinho
2024-06-07 17:29:34 +01:00
parent eef9081aaf
commit 38ef610e5e
36 changed files with 541 additions and 166 deletions

View File

@ -0,0 +1,5 @@
node_modules
.git
.gitignore
dist
data

View File

@ -1,9 +1,9 @@
/storage
.env
package-lock.json
# compiled output
/dist
/node_modules
/data
# Logs
logs

View File

@ -8,10 +8,10 @@
"scripts": {
"build": "nest build",
"format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
"start": "nest start",
"start:dev": "nest start --watch",
"start:debug": "nest start --debug --watch",
"start:prod": "node dist/main",
"start": "cross-env NODE_ENV=development nest start",
"start:dev": "cross-env NODE_ENV=development nest start --watch",
"start:debug": "cross-env NODE_ENV=development nest start --debug --watch",
"start:prod": "cross-env NODE_ENV=production node dist/main",
"email:dev": "email dev -p 5019 -d ./src/integrations/transactional/emails",
"migration:create": "tsx src/database/migrate.ts create",
"migration:up": "tsx src/database/migrate.ts up",
@ -37,6 +37,7 @@
"@nestjs/common": "^10.3.9",
"@nestjs/config": "^3.2.2",
"@nestjs/core": "^10.3.9",
"@nestjs/event-emitter": "^2.0.4",
"@nestjs/jwt": "^10.2.0",
"@nestjs/mapped-types": "^2.0.5",
"@nestjs/passport": "^10.0.3",
@ -45,6 +46,7 @@
"@nestjs/websockets": "^10.3.9",
"@react-email/components": "0.0.19",
"@react-email/render": "^0.0.15",
"@socket.io/redis-adapter": "^8.3.0",
"@types/pg": "^8.11.6",
"bcrypt": "^5.1.1",
"bullmq": "^5.7.14",
@ -64,6 +66,8 @@
"pg": "^8.11.5",
"pg-tsquery": "^8.4.2",
"postmark": "^4.0.2",
"react": "^18.3.1",
"redis": "^4.6.14",
"reflect-metadata": "^0.2.2",
"rxjs": "^7.8.1",
"sanitize-filename-ts": "^1.0.2",

View File

@ -44,7 +44,7 @@ export class TokenService {
async verifyJwt(token: string) {
return this.jwtService.verifyAsync(token, {
secret: this.environmentService.getJwtSecret(),
secret: this.environmentService.getAppSecret(),
});
}
}

View File

@ -20,7 +20,7 @@ export class JwtStrategy extends PassportStrategy(Strategy, 'jwt') {
super({
jwtFromRequest: ExtractJwt.fromAuthHeaderAsBearerToken(),
ignoreExpiration: false,
secretOrKey: environmentService.getJwtSecret(),
secretOrKey: environmentService.getAppSecret(),
passReqToCallback: true,
});
}

View File

@ -21,6 +21,7 @@ import { PageHistoryRepo } from './repos/page/page-history.repo';
import { AttachmentRepo } from './repos/attachment/attachment.repo';
import { KyselyDB } from '@docmost/db/types/kysely.types';
import * as process from 'node:process';
import { MigrationService } from '@docmost/db/services/migration.service';
// https://github.com/brianc/node-postgres/issues/811
types.setTypeParser(types.builtins.INT8, (val) => Number(val));
@ -39,7 +40,7 @@ types.setTypeParser(types.builtins.INT8, (val) => Number(val));
}),
plugins: [new CamelCasePlugin()],
log: (event: LogEvent) => {
if (environmentService.getEnv() !== 'development') return;
if (environmentService.getNodeEnv() !== 'development') return;
if (event.level === 'query') {
// console.log(event.query.sql);
//if (event.query.parameters.length > 0) {
@ -52,6 +53,7 @@ types.setTypeParser(types.builtins.INT8, (val) => Number(val));
}),
],
providers: [
MigrationService,
WorkspaceRepo,
UserRepo,
GroupRepo,
@ -79,10 +81,18 @@ types.setTypeParser(types.builtins.INT8, (val) => Number(val));
export class DatabaseModule implements OnModuleDestroy, OnApplicationBootstrap {
private readonly logger = new Logger(DatabaseModule.name);
constructor(@InjectKysely() private readonly db: KyselyDB) {}
constructor(
@InjectKysely() private readonly db: KyselyDB,
private readonly migrationService: MigrationService,
private readonly environmentService: EnvironmentService,
) {}
async onApplicationBootstrap() {
await this.establishConnection();
if (this.environmentService.getNodeEnv() === 'production') {
await this.migrationService.migrateToLatest();
}
}
async onModuleDestroy(): Promise<void> {
@ -110,7 +120,7 @@ export class DatabaseModule implements OnModuleDestroy, OnApplicationBootstrap {
if (i < retryAttempts - 1) {
this.logger.log(
`Retrying [${i + 1}/${retryAttempts}] in ${retryDelay} ms`,
`Retrying [${i + 1}/${retryAttempts}] in ${retryDelay / 1000} seconds`,
);
await new Promise((resolve) => setTimeout(resolve, retryDelay));
} else {

View File

@ -16,9 +16,7 @@ export async function up(db: Kysely<any>): Promise<void> {
.addColumn('default_role', 'varchar', (col) =>
col.defaultTo(UserRole.MEMBER).notNull(),
)
.addColumn('allowed_email_domains', sql`varchar[]`, (col) =>
col.defaultTo('{}'),
)
.addColumn('email_domains', sql`varchar[]`, (col) => col.defaultTo('{}'))
.addColumn('default_space_id', 'uuid', (col) => col)
.addColumn('created_at', 'timestamptz', (col) =>
col.notNull().defaultTo(sql`now()`),

View File

@ -11,7 +11,7 @@ export async function up(db: Kysely<any>): Promise<void> {
.addColumn('email_verified_at', 'timestamptz', (col) => col)
.addColumn('password', 'varchar', (col) => col)
.addColumn('avatar_url', 'varchar', (col) => col)
.addColumn('role', 'varchar', (col) => col.notNull())
.addColumn('role', 'varchar', (col) => col)
.addColumn('invited_by_id', 'uuid', (col) =>
col.references('users.id').onDelete('set null'),
)

View File

@ -12,7 +12,7 @@ export async function up(db: Kysely<any>): Promise<void> {
.addColumn('slug', 'varchar', (col) => col.notNull())
.addColumn('logo', 'varchar', (col) => col)
.addColumn('visibility', 'varchar', (col) =>
col.defaultTo(SpaceVisibility.OPEN).notNull(),
col.defaultTo(SpaceVisibility.PRIVATE).notNull(),
)
.addColumn('default_role', 'varchar', (col) =>
col.defaultTo(SpaceRole.WRITER).notNull(),

View File

@ -19,6 +19,7 @@ export async function up(db: Kysely<any>): Promise<void> {
.addColumn('workspace_id', 'uuid', (col) =>
col.references('workspaces.id').notNull(),
)
.addColumn('resolved_at', 'timestamptz', (col) => col)
.addColumn('created_at', 'timestamptz', (col) =>
col.notNull().defaultTo(sql`now()`),
)

View File

@ -54,7 +54,7 @@ export class GroupRepo {
): Promise<void> {
await this.db
.updateTable('groups')
.set(updatableGroup)
.set({ ...updatableGroup, updatedAt: new Date() })
.where('id', '=', groupId)
.where('workspaceId', '=', workspaceId)
.execute();

View File

@ -17,8 +17,13 @@ import { DB } from '@docmost/db/types/db';
export class PageHistoryRepo {
constructor(@InjectKysely() private readonly db: KyselyDB) {}
async findById(pageHistoryId: string): Promise<PageHistory> {
return await this.db
async findById(
pageHistoryId: string,
trx?: KyselyTransaction,
): Promise<PageHistory> {
const db = dbOrTx(this.db, trx);
return await db
.selectFrom('pageHistory')
.selectAll()
.select((eb) => this.withLastUpdatedBy(eb))
@ -38,18 +43,21 @@ export class PageHistoryRepo {
.executeTakeFirst();
}
async saveHistory(page: Page): Promise<void> {
await this.insertPageHistory({
pageId: page.id,
slugId: page.slugId,
title: page.title,
content: page.content,
icon: page.icon,
coverPhoto: page.coverPhoto,
lastUpdatedById: page.lastUpdatedById ?? page.creatorId,
spaceId: page.spaceId,
workspaceId: page.workspaceId,
});
async saveHistory(page: Page, trx?: KyselyTransaction): Promise<void> {
await this.insertPageHistory(
{
pageId: page.id,
slugId: page.slugId,
title: page.title,
content: page.content,
icon: page.icon,
coverPhoto: page.coverPhoto,
lastUpdatedById: page.lastUpdatedById ?? page.creatorId,
spaceId: page.spaceId,
workspaceId: page.workspaceId,
},
trx,
);
}
async findPageHistoryByPageId(pageId: string, pagination: PaginationOptions) {
@ -68,6 +76,18 @@ export class PageHistoryRepo {
return result;
}
async findPageLastHistory(pageId: string, trx?: KyselyTransaction) {
const db = dbOrTx(this.db, trx);
return await db
.selectFrom('pageHistory')
.selectAll()
.where('pageId', '=', pageId)
.limit(1)
.orderBy('createdAt', 'desc')
.executeTakeFirst();
}
withLastUpdatedBy(eb: ExpressionBuilder<DB, 'pageHistory'>) {
return jsonObjectFrom(
eb

View File

@ -46,9 +46,13 @@ export class PageRepo {
includeContent?: boolean;
includeYdoc?: boolean;
includeSpace?: boolean;
withLock?: boolean;
trx?: KyselyTransaction;
},
): Promise<Page> {
let query = this.db
const db = dbOrTx(this.db, opts?.trx);
let query = db
.selectFrom('pages')
.select(this.baseFields)
.$if(opts?.includeContent, (qb) => qb.select('content'))
@ -58,6 +62,10 @@ export class PageRepo {
query = query.select((eb) => this.withSpace(eb));
}
if (opts?.withLock && opts?.trx) {
query = query.forUpdate();
}
if (isValidUUID(pageId)) {
query = query.where('id', '=', pageId);
} else {
@ -73,7 +81,9 @@ export class PageRepo {
trx?: KyselyTransaction,
) {
const db = dbOrTx(this.db, trx);
let query = db.updateTable('pages').set(updatablePage);
let query = db
.updateTable('pages')
.set({ ...updatablePage, updatedAt: new Date() });
if (isValidUUID(pageId)) {
query = query.where('id', '=', pageId);

View File

@ -77,7 +77,7 @@ export class SpaceRepo {
const db = dbOrTx(this.db, trx);
return db
.updateTable('spaces')
.set(updatableSpace)
.set({ ...updatableSpace, updatedAt: new Date() })
.where('id', '=', spaceId)
.where('workspaceId', '=', workspaceId)
.returningAll()

View File

@ -80,7 +80,7 @@ export class UserRepo {
return await db
.updateTable('users')
.set(updatableUser)
.set({ ...updatableUser, updatedAt: new Date() })
.where('id', '=', userId)
.where('workspaceId', '=', workspaceId)
.execute();

View File

@ -53,7 +53,7 @@ export class WorkspaceRepo {
const db = dbOrTx(this.db, trx);
return db
.updateTable('workspaces')
.set(updatableWorkspace)
.set({ ...updatableWorkspace, updatedAt: new Date() })
.where('id', '=', workspaceId)
.execute();
}

View File

@ -0,0 +1,47 @@
import { Injectable, Logger } from '@nestjs/common';
import * as path from 'path';
import { promises as fs } from 'fs';
import { Migrator, FileMigrationProvider } from 'kysely';
import { InjectKysely } from 'nestjs-kysely';
import { KyselyDB } from '@docmost/db/types/kysely.types';
@Injectable()
export class MigrationService {
private readonly logger = new Logger(`Database${MigrationService.name}`);
constructor(@InjectKysely() private readonly db: KyselyDB) {}
async migrateToLatest(): Promise<void> {
const migrator = new Migrator({
db: this.db,
provider: new FileMigrationProvider({
fs,
path,
migrationFolder: path.join(__dirname, '..', 'migrations'),
}),
});
const { error, results } = await migrator.migrateToLatest();
if (results && results.length === 0) {
this.logger.log('No pending database migrations');
return;
}
results?.forEach((it) => {
if (it.status === 'Success') {
this.logger.log(
`Migration "${it.migrationName}" executed successfully`,
);
} else if (it.status === 'Error') {
this.logger.error(`Failed to execute migration "${it.migrationName}"`);
}
});
if (error) {
this.logger.error('Failed to run database migration. Exiting program.');
this.logger.error(error);
process.exit(1);
}
}
}

View File

@ -0,0 +1,2 @@
export const APP_DATA_PATH = 'data';
export const LOCAL_STORAGE_PATH = `${APP_DATA_PATH}/storage`;

View File

@ -1,3 +1,4 @@
export * from './utils';
export * from './nanoid.utils';
export * from './file.helper';
export * from './constants';

View File

@ -3,12 +3,6 @@ import * as bcrypt from 'bcrypt';
export const envPath = path.resolve(process.cwd(), '..', '..', '.env');
export function generateHostname(name: string): string {
let hostname = name.replace(/[^a-z0-9]/gi, '').toLowerCase();
hostname = hostname.substring(0, 30);
return hostname;
}
export async function hashPassword(password: string) {
const saltRounds = 12;
return bcrypt.hash(password, saltRounds);
@ -21,15 +15,12 @@ export async function comparePasswordHash(
return bcrypt.compare(plainPassword, passwordHash);
}
export function getRandomInt(min = 4, max = 5) {
return Math.floor(Math.random() * (max - min + 1)) + min;
}
export type RedisConfig = {
host: string;
port: number;
password?: string;
};
export function parseRedisUrl(redisUrl: string): RedisConfig {
// format - redis[s]://[[username][:password]@][host][:port][/db-number]
const { hostname, port, password } = new URL(redisUrl);
@ -37,3 +28,9 @@ export function parseRedisUrl(redisUrl: string): RedisConfig {
return { host: hostname, port: portInt, password };
}
export function createRetryStrategy() {
return function (times: number): number {
return Math.max(Math.min(Math.exp(times), 20000), 3000);
};
}

View File

@ -5,8 +5,8 @@ import { ConfigService } from '@nestjs/config';
export class EnvironmentService {
constructor(private configService: ConfigService) {}
getEnv(): string {
return this.configService.get<string>('NODE_ENV');
getNodeEnv(): string {
return this.configService.get<string>('NODE_ENV', 'development');
}
getAppUrl(): string {
@ -17,7 +17,7 @@ export class EnvironmentService {
}
getPort(): number {
return parseInt(this.configService.get<string>('PORT'));
return parseInt(this.configService.get<string>('PORT', '3000'));
}
getAppSecret(): string {
@ -28,20 +28,19 @@ export class EnvironmentService {
return this.configService.get<string>('DATABASE_URL');
}
getJwtSecret(): string {
return this.configService.get<string>('JWT_SECRET_KEY');
getRedisUrl(): string {
return this.configService.get<string>(
'REDIS_URL',
'redis://localhost:6379',
);
}
getJwtTokenExpiresIn(): string {
return this.configService.get<string>('JWT_TOKEN_EXPIRES_IN');
return this.configService.get<string>('JWT_TOKEN_EXPIRES_IN', '30d');
}
getStorageDriver(): string {
return this.configService.get<string>('STORAGE_DRIVER');
}
getLocalStoragePath(): string {
return this.configService.get<string>('LOCAL_STORAGE_PATH');
return this.configService.get<string>('STORAGE_DRIVER', 'local');
}
getAwsS3AccessKeyId(): string {
@ -68,27 +67,12 @@ export class EnvironmentService {
return this.configService.get<string>('AWS_S3_URL');
}
getAwsS3UsePathStyleEndpoint(): boolean {
return this.configService.get<boolean>('AWS_S3_USE_PATH_STYLE_ENDPOINT');
}
isCloud(): boolean {
const cloudConfig = this.configService
.get<string>('CLOUD', 'false')
.toLowerCase();
return cloudConfig === 'true';
}
isSelfHosted(): boolean {
return !this.isCloud();
}
getMailDriver(): string {
return this.configService.get<string>('MAIL_DRIVER', 'log');
}
getMailHost(): string {
return this.configService.get<string>('MAIL_HOST', '127.0.0.1');
return this.configService.get<string>('MAIL_HOST');
}
getMailPort(): number {
@ -115,10 +99,14 @@ export class EnvironmentService {
return this.configService.get<string>('POSTMARK_TOKEN');
}
getRedisUrl(): string {
return this.configService.get<string>(
'REDIS_URL',
'redis://@127.0.0.1:6379',
);
isCloud(): boolean {
const cloudConfig = this.configService
.get<string>('CLOUD', 'false')
.toLowerCase();
return cloudConfig === 'true';
}
isSelfHosted(): boolean {
return !this.isCloud();
}
}

View File

@ -1,14 +1,12 @@
import { IsString, IsUrl, validateSync } from 'class-validator';
import { IsNotEmpty, IsUrl, validateSync } from 'class-validator';
import { plainToInstance } from 'class-transformer';
export class EnvironmentVariables {
@IsString()
NODE_ENV: string;
@IsNotEmpty()
@IsUrl({ protocols: ['postgres', 'postgresql'], require_tld: false })
DATABASE_URL: string;
@IsString()
@IsNotEmpty()
APP_SECRET: string;
}

View File

@ -0,0 +1,19 @@
import { ConsoleLogger } from '@nestjs/common';
export class InternalLogFilter extends ConsoleLogger {
static contextsToIgnore = [
'InstanceLoader',
'RoutesResolver',
'RouterExplorer',
'WebSocketsController',
];
log(_: any, context?: string): void {
if (
process.env.NODE_ENV !== 'production' ||
!InternalLogFilter.contextsToIgnore.includes(context)
) {
super.log.apply(this, arguments);
}
}
}

View File

@ -1,7 +1,7 @@
import { Global, Module } from '@nestjs/common';
import { BullModule } from '@nestjs/bullmq';
import { EnvironmentService } from '../environment/environment.service';
import { parseRedisUrl } from '../../helpers';
import { createRetryStrategy, parseRedisUrl } from '../../helpers';
import { QueueName } from './constants';
@Global()
@ -15,9 +15,7 @@ import { QueueName } from './constants';
host: redisConfig.host,
port: redisConfig.port,
password: redisConfig.password,
retryStrategy: function (times: number) {
return Math.max(Math.min(Math.exp(times), 20000), 1000);
},
retryStrategy: createRetryStrategy(),
},
defaultJobOptions: {
attempts: 3,

View File

@ -30,7 +30,7 @@ export class StaticModule implements OnModuleInit {
const windowVar = '<!--window-config-->';
const configString = {
ENV: this.environmentService.getEnv(),
ENV: this.environmentService.getNodeEnv(),
APP_URL: this.environmentService.getAppUrl(),
IS_CLOUD: this.environmentService.isCloud(),
};

View File

@ -11,6 +11,8 @@ import {
StorageOption,
} from '../interfaces';
import { LocalDriver, S3Driver } from '../drivers';
import * as process from 'node:process';
import { LOCAL_STORAGE_PATH } from '../../../helpers';
function createStorageDriver(disk: StorageConfig): StorageDriver {
switch (disk.driver) {
@ -33,8 +35,7 @@ export const storageDriverConfigProvider = {
return {
driver,
config: {
storagePath:
process.cwd() + '/' + environmentService.getLocalStoragePath(),
storagePath: process.cwd() + '/' + LOCAL_STORAGE_PATH,
},
};

View File

@ -7,6 +7,8 @@ import {
import { NotFoundException, ValidationPipe } from '@nestjs/common';
import { TransformHttpResponseInterceptor } from './interceptors/http-response.interceptor';
import fastifyMultipart from '@fastify/multipart';
import { WsRedisIoAdapter } from './ws/adapter/ws-redis.adapter';
import { InternalLogFilter } from './integrations/logger/internal-log-filter';
async function bootstrap() {
const app = await NestFactory.create<NestFastifyApplication>(
@ -15,10 +17,18 @@ async function bootstrap() {
ignoreTrailingSlash: true,
ignoreDuplicateSlashes: true,
}),
{
logger: new InternalLogFilter(),
},
);
app.setGlobalPrefix('api');
const redisIoAdapter = new WsRedisIoAdapter(app);
await redisIoAdapter.connectToRedis();
app.useWebSocketAdapter(redisIoAdapter);
await app.register(fastifyMultipart);
app
@ -50,7 +60,7 @@ async function bootstrap() {
app.useGlobalInterceptors(new TransformHttpResponseInterceptor());
app.enableShutdownHooks();
await app.listen(process.env.PORT || 3000);
await app.listen(process.env.PORT || 3000, '0.0.0.0');
}
bootstrap();

View File

@ -0,0 +1,26 @@
import { IoAdapter } from '@nestjs/platform-socket.io';
import { ServerOptions } from 'socket.io';
import { createAdapter } from '@socket.io/redis-adapter';
import Redis, { RedisOptions } from 'ioredis';
import { createRetryStrategy } from '../../helpers';
export class WsRedisIoAdapter extends IoAdapter {
private adapterConstructor: ReturnType<typeof createAdapter>;
async connectToRedis(): Promise<void> {
const options: RedisOptions = {
retryStrategy: createRetryStrategy(),
};
const pubClient = new Redis(process.env.REDIS_URL, options);
const subClient = new Redis(process.env.REDIS_URL, options);
this.adapterConstructor = createAdapter(pubClient, subClient);
}
createIOServer(port: number, options?: ServerOptions): any {
const server = super.createIOServer(port, options);
server.adapter(this.adapterConstructor);
return server;
}
}