Merge branch 'main' into feat/integrations

# Conflicts:
#	apps/client/src/App.tsx
#	apps/server/src/ee
#	apps/server/src/integrations/queue/constants/queue.constants.ts
#	apps/server/src/integrations/queue/queue.module.ts
#	packages/editor-ext/src/index.ts
This commit is contained in:
Philipinho
2026-08-07 01:13:39 +01:00
406 changed files with 33557 additions and 5788 deletions
+80 -77
View File
@@ -1,6 +1,6 @@
{
"name": "server",
"version": "0.90.0",
"version": "0.95.0",
"description": "",
"author": "",
"private": true,
@@ -30,98 +30,98 @@
"test:e2e": "jest --config test/jest-e2e.json"
},
"dependencies": {
"@ai-sdk/google": "^3.0.52",
"@ai-sdk/openai": "^3.0.47",
"@ai-sdk/openai-compatible": "^2.0.37",
"@ai-sdk/google": "3.0.52",
"@ai-sdk/openai": "3.0.47",
"@ai-sdk/openai-compatible": "2.0.37",
"@aws-sdk/client-s3": "3.1050.0",
"@aws-sdk/lib-storage": "3.1050.0",
"@aws-sdk/s3-request-presigner": "3.1050.0",
"@azure/storage-blob": "12.31.0",
"@clickhouse/client": "^1.18.2",
"@clickhouse/client": "1.18.2",
"@docmost/base-formula": "workspace:*",
"@docmost/pdf-inspector": "1.9.6",
"@fastify/cookie": "^11.0.2",
"@fastify/multipart": "^10.0.0",
"@fastify/static": "^9.1.3",
"@keyv/redis": "^5.1.6",
"@fastify/cookie": "11.0.2",
"@fastify/multipart": "10.0.0",
"@fastify/static": "10.1.2",
"@keyv/redis": "5.1.6",
"@langchain/core": "1.1.46",
"@langchain/textsplitters": "1.0.1",
"@modelcontextprotocol/sdk": "1.29.0",
"@nest-lab/throttler-storage-redis": "^1.2.0",
"@nestjs-labs/nestjs-ioredis": "^11.0.4",
"@nestjs/bullmq": "^11.0.4",
"@nestjs/cache-manager": "^3.1.0",
"@nestjs/common": "^11.1.19",
"@nestjs/config": "^4.0.4",
"@nestjs/core": "^11.1.19",
"@nestjs/event-emitter": "^3.0.1",
"@modelcontextprotocol/sdk": "1.30.0",
"@nest-lab/throttler-storage-redis": "1.2.0",
"@nestjs-labs/nestjs-ioredis": "11.0.4",
"@nestjs/bullmq": "11.0.4",
"@nestjs/cache-manager": "3.1.3",
"@nestjs/common": "11.1.27",
"@nestjs/config": "4.0.4",
"@nestjs/core": "11.1.27",
"@nestjs/event-emitter": "3.1.0",
"@nestjs/jwt": "11.0.2",
"@nestjs/mapped-types": "^2.1.1",
"@nestjs/passport": "^11.0.5",
"@nestjs/platform-fastify": "^11.1.19",
"@nestjs/platform-socket.io": "^11.1.19",
"@nestjs/schedule": "^6.1.3",
"@nestjs/terminus": "^11.1.1",
"@nestjs/throttler": "^6.5.0",
"@nestjs/websockets": "^11.1.19",
"@node-saml/passport-saml": "^5.1.0",
"@socket.io/redis-adapter": "^8.3.0",
"ai": "^6.0.134",
"ai-sdk-ollama": "^3.8.1",
"bcrypt": "^6.0.0",
"bowser": "^2.14.1",
"bullmq": "^5.76.10",
"cache-manager": "^7.2.8",
"cheerio": "^1.2.0",
"class-transformer": "^0.5.1",
"class-validator": "^0.15.1",
"cookie": "^1.1.1",
"@nestjs/mapped-types": "2.1.1",
"@nestjs/passport": "11.0.5",
"@nestjs/platform-fastify": "11.1.27",
"@nestjs/platform-socket.io": "11.1.27",
"@nestjs/schedule": "6.1.3",
"@nestjs/terminus": "11.1.1",
"@nestjs/throttler": "6.5.0",
"@nestjs/websockets": "11.1.27",
"@node-saml/passport-saml": "5.1.0",
"@socket.io/redis-adapter": "8.3.0",
"ai": "6.0.134",
"ai-sdk-ollama": "3.8.1",
"bcrypt": "6.0.0",
"bowser": "2.14.1",
"bullmq": "5.79.0",
"cache-manager": "7.2.8",
"cheerio": "1.2.0",
"class-transformer": "0.5.1",
"class-validator": "0.15.1",
"cookie": "1.1.1",
"csv-stringify": "6.8.0",
"fast-bm25": "0.0.5",
"fastify-ip": "^2.0.0",
"fs-extra": "^11.3.4",
"fastify-ip": "2.0.0",
"fs-extra": "11.3.4",
"happy-dom": "20.8.9",
"ioredis": "^5.10.1",
"js-tiktoken": "^1.0.21",
"jsonwebtoken": "^9.0.3",
"kysely": "^0.28.17",
"kysely-migration-cli": "^0.4.2",
"kysely-postgres-js": "^3.0.0",
"ldapts": "^8.1.7",
"lib0": "^0.2.117",
"mammoth": "^1.12.0",
"mime-types": "^3.0.2",
"ioredis": "5.10.1",
"js-tiktoken": "1.0.21",
"jsonwebtoken": "9.0.3",
"kysely": "0.28.17",
"kysely-migration-cli": "0.4.2",
"kysely-postgres-js": "3.0.0",
"ldapts": "8.1.7",
"mammoth": "1.12.0",
"mime-types": "3.0.2",
"msgpackr": "^1.11.9",
"nanoid": "5.1.7",
"nestjs-cls": "^6.2.0",
"nestjs-kysely": "^3.1.2",
"nestjs-pino": "^4.6.1",
"nodemailer": "^8.0.5",
"openid-client": "^6.8.2",
"otpauth": "^9.5.0",
"p-limit": "^7.3.0",
"passport-google-oauth20": "^2.0.0",
"passport-jwt": "^4.0.1",
"pg-tsquery": "^8.4.2",
"nestjs-cls": "6.2.0",
"nestjs-kysely": "3.1.2",
"nestjs-pino": "4.6.1",
"nodemailer": "9.0.1",
"openid-client": "6.8.2",
"otpauth": "9.5.0",
"p-limit": "7.3.0",
"passport-google-oauth20": "2.0.0",
"passport-jwt": "4.0.1",
"pg-tsquery": "8.4.2",
"pgvector": "^0.2.1",
"pino-http": "^11.0.0",
"pino-pretty": "^13.1.3",
"postgres": "^3.4.8",
"postmark": "^4.0.7",
"react": "^18.3.1",
"react-email": "6.0.8",
"reflect-metadata": "^0.2.2",
"pino-http": "11.0.0",
"pino-pretty": "13.1.3",
"postgres": "3.4.8",
"postmark": "4.0.7",
"react": "19.2.7",
"react-email": "6.6.3",
"reflect-metadata": "0.2.2",
"rxjs": "^7.8.2",
"sanitize-filename": "1.6.3",
"scimmy": "1.3.5",
"socket.io": "^4.8.3",
"socket.io": "4.8.3",
"stripe": "^17.7.0",
"tlds": "^1.261.0",
"tmp-promise": "^3.0.3",
"tseep": "^1.3.1",
"typesense": "^3.0.5",
"undici": "7.24.0",
"ws": "^8.20.1",
"yauzl": "^3.2.1",
"zod": "^4.3.6"
"tlds": "1.261.0",
"tmp-promise": "3.0.3",
"typesense": "3.0.5",
"undici": "7.29.0",
"ws": "8.21.0",
"yauzl": "3.4.0",
"zod": "4.3.6"
},
"devDependencies": {
"@eslint/js": "^9.28.0",
@@ -192,7 +192,10 @@
"moduleNameMapper": {
"^@docmost/db/(.*)$": "<rootDir>/database/$1",
"^@docmost/transactional/(.*)$": "<rootDir>/integrations/transactional/$1",
"^@docmost/ee/(.*)$": "<rootDir>/ee/$1"
"^@docmost/ee/(.*)$": "<rootDir>/ee/$1",
"^src/(.*)$": "<rootDir>/$1",
"^@docmost/base-formula/server$": "<rootDir>/../../../packages/base-formula/src/index.server.ts",
"^@docmost/base-formula/client$": "<rootDir>/../../../packages/base-formula/src/index.client.ts"
}
}
}
@@ -15,6 +15,7 @@ import {
RedisSyncExtension,
SerializedHTTPRequest,
} from './extensions/redis-sync';
import { toWebRequest } from './extensions/redis-sync/redis-sync.types';
import { WsSocketWrapper } from './extensions/redis-sync/ws-socket-wrapper';
import RedisClient from 'ioredis';
import { pack, unpack } from 'msgpackr';
@@ -98,34 +99,36 @@ export class CollaborationGateway {
const serializedHTTPRequest = this.serializeRequest(request);
const socketId = serializedHTTPRequest.headers['sec-websocket-key'];
// Create wrapper socket that only receives events via emit()
// This prevents double-handling since Hocuspocus won't listen to raw WebSocket events
const wrappedSocket = new WsSocketWrapper(client);
// Route through RedisSync extension (this calls handleConnection internally)
this.redisSync.onSocketOpen(wrappedSocket as any, serializedHTTPRequest);
this.redisSync.onSocketOpen(wrappedSocket, serializedHTTPRequest);
// Forward raw WebSocket messages to the extension
client.on('message', (data: ArrayBuffer) => {
this.redisSync!.onSocketMessage(
wrappedSocket as any,
serializedHTTPRequest,
data,
);
this.redisSync!.onSocketMessage(serializedHTTPRequest, data);
});
// Forward close events
client.on('close', (code: number, reason: Buffer) => {
this.redisSync!.onSocketClose(socketId, code, reason.buffer as ArrayBuffer);
});
// Forward pong events for keepalive
client.on('pong', (data: Buffer) => {
wrappedSocket.emit('pong', data);
this.redisSync!.onSocketClose(
socketId,
code,
new Uint8Array(reason).buffer,
);
});
} else {
// Fallback to direct Hocuspocus connection
this.hocuspocus.handleConnection(client, request);
const clientConnection = this.hocuspocus.handleConnection(
client,
toWebRequest(this.serializeRequest(request)),
);
client.on('message', (data: Buffer) => {
clientConnection.handleMessage(new Uint8Array(data));
});
client.on('close', (code: number, reason: Buffer) => {
clientConnection.handleClose({ code, reason: reason.toString() });
});
}
}
@@ -178,6 +181,7 @@ export class CollaborationGateway {
if (this.hocuspocus.getDocumentsCount() === 0) resolve('');
this.hocuspocus.closeConnections();
this.hocuspocus.flushPendingStores();
} catch (error) {
console.error(error);
}
@@ -44,6 +44,7 @@ import {
htmlToMarkdown,
TransclusionSource,
TransclusionReference,
BaseEmbed,
} from '@docmost/editor-ext';
import { generateText, getSchema, JSONContent } from '@tiptap/core';
import { generateHTML, generateJSON } from '../common/helpers/prosemirror/html';
@@ -109,6 +110,7 @@ export const tiptapExtensions = [
Status,
TransclusionSource,
TransclusionReference,
BaseEmbed
] as any;
export function jsonToHtml(tiptapJson: any) {
@@ -96,7 +96,7 @@ export class PersistenceExtension implements Extension {
}
async onStoreDocument(data: onStoreDocumentPayload) {
const { documentName, document, context } = data;
const { documentName, document, lastContext } = data;
const pageId = getPageId(documentName);
@@ -151,7 +151,7 @@ export class PersistenceExtension implements Extension {
content: tiptapJson,
textContent: textContent,
ydoc: ydocState,
lastUpdatedById: context.user.id,
lastUpdatedById: lastContext.user.id,
contributorIds: contributorIds,
},
pageId,
@@ -165,6 +165,21 @@ export class PersistenceExtension implements Extension {
}
if (page) {
document.broadcastStateless(
JSON.stringify({
type: 'page.updated',
updatedAt: new Date().toISOString(),
lastUpdatedById: lastContext?.user?.id,
lastUpdatedBy: lastContext?.user
? {
id: lastContext.user?.id,
name: lastContext.user?.name,
avatarUrl: lastContext.user?.avatarUrl,
}
: undefined,
}),
);
await this.syncTransclusion(pageId, page.workspaceId, tiptapJson);
}
@@ -1,61 +1,37 @@
import type RedisClient from 'ioredis';
import { EventEmitter } from 'tseep';
import type {
Pack,
RSAMessageClose,
RSAMessagePing,
RSAMessageSend,
} from './redis-sync.types';
import type { WebSocketLike } from '@hocuspocus/server';
import type { Pack, RSAMessageClose, RSAMessageSend } from './redis-sync.types';
export class CollabProxySocket extends EventEmitter {
// Stands in for the client WebSocket on the server that owns the document.
// Outgoing traffic is relayed over redis to the origin server, which holds the real socket.
export class CollabProxySocket implements WebSocketLike {
private readonly replyTo: string;
private readonly serverChannel: string;
private readonly socketId: string;
private pub: RedisClient;
private readonly pack: Pack;
readyState = 1;
onClose?: (code?: number, reason?: string) => void;
constructor(
pub: RedisClient,
pack: Pack,
replyTo: string,
serverChannel: string,
socketId: string,
) {
super();
constructor(pub: RedisClient, pack: Pack, replyTo: string, socketId: string) {
this.replyTo = replyTo;
this.socketId = socketId;
this.serverChannel = serverChannel;
this.pub = pub;
this.pack = pack;
this.once('close', () => {
this.readyState = 3;
});
}
private publish(msg: RSAMessageClose | RSAMessagePing | RSAMessageSend) {
private publish(msg: RSAMessageClose | RSAMessageSend) {
this.pub.publish(this.replyTo, this.pack(msg));
}
// The origin server already closed the real socket; stop relaying without echoing a close back
markClosed() {
this.readyState = 3;
}
close(code?: number, reason?: string) {
if (this.readyState !== 1) return;
const msg: RSAMessageClose = {
type: 'close',
code,
reason,
socketId: this.socketId,
};
this.publish(msg);
}
ping() {
if (this.readyState !== 1) return;
const msg: RSAMessagePing = {
type: 'ping',
socketId: this.socketId,
replyTo: this.serverChannel,
};
this.publish(msg);
this.readyState = 3;
this.onClose?.(code, reason);
}
send(message: Uint8Array) {
@@ -3,27 +3,30 @@ import {
Extension,
Hocuspocus,
IncomingMessage,
afterUnloadDocumentPayload,
onConfigurePayload,
onLoadDocumentPayload,
afterUnloadDocumentPayload,
WebSocketLike,
} from '@hocuspocus/server';
import { ConnectionTimeout, Unauthorized } from '@hocuspocus/common';
import RedisClient from 'ioredis';
import { readVarString } from 'lib0/decoding.js';
import { CollabProxySocket } from './collab-proxy-socket';
import {
BaseWebSocket,
Configuration,
CustomEvents,
Pack,
RSAMessage,
RSAMessageClose,
RSAMessageCloseProxy,
RSAMessageCustomEventComplete,
RSAMessageCustomEventStart,
RSAMessagePong,
RSAMessageProxy,
RSAMessageUnload,
SerializedHTTPRequest,
Unpack,
OriginConnection,
ProxyConnection,
toWebRequest,
} from './redis-sync.types';
export type { Pack, SerializedHTTPRequest } from './redis-sync.types';
@@ -38,10 +41,10 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
private sub: RedisClient;
private readonly pack: Pack;
private readonly unpack: Unpack;
private originSockets: Record<SocketId, BaseWebSocket> = {};
private originConnections: Record<SocketId, OriginConnection> = {};
private locks: Record<DocumentName, NodeJS.Timeout> = {};
private lockPromises: Record<DocumentName, Promise<ServerId | null>> = {};
private proxySockets: Record<SocketId, CollabProxySocket> = {};
private proxyConnections: Record<SocketId, ProxyConnection> = {};
private readonly prefix: string;
private readonly lockPrefix: string;
private readonly msgChannel: string;
@@ -54,6 +57,9 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
// @ts-ignore
private pendingReplies: Record<number, PromiseWithResolvers<any>['resolve']> =
{};
private deriveContext: (
serializedHTTPRequest: SerializedHTTPRequest,
) => Record<string, any>;
constructor(configuration: Configuration<TCE>) {
const {
@@ -65,6 +71,7 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
prefix,
customEvents,
customEventTTL,
deriveContext,
} = configuration;
this.pub = redis.duplicate();
this.sub = redis.duplicate();
@@ -77,6 +84,7 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
this.lockPrefix = `${this.prefix}Lock`;
this.msgChannel = `${this.prefix}Msg`;
this.customEvents = (customEvents as any) ?? ({} as any as CustomEvents);
this.deriveContext = deriveContext ?? (() => ({}));
this.sub.subscribe(this.msgChannel, `${this.msgChannel}:${this.serverId}`);
this.sub.on('messageBuffer', this.handleRedisMessage);
this.pub.on('error', () => {});
@@ -87,44 +95,63 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
}
private closeProxy(socketId: string) {
const proxySocket = this.proxySockets[socketId];
if (proxySocket) {
proxySocket.emit(
'close',
1000,
Buffer.from('provider_initiated', 'utf-8'),
);
delete this.proxySockets[socketId];
const entry = this.proxyConnections[socketId];
if (entry) {
delete this.proxyConnections[socketId];
const { socket, clientConnection } = entry;
// The origin socket is already gone; don't echo a close message back
socket.markClosed();
clientConnection.handleClose({
code: 1000,
reason: 'provider_initiated',
});
}
}
private pongProxy(socketId: string) {
this.proxySockets[socketId]?.emit('pong');
}
private handleProxyMessage(
msg: Pick<RSAMessageProxy, 'replyTo' | 'message' | 'serializedHTTPRequest'>,
) {
const { replyTo, message, serializedHTTPRequest } = msg;
const { headers } = serializedHTTPRequest;
const socketId = headers['sec-websocket-key']!;
let socket = this.proxySockets[socketId];
if (!socket) {
socket = new CollabProxySocket(
const socketId = headers['sec-websocket-key'];
let entry = this.proxyConnections[socketId];
if (!entry) {
const socket = new CollabProxySocket(
this.pub,
this.pack,
replyTo,
`${this.msgChannel}:${this.serverId}`,
socketId,
);
this.proxySockets[socketId] = socket;
this.instance.handleConnection(
socket as any,
serializedHTTPRequest as any,
{},
// A proxy connection with no live documents (client left the page, auth
// failed, or the origin server crashed) is reaped by hocuspocus' message
// timeout. Dispose it silently in that case: relaying the timeout close
// to the origin would kill the client's real socket, which may be busy
// serving other documents. Genuine protocol closes are still relayed.
socket.onClose = (code, reason) => {
delete this.proxyConnections[socketId];
if (code !== ConnectionTimeout.code) {
const msg: RSAMessageClose = {
type: 'close',
code,
reason,
socketId,
};
this.pub.publish(replyTo, this.pack(msg));
}
};
const clientConnection = this.instance.handleConnection(
socket,
toWebRequest(serializedHTTPRequest),
this.deriveContext(serializedHTTPRequest),
);
entry = { clientConnection, socket };
this.proxyConnections[socketId] = entry;
}
socket.emit('message', message);
entry.clientConnection.handleMessage(message);
}
private getLock(documentName: string) {
return this.pub.get(this.getKey(documentName));
}
private getOrClaimLock(documentName: string) {
@@ -166,10 +193,6 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
this.closeProxy(msg.socketId);
return;
}
if (type === 'pong') {
this.pongProxy(msg.socketId);
return;
}
if (type === 'unload') {
delete this.lockPromises[msg.documentName];
return;
@@ -198,22 +221,14 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
return;
}
const { socketId } = msg;
const socket = this.originSockets[socketId];
if (!socket) {
const entry = this.originConnections[socketId];
if (!entry) {
// origin socket already cleaned up
return;
}
const { socket } = entry;
if (type === 'close') {
socket.close(msg.code, msg.reason);
} else if (type === 'ping') {
// Reply instantly to the proxy socket, without forwarding to client
// The origin socket handles heartbeat for itself
const { replyTo, socketId } = msg;
const reply: RSAMessagePong = {
type: 'pong',
socketId,
};
this.pub.publish(`${replyTo}`, this.pack(reply));
} else if (type === 'send') {
socket.send(msg.message);
}
@@ -251,6 +266,8 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
eventName: TName,
documentName: string,
payload: any,
// if true, don't claim the lock. Useful for targeting pages that are currently open
onlyIfOpen = false,
) {
const isDocLoadedOnInstance = this.instance.documents.has(documentName);
@@ -258,7 +275,14 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
return this.handleEventLocally(eventName, documentName, payload);
}
const proxyTo = await this.getOrClaimLockThrottled(documentName);
const proxyTo = await (onlyIfOpen
? this.getLock(documentName)
: this.getOrClaimLockThrottled(documentName));
if (!proxyTo && onlyIfOpen) {
return;
}
if (proxyTo && proxyTo !== this.serverId) {
++this.replyIdCounter; // bug in biome thinks this.replyIdCounter is not used if written on the line below
const replyId = this.replyIdCounter;
@@ -277,7 +301,8 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
const { promise, resolve, reject } = Promise.withResolvers();
this.pendingReplies[replyId] = resolve;
setTimeout(() => {
reject('TIMEOUT');
delete this.pendingReplies[replyId];
reject(new Error('TIMEOUT'));
}, this.customEventTTL);
return promise as Promise<ReturnType<TCE[TName]>>;
}
@@ -296,36 +321,59 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
/* WebSocket Server Hooks */
onSocketOpen(
ws: BaseWebSocket,
ws: WebSocketLike,
serializedHTTPRequest: SerializedHTTPRequest,
context = {},
) {
const socketId = serializedHTTPRequest.headers['sec-websocket-key']!;
this.originSockets[socketId] = ws;
this.instance.handleConnection(
ws as any,
serializedHTTPRequest as any,
context,
const socketId = serializedHTTPRequest.headers['sec-websocket-key'];
const clientConnection = this.instance.handleConnection(
ws,
toWebRequest(serializedHTTPRequest),
this.deriveContext(serializedHTTPRequest),
);
this.originConnections[socketId] = { clientConnection, socket: ws };
}
async onSocketMessage(
ws: BaseWebSocket,
serializedHTTPRequest: SerializedHTTPRequest,
detachableMsg: ArrayBuffer,
) {
const message = new Uint8Array(detachableMsg.slice());
const tmpMsg = new IncomingMessage(detachableMsg);
const documentName = readVarString(tmpMsg.decoder);
const socketId = serializedHTTPRequest.headers['sec-websocket-key'];
const entry = this.originConnections[socketId];
if (!entry) return;
const { clientConnection } = entry;
let message: Uint8Array;
let documentName: string;
try {
message = new Uint8Array(detachableMsg.slice());
const tmpMsg = new IncomingMessage(detachableMsg);
const documentNameAndSessionId = tmpMsg.readVarString();
// session-aware providers suffix the documentName with \0sessionId
const sepIdx = documentNameAndSessionId.indexOf('\0');
documentName =
sepIdx === -1
? documentNameAndSessionId
: documentNameAndSessionId.slice(0, sepIdx);
} catch (error) {
entry.socket.close(Unauthorized.code, Unauthorized.reason);
return;
}
const isDocLoadedOnInstance = this.instance.documents.has(documentName);
if (isDocLoadedOnInstance) {
ws.emit('message', message);
clientConnection.handleMessage(message);
return;
}
const proxyTo = await this.getOrClaimLockThrottled(documentName);
if (proxyTo && proxyTo !== this.serverId) {
// Proxied messages bypass handleMessage, so refresh the connection's
// liveness fields manually or hocuspocus' message timeout would reap the
// real socket every `timeout` ms. connectionEstablishedAt is the
// reference while unauthenticated (auth for remote docs is proxied too)
// and is private upstream.
clientConnection.lastMessageReceivedAt = Date.now();
(clientConnection as any).connectionEstablishedAt = Date.now();
// another server owns the doc
const proxyMessage: RSAMessageProxy = {
serializedHTTPRequest: serializedHTTPRequest,
@@ -338,16 +386,17 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
return;
}
// This server owns the document, but hocuspocus hasn't loaded it yet
ws.emit('message', message);
clientConnection.handleMessage(message);
}
onSocketClose(socketId: string, code?: number, reason?: ArrayBuffer) {
const socket = this.originSockets[socketId];
if (!socket) return;
// at this point the socket is considered GC'd and we cannot call close
// The origin socket did not set up any connections for the proxy, so none of the hooks will work if we just emit
socket?.emit('close', code, reason);
delete this.originSockets[socketId];
const entry = this.originConnections[socketId];
if (!entry) return;
delete this.originConnections[socketId];
entry.clientConnection.handleClose({
code: code ?? 1000,
reason: reason ? Buffer.from(reason).toString() : '',
});
const msg: RSAMessageCloseProxy = { type: 'closeProxy', socketId };
this.pub.publish(this.msgChannel, this.pack(msg)).catch(() => {});
}
@@ -372,6 +421,7 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
}
async onDestroy() {
this.pendingReplies = {};
this.pub.disconnect(false);
this.sub.disconnect(false);
}
@@ -1,12 +1,13 @@
import EventEmitter from 'node:events';
import { IncomingHttpHeaders } from 'node:http2';
import RedisClient from 'ioredis';
import { CollabProxySocket } from './collab-proxy-socket';
import { type Hocuspocus, type WebSocketLike } from '@hocuspocus/server';
export type SecondParam<T> = T extends (
arg1: unknown,
arg1: any,
arg2: infer A,
...args: unknown[]
) => unknown
...args: any[]
) => any
? A
: never;
@@ -41,17 +42,6 @@ export type RSAMessageClose = {
socketId: string;
};
export type RSAMessagePing = {
type: 'ping';
socketId: string;
replyTo: string;
};
export type RSAMessagePong = {
type: 'pong';
socketId: string;
};
export type RSAMessageSend = {
type: 'send';
// @ts-ignore
@@ -59,7 +49,7 @@ export type RSAMessageSend = {
socketId: string;
};
export type RSAMessageCustomEventStart<TName = string, TPayload = unknown> = {
export type RSAMessageCustomEventStart<TName = string, TPayload = any> = {
type: 'customEventStart';
documentName: string;
eventName: TName;
@@ -71,7 +61,7 @@ export type RSAMessageCustomEventStart<TName = string, TPayload = unknown> = {
export type RSAMessageCustomEventComplete = {
type: 'customEventComplete';
replyId: number;
payload: unknown;
payload: any;
};
export type RSAMessage =
@@ -79,8 +69,6 @@ export type RSAMessage =
| RSAMessageCloseProxy
| RSAMessageUnload
| RSAMessageClose
| RSAMessagePing
| RSAMessagePong
| RSAMessageSend
| RSAMessageCustomEventStart
| RSAMessageCustomEventComplete;
@@ -99,9 +87,20 @@ type CustomEventName = string;
export type CustomEvents = Record<
CustomEventName,
(documentName: string, payload: unknown) => Promise<unknown>
(documentName: string, payload: any) => Promise<any>
>;
// Not exported by @hocuspocus/server
export type ClientConnection = ReturnType<Hocuspocus['handleConnection']>;
export type OriginConnection = {
clientConnection: ClientConnection;
socket: WebSocketLike;
};
export type ProxyConnection = {
clientConnection: ClientConnection;
socket: CollabProxySocket;
};
export interface Configuration<TCE> {
redis: RedisClient;
pack: Pack;
@@ -111,11 +110,29 @@ export interface Configuration<TCE> {
customEventTTL?: number;
prefix?: string;
customEvents?: TCE;
// Derive the hocuspocus context once per socket instead of re-deriving it in a
// per-document hook like onConnect/onAuthenticate. Runs on the origin server when
// the socket opens and on the doc owner when the first proxied message arrives.
deriveContext?: (
serializedHTTPRequest: SerializedHTTPRequest,
) => Record<string, any>;
}
export type BaseWebSocket = EventEmitter & {
readyState: number;
close(code?: number, reason?: string): void;
ping(): void;
send(message: Uint8Array): void;
// Hocuspocus expects a web-standard Request, so rehydrate one from what crossed the wire
export const toWebRequest = (serializedHTTPRequest: SerializedHTTPRequest) => {
const { method, url, headers } = serializedHTTPRequest;
const webHeaders = new Headers();
Object.entries(headers).forEach(([name, value]) => {
if (Array.isArray(value)) {
value.forEach((v) => {
webHeaders.append(name, v);
});
} else if (value !== undefined) {
webHeaders.set(name, value);
}
});
return new Request(new URL(url, 'http://localhost'), {
method,
headers: webHeaders,
});
};
@@ -1,20 +1,17 @@
import { EventEmitter } from 'events';
import type WebSocket from 'ws';
import type { WebSocketLike } from '@hocuspocus/server';
/**
* Wrapper around ws WebSocket that only receives events via emit().
* This prevents double-handling when used with RedisSyncExtension.
* Wrapper around ws WebSocket that Hocuspocus only writes to.
* Incoming socket events are forwarded separately by the gateway,
* which prevents double-handling with RedisSyncExtension.
*/
export class WsSocketWrapper extends EventEmitter {
export class WsSocketWrapper implements WebSocketLike {
private ws: WebSocket;
readyState = 1;
constructor(ws: WebSocket) {
super();
this.ws = ws;
this.once('close', () => {
this.readyState = 3;
});
}
close(code?: number, reason?: string) {
@@ -27,15 +24,6 @@ export class WsSocketWrapper extends EventEmitter {
}
}
ping() {
if (this.readyState !== 1) return;
try {
this.ws.ping();
} catch (e) {
// Socket already closed
}
}
send(message: Uint8Array) {
if (this.readyState !== 1) return;
try {
@@ -15,4 +15,29 @@ export enum EventName {
WORKSPACE_CREATED = 'workspace.created',
WORKSPACE_UPDATED = 'workspace.updated',
WORKSPACE_DELETED = 'workspace.deleted',
BASE_CREATED = 'base.created',
BASE_UPDATED = 'base.updated',
BASE_DELETED = 'base.deleted',
BASE_ROW_CREATED = 'base.row.created',
BASE_ROW_UPDATED = 'base.row.updated',
BASE_ROW_DELETED = 'base.row.deleted',
BASE_ROWS_DELETED = 'base.rows.deleted',
BASE_ROW_RESTORED = 'base.row.restored',
BASE_ROW_REORDERED = 'base.row.reordered',
BASE_PROPERTY_CREATED = 'base.property.created',
BASE_PROPERTY_UPDATED = 'base.property.updated',
BASE_PROPERTY_DELETED = 'base.property.deleted',
BASE_PROPERTY_REORDERED = 'base.property.reordered',
BASE_VIEW_CREATED = 'base.view.created',
BASE_VIEW_UPDATED = 'base.view.updated',
BASE_VIEW_DELETED = 'base.view.deleted',
BASE_SCHEMA_BUMPED = 'base.schema.bumped',
BASE_ROWS_UPDATED = 'base.rows.updated',
BASE_FORMULA_RECOMPUTE_STARTED = 'base.formula.recompute.started',
BASE_FORMULA_RECOMPUTE_COMPLETED = 'base.formula.recompute.completed',
}
+3
View File
@@ -20,6 +20,9 @@ export const Feature = {
VIEWER_COMMENTS: 'comment:viewer',
TEMPLATES: 'templates',
PDF_EXPORT: 'export:pdf',
PERSONAL_SPACES: 'spaces:personal',
DOCX_EXPORT: 'export:docx',
BASES: 'bases',
} as const;
export type FeatureKey = (typeof Feature)[keyof typeof Feature];
+5 -2
View File
@@ -10,6 +10,9 @@ export const LOCAL_STORAGE_PATH = path.resolve(
LOCAL_STORAGE_DIR,
);
export function getPageTitle(title: string | null | undefined): string {
return title || 'untitled';
export function getPageTitle(
title: string | null | undefined,
isBase?: boolean,
): string {
return title || (isBase ? 'Untitled base' : 'untitled');
}
@@ -5,4 +5,9 @@ export const nanoIdGen = customAlphabet(alphabet, 10);
const slugIdAlphabet =
'0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz';
export const generateSlugId = customAlphabet(slugIdAlphabet, 10);
export const generateSlugId = customAlphabet(slugIdAlphabet, 10);
const baseIdSuffix = customAlphabet(alphabet, 9);
export const generateBasePropertyId = (): string => `prp${baseIdSuffix()}`;
export const generateBaseChoiceId = (): string => `opt${baseIdSuffix()}`;
@@ -4,6 +4,11 @@ export enum UserRole {
MEMBER = 'member',
}
export enum InviteUserRole {
ADMIN = 'admin', // can have owner permissions but cannot delete workspace
MEMBER = 'member',
}
export enum SpaceRole {
ADMIN = 'admin', // can manage space settings, members, and delete space
WRITER = 'writer', // can read and write pages in space
@@ -43,7 +43,7 @@ export class AttachmentService {
async uploadFile(opts: {
filePromise: Promise<MultipartFile>;
pageId: string;
pageId?: string;
userId: string;
spaceId: string;
workspaceId: string;
@@ -4,6 +4,8 @@ import { WorkspaceService } from '../../workspace/services/workspace.service';
import { CreateWorkspaceDto } from '../../workspace/dto/create-workspace.dto';
import { CreateAdminUserDto } from '../dto/create-admin-user.dto';
import { UserRepo } from '@docmost/db/repos/user/user.repo';
import { WorkspaceRepo } from '@docmost/db/repos/workspace/workspace.repo';
import { getWorkspaceDefaultPageEditMode } from '../../workspace/workspace.util';
import { KyselyDB, KyselyTransaction } from '@docmost/db/types/kysely.types';
import { executeTx } from '@docmost/db/utils';
import { InjectKysely } from 'nestjs-kysely';
@@ -20,6 +22,7 @@ import {
export class SignupService {
constructor(
private userRepo: UserRepo,
private workspaceRepo: WorkspaceRepo,
private workspaceService: WorkspaceService,
private groupUserRepo: GroupUserRepo,
@InjectKysely() private readonly db: KyselyDB,
@@ -46,12 +49,16 @@ export class SignupService {
this.db,
async (trx) => {
// create user
const workspace = await this.workspaceRepo.findById(workspaceId, {
trx,
});
const user = await this.userRepo.insertUser(
{
...createUserDto,
workspaceId: workspaceId,
},
trx,
{ pageEditMode: getWorkspaceDefaultPageEditMode(workspace) },
);
// add user to workspace
@@ -1,4 +1,5 @@
import {
IsBoolean,
IsIn,
IsOptional,
IsString,
@@ -15,7 +15,7 @@ import {
executeWithCursorPagination,
} from '@docmost/db/pagination/cursor-pagination';
import { InjectKysely } from 'nestjs-kysely';
import { KyselyDB } from '@docmost/db/types/kysely.types';
import { KyselyDB, KyselyTransaction } from '@docmost/db/types/kysely.types';
import { generateJitteredKeyBetween } from 'fractional-indexing-jittered';
import { MovePageDto } from '../dto/move-page.dto';
import { generateSlugId } from '../../../common/helpers';
@@ -92,6 +92,8 @@ export class PageService {
userId: string,
workspaceId: string,
createPageDto: CreatePageDto,
trx?: KyselyTransaction,
isBase: boolean = false,
): Promise<Page> {
let parentPageId = undefined;
@@ -140,21 +142,34 @@ export class PageService {
creatorId: userId,
workspaceId: workspaceId,
lastUpdatedById: userId,
isBase,
content,
textContent,
ydoc,
});
}, trx);
this.generalQueue
.add(QueueJob.ADD_PAGE_WATCHERS, {
userIds: [userId],
pageId: page.id,
spaceId: createPageDto.spaceId,
if (trx) {
// Add the watcher inside the caller's transaction so the async worker
// never inserts against an uncommitted page (FK violation on bases).
await this.watcherService.addPageWatchers(
[userId],
page.id,
createPageDto.spaceId,
workspaceId,
})
.catch((err) =>
this.logger.warn(`Failed to queue add-page-watchers: ${err.message}`),
trx,
);
} else {
this.generalQueue
.add(QueueJob.ADD_PAGE_WATCHERS, {
userIds: [userId],
pageId: page.id,
spaceId: createPageDto.spaceId,
workspaceId,
})
.catch((err) =>
this.logger.warn(`Failed to queue add-page-watchers: ${err.message}`),
);
}
return page;
}
@@ -289,6 +304,7 @@ export class PageService {
'parentPageId',
'spaceId',
'creatorId',
'isBase',
'deletedAt',
])
.select((eb) => this.pageRepo.withHasChildren(eb))
@@ -310,6 +326,7 @@ export class PageService {
expression: 'position',
direction: 'asc',
orderModifier: (ob) => ob.collate('C').asc(),
cursorExpression: sql`position collate "C"`,
},
{ expression: 'id', direction: 'asc' },
],
@@ -429,10 +446,9 @@ export class PageService {
}
if (pageIdsToMove.length > 0) {
// Clear page-level permissions - moved pages inherit destination space permissions
// (page_permissions cascade deletes via foreign key)
await trx
.deleteFrom('pageAccess')
.updateTable('pageAccess')
.set({ spaceId: spaceId })
.where('pageId', 'in', pageIdsToMove)
.execute();
@@ -831,6 +847,7 @@ export class PageService {
'slugId',
'title',
'icon',
'isBase',
'position',
'parentPageId',
'spaceId',
@@ -846,6 +863,7 @@ export class PageService {
'p.slugId',
'p.title',
'p.icon',
'p.isBase',
'p.position',
'p.parentPageId',
'p.spaceId',
@@ -1,320 +0,0 @@
import { Test } from '@nestjs/testing';
import { TransclusionService } from '../transclusion.service';
import { PageTransclusionsRepo } from '@docmost/db/repos/page-transclusions/page-transclusions.repo';
import { PageTransclusionReferencesRepo } from '@docmost/db/repos/page-transclusions/page-transclusion-references.repo';
import { PageRepo } from '@docmost/db/repos/page/page.repo';
import { PagePermissionRepo } from '@docmost/db/repos/page/page-permission.repo';
import { AttachmentRepo } from '@docmost/db/repos/attachment/attachment.repo';
import { StorageService } from '../../../../integrations/storage/storage.service';
import { PageAccessService } from '../../page-access/page-access.service';
describe('TransclusionService.syncPageTransclusions', () => {
let service: TransclusionService;
let repo: jest.Mocked<PageTransclusionsRepo>;
beforeEach(async () => {
const mockRepo: jest.Mocked<Partial<PageTransclusionsRepo>> = {
findByPageId: jest.fn(),
insert: jest.fn(),
update: jest.fn(),
deleteByPageAndTransclusionIds: jest.fn(),
};
const module = await Test.createTestingModule({
providers: [
TransclusionService,
{ provide: PageTransclusionsRepo, useValue: mockRepo },
{ provide: PageTransclusionReferencesRepo, useValue: {} },
{ provide: PageRepo, useValue: {} },
{ provide: PagePermissionRepo, useValue: {} },
{ provide: AttachmentRepo, useValue: {} },
{ provide: StorageService, useValue: {} },
{ provide: PageAccessService, useValue: {} },
],
}).compile();
service = module.get(TransclusionService);
repo = module.get(PageTransclusionsRepo);
});
const pageId = '00000000-0000-0000-0000-000000000001';
const workspaceId = '00000000-0000-0000-0000-000000000099';
it('inserts new transclusions that did not exist before', async () => {
repo.findByPageId.mockResolvedValue([]);
const pm = {
type: 'doc',
content: [
{
type: 'transclusionSource',
attrs: { id: 'a' },
content: [{ type: 'paragraph' }],
},
],
};
const result = await service.syncPageTransclusions(pageId, workspaceId, pm);
expect(result).toEqual({ inserted: 1, updated: 0, deleted: 0 });
expect(repo.insert).toHaveBeenCalledTimes(1);
expect(repo.insert).toHaveBeenCalledWith(
expect.objectContaining({
pageId,
transclusionId: 'a',
}),
undefined,
);
expect(repo.update).not.toHaveBeenCalled();
expect(repo.deleteByPageAndTransclusionIds).not.toHaveBeenCalled();
});
it('updates transclusions whose content changed', async () => {
repo.findByPageId.mockResolvedValue([
{
id: 'row1',
pageId,
transclusionId: 'a',
content: { type: 'doc', content: [{ type: 'paragraph' }] },
createdAt: new Date(),
updatedAt: new Date(),
} as any,
]);
const newContent = {
type: 'doc',
content: [
{ type: 'paragraph', content: [{ type: 'text', text: 'X' }] },
],
};
const pm = {
type: 'doc',
content: [
{
type: 'transclusionSource',
attrs: { id: 'a' },
content: newContent.content,
},
],
};
const result = await service.syncPageTransclusions(pageId, workspaceId, pm);
expect(result).toEqual({ inserted: 0, updated: 1, deleted: 0 });
expect(repo.update).toHaveBeenCalledWith(
pageId,
'a',
expect.objectContaining({ content: newContent }),
undefined,
);
});
it('skips update when content is unchanged', async () => {
const sameContent = {
type: 'doc',
content: [{ type: 'paragraph' }],
};
repo.findByPageId.mockResolvedValue([
{
id: 'row1',
pageId,
transclusionId: 'a',
content: sameContent,
createdAt: new Date(),
updatedAt: new Date(),
} as any,
]);
const pm = {
type: 'doc',
content: [
{
type: 'transclusionSource',
attrs: { id: 'a' },
content: sameContent.content,
},
],
};
const result = await service.syncPageTransclusions(pageId, workspaceId, pm);
expect(result).toEqual({ inserted: 0, updated: 0, deleted: 0 });
expect(repo.update).not.toHaveBeenCalled();
});
it('deletes transclusions that no longer appear in the doc', async () => {
repo.findByPageId.mockResolvedValue([
{
id: 'r',
pageId,
transclusionId: 'gone',
content: { type: 'doc', content: [] },
createdAt: new Date(),
updatedAt: new Date(),
} as any,
]);
const pm = { type: 'doc', content: [{ type: 'paragraph' }] };
const result = await service.syncPageTransclusions(pageId, workspaceId, pm);
expect(result).toEqual({ inserted: 0, updated: 0, deleted: 1 });
expect(repo.deleteByPageAndTransclusionIds).toHaveBeenCalledWith(
pageId,
['gone'],
undefined,
);
});
it('handles empty doc → noop', async () => {
repo.findByPageId.mockResolvedValue([]);
const result = await service.syncPageTransclusions(pageId, workspaceId, null);
expect(result).toEqual({ inserted: 0, updated: 0, deleted: 0 });
expect(repo.insert).not.toHaveBeenCalled();
expect(repo.update).not.toHaveBeenCalled();
expect(repo.deleteByPageAndTransclusionIds).not.toHaveBeenCalled();
});
});
describe('TransclusionService.syncPageReferences', () => {
let service: TransclusionService;
let refRepo: jest.Mocked<PageTransclusionReferencesRepo>;
beforeEach(async () => {
const mockTransclusionsRepo: Partial<PageTransclusionsRepo> = {};
const mockRefRepo: jest.Mocked<Partial<PageTransclusionReferencesRepo>> = {
findByReferencePageId: jest.fn(),
insertMany: jest.fn(),
deleteByReferenceAndKeys: jest.fn(),
};
const module = await Test.createTestingModule({
providers: [
TransclusionService,
{ provide: PageTransclusionsRepo, useValue: mockTransclusionsRepo },
{ provide: PageTransclusionReferencesRepo, useValue: mockRefRepo },
{ provide: PageRepo, useValue: {} },
{ provide: PagePermissionRepo, useValue: {} },
{ provide: AttachmentRepo, useValue: {} },
{ provide: StorageService, useValue: {} },
{ provide: PageAccessService, useValue: {} },
],
}).compile();
service = module.get(TransclusionService);
refRepo = module.get(PageTransclusionReferencesRepo);
});
const referencePageId = '00000000-0000-0000-0000-000000000001';
const workspaceId = '00000000-0000-0000-0000-000000000099';
it('inserts new loose references, no deletes when none existed', async () => {
refRepo.findByReferencePageId.mockResolvedValue([]);
const pm = {
type: 'doc',
content: [
{
type: 'transclusionReference',
attrs: { sourcePageId: 'p1', transclusionId: 'e1' },
},
{
type: 'transclusionReference',
attrs: { sourcePageId: 'p2', transclusionId: 'e2' },
},
],
};
const result = await service.syncPageReferences(referencePageId, workspaceId, pm);
expect(result).toEqual({ inserted: 2, deleted: 0 });
expect(refRepo.insertMany).toHaveBeenCalledWith(
[
{
workspaceId,
referencePageId,
sourcePageId: 'p1',
transclusionId: 'e1',
},
{
workspaceId,
referencePageId,
sourcePageId: 'p2',
transclusionId: 'e2',
},
],
undefined,
);
expect(refRepo.deleteByReferenceAndKeys).not.toHaveBeenCalled();
});
it('ignores references nested inside a source (schema-forbidden)', async () => {
refRepo.findByReferencePageId.mockResolvedValue([]);
const pm = {
type: 'doc',
content: [
{
type: 'transclusionSource',
attrs: { id: 's1' },
content: [
{
type: 'transclusionReference',
attrs: { sourcePageId: 'p2', transclusionId: 'e2' },
},
],
},
],
};
const result = await service.syncPageReferences(referencePageId, workspaceId, pm);
expect(result).toEqual({ inserted: 0, deleted: 0 });
expect(refRepo.insertMany).not.toHaveBeenCalled();
});
it('deletes references that no longer appear', async () => {
refRepo.findByReferencePageId.mockResolvedValue([
{
id: 'r1',
referencePageId,
sourcePageId: 'p1',
transclusionId: 'e1',
createdAt: new Date(),
} as any,
]);
const pm = { type: 'doc', content: [{ type: 'paragraph' }] };
const result = await service.syncPageReferences(referencePageId, workspaceId, pm);
expect(result).toEqual({ inserted: 0, deleted: 1 });
expect(refRepo.deleteByReferenceAndKeys).toHaveBeenCalledWith(
referencePageId,
[
{
sourcePageId: 'p1',
transclusionId: 'e1',
},
],
undefined,
);
expect(refRepo.insertMany).not.toHaveBeenCalled();
});
it('is a no-op when desired matches existing exactly', async () => {
refRepo.findByReferencePageId.mockResolvedValue([
{
id: 'r',
referencePageId,
sourcePageId: 'p1',
transclusionId: 'e1',
createdAt: new Date(),
} as any,
]);
const pm = {
type: 'doc',
content: [
{
type: 'transclusionReference',
attrs: { sourcePageId: 'p1', transclusionId: 'e1' },
},
],
};
const result = await service.syncPageReferences(referencePageId, workspaceId, pm);
expect(result).toEqual({ inserted: 0, deleted: 0 });
expect(refRepo.insertMany).not.toHaveBeenCalled();
expect(refRepo.deleteByReferenceAndKeys).not.toHaveBeenCalled();
});
});
@@ -6,11 +6,13 @@ import {
} from '@nestjs/common';
import { isDeepStrictEqual } from 'node:util';
import { v7 as uuid7 } from 'uuid';
import { KyselyTransaction } from '@docmost/db/types/kysely.types';
import { InjectKysely } from 'nestjs-kysely';
import { KyselyDB, KyselyTransaction } from '@docmost/db/types/kysely.types';
import { PageTransclusionsRepo } from '@docmost/db/repos/page-transclusions/page-transclusions.repo';
import { PageTransclusionReferencesRepo } from '@docmost/db/repos/page-transclusions/page-transclusion-references.repo';
import { PageRepo } from '@docmost/db/repos/page/page.repo';
import { PagePermissionRepo } from '@docmost/db/repos/page/page-permission.repo';
import { SpaceMemberRepo } from '@docmost/db/repos/space/space-member.repo';
import { AttachmentRepo } from '@docmost/db/repos/attachment/attachment.repo';
import { StorageService } from '../../../integrations/storage/storage.service';
import {
@@ -36,10 +38,12 @@ export class TransclusionService {
private readonly logger = new Logger(TransclusionService.name);
constructor(
@InjectKysely() private readonly db: KyselyDB,
private readonly pageTransclusionsRepo: PageTransclusionsRepo,
private readonly pageTransclusionReferencesRepo: PageTransclusionReferencesRepo,
private readonly pageRepo: PageRepo,
private readonly pagePermissionRepo: PagePermissionRepo,
private readonly spaceMemberRepo: SpaceMemberRepo,
private readonly attachmentRepo: AttachmentRepo,
private readonly storageService: StorageService,
private readonly pageAccessService: PageAccessService,
@@ -213,6 +217,40 @@ export class TransclusionService {
return { inserted: rows.length };
}
/**
* Resolve viewer access for source page IDs supplied by an authenticated
* caller. Restricts candidates to pages the viewer can see at the space
* level before applying page-level restrictions, so a workspace member
* cannot read a sync block from a private space they don't belong to via
* an unrestricted source page.
*/
private async filterViewerAccessiblePageIds(
pageIds: string[],
viewerUserId: string,
workspaceId: string,
): Promise<string[]> {
if (pageIds.length === 0) return [];
const spaceVisible = await this.db
.selectFrom('pages')
.select('id')
.where('id', 'in', pageIds)
.where('workspaceId', '=', workspaceId)
.where('deletedAt', 'is', null)
.where(
'spaceId',
'in',
this.spaceMemberRepo.getUserSpaceIdsQuery(viewerUserId),
)
.execute();
if (spaceVisible.length === 0) return [];
return this.pagePermissionRepo.filterAccessiblePageIds({
pageIds: spaceVisible.map((r) => r.id),
userId: viewerUserId,
});
}
async lookup(
references: Array<{ sourcePageId: string; transclusionId: string }>,
viewerUserId: string,
@@ -224,10 +262,11 @@ export class TransclusionService {
new Set(references.map((r) => r.sourcePageId)),
);
const accessibleSet = new Set(
await this.pagePermissionRepo.filterAccessiblePageIds({
pageIds: candidatePageIds,
userId: viewerUserId,
}),
await this.filterViewerAccessiblePageIds(
candidatePageIds,
viewerUserId,
workspaceId,
),
);
return this.lookupWithAccessSet(references, accessibleSet, workspaceId);
@@ -336,10 +375,11 @@ export class TransclusionService {
new Set([sourcePageId, ...referencePageIds]),
);
const accessibleSet = new Set(
await this.pagePermissionRepo.filterAccessiblePageIds({
pageIds: candidatePageIds,
userId: viewerUserId,
}),
await this.filterViewerAccessiblePageIds(
candidatePageIds,
viewerUserId,
workspaceId,
),
);
const accessibleIds = candidatePageIds.filter((id) =>
@@ -4,6 +4,7 @@ import {
IsNumber,
IsOptional,
IsString,
IsUUID,
} from 'class-validator';
export class SearchDTO {
@@ -12,7 +13,7 @@ export class SearchDTO {
query: string;
@IsOptional()
@IsString()
@IsUUID()
spaceId: string;
@IsOptional()
@@ -20,7 +21,7 @@ export class SearchDTO {
shareId?: string;
@IsOptional()
@IsString()
@IsUUID()
creatorId?: string;
@IsOptional()
@@ -38,7 +39,7 @@ export class SearchShareDTO extends SearchDTO {
shareId: string;
@IsOptional()
@IsString()
@IsUUID()
spaceId: string;
}
@@ -1,7 +1,7 @@
import {
IsAlphanumeric,
IsOptional,
IsString,
Matches,
MaxLength,
MinLength,
} from 'class-validator';
@@ -20,6 +20,9 @@ export class CreateSpaceDto {
@MinLength(2)
@MaxLength(100)
@IsAlphanumeric()
@Matches(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/, {
message:
'Space slug must start with a letter or number and may contain hyphens and underscores',
})
slug: string;
}
@@ -48,6 +48,7 @@ export class SpaceService {
workspaceId: string,
createSpaceDto: CreateSpaceDto,
trx?: KyselyTransaction,
options?: { isPersonal?: boolean },
): Promise<Space> {
let space = null;
@@ -59,6 +60,7 @@ export class SpaceService {
workspaceId,
createSpaceDto,
trx,
options,
);
await this.spaceMemberService.addUserToSpace(
@@ -81,6 +83,7 @@ export class SpaceService {
after: {
name: space.name,
slug: space.slug,
...(space.isPersonal ? { isPersonal: true } : {}),
},
},
});
@@ -93,6 +96,7 @@ export class SpaceService {
workspaceId: string,
createSpaceDto: CreateSpaceDto,
trx?: KyselyTransaction,
options?: { isPersonal?: boolean },
): Promise<Space> {
const slugExists = await this.spaceRepo.slugExists(
createSpaceDto.slug,
@@ -112,6 +116,7 @@ export class SpaceService {
creatorId: userId,
workspaceId: workspaceId,
slug: createSpaceDto.slug,
isPersonal: options?.isPersonal ?? false,
},
trx,
);
@@ -11,7 +11,7 @@ import {
MaxLength,
MinLength,
} from 'class-validator';
import { UserRole } from '../../../common/helpers/types/permission';
import { InviteUserRole } from '../../../common/helpers/types/permission';
import { NoUrls } from '../../../common/validators/no-urls.validator';
export class InviteUserDto {
@@ -32,7 +32,7 @@ export class InviteUserDto {
@IsUUID('all', { each: true })
groupIds: string[];
@IsEnum(UserRole)
@IsEnum(InviteUserRole)
role: string;
}
@@ -3,8 +3,10 @@ import { CreateWorkspaceDto } from './create-workspace.dto';
import {
IsArray,
IsBoolean,
IsIn,
IsInt,
IsOptional,
IsString,
Min,
} from 'class-validator';
@@ -57,4 +59,13 @@ export class UpdateWorkspaceDto extends PartialType(CreateWorkspaceDto) {
@IsOptional()
@IsBoolean()
allowMemberTemplates: boolean;
@IsOptional()
@IsBoolean()
allowPersonalSpaces: boolean;
@IsOptional()
@IsString()
@IsIn(['read', 'edit'])
defaultPageEditMode: string;
}
@@ -1,5 +1,6 @@
import {
BadRequestException,
ForbiddenException,
Inject,
Injectable,
Logger,
@@ -40,6 +41,10 @@ import {
AUDIT_SERVICE,
IAuditService,
} from '../../../integrations/audit/audit.service';
import {
getWorkspaceDefaultPageEditMode,
isAdminActingOnOwner,
} from '../workspace.util';
@Injectable()
export class WorkspaceInvitationService {
@@ -119,6 +124,10 @@ export class WorkspaceInvitationService {
): Promise<void> {
const { emails, role, groupIds } = inviteUserDto;
if (isAdminActingOnOwner(authUser.role, role)) {
throw new ForbiddenException();
}
let invites: WorkspaceInvitation[] = [];
try {
@@ -251,6 +260,7 @@ export class WorkspaceInvitationService {
workspaceId: workspace.id,
},
trx,
{ pageEditMode: getWorkspaceDefaultPageEditMode(workspace) },
);
// add user to default group
@@ -30,6 +30,7 @@ import { DomainService } from '../../../integrations/environment/domain.service'
import { jsonArrayFrom } from 'kysely/helpers/postgres';
import { addDays } from 'date-fns';
import { DISALLOWED_HOSTNAMES, WorkspaceStatus } from '../workspace.constants';
import { isAdminActingOnOwner } from '../workspace.util';
import { v4 } from 'uuid';
import { InjectQueue } from '@nestjs/bullmq';
import { QueueJob, QueueName } from '../../../integrations/queue/constants';
@@ -332,7 +333,8 @@ export class WorkspaceService {
typeof updateWorkspaceDto.mcpEnabled !== 'undefined' ||
typeof updateWorkspaceDto.restrictApiToAdmins !== 'undefined' ||
typeof updateWorkspaceDto.allowMemberTemplates !== 'undefined' ||
typeof updateWorkspaceDto.isScimEnabled !== 'undefined'
typeof updateWorkspaceDto.isScimEnabled !== 'undefined' ||
typeof updateWorkspaceDto.allowPersonalSpaces !== 'undefined'
) {
const ws = await this.db
.selectFrom('workspaces')
@@ -360,6 +362,18 @@ export class WorkspaceService {
}
}
if (typeof updateWorkspaceDto.allowPersonalSpaces !== 'undefined') {
if (
!this.licenseCheckService.hasFeature(
ws.licenseKey,
Feature.PERSONAL_SPACES,
ws.plan,
)
) {
throw new ForbiddenException('This feature requires a valid license');
}
}
if (
typeof updateWorkspaceDto.disablePublicSharing !== 'undefined' ||
typeof updateWorkspaceDto.trashRetentionDays !== 'undefined' ||
@@ -499,6 +513,34 @@ export class WorkspaceService {
);
}
if (typeof updateWorkspaceDto.allowPersonalSpaces !== 'undefined') {
const prev = settingsBefore?.spaces?.allowPersonal ?? false;
if (prev !== updateWorkspaceDto.allowPersonalSpaces) {
before.allowPersonalSpaces = prev;
after.allowPersonalSpaces = updateWorkspaceDto.allowPersonalSpaces;
}
await this.workspaceRepo.updateSpaceSettings(
workspaceId,
'allowPersonal',
updateWorkspaceDto.allowPersonalSpaces,
trx,
);
}
if (typeof updateWorkspaceDto.defaultPageEditMode !== 'undefined') {
const prev = settingsBefore?.defaultPageEditMode ?? null;
const next = updateWorkspaceDto.defaultPageEditMode.toLowerCase();
if (prev !== next) {
before.defaultPageEditMode = prev;
after.defaultPageEditMode = next;
}
await this.workspaceRepo.updateDefaultPageEditMode(
workspaceId,
next,
trx,
);
}
delete updateWorkspaceDto.restrictApiToAdmins;
delete updateWorkspaceDto.aiSearch;
delete updateWorkspaceDto.generativeAi;
@@ -506,6 +548,8 @@ export class WorkspaceService {
delete updateWorkspaceDto.mcpEnabled;
delete updateWorkspaceDto.allowMemberTemplates;
delete updateWorkspaceDto.aiChat;
delete updateWorkspaceDto.allowPersonalSpaces;
delete updateWorkspaceDto.defaultPageEditMode;
await this.workspaceRepo.updateWorkspace(
updateWorkspaceDto,
@@ -590,8 +634,8 @@ export class WorkspaceService {
// prevent ADMIN from managing OWNER role
if (
(authUser.role === UserRole.ADMIN && newRole === UserRole.OWNER) ||
(authUser.role === UserRole.ADMIN && user.role === UserRole.OWNER)
isAdminActingOnOwner(authUser.role, newRole) ||
isAdminActingOnOwner(authUser.role, user.role)
) {
throw new ForbiddenException();
}
@@ -695,7 +739,7 @@ export class WorkspaceService {
throw new BadRequestException('You cannot deactivate yourself');
}
if (authUser.role === UserRole.ADMIN && user.role === UserRole.OWNER) {
if (isAdminActingOnOwner(authUser.role, user.role)) {
throw new BadRequestException(
'You cannot deactivate a user with owner role',
);
@@ -753,7 +797,7 @@ export class WorkspaceService {
throw new BadRequestException('User is not deactivated');
}
if (authUser.role === UserRole.ADMIN && user.role === UserRole.OWNER) {
if (isAdminActingOnOwner(authUser.role, user.role)) {
throw new BadRequestException(
'You cannot activate a user with owner role',
);
@@ -805,7 +849,7 @@ export class WorkspaceService {
throw new BadRequestException('You cannot delete yourself');
}
if (authUser.role === UserRole.ADMIN && user.role === UserRole.OWNER) {
if (isAdminActingOnOwner(authUser.role, user.role)) {
throw new BadRequestException('You cannot delete a user with owner role');
}
@@ -0,0 +1,23 @@
import { UserRole } from '../../common/helpers/types/permission';
export function isAdminActingOnOwner(
authUserRole: string,
targetRole: string,
): boolean {
return authUserRole === UserRole.ADMIN && targetRole === UserRole.OWNER;
}
export type PageEditMode = 'read' | 'edit';
export function getWorkspaceDefaultPageEditMode(
workspace: { settings?: unknown } | null | undefined,
): PageEditMode | undefined {
const settings = (workspace?.settings ?? {}) as {
defaultPageEditMode?: unknown;
};
const mode = settings.defaultPageEditMode;
if (mode === 'read' || mode === 'edit') {
return mode;
}
return undefined;
}
@@ -0,0 +1,249 @@
import { type Kysely, sql } from 'kysely';
export async function up(db: Kysely<any>): Promise<void> {
await db.schema
.alterTable('pages')
.addColumn('is_base', 'boolean', (col) =>
col.ifNotExists().notNull().defaultTo(false),
)
.addColumn('base_schema_version', 'integer', (col) =>
col.ifNotExists().notNull().defaultTo(0),
)
.execute();
await sql`
CREATE INDEX IF NOT EXISTS idx_pages_is_base
ON pages (space_id, position COLLATE "C")
WHERE is_base = true AND deleted_at IS NULL
`.execute(db);
await db.schema
.createTable('base_properties')
.ifNotExists()
.addColumn('id', 'varchar', (col) => col.notNull())
.addColumn('page_id', 'uuid', (col) =>
col.references('pages.id').onDelete('cascade').notNull(),
)
.addColumn('name', 'varchar', (col) => col.notNull())
.addColumn('type', 'varchar', (col) => col.notNull())
.addColumn('position', 'varchar', (col) => col.notNull())
.addColumn('type_options', 'jsonb')
.addColumn('pending_type', 'varchar')
.addColumn('pending_type_options', 'jsonb')
.addColumn('pending_token', 'uuid')
.addColumn('is_primary', 'boolean', (col) => col.notNull().defaultTo(false))
.addColumn('schema_version', 'integer', (col) => col.notNull().defaultTo(1))
.addColumn('workspace_id', 'uuid', (col) =>
col.references('workspaces.id').onDelete('cascade').notNull(),
)
.addColumn('created_at', 'timestamptz', (col) =>
col.notNull().defaultTo(sql`now()`),
)
.addColumn('updated_at', 'timestamptz', (col) =>
col.notNull().defaultTo(sql`now()`),
)
.addColumn('deleted_at', 'timestamptz')
.addPrimaryKeyConstraint('base_properties_pkey', ['page_id', 'id'])
.execute();
await sql`CREATE INDEX IF NOT EXISTS idx_base_properties_page_id ON base_properties (page_id)`.execute(
db,
);
await sql`
CREATE INDEX IF NOT EXISTS idx_base_properties_page_alive
ON base_properties (page_id, position COLLATE "C", id)
WHERE deleted_at IS NULL
`.execute(db);
// Match the service-layer name check (name.trim().toLowerCase()) so
// whitespace-padded duplicates also collide. Formulas look properties up by
// name, so the names have to stay unique.
await sql`
CREATE UNIQUE INDEX IF NOT EXISTS base_properties_page_name_alive_unique
ON base_properties (page_id, lower(trim(name)))
WHERE deleted_at IS NULL
`.execute(db);
await db.schema
.createTable('base_rows')
.ifNotExists()
.addColumn('id', 'uuid', (col) =>
col.primaryKey().defaultTo(sql`gen_uuid_v7()`),
)
.addColumn('page_id', 'uuid', (col) =>
col.references('pages.id').onDelete('cascade').notNull(),
)
.addColumn('cells', 'jsonb', (col) =>
col.notNull().defaultTo(sql`'{}'::jsonb`),
)
.addColumn('position', 'varchar', (col) => col.notNull())
.addColumn('creator_id', 'uuid', (col) =>
col.references('users.id').onDelete('set null'),
)
.addColumn('last_updated_by_id', 'uuid', (col) =>
col.references('users.id').onDelete('set null'),
)
.addColumn('workspace_id', 'uuid', (col) =>
col.references('workspaces.id').onDelete('cascade').notNull(),
)
.addColumn('created_at', 'timestamptz', (col) =>
col.notNull().defaultTo(sql`now()`),
)
.addColumn('updated_at', 'timestamptz', (col) =>
col.notNull().defaultTo(sql`now()`),
)
.addColumn('deleted_at', 'timestamptz')
.execute();
await sql`
CREATE INDEX IF NOT EXISTS idx_base_rows_page_alive
ON base_rows (page_id, position COLLATE "C", id)
WHERE deleted_at IS NULL
`.execute(db);
await sql`
CREATE INDEX IF NOT EXISTS idx_base_rows_page_updated
ON base_rows (page_id, updated_at DESC)
WHERE deleted_at IS NULL
`.execute(db);
await sql`
CREATE INDEX IF NOT EXISTS idx_base_rows_page_created
ON base_rows (page_id, created_at DESC)
WHERE deleted_at IS NULL
`.execute(db);
await db.schema
.createTable('base_views')
.ifNotExists()
.addColumn('id', 'uuid', (col) =>
col.primaryKey().defaultTo(sql`gen_uuid_v7()`),
)
.addColumn('page_id', 'uuid', (col) =>
col.references('pages.id').onDelete('cascade').notNull(),
)
.addColumn('name', 'varchar', (col) => col.notNull())
.addColumn('type', 'varchar', (col) => col.notNull().defaultTo('table'))
.addColumn('position', 'varchar', (col) => col.notNull())
.addColumn('config', 'jsonb', (col) =>
col.notNull().defaultTo(sql`'{}'::jsonb`),
)
.addColumn('workspace_id', 'uuid', (col) =>
col.references('workspaces.id').onDelete('cascade').notNull(),
)
.addColumn('creator_id', 'uuid', (col) =>
col.references('users.id').onDelete('set null'),
)
.addColumn('created_at', 'timestamptz', (col) =>
col.notNull().defaultTo(sql`now()`),
)
.addColumn('updated_at', 'timestamptz', (col) =>
col.notNull().defaultTo(sql`now()`),
)
.execute();
await sql`CREATE INDEX IF NOT EXISTS idx_base_views_page_id ON base_views (page_id)`.execute(
db,
);
// Cell extraction helpers for filters and sorts. Return NULL for absent or
// non-castable values.
await sql`
CREATE OR REPLACE FUNCTION base_cell_text(cells jsonb, prop text)
RETURNS text LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE
AS $$ SELECT cells->>prop::text $$
`.execute(db);
await sql`
CREATE OR REPLACE FUNCTION base_cell_numeric(cells jsonb, prop text)
RETURNS numeric LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE
AS $$
SELECT CASE jsonb_typeof(cells->prop::text)
WHEN 'number' THEN (cells->>prop::text)::numeric
WHEN 'string' THEN
CASE
WHEN (cells->>prop::text) ~
'^[[:space:]]*[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?[[:space:]]*$'
THEN (cells->>prop::text)::numeric
END
END
$$
`.execute(db);
// A DATE cell stores an arbitrary string (cell schema is z.string()), so the
// cast can fail on values no regex can pre-validate (e.g. '2024-13-45'). This
// helper uses plpgsql with an EXCEPTION handler to return NULL on failure.
await sql`
CREATE OR REPLACE FUNCTION base_cell_timestamptz(cells jsonb, prop text)
RETURNS timestamptz LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE
AS $$
BEGIN RETURN (cells->>prop::text)::timestamptz;
EXCEPTION WHEN others THEN RETURN NULL; END;
$$
`.execute(db);
await sql`
CREATE OR REPLACE FUNCTION base_cell_bool(cells jsonb, prop text)
RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE
AS $$
SELECT CASE jsonb_typeof(cells->prop::text)
WHEN 'boolean' THEN (cells->>prop::text)::boolean
WHEN 'string' THEN
CASE
WHEN lower(btrim(cells->>prop::text)) IN
('true','t','yes','y','on','1','false','f','no','n','off','0')
THEN (cells->>prop::text)::boolean
END
END
$$
`.execute(db);
await sql`
CREATE OR REPLACE FUNCTION base_cell_array(cells jsonb, prop text)
RETURNS jsonb LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE
AS $$ SELECT cells->prop::text $$
`.execute(db);
// A null patch value deletes the key rather than storing a JSON null.
await sql`
CREATE OR REPLACE FUNCTION jsonb_set_many(target jsonb, patches jsonb)
RETURNS jsonb LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE
AS $$
DECLARE k text; v jsonb; result jsonb := coalesce(target, '{}'::jsonb);
BEGIN
IF patches IS NULL OR jsonb_typeof(patches) <> 'object' THEN
RETURN result;
END IF;
FOR k, v IN SELECT * FROM jsonb_each(patches) LOOP
IF v = 'null'::jsonb THEN
result := result - k;
ELSE
result := jsonb_set(result, ARRAY[k], v, true);
END IF;
END LOOP;
RETURN result;
END;
$$
`.execute(db);
}
export async function down(db: Kysely<any>): Promise<void> {
await db.schema.dropTable('base_views').execute();
await db.schema.dropTable('base_rows').execute();
await db.schema.dropTable('base_properties').execute();
await sql`DROP FUNCTION jsonb_set_many(jsonb, jsonb)`.execute(db);
await sql`DROP FUNCTION base_cell_array(jsonb, text)`.execute(db);
await sql`DROP FUNCTION base_cell_bool(jsonb, text)`.execute(db);
await sql`DROP FUNCTION base_cell_timestamptz(jsonb, text)`.execute(db);
await sql`DROP FUNCTION base_cell_numeric(jsonb, text)`.execute(db);
await sql`DROP FUNCTION base_cell_text(jsonb, text)`.execute(db);
await sql`DROP INDEX idx_pages_is_base`.execute(db);
await db.schema
.alterTable('pages')
.dropColumn('base_schema_version')
.dropColumn('is_base')
.execute();
}
@@ -0,0 +1,24 @@
import { Kysely, sql } from 'kysely';
export async function up(db: Kysely<any>): Promise<void> {
await db.schema
.alterTable('spaces')
.addColumn('is_personal', 'boolean', (col) =>
col.notNull().defaultTo(false),
)
.execute();
await sql`
CREATE UNIQUE INDEX spaces_personal_creator_unique
ON spaces (creator_id)
WHERE is_personal = true AND deleted_at IS NULL
`.execute(db);
}
export async function down(db: Kysely<any>): Promise<void> {
await db.schema
.dropIndex('spaces_personal_creator_unique')
.ifExists()
.execute();
await db.schema.alterTable('spaces').dropColumn('is_personal').execute();
}
@@ -14,12 +14,14 @@ type SortField<DB, TB extends keyof DB, O> =
| (StringReference<DB, TB> & `${string}.${keyof O & string}`);
direction: OrderByDirection;
orderModifier?: OrderByModifiers;
cursorExpression?: ReferenceExpression<DB, TB>;
key?: keyof O & string;
}
| {
expression: ReferenceExpression<DB, TB>;
direction: OrderByDirection;
orderModifier?: OrderByModifiers;
cursorExpression?: ReferenceExpression<DB, TB>;
key: keyof O & string;
};
@@ -202,11 +204,12 @@ export async function executeWithCursorPagination<
const comparison = field.direction === defaultDirection ? '>' : '<';
const value = cursor[field.key as keyof typeof cursor];
const compareExpr = field.cursorExpression ?? field.expression;
const conditions = [eb(field.expression, comparison, value)];
const conditions = [eb(compareExpr, comparison, value)];
if (expression) {
conditions.push(and([eb(field.expression, '=', value), expression]));
conditions.push(and([eb(compareExpr, '=', value), expression]));
}
expression = or(conditions);
@@ -236,6 +236,7 @@ export class FavoriteRepo {
'pages.slugId',
'pages.title',
'pages.icon',
'pages.isBase',
'pages.spaceId',
])
.whereRef('pages.id', '=', 'favorites.pageId'),
@@ -38,6 +38,7 @@ export class PageRepo {
'spaceId',
'workspaceId',
'isLocked',
'isBase',
'createdAt',
'updatedAt',
'deletedAt',
@@ -57,6 +57,22 @@ export class SpaceRepo {
.executeTakeFirst();
}
async findPersonalSpace(
userId: string,
workspaceId: string,
trx?: KyselyTransaction,
): Promise<Space | undefined> {
const db = dbOrTx(this.db, trx);
return db
.selectFrom('spaces')
.selectAll('spaces')
.where('workspaceId', '=', workspaceId)
.where('creatorId', '=', userId)
.where('isPersonal', '=', true)
.where('deletedAt', 'is', null)
.executeTakeFirst();
}
async slugExists(
slug: string,
workspaceId: string,
@@ -112,6 +112,7 @@ export class UserRepo {
async insertUser(
insertableUser: InsertableUser,
trx?: KyselyTransaction,
opts?: { pageEditMode?: string },
): Promise<User> {
const user: InsertableUser = {
name:
@@ -126,7 +127,17 @@ export class UserRepo {
const db = dbOrTx(this.db, trx);
return db
.insertInto('users')
.values({ ...insertableUser, ...user })
.values({
...insertableUser,
...user,
...(opts?.pageEditMode
? {
settings: sql`${JSON.stringify({
preferences: { pageEditMode: opts.pageEditMode },
})}::text::jsonb`,
}
: {}),
})
.returning(this.baseFields)
.executeTakeFirst();
}
@@ -251,4 +251,42 @@ export class WorkspaceRepo {
.executeTakeFirst();
}
async updateSpaceSettings(
workspaceId: string,
prefKey: string,
prefValue: string | boolean,
trx?: KyselyTransaction,
) {
const db = dbOrTx(this.db, trx);
return db
.updateTable('workspaces')
.set({
settings: sql`COALESCE(settings, '{}'::jsonb)
|| jsonb_build_object('spaces', COALESCE(settings->'spaces', '{}'::jsonb)
|| jsonb_build_object('${sql.raw(prefKey)}', ${sql.lit(prefValue)}))`,
updatedAt: new Date(),
})
.where('id', '=', workspaceId)
.returning(this.baseFields)
.executeTakeFirst();
}
async updateDefaultPageEditMode(
workspaceId: string,
pageEditMode: string,
trx?: KyselyTransaction,
) {
const db = dbOrTx(this.db, trx);
return db
.updateTable('workspaces')
.set({
settings: sql`COALESCE(settings, '{}'::jsonb)
|| jsonb_build_object('defaultPageEditMode', ${sql.lit(pageEditMode)})`,
updatedAt: new Date(),
})
.where('id', '=', workspaceId)
.returning(this.baseFields)
.executeTakeFirst();
}
}
+50
View File
@@ -126,6 +126,50 @@ export interface Backlinks {
workspaceId: string;
}
export interface BaseProperties {
createdAt: Generated<Timestamp>;
deletedAt: Timestamp | null;
id: Generated<string>;
isPrimary: Generated<boolean>;
name: string;
pageId: string;
pendingType: string | null;
pendingTypeOptions: Json | null;
pendingToken: string | null;
position: string;
schemaVersion: Generated<number>;
type: string;
typeOptions: Json | null;
updatedAt: Generated<Timestamp>;
workspaceId: string;
}
export interface BaseRows {
cells: Generated<Json>;
createdAt: Generated<Timestamp>;
creatorId: string | null;
deletedAt: Timestamp | null;
id: Generated<string>;
lastUpdatedById: string | null;
pageId: string;
position: string;
updatedAt: Generated<Timestamp>;
workspaceId: string;
}
export interface BaseViews {
config: Generated<Json>;
createdAt: Generated<Timestamp>;
creatorId: string | null;
id: Generated<string>;
name: string;
pageId: string;
position: string;
type: Generated<string>;
updatedAt: Generated<Timestamp>;
workspaceId: string;
}
export interface Billing {
amount: Int8 | null;
billingScheme: string | null;
@@ -275,6 +319,8 @@ export interface Pages {
deletedById: string | null;
icon: string | null;
id: Generated<string>;
isBase: Generated<boolean>;
baseSchemaVersion: Generated<number>;
isLocked: Generated<boolean>;
lastUpdatedById: string | null;
parentPageId: string | null;
@@ -322,6 +368,7 @@ export interface Spaces {
deletedAt: Timestamp | null;
description: string | null;
id: Generated<string>;
isPersonal: Generated<boolean>;
logo: string | null;
name: string | null;
settings: Json | null;
@@ -638,6 +685,9 @@ export interface DB {
authAccounts: AuthAccounts;
authProviders: AuthProviders;
backlinks: Backlinks;
baseProperties: BaseProperties;
baseRows: BaseRows;
baseViews: BaseViews;
billing: Billing;
comments: Comments;
favorites: Favorites;
@@ -3,6 +3,9 @@ import {
AiChats,
AiChatMessages,
Attachments,
BaseProperties,
BaseRows,
BaseViews,
Comments,
Groups,
Integrations as _Integrations,
@@ -261,3 +264,18 @@ export type UpdatableAudit = Updateable<Omit<_Audit, 'id'>>;
export type Template = Selectable<Templates>;
export type InsertableTemplate = Insertable<Templates>;
export type UpdatableTemplate = Updateable<Omit<Templates, 'id'>>;
// Base Property
export type BaseProperty = Selectable<BaseProperties>;
export type InsertableBaseProperty = Insertable<BaseProperties>;
export type UpdatableBaseProperty = Updateable<Omit<BaseProperties, 'id'>>;
// Base Row
export type BaseRow = Selectable<BaseRows>;
export type InsertableBaseRow = Insertable<BaseRows>;
export type UpdatableBaseRow = Updateable<Omit<BaseRows, 'id'>>;
// Base View
export type BaseView = Selectable<BaseViews>;
export type InsertableBaseView = Insertable<BaseViews>;
export type UpdatableBaseView = Updateable<Omit<BaseViews, 'id'>>;
@@ -16,6 +16,7 @@ import {
computeLocalPath,
getExportExtension,
getPageTitle,
getSafePageTitle,
PageExportTree,
replaceInternalLinks,
updateAttachmentUrlsToLocalPaths,
@@ -314,7 +315,7 @@ export class ExportService {
updateAttachmentUrlsToLocalPaths(updatedJsonContent);
}
const pageTitle = getPageTitle(page.title);
const pageTitle = getSafePageTitle(page.title);
const pageExportContent = await this.exportPage(format, {
...page,
content: updatedJsonContent,
+9 -1
View File
@@ -6,6 +6,7 @@ import { validate as isValidUUID } from 'uuid';
import * as path from 'path';
import { Page } from '@docmost/db/types/entity.types';
import { isAttachmentNode } from '../../common/helpers/prosemirror/utils';
import { sanitizeFileName } from '../../common/helpers';
export type PageExportTree = Record<string, Page[]>;
@@ -27,6 +28,13 @@ export function getPageTitle(title: string) {
return title ? title : 'untitled';
}
export function getSafePageTitle(title: string): string {
const sanitized = sanitizeFileName(getPageTitle(title), {
preserveSpaces: true,
});
return sanitized || 'untitled';
}
export function updateAttachmentUrlsToLocalPaths(prosemirrorJson: any) {
const doc = jsonToNode(prosemirrorJson);
if (!doc) return null;
@@ -167,7 +175,7 @@ export function computeLocalPath(
const children = tree[parentPageId] || [];
for (const page of children) {
const title = encodeURIComponent(getPageTitle(page.title));
const title = encodeURIComponent(getSafePageTitle(page.title));
const localPath = `${currentPath}${title}`;
slugIdToPath[page.slugId] = `${localPath}${getExportExtension(format)}`;
@@ -102,8 +102,10 @@ export class ImportService {
throw new BadRequestException(message);
}
const { title, prosemirrorJson } =
this.extractTitleAndRemoveHeading(prosemirrorState);
const { title, prosemirrorJson } = this.extractTitleAndRemoveHeading(
prosemirrorState,
{ anyHeadingLevel: true },
);
const pageTitle = title || fileName;
@@ -246,18 +248,29 @@ export class ImportService {
return null;
}
extractTitleAndRemoveHeading(prosemirrorState: any) {
extractTitleAndRemoveHeading(
prosemirrorState: any,
opts?: { anyHeadingLevel?: boolean },
) {
let title: string | null = null;
const content = prosemirrorState.content ?? [];
const firstNode = content[0];
if (
content.length > 0 &&
content[0].type === 'heading' &&
content[0].attrs?.level === 1
) {
title = content[0].content?.[0]?.text ?? null;
content.shift();
const isTitleHeading =
firstNode?.type === 'heading' &&
(opts?.anyHeadingLevel || firstNode.attrs?.level === 1);
if (isTitleHeading) {
const headingText = (firstNode.content ?? [])
.map((node: any) => node.text ?? '')
.join('')
.trim();
if (headingText) {
title = headingText;
content.shift();
}
}
// ensure at least one paragraph
@@ -31,31 +31,39 @@ export function getFileTaskFolderPath(
}
}
/**
* Extracts a ZIP archive.
*/
const COMPRESSION_HEADROOM = 10;
const MIN_EXTRACTED_BYTES = 256 * 1024 * 1024;
const MAX_ENTRIES = 250_000;
type SizeBudget = { used: number; max: number };
export async function extractZip(
source: string,
target: string,
): Promise<void> {
return extractZipInternal(source, target, true);
const { size: compressedSize } = await fs.promises.stat(source);
const max = Math.max(
compressedSize * COMPRESSION_HEADROOM,
MIN_EXTRACTED_BYTES,
);
return extractZipInternal(source, target, true, { used: 0, max });
}
/**
* Internal helper to extract a ZIP, with optional single-nested-ZIP handling.
* @param source Path to the ZIP file
* @param target Directory to extract into
* @param allowNested Whether to check and unwrap one level of nested ZIP
*/
function extractZipInternal(
source: string,
target: string,
allowNested: boolean,
budget: SizeBudget,
): Promise<void> {
return new Promise((resolve, reject) => {
yauzl.open(
source,
{ lazyEntries: true, decodeStrings: false, autoClose: true },
{
lazyEntries: true,
decodeStrings: false,
autoClose: true,
validateEntrySizes: true,
},
(err, zipfile) => {
if (err) return reject(err);
@@ -73,6 +81,15 @@ function extractZipInternal(
? source.slice(0, -4) + '.inner.zip'
: source + '.inner.zip';
budget.used += entry.uncompressedSize;
if (budget.used > budget.max) {
return reject(
new Error(
'Import archive exceeds the allowed extracted size limit',
),
);
}
zipfile.openReadStream(entry, (openErr, rs) => {
if (openErr) return reject(openErr);
const ws = fs.createWriteStream(nestedPath);
@@ -80,7 +97,7 @@ function extractZipInternal(
ws.on('error', reject);
ws.on('finish', () => {
zipfile.close();
extractZipInternal(nestedPath, target, false)
extractZipInternal(nestedPath, target, false, budget)
.then(() => {
fs.unlinkSync(nestedPath);
resolve();
@@ -91,13 +108,21 @@ function extractZipInternal(
});
} else {
zipfile.close();
extractZipInternal(source, target, false).then(resolve, reject);
extractZipInternal(source, target, false, budget).then(
resolve,
reject,
);
}
});
zipfile.once('error', reject);
return;
}
if (zipfile.entryCount > MAX_ENTRIES) {
zipfile.close();
return reject(new Error('Import archive has too many entries'));
}
// Normal extraction
zipfile.readEntry();
zipfile.on('entry', (entry) => {
@@ -143,6 +168,15 @@ function extractZipInternal(
return;
}
budget.used += entry.uncompressedSize;
if (budget.used > budget.max) {
return reject(
new Error(
'Import archive exceeds the allowed extracted size limit',
),
);
}
// Handle files
try {
fs.mkdirSync(path.dirname(fullPath), { recursive: true });
@@ -14,6 +14,7 @@ export enum QueueName {
// Separate queue for /docmost ask: AI work takes seconds and would
// otherwise starve fast inbound event dispatch.
SLACK_ASK = '{slack-ask}',
BASE_QUEUE = '{base-queue}',
}
export enum QueueJob {
@@ -93,4 +94,8 @@ export enum QueueJob {
INTEGRATION_TOKEN_REFRESH = 'integration-token-refresh',
SLACK_EVENT = 'slack-event',
SLACK_ASK = 'slack-ask',
BASE_TYPE_CONVERSION = 'base-type-conversion',
BASE_CELL_GC = 'base-cell-gc',
BASE_FORMULA_RECOMPUTE = 'base-formula-recompute',
}
@@ -113,3 +113,47 @@ export interface IApprovalRejectedNotificationJob {
requestedById: string;
comment?: string;
}
export interface IBaseTypeConversionJob {
pageId: string;
propertyId: string;
workspaceId: string;
fromType: string;
toType: string;
// Snapshots taken at enqueue time so the job stays correct even if the
// property's current typeOptions drift while the job waits in the queue.
fromTypeOptions: unknown;
toTypeOptions: unknown;
// When true, the job nulls the cell values for that property instead of
// attempting a value conversion. Used for any conversion where the new
// type has no meaningful representation of the old value (e.g. involving
// a system type).
clearMode: boolean;
// Staging identity: guards redelivery and failure cleanup against a
// same-type re-stage made after this job was enqueued.
pendingToken: string;
actorId?: string;
}
export interface IBaseCellGcJob {
pageId: string;
propertyId: string;
workspaceId: string;
}
export interface IBaseFormulaRecomputeJob {
pageId: string;
workspaceId: string;
propertyIds: string[]; // formula properties to recompute
reason:
| 'formula_created'
| 'formula_edited'
| 'dep_type_changed'
| 'dep_deleted'
| 'bulk_import'
| 'manual';
actorId?: string | null;
// When set, scope recompute to these row IDs instead of the whole base.
// Used by the bulk-write path (> FORMULA_INLINE_ROW_THRESHOLD).
rowIds?: string[];
}
@@ -100,6 +100,14 @@ import { GeneralQueueProcessor } from './processors/general-queue.processor';
attempts: 3,
},
}),
BullModule.registerQueue({
name: QueueName.BASE_QUEUE,
defaultJobOptions: {
attempts: 2,
removeOnComplete: { count: 200 },
removeOnFail: { count: 100 },
},
}),
],
exports: [BullModule],
providers: [GeneralQueueProcessor],
@@ -15,13 +15,21 @@ import { getMimeType } from '../../../common/helpers';
import { Upload } from '@aws-sdk/lib-storage';
import { Logger } from '@nestjs/common';
const S3_MAX_SOCKETS = parseInt(process.env.AWS_S3_MAX_SOCKETS) || 200;
export class S3Driver implements StorageDriver {
private readonly s3Client: S3Client;
private readonly config: S3StorageConfig;
constructor(config: S3StorageConfig) {
this.config = config;
this.s3Client = new S3Client(config as any);
this.config = {
...config,
requestHandler: {
httpAgent: { maxSockets: S3_MAX_SOCKETS },
httpsAgent: { maxSockets: S3_MAX_SOCKETS },
},
};
this.s3Client = new S3Client(this.config as any);
}
async upload(filePath: string, file: Buffer | Readable): Promise<void> {
@@ -0,0 +1,49 @@
import { Injectable, Logger } from '@nestjs/common';
import { ModuleRef } from '@nestjs/core';
import { Server, Socket } from 'socket.io';
@Injectable()
export class BaseRealtimeBridge {
private readonly logger = new Logger(BaseRealtimeBridge.name);
private resolved = false;
private svc: any = null;
constructor(private readonly moduleRef: ModuleRef) {}
protected loadServiceClass(): any {
try {
// eslint-disable-next-line @typescript-eslint/no-require-imports
return require('../ee/base/realtime/base-ws.service').BaseWsService;
} catch {
this.logger.debug(
'Base realtime requested but enterprise module not bundled in this build',
);
return null;
}
}
private resolve(): any {
if (this.resolved) return this.svc;
this.resolved = true;
const ServiceClass = this.loadServiceClass();
if (!ServiceClass) return null;
this.svc = this.moduleRef.get(ServiceClass, { strict: false });
return this.svc;
}
setServer(server: Server): void {
this.resolve()?.setServer(server);
}
isBaseEvent(data: any): boolean {
return this.resolve()?.isBaseEvent(data) ?? false;
}
async handleInbound(client: Socket, data: any): Promise<void> {
await this.resolve()?.handleInbound(client, data);
}
async handleDisconnect(client: Socket): Promise<void> {
await this.resolve()?.handleDisconnect(client);
}
}
+19 -1
View File
@@ -1,6 +1,7 @@
import {
MessageBody,
OnGatewayConnection,
OnGatewayDisconnect,
OnGatewayInit,
SubscribeMessage,
WebSocketGateway,
@@ -13,6 +14,7 @@ import { OnModuleDestroy } from '@nestjs/common';
import { SpaceMemberRepo } from '@docmost/db/repos/space/space-member.repo';
import { WsService } from './ws.service';
import { getSpaceRoomName, getUserRoomName } from './ws.utils';
import { BaseRealtimeBridge } from './base-realtime.bridge';
import * as cookie from 'cookie';
@WebSocketGateway({
@@ -20,7 +22,11 @@ import * as cookie from 'cookie';
transports: ['websocket'],
})
export class WsGateway
implements OnGatewayConnection, OnGatewayInit, OnModuleDestroy
implements
OnGatewayConnection,
OnGatewayDisconnect,
OnGatewayInit,
OnModuleDestroy
{
@WebSocketServer()
server: Server;
@@ -29,10 +35,12 @@ export class WsGateway
private tokenService: TokenService,
private spaceMemberRepo: SpaceMemberRepo,
private wsService: WsService,
private baseRealtime: BaseRealtimeBridge,
) {}
afterInit(server: Server): void {
this.wsService.setServer(server);
this.baseRealtime.setServer(server);
}
async handleConnection(client: Socket, ...args: any[]): Promise<void> {
@@ -47,6 +55,7 @@ export class WsGateway
const workspaceId = token.workspaceId;
client.data.userId = userId;
client.data.workspaceId = workspaceId;
const userSpaceIds = await this.spaceMemberRepo.getUserSpaceIds(userId);
@@ -61,10 +70,19 @@ export class WsGateway
}
}
async handleDisconnect(client: Socket): Promise<void> {
await this.baseRealtime.handleDisconnect(client);
}
@SubscribeMessage('message')
async handleMessage(client: Socket, data: any): Promise<void> {
if (this.wsService.isTreeEvent(data)) {
await this.wsService.handleTreeEvent(client, data);
return;
}
if (this.baseRealtime.isBaseEvent(data)) {
await this.baseRealtime.handleInbound(client, data);
return;
}
}
+2 -1
View File
@@ -3,11 +3,12 @@ import { WsGateway } from './ws.gateway';
import { WsService } from './ws.service';
import { WsTreeService } from './ws-tree.service';
import { TokenModule } from '../core/auth/token.module';
import { BaseRealtimeBridge } from './base-realtime.bridge';
@Global()
@Module({
imports: [TokenModule],
providers: [WsGateway, WsService, WsTreeService],
providers: [WsGateway, WsService, WsTreeService, BaseRealtimeBridge],
exports: [WsGateway, WsService, WsTreeService],
})
export class WsModule {}
+11
View File
@@ -16,3 +16,14 @@ export const TREE_EVENTS = new Set([
'deleteTreeNode',
'refetchRootTreeNodeEvent',
]);
export function getBaseRoomName(pageId: string): string {
return `base-${pageId}`;
}
export const BASE_INBOUND_EVENTS = new Set([
'base:subscribe',
'base:unsubscribe',
'base:presence',
'base:presence:leave',
]);
+7 -1
View File
@@ -22,7 +22,13 @@
"paths": {
"@docmost/db/*": ["./src/database/*"],
"@docmost/transactional/*": ["./src/integrations/transactional/*"],
"@docmost/ee/*": ["./src/ee/*"]
"@docmost/ee/*": ["./src/ee/*"],
"@docmost/base-formula/server": [
"../../packages/base-formula/dist/index.server.d.ts"
],
"@docmost/base-formula/client": [
"../../packages/base-formula/dist/index.client.d.ts"
]
}
}
}