mirror of
https://github.com/docmost/docmost.git
synced 2026-08-20 21:01:37 +10:00
Merge branch 'main' into feat/integrations
# Conflicts: # apps/client/src/App.tsx # apps/server/src/ee # apps/server/src/integrations/queue/constants/queue.constants.ts # apps/server/src/integrations/queue/queue.module.ts # packages/editor-ext/src/index.ts
This commit is contained in:
@@ -15,6 +15,7 @@ import {
|
||||
RedisSyncExtension,
|
||||
SerializedHTTPRequest,
|
||||
} from './extensions/redis-sync';
|
||||
import { toWebRequest } from './extensions/redis-sync/redis-sync.types';
|
||||
import { WsSocketWrapper } from './extensions/redis-sync/ws-socket-wrapper';
|
||||
import RedisClient from 'ioredis';
|
||||
import { pack, unpack } from 'msgpackr';
|
||||
@@ -98,34 +99,36 @@ export class CollaborationGateway {
|
||||
const serializedHTTPRequest = this.serializeRequest(request);
|
||||
const socketId = serializedHTTPRequest.headers['sec-websocket-key'];
|
||||
|
||||
// Create wrapper socket that only receives events via emit()
|
||||
// This prevents double-handling since Hocuspocus won't listen to raw WebSocket events
|
||||
const wrappedSocket = new WsSocketWrapper(client);
|
||||
|
||||
// Route through RedisSync extension (this calls handleConnection internally)
|
||||
this.redisSync.onSocketOpen(wrappedSocket as any, serializedHTTPRequest);
|
||||
this.redisSync.onSocketOpen(wrappedSocket, serializedHTTPRequest);
|
||||
|
||||
// Forward raw WebSocket messages to the extension
|
||||
client.on('message', (data: ArrayBuffer) => {
|
||||
this.redisSync!.onSocketMessage(
|
||||
wrappedSocket as any,
|
||||
serializedHTTPRequest,
|
||||
data,
|
||||
);
|
||||
this.redisSync!.onSocketMessage(serializedHTTPRequest, data);
|
||||
});
|
||||
|
||||
// Forward close events
|
||||
client.on('close', (code: number, reason: Buffer) => {
|
||||
this.redisSync!.onSocketClose(socketId, code, reason.buffer as ArrayBuffer);
|
||||
});
|
||||
|
||||
// Forward pong events for keepalive
|
||||
client.on('pong', (data: Buffer) => {
|
||||
wrappedSocket.emit('pong', data);
|
||||
this.redisSync!.onSocketClose(
|
||||
socketId,
|
||||
code,
|
||||
new Uint8Array(reason).buffer,
|
||||
);
|
||||
});
|
||||
} else {
|
||||
// Fallback to direct Hocuspocus connection
|
||||
this.hocuspocus.handleConnection(client, request);
|
||||
const clientConnection = this.hocuspocus.handleConnection(
|
||||
client,
|
||||
toWebRequest(this.serializeRequest(request)),
|
||||
);
|
||||
|
||||
client.on('message', (data: Buffer) => {
|
||||
clientConnection.handleMessage(new Uint8Array(data));
|
||||
});
|
||||
|
||||
client.on('close', (code: number, reason: Buffer) => {
|
||||
clientConnection.handleClose({ code, reason: reason.toString() });
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -178,6 +181,7 @@ export class CollaborationGateway {
|
||||
|
||||
if (this.hocuspocus.getDocumentsCount() === 0) resolve('');
|
||||
this.hocuspocus.closeConnections();
|
||||
this.hocuspocus.flushPendingStores();
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
}
|
||||
|
||||
@@ -44,6 +44,7 @@ import {
|
||||
htmlToMarkdown,
|
||||
TransclusionSource,
|
||||
TransclusionReference,
|
||||
BaseEmbed,
|
||||
} from '@docmost/editor-ext';
|
||||
import { generateText, getSchema, JSONContent } from '@tiptap/core';
|
||||
import { generateHTML, generateJSON } from '../common/helpers/prosemirror/html';
|
||||
@@ -109,6 +110,7 @@ export const tiptapExtensions = [
|
||||
Status,
|
||||
TransclusionSource,
|
||||
TransclusionReference,
|
||||
BaseEmbed
|
||||
] as any;
|
||||
|
||||
export function jsonToHtml(tiptapJson: any) {
|
||||
|
||||
@@ -96,7 +96,7 @@ export class PersistenceExtension implements Extension {
|
||||
}
|
||||
|
||||
async onStoreDocument(data: onStoreDocumentPayload) {
|
||||
const { documentName, document, context } = data;
|
||||
const { documentName, document, lastContext } = data;
|
||||
|
||||
const pageId = getPageId(documentName);
|
||||
|
||||
@@ -151,7 +151,7 @@ export class PersistenceExtension implements Extension {
|
||||
content: tiptapJson,
|
||||
textContent: textContent,
|
||||
ydoc: ydocState,
|
||||
lastUpdatedById: context.user.id,
|
||||
lastUpdatedById: lastContext.user.id,
|
||||
contributorIds: contributorIds,
|
||||
},
|
||||
pageId,
|
||||
@@ -165,6 +165,21 @@ export class PersistenceExtension implements Extension {
|
||||
}
|
||||
|
||||
if (page) {
|
||||
document.broadcastStateless(
|
||||
JSON.stringify({
|
||||
type: 'page.updated',
|
||||
updatedAt: new Date().toISOString(),
|
||||
lastUpdatedById: lastContext?.user?.id,
|
||||
lastUpdatedBy: lastContext?.user
|
||||
? {
|
||||
id: lastContext.user?.id,
|
||||
name: lastContext.user?.name,
|
||||
avatarUrl: lastContext.user?.avatarUrl,
|
||||
}
|
||||
: undefined,
|
||||
}),
|
||||
);
|
||||
|
||||
await this.syncTransclusion(pageId, page.workspaceId, tiptapJson);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,61 +1,37 @@
|
||||
import type RedisClient from 'ioredis';
|
||||
import { EventEmitter } from 'tseep';
|
||||
import type {
|
||||
Pack,
|
||||
RSAMessageClose,
|
||||
RSAMessagePing,
|
||||
RSAMessageSend,
|
||||
} from './redis-sync.types';
|
||||
import type { WebSocketLike } from '@hocuspocus/server';
|
||||
import type { Pack, RSAMessageClose, RSAMessageSend } from './redis-sync.types';
|
||||
|
||||
export class CollabProxySocket extends EventEmitter {
|
||||
// Stands in for the client WebSocket on the server that owns the document.
|
||||
// Outgoing traffic is relayed over redis to the origin server, which holds the real socket.
|
||||
export class CollabProxySocket implements WebSocketLike {
|
||||
private readonly replyTo: string;
|
||||
private readonly serverChannel: string;
|
||||
private readonly socketId: string;
|
||||
private pub: RedisClient;
|
||||
private readonly pack: Pack;
|
||||
readyState = 1;
|
||||
onClose?: (code?: number, reason?: string) => void;
|
||||
|
||||
constructor(
|
||||
pub: RedisClient,
|
||||
pack: Pack,
|
||||
replyTo: string,
|
||||
serverChannel: string,
|
||||
socketId: string,
|
||||
) {
|
||||
super();
|
||||
constructor(pub: RedisClient, pack: Pack, replyTo: string, socketId: string) {
|
||||
this.replyTo = replyTo;
|
||||
this.socketId = socketId;
|
||||
this.serverChannel = serverChannel;
|
||||
this.pub = pub;
|
||||
this.pack = pack;
|
||||
this.once('close', () => {
|
||||
this.readyState = 3;
|
||||
});
|
||||
}
|
||||
|
||||
private publish(msg: RSAMessageClose | RSAMessagePing | RSAMessageSend) {
|
||||
private publish(msg: RSAMessageClose | RSAMessageSend) {
|
||||
this.pub.publish(this.replyTo, this.pack(msg));
|
||||
}
|
||||
|
||||
// The origin server already closed the real socket; stop relaying without echoing a close back
|
||||
markClosed() {
|
||||
this.readyState = 3;
|
||||
}
|
||||
|
||||
close(code?: number, reason?: string) {
|
||||
if (this.readyState !== 1) return;
|
||||
const msg: RSAMessageClose = {
|
||||
type: 'close',
|
||||
code,
|
||||
reason,
|
||||
socketId: this.socketId,
|
||||
};
|
||||
this.publish(msg);
|
||||
}
|
||||
|
||||
ping() {
|
||||
if (this.readyState !== 1) return;
|
||||
const msg: RSAMessagePing = {
|
||||
type: 'ping',
|
||||
socketId: this.socketId,
|
||||
replyTo: this.serverChannel,
|
||||
};
|
||||
this.publish(msg);
|
||||
this.readyState = 3;
|
||||
this.onClose?.(code, reason);
|
||||
}
|
||||
|
||||
send(message: Uint8Array) {
|
||||
|
||||
@@ -3,27 +3,30 @@ import {
|
||||
Extension,
|
||||
Hocuspocus,
|
||||
IncomingMessage,
|
||||
afterUnloadDocumentPayload,
|
||||
onConfigurePayload,
|
||||
onLoadDocumentPayload,
|
||||
afterUnloadDocumentPayload,
|
||||
WebSocketLike,
|
||||
} from '@hocuspocus/server';
|
||||
import { ConnectionTimeout, Unauthorized } from '@hocuspocus/common';
|
||||
import RedisClient from 'ioredis';
|
||||
import { readVarString } from 'lib0/decoding.js';
|
||||
import { CollabProxySocket } from './collab-proxy-socket';
|
||||
import {
|
||||
BaseWebSocket,
|
||||
Configuration,
|
||||
CustomEvents,
|
||||
Pack,
|
||||
RSAMessage,
|
||||
RSAMessageClose,
|
||||
RSAMessageCloseProxy,
|
||||
RSAMessageCustomEventComplete,
|
||||
RSAMessageCustomEventStart,
|
||||
RSAMessagePong,
|
||||
RSAMessageProxy,
|
||||
RSAMessageUnload,
|
||||
SerializedHTTPRequest,
|
||||
Unpack,
|
||||
OriginConnection,
|
||||
ProxyConnection,
|
||||
toWebRequest,
|
||||
} from './redis-sync.types';
|
||||
|
||||
export type { Pack, SerializedHTTPRequest } from './redis-sync.types';
|
||||
@@ -38,10 +41,10 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
|
||||
private sub: RedisClient;
|
||||
private readonly pack: Pack;
|
||||
private readonly unpack: Unpack;
|
||||
private originSockets: Record<SocketId, BaseWebSocket> = {};
|
||||
private originConnections: Record<SocketId, OriginConnection> = {};
|
||||
private locks: Record<DocumentName, NodeJS.Timeout> = {};
|
||||
private lockPromises: Record<DocumentName, Promise<ServerId | null>> = {};
|
||||
private proxySockets: Record<SocketId, CollabProxySocket> = {};
|
||||
private proxyConnections: Record<SocketId, ProxyConnection> = {};
|
||||
private readonly prefix: string;
|
||||
private readonly lockPrefix: string;
|
||||
private readonly msgChannel: string;
|
||||
@@ -54,6 +57,9 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
|
||||
// @ts-ignore
|
||||
private pendingReplies: Record<number, PromiseWithResolvers<any>['resolve']> =
|
||||
{};
|
||||
private deriveContext: (
|
||||
serializedHTTPRequest: SerializedHTTPRequest,
|
||||
) => Record<string, any>;
|
||||
|
||||
constructor(configuration: Configuration<TCE>) {
|
||||
const {
|
||||
@@ -65,6 +71,7 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
|
||||
prefix,
|
||||
customEvents,
|
||||
customEventTTL,
|
||||
deriveContext,
|
||||
} = configuration;
|
||||
this.pub = redis.duplicate();
|
||||
this.sub = redis.duplicate();
|
||||
@@ -77,6 +84,7 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
|
||||
this.lockPrefix = `${this.prefix}Lock`;
|
||||
this.msgChannel = `${this.prefix}Msg`;
|
||||
this.customEvents = (customEvents as any) ?? ({} as any as CustomEvents);
|
||||
this.deriveContext = deriveContext ?? (() => ({}));
|
||||
this.sub.subscribe(this.msgChannel, `${this.msgChannel}:${this.serverId}`);
|
||||
this.sub.on('messageBuffer', this.handleRedisMessage);
|
||||
this.pub.on('error', () => {});
|
||||
@@ -87,44 +95,63 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
|
||||
}
|
||||
|
||||
private closeProxy(socketId: string) {
|
||||
const proxySocket = this.proxySockets[socketId];
|
||||
if (proxySocket) {
|
||||
proxySocket.emit(
|
||||
'close',
|
||||
1000,
|
||||
Buffer.from('provider_initiated', 'utf-8'),
|
||||
);
|
||||
delete this.proxySockets[socketId];
|
||||
const entry = this.proxyConnections[socketId];
|
||||
if (entry) {
|
||||
delete this.proxyConnections[socketId];
|
||||
const { socket, clientConnection } = entry;
|
||||
// The origin socket is already gone; don't echo a close message back
|
||||
socket.markClosed();
|
||||
clientConnection.handleClose({
|
||||
code: 1000,
|
||||
reason: 'provider_initiated',
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private pongProxy(socketId: string) {
|
||||
this.proxySockets[socketId]?.emit('pong');
|
||||
}
|
||||
|
||||
private handleProxyMessage(
|
||||
msg: Pick<RSAMessageProxy, 'replyTo' | 'message' | 'serializedHTTPRequest'>,
|
||||
) {
|
||||
const { replyTo, message, serializedHTTPRequest } = msg;
|
||||
const { headers } = serializedHTTPRequest;
|
||||
const socketId = headers['sec-websocket-key']!;
|
||||
let socket = this.proxySockets[socketId];
|
||||
if (!socket) {
|
||||
socket = new CollabProxySocket(
|
||||
const socketId = headers['sec-websocket-key'];
|
||||
let entry = this.proxyConnections[socketId];
|
||||
if (!entry) {
|
||||
const socket = new CollabProxySocket(
|
||||
this.pub,
|
||||
this.pack,
|
||||
replyTo,
|
||||
`${this.msgChannel}:${this.serverId}`,
|
||||
socketId,
|
||||
);
|
||||
this.proxySockets[socketId] = socket;
|
||||
this.instance.handleConnection(
|
||||
socket as any,
|
||||
serializedHTTPRequest as any,
|
||||
{},
|
||||
// A proxy connection with no live documents (client left the page, auth
|
||||
// failed, or the origin server crashed) is reaped by hocuspocus' message
|
||||
// timeout. Dispose it silently in that case: relaying the timeout close
|
||||
// to the origin would kill the client's real socket, which may be busy
|
||||
// serving other documents. Genuine protocol closes are still relayed.
|
||||
socket.onClose = (code, reason) => {
|
||||
delete this.proxyConnections[socketId];
|
||||
if (code !== ConnectionTimeout.code) {
|
||||
const msg: RSAMessageClose = {
|
||||
type: 'close',
|
||||
code,
|
||||
reason,
|
||||
socketId,
|
||||
};
|
||||
this.pub.publish(replyTo, this.pack(msg));
|
||||
}
|
||||
};
|
||||
const clientConnection = this.instance.handleConnection(
|
||||
socket,
|
||||
toWebRequest(serializedHTTPRequest),
|
||||
this.deriveContext(serializedHTTPRequest),
|
||||
);
|
||||
entry = { clientConnection, socket };
|
||||
this.proxyConnections[socketId] = entry;
|
||||
}
|
||||
socket.emit('message', message);
|
||||
entry.clientConnection.handleMessage(message);
|
||||
}
|
||||
|
||||
private getLock(documentName: string) {
|
||||
return this.pub.get(this.getKey(documentName));
|
||||
}
|
||||
|
||||
private getOrClaimLock(documentName: string) {
|
||||
@@ -166,10 +193,6 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
|
||||
this.closeProxy(msg.socketId);
|
||||
return;
|
||||
}
|
||||
if (type === 'pong') {
|
||||
this.pongProxy(msg.socketId);
|
||||
return;
|
||||
}
|
||||
if (type === 'unload') {
|
||||
delete this.lockPromises[msg.documentName];
|
||||
return;
|
||||
@@ -198,22 +221,14 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
|
||||
return;
|
||||
}
|
||||
const { socketId } = msg;
|
||||
const socket = this.originSockets[socketId];
|
||||
if (!socket) {
|
||||
const entry = this.originConnections[socketId];
|
||||
if (!entry) {
|
||||
// origin socket already cleaned up
|
||||
return;
|
||||
}
|
||||
const { socket } = entry;
|
||||
if (type === 'close') {
|
||||
socket.close(msg.code, msg.reason);
|
||||
} else if (type === 'ping') {
|
||||
// Reply instantly to the proxy socket, without forwarding to client
|
||||
// The origin socket handles heartbeat for itself
|
||||
const { replyTo, socketId } = msg;
|
||||
const reply: RSAMessagePong = {
|
||||
type: 'pong',
|
||||
socketId,
|
||||
};
|
||||
this.pub.publish(`${replyTo}`, this.pack(reply));
|
||||
} else if (type === 'send') {
|
||||
socket.send(msg.message);
|
||||
}
|
||||
@@ -251,6 +266,8 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
|
||||
eventName: TName,
|
||||
documentName: string,
|
||||
payload: any,
|
||||
// if true, don't claim the lock. Useful for targeting pages that are currently open
|
||||
onlyIfOpen = false,
|
||||
) {
|
||||
const isDocLoadedOnInstance = this.instance.documents.has(documentName);
|
||||
|
||||
@@ -258,7 +275,14 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
|
||||
return this.handleEventLocally(eventName, documentName, payload);
|
||||
}
|
||||
|
||||
const proxyTo = await this.getOrClaimLockThrottled(documentName);
|
||||
const proxyTo = await (onlyIfOpen
|
||||
? this.getLock(documentName)
|
||||
: this.getOrClaimLockThrottled(documentName));
|
||||
|
||||
if (!proxyTo && onlyIfOpen) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (proxyTo && proxyTo !== this.serverId) {
|
||||
++this.replyIdCounter; // bug in biome thinks this.replyIdCounter is not used if written on the line below
|
||||
const replyId = this.replyIdCounter;
|
||||
@@ -277,7 +301,8 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
|
||||
const { promise, resolve, reject } = Promise.withResolvers();
|
||||
this.pendingReplies[replyId] = resolve;
|
||||
setTimeout(() => {
|
||||
reject('TIMEOUT');
|
||||
delete this.pendingReplies[replyId];
|
||||
reject(new Error('TIMEOUT'));
|
||||
}, this.customEventTTL);
|
||||
return promise as Promise<ReturnType<TCE[TName]>>;
|
||||
}
|
||||
@@ -296,36 +321,59 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
|
||||
|
||||
/* WebSocket Server Hooks */
|
||||
onSocketOpen(
|
||||
ws: BaseWebSocket,
|
||||
ws: WebSocketLike,
|
||||
serializedHTTPRequest: SerializedHTTPRequest,
|
||||
context = {},
|
||||
) {
|
||||
const socketId = serializedHTTPRequest.headers['sec-websocket-key']!;
|
||||
this.originSockets[socketId] = ws;
|
||||
this.instance.handleConnection(
|
||||
ws as any,
|
||||
serializedHTTPRequest as any,
|
||||
context,
|
||||
const socketId = serializedHTTPRequest.headers['sec-websocket-key'];
|
||||
const clientConnection = this.instance.handleConnection(
|
||||
ws,
|
||||
toWebRequest(serializedHTTPRequest),
|
||||
this.deriveContext(serializedHTTPRequest),
|
||||
);
|
||||
this.originConnections[socketId] = { clientConnection, socket: ws };
|
||||
}
|
||||
|
||||
async onSocketMessage(
|
||||
ws: BaseWebSocket,
|
||||
serializedHTTPRequest: SerializedHTTPRequest,
|
||||
detachableMsg: ArrayBuffer,
|
||||
) {
|
||||
const message = new Uint8Array(detachableMsg.slice());
|
||||
const tmpMsg = new IncomingMessage(detachableMsg);
|
||||
const documentName = readVarString(tmpMsg.decoder);
|
||||
const socketId = serializedHTTPRequest.headers['sec-websocket-key'];
|
||||
const entry = this.originConnections[socketId];
|
||||
if (!entry) return;
|
||||
const { clientConnection } = entry;
|
||||
|
||||
let message: Uint8Array;
|
||||
let documentName: string;
|
||||
try {
|
||||
message = new Uint8Array(detachableMsg.slice());
|
||||
const tmpMsg = new IncomingMessage(detachableMsg);
|
||||
const documentNameAndSessionId = tmpMsg.readVarString();
|
||||
// session-aware providers suffix the documentName with \0sessionId
|
||||
const sepIdx = documentNameAndSessionId.indexOf('\0');
|
||||
documentName =
|
||||
sepIdx === -1
|
||||
? documentNameAndSessionId
|
||||
: documentNameAndSessionId.slice(0, sepIdx);
|
||||
} catch (error) {
|
||||
entry.socket.close(Unauthorized.code, Unauthorized.reason);
|
||||
return;
|
||||
}
|
||||
const isDocLoadedOnInstance = this.instance.documents.has(documentName);
|
||||
|
||||
if (isDocLoadedOnInstance) {
|
||||
ws.emit('message', message);
|
||||
clientConnection.handleMessage(message);
|
||||
return;
|
||||
}
|
||||
|
||||
const proxyTo = await this.getOrClaimLockThrottled(documentName);
|
||||
if (proxyTo && proxyTo !== this.serverId) {
|
||||
// Proxied messages bypass handleMessage, so refresh the connection's
|
||||
// liveness fields manually or hocuspocus' message timeout would reap the
|
||||
// real socket every `timeout` ms. connectionEstablishedAt is the
|
||||
// reference while unauthenticated (auth for remote docs is proxied too)
|
||||
// and is private upstream.
|
||||
clientConnection.lastMessageReceivedAt = Date.now();
|
||||
(clientConnection as any).connectionEstablishedAt = Date.now();
|
||||
// another server owns the doc
|
||||
const proxyMessage: RSAMessageProxy = {
|
||||
serializedHTTPRequest: serializedHTTPRequest,
|
||||
@@ -338,16 +386,17 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
|
||||
return;
|
||||
}
|
||||
// This server owns the document, but hocuspocus hasn't loaded it yet
|
||||
ws.emit('message', message);
|
||||
clientConnection.handleMessage(message);
|
||||
}
|
||||
|
||||
onSocketClose(socketId: string, code?: number, reason?: ArrayBuffer) {
|
||||
const socket = this.originSockets[socketId];
|
||||
if (!socket) return;
|
||||
// at this point the socket is considered GC'd and we cannot call close
|
||||
// The origin socket did not set up any connections for the proxy, so none of the hooks will work if we just emit
|
||||
socket?.emit('close', code, reason);
|
||||
delete this.originSockets[socketId];
|
||||
const entry = this.originConnections[socketId];
|
||||
if (!entry) return;
|
||||
delete this.originConnections[socketId];
|
||||
entry.clientConnection.handleClose({
|
||||
code: code ?? 1000,
|
||||
reason: reason ? Buffer.from(reason).toString() : '',
|
||||
});
|
||||
const msg: RSAMessageCloseProxy = { type: 'closeProxy', socketId };
|
||||
this.pub.publish(this.msgChannel, this.pack(msg)).catch(() => {});
|
||||
}
|
||||
@@ -372,6 +421,7 @@ export class RedisSyncExtension<TCE extends CustomEvents> implements Extension {
|
||||
}
|
||||
|
||||
async onDestroy() {
|
||||
this.pendingReplies = {};
|
||||
this.pub.disconnect(false);
|
||||
this.sub.disconnect(false);
|
||||
}
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
import EventEmitter from 'node:events';
|
||||
import { IncomingHttpHeaders } from 'node:http2';
|
||||
import RedisClient from 'ioredis';
|
||||
import { CollabProxySocket } from './collab-proxy-socket';
|
||||
import { type Hocuspocus, type WebSocketLike } from '@hocuspocus/server';
|
||||
|
||||
export type SecondParam<T> = T extends (
|
||||
arg1: unknown,
|
||||
arg1: any,
|
||||
arg2: infer A,
|
||||
...args: unknown[]
|
||||
) => unknown
|
||||
...args: any[]
|
||||
) => any
|
||||
? A
|
||||
: never;
|
||||
|
||||
@@ -41,17 +42,6 @@ export type RSAMessageClose = {
|
||||
socketId: string;
|
||||
};
|
||||
|
||||
export type RSAMessagePing = {
|
||||
type: 'ping';
|
||||
socketId: string;
|
||||
replyTo: string;
|
||||
};
|
||||
|
||||
export type RSAMessagePong = {
|
||||
type: 'pong';
|
||||
socketId: string;
|
||||
};
|
||||
|
||||
export type RSAMessageSend = {
|
||||
type: 'send';
|
||||
// @ts-ignore
|
||||
@@ -59,7 +49,7 @@ export type RSAMessageSend = {
|
||||
socketId: string;
|
||||
};
|
||||
|
||||
export type RSAMessageCustomEventStart<TName = string, TPayload = unknown> = {
|
||||
export type RSAMessageCustomEventStart<TName = string, TPayload = any> = {
|
||||
type: 'customEventStart';
|
||||
documentName: string;
|
||||
eventName: TName;
|
||||
@@ -71,7 +61,7 @@ export type RSAMessageCustomEventStart<TName = string, TPayload = unknown> = {
|
||||
export type RSAMessageCustomEventComplete = {
|
||||
type: 'customEventComplete';
|
||||
replyId: number;
|
||||
payload: unknown;
|
||||
payload: any;
|
||||
};
|
||||
|
||||
export type RSAMessage =
|
||||
@@ -79,8 +69,6 @@ export type RSAMessage =
|
||||
| RSAMessageCloseProxy
|
||||
| RSAMessageUnload
|
||||
| RSAMessageClose
|
||||
| RSAMessagePing
|
||||
| RSAMessagePong
|
||||
| RSAMessageSend
|
||||
| RSAMessageCustomEventStart
|
||||
| RSAMessageCustomEventComplete;
|
||||
@@ -99,9 +87,20 @@ type CustomEventName = string;
|
||||
|
||||
export type CustomEvents = Record<
|
||||
CustomEventName,
|
||||
(documentName: string, payload: unknown) => Promise<unknown>
|
||||
(documentName: string, payload: any) => Promise<any>
|
||||
>;
|
||||
|
||||
// Not exported by @hocuspocus/server
|
||||
export type ClientConnection = ReturnType<Hocuspocus['handleConnection']>;
|
||||
export type OriginConnection = {
|
||||
clientConnection: ClientConnection;
|
||||
socket: WebSocketLike;
|
||||
};
|
||||
export type ProxyConnection = {
|
||||
clientConnection: ClientConnection;
|
||||
socket: CollabProxySocket;
|
||||
};
|
||||
|
||||
export interface Configuration<TCE> {
|
||||
redis: RedisClient;
|
||||
pack: Pack;
|
||||
@@ -111,11 +110,29 @@ export interface Configuration<TCE> {
|
||||
customEventTTL?: number;
|
||||
prefix?: string;
|
||||
customEvents?: TCE;
|
||||
// Derive the hocuspocus context once per socket instead of re-deriving it in a
|
||||
// per-document hook like onConnect/onAuthenticate. Runs on the origin server when
|
||||
// the socket opens and on the doc owner when the first proxied message arrives.
|
||||
deriveContext?: (
|
||||
serializedHTTPRequest: SerializedHTTPRequest,
|
||||
) => Record<string, any>;
|
||||
}
|
||||
|
||||
export type BaseWebSocket = EventEmitter & {
|
||||
readyState: number;
|
||||
close(code?: number, reason?: string): void;
|
||||
ping(): void;
|
||||
send(message: Uint8Array): void;
|
||||
// Hocuspocus expects a web-standard Request, so rehydrate one from what crossed the wire
|
||||
export const toWebRequest = (serializedHTTPRequest: SerializedHTTPRequest) => {
|
||||
const { method, url, headers } = serializedHTTPRequest;
|
||||
const webHeaders = new Headers();
|
||||
Object.entries(headers).forEach(([name, value]) => {
|
||||
if (Array.isArray(value)) {
|
||||
value.forEach((v) => {
|
||||
webHeaders.append(name, v);
|
||||
});
|
||||
} else if (value !== undefined) {
|
||||
webHeaders.set(name, value);
|
||||
}
|
||||
});
|
||||
return new Request(new URL(url, 'http://localhost'), {
|
||||
method,
|
||||
headers: webHeaders,
|
||||
});
|
||||
};
|
||||
|
||||
@@ -1,20 +1,17 @@
|
||||
import { EventEmitter } from 'events';
|
||||
import type WebSocket from 'ws';
|
||||
import type { WebSocketLike } from '@hocuspocus/server';
|
||||
|
||||
/**
|
||||
* Wrapper around ws WebSocket that only receives events via emit().
|
||||
* This prevents double-handling when used with RedisSyncExtension.
|
||||
* Wrapper around ws WebSocket that Hocuspocus only writes to.
|
||||
* Incoming socket events are forwarded separately by the gateway,
|
||||
* which prevents double-handling with RedisSyncExtension.
|
||||
*/
|
||||
export class WsSocketWrapper extends EventEmitter {
|
||||
export class WsSocketWrapper implements WebSocketLike {
|
||||
private ws: WebSocket;
|
||||
readyState = 1;
|
||||
|
||||
constructor(ws: WebSocket) {
|
||||
super();
|
||||
this.ws = ws;
|
||||
this.once('close', () => {
|
||||
this.readyState = 3;
|
||||
});
|
||||
}
|
||||
|
||||
close(code?: number, reason?: string) {
|
||||
@@ -27,15 +24,6 @@ export class WsSocketWrapper extends EventEmitter {
|
||||
}
|
||||
}
|
||||
|
||||
ping() {
|
||||
if (this.readyState !== 1) return;
|
||||
try {
|
||||
this.ws.ping();
|
||||
} catch (e) {
|
||||
// Socket already closed
|
||||
}
|
||||
}
|
||||
|
||||
send(message: Uint8Array) {
|
||||
if (this.readyState !== 1) return;
|
||||
try {
|
||||
|
||||
@@ -15,4 +15,29 @@ export enum EventName {
|
||||
WORKSPACE_CREATED = 'workspace.created',
|
||||
WORKSPACE_UPDATED = 'workspace.updated',
|
||||
WORKSPACE_DELETED = 'workspace.deleted',
|
||||
|
||||
BASE_CREATED = 'base.created',
|
||||
BASE_UPDATED = 'base.updated',
|
||||
BASE_DELETED = 'base.deleted',
|
||||
|
||||
BASE_ROW_CREATED = 'base.row.created',
|
||||
BASE_ROW_UPDATED = 'base.row.updated',
|
||||
BASE_ROW_DELETED = 'base.row.deleted',
|
||||
BASE_ROWS_DELETED = 'base.rows.deleted',
|
||||
BASE_ROW_RESTORED = 'base.row.restored',
|
||||
BASE_ROW_REORDERED = 'base.row.reordered',
|
||||
|
||||
BASE_PROPERTY_CREATED = 'base.property.created',
|
||||
BASE_PROPERTY_UPDATED = 'base.property.updated',
|
||||
BASE_PROPERTY_DELETED = 'base.property.deleted',
|
||||
BASE_PROPERTY_REORDERED = 'base.property.reordered',
|
||||
|
||||
BASE_VIEW_CREATED = 'base.view.created',
|
||||
BASE_VIEW_UPDATED = 'base.view.updated',
|
||||
BASE_VIEW_DELETED = 'base.view.deleted',
|
||||
|
||||
BASE_SCHEMA_BUMPED = 'base.schema.bumped',
|
||||
BASE_ROWS_UPDATED = 'base.rows.updated',
|
||||
BASE_FORMULA_RECOMPUTE_STARTED = 'base.formula.recompute.started',
|
||||
BASE_FORMULA_RECOMPUTE_COMPLETED = 'base.formula.recompute.completed',
|
||||
}
|
||||
|
||||
@@ -20,6 +20,9 @@ export const Feature = {
|
||||
VIEWER_COMMENTS: 'comment:viewer',
|
||||
TEMPLATES: 'templates',
|
||||
PDF_EXPORT: 'export:pdf',
|
||||
PERSONAL_SPACES: 'spaces:personal',
|
||||
DOCX_EXPORT: 'export:docx',
|
||||
BASES: 'bases',
|
||||
} as const;
|
||||
|
||||
export type FeatureKey = (typeof Feature)[keyof typeof Feature];
|
||||
|
||||
@@ -10,6 +10,9 @@ export const LOCAL_STORAGE_PATH = path.resolve(
|
||||
LOCAL_STORAGE_DIR,
|
||||
);
|
||||
|
||||
export function getPageTitle(title: string | null | undefined): string {
|
||||
return title || 'untitled';
|
||||
export function getPageTitle(
|
||||
title: string | null | undefined,
|
||||
isBase?: boolean,
|
||||
): string {
|
||||
return title || (isBase ? 'Untitled base' : 'untitled');
|
||||
}
|
||||
|
||||
@@ -5,4 +5,9 @@ export const nanoIdGen = customAlphabet(alphabet, 10);
|
||||
|
||||
const slugIdAlphabet =
|
||||
'0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz';
|
||||
export const generateSlugId = customAlphabet(slugIdAlphabet, 10);
|
||||
export const generateSlugId = customAlphabet(slugIdAlphabet, 10);
|
||||
|
||||
const baseIdSuffix = customAlphabet(alphabet, 9);
|
||||
|
||||
export const generateBasePropertyId = (): string => `prp${baseIdSuffix()}`;
|
||||
export const generateBaseChoiceId = (): string => `opt${baseIdSuffix()}`;
|
||||
|
||||
@@ -4,6 +4,11 @@ export enum UserRole {
|
||||
MEMBER = 'member',
|
||||
}
|
||||
|
||||
export enum InviteUserRole {
|
||||
ADMIN = 'admin', // can have owner permissions but cannot delete workspace
|
||||
MEMBER = 'member',
|
||||
}
|
||||
|
||||
export enum SpaceRole {
|
||||
ADMIN = 'admin', // can manage space settings, members, and delete space
|
||||
WRITER = 'writer', // can read and write pages in space
|
||||
|
||||
@@ -43,7 +43,7 @@ export class AttachmentService {
|
||||
|
||||
async uploadFile(opts: {
|
||||
filePromise: Promise<MultipartFile>;
|
||||
pageId: string;
|
||||
pageId?: string;
|
||||
userId: string;
|
||||
spaceId: string;
|
||||
workspaceId: string;
|
||||
|
||||
@@ -4,6 +4,8 @@ import { WorkspaceService } from '../../workspace/services/workspace.service';
|
||||
import { CreateWorkspaceDto } from '../../workspace/dto/create-workspace.dto';
|
||||
import { CreateAdminUserDto } from '../dto/create-admin-user.dto';
|
||||
import { UserRepo } from '@docmost/db/repos/user/user.repo';
|
||||
import { WorkspaceRepo } from '@docmost/db/repos/workspace/workspace.repo';
|
||||
import { getWorkspaceDefaultPageEditMode } from '../../workspace/workspace.util';
|
||||
import { KyselyDB, KyselyTransaction } from '@docmost/db/types/kysely.types';
|
||||
import { executeTx } from '@docmost/db/utils';
|
||||
import { InjectKysely } from 'nestjs-kysely';
|
||||
@@ -20,6 +22,7 @@ import {
|
||||
export class SignupService {
|
||||
constructor(
|
||||
private userRepo: UserRepo,
|
||||
private workspaceRepo: WorkspaceRepo,
|
||||
private workspaceService: WorkspaceService,
|
||||
private groupUserRepo: GroupUserRepo,
|
||||
@InjectKysely() private readonly db: KyselyDB,
|
||||
@@ -46,12 +49,16 @@ export class SignupService {
|
||||
this.db,
|
||||
async (trx) => {
|
||||
// create user
|
||||
const workspace = await this.workspaceRepo.findById(workspaceId, {
|
||||
trx,
|
||||
});
|
||||
const user = await this.userRepo.insertUser(
|
||||
{
|
||||
...createUserDto,
|
||||
workspaceId: workspaceId,
|
||||
},
|
||||
trx,
|
||||
{ pageEditMode: getWorkspaceDefaultPageEditMode(workspace) },
|
||||
);
|
||||
|
||||
// add user to workspace
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import {
|
||||
IsBoolean,
|
||||
IsIn,
|
||||
IsOptional,
|
||||
IsString,
|
||||
|
||||
@@ -15,7 +15,7 @@ import {
|
||||
executeWithCursorPagination,
|
||||
} from '@docmost/db/pagination/cursor-pagination';
|
||||
import { InjectKysely } from 'nestjs-kysely';
|
||||
import { KyselyDB } from '@docmost/db/types/kysely.types';
|
||||
import { KyselyDB, KyselyTransaction } from '@docmost/db/types/kysely.types';
|
||||
import { generateJitteredKeyBetween } from 'fractional-indexing-jittered';
|
||||
import { MovePageDto } from '../dto/move-page.dto';
|
||||
import { generateSlugId } from '../../../common/helpers';
|
||||
@@ -92,6 +92,8 @@ export class PageService {
|
||||
userId: string,
|
||||
workspaceId: string,
|
||||
createPageDto: CreatePageDto,
|
||||
trx?: KyselyTransaction,
|
||||
isBase: boolean = false,
|
||||
): Promise<Page> {
|
||||
let parentPageId = undefined;
|
||||
|
||||
@@ -140,21 +142,34 @@ export class PageService {
|
||||
creatorId: userId,
|
||||
workspaceId: workspaceId,
|
||||
lastUpdatedById: userId,
|
||||
isBase,
|
||||
content,
|
||||
textContent,
|
||||
ydoc,
|
||||
});
|
||||
}, trx);
|
||||
|
||||
this.generalQueue
|
||||
.add(QueueJob.ADD_PAGE_WATCHERS, {
|
||||
userIds: [userId],
|
||||
pageId: page.id,
|
||||
spaceId: createPageDto.spaceId,
|
||||
if (trx) {
|
||||
// Add the watcher inside the caller's transaction so the async worker
|
||||
// never inserts against an uncommitted page (FK violation on bases).
|
||||
await this.watcherService.addPageWatchers(
|
||||
[userId],
|
||||
page.id,
|
||||
createPageDto.spaceId,
|
||||
workspaceId,
|
||||
})
|
||||
.catch((err) =>
|
||||
this.logger.warn(`Failed to queue add-page-watchers: ${err.message}`),
|
||||
trx,
|
||||
);
|
||||
} else {
|
||||
this.generalQueue
|
||||
.add(QueueJob.ADD_PAGE_WATCHERS, {
|
||||
userIds: [userId],
|
||||
pageId: page.id,
|
||||
spaceId: createPageDto.spaceId,
|
||||
workspaceId,
|
||||
})
|
||||
.catch((err) =>
|
||||
this.logger.warn(`Failed to queue add-page-watchers: ${err.message}`),
|
||||
);
|
||||
}
|
||||
|
||||
return page;
|
||||
}
|
||||
@@ -289,6 +304,7 @@ export class PageService {
|
||||
'parentPageId',
|
||||
'spaceId',
|
||||
'creatorId',
|
||||
'isBase',
|
||||
'deletedAt',
|
||||
])
|
||||
.select((eb) => this.pageRepo.withHasChildren(eb))
|
||||
@@ -310,6 +326,7 @@ export class PageService {
|
||||
expression: 'position',
|
||||
direction: 'asc',
|
||||
orderModifier: (ob) => ob.collate('C').asc(),
|
||||
cursorExpression: sql`position collate "C"`,
|
||||
},
|
||||
{ expression: 'id', direction: 'asc' },
|
||||
],
|
||||
@@ -429,10 +446,9 @@ export class PageService {
|
||||
}
|
||||
|
||||
if (pageIdsToMove.length > 0) {
|
||||
// Clear page-level permissions - moved pages inherit destination space permissions
|
||||
// (page_permissions cascade deletes via foreign key)
|
||||
await trx
|
||||
.deleteFrom('pageAccess')
|
||||
.updateTable('pageAccess')
|
||||
.set({ spaceId: spaceId })
|
||||
.where('pageId', 'in', pageIdsToMove)
|
||||
.execute();
|
||||
|
||||
@@ -831,6 +847,7 @@ export class PageService {
|
||||
'slugId',
|
||||
'title',
|
||||
'icon',
|
||||
'isBase',
|
||||
'position',
|
||||
'parentPageId',
|
||||
'spaceId',
|
||||
@@ -846,6 +863,7 @@ export class PageService {
|
||||
'p.slugId',
|
||||
'p.title',
|
||||
'p.icon',
|
||||
'p.isBase',
|
||||
'p.position',
|
||||
'p.parentPageId',
|
||||
'p.spaceId',
|
||||
|
||||
@@ -1,320 +0,0 @@
|
||||
import { Test } from '@nestjs/testing';
|
||||
import { TransclusionService } from '../transclusion.service';
|
||||
import { PageTransclusionsRepo } from '@docmost/db/repos/page-transclusions/page-transclusions.repo';
|
||||
import { PageTransclusionReferencesRepo } from '@docmost/db/repos/page-transclusions/page-transclusion-references.repo';
|
||||
import { PageRepo } from '@docmost/db/repos/page/page.repo';
|
||||
import { PagePermissionRepo } from '@docmost/db/repos/page/page-permission.repo';
|
||||
import { AttachmentRepo } from '@docmost/db/repos/attachment/attachment.repo';
|
||||
import { StorageService } from '../../../../integrations/storage/storage.service';
|
||||
import { PageAccessService } from '../../page-access/page-access.service';
|
||||
|
||||
describe('TransclusionService.syncPageTransclusions', () => {
|
||||
let service: TransclusionService;
|
||||
let repo: jest.Mocked<PageTransclusionsRepo>;
|
||||
|
||||
beforeEach(async () => {
|
||||
const mockRepo: jest.Mocked<Partial<PageTransclusionsRepo>> = {
|
||||
findByPageId: jest.fn(),
|
||||
insert: jest.fn(),
|
||||
update: jest.fn(),
|
||||
deleteByPageAndTransclusionIds: jest.fn(),
|
||||
};
|
||||
const module = await Test.createTestingModule({
|
||||
providers: [
|
||||
TransclusionService,
|
||||
{ provide: PageTransclusionsRepo, useValue: mockRepo },
|
||||
{ provide: PageTransclusionReferencesRepo, useValue: {} },
|
||||
{ provide: PageRepo, useValue: {} },
|
||||
{ provide: PagePermissionRepo, useValue: {} },
|
||||
{ provide: AttachmentRepo, useValue: {} },
|
||||
{ provide: StorageService, useValue: {} },
|
||||
{ provide: PageAccessService, useValue: {} },
|
||||
],
|
||||
}).compile();
|
||||
service = module.get(TransclusionService);
|
||||
repo = module.get(PageTransclusionsRepo);
|
||||
});
|
||||
|
||||
const pageId = '00000000-0000-0000-0000-000000000001';
|
||||
const workspaceId = '00000000-0000-0000-0000-000000000099';
|
||||
|
||||
it('inserts new transclusions that did not exist before', async () => {
|
||||
repo.findByPageId.mockResolvedValue([]);
|
||||
const pm = {
|
||||
type: 'doc',
|
||||
content: [
|
||||
{
|
||||
type: 'transclusionSource',
|
||||
attrs: { id: 'a' },
|
||||
content: [{ type: 'paragraph' }],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = await service.syncPageTransclusions(pageId, workspaceId, pm);
|
||||
|
||||
expect(result).toEqual({ inserted: 1, updated: 0, deleted: 0 });
|
||||
expect(repo.insert).toHaveBeenCalledTimes(1);
|
||||
expect(repo.insert).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
pageId,
|
||||
transclusionId: 'a',
|
||||
}),
|
||||
undefined,
|
||||
);
|
||||
expect(repo.update).not.toHaveBeenCalled();
|
||||
expect(repo.deleteByPageAndTransclusionIds).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('updates transclusions whose content changed', async () => {
|
||||
repo.findByPageId.mockResolvedValue([
|
||||
{
|
||||
id: 'row1',
|
||||
pageId,
|
||||
transclusionId: 'a',
|
||||
content: { type: 'doc', content: [{ type: 'paragraph' }] },
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
} as any,
|
||||
]);
|
||||
const newContent = {
|
||||
type: 'doc',
|
||||
content: [
|
||||
{ type: 'paragraph', content: [{ type: 'text', text: 'X' }] },
|
||||
],
|
||||
};
|
||||
const pm = {
|
||||
type: 'doc',
|
||||
content: [
|
||||
{
|
||||
type: 'transclusionSource',
|
||||
attrs: { id: 'a' },
|
||||
content: newContent.content,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = await service.syncPageTransclusions(pageId, workspaceId, pm);
|
||||
|
||||
expect(result).toEqual({ inserted: 0, updated: 1, deleted: 0 });
|
||||
expect(repo.update).toHaveBeenCalledWith(
|
||||
pageId,
|
||||
'a',
|
||||
expect.objectContaining({ content: newContent }),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it('skips update when content is unchanged', async () => {
|
||||
const sameContent = {
|
||||
type: 'doc',
|
||||
content: [{ type: 'paragraph' }],
|
||||
};
|
||||
repo.findByPageId.mockResolvedValue([
|
||||
{
|
||||
id: 'row1',
|
||||
pageId,
|
||||
transclusionId: 'a',
|
||||
content: sameContent,
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
} as any,
|
||||
]);
|
||||
const pm = {
|
||||
type: 'doc',
|
||||
content: [
|
||||
{
|
||||
type: 'transclusionSource',
|
||||
attrs: { id: 'a' },
|
||||
content: sameContent.content,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = await service.syncPageTransclusions(pageId, workspaceId, pm);
|
||||
|
||||
expect(result).toEqual({ inserted: 0, updated: 0, deleted: 0 });
|
||||
expect(repo.update).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('deletes transclusions that no longer appear in the doc', async () => {
|
||||
repo.findByPageId.mockResolvedValue([
|
||||
{
|
||||
id: 'r',
|
||||
pageId,
|
||||
transclusionId: 'gone',
|
||||
content: { type: 'doc', content: [] },
|
||||
createdAt: new Date(),
|
||||
updatedAt: new Date(),
|
||||
} as any,
|
||||
]);
|
||||
const pm = { type: 'doc', content: [{ type: 'paragraph' }] };
|
||||
|
||||
const result = await service.syncPageTransclusions(pageId, workspaceId, pm);
|
||||
|
||||
expect(result).toEqual({ inserted: 0, updated: 0, deleted: 1 });
|
||||
expect(repo.deleteByPageAndTransclusionIds).toHaveBeenCalledWith(
|
||||
pageId,
|
||||
['gone'],
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it('handles empty doc → noop', async () => {
|
||||
repo.findByPageId.mockResolvedValue([]);
|
||||
const result = await service.syncPageTransclusions(pageId, workspaceId, null);
|
||||
expect(result).toEqual({ inserted: 0, updated: 0, deleted: 0 });
|
||||
expect(repo.insert).not.toHaveBeenCalled();
|
||||
expect(repo.update).not.toHaveBeenCalled();
|
||||
expect(repo.deleteByPageAndTransclusionIds).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('TransclusionService.syncPageReferences', () => {
|
||||
let service: TransclusionService;
|
||||
let refRepo: jest.Mocked<PageTransclusionReferencesRepo>;
|
||||
|
||||
beforeEach(async () => {
|
||||
const mockTransclusionsRepo: Partial<PageTransclusionsRepo> = {};
|
||||
const mockRefRepo: jest.Mocked<Partial<PageTransclusionReferencesRepo>> = {
|
||||
findByReferencePageId: jest.fn(),
|
||||
insertMany: jest.fn(),
|
||||
deleteByReferenceAndKeys: jest.fn(),
|
||||
};
|
||||
const module = await Test.createTestingModule({
|
||||
providers: [
|
||||
TransclusionService,
|
||||
{ provide: PageTransclusionsRepo, useValue: mockTransclusionsRepo },
|
||||
{ provide: PageTransclusionReferencesRepo, useValue: mockRefRepo },
|
||||
{ provide: PageRepo, useValue: {} },
|
||||
{ provide: PagePermissionRepo, useValue: {} },
|
||||
{ provide: AttachmentRepo, useValue: {} },
|
||||
{ provide: StorageService, useValue: {} },
|
||||
{ provide: PageAccessService, useValue: {} },
|
||||
],
|
||||
}).compile();
|
||||
service = module.get(TransclusionService);
|
||||
refRepo = module.get(PageTransclusionReferencesRepo);
|
||||
});
|
||||
|
||||
const referencePageId = '00000000-0000-0000-0000-000000000001';
|
||||
const workspaceId = '00000000-0000-0000-0000-000000000099';
|
||||
|
||||
it('inserts new loose references, no deletes when none existed', async () => {
|
||||
refRepo.findByReferencePageId.mockResolvedValue([]);
|
||||
const pm = {
|
||||
type: 'doc',
|
||||
content: [
|
||||
{
|
||||
type: 'transclusionReference',
|
||||
attrs: { sourcePageId: 'p1', transclusionId: 'e1' },
|
||||
},
|
||||
{
|
||||
type: 'transclusionReference',
|
||||
attrs: { sourcePageId: 'p2', transclusionId: 'e2' },
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = await service.syncPageReferences(referencePageId, workspaceId, pm);
|
||||
|
||||
expect(result).toEqual({ inserted: 2, deleted: 0 });
|
||||
expect(refRepo.insertMany).toHaveBeenCalledWith(
|
||||
[
|
||||
{
|
||||
workspaceId,
|
||||
referencePageId,
|
||||
sourcePageId: 'p1',
|
||||
transclusionId: 'e1',
|
||||
},
|
||||
{
|
||||
workspaceId,
|
||||
referencePageId,
|
||||
sourcePageId: 'p2',
|
||||
transclusionId: 'e2',
|
||||
},
|
||||
],
|
||||
undefined,
|
||||
);
|
||||
expect(refRepo.deleteByReferenceAndKeys).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('ignores references nested inside a source (schema-forbidden)', async () => {
|
||||
refRepo.findByReferencePageId.mockResolvedValue([]);
|
||||
const pm = {
|
||||
type: 'doc',
|
||||
content: [
|
||||
{
|
||||
type: 'transclusionSource',
|
||||
attrs: { id: 's1' },
|
||||
content: [
|
||||
{
|
||||
type: 'transclusionReference',
|
||||
attrs: { sourcePageId: 'p2', transclusionId: 'e2' },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = await service.syncPageReferences(referencePageId, workspaceId, pm);
|
||||
|
||||
expect(result).toEqual({ inserted: 0, deleted: 0 });
|
||||
expect(refRepo.insertMany).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('deletes references that no longer appear', async () => {
|
||||
refRepo.findByReferencePageId.mockResolvedValue([
|
||||
{
|
||||
id: 'r1',
|
||||
referencePageId,
|
||||
sourcePageId: 'p1',
|
||||
transclusionId: 'e1',
|
||||
createdAt: new Date(),
|
||||
} as any,
|
||||
]);
|
||||
const pm = { type: 'doc', content: [{ type: 'paragraph' }] };
|
||||
|
||||
const result = await service.syncPageReferences(referencePageId, workspaceId, pm);
|
||||
|
||||
expect(result).toEqual({ inserted: 0, deleted: 1 });
|
||||
expect(refRepo.deleteByReferenceAndKeys).toHaveBeenCalledWith(
|
||||
referencePageId,
|
||||
[
|
||||
{
|
||||
sourcePageId: 'p1',
|
||||
transclusionId: 'e1',
|
||||
},
|
||||
],
|
||||
undefined,
|
||||
);
|
||||
expect(refRepo.insertMany).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('is a no-op when desired matches existing exactly', async () => {
|
||||
refRepo.findByReferencePageId.mockResolvedValue([
|
||||
{
|
||||
id: 'r',
|
||||
referencePageId,
|
||||
sourcePageId: 'p1',
|
||||
transclusionId: 'e1',
|
||||
createdAt: new Date(),
|
||||
} as any,
|
||||
]);
|
||||
const pm = {
|
||||
type: 'doc',
|
||||
content: [
|
||||
{
|
||||
type: 'transclusionReference',
|
||||
attrs: { sourcePageId: 'p1', transclusionId: 'e1' },
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = await service.syncPageReferences(referencePageId, workspaceId, pm);
|
||||
|
||||
expect(result).toEqual({ inserted: 0, deleted: 0 });
|
||||
expect(refRepo.insertMany).not.toHaveBeenCalled();
|
||||
expect(refRepo.deleteByReferenceAndKeys).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -6,11 +6,13 @@ import {
|
||||
} from '@nestjs/common';
|
||||
import { isDeepStrictEqual } from 'node:util';
|
||||
import { v7 as uuid7 } from 'uuid';
|
||||
import { KyselyTransaction } from '@docmost/db/types/kysely.types';
|
||||
import { InjectKysely } from 'nestjs-kysely';
|
||||
import { KyselyDB, KyselyTransaction } from '@docmost/db/types/kysely.types';
|
||||
import { PageTransclusionsRepo } from '@docmost/db/repos/page-transclusions/page-transclusions.repo';
|
||||
import { PageTransclusionReferencesRepo } from '@docmost/db/repos/page-transclusions/page-transclusion-references.repo';
|
||||
import { PageRepo } from '@docmost/db/repos/page/page.repo';
|
||||
import { PagePermissionRepo } from '@docmost/db/repos/page/page-permission.repo';
|
||||
import { SpaceMemberRepo } from '@docmost/db/repos/space/space-member.repo';
|
||||
import { AttachmentRepo } from '@docmost/db/repos/attachment/attachment.repo';
|
||||
import { StorageService } from '../../../integrations/storage/storage.service';
|
||||
import {
|
||||
@@ -36,10 +38,12 @@ export class TransclusionService {
|
||||
private readonly logger = new Logger(TransclusionService.name);
|
||||
|
||||
constructor(
|
||||
@InjectKysely() private readonly db: KyselyDB,
|
||||
private readonly pageTransclusionsRepo: PageTransclusionsRepo,
|
||||
private readonly pageTransclusionReferencesRepo: PageTransclusionReferencesRepo,
|
||||
private readonly pageRepo: PageRepo,
|
||||
private readonly pagePermissionRepo: PagePermissionRepo,
|
||||
private readonly spaceMemberRepo: SpaceMemberRepo,
|
||||
private readonly attachmentRepo: AttachmentRepo,
|
||||
private readonly storageService: StorageService,
|
||||
private readonly pageAccessService: PageAccessService,
|
||||
@@ -213,6 +217,40 @@ export class TransclusionService {
|
||||
return { inserted: rows.length };
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve viewer access for source page IDs supplied by an authenticated
|
||||
* caller. Restricts candidates to pages the viewer can see at the space
|
||||
* level before applying page-level restrictions, so a workspace member
|
||||
* cannot read a sync block from a private space they don't belong to via
|
||||
* an unrestricted source page.
|
||||
*/
|
||||
private async filterViewerAccessiblePageIds(
|
||||
pageIds: string[],
|
||||
viewerUserId: string,
|
||||
workspaceId: string,
|
||||
): Promise<string[]> {
|
||||
if (pageIds.length === 0) return [];
|
||||
|
||||
const spaceVisible = await this.db
|
||||
.selectFrom('pages')
|
||||
.select('id')
|
||||
.where('id', 'in', pageIds)
|
||||
.where('workspaceId', '=', workspaceId)
|
||||
.where('deletedAt', 'is', null)
|
||||
.where(
|
||||
'spaceId',
|
||||
'in',
|
||||
this.spaceMemberRepo.getUserSpaceIdsQuery(viewerUserId),
|
||||
)
|
||||
.execute();
|
||||
if (spaceVisible.length === 0) return [];
|
||||
|
||||
return this.pagePermissionRepo.filterAccessiblePageIds({
|
||||
pageIds: spaceVisible.map((r) => r.id),
|
||||
userId: viewerUserId,
|
||||
});
|
||||
}
|
||||
|
||||
async lookup(
|
||||
references: Array<{ sourcePageId: string; transclusionId: string }>,
|
||||
viewerUserId: string,
|
||||
@@ -224,10 +262,11 @@ export class TransclusionService {
|
||||
new Set(references.map((r) => r.sourcePageId)),
|
||||
);
|
||||
const accessibleSet = new Set(
|
||||
await this.pagePermissionRepo.filterAccessiblePageIds({
|
||||
pageIds: candidatePageIds,
|
||||
userId: viewerUserId,
|
||||
}),
|
||||
await this.filterViewerAccessiblePageIds(
|
||||
candidatePageIds,
|
||||
viewerUserId,
|
||||
workspaceId,
|
||||
),
|
||||
);
|
||||
|
||||
return this.lookupWithAccessSet(references, accessibleSet, workspaceId);
|
||||
@@ -336,10 +375,11 @@ export class TransclusionService {
|
||||
new Set([sourcePageId, ...referencePageIds]),
|
||||
);
|
||||
const accessibleSet = new Set(
|
||||
await this.pagePermissionRepo.filterAccessiblePageIds({
|
||||
pageIds: candidatePageIds,
|
||||
userId: viewerUserId,
|
||||
}),
|
||||
await this.filterViewerAccessiblePageIds(
|
||||
candidatePageIds,
|
||||
viewerUserId,
|
||||
workspaceId,
|
||||
),
|
||||
);
|
||||
|
||||
const accessibleIds = candidatePageIds.filter((id) =>
|
||||
|
||||
@@ -4,6 +4,7 @@ import {
|
||||
IsNumber,
|
||||
IsOptional,
|
||||
IsString,
|
||||
IsUUID,
|
||||
} from 'class-validator';
|
||||
|
||||
export class SearchDTO {
|
||||
@@ -12,7 +13,7 @@ export class SearchDTO {
|
||||
query: string;
|
||||
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
@IsUUID()
|
||||
spaceId: string;
|
||||
|
||||
@IsOptional()
|
||||
@@ -20,7 +21,7 @@ export class SearchDTO {
|
||||
shareId?: string;
|
||||
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
@IsUUID()
|
||||
creatorId?: string;
|
||||
|
||||
@IsOptional()
|
||||
@@ -38,7 +39,7 @@ export class SearchShareDTO extends SearchDTO {
|
||||
shareId: string;
|
||||
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
@IsUUID()
|
||||
spaceId: string;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import {
|
||||
IsAlphanumeric,
|
||||
IsOptional,
|
||||
IsString,
|
||||
Matches,
|
||||
MaxLength,
|
||||
MinLength,
|
||||
} from 'class-validator';
|
||||
@@ -20,6 +20,9 @@ export class CreateSpaceDto {
|
||||
|
||||
@MinLength(2)
|
||||
@MaxLength(100)
|
||||
@IsAlphanumeric()
|
||||
@Matches(/^[a-zA-Z0-9][a-zA-Z0-9_-]*$/, {
|
||||
message:
|
||||
'Space slug must start with a letter or number and may contain hyphens and underscores',
|
||||
})
|
||||
slug: string;
|
||||
}
|
||||
|
||||
@@ -48,6 +48,7 @@ export class SpaceService {
|
||||
workspaceId: string,
|
||||
createSpaceDto: CreateSpaceDto,
|
||||
trx?: KyselyTransaction,
|
||||
options?: { isPersonal?: boolean },
|
||||
): Promise<Space> {
|
||||
let space = null;
|
||||
|
||||
@@ -59,6 +60,7 @@ export class SpaceService {
|
||||
workspaceId,
|
||||
createSpaceDto,
|
||||
trx,
|
||||
options,
|
||||
);
|
||||
|
||||
await this.spaceMemberService.addUserToSpace(
|
||||
@@ -81,6 +83,7 @@ export class SpaceService {
|
||||
after: {
|
||||
name: space.name,
|
||||
slug: space.slug,
|
||||
...(space.isPersonal ? { isPersonal: true } : {}),
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -93,6 +96,7 @@ export class SpaceService {
|
||||
workspaceId: string,
|
||||
createSpaceDto: CreateSpaceDto,
|
||||
trx?: KyselyTransaction,
|
||||
options?: { isPersonal?: boolean },
|
||||
): Promise<Space> {
|
||||
const slugExists = await this.spaceRepo.slugExists(
|
||||
createSpaceDto.slug,
|
||||
@@ -112,6 +116,7 @@ export class SpaceService {
|
||||
creatorId: userId,
|
||||
workspaceId: workspaceId,
|
||||
slug: createSpaceDto.slug,
|
||||
isPersonal: options?.isPersonal ?? false,
|
||||
},
|
||||
trx,
|
||||
);
|
||||
|
||||
@@ -11,7 +11,7 @@ import {
|
||||
MaxLength,
|
||||
MinLength,
|
||||
} from 'class-validator';
|
||||
import { UserRole } from '../../../common/helpers/types/permission';
|
||||
import { InviteUserRole } from '../../../common/helpers/types/permission';
|
||||
import { NoUrls } from '../../../common/validators/no-urls.validator';
|
||||
|
||||
export class InviteUserDto {
|
||||
@@ -32,7 +32,7 @@ export class InviteUserDto {
|
||||
@IsUUID('all', { each: true })
|
||||
groupIds: string[];
|
||||
|
||||
@IsEnum(UserRole)
|
||||
@IsEnum(InviteUserRole)
|
||||
role: string;
|
||||
}
|
||||
|
||||
|
||||
@@ -3,8 +3,10 @@ import { CreateWorkspaceDto } from './create-workspace.dto';
|
||||
import {
|
||||
IsArray,
|
||||
IsBoolean,
|
||||
IsIn,
|
||||
IsInt,
|
||||
IsOptional,
|
||||
IsString,
|
||||
Min,
|
||||
} from 'class-validator';
|
||||
|
||||
@@ -57,4 +59,13 @@ export class UpdateWorkspaceDto extends PartialType(CreateWorkspaceDto) {
|
||||
@IsOptional()
|
||||
@IsBoolean()
|
||||
allowMemberTemplates: boolean;
|
||||
|
||||
@IsOptional()
|
||||
@IsBoolean()
|
||||
allowPersonalSpaces: boolean;
|
||||
|
||||
@IsOptional()
|
||||
@IsString()
|
||||
@IsIn(['read', 'edit'])
|
||||
defaultPageEditMode: string;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import {
|
||||
BadRequestException,
|
||||
ForbiddenException,
|
||||
Inject,
|
||||
Injectable,
|
||||
Logger,
|
||||
@@ -40,6 +41,10 @@ import {
|
||||
AUDIT_SERVICE,
|
||||
IAuditService,
|
||||
} from '../../../integrations/audit/audit.service';
|
||||
import {
|
||||
getWorkspaceDefaultPageEditMode,
|
||||
isAdminActingOnOwner,
|
||||
} from '../workspace.util';
|
||||
|
||||
@Injectable()
|
||||
export class WorkspaceInvitationService {
|
||||
@@ -119,6 +124,10 @@ export class WorkspaceInvitationService {
|
||||
): Promise<void> {
|
||||
const { emails, role, groupIds } = inviteUserDto;
|
||||
|
||||
if (isAdminActingOnOwner(authUser.role, role)) {
|
||||
throw new ForbiddenException();
|
||||
}
|
||||
|
||||
let invites: WorkspaceInvitation[] = [];
|
||||
|
||||
try {
|
||||
@@ -251,6 +260,7 @@ export class WorkspaceInvitationService {
|
||||
workspaceId: workspace.id,
|
||||
},
|
||||
trx,
|
||||
{ pageEditMode: getWorkspaceDefaultPageEditMode(workspace) },
|
||||
);
|
||||
|
||||
// add user to default group
|
||||
|
||||
@@ -30,6 +30,7 @@ import { DomainService } from '../../../integrations/environment/domain.service'
|
||||
import { jsonArrayFrom } from 'kysely/helpers/postgres';
|
||||
import { addDays } from 'date-fns';
|
||||
import { DISALLOWED_HOSTNAMES, WorkspaceStatus } from '../workspace.constants';
|
||||
import { isAdminActingOnOwner } from '../workspace.util';
|
||||
import { v4 } from 'uuid';
|
||||
import { InjectQueue } from '@nestjs/bullmq';
|
||||
import { QueueJob, QueueName } from '../../../integrations/queue/constants';
|
||||
@@ -332,7 +333,8 @@ export class WorkspaceService {
|
||||
typeof updateWorkspaceDto.mcpEnabled !== 'undefined' ||
|
||||
typeof updateWorkspaceDto.restrictApiToAdmins !== 'undefined' ||
|
||||
typeof updateWorkspaceDto.allowMemberTemplates !== 'undefined' ||
|
||||
typeof updateWorkspaceDto.isScimEnabled !== 'undefined'
|
||||
typeof updateWorkspaceDto.isScimEnabled !== 'undefined' ||
|
||||
typeof updateWorkspaceDto.allowPersonalSpaces !== 'undefined'
|
||||
) {
|
||||
const ws = await this.db
|
||||
.selectFrom('workspaces')
|
||||
@@ -360,6 +362,18 @@ export class WorkspaceService {
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof updateWorkspaceDto.allowPersonalSpaces !== 'undefined') {
|
||||
if (
|
||||
!this.licenseCheckService.hasFeature(
|
||||
ws.licenseKey,
|
||||
Feature.PERSONAL_SPACES,
|
||||
ws.plan,
|
||||
)
|
||||
) {
|
||||
throw new ForbiddenException('This feature requires a valid license');
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
typeof updateWorkspaceDto.disablePublicSharing !== 'undefined' ||
|
||||
typeof updateWorkspaceDto.trashRetentionDays !== 'undefined' ||
|
||||
@@ -499,6 +513,34 @@ export class WorkspaceService {
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof updateWorkspaceDto.allowPersonalSpaces !== 'undefined') {
|
||||
const prev = settingsBefore?.spaces?.allowPersonal ?? false;
|
||||
if (prev !== updateWorkspaceDto.allowPersonalSpaces) {
|
||||
before.allowPersonalSpaces = prev;
|
||||
after.allowPersonalSpaces = updateWorkspaceDto.allowPersonalSpaces;
|
||||
}
|
||||
await this.workspaceRepo.updateSpaceSettings(
|
||||
workspaceId,
|
||||
'allowPersonal',
|
||||
updateWorkspaceDto.allowPersonalSpaces,
|
||||
trx,
|
||||
);
|
||||
}
|
||||
|
||||
if (typeof updateWorkspaceDto.defaultPageEditMode !== 'undefined') {
|
||||
const prev = settingsBefore?.defaultPageEditMode ?? null;
|
||||
const next = updateWorkspaceDto.defaultPageEditMode.toLowerCase();
|
||||
if (prev !== next) {
|
||||
before.defaultPageEditMode = prev;
|
||||
after.defaultPageEditMode = next;
|
||||
}
|
||||
await this.workspaceRepo.updateDefaultPageEditMode(
|
||||
workspaceId,
|
||||
next,
|
||||
trx,
|
||||
);
|
||||
}
|
||||
|
||||
delete updateWorkspaceDto.restrictApiToAdmins;
|
||||
delete updateWorkspaceDto.aiSearch;
|
||||
delete updateWorkspaceDto.generativeAi;
|
||||
@@ -506,6 +548,8 @@ export class WorkspaceService {
|
||||
delete updateWorkspaceDto.mcpEnabled;
|
||||
delete updateWorkspaceDto.allowMemberTemplates;
|
||||
delete updateWorkspaceDto.aiChat;
|
||||
delete updateWorkspaceDto.allowPersonalSpaces;
|
||||
delete updateWorkspaceDto.defaultPageEditMode;
|
||||
|
||||
await this.workspaceRepo.updateWorkspace(
|
||||
updateWorkspaceDto,
|
||||
@@ -590,8 +634,8 @@ export class WorkspaceService {
|
||||
|
||||
// prevent ADMIN from managing OWNER role
|
||||
if (
|
||||
(authUser.role === UserRole.ADMIN && newRole === UserRole.OWNER) ||
|
||||
(authUser.role === UserRole.ADMIN && user.role === UserRole.OWNER)
|
||||
isAdminActingOnOwner(authUser.role, newRole) ||
|
||||
isAdminActingOnOwner(authUser.role, user.role)
|
||||
) {
|
||||
throw new ForbiddenException();
|
||||
}
|
||||
@@ -695,7 +739,7 @@ export class WorkspaceService {
|
||||
throw new BadRequestException('You cannot deactivate yourself');
|
||||
}
|
||||
|
||||
if (authUser.role === UserRole.ADMIN && user.role === UserRole.OWNER) {
|
||||
if (isAdminActingOnOwner(authUser.role, user.role)) {
|
||||
throw new BadRequestException(
|
||||
'You cannot deactivate a user with owner role',
|
||||
);
|
||||
@@ -753,7 +797,7 @@ export class WorkspaceService {
|
||||
throw new BadRequestException('User is not deactivated');
|
||||
}
|
||||
|
||||
if (authUser.role === UserRole.ADMIN && user.role === UserRole.OWNER) {
|
||||
if (isAdminActingOnOwner(authUser.role, user.role)) {
|
||||
throw new BadRequestException(
|
||||
'You cannot activate a user with owner role',
|
||||
);
|
||||
@@ -805,7 +849,7 @@ export class WorkspaceService {
|
||||
throw new BadRequestException('You cannot delete yourself');
|
||||
}
|
||||
|
||||
if (authUser.role === UserRole.ADMIN && user.role === UserRole.OWNER) {
|
||||
if (isAdminActingOnOwner(authUser.role, user.role)) {
|
||||
throw new BadRequestException('You cannot delete a user with owner role');
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
import { UserRole } from '../../common/helpers/types/permission';
|
||||
|
||||
export function isAdminActingOnOwner(
|
||||
authUserRole: string,
|
||||
targetRole: string,
|
||||
): boolean {
|
||||
return authUserRole === UserRole.ADMIN && targetRole === UserRole.OWNER;
|
||||
}
|
||||
|
||||
export type PageEditMode = 'read' | 'edit';
|
||||
|
||||
export function getWorkspaceDefaultPageEditMode(
|
||||
workspace: { settings?: unknown } | null | undefined,
|
||||
): PageEditMode | undefined {
|
||||
const settings = (workspace?.settings ?? {}) as {
|
||||
defaultPageEditMode?: unknown;
|
||||
};
|
||||
const mode = settings.defaultPageEditMode;
|
||||
if (mode === 'read' || mode === 'edit') {
|
||||
return mode;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
import { type Kysely, sql } from 'kysely';
|
||||
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await db.schema
|
||||
.alterTable('pages')
|
||||
.addColumn('is_base', 'boolean', (col) =>
|
||||
col.ifNotExists().notNull().defaultTo(false),
|
||||
)
|
||||
.addColumn('base_schema_version', 'integer', (col) =>
|
||||
col.ifNotExists().notNull().defaultTo(0),
|
||||
)
|
||||
.execute();
|
||||
|
||||
await sql`
|
||||
CREATE INDEX IF NOT EXISTS idx_pages_is_base
|
||||
ON pages (space_id, position COLLATE "C")
|
||||
WHERE is_base = true AND deleted_at IS NULL
|
||||
`.execute(db);
|
||||
|
||||
await db.schema
|
||||
.createTable('base_properties')
|
||||
.ifNotExists()
|
||||
.addColumn('id', 'varchar', (col) => col.notNull())
|
||||
.addColumn('page_id', 'uuid', (col) =>
|
||||
col.references('pages.id').onDelete('cascade').notNull(),
|
||||
)
|
||||
.addColumn('name', 'varchar', (col) => col.notNull())
|
||||
.addColumn('type', 'varchar', (col) => col.notNull())
|
||||
.addColumn('position', 'varchar', (col) => col.notNull())
|
||||
.addColumn('type_options', 'jsonb')
|
||||
.addColumn('pending_type', 'varchar')
|
||||
.addColumn('pending_type_options', 'jsonb')
|
||||
.addColumn('pending_token', 'uuid')
|
||||
.addColumn('is_primary', 'boolean', (col) => col.notNull().defaultTo(false))
|
||||
.addColumn('schema_version', 'integer', (col) => col.notNull().defaultTo(1))
|
||||
.addColumn('workspace_id', 'uuid', (col) =>
|
||||
col.references('workspaces.id').onDelete('cascade').notNull(),
|
||||
)
|
||||
.addColumn('created_at', 'timestamptz', (col) =>
|
||||
col.notNull().defaultTo(sql`now()`),
|
||||
)
|
||||
.addColumn('updated_at', 'timestamptz', (col) =>
|
||||
col.notNull().defaultTo(sql`now()`),
|
||||
)
|
||||
.addColumn('deleted_at', 'timestamptz')
|
||||
.addPrimaryKeyConstraint('base_properties_pkey', ['page_id', 'id'])
|
||||
.execute();
|
||||
|
||||
await sql`CREATE INDEX IF NOT EXISTS idx_base_properties_page_id ON base_properties (page_id)`.execute(
|
||||
db,
|
||||
);
|
||||
|
||||
await sql`
|
||||
CREATE INDEX IF NOT EXISTS idx_base_properties_page_alive
|
||||
ON base_properties (page_id, position COLLATE "C", id)
|
||||
WHERE deleted_at IS NULL
|
||||
`.execute(db);
|
||||
|
||||
// Match the service-layer name check (name.trim().toLowerCase()) so
|
||||
// whitespace-padded duplicates also collide. Formulas look properties up by
|
||||
// name, so the names have to stay unique.
|
||||
await sql`
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS base_properties_page_name_alive_unique
|
||||
ON base_properties (page_id, lower(trim(name)))
|
||||
WHERE deleted_at IS NULL
|
||||
`.execute(db);
|
||||
|
||||
await db.schema
|
||||
.createTable('base_rows')
|
||||
.ifNotExists()
|
||||
.addColumn('id', 'uuid', (col) =>
|
||||
col.primaryKey().defaultTo(sql`gen_uuid_v7()`),
|
||||
)
|
||||
.addColumn('page_id', 'uuid', (col) =>
|
||||
col.references('pages.id').onDelete('cascade').notNull(),
|
||||
)
|
||||
.addColumn('cells', 'jsonb', (col) =>
|
||||
col.notNull().defaultTo(sql`'{}'::jsonb`),
|
||||
)
|
||||
.addColumn('position', 'varchar', (col) => col.notNull())
|
||||
.addColumn('creator_id', 'uuid', (col) =>
|
||||
col.references('users.id').onDelete('set null'),
|
||||
)
|
||||
.addColumn('last_updated_by_id', 'uuid', (col) =>
|
||||
col.references('users.id').onDelete('set null'),
|
||||
)
|
||||
.addColumn('workspace_id', 'uuid', (col) =>
|
||||
col.references('workspaces.id').onDelete('cascade').notNull(),
|
||||
)
|
||||
.addColumn('created_at', 'timestamptz', (col) =>
|
||||
col.notNull().defaultTo(sql`now()`),
|
||||
)
|
||||
.addColumn('updated_at', 'timestamptz', (col) =>
|
||||
col.notNull().defaultTo(sql`now()`),
|
||||
)
|
||||
.addColumn('deleted_at', 'timestamptz')
|
||||
.execute();
|
||||
|
||||
await sql`
|
||||
CREATE INDEX IF NOT EXISTS idx_base_rows_page_alive
|
||||
ON base_rows (page_id, position COLLATE "C", id)
|
||||
WHERE deleted_at IS NULL
|
||||
`.execute(db);
|
||||
|
||||
await sql`
|
||||
CREATE INDEX IF NOT EXISTS idx_base_rows_page_updated
|
||||
ON base_rows (page_id, updated_at DESC)
|
||||
WHERE deleted_at IS NULL
|
||||
`.execute(db);
|
||||
|
||||
await sql`
|
||||
CREATE INDEX IF NOT EXISTS idx_base_rows_page_created
|
||||
ON base_rows (page_id, created_at DESC)
|
||||
WHERE deleted_at IS NULL
|
||||
`.execute(db);
|
||||
|
||||
await db.schema
|
||||
.createTable('base_views')
|
||||
.ifNotExists()
|
||||
.addColumn('id', 'uuid', (col) =>
|
||||
col.primaryKey().defaultTo(sql`gen_uuid_v7()`),
|
||||
)
|
||||
.addColumn('page_id', 'uuid', (col) =>
|
||||
col.references('pages.id').onDelete('cascade').notNull(),
|
||||
)
|
||||
.addColumn('name', 'varchar', (col) => col.notNull())
|
||||
.addColumn('type', 'varchar', (col) => col.notNull().defaultTo('table'))
|
||||
.addColumn('position', 'varchar', (col) => col.notNull())
|
||||
.addColumn('config', 'jsonb', (col) =>
|
||||
col.notNull().defaultTo(sql`'{}'::jsonb`),
|
||||
)
|
||||
.addColumn('workspace_id', 'uuid', (col) =>
|
||||
col.references('workspaces.id').onDelete('cascade').notNull(),
|
||||
)
|
||||
.addColumn('creator_id', 'uuid', (col) =>
|
||||
col.references('users.id').onDelete('set null'),
|
||||
)
|
||||
.addColumn('created_at', 'timestamptz', (col) =>
|
||||
col.notNull().defaultTo(sql`now()`),
|
||||
)
|
||||
.addColumn('updated_at', 'timestamptz', (col) =>
|
||||
col.notNull().defaultTo(sql`now()`),
|
||||
)
|
||||
.execute();
|
||||
|
||||
await sql`CREATE INDEX IF NOT EXISTS idx_base_views_page_id ON base_views (page_id)`.execute(
|
||||
db,
|
||||
);
|
||||
|
||||
// Cell extraction helpers for filters and sorts. Return NULL for absent or
|
||||
// non-castable values.
|
||||
await sql`
|
||||
CREATE OR REPLACE FUNCTION base_cell_text(cells jsonb, prop text)
|
||||
RETURNS text LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE
|
||||
AS $$ SELECT cells->>prop::text $$
|
||||
`.execute(db);
|
||||
|
||||
await sql`
|
||||
CREATE OR REPLACE FUNCTION base_cell_numeric(cells jsonb, prop text)
|
||||
RETURNS numeric LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE
|
||||
AS $$
|
||||
SELECT CASE jsonb_typeof(cells->prop::text)
|
||||
WHEN 'number' THEN (cells->>prop::text)::numeric
|
||||
WHEN 'string' THEN
|
||||
CASE
|
||||
WHEN (cells->>prop::text) ~
|
||||
'^[[:space:]]*[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?[[:space:]]*$'
|
||||
THEN (cells->>prop::text)::numeric
|
||||
END
|
||||
END
|
||||
$$
|
||||
`.execute(db);
|
||||
|
||||
// A DATE cell stores an arbitrary string (cell schema is z.string()), so the
|
||||
// cast can fail on values no regex can pre-validate (e.g. '2024-13-45'). This
|
||||
// helper uses plpgsql with an EXCEPTION handler to return NULL on failure.
|
||||
await sql`
|
||||
CREATE OR REPLACE FUNCTION base_cell_timestamptz(cells jsonb, prop text)
|
||||
RETURNS timestamptz LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE
|
||||
AS $$
|
||||
BEGIN RETURN (cells->>prop::text)::timestamptz;
|
||||
EXCEPTION WHEN others THEN RETURN NULL; END;
|
||||
$$
|
||||
`.execute(db);
|
||||
|
||||
await sql`
|
||||
CREATE OR REPLACE FUNCTION base_cell_bool(cells jsonb, prop text)
|
||||
RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE
|
||||
AS $$
|
||||
SELECT CASE jsonb_typeof(cells->prop::text)
|
||||
WHEN 'boolean' THEN (cells->>prop::text)::boolean
|
||||
WHEN 'string' THEN
|
||||
CASE
|
||||
WHEN lower(btrim(cells->>prop::text)) IN
|
||||
('true','t','yes','y','on','1','false','f','no','n','off','0')
|
||||
THEN (cells->>prop::text)::boolean
|
||||
END
|
||||
END
|
||||
$$
|
||||
`.execute(db);
|
||||
|
||||
await sql`
|
||||
CREATE OR REPLACE FUNCTION base_cell_array(cells jsonb, prop text)
|
||||
RETURNS jsonb LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE
|
||||
AS $$ SELECT cells->prop::text $$
|
||||
`.execute(db);
|
||||
|
||||
// A null patch value deletes the key rather than storing a JSON null.
|
||||
await sql`
|
||||
CREATE OR REPLACE FUNCTION jsonb_set_many(target jsonb, patches jsonb)
|
||||
RETURNS jsonb LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE
|
||||
AS $$
|
||||
DECLARE k text; v jsonb; result jsonb := coalesce(target, '{}'::jsonb);
|
||||
BEGIN
|
||||
IF patches IS NULL OR jsonb_typeof(patches) <> 'object' THEN
|
||||
RETURN result;
|
||||
END IF;
|
||||
FOR k, v IN SELECT * FROM jsonb_each(patches) LOOP
|
||||
IF v = 'null'::jsonb THEN
|
||||
result := result - k;
|
||||
ELSE
|
||||
result := jsonb_set(result, ARRAY[k], v, true);
|
||||
END IF;
|
||||
END LOOP;
|
||||
RETURN result;
|
||||
END;
|
||||
$$
|
||||
`.execute(db);
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await db.schema.dropTable('base_views').execute();
|
||||
await db.schema.dropTable('base_rows').execute();
|
||||
await db.schema.dropTable('base_properties').execute();
|
||||
|
||||
await sql`DROP FUNCTION jsonb_set_many(jsonb, jsonb)`.execute(db);
|
||||
await sql`DROP FUNCTION base_cell_array(jsonb, text)`.execute(db);
|
||||
await sql`DROP FUNCTION base_cell_bool(jsonb, text)`.execute(db);
|
||||
await sql`DROP FUNCTION base_cell_timestamptz(jsonb, text)`.execute(db);
|
||||
await sql`DROP FUNCTION base_cell_numeric(jsonb, text)`.execute(db);
|
||||
await sql`DROP FUNCTION base_cell_text(jsonb, text)`.execute(db);
|
||||
|
||||
await sql`DROP INDEX idx_pages_is_base`.execute(db);
|
||||
await db.schema
|
||||
.alterTable('pages')
|
||||
.dropColumn('base_schema_version')
|
||||
.dropColumn('is_base')
|
||||
.execute();
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
import { Kysely, sql } from 'kysely';
|
||||
|
||||
export async function up(db: Kysely<any>): Promise<void> {
|
||||
await db.schema
|
||||
.alterTable('spaces')
|
||||
.addColumn('is_personal', 'boolean', (col) =>
|
||||
col.notNull().defaultTo(false),
|
||||
)
|
||||
.execute();
|
||||
|
||||
await sql`
|
||||
CREATE UNIQUE INDEX spaces_personal_creator_unique
|
||||
ON spaces (creator_id)
|
||||
WHERE is_personal = true AND deleted_at IS NULL
|
||||
`.execute(db);
|
||||
}
|
||||
|
||||
export async function down(db: Kysely<any>): Promise<void> {
|
||||
await db.schema
|
||||
.dropIndex('spaces_personal_creator_unique')
|
||||
.ifExists()
|
||||
.execute();
|
||||
await db.schema.alterTable('spaces').dropColumn('is_personal').execute();
|
||||
}
|
||||
@@ -14,12 +14,14 @@ type SortField<DB, TB extends keyof DB, O> =
|
||||
| (StringReference<DB, TB> & `${string}.${keyof O & string}`);
|
||||
direction: OrderByDirection;
|
||||
orderModifier?: OrderByModifiers;
|
||||
cursorExpression?: ReferenceExpression<DB, TB>;
|
||||
key?: keyof O & string;
|
||||
}
|
||||
| {
|
||||
expression: ReferenceExpression<DB, TB>;
|
||||
direction: OrderByDirection;
|
||||
orderModifier?: OrderByModifiers;
|
||||
cursorExpression?: ReferenceExpression<DB, TB>;
|
||||
key: keyof O & string;
|
||||
};
|
||||
|
||||
@@ -202,11 +204,12 @@ export async function executeWithCursorPagination<
|
||||
|
||||
const comparison = field.direction === defaultDirection ? '>' : '<';
|
||||
const value = cursor[field.key as keyof typeof cursor];
|
||||
const compareExpr = field.cursorExpression ?? field.expression;
|
||||
|
||||
const conditions = [eb(field.expression, comparison, value)];
|
||||
const conditions = [eb(compareExpr, comparison, value)];
|
||||
|
||||
if (expression) {
|
||||
conditions.push(and([eb(field.expression, '=', value), expression]));
|
||||
conditions.push(and([eb(compareExpr, '=', value), expression]));
|
||||
}
|
||||
|
||||
expression = or(conditions);
|
||||
|
||||
@@ -236,6 +236,7 @@ export class FavoriteRepo {
|
||||
'pages.slugId',
|
||||
'pages.title',
|
||||
'pages.icon',
|
||||
'pages.isBase',
|
||||
'pages.spaceId',
|
||||
])
|
||||
.whereRef('pages.id', '=', 'favorites.pageId'),
|
||||
|
||||
@@ -38,6 +38,7 @@ export class PageRepo {
|
||||
'spaceId',
|
||||
'workspaceId',
|
||||
'isLocked',
|
||||
'isBase',
|
||||
'createdAt',
|
||||
'updatedAt',
|
||||
'deletedAt',
|
||||
|
||||
@@ -57,6 +57,22 @@ export class SpaceRepo {
|
||||
.executeTakeFirst();
|
||||
}
|
||||
|
||||
async findPersonalSpace(
|
||||
userId: string,
|
||||
workspaceId: string,
|
||||
trx?: KyselyTransaction,
|
||||
): Promise<Space | undefined> {
|
||||
const db = dbOrTx(this.db, trx);
|
||||
return db
|
||||
.selectFrom('spaces')
|
||||
.selectAll('spaces')
|
||||
.where('workspaceId', '=', workspaceId)
|
||||
.where('creatorId', '=', userId)
|
||||
.where('isPersonal', '=', true)
|
||||
.where('deletedAt', 'is', null)
|
||||
.executeTakeFirst();
|
||||
}
|
||||
|
||||
async slugExists(
|
||||
slug: string,
|
||||
workspaceId: string,
|
||||
|
||||
@@ -112,6 +112,7 @@ export class UserRepo {
|
||||
async insertUser(
|
||||
insertableUser: InsertableUser,
|
||||
trx?: KyselyTransaction,
|
||||
opts?: { pageEditMode?: string },
|
||||
): Promise<User> {
|
||||
const user: InsertableUser = {
|
||||
name:
|
||||
@@ -126,7 +127,17 @@ export class UserRepo {
|
||||
const db = dbOrTx(this.db, trx);
|
||||
return db
|
||||
.insertInto('users')
|
||||
.values({ ...insertableUser, ...user })
|
||||
.values({
|
||||
...insertableUser,
|
||||
...user,
|
||||
...(opts?.pageEditMode
|
||||
? {
|
||||
settings: sql`${JSON.stringify({
|
||||
preferences: { pageEditMode: opts.pageEditMode },
|
||||
})}::text::jsonb`,
|
||||
}
|
||||
: {}),
|
||||
})
|
||||
.returning(this.baseFields)
|
||||
.executeTakeFirst();
|
||||
}
|
||||
|
||||
@@ -251,4 +251,42 @@ export class WorkspaceRepo {
|
||||
.executeTakeFirst();
|
||||
}
|
||||
|
||||
async updateSpaceSettings(
|
||||
workspaceId: string,
|
||||
prefKey: string,
|
||||
prefValue: string | boolean,
|
||||
trx?: KyselyTransaction,
|
||||
) {
|
||||
const db = dbOrTx(this.db, trx);
|
||||
return db
|
||||
.updateTable('workspaces')
|
||||
.set({
|
||||
settings: sql`COALESCE(settings, '{}'::jsonb)
|
||||
|| jsonb_build_object('spaces', COALESCE(settings->'spaces', '{}'::jsonb)
|
||||
|| jsonb_build_object('${sql.raw(prefKey)}', ${sql.lit(prefValue)}))`,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where('id', '=', workspaceId)
|
||||
.returning(this.baseFields)
|
||||
.executeTakeFirst();
|
||||
}
|
||||
|
||||
async updateDefaultPageEditMode(
|
||||
workspaceId: string,
|
||||
pageEditMode: string,
|
||||
trx?: KyselyTransaction,
|
||||
) {
|
||||
const db = dbOrTx(this.db, trx);
|
||||
return db
|
||||
.updateTable('workspaces')
|
||||
.set({
|
||||
settings: sql`COALESCE(settings, '{}'::jsonb)
|
||||
|| jsonb_build_object('defaultPageEditMode', ${sql.lit(pageEditMode)})`,
|
||||
updatedAt: new Date(),
|
||||
})
|
||||
.where('id', '=', workspaceId)
|
||||
.returning(this.baseFields)
|
||||
.executeTakeFirst();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+50
@@ -126,6 +126,50 @@ export interface Backlinks {
|
||||
workspaceId: string;
|
||||
}
|
||||
|
||||
export interface BaseProperties {
|
||||
createdAt: Generated<Timestamp>;
|
||||
deletedAt: Timestamp | null;
|
||||
id: Generated<string>;
|
||||
isPrimary: Generated<boolean>;
|
||||
name: string;
|
||||
pageId: string;
|
||||
pendingType: string | null;
|
||||
pendingTypeOptions: Json | null;
|
||||
pendingToken: string | null;
|
||||
position: string;
|
||||
schemaVersion: Generated<number>;
|
||||
type: string;
|
||||
typeOptions: Json | null;
|
||||
updatedAt: Generated<Timestamp>;
|
||||
workspaceId: string;
|
||||
}
|
||||
|
||||
export interface BaseRows {
|
||||
cells: Generated<Json>;
|
||||
createdAt: Generated<Timestamp>;
|
||||
creatorId: string | null;
|
||||
deletedAt: Timestamp | null;
|
||||
id: Generated<string>;
|
||||
lastUpdatedById: string | null;
|
||||
pageId: string;
|
||||
position: string;
|
||||
updatedAt: Generated<Timestamp>;
|
||||
workspaceId: string;
|
||||
}
|
||||
|
||||
export interface BaseViews {
|
||||
config: Generated<Json>;
|
||||
createdAt: Generated<Timestamp>;
|
||||
creatorId: string | null;
|
||||
id: Generated<string>;
|
||||
name: string;
|
||||
pageId: string;
|
||||
position: string;
|
||||
type: Generated<string>;
|
||||
updatedAt: Generated<Timestamp>;
|
||||
workspaceId: string;
|
||||
}
|
||||
|
||||
export interface Billing {
|
||||
amount: Int8 | null;
|
||||
billingScheme: string | null;
|
||||
@@ -275,6 +319,8 @@ export interface Pages {
|
||||
deletedById: string | null;
|
||||
icon: string | null;
|
||||
id: Generated<string>;
|
||||
isBase: Generated<boolean>;
|
||||
baseSchemaVersion: Generated<number>;
|
||||
isLocked: Generated<boolean>;
|
||||
lastUpdatedById: string | null;
|
||||
parentPageId: string | null;
|
||||
@@ -322,6 +368,7 @@ export interface Spaces {
|
||||
deletedAt: Timestamp | null;
|
||||
description: string | null;
|
||||
id: Generated<string>;
|
||||
isPersonal: Generated<boolean>;
|
||||
logo: string | null;
|
||||
name: string | null;
|
||||
settings: Json | null;
|
||||
@@ -638,6 +685,9 @@ export interface DB {
|
||||
authAccounts: AuthAccounts;
|
||||
authProviders: AuthProviders;
|
||||
backlinks: Backlinks;
|
||||
baseProperties: BaseProperties;
|
||||
baseRows: BaseRows;
|
||||
baseViews: BaseViews;
|
||||
billing: Billing;
|
||||
comments: Comments;
|
||||
favorites: Favorites;
|
||||
|
||||
@@ -3,6 +3,9 @@ import {
|
||||
AiChats,
|
||||
AiChatMessages,
|
||||
Attachments,
|
||||
BaseProperties,
|
||||
BaseRows,
|
||||
BaseViews,
|
||||
Comments,
|
||||
Groups,
|
||||
Integrations as _Integrations,
|
||||
@@ -261,3 +264,18 @@ export type UpdatableAudit = Updateable<Omit<_Audit, 'id'>>;
|
||||
export type Template = Selectable<Templates>;
|
||||
export type InsertableTemplate = Insertable<Templates>;
|
||||
export type UpdatableTemplate = Updateable<Omit<Templates, 'id'>>;
|
||||
|
||||
// Base Property
|
||||
export type BaseProperty = Selectable<BaseProperties>;
|
||||
export type InsertableBaseProperty = Insertable<BaseProperties>;
|
||||
export type UpdatableBaseProperty = Updateable<Omit<BaseProperties, 'id'>>;
|
||||
|
||||
// Base Row
|
||||
export type BaseRow = Selectable<BaseRows>;
|
||||
export type InsertableBaseRow = Insertable<BaseRows>;
|
||||
export type UpdatableBaseRow = Updateable<Omit<BaseRows, 'id'>>;
|
||||
|
||||
// Base View
|
||||
export type BaseView = Selectable<BaseViews>;
|
||||
export type InsertableBaseView = Insertable<BaseViews>;
|
||||
export type UpdatableBaseView = Updateable<Omit<BaseViews, 'id'>>;
|
||||
|
||||
+1
-1
Submodule apps/server/src/ee updated: 9beaf4a8ab...08bafd4bb9
@@ -16,6 +16,7 @@ import {
|
||||
computeLocalPath,
|
||||
getExportExtension,
|
||||
getPageTitle,
|
||||
getSafePageTitle,
|
||||
PageExportTree,
|
||||
replaceInternalLinks,
|
||||
updateAttachmentUrlsToLocalPaths,
|
||||
@@ -314,7 +315,7 @@ export class ExportService {
|
||||
updateAttachmentUrlsToLocalPaths(updatedJsonContent);
|
||||
}
|
||||
|
||||
const pageTitle = getPageTitle(page.title);
|
||||
const pageTitle = getSafePageTitle(page.title);
|
||||
const pageExportContent = await this.exportPage(format, {
|
||||
...page,
|
||||
content: updatedJsonContent,
|
||||
|
||||
@@ -6,6 +6,7 @@ import { validate as isValidUUID } from 'uuid';
|
||||
import * as path from 'path';
|
||||
import { Page } from '@docmost/db/types/entity.types';
|
||||
import { isAttachmentNode } from '../../common/helpers/prosemirror/utils';
|
||||
import { sanitizeFileName } from '../../common/helpers';
|
||||
|
||||
export type PageExportTree = Record<string, Page[]>;
|
||||
|
||||
@@ -27,6 +28,13 @@ export function getPageTitle(title: string) {
|
||||
return title ? title : 'untitled';
|
||||
}
|
||||
|
||||
export function getSafePageTitle(title: string): string {
|
||||
const sanitized = sanitizeFileName(getPageTitle(title), {
|
||||
preserveSpaces: true,
|
||||
});
|
||||
return sanitized || 'untitled';
|
||||
}
|
||||
|
||||
export function updateAttachmentUrlsToLocalPaths(prosemirrorJson: any) {
|
||||
const doc = jsonToNode(prosemirrorJson);
|
||||
if (!doc) return null;
|
||||
@@ -167,7 +175,7 @@ export function computeLocalPath(
|
||||
const children = tree[parentPageId] || [];
|
||||
|
||||
for (const page of children) {
|
||||
const title = encodeURIComponent(getPageTitle(page.title));
|
||||
const title = encodeURIComponent(getSafePageTitle(page.title));
|
||||
const localPath = `${currentPath}${title}`;
|
||||
slugIdToPath[page.slugId] = `${localPath}${getExportExtension(format)}`;
|
||||
|
||||
|
||||
@@ -102,8 +102,10 @@ export class ImportService {
|
||||
throw new BadRequestException(message);
|
||||
}
|
||||
|
||||
const { title, prosemirrorJson } =
|
||||
this.extractTitleAndRemoveHeading(prosemirrorState);
|
||||
const { title, prosemirrorJson } = this.extractTitleAndRemoveHeading(
|
||||
prosemirrorState,
|
||||
{ anyHeadingLevel: true },
|
||||
);
|
||||
|
||||
const pageTitle = title || fileName;
|
||||
|
||||
@@ -246,18 +248,29 @@ export class ImportService {
|
||||
return null;
|
||||
}
|
||||
|
||||
extractTitleAndRemoveHeading(prosemirrorState: any) {
|
||||
extractTitleAndRemoveHeading(
|
||||
prosemirrorState: any,
|
||||
opts?: { anyHeadingLevel?: boolean },
|
||||
) {
|
||||
let title: string | null = null;
|
||||
|
||||
const content = prosemirrorState.content ?? [];
|
||||
const firstNode = content[0];
|
||||
|
||||
if (
|
||||
content.length > 0 &&
|
||||
content[0].type === 'heading' &&
|
||||
content[0].attrs?.level === 1
|
||||
) {
|
||||
title = content[0].content?.[0]?.text ?? null;
|
||||
content.shift();
|
||||
const isTitleHeading =
|
||||
firstNode?.type === 'heading' &&
|
||||
(opts?.anyHeadingLevel || firstNode.attrs?.level === 1);
|
||||
|
||||
if (isTitleHeading) {
|
||||
const headingText = (firstNode.content ?? [])
|
||||
.map((node: any) => node.text ?? '')
|
||||
.join('')
|
||||
.trim();
|
||||
|
||||
if (headingText) {
|
||||
title = headingText;
|
||||
content.shift();
|
||||
}
|
||||
}
|
||||
|
||||
// ensure at least one paragraph
|
||||
|
||||
@@ -31,31 +31,39 @@ export function getFileTaskFolderPath(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extracts a ZIP archive.
|
||||
*/
|
||||
const COMPRESSION_HEADROOM = 10;
|
||||
const MIN_EXTRACTED_BYTES = 256 * 1024 * 1024;
|
||||
const MAX_ENTRIES = 250_000;
|
||||
|
||||
type SizeBudget = { used: number; max: number };
|
||||
|
||||
export async function extractZip(
|
||||
source: string,
|
||||
target: string,
|
||||
): Promise<void> {
|
||||
return extractZipInternal(source, target, true);
|
||||
const { size: compressedSize } = await fs.promises.stat(source);
|
||||
const max = Math.max(
|
||||
compressedSize * COMPRESSION_HEADROOM,
|
||||
MIN_EXTRACTED_BYTES,
|
||||
);
|
||||
return extractZipInternal(source, target, true, { used: 0, max });
|
||||
}
|
||||
|
||||
/**
|
||||
* Internal helper to extract a ZIP, with optional single-nested-ZIP handling.
|
||||
* @param source Path to the ZIP file
|
||||
* @param target Directory to extract into
|
||||
* @param allowNested Whether to check and unwrap one level of nested ZIP
|
||||
*/
|
||||
function extractZipInternal(
|
||||
source: string,
|
||||
target: string,
|
||||
allowNested: boolean,
|
||||
budget: SizeBudget,
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
yauzl.open(
|
||||
source,
|
||||
{ lazyEntries: true, decodeStrings: false, autoClose: true },
|
||||
{
|
||||
lazyEntries: true,
|
||||
decodeStrings: false,
|
||||
autoClose: true,
|
||||
validateEntrySizes: true,
|
||||
},
|
||||
(err, zipfile) => {
|
||||
if (err) return reject(err);
|
||||
|
||||
@@ -73,6 +81,15 @@ function extractZipInternal(
|
||||
? source.slice(0, -4) + '.inner.zip'
|
||||
: source + '.inner.zip';
|
||||
|
||||
budget.used += entry.uncompressedSize;
|
||||
if (budget.used > budget.max) {
|
||||
return reject(
|
||||
new Error(
|
||||
'Import archive exceeds the allowed extracted size limit',
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
zipfile.openReadStream(entry, (openErr, rs) => {
|
||||
if (openErr) return reject(openErr);
|
||||
const ws = fs.createWriteStream(nestedPath);
|
||||
@@ -80,7 +97,7 @@ function extractZipInternal(
|
||||
ws.on('error', reject);
|
||||
ws.on('finish', () => {
|
||||
zipfile.close();
|
||||
extractZipInternal(nestedPath, target, false)
|
||||
extractZipInternal(nestedPath, target, false, budget)
|
||||
.then(() => {
|
||||
fs.unlinkSync(nestedPath);
|
||||
resolve();
|
||||
@@ -91,13 +108,21 @@ function extractZipInternal(
|
||||
});
|
||||
} else {
|
||||
zipfile.close();
|
||||
extractZipInternal(source, target, false).then(resolve, reject);
|
||||
extractZipInternal(source, target, false, budget).then(
|
||||
resolve,
|
||||
reject,
|
||||
);
|
||||
}
|
||||
});
|
||||
zipfile.once('error', reject);
|
||||
return;
|
||||
}
|
||||
|
||||
if (zipfile.entryCount > MAX_ENTRIES) {
|
||||
zipfile.close();
|
||||
return reject(new Error('Import archive has too many entries'));
|
||||
}
|
||||
|
||||
// Normal extraction
|
||||
zipfile.readEntry();
|
||||
zipfile.on('entry', (entry) => {
|
||||
@@ -143,6 +168,15 @@ function extractZipInternal(
|
||||
return;
|
||||
}
|
||||
|
||||
budget.used += entry.uncompressedSize;
|
||||
if (budget.used > budget.max) {
|
||||
return reject(
|
||||
new Error(
|
||||
'Import archive exceeds the allowed extracted size limit',
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
// Handle files
|
||||
try {
|
||||
fs.mkdirSync(path.dirname(fullPath), { recursive: true });
|
||||
|
||||
@@ -14,6 +14,7 @@ export enum QueueName {
|
||||
// Separate queue for /docmost ask: AI work takes seconds and would
|
||||
// otherwise starve fast inbound event dispatch.
|
||||
SLACK_ASK = '{slack-ask}',
|
||||
BASE_QUEUE = '{base-queue}',
|
||||
}
|
||||
|
||||
export enum QueueJob {
|
||||
@@ -93,4 +94,8 @@ export enum QueueJob {
|
||||
INTEGRATION_TOKEN_REFRESH = 'integration-token-refresh',
|
||||
SLACK_EVENT = 'slack-event',
|
||||
SLACK_ASK = 'slack-ask',
|
||||
|
||||
BASE_TYPE_CONVERSION = 'base-type-conversion',
|
||||
BASE_CELL_GC = 'base-cell-gc',
|
||||
BASE_FORMULA_RECOMPUTE = 'base-formula-recompute',
|
||||
}
|
||||
|
||||
@@ -113,3 +113,47 @@ export interface IApprovalRejectedNotificationJob {
|
||||
requestedById: string;
|
||||
comment?: string;
|
||||
}
|
||||
|
||||
export interface IBaseTypeConversionJob {
|
||||
pageId: string;
|
||||
propertyId: string;
|
||||
workspaceId: string;
|
||||
fromType: string;
|
||||
toType: string;
|
||||
// Snapshots taken at enqueue time so the job stays correct even if the
|
||||
// property's current typeOptions drift while the job waits in the queue.
|
||||
fromTypeOptions: unknown;
|
||||
toTypeOptions: unknown;
|
||||
// When true, the job nulls the cell values for that property instead of
|
||||
// attempting a value conversion. Used for any conversion where the new
|
||||
// type has no meaningful representation of the old value (e.g. involving
|
||||
// a system type).
|
||||
clearMode: boolean;
|
||||
// Staging identity: guards redelivery and failure cleanup against a
|
||||
// same-type re-stage made after this job was enqueued.
|
||||
pendingToken: string;
|
||||
actorId?: string;
|
||||
}
|
||||
|
||||
export interface IBaseCellGcJob {
|
||||
pageId: string;
|
||||
propertyId: string;
|
||||
workspaceId: string;
|
||||
}
|
||||
|
||||
export interface IBaseFormulaRecomputeJob {
|
||||
pageId: string;
|
||||
workspaceId: string;
|
||||
propertyIds: string[]; // formula properties to recompute
|
||||
reason:
|
||||
| 'formula_created'
|
||||
| 'formula_edited'
|
||||
| 'dep_type_changed'
|
||||
| 'dep_deleted'
|
||||
| 'bulk_import'
|
||||
| 'manual';
|
||||
actorId?: string | null;
|
||||
// When set, scope recompute to these row IDs instead of the whole base.
|
||||
// Used by the bulk-write path (> FORMULA_INLINE_ROW_THRESHOLD).
|
||||
rowIds?: string[];
|
||||
}
|
||||
|
||||
@@ -100,6 +100,14 @@ import { GeneralQueueProcessor } from './processors/general-queue.processor';
|
||||
attempts: 3,
|
||||
},
|
||||
}),
|
||||
BullModule.registerQueue({
|
||||
name: QueueName.BASE_QUEUE,
|
||||
defaultJobOptions: {
|
||||
attempts: 2,
|
||||
removeOnComplete: { count: 200 },
|
||||
removeOnFail: { count: 100 },
|
||||
},
|
||||
}),
|
||||
],
|
||||
exports: [BullModule],
|
||||
providers: [GeneralQueueProcessor],
|
||||
|
||||
@@ -15,13 +15,21 @@ import { getMimeType } from '../../../common/helpers';
|
||||
import { Upload } from '@aws-sdk/lib-storage';
|
||||
import { Logger } from '@nestjs/common';
|
||||
|
||||
const S3_MAX_SOCKETS = parseInt(process.env.AWS_S3_MAX_SOCKETS) || 200;
|
||||
|
||||
export class S3Driver implements StorageDriver {
|
||||
private readonly s3Client: S3Client;
|
||||
private readonly config: S3StorageConfig;
|
||||
|
||||
constructor(config: S3StorageConfig) {
|
||||
this.config = config;
|
||||
this.s3Client = new S3Client(config as any);
|
||||
this.config = {
|
||||
...config,
|
||||
requestHandler: {
|
||||
httpAgent: { maxSockets: S3_MAX_SOCKETS },
|
||||
httpsAgent: { maxSockets: S3_MAX_SOCKETS },
|
||||
},
|
||||
};
|
||||
this.s3Client = new S3Client(this.config as any);
|
||||
}
|
||||
|
||||
async upload(filePath: string, file: Buffer | Readable): Promise<void> {
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import { ModuleRef } from '@nestjs/core';
|
||||
import { Server, Socket } from 'socket.io';
|
||||
|
||||
@Injectable()
|
||||
export class BaseRealtimeBridge {
|
||||
private readonly logger = new Logger(BaseRealtimeBridge.name);
|
||||
private resolved = false;
|
||||
private svc: any = null;
|
||||
|
||||
constructor(private readonly moduleRef: ModuleRef) {}
|
||||
|
||||
protected loadServiceClass(): any {
|
||||
try {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
return require('../ee/base/realtime/base-ws.service').BaseWsService;
|
||||
} catch {
|
||||
this.logger.debug(
|
||||
'Base realtime requested but enterprise module not bundled in this build',
|
||||
);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private resolve(): any {
|
||||
if (this.resolved) return this.svc;
|
||||
this.resolved = true;
|
||||
const ServiceClass = this.loadServiceClass();
|
||||
if (!ServiceClass) return null;
|
||||
this.svc = this.moduleRef.get(ServiceClass, { strict: false });
|
||||
return this.svc;
|
||||
}
|
||||
|
||||
setServer(server: Server): void {
|
||||
this.resolve()?.setServer(server);
|
||||
}
|
||||
|
||||
isBaseEvent(data: any): boolean {
|
||||
return this.resolve()?.isBaseEvent(data) ?? false;
|
||||
}
|
||||
|
||||
async handleInbound(client: Socket, data: any): Promise<void> {
|
||||
await this.resolve()?.handleInbound(client, data);
|
||||
}
|
||||
|
||||
async handleDisconnect(client: Socket): Promise<void> {
|
||||
await this.resolve()?.handleDisconnect(client);
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import {
|
||||
MessageBody,
|
||||
OnGatewayConnection,
|
||||
OnGatewayDisconnect,
|
||||
OnGatewayInit,
|
||||
SubscribeMessage,
|
||||
WebSocketGateway,
|
||||
@@ -13,6 +14,7 @@ import { OnModuleDestroy } from '@nestjs/common';
|
||||
import { SpaceMemberRepo } from '@docmost/db/repos/space/space-member.repo';
|
||||
import { WsService } from './ws.service';
|
||||
import { getSpaceRoomName, getUserRoomName } from './ws.utils';
|
||||
import { BaseRealtimeBridge } from './base-realtime.bridge';
|
||||
import * as cookie from 'cookie';
|
||||
|
||||
@WebSocketGateway({
|
||||
@@ -20,7 +22,11 @@ import * as cookie from 'cookie';
|
||||
transports: ['websocket'],
|
||||
})
|
||||
export class WsGateway
|
||||
implements OnGatewayConnection, OnGatewayInit, OnModuleDestroy
|
||||
implements
|
||||
OnGatewayConnection,
|
||||
OnGatewayDisconnect,
|
||||
OnGatewayInit,
|
||||
OnModuleDestroy
|
||||
{
|
||||
@WebSocketServer()
|
||||
server: Server;
|
||||
@@ -29,10 +35,12 @@ export class WsGateway
|
||||
private tokenService: TokenService,
|
||||
private spaceMemberRepo: SpaceMemberRepo,
|
||||
private wsService: WsService,
|
||||
private baseRealtime: BaseRealtimeBridge,
|
||||
) {}
|
||||
|
||||
afterInit(server: Server): void {
|
||||
this.wsService.setServer(server);
|
||||
this.baseRealtime.setServer(server);
|
||||
}
|
||||
|
||||
async handleConnection(client: Socket, ...args: any[]): Promise<void> {
|
||||
@@ -47,6 +55,7 @@ export class WsGateway
|
||||
const workspaceId = token.workspaceId;
|
||||
|
||||
client.data.userId = userId;
|
||||
client.data.workspaceId = workspaceId;
|
||||
|
||||
const userSpaceIds = await this.spaceMemberRepo.getUserSpaceIds(userId);
|
||||
|
||||
@@ -61,10 +70,19 @@ export class WsGateway
|
||||
}
|
||||
}
|
||||
|
||||
async handleDisconnect(client: Socket): Promise<void> {
|
||||
await this.baseRealtime.handleDisconnect(client);
|
||||
}
|
||||
|
||||
@SubscribeMessage('message')
|
||||
async handleMessage(client: Socket, data: any): Promise<void> {
|
||||
if (this.wsService.isTreeEvent(data)) {
|
||||
await this.wsService.handleTreeEvent(client, data);
|
||||
return;
|
||||
}
|
||||
if (this.baseRealtime.isBaseEvent(data)) {
|
||||
await this.baseRealtime.handleInbound(client, data);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,11 +3,12 @@ import { WsGateway } from './ws.gateway';
|
||||
import { WsService } from './ws.service';
|
||||
import { WsTreeService } from './ws-tree.service';
|
||||
import { TokenModule } from '../core/auth/token.module';
|
||||
import { BaseRealtimeBridge } from './base-realtime.bridge';
|
||||
|
||||
@Global()
|
||||
@Module({
|
||||
imports: [TokenModule],
|
||||
providers: [WsGateway, WsService, WsTreeService],
|
||||
providers: [WsGateway, WsService, WsTreeService, BaseRealtimeBridge],
|
||||
exports: [WsGateway, WsService, WsTreeService],
|
||||
})
|
||||
export class WsModule {}
|
||||
|
||||
@@ -16,3 +16,14 @@ export const TREE_EVENTS = new Set([
|
||||
'deleteTreeNode',
|
||||
'refetchRootTreeNodeEvent',
|
||||
]);
|
||||
|
||||
export function getBaseRoomName(pageId: string): string {
|
||||
return `base-${pageId}`;
|
||||
}
|
||||
|
||||
export const BASE_INBOUND_EVENTS = new Set([
|
||||
'base:subscribe',
|
||||
'base:unsubscribe',
|
||||
'base:presence',
|
||||
'base:presence:leave',
|
||||
]);
|
||||
|
||||
Reference in New Issue
Block a user