feat: bases

Add the bases feature: formula engine package, grid/table UI, and the base-embed editor extension, with supporting client and server changes.
This commit is contained in:
Philipinho
2026-06-13 01:26:37 +01:00
parent d86d51c27e
commit 572452c80b
237 changed files with 28203 additions and 146 deletions
+4 -1
View File
@@ -75,6 +75,7 @@
"class-transformer": "^0.5.1",
"class-validator": "^0.15.1",
"cookie": "^1.1.1",
"csv-stringify": "^6",
"fast-bm25": "0.0.5",
"fastify-ip": "^2.0.0",
"fs-extra": "^11.3.4",
@@ -192,7 +193,9 @@
"moduleNameMapper": {
"^@docmost/db/(.*)$": "<rootDir>/database/$1",
"^@docmost/transactional/(.*)$": "<rootDir>/integrations/transactional/$1",
"^@docmost/ee/(.*)$": "<rootDir>/ee/$1"
"^@docmost/ee/(.*)$": "<rootDir>/ee/$1",
"^@docmost/base-formula/server$": "<rootDir>/../../../packages/base-formula/src/index.server.ts",
"^@docmost/base-formula/client$": "<rootDir>/../../../packages/base-formula/src/index.client.ts"
}
}
}
@@ -44,6 +44,7 @@ import {
htmlToMarkdown,
TransclusionSource,
TransclusionReference,
BaseEmbed,
} from '@docmost/editor-ext';
import { generateText, getSchema, JSONContent } from '@tiptap/core';
import { generateHTML, generateJSON } from '../common/helpers/prosemirror/html';
@@ -109,6 +110,7 @@ export const tiptapExtensions = [
Status,
TransclusionSource,
TransclusionReference,
BaseEmbed
] as any;
export function jsonToHtml(tiptapJson: any) {
@@ -15,4 +15,29 @@ export enum EventName {
WORKSPACE_CREATED = 'workspace.created',
WORKSPACE_UPDATED = 'workspace.updated',
WORKSPACE_DELETED = 'workspace.deleted',
BASE_CREATED = 'base.created',
BASE_UPDATED = 'base.updated',
BASE_DELETED = 'base.deleted',
BASE_ROW_CREATED = 'base.row.created',
BASE_ROW_UPDATED = 'base.row.updated',
BASE_ROW_DELETED = 'base.row.deleted',
BASE_ROWS_DELETED = 'base.rows.deleted',
BASE_ROW_RESTORED = 'base.row.restored',
BASE_ROW_REORDERED = 'base.row.reordered',
BASE_PROPERTY_CREATED = 'base.property.created',
BASE_PROPERTY_UPDATED = 'base.property.updated',
BASE_PROPERTY_DELETED = 'base.property.deleted',
BASE_PROPERTY_REORDERED = 'base.property.reordered',
BASE_VIEW_CREATED = 'base.view.created',
BASE_VIEW_UPDATED = 'base.view.updated',
BASE_VIEW_DELETED = 'base.view.deleted',
BASE_SCHEMA_BUMPED = 'base.schema.bumped',
BASE_ROWS_UPDATED = 'base.rows.updated',
BASE_FORMULA_RECOMPUTE_STARTED = 'base.formula.recompute.started',
BASE_FORMULA_RECOMPUTE_COMPLETED = 'base.formula.recompute.completed',
}
+1
View File
@@ -20,6 +20,7 @@ export const Feature = {
VIEWER_COMMENTS: 'comment:viewer',
TEMPLATES: 'templates',
PDF_EXPORT: 'export:pdf',
BASES: 'bases',
} as const;
export type FeatureKey = (typeof Feature)[keyof typeof Feature];
+5 -2
View File
@@ -10,6 +10,9 @@ export const LOCAL_STORAGE_PATH = path.resolve(
LOCAL_STORAGE_DIR,
);
export function getPageTitle(title: string | null | undefined): string {
return title || 'untitled';
export function getPageTitle(
title: string | null | undefined,
isBase?: boolean,
): string {
return title || (isBase ? 'Untitled base' : 'untitled');
}
@@ -43,7 +43,7 @@ export class AttachmentService {
async uploadFile(opts: {
filePromise: Promise<MultipartFile>;
pageId: string;
pageId?: string;
userId: string;
spaceId: string;
workspaceId: string;
@@ -1,4 +1,5 @@
import {
IsBoolean,
IsIn,
IsOptional,
IsString,
@@ -15,7 +15,7 @@ import {
executeWithCursorPagination,
} from '@docmost/db/pagination/cursor-pagination';
import { InjectKysely } from 'nestjs-kysely';
import { KyselyDB } from '@docmost/db/types/kysely.types';
import { KyselyDB, KyselyTransaction } from '@docmost/db/types/kysely.types';
import { generateJitteredKeyBetween } from 'fractional-indexing-jittered';
import { MovePageDto } from '../dto/move-page.dto';
import { generateSlugId } from '../../../common/helpers';
@@ -92,6 +92,8 @@ export class PageService {
userId: string,
workspaceId: string,
createPageDto: CreatePageDto,
trx?: KyselyTransaction,
isBase: boolean = false,
): Promise<Page> {
let parentPageId = undefined;
@@ -140,21 +142,34 @@ export class PageService {
creatorId: userId,
workspaceId: workspaceId,
lastUpdatedById: userId,
isBase,
content,
textContent,
ydoc,
});
}, trx);
this.generalQueue
.add(QueueJob.ADD_PAGE_WATCHERS, {
userIds: [userId],
pageId: page.id,
spaceId: createPageDto.spaceId,
if (trx) {
// Add the watcher inside the caller's transaction so the async worker
// never inserts against an uncommitted page (FK violation on bases).
await this.watcherService.addPageWatchers(
[userId],
page.id,
createPageDto.spaceId,
workspaceId,
})
.catch((err) =>
this.logger.warn(`Failed to queue add-page-watchers: ${err.message}`),
trx,
);
} else {
this.generalQueue
.add(QueueJob.ADD_PAGE_WATCHERS, {
userIds: [userId],
pageId: page.id,
spaceId: createPageDto.spaceId,
workspaceId,
})
.catch((err) =>
this.logger.warn(`Failed to queue add-page-watchers: ${err.message}`),
);
}
return page;
}
@@ -289,6 +304,7 @@ export class PageService {
'parentPageId',
'spaceId',
'creatorId',
'isBase',
'deletedAt',
])
.select((eb) => this.pageRepo.withHasChildren(eb))
@@ -832,6 +848,7 @@ export class PageService {
'slugId',
'title',
'icon',
'isBase',
'position',
'parentPageId',
'spaceId',
@@ -847,6 +864,7 @@ export class PageService {
'p.slugId',
'p.title',
'p.icon',
'p.isBase',
'p.position',
'p.parentPageId',
'p.spaceId',
@@ -0,0 +1,384 @@
import { type Kysely, sql } from 'kysely';
export async function up(db: Kysely<any>): Promise<void> {
await db.schema
.alterTable('pages')
.addColumn('is_base', 'boolean', (col) =>
col.ifNotExists().notNull().defaultTo(false),
)
.addColumn('base_schema_version', 'integer', (col) =>
col.ifNotExists().notNull().defaultTo(0),
)
.execute();
await sql`
CREATE INDEX IF NOT EXISTS idx_pages_is_base
ON pages (space_id, position COLLATE "C")
WHERE is_base = true AND deleted_at IS NULL
`.execute(db);
await db.schema
.createTable('base_properties')
.ifNotExists()
.addColumn('id', 'uuid', (col) =>
col.primaryKey().defaultTo(sql`gen_uuid_v7()`),
)
.addColumn('page_id', 'uuid', (col) =>
col.references('pages.id').onDelete('cascade').notNull(),
)
.addColumn('name', 'varchar', (col) => col.notNull())
.addColumn('type', 'varchar', (col) => col.notNull())
.addColumn('position', 'varchar', (col) => col.notNull())
.addColumn('type_options', 'jsonb')
.addColumn('pending_type', 'varchar')
.addColumn('pending_type_options', 'jsonb')
.addColumn('pending_token', 'uuid')
.addColumn('is_primary', 'boolean', (col) => col.notNull().defaultTo(false))
.addColumn('schema_version', 'integer', (col) => col.notNull().defaultTo(1))
.addColumn('workspace_id', 'uuid', (col) =>
col.references('workspaces.id').onDelete('cascade').notNull(),
)
.addColumn('created_at', 'timestamptz', (col) =>
col.notNull().defaultTo(sql`now()`),
)
.addColumn('updated_at', 'timestamptz', (col) =>
col.notNull().defaultTo(sql`now()`),
)
.addColumn('deleted_at', 'timestamptz')
.execute();
await sql`CREATE INDEX IF NOT EXISTS idx_base_properties_page_id ON base_properties (page_id)`.execute(
db,
);
await sql`
CREATE INDEX IF NOT EXISTS idx_base_properties_page_alive
ON base_properties (page_id, position COLLATE "C", id)
WHERE deleted_at IS NULL
`.execute(db);
// Match the service-layer name check (name.trim().toLowerCase()) so
// whitespace-padded duplicates also collide. Formulas look properties up by
// name, so the names have to stay unique.
await sql`
CREATE UNIQUE INDEX IF NOT EXISTS uq_base_properties_page_name_alive
ON base_properties (page_id, lower(trim(name)))
WHERE deleted_at IS NULL
`.execute(db);
await db.schema
.createTable('base_rows')
.ifNotExists()
.addColumn('id', 'uuid', (col) =>
col.primaryKey().defaultTo(sql`gen_uuid_v7()`),
)
.addColumn('page_id', 'uuid', (col) =>
col.references('pages.id').onDelete('cascade').notNull(),
)
.addColumn('cells', 'jsonb', (col) =>
col.notNull().defaultTo(sql`'{}'::jsonb`),
)
.addColumn('position', 'varchar', (col) => col.notNull())
.addColumn('search_text', 'text')
.addColumn('search_tsv', sql`tsvector`)
.addColumn('creator_id', 'uuid', (col) =>
col.references('users.id').onDelete('set null'),
)
.addColumn('last_updated_by_id', 'uuid', (col) =>
col.references('users.id').onDelete('set null'),
)
.addColumn('workspace_id', 'uuid', (col) =>
col.references('workspaces.id').onDelete('cascade').notNull(),
)
.addColumn('created_at', 'timestamptz', (col) =>
col.notNull().defaultTo(sql`now()`),
)
.addColumn('updated_at', 'timestamptz', (col) =>
col.notNull().defaultTo(sql`now()`),
)
.addColumn('deleted_at', 'timestamptz')
.execute();
// Intentionally no GIN on cells: `cells ? key` lookups are rare and
// page-scoped, while the index doubled the write cost of every cell edit.
await sql`
CREATE INDEX IF NOT EXISTS idx_base_rows_page_alive
ON base_rows (page_id, position COLLATE "C", id)
WHERE deleted_at IS NULL
`.execute(db);
await sql`
CREATE INDEX IF NOT EXISTS idx_base_rows_page_updated
ON base_rows (page_id, updated_at DESC)
WHERE deleted_at IS NULL
`.execute(db);
await sql`
CREATE INDEX IF NOT EXISTS idx_base_rows_page_created
ON base_rows (page_id, created_at DESC)
WHERE deleted_at IS NULL
`.execute(db);
await sql`
CREATE INDEX IF NOT EXISTS idx_base_rows_search_tsv
ON base_rows USING gin (search_tsv)
WHERE deleted_at IS NULL
`.execute(db);
await sql`
CREATE INDEX IF NOT EXISTS idx_base_rows_search_trgm
ON base_rows USING gin (search_text gin_trgm_ops)
WHERE deleted_at IS NULL
`.execute(db);
await db.schema
.createTable('base_views')
.ifNotExists()
.addColumn('id', 'uuid', (col) =>
col.primaryKey().defaultTo(sql`gen_uuid_v7()`),
)
.addColumn('page_id', 'uuid', (col) =>
col.references('pages.id').onDelete('cascade').notNull(),
)
.addColumn('name', 'varchar', (col) => col.notNull())
.addColumn('type', 'varchar', (col) => col.notNull().defaultTo('table'))
.addColumn('position', 'varchar', (col) => col.notNull())
.addColumn('config', 'jsonb', (col) =>
col.notNull().defaultTo(sql`'{}'::jsonb`),
)
.addColumn('workspace_id', 'uuid', (col) =>
col.references('workspaces.id').onDelete('cascade').notNull(),
)
.addColumn('creator_id', 'uuid', (col) =>
col.references('users.id').onDelete('set null'),
)
.addColumn('created_at', 'timestamptz', (col) =>
col.notNull().defaultTo(sql`now()`),
)
.addColumn('updated_at', 'timestamptz', (col) =>
col.notNull().defaultTo(sql`now()`),
)
.execute();
await sql`CREATE INDEX IF NOT EXISTS idx_base_views_page_id ON base_views (page_id)`.execute(
db,
);
// Cell extraction helpers for filters and sorts. Return NULL for absent or
// non-castable values.
await sql`
CREATE OR REPLACE FUNCTION base_cell_text(cells jsonb, prop uuid)
RETURNS text LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE
AS $$ SELECT cells->>prop::text $$
`.execute(db);
await sql`
CREATE OR REPLACE FUNCTION base_cell_numeric(cells jsonb, prop uuid)
RETURNS numeric LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE
AS $$
SELECT CASE jsonb_typeof(cells->prop::text)
WHEN 'number' THEN (cells->>prop::text)::numeric
WHEN 'string' THEN
CASE
WHEN (cells->>prop::text) ~
'^[[:space:]]*[+-]?([0-9]+([.][0-9]*)?|[.][0-9]+)([eE][+-]?[0-9]+)?[[:space:]]*$'
THEN (cells->>prop::text)::numeric
END
END
$$
`.execute(db);
// A DATE cell stores an arbitrary string (cell schema is z.string()), so the
// cast can fail on values no regex can pre-validate (e.g. '2024-13-45'). This
// helper uses plpgsql with an EXCEPTION handler to return NULL on failure.
await sql`
CREATE OR REPLACE FUNCTION base_cell_timestamptz(cells jsonb, prop uuid)
RETURNS timestamptz LANGUAGE plpgsql IMMUTABLE STRICT PARALLEL SAFE
AS $$
BEGIN RETURN (cells->>prop::text)::timestamptz;
EXCEPTION WHEN others THEN RETURN NULL; END;
$$
`.execute(db);
await sql`
CREATE OR REPLACE FUNCTION base_cell_bool(cells jsonb, prop uuid)
RETURNS boolean LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE
AS $$
SELECT CASE jsonb_typeof(cells->prop::text)
WHEN 'boolean' THEN (cells->>prop::text)::boolean
WHEN 'string' THEN
CASE
WHEN lower(btrim(cells->>prop::text)) IN
('true','t','yes','y','on','1','false','f','no','n','off','0')
THEN (cells->>prop::text)::boolean
END
END
$$
`.execute(db);
await sql`
CREATE OR REPLACE FUNCTION base_cell_array(cells jsonb, prop uuid)
RETURNS jsonb LANGUAGE sql IMMUTABLE STRICT PARALLEL SAFE
AS $$ SELECT cells->prop::text $$
`.execute(db);
// A null patch value deletes the key rather than storing a JSON null.
await sql`
CREATE OR REPLACE FUNCTION jsonb_set_many(target jsonb, patches jsonb)
RETURNS jsonb LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE
AS $$
DECLARE k text; v jsonb; result jsonb := coalesce(target, '{}'::jsonb);
BEGIN
IF patches IS NULL OR jsonb_typeof(patches) <> 'object' THEN
RETURN result;
END IF;
FOR k, v IN SELECT * FROM jsonb_each(patches) LOOP
IF v = 'null'::jsonb THEN
result := result - k;
ELSE
result := jsonb_set(result, ARRAY[k], v, true);
END IF;
END LOOP;
RETURN result;
END;
$$
`.execute(db);
// Resolves select/multiSelect cells to their choice names. The property list
// per page is cached in a transaction-local setting so bulk writes look it up
// once instead of per row.
await sql`
CREATE OR REPLACE FUNCTION build_base_row_search_text(
_cells jsonb,
_page_id uuid
) RETURNS text
LANGUAGE plpgsql STABLE PARALLEL SAFE
AS $$
DECLARE
_parts text[] := ARRAY[]::text[];
_prop jsonb;
_value text;
_arr jsonb;
_elem jsonb;
_resolved text;
_cache_key text;
_cached text;
_props jsonb;
BEGIN
IF _cells IS NULL OR _cells = '{}'::jsonb OR _page_id IS NULL THEN
RETURN NULL;
END IF;
_cache_key := 'bases.prop_cache_' || replace(_page_id::text, '-', '_');
_cached := current_setting(_cache_key, true);
IF _cached IS NULL OR _cached = '' THEN
SELECT coalesce(
jsonb_agg(jsonb_build_object(
'id', id,
'type', type,
'type_options', type_options
)),
'[]'::jsonb
)
INTO _props
FROM base_properties
WHERE page_id = _page_id AND deleted_at IS NULL;
PERFORM set_config(_cache_key, _props::text, true);
ELSE
_props := _cached::jsonb;
END IF;
FOR _prop IN SELECT * FROM jsonb_array_elements(_props)
LOOP
IF (_prop->>'type') IN ('text', 'url', 'email') THEN
_value := _cells->>(_prop->>'id');
IF _value IS NOT NULL AND _value <> '' THEN
_parts := array_append(_parts, _value);
END IF;
ELSIF (_prop->>'type') IN ('select', 'status') THEN
_value := _cells->>(_prop->>'id');
IF _value IS NOT NULL AND _value <> '' THEN
SELECT c->>'name' INTO _resolved
FROM jsonb_array_elements(coalesce(_prop->'type_options'->'choices', '[]'::jsonb)) AS c
WHERE c->>'id' = _value
LIMIT 1;
IF _resolved IS NOT NULL AND _resolved <> '' THEN
_parts := array_append(_parts, _resolved);
END IF;
END IF;
ELSIF (_prop->>'type') = 'multiSelect' THEN
_arr := _cells->(_prop->>'id');
IF jsonb_typeof(_arr) = 'array' THEN
FOR _elem IN SELECT * FROM jsonb_array_elements(_arr)
LOOP
SELECT c->>'name' INTO _resolved
FROM jsonb_array_elements(coalesce(_prop->'type_options'->'choices', '[]'::jsonb)) AS c
WHERE c->>'id' = _elem#>>'{}'
LIMIT 1;
IF _resolved IS NOT NULL AND _resolved <> '' THEN
_parts := array_append(_parts, _resolved);
END IF;
END LOOP;
END IF;
END IF;
END LOOP;
IF array_length(_parts, 1) IS NULL THEN
RETURN NULL;
END IF;
RETURN f_unaccent(array_to_string(_parts, ' '));
END;
$$
`.execute(db);
await sql`
CREATE OR REPLACE FUNCTION base_rows_search_trigger() RETURNS trigger
LANGUAGE plpgsql AS $$
BEGIN
NEW.search_text := build_base_row_search_text(NEW.cells, NEW.page_id);
NEW.search_tsv := to_tsvector('english', coalesce(NEW.search_text, ''));
RETURN NEW;
END;
$$
`.execute(db);
await sql`
CREATE OR REPLACE TRIGGER base_rows_search_insert
BEFORE INSERT ON base_rows
FOR EACH ROW EXECUTE FUNCTION base_rows_search_trigger()
`.execute(db);
// Position-only reorders and metadata touches must not pay the
// search_text recompute; OLD is not referenceable on INSERT, hence
// the split triggers.
await sql`
CREATE OR REPLACE TRIGGER base_rows_search_update
BEFORE UPDATE ON base_rows
FOR EACH ROW
WHEN (OLD.cells IS DISTINCT FROM NEW.cells)
EXECUTE FUNCTION base_rows_search_trigger()
`.execute(db);
}
export async function down(db: Kysely<any>): Promise<void> {
await db.schema.dropTable('base_views').execute();
await db.schema.dropTable('base_rows').execute();
await db.schema.dropTable('base_properties').execute();
await sql`DROP FUNCTION base_rows_search_trigger()`.execute(db);
await sql`DROP FUNCTION build_base_row_search_text(jsonb, uuid)`.execute(db);
await sql`DROP FUNCTION jsonb_set_many(jsonb, jsonb)`.execute(db);
await sql`DROP FUNCTION base_cell_array(jsonb, uuid)`.execute(db);
await sql`DROP FUNCTION base_cell_bool(jsonb, uuid)`.execute(db);
await sql`DROP FUNCTION base_cell_timestamptz(jsonb, uuid)`.execute(db);
await sql`DROP FUNCTION base_cell_numeric(jsonb, uuid)`.execute(db);
await sql`DROP FUNCTION base_cell_text(jsonb, uuid)`.execute(db);
await sql`DROP INDEX idx_pages_is_base`.execute(db);
await db.schema
.alterTable('pages')
.dropColumn('base_schema_version')
.dropColumn('is_base')
.execute();
}
@@ -236,6 +236,7 @@ export class FavoriteRepo {
'pages.slugId',
'pages.title',
'pages.icon',
'pages.isBase',
'pages.spaceId',
])
.whereRef('pages.id', '=', 'favorites.pageId'),
@@ -38,6 +38,7 @@ export class PageRepo {
'spaceId',
'workspaceId',
'isLocked',
'isBase',
'createdAt',
'updatedAt',
'deletedAt',
+51
View File
@@ -126,6 +126,52 @@ export interface Backlinks {
workspaceId: string;
}
export interface BaseProperties {
createdAt: Generated<Timestamp>;
deletedAt: Timestamp | null;
id: Generated<string>;
isPrimary: Generated<boolean>;
name: string;
pageId: string;
pendingType: string | null;
pendingTypeOptions: Json | null;
pendingToken: string | null;
position: string;
schemaVersion: Generated<number>;
type: string;
typeOptions: Json | null;
updatedAt: Generated<Timestamp>;
workspaceId: string;
}
export interface BaseRows {
cells: Generated<Json>;
createdAt: Generated<Timestamp>;
creatorId: string | null;
deletedAt: Timestamp | null;
id: Generated<string>;
lastUpdatedById: string | null;
pageId: string;
position: string;
searchText: string | null;
searchTsv: string | null;
updatedAt: Generated<Timestamp>;
workspaceId: string;
}
export interface BaseViews {
config: Generated<Json>;
createdAt: Generated<Timestamp>;
creatorId: string | null;
id: Generated<string>;
name: string;
pageId: string;
position: string;
type: Generated<string>;
updatedAt: Generated<Timestamp>;
workspaceId: string;
}
export interface Billing {
amount: Int8 | null;
billingScheme: string | null;
@@ -275,6 +321,8 @@ export interface Pages {
deletedById: string | null;
icon: string | null;
id: Generated<string>;
isBase: Generated<boolean>;
baseSchemaVersion: Generated<number>;
isLocked: Generated<boolean>;
lastUpdatedById: string | null;
parentPageId: string | null;
@@ -598,6 +646,9 @@ export interface DB {
authAccounts: AuthAccounts;
authProviders: AuthProviders;
backlinks: Backlinks;
baseProperties: BaseProperties;
baseRows: BaseRows;
baseViews: BaseViews;
billing: Billing;
comments: Comments;
favorites: Favorites;
@@ -3,6 +3,9 @@ import {
AiChats,
AiChatMessages,
Attachments,
BaseProperties,
BaseRows,
BaseViews,
Comments,
Groups,
Labels,
@@ -238,3 +241,27 @@ export type UpdatableAudit = Updateable<Omit<_Audit, 'id'>>;
export type Template = Selectable<Templates>;
export type InsertableTemplate = Insertable<Templates>;
export type UpdatableTemplate = Updateable<Omit<Templates, 'id'>>;
// Base Property
export type BaseProperty = Selectable<BaseProperties>;
export type InsertableBaseProperty = Insertable<BaseProperties>;
export type UpdatableBaseProperty = Updateable<Omit<BaseProperties, 'id'>>;
// Base Row
// `searchText` and `searchTsv` are internal fulltext-index columns maintained
// by a trigger. They are omitted from the public types so they never leak into
// HTTP responses or write payloads.
export type BaseRow = Omit<Selectable<BaseRows>, 'searchText' | 'searchTsv'>;
export type InsertableBaseRow = Omit<
Insertable<BaseRows>,
'searchText' | 'searchTsv'
>;
export type UpdatableBaseRow = Omit<
Updateable<Omit<BaseRows, 'id'>>,
'searchText' | 'searchTsv'
>;
// Base View
export type BaseView = Selectable<BaseViews>;
export type InsertableBaseView = Insertable<BaseViews>;
export type UpdatableBaseView = Updateable<Omit<BaseViews, 'id'>>;
@@ -9,6 +9,7 @@ export enum QueueName {
HISTORY_QUEUE = '{history-queue}',
NOTIFICATION_QUEUE = '{notification-queue}',
AUDIT_QUEUE = '{audit-queue}',
BASE_QUEUE = '{base-queue}',
}
export enum QueueJob {
@@ -83,4 +84,8 @@ export enum QueueJob {
PDF_EXPORT_TASK = 'pdf-export-task',
PDF_EXPORT_CLEANUP = 'pdf-export-cleanup',
BASE_TYPE_CONVERSION = 'base-type-conversion',
BASE_CELL_GC = 'base-cell-gc',
BASE_FORMULA_RECOMPUTE = 'base-formula-recompute',
}
@@ -113,3 +113,47 @@ export interface IApprovalRejectedNotificationJob {
requestedById: string;
comment?: string;
}
export interface IBaseTypeConversionJob {
pageId: string;
propertyId: string;
workspaceId: string;
fromType: string;
toType: string;
// Snapshots taken at enqueue time so the job stays correct even if the
// property's current typeOptions drift while the job waits in the queue.
fromTypeOptions: unknown;
toTypeOptions: unknown;
// When true, the job nulls the cell values for that property instead of
// attempting a value conversion. Used for any conversion where the new
// type has no meaningful representation of the old value (e.g. involving
// a system type).
clearMode: boolean;
// Staging identity: guards redelivery and failure cleanup against a
// same-type re-stage made after this job was enqueued.
pendingToken: string;
actorId?: string;
}
export interface IBaseCellGcJob {
pageId: string;
propertyId: string;
workspaceId: string;
}
export interface IBaseFormulaRecomputeJob {
pageId: string;
workspaceId: string;
propertyIds: string[]; // formula properties to recompute
reason:
| 'formula_created'
| 'formula_edited'
| 'dep_type_changed'
| 'dep_deleted'
| 'bulk_import'
| 'manual';
actorId?: string | null;
// When set, scope recompute to these row IDs instead of the whole base.
// Used by the bulk-write path (> FORMULA_INLINE_ROW_THRESHOLD).
rowIds?: string[];
}
@@ -92,6 +92,14 @@ import { GeneralQueueProcessor } from './processors/general-queue.processor';
attempts: 3,
},
}),
BullModule.registerQueue({
name: QueueName.BASE_QUEUE,
defaultJobOptions: {
attempts: 2,
removeOnComplete: { count: 200 },
removeOnFail: { count: 100 },
},
}),
],
exports: [BullModule],
providers: [GeneralQueueProcessor],
@@ -0,0 +1,29 @@
import { BaseRealtimeBridge } from '../base-realtime.bridge';
function makeBridge(svc: any) {
const moduleRef = { get: jest.fn().mockReturnValue(svc) } as any;
const bridge = new BaseRealtimeBridge(moduleRef);
// bypass the require() of the EE module for unit testing by forcing a resolved class token
(bridge as any).loadServiceClass = () => (svc ? class {} : null);
return { bridge, moduleRef };
}
describe('BaseRealtimeBridge', () => {
it('delegates isBaseEvent / handleInbound to the resolved service', async () => {
const svc = {
isBaseEvent: jest.fn().mockReturnValue(true),
handleInbound: jest.fn().mockResolvedValue(undefined),
};
const { bridge } = makeBridge(svc);
expect(bridge.isBaseEvent({ t: 'base' })).toBe(true);
await bridge.handleInbound({} as any, { t: 'base' });
expect(svc.handleInbound).toHaveBeenCalled();
});
it('no-ops safely when the EE base module is absent', async () => {
const { bridge } = makeBridge(null);
expect(bridge.isBaseEvent({ t: 'base' })).toBe(false);
await expect(bridge.handleDisconnect({} as any)).resolves.toBeUndefined();
expect(() => bridge.setServer({} as any)).not.toThrow();
});
});
@@ -0,0 +1,49 @@
import { Injectable, Logger } from '@nestjs/common';
import { ModuleRef } from '@nestjs/core';
import { Server, Socket } from 'socket.io';
@Injectable()
export class BaseRealtimeBridge {
private readonly logger = new Logger(BaseRealtimeBridge.name);
private resolved = false;
private svc: any = null;
constructor(private readonly moduleRef: ModuleRef) {}
protected loadServiceClass(): any {
try {
// eslint-disable-next-line @typescript-eslint/no-require-imports
return require('../ee/base/realtime/base-ws.service').BaseWsService;
} catch {
this.logger.debug(
'Base realtime requested but enterprise module not bundled in this build',
);
return null;
}
}
private resolve(): any {
if (this.resolved) return this.svc;
this.resolved = true;
const ServiceClass = this.loadServiceClass();
if (!ServiceClass) return null;
this.svc = this.moduleRef.get(ServiceClass, { strict: false });
return this.svc;
}
setServer(server: Server): void {
this.resolve()?.setServer(server);
}
isBaseEvent(data: any): boolean {
return this.resolve()?.isBaseEvent(data) ?? false;
}
async handleInbound(client: Socket, data: any): Promise<void> {
await this.resolve()?.handleInbound(client, data);
}
async handleDisconnect(client: Socket): Promise<void> {
await this.resolve()?.handleDisconnect(client);
}
}
+19 -1
View File
@@ -1,6 +1,7 @@
import {
MessageBody,
OnGatewayConnection,
OnGatewayDisconnect,
OnGatewayInit,
SubscribeMessage,
WebSocketGateway,
@@ -13,6 +14,7 @@ import { OnModuleDestroy } from '@nestjs/common';
import { SpaceMemberRepo } from '@docmost/db/repos/space/space-member.repo';
import { WsService } from './ws.service';
import { getSpaceRoomName, getUserRoomName } from './ws.utils';
import { BaseRealtimeBridge } from './base-realtime.bridge';
import * as cookie from 'cookie';
@WebSocketGateway({
@@ -20,7 +22,11 @@ import * as cookie from 'cookie';
transports: ['websocket'],
})
export class WsGateway
implements OnGatewayConnection, OnGatewayInit, OnModuleDestroy
implements
OnGatewayConnection,
OnGatewayDisconnect,
OnGatewayInit,
OnModuleDestroy
{
@WebSocketServer()
server: Server;
@@ -29,10 +35,12 @@ export class WsGateway
private tokenService: TokenService,
private spaceMemberRepo: SpaceMemberRepo,
private wsService: WsService,
private baseRealtime: BaseRealtimeBridge,
) {}
afterInit(server: Server): void {
this.wsService.setServer(server);
this.baseRealtime.setServer(server);
}
async handleConnection(client: Socket, ...args: any[]): Promise<void> {
@@ -47,6 +55,7 @@ export class WsGateway
const workspaceId = token.workspaceId;
client.data.userId = userId;
client.data.workspaceId = workspaceId;
const userSpaceIds = await this.spaceMemberRepo.getUserSpaceIds(userId);
@@ -61,10 +70,19 @@ export class WsGateway
}
}
async handleDisconnect(client: Socket): Promise<void> {
await this.baseRealtime.handleDisconnect(client);
}
@SubscribeMessage('message')
async handleMessage(client: Socket, data: any): Promise<void> {
if (this.wsService.isTreeEvent(data)) {
await this.wsService.handleTreeEvent(client, data);
return;
}
if (this.baseRealtime.isBaseEvent(data)) {
await this.baseRealtime.handleInbound(client, data);
return;
}
}
+2 -1
View File
@@ -3,11 +3,12 @@ import { WsGateway } from './ws.gateway';
import { WsService } from './ws.service';
import { WsTreeService } from './ws-tree.service';
import { TokenModule } from '../core/auth/token.module';
import { BaseRealtimeBridge } from './base-realtime.bridge';
@Global()
@Module({
imports: [TokenModule],
providers: [WsGateway, WsService, WsTreeService],
providers: [WsGateway, WsService, WsTreeService, BaseRealtimeBridge],
exports: [WsGateway, WsService, WsTreeService],
})
export class WsModule {}
+11
View File
@@ -16,3 +16,14 @@ export const TREE_EVENTS = new Set([
'deleteTreeNode',
'refetchRootTreeNodeEvent',
]);
export function getBaseRoomName(pageId: string): string {
return `base-${pageId}`;
}
export const BASE_INBOUND_EVENTS = new Set([
'base:subscribe',
'base:unsubscribe',
'base:presence',
'base:presence:leave',
]);
+7 -1
View File
@@ -22,7 +22,13 @@
"paths": {
"@docmost/db/*": ["./src/database/*"],
"@docmost/transactional/*": ["./src/integrations/transactional/*"],
"@docmost/ee/*": ["./src/ee/*"]
"@docmost/ee/*": ["./src/ee/*"],
"@docmost/base-formula/server": [
"../../packages/base-formula/dist/index.server.d.ts"
],
"@docmost/base-formula/client": [
"../../packages/base-formula/dist/index.client.d.ts"
]
}
}
}