From 89ee3714acc49b251f3db90ff570be8f150235ae Mon Sep 17 00:00:00 2001 From: Philipinho <16838612+Philipinho@users.noreply.github.com> Date: Fri, 24 Apr 2026 12:11:29 +0100 Subject: [PATCH] feat(base): add /bases/rows/count with estimate and capped-exact modes Row-count display on a filtered view shouldn't force a full COUNT(*) on every list fetch. New endpoint returns either an EXPLAIN-plan estimate (default, ~1ms, no execution) or a LIMIT-capped exact count that short- circuits to `{ capped: true }` once the match set passes EXACT_COUNT_CAP. Clients call it in parallel with the rows query so the grid still paints at its own pace. - DTO + repo.countEstimate/countExact reusing the list predicate shape - service picks the mode; controller mirrors the list Read ability check - client hook keyed by filter/search/exact so a "show exact" toggle doesn't clobber the estimate cache --- .../features/base/queries/base-row-query.ts | 32 ++++++ .../features/base/services/base-service.ts | 9 ++ .../src/features/base/types/base.types.ts | 15 +++ .../base/controllers/base-row.controller.ts | 21 ++++ .../src/core/base/dto/update-row.dto.ts | 20 ++++ .../core/base/services/base-row.service.ts | 42 ++++++- .../src/database/repos/base/base-row.repo.ts | 107 +++++++++++++++++- 7 files changed, 244 insertions(+), 2 deletions(-) diff --git a/apps/client/src/features/base/queries/base-row-query.ts b/apps/client/src/features/base/queries/base-row-query.ts index 5350fe7df..55d50550c 100644 --- a/apps/client/src/features/base/queries/base-row-query.ts +++ b/apps/client/src/features/base/queries/base-row-query.ts @@ -1,6 +1,7 @@ import { useInfiniteQuery, useMutation, + useQuery, InfiniteData, } from "@tanstack/react-query"; import { @@ -10,6 +11,7 @@ import { deleteRows, listRows, reorderRow, + countRows, } from "@/features/base/services/base-service"; import { IBaseRow, @@ -21,6 +23,7 @@ import { FilterNode, SearchSpec, ViewSortConfig, + CountRowsResult, } from "@/features/base/types/base.types"; import { notifications } from "@mantine/notifications"; import { queryClient } from "@/main"; @@ -277,6 +280,35 @@ export function useDeleteRowsMutation() { }); } +/* + * Row count for the current view. Fires in parallel with `useBaseRowsQuery` + * — doesn't block first paint. Keyed by filter + search so an independent + * view with a different filter gets its own cached count; `exact` is part + * of the key so a "show exact" toggle doesn't clobber the estimate cache. + */ +export function useBaseRowsCountQuery( + baseId: string | undefined, + filter?: FilterNode, + search?: SearchSpec, + exact = false, +) { + const activeFilter = normalizeFilter(filter); + const activeSearch = search?.query ? search : undefined; + + return useQuery({ + queryKey: ["base-rows-count", baseId, activeFilter, activeSearch, exact], + queryFn: () => + countRows({ + baseId: baseId!, + filter: activeFilter, + search: activeSearch, + exact, + }), + enabled: !!baseId, + staleTime: 30 * 1000, + }); +} + export function useReorderRowMutation() { const { t } = useTranslation(); return useMutation({ diff --git a/apps/client/src/features/base/services/base-service.ts b/apps/client/src/features/base/services/base-service.ts index 8a4adb2f0..1fb78ea2f 100644 --- a/apps/client/src/features/base/services/base-service.ts +++ b/apps/client/src/features/base/services/base-service.ts @@ -23,6 +23,8 @@ import { FilterNode, SearchSpec, ViewSortConfig, + CountRowsInput, + CountRowsResult, } from "@/features/base/types/base.types"; import { IPagination } from "@/lib/types"; @@ -151,6 +153,13 @@ export async function reorderRow(data: ReorderRowInput): Promise { await api.post("/bases/rows/reorder", data); } +export async function countRows( + data: CountRowsInput, +): Promise { + const req = await api.post("/bases/rows/count", data); + return req.data; +} + // --- Views --- export async function createView(data: CreateViewInput): Promise { diff --git a/apps/client/src/features/base/types/base.types.ts b/apps/client/src/features/base/types/base.types.ts index 115a19028..f8fafd6d0 100644 --- a/apps/client/src/features/base/types/base.types.ts +++ b/apps/client/src/features/base/types/base.types.ts @@ -239,6 +239,21 @@ export type ReorderPropertyInput = { requestId?: string; }; +export type CountRowsInput = { + baseId: string; + filter?: FilterNode; + search?: SearchSpec; + // When true the server returns an exact (but capped) count. Otherwise + // a cheap EXPLAIN-plan estimate. Paired with `capped` on the result. + exact?: boolean; +}; + +export type CountRowsResult = { + value: number; + exact: boolean; + capped: boolean; +}; + export type CreateRowInput = { baseId: string; cells?: Record; diff --git a/apps/server/src/core/base/controllers/base-row.controller.ts b/apps/server/src/core/base/controllers/base-row.controller.ts index 42b59d2dc..d5dd21af2 100644 --- a/apps/server/src/core/base/controllers/base-row.controller.ts +++ b/apps/server/src/core/base/controllers/base-row.controller.ts @@ -18,6 +18,7 @@ import { RowIdDto, ListRowsDto, ReorderRowDto, + CountRowsDto, } from '../dto/update-row.dto'; import { AuthUser } from '../../../common/decorators/auth-user.decorator'; import { AuthWorkspace } from '../../../common/decorators/auth-workspace.decorator'; @@ -160,6 +161,26 @@ export class BaseRowController { return this.baseRowService.list(dto, pagination, workspace.id); } + @HttpCode(HttpStatus.OK) + @Post('count') + async count( + @Body() dto: CountRowsDto, + @AuthUser() user: User, + @AuthWorkspace() workspace: Workspace, + ) { + const base = await this.baseRepo.findById(dto.baseId); + if (!base) { + throw new NotFoundException('Base not found'); + } + + const ability = await this.spaceAbility.createForUser(user, base.spaceId); + if (ability.cannot(SpaceCaslAction.Read, SpaceCaslSubject.Base)) { + throw new ForbiddenException(); + } + + return this.baseRowService.count(dto, workspace.id); + } + @HttpCode(HttpStatus.OK) @Post('reorder') async reorder( diff --git a/apps/server/src/core/base/dto/update-row.dto.ts b/apps/server/src/core/base/dto/update-row.dto.ts index 4dd020e1d..e5f26691d 100644 --- a/apps/server/src/core/base/dto/update-row.dto.ts +++ b/apps/server/src/core/base/dto/update-row.dto.ts @@ -1,4 +1,5 @@ import { + IsBoolean, IsIn, IsNotEmpty, IsObject, @@ -83,6 +84,25 @@ export class ListRowsDto { search?: unknown; } +export class CountRowsDto { + @IsUUID() + baseId: string; + + @IsOptional() + @IsObject() + filter?: unknown; + + @IsOptional() + @IsObject() + search?: unknown; + + // Opt-in for an exact (but capped) count. Default is an EXPLAIN-plan + // estimate — cheap and good enough for "≈ N rows" headers. + @IsOptional() + @IsBoolean() + exact?: boolean; +} + export class ReorderRowDto { @IsUUID() rowId: string; diff --git a/apps/server/src/core/base/services/base-row.service.ts b/apps/server/src/core/base/services/base-row.service.ts index 4453d8fa2..bb52cf801 100644 --- a/apps/server/src/core/base/services/base-row.service.ts +++ b/apps/server/src/core/base/services/base-row.service.ts @@ -19,6 +19,7 @@ import { DeleteRowsDto, ListRowsDto, ReorderRowDto, + CountRowsDto, } from '../dto/update-row.dto'; import { BasePropertyTypeValue, @@ -46,6 +47,12 @@ import { } from '../events/base-events'; import { FormulaService } from '../formula/formula.service'; +// Cap for `count({ exact: true })`. Beyond this we return `capped: true` and +// the UI shows "N+". Chosen so a cell-wise scan stays well under 100ms on +// the existing `idx_base_rows_cells_gin_path_ops` index; callers that need +// true totals should fall back to `exact: false` (planner estimate). +const EXACT_COUNT_CAP = 10_000; + @Injectable() export class BaseRowService { constructor( @@ -244,6 +251,39 @@ export class BaseRowService { }); } + async count(dto: CountRowsDto, workspaceId: string) { + const properties = await this.basePropertyRepo.findByBaseId(dto.baseId); + const schema: PropertySchema = new Map(properties.map((p) => [p.id, p])); + + const filter = this.normaliseFilter({ filter: dto.filter }); + const search = this.normaliseSearch(dto.search); + + if (dto.exact) { + const { value, capped } = await this.baseRowRepo.countExact({ + baseId: dto.baseId, + workspaceId, + filter, + search, + schema, + cap: EXACT_COUNT_CAP, + }); + return { value, exact: true as const, capped }; + } + + const estimate = await this.baseRowRepo.countEstimate({ + baseId: dto.baseId, + workspaceId, + filter, + search, + schema, + }); + return { + value: estimate ?? 0, + exact: false as const, + capped: false, + }; + } + async reorder(dto: ReorderRowDto, workspaceId: string, userId?: string) { const row = await this.baseRowRepo.findById(dto.rowId, { workspaceId }); if (!row || row.baseId !== dto.baseId) { @@ -274,7 +314,7 @@ export class BaseRowService { // --- private helpers ------------------------------------------------ - private normaliseFilter(dto: ListRowsDto): FilterNode | undefined { + private normaliseFilter(dto: { filter?: unknown }): FilterNode | undefined { if (!dto.filter) return undefined; const parsed = filterGroupSchema.safeParse(dto.filter); diff --git a/apps/server/src/database/repos/base/base-row.repo.ts b/apps/server/src/database/repos/base/base-row.repo.ts index 9f709abf5..6778f7ab0 100644 --- a/apps/server/src/database/repos/base/base-row.repo.ts +++ b/apps/server/src/database/repos/base/base-row.repo.ts @@ -11,12 +11,14 @@ import { CursorPaginationResult, executeWithCursorPagination, } from '@docmost/db/pagination/cursor-pagination'; -import { sql, SqlBool } from 'kysely'; +import { CompiledQuery, sql, SqlBool } from 'kysely'; import { FilterNode, PropertySchema, SearchSpec, SortSpec, + buildSearch, + buildWhere, runListQuery, } from '../../../core/base/engine'; @@ -128,6 +130,109 @@ export class BaseRowRepo { }); } + /* + * EXPLAIN-based row estimate for the same predicate shape `list` uses. + * Doesn't execute the query — ~1ms even on large bases. Returns `null` + * when the plan JSON is unexpected. Caller treats the number as + * approximate; planner accuracy depends on fresh stats (ANALYZE). + */ + async countEstimate(opts: { + baseId: string; + workspaceId: string; + filter?: FilterNode; + search?: SearchSpec; + schema: PropertySchema; + trx?: KyselyTransaction; + }): Promise { + const db = dbOrTx(this.db, opts.trx); + + let qb = db + .selectFrom('baseRows') + .select(sql`1`.as('x')) + .where('baseId', '=', opts.baseId) + .where('workspaceId', '=', opts.workspaceId) + .where('deletedAt', 'is', null); + + if (opts.search) { + const spec = opts.search; + qb = qb.where((eb) => buildSearch(eb, spec)); + } + if (opts.filter) { + const filter = opts.filter; + const schema = opts.schema; + qb = qb.where((eb) => buildWhere(eb, filter, schema)); + } + + // Embedding `qb` inside a `sql` template would wrap it in parens as a + // subquery, which PG rejects after EXPLAIN. Splice the raw SQL + reuse + // the compiled parameters via a hand-built CompiledQuery instead. + const compiled = qb.compile(); + const explain: CompiledQuery = { + ...compiled, + sql: `EXPLAIN (FORMAT JSON) ${compiled.sql}`, + }; + const res = await db.executeQuery<{ 'QUERY PLAN': unknown }>(explain); + + // `QUERY PLAN` is a json column — postgres.js auto-parses it into an + // array. Fall back to JSON.parse for drivers that return it as text. + let plan = (res.rows[0] as any)?.['QUERY PLAN']; + if (typeof plan === 'string') { + try { + plan = JSON.parse(plan); + } catch { + return null; + } + } + const n = Array.isArray(plan) ? plan[0]?.Plan?.['Plan Rows'] : undefined; + return typeof n === 'number' ? Math.max(0, Math.round(n)) : null; + } + + /* + * Capped exact count. Wraps the filtered scan in a `LIMIT cap + 1` + * subquery so PG stops touching rows once we've proven the filter + * matches more than `cap` — callers render "cap+" in the UI instead. + */ + async countExact(opts: { + baseId: string; + workspaceId: string; + filter?: FilterNode; + search?: SearchSpec; + schema: PropertySchema; + cap: number; + trx?: KyselyTransaction; + }): Promise<{ value: number; capped: boolean }> { + const db = dbOrTx(this.db, opts.trx); + + let inner = db + .selectFrom('baseRows') + .select(sql`1`.as('x')) + .where('baseId', '=', opts.baseId) + .where('workspaceId', '=', opts.workspaceId) + .where('deletedAt', 'is', null); + + if (opts.search) { + const spec = opts.search; + inner = inner.where((eb) => buildSearch(eb, spec)); + } + if (opts.filter) { + const filter = opts.filter; + const schema = opts.schema; + inner = inner.where((eb) => buildWhere(eb, filter, schema)); + } + + inner = inner.limit(opts.cap + 1); + const compiled = inner.compile(); + const wrapped: CompiledQuery = { + ...compiled, + sql: `SELECT count(*)::text AS n FROM (${compiled.sql}) AS t`, + }; + const res = await db.executeQuery<{ n: string }>(wrapped); + + const n = Number(res.rows[0]?.n ?? 0); + if (n > opts.cap) return { value: opts.cap, capped: true }; + return { value: n, capped: false }; + } + async getLastPosition( baseId: string, opts: WorkspaceOpts,