mirror of
https://github.com/docmost/docmost.git
synced 2026-07-27 10:34:52 +10:00
feat(base): add /bases/rows/count with estimate and capped-exact modes
Row-count display on a filtered view shouldn't force a full COUNT(*) on
every list fetch. New endpoint returns either an EXPLAIN-plan estimate
(default, ~1ms, no execution) or a LIMIT-capped exact count that short-
circuits to `{ capped: true }` once the match set passes EXACT_COUNT_CAP.
Clients call it in parallel with the rows query so the grid still paints
at its own pace.
- DTO + repo.countEstimate/countExact reusing the list predicate shape
- service picks the mode; controller mirrors the list Read ability check
- client hook keyed by filter/search/exact so a "show exact" toggle
doesn't clobber the estimate cache
This commit is contained in:
@@ -1,6 +1,7 @@
|
|||||||
import {
|
import {
|
||||||
useInfiniteQuery,
|
useInfiniteQuery,
|
||||||
useMutation,
|
useMutation,
|
||||||
|
useQuery,
|
||||||
InfiniteData,
|
InfiniteData,
|
||||||
} from "@tanstack/react-query";
|
} from "@tanstack/react-query";
|
||||||
import {
|
import {
|
||||||
@@ -10,6 +11,7 @@ import {
|
|||||||
deleteRows,
|
deleteRows,
|
||||||
listRows,
|
listRows,
|
||||||
reorderRow,
|
reorderRow,
|
||||||
|
countRows,
|
||||||
} from "@/features/base/services/base-service";
|
} from "@/features/base/services/base-service";
|
||||||
import {
|
import {
|
||||||
IBaseRow,
|
IBaseRow,
|
||||||
@@ -21,6 +23,7 @@ import {
|
|||||||
FilterNode,
|
FilterNode,
|
||||||
SearchSpec,
|
SearchSpec,
|
||||||
ViewSortConfig,
|
ViewSortConfig,
|
||||||
|
CountRowsResult,
|
||||||
} from "@/features/base/types/base.types";
|
} from "@/features/base/types/base.types";
|
||||||
import { notifications } from "@mantine/notifications";
|
import { notifications } from "@mantine/notifications";
|
||||||
import { queryClient } from "@/main";
|
import { queryClient } from "@/main";
|
||||||
@@ -277,6 +280,35 @@ export function useDeleteRowsMutation() {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Row count for the current view. Fires in parallel with `useBaseRowsQuery`
|
||||||
|
* — doesn't block first paint. Keyed by filter + search so an independent
|
||||||
|
* view with a different filter gets its own cached count; `exact` is part
|
||||||
|
* of the key so a "show exact" toggle doesn't clobber the estimate cache.
|
||||||
|
*/
|
||||||
|
export function useBaseRowsCountQuery(
|
||||||
|
baseId: string | undefined,
|
||||||
|
filter?: FilterNode,
|
||||||
|
search?: SearchSpec,
|
||||||
|
exact = false,
|
||||||
|
) {
|
||||||
|
const activeFilter = normalizeFilter(filter);
|
||||||
|
const activeSearch = search?.query ? search : undefined;
|
||||||
|
|
||||||
|
return useQuery<CountRowsResult>({
|
||||||
|
queryKey: ["base-rows-count", baseId, activeFilter, activeSearch, exact],
|
||||||
|
queryFn: () =>
|
||||||
|
countRows({
|
||||||
|
baseId: baseId!,
|
||||||
|
filter: activeFilter,
|
||||||
|
search: activeSearch,
|
||||||
|
exact,
|
||||||
|
}),
|
||||||
|
enabled: !!baseId,
|
||||||
|
staleTime: 30 * 1000,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export function useReorderRowMutation() {
|
export function useReorderRowMutation() {
|
||||||
const { t } = useTranslation();
|
const { t } = useTranslation();
|
||||||
return useMutation<void, Error, ReorderRowInput, RowCacheContext>({
|
return useMutation<void, Error, ReorderRowInput, RowCacheContext>({
|
||||||
|
|||||||
@@ -23,6 +23,8 @@ import {
|
|||||||
FilterNode,
|
FilterNode,
|
||||||
SearchSpec,
|
SearchSpec,
|
||||||
ViewSortConfig,
|
ViewSortConfig,
|
||||||
|
CountRowsInput,
|
||||||
|
CountRowsResult,
|
||||||
} from "@/features/base/types/base.types";
|
} from "@/features/base/types/base.types";
|
||||||
import { IPagination } from "@/lib/types";
|
import { IPagination } from "@/lib/types";
|
||||||
|
|
||||||
@@ -151,6 +153,13 @@ export async function reorderRow(data: ReorderRowInput): Promise<void> {
|
|||||||
await api.post("/bases/rows/reorder", data);
|
await api.post("/bases/rows/reorder", data);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function countRows(
|
||||||
|
data: CountRowsInput,
|
||||||
|
): Promise<CountRowsResult> {
|
||||||
|
const req = await api.post<CountRowsResult>("/bases/rows/count", data);
|
||||||
|
return req.data;
|
||||||
|
}
|
||||||
|
|
||||||
// --- Views ---
|
// --- Views ---
|
||||||
|
|
||||||
export async function createView(data: CreateViewInput): Promise<IBaseView> {
|
export async function createView(data: CreateViewInput): Promise<IBaseView> {
|
||||||
|
|||||||
@@ -239,6 +239,21 @@ export type ReorderPropertyInput = {
|
|||||||
requestId?: string;
|
requestId?: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export type CountRowsInput = {
|
||||||
|
baseId: string;
|
||||||
|
filter?: FilterNode;
|
||||||
|
search?: SearchSpec;
|
||||||
|
// When true the server returns an exact (but capped) count. Otherwise
|
||||||
|
// a cheap EXPLAIN-plan estimate. Paired with `capped` on the result.
|
||||||
|
exact?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
|
export type CountRowsResult = {
|
||||||
|
value: number;
|
||||||
|
exact: boolean;
|
||||||
|
capped: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
export type CreateRowInput = {
|
export type CreateRowInput = {
|
||||||
baseId: string;
|
baseId: string;
|
||||||
cells?: Record<string, unknown>;
|
cells?: Record<string, unknown>;
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ import {
|
|||||||
RowIdDto,
|
RowIdDto,
|
||||||
ListRowsDto,
|
ListRowsDto,
|
||||||
ReorderRowDto,
|
ReorderRowDto,
|
||||||
|
CountRowsDto,
|
||||||
} from '../dto/update-row.dto';
|
} from '../dto/update-row.dto';
|
||||||
import { AuthUser } from '../../../common/decorators/auth-user.decorator';
|
import { AuthUser } from '../../../common/decorators/auth-user.decorator';
|
||||||
import { AuthWorkspace } from '../../../common/decorators/auth-workspace.decorator';
|
import { AuthWorkspace } from '../../../common/decorators/auth-workspace.decorator';
|
||||||
@@ -160,6 +161,26 @@ export class BaseRowController {
|
|||||||
return this.baseRowService.list(dto, pagination, workspace.id);
|
return this.baseRowService.list(dto, pagination, workspace.id);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@HttpCode(HttpStatus.OK)
|
||||||
|
@Post('count')
|
||||||
|
async count(
|
||||||
|
@Body() dto: CountRowsDto,
|
||||||
|
@AuthUser() user: User,
|
||||||
|
@AuthWorkspace() workspace: Workspace,
|
||||||
|
) {
|
||||||
|
const base = await this.baseRepo.findById(dto.baseId);
|
||||||
|
if (!base) {
|
||||||
|
throw new NotFoundException('Base not found');
|
||||||
|
}
|
||||||
|
|
||||||
|
const ability = await this.spaceAbility.createForUser(user, base.spaceId);
|
||||||
|
if (ability.cannot(SpaceCaslAction.Read, SpaceCaslSubject.Base)) {
|
||||||
|
throw new ForbiddenException();
|
||||||
|
}
|
||||||
|
|
||||||
|
return this.baseRowService.count(dto, workspace.id);
|
||||||
|
}
|
||||||
|
|
||||||
@HttpCode(HttpStatus.OK)
|
@HttpCode(HttpStatus.OK)
|
||||||
@Post('reorder')
|
@Post('reorder')
|
||||||
async reorder(
|
async reorder(
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import {
|
import {
|
||||||
|
IsBoolean,
|
||||||
IsIn,
|
IsIn,
|
||||||
IsNotEmpty,
|
IsNotEmpty,
|
||||||
IsObject,
|
IsObject,
|
||||||
@@ -83,6 +84,25 @@ export class ListRowsDto {
|
|||||||
search?: unknown;
|
search?: unknown;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export class CountRowsDto {
|
||||||
|
@IsUUID()
|
||||||
|
baseId: string;
|
||||||
|
|
||||||
|
@IsOptional()
|
||||||
|
@IsObject()
|
||||||
|
filter?: unknown;
|
||||||
|
|
||||||
|
@IsOptional()
|
||||||
|
@IsObject()
|
||||||
|
search?: unknown;
|
||||||
|
|
||||||
|
// Opt-in for an exact (but capped) count. Default is an EXPLAIN-plan
|
||||||
|
// estimate — cheap and good enough for "≈ N rows" headers.
|
||||||
|
@IsOptional()
|
||||||
|
@IsBoolean()
|
||||||
|
exact?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
export class ReorderRowDto {
|
export class ReorderRowDto {
|
||||||
@IsUUID()
|
@IsUUID()
|
||||||
rowId: string;
|
rowId: string;
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ import {
|
|||||||
DeleteRowsDto,
|
DeleteRowsDto,
|
||||||
ListRowsDto,
|
ListRowsDto,
|
||||||
ReorderRowDto,
|
ReorderRowDto,
|
||||||
|
CountRowsDto,
|
||||||
} from '../dto/update-row.dto';
|
} from '../dto/update-row.dto';
|
||||||
import {
|
import {
|
||||||
BasePropertyTypeValue,
|
BasePropertyTypeValue,
|
||||||
@@ -46,6 +47,12 @@ import {
|
|||||||
} from '../events/base-events';
|
} from '../events/base-events';
|
||||||
import { FormulaService } from '../formula/formula.service';
|
import { FormulaService } from '../formula/formula.service';
|
||||||
|
|
||||||
|
// Cap for `count({ exact: true })`. Beyond this we return `capped: true` and
|
||||||
|
// the UI shows "N+". Chosen so a cell-wise scan stays well under 100ms on
|
||||||
|
// the existing `idx_base_rows_cells_gin_path_ops` index; callers that need
|
||||||
|
// true totals should fall back to `exact: false` (planner estimate).
|
||||||
|
const EXACT_COUNT_CAP = 10_000;
|
||||||
|
|
||||||
@Injectable()
|
@Injectable()
|
||||||
export class BaseRowService {
|
export class BaseRowService {
|
||||||
constructor(
|
constructor(
|
||||||
@@ -244,6 +251,39 @@ export class BaseRowService {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async count(dto: CountRowsDto, workspaceId: string) {
|
||||||
|
const properties = await this.basePropertyRepo.findByBaseId(dto.baseId);
|
||||||
|
const schema: PropertySchema = new Map(properties.map((p) => [p.id, p]));
|
||||||
|
|
||||||
|
const filter = this.normaliseFilter({ filter: dto.filter });
|
||||||
|
const search = this.normaliseSearch(dto.search);
|
||||||
|
|
||||||
|
if (dto.exact) {
|
||||||
|
const { value, capped } = await this.baseRowRepo.countExact({
|
||||||
|
baseId: dto.baseId,
|
||||||
|
workspaceId,
|
||||||
|
filter,
|
||||||
|
search,
|
||||||
|
schema,
|
||||||
|
cap: EXACT_COUNT_CAP,
|
||||||
|
});
|
||||||
|
return { value, exact: true as const, capped };
|
||||||
|
}
|
||||||
|
|
||||||
|
const estimate = await this.baseRowRepo.countEstimate({
|
||||||
|
baseId: dto.baseId,
|
||||||
|
workspaceId,
|
||||||
|
filter,
|
||||||
|
search,
|
||||||
|
schema,
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
value: estimate ?? 0,
|
||||||
|
exact: false as const,
|
||||||
|
capped: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
async reorder(dto: ReorderRowDto, workspaceId: string, userId?: string) {
|
async reorder(dto: ReorderRowDto, workspaceId: string, userId?: string) {
|
||||||
const row = await this.baseRowRepo.findById(dto.rowId, { workspaceId });
|
const row = await this.baseRowRepo.findById(dto.rowId, { workspaceId });
|
||||||
if (!row || row.baseId !== dto.baseId) {
|
if (!row || row.baseId !== dto.baseId) {
|
||||||
@@ -274,7 +314,7 @@ export class BaseRowService {
|
|||||||
|
|
||||||
// --- private helpers ------------------------------------------------
|
// --- private helpers ------------------------------------------------
|
||||||
|
|
||||||
private normaliseFilter(dto: ListRowsDto): FilterNode | undefined {
|
private normaliseFilter(dto: { filter?: unknown }): FilterNode | undefined {
|
||||||
if (!dto.filter) return undefined;
|
if (!dto.filter) return undefined;
|
||||||
|
|
||||||
const parsed = filterGroupSchema.safeParse(dto.filter);
|
const parsed = filterGroupSchema.safeParse(dto.filter);
|
||||||
|
|||||||
@@ -11,12 +11,14 @@ import {
|
|||||||
CursorPaginationResult,
|
CursorPaginationResult,
|
||||||
executeWithCursorPagination,
|
executeWithCursorPagination,
|
||||||
} from '@docmost/db/pagination/cursor-pagination';
|
} from '@docmost/db/pagination/cursor-pagination';
|
||||||
import { sql, SqlBool } from 'kysely';
|
import { CompiledQuery, sql, SqlBool } from 'kysely';
|
||||||
import {
|
import {
|
||||||
FilterNode,
|
FilterNode,
|
||||||
PropertySchema,
|
PropertySchema,
|
||||||
SearchSpec,
|
SearchSpec,
|
||||||
SortSpec,
|
SortSpec,
|
||||||
|
buildSearch,
|
||||||
|
buildWhere,
|
||||||
runListQuery,
|
runListQuery,
|
||||||
} from '../../../core/base/engine';
|
} from '../../../core/base/engine';
|
||||||
|
|
||||||
@@ -128,6 +130,109 @@ export class BaseRowRepo {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* EXPLAIN-based row estimate for the same predicate shape `list` uses.
|
||||||
|
* Doesn't execute the query — ~1ms even on large bases. Returns `null`
|
||||||
|
* when the plan JSON is unexpected. Caller treats the number as
|
||||||
|
* approximate; planner accuracy depends on fresh stats (ANALYZE).
|
||||||
|
*/
|
||||||
|
async countEstimate(opts: {
|
||||||
|
baseId: string;
|
||||||
|
workspaceId: string;
|
||||||
|
filter?: FilterNode;
|
||||||
|
search?: SearchSpec;
|
||||||
|
schema: PropertySchema;
|
||||||
|
trx?: KyselyTransaction;
|
||||||
|
}): Promise<number | null> {
|
||||||
|
const db = dbOrTx(this.db, opts.trx);
|
||||||
|
|
||||||
|
let qb = db
|
||||||
|
.selectFrom('baseRows')
|
||||||
|
.select(sql<number>`1`.as('x'))
|
||||||
|
.where('baseId', '=', opts.baseId)
|
||||||
|
.where('workspaceId', '=', opts.workspaceId)
|
||||||
|
.where('deletedAt', 'is', null);
|
||||||
|
|
||||||
|
if (opts.search) {
|
||||||
|
const spec = opts.search;
|
||||||
|
qb = qb.where((eb) => buildSearch(eb, spec));
|
||||||
|
}
|
||||||
|
if (opts.filter) {
|
||||||
|
const filter = opts.filter;
|
||||||
|
const schema = opts.schema;
|
||||||
|
qb = qb.where((eb) => buildWhere(eb, filter, schema));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Embedding `qb` inside a `sql` template would wrap it in parens as a
|
||||||
|
// subquery, which PG rejects after EXPLAIN. Splice the raw SQL + reuse
|
||||||
|
// the compiled parameters via a hand-built CompiledQuery instead.
|
||||||
|
const compiled = qb.compile();
|
||||||
|
const explain: CompiledQuery = {
|
||||||
|
...compiled,
|
||||||
|
sql: `EXPLAIN (FORMAT JSON) ${compiled.sql}`,
|
||||||
|
};
|
||||||
|
const res = await db.executeQuery<{ 'QUERY PLAN': unknown }>(explain);
|
||||||
|
|
||||||
|
// `QUERY PLAN` is a json column — postgres.js auto-parses it into an
|
||||||
|
// array. Fall back to JSON.parse for drivers that return it as text.
|
||||||
|
let plan = (res.rows[0] as any)?.['QUERY PLAN'];
|
||||||
|
if (typeof plan === 'string') {
|
||||||
|
try {
|
||||||
|
plan = JSON.parse(plan);
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const n = Array.isArray(plan) ? plan[0]?.Plan?.['Plan Rows'] : undefined;
|
||||||
|
return typeof n === 'number' ? Math.max(0, Math.round(n)) : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Capped exact count. Wraps the filtered scan in a `LIMIT cap + 1`
|
||||||
|
* subquery so PG stops touching rows once we've proven the filter
|
||||||
|
* matches more than `cap` — callers render "cap+" in the UI instead.
|
||||||
|
*/
|
||||||
|
async countExact(opts: {
|
||||||
|
baseId: string;
|
||||||
|
workspaceId: string;
|
||||||
|
filter?: FilterNode;
|
||||||
|
search?: SearchSpec;
|
||||||
|
schema: PropertySchema;
|
||||||
|
cap: number;
|
||||||
|
trx?: KyselyTransaction;
|
||||||
|
}): Promise<{ value: number; capped: boolean }> {
|
||||||
|
const db = dbOrTx(this.db, opts.trx);
|
||||||
|
|
||||||
|
let inner = db
|
||||||
|
.selectFrom('baseRows')
|
||||||
|
.select(sql<number>`1`.as('x'))
|
||||||
|
.where('baseId', '=', opts.baseId)
|
||||||
|
.where('workspaceId', '=', opts.workspaceId)
|
||||||
|
.where('deletedAt', 'is', null);
|
||||||
|
|
||||||
|
if (opts.search) {
|
||||||
|
const spec = opts.search;
|
||||||
|
inner = inner.where((eb) => buildSearch(eb, spec));
|
||||||
|
}
|
||||||
|
if (opts.filter) {
|
||||||
|
const filter = opts.filter;
|
||||||
|
const schema = opts.schema;
|
||||||
|
inner = inner.where((eb) => buildWhere(eb, filter, schema));
|
||||||
|
}
|
||||||
|
|
||||||
|
inner = inner.limit(opts.cap + 1);
|
||||||
|
const compiled = inner.compile();
|
||||||
|
const wrapped: CompiledQuery = {
|
||||||
|
...compiled,
|
||||||
|
sql: `SELECT count(*)::text AS n FROM (${compiled.sql}) AS t`,
|
||||||
|
};
|
||||||
|
const res = await db.executeQuery<{ n: string }>(wrapped);
|
||||||
|
|
||||||
|
const n = Number(res.rows[0]?.n ?? 0);
|
||||||
|
if (n > opts.cap) return { value: opts.cap, capped: true };
|
||||||
|
return { value: n, capped: false };
|
||||||
|
}
|
||||||
|
|
||||||
async getLastPosition(
|
async getLastPosition(
|
||||||
baseId: string,
|
baseId: string,
|
||||||
opts: WorkspaceOpts,
|
opts: WorkspaceOpts,
|
||||||
|
|||||||
Reference in New Issue
Block a user