feat: add ai detection for recipients and fields (#2271)

Use Gemini to handle detection of recipients and fields within
documents.

Opt in using organisation or team settings.

Replaces #2128 since the branch was cursed and would include
dependencies that weren't even in the lock file.



https://github.com/user-attachments/assets/e6cbb58f-62b9-4079-a9ae-7af5c4f2e4ec
This commit is contained in:
Lucas Smith
2025-12-03 23:39:41 +11:00
committed by GitHub
parent e39924714a
commit 7a94ee3b83
51 changed files with 4202 additions and 68 deletions
@@ -0,0 +1,161 @@
import { z } from 'zod';
import {
type TDetectFieldsRequest,
ZNormalizedFieldWithContextSchema,
} from './detect-fields.types';
export type { TDetectFieldsRequest };
// Stream event schemas
const ZProgressEventSchema = z.object({
type: z.literal('progress'),
pagesProcessed: z.number(),
totalPages: z.number(),
fieldsDetected: z.number(),
});
const ZKeepaliveEventSchema = z.object({
type: z.literal('keepalive'),
});
const ZErrorEventSchema = z.object({
type: z.literal('error'),
message: z.string(),
});
const ZCompleteEventSchema = z.object({
type: z.literal('complete'),
fields: z.array(ZNormalizedFieldWithContextSchema),
});
const ZStreamEventSchema = z.discriminatedUnion('type', [
ZProgressEventSchema,
ZKeepaliveEventSchema,
ZErrorEventSchema,
ZCompleteEventSchema,
]);
export type DetectFieldsProgressEvent = z.infer<typeof ZProgressEventSchema>;
export type DetectFieldsCompleteEvent = z.infer<typeof ZCompleteEventSchema>;
export type DetectFieldsStreamEvent = z.infer<typeof ZStreamEventSchema>;
const ZApiErrorResponseSchema = z.object({
error: z.string(),
});
export class AiApiError extends Error {
constructor(
message: string,
public status: number,
) {
super(message);
this.name = 'AiApiError';
}
}
export type DetectFieldsOptions = {
request: TDetectFieldsRequest;
onProgress?: (event: DetectFieldsProgressEvent) => void;
onComplete: (event: DetectFieldsCompleteEvent) => void;
onError: (error: AiApiError) => void;
signal?: AbortSignal;
};
/**
* Detect fields from an envelope using AI with streaming support.
*/
export const detectFields = async ({
request,
onProgress,
onComplete,
onError,
signal,
}: DetectFieldsOptions): Promise<void> => {
const response = await fetch('/api/ai/detect-fields', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(request),
signal,
});
// Handle non-streaming error responses (auth failures, etc.)
if (!response.ok) {
const text = await response.text();
try {
const parsed = ZApiErrorResponseSchema.parse(JSON.parse(text));
throw new AiApiError(parsed.error, response.status);
} catch (e) {
if (e instanceof AiApiError) {
throw e;
}
throw new AiApiError('Failed to detect fields', response.status);
}
}
// Handle streaming response
const reader = response.body?.getReader();
if (!reader) {
throw new AiApiError('No response body', 500);
}
const decoder = new TextDecoder();
let buffer = '';
try {
let done = false;
while (!done) {
const result = await reader.read();
done = result.done;
if (done) {
break;
}
const value = result.value;
buffer += decoder.decode(value, { stream: true });
// Process complete lines
const lines = buffer.split('\n');
buffer = lines.pop() || ''; // Keep incomplete line in buffer
for (const line of lines) {
if (!line.trim()) {
continue;
}
try {
const event = ZStreamEventSchema.parse(JSON.parse(line));
switch (event.type) {
case 'progress':
onProgress?.(event);
break;
case 'keepalive':
// Ignore keepalive, it's just to prevent timeout
break;
case 'error':
onError(new AiApiError(event.message, 500));
return;
case 'complete':
onComplete(event);
return;
}
} catch {
// Ignore malformed lines
console.warn('Failed to parse stream event:', line);
}
}
}
} finally {
reader.releaseLock();
}
};
+150
View File
@@ -0,0 +1,150 @@
import { sValidator } from '@hono/standard-validator';
import { Hono } from 'hono';
import { streamText } from 'hono/streaming';
import { getSession } from '@documenso/auth/server/lib/utils/get-session';
import { IS_AI_FEATURES_CONFIGURED } from '@documenso/lib/constants/app';
import { AppError, AppErrorCode } from '@documenso/lib/errors/app-error';
import { detectFieldsFromEnvelope } from '@documenso/lib/server-only/ai/envelope/detect-fields';
import { getTeamById } from '@documenso/lib/server-only/team/get-team';
import type { HonoEnv } from '../../router';
import { ZDetectFieldsRequestSchema } from './detect-fields.types';
const KEEPALIVE_INTERVAL_MS = 5000;
export const detectFieldsRoute = new Hono<HonoEnv>().post(
'/',
sValidator('json', ZDetectFieldsRequestSchema),
async (c) => {
const logger = c.get('logger');
try {
const { envelopeId, teamId, context } = c.req.valid('json');
const session = await getSession(c);
if (!session.user) {
throw new AppError(AppErrorCode.UNAUTHORIZED, {
message: 'You must be logged in to detect fields',
});
}
// Verify user has access to the team (abort early)
const team = await getTeamById({
userId: session.user.id,
teamId,
}).catch(() => null);
if (!team) {
throw new AppError(AppErrorCode.UNAUTHORIZED, {
message: 'You do not have access to this team',
});
}
// Check if AI features are enabled for the team
const { aiFeaturesEnabled } = team.derivedSettings;
if (!aiFeaturesEnabled) {
throw new AppError(AppErrorCode.UNAUTHORIZED, {
message: 'AI features are not enabled for this team',
});
}
if (!IS_AI_FEATURES_CONFIGURED()) {
throw new AppError(AppErrorCode.INVALID_REQUEST, {
message: 'AI features are not configured. Please contact support to enable AI features.',
});
}
logger.info({
event: 'ai.detect-fields.start',
envelopeId,
userId: session.user.id,
teamId: team.id,
hasContext: !!context,
});
// Return streaming response with NDJSON
return streamText(c, async (stream) => {
// Start keepalive to prevent connection timeout
let interval: NodeJS.Timeout | null = setInterval(() => {
void stream.writeln(JSON.stringify({ type: 'keepalive' }));
}, KEEPALIVE_INTERVAL_MS);
try {
const allFields = await detectFieldsFromEnvelope({
context,
envelopeId,
userId: session.user.id,
teamId: team.id,
onProgress: (progress) => {
void stream.writeln(
JSON.stringify({
type: 'progress',
pagesProcessed: progress.pagesProcessed,
totalPages: progress.totalPages,
fieldsDetected: progress.fieldsDetected,
}),
);
},
});
// Clear keepalive before sending final response
if (interval) {
clearInterval(interval);
interval = null;
}
logger.info({
event: 'ai.detect-fields.complete',
envelopeId,
userId: session.user.id,
teamId: team.id,
fieldCount: allFields.length,
});
await stream.writeln(
JSON.stringify({
type: 'complete',
fields: allFields,
}),
);
} catch (error) {
// Clear keepalive on error
if (interval) {
clearInterval(interval);
}
logger.error({
event: 'ai.detect-fields.error',
error,
});
const message = error instanceof AppError ? error.message : 'Failed to detect fields';
await stream.writeln(
JSON.stringify({
type: 'error',
message,
}),
);
}
});
} catch (error) {
// Handle errors that occur before streaming starts
logger.error({
event: 'ai.detect-fields.error',
error,
});
if (error instanceof AppError) {
const { status, body } = AppError.toRestAPIError(error);
return c.json(body, status);
}
return c.json({ error: 'Failed to detect fields' }, 500);
}
},
);
@@ -0,0 +1,54 @@
import { z } from 'zod';
import {
ZConfidenceLevel,
ZDetectableFieldType,
} from '@documenso/lib/server-only/ai/envelope/detect-fields/schema';
export const ZDetectFieldsRequestSchema = z.object({
envelopeId: z.string().min(1).describe('The ID of the envelope to detect fields from.'),
teamId: z.number().describe('The ID of the team the envelope belongs to.'),
context: z
.string()
.optional()
.describe(
'Optional context about recipients to help map fields (e.g., "David is the Employee, Lucas is the Manager").',
),
});
export type TDetectFieldsRequest = z.infer<typeof ZDetectFieldsRequestSchema>;
// Schema for fields returned from streaming API (before recipient resolution)
export const ZNormalizedFieldWithPageSchema = z.object({
type: ZDetectableFieldType,
recipientKey: z.string(),
positionX: z.number(),
positionY: z.number(),
width: z.number(),
height: z.number(),
confidence: ZConfidenceLevel,
pageNumber: z.number(),
});
export type TNormalizedFieldWithPage = z.infer<typeof ZNormalizedFieldWithPageSchema>;
// Schema for fields after recipient resolution
export const ZNormalizedFieldWithContextSchema = z.object({
type: ZDetectableFieldType,
positionX: z.number(),
positionY: z.number(),
width: z.number(),
height: z.number(),
confidence: ZConfidenceLevel,
pageNumber: z.number(),
recipientId: z.number(),
envelopeItemId: z.string(),
});
export type TNormalizedFieldWithContext = z.infer<typeof ZNormalizedFieldWithContextSchema>;
export const ZDetectFieldsResponseSchema = z.object({
fields: z.array(ZNormalizedFieldWithContextSchema),
});
export type TDetectFieldsResponse = z.infer<typeof ZDetectFieldsResponseSchema>;
@@ -0,0 +1,160 @@
import { z } from 'zod';
import { ZDetectedRecipientSchema } from '@documenso/lib/server-only/ai/envelope/detect-recipients/schema';
import { type TDetectRecipientsRequest } from './detect-recipients.types';
export type { TDetectRecipientsRequest };
// Stream event schemas
const ZProgressEventSchema = z.object({
type: z.literal('progress'),
pagesProcessed: z.number(),
totalPages: z.number(),
recipientsDetected: z.number(),
});
const ZKeepaliveEventSchema = z.object({
type: z.literal('keepalive'),
});
const ZErrorEventSchema = z.object({
type: z.literal('error'),
message: z.string(),
});
const ZCompleteEventSchema = z.object({
type: z.literal('complete'),
recipients: z.array(ZDetectedRecipientSchema),
});
const ZStreamEventSchema = z.discriminatedUnion('type', [
ZProgressEventSchema,
ZKeepaliveEventSchema,
ZErrorEventSchema,
ZCompleteEventSchema,
]);
export type DetectRecipientsProgressEvent = z.infer<typeof ZProgressEventSchema>;
export type DetectRecipientsCompleteEvent = z.infer<typeof ZCompleteEventSchema>;
export type DetectRecipientsStreamEvent = z.infer<typeof ZStreamEventSchema>;
const ZApiErrorResponseSchema = z.object({
error: z.string(),
});
export class AiApiError extends Error {
constructor(
message: string,
public status: number,
) {
super(message);
this.name = 'AiApiError';
}
}
export type DetectRecipientsOptions = {
request: TDetectRecipientsRequest;
onProgress?: (event: DetectRecipientsProgressEvent) => void;
onComplete: (event: DetectRecipientsCompleteEvent) => void;
onError: (error: AiApiError) => void;
signal?: AbortSignal;
};
/**
* Detect recipients from an envelope using AI with streaming support.
*/
export const detectRecipients = async ({
request,
onProgress,
onComplete,
onError,
signal,
}: DetectRecipientsOptions): Promise<void> => {
const response = await fetch('/api/ai/detect-recipients', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(request),
signal,
});
// Handle non-streaming error responses (auth failures, etc.)
if (!response.ok) {
const text = await response.text();
try {
const parsed = ZApiErrorResponseSchema.parse(JSON.parse(text));
throw new AiApiError(parsed.error, response.status);
} catch (e) {
if (e instanceof AiApiError) {
throw e;
}
throw new AiApiError('Failed to detect recipients', response.status);
}
}
// Handle streaming response
const reader = response.body?.getReader();
if (!reader) {
throw new AiApiError('No response body', 500);
}
const decoder = new TextDecoder();
let buffer = '';
try {
let done = false;
while (!done) {
const result = await reader.read();
done = result.done;
if (done) {
break;
}
const value = result.value;
buffer += decoder.decode(value, { stream: true });
// Process complete lines
const lines = buffer.split('\n');
buffer = lines.pop() || ''; // Keep incomplete line in buffer
for (const line of lines) {
if (!line.trim()) {
continue;
}
try {
const event = ZStreamEventSchema.parse(JSON.parse(line));
switch (event.type) {
case 'progress':
onProgress?.(event);
break;
case 'keepalive':
// Ignore keepalive, it's just to prevent timeout
break;
case 'error':
onError(new AiApiError(event.message, 500));
return;
case 'complete':
onComplete(event);
return;
}
} catch {
// Ignore malformed lines
console.warn('Failed to parse stream event:', line);
}
}
}
} finally {
reader.releaseLock();
}
};
@@ -0,0 +1,148 @@
import { sValidator } from '@hono/standard-validator';
import { Hono } from 'hono';
import { streamText } from 'hono/streaming';
import { getSession } from '@documenso/auth/server/lib/utils/get-session';
import { IS_AI_FEATURES_CONFIGURED } from '@documenso/lib/constants/app';
import { AppError, AppErrorCode } from '@documenso/lib/errors/app-error';
import { detectRecipientsFromEnvelope } from '@documenso/lib/server-only/ai/envelope/detect-recipients';
import { getTeamById } from '@documenso/lib/server-only/team/get-team';
import type { HonoEnv } from '../../router';
import { ZDetectRecipientsRequestSchema } from './detect-recipients.types';
const KEEPALIVE_INTERVAL_MS = 5000;
export const detectRecipientsRoute = new Hono<HonoEnv>().post(
'/',
sValidator('json', ZDetectRecipientsRequestSchema),
async (c) => {
const logger = c.get('logger');
try {
const { envelopeId, teamId } = c.req.valid('json');
const session = await getSession(c);
if (!session.user) {
throw new AppError(AppErrorCode.UNAUTHORIZED, {
message: 'You must be logged in to detect recipients',
});
}
// Verify user has access to the team (abort early)
const team = await getTeamById({
userId: session.user.id,
teamId,
}).catch(() => null);
if (!team) {
throw new AppError(AppErrorCode.UNAUTHORIZED, {
message: 'You do not have access to this team',
});
}
// Check if AI features are enabled for the team
const { aiFeaturesEnabled } = team.derivedSettings;
if (!aiFeaturesEnabled) {
throw new AppError(AppErrorCode.UNAUTHORIZED, {
message: 'AI features are not enabled for this team',
});
}
if (!IS_AI_FEATURES_CONFIGURED()) {
throw new AppError(AppErrorCode.INVALID_REQUEST, {
message: 'AI features are not configured. Please contact support to enable AI features.',
});
}
logger.info({
event: 'ai.detect-recipients.start',
envelopeId,
userId: session.user.id,
teamId: team.id,
});
// Return streaming response with NDJSON
return streamText(c, async (stream) => {
// Start keepalive to prevent connection timeout
let interval: NodeJS.Timeout | null = setInterval(() => {
void stream.writeln(JSON.stringify({ type: 'keepalive' }));
}, KEEPALIVE_INTERVAL_MS);
try {
const recipients = await detectRecipientsFromEnvelope({
envelopeId,
userId: session.user.id,
teamId: team.id,
onProgress: (progress) => {
void stream.writeln(
JSON.stringify({
type: 'progress',
pagesProcessed: progress.pagesProcessed,
totalPages: progress.totalPages,
recipientsDetected: progress.recipientsDetected,
}),
);
},
});
// Clear keepalive before sending final response
if (interval) {
clearInterval(interval);
interval = null;
}
logger.info({
event: 'ai.detect-recipients.complete',
envelopeId,
userId: session.user.id,
teamId: team.id,
recipientCount: recipients.length,
});
await stream.writeln(
JSON.stringify({
type: 'complete',
recipients,
}),
);
} catch (error) {
// Clear keepalive on error
if (interval) {
clearInterval(interval);
}
logger.error({
event: 'ai.detect-recipients.error',
error,
});
const message = error instanceof AppError ? error.message : 'Failed to detect recipients';
await stream.writeln(
JSON.stringify({
type: 'error',
message,
}),
);
}
});
} catch (error) {
// Handle errors that occur before streaming starts
logger.error({
event: 'ai.detect-recipients.error',
error,
});
if (error instanceof AppError) {
const { status, body } = AppError.toRestAPIError(error);
return c.json(body, status);
}
return c.json({ error: 'Failed to detect recipients' }, 500);
}
},
);
@@ -0,0 +1,16 @@
import { z } from 'zod';
import { ZDetectedRecipientSchema } from '@documenso/lib/server-only/ai/envelope/detect-recipients/schema';
export const ZDetectRecipientsRequestSchema = z.object({
envelopeId: z.string().min(1).describe('The ID of the envelope to detect recipients from.'),
teamId: z.number().describe('The ID of the team the envelope belongs to.'),
});
export type TDetectRecipientsRequest = z.infer<typeof ZDetectRecipientsRequestSchema>;
export const ZDetectRecipientsResponseSchema = z.object({
recipients: z.array(ZDetectedRecipientSchema),
});
export type TDetectRecipientsResponse = z.infer<typeof ZDetectRecipientsResponseSchema>;
+9
View File
@@ -0,0 +1,9 @@
import { Hono } from 'hono';
import type { HonoEnv } from '../../router';
import { detectFieldsRoute } from './detect-fields';
import { detectRecipientsRoute } from './detect-recipients';
export const aiRoute = new Hono<HonoEnv>()
.route('/detect-recipients', detectRecipientsRoute)
.route('/detect-fields', detectFieldsRoute);