mirror of
https://github.com/AmruthPillai/Reactive-Resume.git
synced 2026-10-03 18:23:47 +10:00
feat: add shared AI and Firecrawl job search integrations
Add managed credentials, job posting search and scraping, database migration, translations, and self-hosting documentation. Fix empty resume-copy recovery and ensure every sheet popup renders inside a drawer viewport.
This commit is contained in:
@@ -44,6 +44,7 @@ export const aiProvidersRouter = {
|
||||
.output(type<AiProviderResponse>())
|
||||
.errors({
|
||||
BAD_REQUEST: { message: "Invalid AI provider configuration.", status: 400 },
|
||||
FORBIDDEN: { message: "AI is managed by the server.", status: 403 },
|
||||
PRECONDITION_FAILED: { message: "AI agent workspace is not configured.", status: 412 },
|
||||
})
|
||||
.handler(async ({ context, input }) => {
|
||||
@@ -76,6 +77,7 @@ export const aiProvidersRouter = {
|
||||
.output(type<AiProviderResponse>())
|
||||
.errors({
|
||||
BAD_REQUEST: { message: "Invalid AI provider configuration.", status: 400 },
|
||||
FORBIDDEN: { message: "AI is managed by the server.", status: 403 },
|
||||
NOT_FOUND: { message: "AI provider was not found.", status: 404 },
|
||||
PRECONDITION_FAILED: { message: "AI agent workspace is not configured.", status: 412 },
|
||||
})
|
||||
@@ -109,6 +111,7 @@ export const aiProvidersRouter = {
|
||||
.input(z.object({ id: z.string() }))
|
||||
.output(z.void())
|
||||
.errors({
|
||||
FORBIDDEN: { message: "AI is managed by the server.", status: 403 },
|
||||
PRECONDITION_FAILED: { message: "AI agent workspace is not configured.", status: 412 },
|
||||
})
|
||||
.handler(({ context, input }) => aiProvidersService.delete({ id: input.id, userId: context.user.id })),
|
||||
@@ -127,6 +130,7 @@ export const aiProvidersRouter = {
|
||||
.use(aiRequestRateLimit)
|
||||
.errors({
|
||||
BAD_REQUEST: { message: "Invalid AI provider configuration.", status: 400 },
|
||||
FORBIDDEN: { message: "AI is managed by the server.", status: 403 },
|
||||
BAD_GATEWAY: { message: "The AI provider returned an error or is unreachable.", status: 502 },
|
||||
NOT_FOUND: { message: "AI provider was not found.", status: 404 },
|
||||
PRECONDITION_FAILED: { message: "AI agent workspace is not configured.", status: 412 },
|
||||
|
||||
@@ -53,8 +53,10 @@ vi.mock("drizzle-orm", () => ({
|
||||
asc: (value: unknown) => ({ type: "asc", value }),
|
||||
desc: (value: unknown) => ({ type: "desc", value }),
|
||||
eq: (left: unknown, right: unknown) => ({ type: "eq", left, right }),
|
||||
ne: (left: unknown, right: unknown) => ({ type: "ne", left, right }),
|
||||
sql: (strings: TemplateStringsArray, ...values: unknown[]) => ({ type: "sql", strings: [...strings], values }),
|
||||
}));
|
||||
vi.mock("@reactive-resume/env/server", () => ({ env: {} }));
|
||||
vi.mock("../ai/credentials", () => ({
|
||||
assertCredentialEncryptionConfigured: vi.fn(),
|
||||
decryptCredential: vi.fn(() => "decrypted-key"),
|
||||
@@ -110,6 +112,7 @@ describe("aiProvidersService", () => {
|
||||
{ type: "eq", left: "ai_provider.user_id", right: "user-1" },
|
||||
{ type: "eq", left: "ai_provider.enabled", right: true },
|
||||
{ type: "eq", left: "ai_provider.test_status", right: "success" },
|
||||
{ type: "ne", left: "ai_provider.id", right: "server-ai:user-1" },
|
||||
],
|
||||
});
|
||||
expect(queryState.orderByArgs).toEqual([
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import type { AIProvider } from "@reactive-resume/ai/types";
|
||||
import { ORPCError } from "@orpc/client";
|
||||
import { and, asc, desc, eq, sql } from "drizzle-orm";
|
||||
import { and, asc, desc, eq, ne, sql } from "drizzle-orm";
|
||||
import { aiProviderSchema } from "@reactive-resume/ai/types";
|
||||
import { db } from "@reactive-resume/db/client";
|
||||
import * as schema from "@reactive-resume/db/schema";
|
||||
import { env } from "@reactive-resume/env/server";
|
||||
import { assertCredentialEncryptionConfigured, decryptCredential, encryptCredential } from "../ai/credentials";
|
||||
import { testConnection } from "../ai/service";
|
||||
import { resolveAiBaseUrl } from "../ai/url-policy";
|
||||
@@ -11,6 +12,7 @@ import { resolveAiBaseUrl } from "../ai/url-policy";
|
||||
type AiProviderRecord = typeof schema.aiProvider.$inferSelect;
|
||||
|
||||
export type AiProviderResponse = {
|
||||
managed: boolean;
|
||||
id: string;
|
||||
label: string;
|
||||
provider: AIProvider;
|
||||
@@ -51,6 +53,7 @@ function toResponse(row: AiProviderRecord): AiProviderResponse {
|
||||
const provider = aiProviderSchema.parse(row.provider);
|
||||
|
||||
return {
|
||||
managed: row.id === serverProviderId(row.userId),
|
||||
id: row.id,
|
||||
label: row.label,
|
||||
provider,
|
||||
@@ -77,6 +80,7 @@ function normalizeBaseUrl(input: { provider: AIProvider; baseURL?: string | null
|
||||
}
|
||||
|
||||
async function getOwnedProvider(input: { id: string; userId: string }) {
|
||||
if (input.id === serverProviderId(input.userId)) throw new ORPCError("NOT_FOUND");
|
||||
const [provider] = await db
|
||||
.select()
|
||||
.from(schema.aiProvider)
|
||||
@@ -88,14 +92,50 @@ async function getOwnedProvider(input: { id: string; userId: string }) {
|
||||
return provider;
|
||||
}
|
||||
|
||||
const serverProviderId = (userId: string) => `server-ai:${userId}`;
|
||||
|
||||
function assertPersonalProvidersAllowed() {
|
||||
if (env.AI_PROVIDER) throw new ORPCError("FORBIDDEN", { message: "AI is managed by the server." });
|
||||
}
|
||||
|
||||
/** A persisted reference keeps Assistant thread foreign keys valid; server credentials stay in the environment. */
|
||||
async function serverProvider(userId: string) {
|
||||
if (!env.AI_PROVIDER || !env.AI_MODEL) return null;
|
||||
const values = {
|
||||
userId,
|
||||
label: "Server AI",
|
||||
provider: env.AI_PROVIDER,
|
||||
model: env.AI_MODEL,
|
||||
baseUrl: env.AI_BASE_URL ?? null,
|
||||
enabled: true,
|
||||
testStatus: "success",
|
||||
encryptedApiKey: "",
|
||||
apiKeySalt: "",
|
||||
apiKeyHash: "",
|
||||
apiKeyPreview: "",
|
||||
};
|
||||
const [row] = await db
|
||||
.insert(schema.aiProvider)
|
||||
.values({ id: serverProviderId(userId), ...values })
|
||||
.onConflictDoUpdate({ target: schema.aiProvider.id, set: values })
|
||||
.returning();
|
||||
if (!row) throw new Error("SERVER_AI_PROVIDER_UNAVAILABLE");
|
||||
return { ...toResponse(row), apiKey: env.AI_API_KEY ?? "", baseURL: env.AI_BASE_URL ?? "" };
|
||||
}
|
||||
|
||||
export const aiProvidersService = {
|
||||
list: async (input: { userId: string }) => {
|
||||
const global = await serverProvider(input.userId);
|
||||
if (global) {
|
||||
const { apiKey: _apiKey, ...response } = global;
|
||||
return [response];
|
||||
}
|
||||
assertCredentialEncryptionConfigured();
|
||||
|
||||
const providers = await db
|
||||
.select()
|
||||
.from(schema.aiProvider)
|
||||
.where(eq(schema.aiProvider.userId, input.userId))
|
||||
.where(and(eq(schema.aiProvider.userId, input.userId), ne(schema.aiProvider.id, serverProviderId(input.userId))))
|
||||
.orderBy(
|
||||
desc(sql<Date>`coalesce(${schema.aiProvider.lastUsedAt}, '1970-01-01T00:00:00.000Z'::timestamptz)`),
|
||||
asc(schema.aiProvider.createdAt),
|
||||
@@ -105,6 +145,8 @@ export const aiProvidersService = {
|
||||
},
|
||||
|
||||
getRunnableById: async (input: { id: string; userId: string }) => {
|
||||
const global = await serverProvider(input.userId);
|
||||
if (global) return global;
|
||||
assertCredentialEncryptionConfigured();
|
||||
|
||||
const provider = await getOwnedProvider(input);
|
||||
@@ -120,6 +162,8 @@ export const aiProvidersService = {
|
||||
},
|
||||
|
||||
getDefaultRunnable: async (input: { userId: string }) => {
|
||||
const global = await serverProvider(input.userId);
|
||||
if (global) return global;
|
||||
assertCredentialEncryptionConfigured();
|
||||
|
||||
const [provider] = await db
|
||||
@@ -130,6 +174,7 @@ export const aiProvidersService = {
|
||||
eq(schema.aiProvider.userId, input.userId),
|
||||
eq(schema.aiProvider.enabled, true),
|
||||
eq(schema.aiProvider.testStatus, "success"),
|
||||
ne(schema.aiProvider.id, serverProviderId(input.userId)),
|
||||
),
|
||||
)
|
||||
.orderBy(
|
||||
@@ -148,6 +193,7 @@ export const aiProvidersService = {
|
||||
},
|
||||
|
||||
create: async (input: CreateAiProviderInput) => {
|
||||
assertPersonalProvidersAllowed();
|
||||
assertCredentialEncryptionConfigured();
|
||||
|
||||
const encrypted = encryptCredential(input.apiKey.trim());
|
||||
@@ -169,6 +215,7 @@ export const aiProvidersService = {
|
||||
},
|
||||
|
||||
update: async (input: UpdateAiProviderInput) => {
|
||||
assertPersonalProvidersAllowed();
|
||||
assertCredentialEncryptionConfigured();
|
||||
|
||||
const existing = await getOwnedProvider(input);
|
||||
@@ -206,6 +253,7 @@ export const aiProvidersService = {
|
||||
},
|
||||
|
||||
delete: async (input: { id: string; userId: string }) => {
|
||||
assertPersonalProvidersAllowed();
|
||||
assertCredentialEncryptionConfigured();
|
||||
|
||||
await db
|
||||
@@ -214,6 +262,7 @@ export const aiProvidersService = {
|
||||
},
|
||||
|
||||
test: async (input: { id: string; userId: string }) => {
|
||||
assertPersonalProvidersAllowed();
|
||||
assertCredentialEncryptionConfigured();
|
||||
|
||||
const provider = await getOwnedProvider(input);
|
||||
|
||||
@@ -88,8 +88,8 @@ export function assertCredentialEncryptionConfigured() {
|
||||
if (!isCredentialEncryptionConfigured()) throw new Error("AI_CREDENTIAL_ENCRYPTION_UNAVAILABLE");
|
||||
}
|
||||
|
||||
// The assistant needs stored keys, so ENCRYPTION_SECRET. Redis is optional: with it, replies survive a reload
|
||||
// Personal keys need ENCRYPTION_SECRET; server AI uses environment credentials. Redis is optional: replies survive a reload
|
||||
// and Stop reaches a run on another server; without it, both work within one server.
|
||||
export function assertAgentEnvironment() {
|
||||
if (!isCredentialEncryptionConfigured()) throw new Error("AGENT_ENVIRONMENT_UNAVAILABLE");
|
||||
if (!isCredentialEncryptionConfigured() && !env.AI_PROVIDER) throw new Error("AGENT_ENVIRONMENT_UNAVAILABLE");
|
||||
}
|
||||
|
||||
@@ -9,14 +9,15 @@ import { generateJson as sharedGenerateJson } from "../ai/generate-json";
|
||||
import { getModel } from "../ai/service";
|
||||
import { aiProvidersService } from "../ai-providers/service";
|
||||
import { coverLetterService } from "../cover-letters/service";
|
||||
import { firecrawlService } from "../firecrawl/service";
|
||||
import { resumeService } from "../resume/service";
|
||||
import {
|
||||
fetchPostingPage,
|
||||
htmlToText,
|
||||
fetchJobPosting,
|
||||
isPostingLink,
|
||||
MAX_POSTING_CHARS,
|
||||
PostingFetchError,
|
||||
readJobPosting,
|
||||
postingSearchResult,
|
||||
searchJobPostings,
|
||||
} from "./posting";
|
||||
import { applicationService } from "./service";
|
||||
|
||||
@@ -151,6 +152,34 @@ const aiErrors = {
|
||||
};
|
||||
|
||||
export const aiRouter = {
|
||||
searchPostings: protectedProcedure
|
||||
.route({
|
||||
method: "POST",
|
||||
path: "/applications/ai/search-postings",
|
||||
operationId: "searchApplicationPostings",
|
||||
summary: "Search job postings",
|
||||
description:
|
||||
"Searches the web for job postings using server-configured Firecrawl or the user's Firecrawl Cloud key. Returns up to five public https links to review and import. Requires authentication.",
|
||||
...reserved,
|
||||
})
|
||||
.input(z.object({ query: z.string().trim().min(2).max(500) }))
|
||||
.use(aiRequestRateLimit)
|
||||
.output(z.array(postingSearchResult))
|
||||
.errors({
|
||||
SEARCH_UNAVAILABLE: { message: "Job search isn't configured on this server.", status: 503 },
|
||||
SEARCH_FAILED: { message: "Job search couldn't be reached. Try again or paste a posting link.", status: 502 },
|
||||
})
|
||||
.handler(async ({ context, input }) => {
|
||||
const config = await firecrawlService.resolve(context.user.id);
|
||||
if (!config) throw new ORPCError("SEARCH_UNAVAILABLE", { status: 503 });
|
||||
try {
|
||||
return await searchJobPostings(input.query, config);
|
||||
} catch {
|
||||
// Provider errors can include request credentials or echo them in their response.
|
||||
throw new ORPCError("SEARCH_FAILED", { status: 502 });
|
||||
}
|
||||
}),
|
||||
|
||||
// Reads a pasted link or posting into an application's fields. A link is fetched on the server (public https
|
||||
// pages only); the page's own job data fills what it can, and an AI provider, when one is set up, reads the rest.
|
||||
parsePosting: protectedProcedure
|
||||
@@ -173,13 +202,11 @@ export const aiRouter = {
|
||||
.handler(async ({ context, input }) => {
|
||||
const link = isPostingLink(input.input) ? input.input.trim() : null;
|
||||
let text = input.input;
|
||||
let page: ReturnType<typeof readJobPosting> = null;
|
||||
let page: Awaited<ReturnType<typeof fetchJobPosting>>["page"] = null;
|
||||
|
||||
if (link) {
|
||||
try {
|
||||
const html = await fetchPostingPage(link);
|
||||
page = readJobPosting(html);
|
||||
text = page?.description || htmlToText(html);
|
||||
({ page, text } = await fetchJobPosting(link, await firecrawlService.resolve(context.user.id)));
|
||||
} catch (error) {
|
||||
if (error instanceof PostingFetchError)
|
||||
throw new ORPCError("POSTING_UNREADABLE", { status: 422, cause: error });
|
||||
|
||||
@@ -1,5 +1,34 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { allPublic, assertPublicPageUrl, htmlToText, readJobPosting } from "./posting";
|
||||
import { lookup } from "node:dns/promises";
|
||||
import { EventEmitter } from "node:events";
|
||||
import { createServer } from "node:http";
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
allPublic,
|
||||
assertPublicPageUrl,
|
||||
fetchJobPosting,
|
||||
htmlToText,
|
||||
readJobPosting,
|
||||
searchJobPostings,
|
||||
} from "./posting";
|
||||
|
||||
const config = { apiUrl: "", apiKey: "" };
|
||||
vi.mock("node:dns/promises", () => ({ lookup: vi.fn() }));
|
||||
vi.mock("node:https", () => ({
|
||||
request: (_url: unknown, _options: unknown, callback: (response: EventEmitter) => void) => {
|
||||
const request = new EventEmitter();
|
||||
return Object.assign(request, {
|
||||
end: () => {
|
||||
const response = Object.assign(new EventEmitter(), {
|
||||
statusCode: 200,
|
||||
headers: { "content-type": "text/html" },
|
||||
});
|
||||
callback(response);
|
||||
response.emit("data", Buffer.from("<h1>Direct posting</h1><p>Work on accessible products.</p>"));
|
||||
response.emit("end");
|
||||
},
|
||||
});
|
||||
},
|
||||
}));
|
||||
|
||||
describe("posting links", () => {
|
||||
it("accepts only public https pages", () => {
|
||||
@@ -55,3 +84,122 @@ describe("readJobPosting", () => {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("optional Firecrawl posting reader", () => {
|
||||
let endpoint = "";
|
||||
let status = 200;
|
||||
let response: unknown;
|
||||
const requests: { path: string; authorization: string | undefined; body: Record<string, unknown> }[] = [];
|
||||
const server = createServer(async (request, result) => {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of request) chunks.push(chunk);
|
||||
requests.push({
|
||||
path: request.url ?? "",
|
||||
authorization: request.headers.authorization,
|
||||
body: JSON.parse(Buffer.concat(chunks).toString("utf8")),
|
||||
});
|
||||
result.writeHead(status, { "content-type": "application/json" });
|
||||
result.end(JSON.stringify(response));
|
||||
});
|
||||
beforeAll(async () => {
|
||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") throw new Error("No test server address");
|
||||
endpoint = `http://127.0.0.1:${address.port}`;
|
||||
});
|
||||
afterAll(async () => {
|
||||
await new Promise<void>((resolve, reject) => server.close((error) => (error ? reject(error) : resolve())));
|
||||
});
|
||||
beforeEach(() => {
|
||||
config.apiUrl = endpoint;
|
||||
config.apiKey = "test-firecrawl-key";
|
||||
vi.mocked(lookup).mockResolvedValue([{ address: "93.184.216.34", family: 4 }] as never);
|
||||
requests.length = 0;
|
||||
status = 200;
|
||||
response = { success: true, data: { markdown: "# Rendered role\n\nReact and accessibility" } };
|
||||
});
|
||||
|
||||
it("reads rendered Markdown through the v2 API and caps saved text", async () => {
|
||||
response = { success: true, data: { markdown: `# Rendered role\n${"x".repeat(25_000)}` } };
|
||||
const posting = await fetchJobPosting("https://jobs.example.com/role#apply", config);
|
||||
expect(posting.page).toBeNull();
|
||||
expect(posting.text).toHaveLength(20_000);
|
||||
expect(posting.text).toMatch(/^# Rendered role/);
|
||||
expect(requests[0]).toMatchObject({
|
||||
path: "/v2/scrape",
|
||||
authorization: "Bearer test-firecrawl-key",
|
||||
body: { url: "https://jobs.example.com/role", formats: ["markdown", "rawHtml"], skipTlsVerification: false },
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps page fields available without an AI provider and accepts a keyless local service", async () => {
|
||||
config.apiKey = "";
|
||||
response = {
|
||||
success: true,
|
||||
data: {
|
||||
markdown: "Navigation and other content",
|
||||
rawHtml: `<script type="application/ld+json">${JSON.stringify({
|
||||
"@type": "JobPosting",
|
||||
title: "Designer",
|
||||
hiringOrganization: { name: "Example" },
|
||||
description: "<p>Design accessible products.</p>",
|
||||
})}</script>`,
|
||||
},
|
||||
};
|
||||
expect(await fetchJobPosting("https://jobs.example.com/role", config)).toMatchObject({
|
||||
page: { role: "Designer", company: "Example" },
|
||||
text: "Design accessible products.",
|
||||
});
|
||||
expect(requests[0]?.authorization).toBeUndefined();
|
||||
});
|
||||
|
||||
it.each(["disabled", "http-error", "empty", "malformed", "too-large"])(
|
||||
"uses the direct reader when Firecrawl is %s",
|
||||
async (failure) => {
|
||||
if (failure === "disabled") {
|
||||
config.apiUrl = "";
|
||||
config.apiKey = "";
|
||||
}
|
||||
if (failure === "http-error") status = 503;
|
||||
if (failure === "empty") response = { success: true, data: { markdown: " " } };
|
||||
if (failure === "malformed") response = { success: false };
|
||||
if (failure === "too-large") response = { success: true, data: { markdown: "x".repeat(2_000_001) } };
|
||||
expect(await fetchJobPosting("https://jobs.example.com/role", failure === "disabled" ? null : config)).toEqual({
|
||||
page: null,
|
||||
text: "Direct posting\nWork on accessible products.",
|
||||
});
|
||||
if (failure === "disabled") expect(requests).toHaveLength(0);
|
||||
},
|
||||
);
|
||||
|
||||
it("blocks private targets and DNS answers before sending anything to Firecrawl", async () => {
|
||||
await expect(fetchJobPosting("https://127.0.0.1/private", config)).rejects.toMatchObject({ reason: "unsafe-url" });
|
||||
vi.mocked(lookup).mockResolvedValue([{ address: "10.0.0.1", family: 4 }] as never);
|
||||
await expect(fetchJobPosting("https://jobs.example.com/private", config)).rejects.toMatchObject({
|
||||
reason: "unsafe-url",
|
||||
});
|
||||
expect(requests).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("returns public posting links from v2 search, dropping unsafe and malformed results", async () => {
|
||||
response = {
|
||||
success: true,
|
||||
data: {
|
||||
web: [
|
||||
{ url: "https://jobs.example.com/designer", title: "Designer", description: "Berlin" },
|
||||
{ url: "https://localhost/private", title: "Internal" },
|
||||
{ url: "javascript:alert(1)", title: "Unsafe" },
|
||||
{ url: "http://example.com/insecure", title: "Insecure" },
|
||||
{ url: "https://jobs.example.com/other" },
|
||||
],
|
||||
},
|
||||
};
|
||||
expect(await searchJobPostings("designer Berlin", config)).toEqual([
|
||||
{ url: "https://jobs.example.com/designer", title: "Designer", description: "Berlin" },
|
||||
]);
|
||||
expect(requests[0]).toMatchObject({
|
||||
path: "/v2/search",
|
||||
body: { query: "designer Berlin job posting", limit: 5, sources: ["web"] },
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
import type { LookupAddress } from "node:dns";
|
||||
import type { LookupFunction } from "node:net";
|
||||
import type { FirecrawlConfig } from "../firecrawl/service";
|
||||
import { lookup } from "node:dns";
|
||||
import { lookup as lookupAddresses } from "node:dns/promises";
|
||||
import { request } from "node:https";
|
||||
import { Firecrawl } from "firecrawl";
|
||||
import sanitizeHtml from "sanitize-html";
|
||||
import { z } from "zod";
|
||||
import { isPrivateOrLoopbackHost, parseUrl } from "@reactive-resume/utils/url-security.node";
|
||||
|
||||
/** Matches the applications feature's cap on a saved posting. */
|
||||
@@ -104,6 +108,73 @@ export function fetchPostingPage(input: string, redirects = 0): Promise<string>
|
||||
});
|
||||
}
|
||||
|
||||
const firecrawl = (config: FirecrawlConfig) => new Firecrawl({ ...config, timeoutMs: 65_000, maxRetries: 1 });
|
||||
|
||||
const scrapeResponse = z.object({
|
||||
markdown: z.string().max(MAX_PAGE_BYTES).optional(),
|
||||
rawHtml: z.string().max(MAX_PAGE_BYTES).optional(),
|
||||
metadata: z.object({ statusCode: z.number().optional() }).optional(),
|
||||
});
|
||||
|
||||
/** Firecrawl renders dynamic pages; the built-in reader remains the fallback when it is unavailable. */
|
||||
export async function fetchJobPosting(
|
||||
input: string,
|
||||
config: FirecrawlConfig | null,
|
||||
): Promise<{ page: PagePosting | null; text: string }> {
|
||||
const url = assertPublicPageUrl(input);
|
||||
if (config) {
|
||||
// Preflight before delegating. Firecrawl must also enforce public destinations on redirects and at connect time.
|
||||
const addresses = await lookupAddresses(url.hostname, { all: true }).catch(() => []);
|
||||
if (!allPublic(addresses)) throw new PostingFetchError("unsafe-url");
|
||||
try {
|
||||
const data = scrapeResponse.parse(
|
||||
await firecrawl(config).scrape(url.toString(), {
|
||||
formats: ["markdown", "rawHtml"],
|
||||
onlyMainContent: true,
|
||||
skipTlsVerification: false,
|
||||
timeout: 60_000,
|
||||
autoResume: false,
|
||||
}),
|
||||
);
|
||||
if (data.metadata?.statusCode && data.metadata.statusCode >= 400) throw new PostingFetchError("unreachable");
|
||||
const page = readJobPosting(data.rawHtml ?? "");
|
||||
const text = page?.description || data.markdown?.trim() || htmlToText(data.rawHtml ?? "");
|
||||
if (text) return { page, text: text.slice(0, MAX_POSTING_CHARS) };
|
||||
} catch {
|
||||
// Preserve URL import when Firecrawl is down, rate limited, or cannot read this page.
|
||||
}
|
||||
}
|
||||
const html = await fetchPostingPage(url.toString());
|
||||
const page = readJobPosting(html);
|
||||
return { page, text: (page?.description || htmlToText(html)).slice(0, MAX_POSTING_CHARS) };
|
||||
}
|
||||
|
||||
export const postingSearchResult = z.object({
|
||||
url: z.string(),
|
||||
title: z.string().max(1_000),
|
||||
description: z.string().max(5_000).default(""),
|
||||
});
|
||||
|
||||
/** Five web results for an explicit job query; no postings are scraped until the user selects one. */
|
||||
export async function searchJobPostings(query: string, config: FirecrawlConfig) {
|
||||
const response = await firecrawl(config).search(`${query} job posting`, {
|
||||
sources: ["web"],
|
||||
limit: 5,
|
||||
timeout: 60_000,
|
||||
});
|
||||
return (response.web ?? [])
|
||||
.flatMap((item) => {
|
||||
const result = postingSearchResult.safeParse(item);
|
||||
if (!result.success) return [];
|
||||
try {
|
||||
return [{ ...result.data, url: assertPublicPageUrl(result.data.url).toString() }];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
})
|
||||
.slice(0, 5);
|
||||
}
|
||||
|
||||
const ENTITIES: Record<string, string> = { amp: "&", lt: "<", gt: ">", quot: '"', apos: "'", nbsp: " " };
|
||||
|
||||
const decodeEntities = (text: string) =>
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import z from "zod";
|
||||
import { protectedProcedure } from "../../context";
|
||||
import { firecrawlService } from "./service";
|
||||
|
||||
const route = { tags: ["Integrations"] };
|
||||
const errors = {
|
||||
FORBIDDEN: { message: "Firecrawl is managed by the server.", status: 403 },
|
||||
PRECONDITION_FAILED: { message: "Credential encryption is not configured.", status: 412 },
|
||||
};
|
||||
|
||||
export const firecrawlRouter = {
|
||||
status: protectedProcedure
|
||||
.route({
|
||||
...route,
|
||||
method: "GET",
|
||||
path: "/integrations/firecrawl",
|
||||
operationId: "getFirecrawlStatus",
|
||||
summary: "Get Firecrawl availability",
|
||||
})
|
||||
.output(z.object({ managed: z.boolean(), configured: z.boolean(), canSave: z.boolean() }))
|
||||
.handler(({ context }) => firecrawlService.status(context.user.id)),
|
||||
save: protectedProcedure
|
||||
.route({
|
||||
...route,
|
||||
method: "PUT",
|
||||
path: "/integrations/firecrawl",
|
||||
operationId: "saveFirecrawlKey",
|
||||
summary: "Save a personal Firecrawl Cloud key",
|
||||
})
|
||||
.input(z.object({ apiKey: z.string().trim().min(1).max(2_000) }))
|
||||
.errors(errors)
|
||||
.output(z.void())
|
||||
.handler(({ context, input }) => firecrawlService.save(context.user.id, input.apiKey)),
|
||||
delete: protectedProcedure
|
||||
.route({
|
||||
...route,
|
||||
method: "DELETE",
|
||||
path: "/integrations/firecrawl",
|
||||
operationId: "deleteFirecrawlKey",
|
||||
summary: "Delete a personal Firecrawl Cloud key",
|
||||
})
|
||||
.errors(errors)
|
||||
.output(z.void())
|
||||
.handler(({ context }) => firecrawlService.delete(context.user.id)),
|
||||
};
|
||||
@@ -0,0 +1,147 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { drizzle } from "drizzle-orm/node-postgres";
|
||||
import { Pool } from "pg";
|
||||
|
||||
const fixture = vi.hoisted(() => ({ db: undefined as ReturnType<typeof drizzle> | undefined }));
|
||||
const env = vi.hoisted(() => ({
|
||||
ENCRYPTION_SECRET: "integration-test-encryption-secret-32-chars",
|
||||
FIRECRAWL_API_URL: "",
|
||||
FIRECRAWL_API_KEY: "",
|
||||
AI_PROVIDER: "",
|
||||
AI_MODEL: "",
|
||||
AI_API_KEY: "",
|
||||
AI_BASE_URL: "",
|
||||
}));
|
||||
vi.mock("@reactive-resume/env/server", () => ({ env }));
|
||||
vi.mock("@reactive-resume/db/client", () => ({
|
||||
get db() {
|
||||
return fixture.db;
|
||||
},
|
||||
}));
|
||||
|
||||
// Opt in with a disposable PostgreSQL database. Each run owns and removes a separate schema.
|
||||
describe.skipIf(!process.env.INTEGRATIONS_TEST_DATABASE_URL)("integration credentials and server precedence", () => {
|
||||
let firecrawl: typeof import("./service").firecrawlService;
|
||||
let ai: typeof import("../ai-providers/service").aiProvidersService;
|
||||
let pool: Pool;
|
||||
let admin: Pool;
|
||||
const schemaName = `integrations_test_${randomUUID().replaceAll("-", "")}`;
|
||||
beforeAll(async () => {
|
||||
admin = new Pool({ connectionString: process.env.INTEGRATIONS_TEST_DATABASE_URL });
|
||||
await admin.query(`CREATE SCHEMA ${schemaName}`);
|
||||
pool = new Pool({
|
||||
connectionString: process.env.INTEGRATIONS_TEST_DATABASE_URL,
|
||||
options: `-c search_path=${schemaName}`,
|
||||
});
|
||||
fixture.db = drizzle({ client: pool });
|
||||
await pool.query('CREATE TABLE "user" (id text PRIMARY KEY); CREATE TABLE resume (id text PRIMARY KEY)');
|
||||
for (const name of ["20260513181752_bent_human_cannonball", "20260930140759_perpetual_drax"]) {
|
||||
const sql = await readFile(new URL(`../../../../../migrations/${name}/migration.sql`, import.meta.url), "utf8");
|
||||
await pool.query(sql.replaceAll('"public".', ""));
|
||||
}
|
||||
firecrawl = (await import("./service")).firecrawlService;
|
||||
ai = (await import("../ai-providers/service")).aiProvidersService;
|
||||
});
|
||||
afterAll(async () => {
|
||||
await pool?.end();
|
||||
await admin?.query(`DROP SCHEMA IF EXISTS ${schemaName} CASCADE`);
|
||||
await admin?.end();
|
||||
});
|
||||
beforeEach(async () => {
|
||||
Object.assign(env, {
|
||||
ENCRYPTION_SECRET: "integration-test-encryption-secret-32-chars",
|
||||
FIRECRAWL_API_URL: "",
|
||||
FIRECRAWL_API_KEY: "",
|
||||
AI_PROVIDER: "",
|
||||
AI_MODEL: "",
|
||||
AI_API_KEY: "",
|
||||
AI_BASE_URL: "",
|
||||
});
|
||||
await pool.query("TRUNCATE \"user\" CASCADE; INSERT INTO \"user\" VALUES ('alice'),('bob')");
|
||||
});
|
||||
|
||||
it("encrypts personal Cloud keys, isolates accounts, and locks personal writes under server configuration", async () => {
|
||||
expect(await firecrawl.resolve("alice")).toBeNull();
|
||||
await firecrawl.save("alice", "fc-alice-secret");
|
||||
await firecrawl.save("bob", "fc-bob-secret");
|
||||
expect(await firecrawl.resolve("alice")).toEqual({
|
||||
apiUrl: "https://api.firecrawl.dev",
|
||||
apiKey: "fc-alice-secret",
|
||||
});
|
||||
expect(await firecrawl.resolve("bob")).toEqual({ apiUrl: "https://api.firecrawl.dev", apiKey: "fc-bob-secret" });
|
||||
const saved = (await pool.query("SELECT encrypted_api_key FROM firecrawl_credentials")).rows;
|
||||
expect(JSON.stringify(saved)).not.toContain("fc-alice-secret");
|
||||
expect(JSON.stringify(saved)).not.toContain("fc-bob-secret");
|
||||
await firecrawl.delete("alice");
|
||||
expect(await firecrawl.resolve("alice")).toBeNull();
|
||||
expect(await firecrawl.status("bob")).toEqual({ managed: false, configured: true, canSave: true });
|
||||
|
||||
env.FIRECRAWL_API_URL = "http://firecrawl:3002";
|
||||
env.FIRECRAWL_API_KEY = "server-key";
|
||||
expect(await firecrawl.resolve("bob")).toEqual({ apiUrl: "http://firecrawl:3002", apiKey: "server-key" });
|
||||
expect(await firecrawl.status("alice")).toEqual({ managed: true, configured: true, canSave: false });
|
||||
await expect(firecrawl.save("bob", "override")).rejects.toMatchObject({ code: "FORBIDDEN" });
|
||||
await expect(firecrawl.delete("bob")).rejects.toMatchObject({ code: "FORBIDDEN" });
|
||||
env.FIRECRAWL_API_KEY = "";
|
||||
env.ENCRYPTION_SECRET = "";
|
||||
expect(await firecrawl.resolve("alice")).toEqual({ apiUrl: "http://firecrawl:3002", apiKey: "" });
|
||||
env.FIRECRAWL_API_URL = "";
|
||||
expect(await firecrawl.status("alice")).toEqual({ managed: false, configured: false, canSave: false });
|
||||
await expect(firecrawl.save("alice", "secret")).rejects.toMatchObject({ code: "PRECONDITION_FAILED" });
|
||||
});
|
||||
|
||||
it("routes AI through server credentials, keeps thread references valid, and restores personal providers when disabled", async () => {
|
||||
const personal = await ai.create({
|
||||
userId: "alice",
|
||||
label: "Personal",
|
||||
provider: "openai",
|
||||
model: "personal-model",
|
||||
apiKey: "personal-key",
|
||||
});
|
||||
await pool.query("UPDATE ai_providers SET enabled=true, test_status='success' WHERE id=$1", [personal.id]);
|
||||
Object.assign(env, {
|
||||
AI_PROVIDER: "openai",
|
||||
AI_MODEL: "server-model",
|
||||
AI_API_KEY: "server-secret",
|
||||
AI_BASE_URL: "https://ai.example.com/v1",
|
||||
ENCRYPTION_SECRET: "",
|
||||
});
|
||||
const global = await ai.getRunnableById({ userId: "alice", id: personal.id });
|
||||
expect(global).toMatchObject({
|
||||
managed: true,
|
||||
provider: "openai",
|
||||
model: "server-model",
|
||||
apiKey: "server-secret",
|
||||
baseURL: "https://ai.example.com/v1",
|
||||
});
|
||||
expect((await ai.getDefaultRunnable({ userId: "alice" }))?.id).toBe(global.id);
|
||||
const listed = await ai.list({ userId: "alice" });
|
||||
expect(listed).toHaveLength(1);
|
||||
expect(listed[0]).toMatchObject({ managed: true, enabled: true });
|
||||
expect(JSON.stringify(listed)).not.toContain("server-secret");
|
||||
expect(JSON.stringify((await pool.query("SELECT * FROM ai_providers")).rows)).not.toContain("server-secret");
|
||||
await pool.query(
|
||||
"INSERT INTO agent_threads (id,user_id,ai_provider_id,title) VALUES ('thread','alice',$1,'Server AI')",
|
||||
[global.id],
|
||||
);
|
||||
for (const action of [
|
||||
() => ai.create({ userId: "alice", label: "Override", provider: "openai", model: "other", apiKey: "other" }),
|
||||
() => ai.update({ userId: "alice", id: personal.id, apiKey: "override" }),
|
||||
() => ai.test({ userId: "alice", id: personal.id }),
|
||||
() => ai.delete({ userId: "alice", id: personal.id }),
|
||||
])
|
||||
await expect(action()).rejects.toMatchObject({ code: "FORBIDDEN" });
|
||||
Object.assign(env, {
|
||||
AI_PROVIDER: "",
|
||||
AI_MODEL: "",
|
||||
AI_API_KEY: "",
|
||||
AI_BASE_URL: "",
|
||||
ENCRYPTION_SECRET: "integration-test-encryption-secret-32-chars",
|
||||
});
|
||||
expect(await ai.list({ userId: "alice" })).toMatchObject([{ id: personal.id, managed: false }]);
|
||||
expect((await ai.getDefaultRunnable({ userId: "alice" }))?.apiKey).toBe("personal-key");
|
||||
await expect(ai.getRunnableById({ userId: "alice", id: global.id })).rejects.toMatchObject({ code: "NOT_FOUND" });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,55 @@
|
||||
import { ORPCError } from "@orpc/client";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { db } from "@reactive-resume/db/client";
|
||||
import { firecrawlCredential } from "@reactive-resume/db/schema";
|
||||
import { env } from "@reactive-resume/env/server";
|
||||
import { decryptCredential, encryptCredential } from "../ai/credentials";
|
||||
|
||||
export type FirecrawlConfig = { apiUrl: string; apiKey: string };
|
||||
|
||||
const serverConfig = (): FirecrawlConfig | null =>
|
||||
env.FIRECRAWL_API_URL || env.FIRECRAWL_API_KEY
|
||||
? { apiUrl: env.FIRECRAWL_API_URL || "https://api.firecrawl.dev", apiKey: env.FIRECRAWL_API_KEY || "" }
|
||||
: null;
|
||||
|
||||
async function savedCredential(userId: string) {
|
||||
const [credential] = await db
|
||||
.select()
|
||||
.from(firecrawlCredential)
|
||||
.where(eq(firecrawlCredential.userId, userId))
|
||||
.limit(1);
|
||||
return credential;
|
||||
}
|
||||
|
||||
function assertPersonalKeysAllowed() {
|
||||
if (serverConfig()) throw new ORPCError("FORBIDDEN", { message: "Firecrawl is managed by the server." });
|
||||
if (!env.ENCRYPTION_SECRET)
|
||||
throw new ORPCError("PRECONDITION_FAILED", { message: "Credential encryption is not configured." });
|
||||
}
|
||||
|
||||
export const firecrawlService = {
|
||||
status: async (userId: string) => {
|
||||
const managed = !!serverConfig();
|
||||
const configured = managed || (!!env.ENCRYPTION_SECRET && !!(await savedCredential(userId)));
|
||||
return { managed, configured, canSave: !managed && !!env.ENCRYPTION_SECRET };
|
||||
},
|
||||
resolve: async (userId: string): Promise<FirecrawlConfig | null> => {
|
||||
const global = serverConfig();
|
||||
if (global) return global;
|
||||
if (!env.ENCRYPTION_SECRET) return null;
|
||||
const saved = await savedCredential(userId);
|
||||
return saved ? { apiUrl: "https://api.firecrawl.dev", apiKey: decryptCredential(saved.encryptedApiKey) } : null;
|
||||
},
|
||||
save: async (userId: string, apiKey: string) => {
|
||||
assertPersonalKeysAllowed();
|
||||
const { encryptedApiKey } = encryptCredential(apiKey.trim());
|
||||
await db.insert(firecrawlCredential).values({ userId, encryptedApiKey }).onConflictDoUpdate({
|
||||
target: firecrawlCredential.userId,
|
||||
set: { encryptedApiKey },
|
||||
});
|
||||
},
|
||||
delete: async (userId: string) => {
|
||||
assertPersonalKeysAllowed();
|
||||
await db.delete(firecrawlCredential).where(eq(firecrawlCredential.userId, userId));
|
||||
},
|
||||
};
|
||||
@@ -5,6 +5,7 @@ import { applicationsRouter } from "../features/applications/router";
|
||||
import { authRouter } from "../features/auth/router";
|
||||
import { coverLettersRouter } from "../features/cover-letters/router";
|
||||
import { documentsRouter } from "../features/documents/router";
|
||||
import { firecrawlRouter } from "../features/firecrawl/router";
|
||||
import { flagsRouter } from "../features/flags/router";
|
||||
import { resumeRouter } from "../features/resume/router";
|
||||
import { statisticsRouter } from "../features/statistics/router";
|
||||
@@ -19,6 +20,7 @@ export default {
|
||||
coverLetters: coverLettersRouter,
|
||||
documents: documentsRouter,
|
||||
flags: flagsRouter,
|
||||
firecrawl: firecrawlRouter,
|
||||
resume: resumeRouter,
|
||||
statistics: statisticsRouter,
|
||||
storage: storageRouter,
|
||||
|
||||
Reference in New Issue
Block a user