fix(server): verify migrated schema at startup (#3513)

Co-authored-by: Amruth Pillai <im.amruth@gmail.com>

.env.example

@@ -18,6 +18,11 @@ APP_URL="http://localhost:3000"
 # when running directly on your machine, `localhost` is typical.
 DATABASE_URL="postgresql://postgres:postgres@postgres:5432/postgres"
 
+# When "true", the server refuses to boot if the live database schema has drifted from
+# the migration ledger (e.g. a table dropped outside migrations). Default "false" logs
+# the drift loudly at startup and continues.
+STRICT_SCHEMA_CHECK="false"
+
 # --- Authentication ---
 # Generated using `openssl rand -hex 32`
 AUTH_SECRET="change-me-to-a-secure-secret-key-in-production"

apps/server/src/startup/checks.ts

@@ -7,6 +7,7 @@ import { migrate } from "drizzle-orm/node-postgres/migrator";
 import { Pool } from "pg";
 import { env } from "@reactive-resume/env/server";
 import { getLocalDataDirectory } from "@reactive-resume/utils/monorepo.node";
+import { verifyMigratedSchema } from "./schema-check";
 
 function resolveFromCurrentModule(relativePath: string) {
 	return fileURLToPath(new URL(relativePath, import.meta.url));
@@ -31,11 +32,26 @@ async function runDatabaseMigrations() {
 	const db = drizzle({ client: pool });
 
 	try {
-		await migrate(db, { migrationsFolder: resolveWorkspaceFolder("migrations") });
-		console.info("Database migrations completed");
-	} catch (error) {
-		console.error("Database migrations failed", { error });
-		throw error;
+		try {
+			await migrate(db, { migrationsFolder: resolveWorkspaceFolder("migrations") });
+			console.info("Database migrations completed");
+		} catch (error) {
+			console.error("Database migrations failed", { error });
+			throw error;
+		}
+
+		// Post-migration verification is not a migration failure, so it gets its own log
+		// message. A drifted schema still lets the server boot; STRICT_SCHEMA_CHECK=true
+		// makes the drift fatal instead.
+		try {
+			await verifyMigratedSchema(pool);
+		} catch (error) {
+			console.error("Database schema verification failed", { error });
+			if (env.STRICT_SCHEMA_CHECK) throw error;
+			console.error(
+				"Continuing with a drifted database schema; set STRICT_SCHEMA_CHECK=true to refuse startup instead.",
+			);
+		}
 	} finally {
 		await pool.end();
 	}

apps/server/src/startup/schema-check.test.ts

@@ -0,0 +1,50 @@
+import { describe, expect, it } from "vitest";
+import { collectExpectedColumns, verifyMigratedSchema } from "./schema-check";
+
+describe("collectExpectedColumns", () => {
+	it("collects every column of every schema table", () => {
+		const expected = collectExpectedColumns();
+		expect(expected.length).toBeGreaterThan(0);
+		expect(expected).toContainEqual({ tableName: "ai_providers", columnName: "user_id" });
+		expect(expected).toContainEqual({ tableName: "user", columnName: "id" });
+	});
+});
+
+describe("verifyMigratedSchema", () => {
+	it("passes when the catalog reports nothing missing", async () => {
+		const queryable = { query: async () => ({ rows: [] }) };
+		await expect(verifyMigratedSchema(queryable)).resolves.toBeUndefined();
+	});
+
+	it("fails with the table name when every column of a table is missing", async () => {
+		const rows = collectExpectedColumns()
+			.filter((e) => e.tableName === "ai_providers")
+			.map((e) => ({ table_name: e.tableName, column_name: e.columnName }));
+
+		const queryable = { query: async () => ({ rows }) };
+		await expect(verifyMigratedSchema(queryable)).rejects.toThrow('table "ai_providers"');
+	});
+
+	it("fails with the qualified column name when only some columns are missing", async () => {
+		const queryable = { query: async () => ({ rows: [{ table_name: "user", column_name: "role" }] }) };
+		await expect(verifyMigratedSchema(queryable)).rejects.toThrow('"user"."role"');
+	});
+
+	it("passes the expected table and column lists to the catalog query", async () => {
+		let captured: unknown[] | undefined;
+		const queryable = {
+			query: (_text: string, values?: unknown[]) => {
+				captured = values;
+				return Promise.resolve({ rows: [] });
+			},
+		};
+
+		await verifyMigratedSchema(queryable);
+
+		const [tables, columns] = captured as [string[], string[]];
+		// The query relies on $1/$2 being index-aligned, so each table name must pair
+		// with its own column name at the same index.
+		const pairs = tables.map((table, index) => `${table}.${columns[index]}`);
+		expect(pairs).toContain("ai_providers.user_id");
+	});
+});

apps/server/src/startup/schema-check.ts

@@ -0,0 +1,71 @@
+import { is } from "drizzle-orm";
+import { getTableConfig, PgTable } from "drizzle-orm/pg-core";
+import * as schema from "@reactive-resume/db/schema";
+
+interface SchemaQueryable {
+	query(text: string, values?: unknown[]): Promise<{ rows: { table_name: string; column_name: string }[] }>;
+}
+
+export function collectExpectedColumns() {
+	const expected: { tableName: string; columnName: string }[] = [];
+
+	for (const value of Object.values(schema)) {
+		if (!is(value, PgTable)) continue;
+		const config = getTableConfig(value);
+		for (const column of config.columns) expected.push({ tableName: config.name, columnName: column.name });
+	}
+
+	return expected;
+}
+
+// The migration ledger (drizzle.__drizzle_migrations) only records that a migration ran; it
+// cannot detect objects that were dropped or lost outside the migrator (a partial restore,
+// a manual DROP TABLE, or a recreated "public" schema while the "drizzle" schema survives).
+// Comparing the live catalog with the declared schema turns that silent drift into a startup
+// failure instead of runtime "relation does not exist" (42P01) errors. The comparison covers
+// tables and columns only — indexes, constraints, and enums are intentionally out of scope.
+export async function verifyMigratedSchema(queryable: SchemaQueryable): Promise<void> {
+	const expected = collectExpectedColumns();
+	if (expected.length === 0) return;
+
+	// $1 and $2 are index-aligned: $1[i] is the name of the table expected to contain $2[i].
+	// Names are qualified as "public.<table>" so the lookup does not follow the connection's
+	// search_path — migrations always create these tables in the public schema.
+	const result = await queryable.query(
+		`select e.table_name, e.column_name
+			from unnest($1::text[], $2::text[]) as e(table_name, column_name)
+			where to_regclass('public.' || e.table_name) is null
+				or not exists (
+					select 1 from pg_catalog.pg_attribute a
+					where a.attrelid = to_regclass('public.' || e.table_name)
+						and a.attname = e.column_name
+						and a.attnum > 0 and not a.attisdropped
+				)
+			order by e.table_name, e.column_name`,
+		[expected.map((e) => e.tableName), expected.map((e) => e.columnName)],
+	);
+	if (result.rows.length === 0) return;
+
+	const expectedPerTable = new Map<string, number>();
+	for (const e of expected) expectedPerTable.set(e.tableName, (expectedPerTable.get(e.tableName) ?? 0) + 1);
+
+	const missingByTable = new Map<string, Set<string>>();
+	for (const row of result.rows) {
+		const columns = missingByTable.get(row.table_name) ?? new Set<string>();
+		columns.add(row.column_name);
+		missingByTable.set(row.table_name, columns);
+	}
+
+	const missing = [...missingByTable.entries()].map(([table, columns]) =>
+		columns.size === expectedPerTable.get(table)
+			? `table "${table}"`
+			: `column(s) ${[...columns].map((column) => `"${table}"."${column}"`).join(", ")}`,
+	);
+
+	throw new Error(
+		`Database schema does not match the migration ledger: ${missing.join(", ")} ` +
+			"missing even though all migrations are marked as applied. This usually means the database was " +
+			"restored from a backup that did not include these objects, or they were dropped outside of " +
+			"migrations. Restore a consistent backup or recreate the missing objects, then restart the server.",
+	);
+}

packages/env/src/server.ts

@@ -28,6 +28,7 @@ export const env = createEnv({
 
 		// Database
 		DATABASE_URL: z.url({ protocol: /postgres(ql)?/ }),
+		STRICT_SCHEMA_CHECK: z.stringbool().default(false),
 
 		// Authentication
 		AUTH_SECRET: z.string().min(1),

turbo.json

@@ -90,7 +90,8 @@
 		"FLAG_ALLOW_UNSAFE_AI_BASE_URL",
 		"AI_TEST_TIMEOUT_MS",
 		"OFFLINE_FONT_DIAGNOSTIC",
-		"OFFLINE_FONT_DIAGNOSTIC_SERVER_RESTARTED"
+		"OFFLINE_FONT_DIAGNOSTIC_SERVER_RESTARTED",
+		"STRICT_SCHEMA_CHECK"
 	],
 	"tasks": {
 		"transit": {