mirror of
https://github.com/AmruthPillai/Reactive-Resume.git
synced 2026-07-24 00:43:29 +10:00
use vite+
This commit is contained in:
Amruth Pillai
+12
-12
@@ -5,21 +5,21 @@ import { Pool } from "pg";
|
||||
import { env } from "@/utils/env";
|
||||
|
||||
export async function migrateDatabase() {
|
||||
console.log("⌛ Running database migrations...");
|
||||
console.log("⌛ Running database migrations...");
|
||||
|
||||
const pool = new Pool({ connectionString: env.DATABASE_URL });
|
||||
const db = drizzle({ client: pool });
|
||||
const pool = new Pool({ connectionString: env.DATABASE_URL });
|
||||
const db = drizzle({ client: pool });
|
||||
|
||||
try {
|
||||
await migrate(db, { migrationsFolder: "./migrations" });
|
||||
console.log("✅ Database migrations completed");
|
||||
} catch (error) {
|
||||
console.error("🚨 Database migrations failed:", error);
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
try {
|
||||
await migrate(db, { migrationsFolder: "./migrations" });
|
||||
console.log("✅ Database migrations completed");
|
||||
} catch (error) {
|
||||
console.error("🚨 Database migrations failed:", error);
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
await migrateDatabase();
|
||||
await migrateDatabase();
|
||||
}
|
||||
|
||||
+19
-19
@@ -5,30 +5,30 @@ import { Pool } from "pg";
|
||||
import { env } from "@/utils/env";
|
||||
|
||||
export async function resetDatabase() {
|
||||
console.log("⌛ Resetting database...");
|
||||
console.log("⌛ Resetting database...");
|
||||
|
||||
const pool = new Pool({ connectionString: env.DATABASE_URL });
|
||||
const db = drizzle({ client: pool });
|
||||
const pool = new Pool({ connectionString: env.DATABASE_URL });
|
||||
const db = drizzle({ client: pool });
|
||||
|
||||
try {
|
||||
await db.transaction(async (tx) => {
|
||||
await tx.execute(sql`DROP SCHEMA drizzle CASCADE`);
|
||||
await tx.execute(sql`CREATE SCHEMA drizzle`);
|
||||
await tx.execute(sql`GRANT ALL ON SCHEMA drizzle TO postgres`);
|
||||
try {
|
||||
await db.transaction(async (tx) => {
|
||||
await tx.execute(sql`DROP SCHEMA drizzle CASCADE`);
|
||||
await tx.execute(sql`CREATE SCHEMA drizzle`);
|
||||
await tx.execute(sql`GRANT ALL ON SCHEMA drizzle TO postgres`);
|
||||
|
||||
await tx.execute(sql`DROP SCHEMA public CASCADE`);
|
||||
await tx.execute(sql`CREATE SCHEMA public`);
|
||||
await tx.execute(sql`GRANT ALL ON SCHEMA public TO postgres`);
|
||||
});
|
||||
await tx.execute(sql`DROP SCHEMA public CASCADE`);
|
||||
await tx.execute(sql`CREATE SCHEMA public`);
|
||||
await tx.execute(sql`GRANT ALL ON SCHEMA public TO postgres`);
|
||||
});
|
||||
|
||||
console.log("✅ Database reset completed");
|
||||
} catch (error) {
|
||||
console.error("🚨 Database reset failed:", error);
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
console.log("✅ Database reset completed");
|
||||
} catch (error) {
|
||||
console.error("🚨 Database reset failed:", error);
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
await resetDatabase();
|
||||
await resetDatabase();
|
||||
}
|
||||
|
||||
+123
-123
@@ -25,35 +25,35 @@ const WEBFONTLIST_FILE = `${FONTS_DIR}/webfontlist.json`;
|
||||
|
||||
/** Returns JSON from Google Fonts API or (unless --force) from local cache if it exists */
|
||||
async function getGoogleFontsJSON() {
|
||||
let contents: string | null = null;
|
||||
let contents: string | null = null;
|
||||
|
||||
try {
|
||||
contents = await readFile(RESPONSE_FILE, "utf-8");
|
||||
} catch {
|
||||
// If the file doesn't exist or there's an error reading, just continue.
|
||||
}
|
||||
try {
|
||||
contents = await readFile(RESPONSE_FILE, "utf-8");
|
||||
} catch {
|
||||
// If the file doesn't exist or there's an error reading, just continue.
|
||||
}
|
||||
|
||||
if (!argForce && contents) return JSON.parse(contents) as APIResponse;
|
||||
if (!argForce && contents) return JSON.parse(contents) as APIResponse;
|
||||
|
||||
const apiKey = process.env.GOOGLE_CLOUD_API_KEY;
|
||||
if (!apiKey) throw new Error("GOOGLE_CLOUD_API_KEY is not set");
|
||||
const apiKey = process.env.GOOGLE_CLOUD_API_KEY;
|
||||
if (!apiKey) throw new Error("GOOGLE_CLOUD_API_KEY is not set");
|
||||
|
||||
const url = `https://www.googleapis.com/webfonts/v1/webfonts?key=${apiKey}&sort=popularity`;
|
||||
const response = await fetch(url);
|
||||
const data = (await response.json()) as APIResponse;
|
||||
const url = `https://www.googleapis.com/webfonts/v1/webfonts?key=${apiKey}&sort=popularity`;
|
||||
const response = await fetch(url);
|
||||
const data = (await response.json()) as APIResponse;
|
||||
|
||||
const jsonString = argCompress ? JSON.stringify(data) : JSON.stringify(data, null, 2);
|
||||
await mkdir(FONTS_DIR, { recursive: true });
|
||||
await writeFile(RESPONSE_FILE, jsonString, "utf-8");
|
||||
const jsonString = argCompress ? JSON.stringify(data) : JSON.stringify(data, null, 2);
|
||||
await mkdir(FONTS_DIR, { recursive: true });
|
||||
await writeFile(RESPONSE_FILE, jsonString, "utf-8");
|
||||
|
||||
return data;
|
||||
return data;
|
||||
}
|
||||
|
||||
/** Map Google Fonts API variant strings to simple weights */
|
||||
function variantToWeight(variant: Variant): Weight | null {
|
||||
if (["100", "200", "300", "500", "600", "700", "800", "900"].includes(variant)) return variant as Weight;
|
||||
if (variant === "regular") return "400";
|
||||
return null;
|
||||
if (["100", "200", "300", "500", "600", "700", "800", "900"].includes(variant)) return variant as Weight;
|
||||
if (variant === "regular") return "400";
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -62,124 +62,124 @@ function variantToWeight(variant: Variant): Weight | null {
|
||||
* Files are delivered via jsDelivr CDN.
|
||||
*/
|
||||
function getComputerModernWebFonts(): WebFont[] {
|
||||
const CDN = "https://cdn.jsdelivr.net/gh/bitmaks/cm-web-fonts@latest/font";
|
||||
const CDN = "https://cdn.jsdelivr.net/gh/bitmaks/cm-web-fonts@latest/font";
|
||||
|
||||
return [
|
||||
{
|
||||
type: "web",
|
||||
category: "display",
|
||||
family: "Computer Modern Bright",
|
||||
weights: ["400", "700"],
|
||||
preview: `${CDN}/Bright/cmunbmr.woff`,
|
||||
files: {
|
||||
"400": `${CDN}/Bright/cmunbmr.woff`,
|
||||
"400italic": `${CDN}/Bright/cmunbmo.woff`,
|
||||
"700": `${CDN}/Bright/cmunbbx.woff`,
|
||||
"700italic": `${CDN}/Bright/cmunbxo.woff`,
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "web",
|
||||
category: "serif",
|
||||
family: "Computer Modern Concrete",
|
||||
weights: ["400", "700"],
|
||||
preview: `${CDN}/Concrete/cmunorm.woff`,
|
||||
files: {
|
||||
"400": `${CDN}/Concrete/cmunorm.woff`,
|
||||
"400italic": `${CDN}/Concrete/cmunobi.woff`,
|
||||
"700": `${CDN}/Concrete/cmunobx.woff`,
|
||||
"700italic": `${CDN}/Concrete/cmunoti.woff`,
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "web",
|
||||
category: "sans-serif",
|
||||
family: "Computer Modern Sans",
|
||||
weights: ["400", "700"],
|
||||
preview: `${CDN}/Sans/cmunss.woff`,
|
||||
files: {
|
||||
"400": `${CDN}/Sans/cmunss.woff`,
|
||||
"400italic": `${CDN}/Sans/cmunsl.woff`,
|
||||
"700": `${CDN}/Sans/cmunsx.woff`,
|
||||
"700italic": `${CDN}/Sans/cmunsi.woff`,
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "web",
|
||||
category: "serif",
|
||||
family: "Computer Modern Serif",
|
||||
weights: ["400", "700"],
|
||||
preview: `${CDN}/Serif/cmunrm.woff`,
|
||||
files: {
|
||||
"400": `${CDN}/Serif/cmunrm.woff`,
|
||||
"400italic": `${CDN}/Serif/cmunti.woff`,
|
||||
"700": `${CDN}/Serif/cmunbx.woff`,
|
||||
"700italic": `${CDN}/Serif/cmunbi.woff`,
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "web",
|
||||
category: "monospace",
|
||||
family: "Computer Modern Typewriter",
|
||||
weights: ["400", "700"],
|
||||
preview: `${CDN}/Typewriter/cmuntt.woff`,
|
||||
files: {
|
||||
"400": `${CDN}/Typewriter/cmuntt.woff`,
|
||||
"400italic": `${CDN}/Typewriter/cmunit.woff`,
|
||||
"700": `${CDN}/Typewriter/cmuntx.woff`,
|
||||
"700italic": `${CDN}/Typewriter/cmuntb.woff`,
|
||||
},
|
||||
},
|
||||
];
|
||||
return [
|
||||
{
|
||||
type: "web",
|
||||
category: "display",
|
||||
family: "Computer Modern Bright",
|
||||
weights: ["400", "700"],
|
||||
preview: `${CDN}/Bright/cmunbmr.woff`,
|
||||
files: {
|
||||
"400": `${CDN}/Bright/cmunbmr.woff`,
|
||||
"400italic": `${CDN}/Bright/cmunbmo.woff`,
|
||||
"700": `${CDN}/Bright/cmunbbx.woff`,
|
||||
"700italic": `${CDN}/Bright/cmunbxo.woff`,
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "web",
|
||||
category: "serif",
|
||||
family: "Computer Modern Concrete",
|
||||
weights: ["400", "700"],
|
||||
preview: `${CDN}/Concrete/cmunorm.woff`,
|
||||
files: {
|
||||
"400": `${CDN}/Concrete/cmunorm.woff`,
|
||||
"400italic": `${CDN}/Concrete/cmunobi.woff`,
|
||||
"700": `${CDN}/Concrete/cmunobx.woff`,
|
||||
"700italic": `${CDN}/Concrete/cmunoti.woff`,
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "web",
|
||||
category: "sans-serif",
|
||||
family: "Computer Modern Sans",
|
||||
weights: ["400", "700"],
|
||||
preview: `${CDN}/Sans/cmunss.woff`,
|
||||
files: {
|
||||
"400": `${CDN}/Sans/cmunss.woff`,
|
||||
"400italic": `${CDN}/Sans/cmunsl.woff`,
|
||||
"700": `${CDN}/Sans/cmunsx.woff`,
|
||||
"700italic": `${CDN}/Sans/cmunsi.woff`,
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "web",
|
||||
category: "serif",
|
||||
family: "Computer Modern Serif",
|
||||
weights: ["400", "700"],
|
||||
preview: `${CDN}/Serif/cmunrm.woff`,
|
||||
files: {
|
||||
"400": `${CDN}/Serif/cmunrm.woff`,
|
||||
"400italic": `${CDN}/Serif/cmunti.woff`,
|
||||
"700": `${CDN}/Serif/cmunbx.woff`,
|
||||
"700italic": `${CDN}/Serif/cmunbi.woff`,
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "web",
|
||||
category: "monospace",
|
||||
family: "Computer Modern Typewriter",
|
||||
weights: ["400", "700"],
|
||||
preview: `${CDN}/Typewriter/cmuntt.woff`,
|
||||
files: {
|
||||
"400": `${CDN}/Typewriter/cmuntt.woff`,
|
||||
"400italic": `${CDN}/Typewriter/cmunit.woff`,
|
||||
"700": `${CDN}/Typewriter/cmuntx.woff`,
|
||||
"700italic": `${CDN}/Typewriter/cmuntb.woff`,
|
||||
},
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
export async function generateFonts() {
|
||||
const response = await getGoogleFontsJSON();
|
||||
console.log(`Found ${response.items.length} fonts in total (Google Fonts).`);
|
||||
const response = await getGoogleFontsJSON();
|
||||
console.log(`Found ${response.items.length} fonts in total (Google Fonts).`);
|
||||
|
||||
const filteredItems = response.items.filter(
|
||||
(item) => !skippedFamilies.some((family) => item.family.includes(family)),
|
||||
);
|
||||
const filteredItems = response.items.filter(
|
||||
(item) => !skippedFamilies.some((family) => item.family.includes(family)),
|
||||
);
|
||||
|
||||
const googleFontResults: WebFont[] = filteredItems.slice(0, argLimit).map((item) => {
|
||||
// 1. weights: Only non-italic, convert "regular" to "400"
|
||||
const weights: Weight[] = item.variants.map((v) => variantToWeight(v)).filter((w): w is Weight => !!w);
|
||||
const googleFontResults: WebFont[] = filteredItems.slice(0, argLimit).map((item) => {
|
||||
// 1. weights: Only non-italic, convert "regular" to "400"
|
||||
const weights: Weight[] = item.variants.map((v) => variantToWeight(v)).filter((w): w is Weight => !!w);
|
||||
|
||||
// 2. files: all files, but change "regular"->"400", "italic"->"400italic"
|
||||
const files: Record<string, string> = {};
|
||||
// 2. files: all files, but change "regular"->"400", "italic"->"400italic"
|
||||
const files: Record<string, string> = {};
|
||||
|
||||
for (const [variant, url] of Object.entries(item.files)) {
|
||||
let key = variant;
|
||||
if (variant === "regular") key = "400";
|
||||
else if (variant === "italic") key = "400italic";
|
||||
files[key] = url;
|
||||
}
|
||||
for (const [variant, url] of Object.entries(item.files)) {
|
||||
let key = variant;
|
||||
if (variant === "regular") key = "400";
|
||||
else if (variant === "italic") key = "400italic";
|
||||
files[key] = url;
|
||||
}
|
||||
|
||||
return {
|
||||
type: "web",
|
||||
category: item.category,
|
||||
family: item.family,
|
||||
weights,
|
||||
preview: item.menu,
|
||||
files,
|
||||
} satisfies WebFont;
|
||||
});
|
||||
return {
|
||||
type: "web",
|
||||
category: item.category,
|
||||
family: item.family,
|
||||
weights,
|
||||
preview: item.menu,
|
||||
files,
|
||||
} satisfies WebFont;
|
||||
});
|
||||
|
||||
// Manually append Computer Modern web fonts
|
||||
const computerModernFonts = getComputerModernWebFonts();
|
||||
const allWebFonts: WebFont[] = [...computerModernFonts, ...googleFontResults];
|
||||
// Manually append Computer Modern web fonts
|
||||
const computerModernFonts = getComputerModernWebFonts();
|
||||
const allWebFonts: WebFont[] = [...computerModernFonts, ...googleFontResults];
|
||||
|
||||
console.log(
|
||||
`Added ${computerModernFonts.length} Computer Modern Web Fonts. Total output: ${allWebFonts.length} web fonts.`,
|
||||
);
|
||||
console.log(
|
||||
`Added ${computerModernFonts.length} Computer Modern Web Fonts. Total output: ${allWebFonts.length} web fonts.`,
|
||||
);
|
||||
|
||||
const jsonString = argCompress ? JSON.stringify(allWebFonts) : JSON.stringify(allWebFonts, null, 2);
|
||||
await mkdir(FONTS_DIR, { recursive: true });
|
||||
await writeFile(WEBFONTLIST_FILE, jsonString, "utf-8");
|
||||
const jsonString = argCompress ? JSON.stringify(allWebFonts) : JSON.stringify(allWebFonts, null, 2);
|
||||
await mkdir(FONTS_DIR, { recursive: true });
|
||||
await writeFile(WEBFONTLIST_FILE, jsonString, "utf-8");
|
||||
|
||||
console.log(`Generated ${allWebFonts.length} fonts in the list (including Computer Modern web fonts).`);
|
||||
console.log(`Generated ${allWebFonts.length} fonts in the list (including Computer Modern web fonts).`);
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
await generateFonts();
|
||||
await generateFonts();
|
||||
}
|
||||
|
||||
+36
-36
@@ -1,51 +1,51 @@
|
||||
type Category = "display" | "handwriting" | "monospace" | "serif" | "sans-serif";
|
||||
|
||||
export type Variant =
|
||||
| "100"
|
||||
| "100italic"
|
||||
| "200"
|
||||
| "200italic"
|
||||
| "300"
|
||||
| "300italic"
|
||||
| "regular"
|
||||
| "italic"
|
||||
| "500"
|
||||
| "500italic"
|
||||
| "600"
|
||||
| "600italic"
|
||||
| "700"
|
||||
| "700italic"
|
||||
| "800"
|
||||
| "800italic"
|
||||
| "900"
|
||||
| "900italic";
|
||||
| "100"
|
||||
| "100italic"
|
||||
| "200"
|
||||
| "200italic"
|
||||
| "300"
|
||||
| "300italic"
|
||||
| "regular"
|
||||
| "italic"
|
||||
| "500"
|
||||
| "500italic"
|
||||
| "600"
|
||||
| "600italic"
|
||||
| "700"
|
||||
| "700italic"
|
||||
| "800"
|
||||
| "800italic"
|
||||
| "900"
|
||||
| "900italic";
|
||||
|
||||
type Item = {
|
||||
kind: "webfonts#webfont";
|
||||
menu: string;
|
||||
family: string;
|
||||
version: string;
|
||||
category: Category;
|
||||
lastModified: string;
|
||||
subsets: string[];
|
||||
variants: Variant[];
|
||||
colorCapabilities?: string[];
|
||||
files: Partial<Record<Variant, string>>;
|
||||
kind: "webfonts#webfont";
|
||||
menu: string;
|
||||
family: string;
|
||||
version: string;
|
||||
category: Category;
|
||||
lastModified: string;
|
||||
subsets: string[];
|
||||
variants: Variant[];
|
||||
colorCapabilities?: string[];
|
||||
files: Partial<Record<Variant, string>>;
|
||||
};
|
||||
|
||||
export type APIResponse = {
|
||||
kind: "webfonts#webfontList";
|
||||
items: Item[];
|
||||
kind: "webfonts#webfontList";
|
||||
items: Item[];
|
||||
};
|
||||
|
||||
export type Weight = "100" | "200" | "300" | "400" | "500" | "600" | "700" | "800" | "900";
|
||||
type FileWeight = Weight | `${Weight}italic`;
|
||||
|
||||
export type WebFont = {
|
||||
type: "web";
|
||||
category: Category;
|
||||
family: string;
|
||||
weights: Weight[];
|
||||
preview: string;
|
||||
files: Partial<Record<FileWeight, string>>;
|
||||
type: "web";
|
||||
category: Category;
|
||||
family: string;
|
||||
weights: Weight[];
|
||||
preview: string;
|
||||
files: Partial<Record<FileWeight, string>>;
|
||||
};
|
||||
|
||||
+361
-361
@@ -12,24 +12,24 @@ import { generateId } from "@/utils/string";
|
||||
type Visibility = "public" | "private";
|
||||
|
||||
interface ProductionResume {
|
||||
id: string;
|
||||
title: string;
|
||||
slug: string;
|
||||
data: unknown; // JSON data
|
||||
visibility: Visibility;
|
||||
locked: boolean;
|
||||
userId: string;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
id: string;
|
||||
title: string;
|
||||
slug: string;
|
||||
data: unknown; // JSON data
|
||||
visibility: Visibility;
|
||||
locked: boolean;
|
||||
userId: string;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
}
|
||||
|
||||
interface ProductionStatistics {
|
||||
id: string;
|
||||
views: number;
|
||||
downloads: number;
|
||||
resumeId: string;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
id: string;
|
||||
views: number;
|
||||
downloads: number;
|
||||
resumeId: string;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
}
|
||||
|
||||
const productionUrl = process.env.PRODUCTION_DATABASE_URL;
|
||||
@@ -62,444 +62,444 @@ const INSERT_CHUNK_SIZE = 1000;
|
||||
// == Progress checkpoint interface ==
|
||||
// Uses cursor-based pagination with (createdAt, id) composite key for efficiency
|
||||
interface MigrationProgress {
|
||||
// Cursor for pagination - last seen createdAt timestamp
|
||||
lastSeenCreatedAt: string | null;
|
||||
// Cursor for pagination - last seen id (for tiebreaker when timestamps are equal)
|
||||
lastSeenId: string | null;
|
||||
resumesCreated: number;
|
||||
statisticsCreated: number;
|
||||
skipped: number;
|
||||
totalResumesProcessed: number;
|
||||
errors: number;
|
||||
lastUpdated: string;
|
||||
// Cursor for pagination - last seen createdAt timestamp
|
||||
lastSeenCreatedAt: string | null;
|
||||
// Cursor for pagination - last seen id (for tiebreaker when timestamps are equal)
|
||||
lastSeenId: string | null;
|
||||
resumesCreated: number;
|
||||
statisticsCreated: number;
|
||||
skipped: number;
|
||||
totalResumesProcessed: number;
|
||||
errors: number;
|
||||
lastUpdated: string;
|
||||
}
|
||||
|
||||
// Flag to track if shutdown was requested
|
||||
let shutdownRequested = false;
|
||||
|
||||
async function loadProgress(): Promise<MigrationProgress | null> {
|
||||
try {
|
||||
const text = await fs.readFile(PROGRESS_FILE, { encoding: "utf-8" });
|
||||
const progress = JSON.parse(text) as MigrationProgress;
|
||||
console.log(`📂 Found existing progress file. Last updated: ${progress.lastUpdated}`);
|
||||
console.log(` Resuming from cursor (createdAt: ${progress.lastSeenCreatedAt}, id: ${progress.lastSeenId})...`);
|
||||
return progress;
|
||||
} catch (e) {
|
||||
console.warn("⚠️ Failed to load progress file, starting from beginning.", e);
|
||||
}
|
||||
return null;
|
||||
try {
|
||||
const text = await fs.readFile(PROGRESS_FILE, { encoding: "utf-8" });
|
||||
const progress = JSON.parse(text) as MigrationProgress;
|
||||
console.log(`📂 Found existing progress file. Last updated: ${progress.lastUpdated}`);
|
||||
console.log(` Resuming from cursor (createdAt: ${progress.lastSeenCreatedAt}, id: ${progress.lastSeenId})...`);
|
||||
return progress;
|
||||
} catch (e) {
|
||||
console.warn("⚠️ Failed to load progress file, starting from beginning.", e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function saveProgress(progress: MigrationProgress): Promise<void> {
|
||||
try {
|
||||
progress.lastUpdated = new Date().toISOString();
|
||||
await fs.writeFile(PROGRESS_FILE, JSON.stringify(progress, null, 2), { encoding: "utf-8" });
|
||||
console.log(`💾 Progress saved at cursor (createdAt: ${progress.lastSeenCreatedAt}, id: ${progress.lastSeenId})`);
|
||||
} catch (e) {
|
||||
console.error("🚨 Failed to save progress:", e);
|
||||
}
|
||||
try {
|
||||
progress.lastUpdated = new Date().toISOString();
|
||||
await fs.writeFile(PROGRESS_FILE, JSON.stringify(progress, null, 2), { encoding: "utf-8" });
|
||||
console.log(`💾 Progress saved at cursor (createdAt: ${progress.lastSeenCreatedAt}, id: ${progress.lastSeenId})`);
|
||||
} catch (e) {
|
||||
console.error("🚨 Failed to save progress:", e);
|
||||
}
|
||||
}
|
||||
|
||||
async function clearProgress(): Promise<void> {
|
||||
try {
|
||||
await fs.unlink(PROGRESS_FILE);
|
||||
console.log("🗑️ Progress file cleared.");
|
||||
} catch {
|
||||
// Ignore errors when clearing
|
||||
}
|
||||
try {
|
||||
await fs.unlink(PROGRESS_FILE);
|
||||
console.log("🗑️ Progress file cleared.");
|
||||
} catch {
|
||||
// Ignore errors when clearing
|
||||
}
|
||||
}
|
||||
|
||||
async function loadUserIdMapFromFile(): Promise<Map<string, string>> {
|
||||
try {
|
||||
const text = await fs.readFile(USER_ID_MAP_FILE, { encoding: "utf-8" });
|
||||
const obj = JSON.parse(text);
|
||||
return new Map(Object.entries(obj));
|
||||
} catch (e) {
|
||||
console.warn("⚠️ Failed to load userIdMap from disk, continuing with empty map.", e);
|
||||
}
|
||||
return new Map<string, string>();
|
||||
try {
|
||||
const text = await fs.readFile(USER_ID_MAP_FILE, { encoding: "utf-8" });
|
||||
const obj = JSON.parse(text);
|
||||
return new Map(Object.entries(obj));
|
||||
} catch (e) {
|
||||
console.warn("⚠️ Failed to load userIdMap from disk, continuing with empty map.", e);
|
||||
}
|
||||
return new Map<string, string>();
|
||||
}
|
||||
|
||||
async function loadResumeIdMapFromFile(): Promise<Map<string, string>> {
|
||||
try {
|
||||
const text = await fs.readFile(RESUME_ID_MAP_FILE, { encoding: "utf-8" });
|
||||
const obj = JSON.parse(text);
|
||||
return new Map(Object.entries(obj));
|
||||
} catch (e) {
|
||||
console.warn("⚠️ Failed to load resumeIdMap from disk, continuing with empty map.", e);
|
||||
}
|
||||
return new Map<string, string>();
|
||||
try {
|
||||
const text = await fs.readFile(RESUME_ID_MAP_FILE, { encoding: "utf-8" });
|
||||
const obj = JSON.parse(text);
|
||||
return new Map(Object.entries(obj));
|
||||
} catch (e) {
|
||||
console.warn("⚠️ Failed to load resumeIdMap from disk, continuing with empty map.", e);
|
||||
}
|
||||
return new Map<string, string>();
|
||||
}
|
||||
|
||||
async function saveResumeIdMapToFile(resumeIdMap: Map<string, string>) {
|
||||
const obj: Record<string, string> = Object.fromEntries(resumeIdMap.entries());
|
||||
await fs.writeFile(RESUME_ID_MAP_FILE, JSON.stringify(obj, null, "\t"), { encoding: "utf-8" });
|
||||
const obj: Record<string, string> = Object.fromEntries(resumeIdMap.entries());
|
||||
await fs.writeFile(RESUME_ID_MAP_FILE, JSON.stringify(obj, null, "\t"), { encoding: "utf-8" });
|
||||
}
|
||||
|
||||
export async function migrateResumes() {
|
||||
const migrationStart = performance.now();
|
||||
console.log("⌛ Starting resume migration...");
|
||||
const migrationStart = performance.now();
|
||||
console.log("⌛ Starting resume migration...");
|
||||
|
||||
let hasMore = true;
|
||||
let hasMore = true;
|
||||
|
||||
// Cursor-based pagination state
|
||||
let lastSeenCreatedAt: string | null = null;
|
||||
let lastSeenId: string | null = null;
|
||||
// Cursor-based pagination state
|
||||
let lastSeenCreatedAt: string | null = null;
|
||||
let lastSeenId: string | null = null;
|
||||
|
||||
// Load persistent ID maps from file
|
||||
const userIdMap = await loadUserIdMapFromFile();
|
||||
const resumeIdMap = await loadResumeIdMapFromFile();
|
||||
// Load persistent ID maps from file
|
||||
const userIdMap = await loadUserIdMapFromFile();
|
||||
const resumeIdMap = await loadResumeIdMapFromFile();
|
||||
|
||||
// Track migration stats
|
||||
let resumesCreated = 0;
|
||||
let statisticsCreated = 0;
|
||||
let skipped = 0;
|
||||
let totalResumesProcessed = 0;
|
||||
let errors = 0;
|
||||
// Track migration stats
|
||||
let resumesCreated = 0;
|
||||
let statisticsCreated = 0;
|
||||
let skipped = 0;
|
||||
let totalResumesProcessed = 0;
|
||||
let errors = 0;
|
||||
|
||||
// Load saved progress if exists
|
||||
const savedProgress = await loadProgress();
|
||||
if (savedProgress) {
|
||||
lastSeenCreatedAt = savedProgress.lastSeenCreatedAt;
|
||||
lastSeenId = savedProgress.lastSeenId;
|
||||
resumesCreated = savedProgress.resumesCreated;
|
||||
statisticsCreated = savedProgress.statisticsCreated;
|
||||
skipped = savedProgress.skipped;
|
||||
totalResumesProcessed = savedProgress.totalResumesProcessed;
|
||||
errors = savedProgress.errors;
|
||||
}
|
||||
// Load saved progress if exists
|
||||
const savedProgress = await loadProgress();
|
||||
if (savedProgress) {
|
||||
lastSeenCreatedAt = savedProgress.lastSeenCreatedAt;
|
||||
lastSeenId = savedProgress.lastSeenId;
|
||||
resumesCreated = savedProgress.resumesCreated;
|
||||
statisticsCreated = savedProgress.statisticsCreated;
|
||||
skipped = savedProgress.skipped;
|
||||
totalResumesProcessed = savedProgress.totalResumesProcessed;
|
||||
errors = savedProgress.errors;
|
||||
}
|
||||
|
||||
// Helper to get current progress object
|
||||
const getCurrentProgress = (): MigrationProgress => ({
|
||||
lastSeenCreatedAt,
|
||||
lastSeenId,
|
||||
resumesCreated,
|
||||
statisticsCreated,
|
||||
skipped,
|
||||
totalResumesProcessed,
|
||||
errors,
|
||||
lastUpdated: new Date().toISOString(),
|
||||
});
|
||||
// Helper to get current progress object
|
||||
const getCurrentProgress = (): MigrationProgress => ({
|
||||
lastSeenCreatedAt,
|
||||
lastSeenId,
|
||||
resumesCreated,
|
||||
statisticsCreated,
|
||||
skipped,
|
||||
totalResumesProcessed,
|
||||
errors,
|
||||
lastUpdated: new Date().toISOString(),
|
||||
});
|
||||
|
||||
// Setup graceful shutdown handler
|
||||
const handleShutdown = async () => {
|
||||
if (shutdownRequested) return;
|
||||
shutdownRequested = true;
|
||||
console.log("\n⚠️ Shutdown requested. Saving progress...");
|
||||
await saveProgress(getCurrentProgress());
|
||||
await saveResumeIdMapToFile(resumeIdMap);
|
||||
console.log("👋 Exiting. Run the script again to resume from where you left off.");
|
||||
process.exit(0);
|
||||
};
|
||||
// Setup graceful shutdown handler
|
||||
const handleShutdown = async () => {
|
||||
if (shutdownRequested) return;
|
||||
shutdownRequested = true;
|
||||
console.log("\n⚠️ Shutdown requested. Saving progress...");
|
||||
await saveProgress(getCurrentProgress());
|
||||
await saveResumeIdMapToFile(resumeIdMap);
|
||||
console.log("👋 Exiting. Run the script again to resume from where you left off.");
|
||||
process.exit(0);
|
||||
};
|
||||
|
||||
process.on("SIGINT", handleShutdown);
|
||||
process.on("SIGTERM", handleShutdown);
|
||||
process.on("SIGINT", handleShutdown);
|
||||
process.on("SIGTERM", handleShutdown);
|
||||
|
||||
// Initialize the importer
|
||||
const importer = new ReactiveResumeV4JSONImporter();
|
||||
// Initialize the importer
|
||||
const importer = new ReactiveResumeV4JSONImporter();
|
||||
|
||||
while (hasMore) {
|
||||
// Check if shutdown was requested
|
||||
if (shutdownRequested) break;
|
||||
while (hasMore) {
|
||||
// Check if shutdown was requested
|
||||
if (shutdownRequested) break;
|
||||
|
||||
console.log(
|
||||
`📥 Fetching resumes batch from production database (cursor: createdAt=${lastSeenCreatedAt}, id=${lastSeenId})...`,
|
||||
);
|
||||
console.log(
|
||||
`📥 Fetching resumes batch from production database (cursor: createdAt=${lastSeenCreatedAt}, id=${lastSeenId})...`,
|
||||
);
|
||||
|
||||
// Use cursor-based pagination for better performance
|
||||
// Tuple comparison syntax allows Postgres to use composite index efficiently
|
||||
let resumes: ProductionResume[];
|
||||
// Use cursor-based pagination for better performance
|
||||
// Tuple comparison syntax allows Postgres to use composite index efficiently
|
||||
let resumes: ProductionResume[];
|
||||
|
||||
if (lastSeenCreatedAt && lastSeenId) {
|
||||
const result = (await productionDb.execute(sql`
|
||||
if (lastSeenCreatedAt && lastSeenId) {
|
||||
const result = (await productionDb.execute(sql`
|
||||
SELECT id, title, slug, data, visibility, locked, "userId", "createdAt", "updatedAt"
|
||||
FROM "Resume"
|
||||
WHERE ("createdAt", id) < (${lastSeenCreatedAt}::timestamp, ${lastSeenId})
|
||||
ORDER BY "createdAt" DESC, id DESC
|
||||
LIMIT ${BATCH_SIZE}
|
||||
`)) as unknown as QueryResult<ProductionResume>;
|
||||
resumes = result.rows;
|
||||
} else {
|
||||
const result = (await productionDb.execute(sql`
|
||||
resumes = result.rows;
|
||||
} else {
|
||||
const result = (await productionDb.execute(sql`
|
||||
SELECT id, title, slug, data, visibility, locked, "userId", "createdAt", "updatedAt"
|
||||
FROM "Resume"
|
||||
ORDER BY "createdAt" DESC, id DESC
|
||||
LIMIT ${BATCH_SIZE}
|
||||
`)) as unknown as QueryResult<ProductionResume>;
|
||||
resumes = result.rows;
|
||||
}
|
||||
resumes = result.rows;
|
||||
}
|
||||
|
||||
console.log(`📋 Found ${resumes.length} resumes in this batch.`);
|
||||
console.log(`📋 Found ${resumes.length} resumes in this batch.`);
|
||||
|
||||
if (resumes.length === 0) {
|
||||
hasMore = false;
|
||||
break;
|
||||
}
|
||||
if (resumes.length === 0) {
|
||||
hasMore = false;
|
||||
break;
|
||||
}
|
||||
|
||||
// Fetch statistics only for these resumes in this batch
|
||||
const resumeIds = resumes.map((r) => r.id);
|
||||
// Fetch statistics only for these resumes in this batch
|
||||
const resumeIds = resumes.map((r) => r.id);
|
||||
|
||||
// Drizzle does not interpolate arrays, so we join and use a custom SQL string
|
||||
// Escape single quotes in IDs (though UUIDs shouldn't contain them, this is safer)
|
||||
const resumeIdsForSql = resumeIds.map((id) => `'${id.replace(/'/g, "''")}'`).join(", ");
|
||||
// Drizzle does not interpolate arrays, so we join and use a custom SQL string
|
||||
// Escape single quotes in IDs (though UUIDs shouldn't contain them, this is safer)
|
||||
const resumeIdsForSql = resumeIds.map((id) => `'${id.replace(/'/g, "''")}'`).join(", ");
|
||||
|
||||
const { rows: statistics } = (await productionDb.execute(sql`
|
||||
const { rows: statistics } = (await productionDb.execute(sql`
|
||||
SELECT id, views, downloads, "resumeId", "createdAt", "updatedAt"
|
||||
FROM "Statistics"
|
||||
WHERE "resumeId" IN (${sql.raw(resumeIdsForSql)})
|
||||
`)) as unknown as QueryResult<ProductionStatistics>;
|
||||
|
||||
// Create a map of resumeId -> statistics for quick lookup
|
||||
const statisticsMap = new Map<string, ProductionStatistics>();
|
||||
for (const stat of statistics) {
|
||||
statisticsMap.set(stat.resumeId, stat);
|
||||
}
|
||||
// Create a map of resumeId -> statistics for quick lookup
|
||||
const statisticsMap = new Map<string, ProductionStatistics>();
|
||||
for (const stat of statistics) {
|
||||
statisticsMap.set(stat.resumeId, stat);
|
||||
}
|
||||
|
||||
// Filter out resumes where userId is not in userIdMap
|
||||
const resumesToProcess = resumes
|
||||
.map((resume) => {
|
||||
const newUserId = userIdMap.get(resume.userId);
|
||||
if (!newUserId) {
|
||||
skipped++;
|
||||
return null;
|
||||
}
|
||||
return { resume, newUserId };
|
||||
})
|
||||
.filter((item): item is NonNullable<typeof item> => item !== null);
|
||||
// Filter out resumes where userId is not in userIdMap
|
||||
const resumesToProcess = resumes
|
||||
.map((resume) => {
|
||||
const newUserId = userIdMap.get(resume.userId);
|
||||
if (!newUserId) {
|
||||
skipped++;
|
||||
return null;
|
||||
}
|
||||
return { resume, newUserId };
|
||||
})
|
||||
.filter((item): item is NonNullable<typeof item> => item !== null);
|
||||
|
||||
if (resumesToProcess.length === 0) {
|
||||
console.log(`⏭️ All resumes in this batch have userIds not found in userIdMap.`);
|
||||
// Update cursor to the last resume in this batch
|
||||
const lastResume = resumes[resumes.length - 1];
|
||||
if (lastResume) {
|
||||
lastSeenCreatedAt =
|
||||
lastResume.createdAt instanceof Date ? lastResume.createdAt.toISOString() : String(lastResume.createdAt);
|
||||
lastSeenId = lastResume.id;
|
||||
}
|
||||
totalResumesProcessed += resumes.length;
|
||||
await saveProgress(getCurrentProgress());
|
||||
continue;
|
||||
}
|
||||
if (resumesToProcess.length === 0) {
|
||||
console.log(`⏭️ All resumes in this batch have userIds not found in userIdMap.`);
|
||||
// Update cursor to the last resume in this batch
|
||||
const lastResume = resumes[resumes.length - 1];
|
||||
if (lastResume) {
|
||||
lastSeenCreatedAt =
|
||||
lastResume.createdAt instanceof Date ? lastResume.createdAt.toISOString() : String(lastResume.createdAt);
|
||||
lastSeenId = lastResume.id;
|
||||
}
|
||||
totalResumesProcessed += resumes.length;
|
||||
await saveProgress(getCurrentProgress());
|
||||
continue;
|
||||
}
|
||||
|
||||
// Get unique userIds and bulk check if they exist in local database
|
||||
const uniqueUserIds = [...new Set(resumesToProcess.map((r) => r.newUserId))];
|
||||
const existingUsers = await localDb.select().from(schema.user).where(inArray(schema.user.id, uniqueUserIds));
|
||||
// Get unique userIds and bulk check if they exist in local database
|
||||
const uniqueUserIds = [...new Set(resumesToProcess.map((r) => r.newUserId))];
|
||||
const existingUsers = await localDb.select().from(schema.user).where(inArray(schema.user.id, uniqueUserIds));
|
||||
|
||||
const existingUserIds = new Set(existingUsers.map((u) => u.id));
|
||||
const existingUserIds = new Set(existingUsers.map((u) => u.id));
|
||||
|
||||
// Filter out resumes where user doesn't exist
|
||||
const resumesWithValidUsers = resumesToProcess.filter(({ newUserId }) => {
|
||||
if (!existingUserIds.has(newUserId)) {
|
||||
skipped++;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
// Filter out resumes where user doesn't exist
|
||||
const resumesWithValidUsers = resumesToProcess.filter(({ newUserId }) => {
|
||||
if (!existingUserIds.has(newUserId)) {
|
||||
skipped++;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
if (resumesWithValidUsers.length === 0) {
|
||||
console.log(`⏭️ All resumes in this batch have userIds not found in local database.`);
|
||||
// Update cursor to the last resume in this batch
|
||||
const lastResume = resumes[resumes.length - 1];
|
||||
if (lastResume) {
|
||||
lastSeenCreatedAt =
|
||||
lastResume.createdAt instanceof Date ? lastResume.createdAt.toISOString() : String(lastResume.createdAt);
|
||||
lastSeenId = lastResume.id;
|
||||
}
|
||||
totalResumesProcessed += resumes.length;
|
||||
await saveProgress(getCurrentProgress());
|
||||
continue;
|
||||
}
|
||||
if (resumesWithValidUsers.length === 0) {
|
||||
console.log(`⏭️ All resumes in this batch have userIds not found in local database.`);
|
||||
// Update cursor to the last resume in this batch
|
||||
const lastResume = resumes[resumes.length - 1];
|
||||
if (lastResume) {
|
||||
lastSeenCreatedAt =
|
||||
lastResume.createdAt instanceof Date ? lastResume.createdAt.toISOString() : String(lastResume.createdAt);
|
||||
lastSeenId = lastResume.id;
|
||||
}
|
||||
totalResumesProcessed += resumes.length;
|
||||
await saveProgress(getCurrentProgress());
|
||||
continue;
|
||||
}
|
||||
|
||||
// Bulk check for existing resumes (by slug + userId)
|
||||
// We need to check each unique combination
|
||||
const slugUserIdPairs = resumesWithValidUsers.map(({ resume, newUserId }) => ({
|
||||
slug: resume.slug,
|
||||
userId: newUserId,
|
||||
}));
|
||||
// Bulk check for existing resumes (by slug + userId)
|
||||
// We need to check each unique combination
|
||||
const slugUserIdPairs = resumesWithValidUsers.map(({ resume, newUserId }) => ({
|
||||
slug: resume.slug,
|
||||
userId: newUserId,
|
||||
}));
|
||||
|
||||
// Get all unique slugs and userIds
|
||||
const uniqueSlugs = [...new Set(slugUserIdPairs.map((p) => p.slug))];
|
||||
const userIdsForSlugCheck = [...new Set(slugUserIdPairs.map((p) => p.userId))];
|
||||
// Get all unique slugs and userIds
|
||||
const uniqueSlugs = [...new Set(slugUserIdPairs.map((p) => p.slug))];
|
||||
const userIdsForSlugCheck = [...new Set(slugUserIdPairs.map((p) => p.userId))];
|
||||
|
||||
// Fetch all existing resumes that match any of our slugs and userIds
|
||||
const existingResumes = await localDb
|
||||
.select()
|
||||
.from(schema.resume)
|
||||
.where(and(inArray(schema.resume.slug, uniqueSlugs), inArray(schema.resume.userId, userIdsForSlugCheck)));
|
||||
// Fetch all existing resumes that match any of our slugs and userIds
|
||||
const existingResumes = await localDb
|
||||
.select()
|
||||
.from(schema.resume)
|
||||
.where(and(inArray(schema.resume.slug, uniqueSlugs), inArray(schema.resume.userId, userIdsForSlugCheck)));
|
||||
|
||||
// Create a set of existing slug+userId combinations
|
||||
const existingResumeKeys = new Set(existingResumes.map((r) => `${r.slug}:${r.userId}`));
|
||||
// Create a set of existing slug+userId combinations
|
||||
const existingResumeKeys = new Set(existingResumes.map((r) => `${r.slug}:${r.userId}`));
|
||||
|
||||
// Filter out resumes that already exist
|
||||
const resumesToInsert = resumesWithValidUsers.filter(({ resume, newUserId }) => {
|
||||
const key = `${resume.slug}:${newUserId}`;
|
||||
if (existingResumeKeys.has(key)) {
|
||||
skipped++;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
// Filter out resumes that already exist
|
||||
const resumesToInsert = resumesWithValidUsers.filter(({ resume, newUserId }) => {
|
||||
const key = `${resume.slug}:${newUserId}`;
|
||||
if (existingResumeKeys.has(key)) {
|
||||
skipped++;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
if (resumesToInsert.length === 0) {
|
||||
console.log(`⏭️ All resumes in this batch already exist in target DB.`);
|
||||
// Update cursor to the last resume in this batch
|
||||
const lastResume = resumes[resumes.length - 1];
|
||||
if (lastResume) {
|
||||
lastSeenCreatedAt =
|
||||
lastResume.createdAt instanceof Date ? lastResume.createdAt.toISOString() : String(lastResume.createdAt);
|
||||
lastSeenId = lastResume.id;
|
||||
}
|
||||
totalResumesProcessed += resumes.length;
|
||||
await saveProgress(getCurrentProgress());
|
||||
continue;
|
||||
}
|
||||
if (resumesToInsert.length === 0) {
|
||||
console.log(`⏭️ All resumes in this batch already exist in target DB.`);
|
||||
// Update cursor to the last resume in this batch
|
||||
const lastResume = resumes[resumes.length - 1];
|
||||
if (lastResume) {
|
||||
lastSeenCreatedAt =
|
||||
lastResume.createdAt instanceof Date ? lastResume.createdAt.toISOString() : String(lastResume.createdAt);
|
||||
lastSeenId = lastResume.id;
|
||||
}
|
||||
totalResumesProcessed += resumes.length;
|
||||
await saveProgress(getCurrentProgress());
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log(`📝 Preparing to bulk insert ${resumesToInsert.length} resumes...`);
|
||||
console.log(`📝 Preparing to bulk insert ${resumesToInsert.length} resumes...`);
|
||||
|
||||
// Prepare bulk insert data
|
||||
const batchStart = performance.now();
|
||||
try {
|
||||
const resumesToInsertData = resumesToInsert.map(({ resume, newUserId }) => {
|
||||
// Transform the data using the V4 importer
|
||||
let transformedData = defaultResumeData;
|
||||
try {
|
||||
const dataJson = typeof resume.data === "string" ? resume.data : JSON.stringify(resume.data);
|
||||
transformedData = importer.parse(dataJson);
|
||||
} catch (error) {
|
||||
console.error(`⚠️ Failed to parse resume data for resume ${resume.id}, using default data:`, error);
|
||||
// Use default data if parsing fails
|
||||
transformedData = defaultResumeData;
|
||||
}
|
||||
// Prepare bulk insert data
|
||||
const batchStart = performance.now();
|
||||
try {
|
||||
const resumesToInsertData = resumesToInsert.map(({ resume, newUserId }) => {
|
||||
// Transform the data using the V4 importer
|
||||
let transformedData = defaultResumeData;
|
||||
try {
|
||||
const dataJson = typeof resume.data === "string" ? resume.data : JSON.stringify(resume.data);
|
||||
transformedData = importer.parse(dataJson);
|
||||
} catch (error) {
|
||||
console.error(`⚠️ Failed to parse resume data for resume ${resume.id}, using default data:`, error);
|
||||
// Use default data if parsing fails
|
||||
transformedData = defaultResumeData;
|
||||
}
|
||||
|
||||
// Map visibility to isPublic (visibility === "public" -> isPublic = true)
|
||||
const isPublic = resume.visibility === "public";
|
||||
// Map visibility to isPublic (visibility === "public" -> isPublic = true)
|
||||
const isPublic = resume.visibility === "public";
|
||||
|
||||
const newResumeId = generateId();
|
||||
const newResumeId = generateId();
|
||||
|
||||
// Track the ID mapping for future reference
|
||||
resumeIdMap.set(resume.id, newResumeId);
|
||||
// Track the ID mapping for future reference
|
||||
resumeIdMap.set(resume.id, newResumeId);
|
||||
|
||||
return {
|
||||
resumeData: {
|
||||
id: newResumeId,
|
||||
name: resume.title,
|
||||
slug: resume.slug,
|
||||
tags: [], // Default empty array
|
||||
isPublic: isPublic,
|
||||
isLocked: resume.locked,
|
||||
password: null, // No password in old schema
|
||||
data: transformedData,
|
||||
userId: newUserId,
|
||||
createdAt: resume.createdAt,
|
||||
updatedAt: resume.updatedAt,
|
||||
},
|
||||
originalResumeId: resume.id,
|
||||
newResumeId: newResumeId,
|
||||
};
|
||||
});
|
||||
return {
|
||||
resumeData: {
|
||||
id: newResumeId,
|
||||
name: resume.title,
|
||||
slug: resume.slug,
|
||||
tags: [], // Default empty array
|
||||
isPublic: isPublic,
|
||||
isLocked: resume.locked,
|
||||
password: null, // No password in old schema
|
||||
data: transformedData,
|
||||
userId: newUserId,
|
||||
createdAt: resume.createdAt,
|
||||
updatedAt: resume.updatedAt,
|
||||
},
|
||||
originalResumeId: resume.id,
|
||||
newResumeId: newResumeId,
|
||||
};
|
||||
});
|
||||
|
||||
// Bulk insert resumes (chunked to avoid PostgreSQL message size limits)
|
||||
// Resumes contain large JSONB data, so we use smaller chunks
|
||||
const resumeDataList = resumesToInsertData.map(({ resumeData }) => resumeData);
|
||||
for (let i = 0; i < resumeDataList.length; i += INSERT_CHUNK_SIZE) {
|
||||
const chunk = resumeDataList.slice(i, i + INSERT_CHUNK_SIZE);
|
||||
await localDb.insert(schema.resume).values(chunk);
|
||||
}
|
||||
resumesCreated += resumesToInsertData.length;
|
||||
// Bulk insert resumes (chunked to avoid PostgreSQL message size limits)
|
||||
// Resumes contain large JSONB data, so we use smaller chunks
|
||||
const resumeDataList = resumesToInsertData.map(({ resumeData }) => resumeData);
|
||||
for (let i = 0; i < resumeDataList.length; i += INSERT_CHUNK_SIZE) {
|
||||
const chunk = resumeDataList.slice(i, i + INSERT_CHUNK_SIZE);
|
||||
await localDb.insert(schema.resume).values(chunk);
|
||||
}
|
||||
resumesCreated += resumesToInsertData.length;
|
||||
|
||||
// Prepare statistics for bulk insert
|
||||
const statisticsToInsert = resumesToInsertData
|
||||
.map(({ originalResumeId, newResumeId }) => {
|
||||
const resumeStatistics = statisticsMap.get(originalResumeId);
|
||||
if (!resumeStatistics) return null;
|
||||
// Prepare statistics for bulk insert
|
||||
const statisticsToInsert = resumesToInsertData
|
||||
.map(({ originalResumeId, newResumeId }) => {
|
||||
const resumeStatistics = statisticsMap.get(originalResumeId);
|
||||
if (!resumeStatistics) return null;
|
||||
|
||||
return {
|
||||
id: generateId(),
|
||||
views: resumeStatistics.views,
|
||||
downloads: resumeStatistics.downloads,
|
||||
lastViewedAt: null, // Not available in old schema
|
||||
lastDownloadedAt: null, // Not available in old schema
|
||||
resumeId: newResumeId,
|
||||
createdAt: resumeStatistics.createdAt,
|
||||
updatedAt: resumeStatistics.updatedAt,
|
||||
};
|
||||
})
|
||||
.filter((stat): stat is NonNullable<typeof stat> => stat !== null);
|
||||
return {
|
||||
id: generateId(),
|
||||
views: resumeStatistics.views,
|
||||
downloads: resumeStatistics.downloads,
|
||||
lastViewedAt: null, // Not available in old schema
|
||||
lastDownloadedAt: null, // Not available in old schema
|
||||
resumeId: newResumeId,
|
||||
createdAt: resumeStatistics.createdAt,
|
||||
updatedAt: resumeStatistics.updatedAt,
|
||||
};
|
||||
})
|
||||
.filter((stat): stat is NonNullable<typeof stat> => stat !== null);
|
||||
|
||||
// Bulk insert statistics (chunked)
|
||||
if (statisticsToInsert.length > 0) {
|
||||
for (let i = 0; i < statisticsToInsert.length; i += INSERT_CHUNK_SIZE) {
|
||||
const chunk = statisticsToInsert.slice(i, i + INSERT_CHUNK_SIZE);
|
||||
await localDb.insert(schema.resumeStatistics).values(chunk);
|
||||
}
|
||||
statisticsCreated += statisticsToInsert.length;
|
||||
}
|
||||
// Bulk insert statistics (chunked)
|
||||
if (statisticsToInsert.length > 0) {
|
||||
for (let i = 0; i < statisticsToInsert.length; i += INSERT_CHUNK_SIZE) {
|
||||
const chunk = statisticsToInsert.slice(i, i + INSERT_CHUNK_SIZE);
|
||||
await localDb.insert(schema.resumeStatistics).values(chunk);
|
||||
}
|
||||
statisticsCreated += statisticsToInsert.length;
|
||||
}
|
||||
|
||||
const batchEnd = performance.now();
|
||||
const batchTimeMs = batchEnd - batchStart;
|
||||
console.log(
|
||||
`✅ Bulk inserted ${resumesToInsertData.length} resumes in ${batchTimeMs.toFixed(1)} ms (avg ${(batchTimeMs / resumesToInsertData.length).toFixed(1)} ms/resume)`,
|
||||
);
|
||||
const batchEnd = performance.now();
|
||||
const batchTimeMs = batchEnd - batchStart;
|
||||
console.log(
|
||||
`✅ Bulk inserted ${resumesToInsertData.length} resumes in ${batchTimeMs.toFixed(1)} ms (avg ${(batchTimeMs / resumesToInsertData.length).toFixed(1)} ms/resume)`,
|
||||
);
|
||||
|
||||
// Save resume ID map after each successful batch
|
||||
await saveResumeIdMapToFile(resumeIdMap);
|
||||
} catch (error) {
|
||||
console.error(`🚨 Failed to bulk insert resumes batch:`, error);
|
||||
errors++;
|
||||
// Continue with next batch even if this one fails
|
||||
}
|
||||
// Save resume ID map after each successful batch
|
||||
await saveResumeIdMapToFile(resumeIdMap);
|
||||
} catch (error) {
|
||||
console.error(`🚨 Failed to bulk insert resumes batch:`, error);
|
||||
errors++;
|
||||
// Continue with next batch even if this one fails
|
||||
}
|
||||
|
||||
// Update cursor to the last resume in this batch
|
||||
const lastResume = resumes[resumes.length - 1];
|
||||
if (lastResume) {
|
||||
lastSeenCreatedAt =
|
||||
lastResume.createdAt instanceof Date ? lastResume.createdAt.toISOString() : String(lastResume.createdAt);
|
||||
lastSeenId = lastResume.id;
|
||||
}
|
||||
// Update cursor to the last resume in this batch
|
||||
const lastResume = resumes[resumes.length - 1];
|
||||
if (lastResume) {
|
||||
lastSeenCreatedAt =
|
||||
lastResume.createdAt instanceof Date ? lastResume.createdAt.toISOString() : String(lastResume.createdAt);
|
||||
lastSeenId = lastResume.id;
|
||||
}
|
||||
|
||||
totalResumesProcessed += resumes.length;
|
||||
console.log(`📦 Processed ${totalResumesProcessed} resumes so far...\n`);
|
||||
totalResumesProcessed += resumes.length;
|
||||
console.log(`📦 Processed ${totalResumesProcessed} resumes so far...\n`);
|
||||
|
||||
// Save progress after each batch
|
||||
await saveProgress(getCurrentProgress());
|
||||
}
|
||||
// Save progress after each batch
|
||||
await saveProgress(getCurrentProgress());
|
||||
}
|
||||
|
||||
// Remove signal handlers
|
||||
process.off("SIGINT", handleShutdown);
|
||||
process.off("SIGTERM", handleShutdown);
|
||||
// Remove signal handlers
|
||||
process.off("SIGINT", handleShutdown);
|
||||
process.off("SIGTERM", handleShutdown);
|
||||
|
||||
const migrationEnd = performance.now();
|
||||
const migrationDurationMs = migrationEnd - migrationStart;
|
||||
const migrationEnd = performance.now();
|
||||
const migrationDurationMs = migrationEnd - migrationStart;
|
||||
|
||||
console.log("\n📊 Migration Summary:");
|
||||
console.log(` Resumes created: ${resumesCreated}`);
|
||||
console.log(` Statistics created: ${statisticsCreated}`);
|
||||
console.log(` Skipped (userId not found or already exist): ${skipped}`);
|
||||
console.log(` Errors: ${errors}`);
|
||||
console.log(
|
||||
`⏱️ Total migration time: ${migrationDurationMs.toFixed(1)} ms (${(migrationDurationMs / 1000).toFixed(2)} seconds)`,
|
||||
);
|
||||
console.log("\n📊 Migration Summary:");
|
||||
console.log(` Resumes created: ${resumesCreated}`);
|
||||
console.log(` Statistics created: ${statisticsCreated}`);
|
||||
console.log(` Skipped (userId not found or already exist): ${skipped}`);
|
||||
console.log(` Errors: ${errors}`);
|
||||
console.log(
|
||||
`⏱️ Total migration time: ${migrationDurationMs.toFixed(1)} ms (${(migrationDurationMs / 1000).toFixed(2)} seconds)`,
|
||||
);
|
||||
|
||||
// Final save of the mapping (ensures up-to-date state)
|
||||
await saveResumeIdMapToFile(resumeIdMap);
|
||||
// Final save of the mapping (ensures up-to-date state)
|
||||
await saveResumeIdMapToFile(resumeIdMap);
|
||||
|
||||
// Clear progress file on successful completion (only if not interrupted)
|
||||
if (!shutdownRequested) {
|
||||
await clearProgress();
|
||||
console.log("✅ Resume migration complete!");
|
||||
} else {
|
||||
console.log("⏸️ Migration paused. Run again to resume.");
|
||||
}
|
||||
// Clear progress file on successful completion (only if not interrupted)
|
||||
if (!shutdownRequested) {
|
||||
await clearProgress();
|
||||
console.log("✅ Resume migration complete!");
|
||||
} else {
|
||||
console.log("⏸️ Migration paused. Run again to resume.");
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
// Reset shutdown flag for fresh run
|
||||
shutdownRequested = false;
|
||||
// Reset shutdown flag for fresh run
|
||||
shutdownRequested = false;
|
||||
|
||||
try {
|
||||
await migrateResumes();
|
||||
} finally {
|
||||
await productionPool.end();
|
||||
await localPool.end();
|
||||
}
|
||||
try {
|
||||
await migrateResumes();
|
||||
} finally {
|
||||
await productionPool.end();
|
||||
await localPool.end();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,16 +13,16 @@ const localPool = new Pool({ connectionString: localUrl });
|
||||
const localDb = drizzle({ client: localPool });
|
||||
|
||||
try {
|
||||
const productionResult = await productionDb.execute(sql`SELECT 1 as connected`);
|
||||
console.log("✅ Production database connection successful", JSON.stringify(productionResult));
|
||||
const productionResult = await productionDb.execute(sql`SELECT 1 as connected`);
|
||||
console.log("✅ Production database connection successful", JSON.stringify(productionResult));
|
||||
|
||||
const localResult = await localDb.execute(sql`SELECT 1 as connected`);
|
||||
console.log("✅ Local database connection successful", JSON.stringify(localResult));
|
||||
const localResult = await localDb.execute(sql`SELECT 1 as connected`);
|
||||
console.log("✅ Local database connection successful", JSON.stringify(localResult));
|
||||
} catch (error) {
|
||||
console.error("🚨 Database connection failed:", error);
|
||||
process.exit(1);
|
||||
console.error("🚨 Database connection failed:", error);
|
||||
process.exit(1);
|
||||
} finally {
|
||||
await productionPool.end();
|
||||
await localPool.end();
|
||||
process.exit(0);
|
||||
await productionPool.end();
|
||||
await localPool.end();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
+317
-317
@@ -12,37 +12,37 @@ type Provider = "credential" | "google" | "github" | "custom";
|
||||
type ProductionProvider = "email" | "github" | "google" | "openid";
|
||||
|
||||
interface ProductionUser {
|
||||
id: string;
|
||||
name: string;
|
||||
picture: string | null;
|
||||
username: string;
|
||||
email: string;
|
||||
locale: string;
|
||||
emailVerified: boolean;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
provider: ProductionProvider;
|
||||
id: string;
|
||||
name: string;
|
||||
picture: string | null;
|
||||
username: string;
|
||||
email: string;
|
||||
locale: string;
|
||||
emailVerified: boolean;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
provider: ProductionProvider;
|
||||
}
|
||||
|
||||
interface ProductionSecrets {
|
||||
id: string;
|
||||
password: string | null;
|
||||
lastSignedIn: Date;
|
||||
userId: string;
|
||||
id: string;
|
||||
password: string | null;
|
||||
lastSignedIn: Date;
|
||||
userId: string;
|
||||
}
|
||||
|
||||
// Map old provider to new providerId
|
||||
function mapProviderId(provider: ProductionProvider): Provider {
|
||||
switch (provider) {
|
||||
case "email":
|
||||
return "credential";
|
||||
case "google":
|
||||
return "google";
|
||||
case "github":
|
||||
return "github";
|
||||
default:
|
||||
return "custom";
|
||||
}
|
||||
switch (provider) {
|
||||
case "email":
|
||||
return "credential";
|
||||
case "google":
|
||||
return "google";
|
||||
case "github":
|
||||
return "github";
|
||||
default:
|
||||
return "custom";
|
||||
}
|
||||
}
|
||||
|
||||
const productionUrl = process.env.PRODUCTION_DATABASE_URL;
|
||||
@@ -72,380 +72,380 @@ const INSERT_CHUNK_SIZE = 1000;
|
||||
// == Progress checkpoint interface ==
|
||||
// Uses cursor-based pagination with (createdAt, id) composite key for efficiency
|
||||
interface MigrationProgress {
|
||||
// Cursor for pagination - last seen createdAt timestamp
|
||||
lastSeenCreatedAt: string | null;
|
||||
// Cursor for pagination - last seen id (for tiebreaker when timestamps are equal)
|
||||
lastSeenId: string | null;
|
||||
usersCreated: number;
|
||||
accountsCreated: number;
|
||||
skipped: number;
|
||||
totalUsersProcessed: number;
|
||||
lastUpdated: string;
|
||||
// Cursor for pagination - last seen createdAt timestamp
|
||||
lastSeenCreatedAt: string | null;
|
||||
// Cursor for pagination - last seen id (for tiebreaker when timestamps are equal)
|
||||
lastSeenId: string | null;
|
||||
usersCreated: number;
|
||||
accountsCreated: number;
|
||||
skipped: number;
|
||||
totalUsersProcessed: number;
|
||||
lastUpdated: string;
|
||||
}
|
||||
|
||||
// Flag to track if shutdown was requested
|
||||
let shutdownRequested = false;
|
||||
|
||||
async function loadProgress(): Promise<MigrationProgress | null> {
|
||||
try {
|
||||
const text = await fs.readFile(PROGRESS_FILE, { encoding: "utf-8" });
|
||||
const progress = JSON.parse(text) as MigrationProgress;
|
||||
console.log(`📂 Found existing progress file. Last updated: ${progress.lastUpdated}`);
|
||||
console.log(` Resuming from cursor (createdAt: ${progress.lastSeenCreatedAt}, id: ${progress.lastSeenId})...`);
|
||||
return progress;
|
||||
} catch (e) {
|
||||
console.warn("⚠️ Failed to load progress file, starting from beginning.", e);
|
||||
}
|
||||
return null;
|
||||
try {
|
||||
const text = await fs.readFile(PROGRESS_FILE, { encoding: "utf-8" });
|
||||
const progress = JSON.parse(text) as MigrationProgress;
|
||||
console.log(`📂 Found existing progress file. Last updated: ${progress.lastUpdated}`);
|
||||
console.log(` Resuming from cursor (createdAt: ${progress.lastSeenCreatedAt}, id: ${progress.lastSeenId})...`);
|
||||
return progress;
|
||||
} catch (e) {
|
||||
console.warn("⚠️ Failed to load progress file, starting from beginning.", e);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function saveProgress(progress: MigrationProgress): Promise<void> {
|
||||
try {
|
||||
progress.lastUpdated = new Date().toISOString();
|
||||
await fs.writeFile(PROGRESS_FILE, JSON.stringify(progress, null, 2), { encoding: "utf-8" });
|
||||
console.log(`💾 Progress saved at cursor (createdAt: ${progress.lastSeenCreatedAt}, id: ${progress.lastSeenId})`);
|
||||
} catch (e) {
|
||||
console.error("🚨 Failed to save progress:", e);
|
||||
}
|
||||
try {
|
||||
progress.lastUpdated = new Date().toISOString();
|
||||
await fs.writeFile(PROGRESS_FILE, JSON.stringify(progress, null, 2), { encoding: "utf-8" });
|
||||
console.log(`💾 Progress saved at cursor (createdAt: ${progress.lastSeenCreatedAt}, id: ${progress.lastSeenId})`);
|
||||
} catch (e) {
|
||||
console.error("🚨 Failed to save progress:", e);
|
||||
}
|
||||
}
|
||||
|
||||
async function clearProgress(): Promise<void> {
|
||||
try {
|
||||
await fs.unlink(PROGRESS_FILE);
|
||||
console.log("🗑️ Progress file cleared.");
|
||||
} catch {
|
||||
// Ignore errors when clearing
|
||||
}
|
||||
try {
|
||||
await fs.unlink(PROGRESS_FILE);
|
||||
console.log("🗑️ Progress file cleared.");
|
||||
} catch {
|
||||
// Ignore errors when clearing
|
||||
}
|
||||
}
|
||||
|
||||
async function loadUserIdMapFromFile(): Promise<Map<string, string>> {
|
||||
try {
|
||||
const text = await fs.readFile(USER_ID_MAP_FILE, { encoding: "utf-8" });
|
||||
const obj = JSON.parse(text);
|
||||
return new Map(Object.entries(obj));
|
||||
} catch (e) {
|
||||
console.warn("⚠️ Failed to load userIdMap from disk, continuing with empty map.", e);
|
||||
}
|
||||
return new Map<string, string>();
|
||||
try {
|
||||
const text = await fs.readFile(USER_ID_MAP_FILE, { encoding: "utf-8" });
|
||||
const obj = JSON.parse(text);
|
||||
return new Map(Object.entries(obj));
|
||||
} catch (e) {
|
||||
console.warn("⚠️ Failed to load userIdMap from disk, continuing with empty map.", e);
|
||||
}
|
||||
return new Map<string, string>();
|
||||
}
|
||||
|
||||
async function saveUserIdMapToFile(userIdMap: Map<string, string>) {
|
||||
const obj: Record<string, string> = Object.fromEntries(userIdMap.entries());
|
||||
await fs.writeFile(USER_ID_MAP_FILE, JSON.stringify(obj, null, "\t"), { encoding: "utf-8" });
|
||||
const obj: Record<string, string> = Object.fromEntries(userIdMap.entries());
|
||||
await fs.writeFile(USER_ID_MAP_FILE, JSON.stringify(obj, null, "\t"), { encoding: "utf-8" });
|
||||
}
|
||||
|
||||
export async function migrateUsers() {
|
||||
const migrationStart = performance.now();
|
||||
console.log("⌛ Starting user migration...");
|
||||
const migrationStart = performance.now();
|
||||
console.log("⌛ Starting user migration...");
|
||||
|
||||
let hasMore = true;
|
||||
let hasMore = true;
|
||||
|
||||
// Cursor-based pagination state
|
||||
let lastSeenCreatedAt: string | null = null;
|
||||
let lastSeenId: string | null = null;
|
||||
// Cursor-based pagination state
|
||||
let lastSeenCreatedAt: string | null = null;
|
||||
let lastSeenId: string | null = null;
|
||||
|
||||
// Load persistent userIdMap from file
|
||||
const userIdMap = await loadUserIdMapFromFile();
|
||||
// Load persistent userIdMap from file
|
||||
const userIdMap = await loadUserIdMapFromFile();
|
||||
|
||||
// Track migration stats
|
||||
let usersCreated = 0;
|
||||
let accountsCreated = 0;
|
||||
let skipped = 0;
|
||||
let totalUsersProcessed = 0;
|
||||
// Track migration stats
|
||||
let usersCreated = 0;
|
||||
let accountsCreated = 0;
|
||||
let skipped = 0;
|
||||
let totalUsersProcessed = 0;
|
||||
|
||||
// Load saved progress if exists
|
||||
const savedProgress = await loadProgress();
|
||||
if (savedProgress) {
|
||||
lastSeenCreatedAt = savedProgress.lastSeenCreatedAt;
|
||||
lastSeenId = savedProgress.lastSeenId;
|
||||
usersCreated = savedProgress.usersCreated;
|
||||
accountsCreated = savedProgress.accountsCreated;
|
||||
skipped = savedProgress.skipped;
|
||||
totalUsersProcessed = savedProgress.totalUsersProcessed;
|
||||
}
|
||||
// Load saved progress if exists
|
||||
const savedProgress = await loadProgress();
|
||||
if (savedProgress) {
|
||||
lastSeenCreatedAt = savedProgress.lastSeenCreatedAt;
|
||||
lastSeenId = savedProgress.lastSeenId;
|
||||
usersCreated = savedProgress.usersCreated;
|
||||
accountsCreated = savedProgress.accountsCreated;
|
||||
skipped = savedProgress.skipped;
|
||||
totalUsersProcessed = savedProgress.totalUsersProcessed;
|
||||
}
|
||||
|
||||
// Helper to get current progress object
|
||||
const getCurrentProgress = (): MigrationProgress => ({
|
||||
lastSeenCreatedAt,
|
||||
lastSeenId,
|
||||
usersCreated,
|
||||
accountsCreated,
|
||||
skipped,
|
||||
totalUsersProcessed,
|
||||
lastUpdated: new Date().toISOString(),
|
||||
});
|
||||
// Helper to get current progress object
|
||||
const getCurrentProgress = (): MigrationProgress => ({
|
||||
lastSeenCreatedAt,
|
||||
lastSeenId,
|
||||
usersCreated,
|
||||
accountsCreated,
|
||||
skipped,
|
||||
totalUsersProcessed,
|
||||
lastUpdated: new Date().toISOString(),
|
||||
});
|
||||
|
||||
// Setup graceful shutdown handler
|
||||
const handleShutdown = async () => {
|
||||
if (shutdownRequested) return;
|
||||
shutdownRequested = true;
|
||||
console.log("\n⚠️ Shutdown requested. Saving progress...");
|
||||
await saveProgress(getCurrentProgress());
|
||||
await saveUserIdMapToFile(userIdMap);
|
||||
console.log("👋 Exiting. Run the script again to resume from where you left off.");
|
||||
process.exit(0);
|
||||
};
|
||||
// Setup graceful shutdown handler
|
||||
const handleShutdown = async () => {
|
||||
if (shutdownRequested) return;
|
||||
shutdownRequested = true;
|
||||
console.log("\n⚠️ Shutdown requested. Saving progress...");
|
||||
await saveProgress(getCurrentProgress());
|
||||
await saveUserIdMapToFile(userIdMap);
|
||||
console.log("👋 Exiting. Run the script again to resume from where you left off.");
|
||||
process.exit(0);
|
||||
};
|
||||
|
||||
process.on("SIGINT", handleShutdown);
|
||||
process.on("SIGTERM", handleShutdown);
|
||||
process.on("SIGINT", handleShutdown);
|
||||
process.on("SIGTERM", handleShutdown);
|
||||
|
||||
while (hasMore) {
|
||||
// Check if shutdown was requested
|
||||
if (shutdownRequested) break;
|
||||
while (hasMore) {
|
||||
// Check if shutdown was requested
|
||||
if (shutdownRequested) break;
|
||||
|
||||
console.log(
|
||||
`📥 Fetching users batch from production database (cursor: createdAt=${lastSeenCreatedAt}, id=${lastSeenId})...`,
|
||||
);
|
||||
console.log(
|
||||
`📥 Fetching users batch from production database (cursor: createdAt=${lastSeenCreatedAt}, id=${lastSeenId})...`,
|
||||
);
|
||||
|
||||
let users: ProductionUser[];
|
||||
let users: ProductionUser[];
|
||||
|
||||
if (lastSeenCreatedAt && lastSeenId) {
|
||||
const result = (await productionDb.execute(sql`
|
||||
if (lastSeenCreatedAt && lastSeenId) {
|
||||
const result = (await productionDb.execute(sql`
|
||||
SELECT id, name, picture, username, email, locale, "emailVerified", "createdAt", "updatedAt", provider
|
||||
FROM "User"
|
||||
WHERE ("createdAt", id) < (${lastSeenCreatedAt}::timestamp, ${lastSeenId})
|
||||
ORDER BY "createdAt" DESC, id DESC
|
||||
LIMIT ${BATCH_SIZE}
|
||||
`)) as unknown as QueryResult<ProductionUser>;
|
||||
users = result.rows;
|
||||
} else {
|
||||
const result = (await productionDb.execute(sql`
|
||||
users = result.rows;
|
||||
} else {
|
||||
const result = (await productionDb.execute(sql`
|
||||
SELECT id, name, picture, username, email, locale, "emailVerified", "createdAt", "updatedAt", provider
|
||||
FROM "User"
|
||||
ORDER BY "createdAt" DESC, id DESC
|
||||
LIMIT ${BATCH_SIZE}
|
||||
`)) as unknown as QueryResult<ProductionUser>;
|
||||
users = result.rows;
|
||||
}
|
||||
users = result.rows;
|
||||
}
|
||||
|
||||
console.log(`📋 Found ${users.length} users in this batch.`);
|
||||
console.log(`📋 Found ${users.length} users in this batch.`);
|
||||
|
||||
if (users.length === 0) {
|
||||
hasMore = false;
|
||||
break;
|
||||
}
|
||||
if (users.length === 0) {
|
||||
hasMore = false;
|
||||
break;
|
||||
}
|
||||
|
||||
// Fetch secrets only for these users in this batch
|
||||
const userIds = users.map((u) => u.id);
|
||||
// Fetch secrets only for these users in this batch
|
||||
const userIds = users.map((u) => u.id);
|
||||
|
||||
// Drizzle does not interpolate arrays, so we join and use a custom SQL string
|
||||
// Escape single quotes in IDs (though UUIDs shouldn't contain them, this is safer)
|
||||
const userIdsForSql = userIds.map((id) => `'${id.replace(/'/g, "''")}'`).join(", ");
|
||||
// Drizzle does not interpolate arrays, so we join and use a custom SQL string
|
||||
// Escape single quotes in IDs (though UUIDs shouldn't contain them, this is safer)
|
||||
const userIdsForSql = userIds.map((id) => `'${id.replace(/'/g, "''")}'`).join(", ");
|
||||
|
||||
const { rows: secrets } = (await productionDb.execute(sql`
|
||||
const { rows: secrets } = (await productionDb.execute(sql`
|
||||
SELECT id, password, "lastSignedIn", "userId"
|
||||
FROM "Secrets"
|
||||
WHERE "userId" IN (${sql.raw(userIdsForSql)})
|
||||
`)) as unknown as QueryResult<ProductionSecrets>;
|
||||
|
||||
// Create a map of userId -> secrets for quick lookup
|
||||
const secretsMap = new Map<string, ProductionSecrets>();
|
||||
for (const secret of secrets) {
|
||||
secretsMap.set(secret.userId, secret);
|
||||
}
|
||||
// Create a map of userId -> secrets for quick lookup
|
||||
const secretsMap = new Map<string, ProductionSecrets>();
|
||||
for (const secret of secrets) {
|
||||
secretsMap.set(secret.userId, secret);
|
||||
}
|
||||
|
||||
// Filter out users already in userIdMap (previously migrated)
|
||||
const usersToProcess = users.filter((user) => {
|
||||
if (userIdMap.has(user.id)) {
|
||||
skipped++;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
// Filter out users already in userIdMap (previously migrated)
|
||||
const usersToProcess = users.filter((user) => {
|
||||
if (userIdMap.has(user.id)) {
|
||||
skipped++;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
if (usersToProcess.length === 0) {
|
||||
console.log(`⏭️ All users in this batch were already migrated.`);
|
||||
// Update cursor to the last user in this batch
|
||||
const lastUser = users[users.length - 1];
|
||||
if (lastUser) {
|
||||
lastSeenCreatedAt =
|
||||
lastUser.createdAt instanceof Date ? lastUser.createdAt.toISOString() : String(lastUser.createdAt);
|
||||
lastSeenId = lastUser.id;
|
||||
}
|
||||
totalUsersProcessed += users.length;
|
||||
await saveProgress(getCurrentProgress());
|
||||
continue;
|
||||
}
|
||||
if (usersToProcess.length === 0) {
|
||||
console.log(`⏭️ All users in this batch were already migrated.`);
|
||||
// Update cursor to the last user in this batch
|
||||
const lastUser = users[users.length - 1];
|
||||
if (lastUser) {
|
||||
lastSeenCreatedAt =
|
||||
lastUser.createdAt instanceof Date ? lastUser.createdAt.toISOString() : String(lastUser.createdAt);
|
||||
lastSeenId = lastUser.id;
|
||||
}
|
||||
totalUsersProcessed += users.length;
|
||||
await saveProgress(getCurrentProgress());
|
||||
continue;
|
||||
}
|
||||
|
||||
// Prepare usernames for all users
|
||||
const userData = usersToProcess.map((user) => ({
|
||||
user,
|
||||
username: toUsername(user.username),
|
||||
displayUsername: user.username,
|
||||
}));
|
||||
// Prepare usernames for all users
|
||||
const userData = usersToProcess.map((user) => ({
|
||||
user,
|
||||
username: toUsername(user.username),
|
||||
displayUsername: user.username,
|
||||
}));
|
||||
|
||||
// Bulk check for existing users (by email or username)
|
||||
const emails = userData.map((u) => u.user.email);
|
||||
const usernames = userData.map((u) => u.username);
|
||||
const displayUsernames = userData.map((u) => u.displayUsername);
|
||||
// Bulk check for existing users (by email or username)
|
||||
const emails = userData.map((u) => u.user.email);
|
||||
const usernames = userData.map((u) => u.username);
|
||||
const displayUsernames = userData.map((u) => u.displayUsername);
|
||||
|
||||
const existingUsers = await localDb
|
||||
.select()
|
||||
.from(schema.user)
|
||||
.where(
|
||||
or(
|
||||
inArray(schema.user.email, emails),
|
||||
inArray(schema.user.username, usernames),
|
||||
inArray(schema.user.displayUsername, displayUsernames),
|
||||
),
|
||||
);
|
||||
const existingUsers = await localDb
|
||||
.select()
|
||||
.from(schema.user)
|
||||
.where(
|
||||
or(
|
||||
inArray(schema.user.email, emails),
|
||||
inArray(schema.user.username, usernames),
|
||||
inArray(schema.user.displayUsername, displayUsernames),
|
||||
),
|
||||
);
|
||||
|
||||
const existingEmails = new Set(existingUsers.map((u) => u.email));
|
||||
const existingUsernames = new Set(existingUsers.map((u) => u.username));
|
||||
const existingDisplayUsernames = new Set(existingUsers.map((u) => u.displayUsername));
|
||||
const existingEmails = new Set(existingUsers.map((u) => u.email));
|
||||
const existingUsernames = new Set(existingUsers.map((u) => u.username));
|
||||
const existingDisplayUsernames = new Set(existingUsers.map((u) => u.displayUsername));
|
||||
|
||||
// Filter out users that already exist
|
||||
const usersToInsert = userData.filter(({ user, username, displayUsername }) => {
|
||||
if (
|
||||
existingEmails.has(user.email) ||
|
||||
existingUsernames.has(username) ||
|
||||
existingDisplayUsernames.has(displayUsername)
|
||||
) {
|
||||
skipped++;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
// Filter out users that already exist
|
||||
const usersToInsert = userData.filter(({ user, username, displayUsername }) => {
|
||||
if (
|
||||
existingEmails.has(user.email) ||
|
||||
existingUsernames.has(username) ||
|
||||
existingDisplayUsernames.has(displayUsername)
|
||||
) {
|
||||
skipped++;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
|
||||
if (usersToInsert.length === 0) {
|
||||
console.log(`⏭️ All users in this batch already exist in target DB.`);
|
||||
// Update cursor to the last user in this batch
|
||||
const lastUser = users[users.length - 1];
|
||||
if (lastUser) {
|
||||
lastSeenCreatedAt =
|
||||
lastUser.createdAt instanceof Date ? lastUser.createdAt.toISOString() : String(lastUser.createdAt);
|
||||
lastSeenId = lastUser.id;
|
||||
}
|
||||
totalUsersProcessed += users.length;
|
||||
await saveUserIdMapToFile(userIdMap);
|
||||
await saveProgress(getCurrentProgress());
|
||||
continue;
|
||||
}
|
||||
if (usersToInsert.length === 0) {
|
||||
console.log(`⏭️ All users in this batch already exist in target DB.`);
|
||||
// Update cursor to the last user in this batch
|
||||
const lastUser = users[users.length - 1];
|
||||
if (lastUser) {
|
||||
lastSeenCreatedAt =
|
||||
lastUser.createdAt instanceof Date ? lastUser.createdAt.toISOString() : String(lastUser.createdAt);
|
||||
lastSeenId = lastUser.id;
|
||||
}
|
||||
totalUsersProcessed += users.length;
|
||||
await saveUserIdMapToFile(userIdMap);
|
||||
await saveProgress(getCurrentProgress());
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log(`📝 Preparing to bulk insert ${usersToInsert.length} users...`);
|
||||
console.log(`📝 Preparing to bulk insert ${usersToInsert.length} users...`);
|
||||
|
||||
// Prepare bulk insert data
|
||||
const usersToInsertData = usersToInsert.map(({ user, username, displayUsername }) => {
|
||||
const newUserId = generateId();
|
||||
userIdMap.set(user.id, newUserId);
|
||||
return {
|
||||
userData: {
|
||||
id: newUserId,
|
||||
name: user.name,
|
||||
email: user.email,
|
||||
image: user.picture,
|
||||
username: username,
|
||||
displayUsername: displayUsername,
|
||||
emailVerified: user.emailVerified,
|
||||
twoFactorEnabled: false, // All users start with 2FA disabled in the new system
|
||||
createdAt: user.createdAt,
|
||||
updatedAt: user.updatedAt,
|
||||
},
|
||||
originalUser: user,
|
||||
newUserId: newUserId,
|
||||
};
|
||||
});
|
||||
// Prepare bulk insert data
|
||||
const usersToInsertData = usersToInsert.map(({ user, username, displayUsername }) => {
|
||||
const newUserId = generateId();
|
||||
userIdMap.set(user.id, newUserId);
|
||||
return {
|
||||
userData: {
|
||||
id: newUserId,
|
||||
name: user.name,
|
||||
email: user.email,
|
||||
image: user.picture,
|
||||
username: username,
|
||||
displayUsername: displayUsername,
|
||||
emailVerified: user.emailVerified,
|
||||
twoFactorEnabled: false, // All users start with 2FA disabled in the new system
|
||||
createdAt: user.createdAt,
|
||||
updatedAt: user.updatedAt,
|
||||
},
|
||||
originalUser: user,
|
||||
newUserId: newUserId,
|
||||
};
|
||||
});
|
||||
|
||||
// Bulk insert users (chunked to avoid PostgreSQL message size limits)
|
||||
const batchStart = performance.now();
|
||||
try {
|
||||
const userDataList = usersToInsertData.map(({ userData }) => userData);
|
||||
for (let i = 0; i < userDataList.length; i += INSERT_CHUNK_SIZE) {
|
||||
const chunk = userDataList.slice(i, i + INSERT_CHUNK_SIZE);
|
||||
await localDb.insert(schema.user).values(chunk);
|
||||
}
|
||||
usersCreated += usersToInsertData.length;
|
||||
// Bulk insert users (chunked to avoid PostgreSQL message size limits)
|
||||
const batchStart = performance.now();
|
||||
try {
|
||||
const userDataList = usersToInsertData.map(({ userData }) => userData);
|
||||
for (let i = 0; i < userDataList.length; i += INSERT_CHUNK_SIZE) {
|
||||
const chunk = userDataList.slice(i, i + INSERT_CHUNK_SIZE);
|
||||
await localDb.insert(schema.user).values(chunk);
|
||||
}
|
||||
usersCreated += usersToInsertData.length;
|
||||
|
||||
// Prepare accounts for bulk insert
|
||||
const accountsToInsert = usersToInsertData.map(({ originalUser, newUserId, userData }) => {
|
||||
const userSecrets = secretsMap.get(originalUser.id);
|
||||
const providerId = mapProviderId(originalUser.provider);
|
||||
const accountId = providerId === "credential" ? newUserId : originalUser.id;
|
||||
// Prepare accounts for bulk insert
|
||||
const accountsToInsert = usersToInsertData.map(({ originalUser, newUserId, userData }) => {
|
||||
const userSecrets = secretsMap.get(originalUser.id);
|
||||
const providerId = mapProviderId(originalUser.provider);
|
||||
const accountId = providerId === "credential" ? newUserId : originalUser.id;
|
||||
|
||||
return {
|
||||
id: generateId(),
|
||||
userId: newUserId,
|
||||
accountId: accountId,
|
||||
providerId: providerId,
|
||||
password: userSecrets?.password ?? null,
|
||||
createdAt: userData.createdAt,
|
||||
updatedAt: userData.updatedAt,
|
||||
};
|
||||
});
|
||||
return {
|
||||
id: generateId(),
|
||||
userId: newUserId,
|
||||
accountId: accountId,
|
||||
providerId: providerId,
|
||||
password: userSecrets?.password ?? null,
|
||||
createdAt: userData.createdAt,
|
||||
updatedAt: userData.updatedAt,
|
||||
};
|
||||
});
|
||||
|
||||
// Bulk insert accounts (chunked)
|
||||
for (let i = 0; i < accountsToInsert.length; i += INSERT_CHUNK_SIZE) {
|
||||
const chunk = accountsToInsert.slice(i, i + INSERT_CHUNK_SIZE);
|
||||
await localDb.insert(schema.account).values(chunk);
|
||||
}
|
||||
accountsCreated += accountsToInsert.length;
|
||||
// Bulk insert accounts (chunked)
|
||||
for (let i = 0; i < accountsToInsert.length; i += INSERT_CHUNK_SIZE) {
|
||||
const chunk = accountsToInsert.slice(i, i + INSERT_CHUNK_SIZE);
|
||||
await localDb.insert(schema.account).values(chunk);
|
||||
}
|
||||
accountsCreated += accountsToInsert.length;
|
||||
|
||||
const batchEnd = performance.now();
|
||||
const batchTimeMs = batchEnd - batchStart;
|
||||
console.log(
|
||||
`✅ Bulk inserted ${usersToInsertData.length} users in ${batchTimeMs.toFixed(1)} ms (avg ${(batchTimeMs / usersToInsertData.length).toFixed(1)} ms/user)`,
|
||||
);
|
||||
const batchEnd = performance.now();
|
||||
const batchTimeMs = batchEnd - batchStart;
|
||||
console.log(
|
||||
`✅ Bulk inserted ${usersToInsertData.length} users in ${batchTimeMs.toFixed(1)} ms (avg ${(batchTimeMs / usersToInsertData.length).toFixed(1)} ms/user)`,
|
||||
);
|
||||
|
||||
// Save progress after each batch
|
||||
await saveUserIdMapToFile(userIdMap);
|
||||
} catch (error) {
|
||||
console.error(`🚨 Failed to bulk insert users batch:`, error);
|
||||
// Continue with next batch even if this one fails
|
||||
}
|
||||
// Save progress after each batch
|
||||
await saveUserIdMapToFile(userIdMap);
|
||||
} catch (error) {
|
||||
console.error(`🚨 Failed to bulk insert users batch:`, error);
|
||||
// Continue with next batch even if this one fails
|
||||
}
|
||||
|
||||
// Update cursor to the last user in this batch
|
||||
const lastUser = users[users.length - 1];
|
||||
if (lastUser) {
|
||||
lastSeenCreatedAt =
|
||||
lastUser.createdAt instanceof Date ? lastUser.createdAt.toISOString() : String(lastUser.createdAt);
|
||||
lastSeenId = lastUser.id;
|
||||
}
|
||||
// Update cursor to the last user in this batch
|
||||
const lastUser = users[users.length - 1];
|
||||
if (lastUser) {
|
||||
lastSeenCreatedAt =
|
||||
lastUser.createdAt instanceof Date ? lastUser.createdAt.toISOString() : String(lastUser.createdAt);
|
||||
lastSeenId = lastUser.id;
|
||||
}
|
||||
|
||||
totalUsersProcessed += users.length;
|
||||
console.log(`📦 Processed ${totalUsersProcessed} users so far...\n`);
|
||||
totalUsersProcessed += users.length;
|
||||
console.log(`📦 Processed ${totalUsersProcessed} users so far...\n`);
|
||||
|
||||
// Save progress after updating cursor
|
||||
await saveProgress(getCurrentProgress());
|
||||
}
|
||||
// Save progress after updating cursor
|
||||
await saveProgress(getCurrentProgress());
|
||||
}
|
||||
|
||||
// Remove signal handlers
|
||||
process.off("SIGINT", handleShutdown);
|
||||
process.off("SIGTERM", handleShutdown);
|
||||
// Remove signal handlers
|
||||
process.off("SIGINT", handleShutdown);
|
||||
process.off("SIGTERM", handleShutdown);
|
||||
|
||||
const migrationEnd = performance.now();
|
||||
const migrationDurationMs = migrationEnd - migrationStart;
|
||||
const migrationEnd = performance.now();
|
||||
const migrationDurationMs = migrationEnd - migrationStart;
|
||||
|
||||
console.log("\n📊 Migration Summary:");
|
||||
console.log(` Users created: ${usersCreated}`);
|
||||
console.log(` Accounts created: ${accountsCreated}`);
|
||||
console.log(` Skipped (already exist): ${skipped}`);
|
||||
console.log(
|
||||
`⏱️ Total migration time: ${migrationDurationMs.toFixed(1)} ms (${(migrationDurationMs / 1000).toFixed(2)} seconds)`,
|
||||
);
|
||||
console.log("\n📊 Migration Summary:");
|
||||
console.log(` Users created: ${usersCreated}`);
|
||||
console.log(` Accounts created: ${accountsCreated}`);
|
||||
console.log(` Skipped (already exist): ${skipped}`);
|
||||
console.log(
|
||||
`⏱️ Total migration time: ${migrationDurationMs.toFixed(1)} ms (${(migrationDurationMs / 1000).toFixed(2)} seconds)`,
|
||||
);
|
||||
|
||||
// Final save of the mapping (ensures up-to-date state)
|
||||
await saveUserIdMapToFile(userIdMap);
|
||||
// Final save of the mapping (ensures up-to-date state)
|
||||
await saveUserIdMapToFile(userIdMap);
|
||||
|
||||
// Clear progress file on successful completion (only if not interrupted)
|
||||
if (!shutdownRequested) {
|
||||
await clearProgress();
|
||||
console.log("✅ User migration complete!");
|
||||
} else {
|
||||
console.log("⏸️ Migration paused. Run again to resume.");
|
||||
}
|
||||
// Clear progress file on successful completion (only if not interrupted)
|
||||
if (!shutdownRequested) {
|
||||
await clearProgress();
|
||||
console.log("✅ User migration complete!");
|
||||
} else {
|
||||
console.log("⏸️ Migration paused. Run again to resume.");
|
||||
}
|
||||
|
||||
// Return the ID mapping for use in other migrations (e.g., resumes)
|
||||
return userIdMap;
|
||||
// Return the ID mapping for use in other migrations (e.g., resumes)
|
||||
return userIdMap;
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
// Reset shutdown flag for fresh run
|
||||
shutdownRequested = false;
|
||||
// Reset shutdown flag for fresh run
|
||||
shutdownRequested = false;
|
||||
|
||||
try {
|
||||
await migrateUsers();
|
||||
} finally {
|
||||
await productionPool.end();
|
||||
await localPool.end();
|
||||
}
|
||||
try {
|
||||
await migrateUsers();
|
||||
} finally {
|
||||
await productionPool.end();
|
||||
await localPool.end();
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user