mirror of
https://github.com/AmruthPillai/Reactive-Resume.git
synced 2026-07-12 05:55:02 +10:00
lot of bugfixes, better migration script
This commit is contained in:
@@ -9,15 +9,27 @@ export async function resetDatabase() {
|
||||
const pool = new Pool({ connectionString: env.DATABASE_URL });
|
||||
const db = drizzle({ client: pool });
|
||||
|
||||
// Extract the username from a PostgreSQL connection string like: postgresql://postgres:password@localhost:5432/db
|
||||
const username = (() => {
|
||||
try {
|
||||
const match = env.DATABASE_URL.match(/^postgres(?:ql)?:\/\/([^:]+):[^@]+@/);
|
||||
return match ? match[1] : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
})();
|
||||
|
||||
console.log("🔑 Username:", username);
|
||||
|
||||
try {
|
||||
await db.transaction(async (tx) => {
|
||||
await tx.execute(sql`DROP SCHEMA drizzle CASCADE`);
|
||||
await tx.execute(sql`CREATE SCHEMA drizzle`);
|
||||
await tx.execute(sql`GRANT ALL ON SCHEMA drizzle TO postgres`);
|
||||
await tx.execute(sql.raw(`GRANT ALL ON SCHEMA drizzle TO ${username}`));
|
||||
|
||||
await tx.execute(sql`DROP SCHEMA public CASCADE`);
|
||||
await tx.execute(sql`CREATE SCHEMA public`);
|
||||
await tx.execute(sql`GRANT ALL ON SCHEMA public TO postgres`);
|
||||
await tx.execute(sql.raw(`GRANT ALL ON SCHEMA public TO ${username}`));
|
||||
});
|
||||
|
||||
console.log("✅ Database reset completed");
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
import { drizzle } from "drizzle-orm/node-postgres";
|
||||
import { Pool } from "pg";
|
||||
import { schema } from "@/integrations/drizzle";
|
||||
import { env } from "@/utils/env";
|
||||
import { hashPassword } from "@/utils/password";
|
||||
import { generateId } from "@/utils/string";
|
||||
|
||||
export async function seedDatabase() {
|
||||
console.log("⌛ Seeding database...");
|
||||
|
||||
const pool = new Pool({ connectionString: env.DATABASE_URL });
|
||||
const db = drizzle({ client: pool, schema });
|
||||
|
||||
try {
|
||||
const userId = generateId();
|
||||
|
||||
await db.insert(schema.user).values({
|
||||
id: userId,
|
||||
name: "Test User",
|
||||
email: "test@test.com",
|
||||
username: "test",
|
||||
displayUsername: "test",
|
||||
emailVerified: true,
|
||||
image: "https://i.pravatar.cc/300",
|
||||
});
|
||||
|
||||
await db.insert(schema.account).values({
|
||||
id: generateId(),
|
||||
userId,
|
||||
accountId: userId,
|
||||
password: await hashPassword("password"),
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("🚨 Database seeding failed:", error);
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
await seedDatabase();
|
||||
}
|
||||
@@ -13,10 +13,20 @@ const argForce = args.includes("--force");
|
||||
const argCompress = args.includes("--compress");
|
||||
const argLimit = args.includes("--limit") ? parseInt(args[args.indexOf("--limit") + 1], 10) : 500;
|
||||
|
||||
const blacklistedFonts = ["Material Icons", "Material Symbols", "Noto Color Emoji"];
|
||||
const skippedFamilies = ["Material Icons", "Material Symbols", "Noto Color Emoji"];
|
||||
|
||||
const FONTS_DIR = "./scripts/fonts";
|
||||
const RESPONSE_FILE = `${FONTS_DIR}/response.json`;
|
||||
const WEBFONTLIST_FILE = `${FONTS_DIR}/webfontlist.json`;
|
||||
|
||||
async function getGoogleFontsJSON() {
|
||||
const contents = await readFile("data/fonts/response.json", "utf-8");
|
||||
let contents: string | null = null;
|
||||
|
||||
try {
|
||||
contents = await readFile(RESPONSE_FILE, "utf-8");
|
||||
} catch {
|
||||
// If the file doesn't exist or there's an error reading, just continue.
|
||||
}
|
||||
|
||||
if (!argForce && contents) return JSON.parse(contents) as APIResponse;
|
||||
|
||||
@@ -28,8 +38,8 @@ async function getGoogleFontsJSON() {
|
||||
const data = (await response.json()) as APIResponse;
|
||||
|
||||
const jsonString = argCompress ? JSON.stringify(data) : JSON.stringify(data, null, 2);
|
||||
await mkdir("data/fonts", { recursive: true });
|
||||
await writeFile("data/fonts/response.json", jsonString, "utf-8");
|
||||
await mkdir(FONTS_DIR, { recursive: true });
|
||||
await writeFile(RESPONSE_FILE, jsonString, "utf-8");
|
||||
|
||||
return data;
|
||||
}
|
||||
@@ -45,7 +55,7 @@ export async function generateFonts() {
|
||||
console.log(`Found ${response.items.length} fonts in total.`);
|
||||
|
||||
const filteredItems = response.items.filter(
|
||||
(item) => !blacklistedFonts.some((blacklist) => item.family.includes(blacklist)),
|
||||
(item) => !skippedFamilies.some((family) => item.family.includes(family)),
|
||||
);
|
||||
|
||||
const result: WebFont[] = filteredItems.slice(0, argLimit).map((item) => {
|
||||
@@ -54,6 +64,7 @@ export async function generateFonts() {
|
||||
|
||||
// 2. files: all files, but change "regular"->"400", "italic"->"400italic"
|
||||
const files: Record<string, string> = {};
|
||||
|
||||
for (const [variant, url] of Object.entries(item.files)) {
|
||||
let key = variant;
|
||||
if (variant === "regular") key = "400";
|
||||
@@ -62,17 +73,18 @@ export async function generateFonts() {
|
||||
}
|
||||
|
||||
return {
|
||||
type: "web",
|
||||
category: item.category,
|
||||
family: item.family,
|
||||
weights,
|
||||
preview: item.menu,
|
||||
files,
|
||||
};
|
||||
} satisfies WebFont;
|
||||
});
|
||||
|
||||
const jsonString = argCompress ? JSON.stringify(result) : JSON.stringify(result, null, 2);
|
||||
await mkdir("data/fonts", { recursive: true });
|
||||
await writeFile("data/fonts/webfontlist.json", jsonString, "utf-8");
|
||||
await mkdir(FONTS_DIR, { recursive: true });
|
||||
await writeFile(WEBFONTLIST_FILE, jsonString, "utf-8");
|
||||
|
||||
console.log(`Generated ${result.length} fonts in the list.`);
|
||||
}
|
||||
|
||||
@@ -42,6 +42,7 @@ export type Weight = "100" | "200" | "300" | "400" | "500" | "600" | "700" | "80
|
||||
type FileWeight = Weight | `${Weight}italic`;
|
||||
|
||||
export type WebFont = {
|
||||
type: "web";
|
||||
category: Category;
|
||||
family: string;
|
||||
weights: Weight[];
|
||||
|
||||
+106
-23
@@ -1,7 +1,7 @@
|
||||
import fs from "node:fs/promises";
|
||||
import { and, inArray, sql } from "drizzle-orm";
|
||||
import { drizzle } from "drizzle-orm/node-postgres";
|
||||
import { Pool } from "pg";
|
||||
import { Pool, type QueryResult } from "pg";
|
||||
import { schema } from "@/integrations/drizzle";
|
||||
import { ReactiveResumeV4JSONImporter } from "@/integrations/import/reactive-resume-v4-json";
|
||||
import { defaultResumeData } from "@/schema/resume/data";
|
||||
@@ -43,23 +43,28 @@ const localPool = new Pool({ connectionString: localUrl });
|
||||
const productionDb = drizzle({ client: productionPool });
|
||||
const localDb = drizzle({ client: localPool, schema });
|
||||
|
||||
// == Persistent mapping file path ==
|
||||
// == Persistent mapping file paths ==
|
||||
const USER_ID_MAP_FILE = "./scripts/migration/user-id-map.json";
|
||||
const RESUME_ID_MAP_FILE = "./scripts/migration/resume-id-map.json";
|
||||
|
||||
// == Progress checkpoint file path ==
|
||||
const PROGRESS_FILE = "./scripts/migration/resume-progress.json";
|
||||
|
||||
// You may tune this for your use case
|
||||
// Reduced from 10000 to avoid PostgreSQL message format errors
|
||||
const BATCH_SIZE = 5000;
|
||||
const BATCH_SIZE = 10_000;
|
||||
|
||||
// Chunk size for actual inserts - smaller to avoid PostgreSQL message size limits
|
||||
// Especially important for resumes as they contain large JSONB data
|
||||
const INSERT_CHUNK_SIZE = 1000;
|
||||
|
||||
// == Progress checkpoint interface ==
|
||||
// Uses cursor-based pagination with (createdAt, id) composite key for efficiency
|
||||
interface MigrationProgress {
|
||||
currentOffset: number;
|
||||
// Cursor for pagination - last seen createdAt timestamp
|
||||
lastSeenCreatedAt: string | null;
|
||||
// Cursor for pagination - last seen id (for tiebreaker when timestamps are equal)
|
||||
lastSeenId: string | null;
|
||||
resumesCreated: number;
|
||||
statisticsCreated: number;
|
||||
skipped: number;
|
||||
@@ -76,7 +81,7 @@ async function loadProgress(): Promise<MigrationProgress | null> {
|
||||
const text = await fs.readFile(PROGRESS_FILE, { encoding: "utf-8" });
|
||||
const progress = JSON.parse(text) as MigrationProgress;
|
||||
console.log(`📂 Found existing progress file. Last updated: ${progress.lastUpdated}`);
|
||||
console.log(` Resuming from offset ${progress.currentOffset}...`);
|
||||
console.log(` Resuming from cursor (createdAt: ${progress.lastSeenCreatedAt}, id: ${progress.lastSeenId})...`);
|
||||
return progress;
|
||||
} catch (e) {
|
||||
console.warn("⚠️ Failed to load progress file, starting from beginning.", e);
|
||||
@@ -88,7 +93,7 @@ async function saveProgress(progress: MigrationProgress): Promise<void> {
|
||||
try {
|
||||
progress.lastUpdated = new Date().toISOString();
|
||||
await fs.writeFile(PROGRESS_FILE, JSON.stringify(progress, null, 2), { encoding: "utf-8" });
|
||||
console.log(`💾 Progress saved at offset ${progress.currentOffset}`);
|
||||
console.log(`💾 Progress saved at cursor (createdAt: ${progress.lastSeenCreatedAt}, id: ${progress.lastSeenId})`);
|
||||
} catch (e) {
|
||||
console.error("🚨 Failed to save progress:", e);
|
||||
}
|
||||
@@ -114,15 +119,35 @@ async function loadUserIdMapFromFile(): Promise<Map<string, string>> {
|
||||
return new Map<string, string>();
|
||||
}
|
||||
|
||||
async function loadResumeIdMapFromFile(): Promise<Map<string, string>> {
|
||||
try {
|
||||
const text = await fs.readFile(RESUME_ID_MAP_FILE, { encoding: "utf-8" });
|
||||
const obj = JSON.parse(text);
|
||||
return new Map(Object.entries(obj));
|
||||
} catch (e) {
|
||||
console.warn("⚠️ Failed to load resumeIdMap from disk, continuing with empty map.", e);
|
||||
}
|
||||
return new Map<string, string>();
|
||||
}
|
||||
|
||||
async function saveResumeIdMapToFile(resumeIdMap: Map<string, string>) {
|
||||
const obj: Record<string, string> = Object.fromEntries(resumeIdMap.entries());
|
||||
await fs.writeFile(RESUME_ID_MAP_FILE, JSON.stringify(obj, null, "\t"), { encoding: "utf-8" });
|
||||
}
|
||||
|
||||
export async function migrateResumes() {
|
||||
const migrationStart = performance.now();
|
||||
console.log("⌛ Starting resume migration...");
|
||||
|
||||
let hasMore = true;
|
||||
let currentOffset = 0;
|
||||
|
||||
// Load persistent userIdMap from file
|
||||
// Cursor-based pagination state
|
||||
let lastSeenCreatedAt: string | null = null;
|
||||
let lastSeenId: string | null = null;
|
||||
|
||||
// Load persistent ID maps from file
|
||||
const userIdMap = await loadUserIdMapFromFile();
|
||||
const resumeIdMap = await loadResumeIdMapFromFile();
|
||||
|
||||
// Track migration stats
|
||||
let resumesCreated = 0;
|
||||
@@ -134,7 +159,8 @@ export async function migrateResumes() {
|
||||
// Load saved progress if exists
|
||||
const savedProgress = await loadProgress();
|
||||
if (savedProgress) {
|
||||
currentOffset = savedProgress.currentOffset;
|
||||
lastSeenCreatedAt = savedProgress.lastSeenCreatedAt;
|
||||
lastSeenId = savedProgress.lastSeenId;
|
||||
resumesCreated = savedProgress.resumesCreated;
|
||||
statisticsCreated = savedProgress.statisticsCreated;
|
||||
skipped = savedProgress.skipped;
|
||||
@@ -144,7 +170,8 @@ export async function migrateResumes() {
|
||||
|
||||
// Helper to get current progress object
|
||||
const getCurrentProgress = (): MigrationProgress => ({
|
||||
currentOffset,
|
||||
lastSeenCreatedAt,
|
||||
lastSeenId,
|
||||
resumesCreated,
|
||||
statisticsCreated,
|
||||
skipped,
|
||||
@@ -159,6 +186,7 @@ export async function migrateResumes() {
|
||||
shutdownRequested = true;
|
||||
console.log("\n⚠️ Shutdown requested. Saving progress...");
|
||||
await saveProgress(getCurrentProgress());
|
||||
await saveResumeIdMapToFile(resumeIdMap);
|
||||
console.log("👋 Exiting. Run the script again to resume from where you left off.");
|
||||
process.exit(0);
|
||||
};
|
||||
@@ -173,14 +201,32 @@ export async function migrateResumes() {
|
||||
// Check if shutdown was requested
|
||||
if (shutdownRequested) break;
|
||||
|
||||
console.log(`📥 Fetching resumes batch from production database (OFFSET ${currentOffset})...`);
|
||||
console.log(
|
||||
`📥 Fetching resumes batch from production database (cursor: createdAt=${lastSeenCreatedAt}, id=${lastSeenId})...`,
|
||||
);
|
||||
|
||||
const resumes = (await productionDb.execute(sql`
|
||||
SELECT id, title, slug, data, visibility, locked, "userId", "createdAt", "updatedAt"
|
||||
FROM "Resume"
|
||||
ORDER BY "id" ASC
|
||||
LIMIT ${BATCH_SIZE} OFFSET ${currentOffset}
|
||||
`)) as unknown as ProductionResume[];
|
||||
// Use cursor-based pagination for better performance
|
||||
// Tuple comparison syntax allows Postgres to use composite index efficiently
|
||||
let resumes: ProductionResume[];
|
||||
|
||||
if (lastSeenCreatedAt && lastSeenId) {
|
||||
const result = (await productionDb.execute(sql`
|
||||
SELECT id, title, slug, data, visibility, locked, "userId", "createdAt", "updatedAt"
|
||||
FROM "Resume"
|
||||
WHERE ("createdAt", id) < (${lastSeenCreatedAt}::timestamp, ${lastSeenId})
|
||||
ORDER BY "createdAt" DESC, id DESC
|
||||
LIMIT ${BATCH_SIZE}
|
||||
`)) as unknown as QueryResult<ProductionResume>;
|
||||
resumes = result.rows;
|
||||
} else {
|
||||
const result = (await productionDb.execute(sql`
|
||||
SELECT id, title, slug, data, visibility, locked, "userId", "createdAt", "updatedAt"
|
||||
FROM "Resume"
|
||||
ORDER BY "createdAt" DESC, id DESC
|
||||
LIMIT ${BATCH_SIZE}
|
||||
`)) as unknown as QueryResult<ProductionResume>;
|
||||
resumes = result.rows;
|
||||
}
|
||||
|
||||
console.log(`📋 Found ${resumes.length} resumes in this batch.`);
|
||||
|
||||
@@ -196,11 +242,11 @@ export async function migrateResumes() {
|
||||
// Escape single quotes in IDs (though UUIDs shouldn't contain them, this is safer)
|
||||
const resumeIdsForSql = resumeIds.map((id) => `'${id.replace(/'/g, "''")}'`).join(", ");
|
||||
|
||||
const statistics = (await productionDb.execute(sql`
|
||||
const { rows: statistics } = (await productionDb.execute(sql`
|
||||
SELECT id, views, downloads, "resumeId", "createdAt", "updatedAt"
|
||||
FROM "Statistics"
|
||||
WHERE "resumeId" IN (${sql.raw(resumeIdsForSql)})
|
||||
`)) as unknown as ProductionStatistics[];
|
||||
`)) as unknown as QueryResult<ProductionStatistics>;
|
||||
|
||||
// Create a map of resumeId -> statistics for quick lookup
|
||||
const statisticsMap = new Map<string, ProductionStatistics>();
|
||||
@@ -222,8 +268,15 @@ export async function migrateResumes() {
|
||||
|
||||
if (resumesToProcess.length === 0) {
|
||||
console.log(`⏭️ All resumes in this batch have userIds not found in userIdMap.`);
|
||||
currentOffset += resumes.length;
|
||||
// Update cursor to the last resume in this batch
|
||||
const lastResume = resumes[resumes.length - 1];
|
||||
if (lastResume) {
|
||||
lastSeenCreatedAt =
|
||||
lastResume.createdAt instanceof Date ? lastResume.createdAt.toISOString() : String(lastResume.createdAt);
|
||||
lastSeenId = lastResume.id;
|
||||
}
|
||||
totalResumesProcessed += resumes.length;
|
||||
await saveProgress(getCurrentProgress());
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -244,8 +297,15 @@ export async function migrateResumes() {
|
||||
|
||||
if (resumesWithValidUsers.length === 0) {
|
||||
console.log(`⏭️ All resumes in this batch have userIds not found in local database.`);
|
||||
currentOffset += resumes.length;
|
||||
// Update cursor to the last resume in this batch
|
||||
const lastResume = resumes[resumes.length - 1];
|
||||
if (lastResume) {
|
||||
lastSeenCreatedAt =
|
||||
lastResume.createdAt instanceof Date ? lastResume.createdAt.toISOString() : String(lastResume.createdAt);
|
||||
lastSeenId = lastResume.id;
|
||||
}
|
||||
totalResumesProcessed += resumes.length;
|
||||
await saveProgress(getCurrentProgress());
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -281,8 +341,15 @@ export async function migrateResumes() {
|
||||
|
||||
if (resumesToInsert.length === 0) {
|
||||
console.log(`⏭️ All resumes in this batch already exist in target DB.`);
|
||||
currentOffset += resumes.length;
|
||||
// Update cursor to the last resume in this batch
|
||||
const lastResume = resumes[resumes.length - 1];
|
||||
if (lastResume) {
|
||||
lastSeenCreatedAt =
|
||||
lastResume.createdAt instanceof Date ? lastResume.createdAt.toISOString() : String(lastResume.createdAt);
|
||||
lastSeenId = lastResume.id;
|
||||
}
|
||||
totalResumesProcessed += resumes.length;
|
||||
await saveProgress(getCurrentProgress());
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -308,6 +375,9 @@ export async function migrateResumes() {
|
||||
|
||||
const newResumeId = generateId();
|
||||
|
||||
// Track the ID mapping for future reference
|
||||
resumeIdMap.set(resume.id, newResumeId);
|
||||
|
||||
return {
|
||||
resumeData: {
|
||||
id: newResumeId,
|
||||
@@ -369,13 +439,23 @@ export async function migrateResumes() {
|
||||
console.log(
|
||||
`✅ Bulk inserted ${resumesToInsertData.length} resumes in ${batchTimeMs.toFixed(1)} ms (avg ${(batchTimeMs / resumesToInsertData.length).toFixed(1)} ms/resume)`,
|
||||
);
|
||||
|
||||
// Save resume ID map after each successful batch
|
||||
await saveResumeIdMapToFile(resumeIdMap);
|
||||
} catch (error) {
|
||||
console.error(`🚨 Failed to bulk insert resumes batch:`, error);
|
||||
errors++;
|
||||
// Continue with next batch even if this one fails
|
||||
}
|
||||
|
||||
currentOffset += resumes.length;
|
||||
// Update cursor to the last resume in this batch
|
||||
const lastResume = resumes[resumes.length - 1];
|
||||
if (lastResume) {
|
||||
lastSeenCreatedAt =
|
||||
lastResume.createdAt instanceof Date ? lastResume.createdAt.toISOString() : String(lastResume.createdAt);
|
||||
lastSeenId = lastResume.id;
|
||||
}
|
||||
|
||||
totalResumesProcessed += resumes.length;
|
||||
console.log(`📦 Processed ${totalResumesProcessed} resumes so far...\n`);
|
||||
|
||||
@@ -399,6 +479,9 @@ export async function migrateResumes() {
|
||||
`⏱️ Total migration time: ${migrationDurationMs.toFixed(1)} ms (${(migrationDurationMs / 1000).toFixed(2)} seconds)`,
|
||||
);
|
||||
|
||||
// Final save of the mapping (ensures up-to-date state)
|
||||
await saveResumeIdMapToFile(resumeIdMap);
|
||||
|
||||
// Clear progress file on successful completion (only if not interrupted)
|
||||
if (!shutdownRequested) {
|
||||
await clearProgress();
|
||||
|
||||
@@ -1,18 +1,28 @@
|
||||
import { sql } from "drizzle-orm";
|
||||
import { drizzle } from "drizzle-orm/node-postgres";
|
||||
import { Pool } from "pg";
|
||||
import { env } from "@/utils/env";
|
||||
|
||||
const pool = new Pool({ connectionString: env.DATABASE_URL });
|
||||
const db = drizzle({ client: pool });
|
||||
const productionUrl = process.env.PRODUCTION_DATABASE_URL;
|
||||
if (!productionUrl) throw new Error("PRODUCTION_DATABASE_URL is not set");
|
||||
const productionPool = new Pool({ connectionString: productionUrl });
|
||||
const productionDb = drizzle({ client: productionPool });
|
||||
|
||||
const localUrl = process.env.DATABASE_URL;
|
||||
if (!localUrl) throw new Error("DATABASE_URL is not set");
|
||||
const localPool = new Pool({ connectionString: localUrl });
|
||||
const localDb = drizzle({ client: localPool });
|
||||
|
||||
try {
|
||||
const result = await db.execute(sql`SELECT 1 as connected`);
|
||||
console.log("✅ Database connection successful", JSON.stringify(result));
|
||||
const productionResult = await productionDb.execute(sql`SELECT 1 as connected`);
|
||||
console.log("✅ Production database connection successful", JSON.stringify(productionResult));
|
||||
|
||||
const localResult = await localDb.execute(sql`SELECT 1 as connected`);
|
||||
console.log("✅ Local database connection successful", JSON.stringify(localResult));
|
||||
} catch (error) {
|
||||
console.error("🚨 Database connection failed:", error);
|
||||
process.exit(1);
|
||||
} finally {
|
||||
await pool.end();
|
||||
await productionPool.end();
|
||||
await localPool.end();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
+71
-64
@@ -1,7 +1,7 @@
|
||||
import fs from "node:fs/promises";
|
||||
import { inArray, or, sql } from "drizzle-orm";
|
||||
import { drizzle } from "drizzle-orm/node-postgres";
|
||||
import { Pool } from "pg";
|
||||
import { Pool, type QueryResult } from "pg";
|
||||
import { schema } from "@/integrations/drizzle";
|
||||
import { generateId, toUsername } from "@/utils/string";
|
||||
|
||||
@@ -18,7 +18,6 @@ interface ProductionUser {
|
||||
email: string;
|
||||
locale: string;
|
||||
emailVerified: boolean;
|
||||
twoFactorEnabled: boolean;
|
||||
createdAt: Date;
|
||||
updatedAt: Date;
|
||||
provider: ProductionProvider;
|
||||
@@ -28,11 +27,6 @@ interface ProductionSecrets {
|
||||
id: string;
|
||||
password: string | null;
|
||||
lastSignedIn: Date;
|
||||
verificationToken: string | null;
|
||||
twoFactorSecret: string | null;
|
||||
twoFactorBackupCodes: string[];
|
||||
refreshToken: string | null;
|
||||
resetToken: string | null;
|
||||
userId: string;
|
||||
}
|
||||
|
||||
@@ -69,18 +63,20 @@ const USER_ID_MAP_FILE = "./scripts/migration/user-id-map.json";
|
||||
const PROGRESS_FILE = "./scripts/migration/user-progress.json";
|
||||
|
||||
// You may tune this for your use case
|
||||
// Reduced from 10000 to avoid PostgreSQL message format errors
|
||||
const BATCH_SIZE = 5000;
|
||||
const BATCH_SIZE = 10_000;
|
||||
|
||||
// Chunk size for actual inserts - smaller to avoid PostgreSQL message size limits
|
||||
const INSERT_CHUNK_SIZE = 1000;
|
||||
|
||||
// == Progress checkpoint interface ==
|
||||
// Uses cursor-based pagination with (createdAt, id) composite key for efficiency
|
||||
interface MigrationProgress {
|
||||
currentOffset: number;
|
||||
// Cursor for pagination - last seen createdAt timestamp
|
||||
lastSeenCreatedAt: string | null;
|
||||
// Cursor for pagination - last seen id (for tiebreaker when timestamps are equal)
|
||||
lastSeenId: string | null;
|
||||
usersCreated: number;
|
||||
accountsCreated: number;
|
||||
twoFactorCreated: number;
|
||||
skipped: number;
|
||||
totalUsersProcessed: number;
|
||||
lastUpdated: string;
|
||||
@@ -94,7 +90,7 @@ async function loadProgress(): Promise<MigrationProgress | null> {
|
||||
const text = await fs.readFile(PROGRESS_FILE, { encoding: "utf-8" });
|
||||
const progress = JSON.parse(text) as MigrationProgress;
|
||||
console.log(`📂 Found existing progress file. Last updated: ${progress.lastUpdated}`);
|
||||
console.log(` Resuming from offset ${progress.currentOffset}...`);
|
||||
console.log(` Resuming from cursor (createdAt: ${progress.lastSeenCreatedAt}, id: ${progress.lastSeenId})...`);
|
||||
return progress;
|
||||
} catch (e) {
|
||||
console.warn("⚠️ Failed to load progress file, starting from beginning.", e);
|
||||
@@ -106,7 +102,7 @@ async function saveProgress(progress: MigrationProgress): Promise<void> {
|
||||
try {
|
||||
progress.lastUpdated = new Date().toISOString();
|
||||
await fs.writeFile(PROGRESS_FILE, JSON.stringify(progress, null, 2), { encoding: "utf-8" });
|
||||
console.log(`💾 Progress saved at offset ${progress.currentOffset}`);
|
||||
console.log(`💾 Progress saved at cursor (createdAt: ${progress.lastSeenCreatedAt}, id: ${progress.lastSeenId})`);
|
||||
} catch (e) {
|
||||
console.error("🚨 Failed to save progress:", e);
|
||||
}
|
||||
@@ -142,7 +138,10 @@ export async function migrateUsers() {
|
||||
console.log("⌛ Starting user migration...");
|
||||
|
||||
let hasMore = true;
|
||||
let currentOffset = 0;
|
||||
|
||||
// Cursor-based pagination state
|
||||
let lastSeenCreatedAt: string | null = null;
|
||||
let lastSeenId: string | null = null;
|
||||
|
||||
// Load persistent userIdMap from file
|
||||
const userIdMap = await loadUserIdMapFromFile();
|
||||
@@ -150,27 +149,26 @@ export async function migrateUsers() {
|
||||
// Track migration stats
|
||||
let usersCreated = 0;
|
||||
let accountsCreated = 0;
|
||||
let twoFactorCreated = 0;
|
||||
let skipped = 0;
|
||||
let totalUsersProcessed = 0;
|
||||
|
||||
// Load saved progress if exists
|
||||
const savedProgress = await loadProgress();
|
||||
if (savedProgress) {
|
||||
currentOffset = savedProgress.currentOffset;
|
||||
lastSeenCreatedAt = savedProgress.lastSeenCreatedAt;
|
||||
lastSeenId = savedProgress.lastSeenId;
|
||||
usersCreated = savedProgress.usersCreated;
|
||||
accountsCreated = savedProgress.accountsCreated;
|
||||
twoFactorCreated = savedProgress.twoFactorCreated;
|
||||
skipped = savedProgress.skipped;
|
||||
totalUsersProcessed = savedProgress.totalUsersProcessed;
|
||||
}
|
||||
|
||||
// Helper to get current progress object
|
||||
const getCurrentProgress = (): MigrationProgress => ({
|
||||
currentOffset,
|
||||
lastSeenCreatedAt,
|
||||
lastSeenId,
|
||||
usersCreated,
|
||||
accountsCreated,
|
||||
twoFactorCreated,
|
||||
skipped,
|
||||
totalUsersProcessed,
|
||||
lastUpdated: new Date().toISOString(),
|
||||
@@ -194,14 +192,30 @@ export async function migrateUsers() {
|
||||
// Check if shutdown was requested
|
||||
if (shutdownRequested) break;
|
||||
|
||||
console.log(`📥 Fetching users batch from production database (OFFSET ${currentOffset})...`);
|
||||
console.log(
|
||||
`📥 Fetching users batch from production database (cursor: createdAt=${lastSeenCreatedAt}, id=${lastSeenId})...`,
|
||||
);
|
||||
|
||||
const users = (await productionDb.execute(sql`
|
||||
SELECT id, name, picture, username, email, locale, "emailVerified", "twoFactorEnabled", "createdAt", "updatedAt", provider
|
||||
FROM "User"
|
||||
ORDER BY "id" ASC
|
||||
LIMIT ${BATCH_SIZE} OFFSET ${currentOffset}
|
||||
`)) as unknown as ProductionUser[];
|
||||
let users: ProductionUser[];
|
||||
|
||||
if (lastSeenCreatedAt && lastSeenId) {
|
||||
const result = (await productionDb.execute(sql`
|
||||
SELECT id, name, picture, username, email, locale, "emailVerified", "createdAt", "updatedAt", provider
|
||||
FROM "User"
|
||||
WHERE ("createdAt", id) < (${lastSeenCreatedAt}::timestamp, ${lastSeenId})
|
||||
ORDER BY "createdAt" DESC, id DESC
|
||||
LIMIT ${BATCH_SIZE}
|
||||
`)) as unknown as QueryResult<ProductionUser>;
|
||||
users = result.rows;
|
||||
} else {
|
||||
const result = (await productionDb.execute(sql`
|
||||
SELECT id, name, picture, username, email, locale, "emailVerified", "createdAt", "updatedAt", provider
|
||||
FROM "User"
|
||||
ORDER BY "createdAt" DESC, id DESC
|
||||
LIMIT ${BATCH_SIZE}
|
||||
`)) as unknown as QueryResult<ProductionUser>;
|
||||
users = result.rows;
|
||||
}
|
||||
|
||||
console.log(`📋 Found ${users.length} users in this batch.`);
|
||||
|
||||
@@ -217,11 +231,11 @@ export async function migrateUsers() {
|
||||
// Escape single quotes in IDs (though UUIDs shouldn't contain them, this is safer)
|
||||
const userIdsForSql = userIds.map((id) => `'${id.replace(/'/g, "''")}'`).join(", ");
|
||||
|
||||
const secrets = (await productionDb.execute(sql`
|
||||
SELECT id, password, "lastSignedIn", "verificationToken", "twoFactorSecret", "twoFactorBackupCodes", "refreshToken", "resetToken", "userId"
|
||||
const { rows: secrets } = (await productionDb.execute(sql`
|
||||
SELECT id, password, "lastSignedIn", "userId"
|
||||
FROM "Secrets"
|
||||
WHERE "userId" IN (${sql.raw(userIdsForSql)})
|
||||
`)) as unknown as ProductionSecrets[];
|
||||
`)) as unknown as QueryResult<ProductionSecrets>;
|
||||
|
||||
// Create a map of userId -> secrets for quick lookup
|
||||
const secretsMap = new Map<string, ProductionSecrets>();
|
||||
@@ -240,8 +254,15 @@ export async function migrateUsers() {
|
||||
|
||||
if (usersToProcess.length === 0) {
|
||||
console.log(`⏭️ All users in this batch were already migrated.`);
|
||||
currentOffset += users.length;
|
||||
// Update cursor to the last user in this batch
|
||||
const lastUser = users[users.length - 1];
|
||||
if (lastUser) {
|
||||
lastSeenCreatedAt =
|
||||
lastUser.createdAt instanceof Date ? lastUser.createdAt.toISOString() : String(lastUser.createdAt);
|
||||
lastSeenId = lastUser.id;
|
||||
}
|
||||
totalUsersProcessed += users.length;
|
||||
await saveProgress(getCurrentProgress());
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -287,9 +308,16 @@ export async function migrateUsers() {
|
||||
|
||||
if (usersToInsert.length === 0) {
|
||||
console.log(`⏭️ All users in this batch already exist in target DB.`);
|
||||
currentOffset += users.length;
|
||||
// Update cursor to the last user in this batch
|
||||
const lastUser = users[users.length - 1];
|
||||
if (lastUser) {
|
||||
lastSeenCreatedAt =
|
||||
lastUser.createdAt instanceof Date ? lastUser.createdAt.toISOString() : String(lastUser.createdAt);
|
||||
lastSeenId = lastUser.id;
|
||||
}
|
||||
totalUsersProcessed += users.length;
|
||||
await saveUserIdMapToFile(userIdMap);
|
||||
await saveProgress(getCurrentProgress());
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -308,7 +336,7 @@ export async function migrateUsers() {
|
||||
username: username,
|
||||
displayUsername: displayUsername,
|
||||
emailVerified: user.emailVerified,
|
||||
twoFactorEnabled: user.twoFactorEnabled,
|
||||
twoFactorEnabled: false, // All users start with 2FA disabled in the new system
|
||||
createdAt: user.createdAt,
|
||||
updatedAt: user.updatedAt,
|
||||
},
|
||||
@@ -339,7 +367,6 @@ export async function migrateUsers() {
|
||||
accountId: accountId,
|
||||
providerId: providerId,
|
||||
password: userSecrets?.password ?? null,
|
||||
refreshToken: userSecrets?.refreshToken ?? null,
|
||||
createdAt: userData.createdAt,
|
||||
updatedAt: userData.updatedAt,
|
||||
};
|
||||
@@ -352,34 +379,6 @@ export async function migrateUsers() {
|
||||
}
|
||||
accountsCreated += accountsToInsert.length;
|
||||
|
||||
// Prepare two-factor entries for bulk insert
|
||||
const twoFactorToInsert = usersToInsertData
|
||||
.map(({ originalUser, newUserId, userData }) => {
|
||||
const userSecrets = secretsMap.get(originalUser.id);
|
||||
|
||||
if (originalUser.twoFactorEnabled && userSecrets?.twoFactorSecret) {
|
||||
return {
|
||||
id: generateId(),
|
||||
userId: newUserId,
|
||||
secret: userSecrets.twoFactorSecret,
|
||||
backupCodes: userSecrets.twoFactorBackupCodes.join(","),
|
||||
createdAt: userData.createdAt,
|
||||
updatedAt: userData.updatedAt,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
})
|
||||
.filter((entry): entry is NonNullable<typeof entry> => entry !== null);
|
||||
|
||||
// Bulk insert two-factor entries (chunked)
|
||||
if (twoFactorToInsert.length > 0) {
|
||||
for (let i = 0; i < twoFactorToInsert.length; i += INSERT_CHUNK_SIZE) {
|
||||
const chunk = twoFactorToInsert.slice(i, i + INSERT_CHUNK_SIZE);
|
||||
await localDb.insert(schema.twoFactor).values(chunk);
|
||||
}
|
||||
twoFactorCreated += twoFactorToInsert.length;
|
||||
}
|
||||
|
||||
const batchEnd = performance.now();
|
||||
const batchTimeMs = batchEnd - batchStart;
|
||||
console.log(
|
||||
@@ -388,15 +387,24 @@ export async function migrateUsers() {
|
||||
|
||||
// Save progress after each batch
|
||||
await saveUserIdMapToFile(userIdMap);
|
||||
await saveProgress(getCurrentProgress());
|
||||
} catch (error) {
|
||||
console.error(`🚨 Failed to bulk insert users batch:`, error);
|
||||
// Continue with next batch even if this one fails
|
||||
}
|
||||
|
||||
currentOffset += users.length;
|
||||
// Update cursor to the last user in this batch
|
||||
const lastUser = users[users.length - 1];
|
||||
if (lastUser) {
|
||||
lastSeenCreatedAt =
|
||||
lastUser.createdAt instanceof Date ? lastUser.createdAt.toISOString() : String(lastUser.createdAt);
|
||||
lastSeenId = lastUser.id;
|
||||
}
|
||||
|
||||
totalUsersProcessed += users.length;
|
||||
console.log(`📦 Processed ${totalUsersProcessed} users so far...\n`);
|
||||
|
||||
// Save progress after updating cursor
|
||||
await saveProgress(getCurrentProgress());
|
||||
}
|
||||
|
||||
// Remove signal handlers
|
||||
@@ -409,7 +417,6 @@ export async function migrateUsers() {
|
||||
console.log("\n📊 Migration Summary:");
|
||||
console.log(` Users created: ${usersCreated}`);
|
||||
console.log(` Accounts created: ${accountsCreated}`);
|
||||
console.log(` Two-factor entries created: ${twoFactorCreated}`);
|
||||
console.log(` Skipped (already exist): ${skipped}`);
|
||||
console.log(
|
||||
`⏱️ Total migration time: ${migrationDurationMs.toFixed(1)} ms (${(migrationDurationMs / 1000).toFixed(2)} seconds)`,
|
||||
|
||||
Reference in New Issue
Block a user