lot of bugfixes, better migration script

This commit is contained in:
Amruth Pillai
2026-01-23 01:07:52 +01:00
parent a10954ed14
commit 35e2daa807
95 changed files with 3661 additions and 3645 deletions
+14 -2
View File
@@ -9,15 +9,27 @@ export async function resetDatabase() {
const pool = new Pool({ connectionString: env.DATABASE_URL });
const db = drizzle({ client: pool });
// Extract the username from a PostgreSQL connection string like: postgresql://postgres:password@localhost:5432/db
const username = (() => {
try {
const match = env.DATABASE_URL.match(/^postgres(?:ql)?:\/\/([^:]+):[^@]+@/);
return match ? match[1] : undefined;
} catch {
return undefined;
}
})();
console.log("🔑 Username:", username);
try {
await db.transaction(async (tx) => {
await tx.execute(sql`DROP SCHEMA drizzle CASCADE`);
await tx.execute(sql`CREATE SCHEMA drizzle`);
await tx.execute(sql`GRANT ALL ON SCHEMA drizzle TO postgres`);
await tx.execute(sql.raw(`GRANT ALL ON SCHEMA drizzle TO ${username}`));
await tx.execute(sql`DROP SCHEMA public CASCADE`);
await tx.execute(sql`CREATE SCHEMA public`);
await tx.execute(sql`GRANT ALL ON SCHEMA public TO postgres`);
await tx.execute(sql.raw(`GRANT ALL ON SCHEMA public TO ${username}`));
});
console.log("✅ Database reset completed");
-42
View File
@@ -1,42 +0,0 @@
import { drizzle } from "drizzle-orm/node-postgres";
import { Pool } from "pg";
import { schema } from "@/integrations/drizzle";
import { env } from "@/utils/env";
import { hashPassword } from "@/utils/password";
import { generateId } from "@/utils/string";
export async function seedDatabase() {
console.log("⌛ Seeding database...");
const pool = new Pool({ connectionString: env.DATABASE_URL });
const db = drizzle({ client: pool, schema });
try {
const userId = generateId();
await db.insert(schema.user).values({
id: userId,
name: "Test User",
email: "test@test.com",
username: "test",
displayUsername: "test",
emailVerified: true,
image: "https://i.pravatar.cc/300",
});
await db.insert(schema.account).values({
id: generateId(),
userId,
accountId: userId,
password: await hashPassword("password"),
});
} catch (error) {
console.error("🚨 Database seeding failed:", error);
} finally {
await pool.end();
}
}
if (import.meta.main) {
await seedDatabase();
}
+20 -8
View File
@@ -13,10 +13,20 @@ const argForce = args.includes("--force");
const argCompress = args.includes("--compress");
const argLimit = args.includes("--limit") ? parseInt(args[args.indexOf("--limit") + 1], 10) : 500;
const blacklistedFonts = ["Material Icons", "Material Symbols", "Noto Color Emoji"];
const skippedFamilies = ["Material Icons", "Material Symbols", "Noto Color Emoji"];
const FONTS_DIR = "./scripts/fonts";
const RESPONSE_FILE = `${FONTS_DIR}/response.json`;
const WEBFONTLIST_FILE = `${FONTS_DIR}/webfontlist.json`;
async function getGoogleFontsJSON() {
const contents = await readFile("data/fonts/response.json", "utf-8");
let contents: string | null = null;
try {
contents = await readFile(RESPONSE_FILE, "utf-8");
} catch {
// If the file doesn't exist or there's an error reading, just continue.
}
if (!argForce && contents) return JSON.parse(contents) as APIResponse;
@@ -28,8 +38,8 @@ async function getGoogleFontsJSON() {
const data = (await response.json()) as APIResponse;
const jsonString = argCompress ? JSON.stringify(data) : JSON.stringify(data, null, 2);
await mkdir("data/fonts", { recursive: true });
await writeFile("data/fonts/response.json", jsonString, "utf-8");
await mkdir(FONTS_DIR, { recursive: true });
await writeFile(RESPONSE_FILE, jsonString, "utf-8");
return data;
}
@@ -45,7 +55,7 @@ export async function generateFonts() {
console.log(`Found ${response.items.length} fonts in total.`);
const filteredItems = response.items.filter(
(item) => !blacklistedFonts.some((blacklist) => item.family.includes(blacklist)),
(item) => !skippedFamilies.some((family) => item.family.includes(family)),
);
const result: WebFont[] = filteredItems.slice(0, argLimit).map((item) => {
@@ -54,6 +64,7 @@ export async function generateFonts() {
// 2. files: all files, but change "regular"->"400", "italic"->"400italic"
const files: Record<string, string> = {};
for (const [variant, url] of Object.entries(item.files)) {
let key = variant;
if (variant === "regular") key = "400";
@@ -62,17 +73,18 @@ export async function generateFonts() {
}
return {
type: "web",
category: item.category,
family: item.family,
weights,
preview: item.menu,
files,
};
} satisfies WebFont;
});
const jsonString = argCompress ? JSON.stringify(result) : JSON.stringify(result, null, 2);
await mkdir("data/fonts", { recursive: true });
await writeFile("data/fonts/webfontlist.json", jsonString, "utf-8");
await mkdir(FONTS_DIR, { recursive: true });
await writeFile(WEBFONTLIST_FILE, jsonString, "utf-8");
console.log(`Generated ${result.length} fonts in the list.`);
}
+1
View File
@@ -42,6 +42,7 @@ export type Weight = "100" | "200" | "300" | "400" | "500" | "600" | "700" | "80
type FileWeight = Weight | `${Weight}italic`;
export type WebFont = {
type: "web";
category: Category;
family: string;
weights: Weight[];
+106 -23
View File
@@ -1,7 +1,7 @@
import fs from "node:fs/promises";
import { and, inArray, sql } from "drizzle-orm";
import { drizzle } from "drizzle-orm/node-postgres";
import { Pool } from "pg";
import { Pool, type QueryResult } from "pg";
import { schema } from "@/integrations/drizzle";
import { ReactiveResumeV4JSONImporter } from "@/integrations/import/reactive-resume-v4-json";
import { defaultResumeData } from "@/schema/resume/data";
@@ -43,23 +43,28 @@ const localPool = new Pool({ connectionString: localUrl });
const productionDb = drizzle({ client: productionPool });
const localDb = drizzle({ client: localPool, schema });
// == Persistent mapping file path ==
// == Persistent mapping file paths ==
const USER_ID_MAP_FILE = "./scripts/migration/user-id-map.json";
const RESUME_ID_MAP_FILE = "./scripts/migration/resume-id-map.json";
// == Progress checkpoint file path ==
const PROGRESS_FILE = "./scripts/migration/resume-progress.json";
// You may tune this for your use case
// Reduced from 10000 to avoid PostgreSQL message format errors
const BATCH_SIZE = 5000;
const BATCH_SIZE = 10_000;
// Chunk size for actual inserts - smaller to avoid PostgreSQL message size limits
// Especially important for resumes as they contain large JSONB data
const INSERT_CHUNK_SIZE = 1000;
// == Progress checkpoint interface ==
// Uses cursor-based pagination with (createdAt, id) composite key for efficiency
interface MigrationProgress {
currentOffset: number;
// Cursor for pagination - last seen createdAt timestamp
lastSeenCreatedAt: string | null;
// Cursor for pagination - last seen id (for tiebreaker when timestamps are equal)
lastSeenId: string | null;
resumesCreated: number;
statisticsCreated: number;
skipped: number;
@@ -76,7 +81,7 @@ async function loadProgress(): Promise<MigrationProgress | null> {
const text = await fs.readFile(PROGRESS_FILE, { encoding: "utf-8" });
const progress = JSON.parse(text) as MigrationProgress;
console.log(`📂 Found existing progress file. Last updated: ${progress.lastUpdated}`);
console.log(` Resuming from offset ${progress.currentOffset}...`);
console.log(` Resuming from cursor (createdAt: ${progress.lastSeenCreatedAt}, id: ${progress.lastSeenId})...`);
return progress;
} catch (e) {
console.warn("⚠️ Failed to load progress file, starting from beginning.", e);
@@ -88,7 +93,7 @@ async function saveProgress(progress: MigrationProgress): Promise<void> {
try {
progress.lastUpdated = new Date().toISOString();
await fs.writeFile(PROGRESS_FILE, JSON.stringify(progress, null, 2), { encoding: "utf-8" });
console.log(`💾 Progress saved at offset ${progress.currentOffset}`);
console.log(`💾 Progress saved at cursor (createdAt: ${progress.lastSeenCreatedAt}, id: ${progress.lastSeenId})`);
} catch (e) {
console.error("🚨 Failed to save progress:", e);
}
@@ -114,15 +119,35 @@ async function loadUserIdMapFromFile(): Promise<Map<string, string>> {
return new Map<string, string>();
}
async function loadResumeIdMapFromFile(): Promise<Map<string, string>> {
try {
const text = await fs.readFile(RESUME_ID_MAP_FILE, { encoding: "utf-8" });
const obj = JSON.parse(text);
return new Map(Object.entries(obj));
} catch (e) {
console.warn("⚠️ Failed to load resumeIdMap from disk, continuing with empty map.", e);
}
return new Map<string, string>();
}
async function saveResumeIdMapToFile(resumeIdMap: Map<string, string>) {
const obj: Record<string, string> = Object.fromEntries(resumeIdMap.entries());
await fs.writeFile(RESUME_ID_MAP_FILE, JSON.stringify(obj, null, "\t"), { encoding: "utf-8" });
}
export async function migrateResumes() {
const migrationStart = performance.now();
console.log("⌛ Starting resume migration...");
let hasMore = true;
let currentOffset = 0;
// Load persistent userIdMap from file
// Cursor-based pagination state
let lastSeenCreatedAt: string | null = null;
let lastSeenId: string | null = null;
// Load persistent ID maps from file
const userIdMap = await loadUserIdMapFromFile();
const resumeIdMap = await loadResumeIdMapFromFile();
// Track migration stats
let resumesCreated = 0;
@@ -134,7 +159,8 @@ export async function migrateResumes() {
// Load saved progress if exists
const savedProgress = await loadProgress();
if (savedProgress) {
currentOffset = savedProgress.currentOffset;
lastSeenCreatedAt = savedProgress.lastSeenCreatedAt;
lastSeenId = savedProgress.lastSeenId;
resumesCreated = savedProgress.resumesCreated;
statisticsCreated = savedProgress.statisticsCreated;
skipped = savedProgress.skipped;
@@ -144,7 +170,8 @@ export async function migrateResumes() {
// Helper to get current progress object
const getCurrentProgress = (): MigrationProgress => ({
currentOffset,
lastSeenCreatedAt,
lastSeenId,
resumesCreated,
statisticsCreated,
skipped,
@@ -159,6 +186,7 @@ export async function migrateResumes() {
shutdownRequested = true;
console.log("\n⚠️ Shutdown requested. Saving progress...");
await saveProgress(getCurrentProgress());
await saveResumeIdMapToFile(resumeIdMap);
console.log("👋 Exiting. Run the script again to resume from where you left off.");
process.exit(0);
};
@@ -173,14 +201,32 @@ export async function migrateResumes() {
// Check if shutdown was requested
if (shutdownRequested) break;
console.log(`📥 Fetching resumes batch from production database (OFFSET ${currentOffset})...`);
console.log(
`📥 Fetching resumes batch from production database (cursor: createdAt=${lastSeenCreatedAt}, id=${lastSeenId})...`,
);
const resumes = (await productionDb.execute(sql`
SELECT id, title, slug, data, visibility, locked, "userId", "createdAt", "updatedAt"
FROM "Resume"
ORDER BY "id" ASC
LIMIT ${BATCH_SIZE} OFFSET ${currentOffset}
`)) as unknown as ProductionResume[];
// Use cursor-based pagination for better performance
// Tuple comparison syntax allows Postgres to use composite index efficiently
let resumes: ProductionResume[];
if (lastSeenCreatedAt && lastSeenId) {
const result = (await productionDb.execute(sql`
SELECT id, title, slug, data, visibility, locked, "userId", "createdAt", "updatedAt"
FROM "Resume"
WHERE ("createdAt", id) < (${lastSeenCreatedAt}::timestamp, ${lastSeenId})
ORDER BY "createdAt" DESC, id DESC
LIMIT ${BATCH_SIZE}
`)) as unknown as QueryResult<ProductionResume>;
resumes = result.rows;
} else {
const result = (await productionDb.execute(sql`
SELECT id, title, slug, data, visibility, locked, "userId", "createdAt", "updatedAt"
FROM "Resume"
ORDER BY "createdAt" DESC, id DESC
LIMIT ${BATCH_SIZE}
`)) as unknown as QueryResult<ProductionResume>;
resumes = result.rows;
}
console.log(`📋 Found ${resumes.length} resumes in this batch.`);
@@ -196,11 +242,11 @@ export async function migrateResumes() {
// Escape single quotes in IDs (though UUIDs shouldn't contain them, this is safer)
const resumeIdsForSql = resumeIds.map((id) => `'${id.replace(/'/g, "''")}'`).join(", ");
const statistics = (await productionDb.execute(sql`
const { rows: statistics } = (await productionDb.execute(sql`
SELECT id, views, downloads, "resumeId", "createdAt", "updatedAt"
FROM "Statistics"
WHERE "resumeId" IN (${sql.raw(resumeIdsForSql)})
`)) as unknown as ProductionStatistics[];
`)) as unknown as QueryResult<ProductionStatistics>;
// Create a map of resumeId -> statistics for quick lookup
const statisticsMap = new Map<string, ProductionStatistics>();
@@ -222,8 +268,15 @@ export async function migrateResumes() {
if (resumesToProcess.length === 0) {
console.log(`⏭️ All resumes in this batch have userIds not found in userIdMap.`);
currentOffset += resumes.length;
// Update cursor to the last resume in this batch
const lastResume = resumes[resumes.length - 1];
if (lastResume) {
lastSeenCreatedAt =
lastResume.createdAt instanceof Date ? lastResume.createdAt.toISOString() : String(lastResume.createdAt);
lastSeenId = lastResume.id;
}
totalResumesProcessed += resumes.length;
await saveProgress(getCurrentProgress());
continue;
}
@@ -244,8 +297,15 @@ export async function migrateResumes() {
if (resumesWithValidUsers.length === 0) {
console.log(`⏭️ All resumes in this batch have userIds not found in local database.`);
currentOffset += resumes.length;
// Update cursor to the last resume in this batch
const lastResume = resumes[resumes.length - 1];
if (lastResume) {
lastSeenCreatedAt =
lastResume.createdAt instanceof Date ? lastResume.createdAt.toISOString() : String(lastResume.createdAt);
lastSeenId = lastResume.id;
}
totalResumesProcessed += resumes.length;
await saveProgress(getCurrentProgress());
continue;
}
@@ -281,8 +341,15 @@ export async function migrateResumes() {
if (resumesToInsert.length === 0) {
console.log(`⏭️ All resumes in this batch already exist in target DB.`);
currentOffset += resumes.length;
// Update cursor to the last resume in this batch
const lastResume = resumes[resumes.length - 1];
if (lastResume) {
lastSeenCreatedAt =
lastResume.createdAt instanceof Date ? lastResume.createdAt.toISOString() : String(lastResume.createdAt);
lastSeenId = lastResume.id;
}
totalResumesProcessed += resumes.length;
await saveProgress(getCurrentProgress());
continue;
}
@@ -308,6 +375,9 @@ export async function migrateResumes() {
const newResumeId = generateId();
// Track the ID mapping for future reference
resumeIdMap.set(resume.id, newResumeId);
return {
resumeData: {
id: newResumeId,
@@ -369,13 +439,23 @@ export async function migrateResumes() {
console.log(
`✅ Bulk inserted ${resumesToInsertData.length} resumes in ${batchTimeMs.toFixed(1)} ms (avg ${(batchTimeMs / resumesToInsertData.length).toFixed(1)} ms/resume)`,
);
// Save resume ID map after each successful batch
await saveResumeIdMapToFile(resumeIdMap);
} catch (error) {
console.error(`🚨 Failed to bulk insert resumes batch:`, error);
errors++;
// Continue with next batch even if this one fails
}
currentOffset += resumes.length;
// Update cursor to the last resume in this batch
const lastResume = resumes[resumes.length - 1];
if (lastResume) {
lastSeenCreatedAt =
lastResume.createdAt instanceof Date ? lastResume.createdAt.toISOString() : String(lastResume.createdAt);
lastSeenId = lastResume.id;
}
totalResumesProcessed += resumes.length;
console.log(`📦 Processed ${totalResumesProcessed} resumes so far...\n`);
@@ -399,6 +479,9 @@ export async function migrateResumes() {
`⏱️ Total migration time: ${migrationDurationMs.toFixed(1)} ms (${(migrationDurationMs / 1000).toFixed(2)} seconds)`,
);
// Final save of the mapping (ensures up-to-date state)
await saveResumeIdMapToFile(resumeIdMap);
// Clear progress file on successful completion (only if not interrupted)
if (!shutdownRequested) {
await clearProgress();
+16 -6
View File
@@ -1,18 +1,28 @@
import { sql } from "drizzle-orm";
import { drizzle } from "drizzle-orm/node-postgres";
import { Pool } from "pg";
import { env } from "@/utils/env";
const pool = new Pool({ connectionString: env.DATABASE_URL });
const db = drizzle({ client: pool });
const productionUrl = process.env.PRODUCTION_DATABASE_URL;
if (!productionUrl) throw new Error("PRODUCTION_DATABASE_URL is not set");
const productionPool = new Pool({ connectionString: productionUrl });
const productionDb = drizzle({ client: productionPool });
const localUrl = process.env.DATABASE_URL;
if (!localUrl) throw new Error("DATABASE_URL is not set");
const localPool = new Pool({ connectionString: localUrl });
const localDb = drizzle({ client: localPool });
try {
const result = await db.execute(sql`SELECT 1 as connected`);
console.log("✅ Database connection successful", JSON.stringify(result));
const productionResult = await productionDb.execute(sql`SELECT 1 as connected`);
console.log("✅ Production database connection successful", JSON.stringify(productionResult));
const localResult = await localDb.execute(sql`SELECT 1 as connected`);
console.log("✅ Local database connection successful", JSON.stringify(localResult));
} catch (error) {
console.error("🚨 Database connection failed:", error);
process.exit(1);
} finally {
await pool.end();
await productionPool.end();
await localPool.end();
process.exit(0);
}
+71 -64
View File
@@ -1,7 +1,7 @@
import fs from "node:fs/promises";
import { inArray, or, sql } from "drizzle-orm";
import { drizzle } from "drizzle-orm/node-postgres";
import { Pool } from "pg";
import { Pool, type QueryResult } from "pg";
import { schema } from "@/integrations/drizzle";
import { generateId, toUsername } from "@/utils/string";
@@ -18,7 +18,6 @@ interface ProductionUser {
email: string;
locale: string;
emailVerified: boolean;
twoFactorEnabled: boolean;
createdAt: Date;
updatedAt: Date;
provider: ProductionProvider;
@@ -28,11 +27,6 @@ interface ProductionSecrets {
id: string;
password: string | null;
lastSignedIn: Date;
verificationToken: string | null;
twoFactorSecret: string | null;
twoFactorBackupCodes: string[];
refreshToken: string | null;
resetToken: string | null;
userId: string;
}
@@ -69,18 +63,20 @@ const USER_ID_MAP_FILE = "./scripts/migration/user-id-map.json";
const PROGRESS_FILE = "./scripts/migration/user-progress.json";
// You may tune this for your use case
// Reduced from 10000 to avoid PostgreSQL message format errors
const BATCH_SIZE = 5000;
const BATCH_SIZE = 10_000;
// Chunk size for actual inserts - smaller to avoid PostgreSQL message size limits
const INSERT_CHUNK_SIZE = 1000;
// == Progress checkpoint interface ==
// Uses cursor-based pagination with (createdAt, id) composite key for efficiency
interface MigrationProgress {
currentOffset: number;
// Cursor for pagination - last seen createdAt timestamp
lastSeenCreatedAt: string | null;
// Cursor for pagination - last seen id (for tiebreaker when timestamps are equal)
lastSeenId: string | null;
usersCreated: number;
accountsCreated: number;
twoFactorCreated: number;
skipped: number;
totalUsersProcessed: number;
lastUpdated: string;
@@ -94,7 +90,7 @@ async function loadProgress(): Promise<MigrationProgress | null> {
const text = await fs.readFile(PROGRESS_FILE, { encoding: "utf-8" });
const progress = JSON.parse(text) as MigrationProgress;
console.log(`📂 Found existing progress file. Last updated: ${progress.lastUpdated}`);
console.log(` Resuming from offset ${progress.currentOffset}...`);
console.log(` Resuming from cursor (createdAt: ${progress.lastSeenCreatedAt}, id: ${progress.lastSeenId})...`);
return progress;
} catch (e) {
console.warn("⚠️ Failed to load progress file, starting from beginning.", e);
@@ -106,7 +102,7 @@ async function saveProgress(progress: MigrationProgress): Promise<void> {
try {
progress.lastUpdated = new Date().toISOString();
await fs.writeFile(PROGRESS_FILE, JSON.stringify(progress, null, 2), { encoding: "utf-8" });
console.log(`💾 Progress saved at offset ${progress.currentOffset}`);
console.log(`💾 Progress saved at cursor (createdAt: ${progress.lastSeenCreatedAt}, id: ${progress.lastSeenId})`);
} catch (e) {
console.error("🚨 Failed to save progress:", e);
}
@@ -142,7 +138,10 @@ export async function migrateUsers() {
console.log("⌛ Starting user migration...");
let hasMore = true;
let currentOffset = 0;
// Cursor-based pagination state
let lastSeenCreatedAt: string | null = null;
let lastSeenId: string | null = null;
// Load persistent userIdMap from file
const userIdMap = await loadUserIdMapFromFile();
@@ -150,27 +149,26 @@ export async function migrateUsers() {
// Track migration stats
let usersCreated = 0;
let accountsCreated = 0;
let twoFactorCreated = 0;
let skipped = 0;
let totalUsersProcessed = 0;
// Load saved progress if exists
const savedProgress = await loadProgress();
if (savedProgress) {
currentOffset = savedProgress.currentOffset;
lastSeenCreatedAt = savedProgress.lastSeenCreatedAt;
lastSeenId = savedProgress.lastSeenId;
usersCreated = savedProgress.usersCreated;
accountsCreated = savedProgress.accountsCreated;
twoFactorCreated = savedProgress.twoFactorCreated;
skipped = savedProgress.skipped;
totalUsersProcessed = savedProgress.totalUsersProcessed;
}
// Helper to get current progress object
const getCurrentProgress = (): MigrationProgress => ({
currentOffset,
lastSeenCreatedAt,
lastSeenId,
usersCreated,
accountsCreated,
twoFactorCreated,
skipped,
totalUsersProcessed,
lastUpdated: new Date().toISOString(),
@@ -194,14 +192,30 @@ export async function migrateUsers() {
// Check if shutdown was requested
if (shutdownRequested) break;
console.log(`📥 Fetching users batch from production database (OFFSET ${currentOffset})...`);
console.log(
`📥 Fetching users batch from production database (cursor: createdAt=${lastSeenCreatedAt}, id=${lastSeenId})...`,
);
const users = (await productionDb.execute(sql`
SELECT id, name, picture, username, email, locale, "emailVerified", "twoFactorEnabled", "createdAt", "updatedAt", provider
FROM "User"
ORDER BY "id" ASC
LIMIT ${BATCH_SIZE} OFFSET ${currentOffset}
`)) as unknown as ProductionUser[];
let users: ProductionUser[];
if (lastSeenCreatedAt && lastSeenId) {
const result = (await productionDb.execute(sql`
SELECT id, name, picture, username, email, locale, "emailVerified", "createdAt", "updatedAt", provider
FROM "User"
WHERE ("createdAt", id) < (${lastSeenCreatedAt}::timestamp, ${lastSeenId})
ORDER BY "createdAt" DESC, id DESC
LIMIT ${BATCH_SIZE}
`)) as unknown as QueryResult<ProductionUser>;
users = result.rows;
} else {
const result = (await productionDb.execute(sql`
SELECT id, name, picture, username, email, locale, "emailVerified", "createdAt", "updatedAt", provider
FROM "User"
ORDER BY "createdAt" DESC, id DESC
LIMIT ${BATCH_SIZE}
`)) as unknown as QueryResult<ProductionUser>;
users = result.rows;
}
console.log(`📋 Found ${users.length} users in this batch.`);
@@ -217,11 +231,11 @@ export async function migrateUsers() {
// Escape single quotes in IDs (though UUIDs shouldn't contain them, this is safer)
const userIdsForSql = userIds.map((id) => `'${id.replace(/'/g, "''")}'`).join(", ");
const secrets = (await productionDb.execute(sql`
SELECT id, password, "lastSignedIn", "verificationToken", "twoFactorSecret", "twoFactorBackupCodes", "refreshToken", "resetToken", "userId"
const { rows: secrets } = (await productionDb.execute(sql`
SELECT id, password, "lastSignedIn", "userId"
FROM "Secrets"
WHERE "userId" IN (${sql.raw(userIdsForSql)})
`)) as unknown as ProductionSecrets[];
`)) as unknown as QueryResult<ProductionSecrets>;
// Create a map of userId -> secrets for quick lookup
const secretsMap = new Map<string, ProductionSecrets>();
@@ -240,8 +254,15 @@ export async function migrateUsers() {
if (usersToProcess.length === 0) {
console.log(`⏭️ All users in this batch were already migrated.`);
currentOffset += users.length;
// Update cursor to the last user in this batch
const lastUser = users[users.length - 1];
if (lastUser) {
lastSeenCreatedAt =
lastUser.createdAt instanceof Date ? lastUser.createdAt.toISOString() : String(lastUser.createdAt);
lastSeenId = lastUser.id;
}
totalUsersProcessed += users.length;
await saveProgress(getCurrentProgress());
continue;
}
@@ -287,9 +308,16 @@ export async function migrateUsers() {
if (usersToInsert.length === 0) {
console.log(`⏭️ All users in this batch already exist in target DB.`);
currentOffset += users.length;
// Update cursor to the last user in this batch
const lastUser = users[users.length - 1];
if (lastUser) {
lastSeenCreatedAt =
lastUser.createdAt instanceof Date ? lastUser.createdAt.toISOString() : String(lastUser.createdAt);
lastSeenId = lastUser.id;
}
totalUsersProcessed += users.length;
await saveUserIdMapToFile(userIdMap);
await saveProgress(getCurrentProgress());
continue;
}
@@ -308,7 +336,7 @@ export async function migrateUsers() {
username: username,
displayUsername: displayUsername,
emailVerified: user.emailVerified,
twoFactorEnabled: user.twoFactorEnabled,
twoFactorEnabled: false, // All users start with 2FA disabled in the new system
createdAt: user.createdAt,
updatedAt: user.updatedAt,
},
@@ -339,7 +367,6 @@ export async function migrateUsers() {
accountId: accountId,
providerId: providerId,
password: userSecrets?.password ?? null,
refreshToken: userSecrets?.refreshToken ?? null,
createdAt: userData.createdAt,
updatedAt: userData.updatedAt,
};
@@ -352,34 +379,6 @@ export async function migrateUsers() {
}
accountsCreated += accountsToInsert.length;
// Prepare two-factor entries for bulk insert
const twoFactorToInsert = usersToInsertData
.map(({ originalUser, newUserId, userData }) => {
const userSecrets = secretsMap.get(originalUser.id);
if (originalUser.twoFactorEnabled && userSecrets?.twoFactorSecret) {
return {
id: generateId(),
userId: newUserId,
secret: userSecrets.twoFactorSecret,
backupCodes: userSecrets.twoFactorBackupCodes.join(","),
createdAt: userData.createdAt,
updatedAt: userData.updatedAt,
};
}
return null;
})
.filter((entry): entry is NonNullable<typeof entry> => entry !== null);
// Bulk insert two-factor entries (chunked)
if (twoFactorToInsert.length > 0) {
for (let i = 0; i < twoFactorToInsert.length; i += INSERT_CHUNK_SIZE) {
const chunk = twoFactorToInsert.slice(i, i + INSERT_CHUNK_SIZE);
await localDb.insert(schema.twoFactor).values(chunk);
}
twoFactorCreated += twoFactorToInsert.length;
}
const batchEnd = performance.now();
const batchTimeMs = batchEnd - batchStart;
console.log(
@@ -388,15 +387,24 @@ export async function migrateUsers() {
// Save progress after each batch
await saveUserIdMapToFile(userIdMap);
await saveProgress(getCurrentProgress());
} catch (error) {
console.error(`🚨 Failed to bulk insert users batch:`, error);
// Continue with next batch even if this one fails
}
currentOffset += users.length;
// Update cursor to the last user in this batch
const lastUser = users[users.length - 1];
if (lastUser) {
lastSeenCreatedAt =
lastUser.createdAt instanceof Date ? lastUser.createdAt.toISOString() : String(lastUser.createdAt);
lastSeenId = lastUser.id;
}
totalUsersProcessed += users.length;
console.log(`📦 Processed ${totalUsersProcessed} users so far...\n`);
// Save progress after updating cursor
await saveProgress(getCurrentProgress());
}
// Remove signal handlers
@@ -409,7 +417,6 @@ export async function migrateUsers() {
console.log("\n📊 Migration Summary:");
console.log(` Users created: ${usersCreated}`);
console.log(` Accounts created: ${accountsCreated}`);
console.log(` Two-factor entries created: ${twoFactorCreated}`);
console.log(` Skipped (already exist): ${skipped}`);
console.log(
`⏱️ Total migration time: ${migrationDurationMs.toFixed(1)} ms (${(migrationDurationMs / 1000).toFixed(2)} seconds)`,