Database-level multi-library support #48 (#58)

* feat: start of library backends

* feat: update backend routes and create initializer

* feat: add legacy library creation

* fix: resolve frontend type errors

* fix: runtime errors

* fix: lint
This commit is contained in:
DecDuck
2025-06-01 16:05:05 +10:00
committed by GitHub
parent 490afd0bb7
commit 3e5c3678d5
21 changed files with 664 additions and 298 deletions

View File

@ -16,7 +16,7 @@
"lint:fix": "eslint . --fix && prettier --write --list-different ."
},
"dependencies": {
"@drop-oss/droplet": "^0.7.2",
"@drop-oss/droplet": "^1.3.1",
"@headlessui/vue": "^1.7.23",
"@heroicons/vue": "^2.1.5",
"@lobomfz/prismark": "0.0.3",

View File

@ -27,10 +27,7 @@
</NuxtLink>
</p>
<div
v-if="libraryState.unimportedGames.length > 0"
class="mt-2 rounded-md bg-blue-600/10 p-4"
>
<div v-if="toImport" class="mt-2 rounded-md bg-blue-600/10 p-4">
<div class="flex">
<div class="flex-shrink-0">
<InformationCircleIcon
@ -177,4 +174,5 @@ useHead({
});
const libraryState = await $dropFetch("/api/v1/admin/library");
const toImport = Object.entries(libraryState.unimportedGames).length > 0;
</script>

View File

@ -13,7 +13,7 @@
class="relative w-full cursor-default rounded-md bg-zinc-950 py-1.5 pl-3 pr-10 text-left text-zinc-100 shadow-sm ring-1 ring-inset ring-zinc-800 focus:outline-none focus:ring-2 focus:ring-blue-600 sm:text-sm sm:leading-6"
>
<span v-if="currentlySelectedGame != -1" class="block truncate">{{
games.unimportedGames[currentlySelectedGame]
games.unimportedGames[currentlySelectedGame].game
}}</span>
<span v-else class="block truncate text-zinc-400"
>Please select a directory...</span
@ -37,7 +37,7 @@
class="absolute z-10 mt-1 max-h-60 w-full overflow-auto rounded-md bg-zinc-900 py-1 text-base shadow-lg ring-1 ring-zinc-800 focus:outline-none sm:text-sm"
>
<ListboxOption
v-for="(game, gameIdx) in games.unimportedGames"
v-for="({ game }, gameIdx) in games.unimportedGames"
:key="game"
v-slot="{ active, selected }"
as="template"
@ -275,7 +275,6 @@ definePageMeta({
});
const games = await $dropFetch("/api/v1/admin/import/game");
const currentlySelectedGame = ref(-1);
const gameSearchResultsLoading = ref(false);
const gameSearchResultsError = ref<string | undefined>();
@ -286,12 +285,12 @@ async function updateSelectedGame(value: number) {
if (currentlySelectedGame.value == value) return;
currentlySelectedGame.value = value;
if (currentlySelectedGame.value == -1) return;
const game = games.unimportedGames[currentlySelectedGame.value];
if (!game) return;
const option = games.unimportedGames[currentlySelectedGame.value];
if (!option) return;
metadataResults.value = undefined;
currentlySelectedMetadata.value = -1;
gameSearchTerm.value = game;
gameSearchTerm.value = option.game;
await searchGame();
}
@ -324,17 +323,21 @@ const router = useRouter();
const importLoading = ref(false);
const importError = ref<string | undefined>();
async function importGame(metadata: boolean) {
if (!metadataResults.value && metadata) return;
async function importGame(useMetadata: boolean) {
if (!metadataResults.value && useMetadata) return;
const metadata =
useMetadata && metadataResults.value
? metadataResults.value[currentlySelectedMetadata.value]
: undefined;
const option = games.unimportedGames[currentlySelectedGame.value];
const game = await $dropFetch("/api/v1/admin/import/game", {
method: "POST",
body: {
path: games.unimportedGames[currentlySelectedGame.value],
metadata:
metadata && metadataResults.value
? metadataResults.value[currentlySelectedMetadata.value]
: undefined,
path: option.game,
library: option.library,
metadata,
},
});

View File

@ -14,10 +14,7 @@
version.
</p>
</div>
<div
v-if="libraryState.unimportedGames.length > 0"
class="rounded-md bg-blue-600/10 p-4"
>
<div v-if="toImport" class="rounded-md bg-blue-600/10 p-4">
<div class="flex">
<div class="flex-shrink-0">
<InformationCircleIcon
@ -186,6 +183,9 @@ useHead({
const searchQuery = ref("");
const libraryState = await $dropFetch("/api/v1/admin/library");
const toImport = ref(Object.entries(libraryState.unimportedGames).length > 0);
const libraryGames = ref(
libraryState.games.map((e) => {
const noVersions = e.status.noVersions;
@ -219,5 +219,6 @@ async function deleteGame(id: string) {
await $dropFetch(`/api/v1/admin/game?id=${id}`, { method: "DELETE" });
const index = libraryGames.value.findIndex((e) => e.id === id);
libraryGames.value.splice(index, 1);
toImport.value = true;
}
</script>

View File

@ -0,0 +1,53 @@
/*
Warnings:
- You are about to drop the `ClientPeerAPIConfiguration` table. If the table is not empty, all the data it contains will be lost.
*/
-- CreateEnum
CREATE TYPE "LibraryBackend" AS ENUM ('Filesystem');
-- AlterEnum
ALTER TYPE "ClientCapabilities" ADD VALUE 'trackPlaytime';
-- DropForeignKey
ALTER TABLE "ClientPeerAPIConfiguration" DROP CONSTRAINT "ClientPeerAPIConfiguration_clientId_fkey";
-- AlterTable
ALTER TABLE "Screenshot" ALTER COLUMN "private" DROP DEFAULT;
-- DropTable
DROP TABLE "ClientPeerAPIConfiguration";
-- CreateTable
CREATE TABLE "Library" (
"id" TEXT NOT NULL,
"name" TEXT NOT NULL,
"backend" "LibraryBackend" NOT NULL,
"options" JSONB NOT NULL,
CONSTRAINT "Library_pkey" PRIMARY KEY ("id")
);
-- CreateTable
CREATE TABLE "Playtime" (
"gameId" TEXT NOT NULL,
"userId" TEXT NOT NULL,
"seconds" INTEGER NOT NULL,
"updatedAt" TIMESTAMPTZ(6) NOT NULL,
"createdAt" TIMESTAMPTZ(6) NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT "Playtime_pkey" PRIMARY KEY ("gameId","userId")
);
-- CreateIndex
CREATE INDEX "Playtime_userId_idx" ON "Playtime"("userId");
-- CreateIndex
CREATE INDEX "Screenshot_userId_idx" ON "Screenshot"("userId");
-- AddForeignKey
ALTER TABLE "Playtime" ADD CONSTRAINT "Playtime_gameId_fkey" FOREIGN KEY ("gameId") REFERENCES "Game"("id") ON DELETE CASCADE ON UPDATE CASCADE;
-- AddForeignKey
ALTER TABLE "Playtime" ADD CONSTRAINT "Playtime_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE;

View File

@ -0,0 +1,17 @@
/*
Warnings:
- You are about to drop the column `libraryBasePath` on the `Game` table. All the data in the column will be lost.
*/
-- DropIndex
DROP INDEX "Game_libraryBasePath_key";
-- AlterTable
ALTER TABLE "Game" RENAME COLUMN "libraryBasePath" TO "libraryPath";
ALTER TABLE "Game" ADD COLUMN "libraryId" TEXT;
-- AddForeignKey
ALTER TABLE "Game"
ADD CONSTRAINT "Game_libraryId_fkey" FOREIGN KEY ("libraryId") REFERENCES "Library" ("id") ON DELETE SET NULL ON UPDATE CASCADE;

View File

@ -0,0 +1,8 @@
/*
Warnings:
- A unique constraint covering the columns `[libraryId,libraryPath]` on the table `Game` will be added. If there are existing duplicate values, this will fail.
*/
-- CreateIndex
CREATE UNIQUE INDEX "Game_libraryId_libraryPath_key" ON "Game"("libraryId", "libraryPath");

View File

@ -1,7 +1,7 @@
model ApplicationSettings {
timestamp DateTime @id @default(now())
metadataProviders String[]
metadataProviders String[]
saveSlotCountLimit Int @default(5)
saveSlotSizeLimit Float @default(10) // MB
@ -13,3 +13,17 @@ enum Platform {
Linux @map("linux")
macOS @map("macos")
}
enum LibraryBackend {
Filesystem
}
model Library {
id String @id @default(uuid())
name String
backend LibraryBackend
options Json
games Game[]
}

View File

@ -29,8 +29,13 @@ model Game {
mImageCarouselObjectIds String[] // linked to below array
mImageLibraryObjectIds String[] // linked to objects in s3
versions GameVersion[]
libraryBasePath String @unique // Base dir for all the game versions
versions GameVersion[]
// These fields will not be optional in the next version
// Any game without a library ID will be assigned one at startup, based on the defaults
libraryId String?
library Library? @relation(fields: [libraryId], references: [id])
libraryPath String
collections CollectionEntry[]
saves SaveSlot[]
@ -42,6 +47,7 @@ model Game {
publishers Company[] @relation(name: "publishers")
@@unique([metadataSource, metadataId], name: "metadataKey")
@@unique([libraryId, libraryPath], name: "libraryKey")
}
model GameRating {

View File

@ -33,11 +33,12 @@ export default defineEventHandler(async (h3) => {
},
});
if (!game)
if (!game || !game.libraryId)
throw createError({ statusCode: 404, statusMessage: "Game ID not found" });
const unimportedVersions = await libraryManager.fetchUnimportedVersions(
game.id,
const unimportedVersions = await libraryManager.fetchUnimportedGameVersions(
game.libraryId,
game.libraryPath,
);
return { game, unimportedVersions };

View File

@ -6,5 +6,10 @@ export default defineEventHandler(async (h3) => {
if (!allowed) throw createError({ statusCode: 403 });
const unimportedGames = await libraryManager.fetchAllUnimportedGames();
return { unimportedGames };
const iterableUnimportedGames = Object.entries(unimportedGames)
.map(([libraryId, gameArray]) =>
gameArray.map((e) => ({ game: e, library: libraryId })),
)
.flat();
return { unimportedGames: iterableUnimportedGames };
});

View File

@ -1,37 +1,46 @@
import { type } from "arktype";
import { throwingArktype } from "~/server/arktype";
import aclManager from "~/server/internal/acls";
import libraryManager from "~/server/internal/library";
import metadataHandler from "~/server/internal/metadata";
import type {
GameMetadataSearchResult,
GameMetadataSource,
} from "~/server/internal/metadata/types";
export default defineEventHandler(async (h3) => {
const allowed = await aclManager.allowSystemACL(h3, ["import:game:new"]);
if (!allowed) throw createError({ statusCode: 403 });
const ImportGameBody = type({
library: "string",
path: "string",
["metadata?"]: {
id: "string",
sourceId: "string",
name: "string",
},
}).configure(throwingArktype);
const body = await readBody(h3);
export default defineEventHandler<{ body: typeof ImportGameBody.infer }>(
async (h3) => {
const allowed = await aclManager.allowSystemACL(h3, ["import:game:new"]);
if (!allowed) throw createError({ statusCode: 403 });
const path = body.path;
const metadata = body.metadata as GameMetadataSearchResult &
GameMetadataSource;
if (!path)
throw createError({
statusCode: 400,
statusMessage: "Path missing from body",
});
const { library, path, metadata } = await readValidatedBody(
h3,
ImportGameBody,
);
const validPath = await libraryManager.checkUnimportedGamePath(path);
if (!validPath)
throw createError({
statusCode: 400,
statusMessage: "Invalid unimported game path",
});
if (!path)
throw createError({
statusCode: 400,
statusMessage: "Path missing from body",
});
if (!metadata || !metadata.id || !metadata.sourceId) {
console.log(metadata);
return await metadataHandler.createGameWithoutMetadata(path);
} else {
return await metadataHandler.createGame(metadata, path);
}
});
const valid = await libraryManager.checkUnimportedGamePath(library, path);
if (!valid)
throw createError({
statusCode: 400,
statusMessage: "Invalid library or game.",
});
if (!metadata) {
return await metadataHandler.createGameWithoutMetadata(library, path);
} else {
return await metadataHandler.createGame(metadata, library, path);
}
},
);

View File

@ -1,4 +1,5 @@
import aclManager from "~/server/internal/acls";
import prisma from "~/server/internal/db/database";
import libraryManager from "~/server/internal/library";
export default defineEventHandler(async (h3) => {
@ -13,8 +14,17 @@ export default defineEventHandler(async (h3) => {
statusMessage: "Missing id in request params",
});
const unimportedVersions =
await libraryManager.fetchUnimportedVersions(gameId);
const game = await prisma.game.findUnique({
where: { id: gameId },
select: { libraryId: true, libraryPath: true },
});
if (!game || !game.libraryId)
throw createError({ statusCode: 404, statusMessage: "Game not found" });
const unimportedVersions = await libraryManager.fetchUnimportedGameVersions(
game.libraryId,
game.libraryPath,
);
if (!unimportedVersions)
throw createError({ statusCode: 400, statusMessage: "Invalid game ID" });

View File

@ -9,31 +9,31 @@ const ImportVersion = type({
version: "string",
platform: "string",
launch: "string?",
launchArgs: "string?",
setup: "string?",
setupArgs: "string?",
onlySetup: "boolean?",
delta: "boolean?",
umuId: "string?",
launch: "string = ''",
launchArgs: "string = ''",
setup: "string = ''",
setupArgs: "string = ''",
onlySetup: "boolean = false",
delta: "boolean = false",
umuId: "string = ''",
});
export default defineEventHandler(async (h3) => {
const allowed = await aclManager.allowSystemACL(h3, ["import:version:new"]);
if (!allowed) throw createError({ statusCode: 403 });
const body = await readValidatedBody(h3, ImportVersion);
const gameId = body.id;
const versionName = body.version;
const platform = body.platform;
const launch = body.launch ?? "";
const launchArgs = body.launchArgs ?? "";
const setup = body.setup ?? "";
const setupArgs = body.setupArgs ?? "";
const onlySetup = body.onlySetup ?? false;
const delta = body.delta ?? false;
const umuId = body.umuId ?? "";
const {
id,
version,
platform,
launch,
launchArgs,
setup,
setupArgs,
onlySetup,
delta,
umuId,
} = await readValidatedBody(h3, ImportVersion);
const platformParsed = parsePlatform(platform);
if (!platformParsed)
@ -41,7 +41,7 @@ export default defineEventHandler(async (h3) => {
if (delta) {
const validOverlayVersions = await prisma.gameVersion.count({
where: { gameId: gameId, platform: platformParsed, delta: false },
where: { gameId: id, platform: platformParsed, delta: false },
});
if (validOverlayVersions == 0)
throw createError({
@ -66,7 +66,7 @@ export default defineEventHandler(async (h3) => {
}
// startup & delta require more complex checking logic
const taskId = await libraryManager.importVersion(gameId, versionName, {
const taskId = await libraryManager.importVersion(id, version, {
platform,
onlySetup,

View File

@ -1,10 +1,14 @@
import cacheHandler from "~/server/internal/cache";
import prisma from "~/server/internal/db/database";
import fs from "fs";
import path from "path";
import libraryManager from "~/server/internal/library";
const chunkSize = 1024 * 1024 * 64;
const gameLookupCache = cacheHandler.createCache<{
libraryId: string | null;
libraryPath: string;
}>("downloadGameLookupCache");
export default defineEventHandler(async (h3) => {
const query = getQuery(h3);
const gameId = query.id?.toString();
@ -18,36 +22,40 @@ export default defineEventHandler(async (h3) => {
statusMessage: "Invalid chunk arguments",
});
const game = await prisma.game.findUnique({
where: {
id: gameId,
},
select: {
libraryBasePath: true,
},
});
if (!game)
throw createError({ statusCode: 400, statusMessage: "Invalid game ID" });
let game = await gameLookupCache.getItem(gameId);
if (!game) {
game = await prisma.game.findUnique({
where: {
id: gameId,
},
select: {
libraryId: true,
libraryPath: true,
},
});
if (!game || !game.libraryId)
throw createError({ statusCode: 400, statusMessage: "Invalid game ID" });
const versionDir = path.join(
libraryManager.fetchLibraryPath(),
game.libraryBasePath,
versionName,
);
if (!fs.existsSync(versionDir))
await gameLookupCache.setItem(gameId, game);
}
if (!game.libraryId)
throw createError({
statusCode: 400,
statusMessage: "Invalid version name",
statusCode: 500,
statusMessage: "Somehow, we got here.",
});
const gameFile = path.join(versionDir, filename);
if (!fs.existsSync(gameFile))
throw createError({ statusCode: 400, statusMessage: "Invalid game file" });
const gameFileStats = fs.statSync(gameFile);
const peek = await libraryManager.peekFile(
game.libraryId,
game.libraryPath,
versionName,
filename,
);
if (!peek)
throw createError({ status: 400, statusMessage: "Failed to peek file" });
const start = chunkIndex * chunkSize;
const end = Math.min((chunkIndex + 1) * chunkSize, gameFileStats.size);
const end = Math.min((chunkIndex + 1) * chunkSize, peek.size);
const currentChunkSize = end - start;
setHeader(h3, "Content-Length", currentChunkSize);
@ -57,7 +65,18 @@ export default defineEventHandler(async (h3) => {
statusMessage: "Invalid chunk index",
});
const gameReadStream = fs.createReadStream(gameFile, { start, end: end - 1 }); // end needs to be offset by 1
const gameReadStream = await libraryManager.readFile(
game.libraryId,
game.libraryPath,
versionName,
filename,
{ start, end: end - 1 },
); // end needs to be offset by 1
if (!gameReadStream)
throw createError({
statusCode: 400,
statusMessage: "Failed to create stream",
});
return sendStream(h3, gameReadStream);
});

View File

@ -0,0 +1,107 @@
import { ArkErrors, type } from "arktype";
import {
GameNotFoundError,
VersionNotFoundError,
type LibraryProvider,
} from "./provider";
import { LibraryBackend } from "~/prisma/client";
import fs from "fs";
import path from "path";
import droplet from "@drop-oss/droplet";
import type { Readable } from "stream";
export const FilesystemProviderConfig = type({
baseDir: "string",
});
export class FilesystemProvider
implements LibraryProvider<typeof FilesystemProviderConfig.infer>
{
private config: typeof FilesystemProviderConfig.infer;
private myId: string;
constructor(rawConfig: unknown, id: string) {
const config = FilesystemProviderConfig(rawConfig);
if (config instanceof ArkErrors) {
throw new Error(
`Failed to create filesystem provider: ${config.summary}`,
);
}
this.myId = id;
this.config = config;
fs.mkdirSync(this.config.baseDir, { recursive: true });
}
id(): string {
return this.myId;
}
type(): LibraryBackend {
return LibraryBackend.Filesystem;
}
async listGames(): Promise<string[]> {
const dirs = fs.readdirSync(this.config.baseDir);
const folderDirs = dirs.filter((e) => {
const fullDir = path.join(this.config.baseDir, e);
return fs.lstatSync(fullDir).isDirectory();
});
return folderDirs;
}
async listVersions(game: string): Promise<string[]> {
const gameDir = path.join(this.config.baseDir, game);
if (!fs.existsSync(gameDir)) throw new GameNotFoundError();
const versionDirs = fs.readdirSync(gameDir);
const validVersionDirs = versionDirs.filter((e) => {
const fullDir = path.join(this.config.baseDir, game, e);
return droplet.hasBackendForPath(fullDir);
});
return validVersionDirs;
}
async versionReaddir(game: string, version: string): Promise<string[]> {
const versionDir = path.join(this.config.baseDir, game, version);
if (!fs.existsSync(versionDir)) throw new VersionNotFoundError();
return droplet.listFiles(versionDir);
}
async generateDropletManifest(
game: string,
version: string,
progress: (err: Error | null, v: number) => void,
log: (err: Error | null, v: string) => void,
): Promise<string> {
const versionDir = path.join(this.config.baseDir, game, version);
if (!fs.existsSync(versionDir)) throw new VersionNotFoundError();
const manifest = await new Promise<string>((r, j) =>
droplet.generateManifest(versionDir, progress, log, (err, result) => {
if (err) return j(err);
r(result);
}),
);
return manifest;
}
// TODO: move this over to the droplet.readfile function it works
async readFile(
game: string,
version: string,
filename: string,
options?: { start?: number; end?: number },
): Promise<Readable | undefined> {
const filepath = path.join(this.config.baseDir, game, version, filename);
if (!fs.existsSync(filepath)) return undefined;
const stream = fs.createReadStream(filepath, options);
return stream;
}
async peekFile(game: string, version: string, filename: string) {
const filepath = path.join(this.config.baseDir, game, version, filename);
if (!fs.existsSync(filepath)) return undefined;
const stat = fs.statSync(filepath);
return { size: stat.size };
}
}

View File

@ -5,60 +5,63 @@
* It also provides the endpoints with information about unmatched games
*/
import fs from "fs";
import path from "path";
import prisma from "../db/database";
import type { GameVersion } from "~/prisma/client";
import { fuzzy } from "fast-fuzzy";
import { recursivelyReaddir } from "../utils/recursivedirs";
import taskHandler from "../tasks";
import { parsePlatform } from "../utils/parseplatform";
import droplet from "@drop-oss/droplet";
import notificationSystem from "../notifications";
import { systemConfig } from "../config/sys-conf";
import type { LibraryProvider } from "./provider";
class LibraryManager {
private basePath: string;
private libraries: Map<string, LibraryProvider<unknown>> = new Map();
constructor() {
this.basePath = systemConfig.getLibraryFolder();
fs.mkdirSync(this.basePath, { recursive: true });
}
fetchLibraryPath() {
return this.basePath;
addLibrary(library: LibraryProvider<unknown>) {
this.libraries.set(library.id(), library);
}
async fetchAllUnimportedGames() {
const dirs = fs.readdirSync(this.basePath).filter((e) => {
const fullDir = path.join(this.basePath, e);
return fs.lstatSync(fullDir).isDirectory();
});
const unimportedGames: { [key: string]: string[] } = {};
const validGames = await prisma.game.findMany({
where: {
libraryBasePath: { in: dirs },
},
select: {
libraryBasePath: true,
},
});
const validGameDirs = validGames.map((e) => e.libraryBasePath);
for (const [id, library] of this.libraries.entries()) {
const games = await library.listGames();
const validGames = await prisma.game.findMany({
where: {
libraryId: id,
libraryPath: { in: games },
},
select: {
libraryPath: true,
},
});
const providerUnimportedGames = games.filter(
(e) => validGames.findIndex((v) => v.libraryPath == e) == -1,
);
unimportedGames[id] = providerUnimportedGames;
}
const unregisteredGames = dirs.filter((e) => !validGameDirs.includes(e));
return unregisteredGames;
return unimportedGames;
}
async fetchUnimportedGameVersions(
libraryBasePath: string,
versions: Array<GameVersion>,
) {
const gameDir = path.join(this.basePath, libraryBasePath);
const versionsDirs = fs.readdirSync(gameDir);
const importedVersionDirs = versions.map((e) => e.versionName);
const unimportedVersions = versionsDirs.filter(
(e) => !importedVersionDirs.includes(e),
async fetchUnimportedGameVersions(libraryId: string, libraryPath: string) {
const provider = this.libraries.get(libraryId);
if (!provider) return undefined;
const game = await prisma.game.findUnique({
where: {
libraryKey: {
libraryId,
libraryPath,
},
},
select: {
versions: true,
},
});
if (!game) return undefined;
const versions = await provider.listVersions(libraryPath);
const unimportedVersions = versions.filter(
(e) => game.versions.findIndex((v) => v.versionName == e) == -1,
);
return unimportedVersions;
@ -73,7 +76,8 @@ class LibraryManager {
mShortDescription: true,
metadataSource: true,
mIconObjectId: true,
libraryBasePath: true,
libraryId: true,
libraryPath: true,
},
orderBy: {
mName: "asc",
@ -85,60 +89,24 @@ class LibraryManager {
game: e,
status: {
noVersions: e.versions.length == 0,
unimportedVersions: await this.fetchUnimportedGameVersions(
e.libraryBasePath,
e.versions,
),
unimportedVersions: (await this.fetchUnimportedGameVersions(
e.libraryId ?? "",
e.libraryPath,
))!,
},
})),
);
}
async fetchUnimportedVersions(gameId: string) {
const game = await prisma.game.findUnique({
where: { id: gameId },
select: {
versions: {
select: {
versionName: true,
},
},
libraryBasePath: true,
},
});
if (!game) return undefined;
const targetDir = path.join(this.basePath, game.libraryBasePath);
if (!fs.existsSync(targetDir))
throw new Error(
"Game in database, but no physical directory? Something is very very wrong...",
);
const versions = fs.readdirSync(targetDir);
const validVersions = versions.filter((versionDir) => {
const versionPath = path.join(targetDir, versionDir);
const stat = fs.statSync(versionPath);
return stat.isDirectory();
});
const currentVersions = game.versions.map((e) => e.versionName);
const unimportedVersions = validVersions.filter(
(e) => !currentVersions.includes(e),
);
return unimportedVersions;
}
async fetchUnimportedVersionInformation(gameId: string, versionName: string) {
const game = await prisma.game.findUnique({
where: { id: gameId },
select: { libraryBasePath: true, mName: true },
select: { libraryPath: true, libraryId: true, mName: true },
});
if (!game) return undefined;
const targetDir = path.join(
this.basePath,
game.libraryBasePath,
versionName,
);
if (!fs.existsSync(targetDir)) return undefined;
if (!game || !game.libraryId) return undefined;
const library = this.libraries.get(game.libraryId);
if (!library) return undefined;
const fileExts: { [key: string]: string[] } = {
Linux: [
@ -165,7 +133,7 @@ class LibraryManager {
match: number;
}> = [];
const files = recursivelyReaddir(targetDir, 2);
const files = await library.versionReaddir(game.libraryPath, versionName);
for (const file of files) {
const filename = path.basename(file);
const dotLocation = file.lastIndexOf(".");
@ -174,10 +142,9 @@ class LibraryManager {
for (const checkExt of checkExts) {
if (checkExt != ext) continue;
const fuzzyValue = fuzzy(filename, game.mName);
const relative = path.relative(targetDir, file);
options.push({
filename: relative,
platform: platform,
filename,
platform,
match: fuzzyValue,
});
}
@ -190,17 +157,22 @@ class LibraryManager {
}
// Checks are done in least to most expensive order
async checkUnimportedGamePath(targetPath: string) {
const targetDir = path.join(this.basePath, targetPath);
if (!fs.existsSync(targetDir)) return false;
async checkUnimportedGamePath(libraryId: string, libraryPath: string) {
const hasGame =
(await prisma.game.count({ where: { libraryBasePath: targetPath } })) > 0;
(await prisma.game.count({ where: { libraryId, libraryPath } })) > 0;
if (hasGame) return false;
return true;
}
/*
Game creation happens in metadata, because it's primarily a metadata object
async createGame(libraryId: string, libraryPath: string, game: Omit<Game, "libraryId" | "libraryPath">) {
}
*/
async importVersion(
gameId: string,
versionName: string,
@ -224,12 +196,12 @@ class LibraryManager {
const game = await prisma.game.findUnique({
where: { id: gameId },
select: { mName: true, libraryBasePath: true },
select: { mName: true, libraryId: true, libraryPath: true },
});
if (!game) return undefined;
if (!game || !game.libraryId) return undefined;
const baseDir = path.join(this.basePath, game.libraryBasePath, versionName);
if (!fs.existsSync(baseDir)) return undefined;
const library = this.libraries.get(game.libraryId);
if (!library) return undefined;
taskHandler.create({
id: taskId,
@ -238,23 +210,18 @@ class LibraryManager {
async run({ progress, log }) {
// First, create the manifest via droplet.
// This takes up 90% of our progress, so we wrap it in a *0.9
const manifest = await new Promise<string>((resolve, reject) => {
droplet.generateManifest(
baseDir,
(err, value) => {
if (err) return reject(err);
progress(value * 0.9);
},
(err, line) => {
if (err) return reject(err);
log(line);
},
(err, manifest) => {
if (err) return reject(err);
resolve(manifest);
},
);
});
const manifest = await library.generateDropletManifest(
game.libraryPath,
versionName,
(err, value) => {
if (err) throw err;
progress(value * 0.9);
},
(err, value) => {
if (err) throw err;
log(value);
},
);
log("Created manifest successfully!");
@ -315,6 +282,29 @@ class LibraryManager {
return taskId;
}
async peekFile(
libraryId: string,
game: string,
version: string,
filename: string,
) {
const library = this.libraries.get(libraryId);
if (!library) return undefined;
return library.peekFile(game, version, filename);
}
async readFile(
libraryId: string,
game: string,
version: string,
filename: string,
options?: { start?: number; end?: number },
) {
const library = this.libraries.get(libraryId);
if (!library) return undefined;
return library.readFile(game, version, filename, options);
}
}
export const libraryManager = new LibraryManager();

View File

@ -0,0 +1,64 @@
import type { Readable } from "stream";
import type { LibraryBackend } from "~/prisma/client";
export abstract class LibraryProvider<CFG> {
constructor(_config: CFG, _id: string) {
throw new Error("Library doesn't have a proper constructor");
}
/**
* @returns ID of the current library provider (fs, smb, s3, etc)
*/
abstract type(): LibraryBackend;
/**
* @returns the specific ID of this current provider
*/
abstract id(): string;
/**
* @returns list of (usually) top-level game folder names
*/
abstract listGames(): Promise<string[]>;
/**
* @param game folder name of the game to list versions for
* @returns list of version folder names
*/
abstract listVersions(game: string): Promise<string[]>;
/**
* @param game folder name of the game
* @param version folder name of the version
* @returns recursive list of all files in version, relative to the version folder (e.g. ./setup.exe)
*/
abstract versionReaddir(game: string, version: string): Promise<string[]>;
/**
* @param game folder name of the game
* @param version folder name of the version
* @returns string of JSON of the droplet manifest
*/
abstract generateDropletManifest(
game: string,
version: string,
progress: (err: Error | null, v: number) => void,
log: (err: Error | null, v: string) => void,
): Promise<string>;
abstract peekFile(
game: string,
version: string,
filename: string,
): Promise<{ size: number } | undefined>;
abstract readFile(
game: string,
version: string,
filename: string,
options?: { start?: number; end?: number },
): Promise<Readable | undefined>;
}
export class GameNotFoundError extends Error {}
export class VersionNotFoundError extends Error {}

View File

@ -97,18 +97,15 @@ export class MetadataHandler {
return successfulResults;
}
async createGameWithoutMetadata(libraryBasePath: string) {
async createGameWithoutMetadata(libraryId: string, libraryPath: string) {
return await this.createGame(
{
id: "",
name: libraryBasePath,
icon: "",
description: "",
year: 0,
name: libraryPath,
sourceId: "manual",
sourceName: "Manual",
},
libraryBasePath,
libraryId,
libraryPath,
);
}
@ -165,8 +162,9 @@ export class MetadataHandler {
}
async createGame(
result: InternalGameMetadataResult,
libraryBasePath: string,
result: { sourceId: string; id: string; name: string },
libraryId: string,
libraryPath: string,
) {
const provider = this.providers.get(result.sourceId);
if (!provider)
@ -231,7 +229,8 @@ export class MetadataHandler {
connectOrCreate: this.parseTags(metadata.tags),
},
libraryBasePath,
libraryId,
libraryPath,
},
});

View File

@ -0,0 +1,74 @@
import { LibraryBackend } from "~/prisma/client";
import prisma from "../internal/db/database";
import type { JsonValue } from "@prisma/client/runtime/library";
import type { LibraryProvider } from "../internal/library/provider";
import type { FilesystemProviderConfig } from "../internal/library/filesystem";
import { FilesystemProvider } from "../internal/library/filesystem";
import libraryManager from "../internal/library";
import path from "path";
const libraryConstructors: {
[key in LibraryBackend]: (
value: JsonValue,
id: string,
) => LibraryProvider<unknown>;
} = {
Filesystem: function (
value: JsonValue,
id: string,
): LibraryProvider<unknown> {
return new FilesystemProvider(value, id);
},
};
export default defineNitroPlugin(async () => {
let successes = 0;
const libraries = await prisma.library.findMany({});
// Add migration handler
const legacyPath = process.env.LIBRARY;
if (legacyPath && libraries.length == 0) {
const options: typeof FilesystemProviderConfig.infer = {
baseDir: path.resolve(legacyPath),
};
const library = await prisma.library.create({
data: {
name: "Auto-created",
backend: LibraryBackend.Filesystem,
options,
},
});
libraries.push(library);
// Update all existing games
await prisma.game.updateMany({
where: {
libraryId: null,
},
data: {
libraryId: library.id,
},
});
}
for (const library of libraries) {
const constructor = libraryConstructors[library.backend];
try {
const provider = constructor(library.options, library.id);
libraryManager.addLibrary(provider);
successes++;
} catch (e) {
console.warn(
`Failed to create library (${library.id}) of type ${library.backend}:\n ${e}`,
);
}
}
if (successes == 0) {
console.warn(
"No library was successfully initialised. Please check for errors. If you have just set up an instance, this is normal.",
);
}
});

120
yarn.lock
View File

@ -305,83 +305,71 @@
dependencies:
mime "^3.0.0"
"@drop-oss/droplet-darwin-arm64@0.7.2":
version "0.7.2"
resolved "https://registry.yarnpkg.com/@drop-oss/droplet-darwin-arm64/-/droplet-darwin-arm64-0.7.2.tgz#fb714d3bf83dbf5e0ee6068ce2fdc74652a9d073"
integrity sha512-g1IiaSWYd+NDhyRbEKxSxrKFieJV/bwijcFfzP5VLHbTohDu5zJLe6Exc/IXbIb+Ex70Rfsk8Sf9n1zfHCD+Fg==
"@drop-oss/droplet-darwin-arm64@1.3.1":
version "1.3.1"
resolved "https://registry.yarnpkg.com/@drop-oss/droplet-darwin-arm64/-/droplet-darwin-arm64-1.3.1.tgz#672484c15419dbc068950f2d53f130ccbea3ff17"
integrity sha512-rarsZtIiZhv2hb3bAZSJjxwnme+rWUFY+FY79MRrMnz7EuNBez063pFBqDhwFCz+0QqDBz7zKDUQR6v+6gnVFw==
"@drop-oss/droplet-darwin-universal@0.7.2":
version "0.7.2"
resolved "https://registry.yarnpkg.com/@drop-oss/droplet-darwin-universal/-/droplet-darwin-universal-0.7.2.tgz#237d70fab92b892e4d40855d13fd54ad55cf026e"
integrity sha512-wVVkMi0uwOob876xNFc37/5dGusKjlsWc4Z9bTUtTGeWo9gx5BkEpHBRrwD9NBAklr0Eu7Kmin3niB7pfx9vTw==
"@drop-oss/droplet-darwin-universal@1.3.1":
version "1.3.1"
resolved "https://registry.yarnpkg.com/@drop-oss/droplet-darwin-universal/-/droplet-darwin-universal-1.3.1.tgz#0a3e1663125349b2d443dbfaba1bb7b8d2f9c64d"
integrity sha512-PuN5FdotwYuZ7O2r1aAWiE8hH/gH7CrH+j33OdgS4FI4XIOeW6qq+14JECZp6JgWv0863/C7tD5Ll4yMgIRvUQ==
"@drop-oss/droplet-darwin-x64@0.7.2":
version "0.7.2"
resolved "https://registry.yarnpkg.com/@drop-oss/droplet-darwin-x64/-/droplet-darwin-x64-0.7.2.tgz#ed12ced467ff38f7eb2419b0ae6b1c508d828b84"
integrity sha512-/p53OVesFG1Q/3+kYImitduGvZFfrfyVgdW+twoy+DYTX5EE1XZKaLZs2PSnbFSnnFJTmWvfnGqN5s+Dh12AKw==
"@drop-oss/droplet-darwin-x64@1.3.1":
version "1.3.1"
resolved "https://registry.yarnpkg.com/@drop-oss/droplet-darwin-x64/-/droplet-darwin-x64-1.3.1.tgz#755db12988b431eff24df9a7177453178f29ce47"
integrity sha512-IloUIHnEI67S38vJxADbcXk81tR8b4fFTTpyCNUlAwIXHGbhEjFrfu+sLdK94MHN/vjlafMBf0APwYF2vclCkw==
"@drop-oss/droplet-linux-arm-gnueabihf@0.7.2":
version "0.7.2"
resolved "https://registry.yarnpkg.com/@drop-oss/droplet-linux-arm-gnueabihf/-/droplet-linux-arm-gnueabihf-0.7.2.tgz#2fb0bfae2cb5fd08942d4f490f25046f006123ce"
integrity sha512-hZtkKhgMkSqhueOEBRBZlSWE6uawM9M31gPmajrYHNOEnnmt8oUtZriPvC1ffZwZnQb4LL7IMGUZmXTl6guZXQ==
"@drop-oss/droplet-linux-arm64-gnu@1.3.1":
version "1.3.1"
resolved "https://registry.yarnpkg.com/@drop-oss/droplet-linux-arm64-gnu/-/droplet-linux-arm64-gnu-1.3.1.tgz#26ac0b24a08e6785742a19cbf97003f23d3b152d"
integrity sha512-aiesHfQushi+EGmTj970bxZvhNsBh90kzKbg14vdgFTL0/mhcturJSHa0VhJ2/m4qIg10IlmJpbuEm165Q25rQ==
"@drop-oss/droplet-linux-arm-musleabihf@0.7.2":
version "0.7.2"
resolved "https://registry.yarnpkg.com/@drop-oss/droplet-linux-arm-musleabihf/-/droplet-linux-arm-musleabihf-0.7.2.tgz#cbf54f5ff271a9e4601f6f6489cb6f630c6e9cbc"
integrity sha512-FBy8GE06mWSlv/t3d7iOF2wP9jvvPTePwPpIQyMpmEOz5MmdwF3/PFFncV4WcmxQ/RHUhIrZ3M9Dfq8WCiXPgw==
"@drop-oss/droplet-linux-arm64-musl@1.3.1":
version "1.3.1"
resolved "https://registry.yarnpkg.com/@drop-oss/droplet-linux-arm64-musl/-/droplet-linux-arm64-musl-1.3.1.tgz#1449cd59a75363a01f2ed68c1921e6d93fbafccd"
integrity sha512-Oa6HvvBbflxoG1nmdYbcgMAf29aTZ6xCxC84X+Q8TjiO5Qx2PHI+nX+KKK8rrJdQszrqpdT9wZbnD4zDtLzSeQ==
"@drop-oss/droplet-linux-arm64-gnu@0.7.2":
version "0.7.2"
resolved "https://registry.yarnpkg.com/@drop-oss/droplet-linux-arm64-gnu/-/droplet-linux-arm64-gnu-0.7.2.tgz#96062bf8a63de742995d89b782fa7e11f26d984f"
integrity sha512-Ev+WOUwazMgzz3tcHZefCaELSQ/dUJA795eXiNp0jDFRhddeybulxabte9hM9XjP5Yg/pZ0GpenWMjcWvxVaIQ==
"@drop-oss/droplet-linux-riscv64-gnu@1.3.1":
version "1.3.1"
resolved "https://registry.yarnpkg.com/@drop-oss/droplet-linux-riscv64-gnu/-/droplet-linux-riscv64-gnu-1.3.1.tgz#6300be23128ed4fd8895ca610cc2fb9bdf9abc05"
integrity sha512-vAVUiMixfB/oXIZ7N6QhJB1N+lb96JLrs2EjZiPGNSgwKGMV0H+84ZI+5NJ30qoytm7WB8mm2beezoCpM8frjg==
"@drop-oss/droplet-linux-arm64-musl@0.7.2":
version "0.7.2"
resolved "https://registry.yarnpkg.com/@drop-oss/droplet-linux-arm64-musl/-/droplet-linux-arm64-musl-0.7.2.tgz#cabb0305e3337dc8fabe4d46d21756fe2d93bde6"
integrity sha512-uJ0oOjPNNsNrqc8kJhlOxetz+lYb1QUOIKyKjpmTKVHYjNXj8bvc/FSDYwQjCPRs0r9qrEszF8hW6lsibQ92/g==
"@drop-oss/droplet-linux-x64-gnu@1.3.1":
version "1.3.1"
resolved "https://registry.yarnpkg.com/@drop-oss/droplet-linux-x64-gnu/-/droplet-linux-x64-gnu-1.3.1.tgz#5f59e8816978af444301ad7116a0377f9aa2633a"
integrity sha512-R3UtBIw5amY1ExaX8fZMcS1zLv0DF9Y8YoBgqk+VbQrHMVfiQKiktv/dXRp+9iWzLB/m5aG/Se5QuJazOMlwtA==
"@drop-oss/droplet-linux-riscv64-gnu@0.7.2":
version "0.7.2"
resolved "https://registry.yarnpkg.com/@drop-oss/droplet-linux-riscv64-gnu/-/droplet-linux-riscv64-gnu-0.7.2.tgz#5ea5b99df8677def14da6099dec9577433736655"
integrity sha512-5xdbTvEs8MiOL3ren+QyCXvcLmKWa7NSAehdunaD82qIwV19Xz+/C7OC1jN2zGgAQ0TBM/HcbkmWITNEQB7Oiw==
"@drop-oss/droplet-linux-x64-musl@1.3.1":
version "1.3.1"
resolved "https://registry.yarnpkg.com/@drop-oss/droplet-linux-x64-musl/-/droplet-linux-x64-musl-1.3.1.tgz#84b136665dc713c66a3e5b2be6ce5d97514dcb25"
integrity sha512-rDtmTYzx39Y1xHyRvm2AW97GkHy4ZfhXsmYWSjqo0dmoM5BY/nHfmNO6kWOABg4WP6mr3NPZKJTe885JVNilcg==
"@drop-oss/droplet-linux-x64-gnu@0.7.2":
version "0.7.2"
resolved "https://registry.yarnpkg.com/@drop-oss/droplet-linux-x64-gnu/-/droplet-linux-x64-gnu-0.7.2.tgz#244f137f6f301c307414e43b7cbd42fbd3e3247f"
integrity sha512-xM7tEzAR/yGFpO3C3lLpyOiqCD84MqwXQS6I1aR+z7IU+tAVwX1JYmu4HYGw1pxPCHpK/9w8NtAwzgSiw5d2jQ==
"@drop-oss/droplet-win32-arm64-msvc@1.3.1":
version "1.3.1"
resolved "https://registry.yarnpkg.com/@drop-oss/droplet-win32-arm64-msvc/-/droplet-win32-arm64-msvc-1.3.1.tgz#77948fe25f27dedda979367feadbe89622aa6b19"
integrity sha512-ZQVLgloMd7NIW3j1cvL7DKp9164K8luLxb692yuXRF6pQ7ok8IPWgwiWoeqQ1OE/msPkgXEC7hHupwWtfX6tHw==
"@drop-oss/droplet-linux-x64-musl@0.7.2":
version "0.7.2"
resolved "https://registry.yarnpkg.com/@drop-oss/droplet-linux-x64-musl/-/droplet-linux-x64-musl-0.7.2.tgz#37bc2d079cc63949bd5ac5194be65cb9c769feb0"
integrity sha512-s9YbnqPQhz468py49icPO74ezXF+EGKt7DX9vMs7XIp2Uyz+pWejRkerSj70WTypy5UcSNgcIBOB6kfD/FMMAQ==
"@drop-oss/droplet-win32-x64-msvc@1.3.1":
version "1.3.1"
resolved "https://registry.yarnpkg.com/@drop-oss/droplet-win32-x64-msvc/-/droplet-win32-x64-msvc-1.3.1.tgz#35f2dca041af48dec6743bf5c62a35582e25715d"
integrity sha512-NJsZM4g40I0b/MHFTvur3t30ULiU8D3DfhZTlLzyT+btiQ/8PdjCKRM4CPJJhs7JG8Bp30cl2n54XnsnyaFtJA==
"@drop-oss/droplet-win32-arm64-msvc@0.7.2":
version "0.7.2"
resolved "https://registry.yarnpkg.com/@drop-oss/droplet-win32-arm64-msvc/-/droplet-win32-arm64-msvc-0.7.2.tgz#c0a6048b9dc89596bf230346c5bbe86fcdc27009"
integrity sha512-E0isKXZIt/mFUAfziZ9hat84uol4hWHcEZ86xxfz4L8/wljrKU7Vbw9yaYznk4FvKRHnwoccymtOTLrSq2Ju4Q==
"@drop-oss/droplet-win32-x64-msvc@0.7.2":
version "0.7.2"
resolved "https://registry.yarnpkg.com/@drop-oss/droplet-win32-x64-msvc/-/droplet-win32-x64-msvc-0.7.2.tgz#2835e05bcf9923eb23e04b94c298520ecb6299d0"
integrity sha512-O5t2B/3Ld+17q1qDPVds3V/Ex2as2l8piVBgEKIkEL51wJYu7ucwMwWrfdMWKXRn17Fl5ueeujZLuD3iySRkLw==
"@drop-oss/droplet@^0.7.2":
version "0.7.2"
resolved "https://registry.yarnpkg.com/@drop-oss/droplet/-/droplet-0.7.2.tgz#a914dbee85cb3b3a0c9dd90d9cebec5bb0575bce"
integrity sha512-XxKUuRMYMdTVT4IaetNRN07iUpHJkXdS1LKfPBDrNkjszfG0SGjqCd1PVw7p6ugPWdezS8ygGODR6c/cAOQ4kw==
"@drop-oss/droplet@^1.3.1":
version "1.3.1"
resolved "https://registry.yarnpkg.com/@drop-oss/droplet/-/droplet-1.3.1.tgz#360faadadf50dbe3133ed8aadce6a077d289a48d"
integrity sha512-wXQof5rUiUujAiVwJovAu4Tj2Rhlb0pE/lHSEHF3ywcOLQFcSrE1naQVz3RQ7at+ZkwmDL9fFMXmGJkEKI8jog==
optionalDependencies:
"@drop-oss/droplet-darwin-arm64" "0.7.2"
"@drop-oss/droplet-darwin-universal" "0.7.2"
"@drop-oss/droplet-darwin-x64" "0.7.2"
"@drop-oss/droplet-linux-arm-gnueabihf" "0.7.2"
"@drop-oss/droplet-linux-arm-musleabihf" "0.7.2"
"@drop-oss/droplet-linux-arm64-gnu" "0.7.2"
"@drop-oss/droplet-linux-arm64-musl" "0.7.2"
"@drop-oss/droplet-linux-riscv64-gnu" "0.7.2"
"@drop-oss/droplet-linux-x64-gnu" "0.7.2"
"@drop-oss/droplet-linux-x64-musl" "0.7.2"
"@drop-oss/droplet-win32-arm64-msvc" "0.7.2"
"@drop-oss/droplet-win32-x64-msvc" "0.7.2"
"@drop-oss/droplet-darwin-arm64" "1.3.1"
"@drop-oss/droplet-darwin-universal" "1.3.1"
"@drop-oss/droplet-darwin-x64" "1.3.1"
"@drop-oss/droplet-linux-arm64-gnu" "1.3.1"
"@drop-oss/droplet-linux-arm64-musl" "1.3.1"
"@drop-oss/droplet-linux-riscv64-gnu" "1.3.1"
"@drop-oss/droplet-linux-x64-gnu" "1.3.1"
"@drop-oss/droplet-linux-x64-musl" "1.3.1"
"@drop-oss/droplet-win32-arm64-msvc" "1.3.1"
"@drop-oss/droplet-win32-x64-msvc" "1.3.1"
"@emnapi/core@^1.4.0":
version "1.4.0"