Ryan/drop
1
0
Fork 0
mirror of https://github.com/Drop-OSS/drop.git synced 2025-11-13 16:22:39 +10:00
Code Issues Packages Projects Releases Wiki Activity

Migrate game metadata import to task system #90 (#103)

Browse Source 
* feat: move game import to new task system

* fix: sizing issue with new task UI

* fix: lint

* feat: add pcgamingwiki task
This commit is contained in:
DecDuck
2025-06-08 11:37:24 +10:00
committed by GitHub
parent 9f8890020f
commit de438b93d5
9 changed files with 307 additions and 187 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pages/admin/library/import.vue
View File

@ -334,7 +334,7 @@ async function importGame(useMetadata: boolean) {
: undefined;
const option = games.unimportedGames[currentlySelectedGame.value];
const game = await $dropFetch("/api/v1/admin/import/game", {
const { taskId } = await $dropFetch("/api/v1/admin/import/game", {
method: "POST",
body: {
path: option.game,
@ -343,7 +343,7 @@ async function importGame(useMetadata: boolean) {
},
});
router.push(`/admin/library/${game.id}`);
router.push(`/admin/task/${taskId}`);
}
function importGame_wrapper(metadata = true) {
importLoading.value = true;

6
pages/admin/task/[id]/index.vue
View File

@ -57,11 +57,7 @@
<pre v-for="(line, idx) in task.log" :key="idx">{{ line }}</pre>
</div>
</div>
<div
v-else
role="status"
class="w-full h-screen flex items-center justify-center"
>
<div v-else role="status" class="w-full flex items-center justify-center">
<svg
aria-hidden="true"
class="size-8 text-transparent animate-spin fill-white"

17
server/api/v1/admin/import/game/index.post.ts
View File

@ -37,10 +37,17 @@ export default defineEventHandler<{ body: typeof ImportGameBody.infer }>(
statusMessage: "Invalid library or game.",
});
if (!metadata) {
return await metadataHandler.createGameWithoutMetadata(library, path);
} else {
return await metadataHandler.createGame(metadata, library, path);
}
const taskId = metadata
? await metadataHandler.createGame(metadata, library, path)
: await metadataHandler.createGameWithoutMetadata(library, path);
if (!taskId)
throw createError({
statusCode: 400,
statusMessage:
"Duplicate metadata import. Please chose a different game or metadata provider.",
});
return { taskId };
},
);

30
server/internal/metadata/giantbomb.ts
View File

@ -12,6 +12,7 @@ import type {
import axios, { type AxiosRequestConfig } from "axios";
import TurndownService from "turndown";
import { DateTime } from "luxon";
import type { TaskRunContext } from "../tasks";
interface GiantBombResponseType<T> {
error: "OK" | string;
@ -164,12 +165,12 @@ export class GiantBombProvider implements MetadataProvider {
return mapped;
}
async fetchGame({
id,
publisher,
developer,
createObject,
}: _FetchGameMetadataParams): Promise<GameMetadata> {
async fetchGame(
{ id, publisher, developer, createObject }: _FetchGameMetadataParams,
context?: TaskRunContext,
): Promise<GameMetadata> {
context?.log("Using GiantBomb provider");
const result = await this.request<GameResult>("game", id, {});
const gameData = result.data.results;
@ -180,21 +181,29 @@ export class GiantBombProvider implements MetadataProvider {
const publishers: Company[] = [];
if (gameData.publishers) {
for (const pub of gameData.publishers) {
context?.log(`Importing publisher "${pub.name}"`);
const res = await publisher(pub.name);
if (res === undefined) continue;
publishers.push(res);
}
}
context?.progress(35);
const developers: Company[] = [];
if (gameData.developers) {
for (const dev of gameData.developers) {
context?.log(`Importing developer "${dev.name}"`);
const res = await developer(dev.name);
if (res === undefined) continue;
developers.push(res);
}
}
context?.progress(70);
const icon = createObject(gameData.image.icon_url);
const banner = createObject(gameData.image.screen_large_url);
@ -202,6 +211,8 @@ export class GiantBombProvider implements MetadataProvider {
const images = [banner, ...imageURLs.map(createObject)];
context?.log(`Found all images. Total of ${images.length + 1}.`);
const releaseDate = gameData.original_release_date
? DateTime.fromISO(gameData.original_release_date).toJSDate()
: DateTime.fromISO(
@ -210,8 +221,11 @@ export class GiantBombProvider implements MetadataProvider {
}-${gameData.expected_release_day ?? 1}`,
).toJSDate();
context?.progress(85);
const reviews: GameMetadataRating[] = [];
if (gameData.reviews) {
context?.log("Found reviews, importing...");
for (const { api_detail_url } of gameData.reviews) {
const reviewId = api_detail_url.split("/").at(-2);
if (!reviewId) continue;
@ -225,6 +239,7 @@ export class GiantBombProvider implements MetadataProvider {
});
}
}
const metadata: GameMetadata = {
id: gameData.guid,
name: gameData.name,
@ -245,6 +260,9 @@ export class GiantBombProvider implements MetadataProvider {
images,
};
context?.log("GiantBomb provider finished.");
context?.progress(100);
return metadata;
}
async fetchCompany({

93
server/internal/metadata/igdb.ts
View File

@ -13,6 +13,7 @@ import type { AxiosRequestConfig } from "axios";
import axios from "axios";
import { DateTime } from "luxon";
import * as jdenticon from "jdenticon";
import type { TaskRunContext } from "../tasks";
type IGDBID = number;
@ -345,46 +346,50 @@ export class IGDBProvider implements MetadataProvider {
return results;
}
async fetchGame({
id,
publisher,
developer,
createObject,
}: _FetchGameMetadataParams): Promise<GameMetadata> {
async fetchGame(
{ id, publisher, developer, createObject }: _FetchGameMetadataParams,
context?: TaskRunContext,
): Promise<GameMetadata> {
const body = `where id = ${id}; fields *;`;
const response = await this.request<IGDBGameFull>("games", body);
const currentGame = (await this.request<IGDBGameFull>("games", body)).at(0);
if (!currentGame) throw new Error("No game found on IGDB with that id");
for (let i = 0; i < response.length; i++) {
const currentGame = response[i];
if (!currentGame) continue;
context?.log("Using IDGB provider.");
let iconRaw;
const cover = currentGame.cover;
if (cover !== undefined) {
context?.log("Found cover URL, using...");
iconRaw = await this.getCoverURL(cover);
} else {
context?.log("Missing cover URL, using fallback...");
iconRaw = jdenticon.toPng(id, 512);
}
const icon = createObject(iconRaw);
let banner = "";
let banner;
const images = [icon];
for (const art of currentGame.artworks ?? []) {
// if banner not set
if (banner.length <= 0) {
banner = createObject(await this.getArtworkURL(art));
images.push(banner);
} else {
images.push(createObject(await this.getArtworkURL(art)));
const objectId = createObject(await this.getArtworkURL(art));
if (!banner) {
banner = objectId;
}
images.push(objectId);
}
if (!banner) {
banner = createObject(jdenticon.toPng(id, 512));
}
context?.progress(20);
const publishers: Company[] = [];
const developers: Company[] = [];
for (const involvedCompany of currentGame.involved_companies ?? []) {
// get details about the involved company
const involved_company_response =
await this.request<IGDBInvolvedCompany>(
const involved_company_response = await this.request<IGDBInvolvedCompany>(
"involved_companies",
`where id = ${involvedCompany}; fields *;`,
);
@ -395,6 +400,10 @@ export class IGDBProvider implements MetadataProvider {
>("companies", `where id = ${foundInvolved.company}; fields name;`);
for (const company of findCompanyResponse) {
context?.log(
`Found involved company "${company.name}" as: ${foundInvolved.developer ? "developer, " : ""}${foundInvolved.publisher ? "publisher" : ""}`,
);
// if company was a dev or publisher
// CANNOT use else since a company can be both
if (foundInvolved.developer) {
@ -402,6 +411,7 @@ export class IGDBProvider implements MetadataProvider {
if (res === undefined) continue;
developers.push(res);
}
if (foundInvolved.publisher) {
const res = await publisher(company.name);
if (res === undefined) continue;
@ -411,41 +421,50 @@ export class IGDBProvider implements MetadataProvider {
}
}
const firstReleaseDate = currentGame.first_release_date;
context?.progress(80);
return {
id: "" + response[i].id,
name: response[i].name,
shortDescription: this.trimMessage(currentGame.summary, 280),
description: currentGame.summary,
released:
const firstReleaseDate = currentGame.first_release_date;
const released =
firstReleaseDate === undefined
? new Date()
: DateTime.fromSeconds(firstReleaseDate).toJSDate(),
: DateTime.fromSeconds(firstReleaseDate).toJSDate();
reviews: [
{
metadataId: "" + currentGame.id,
const review = {
metadataId: currentGame.id.toString(),
metadataSource: MetadataSource.IGDB,
mReviewCount: currentGame.total_rating_count ?? 0,
mReviewRating: (currentGame.total_rating ?? 0) / 100,
mReviewHref: currentGame.url,
},
],
};
publishers: [],
developers: [],
const tags = await this.getGenres(currentGame.genres);
tags: await this.getGenres(currentGame.genres),
const deck = this.trimMessage(currentGame.summary, 280);
const metadata = {
id: currentGame.id.toString(),
name: currentGame.name,
shortDescription: deck,
description: currentGame.summary,
released,
reviews: [review],
publishers,
developers,
tags,
icon,
bannerId: banner,
coverId: icon,
images,
};
}
throw new Error("No game found on igdb with that id");
context?.log("IGDB provider finished.");
context?.progress(100);
return metadata;
}
async fetchCompany({
query,

95
server/internal/metadata/index.ts
View File

@ -1,4 +1,4 @@
import { MetadataSource, type GameRating } from "~/prisma/client";
import { type Prisma, MetadataSource } from "~/prisma/client";
import prisma from "../db/database";
import type {
_FetchGameMetadataParams,
@ -12,6 +12,10 @@ import type {
import { ObjectTransactionalHandler } from "../objects/transactional";
import { PriorityListIndexed } from "../utils/prioritylist";
import { systemConfig } from "../config/sys-conf";
import type { TaskRunContext } from "../tasks";
import taskHandler, { wrapTaskContext } from "../tasks";
import { randomUUID } from "crypto";
import { fuzzy } from "fast-fuzzy";
export class MissingMetadataProviderConfig extends Error {
private providerName: string;
@ -34,9 +38,13 @@ export abstract class MetadataProvider {
abstract source(): MetadataSource;
abstract search(query: string): Promise<GameMetadataSearchResult[]>;
abstract fetchGame(params: _FetchGameMetadataParams): Promise<GameMetadata>;
abstract fetchGame(
params: _FetchGameMetadataParams,
taskRunContext?: TaskRunContext,
): Promise<GameMetadata>;
abstract fetchCompany(
params: _FetchCompanyMetadataParams,
taskRunContext?: TaskRunContext,
): Promise<CompanyMetadata | undefined>;
}
@ -92,7 +100,12 @@ export class MetadataHandler {
const successfulResults = results
.filter((result) => result.status === "fulfilled")
.map((result) => result.value)
.flat();
.flat()
.map((result) => {
const match = fuzzy(query, result.name);
return { ...result, fuzzy: match };
})
.sort((a, b) => b.fuzzy - a.fuzzy);
return successfulResults;
}
@ -110,14 +123,7 @@ export class MetadataHandler {
}
private parseTags(tags: string[]) {
const results: {
where: {
name: string;
};
create: {
name: string;
};
}[] = [];
const results: Array<Prisma.TagCreateOrConnectWithoutGamesInput> = [];
tags.forEach((t) =>
results.push({
@ -134,15 +140,7 @@ export class MetadataHandler {
}
private parseRatings(ratings: GameMetadataRating[]) {
const results: {
where: {
metadataKey: {
metadataId: string;
metadataSource: MetadataSource;
};
};
create: Omit<GameRating, "gameId" | "created" | "id">;
}[] = [];
const results: Array<Prisma.GameRatingCreateOrConnectWithoutGameInput> = [];
ratings.forEach((r) => {
results.push({
@ -178,30 +176,59 @@ export class MetadataHandler {
},
},
});
if (existing) return existing;
if (existing) return undefined;
const [createObject, pullObjects, dumpObjects] = this.objectHandler.new(
const gameId = randomUUID();
const taskId = `import:${gameId}`;
await taskHandler.create({
name: `Import game "${result.name}" (${libraryPath})`,
id: taskId,
taskGroup: "import:game",
acls: ["system:import:game:read"],
async run(context) {
const { progress, log } = context;
progress(0);
const [createObject, pullObjects, dumpObjects] =
metadataHandler.objectHandler.new(
{},
["internal:read"],
wrapTaskContext(context, {
min: 63,
max: 100,
prefix: "[object import] ",
}),
);
let metadata: GameMetadata | undefined = undefined;
try {
metadata = await provider.fetchGame({
metadata = await provider.fetchGame(
{
id: result.id,
name: result.name,
// wrap in anonymous functions to keep references to this
publisher: (name: string) => this.fetchCompany(name),
developer: (name: string) => this.fetchCompany(name),
publisher: (name: string) => metadataHandler.fetchCompany(name),
developer: (name: string) => metadataHandler.fetchCompany(name),
createObject,
});
},
wrapTaskContext(context, {
min: 0,
max: 60,
prefix: "[metadata import] ",
}),
);
} catch (e) {
dumpObjects();
throw e;
}
const game = await prisma.game.create({
context?.progress(60);
await prisma.game.create({
data: {
id: gameId,
metadataSource: provider.source(),
metadataId: metadata.id,
@ -223,10 +250,10 @@ export class MetadataHandler {
},
ratings: {
connectOrCreate: this.parseRatings(metadata.reviews),
connectOrCreate: metadataHandler.parseRatings(metadata.reviews),
},
tags: {
connectOrCreate: this.parseTags(metadata.tags),
connectOrCreate: metadataHandler.parseTags(metadata.tags),
},
libraryId,
@ -234,9 +261,17 @@ export class MetadataHandler {
},
});
progress(63);
log(`Successfully fetched all metadata.`);
log(`Importing objects...`);
await pullObjects();
return game;
log(`Finished game import.`);
},
});
return taskId;
}
// Careful with this function, it has no typechecking

42
server/internal/metadata/pcgamingwiki.ts
View File

@ -15,6 +15,7 @@ import * as jdenticon from "jdenticon";
import { DateTime } from "luxon";
import * as cheerio from "cheerio";
import { type } from "arktype";
import type { TaskRunContext } from "../tasks";
interface PCGamingWikiParseRawPage {
parse: {
@ -369,13 +370,13 @@ export class PCGamingWikiProvider implements MetadataProvider {
return results;
}
async fetchGame({
id,
name,
publisher,
developer,
createObject,
}: _FetchGameMetadataParams): Promise<GameMetadata> {
async fetchGame(
{ id, name, publisher, developer, createObject }: _FetchGameMetadataParams,
context?: TaskRunContext,
): Promise<GameMetadata> {
context?.log("Using PCGamingWiki provider");
context?.progress(0);
const searchParams = new URLSearchParams({
action: "cargoquery",
tables: "Infobox_game",
@ -396,37 +397,49 @@ export class PCGamingWikiProvider implements MetadataProvider {
const publishers: Company[] = [];
if (game.Publishers !== null) {
context?.log("Found publishers, importing...");
const pubListClean = this.parseWikiStringArray(game.Publishers);
for (const pub of pubListClean) {
context?.log(`Importing "${pub}"...`);
const res = await publisher(pub);
if (res === undefined) continue;
publishers.push(res);
}
}
context?.progress(40);
const developers: Company[] = [];
if (game.Developers !== null) {
context?.log("Found developers, importing...");
const devListClean = this.parseWikiStringArray(game.Developers);
for (const dev of devListClean) {
context?.log(`Importing "${dev}"...`);
const res = await developer(dev);
if (res === undefined) continue;
developers.push(res);
}
}
const icon =
context?.progress(80);
const icon = createObject(
game["Cover URL"] !== null
? createObject(game["Cover URL"])
: createObject(jdenticon.toPng(name, 512));
? game["Cover URL"]
: jdenticon.toPng(name, 512),
);
const released = game.Released
? DateTime.fromISO(game.Released.split(";")[0]).toJSDate()
: new Date();
const metadata: GameMetadata = {
id: game.PageID,
name: game.PageName,
shortDescription: pageContent.shortIntro,
description: pageContent.introduction,
released: game.Released
? DateTime.fromISO(game.Released.split(";")[0]).toJSDate()
: new Date(),
released,
tags: this.compileTags(game),
@ -440,6 +453,9 @@ export class PCGamingWikiProvider implements MetadataProvider {
images: [icon],
};
context?.log("PCGamingWiki provider finished.");
context?.progress(100);
return metadata;
}

13
server/internal/objects/transactional.ts
View File

@ -5,6 +5,7 @@ This is used as a utility in metadata handling, so we only fetch the objects if
import type { Readable } from "stream";
import { randomUUID } from "node:crypto";
import objectHandler from ".";
import type { TaskRunContext } from "../tasks";
export type TransactionDataType = string | Readable | Buffer;
type TransactionTable = Map<string, TransactionDataType>; // ID to data
@ -20,6 +21,7 @@ export class ObjectTransactionalHandler {
new(
metadata: { [key: string]: string },
permissions: Array<string>,
context?: TaskRunContext,
): [Register, Pull, Dump] {
const transactionId = randomUUID();
@ -35,7 +37,16 @@ export class ObjectTransactionalHandler {
const pull = async () => {
const transaction = this.record.get(transactionId);
if (!transaction) return;
let progress = 0;
const increment = (1 / transaction.size) * 100;
for (const [id, data] of transaction) {
if (typeof data === "string") {
context?.log(`Importing object from "${data}"`);
} else {
context?.log(`Importing raw object...`);
}
await objectHandler.createFromSource(
id,
() => {
@ -47,6 +58,8 @@ export class ObjectTransactionalHandler {
metadata,
permissions,
);
progress += increment;
context?.progress(progress);
}
};

16
server/internal/tasks/index.ts
View File

@ -332,6 +332,22 @@ export type TaskRunContext = {
log: (message: string) => void;
};
export function wrapTaskContext(
context: TaskRunContext,
options: { min: number; max: number; prefix: string },
): TaskRunContext {
return {
progress(progress) {
const scalar = 100 / (options.max - options.min);
const adjustedProgress = progress * scalar + options.min;
return context.progress(adjustedProgress);
},
log(message) {
return context.log(options.prefix + message);
},
};
}
export interface Task {
id: string;
taskGroup: TaskGroup;