Migrate game metadata import to task system #90 (#103)

* feat: move game import to new task system

* fix: sizing issue with new task UI

* fix: lint

* feat: add pcgamingwiki task

pages/admin/library/import.vue

@@ -334,7 +334,7 @@ async function importGame(useMetadata: boolean) {
       : undefined;
   const option = games.unimportedGames[currentlySelectedGame.value];
 
-  const game = await $dropFetch("/api/v1/admin/import/game", {
+  const { taskId } = await $dropFetch("/api/v1/admin/import/game", {
     method: "POST",
     body: {
       path: option.game,
@@ -343,7 +343,7 @@ async function importGame(useMetadata: boolean) {
     },
   });
 
-  router.push(`/admin/library/${game.id}`);
+  router.push(`/admin/task/${taskId}`);
 }
 function importGame_wrapper(metadata = true) {
   importLoading.value = true;

pages/admin/task/[id]/index.vue

@@ -57,11 +57,7 @@
         <pre v-for="(line, idx) in task.log" :key="idx">{{ line }}</pre>
       </div>
     </div>
-    <div
+    <div v-else role="status" class="w-full flex items-center justify-center">
-      v-else
-      role="status"
-      class="w-full h-screen flex items-center justify-center"
-    >
       <svg
         aria-hidden="true"
         class="size-8 text-transparent animate-spin fill-white"

server/api/v1/admin/import/game/index.post.ts

@@ -37,10 +37,17 @@ export default defineEventHandler<{ body: typeof ImportGameBody.infer }>(
         statusMessage: "Invalid library or game.",
       });
 
-    if (!metadata) {
+    const taskId = metadata
-      return await metadataHandler.createGameWithoutMetadata(library, path);
+      ? await metadataHandler.createGame(metadata, library, path)
-    } else {
+      : await metadataHandler.createGameWithoutMetadata(library, path);
-      return await metadataHandler.createGame(metadata, library, path);
+
-    }
+    if (!taskId)
+      throw createError({
+        statusCode: 400,
+        statusMessage:
+          "Duplicate metadata import. Please chose a different game or metadata provider.",
+      });
+
+    return { taskId };
   },
 );

server/internal/metadata/giantbomb.ts

@@ -12,6 +12,7 @@ import type {
 import axios, { type AxiosRequestConfig } from "axios";
 import TurndownService from "turndown";
 import { DateTime } from "luxon";
+import type { TaskRunContext } from "../tasks";
 
 interface GiantBombResponseType<T> {
   error: "OK" | string;
@@ -164,12 +165,12 @@ export class GiantBombProvider implements MetadataProvider {
 
     return mapped;
   }
-  async fetchGame({
+  async fetchGame(
-    id,
+    { id, publisher, developer, createObject }: _FetchGameMetadataParams,
-    publisher,
+    context?: TaskRunContext,
-    developer,
+  ): Promise<GameMetadata> {
-    createObject,
+    context?.log("Using GiantBomb provider");
-  }: _FetchGameMetadataParams): Promise<GameMetadata> {
+
     const result = await this.request<GameResult>("game", id, {});
     const gameData = result.data.results;
 
@@ -180,21 +181,29 @@ export class GiantBombProvider implements MetadataProvider {
     const publishers: Company[] = [];
     if (gameData.publishers) {
       for (const pub of gameData.publishers) {
+        context?.log(`Importing publisher "${pub.name}"`);
+
         const res = await publisher(pub.name);
         if (res === undefined) continue;
         publishers.push(res);
       }
     }
 
+    context?.progress(35);
+
     const developers: Company[] = [];
     if (gameData.developers) {
       for (const dev of gameData.developers) {
+        context?.log(`Importing developer "${dev.name}"`);
+
         const res = await developer(dev.name);
         if (res === undefined) continue;
         developers.push(res);
       }
     }
 
+    context?.progress(70);
+
     const icon = createObject(gameData.image.icon_url);
     const banner = createObject(gameData.image.screen_large_url);
 
@@ -202,6 +211,8 @@ export class GiantBombProvider implements MetadataProvider {
 
     const images = [banner, ...imageURLs.map(createObject)];
 
+    context?.log(`Found all images. Total of ${images.length + 1}.`);
+
     const releaseDate = gameData.original_release_date
       ? DateTime.fromISO(gameData.original_release_date).toJSDate()
       : DateTime.fromISO(
@@ -210,8 +221,11 @@ export class GiantBombProvider implements MetadataProvider {
           }-${gameData.expected_release_day ?? 1}`,
         ).toJSDate();
 
+    context?.progress(85);
+
     const reviews: GameMetadataRating[] = [];
     if (gameData.reviews) {
+      context?.log("Found reviews, importing...");
       for (const { api_detail_url } of gameData.reviews) {
         const reviewId = api_detail_url.split("/").at(-2);
         if (!reviewId) continue;
@@ -225,6 +239,7 @@ export class GiantBombProvider implements MetadataProvider {
         });
       }
     }
+
     const metadata: GameMetadata = {
       id: gameData.guid,
       name: gameData.name,
@@ -245,6 +260,9 @@ export class GiantBombProvider implements MetadataProvider {
       images,
     };
 
+    context?.log("GiantBomb provider finished.");
+    context?.progress(100);
+
     return metadata;
   }
   async fetchCompany({

server/internal/metadata/igdb.ts

@@ -13,6 +13,7 @@ import type { AxiosRequestConfig } from "axios";
 import axios from "axios";
 import { DateTime } from "luxon";
 import * as jdenticon from "jdenticon";
+import type { TaskRunContext } from "../tasks";
 
 type IGDBID = number;
 
@@ -345,46 +346,50 @@ export class IGDBProvider implements MetadataProvider {
 
     return results;
   }
-  async fetchGame({
+  async fetchGame(
-    id,
+    { id, publisher, developer, createObject }: _FetchGameMetadataParams,
-    publisher,
+    context?: TaskRunContext,
-    developer,
+  ): Promise<GameMetadata> {
-    createObject,
-  }: _FetchGameMetadataParams): Promise<GameMetadata> {
     const body = `where id = ${id}; fields *;`;
-    const response = await this.request<IGDBGameFull>("games", body);
+    const currentGame = (await this.request<IGDBGameFull>("games", body)).at(0);
+    if (!currentGame) throw new Error("No game found on IGDB with that id");
 
-    for (let i = 0; i < response.length; i++) {
+    context?.log("Using IDGB provider.");
-      const currentGame = response[i];
-      if (!currentGame) continue;
 
     let iconRaw;
     const cover = currentGame.cover;
+
     if (cover !== undefined) {
+      context?.log("Found cover URL, using...");
       iconRaw = await this.getCoverURL(cover);
     } else {
+      context?.log("Missing cover URL, using fallback...");
       iconRaw = jdenticon.toPng(id, 512);
     }
+
     const icon = createObject(iconRaw);
-      let banner = "";
+    let banner;
 
     const images = [icon];
     for (const art of currentGame.artworks ?? []) {
-        // if banner not set
+      const objectId = createObject(await this.getArtworkURL(art));
-        if (banner.length <= 0) {
+      if (!banner) {
-          banner = createObject(await this.getArtworkURL(art));
+        banner = objectId;
-          images.push(banner);
-        } else {
-          images.push(createObject(await this.getArtworkURL(art)));
       }
+      images.push(objectId);
     }
 
+    if (!banner) {
+      banner = createObject(jdenticon.toPng(id, 512));
+    }
+
+    context?.progress(20);
+
     const publishers: Company[] = [];
     const developers: Company[] = [];
     for (const involvedCompany of currentGame.involved_companies ?? []) {
       // get details about the involved company
-        const involved_company_response =
+      const involved_company_response = await this.request<IGDBInvolvedCompany>(
-          await this.request<IGDBInvolvedCompany>(
         "involved_companies",
         `where id = ${involvedCompany}; fields *;`,
       );
@@ -395,6 +400,10 @@ export class IGDBProvider implements MetadataProvider {
         >("companies", `where id = ${foundInvolved.company}; fields name;`);
 
         for (const company of findCompanyResponse) {
+          context?.log(
+            `Found involved company "${company.name}" as: ${foundInvolved.developer ? "developer, " : ""}${foundInvolved.publisher ? "publisher" : ""}`,
+          );
+
           // if company was a dev or publisher
           // CANNOT use else since a company can be both
           if (foundInvolved.developer) {
@@ -402,6 +411,7 @@ export class IGDBProvider implements MetadataProvider {
             if (res === undefined) continue;
             developers.push(res);
           }
+
           if (foundInvolved.publisher) {
             const res = await publisher(company.name);
             if (res === undefined) continue;
@@ -411,41 +421,50 @@ export class IGDBProvider implements MetadataProvider {
       }
     }
 
-      const firstReleaseDate = currentGame.first_release_date;
+    context?.progress(80);
 
-      return {
+    const firstReleaseDate = currentGame.first_release_date;
-        id: "" + response[i].id,
+    const released =
-        name: response[i].name,
-        shortDescription: this.trimMessage(currentGame.summary, 280),
-        description: currentGame.summary,
-        released:
       firstReleaseDate === undefined
         ? new Date()
-            : DateTime.fromSeconds(firstReleaseDate).toJSDate(),
+        : DateTime.fromSeconds(firstReleaseDate).toJSDate();
 
-        reviews: [
+    const review = {
-          {
+      metadataId: currentGame.id.toString(),
-            metadataId: "" + currentGame.id,
       metadataSource: MetadataSource.IGDB,
       mReviewCount: currentGame.total_rating_count ?? 0,
       mReviewRating: (currentGame.total_rating ?? 0) / 100,
       mReviewHref: currentGame.url,
-          },
+    };
-        ],
 
-        publishers: [],
+    const tags = await this.getGenres(currentGame.genres);
-        developers: [],
 
-        tags: await this.getGenres(currentGame.genres),
+    const deck = this.trimMessage(currentGame.summary, 280);
+
+    const metadata = {
+      id: currentGame.id.toString(),
+      name: currentGame.name,
+      shortDescription: deck,
+      description: currentGame.summary,
+      released,
+
+      reviews: [review],
+
+      publishers,
+      developers,
+
+      tags,
 
       icon,
       bannerId: banner,
       coverId: icon,
       images,
     };
-    }
 
-    throw new Error("No game found on igdb with that id");
+    context?.log("IGDB provider finished.");
+    context?.progress(100);
+
+    return metadata;
   }
   async fetchCompany({
     query,

server/internal/metadata/index.ts

@@ -1,4 +1,4 @@
-import { MetadataSource, type GameRating } from "~/prisma/client";
+import { type Prisma, MetadataSource } from "~/prisma/client";
 import prisma from "../db/database";
 import type {
   _FetchGameMetadataParams,
@@ -12,6 +12,10 @@ import type {
 import { ObjectTransactionalHandler } from "../objects/transactional";
 import { PriorityListIndexed } from "../utils/prioritylist";
 import { systemConfig } from "../config/sys-conf";
+import type { TaskRunContext } from "../tasks";
+import taskHandler, { wrapTaskContext } from "../tasks";
+import { randomUUID } from "crypto";
+import { fuzzy } from "fast-fuzzy";
 
 export class MissingMetadataProviderConfig extends Error {
   private providerName: string;
@@ -34,9 +38,13 @@ export abstract class MetadataProvider {
   abstract source(): MetadataSource;
 
   abstract search(query: string): Promise<GameMetadataSearchResult[]>;
-  abstract fetchGame(params: _FetchGameMetadataParams): Promise<GameMetadata>;
+  abstract fetchGame(
+    params: _FetchGameMetadataParams,
+    taskRunContext?: TaskRunContext,
+  ): Promise<GameMetadata>;
   abstract fetchCompany(
     params: _FetchCompanyMetadataParams,
+    taskRunContext?: TaskRunContext,
   ): Promise<CompanyMetadata | undefined>;
 }
 
@@ -92,7 +100,12 @@ export class MetadataHandler {
     const successfulResults = results
       .filter((result) => result.status === "fulfilled")
       .map((result) => result.value)
-      .flat();
+      .flat()
+      .map((result) => {
+        const match = fuzzy(query, result.name);
+        return { ...result, fuzzy: match };
+      })
+      .sort((a, b) => b.fuzzy - a.fuzzy);
 
     return successfulResults;
   }
@@ -110,14 +123,7 @@ export class MetadataHandler {
   }
 
   private parseTags(tags: string[]) {
-    const results: {
+    const results: Array<Prisma.TagCreateOrConnectWithoutGamesInput> = [];
-      where: {
-        name: string;
-      };
-      create: {
-        name: string;
-      };
-    }[] = [];
 
     tags.forEach((t) =>
       results.push({
@@ -134,15 +140,7 @@ export class MetadataHandler {
   }
 
   private parseRatings(ratings: GameMetadataRating[]) {
-    const results: {
+    const results: Array<Prisma.GameRatingCreateOrConnectWithoutGameInput> = [];
-      where: {
-        metadataKey: {
-          metadataId: string;
-          metadataSource: MetadataSource;
-        };
-      };
-      create: Omit<GameRating, "gameId" | "created" | "id">;
-    }[] = [];
 
     ratings.forEach((r) => {
       results.push({
@@ -178,30 +176,59 @@ export class MetadataHandler {
         },
       },
     });
-    if (existing) return existing;
+    if (existing) return undefined;
 
-    const [createObject, pullObjects, dumpObjects] = this.objectHandler.new(
+    const gameId = randomUUID();
+
+    const taskId = `import:${gameId}`;
+    await taskHandler.create({
+      name: `Import game "${result.name}" (${libraryPath})`,
+      id: taskId,
+      taskGroup: "import:game",
+      acls: ["system:import:game:read"],
+      async run(context) {
+        const { progress, log } = context;
+
+        progress(0);
+
+        const [createObject, pullObjects, dumpObjects] =
+          metadataHandler.objectHandler.new(
             {},
             ["internal:read"],
+            wrapTaskContext(context, {
+              min: 63,
+              max: 100,
+              prefix: "[object import] ",
+            }),
           );
 
         let metadata: GameMetadata | undefined = undefined;
         try {
-      metadata = await provider.fetchGame({
+          metadata = await provider.fetchGame(
+            {
               id: result.id,
               name: result.name,
               // wrap in anonymous functions to keep references to this
-        publisher: (name: string) => this.fetchCompany(name),
+              publisher: (name: string) => metadataHandler.fetchCompany(name),
-        developer: (name: string) => this.fetchCompany(name),
+              developer: (name: string) => metadataHandler.fetchCompany(name),
               createObject,
-      });
+            },
+            wrapTaskContext(context, {
+              min: 0,
+              max: 60,
+              prefix: "[metadata import] ",
+            }),
+          );
         } catch (e) {
           dumpObjects();
           throw e;
         }
 
-    const game = await prisma.game.create({
+        context?.progress(60);
+
+        await prisma.game.create({
           data: {
+            id: gameId,
             metadataSource: provider.source(),
             metadataId: metadata.id,
 
@@ -223,10 +250,10 @@ export class MetadataHandler {
             },
 
             ratings: {
-          connectOrCreate: this.parseRatings(metadata.reviews),
+              connectOrCreate: metadataHandler.parseRatings(metadata.reviews),
             },
             tags: {
-          connectOrCreate: this.parseTags(metadata.tags),
+              connectOrCreate: metadataHandler.parseTags(metadata.tags),
             },
 
             libraryId,
@@ -234,9 +261,17 @@ export class MetadataHandler {
           },
         });
 
+        progress(63);
+        log(`Successfully fetched all metadata.`);
+        log(`Importing objects...`);
+
         await pullObjects();
 
-    return game;
+        log(`Finished game import.`);
+      },
+    });
+
+    return taskId;
   }
 
   // Careful with this function, it has no typechecking

server/internal/metadata/pcgamingwiki.ts

@@ -15,6 +15,7 @@ import * as jdenticon from "jdenticon";
 import { DateTime } from "luxon";
 import * as cheerio from "cheerio";
 import { type } from "arktype";
+import type { TaskRunContext } from "../tasks";
 
 interface PCGamingWikiParseRawPage {
   parse: {
@@ -369,13 +370,13 @@ export class PCGamingWikiProvider implements MetadataProvider {
     return results;
   }
 
-  async fetchGame({
+  async fetchGame(
-    id,
+    { id, name, publisher, developer, createObject }: _FetchGameMetadataParams,
-    name,
+    context?: TaskRunContext,
-    publisher,
+  ): Promise<GameMetadata> {
-    developer,
+    context?.log("Using PCGamingWiki provider");
-    createObject,
+    context?.progress(0);
-  }: _FetchGameMetadataParams): Promise<GameMetadata> {
+
     const searchParams = new URLSearchParams({
       action: "cargoquery",
       tables: "Infobox_game",
@@ -396,37 +397,49 @@ export class PCGamingWikiProvider implements MetadataProvider {
 
     const publishers: Company[] = [];
     if (game.Publishers !== null) {
+      context?.log("Found publishers, importing...");
       const pubListClean = this.parseWikiStringArray(game.Publishers);
       for (const pub of pubListClean) {
+        context?.log(`Importing "${pub}"...`);
+
         const res = await publisher(pub);
         if (res === undefined) continue;
         publishers.push(res);
       }
     }
 
+    context?.progress(40);
+
     const developers: Company[] = [];
     if (game.Developers !== null) {
+      context?.log("Found developers, importing...");
       const devListClean = this.parseWikiStringArray(game.Developers);
       for (const dev of devListClean) {
+        context?.log(`Importing "${dev}"...`);
         const res = await developer(dev);
         if (res === undefined) continue;
         developers.push(res);
       }
     }
 
-    const icon =
+    context?.progress(80);
+
+    const icon = createObject(
       game["Cover URL"] !== null
-        ? createObject(game["Cover URL"])
+        ? game["Cover URL"]
-        : createObject(jdenticon.toPng(name, 512));
+        : jdenticon.toPng(name, 512),
+    );
+
+    const released = game.Released
+      ? DateTime.fromISO(game.Released.split(";")[0]).toJSDate()
+      : new Date();
 
     const metadata: GameMetadata = {
       id: game.PageID,
       name: game.PageName,
       shortDescription: pageContent.shortIntro,
       description: pageContent.introduction,
-      released: game.Released
+      released,
-        ? DateTime.fromISO(game.Released.split(";")[0]).toJSDate()
-        : new Date(),
 
       tags: this.compileTags(game),
 
@@ -440,6 +453,9 @@ export class PCGamingWikiProvider implements MetadataProvider {
       images: [icon],
     };
 
+    context?.log("PCGamingWiki provider finished.");
+    context?.progress(100);
+
     return metadata;
   }
 

server/internal/objects/transactional.ts

@@ -5,6 +5,7 @@ This is used as a utility in metadata handling, so we only fetch the objects if
 import type { Readable } from "stream";
 import { randomUUID } from "node:crypto";
 import objectHandler from ".";
+import type { TaskRunContext } from "../tasks";
 
 export type TransactionDataType = string | Readable | Buffer;
 type TransactionTable = Map<string, TransactionDataType>; // ID to data
@@ -20,6 +21,7 @@ export class ObjectTransactionalHandler {
   new(
     metadata: { [key: string]: string },
     permissions: Array<string>,
+    context?: TaskRunContext,
   ): [Register, Pull, Dump] {
     const transactionId = randomUUID();
 
@@ -35,7 +37,16 @@ export class ObjectTransactionalHandler {
     const pull = async () => {
       const transaction = this.record.get(transactionId);
       if (!transaction) return;
+
+      let progress = 0;
+      const increment = (1 / transaction.size) * 100;
+
       for (const [id, data] of transaction) {
+        if (typeof data === "string") {
+          context?.log(`Importing object from "${data}"`);
+        } else {
+          context?.log(`Importing raw object...`);
+        }
         await objectHandler.createFromSource(
           id,
           () => {
@@ -47,6 +58,8 @@ export class ObjectTransactionalHandler {
           metadata,
           permissions,
         );
+        progress += increment;
+        context?.progress(progress);
       }
     };
 

server/internal/tasks/index.ts

@@ -332,6 +332,22 @@ export type TaskRunContext = {
   log: (message: string) => void;
 };
 
+export function wrapTaskContext(
+  context: TaskRunContext,
+  options: { min: number; max: number; prefix: string },
+): TaskRunContext {
+  return {
+    progress(progress) {
+      const scalar = 100 / (options.max - options.min);
+      const adjustedProgress = progress * scalar + options.min;
+      return context.progress(adjustedProgress);
+    },
+    log(message) {
+      return context.log(options.prefix + message);
+    },
+  };
+}
+
 export interface Task {
   id: string;
   taskGroup: TaskGroup;