Migrate game metadata import to task system #90 (#103)

* feat: move game import to new task system

* fix: sizing issue with new task UI

* fix: lint

* feat: add pcgamingwiki task

pages/admin/library/import.vue

@@ -334,7 +334,7 @@ async function importGame(useMetadata: boolean) {
       : undefined;
   const option = games.unimportedGames[currentlySelectedGame.value];
 
-  const game = await $dropFetch("/api/v1/admin/import/game", {
+  const { taskId } = await $dropFetch("/api/v1/admin/import/game", {
     method: "POST",
     body: {
       path: option.game,
@@ -343,7 +343,7 @@ async function importGame(useMetadata: boolean) {
     },
   });
 
-  router.push(`/admin/library/${game.id}`);
+  router.push(`/admin/task/${taskId}`);
 }
 function importGame_wrapper(metadata = true) {
   importLoading.value = true;

pages/admin/task/[id]/index.vue

@@ -57,11 +57,7 @@
         <pre v-for="(line, idx) in task.log" :key="idx">{{ line }}</pre>
       </div>
     </div>
-    <div
+    <div v-else role="status" class="w-full flex items-center justify-center">
-      v-else
-      role="status"
-      class="w-full h-screen flex items-center justify-center"
-    >
       <svg
         aria-hidden="true"
         class="size-8 text-transparent animate-spin fill-white"

server/api/v1/admin/import/game/index.post.ts

@@ -37,10 +37,17 @@ export default defineEventHandler<{ body: typeof ImportGameBody.infer }>(
         statusMessage: "Invalid library or game.",
       });
 
-    if (!metadata) {
+    const taskId = metadata
-      return await metadataHandler.createGameWithoutMetadata(library, path);
+      ? await metadataHandler.createGame(metadata, library, path)
-    } else {
+      : await metadataHandler.createGameWithoutMetadata(library, path);
-      return await metadataHandler.createGame(metadata, library, path);
+
-    }
+    if (!taskId)
+      throw createError({
+        statusCode: 400,
+        statusMessage:
+          "Duplicate metadata import. Please chose a different game or metadata provider.",
+      });
+
+    return { taskId };
   },
 );

server/internal/metadata/giantbomb.ts

@@ -12,6 +12,7 @@ import type {
 import axios, { type AxiosRequestConfig } from "axios";
 import TurndownService from "turndown";
 import { DateTime } from "luxon";
+import type { TaskRunContext } from "../tasks";
 
 interface GiantBombResponseType<T> {
   error: "OK" | string;
@@ -164,12 +165,12 @@ export class GiantBombProvider implements MetadataProvider {
 
     return mapped;
   }
-  async fetchGame({
+  async fetchGame(
-    id,
+    { id, publisher, developer, createObject }: _FetchGameMetadataParams,
-    publisher,
+    context?: TaskRunContext,
-    developer,
+  ): Promise<GameMetadata> {
-    createObject,
+    context?.log("Using GiantBomb provider");
-  }: _FetchGameMetadataParams): Promise<GameMetadata> {
+
     const result = await this.request<GameResult>("game", id, {});
     const gameData = result.data.results;
 
@@ -180,21 +181,29 @@ export class GiantBombProvider implements MetadataProvider {
     const publishers: Company[] = [];
     if (gameData.publishers) {
       for (const pub of gameData.publishers) {
+        context?.log(`Importing publisher "${pub.name}"`);
+
         const res = await publisher(pub.name);
         if (res === undefined) continue;
         publishers.push(res);
       }
     }
 
+    context?.progress(35);
+
     const developers: Company[] = [];
     if (gameData.developers) {
       for (const dev of gameData.developers) {
+        context?.log(`Importing developer "${dev.name}"`);
+
         const res = await developer(dev.name);
         if (res === undefined) continue;
         developers.push(res);
       }
     }
 
+    context?.progress(70);
+
     const icon = createObject(gameData.image.icon_url);
     const banner = createObject(gameData.image.screen_large_url);
 
@@ -202,6 +211,8 @@ export class GiantBombProvider implements MetadataProvider {
 
     const images = [banner, ...imageURLs.map(createObject)];
 
+    context?.log(`Found all images. Total of ${images.length + 1}.`);
+
     const releaseDate = gameData.original_release_date
       ? DateTime.fromISO(gameData.original_release_date).toJSDate()
       : DateTime.fromISO(
@@ -210,8 +221,11 @@ export class GiantBombProvider implements MetadataProvider {
           }-${gameData.expected_release_day ?? 1}`,
         ).toJSDate();
 
+    context?.progress(85);
+
     const reviews: GameMetadataRating[] = [];
     if (gameData.reviews) {
+      context?.log("Found reviews, importing...");
       for (const { api_detail_url } of gameData.reviews) {
         const reviewId = api_detail_url.split("/").at(-2);
         if (!reviewId) continue;
@@ -225,6 +239,7 @@ export class GiantBombProvider implements MetadataProvider {
         });
       }
     }
+
     const metadata: GameMetadata = {
       id: gameData.guid,
       name: gameData.name,
@@ -245,6 +260,9 @@ export class GiantBombProvider implements MetadataProvider {
       images,
     };
 
+    context?.log("GiantBomb provider finished.");
+    context?.progress(100);
+
     return metadata;
   }
   async fetchCompany({

server/internal/metadata/igdb.ts

@@ -13,6 +13,7 @@ import type { AxiosRequestConfig } from "axios";
 import axios from "axios";
 import { DateTime } from "luxon";
 import * as jdenticon from "jdenticon";
+import type { TaskRunContext } from "../tasks";
 
 type IGDBID = number;
 
@@ -345,107 +346,125 @@ export class IGDBProvider implements MetadataProvider {
 
     return results;
   }
-  async fetchGame({
+  async fetchGame(
-    id,
+    { id, publisher, developer, createObject }: _FetchGameMetadataParams,
-    publisher,
+    context?: TaskRunContext,
-    developer,
+  ): Promise<GameMetadata> {
-    createObject,
-  }: _FetchGameMetadataParams): Promise<GameMetadata> {
     const body = `where id = ${id}; fields *;`;
-    const response = await this.request<IGDBGameFull>("games", body);
+    const currentGame = (await this.request<IGDBGameFull>("games", body)).at(0);
+    if (!currentGame) throw new Error("No game found on IGDB with that id");
 
-    for (let i = 0; i < response.length; i++) {
+    context?.log("Using IDGB provider.");
-      const currentGame = response[i];
-      if (!currentGame) continue;
 
-      let iconRaw;
+    let iconRaw;
-      const cover = currentGame.cover;
+    const cover = currentGame.cover;
-      if (cover !== undefined) {
+
-        iconRaw = await this.getCoverURL(cover);
+    if (cover !== undefined) {
-      } else {
+      context?.log("Found cover URL, using...");
-        iconRaw = jdenticon.toPng(id, 512);
+      iconRaw = await this.getCoverURL(cover);
+    } else {
+      context?.log("Missing cover URL, using fallback...");
+      iconRaw = jdenticon.toPng(id, 512);
+    }
+
+    const icon = createObject(iconRaw);
+    let banner;
+
+    const images = [icon];
+    for (const art of currentGame.artworks ?? []) {
+      const objectId = createObject(await this.getArtworkURL(art));
+      if (!banner) {
+        banner = objectId;
       }
-      const icon = createObject(iconRaw);
+      images.push(objectId);
-      let banner = "";
+    }
 
-      const images = [icon];
+    if (!banner) {
-      for (const art of currentGame.artworks ?? []) {
+      banner = createObject(jdenticon.toPng(id, 512));
-        // if banner not set
+    }
-        if (banner.length <= 0) {
-          banner = createObject(await this.getArtworkURL(art));
-          images.push(banner);
-        } else {
-          images.push(createObject(await this.getArtworkURL(art)));
-        }
-      }
 
-      const publishers: Company[] = [];
+    context?.progress(20);
-      const developers: Company[] = [];
+
-      for (const involvedCompany of currentGame.involved_companies ?? []) {
+    const publishers: Company[] = [];
-        // get details about the involved company
+    const developers: Company[] = [];
-        const involved_company_response =
+    for (const involvedCompany of currentGame.involved_companies ?? []) {
-          await this.request<IGDBInvolvedCompany>(
+      // get details about the involved company
-            "involved_companies",
+      const involved_company_response = await this.request<IGDBInvolvedCompany>(
-            `where id = ${involvedCompany}; fields *;`,
+        "involved_companies",
+        `where id = ${involvedCompany}; fields *;`,
+      );
+      for (const foundInvolved of involved_company_response) {
+        // now we need to get the actual company so we can get the name
+        const findCompanyResponse = await this.request<
+          { name: string } & IGDBItem
+        >("companies", `where id = ${foundInvolved.company}; fields name;`);
+
+        for (const company of findCompanyResponse) {
+          context?.log(
+            `Found involved company "${company.name}" as: ${foundInvolved.developer ? "developer, " : ""}${foundInvolved.publisher ? "publisher" : ""}`,
           );
-        for (const foundInvolved of involved_company_response) {
-          // now we need to get the actual company so we can get the name
-          const findCompanyResponse = await this.request<
-            { name: string } & IGDBItem
-          >("companies", `where id = ${foundInvolved.company}; fields name;`);
 
-          for (const company of findCompanyResponse) {
+          // if company was a dev or publisher
-            // if company was a dev or publisher
+          // CANNOT use else since a company can be both
-            // CANNOT use else since a company can be both
+          if (foundInvolved.developer) {
-            if (foundInvolved.developer) {
+            const res = await developer(company.name);
-              const res = await developer(company.name);
+            if (res === undefined) continue;
-              if (res === undefined) continue;
+            developers.push(res);
-              developers.push(res);
+          }
-            }
+
-            if (foundInvolved.publisher) {
+          if (foundInvolved.publisher) {
-              const res = await publisher(company.name);
+            const res = await publisher(company.name);
-              if (res === undefined) continue;
+            if (res === undefined) continue;
-              publishers.push(res);
+            publishers.push(res);
-            }
           }
         }
       }
-
-      const firstReleaseDate = currentGame.first_release_date;
-
-      return {
-        id: "" + response[i].id,
-        name: response[i].name,
-        shortDescription: this.trimMessage(currentGame.summary, 280),
-        description: currentGame.summary,
-        released:
-          firstReleaseDate === undefined
-            ? new Date()
-            : DateTime.fromSeconds(firstReleaseDate).toJSDate(),
-
-        reviews: [
-          {
-            metadataId: "" + currentGame.id,
-            metadataSource: MetadataSource.IGDB,
-            mReviewCount: currentGame.total_rating_count ?? 0,
-            mReviewRating: (currentGame.total_rating ?? 0) / 100,
-            mReviewHref: currentGame.url,
-          },
-        ],
-
-        publishers: [],
-        developers: [],
-
-        tags: await this.getGenres(currentGame.genres),
-
-        icon,
-        bannerId: banner,
-        coverId: icon,
-        images,
-      };
     }
 
-    throw new Error("No game found on igdb with that id");
+    context?.progress(80);
+
+    const firstReleaseDate = currentGame.first_release_date;
+    const released =
+      firstReleaseDate === undefined
+        ? new Date()
+        : DateTime.fromSeconds(firstReleaseDate).toJSDate();
+
+    const review = {
+      metadataId: currentGame.id.toString(),
+      metadataSource: MetadataSource.IGDB,
+      mReviewCount: currentGame.total_rating_count ?? 0,
+      mReviewRating: (currentGame.total_rating ?? 0) / 100,
+      mReviewHref: currentGame.url,
+    };
+
+    const tags = await this.getGenres(currentGame.genres);
+
+    const deck = this.trimMessage(currentGame.summary, 280);
+
+    const metadata = {
+      id: currentGame.id.toString(),
+      name: currentGame.name,
+      shortDescription: deck,
+      description: currentGame.summary,
+      released,
+
+      reviews: [review],
+
+      publishers,
+      developers,
+
+      tags,
+
+      icon,
+      bannerId: banner,
+      coverId: icon,
+      images,
+    };
+
+    context?.log("IGDB provider finished.");
+    context?.progress(100);
+
+    return metadata;
   }
   async fetchCompany({
     query,

server/internal/metadata/index.ts

@@ -1,4 +1,4 @@
-import { MetadataSource, type GameRating } from "~/prisma/client";
+import { type Prisma, MetadataSource } from "~/prisma/client";
 import prisma from "../db/database";
 import type {
   _FetchGameMetadataParams,
@@ -12,6 +12,10 @@ import type {
 import { ObjectTransactionalHandler } from "../objects/transactional";
 import { PriorityListIndexed } from "../utils/prioritylist";
 import { systemConfig } from "../config/sys-conf";
+import type { TaskRunContext } from "../tasks";
+import taskHandler, { wrapTaskContext } from "../tasks";
+import { randomUUID } from "crypto";
+import { fuzzy } from "fast-fuzzy";
 
 export class MissingMetadataProviderConfig extends Error {
   private providerName: string;
@@ -34,9 +38,13 @@ export abstract class MetadataProvider {
   abstract source(): MetadataSource;
 
   abstract search(query: string): Promise<GameMetadataSearchResult[]>;
-  abstract fetchGame(params: _FetchGameMetadataParams): Promise<GameMetadata>;
+  abstract fetchGame(
+    params: _FetchGameMetadataParams,
+    taskRunContext?: TaskRunContext,
+  ): Promise<GameMetadata>;
   abstract fetchCompany(
     params: _FetchCompanyMetadataParams,
+    taskRunContext?: TaskRunContext,
   ): Promise<CompanyMetadata | undefined>;
 }
 
@@ -92,7 +100,12 @@ export class MetadataHandler {
     const successfulResults = results
       .filter((result) => result.status === "fulfilled")
       .map((result) => result.value)
-      .flat();
+      .flat()
+      .map((result) => {
+        const match = fuzzy(query, result.name);
+        return { ...result, fuzzy: match };
+      })
+      .sort((a, b) => b.fuzzy - a.fuzzy);
 
     return successfulResults;
   }
@@ -110,14 +123,7 @@ export class MetadataHandler {
   }
 
   private parseTags(tags: string[]) {
-    const results: {
+    const results: Array<Prisma.TagCreateOrConnectWithoutGamesInput> = [];
-      where: {
-        name: string;
-      };
-      create: {
-        name: string;
-      };
-    }[] = [];
 
     tags.forEach((t) =>
       results.push({
@@ -134,15 +140,7 @@ export class MetadataHandler {
   }
 
   private parseRatings(ratings: GameMetadataRating[]) {
-    const results: {
+    const results: Array<Prisma.GameRatingCreateOrConnectWithoutGameInput> = [];
-      where: {
-        metadataKey: {
-          metadataId: string;
-          metadataSource: MetadataSource;
-        };
-      };
-      create: Omit<GameRating, "gameId" | "created" | "id">;
-    }[] = [];
 
     ratings.forEach((r) => {
       results.push({
@@ -178,65 +176,102 @@ export class MetadataHandler {
         },
       },
     });
-    if (existing) return existing;
+    if (existing) return undefined;
 
-    const [createObject, pullObjects, dumpObjects] = this.objectHandler.new(
+    const gameId = randomUUID();
-      {},
-      ["internal:read"],
-    );
 
-    let metadata: GameMetadata | undefined = undefined;
+    const taskId = `import:${gameId}`;
-    try {
+    await taskHandler.create({
-      metadata = await provider.fetchGame({
+      name: `Import game "${result.name}" (${libraryPath})`,
-        id: result.id,
+      id: taskId,
-        name: result.name,
+      taskGroup: "import:game",
-        // wrap in anonymous functions to keep references to this
+      acls: ["system:import:game:read"],
-        publisher: (name: string) => this.fetchCompany(name),
+      async run(context) {
-        developer: (name: string) => this.fetchCompany(name),
+        const { progress, log } = context;
-        createObject,
-      });
-    } catch (e) {
-      dumpObjects();
-      throw e;
-    }
 
-    const game = await prisma.game.create({
+        progress(0);
-      data: {
-        metadataSource: provider.source(),
-        metadataId: metadata.id,
 
-        mName: metadata.name,
+        const [createObject, pullObjects, dumpObjects] =
-        mShortDescription: metadata.shortDescription,
+          metadataHandler.objectHandler.new(
-        mDescription: metadata.description,
+            {},
-        mReleased: metadata.released,
+            ["internal:read"],
+            wrapTaskContext(context, {
+              min: 63,
+              max: 100,
+              prefix: "[object import] ",
+            }),
+          );
 
-        mIconObjectId: metadata.icon,
+        let metadata: GameMetadata | undefined = undefined;
-        mBannerObjectId: metadata.bannerId,
+        try {
-        mCoverObjectId: metadata.coverId,
+          metadata = await provider.fetchGame(
-        mImageLibraryObjectIds: metadata.images,
+            {
+              id: result.id,
+              name: result.name,
+              // wrap in anonymous functions to keep references to this
+              publisher: (name: string) => metadataHandler.fetchCompany(name),
+              developer: (name: string) => metadataHandler.fetchCompany(name),
+              createObject,
+            },
+            wrapTaskContext(context, {
+              min: 0,
+              max: 60,
+              prefix: "[metadata import] ",
+            }),
+          );
+        } catch (e) {
+          dumpObjects();
+          throw e;
+        }
 
-        publishers: {
+        context?.progress(60);
-          connect: metadata.publishers,
-        },
-        developers: {
-          connect: metadata.developers,
-        },
 
-        ratings: {
+        await prisma.game.create({
-          connectOrCreate: this.parseRatings(metadata.reviews),
+          data: {
-        },
+            id: gameId,
-        tags: {
+            metadataSource: provider.source(),
-          connectOrCreate: this.parseTags(metadata.tags),
+            metadataId: metadata.id,
-        },
 
-        libraryId,
+            mName: metadata.name,
-        libraryPath,
+            mShortDescription: metadata.shortDescription,
+            mDescription: metadata.description,
+            mReleased: metadata.released,
+
+            mIconObjectId: metadata.icon,
+            mBannerObjectId: metadata.bannerId,
+            mCoverObjectId: metadata.coverId,
+            mImageLibraryObjectIds: metadata.images,
+
+            publishers: {
+              connect: metadata.publishers,
+            },
+            developers: {
+              connect: metadata.developers,
+            },
+
+            ratings: {
+              connectOrCreate: metadataHandler.parseRatings(metadata.reviews),
+            },
+            tags: {
+              connectOrCreate: metadataHandler.parseTags(metadata.tags),
+            },
+
+            libraryId,
+            libraryPath,
+          },
+        });
+
+        progress(63);
+        log(`Successfully fetched all metadata.`);
+        log(`Importing objects...`);
+
+        await pullObjects();
+
+        log(`Finished game import.`);
       },
     });
 
-    await pullObjects();
+    return taskId;
-
-    return game;
   }
 
   // Careful with this function, it has no typechecking

server/internal/metadata/pcgamingwiki.ts

@@ -15,6 +15,7 @@ import * as jdenticon from "jdenticon";
 import { DateTime } from "luxon";
 import * as cheerio from "cheerio";
 import { type } from "arktype";
+import type { TaskRunContext } from "../tasks";
 
 interface PCGamingWikiParseRawPage {
   parse: {
@@ -369,13 +370,13 @@ export class PCGamingWikiProvider implements MetadataProvider {
     return results;
   }
 
-  async fetchGame({
+  async fetchGame(
-    id,
+    { id, name, publisher, developer, createObject }: _FetchGameMetadataParams,
-    name,
+    context?: TaskRunContext,
-    publisher,
+  ): Promise<GameMetadata> {
-    developer,
+    context?.log("Using PCGamingWiki provider");
-    createObject,
+    context?.progress(0);
-  }: _FetchGameMetadataParams): Promise<GameMetadata> {
+
     const searchParams = new URLSearchParams({
       action: "cargoquery",
       tables: "Infobox_game",
@@ -396,37 +397,49 @@ export class PCGamingWikiProvider implements MetadataProvider {
 
     const publishers: Company[] = [];
     if (game.Publishers !== null) {
+      context?.log("Found publishers, importing...");
       const pubListClean = this.parseWikiStringArray(game.Publishers);
       for (const pub of pubListClean) {
+        context?.log(`Importing "${pub}"...`);
+
         const res = await publisher(pub);
         if (res === undefined) continue;
         publishers.push(res);
       }
     }
 
+    context?.progress(40);
+
     const developers: Company[] = [];
     if (game.Developers !== null) {
+      context?.log("Found developers, importing...");
       const devListClean = this.parseWikiStringArray(game.Developers);
       for (const dev of devListClean) {
+        context?.log(`Importing "${dev}"...`);
         const res = await developer(dev);
         if (res === undefined) continue;
         developers.push(res);
       }
     }
 
-    const icon =
+    context?.progress(80);
+
+    const icon = createObject(
       game["Cover URL"] !== null
-        ? createObject(game["Cover URL"])
+        ? game["Cover URL"]
-        : createObject(jdenticon.toPng(name, 512));
+        : jdenticon.toPng(name, 512),
+    );
+
+    const released = game.Released
+      ? DateTime.fromISO(game.Released.split(";")[0]).toJSDate()
+      : new Date();
 
     const metadata: GameMetadata = {
       id: game.PageID,
       name: game.PageName,
       shortDescription: pageContent.shortIntro,
       description: pageContent.introduction,
-      released: game.Released
+      released,
-        ? DateTime.fromISO(game.Released.split(";")[0]).toJSDate()
-        : new Date(),
 
       tags: this.compileTags(game),
 
@@ -440,6 +453,9 @@ export class PCGamingWikiProvider implements MetadataProvider {
       images: [icon],
     };
 
+    context?.log("PCGamingWiki provider finished.");
+    context?.progress(100);
+
     return metadata;
   }
 

server/internal/objects/transactional.ts

@@ -5,6 +5,7 @@ This is used as a utility in metadata handling, so we only fetch the objects if
 import type { Readable } from "stream";
 import { randomUUID } from "node:crypto";
 import objectHandler from ".";
+import type { TaskRunContext } from "../tasks";
 
 export type TransactionDataType = string | Readable | Buffer;
 type TransactionTable = Map<string, TransactionDataType>; // ID to data
@@ -20,6 +21,7 @@ export class ObjectTransactionalHandler {
   new(
     metadata: { [key: string]: string },
     permissions: Array<string>,
+    context?: TaskRunContext,
   ): [Register, Pull, Dump] {
     const transactionId = randomUUID();
 
@@ -35,7 +37,16 @@ export class ObjectTransactionalHandler {
     const pull = async () => {
       const transaction = this.record.get(transactionId);
       if (!transaction) return;
+
+      let progress = 0;
+      const increment = (1 / transaction.size) * 100;
+
       for (const [id, data] of transaction) {
+        if (typeof data === "string") {
+          context?.log(`Importing object from "${data}"`);
+        } else {
+          context?.log(`Importing raw object...`);
+        }
         await objectHandler.createFromSource(
           id,
           () => {
@@ -47,6 +58,8 @@ export class ObjectTransactionalHandler {
           metadata,
           permissions,
         );
+        progress += increment;
+        context?.progress(progress);
       }
     };
 

server/internal/tasks/index.ts

@@ -332,6 +332,22 @@ export type TaskRunContext = {
   log: (message: string) => void;
 };
 
+export function wrapTaskContext(
+  context: TaskRunContext,
+  options: { min: number; max: number; prefix: string },
+): TaskRunContext {
+  return {
+    progress(progress) {
+      const scalar = 100 / (options.max - options.min);
+      const adjustedProgress = progress * scalar + options.min;
+      return context.progress(adjustedProgress);
+    },
+    log(message) {
+      return context.log(options.prefix + message);
+    },
+  };
+}
+
 export interface Task {
   id: string;
   taskGroup: TaskGroup;