Merge branch 'metadata-improvements' of https://github.com/Huskydog9988/drop into Huskydog9988-metadata-improvements

server/internal/metadata/giantbomb.ts

@@ -1,5 +1,5 @@
 import { Developer, MetadataSource, Publisher } from "@prisma/client";
-import { MetadataProvider } from ".";
+import { MetadataProvider, MissingMetadataProviderConfig } from ".";
 import {
   GameMetadataSearchResult,
   _FetchGameMetadataParams,
@@ -74,13 +74,18 @@ interface CompanySearchResult {
   };
 }
 
+// Api Docs: https://www.giantbomb.com/api/
 export class GiantBombProvider implements MetadataProvider {
   private apikey: string;
   private turndown: TurndownService;
 
   constructor() {
     const apikey = process.env.GIANT_BOMB_API_KEY;
-    if (!apikey) throw new Error("No GIANT_BOMB_API_KEY in environment");
+    if (!apikey)
+      throw new MissingMetadataProviderConfig(
+        "GIANT_BOMB_API_KEY",
+        this.name()
+      );
 
     this.apikey = apikey;
 
@@ -96,18 +101,14 @@ export class GiantBombProvider implements MetadataProvider {
   private async request<T>(
     resource: string,
     url: string,
-    query: { [key: string]: string | Array<string> },
+    query: { [key: string]: string },
     options?: AxiosRequestConfig
   ) {
-    const queryOptions = { ...query, api_key: this.apikey, format: "json" };
-    const queryString = Object.entries(queryOptions)
-      .map(([key, value]) => {
-        if (Array.isArray(value)) {
-          return `${key}=${value.map(encodeURIComponent).join(",")}`;
-        }
-        return `${key}=${encodeURIComponent(value)}`;
-      })
-      .join("&");
+    const queryString = new URLSearchParams({
+      ...query,
+      api_key: this.apikey,
+      format: "json",
+    }).toString();
 
     const finalURL = `https://www.giantbomb.com/api/${resource}/${url}?${queryString}`;
 
@@ -134,7 +135,7 @@ export class GiantBombProvider implements MetadataProvider {
   async search(query: string): Promise<GameMetadataSearchResult[]> {
     const results = await this.request<Array<GameSearchResult>>("search", "", {
       query: query,
-      resources: ["game"],
+      resources: ["game"].join(","),
     });
     const mapped = results.data.results.map((result) => {
       const date =

server/internal/metadata/igdb.ts

@@ -0,0 +1,406 @@
+import { Developer, MetadataSource, Publisher } from "@prisma/client";
+import { MetadataProvider, MissingMetadataProviderConfig } from ".";
+import {
+  GameMetadataSearchResult,
+  _FetchGameMetadataParams,
+  GameMetadata,
+  _FetchPublisherMetadataParams,
+  PublisherMetadata,
+  _FetchDeveloperMetadataParams,
+  DeveloperMetadata,
+} from "./types";
+import axios, { AxiosRequestConfig } from "axios";
+import { inspect } from "util";
+import moment from "moment";
+
+type IGDBID = number;
+
+interface TwitchAuthResponse {
+  access_token: string;
+  expires_in: number;
+  token_type: string; // likely 'bearer'
+}
+
+interface IGDBErrorResponse {
+  title: string;
+  status: number;
+  cause: string;
+}
+
+interface IGDBItem {
+  id: IGDBID;
+}
+
+// denotes role a company had in a game
+interface IGDBInvolvedCompany extends IGDBItem {
+  company: IGDBID;
+  game: IGDBID;
+
+  developer: boolean;
+  porting: boolean;
+  publisher: boolean;
+  supporting: boolean;
+
+  created_at: number;
+  updated_at: number;
+}
+
+interface IGDBCompany extends IGDBItem {
+  name: string;
+  country: number; // ISO 3166-1 country code
+  description: string;
+  logo: IGDBID;
+  parent: IGDBID;
+  slug: string;
+  start_date: number;
+  status: IGDBID;
+  websites: IGDBID[];
+}
+
+interface IGDBCompanyWebsite extends IGDBItem {
+  trusted: boolean;
+  url: string;
+}
+
+interface IGDBCover extends IGDBItem {
+  url: string;
+}
+
+interface IGDBSearchStub extends IGDBItem {
+  name: string;
+  cover: IGDBID;
+  first_release_date: number; // unix timestamp
+  summary: string;
+}
+
+// https://api-docs.igdb.com/?shell#game
+interface IGDBGameFull extends IGDBSearchStub {
+  age_ratings?: IGDBID[];
+  aggregated_rating?: number;
+  aggregated_rating_count?: number;
+  alternative_names?: IGDBID[];
+  artworks?: IGDBID[];
+  bundles?: IGDBID[];
+  checksum?: string;
+  collections?: IGDBID[];
+  created_at: number; // unix timestamp
+  dlcs?: IGDBID[];
+  expanded_games?: IGDBID[];
+  expansions?: IGDBID[];
+  external_games?: IGDBID[];
+  forks?: IGDBID[];
+  franchise?: IGDBID;
+  franchises?: IGDBID[];
+  game_engines?: IGDBID[];
+  game_localizations?: IGDBID[];
+  game_modes?: IGDBID[];
+  game_status?: IGDBID;
+  game_type?: IGDBID;
+  genres?: IGDBID[];
+  hypes?: number;
+  involved_companies?: IGDBID[];
+  keywords?: IGDBID[];
+  language_supports?: IGDBID[];
+  multiplayer_modes?: IGDBID[];
+  platforms?: IGDBID[];
+  player_perspectives?: IGDBID[];
+  ports?: IGDBID[];
+  rating?: number;
+  rating_count?: number;
+  release_dates?: IGDBID[];
+  remakes?: IGDBID[];
+  remasters?: IGDBID[];
+  screenshots?: IGDBID[];
+  similar_games?: IGDBID[];
+  slug: string;
+  standalone_expansions?: IGDBID[];
+  storyline?: string;
+  tags?: IGDBID[];
+  themes?: IGDBID[];
+  total_rating?: number;
+  total_rating_count?: number;
+  updated_at: number;
+  url: string;
+  version_parent?: IGDBID;
+  version_title?: string;
+  videos?: IGDBID[];
+  websites?: IGDBID[];
+}
+
+// Api Docs: https://api-docs.igdb.com/
+export class IGDBProvider implements MetadataProvider {
+  private client_id: string;
+  private client_secret: string;
+  private access_token: string;
+  private access_token_expire: moment.Moment;
+
+  constructor() {
+    const client_id = process.env.IGDB_CLIENT_ID;
+    if (!client_id)
+      throw new MissingMetadataProviderConfig("IGDB_CLIENT_ID", this.name());
+    const client_secret = process.env.IGDB_CLIENT_SECRET;
+    if (!client_secret)
+      throw new MissingMetadataProviderConfig(
+        "IGDB_CLIENT_SECRET",
+        this.name()
+      );
+
+    this.client_id = client_id;
+    this.client_secret = client_secret;
+
+    this.access_token = "";
+    this.access_token_expire = moment();
+    this.authWithTwitch();
+  }
+
+  private async authWithTwitch() {
+    const params = new URLSearchParams({
+      client_id: this.client_id,
+      client_secret: this.client_secret,
+      grant_type: "client_credentials",
+    });
+
+    const response = await axios.request<TwitchAuthResponse>({
+      url: `https://id.twitch.tv/oauth2/token?${params.toString()}`,
+      baseURL: "",
+      method: "POST",
+    });
+
+    this.access_token = response.data.access_token;
+    this.access_token_expire = moment().add(
+      response.data.expires_in,
+      "seconds"
+    );
+  }
+
+  private async refreshCredentials() {
+    const futureTime = moment().add(1, "day");
+
+    // if the token expires before this future time (aka soon), refresh
+    if (this.access_token_expire.isBefore(futureTime)) this.authWithTwitch();
+  }
+
+  private async request<T extends Object>(
+    resource: string,
+    body: string,
+    options?: AxiosRequestConfig
+  ) {
+    await this.refreshCredentials();
+
+    // prevent calling api before auth is complete
+    if (this.access_token.length <= 0)
+      throw new Error(
+        "IGDB either failed to authenticate, or has not done so yet"
+      );
+
+    const finalURL = `https://api.igdb.com/v4/${resource}`;
+
+    const overlay: AxiosRequestConfig = {
+      url: finalURL,
+      baseURL: "",
+      method: "POST",
+      data: body,
+      headers: {
+        Accept: "application/json",
+        "Client-ID": this.client_id,
+        Authorization: `Bearer ${this.access_token}`,
+        "content-type": "text/plain",
+      },
+    };
+    const response = await axios.request<T[] | IGDBErrorResponse[]>(
+      Object.assign({}, options, overlay)
+    );
+
+    if (response.status !== 200) {
+      let cause = "";
+
+      response.data.forEach((item) => {
+        if ("cause" in item) cause = item.cause;
+      });
+
+      throw new Error(
+        `Error in igdb \nStatus Code: ${response.status} \nCause: ${cause}`
+      );
+    }
+
+    // should not have an error object if the status code is 200
+    return <T[]>response.data;
+  }
+
+  private async _getMediaInternal(mediaID: IGDBID, type: string) {
+    const body = `where id = ${mediaID}; fields url;`;
+    const response = await this.request<IGDBCover>(type, body);
+
+    let result = "";
+
+    response.forEach((cover) => {
+      if (cover.url.startsWith("https:")) {
+        result = cover.url;
+      } else {
+        // twitch *sometimes* provides it in the format "//images.igdb.com"
+        result = `https:${cover.url}`;
+      }
+    });
+    return result;
+  }
+
+  private async getCoverURL(id: IGDBID) {
+    return await this._getMediaInternal(id, "covers");
+  }
+
+  private async getArtworkURL(id: IGDBID) {
+    return await this._getMediaInternal(id, "artworks");
+  }
+
+  private async getCompanyLogoURl(id: IGDBID) {
+    return await this._getMediaInternal(id, "company_logos");
+  }
+
+  private trimMessage(msg: string, len: number) {
+    return msg.length > len ? msg.substring(0, 280) + "..." : msg;
+  }
+
+  id() {
+    return "igdb";
+  }
+  name() {
+    return "IGDB";
+  }
+  source() {
+    return MetadataSource.IGDB;
+  }
+
+  async search(query: string): Promise<GameMetadataSearchResult[]> {
+    const body = `search "${query}"; fields name,cover,first_release_date,summary; limit 3;`;
+    const response = await this.request<IGDBSearchStub>("games", body);
+
+    const results: GameMetadataSearchResult[] = [];
+    for (let i = 0; i < response.length; i++) {
+      results.push({
+        id: "" + response[i].id,
+        name: response[i].name,
+        icon: await this.getCoverURL(response[i].cover),
+        description: response[i].summary,
+        year: moment.unix(response[i].first_release_date).year(),
+      });
+    }
+
+    return results;
+  }
+  async fetchGame({
+    id,
+    publisher,
+    developer,
+    createObject,
+  }: _FetchGameMetadataParams): Promise<GameMetadata> {
+    const body = `where id = ${id}; fields *;`;
+    const response = await this.request<IGDBGameFull>("games", body);
+
+    for (let i = 0; i < response.length; i++) {
+      const icon = createObject(await this.getCoverURL(response[i].cover));
+      let banner = "";
+
+      const images = [icon];
+      for (const art of response[i]?.artworks ?? []) {
+        // if banner not set
+        if (banner.length <= 0) {
+          banner = createObject(await this.getArtworkURL(art));
+          images.push(banner);
+        } else {
+          images.push(createObject(await this.getArtworkURL(art)));
+        }
+      }
+
+      const publishers: Publisher[] = [];
+      const developers: Developer[] = [];
+      for (const involvedCompany of response[i]?.involved_companies ?? []) {
+        // get details about the involved company
+        const involved_company_response =
+          await this.request<IGDBInvolvedCompany>(
+            "involved_companies",
+            `where id = ${involvedCompany}; fields *;`
+          );
+        for (const foundInvolved of involved_company_response) {
+          // now we need to get the actual company so we can get the name
+          const findCompanyResponse = await this.request<
+            { name: string } & IGDBItem
+          >("companies", `where id = ${foundInvolved.company}; fields name;`);
+
+          for (const company of findCompanyResponse) {
+            // if company was a dev or publisher
+            // CANNOT use else since a company can be both
+            if (foundInvolved.developer)
+              developers.push(await developer(company.name));
+            if (foundInvolved.publisher)
+              publishers.push(await publisher(company.name));
+          }
+        }
+      }
+
+      return {
+        id: "" + response[i].id,
+        name: response[i].name,
+        shortDescription: this.trimMessage(response[i].summary, 280),
+        description: response[i].summary,
+        released: moment.unix(response[i].first_release_date).toDate(),
+
+        reviewCount: response[i]?.total_rating_count ?? 0,
+        reviewRating: (response[i]?.total_rating ?? 0) / 100,
+
+        publishers: [],
+        developers: [],
+
+        icon,
+        bannerId: banner,
+        coverId: icon,
+        images,
+      };
+    }
+
+    throw new Error("No game found on igdb with that id");
+  }
+  async fetchPublisher({
+    query,
+    createObject,
+  }: _FetchPublisherMetadataParams): Promise<PublisherMetadata> {
+    const response = await this.request<IGDBCompany>(
+      "companies",
+      `where name = "${query}"; fields *; limit 1;`
+    );
+
+    for (const company of response) {
+      const logo = createObject(await this.getCompanyLogoURl(company.logo));
+
+      let company_url = "";
+      for (const companySite of company.websites) {
+        const companySiteRes = await this.request<IGDBCompanyWebsite>(
+          "company_websites",
+          `where id = ${companySite}; fields *;`
+        );
+
+        for (const site of companySiteRes) {
+          if (company_url.length <= 0) company_url = site.url;
+        }
+      }
+      const metadata: PublisherMetadata = {
+        id: "" + company.id,
+        name: company.name,
+        shortDescription: this.trimMessage(company.description, 280),
+        description: company.description,
+        website: company_url,
+
+        logo: logo,
+        banner: logo,
+      };
+
+      return metadata;
+    }
+
+    throw new Error("No results found");
+  }
+  async fetchDeveloper(
+    params: _FetchDeveloperMetadataParams
+  ): Promise<DeveloperMetadata> {
+    return await this.fetchPublisher(params);
+  }
+}

server/internal/metadata/index.ts

@@ -19,6 +19,24 @@ import { ObjectTransactionalHandler } from "../objects/transactional";
 import { PriorityList, PriorityListIndexed } from "../utils/prioritylist";
 import { GiantBombProvider } from "./giantbomb";
 import { ManualMetadataProvider } from "./manual";
+import { PCGamingWikiProvider } from "./pcgamingwiki";
+import { IGDBProvider } from "./igdb";
+
+export class MissingMetadataProviderConfig extends Error {
+  private providerName: string;
+
+  constructor(configKey: string, providerName: string) {
+    super(`Missing config item ${configKey} for ${providerName}`);
+    this.providerName = providerName;
+  }
+
+  getProviderName() {
+    return this.providerName;
+  }
+}
+
+// TODO: add useragent to all outbound api calls (best practice)
+export const DropUserAgent = "Drop/0.2";
 
 export abstract class MetadataProvider {
   abstract id(): string;
@@ -46,7 +64,6 @@ export class MetadataHandler {
     this.providers.push(provider, priority);
   }
 
-  
   /**
    * Returns provider IDs, used to save to applicationConfig
    * @returns The provider IDs in order, missing manual
@@ -143,7 +160,6 @@ export class MetadataHandler {
       throw e;
     }
 
-    await pullObjects();
     const game = await prisma.game.create({
       data: {
         metadataSource: provider.source(),
@@ -171,6 +187,7 @@ export class MetadataHandler {
         libraryBasePath,
       },
     });
+    await pullObjects();
 
     return game;
   }
@@ -195,8 +212,8 @@ export class MetadataHandler {
   // Type-checking this thing is impossible
   private async fetchDeveloperPublisher(
     query: string,
-    functionName: any,
-    databaseName: any
+    functionName: "fetchDeveloper" | "fetchPublisher",
+    databaseName: "developer" | "publisher"
   ) {
     const existing = await (prisma as any)[databaseName].findFirst({
       where: {
@@ -205,12 +222,15 @@ export class MetadataHandler {
     });
     if (existing) return existing;
 
-    for (const provider of this.providers.values() as any) {
+    for (const provider of this.providers.values()) {
+      // don't allow manual provider to "fetch" metadata
+      if (provider.source() === MetadataSource.Manual) continue;
+
       const [createObject, pullObjects, dumpObjects] = this.objectHandler.new(
         {},
         ["internal:read"]
       );
-      let result;
+      let result: PublisherMetadata;
       try {
         result = await provider[functionName]({ query, createObject });
       } catch (e) {

server/internal/metadata/pcgamingwiki.ts

@@ -0,0 +1,259 @@
+import { Developer, MetadataSource, Publisher } from "@prisma/client";
+import { MetadataProvider, MissingMetadataProviderConfig } from ".";
+import {
+  GameMetadataSearchResult,
+  _FetchGameMetadataParams,
+  GameMetadata,
+  _FetchPublisherMetadataParams,
+  PublisherMetadata,
+  _FetchDeveloperMetadataParams,
+  DeveloperMetadata,
+} from "./types";
+import axios, { AxiosRequestConfig } from "axios";
+import moment from "moment";
+import * as jdenticon from "jdenticon";
+
+interface PCGamingWikiPage {
+  PageID: string;
+  PageName: string;
+}
+
+interface PCGamingWikiSearchStub extends PCGamingWikiPage {
+  "Cover URL": string | null;
+  Released: string | null;
+  Released__precision: string | null;
+}
+
+interface PCGamingWikiGame extends PCGamingWikiSearchStub {
+  Developers: string | null;
+  Genres: string | null;
+  Publishers: string | null;
+  Themes: string | null;
+  Series: string | null;
+  Modes: string | null;
+}
+
+interface PCGamingWikiCompany extends PCGamingWikiPage {
+  Parent: string | null;
+  Founded: string | null;
+  Website: string | null;
+  Founded__precision: string | null;
+  Defunct__precision: string | null;
+}
+
+interface PCGamingWikiCargoResult<T> {
+  cargoquery: [
+    {
+      title: T;
+    }
+  ];
+  error?: {
+    code?: string;
+    info?: string;
+    errorclass?: string;
+    "*"?: string;
+  };
+}
+
+// Api Docs: https://www.pcgamingwiki.com/wiki/PCGamingWiki:API
+// Good tool for helping build cargo queries: https://www.pcgamingwiki.com/wiki/Special:CargoQuery
+export class PCGamingWikiProvider implements MetadataProvider {
+  constructor() {}
+
+  id() {
+    return "pcgamingwiki";
+  }
+  name() {
+    return "PCGamingWiki";
+  }
+  source() {
+    return MetadataSource.PCGamingWiki;
+  }
+
+  private async request<T>(
+    query: URLSearchParams,
+    options?: AxiosRequestConfig
+  ) {
+    const finalURL = `https://www.pcgamingwiki.com/w/api.php?${query.toString()}`;
+
+    const overlay: AxiosRequestConfig = {
+      url: finalURL,
+      baseURL: "",
+    };
+    const response = await axios.request<PCGamingWikiCargoResult<T>>(
+      Object.assign({}, options, overlay)
+    );
+
+    if (response.status !== 200)
+      throw new Error(
+        `Error in pcgamingwiki \nStatus Code: ${response.status}`
+      );
+    else if (response.data.error !== undefined)
+      throw new Error(`Error in pcgamingwiki, malformed query`);
+
+    return response;
+  }
+
+  async search(query: string) {
+    const searchParams = new URLSearchParams({
+      action: "cargoquery",
+      tables: "Infobox_game",
+      fields:
+        "Infobox_game._pageID=PageID,Infobox_game._pageName=PageName,Infobox_game.Cover_URL,Infobox_game.Released",
+      where: `Infobox_game._pageName="${query}"`,
+      format: "json",
+    });
+
+    const res = await this.request<PCGamingWikiSearchStub>(searchParams);
+
+    const mapped = res.data.cargoquery.map((result) => {
+      const game = result.title;
+
+      const metadata: GameMetadataSearchResult = {
+        id: game.PageID,
+        name: game.PageName,
+        icon: game["Cover URL"] ?? "",
+        description: "", // TODO: need to render the `Introduction` template somehow (or we could just hardcode it)
+        year:
+          game.Released !== null && game.Released.length > 0
+            ? moment(game.Released).year()
+            : 0,
+      };
+      return metadata;
+    });
+
+    return mapped;
+  }
+
+  /**
+   * Parses the specific format that the wiki returns when specifying a company
+   * @param companyStr
+   * @returns
+   */
+  private parseCompanyStr(companyStr: string): string[] {
+    const results: string[] = [];
+    // provides the string as a list of companies
+    // ie: "Company:Digerati Distribution,Company:Greylock Studio"
+    const items = companyStr.split(",");
+
+    items.forEach((item) => {
+      // remove the `Company:` and trim and whitespace
+      results.push(item.replace("Company:", "").trim());
+    });
+
+    return results;
+  }
+
+  async fetchGame({
+    id,
+    name,
+    publisher,
+    developer,
+    createObject,
+  }: _FetchGameMetadataParams): Promise<GameMetadata> {
+    const searchParams = new URLSearchParams({
+      action: "cargoquery",
+      tables: "Infobox_game",
+      fields:
+        "Infobox_game._pageID=PageID,Infobox_game._pageName=PageName,Infobox_game.Cover_URL,Infobox_game.Developers,Infobox_game.Released,Infobox_game.Genres,Infobox_game.Publishers,Infobox_game.Themes,Infobox_game.Series,Infobox_game.Modes",
+      where: `Infobox_game._pageID="${id}"`,
+      format: "json",
+    });
+
+    const res = await this.request<PCGamingWikiGame>(searchParams);
+    if (res.data.cargoquery.length < 1)
+      throw new Error("Error in pcgamingwiki, no game");
+
+    const game = res.data.cargoquery[0].title;
+
+    const publishers: Publisher[] = [];
+    if (game.Publishers !== null) {
+      const pubListClean = this.parseCompanyStr(game.Publishers);
+      for (const pub of pubListClean) {
+        publishers.push(await publisher(pub));
+      }
+    }
+
+    const developers: Developer[] = [];
+    if (game.Developers !== null) {
+      const devListClean = this.parseCompanyStr(game.Developers);
+      for (const dev of devListClean) {
+        developers.push(await developer(dev));
+      }
+    }
+
+    const icon =
+      game["Cover URL"] !== null
+        ? createObject(game["Cover URL"])
+        : createObject(jdenticon.toPng(name, 512));
+
+    const metadata: GameMetadata = {
+      id: game.PageID,
+      name: game.PageName,
+      shortDescription: "", // TODO: (again) need to render the `Introduction` template somehow (or we could just hardcode it)
+      description: "",
+      released: game.Released
+        ? moment(game.Released.split(";").at(0)).toDate()
+        : new Date(),
+
+      reviewCount: 0,
+      reviewRating: 0,
+
+      publishers,
+      developers,
+
+      icon: icon,
+      bannerId: icon,
+      coverId: icon,
+      images: [icon],
+    };
+
+    return metadata;
+  }
+
+  async fetchPublisher({
+    query,
+    createObject,
+  }: _FetchPublisherMetadataParams): Promise<PublisherMetadata> {
+    const searchParams = new URLSearchParams({
+      action: "cargoquery",
+      tables: "Company",
+      fields:
+        "Company.Parent,Company.Founded,Company.Defunct,Company.Website,Company._pageName=PageName,Company._pageID=pageID",
+      where: `Company._pageName="Company:${query}"`,
+      format: "json",
+    });
+
+    const res = await this.request<PCGamingWikiCompany>(searchParams);
+
+    // TODO: replace
+    const icon = createObject(jdenticon.toPng(query, 512));
+
+    for (let i = 0; i < res.data.cargoquery.length; i++) {
+      const company = res.data.cargoquery[i].title;
+
+      const fixedCompanyName =
+        company.PageName.split("Company:").at(1) ?? company.PageName;
+
+      const metadata: PublisherMetadata = {
+        id: company.PageID,
+        name: fixedCompanyName,
+        shortDescription: "",
+        description: "",
+        website: company?.Website ?? "",
+
+        logo: icon,
+        banner: icon,
+      };
+      return metadata;
+    }
+
+    throw new Error("Error in pcgamingwiki, no publisher");
+  }
+
+  async fetchDeveloper(
+    params: _FetchDeveloperMetadataParams
+  ): Promise<DeveloperMetadata> {
+    return await this.fetchPublisher(params);
+  }
+}