v2 download API and Admin UI fixes (#177)

* fix: small ui fixes

* feat: #171

* fix: improvements to library scanning on admin UI

* feat: v2 download API

* fix: add download context cleanup

* fix: lint

server/api/v1/admin/company/index.post.ts

@@ -0,0 +1,47 @@
+import { type } from "arktype";
+import { readDropValidatedBody, throwingArktype } from "~/server/arktype";
+import aclManager from "~/server/internal/acls";
+import * as jdenticon from "jdenticon";
+import { ObjectTransactionalHandler } from "~/server/internal/objects/transactional";
+import prisma from "~/server/internal/db/database";
+import { MetadataSource } from "~/prisma/client/enums";
+
+const CompanyCreate = type({
+  name: "string",
+  description: "string",
+  website: "string",
+}).configure(throwingArktype);
+
+export default defineEventHandler(async (h3) => {
+  const allowed = await aclManager.allowSystemACL(h3, ["company:create"]);
+  if (!allowed) throw createError({ statusCode: 403 });
+
+  const body = await readDropValidatedBody(h3, CompanyCreate);
+  const obj = new ObjectTransactionalHandler();
+  const [register, pull, _] = obj.new({}, ["internal:read"]);
+
+  const icon = jdenticon.toPng(body.name, 512);
+  const logoId = register(icon);
+
+  const banner = jdenticon.toPng(body.description, 1024);
+  const bannerId = register(banner);
+
+  const company = await prisma.company.create({
+    data: {
+      metadataSource: MetadataSource.Manual,
+      metadataId: crypto.randomUUID(),
+      metadataOriginalQuery: "",
+
+      mName: body.name,
+      mShortDescription: body.description,
+      mDescription: "",
+      mLogoObjectId: logoId,
+      mBannerObjectId: bannerId,
+      mWebsite: body.website,
+    },
+  });
+
+  await pull();
+
+  return company;
+});

server/api/v1/index.get.ts

@@ -5,5 +5,6 @@ export default defineEventHandler((_h3) => {
     appName: "Drop",
     version: systemConfig.getDropVersion(),
     gitRef: `#${systemConfig.getGitRef()}`,
+    external: systemConfig.getExternalUrl(),
   };
 });

server/api/v2/client/chunk.post.ts

@@ -0,0 +1,73 @@
+import { type } from "arktype";
+import { readDropValidatedBody, throwingArktype } from "~/server/arktype";
+import contextManager from "~/server/internal/downloads/coordinator";
+import libraryManager from "~/server/internal/library";
+
+const GetChunk = type({
+  context: "string",
+  files: type({
+    filename: "string",
+    chunkIndex: "number",
+  }).array(),
+}).configure(throwingArktype);
+
+export default defineEventHandler(async (h3) => {
+  const body = await readDropValidatedBody(h3, GetChunk);
+
+  const context = await contextManager.fetchContext(body.context);
+  if (!context)
+    throw createError({
+      statusCode: 400,
+      statusMessage: "Invalid download context.",
+    });
+
+  const streamFiles = [];
+
+  for (const file of body.files) {
+    const manifestFile = context.manifest[file.filename];
+    if (!manifestFile)
+      throw createError({
+        statusCode: 400,
+        statusMessage: `Unknown file: ${file.filename}`,
+      });
+
+    const start = manifestFile.lengths
+      .slice(0, file.chunkIndex)
+      .reduce((a, b) => a + b, 0);
+    const end = start + manifestFile.lengths[file.chunkIndex];
+
+    streamFiles.push({ filename: file.filename, start, end });
+  }
+
+  setHeader(
+    h3,
+    "Content-Lengths",
+    streamFiles.map((e) => e.end - e.start).join(","),
+  ); // Non-standard header, but we're cool like that 😎
+
+  for (const file of streamFiles) {
+    const gameReadStream = await libraryManager.readFile(
+      context.libraryId,
+      context.libraryPath,
+      context.versionName,
+      file.filename,
+      { start: file.start, end: file.end },
+    );
+    if (!gameReadStream)
+      throw createError({
+        statusCode: 500,
+        statusMessage: "Failed to create read stream",
+      });
+    await gameReadStream.pipeTo(
+      new WritableStream({
+        write(chunk) {
+          h3.node.res.write(chunk);
+        },
+      }),
+    );
+  }
+
+  await h3.node.res.end();
+
+  return;
+});

server/api/v2/client/context.post.ts

@@ -0,0 +1,22 @@
+import { type } from "arktype";
+import { readDropValidatedBody, throwingArktype } from "~/server/arktype";
+import { defineClientEventHandler } from "~/server/internal/clients/event-handler";
+import contextManager from "~/server/internal/downloads/coordinator";
+
+const CreateContext = type({
+  game: "string",
+  version: "string",
+}).configure(throwingArktype);
+
+export default defineClientEventHandler(async (h3) => {
+  const body = await readDropValidatedBody(h3, CreateContext);
+
+  const context = await contextManager.createContext(body.game, body.version);
+  if (!context)
+    throw createError({
+      statusCode: 400,
+      statusMessage: "Invalid game or version",
+    });
+
+  return { context };
+});

server/internal/downloads/coordinator.ts

@@ -1,9 +1,68 @@
-/*
-The download co-ordinator's job is to keep track of all the currently online clients.
+import prisma from "../db/database";
+import type { DropManifest } from "./manifest";
 
-When a client signs on and registers itself as a peer
+const TIMEOUT = 1000 * 60 * 60 * 1; // 1 hour
 
-*/
+class DownloadContextManager {
+  private contexts: Map<
+    string,
+    {
+      timeout: Date;
+      manifest: DropManifest;
+      versionName: string;
+      libraryId: string;
+      libraryPath: string;
+    }
+  > = new Map();
 
-// eslint-disable-next-line @typescript-eslint/no-extraneous-class, @typescript-eslint/no-unused-vars
-class DownloadCoordinator {}
+  async createContext(game: string, versionName: string) {
+    const version = await prisma.gameVersion.findUnique({
+      where: {
+        gameId_versionName: {
+          gameId: game,
+          versionName,
+        },
+      },
+      include: {
+        game: {
+          select: {
+            libraryId: true,
+            libraryPath: true,
+          },
+        },
+      },
+    });
+    if (!version) return undefined;
+
+    const contextId = crypto.randomUUID();
+    this.contexts.set(contextId, {
+      timeout: new Date(),
+      manifest: JSON.parse(version.dropletManifest as string) as DropManifest,
+      versionName,
+      libraryId: version.game.libraryId!,
+      libraryPath: version.game.libraryPath,
+    });
+
+    return contextId;
+  }
+
+  async fetchContext(contextId: string) {
+    const context = this.contexts.get(contextId);
+    if (!context) return undefined;
+    context.timeout = new Date();
+    this.contexts.set(contextId, context);
+    return context;
+  }
+
+  async cleanup() {
+    for (const key of this.contexts.keys()) {
+      const context = this.contexts.get(key)!;
+      if (context.timeout.getDate() + TIMEOUT < Date.now()) {
+        this.contexts.delete(key);
+      }
+    }
+  }
+}
+
+export const contextManager = new DownloadContextManager();
+export default contextManager;

server/internal/downloads/manifest.ts

@@ -5,7 +5,7 @@ export type DropChunk = {
   permissions: number;
   ids: string[];
   checksums: string[];
-  lengths: string[];
+  lengths: number[];
 };
 
 export type DropManifest = {

server/internal/library/index.ts

@@ -13,6 +13,7 @@ import { parsePlatform } from "../utils/parseplatform";
 import notificationSystem from "../notifications";
 import { GameNotFoundError, type LibraryProvider } from "./provider";
 import { logger } from "../logging";
+import type { GameModel } from "~/prisma/client/models";
 
 class LibraryManager {
   private libraries: Map<string, LibraryProvider<unknown>> = new Map();
@@ -37,24 +38,32 @@ class LibraryManager {
     return libraryWithMetadata;
   }
 
+  async fetchGamesByLibrary() {
+    const results: { [key: string]: { [key: string]: GameModel } } = {};
+    const games = await prisma.game.findMany({});
+    for (const game of games) {
+      const libraryId = game.libraryId!;
+      const libraryPath = game.libraryPath!;
+
+      results[libraryId] ??= {};
+      results[libraryId][libraryPath] = game;
+    }
+
+    return results;
+  }
+
   async fetchUnimportedGames() {
     const unimportedGames: { [key: string]: string[] } = {};
+    const instanceGames = await this.fetchGamesByLibrary();
 
     for (const [id, library] of this.libraries.entries()) {
-      const games = await library.listGames();
-      const validGames = await prisma.game.findMany({
-        where: {
-          libraryId: id,
-          libraryPath: { in: games },
-        },
-        select: {
-          libraryPath: true,
-        },
-      });
-      const providerUnimportedGames = games.filter(
-        (e) =>
-          validGames.findIndex((v) => v.libraryPath == e) == -1 &&
-          !(this.gameImportLocks.get(id) ?? []).includes(e),
+      const providerGames = await library.listGames();
+      const locks = this.gameImportLocks.get(id) ?? [];
+      const providerUnimportedGames = providerGames.filter(
+        (libraryPath) =>
+          instanceGames[id] &&
+          !instanceGames[id][libraryPath] &&
+          !locks.includes(libraryPath),
       );
       unimportedGames[id] = providerUnimportedGames;
     }

server/middleware/latency.ts

@@ -0,0 +1,3 @@
+export default defineEventHandler(async () => {
+  // await new Promise((r) => setTimeout(r, 700));
+});

server/tasks/downloadCleanup.ts

@@ -0,0 +1,11 @@
+import contextManager from "../internal/downloads/coordinator";
+
+export default defineTask({
+  meta: {
+    name: "downloadCleanup",
+  },
+  async run() {
+    await contextManager.cleanup();
+    return { result: true };
+  },
+});