v2 download API and Admin UI fixes (#177)

* fix: small ui fixes

* feat: #171

* fix: improvements to library scanning on admin UI

* feat: v2 download API

* fix: add download context cleanup

* fix: lint

app.vue

@@ -4,10 +4,52 @@
     <NuxtPage />
   </NuxtLayout>
   <ModalStack />
+  <div
+    v-if="showExternalUrlWarning"
+    class="fixed flex flex-row gap-x-2 right-0 bottom-0 m-2 px-2 py-2 z-50 text-right bg-red-700/90 rounded-lg"
+  >
+    <div class="flex flex-col">
+      <span class="text-sm text-zinc-200 font-bold font-display">{{
+        $t("errors.externalUrl.title")
+      }}</span>
+      <span class="text-xs text-red-400">{{
+        $t("errors.externalUrl.subtitle")
+      }}</span>
+    </div>
+    <button class="text-red-200" @click="() => hideExternalURL()">
+      <XMarkIcon class="size-5" />
+    </button>
+  </div>
 </template>
 
 <script setup lang="ts">
+import { XMarkIcon } from "@heroicons/vue/24/outline";
+
 await updateUser();
+
+const user = useUser();
+const apiDetails = await $dropFetch("/api/v1");
+
+const showExternalUrlWarning = ref(false);
+function checkExternalUrl() {
+  if (!import.meta.client) return;
+  const realOrigin = window.location.origin.trim();
+  const chosenOrigin = apiDetails.external.trim();
+  const ignore = window.localStorage.getItem("ignoreExternalUrl");
+  if (ignore && ignore == "true") return;
+  showExternalUrlWarning.value = !(realOrigin == chosenOrigin);
+}
+
+function hideExternalURL() {
+  window.localStorage.setItem("ignoreExternalUrl", "true");
+  showExternalUrlWarning.value = false;
+}
+
+if (user.value?.admin) {
+  onMounted(() => {
+    checkExternalUrl();
+  });
+}
 </script>
 
 <style scoped>

components/Modal/CreateCompany.vue

@@ -0,0 +1,148 @@
+<template>
+  <ModalTemplate v-model="open">
+    <template #default>
+      <div>
+        <h3 class="text-lg font-medium leading-6 text-white">
+          {{ $t("library.admin.metadata.companies.modals.createTitle") }}
+        </h3>
+        <p class="mt-1 text-zinc-400 text-sm">
+          {{ $t("library.admin.metadata.companies.modals.createDescription") }}
+        </p>
+      </div>
+      <div class="mt-2">
+        <form class="space-y-4" @submit.prevent="() => createCompany()">
+          <div>
+            <label
+              for="name"
+              class="block text-sm/6 font-medium text-zinc-100"
+              >{{
+                $t("library.admin.metadata.companies.modals.createFieldName")
+              }}</label
+            >
+            <div class="mt-2">
+              <input
+                id="name"
+                v-model="companyName"
+                type="text"
+                name="name"
+                :placeholder="
+                  $t(
+                    'library.admin.metadata.companies.modals.createFieldNamePlaceholder',
+                  )
+                "
+                class="block w-full rounded-md bg-zinc-800 px-3 py-1.5 text-base text-zinc-100 outline outline-1 -outline-offset-1 outline-zinc-700 placeholder:text-zinc-400 focus:outline focus:outline-2 focus:-outline-offset-2 focus:outline-blue-600 sm:text-sm/6"
+              />
+            </div>
+          </div>
+          <div>
+            <label
+              for="description"
+              class="block text-sm/6 font-medium text-zinc-100"
+              >{{
+                $t(
+                  "library.admin.metadata.companies.modals.createFieldDescription",
+                )
+              }}</label
+            >
+            <div class="mt-2">
+              <input
+                id="description"
+                v-model="companyDescription"
+                type="text"
+                name="description"
+                :placeholder="
+                  $t(
+                    'library.admin.metadata.companies.modals.createFieldDescriptionPlaceholder',
+                  )
+                "
+                class="block w-full rounded-md bg-zinc-800 px-3 py-1.5 text-base text-zinc-100 outline outline-1 -outline-offset-1 outline-zinc-700 placeholder:text-zinc-400 focus:outline focus:outline-2 focus:-outline-offset-2 focus:outline-blue-600 sm:text-sm/6"
+              />
+            </div>
+          </div>
+          <div>
+            <label
+              for="website"
+              class="block text-sm/6 font-medium text-zinc-100"
+              >{{
+                $t("library.admin.metadata.companies.modals.createFieldWebsite")
+              }}</label
+            >
+            <div class="mt-2">
+              <input
+                id="website"
+                v-model="companyWebsite"
+                type="text"
+                name="website"
+                :placeholder="
+                  $t(
+                    'library.admin.metadata.companies.modals.createFieldWebsitePlaceholder',
+                  )
+                "
+                class="block w-full rounded-md bg-zinc-800 px-3 py-1.5 text-base text-zinc-100 outline outline-1 -outline-offset-1 outline-zinc-700 placeholder:text-zinc-400 focus:outline focus:outline-2 focus:-outline-offset-2 focus:outline-blue-600 sm:text-sm/6"
+              />
+            </div>
+          </div>
+
+          <button class="hidden" type="submit" />
+        </form>
+      </div>
+    </template>
+
+    <template #buttons="{ close }">
+      <LoadingButton
+        :loading="loading"
+        :disabled="!companyValid"
+        class="w-full sm:w-fit"
+        @click="() => createCompany()"
+      >
+        {{ $t("common.create") }}
+      </LoadingButton>
+      <button
+        ref="cancelButtonRef"
+        type="button"
+        class="mt-3 inline-flex w-full justify-center rounded-md bg-zinc-800 px-3 py-2 text-sm font-semibold text-zinc-100 shadow-sm ring-1 ring-inset ring-zinc-800 hover:bg-zinc-900 sm:mt-0 sm:w-auto"
+        @click="() => close()"
+      >
+        {{ $t("cancel") }}
+      </button>
+    </template>
+  </ModalTemplate>
+</template>
+
+<script setup lang="ts">
+import type { CompanyModel } from "~/prisma/client/models";
+
+const open = defineModel<boolean>({ required: true });
+
+const emit = defineEmits<{
+  created: [company: CompanyModel];
+}>();
+
+const companyName = ref("");
+const companyDescription = ref("");
+const companyWebsite = ref("");
+
+const loading = ref(false);
+const companyValid = computed(
+  () => companyName.value && companyDescription.value,
+);
+async function createCompany() {
+  loading.value = true;
+  try {
+    const newCompany = await $dropFetch("/api/v1/admin/company", {
+      method: "POST",
+      body: {
+        name: companyName.value,
+        description: companyDescription.value,
+        website: companyWebsite.value,
+      },
+      failTitle: "Failed to create new company",
+    });
+    open.value = false;
+    emit("created", newCompany);
+  } finally {
+    /* empty */
+  }
+  loading.value = false;
+}
+</script>

components/StoreView.vue

@@ -292,7 +292,7 @@
             <div
               v-if="games?.length ?? 0 > 0"
               ref="product-grid"
-              class="relative lg:col-span-4 grid grid-cols-2 sm:grid-cols-3 md:grid-cols-4 lg:grid-cols-4 xl:grid-cols-5 2xl:grid-cols-6 gap-4"
+              class="relative lg:col-span-4 grid grid-cols-2 sm:grid-cols-3 md:grid-cols-4 lg:grid-cols-4 xl:grid-cols-6 2xl:grid-cols-7 gap-4"
             >
               <!-- Your content -->
               <GamePanel

i18n/locales/en_us.json

@@ -212,6 +212,10 @@
         "desc": "Drop encountered an error while updating the version: {error}",
         "title": "There an error while updating the version order"
       }
+    },
+    "externalUrl": {
+      "title": "Accessing over different EXTERNAL_URL. Please check the docs.",
+      "subtitle": "This message is only visible to admins."
     }
   },
   "footer": {
@@ -329,7 +333,9 @@
             "noDescription": "(no description)",
             "published": "Published",
             "uploadBanner": "Upload banner",
-            "uploadIcon": "Upload icon"
+            "uploadIcon": "Upload icon",
+            "descriptionPlaceholder": "{'<'}description{'>'}",
+            "websitePlaceholder": "{'<'}website{'>'}"
           },
           "modals": {
             "nameDescription": "Edit the company's name. Used to match to new game imports.",
@@ -337,7 +343,16 @@
             "shortDeckDescription": "Edit the company's description. Doesn't affect long (markdown) description.",
             "shortDeckTitle": "Edit company description",
             "websiteDescription": "Edit the company's website. Note: this will be a link, and won't have redirect protection.",
-            "websiteTitle": "Edit company website"
+            "websiteTitle": "Edit company website",
+
+            "createTitle": "Create a company",
+            "createDescription": "Create a company to further organize your games.",
+            "createFieldName": "Company Name",
+            "createFieldNamePlaceholder": "My New Company...",
+            "createFieldDescription": "Company Description",
+            "createFieldDescriptionPlaceholder": "A small indie studio that...",
+            "createFieldWebsite": "Company Website",
+            "createFieldWebsitePlaceholder": "https://example.com/"
           },
           "noCompanies": "No companies",
           "noGames": "No games",
@@ -499,7 +514,9 @@
     "images": "Game Images",
     "lookAt": "Check it out",
     "noDevelopers": "No developers",
-    "noGame": "no game",
+    "noGame": "NO GAME",
+    "noFeatured": "NO FEATURED GAMES",
+    "openFeatured": "Star games in Admin Library {arrow}",
     "noImages": "No images",
     "noPublishers": "No publishers.",
     "noTags": "No tags",

nuxt.config.ts

@@ -138,6 +138,7 @@ export default defineNuxtConfig({
 
     scheduledTasks: {
       "0 * * * *": ["dailyTasks"],
+      "*/30 * * * *": ["downloadCleanup"],
     },
 
     storage: {

pages/admin/metadata/companies/[id]/index.vue

@@ -30,7 +30,7 @@
             {{ company.mName }}
             <button @click="() => editName()">
               <PencilIcon
-                class="transition duration-200 opacity-0 group-hover/name:opacity-100 size-8"
+                class="transition duration-200 xl:opacity-0 group-hover/name:opacity-100 size-8"
               />
             </button>
           </h1>
@@ -43,17 +43,20 @@
             }}
             <button @click="() => editShortDescription()">
               <PencilIcon
-                class="transition duration-200 opacity-0 group-hover/description:opacity-100 size-5"
+                class="transition duration-200 xl:opacity-0 group-hover/description:opacity-100 size-5"
               />
             </button>
           </p>
           <p
             class="group/website mt-1 text-zinc-500 inline-flex items-center gap-x-3"
           >
-            {{ company.mWebsite }}
+            {{
+              company.mWebsite ||
+              $t("library.admin.metadata.companies.editor.websitePlaceholder")
+            }}
             <button @click="() => editWebsite()">
               <PencilIcon
-                class="transition duration-200 opacity-0 group-hover/website:opacity-100 size-4"
+                class="transition duration-200 xl:opacity-0 group-hover/website:opacity-100 size-4"
               />
             </button>
           </p>

pages/admin/metadata/companies/index.vue

@@ -10,20 +10,12 @@
         </p>
       </div>
       <div class="mt-4 sm:ml-16 sm:mt-0 sm:flex-none">
-        <NuxtLink
-          to="/admin/library/sources"
+        <button
           class="block rounded-md bg-blue-600 px-3 py-2 text-center text-sm font-semibold text-white shadow-sm transition-all duration-200 hover:bg-blue-500 hover:scale-105 hover:shadow-lg hover:shadow-blue-500/25 active:scale-95 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-blue-600"
+          @click="() => (createCompanyOpen = true)"
         >
-          <i18n-t
-            keypath="library.admin.sources.link"
-            tag="span"
-            scope="global"
-          >
-            <template #arrow>
-              <span aria-hidden="true">{{ $t("chars.arrow") }}</span>
-            </template>
-          </i18n-t>
-        </NuxtLink>
+          {{ $t("common.create") }}
+        </button>
       </div>
     </div>
     <div class="mt-2 grid grid-cols-1">
@@ -105,6 +97,10 @@
         {{ $t("library.admin.metadata.companies.noCompanies") }}
       </p>
     </ul>
+    <ModalCreateCompany
+      v-model="createCompanyOpen"
+      @created="(company) => createCompany(company)"
+    />
   </div>
 </template>
 
@@ -122,9 +118,12 @@ useHead({
   title: t("library.admin.metadata.companies.title"),
 });
 
+const createCompanyOpen = ref(false);
+
 const searchQuery = ref("");
 
-const companies = ref(await $dropFetch("/api/v1/admin/company"));
+const rawCompanies = await $dropFetch("/api/v1/admin/company");
+const companies = ref(rawCompanies);
 
 const filteredCompanies = computed(() =>
   companies.value.filter((e: CompanyModel) => {
@@ -147,4 +146,8 @@ async function deleteCompany(id: string) {
   const index = companies.value.findIndex((e) => e.id === id);
   companies.value.splice(index, 1);
 }
+
+function createCompany(company: (typeof companies.value)[number]) {
+  companies.value.push(company);
+}
 </script>

pages/store/index.vue

@@ -59,13 +59,30 @@
     </VueCarousel>
     <div
       v-else
-      class="w-full h-full flex items-center justify-center bg-zinc-950/50 px-6 py-32 sm:px-12 sm:py-40 lg:px-16"
+      class="w-full h-full flex flex-col items-center justify-center bg-zinc-950/50 px-6 py-32 sm:px-12 sm:py-40 lg:px-16 gap-4"
     >
       <h2
         class="uppercase text-xl font-bold tracking-tight text-zinc-700 sm:text-3xl"
       >
-        {{ $t("store.noGame") }}
+        {{ $t("store.noFeatured") }}
       </h2>
+      <NuxtLink
+        v-if="user?.admin"
+        to="/admin/library"
+        type="button"
+        class="inline-flex items-center gap-x-2 rounded-md bg-zinc-800 px-3 py-1 text-sm font-semibold font-display text-white shadow-sm hover:bg-zinc-700 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-blue-600 duration-200 hover:scale-105 active:scale-95"
+      >
+        <i18n-t
+          keypath="store.openFeatured"
+          tag="span"
+          scope="global"
+          class="inline-flex items-center gap-x-1"
+        >
+          <template #arrow>
+            <ArrowTopRightOnSquareIcon class="size-4" />
+          </template>
+        </i18n-t>
+      </NuxtLink>
     </div>
 
     <StoreView />
@@ -73,8 +90,12 @@
 </template>
 
 <script setup lang="ts">
+import { ArrowTopRightOnSquareIcon } from "@heroicons/vue/24/outline";
+
 const recent = await $dropFetch("/api/v1/store/featured");
 
+const user = useUser();
+
 const { t } = useI18n();
 
 useHead({

server/api/v1/admin/company/index.post.ts

@@ -0,0 +1,47 @@
+import { type } from "arktype";
+import { readDropValidatedBody, throwingArktype } from "~/server/arktype";
+import aclManager from "~/server/internal/acls";
+import * as jdenticon from "jdenticon";
+import { ObjectTransactionalHandler } from "~/server/internal/objects/transactional";
+import prisma from "~/server/internal/db/database";
+import { MetadataSource } from "~/prisma/client/enums";
+
+const CompanyCreate = type({
+  name: "string",
+  description: "string",
+  website: "string",
+}).configure(throwingArktype);
+
+export default defineEventHandler(async (h3) => {
+  const allowed = await aclManager.allowSystemACL(h3, ["company:create"]);
+  if (!allowed) throw createError({ statusCode: 403 });
+
+  const body = await readDropValidatedBody(h3, CompanyCreate);
+  const obj = new ObjectTransactionalHandler();
+  const [register, pull, _] = obj.new({}, ["internal:read"]);
+
+  const icon = jdenticon.toPng(body.name, 512);
+  const logoId = register(icon);
+
+  const banner = jdenticon.toPng(body.description, 1024);
+  const bannerId = register(banner);
+
+  const company = await prisma.company.create({
+    data: {
+      metadataSource: MetadataSource.Manual,
+      metadataId: crypto.randomUUID(),
+      metadataOriginalQuery: "",
+
+      mName: body.name,
+      mShortDescription: body.description,
+      mDescription: "",
+      mLogoObjectId: logoId,
+      mBannerObjectId: bannerId,
+      mWebsite: body.website,
+    },
+  });
+
+  await pull();
+
+  return company;
+});

server/api/v1/index.get.ts

@@ -5,5 +5,6 @@ export default defineEventHandler((_h3) => {
     appName: "Drop",
     version: systemConfig.getDropVersion(),
     gitRef: `#${systemConfig.getGitRef()}`,
+    external: systemConfig.getExternalUrl(),
   };
 });

server/api/v2/client/chunk.post.ts

@@ -0,0 +1,73 @@
+import { type } from "arktype";
+import { readDropValidatedBody, throwingArktype } from "~/server/arktype";
+import contextManager from "~/server/internal/downloads/coordinator";
+import libraryManager from "~/server/internal/library";
+
+const GetChunk = type({
+  context: "string",
+  files: type({
+    filename: "string",
+    chunkIndex: "number",
+  }).array(),
+}).configure(throwingArktype);
+
+export default defineEventHandler(async (h3) => {
+  const body = await readDropValidatedBody(h3, GetChunk);
+
+  const context = await contextManager.fetchContext(body.context);
+  if (!context)
+    throw createError({
+      statusCode: 400,
+      statusMessage: "Invalid download context.",
+    });
+
+  const streamFiles = [];
+
+  for (const file of body.files) {
+    const manifestFile = context.manifest[file.filename];
+    if (!manifestFile)
+      throw createError({
+        statusCode: 400,
+        statusMessage: `Unknown file: ${file.filename}`,
+      });
+
+    const start = manifestFile.lengths
+      .slice(0, file.chunkIndex)
+      .reduce((a, b) => a + b, 0);
+    const end = start + manifestFile.lengths[file.chunkIndex];
+
+    streamFiles.push({ filename: file.filename, start, end });
+  }
+
+  setHeader(
+    h3,
+    "Content-Lengths",
+    streamFiles.map((e) => e.end - e.start).join(","),
+  ); // Non-standard header, but we're cool like that 😎
+
+  for (const file of streamFiles) {
+    const gameReadStream = await libraryManager.readFile(
+      context.libraryId,
+      context.libraryPath,
+      context.versionName,
+      file.filename,
+      { start: file.start, end: file.end },
+    );
+    if (!gameReadStream)
+      throw createError({
+        statusCode: 500,
+        statusMessage: "Failed to create read stream",
+      });
+    await gameReadStream.pipeTo(
+      new WritableStream({
+        write(chunk) {
+          h3.node.res.write(chunk);
+        },
+      }),
+    );
+  }
+
+  await h3.node.res.end();
+
+  return;
+});

server/api/v2/client/context.post.ts

@@ -0,0 +1,22 @@
+import { type } from "arktype";
+import { readDropValidatedBody, throwingArktype } from "~/server/arktype";
+import { defineClientEventHandler } from "~/server/internal/clients/event-handler";
+import contextManager from "~/server/internal/downloads/coordinator";
+
+const CreateContext = type({
+  game: "string",
+  version: "string",
+}).configure(throwingArktype);
+
+export default defineClientEventHandler(async (h3) => {
+  const body = await readDropValidatedBody(h3, CreateContext);
+
+  const context = await contextManager.createContext(body.game, body.version);
+  if (!context)
+    throw createError({
+      statusCode: 400,
+      statusMessage: "Invalid game or version",
+    });
+
+  return { context };
+});

server/internal/downloads/coordinator.ts

@@ -1,9 +1,68 @@
-/*
-The download co-ordinator's job is to keep track of all the currently online clients.
+import prisma from "../db/database";
+import type { DropManifest } from "./manifest";
 
-When a client signs on and registers itself as a peer
+const TIMEOUT = 1000 * 60 * 60 * 1; // 1 hour
 
-*/
+class DownloadContextManager {
+  private contexts: Map<
+    string,
+    {
+      timeout: Date;
+      manifest: DropManifest;
+      versionName: string;
+      libraryId: string;
+      libraryPath: string;
+    }
+  > = new Map();
 
-// eslint-disable-next-line @typescript-eslint/no-extraneous-class, @typescript-eslint/no-unused-vars
-class DownloadCoordinator {}
+  async createContext(game: string, versionName: string) {
+    const version = await prisma.gameVersion.findUnique({
+      where: {
+        gameId_versionName: {
+          gameId: game,
+          versionName,
+        },
+      },
+      include: {
+        game: {
+          select: {
+            libraryId: true,
+            libraryPath: true,
+          },
+        },
+      },
+    });
+    if (!version) return undefined;
+
+    const contextId = crypto.randomUUID();
+    this.contexts.set(contextId, {
+      timeout: new Date(),
+      manifest: JSON.parse(version.dropletManifest as string) as DropManifest,
+      versionName,
+      libraryId: version.game.libraryId!,
+      libraryPath: version.game.libraryPath,
+    });
+
+    return contextId;
+  }
+
+  async fetchContext(contextId: string) {
+    const context = this.contexts.get(contextId);
+    if (!context) return undefined;
+    context.timeout = new Date();
+    this.contexts.set(contextId, context);
+    return context;
+  }
+
+  async cleanup() {
+    for (const key of this.contexts.keys()) {
+      const context = this.contexts.get(key)!;
+      if (context.timeout.getDate() + TIMEOUT < Date.now()) {
+        this.contexts.delete(key);
+      }
+    }
+  }
+}
+
+export const contextManager = new DownloadContextManager();
+export default contextManager;

server/internal/downloads/manifest.ts

@@ -5,7 +5,7 @@ export type DropChunk = {
   permissions: number;
   ids: string[];
   checksums: string[];
-  lengths: string[];
+  lengths: number[];
 };
 
 export type DropManifest = {

server/internal/library/index.ts

@@ -13,6 +13,7 @@ import { parsePlatform } from "../utils/parseplatform";
 import notificationSystem from "../notifications";
 import { GameNotFoundError, type LibraryProvider } from "./provider";
 import { logger } from "../logging";
+import type { GameModel } from "~/prisma/client/models";
 
 class LibraryManager {
   private libraries: Map<string, LibraryProvider<unknown>> = new Map();
@@ -37,24 +38,32 @@ class LibraryManager {
     return libraryWithMetadata;
   }
 
+  async fetchGamesByLibrary() {
+    const results: { [key: string]: { [key: string]: GameModel } } = {};
+    const games = await prisma.game.findMany({});
+    for (const game of games) {
+      const libraryId = game.libraryId!;
+      const libraryPath = game.libraryPath!;
+
+      results[libraryId] ??= {};
+      results[libraryId][libraryPath] = game;
+    }
+
+    return results;
+  }
+
   async fetchUnimportedGames() {
     const unimportedGames: { [key: string]: string[] } = {};
+    const instanceGames = await this.fetchGamesByLibrary();
 
     for (const [id, library] of this.libraries.entries()) {
-      const games = await library.listGames();
-      const validGames = await prisma.game.findMany({
-        where: {
-          libraryId: id,
-          libraryPath: { in: games },
-        },
-        select: {
-          libraryPath: true,
-        },
-      });
-      const providerUnimportedGames = games.filter(
-        (e) =>
-          validGames.findIndex((v) => v.libraryPath == e) == -1 &&
-          !(this.gameImportLocks.get(id) ?? []).includes(e),
+      const providerGames = await library.listGames();
+      const locks = this.gameImportLocks.get(id) ?? [];
+      const providerUnimportedGames = providerGames.filter(
+        (libraryPath) =>
+          instanceGames[id] &&
+          !instanceGames[id][libraryPath] &&
+          !locks.includes(libraryPath),
       );
       unimportedGames[id] = providerUnimportedGames;
     }

server/middleware/latency.ts

@@ -0,0 +1,3 @@
+export default defineEventHandler(async () => {
+  // await new Promise((r) => setTimeout(r, 700));
+});

server/tasks/downloadCleanup.ts

@@ -0,0 +1,11 @@
+import contextManager from "../internal/downloads/coordinator";
+
+export default defineTask({
+  meta: {
+    name: "downloadCleanup",
+  },
+  async run() {
+    await contextManager.cleanup();
+    return { result: true };
+  },
+});