Check integrity task (#364)

* fix: validation and issuer checks

* feat: query param util

* fix: lint

components/GameEditor/Metadata.vue

@@ -30,7 +30,11 @@
         </div>
 
         <div class="grid grid-cols-1 lg:grid-cols-3 gap-4 pt-8">
-          <SelectorMultiItem v-model="currentTags" :items="tags" />
+          <SelectorMultiItem
+            v-model="currentTags"
+            :items="tags"
+            :create="createTag"
+          />
           <div class="flex flex-col">
             <label
               for="releaseDate"
@@ -493,8 +497,9 @@ if (!game.value)
 const currentTags = ref<{ [key: string]: boolean }>(
   Object.fromEntries(game.value.tags.map((e) => [e.id, true])),
 );
-const tags = (await $dropFetch("/api/v1/admin/tags")).map(
-  (e) => ({ name: e.name, param: e.id }) satisfies StoreSortOption,
+const rawTags = await $dropFetch("/api/v1/admin/tags");
+const tags = ref(
+  rawTags.map((e) => ({ name: e.name, param: e.id }) satisfies StoreSortOption),
 );
 
 watch(
@@ -505,7 +510,11 @@ watch(
       params: {
         id: game.value.id,
       },
-      body: { tags: Object.keys(v) },
+      body: {
+        tags: Object.entries(v)
+          .filter((v) => v[1])
+          .map((v) => v[0]),
+      },
       failTitle: "Failed to update game tags",
     });
   },
@@ -816,4 +825,15 @@ async function updateImageCarousel() {
     );
   }
 }
+
+async function createTag(value: string): Promise<string> {
+  const tag = await $dropFetch(`/api/v1/admin/tags`, {
+    method: "POST",
+    body: {
+      name: value,
+    },
+  });
+  tags.value.push({ name: tag.name, param: tag.id });
+  return tag.id;
+}
 </script>

components/Selector/MultiItem.vue

@@ -34,7 +34,7 @@
         <ComboboxInput
           class="block w-full rounded-md bg-zinc-900 py-1.5 pr-12 pl-3 text-base text-zinc-100 outline-1 -outline-offset-1 outline-zinc-700 placeholder:text-zinc-500 focus:outline-2 focus:-outline-offset-2 focus:outline-blue-600 sm:text-sm/6"
           :display-value="(item) => (item as StoreSortOption)?.name"
-          placeholder="Start typing..."
+          :placeholder="$t('common.components.multiitem.placeholder')"
           @change="search = $event.target.value"
           @blur="search = ''"
         />
@@ -68,7 +68,51 @@
               </span>
             </li>
           </ComboboxOption>
+          <ComboboxOption
+            v-if="$props.create"
+            v-slot="{ active }"
+            :value="CREATE_PREFIX + search"
+            as="template"
+          >
+            <li
+              :class="[
+                'relative cursor-default py-2 pr-9 pl-3 select-none',
+                active
+                  ? 'bg-blue-600 text-white outline-hidden'
+                  : 'text-zinc-100',
+              ]"
+            >
+              <span class="block truncate">
+                {{ $t("common.components.multiitem.new", [search]) }}
+              </span>
+            </li>
+          </ComboboxOption>
         </ComboboxOptions>
+
+        <div
+          v-if="createLoading"
+          class="absolute inset-0 bg-zinc-950 flex items-center justify-center"
+        >
+          <div role="status">
+            <svg
+              aria-hidden="true"
+              class="size-8 text-transparent animate-spin fill-white"
+              viewBox="0 0 100 101"
+              fill="none"
+              xmlns="http://www.w3.org/2000/svg"
+            >
+              <path
+                d="M100 50.5908C100 78.2051 77.6142 100.591 50 100.591C22.3858 100.591 0 78.2051 0 50.5908C0 22.9766 22.3858 0.59082 50 0.59082C77.6142 0.59082 100 22.9766 100 50.5908ZM9.08144 50.5908C9.08144 73.1895 27.4013 91.5094 50 91.5094C72.5987 91.5094 90.9186 73.1895 90.9186 50.5908C90.9186 27.9921 72.5987 9.67226 50 9.67226C27.4013 9.67226 9.08144 27.9921 9.08144 50.5908Z"
+                fill="currentColor"
+              />
+              <path
+                d="M93.9676 39.0409C96.393 38.4038 97.8624 35.9116 97.0079 33.5539C95.2932 28.8227 92.871 24.3692 89.8167 20.348C85.8452 15.1192 80.8826 10.7238 75.2124 7.41289C69.5422 4.10194 63.2754 1.94025 56.7698 1.05124C51.7666 0.367541 46.6976 0.446843 41.7345 1.27873C39.2613 1.69328 37.813 4.19778 38.4501 6.62326C39.0873 9.04874 41.5694 10.4717 44.0505 10.1071C47.8511 9.54855 51.7191 9.52689 55.5402 10.0491C60.8642 10.7766 65.9928 12.5457 70.6331 15.2552C75.2735 17.9648 79.3347 21.5619 82.5849 25.841C84.9175 28.9121 86.7997 32.2913 88.1811 35.8758C89.083 38.2158 91.5421 39.6781 93.9676 39.0409Z"
+                fill="currentFill"
+              />
+            </svg>
+            <span class="sr-only">{{ $t("common.srLoading") }}</span>
+          </div>
+        </div>
       </div>
     </Combobox>
   </div>
@@ -85,6 +129,7 @@ import {
 } from "@headlessui/vue";
 const props = defineProps<{
   items: Array<StoreSortOption>;
+  create?: (value: string) => Promise<string>;
 }>();
 
 const model = defineModel<{ [key: string]: boolean }>();
@@ -102,7 +147,37 @@ const enabledItems = computed(() =>
   props.items.filter((e) => model.value?.[e.param]),
 );
 
+// I do not love how this works, but it's okay for now
+const CREATE_PREFIX = "CREATE";
+
+const createLoading = ref(false);
 function add(item: string) {
+  if (item.startsWith(CREATE_PREFIX)) {
+    if (!props.create) return;
+    const value = item.substring(CREATE_PREFIX.length);
+    createLoading.value = true;
+    props
+      .create(value)
+      .then(
+        (result) => {
+          add(result);
+        },
+        (err) => {
+          createModal(
+            ModalType.Notification,
+            {
+              title: "Failed to create value",
+              description: err,
+            },
+            (_, c) => c(),
+          );
+        },
+      )
+      .finally(() => {
+        createLoading.value = false;
+      });
+    return;
+  }
   search.value = "";
   model.value ??= {};
   model.value[item] = true;

i18n/locales/en_us.json

@@ -160,6 +160,12 @@
     "add": "Add",
     "cannotUndo": "This action cannot be undone.",
     "close": "Close",
+    "components": {
+      "multiitem": {
+        "new": "Create new: \"{0}\"",
+        "placeholder": "Start typing..."
+      }
+    },
     "create": "Create",
     "date": "Date",
     "delete": "Delete",
@@ -749,6 +755,7 @@
         "cleanupSessionsDescription": "Cleans up expired sessions to save space and ensure security.",
         "cleanupSessionsName": "Clean up sessions."
       },
+      "utilityTitle": "Utility tasks",
       "viewTask": "View {arrow}",
       "weeklyScheduledTitle": "Weekly scheduled tasks"
     }

pages/admin/task/index.vue

@@ -166,6 +166,44 @@
             </div>
           </li>
         </ul>
+        <h2 class="text-sm font-medium text-zinc-400 mt-8">
+          {{ $t("tasks.admin.utilityTitle") }}
+        </h2>
+        <ul role="list" class="mt-4 grid grid-cols-1 lg:grid-cols-2 gap-6">
+          <li
+            v-for="task in other"
+            :key="task"
+            class="col-span-1 divide-y divide-gray-200 rounded-lg bg-zinc-800 border border-zinc-700 shadow-sm"
+          >
+            <div class="flex w-full items-center justify-between space-x-6 p-6">
+              <div class="flex-1">
+                <div class="flex items-center space-x-2">
+                  <h3 class="text-sm font-medium text-zinc-100">
+                    {{ scheduledTasks[task].name }}
+                  </h3>
+                </div>
+                <p class="mt-1 text-sm text-zinc-400">
+                  {{ scheduledTasks[task].description }}
+                </p>
+                <button
+                  class="mt-3 rounded-md text-xs font-medium text-zinc-100 hover:text-zinc-300 focus:outline-none focus:ring-2 focus:ring-zinc-100 focus:ring-offset-2"
+                  @click="() => startTask(task)"
+                >
+                  <i18n-t
+                    keypath="tasks.admin.execute"
+                    tag="span"
+                    scope="global"
+                    class="inline-flex items-center gap-x-1"
+                  >
+                    <template #arrow>
+                      <PlayIcon class="size-4" aria-hidden="true" />
+                    </template>
+                  </i18n-t>
+                </button>
+              </div>
+            </div>
+          </li>
+        </ul>
       </div>
     </div>
   </div>
@@ -185,7 +223,7 @@ definePageMeta({
 
 const { t } = useI18n();
 
-const { runningTasks, historicalTasks, dailyTasks, weeklyTasks } =
+const { runningTasks, historicalTasks, dailyTasks, weeklyTasks, other } =
   await $dropFetch("/api/v1/admin/task");
 
 const liveRunningTasks = ref(
@@ -219,9 +257,9 @@ const scheduledTasks: {
     name: "",
     description: "",
   },
-  debug: {
-    name: "",
-    description: "",
+  "import:check-integrity": {
+    name: "Check Integrity",
+    description: "Re-imports all versions and updates their manifests.",
   },
 };
 

server/api/v1/admin/game/[id]/tags.patch.ts

@@ -16,10 +16,19 @@ export default defineEventHandler(async (h3) => {
 
   const game = await prisma.game.findUnique({
     where: { id },
-    select: { id: true },
+    select: { id: true, tags: { select: { id: true } } },
   });
   if (!game) throw createError({ statusCode: 404, message: "Game not found" });
 
+  const tagSet = new Set(game.tags.map((v) => v.id));
+  const toConnect = body.tags.filter((v) => !tagSet.has(v));
+
+  const bodyTagSet = new Set(body.tags);
+  const toDisconnect = tagSet
+    .values()
+    .filter((v) => !bodyTagSet.has(v))
+    .toArray();
+
   // SAFETY: Okay to disable due to check above
   // eslint-disable-next-line drop/no-prisma-delete
   await prisma.game.update({
@@ -28,7 +37,8 @@ export default defineEventHandler(async (h3) => {
     },
     data: {
       tags: {
-        connect: body.tags.map((e) => ({ id: e })),
+        connect: toConnect.map((e) => ({ id: e })),
+        disconnect: toDisconnect.map((e) => ({ id: e })),
       },
     },
   });

server/api/v1/admin/task/index.get.ts

@@ -1,6 +1,7 @@
 import aclManager from "~/server/internal/acls";
 import prisma from "~/server/internal/db/database";
 import taskHandler from "~/server/internal/tasks";
+import type { TaskGroup } from "~/server/internal/tasks/group";
 
 export default defineEventHandler(async (h3) => {
   const allowed = await aclManager.allowSystemACL(h3, ["task:read"]);
@@ -38,6 +39,7 @@ export default defineEventHandler(async (h3) => {
   });
   const dailyTasks = await taskHandler.dailyTasks();
   const weeklyTasks = await taskHandler.weeklyTasks();
+  const other: TaskGroup[] = ["import:check-integrity"];
 
-  return { runningTasks, historicalTasks, dailyTasks, weeklyTasks };
+  return { runningTasks, historicalTasks, dailyTasks, weeklyTasks, other };
 });

server/api/v1/store/index.get.ts

@@ -53,24 +53,34 @@ export default defineEventHandler(async (h3) => {
     : undefined;
   const platformFilter = filterPlatforms
     ? ({
-        versions: {
-          some: {
-            launches: {
+        OR: [
+          {
+            versions: {
               some: {
-                platform: {
-                  in: filterPlatforms,
-                },
-              },
-            },
-            setups: {
-              some: {
-                platform: {
-                  in: filterPlatforms,
+                setups: {
+                  some: {
+                    platform: {
+                      in: filterPlatforms,
+                    },
+                  },
                 },
               },
             },
           },
-        },
+          {
+            versions: {
+              some: {
+                launches: {
+                  some: {
+                    platform: {
+                      in: filterPlatforms,
+                    },
+                  },
+                },
+              },
+            },
+          },
+        ],
       } satisfies Prisma.GameWhereInput)
     : undefined;
 

server/internal/auth/index.ts

@@ -30,7 +30,9 @@ class AuthManager {
         (this.authProviders as any)[key] = object;
         logger.info(`enabled auth: ${key}`);
       } catch (e) {
-        logger.warn((e as string).toString());
+        logger.warn(
+          `failed to enable auth ${key}: ${(e as string).toString()}`,
+        );
       }
     }
 

server/internal/auth/oidc/index.ts

@@ -12,28 +12,22 @@ import * as jose from "jose";
 // import { inspect } from "util";
 import sessionHandler from "../../session";
 import type { SessionSearchTerms } from "../../session/types";
+import { queryParamBuilder } from "../../utils/query";
 
 // TODO: monitor https://github.com/goauthentik/authentik/issues/8751 for easier?? OIDC setup by end users
 
 // Schema for OIDC well-known configuration
 const OIDCWellKnownV1 = type({
-  issuer: "string.url.parse",
+  issuer: "string",
   authorization_endpoint: "string.url.parse",
   token_endpoint: "string.url.parse",
-  userinfo_endpoint: "string.url.parse?",
+  userinfo_endpoint: "string.url.parse",
   jwks_uri: "string.url.parse",
-  scopes_supported: "string[]?",
+  scopes_supported: "string[]",
 });
 
 // Represents required OIDC configuration
-interface OIDCConfiguration {
-  issuer: URL;
-  authorization_endpoint: URL;
-  token_endpoint: URL;
-  userinfo_endpoint: URL;
-  jwks_uri: URL;
-  scopes_supported: string[];
-}
+type OIDCConfiguration = typeof OIDCWellKnownV1.infer;
 
 interface OIDCAuthSessionOptions {
   redirect: string | undefined;
@@ -153,14 +147,14 @@ export class OIDCManager {
 
     this.JWKS = jose.createRemoteJWKSet(this.oidcConfiguration.jwks_uri);
     this.redirectUrl = new URL(
-      `${this.externalUrl.toString()}api/v1/auth/odic/callback`,
+      `${this.externalUrl.toString()}api/v1/auth/oidc/callback`,
     );
   }
 
   static async create() {
     if (!systemConfig.shouldOidcRequireHttps()) {
       console.warn(
-        "Disabling HTTPS requirement for ODIC provider, not recommened in production enviroments",
+        "Disabling HTTPS requirement for OIDC provider, not recommened in production enviroments",
       );
     }
 
@@ -241,7 +235,7 @@ export class OIDCManager {
         token_endpoint: new URL(tokenEndpoint),
         userinfo_endpoint: new URL(userinfoEndpoint),
         scopes_supported: scopes.split(","),
-        issuer: new URL(issuer),
+        issuer: issuer,
         jwks_uri: new URL(jwksEndpoint),
       };
     }
@@ -294,7 +288,15 @@ export class OIDCManager {
       this.oidcConfiguration.authorization_endpoint,
     ).toString();
 
-    const finalUrl = `${normalisedUrl}?client_id=${this.clientId}&redirect_uri=${encodeURIComponent(this.redirectUrl.toString())}&state=${stateKey}&response_type=code&scope=${encodeURIComponent(this.oidcConfiguration.scopes_supported.join(" "))}`;
+    const queryParams = queryParamBuilder({
+      client_id: this.clientId,
+      redirect_uri: this.redirectUrl.toString(),
+      state: stateKey,
+      response_type: "code",
+      scope: this.oidcConfiguration.scopes_supported.join(" "),
+    });
+
+    const finalUrl = `${normalisedUrl}?${queryParams}`;
 
     const session: OIDCAuthSession = {
       redirectUrl: finalUrl,
@@ -549,7 +551,8 @@ export class OIDCManager {
   }
 }
 
-function isHttps(url: URL): boolean {
-  if (url.protocol === "https:") return true;
+function isHttps(url: URL | string): boolean {
+  const parsedUrl = typeof url === "string" ? new URL(url) : url;
+  if (parsedUrl.protocol === "https:") return true;
   else return false;
 }

server/internal/config/sys-conf.ts

@@ -12,7 +12,7 @@ class SystemConfig {
   );
   private dropVersion: string;
   private gitRef: string;
-  private odicRequireHttps;
+  private oidcRequireHttps;
 
   private checkForUpdates = getUpdateCheckConfig();
 
@@ -22,14 +22,14 @@ class SystemConfig {
     this.dropVersion = config.dropVersion;
     this.gitRef = config.gitRef;
 
-    const odicRequireHttps = process.env.OIDC_REQUIRE_HTTPS as
+    const oidcRequireHttps = process.env.OIDC_REQUIRE_HTTPS as
       | string
       | undefined;
 
     // default to true if not set
-    this.odicRequireHttps =
-      odicRequireHttps !== undefined &&
-      odicRequireHttps.toLocaleLowerCase() === "false"
+    this.oidcRequireHttps =
+      oidcRequireHttps !== undefined &&
+      oidcRequireHttps.toLocaleLowerCase() === "false"
         ? false
         : true;
   }
@@ -64,7 +64,7 @@ class SystemConfig {
 
   // if oidc should require https for endpoints
   shouldOidcRequireHttps() {
-    return this.odicRequireHttps;
+    return this.oidcRequireHttps;
   }
 }
 

server/internal/services/torrential/droplet-interface.ts

@@ -217,7 +217,7 @@ class DropletInterfaceManager {
       run: async (message) => {
         const callbacks = this.callbacks.get(message.messageId);
         if (!callbacks) {
-          logger.warn(
+          logger.debug(
             `got a droplet message with old message id: ${message.type}, ${message.messageId}`,
           );
           return undefined;

server/internal/tasks/group.ts

@@ -17,8 +17,8 @@ export const taskGroups = {
   "import:version": {
     concurrency: true,
   },
-  debug: {
-    concurrency: true,
+  "import:check-integrity": {
+    concurrency: false,
   },
 } as const;
 

server/internal/tasks/index.ts

@@ -1,11 +1,6 @@
 import type { MinimumRequestObject } from "~/server/h3";
 import type { GlobalACL } from "../acls";
 import aclManager from "../acls";
-
-import cleanupInvites from "./registry/invitations";
-import cleanupSessions from "./registry/sessions";
-import checkUpdate from "./registry/update";
-import cleanupObjects from "./registry/objects";
 import { taskGroups, type TaskGroup } from "./group";
 import prisma from "../db/database";
 import { ArkErrors, type } from "arktype";
@@ -13,6 +8,12 @@ import pino from "pino";
 import { logger } from "~/server/internal/logging";
 import { Writable } from "node:stream";
 
+import cleanupInvites from "./registry/invitations";
+import cleanupSessions from "./registry/sessions";
+import checkUpdate from "./registry/update";
+import cleanupObjects from "./registry/objects";
+import checkIntegrity from "./registry/check-integrity";
+
 type TaskActionLink = `${string}:${string}`;
 
 // a task that has been run
@@ -65,7 +66,7 @@ class TaskHandler {
     this.saveScheduledTask(cleanupSessions);
     this.saveScheduledTask(checkUpdate);
     this.saveScheduledTask(cleanupObjects);
-    //this.saveScheduledTask(debug);
+    this.saveScheduledTask(checkIntegrity);
   }
 
   /**

server/internal/tasks/registry/check-integrity.ts

@@ -0,0 +1,80 @@
+import prisma from "~/server/internal/db/database";
+import { defineDropTask, wrapTaskContext } from "..";
+import { libraryManager } from "../../library";
+
+export default defineDropTask({
+  buildId: () => `import:check-integrity:${new Date().toISOString()}`,
+  name: "Check version integrity",
+  acls: ["system:import:version:read"],
+  taskGroup: "import:check-integrity",
+  async run({ progress, logger, addAction }) {
+    const versions = await prisma.gameVersion.findMany({
+      where: {
+        versionPath: {
+          not: null,
+        },
+      },
+      select: {
+        versionId: true,
+        versionPath: true,
+        displayName: true,
+        game: {
+          select: {
+            libraryId: true,
+            libraryPath: true,
+            mName: true,
+          },
+        },
+      },
+    });
+
+    logger.info(`Checking version integrity for ${versions.length} versions`);
+
+    let i = 0;
+    const progressStep = 100 / versions.length;
+    for (const version of versions) {
+      const displayName = `${version.game.mName} ${version.displayName ?? version.versionPath}`;
+      logger.info(`Starting integrity check for ${displayName}`);
+
+      const library = await libraryManager.getLibrary(version.game.libraryId);
+      if (!library) {
+        logger.warn(`No library for ${displayName}`);
+        continue;
+      }
+
+      const min = i * progressStep;
+      const max = (i + 1) * progressStep;
+      const taskContext = wrapTaskContext(
+        { progress, logger, addAction },
+        { min, max, prefix: `re-check ${displayName}` },
+      );
+
+      const manifest = await library.generateDropletManifest(
+        version.game.libraryPath,
+        version.versionPath!,
+        taskContext.progress,
+        (value) => {
+          taskContext.logger.info(value);
+        },
+      );
+
+      // SAFETY: this is requested from the database
+      // eslint-disable-next-line drop/no-prisma-delete
+      await prisma.gameVersion.update({
+        where: {
+          versionId: version.versionId,
+        },
+        data: {
+          versionId: crypto.randomUUID(),
+          dropletManifest: manifest,
+        },
+      });
+
+      logger.info(`Finished integrity check for ${displayName}`);
+      i++;
+    }
+
+    logger.info("Done");
+    progress(100);
+  },
+});

server/internal/tasks/registry/debug.ts

@@ -1,18 +0,0 @@
-import { defineDropTask } from "..";
-
-export default defineDropTask({
-  buildId: () => `debug:${new Date().toISOString()}`,
-  name: "Debug Task",
-  acls: ["system:maintenance:read"],
-  taskGroup: "debug",
-  async run({ progress, logger }) {
-    const amount = 1000;
-    for (let i = 0; i < amount; i++) {
-      progress((i / amount) * 100);
-      logger.info(`dajksdkajd ${i}`);
-      logger.warn("warning");
-      logger.error("error\nmultiline and stuff\nwoah more lines");
-      await new Promise((r) => setTimeout(r, 1500));
-    }
-  },
-});

server/internal/utils/query.ts

@@ -0,0 +1,7 @@
+export function queryParamBuilder(params: { [key: string]: string }) {
+  const list = Object.entries(params).map(
+    ([key, value]) => `${key}=${encodeURIComponent(value)}`,
+  );
+  const str = list.join("&");
+  return str;
+}