i18n Support and Task improvements (#80)

* fix: release workflow

* feat: move mostly to internal tasks system

* feat: migrate object clean to new task system

* fix: release not  getting good base version

* chore: set version v0.3.0

* chore: style

* feat: basic task concurrency

* feat: temp pages to fill in page links

* feat: inital i18n support

* feat: localize store page

* chore: style

* fix: weblate doesn't like multifile thing

* fix: update nuxt

* feat: improved error logging

* fix: using old task api

* feat: basic translation docs

* feat: add i18n eslint plugin

* feat: translate store and auth pages

* feat: more translation progress

* feat: admin dash i18n progress

* feat: enable update check by default in prod

* fix: using wrong i18n keys

* fix: crash in library sources page

* feat: finish i18n work

* fix: missing i18n translations

* feat: use twemoji for emojis

* feat: sanatize object ids

* fix: EmojiText's alt text

* fix: UserWidget not using links

* feat: cache and auth for emoji api

* fix: add more missing translations

server/internal/config/sys-conf.ts

@@ -5,11 +5,7 @@ class SystemConfig {
   private dropVersion;
   private gitRef;
 
-  private checkForUpdates =
-    process.env.CHECK_FOR_UPDATES !== undefined &&
-    process.env.CHECK_FOR_UPDATES.toLocaleLowerCase() === "true"
-      ? true
-      : false;
+  private checkForUpdates = getUpdateCheckConfig();
 
   constructor() {
     // get drop version and git ref from nuxt config
@@ -40,3 +36,26 @@ class SystemConfig {
 }
 
 export const systemConfig = new SystemConfig();
+
+/**
+ * Gets the configuration for checking updates based on various conditions
+ * @returns true if updates should be checked, false otherwise.
+ */
+function getUpdateCheckConfig(): boolean {
+  const envCheckUpdates = process.env.CHECK_FOR_UPDATES;
+
+  // Check environment variable
+  if (envCheckUpdates !== undefined) {
+    // if explicitly set to true or false, return that value
+    if (envCheckUpdates.toLocaleLowerCase() === "true") {
+      return true;
+    } else if (envCheckUpdates.toLocaleLowerCase() === "false") {
+      return false;
+    }
+  } else if (process.env.NODE_ENV === "production") {
+    // default to true in production
+    return true;
+  }
+
+  return false;
+}

server/internal/library/index.ts

@@ -218,6 +218,7 @@ class LibraryManager {
 
     taskHandler.create({
       id: taskId,
+      taskGroup: "import:game",
       name: `Importing version ${versionName} for ${game.mName}`,
       acls: ["system:import:version:read"],
       async run({ progress, log }) {

server/internal/tasks/group.ts

@@ -0,0 +1,19 @@
+export const taskGroups = {
+  "cleanup:invitations": {
+    concurrency: false,
+  },
+  "cleanup:objects": {
+    concurrency: false,
+  },
+  "cleanup:sessions": {
+    concurrency: false,
+  },
+  "check:update": {
+    concurrency: false,
+  },
+  "import:game": {
+    concurrency: true,
+  },
+} as const;
+
+export type TaskGroup = keyof typeof taskGroups;

server/internal/tasks/index.ts

@@ -2,34 +2,86 @@ import droplet from "@drop-oss/droplet";
 import type { MinimumRequestObject } from "~/server/h3";
 import aclManager from "../acls";
 
+import cleanupInvites from "./registry/invitations";
+import cleanupSessions from "./registry/sessions";
+import checkUpdate from "./registry/update";
+import cleanupObjects from "./registry/objects";
+import { taskGroups, type TaskGroup } from "./group";
+
+// a task that has been run
+type FinishedTask = {
+  success: boolean;
+  progress: number;
+  log: string[];
+  error: { title: string; description: string } | undefined;
+  name: string;
+  taskGroup: TaskGroup;
+  acls: string[];
+
+  // ISO timestamp of when the task started
+  startTime: string;
+  // ISO timestamp of when the task ended
+  endTime: string | undefined;
+};
+
+// a currently running task in the pool
+type TaskPoolEntry = FinishedTask & {
+  clients: Map<string, boolean>;
+};
+
 /**
  * The TaskHandler setups up two-way connections to web clients and manages the state for them
  * This allows long-running tasks (like game imports and such) to report progress, success and error states
  * easily without re-inventing the wheel every time.
  */
-
-type TaskRegistryEntry = {
-  success: boolean;
-  progress: number;
-  log: string[];
-  error: { title: string; description: string } | undefined;
-  clients: Map<string, boolean>;
-  name: string;
-  acls: string[];
-};
-
 class TaskHandler {
-  // TODO: make these maps, using objects like this has performance impacts
-  // https://typescript-eslint.io/rules/no-dynamic-delete/
-  private taskRegistry = new Map<string, TaskRegistryEntry>();
+  // registry of schedualed tasks to be created
+  private scheduledTasks: Map<TaskGroup, () => Task> = new Map();
+  // list of all finished tasks
+  private finishedTasks: Map<string, FinishedTask> = new Map();
+
+  // list of all currently running tasks
+  private taskPool = new Map<string, TaskPoolEntry>();
+  // list of all clients currently connected to tasks
   private clientRegistry = new Map<string, PeerImpl>();
-  startTasks: (() => void)[] = [];
+
+  constructor() {
+    // register the cleanup invitations task
+    this.saveScheduledTask(cleanupInvites);
+    this.saveScheduledTask(cleanupSessions);
+    this.saveScheduledTask(checkUpdate);
+    this.saveScheduledTask(cleanupObjects);
+  }
+
+  /**
+   * Saves scheduled task to the registry
+   * @param createTask
+   */
+  private saveScheduledTask(task: DropTask) {
+    this.scheduledTasks.set(task.taskGroup, task.build);
+  }
 
   create(task: Task) {
     let updateCollectTimeout: NodeJS.Timeout | undefined;
     let updateCollectResolves: Array<(value: unknown) => void> = [];
     let logOffset: number = 0;
 
+    // if taskgroup disallows concurrency
+    if (!taskGroups[task.taskGroup].concurrency) {
+      for (const existingTask of this.taskPool.values()) {
+        // if a task is already running, we don't want to start another
+        if (existingTask.taskGroup === task.taskGroup) {
+          // TODO: handle this more gracefully, maybe with a queue? should be configurable
+          console.warn(
+            `Task group ${task.taskGroup} does not allow concurrent tasks. Task ${task.id} will not be started.`,
+          );
+          throw new Error(
+            `Task group ${task.taskGroup} does not allow concurrent tasks.`,
+          );
+        }
+      }
+    }
+
     const updateAllClients = (reset = false) =>
       new Promise((r) => {
         if (updateCollectTimeout) {
@@ -37,7 +89,7 @@ class TaskHandler {
           return;
         }
         updateCollectTimeout = setTimeout(() => {
-          const taskEntry = this.taskRegistry.get(task.id);
+          const taskEntry = this.taskPool.get(task.id);
           if (!taskEntry) return;
 
           const taskMessage: TaskMessage = {
@@ -67,33 +119,37 @@ class TaskHandler {
       });
 
     const progress = (progress: number) => {
-      const taskEntry = this.taskRegistry.get(task.id);
+      const taskEntry = this.taskPool.get(task.id);
       if (!taskEntry) return;
       taskEntry.progress = progress;
       updateAllClients();
     };
 
     const log = (entry: string) => {
-      const taskEntry = this.taskRegistry.get(task.id);
+      const taskEntry = this.taskPool.get(task.id);
       if (!taskEntry) return;
       taskEntry.log.push(entry);
+      console.log(`[Task ${task.taskGroup}]: ${entry}`);
       updateAllClients();
     };
 
-    this.taskRegistry.set(task.id, {
+    this.taskPool.set(task.id, {
       name: task.name,
+      taskGroup: task.taskGroup,
       success: false,
       progress: 0,
       error: undefined,
       log: [],
       clients: new Map(),
       acls: task.acls,
+      startTime: new Date().toISOString(),
+      endTime: undefined,
     });
 
     updateAllClients(true);
 
     droplet.callAltThreadFunc(async () => {
-      const taskEntry = this.taskRegistry.get(task.id);
+      const taskEntry = this.taskPool.get(task.id);
       if (!taskEntry) throw new Error("No task entry");
 
       try {
@@ -106,13 +162,22 @@ class TaskHandler {
           description: (error as string).toString(),
         };
       }
+
+      taskEntry.endTime = new Date().toISOString();
       await updateAllClients();
 
       for (const clientId of taskEntry.clients.keys()) {
         if (!this.clientRegistry.get(clientId)) continue;
         this.disconnect(clientId, task.id);
       }
-      this.taskRegistry.delete(task.id);
+
+      // so we can drop the clients from the task entry
+      const { clients, ...copied } = taskEntry;
+      this.finishedTasks.set(task.id, {
+        ...copied,
+      });
+
+      this.taskPool.delete(task.id);
     });
   }
 
@@ -122,7 +187,7 @@ class TaskHandler {
     peer: PeerImpl,
     request: MinimumRequestObject,
   ) {
-    const task = this.taskRegistry.get(taskId);
+    const task = this.taskPool.get(taskId);
     if (!task) {
       peer.send(
         `error/${taskId}/Unknown task/Drop couldn't find the task you're looking for.`,
@@ -160,8 +225,8 @@ class TaskHandler {
   }
 
   disconnectAll(id: string) {
-    for (const taskId of this.taskRegistry.keys()) {
-      this.taskRegistry.get(taskId)?.clients.delete(id);
+    for (const taskId of this.taskPool.keys()) {
+      this.taskPool.get(taskId)?.clients.delete(id);
       this.sendDisconnectEvent(id, taskId);
     }
 
@@ -169,13 +234,13 @@ class TaskHandler {
   }
 
   disconnect(id: string, taskId: string) {
-    const task = this.taskRegistry.get(taskId);
+    const task = this.taskPool.get(taskId);
     if (!task) return false;
 
     task.clients.delete(id);
     this.sendDisconnectEvent(id, taskId);
 
-    const allClientIds = this.taskRegistry
+    const allClientIds = this.taskPool
       .values()
       .toArray()
       .map((e) => e.clients.keys().toArray())
@@ -187,6 +252,24 @@ class TaskHandler {
 
     return true;
   }
+
+  runTaskGroupByName(name: TaskGroup) {
+    const task = this.scheduledTasks.get(name);
+    if (!task) {
+      console.warn(`No task found for group ${name}`);
+      return;
+    }
+    this.create(task());
+  }
+
+  /**]
+   * Runs all daily tasks that are scheduled to run once a day.
+   */
+  triggerDailyTasks() {
+    this.runTaskGroupByName("cleanup:invitations");
+    this.runTaskGroupByName("cleanup:sessions");
+    this.runTaskGroupByName("check:update");
+  }
 }
 
 export type TaskRunContext = {
@@ -196,6 +279,7 @@ export type TaskRunContext = {
 
 export interface Task {
   id: string;
+  taskGroup: TaskGroup;
   name: string;
   run: (context: TaskRunContext) => Promise<void>;
   acls: string[];
@@ -215,5 +299,33 @@ export type PeerImpl = {
   send: (message: string) => void;
 };
 
+export interface BuildTask {
+  buildId: () => string;
+  taskGroup: TaskGroup;
+  name: string;
+  run: (context: TaskRunContext) => Promise<void>;
+  acls: string[];
+}
+
+interface DropTask {
+  taskGroup: TaskGroup;
+  build: () => Task;
+}
+
+export function defineDropTask(buildTask: BuildTask): DropTask {
+  // TODO: only let one task with the same taskGroup run at the same time if specified
+
+  return {
+    taskGroup: buildTask.taskGroup,
+    build: () => ({
+      id: buildTask.buildId(),
+      taskGroup: buildTask.taskGroup,
+      name: buildTask.name,
+      run: buildTask.run,
+      acls: buildTask.acls,
+    }),
+  };
+}
+
 export const taskHandler = new TaskHandler();
 export default taskHandler;

server/internal/tasks/registry/invitations.ts

@@ -0,0 +1,24 @@
+import prisma from "~/server/internal/db/database";
+import { defineDropTask } from "..";
+
+export default defineDropTask({
+  buildId: () => `cleanup:invitations:${new Date().toISOString()}`,
+  name: "Cleanup Invitations",
+  acls: [],
+  taskGroup: "cleanup:invitations",
+  async run({ log }) {
+    log("Cleaning invitations");
+
+    const now = new Date();
+
+    await prisma.invitation.deleteMany({
+      where: {
+        expires: {
+          lt: now,
+        },
+      },
+    });
+
+    log("Done");
+  },
+});

server/internal/tasks/registry/objects.ts

@@ -0,0 +1,159 @@
+import prisma from "~/server/internal/db/database";
+import objectHandler from "~/server/internal/objects";
+import { defineDropTask } from "..";
+
+type FieldReferenceMap = {
+  [modelName: string]: {
+    model: unknown; // Prisma model
+    fields: string[]; // Fields that may contain IDs
+    arrayFields: string[]; // Fields that are arrays that may contain IDs
+  };
+};
+
+export default defineDropTask({
+  buildId: () => `cleanup:objects:${new Date().toISOString()}`,
+  name: "Cleanup Objects",
+  acls: [],
+  taskGroup: "cleanup:objects",
+  async run({ progress, log }) {
+    log("Cleaning unreferenced objects");
+
+    // get all objects
+    const objects = await objectHandler.listAll();
+    log(`searching for ${objects.length} objects`);
+    progress(30);
+
+    // find unreferenced objects
+    const refMap = buildRefMap();
+    log("Building reference map");
+    log(`Found ${Object.keys(refMap).length} models with reference fields`);
+    log("Searching for unreferenced objects");
+    const unrefedObjects = await findUnreferencedStrings(objects, refMap);
+    log(`found ${unrefedObjects.length} Unreferenced objects`);
+    // console.log(unrefedObjects);
+    progress(60);
+
+    // remove objects
+    const deletePromises: Promise<boolean>[] = [];
+    for (const obj of unrefedObjects) {
+      log(`Deleting object ${obj}`);
+      deletePromises.push(objectHandler.deleteAsSystem(obj));
+    }
+    await Promise.all(deletePromises);
+
+    // Remove any possible leftover metadata
+    objectHandler.cleanupMetadata();
+
+    log("Done");
+    progress(100);
+  },
+});
+
+/**
+ * Builds a map of Prisma models and their fields that may contain object IDs
+ * @returns
+ */
+function buildRefMap(): FieldReferenceMap {
+  const tables = Object.keys(prisma).filter(
+    (v) => !(v.startsWith("$") || v.startsWith("_") || v === "constructor"),
+  );
+  // type test = Prisma.ModelName
+  // prisma.game.fields.mIconId.
+
+  const result: FieldReferenceMap = {};
+
+  for (const model of tables) {
+    // @ts-expect-error can't get model to typematch key names
+    const fields = Object.keys(prisma[model]["fields"]);
+
+    const single = fields.filter((v) => v.toLowerCase().endsWith("objectid"));
+    const array = fields.filter((v) => v.toLowerCase().endsWith("objectids"));
+
+    result[model] = {
+      // @ts-expect-error im not dealing with this
+      model: prisma[model],
+      fields: single,
+      arrayFields: array,
+    };
+  }
+
+  return result;
+}
+
+/**
+ * Searches all models for a given id in their fields
+ * @param id
+ * @param fieldRefMap
+ * @returns
+ */
+async function isReferencedInModelFields(
+  id: string,
+  fieldRefMap: FieldReferenceMap,
+): Promise<boolean> {
+  // TODO: optimize the built queries
+  // rn it runs a query for every id over each db table
+  for (const { model, fields, arrayFields } of Object.values(fieldRefMap)) {
+    const singleFieldOrConditions = fields
+      ? fields.map((field) => ({
+          [field]: {
+            equals: id,
+          },
+        }))
+      : [];
+    const arrayFieldOrConditions = arrayFields
+      ? arrayFields.map((field) => ({
+          [field]: {
+            has: id,
+          },
+        }))
+      : [];
+
+    // prisma.game.findFirst({
+    //   where: {
+    //     OR: [
+    //       // single item
+    //       {
+    //         mIconId: {
+    //           equals: "",
+    //         },
+    //       },
+    //       // array
+    //       {
+    //         mImageCarousel: {
+    //           has: "",
+    //         },
+    //       },
+    //     ],
+    //   },
+    // });
+
+    // @ts-expect-error using unknown because im not typing this mess omg
+    const found = await model.findFirst({
+      where: { OR: [...singleFieldOrConditions, ...arrayFieldOrConditions] },
+    });
+
+    if (found) return true;
+  }
+
+  return false;
+}
+
+/**
+ * Takes a list of objects and checks if they are referenced in any model fields
+ * @param objects
+ * @param fieldRefMap
+ * @returns
+ */
+async function findUnreferencedStrings(
+  objects: string[],
+  fieldRefMap: FieldReferenceMap,
+): Promise<string[]> {
+  const unreferenced: string[] = [];
+
+  for (const obj of objects) {
+    const isRef = await isReferencedInModelFields(obj, fieldRefMap);
+    if (!isRef) unreferenced.push(obj);
+  }
+
+  return unreferenced;
+}

server/internal/tasks/registry/sessions.ts

@@ -0,0 +1,14 @@
+import sessionHandler from "~/server/internal/session";
+import { defineDropTask } from "..";
+
+export default defineDropTask({
+  buildId: () => `cleanup:sessions:${new Date().toISOString()}`,
+  name: "Cleanup Sessions",
+  acls: [],
+  taskGroup: "cleanup:sessions",
+  async run({ log }) {
+    log("Cleaning up sessions");
+    await sessionHandler.cleanupSessions();
+    log("Done");
+  },
+});

server/internal/tasks/registry/update.ts

@@ -0,0 +1,98 @@
+import { type } from "arktype";
+import * as semver from "semver";
+import { defineDropTask } from "..";
+import { systemConfig } from "../../config/sys-conf";
+import notificationSystem from "../../notifications";
+
+const latestRelease = type({
+  url: "string", // api url for specific release
+  html_url: "string", // user facing url
+  id: "number", // release id
+  tag_name: "string", // tag used for release
+  name: "string", // release name
+  draft: "boolean",
+  prerelease: "boolean",
+  created_at: "string",
+  published_at: "string",
+});
+
+export default defineDropTask({
+  buildId: () => `check:update:${new Date().toISOString()}`,
+  name: "Check for Update",
+  acls: [],
+  taskGroup: "check:update",
+  async run({ log }) {
+    // TODO: maybe implement some sort of rate limit thing to prevent this from calling github api a bunch in the event of crashloop or whatever?
+    // probably will require custom task scheduler for object cleanup anyway, so something to thing about
+
+    if (!systemConfig.shouldCheckForUpdates()) {
+      log("Update check is disabled by configuration");
+      return;
+    }
+
+    log("Checking for update");
+
+    const currVerStr = systemConfig.getDropVersion();
+    const currVer = semver.coerce(currVerStr);
+    if (currVer === null) {
+      const msg = "Drop provided a invalid semver tag";
+      log(msg);
+      throw new Error(msg);
+    }
+
+    const response = await fetch(
+      "https://api.github.com/repos/Drop-OSS/drop/releases/latest",
+    );
+
+    // if response failed somehow
+    if (!response.ok) {
+      log(
+        "Failed to check for update " +
+          JSON.stringify({
+            status: response.status,
+            body: response.body,
+          }),
+      );
+
+      throw new Error(
+        `Failed to check for update: ${response.status} ${response.body}`,
+      );
+    }
+
+    // parse and validate response
+    const resJson = await response.json();
+    const body = latestRelease(resJson);
+    if (body instanceof type.errors) {
+      log(body.summary);
+      log("GitHub Api response" + JSON.stringify(resJson));
+      throw new Error(
+        `GitHub Api response did not match expected schema: ${body.summary}`,
+      );
+    }
+
+    // parse remote version
+    const latestVer = semver.coerce(body.tag_name);
+    if (latestVer === null) {
+      const msg = "Github Api returned invalid semver tag";
+      log(msg);
+      throw new Error(msg);
+    }
+
+    // TODO: handle prerelease identifiers https://github.com/npm/node-semver#prerelease-identifiers
+    // check if is newer version
+    if (semver.gt(latestVer, currVer)) {
+      log("Update available");
+      notificationSystem.systemPush({
+        nonce: `drop-update-available-${currVer}-to-${latestVer}`,
+        title: `Update available to v${latestVer}`,
+        description: `A new version of Drop is available v${latestVer}`,
+        actions: [`View|${body.html_url}`],
+        acls: ["system:notifications:read"],
+      });
+    } else {
+      log("no update available");
+    }
+
+    log("Done");
+  },
+});