i18n Support and Task improvements (#80)

* fix: release workflow

* feat: move mostly to internal tasks system

* feat: migrate object clean to new task system

* fix: release not  getting good base version

* chore: set version v0.3.0

* chore: style

* feat: basic task concurrency

* feat: temp pages to fill in page links

* feat: inital i18n support

* feat: localize store page

* chore: style

* fix: weblate doesn't like multifile thing

* fix: update nuxt

* feat: improved error logging

* fix: using old task api

* feat: basic translation docs

* feat: add i18n eslint plugin

* feat: translate store and auth pages

* feat: more translation progress

* feat: admin dash i18n progress

* feat: enable update check by default in prod

* fix: using wrong i18n keys

* fix: crash in library sources page

* feat: finish i18n work

* fix: missing i18n translations

* feat: use twemoji for emojis

* feat: sanatize object ids

* fix: EmojiText's alt text

* fix: UserWidget not using links

* feat: cache and auth for emoji api

* fix: add more missing translations

server/internal/tasks/registry/invitations.ts

@@ -0,0 +1,24 @@
+import prisma from "~/server/internal/db/database";
+import { defineDropTask } from "..";
+
+export default defineDropTask({
+  buildId: () => `cleanup:invitations:${new Date().toISOString()}`,
+  name: "Cleanup Invitations",
+  acls: [],
+  taskGroup: "cleanup:invitations",
+  async run({ log }) {
+    log("Cleaning invitations");
+
+    const now = new Date();
+
+    await prisma.invitation.deleteMany({
+      where: {
+        expires: {
+          lt: now,
+        },
+      },
+    });
+
+    log("Done");
+  },
+});

server/internal/tasks/registry/objects.ts

@@ -0,0 +1,159 @@
+import prisma from "~/server/internal/db/database";
+import objectHandler from "~/server/internal/objects";
+import { defineDropTask } from "..";
+
+type FieldReferenceMap = {
+  [modelName: string]: {
+    model: unknown; // Prisma model
+    fields: string[]; // Fields that may contain IDs
+    arrayFields: string[]; // Fields that are arrays that may contain IDs
+  };
+};
+
+export default defineDropTask({
+  buildId: () => `cleanup:objects:${new Date().toISOString()}`,
+  name: "Cleanup Objects",
+  acls: [],
+  taskGroup: "cleanup:objects",
+  async run({ progress, log }) {
+    log("Cleaning unreferenced objects");
+
+    // get all objects
+    const objects = await objectHandler.listAll();
+    log(`searching for ${objects.length} objects`);
+    progress(30);
+
+    // find unreferenced objects
+    const refMap = buildRefMap();
+    log("Building reference map");
+    log(`Found ${Object.keys(refMap).length} models with reference fields`);
+    log("Searching for unreferenced objects");
+    const unrefedObjects = await findUnreferencedStrings(objects, refMap);
+    log(`found ${unrefedObjects.length} Unreferenced objects`);
+    // console.log(unrefedObjects);
+    progress(60);
+
+    // remove objects
+    const deletePromises: Promise<boolean>[] = [];
+    for (const obj of unrefedObjects) {
+      log(`Deleting object ${obj}`);
+      deletePromises.push(objectHandler.deleteAsSystem(obj));
+    }
+    await Promise.all(deletePromises);
+
+    // Remove any possible leftover metadata
+    objectHandler.cleanupMetadata();
+
+    log("Done");
+    progress(100);
+  },
+});
+
+/**
+ * Builds a map of Prisma models and their fields that may contain object IDs
+ * @returns
+ */
+function buildRefMap(): FieldReferenceMap {
+  const tables = Object.keys(prisma).filter(
+    (v) => !(v.startsWith("$") || v.startsWith("_") || v === "constructor"),
+  );
+  // type test = Prisma.ModelName
+  // prisma.game.fields.mIconId.
+
+  const result: FieldReferenceMap = {};
+
+  for (const model of tables) {
+    // @ts-expect-error can't get model to typematch key names
+    const fields = Object.keys(prisma[model]["fields"]);
+
+    const single = fields.filter((v) => v.toLowerCase().endsWith("objectid"));
+    const array = fields.filter((v) => v.toLowerCase().endsWith("objectids"));
+
+    result[model] = {
+      // @ts-expect-error im not dealing with this
+      model: prisma[model],
+      fields: single,
+      arrayFields: array,
+    };
+  }
+
+  return result;
+}
+
+/**
+ * Searches all models for a given id in their fields
+ * @param id
+ * @param fieldRefMap
+ * @returns
+ */
+async function isReferencedInModelFields(
+  id: string,
+  fieldRefMap: FieldReferenceMap,
+): Promise<boolean> {
+  // TODO: optimize the built queries
+  // rn it runs a query for every id over each db table
+  for (const { model, fields, arrayFields } of Object.values(fieldRefMap)) {
+    const singleFieldOrConditions = fields
+      ? fields.map((field) => ({
+          [field]: {
+            equals: id,
+          },
+        }))
+      : [];
+    const arrayFieldOrConditions = arrayFields
+      ? arrayFields.map((field) => ({
+          [field]: {
+            has: id,
+          },
+        }))
+      : [];
+
+    // prisma.game.findFirst({
+    //   where: {
+    //     OR: [
+    //       // single item
+    //       {
+    //         mIconId: {
+    //           equals: "",
+    //         },
+    //       },
+    //       // array
+    //       {
+    //         mImageCarousel: {
+    //           has: "",
+    //         },
+    //       },
+    //     ],
+    //   },
+    // });
+
+    // @ts-expect-error using unknown because im not typing this mess omg
+    const found = await model.findFirst({
+      where: { OR: [...singleFieldOrConditions, ...arrayFieldOrConditions] },
+    });
+
+    if (found) return true;
+  }
+
+  return false;
+}
+
+/**
+ * Takes a list of objects and checks if they are referenced in any model fields
+ * @param objects
+ * @param fieldRefMap
+ * @returns
+ */
+async function findUnreferencedStrings(
+  objects: string[],
+  fieldRefMap: FieldReferenceMap,
+): Promise<string[]> {
+  const unreferenced: string[] = [];
+
+  for (const obj of objects) {
+    const isRef = await isReferencedInModelFields(obj, fieldRefMap);
+    if (!isRef) unreferenced.push(obj);
+  }
+
+  return unreferenced;
+}

server/internal/tasks/registry/sessions.ts

@@ -0,0 +1,14 @@
+import sessionHandler from "~/server/internal/session";
+import { defineDropTask } from "..";
+
+export default defineDropTask({
+  buildId: () => `cleanup:sessions:${new Date().toISOString()}`,
+  name: "Cleanup Sessions",
+  acls: [],
+  taskGroup: "cleanup:sessions",
+  async run({ log }) {
+    log("Cleaning up sessions");
+    await sessionHandler.cleanupSessions();
+    log("Done");
+  },
+});

server/internal/tasks/registry/update.ts

@@ -0,0 +1,98 @@
+import { type } from "arktype";
+import * as semver from "semver";
+import { defineDropTask } from "..";
+import { systemConfig } from "../../config/sys-conf";
+import notificationSystem from "../../notifications";
+
+const latestRelease = type({
+  url: "string", // api url for specific release
+  html_url: "string", // user facing url
+  id: "number", // release id
+  tag_name: "string", // tag used for release
+  name: "string", // release name
+  draft: "boolean",
+  prerelease: "boolean",
+  created_at: "string",
+  published_at: "string",
+});
+
+export default defineDropTask({
+  buildId: () => `check:update:${new Date().toISOString()}`,
+  name: "Check for Update",
+  acls: [],
+  taskGroup: "check:update",
+  async run({ log }) {
+    // TODO: maybe implement some sort of rate limit thing to prevent this from calling github api a bunch in the event of crashloop or whatever?
+    // probably will require custom task scheduler for object cleanup anyway, so something to thing about
+
+    if (!systemConfig.shouldCheckForUpdates()) {
+      log("Update check is disabled by configuration");
+      return;
+    }
+
+    log("Checking for update");
+
+    const currVerStr = systemConfig.getDropVersion();
+    const currVer = semver.coerce(currVerStr);
+    if (currVer === null) {
+      const msg = "Drop provided a invalid semver tag";
+      log(msg);
+      throw new Error(msg);
+    }
+
+    const response = await fetch(
+      "https://api.github.com/repos/Drop-OSS/drop/releases/latest",
+    );
+
+    // if response failed somehow
+    if (!response.ok) {
+      log(
+        "Failed to check for update " +
+          JSON.stringify({
+            status: response.status,
+            body: response.body,
+          }),
+      );
+
+      throw new Error(
+        `Failed to check for update: ${response.status} ${response.body}`,
+      );
+    }
+
+    // parse and validate response
+    const resJson = await response.json();
+    const body = latestRelease(resJson);
+    if (body instanceof type.errors) {
+      log(body.summary);
+      log("GitHub Api response" + JSON.stringify(resJson));
+      throw new Error(
+        `GitHub Api response did not match expected schema: ${body.summary}`,
+      );
+    }
+
+    // parse remote version
+    const latestVer = semver.coerce(body.tag_name);
+    if (latestVer === null) {
+      const msg = "Github Api returned invalid semver tag";
+      log(msg);
+      throw new Error(msg);
+    }
+
+    // TODO: handle prerelease identifiers https://github.com/npm/node-semver#prerelease-identifiers
+    // check if is newer version
+    if (semver.gt(latestVer, currVer)) {
+      log("Update available");
+      notificationSystem.systemPush({
+        nonce: `drop-update-available-${currVer}-to-${latestVer}`,
+        title: `Update available to v${latestVer}`,
+        description: `A new version of Drop is available v${latestVer}`,
+        actions: [`View|${body.html_url}`],
+        acls: ["system:notifications:read"],
+      });
+    } else {
+      log("no update available");
+    }
+
+    log("Done");
+  },
+});