Merge branch 'Huskydog9988-more-stuff' into develop

components/UserHeader.vue

@@ -1,7 +1,7 @@
 <template>
   <div class="hidden lg:flex bg-zinc-950 flex-row px-12 xl:px-48 py-5">
     <div class="grow inline-flex items-center gap-x-20">
-      <NuxtLink to="/store">
+      <NuxtLink :to="homepageURL">
         <DropWordmark class="h-8" />
       </NuxtLink>
       <nav class="inline-flex items-center">
@@ -62,7 +62,10 @@
   <div
     class="sticky lg:hidden top-0 z-40 flex h-16 justify-between items-center gap-x-4 border-b border-zinc-700 bg-zinc-950 px-4 shadow-sm sm:gap-x-6 sm:px-6 lg:px-8"
   >
+    <NuxtLink :to="homepageURL">
     <DropWordmark class="mb-0.5" />
+    </NuxtLink>
+
     <div class="flex gap-x-4 lg:gap-x-6">
       <div class="flex items-center gap-x-3">
         <!-- Profile dropdown -->
@@ -132,7 +135,7 @@
               class="flex grow flex-col gap-y-5 overflow-y-auto bg-zinc-950 px-6 pb-4"
             >
               <div class="flex shrink-0 h-16 items-center justify-between">
-                <NuxtLink to="/store">
+                <NuxtLink :to="homepageURL">
                   <DropLogo class="h-8 w-auto" />
                 </NuxtLink>
 
@@ -180,7 +183,11 @@
 </template>
 
 <script setup lang="ts">
-import { BellIcon, UserGroupIcon } from "@heroicons/vue/16/solid";
+import {
+  BellIcon,
+  UserGroupIcon,
+  ArrowDownTrayIcon,
+} from "@heroicons/vue/16/solid";
 import {
   Dialog,
   DialogPanel,
@@ -196,6 +203,7 @@ import { XMarkIcon } from "@heroicons/vue/24/solid";
 
 const router = useRouter();
 
+const homepageURL = "/store";
 const navigation: Array<NavigationItem> = [
   {
     prefix: "/store",

nuxt.config.ts

@@ -9,7 +9,8 @@ export default defineNuxtConfig({
     enabled: true,
     telemetry: false,
     timeline: {
-      // seems to break things
+      // this seems to be the tracking issue, composables not registered
+      // https://github.com/nuxt/devtools/issues/662
       enabled: false,
     },
   },
@@ -47,6 +48,20 @@ export default defineNuxtConfig({
     },
 
     compressPublicAssets: true,
+
+    storage: {
+      appCache: {
+        driver: "lru-cache",
+      },
+    },
+
+    devStorage: {
+      appCache: {
+        // store cache on fs to handle dev server restarts
+        driver: "fs",
+        base: "./.data/appCache",
+      },
+    },
   },
 
   typescript: {

package.json

@@ -31,7 +31,6 @@
     "fast-fuzzy": "^1.12.0",
     "file-type-mime": "^0.4.3",
     "jdenticon": "^3.3.0",
-    "lru-cache": "^11.1.0",
     "luxon": "^3.6.1",
     "micromark": "^4.0.1",
     "nuxt": "^3.16.2",
@@ -40,6 +39,7 @@
     "sharp": "^0.33.5",
     "stream-mime-type": "^2.0.0",
     "turndown": "^7.2.0",
+    "unstorage": "^1.15.0",
     "vue": "latest",
     "vue-router": "latest",
     "vue3-carousel": "^0.15.0",
@@ -75,4 +75,4 @@
   "prisma": {
     "schema": "./prisma"
   }
-}
+}

prisma/schema.prisma

@@ -3,7 +3,7 @@
 
 generator client {
   provider        = "prisma-client-js"
-  previewFeatures = ["prismaSchemaFolder", "omitApi", "fullTextSearchPostgres"]
+  previewFeatures = ["prismaSchemaFolder", "fullTextSearchPostgres"]
 }
 
 datasource db {

server/api/v1/notifications/ws.get.ts

@@ -1,9 +1,9 @@
 import notificationSystem from "~/server/internal/notifications";
 import aclManager from "~/server/internal/acls";
+import cacheHandler from "~/server/internal/cache";
 
-// TODO add web socket sessions for horizontal scaling
 // Peer ID to user ID
-const socketSessions = new Map<string, string>();
+const socketSessions = cacheHandler.createCache<string>("notificationSocketSessions");
 
 export default defineWebSocketHandler({
   async open(peer) {
@@ -23,7 +23,7 @@ export default defineWebSocketHandler({
       userIds.push("system");
     }
 
-    socketSessions.set(peer.id, userId);
+    await socketSessions.set(peer.id, userId);
 
     for (const listenUserId of userIds) {
       notificationSystem.listen(listenUserId, peer.id, (notification) => {
@@ -32,7 +32,7 @@ export default defineWebSocketHandler({
     }
   },
   async close(peer, _details) {
-    const userId = socketSessions.get(peer.id);
+    const userId = await socketSessions.get(peer.id);
     if (!userId) {
       console.log(`skipping websocket close for ${peer.id}`);
       return;
@@ -40,6 +40,6 @@ export default defineWebSocketHandler({
 
     notificationSystem.unlisten(userId, peer.id);
     notificationSystem.unlisten("system", peer.id); // In case we were listening as 'system'
-    socketSessions.delete(peer.id);
+    await socketSessions.remove(peer.id);
   },
 });

server/api/v1/task/index.get.ts

@@ -1,9 +1,9 @@
 import taskHandler from "~/server/internal/tasks";
 import type { MinimumRequestObject } from "~/server/h3";
+import cacheHandler from "~/server/internal/cache";
 
-// TODO add web socket sessions for horizontal scaling
 // ID to admin
-const socketHeaders = new Map<string, MinimumRequestObject>();
+const socketHeaders = cacheHandler.createCache<MinimumRequestObject>("taskSocketHeaders");
 
 export default defineWebSocketHandler({
   async open(peer) {
@@ -13,15 +13,15 @@ export default defineWebSocketHandler({
       return;
     }
 
-    socketHeaders.set(peer.id, {
+    await socketHeaders.set(peer.id, {
       headers: request.headers ?? new Headers(),
     });
     peer.send(`connect`);
   },
-  message(peer, message) {
+  async message(peer, message) {
     if (!peer.id) return;
-    const headers = socketHeaders.get(peer.id);
-    if (headers === undefined) return;
+    const headers = await socketHeaders.get(peer.id);
+    if (!headers) return;
     const text = message.text();
     if (text.startsWith("connect/")) {
       const id = text.substring("connect/".length);
@@ -29,10 +29,10 @@ export default defineWebSocketHandler({
       return;
     }
   },
-  close(peer, _details) {
+  async close(peer, _details) {
     if (!peer.id) return;
     if (!socketHeaders.has(peer.id)) return;
-    socketHeaders.delete(peer.id);
+    await socketHeaders.remove(peer.id);
 
     taskHandler.disconnectAll(peer.id);
   },

server/internal/cache/cacheHandler.ts

@@ -0,0 +1,33 @@
+import { prefixStorage, type StorageValue, type Storage } from "unstorage";
+
+export interface CacheProviderOptions {
+  /**
+   * Max number of items in the cache
+   */
+  max?: number;
+
+  /**
+   * Time to live (in ms)
+   */
+  ttl?: number;
+}
+
+/**
+ * Creates and manages the lifecycles of various caches
+ */
+export class CacheHandler {
+  private caches = new Map<string, Storage<StorageValue>>();
+
+  /**
+   * Create a new cache
+   * @param name
+   * @returns
+   */
+  createCache<V extends StorageValue>(name: string) {
+    // will allow us to dynamicing use redis in the future just by changing the storage used
+    const provider = prefixStorage<V>(useStorage<V>("appCache"), name);
+    // hack to let ts have us store cache
+    this.caches.set(name, provider as unknown as Storage<StorageValue>);
+    return provider;
+  }
+}

server/internal/cache/index.ts

@@ -0,0 +1,4 @@
+import { CacheHandler } from "./cacheHandler";
+
+export const cacheHandler = new CacheHandler();
+export default cacheHandler;

server/internal/objects/fsBackend.ts

@@ -1,12 +1,12 @@
 import type { ObjectMetadata, ObjectReference, Source } from "./objectHandler";
 import { ObjectBackend } from "./objectHandler";
 
-import { LRUCache } from "lru-cache";
 import fs from "fs";
 import path from "path";
 import { Readable } from "stream";
 import { createHash } from "crypto";
 import prisma from "../db/database";
+import cacheHandler from "../cache";
 
 export class FsObjectBackend extends ObjectBackend {
   private baseObjectPath: string;
@@ -34,7 +34,7 @@ export class FsObjectBackend extends ObjectBackend {
     if (!fs.existsSync(objectPath)) return false;
 
     // remove item from cache
-    this.hashStore.delete(id);
+    await this.hashStore.delete(id);
 
     if (source instanceof Readable) {
       const outputStream = fs.createWriteStream(objectPath);
@@ -54,7 +54,7 @@ export class FsObjectBackend extends ObjectBackend {
     const objectPath = path.join(this.baseObjectPath, id);
     if (!fs.existsSync(objectPath)) return undefined;
     // remove item from cache
-    this.hashStore.delete(id);
+    await this.hashStore.delete(id);
     return fs.createWriteStream(objectPath);
   }
   async create(
@@ -99,7 +99,7 @@ export class FsObjectBackend extends ObjectBackend {
     if (!fs.existsSync(objectPath)) return true;
     fs.rmSync(objectPath);
     // remove item from cache
-    this.hashStore.delete(id);
+    await this.hashStore.delete(id);
     return true;
   }
   async fetchMetadata(
@@ -121,36 +121,41 @@ export class FsObjectBackend extends ObjectBackend {
   }
   async fetchHash(id: ObjectReference): Promise<string | undefined> {
     const cacheResult = await this.hashStore.get(id);
-    if (cacheResult !== undefined) return cacheResult;
+    if (cacheResult !== null) return cacheResult;
 
     const obj = await this.fetch(id);
     if (obj === undefined) return;
 
-    // local variable to point to object
-    const cache = this.hashStore;
-
     // hash object
     const hash = createHash("md5");
     hash.setEncoding("hex");
 
-    // read obj into hash
-    obj.pipe(hash);
-    await new Promise<void>((r) => {
-      obj.on("end", function () {
+    // local variable to point to object
+    const store = this.hashStore;
+    let hashResult = "";
+
+    const objEnd = new Promise<void>((r) => {
+      obj.on("end", async function () {
         hash.end();
-        cache.save(id, hash.read());
+        hashResult = hash.read();
         r();
       });
     });
+    // read obj into hash
+    obj.pipe(hash);
+    await objEnd;
 
-    return await this.hashStore.get(id);
+    // if hash isn't a string somehow, mark as unknown hash
+    if (typeof hashResult !== "string") {
+      return undefined;
+    }
+    await store.save(id, hashResult);
+    return typeof hashResult;
   }
 }
 
 class FsHashStore {
-  private cache = new LRUCache<string, string>({
-    max: 1000, // number of items
-  });
+  private cache = cacheHandler.createCache<string>("ObjectHashStore");
 
   /**
    * Gets hash of object
@@ -158,8 +163,10 @@ class FsHashStore {
    * @returns
    */
   async get(id: ObjectReference) {
-    const cacheRes = this.cache.get(id);
-    if (cacheRes !== undefined) return cacheRes;
+    const cacheRes = await this.cache.get(id);
+    if (cacheRes !== null) {
+      return cacheRes;
+    }
 
     const objectHash = await prisma.objectHash.findUnique({
       where: {
@@ -170,7 +177,7 @@ class FsHashStore {
       },
     });
     if (objectHash === null) return undefined;
-    this.cache.set(id, objectHash.hash);
+    await this.cache.set(id, objectHash.hash);
     return objectHash.hash;
   }
 
@@ -191,7 +198,7 @@ class FsHashStore {
         hash,
       },
     });
-    this.cache.set(id, hash);
+    await this.cache.set(id, hash);
   }
 
   /**
@@ -199,7 +206,7 @@ class FsHashStore {
    * @param id
    */
   async delete(id: ObjectReference) {
-    this.cache.delete(id);
+    await this.cache.remove(id);
 
     try {
       // need to catch in case the object doesn't exist

server/internal/session/db.ts

@@ -1,16 +1,13 @@
-import { LRUCache } from "lru-cache";
 import prisma from "../db/database";
 import type { Session, SessionProvider } from "./types";
+import cacheHandler from "../cache";
 
 export default function createDBSessionHandler(): SessionProvider {
-  const cache = new LRUCache<string, Session>({
-    max: 50, // number of items
-    ttl: 30 * 100, // 30s (in ms)
-  });
+  const cache = cacheHandler.createCache<Session>("DBSession");
 
   return {
     async setSession(token, session) {
-      cache.set(token, session);
+      await cache.set(token, session);
 
       //   const strData = JSON.stringify(data);
       await prisma.session.upsert({
@@ -29,8 +26,8 @@ export default function createDBSessionHandler(): SessionProvider {
       return await this.setSession(token, data);
     },
     async getSession<T extends Session>(token: string) {
-      const cached = cache.get(token);
-      if (cached !== undefined) return cached as T;
+      const cached = await cache.get(token);
+      if (cached !== null) return cached as T;
 
       const result = await prisma.session.findUnique({
         where: {
@@ -45,7 +42,7 @@ export default function createDBSessionHandler(): SessionProvider {
       return result as unknown as T;
     },
     async removeSession(token) {
-      cache.delete(token);
+      await cache.remove(token);
       await prisma.session.delete({
         where: {
           token,

server/internal/userlibrary/index.ts

@@ -2,15 +2,17 @@
 Handles managing collections
 */
 
+import cacheHandler from "../cache";
 import prisma from "../db/database";
 
 class UserLibraryManager {
   // Caches the user's core library
-  private userCoreLibraryCache: { [key: string]: string } = {};
+  private coreLibraryCache =
+    cacheHandler.createCache<string>("UserCoreLibrary");
 
   private async fetchUserLibrary(userId: string) {
-    if (this.userCoreLibraryCache[userId])
-      return this.userCoreLibraryCache[userId];
+    const cached = await this.coreLibraryCache.get(userId);
+    if (cached !== null) return cached;
 
     let collection = await prisma.collection.findFirst({
       where: {
@@ -28,7 +30,7 @@ class UserLibraryManager {
         },
       });
 
-    this.userCoreLibraryCache[userId] = collection.id;
+    await this.coreLibraryCache.set(userId, collection.id);
 
     return collection.id;
   }

server/plugins/01.system-init.ts

@@ -1,4 +1,4 @@
-import prisma from "../internal/db/database";
+import prisma from "~/server/internal/db/database";
 
 export default defineNitroPlugin(async (_nitro) => {
   // Ensure system user exists

server/plugins/02.setup-admin.ts

@@ -1,4 +1,4 @@
-import prisma from "../internal/db/database";
+import prisma from "~/server/internal/db/database";
 
 export default defineNitroPlugin(async (_nitro) => {
   const userCount = await prisma.user.count({

server/plugins/tasks.ts

@@ -0,0 +1,7 @@
+export default defineNitroPlugin(async (_nitro) => {
+  // all tasks we should run on server boot
+  await Promise.all([
+    runTask("cleanup:invitations"),
+    runTask("cleanup:sessions"),
+  ]);
+});

server/tasks/cleanup/invitations.ts

@@ -5,6 +5,8 @@ export default defineTask({
     name: "cleanup:invitations",
   },
   async run() {
+    console.log("[Task cleanup:invitations]: Cleaning invitations");
+
     const now = new Date();
 
     await prisma.invitation.deleteMany({
@@ -15,6 +17,7 @@ export default defineTask({
       },
     });
 
+    console.log("[Task cleanup:invitations]: Done");
     return { result: true };
   },
 });

server/tasks/cleanup/sessions.ts

@@ -2,11 +2,12 @@ import sessionHandler from "~/server/internal/session";
 
 export default defineTask({
   meta: {
-    name: "cleanup:invitations",
+    name: "cleanup:sessions",
   },
   async run() {
+    console.log("[Task cleanup:sessions]: Cleaning up sessions");
     await sessionHandler.cleanupSessions();
-
+    console.log("[Task cleanup:sessions]: Done");
     return { result: true };
   },
 });