diff --git a/server/internal/objects/fsBackend.ts b/server/internal/objects/fsBackend.ts
index de0ea0a..e8b070a 100644
--- a/server/internal/objects/fsBackend.ts
+++ b/server/internal/objects/fsBackend.ts
@@ -174,6 +174,30 @@ export class FsObjectBackend extends ObjectBackend {
   async listAll(): Promise<string[]> {
     return fs.readdirSync(this.baseObjectPath);
   }
+
+  async cleanupMetadata() {
+    const metadataFiles = fs.readdirSync(this.baseMetadataPath);
+    const objects = await this.listAll();
+
+    const extraFiles = metadataFiles.filter(
+      (file) => !objects.includes(file.replace(/\.json$/, "")),
+    );
+    console.log(
+      `[FsObjectBackend#cleanupMetadata]: Found ${extraFiles.length} metadata files without corresponding objects.`,
+    );
+    for (const file of extraFiles) {
+      const filePath = path.join(this.baseMetadataPath, file);
+      try {
+        fs.rmSync(filePath);
+        console.log(`[FsObjectBackend#cleanupMetadata]: Removed ${file}`);
+      } catch (error) {
+        console.error(
+          `[FsObjectBackend#cleanupMetadata]: Failed to remove ${file}`,
+          error,
+        );
+      }
+    }
+  }
 }
 
 class FsHashStore {
diff --git a/server/internal/objects/objectHandler.ts b/server/internal/objects/objectHandler.ts
index 81cb11f..0506f41 100644
--- a/server/internal/objects/objectHandler.ts
+++ b/server/internal/objects/objectHandler.ts
@@ -71,6 +71,7 @@ export abstract class ObjectBackend {
   ): Promise<boolean>;
   abstract fetchHash(id: ObjectReference): Promise<string | undefined>;
   abstract listAll(): Promise<string[]>;
+  abstract cleanupMetadata(): Promise<void>;
 }
 
 export class ObjectHandler {
@@ -257,4 +258,13 @@ export class ObjectHandler {
   async listAll() {
     return await this.backend.listAll();
   }
+
+  /**
+   * Purges metadata for objects that no longer exist
+   * This is useful for cleaning up metadata files that are left behinds
+   * @returns
+   */
+  async cleanupMetadata() {
+    return await this.backend.cleanupMetadata();
+  }
 }
diff --git a/server/tasks/cleanup/objects.ts b/server/tasks/cleanup/objects.ts
index d43f08f..44cd7f3 100644
--- a/server/tasks/cleanup/objects.ts
+++ b/server/tasks/cleanup/objects.ts
@@ -44,6 +44,9 @@ export default defineTask<TaskReturn>({
     }
     await Promise.all(deletePromises);
 
+    // Remove any possible leftover metadata
+    objectHandler.cleanupMetadata();
+
     console.log("[Task cleanup:objects]: Done");
     return {
       result: {