Ryan/drop
1
0
Fork 0
mirror of https://github.com/Drop-OSS/drop.git synced 2025-11-10 04:22:09 +10:00
Code Issues Packages Projects Releases Wiki Activity

Various fixes (#186)

Browse Source 
* fix: #181

* fix: use taskHandler as source of truth for imports

* fix: task formatting

* fix: zip downloads

* feat: re-enable import version button on delete + lint
This commit is contained in:
DecDuck
2025-08-15 22:57:56 +10:00
committed by GitHub
parent 9ff541059d
commit abec952e39
14 changed files with 180 additions and 185 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
server/api/v2/client/chunk.post.ts
View File

@ -2,6 +2,7 @@ import { type } from "arktype";
import { readDropValidatedBody, throwingArktype } from "~/server/arktype";
import contextManager from "~/server/internal/downloads/coordinator";
import libraryManager from "~/server/internal/library";
import { logger } from "~/server/internal/logging";
const GetChunk = type({
context: "string",
@ -58,13 +59,25 @@ export default defineEventHandler(async (h3) => {
statusCode: 500,
statusMessage: "Failed to create read stream",
});
let length = 0;
await gameReadStream.pipeTo(
new WritableStream({
write(chunk) {
h3.node.res.write(chunk);
length += chunk.length;
},
}),
);
if (length != file.end - file.start) {
logger.warn(
`failed to read enough from ${file.filename}. read ${length}, required: ${file.end - file.start}`,
);
throw createError({
statusCode: 500,
statusMessage: "Failed to read enough from stream.",
});
}
}
await h3.node.res.end();

97
server/internal/library/index.ts
View File

@ -15,12 +15,17 @@ import { GameNotFoundError, type LibraryProvider } from "./provider";
import { logger } from "../logging";
import type { GameModel } from "~/prisma/client/models";
export function createGameImportTaskId(libraryId: string, libraryPath: string) {
return btoa(`import:${libraryId}:${libraryPath}`);
}
export function createVersionImportTaskId(gameId: string, versionName: string) {
return btoa(`import:${gameId}:${versionName}`);
}
class LibraryManager {
private libraries: Map<string, LibraryProvider<unknown>> = new Map();
private gameImportLocks: Map<string, Array<string>> = new Map(); // Library ID to Library Path
private versionImportLocks: Map<string, Array<string>> = new Map(); // Game ID to Version Name
addLibrary(library: LibraryProvider<unknown>) {
this.libraries.set(library.id(), library);
}
@ -58,12 +63,10 @@ class LibraryManager {
for (const [id, library] of this.libraries.entries()) {
const providerGames = await library.listGames();
const locks = this.gameImportLocks.get(id) ?? [];
const providerUnimportedGames = providerGames.filter(
(libraryPath) =>
instanceGames[id] &&
!instanceGames[id][libraryPath] &&
!locks.includes(libraryPath),
!instanceGames[id]?.[libraryPath] &&
!taskHandler.hasTask(createGameImportTaskId(id, libraryPath)),
);
unimportedGames[id] = providerUnimportedGames;
}
@ -93,7 +96,7 @@ class LibraryManager {
const unimportedVersions = versions.filter(
(e) =>
game.versions.findIndex((v) => v.versionName == e) == -1 &&
!(this.versionImportLocks.get(game.id) ?? []).includes(e),
!taskHandler.hasTask(createVersionImportTaskId(game.id, e)),
);
return unimportedVersions;
} catch (e) {
@ -177,7 +180,8 @@ class LibraryManager {
for (const filename of files) {
const basename = path.basename(filename);
const dotLocation = filename.lastIndexOf(".");
const ext = dotLocation == -1 ? "" : filename.slice(dotLocation);
const ext =
dotLocation == -1 ? "" : filename.slice(dotLocation).toLowerCase();
for (const [platform, checkExts] of Object.entries(fileExts)) {
for (const checkExt of checkExts) {
if (checkExt != ext) continue;
@ -215,70 +219,6 @@ class LibraryManager {
}
*/
/**
* Locks the game so you can't be imported
* @param libraryId
* @param libraryPath
*/
async lockGame(libraryId: string, libraryPath: string) {
let games = this.gameImportLocks.get(libraryId);
if (!games) this.gameImportLocks.set(libraryId, (games = []));
if (!games.includes(libraryPath)) games.push(libraryPath);
this.gameImportLocks.set(libraryId, games);
}
/**
* Unlocks the game, call once imported
* @param libraryId
* @param libraryPath
*/
async unlockGame(libraryId: string, libraryPath: string) {
let games = this.gameImportLocks.get(libraryId);
if (!games) this.gameImportLocks.set(libraryId, (games = []));
if (games.includes(libraryPath))
games.splice(
games.findIndex((e) => e === libraryPath),
1,
);
this.gameImportLocks.set(libraryId, games);
}
/**
* Locks a version so it can't be imported
* @param gameId
* @param versionName
*/
async lockVersion(gameId: string, versionName: string) {
let versions = this.versionImportLocks.get(gameId);
if (!versions) this.versionImportLocks.set(gameId, (versions = []));
if (!versions.includes(versionName)) versions.push(versionName);
this.versionImportLocks.set(gameId, versions);
}
/**
* Unlocks the version, call once imported
* @param libraryId
* @param libraryPath
*/
async unlockVersion(gameId: string, versionName: string) {
let versions = this.versionImportLocks.get(gameId);
if (!versions) this.versionImportLocks.set(gameId, (versions = []));
if (versions.includes(gameId))
versions.splice(
versions.findIndex((e) => e === versionName),
1,
);
this.versionImportLocks.set(gameId, versions);
}
async importVersion(
gameId: string,
versionName: string,
@ -295,7 +235,7 @@ class LibraryManager {
umuId: string;
},
) {
const taskId = `import:${gameId}:${versionName}`;
const taskId = createVersionImportTaskId(gameId, versionName);
const platform = parsePlatform(metadata.platform);
if (!platform) return undefined;
@ -309,8 +249,6 @@ class LibraryManager {
const library = this.libraries.get(game.libraryId);
if (!library) return undefined;
await this.lockVersion(gameId, versionName);
taskHandler.create({
id: taskId,
taskGroup: "import:game",
@ -387,9 +325,6 @@ class LibraryManager {
progress(100);
},
async finally() {
await libraryManager.unlockVersion(gameId, versionName);
},
});
return taskId;
@ -403,7 +338,7 @@ class LibraryManager {
) {
const library = this.libraries.get(libraryId);
if (!library) return undefined;
return library.peekFile(game, version, filename);
return await library.peekFile(game, version, filename);
}
async readFile(
@ -415,7 +350,7 @@ class LibraryManager {
) {
const library = this.libraries.get(libraryId);
if (!library) return undefined;
return library.readFile(game, version, filename, options);
return await library.readFile(game, version, filename, options);
}
}

42
server/internal/library/providers/filesystem.ts
View File

@ -7,12 +7,14 @@ import {
import { LibraryBackend } from "~/prisma/client/enums";
import fs from "fs";
import path from "path";
import droplet from "@drop-oss/droplet";
import droplet, { DropletHandler } from "@drop-oss/droplet";
export const FilesystemProviderConfig = type({
baseDir: "string",
});
export const DROPLET_HANDLER = new DropletHandler();
export class FilesystemProvider
implements LibraryProvider<typeof FilesystemProviderConfig.infer>
{
@ -57,7 +59,7 @@ export class FilesystemProvider
const versionDirs = fs.readdirSync(gameDir);
const validVersionDirs = versionDirs.filter((e) => {
const fullDir = path.join(this.config.baseDir, game, e);
return droplet.hasBackendForPath(fullDir);
return DROPLET_HANDLER.hasBackendForPath(fullDir);
});
return validVersionDirs;
}
@ -65,7 +67,7 @@ export class FilesystemProvider
async versionReaddir(game: string, version: string): Promise<string[]> {
const versionDir = path.join(this.config.baseDir, game, version);
if (!fs.existsSync(versionDir)) throw new VersionNotFoundError();
return droplet.listFiles(versionDir);
return DROPLET_HANDLER.listFiles(versionDir);
}
async generateDropletManifest(
@ -77,10 +79,16 @@ export class FilesystemProvider
const versionDir = path.join(this.config.baseDir, game, version);
if (!fs.existsSync(versionDir)) throw new VersionNotFoundError();
const manifest = await new Promise<string>((r, j) =>
droplet.generateManifest(versionDir, progress, log, (err, result) => {
if (err) return j(err);
r(result);
}),
droplet.generateManifest(
DROPLET_HANDLER,
versionDir,
progress,
log,
(err, result) => {
if (err) return j(err);
r(result);
},
),
);
return manifest;
}
@ -88,7 +96,7 @@ export class FilesystemProvider
async peekFile(game: string, version: string, filename: string) {
const filepath = path.join(this.config.baseDir, game, version);
if (!fs.existsSync(filepath)) return undefined;
const stat = droplet.peekFile(filepath, filename);
const stat = DROPLET_HANDLER.peekFile(filepath, filename);
return { size: Number(stat) };
}
@ -100,13 +108,17 @@ export class FilesystemProvider
) {
const filepath = path.join(this.config.baseDir, game, version);
if (!fs.existsSync(filepath)) return undefined;
const stream = droplet.readFile(
filepath,
filename,
options?.start ? BigInt(options.start) : undefined,
options?.end ? BigInt(options.end) : undefined,
);
if (!stream) return undefined;
let stream;
while (!(stream instanceof ReadableStream)) {
const v = DROPLET_HANDLER.readFile(
filepath,
filename,
options?.start ? BigInt(options.start) : undefined,
options?.end ? BigInt(options.end) : undefined,
);
if (!v) return undefined;
stream = v.getStream() as ReadableStream<unknown>;
}
return stream;
}

25
server/internal/library/providers/flat.ts
View File

@ -5,6 +5,7 @@ import { LibraryBackend } from "~/prisma/client/enums";
import fs from "fs";
import path from "path";
import droplet from "@drop-oss/droplet";
import { DROPLET_HANDLER } from "./filesystem";
export const FlatFilesystemProviderConfig = type({
baseDir: "string",
@ -46,7 +47,7 @@ export class FlatFilesystemProvider
const versionDirs = fs.readdirSync(this.config.baseDir);
const validVersionDirs = versionDirs.filter((e) => {
const fullDir = path.join(this.config.baseDir, e);
return droplet.hasBackendForPath(fullDir);
return DROPLET_HANDLER.hasBackendForPath(fullDir);
});
return validVersionDirs;
}
@ -63,7 +64,7 @@ export class FlatFilesystemProvider
async versionReaddir(game: string, _version: string) {
const versionDir = path.join(this.config.baseDir, game);
if (!fs.existsSync(versionDir)) throw new VersionNotFoundError();
return droplet.listFiles(versionDir);
return DROPLET_HANDLER.listFiles(versionDir);
}
async generateDropletManifest(
@ -75,17 +76,23 @@ export class FlatFilesystemProvider
const versionDir = path.join(this.config.baseDir, game);
if (!fs.existsSync(versionDir)) throw new VersionNotFoundError();
const manifest = await new Promise<string>((r, j) =>
droplet.generateManifest(versionDir, progress, log, (err, result) => {
if (err) return j(err);
r(result);
}),
droplet.generateManifest(
DROPLET_HANDLER,
versionDir,
progress,
log,
(err, result) => {
if (err) return j(err);
r(result);
},
),
);
return manifest;
}
async peekFile(game: string, _version: string, filename: string) {
const filepath = path.join(this.config.baseDir, game);
if (!fs.existsSync(filepath)) return undefined;
const stat = droplet.peekFile(filepath, filename);
const stat = DROPLET_HANDLER.peekFile(filepath, filename);
return { size: Number(stat) };
}
async readFile(
@ -96,7 +103,7 @@ export class FlatFilesystemProvider
) {
const filepath = path.join(this.config.baseDir, game);
if (!fs.existsSync(filepath)) return undefined;
const stream = droplet.readFile(
const stream = DROPLET_HANDLER.readFile(
filepath,
filename,
options?.start ? BigInt(options.start) : undefined,
@ -104,6 +111,6 @@ export class FlatFilesystemProvider
);
if (!stream) return undefined;
return stream;
return stream.getStream();
}
}

9
server/internal/metadata/index.ts
View File

@ -18,7 +18,7 @@ import taskHandler, { wrapTaskContext } from "../tasks";
import { randomUUID } from "crypto";
import { fuzzy } from "fast-fuzzy";
import { logger } from "~/server/internal/logging";
import libraryManager from "../library";
import { createGameImportTaskId } from "../library";
import type { GameTagModel } from "~/prisma/client/models";
export class MissingMetadataProviderConfig extends Error {
@ -185,11 +185,9 @@ export class MetadataHandler {
});
if (existing) return undefined;
await libraryManager.lockGame(libraryId, libraryPath);
const gameId = randomUUID();
const taskId = `import:${gameId}`;
const taskId = createGameImportTaskId(libraryId, libraryPath);
await taskHandler.create({
name: `Import game "${result.name}" (${libraryPath})`,
id: taskId,
@ -280,9 +278,6 @@ export class MetadataHandler {
logger.info(`Finished game import.`);
progress(100);
},
async finally() {
await libraryManager.unlockGame(libraryId, libraryPath);
},
});
return taskId;

14
server/internal/tasks/index.ts
View File

@ -73,6 +73,8 @@ class TaskHandler {
}
async create(task: Task) {
if (this.hasTask(task.id)) throw new Error("Task with ID already exists.");
let updateCollectTimeout: NodeJS.Timeout | undefined;
let updateCollectResolves: Array<(value: unknown) => void> = [];
let logOffset: number = 0;
@ -206,8 +208,6 @@ class TaskHandler {
};
}
if (task.finally) await task.finally();
taskEntry.endTime = new Date().toISOString();
await updateAllClients();
@ -247,7 +247,10 @@ class TaskHandler {
) {
const task =
this.taskPool.get(taskId) ??
(await prisma.task.findUnique({ where: { id: taskId } }));
(await prisma.task.findFirst({
where: { id: taskId },
orderBy: { started: "desc" },
}));
if (!task) {
peer.send(
`error/${taskId}/Unknown task/Drop couldn't find the task you're looking for.`,
@ -324,6 +327,10 @@ class TaskHandler {
.toArray();
}
hasTask(id: string) {
return this.taskPool.has(id);
}
dailyTasks() {
return this.dailyScheduledTasks;
}
@ -429,7 +436,6 @@ export interface Task {
taskGroup: TaskGroup;
name: string;
run: (context: TaskRunContext) => Promise<void>;
finally?: () => Promise<void> | void;
acls: GlobalACL[];
}