feat: universal upload

Implementation of a universal upload allowing for multiple storage backends
starting with `database` and `s3`.

Allows clients to put and retrieve files from either client or server using
a blend of client and server actions.

packages/email/package.json

@@ -16,13 +16,12 @@
     "worker:test": "tsup worker/index.ts --format esm"
   },
   "dependencies": {
-    "@documenso/tsconfig": "*",
-    "@documenso/tailwind-config": "*",
-    "@documenso/ui": "*",
     "@react-email/components": "^0.0.7",
     "nodemailer": "^6.9.3"
   },
   "devDependencies": {
+    "@documenso/tsconfig": "*",
+    "@documenso/tailwind-config": "*",
     "@types/nodemailer": "^6.4.8",
     "tsup": "^7.1.0"
   }

packages/email/tailwind.config.js

@@ -4,8 +4,5 @@ const path = require('path');
 
 module.exports = {
   ...baseConfig,
-  content: [
-    `templates/**/*.{ts,tsx}`,
-    `${path.join(require.resolve('@documenso/ui'), '..')}/**/*.{ts,tsx}`,
-  ],
+  content: [`templates/**/*.{ts,tsx}`],
 };

packages/lib/constants/time.ts

@@ -0,0 +1,5 @@
+export const ONE_SECOND = 1000;
+export const ONE_MINUTE = ONE_SECOND * 60;
+export const ONE_HOUR = ONE_MINUTE * 60;
+export const ONE_DAY = ONE_HOUR * 24;
+export const ONE_WEEK = ONE_DAY * 7;

packages/lib/package.json

@@ -12,12 +12,15 @@
   ],
   "scripts": {},
   "dependencies": {
-    "@aws-sdk/s3-request-presigner": "^3.405.0",
-    "@aws-sdk/client-s3": "^3.405.0",
+    "@aws-sdk/client-s3": "^3.410.0",
+    "@aws-sdk/s3-request-presigner": "^3.410.0",
+    "@aws-sdk/signature-v4-crt": "^3.410.0",
     "@documenso/email": "*",
     "@documenso/prisma": "*",
     "@next-auth/prisma-adapter": "1.0.7",
     "@pdf-lib/fontkit": "^1.1.1",
+    "@scure/base": "^1.1.3",
+    "@sindresorhus/slugify": "^2.2.1",
     "@upstash/redis": "^1.20.6",
     "bcrypt": "^5.1.0",
     "luxon": "^3.4.0",

packages/lib/server-only/document-data/create-document-data.ts

@@ -0,0 +1,19 @@
+'use server';
+
+import { prisma } from '@documenso/prisma';
+import { DocumentDataType } from '@documenso/prisma/client';
+
+export type CreateDocumentDataOptions = {
+  type: DocumentDataType;
+  data: string;
+};
+
+export const createDocumentData = async ({ type, data }: CreateDocumentDataOptions) => {
+  return await prisma.documentData.create({
+    data: {
+      type,
+      data,
+      initialData: data,
+    },
+  });
+};

packages/lib/server-only/document/create-document.ts

@@ -1,10 +1,19 @@
 'use server';
 
+import { prisma } from '@documenso/prisma';
+
 export type CreateDocumentOptions = {
+  title: string;
   userId: number;
-  fileName: string;
+  documentDataId: string;
 };
 
-export const createDocument = () => {
-  //
+export const createDocument = async ({ userId, title, documentDataId }: CreateDocumentOptions) => {
+  return await prisma.document.create({
+    data: {
+      title,
+      documentDataId,
+      userId,
+    },
+  });
 };

packages/lib/server-only/document/seal-document.ts

@@ -1,10 +1,13 @@
 'use server';
 
+import path from 'node:path';
 import { PDFDocument } from 'pdf-lib';
 
 import { prisma } from '@documenso/prisma';
 import { DocumentStatus, SigningStatus } from '@documenso/prisma/client';
 
+import { getFile } from '../../universal/upload/get-file';
+import { putFile } from '../../universal/upload/put-file';
 import { insertFieldInPDF } from '../pdf/insert-field-in-pdf';
 
 export type SealDocumentOptions = {
@@ -23,7 +26,9 @@ export const sealDocument = async ({ documentId }: SealDocumentOptions) => {
     },
   });
 
-  if (!document.documentData) {
+  const { documentData } = document;
+
+  if (!documentData) {
     throw new Error(`Document ${document.id} has no document data`);
   }
 
@@ -55,7 +60,7 @@ export const sealDocument = async ({ documentId }: SealDocumentOptions) => {
   }
 
   // !: Need to write the fields onto the document as a hard copy
-  const { data: pdfData } = document.documentData;
+  const pdfData = await getFile(documentData);
 
   const doc = await PDFDocument.load(pdfData);
 
@@ -65,17 +70,20 @@ export const sealDocument = async ({ documentId }: SealDocumentOptions) => {
 
   const pdfBytes = await doc.save();
 
-  await prisma.document.update({
+  const { name, ext } = path.parse(document.title);
+
+  const { data: newData } = await putFile({
+    name: `${name}_signed${ext}`,
+    type: 'application/pdf',
+    arrayBuffer: async () => Promise.resolve(Buffer.from(pdfBytes)),
+  });
+
+  await prisma.documentData.update({
     where: {
-      id: document.id,
-      status: DocumentStatus.COMPLETED,
+      id: documentData.id,
     },
     data: {
-      documentData: {
-        update: {
-          data: Buffer.from(pdfBytes).toString('base64'),
-        },
-      },
+      data: newData,
     },
   });
 };

packages/lib/server-only/recipient/set-recipients-for-document.ts

@@ -1,8 +1,8 @@
-import { nanoid } from 'nanoid';
-
 import { prisma } from '@documenso/prisma';
 import { SendStatus, SigningStatus } from '@documenso/prisma/client';
 
+import { nanoid } from '../../universal/id';
+
 export interface SetRecipientsForDocumentOptions {
   userId: number;
   documentId: number;

packages/lib/tsconfig.json

@@ -1,5 +1,8 @@
 {
   "extends": "@documenso/tsconfig/react-library.json",
+  "compilerOptions": {
+    "types": ["@documenso/tsconfig/process-env.d.ts"]
+  },
   "include": ["**/*.ts", "**/*.tsx", "**/*.d.ts"],
   "exclude": ["dist", "build", "node_modules"]
 }

packages/lib/universal/id.ts

@@ -0,0 +1,5 @@
+import { customAlphabet } from 'nanoid';
+
+export const alphaid = customAlphabet('0123456789abcdefghijklmnopqrstuvwxyz', 10);
+
+export { nanoid } from 'nanoid';

packages/lib/universal/upload/delete-file.ts

@@ -0,0 +1,22 @@
+import { match } from 'ts-pattern';
+
+import { DocumentDataType } from '@documenso/prisma/client';
+
+import { deleteS3File } from './server-actions';
+
+export type DeleteFileOptions = {
+  type: DocumentDataType;
+  data: string;
+};
+
+export const deleteFile = async ({ type, data }: DeleteFileOptions) => {
+  return await match(type)
+    .with(DocumentDataType.S3_PATH, async () => deleteFileFromS3(data))
+    .otherwise(() => {
+      return;
+    });
+};
+
+const deleteFileFromS3 = async (key: string) => {
+  await deleteS3File(key);
+};

packages/lib/universal/upload/get-file.ts

@@ -0,0 +1,45 @@
+import { base64 } from '@scure/base';
+import { match } from 'ts-pattern';
+
+import { DocumentDataType } from '@documenso/prisma/client';
+
+import { getPresignGetUrl } from './server-actions';
+
+export type GetFileOptions = {
+  type: DocumentDataType;
+  data: string;
+};
+
+export const getFile = async ({ type, data }: GetFileOptions) => {
+  return await match(type)
+    .with(DocumentDataType.BYTES, () => getFileFromBytes(data))
+    .with(DocumentDataType.BYTES_64, () => getFileFromBytes64(data))
+    .with(DocumentDataType.S3_PATH, async () => getFileFromS3(data))
+    .exhaustive();
+};
+
+const getFileFromBytes = (data: string) => {
+  const encoder = new TextEncoder();
+
+  const binaryData = encoder.encode(data);
+
+  return binaryData;
+};
+
+const getFileFromBytes64 = (data: string) => {
+  const binaryData = base64.decode(data);
+
+  return binaryData;
+};
+
+const getFileFromS3 = async (key: string) => {
+  const { url } = await getPresignGetUrl(key);
+
+  const buffer = await fetch(url, {
+    method: 'GET',
+  }).then(async (res) => res.arrayBuffer());
+
+  const binaryData = new Uint8Array(buffer);
+
+  return binaryData;
+};

packages/lib/universal/upload/put-file.ts

@@ -0,0 +1,53 @@
+import { base64 } from '@scure/base';
+import { match } from 'ts-pattern';
+
+import { DocumentDataType } from '@documenso/prisma/client';
+
+import { createDocumentData } from '../../server-only/document-data/create-document-data';
+import { getPresignPostUrl } from './server-actions';
+
+type File = {
+  name: string;
+  type: string;
+  arrayBuffer: () => Promise<ArrayBuffer>;
+};
+
+export const putFile = async (file: File) => {
+  const { type, data } = await match(process.env.NEXT_PUBLIC_UPLOAD_TRANSPORT)
+    .with('s3', async () => putFileInS3(file))
+    .otherwise(async () => putFileInDatabase(file));
+
+  return await createDocumentData({ type, data });
+};
+
+const putFileInDatabase = async (file: File) => {
+  const contents = await file.arrayBuffer();
+
+  const binaryData = new Uint8Array(contents);
+
+  const asciiData = base64.encode(binaryData);
+
+  return {
+    type: DocumentDataType.BYTES_64,
+    data: asciiData,
+  };
+};
+
+const putFileInS3 = async (file: File) => {
+  const { url, key } = await getPresignPostUrl(file.name, file.type);
+
+  const body = await file.arrayBuffer();
+
+  await fetch(url, {
+    method: 'PUT',
+    headers: {
+      'Content-Type': 'application/octet-stream',
+    },
+    body,
+  });
+
+  return {
+    type: DocumentDataType.S3_PATH,
+    data: key,
+  };
+};

packages/lib/universal/upload/server-actions.ts

@@ -0,0 +1,104 @@
+'use server';
+
+import {
+  DeleteObjectCommand,
+  GetObjectCommand,
+  PutObjectCommand,
+  S3Client,
+} from '@aws-sdk/client-s3';
+import { getSignedUrl } from '@aws-sdk/s3-request-presigner';
+import slugify from '@sindresorhus/slugify';
+import path from 'node:path';
+
+import { ONE_HOUR, ONE_SECOND } from '../../constants/time';
+import { getServerComponentSession } from '../../next-auth/get-server-session';
+import { alphaid } from '../id';
+
+export const getPresignPostUrl = async (fileName: string, contentType: string) => {
+  const client = getS3Client();
+
+  const user = await getServerComponentSession();
+
+  // Get the basename and extension for the file
+  const { name, ext } = path.parse(fileName);
+
+  let key = `${alphaid(12)}/${slugify(name)}${ext}`;
+
+  if (user) {
+    key = `${user.id}/${key}`;
+  }
+
+  const putObjectCommand = new PutObjectCommand({
+    Bucket: process.env.NEXT_PRIVATE_UPLOAD_BUCKET,
+    Key: key,
+    ContentType: contentType,
+  });
+
+  const url = await getSignedUrl(client, putObjectCommand, {
+    expiresIn: ONE_HOUR / ONE_SECOND,
+  });
+
+  return { key, url };
+};
+
+export const getAbsolutePresignPostUrl = async (key: string) => {
+  const client = getS3Client();
+
+  const putObjectCommand = new PutObjectCommand({
+    Bucket: process.env.NEXT_PRIVATE_UPLOAD_BUCKET,
+    Key: key,
+  });
+
+  const url = await getSignedUrl(client, putObjectCommand, {
+    expiresIn: ONE_HOUR / ONE_SECOND,
+  });
+
+  return { key, url };
+};
+
+export const getPresignGetUrl = async (key: string) => {
+  const client = getS3Client();
+
+  const getObjectCommand = new GetObjectCommand({
+    Bucket: process.env.NEXT_PRIVATE_UPLOAD_BUCKET,
+    Key: key,
+  });
+
+  const url = await getSignedUrl(client, getObjectCommand, {
+    expiresIn: ONE_HOUR / ONE_SECOND,
+  });
+
+  return { key, url };
+};
+
+export const deleteS3File = async (key: string) => {
+  const client = getS3Client();
+
+  await client.send(
+    new DeleteObjectCommand({
+      Bucket: process.env.NEXT_PRIVATE_UPLOAD_BUCKET,
+      Key: key,
+    }),
+  );
+};
+
+const getS3Client = () => {
+  if (process.env.NEXT_PUBLIC_UPLOAD_TRANSPORT !== 's3') {
+    throw new Error('Invalid upload transport');
+  }
+
+  const hasCredentials =
+    process.env.NEXT_PRIVATE_UPLOAD_ACCESS_KEY_ID &&
+    process.env.NEXT_PRIVATE_UPLOAD_SECRET_ACCESS_KEY;
+
+  return new S3Client({
+    endpoint: process.env.NEXT_PRIVATE_UPLOAD_ENDPOINT || undefined,
+    region: process.env.NEXT_PRIVATE_UPLOAD_REGION || 'us-east-1',
+    credentials: hasCredentials
+      ? {
+          accessKeyId: String(process.env.NEXT_PRIVATE_UPLOAD_ACCESS_KEY_ID),
+          secretAccessKey: String(process.env.NEXT_PRIVATE_UPLOAD_SECRET_ACCESS_KEY),
+        }
+      : undefined,
+  });
+};

packages/lib/universal/upload/update-file.ts

@@ -0,0 +1,54 @@
+import { base64 } from '@scure/base';
+import { match } from 'ts-pattern';
+
+import { DocumentDataType } from '@documenso/prisma/client';
+
+import { getAbsolutePresignPostUrl } from './server-actions';
+
+export type UpdateFileOptions = {
+  type: DocumentDataType;
+  oldData: string;
+  newData: string;
+};
+
+export const updateFile = async ({ type, oldData, newData }: UpdateFileOptions) => {
+  return await match(type)
+    .with(DocumentDataType.BYTES, () => updateFileWithBytes(newData))
+    .with(DocumentDataType.BYTES_64, () => updateFileWithBytes64(newData))
+    .with(DocumentDataType.S3_PATH, async () => updateFileWithS3(oldData, newData))
+    .exhaustive();
+};
+
+const updateFileWithBytes = (data: string) => {
+  return {
+    type: DocumentDataType.BYTES,
+    data,
+  };
+};
+
+const updateFileWithBytes64 = (data: string) => {
+  const encoder = new TextEncoder();
+
+  const binaryData = encoder.encode(data);
+
+  const asciiData = base64.encode(binaryData);
+
+  return {
+    type: DocumentDataType.BYTES_64,
+    data: asciiData,
+  };
+};
+
+const updateFileWithS3 = async (key: string, data: string) => {
+  const { url } = await getAbsolutePresignPostUrl(key);
+
+  await fetch(url, {
+    method: 'PUT',
+    body: data,
+  });
+
+  return {
+    type: DocumentDataType.S3_PATH,
+    data: key,
+  };
+};

packages/prisma/migrations/20230912011344_reverse_document_data_relation/migration.sql

@@ -0,0 +1,23 @@
+-- DropForeignKey
+ALTER TABLE "DocumentData" DROP CONSTRAINT "DocumentData_documentId_fkey";
+
+-- DropIndex
+DROP INDEX "DocumentData_documentId_key";
+
+-- AlterTable
+ALTER TABLE "Document" ADD COLUMN "documentDataId" TEXT;
+
+-- Reverse relation foreign key ids
+UPDATE "Document" SET "documentDataId" = "DocumentData"."id" FROM "DocumentData" WHERE "Document"."id" = "DocumentData"."documentId";
+
+-- AlterColumn
+ALTER TABLE "Document" ALTER COLUMN "documentDataId" SET NOT NULL;
+
+-- AlterTable
+ALTER TABLE "DocumentData" DROP COLUMN "documentId";
+
+-- CreateIndex
+CREATE UNIQUE INDEX "Document_documentDataId_key" ON "Document"("documentDataId");
+
+-- AddForeignKey
+ALTER TABLE "Document" ADD CONSTRAINT "Document_documentDataId_fkey" FOREIGN KEY ("documentDataId") REFERENCES "DocumentData"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/prisma/schema.prisma

@@ -91,17 +91,20 @@ enum DocumentStatus {
 }
 
 model Document {
-  id           Int            @id @default(autoincrement())
-  created      DateTime       @default(now())
-  userId       Int
-  User         User           @relation(fields: [userId], references: [id], onDelete: Cascade)
-  title        String
-  status       DocumentStatus @default(DRAFT)
-  Recipient    Recipient[]
-  Field        Field[]
-  documentData DocumentData?
-  createdAt    DateTime       @default(now())
-  updatedAt    DateTime       @updatedAt @default(now())
+  id             Int            @id @default(autoincrement())
+  created        DateTime       @default(now())
+  userId         Int
+  User           User           @relation(fields: [userId], references: [id], onDelete: Cascade)
+  title          String
+  status         DocumentStatus @default(DRAFT)
+  Recipient      Recipient[]
+  Field          Field[]
+  documentDataId String
+  documentData   DocumentData   @relation(fields: [documentDataId], references: [id], onDelete: Cascade)
+  createdAt      DateTime       @default(now())
+  updatedAt      DateTime       @default(now()) @updatedAt
+
+  @@unique([documentDataId])
 }
 
 enum DocumentDataType {
@@ -115,11 +118,7 @@ model DocumentData {
   type        DocumentDataType
   data        String
   initialData String
-  documentId  Int
-
-  Document Document @relation(fields: [documentId], references: [id], onDelete: Cascade)
-
-  @@unique([documentId])
+  Document    Document?
 }
 
 enum ReadStatus {

packages/trpc/server/document-router/router.ts

@@ -1,5 +1,6 @@
 import { TRPCError } from '@trpc/server';
 
+import { createDocument } from '@documenso/lib/server-only/document/create-document';
 import { getDocumentById } from '@documenso/lib/server-only/document/get-document-by-id';
 import { getDocumentAndSenderByToken } from '@documenso/lib/server-only/document/get-document-by-token';
 import { sendDocument } from '@documenso/lib/server-only/document/send-document';
@@ -8,6 +9,7 @@ import { setRecipientsForDocument } from '@documenso/lib/server-only/recipient/s
 
 import { authenticatedProcedure, procedure, router } from '../trpc';
 import {
+  ZCreateDocumentMutationSchema,
   ZGetDocumentByIdQuerySchema,
   ZGetDocumentByTokenQuerySchema,
   ZSendDocumentMutationSchema,
@@ -22,11 +24,6 @@ export const documentRouter = router({
       try {
         const { id } = input;
 
-        console.log({
-          id,
-          userId: ctx.user.id,
-        });
-
         return await getDocumentById({
           id,
           userId: ctx.user.id,
@@ -58,6 +55,27 @@ export const documentRouter = router({
     }
   }),
 
+  createDocument: authenticatedProcedure
+    .input(ZCreateDocumentMutationSchema)
+    .mutation(async ({ input, ctx }) => {
+      try {
+        const { title, documentDataId } = input;
+
+        return await createDocument({
+          userId: ctx.user.id,
+          title,
+          documentDataId,
+        });
+      } catch (err) {
+        console.error(err);
+
+        throw new TRPCError({
+          code: 'BAD_REQUEST',
+          message: 'We were unable to create this document. Please try again later.',
+        });
+      }
+    }),
+
   setRecipientsForDocument: authenticatedProcedure
     .input(ZSetRecipientsForDocumentMutationSchema)
     .mutation(async ({ input, ctx }) => {

packages/trpc/server/document-router/schema.ts

@@ -14,6 +14,13 @@ export const ZGetDocumentByTokenQuerySchema = z.object({
 
 export type TGetDocumentByTokenQuerySchema = z.infer<typeof ZGetDocumentByTokenQuerySchema>;
 
+export const ZCreateDocumentMutationSchema = z.object({
+  title: z.string().min(1),
+  documentDataId: z.string().min(1),
+});
+
+export type TCreateDocumentMutationSchema = z.infer<typeof ZCreateDocumentMutationSchema>;
+
 export const ZSetRecipientsForDocumentMutationSchema = z.object({
   documentId: z.number(),
   recipients: z.array(

packages/tsconfig/process-env.d.ts

@@ -13,7 +13,7 @@ declare namespace NodeJS {
     NEXT_PRIVATE_STRIPE_API_KEY: string;
     NEXT_PRIVATE_STRIPE_WEBHOOK_SECRET: string;
 
-    NEXT_PRIVATE_UPLOAD_TRANSPORT?: 'database' | 's3';
+    NEXT_PUBLIC_UPLOAD_TRANSPORT?: 'database' | 's3';
     NEXT_PRIVATE_UPLOAD_ENDPOINT?: string;
     NEXT_PRIVATE_UPLOAD_REGION?: string;
     NEXT_PRIVATE_UPLOAD_BUCKET?: string;

packages/ui/package.json

@@ -15,6 +15,7 @@
     "lint": "eslint \"**/*.ts*\""
   },
   "devDependencies": {
+    "@documenso/tailwind-config": "*",
     "@documenso/tsconfig": "*",
     "@types/react": "18.2.18",
     "@types/react-dom": "18.2.7",
@@ -22,6 +23,7 @@
     "typescript": "^5.1.6"
   },
   "dependencies": {
+    "@documenso/lib": "*",
     "@radix-ui/react-accordion": "^1.1.1",
     "@radix-ui/react-alert-dialog": "^1.0.3",
     "@radix-ui/react-aspect-ratio": "^1.0.2",
@@ -51,7 +53,6 @@
     "class-variance-authority": "^0.6.0",
     "clsx": "^1.2.1",
     "cmdk": "^0.2.0",
-    "date-fns": "^2.30.0",
     "framer-motion": "^10.12.8",
     "lucide-react": "^0.214.0",
     "next": "13.4.12",
@@ -61,4 +62,4 @@
     "tailwind-merge": "^1.12.0",
     "tailwindcss-animate": "^1.0.5"
   }
-}
+}

packages/ui/primitives/document-flow/add-fields.tsx

@@ -5,12 +5,12 @@ import { useCallback, useEffect, useRef, useState } from 'react';
 import { Caveat } from 'next/font/google';
 
 import { Check, ChevronsUpDown, Info } from 'lucide-react';
-import { nanoid } from 'nanoid';
 import { useFieldArray, useForm } from 'react-hook-form';
 
 import { getBoundingClientRect } from '@documenso/lib/client-only/get-bounding-client-rect';
 import { useDocumentElement } from '@documenso/lib/client-only/hooks/use-document-element';
 import { PDF_VIEWER_PAGE_SELECTOR } from '@documenso/lib/constants/pdf-viewer';
+import { nanoid } from '@documenso/lib/universal/id';
 import { Field, FieldType, Recipient, SendStatus } from '@documenso/prisma/client';
 import { cn } from '@documenso/ui/lib/utils';
 import { Button } from '@documenso/ui/primitives/button';

packages/ui/primitives/document-flow/add-signers.tsx

@@ -5,9 +5,9 @@ import React, { useId } from 'react';
 import { zodResolver } from '@hookform/resolvers/zod';
 import { AnimatePresence, motion } from 'framer-motion';
 import { Plus, Trash } from 'lucide-react';
-import { nanoid } from 'nanoid';
 import { Controller, useFieldArray, useForm } from 'react-hook-form';
 
+import { nanoid } from '@documenso/lib/universal/id';
 import { Field, Recipient, SendStatus } from '@documenso/prisma/client';
 import { Button } from '@documenso/ui/primitives/button';
 import { FormErrorMessage } from '@documenso/ui/primitives/form/form-error-message';