feat: universal upload

Implementation of a universal upload allowing for multiple storage backends
starting with `database` and `s3`.

Allows clients to put and retrieve files from either client or server using
a blend of client and server actions.

apps/marketing/next.config.js

@@ -8,9 +8,16 @@ const { parsed: env } = require('dotenv').config({
 
 /** @type {import('next').NextConfig} */
 const config = {
+  experimental: {
+    serverActions: true,
+  },
   reactStrictMode: true,
   transpilePackages: ['@documenso/lib', '@documenso/prisma', '@documenso/trpc', '@documenso/ui'],
-  env,
+  modularizeImports: {
+    'lucide-react': {
+      transform: 'lucide-react/dist/esm/icons/{{ kebabCase member }}',
+    },
+  },
 };
 
 module.exports = withContentlayer(config);

apps/marketing/src/pages/api/stripe/webhook/index.ts

@@ -8,6 +8,8 @@ import { insertImageInPDF } from '@documenso/lib/server-only/pdf/insert-image-in
 import { insertTextInPDF } from '@documenso/lib/server-only/pdf/insert-text-in-pdf';
 import { redis } from '@documenso/lib/server-only/redis';
 import { Stripe, stripe } from '@documenso/lib/server-only/stripe';
+import { getFile } from '@documenso/lib/universal/upload/get-file';
+import { updateFile } from '@documenso/lib/universal/upload/update-file';
 import { prisma } from '@documenso/prisma';
 import {
   DocumentDataType,
@@ -88,19 +90,21 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
 
       const bytes64 = readFileSync('./public/documenso-supporter-pledge.pdf').toString('base64');
 
+      const { id: documentDataId } = await prisma.documentData.create({
+        data: {
+          type: DocumentDataType.BYTES_64,
+          data: bytes64,
+          initialData: bytes64,
+        },
+      });
+
       const document = await prisma.document.create({
         data: {
           title: 'Documenso Supporter Pledge.pdf',
           status: DocumentStatus.COMPLETED,
           userId: user.id,
           created: now,
-          documentData: {
-            create: {
-              type: DocumentDataType.BYTES_64,
-              data: bytes64,
-              initialData: bytes64,
-            },
-          },
+          documentDataId,
         },
         include: {
           documentData: true,
@@ -139,17 +143,21 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
         },
       });
 
+      let pdfData = await getFile(documentData).then((data) =>
+        Buffer.from(data).toString('base64'),
+      );
+
       if (signatureDataUrl) {
-        documentData.data = await insertImageInPDF(
-          documentData.data,
+        pdfData = await insertImageInPDF(
+          pdfData,
           signatureDataUrl,
           Number(field.positionX),
           Number(field.positionY),
           field.page,
         );
       } else {
-        documentData.data = await insertTextInPDF(
-          documentData.data,
+        pdfData = await insertTextInPDF(
+          pdfData,
           signatureText ?? '',
           Number(field.positionX),
           Number(field.positionY),
@@ -157,6 +165,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
         );
       }
 
+      const { data: newData } = await updateFile({
+        type: documentData.type,
+        oldData: documentData.initialData,
+        newData: Buffer.from(pdfData, 'base64').toString('binary'),
+      });
+
       await Promise.all([
         prisma.signature.create({
           data: {
@@ -166,16 +180,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
             typedSignature: signatureDataUrl ? '' : signatureText,
           },
         }),
-        prisma.document.update({
+        prisma.documentData.update({
           where: {
-            id: document.id,
+            id: documentData.id,
           },
           data: {
-            documentData: {
-              update: {
-                data: documentData.data,
-              },
-            },
+            data: newData,
           },
         }),
       ]);

apps/web/next.config.js

@@ -9,6 +9,7 @@ const { parsed: env } = require('dotenv').config({
 const config = {
   experimental: {
     serverActions: true,
+    serverActionsBodySizeLimit: '50mb',
   },
   reactStrictMode: true,
   transpilePackages: [
@@ -18,7 +19,6 @@ const config = {
     '@documenso/ui',
     '@documenso/email',
   ],
-  env,
   modularizeImports: {
     'lucide-react': {
       transform: 'lucide-react/dist/esm/icons/{{ kebabCase member }}',

apps/web/package.json

@@ -24,7 +24,6 @@
     "lucide-react": "^0.214.0",
     "luxon": "^3.4.0",
     "micro": "^10.0.1",
-    "nanoid": "^4.0.2",
     "next": "13.4.12",
     "next-auth": "4.22.3",
     "next-plausible": "^3.10.1",

apps/web/src/api/document/create/fetcher.ts

@@ -1,34 +0,0 @@
-import { useMutation } from '@tanstack/react-query';
-
-import { TCreateDocumentRequestSchema, ZCreateDocumentResponseSchema } from './types';
-
-export const useCreateDocument = () => {
-  return useMutation(async ({ file }: TCreateDocumentRequestSchema) => {
-    const formData = new FormData();
-
-    formData.set('file', file);
-
-    const response = await fetch('/api/document/create', {
-      method: 'POST',
-      body: formData,
-    });
-
-    const body = await response.json();
-
-    if (response.status !== 200) {
-      throw new Error('Failed to create document');
-    }
-
-    const safeBody = ZCreateDocumentResponseSchema.safeParse(body);
-
-    if (!safeBody.success) {
-      throw new Error('Failed to create document');
-    }
-
-    if ('error' in safeBody.data) {
-      throw new Error(safeBody.data.error);
-    }
-
-    return safeBody.data;
-  });
-};

apps/web/src/api/document/create/types.ts

@@ -1,19 +0,0 @@
-import { z } from 'zod';
-
-export const ZCreateDocumentRequestSchema = z.object({
-  file: z.instanceof(File),
-});
-
-export type TCreateDocumentRequestSchema = z.infer<typeof ZCreateDocumentRequestSchema>;
-
-export const ZCreateDocumentResponseSchema = z
-  .object({
-    id: z.number(),
-  })
-  .or(
-    z.object({
-      error: z.string(),
-    }),
-  );
-
-export type TCreateDocumentResponseSchema = z.infer<typeof ZCreateDocumentResponseSchema>;

apps/web/src/app/(dashboard)/documents/[id]/edit-document.tsx

@@ -32,6 +32,7 @@ export type EditDocumentFormProps = {
   document: DocumentWithData;
   recipients: Recipient[];
   fields: Field[];
+  dataUrl: string;
 };
 
 type EditDocumentStep = 'signers' | 'fields' | 'subject';
@@ -42,16 +43,13 @@ export const EditDocumentForm = ({
   recipients,
   fields,
   user: _user,
+  dataUrl,
 }: EditDocumentFormProps) => {
   const { toast } = useToast();
   const router = useRouter();
 
-  const { documentData } = document;
-
   const [step, setStep] = useState<EditDocumentStep>('signers');
 
-  const documentUrl = `data:application/pdf;base64,${documentData?.data}`;
-
   const documentFlow: Record<EditDocumentStep, DocumentFlowStep> = {
     signers: {
       title: 'Add Signers',
@@ -154,11 +152,11 @@ export const EditDocumentForm = ({
   return (
     <div className={cn('grid w-full grid-cols-12 gap-8', className)}>
       <Card
-        className="col-span-12 rounded-xl before:rounded-xl lg:col-span-6 xl:col-span-7"
+        className="relative col-span-12 rounded-xl before:rounded-xl lg:col-span-6 xl:col-span-7"
         gradient
       >
         <CardContent className="p-2">
-          <LazyPDFViewer document={documentUrl} />
+          <LazyPDFViewer document={dataUrl} />
         </CardContent>
       </Card>
 

apps/web/src/app/(dashboard)/documents/[id]/loadable-pdf-card.tsx

@@ -1,20 +0,0 @@
-'use client';
-
-import { Card, CardContent } from '@documenso/ui/primitives/card';
-import { LazyPDFViewer } from '@documenso/ui/primitives/lazy-pdf-viewer';
-import { PDFViewerProps } from '@documenso/ui/primitives/pdf-viewer';
-
-export type LoadablePDFCard = PDFViewerProps & {
-  className?: string;
-  pdfClassName?: string;
-};
-
-export const LoadablePDFCard = ({ className, pdfClassName, ...props }: LoadablePDFCard) => {
-  return (
-    <Card className={className} gradient {...props}>
-      <CardContent className="p-2">
-        <LazyPDFViewer className={pdfClassName} {...props} />
-      </CardContent>
-    </Card>
-  );
-};

apps/web/src/app/(dashboard)/documents/[id]/page.tsx

@@ -7,6 +7,7 @@ import { getRequiredServerComponentSession } from '@documenso/lib/next-auth/get-
 import { getDocumentById } from '@documenso/lib/server-only/document/get-document-by-id';
 import { getFieldsForDocument } from '@documenso/lib/server-only/field/get-fields-for-document';
 import { getRecipientsForDocument } from '@documenso/lib/server-only/recipient/get-recipients-for-document';
+import { getFile } from '@documenso/lib/universal/upload/get-file';
 import { DocumentStatus as InternalDocumentStatus } from '@documenso/prisma/client';
 import { LazyPDFViewer } from '@documenso/ui/primitives/lazy-pdf-viewer';
 
@@ -42,6 +43,12 @@ export default async function DocumentPage({ params }: DocumentPageProps) {
 
   const { documentData } = document;
 
+  const documentDataUrl = await getFile(documentData)
+    .then((buffer) => Buffer.from(buffer).toString('base64'))
+    .then((data) => `data:application/pdf;base64,${data}`);
+
+  console.log({ documentDataUrl: documentDataUrl.slice(0, 40) });
+
   const [recipients, fields] = await Promise.all([
     await getRecipientsForDocument({
       documentId,
@@ -88,12 +95,13 @@ export default async function DocumentPage({ params }: DocumentPageProps) {
           user={session}
           recipients={recipients}
           fields={fields}
+          dataUrl={documentDataUrl}
         />
       )}
 
       {document.status === InternalDocumentStatus.COMPLETED && (
         <div className="mx-auto mt-12 max-w-2xl">
-          <LazyPDFViewer document={`data:application/pdf;base64,${documentData.data}`} />
+          <LazyPDFViewer document={documentDataUrl} />
         </div>
       )}
     </div>

apps/web/src/app/(dashboard)/documents/upload-document.tsx

@@ -1,29 +1,45 @@
 'use client';
 
+import { useState } from 'react';
+
 import { useRouter } from 'next/navigation';
 
 import { Loader } from 'lucide-react';
 
+import { createDocumentData } from '@documenso/lib/server-only/document-data/create-document-data';
+import { putFile } from '@documenso/lib/universal/upload/put-file';
+import { trpc } from '@documenso/trpc/react';
 import { cn } from '@documenso/ui/lib/utils';
 import { DocumentDropzone } from '@documenso/ui/primitives/document-dropzone';
 import { useToast } from '@documenso/ui/primitives/use-toast';
 
-import { useCreateDocument } from '~/api/document/create/fetcher';
-
 export type UploadDocumentProps = {
   className?: string;
 };
 
 export const UploadDocument = ({ className }: UploadDocumentProps) => {
-  const { toast } = useToast();
   const router = useRouter();
 
-  const { isLoading, mutateAsync: createDocument } = useCreateDocument();
+  const { toast } = useToast();
+
+  const [isLoading, setIsLoading] = useState(false);
+
+  const { mutateAsync: createDocument } = trpc.document.createDocument.useMutation();
 
   const onFileDrop = async (file: File) => {
     try {
+      setIsLoading(true);
+
+      const { type, data } = await putFile(file);
+
+      const { id: documentDataId } = await createDocumentData({
+        type,
+        data,
+      });
+
       const { id } = await createDocument({
-        file: file,
+        title: file.name,
+        documentDataId,
       });
 
       toast({
@@ -41,6 +57,8 @@ export const UploadDocument = ({ className }: UploadDocumentProps) => {
         description: 'An error occurred while uploading your document.',
         variant: 'destructive',
       });
+    } finally {
+      setIsLoading(false);
     }
   };
 

apps/web/src/app/(signing)/sign/[token]/complete/download-button.tsx

@@ -1,55 +1,55 @@
 'use client';
 
-import { HTMLAttributes } from 'react';
+import { HTMLAttributes, useState } from 'react';
 
 import { Download } from 'lucide-react';
 
+import { getFile } from '@documenso/lib/universal/upload/get-file';
+import { DocumentData } from '@documenso/prisma/client';
 import { Button } from '@documenso/ui/primitives/button';
 
 export type DownloadButtonProps = HTMLAttributes<HTMLButtonElement> & {
   disabled?: boolean;
   fileName?: string;
-  document?: string;
+  documentData?: DocumentData;
 };
 
 export const DownloadButton = ({
   className,
   fileName,
-  document,
+  documentData,
   disabled,
   ...props
 }: DownloadButtonProps) => {
-  /**
-   * Convert the document from base64 to a blob and download it.
-   */
-  const onDownloadClick = () => {
-    if (!document) {
-      return;
-    }
-
-    let decodedDocument = document;
+  const [isLoading, setIsLoading] = useState(false);
 
+  const onDownloadClick = async () => {
     try {
-      decodedDocument = atob(document);
+      setIsLoading(true);
+
+      if (!documentData) {
+        return;
+      }
+
+      const bytes = await getFile(documentData);
+
+      const blob = new Blob([bytes], {
+        type: 'application/pdf',
+      });
+
+      const link = window.document.createElement('a');
+
+      link.href = window.URL.createObjectURL(blob);
+      link.download = fileName || 'document.pdf';
+
+      link.click();
+
+      window.URL.revokeObjectURL(link.href);
     } catch (err) {
-      // We're just going to ignore this error and try to download the document
       console.error(err);
+    } finally {
+      setIsLoading(false);
     }
-
-    const documentBytes = Uint8Array.from(decodedDocument.split('').map((c) => c.charCodeAt(0)));
-
-    const blob = new Blob([documentBytes], {
-      type: 'application/pdf',
-    });
-
-    const link = window.document.createElement('a');
-
-    link.href = window.URL.createObjectURL(blob);
-    link.download = fileName || 'document.pdf';
-
-    link.click();
-
-    window.URL.revokeObjectURL(link.href);
   };
 
   return (
@@ -57,8 +57,9 @@ export const DownloadButton = ({
       type="button"
       variant="outline"
       className={className}
-      disabled={disabled || !document}
+      disabled={disabled || !documentData}
       onClick={onDownloadClick}
+      loading={isLoading}
       {...props}
     >
       <Download className="mr-2 h-5 w-5" />

apps/web/src/app/(signing)/sign/[token]/complete/page.tsx

@@ -38,9 +38,13 @@ export default async function CompletedSigningPage({
 
   const [fields, recipient] = await Promise.all([
     getFieldsForToken({ token }),
-    getRecipientByToken({ token }),
+    getRecipientByToken({ token }).catch(() => null),
   ]);
 
+  if (!recipient) {
+    return notFound();
+  }
+
   const recipientName =
     recipient.name ||
     fields.find((field) => field.type === FieldType.NAME)?.customText ||
@@ -93,7 +97,7 @@ export default async function CompletedSigningPage({
         <DownloadButton
           className="flex-1"
           fileName={document.title}
-          document={document.status === DocumentStatus.COMPLETED ? documentData.data : undefined}
+          documentData={documentData}
           disabled={document.status !== DocumentStatus.COMPLETED}
         />
       </div>

apps/web/src/app/(signing)/sign/[token]/page.tsx

@@ -8,6 +8,7 @@ import { getDocumentAndSenderByToken } from '@documenso/lib/server-only/document
 import { viewedDocument } from '@documenso/lib/server-only/document/viewed-document';
 import { getFieldsForToken } from '@documenso/lib/server-only/field/get-fields-for-token';
 import { getRecipientByToken } from '@documenso/lib/server-only/recipient/get-recipient-by-token';
+import { getFile } from '@documenso/lib/universal/upload/get-file';
 import { FieldType } from '@documenso/prisma/client';
 import { Card, CardContent } from '@documenso/ui/primitives/card';
 import { ElementVisible } from '@documenso/ui/primitives/element-visible';
@@ -36,19 +37,21 @@ export default async function SigningPage({ params: { token } }: SigningPageProp
       token,
     }).catch(() => null),
     getFieldsForToken({ token }),
-    getRecipientByToken({ token }),
+    getRecipientByToken({ token }).catch(() => null),
     viewedDocument({ token }),
   ]);
 
-  if (!document || !document.documentData) {
+  if (!document || !document.documentData || !recipient) {
     return notFound();
   }
 
   const { documentData } = document;
 
-  const user = await getServerComponentSession();
+  const documentDataUrl = await getFile(documentData)
+    .then((buffer) => Buffer.from(buffer).toString('base64'))
+    .then((data) => `data:application/pdf;base64,${data}`);
 
-  const documentUrl = `data:application/pdf;base64,${documentData.data}`;
+  const user = await getServerComponentSession();
 
   return (
     <SigningProvider email={recipient.email} fullName={recipient.name} signature={user?.signature}>
@@ -69,7 +72,7 @@ export default async function SigningPage({ params: { token } }: SigningPageProp
             gradient
           >
             <CardContent className="p-2">
-              <LazyPDFViewer document={documentUrl} />
+              <LazyPDFViewer document={documentDataUrl} />
             </CardContent>
           </Card>
 

apps/web/src/pages/api/document/create.ts

@@ -1,96 +0,0 @@
-import { NextApiRequest, NextApiResponse } from 'next';
-
-import formidable, { type File } from 'formidable';
-import { readFileSync } from 'fs';
-
-import { getServerSession } from '@documenso/lib/next-auth/get-server-session';
-import { prisma } from '@documenso/prisma';
-import { DocumentDataType, DocumentStatus } from '@documenso/prisma/client';
-
-import {
-  TCreateDocumentRequestSchema,
-  TCreateDocumentResponseSchema,
-} from '~/api/document/create/types';
-
-export const config = {
-  api: {
-    bodyParser: false,
-  },
-};
-
-export type TFormidableCreateDocumentRequestSchema = {
-  file: File;
-};
-
-export default async function handler(
-  req: NextApiRequest,
-  res: NextApiResponse<TCreateDocumentResponseSchema>,
-) {
-  const user = await getServerSession({ req, res });
-
-  if (!user) {
-    return res.status(401).json({
-      error: 'Unauthorized',
-    });
-  }
-
-  try {
-    const form = formidable();
-
-    const { file } = await new Promise<TFormidableCreateDocumentRequestSchema>(
-      (resolve, reject) => {
-        form.parse(req, (err, fields, files) => {
-          if (err) {
-            reject(err);
-          }
-
-          // We had intended to do this with Zod but we can only validate it
-          // as a persistent file which does not include the properties that we
-          // need.
-          // eslint-disable-next-line @typescript-eslint/consistent-type-assertions, @typescript-eslint/no-explicit-any
-          resolve({ ...fields, ...files } as any);
-        });
-      },
-    );
-
-    const fileBuffer = readFileSync(file.filepath);
-
-    const bytes64 = fileBuffer.toString('base64');
-
-    const document = await prisma.document.create({
-      data: {
-        title: file.originalFilename ?? file.newFilename,
-        status: DocumentStatus.DRAFT,
-        userId: user.id,
-        documentData: {
-          create: {
-            type: DocumentDataType.BYTES_64,
-            data: bytes64,
-            initialData: bytes64,
-          },
-        },
-        created: new Date(),
-      },
-    });
-
-    return res.status(200).json({
-      id: document.id,
-    });
-  } catch (err) {
-    console.error(err);
-
-    return res.status(500).json({
-      error: 'Internal server error',
-    });
-  }
-}
-
-/**
- * This is a hack to ensure that the types are correct.
- */
-type FormidableSatisfiesCreateDocument =
-  keyof TCreateDocumentRequestSchema extends keyof TFormidableCreateDocumentRequestSchema
-    ? true
-    : never;
-
-true satisfies FormidableSatisfiesCreateDocument;

apps/web/src/pages/api/feature-flag/get.ts

@@ -1,9 +1,9 @@
 import { NextRequest, NextResponse } from 'next/server';
 
-import { nanoid } from 'nanoid';
 import { JWT, getToken } from 'next-auth/jwt';
 
 import { LOCAL_FEATURE_FLAGS, extractPostHogConfig } from '@documenso/lib/constants/feature-flags';
+import { nanoid } from '@documenso/lib/universal/id';
 
 import PostHogServerClient from '~/helpers/get-post-hog-server-client';
 

apps/web/src/pages/api/stripe/webhook/index.ts

@@ -88,19 +88,21 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
 
       const bytes64 = readFileSync('./public/documenso-supporter-pledge.pdf').toString('base64');
 
+      const { id: documentDataId } = await prisma.documentData.create({
+        data: {
+          type: DocumentDataType.BYTES_64,
+          data: bytes64,
+          initialData: bytes64,
+        },
+      });
+
       const document = await prisma.document.create({
         data: {
           title: 'Documenso Supporter Pledge.pdf',
           status: DocumentStatus.COMPLETED,
           userId: user.id,
           created: now,
-          documentData: {
-            create: {
-              type: DocumentDataType.BYTES_64,
-              data: bytes64,
-              initialData: bytes64,
-            },
-          },
+          documentDataId,
         },
         include: {
           documentData: true,