feat: separate document data from document

apps/marketing/src/pages/api/stripe/webhook/index.ts

@@ -10,6 +10,7 @@ import { redis } from '@documenso/lib/server-only/redis';
 import { Stripe, stripe } from '@documenso/lib/server-only/stripe';
 import { prisma } from '@documenso/prisma';
 import {
+  DocumentDataType,
   DocumentStatus,
   FieldType,
   ReadStatus,
@@ -85,16 +86,33 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
 
       const now = new Date();
 
+      const bytes64 = readFileSync('./public/documenso-supporter-pledge.pdf').toString('base64');
+
       const document = await prisma.document.create({
         data: {
           title: 'Documenso Supporter Pledge.pdf',
           status: DocumentStatus.COMPLETED,
           userId: user.id,
-          document: readFileSync('./public/documenso-supporter-pledge.pdf').toString('base64'),
           created: now,
+          documentData: {
+            create: {
+              type: DocumentDataType.BYTES_64,
+              data: bytes64,
+              initialData: bytes64,
+            },
+          },
+        },
+        include: {
+          documentData: true,
         },
       });
 
+      const { documentData } = document;
+
+      if (!documentData) {
+        throw new Error(`Document ${document.id} has no document data`);
+      }
+
       const recipient = await prisma.recipient.create({
         data: {
           name: user.name ?? '',
@@ -122,16 +140,16 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       });
 
       if (signatureDataUrl) {
-        document.document = await insertImageInPDF(
+        documentData.data = await insertImageInPDF(
-          document.document,
+          documentData.data,
           signatureDataUrl,
           Number(field.positionX),
           Number(field.positionY),
           field.page,
         );
       } else {
-        document.document = await insertTextInPDF(
+        documentData.data = await insertTextInPDF(
-          document.document,
+          documentData.data,
           signatureText ?? '',
           Number(field.positionX),
           Number(field.positionY),
@@ -153,7 +171,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
             id: document.id,
           },
           data: {
-            document: document.document,
+            documentData: {
+              update: {
+                data: documentData.data,
+              },
+            },
           },
         }),
       ]);

apps/web/src/app/(dashboard)/documents/[id]/edit-document.tsx

@@ -4,7 +4,8 @@ import { useState } from 'react';
 
 import { useRouter } from 'next/navigation';
 
-import { Document, Field, Recipient, User } from '@documenso/prisma/client';
+import { Field, Recipient, User } from '@documenso/prisma/client';
+import { DocumentWithData } from '@documenso/prisma/types/document-with-data';
 import { cn } from '@documenso/ui/lib/utils';
 import { Card, CardContent } from '@documenso/ui/primitives/card';
 import { AddFieldsFormPartial } from '@documenso/ui/primitives/document-flow/add-fields';
@@ -28,7 +29,7 @@ import { completeDocument } from '~/components/forms/edit-document/add-subject.a
 export type EditDocumentFormProps = {
   className?: string;
   user: User;
-  document: Document;
+  document: DocumentWithData;
   recipients: Recipient[];
   fields: Field[];
 };
@@ -45,9 +46,11 @@ export const EditDocumentForm = ({
   const { toast } = useToast();
   const router = useRouter();
 
+  const { documentData } = document;
+
   const [step, setStep] = useState<EditDocumentStep>('signers');
 
-  const documentUrl = `data:application/pdf;base64,${document.document}`;
+  const documentUrl = `data:application/pdf;base64,${documentData?.data}`;
 
   const documentFlow: Record<EditDocumentStep, DocumentFlowStep> = {
     signers: {

apps/web/src/app/(dashboard)/documents/[id]/page.tsx

@@ -36,10 +36,12 @@ export default async function DocumentPage({ params }: DocumentPageProps) {
     userId: session.id,
   }).catch(() => null);
 
-  if (!document) {
+  if (!document || !document.documentData) {
     redirect('/documents');
   }
 
+  const { documentData } = document;
+
   const [recipients, fields] = await Promise.all([
     await getRecipientsForDocument({
       documentId,
@@ -91,7 +93,7 @@ export default async function DocumentPage({ params }: DocumentPageProps) {
 
       {document.status === InternalDocumentStatus.COMPLETED && (
         <div className="mx-auto mt-12 max-w-2xl">
-          <LazyPDFViewer document={`data:application/pdf;base64,${document.document}`} />
+          <LazyPDFViewer document={`data:application/pdf;base64,${documentData.data}`} />
         </div>
       )}
     </div>

apps/web/src/app/(dashboard)/documents/data-table-action-dropdown.tsx

@@ -14,8 +14,17 @@ import {
   XCircle,
 } from 'lucide-react';
 import { useSession } from 'next-auth/react';
+import { match } from 'ts-pattern';
 
-import { Document, DocumentStatus, Recipient, User } from '@documenso/prisma/client';
+import {
+  Document,
+  DocumentDataType,
+  DocumentStatus,
+  Recipient,
+  User,
+} from '@documenso/prisma/client';
+import { DocumentWithData } from '@documenso/prisma/types/document-with-data';
+import { trpc } from '@documenso/trpc/client';
 import {
   DropdownMenu,
   DropdownMenuContent,
@@ -47,17 +56,42 @@ export const DataTableActionDropdown = ({ row }: DataTableActionDropdownProps) =
   const isComplete = row.status === DocumentStatus.COMPLETED;
   // const isSigned = recipient?.signingStatus === SigningStatus.SIGNED;
 
-  const onDownloadClick = () => {
+  const onDownloadClick = async () => {
-    let decodedDocument = row.document;
+    let document: DocumentWithData | null = null;
 
-    try {
+    if (!recipient) {
-      decodedDocument = atob(decodedDocument);
+      document = await trpc.document.getDocumentById.query({
-    } catch (err) {
+        id: row.id,
-      // We're just going to ignore this error and try to download the document
+      });
-      console.error(err);
+    } else {
+      document = await trpc.document.getDocumentByToken.query({
+        token: recipient.token,
+      });
     }
 
-    const documentBytes = Uint8Array.from(decodedDocument.split('').map((c) => c.charCodeAt(0)));
+    const documentData = document?.documentData;
+
+    if (!documentData) {
+      return;
+    }
+
+    const documentBytes = await match(documentData.type)
+      .with(DocumentDataType.BYTES, () =>
+        Uint8Array.from(documentData.data, (c) => c.charCodeAt(0)),
+      )
+      .with(DocumentDataType.BYTES_64, () =>
+        Uint8Array.from(
+          atob(documentData.data)
+            .split('')
+            .map((c) => c.charCodeAt(0)),
+        ),
+      )
+      .with(DocumentDataType.S3_PATH, async () =>
+        fetch(documentData.data)
+          .then(async (res) => res.arrayBuffer())
+          .then((buffer) => new Uint8Array(buffer)),
+      )
+      .exhaustive();
 
     const blob = new Blob([documentBytes], {
       type: 'application/pdf',

apps/web/src/app/(signing)/sign/[token]/complete/page.tsx

@@ -30,10 +30,12 @@ export default async function CompletedSigningPage({
     token,
   }).catch(() => null);
 
-  if (!document) {
+  if (!document || !document.documentData) {
     return notFound();
   }
 
+  const { documentData } = document;
+
   const [fields, recipient] = await Promise.all([
     getFieldsForToken({ token }),
     getRecipientByToken({ token }),
@@ -91,7 +93,7 @@ export default async function CompletedSigningPage({
         <DownloadButton
           className="flex-1"
           fileName={document.title}
-          document={document.status === DocumentStatus.COMPLETED ? document.document : undefined}
+          document={document.status === DocumentStatus.COMPLETED ? documentData.data : undefined}
           disabled={document.status !== DocumentStatus.COMPLETED}
         />
       </div>

apps/web/src/app/(signing)/sign/[token]/page.tsx

@@ -40,13 +40,15 @@ export default async function SigningPage({ params: { token } }: SigningPageProp
     viewedDocument({ token }),
   ]);
 
-  if (!document) {
+  if (!document || !document.documentData) {
     return notFound();
   }
 
+  const { documentData } = document;
+
   const user = await getServerComponentSession();
 
-  const documentUrl = `data:application/pdf;base64,${document.document}`;
+  const documentUrl = `data:application/pdf;base64,${documentData.data}`;
 
   return (
     <SigningProvider email={recipient.email} fullName={recipient.name} signature={user?.signature}>

apps/web/src/pages/api/document/create.ts

@@ -5,7 +5,7 @@ import { readFileSync } from 'fs';
 
 import { getServerSession } from '@documenso/lib/next-auth/get-server-session';
 import { prisma } from '@documenso/prisma';
-import { DocumentStatus } from '@documenso/prisma/client';
+import { DocumentDataType, DocumentStatus } from '@documenso/prisma/client';
 
 import {
   TCreateDocumentRequestSchema,
@@ -55,12 +55,20 @@ export default async function handler(
 
     const fileBuffer = readFileSync(file.filepath);
 
+    const bytes64 = fileBuffer.toString('base64');
+
     const document = await prisma.document.create({
       data: {
         title: file.originalFilename ?? file.newFilename,
         status: DocumentStatus.DRAFT,
         userId: user.id,
-        document: fileBuffer.toString('base64'),
+        documentData: {
+          create: {
+            type: DocumentDataType.BYTES_64,
+            data: bytes64,
+            initialData: bytes64,
+          },
+        },
         created: new Date(),
       },
     });

apps/web/src/pages/api/stripe/webhook/index.ts

@@ -10,6 +10,7 @@ import { redis } from '@documenso/lib/server-only/redis';
 import { Stripe, stripe } from '@documenso/lib/server-only/stripe';
 import { prisma } from '@documenso/prisma';
 import {
+  DocumentDataType,
   DocumentStatus,
   FieldType,
   ReadStatus,
@@ -85,16 +86,33 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
 
       const now = new Date();
 
+      const bytes64 = readFileSync('./public/documenso-supporter-pledge.pdf').toString('base64');
+
       const document = await prisma.document.create({
         data: {
           title: 'Documenso Supporter Pledge.pdf',
           status: DocumentStatus.COMPLETED,
           userId: user.id,
-          document: readFileSync('./public/documenso-supporter-pledge.pdf').toString('base64'),
           created: now,
+          documentData: {
+            create: {
+              type: DocumentDataType.BYTES_64,
+              data: bytes64,
+              initialData: bytes64,
+            },
+          },
+        },
+        include: {
+          documentData: true,
         },
       });
 
+      const { documentData } = document;
+
+      if (!documentData) {
+        throw new Error(`Document ${document.id} has no document data`);
+      }
+
       const recipient = await prisma.recipient.create({
         data: {
           name: user.name ?? '',
@@ -122,16 +140,16 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
       });
 
       if (signatureDataUrl) {
-        document.document = await insertImageInPDF(
+        documentData.data = await insertImageInPDF(
-          document.document,
+          documentData.data,
           signatureDataUrl,
           field.positionX.toNumber(),
           field.positionY.toNumber(),
           field.page,
         );
       } else {
-        document.document = await insertTextInPDF(
+        documentData.data = await insertTextInPDF(
-          document.document,
+          documentData.data,
           signatureText ?? '',
           field.positionX.toNumber(),
           field.positionY.toNumber(),
@@ -153,7 +171,11 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse)
             id: document.id,
           },
           data: {
-            document: document.document,
+            documentData: {
+              update: {
+                data: documentData.data,
+              },
+            },
           },
         }),
       ]);

packages/lib/package.json

@@ -12,6 +12,8 @@
   ],
   "scripts": {},
   "dependencies": {
+    "@aws-sdk/s3-request-presigner": "^3.405.0",
+    "@aws-sdk/client-s3": "^3.405.0",
     "@documenso/email": "*",
     "@documenso/prisma": "*",
     "@next-auth/prisma-adapter": "1.0.7",

packages/lib/server-only/document/create-document.ts

@@ -0,0 +1,10 @@
+'use server';
+
+export type CreateDocumentOptions = {
+  userId: number;
+  fileName: string;
+};
+
+export const createDocument = () => {
+  //
+};

packages/lib/server-only/document/get-document-by-id.ts

@@ -11,5 +11,8 @@ export const getDocumentById = async ({ id, userId }: GetDocumentByIdOptions) =>
       id,
       userId,
     },
+    include: {
+      documentData: true,
+    },
   });
 };

packages/lib/server-only/document/get-document-by-token.ts

@@ -17,6 +17,7 @@ export const getDocumentAndSenderByToken = async ({
     },
     include: {
       User: true,
+      documentData: true,
     },
   });
 

packages/lib/server-only/document/seal-document.ts

@@ -18,8 +18,15 @@ export const sealDocument = async ({ documentId }: SealDocumentOptions) => {
     where: {
       id: documentId,
     },
+    include: {
+      documentData: true,
+    },
   });
 
+  if (!document.documentData) {
+    throw new Error(`Document ${document.id} has no document data`);
+  }
+
   if (document.status !== DocumentStatus.COMPLETED) {
     throw new Error(`Document ${document.id} has not been completed`);
   }
@@ -48,7 +55,7 @@ export const sealDocument = async ({ documentId }: SealDocumentOptions) => {
   }
 
   // !: Need to write the fields onto the document as a hard copy
-  const { document: pdfData } = document;
+  const { data: pdfData } = document.documentData;
 
   const doc = await PDFDocument.load(pdfData);
 
@@ -64,7 +71,11 @@ export const sealDocument = async ({ documentId }: SealDocumentOptions) => {
       status: DocumentStatus.COMPLETED,
     },
     data: {
-      document: Buffer.from(pdfBytes).toString('base64'),
+      documentData: {
+        update: {
+          data: Buffer.from(pdfBytes).toString('base64'),
+        },
+      },
     },
   });
 };

packages/prisma/migrations/20230907041233_add_document_data_table/migration.sql

@@ -0,0 +1,19 @@
+-- CreateEnum
+CREATE TYPE "DocumentDataType" AS ENUM ('S3_PATH', 'BYTES', 'BYTES_64');
+
+-- CreateTable
+CREATE TABLE "DocumentData" (
+    "id" TEXT NOT NULL,
+    "type" "DocumentDataType" NOT NULL,
+    "data" TEXT NOT NULL,
+    "initialData" TEXT NOT NULL,
+    "documentId" INTEGER NOT NULL,
+
+    CONSTRAINT "DocumentData_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "DocumentData_documentId_key" ON "DocumentData"("documentId");
+
+-- AddForeignKey
+ALTER TABLE "DocumentData" ADD CONSTRAINT "DocumentData_documentId_fkey" FOREIGN KEY ("documentId") REFERENCES "Document"("id") ON DELETE CASCADE ON UPDATE CASCADE;

packages/prisma/migrations/20230907074451_insert_old_data_into_document_data_table/migration.sql

@@ -0,0 +1,14 @@
+INSERT INTO
+  "DocumentData" ("id", "type", "data", "initialData", "documentId") (
+    SELECT
+      CAST(gen_random_uuid() AS TEXT),
+      'BYTES_64',
+      d."document",
+      d."document",
+      d."id"
+    FROM
+      "Document" d
+    WHERE
+      d."id" IS NOT NULL
+      AND d."document" IS NOT NULL
+  );

packages/prisma/migrations/20230907080056_add_created_at_and_updated_at_columns/migration.sql

@@ -0,0 +1,3 @@
+-- AlterTable
+ALTER TABLE "Document" ADD COLUMN     "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ADD COLUMN     "updatedAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP;

packages/prisma/migrations/20230907082622_remove_old_document_data/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - You are about to drop the column `document` on the `Document` table. All the data in the column will be lost.
+
+*/
+-- AlterTable
+ALTER TABLE "Document" DROP COLUMN "document";

packages/prisma/schema.prisma

@@ -91,15 +91,35 @@ enum DocumentStatus {
 }
 
 model Document {
-  id        Int            @id @default(autoincrement())
+  id           Int            @id @default(autoincrement())
-  created   DateTime       @default(now())
+  created      DateTime       @default(now())
-  userId    Int
+  userId       Int
-  User      User           @relation(fields: [userId], references: [id], onDelete: Cascade)
+  User         User           @relation(fields: [userId], references: [id], onDelete: Cascade)
-  title     String
+  title        String
-  status    DocumentStatus @default(DRAFT)
+  status       DocumentStatus @default(DRAFT)
-  document  String
+  Recipient    Recipient[]
-  Recipient Recipient[]
+  Field        Field[]
-  Field     Field[]
+  documentData DocumentData?
+  createdAt    DateTime       @default(now())
+  updatedAt    DateTime       @updatedAt @default(now())
+}
+
+enum DocumentDataType {
+  S3_PATH
+  BYTES
+  BYTES_64
+}
+
+model DocumentData {
+  id          String           @id @default(cuid())
+  type        DocumentDataType
+  data        String
+  initialData String
+  documentId  Int
+
+  Document Document @relation(fields: [documentId], references: [id], onDelete: Cascade)
+
+  @@unique([documentId])
 }
 
 enum ReadStatus {

packages/prisma/types/document-with-data.ts

@@ -0,0 +1,5 @@
+import { Document, DocumentData } from '@documenso/prisma/client';
+
+export type DocumentWithData = Document & {
+  documentData?: DocumentData | null;
+};

packages/trpc/server/document-router/router.ts

@@ -1,17 +1,63 @@
 import { TRPCError } from '@trpc/server';
 
+import { getDocumentById } from '@documenso/lib/server-only/document/get-document-by-id';
+import { getDocumentAndSenderByToken } from '@documenso/lib/server-only/document/get-document-by-token';
 import { sendDocument } from '@documenso/lib/server-only/document/send-document';
 import { setFieldsForDocument } from '@documenso/lib/server-only/field/set-fields-for-document';
 import { setRecipientsForDocument } from '@documenso/lib/server-only/recipient/set-recipients-for-document';
 
-import { authenticatedProcedure, router } from '../trpc';
+import { authenticatedProcedure, procedure, router } from '../trpc';
 import {
+  ZGetDocumentByIdQuerySchema,
+  ZGetDocumentByTokenQuerySchema,
   ZSendDocumentMutationSchema,
   ZSetFieldsForDocumentMutationSchema,
   ZSetRecipientsForDocumentMutationSchema,
 } from './schema';
 
 export const documentRouter = router({
+  getDocumentById: authenticatedProcedure
+    .input(ZGetDocumentByIdQuerySchema)
+    .query(async ({ input, ctx }) => {
+      try {
+        const { id } = input;
+
+        console.log({
+          id,
+          userId: ctx.user.id,
+        });
+
+        return await getDocumentById({
+          id,
+          userId: ctx.user.id,
+        });
+      } catch (err) {
+        console.error(err);
+
+        throw new TRPCError({
+          code: 'BAD_REQUEST',
+          message: 'We were unable to find this document. Please try again later.',
+        });
+      }
+    }),
+
+  getDocumentByToken: procedure.input(ZGetDocumentByTokenQuerySchema).query(async ({ input }) => {
+    try {
+      const { token } = input;
+
+      return await getDocumentAndSenderByToken({
+        token,
+      });
+    } catch (err) {
+      console.error(err);
+
+      throw new TRPCError({
+        code: 'BAD_REQUEST',
+        message: 'We were unable to find this document. Please try again later.',
+      });
+    }
+  }),
+
   setRecipientsForDocument: authenticatedProcedure
     .input(ZSetRecipientsForDocumentMutationSchema)
     .mutation(async ({ input, ctx }) => {

packages/trpc/server/document-router/schema.ts

@@ -2,6 +2,18 @@ import { z } from 'zod';
 
 import { FieldType } from '@documenso/prisma/client';
 
+export const ZGetDocumentByIdQuerySchema = z.object({
+  id: z.number().min(1),
+});
+
+export type TGetDocumentByIdQuerySchema = z.infer<typeof ZGetDocumentByIdQuerySchema>;
+
+export const ZGetDocumentByTokenQuerySchema = z.object({
+  token: z.string().min(1),
+});
+
+export type TGetDocumentByTokenQuerySchema = z.infer<typeof ZGetDocumentByTokenQuerySchema>;
+
 export const ZSetRecipientsForDocumentMutationSchema = z.object({
   documentId: z.number(),
   recipients: z.array(

packages/tsconfig/process-env.d.ts

@@ -13,6 +13,13 @@ declare namespace NodeJS {
     NEXT_PRIVATE_STRIPE_API_KEY: string;
     NEXT_PRIVATE_STRIPE_WEBHOOK_SECRET: string;
 
+    NEXT_PRIVATE_UPLOAD_TRANSPORT?: 'database' | 's3';
+    NEXT_PRIVATE_UPLOAD_ENDPOINT?: string;
+    NEXT_PRIVATE_UPLOAD_REGION?: string;
+    NEXT_PRIVATE_UPLOAD_BUCKET?: string;
+    NEXT_PRIVATE_UPLOAD_ACCESS_KEY_ID?: string;
+    NEXT_PRIVATE_UPLOAD_SECRET_ACCESS_KEY?: string;
+
     NEXT_PRIVATE_SMTP_TRANSPORT?: 'mailchannels' | 'smtp-auth' | 'smtp-api';
 
     NEXT_PRIVATE_MAILCHANNELS_API_KEY?: string;