fix: normalize and flatten annotations (#1062)

This change flattens and normalizes annotation and widget layers within
the PDF document removing items that can be accidentally modified after
signing which would void the signature attached to the document.

Initially this change was just to assign to an ArcoForm object in the
document catalog if it existed but quickly turned into the above.

When annotations aren't flattened Adobe PDF will say that the signature
needs to be validated and upon doing so will become invalid due to the
annotation layers being touched.

To resolve this I set out to flatten and remove the annotations by
pulling out their normal appearances if they are present, converting
them into xobjects and then drawing those using the drawObject operator.

This resolves a critical issue the users experienced during the signing
flow when they had marked up a document using annotations in pdf
editors.

packages/lib/server-only/document/seal-document.ts

@@ -2,7 +2,7 @@
 
 import { nanoid } from 'nanoid';
 import path from 'node:path';
-import { PDFDocument, PDFSignature, rectangle } from 'pdf-lib';
+import { PDFDocument } from 'pdf-lib';
 
 import PostHogServerClient from '@documenso/lib/server-only/feature-flags/get-post-hog-server-client';
 import { DOCUMENT_AUDIT_LOG_TYPE } from '@documenso/lib/types/document-audit-logs';
@@ -15,7 +15,9 @@ import { signPdf } from '@documenso/signing';
 import type { RequestMetadata } from '../../universal/extract-request-metadata';
 import { getFile } from '../../universal/upload/get-file';
 import { putFile } from '../../universal/upload/put-file';
+import { flattenAnnotations } from '../pdf/flatten-annotations';
 import { insertFieldInPDF } from '../pdf/insert-field-in-pdf';
+import { normalizeSignatureAppearances } from '../pdf/normalize-signature-appearances';
 import { triggerWebhook } from '../webhooks/trigger/trigger-webhook';
 import { sendCompletedEmail } from './send-completed-email';
 
@@ -91,31 +93,10 @@ export const sealDocument = async ({
 
   const doc = await PDFDocument.load(pdfData);
 
-  const form = doc.getForm();
+  // Normalize and flatten layers that could cause issues with the signature
-
+  normalizeSignatureAppearances(doc);
-  // Remove old signatures
+  flattenAnnotations(doc);
-  for (const field of form.getFields()) {
+  doc.getForm().flatten();
-    if (field instanceof PDFSignature) {
-      field.acroField.getWidgets().forEach((widget) => {
-        widget.ensureAP();
-
-        try {
-          widget.getNormalAppearance();
-        } catch (e) {
-          const { context } = widget.dict;
-
-          const xobj = context.formXObject([rectangle(0, 0, 0, 0)]);
-
-          const streamRef = context.register(xobj);
-
-          widget.setNormalAppearance(streamRef);
-        }
-      });
-    }
-  }
-
-  // Flatten the form to stop annotation layers from appearing above documenso fields
-  form.flatten();
 
   for (const field of fields) {
     await insertFieldInPDF(doc, field);

packages/lib/server-only/pdf/flatten-annotations.ts

@@ -0,0 +1,63 @@
+import { PDFAnnotation, PDFRef } from 'pdf-lib';
+import {
+  PDFDict,
+  type PDFDocument,
+  PDFName,
+  drawObject,
+  popGraphicsState,
+  pushGraphicsState,
+  rotateInPlace,
+  translate,
+} from 'pdf-lib';
+
+export const flattenAnnotations = (document: PDFDocument) => {
+  const pages = document.getPages();
+
+  for (const page of pages) {
+    const annotations = page.node.Annots()?.asArray() ?? [];
+
+    annotations.forEach((annotation) => {
+      if (!(annotation instanceof PDFRef)) {
+        return;
+      }
+
+      const actualAnnotation = page.node.context.lookup(annotation);
+
+      if (!(actualAnnotation instanceof PDFDict)) {
+        return;
+      }
+
+      const pdfAnnot = PDFAnnotation.fromDict(actualAnnotation);
+
+      const appearance = pdfAnnot.ensureAP();
+
+      // Skip annotations without a normal appearance
+      if (!appearance.has(PDFName.of('N'))) {
+        return;
+      }
+
+      const normalAppearance = pdfAnnot.getNormalAppearance();
+      const rectangle = pdfAnnot.getRectangle();
+
+      if (!(normalAppearance instanceof PDFRef)) {
+        // Not sure how to get the reference to the normal appearance yet
+        // so we should skip this annotation for now
+        return;
+      }
+
+      const xobj = page.node.newXObject('FlatAnnot', normalAppearance);
+
+      const operators = [
+        pushGraphicsState(),
+        translate(rectangle.x, rectangle.y),
+        ...rotateInPlace({ ...rectangle, rotation: 0 }),
+        drawObject(xobj),
+        popGraphicsState(),
+      ].filter((op) => !!op);
+
+      page.pushOperators(...operators);
+
+      page.node.removeAnnot(annotation);
+    });
+  }
+};

packages/lib/server-only/pdf/normalize-signature-appearances.ts

@@ -0,0 +1,26 @@
+import type { PDFDocument } from 'pdf-lib';
+import { PDFSignature, rectangle } from 'pdf-lib';
+
+export const normalizeSignatureAppearances = (document: PDFDocument) => {
+  const form = document.getForm();
+
+  for (const field of form.getFields()) {
+    if (field instanceof PDFSignature) {
+      field.acroField.getWidgets().forEach((widget) => {
+        widget.ensureAP();
+
+        try {
+          widget.getNormalAppearance();
+        } catch {
+          const { context } = widget.dict;
+
+          const xobj = context.formXObject([rectangle(0, 0, 0, 0)]);
+
+          const streamRef = context.register(xobj);
+
+          widget.setNormalAppearance(streamRef);
+        }
+      });
+    }
+  }
+};

packages/signing/helpers/add-signing-placeholder.ts

@@ -1,5 +1,6 @@
 import {
   PDFArray,
+  PDFDict,
   PDFDocument,
   PDFHexString,
   PDFName,
@@ -16,7 +17,7 @@ export type AddSigningPlaceholderOptions = {
 
 export const addSigningPlaceholder = async ({ pdf }: AddSigningPlaceholderOptions) => {
   const doc = await PDFDocument.load(pdf);
-  const pages = doc.getPages();
+  const [firstPage] = doc.getPages();
 
   const byteRange = PDFArray.withContext(doc.context);
 
@@ -25,7 +26,8 @@ export const addSigningPlaceholder = async ({ pdf }: AddSigningPlaceholderOption
   byteRange.push(PDFName.of(BYTE_RANGE_PLACEHOLDER));
   byteRange.push(PDFName.of(BYTE_RANGE_PLACEHOLDER));
 
-  const signature = doc.context.obj({
+  const signature = doc.context.register(
+    doc.context.obj({
       Type: 'Sig',
       Filter: 'Adobe.PPKLite',
       SubFilter: 'adbe.pkcs7.detached',
@@ -33,56 +35,62 @@ export const addSigningPlaceholder = async ({ pdf }: AddSigningPlaceholderOption
       Contents: PDFHexString.fromText(' '.repeat(8192)),
       Reason: PDFString.of('Signed by Documenso'),
       M: PDFString.fromDate(new Date()),
-  });
+    }),
+  );
 
-  const signatureRef = doc.context.register(signature);
+  const widget = doc.context.register(
-
+    doc.context.obj({
-  const widget = doc.context.obj({
       Type: 'Annot',
       Subtype: 'Widget',
       FT: 'Sig',
       Rect: [0, 0, 0, 0],
-    V: signatureRef,
+      V: signature,
       T: PDFString.of('Signature1'),
       F: 4,
-    P: pages[0].ref,
+      P: firstPage.ref,
-  });
+      AP: doc.context.obj({
-
+        N: doc.context.register(doc.context.formXObject([rectangle(0, 0, 0, 0)])),
-  const xobj = widget.context.formXObject([rectangle(0, 0, 0, 0)]);
+      }),
-
-  const streamRef = widget.context.register(xobj);
-
-  widget.set(PDFName.of('AP'), widget.context.obj({ N: streamRef }));
-
-  const widgetRef = doc.context.register(widget);
-
-  let widgets = pages[0].node.get(PDFName.of('Annots'));
-
-  if (widgets instanceof PDFArray) {
-    widgets.push(widgetRef);
-  } else {
-    const newWidgets = PDFArray.withContext(doc.context);
-
-    newWidgets.push(widgetRef);
-
-    pages[0].node.set(PDFName.of('Annots'), newWidgets);
-
-    widgets = pages[0].node.get(PDFName.of('Annots'));
-  }
-
-  if (!widgets) {
-    throw new Error('No widgets');
-  }
-
-  pages[0].node.set(PDFName.of('Annots'), widgets);
-
-  doc.catalog.set(
-    PDFName.of('AcroForm'),
-    doc.context.obj({
-      SigFlags: 3,
-      Fields: [widgetRef],
     }),
   );
 
+  let widgets: PDFArray;
+
+  try {
+    widgets = firstPage.node.lookup(PDFName.of('Annots'), PDFArray);
+  } catch {
+    widgets = PDFArray.withContext(doc.context);
+
+    firstPage.node.set(PDFName.of('Annots'), widgets);
+  }
+
+  widgets.push(widget);
+
+  let arcoForm: PDFDict;
+
+  try {
+    arcoForm = doc.catalog.lookup(PDFName.of('AcroForm'), PDFDict);
+  } catch {
+    arcoForm = doc.context.obj({
+      Fields: PDFArray.withContext(doc.context),
+    });
+
+    doc.catalog.set(PDFName.of('AcroForm'), arcoForm);
+  }
+
+  let fields: PDFArray;
+
+  try {
+    fields = arcoForm.lookup(PDFName.of('Fields'), PDFArray);
+  } catch {
+    fields = PDFArray.withContext(doc.context);
+
+    arcoForm.set(PDFName.of('Fields'), fields);
+  }
+
+  fields.push(widget);
+
+  arcoForm.set(PDFName.of('SigFlags'), PDFNumber.of(3));
+
   return Buffer.from(await doc.save({ useObjectStreams: false }));
 };