feat: add job queue

docker/development/compose.yml

@@ -11,6 +11,17 @@ services:
     ports:
       - 54320:5432
 
+  queue:
+    image: postgres:15
+    container_name: queue
+    user: postgres
+    command: -c jit=off
+    environment:
+      POSTGRES_PASSWORD: password
+      POSTGRES_DB: queue
+    ports:
+      - 54321:5432
+
   inbucket:
     image: inbucket/inbucket
     container_name: mailserver

package-lock.json

@@ -8443,6 +8443,18 @@
         "node": ">= 6.0.0"
       }
     },
+    "node_modules/aggregate-error": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz",
+      "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==",
+      "dependencies": {
+        "clean-stack": "^2.0.0",
+        "indent-string": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/ajv": {
       "version": "8.12.0",
       "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.12.0.tgz",
@@ -9417,6 +9429,14 @@
       "resolved": "https://registry.npmjs.org/classnames/-/classnames-2.5.1.tgz",
       "integrity": "sha512-saHYOzhIQs6wy2sVxTM6bUDsQO4F50V9RQ22qBpEdCW+I+/Wmke2HOl6lS6dTpdxVhb88/I6+Hs+438c3lfUow=="
     },
+    "node_modules/clean-stack": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz",
+      "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==",
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/cli-cursor": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-4.0.0.tgz",
@@ -10192,6 +10212,17 @@
       "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
       "devOptional": true
     },
+    "node_modules/cron-parser": {
+      "version": "4.9.0",
+      "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz",
+      "integrity": "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==",
+      "dependencies": {
+        "luxon": "^3.2.1"
+      },
+      "engines": {
+        "node": ">=12.0.0"
+      }
+    },
     "node_modules/cross-fetch": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-4.0.0.tgz",
@@ -10558,6 +10589,17 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/delay": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/delay/-/delay-5.0.0.tgz",
+      "integrity": "sha512-ReEBKkIfe4ya47wlPYf/gu5ib6yUG0/Aez0JQZQz94kiWtRQvZIQbTiehsnwHvLSWJnQdhVeqYue7Id1dKr0qw==",
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/delayed-stream": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
@@ -13672,7 +13714,6 @@
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz",
       "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==",
-      "dev": true,
       "engines": {
         "node": ">=8"
       }
@@ -17250,6 +17291,20 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/p-map": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz",
+      "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==",
+      "dependencies": {
+        "aggregate-error": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/p-try": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
@@ -17559,6 +17614,124 @@
         "is-reference": "^3.0.0"
       }
     },
+    "node_modules/pg": {
+      "version": "8.11.5",
+      "resolved": "https://registry.npmjs.org/pg/-/pg-8.11.5.tgz",
+      "integrity": "sha512-jqgNHSKL5cbDjFlHyYsCXmQDrfIX/3RsNwYqpd4N0Kt8niLuNoRNH+aazv6cOd43gPh9Y4DjQCtb+X0MH0Hvnw==",
+      "dependencies": {
+        "pg-connection-string": "^2.6.4",
+        "pg-pool": "^3.6.2",
+        "pg-protocol": "^1.6.1",
+        "pg-types": "^2.1.0",
+        "pgpass": "1.x"
+      },
+      "engines": {
+        "node": ">= 8.0.0"
+      },
+      "optionalDependencies": {
+        "pg-cloudflare": "^1.1.1"
+      },
+      "peerDependencies": {
+        "pg-native": ">=3.0.1"
+      },
+      "peerDependenciesMeta": {
+        "pg-native": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/pg-boss": {
+      "version": "9.0.3",
+      "resolved": "https://registry.npmjs.org/pg-boss/-/pg-boss-9.0.3.tgz",
+      "integrity": "sha512-cUWUiv3sr563yNy0nCZ25Tv5U0m59Y9MhX/flm0vTR012yeVCrqpfboaZP4xFOQPdWipMJpuu4g94HR0SncTgw==",
+      "dependencies": {
+        "cron-parser": "^4.0.0",
+        "delay": "^5.0.0",
+        "lodash.debounce": "^4.0.8",
+        "p-map": "^4.0.0",
+        "pg": "^8.5.1",
+        "serialize-error": "^8.1.0",
+        "uuid": "^9.0.0"
+      },
+      "engines": {
+        "node": ">=16"
+      }
+    },
+    "node_modules/pg-boss/node_modules/uuid": {
+      "version": "9.0.1",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz",
+      "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==",
+      "funding": [
+        "https://github.com/sponsors/broofa",
+        "https://github.com/sponsors/ctavan"
+      ],
+      "bin": {
+        "uuid": "dist/bin/uuid"
+      }
+    },
+    "node_modules/pg-cloudflare": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz",
+      "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==",
+      "optional": true
+    },
+    "node_modules/pg-connection-string": {
+      "version": "2.6.4",
+      "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.4.tgz",
+      "integrity": "sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA=="
+    },
+    "node_modules/pg-int8": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz",
+      "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==",
+      "engines": {
+        "node": ">=4.0.0"
+      }
+    },
+    "node_modules/pg-pool": {
+      "version": "3.6.2",
+      "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.2.tgz",
+      "integrity": "sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg==",
+      "peerDependencies": {
+        "pg": ">=8.0"
+      }
+    },
+    "node_modules/pg-protocol": {
+      "version": "1.6.1",
+      "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.1.tgz",
+      "integrity": "sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg=="
+    },
+    "node_modules/pg-types": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz",
+      "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==",
+      "dependencies": {
+        "pg-int8": "1.0.1",
+        "postgres-array": "~2.0.0",
+        "postgres-bytea": "~1.0.0",
+        "postgres-date": "~1.0.4",
+        "postgres-interval": "^1.1.0"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/pgpass": {
+      "version": "1.0.5",
+      "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz",
+      "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==",
+      "dependencies": {
+        "split2": "^4.1.0"
+      }
+    },
+    "node_modules/pgpass/node_modules/split2": {
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",
+      "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==",
+      "engines": {
+        "node": ">= 10.x"
+      }
+    },
     "node_modules/picocolors": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz",
@@ -17817,6 +17990,41 @@
       "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz",
       "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ=="
     },
+    "node_modules/postgres-array": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz",
+      "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==",
+      "engines": {
+        "node": ">=4"
+      }
+    },
+    "node_modules/postgres-bytea": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.0.tgz",
+      "integrity": "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w==",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/postgres-date": {
+      "version": "1.0.7",
+      "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz",
+      "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/postgres-interval": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz",
+      "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==",
+      "dependencies": {
+        "xtend": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/posthog-js": {
       "version": "1.93.2",
       "resolved": "https://registry.npmjs.org/posthog-js/-/posthog-js-1.93.2.tgz",
@@ -24926,6 +25134,7 @@
         "next-auth": "4.24.5",
         "oslo": "^0.17.0",
         "pdf-lib": "^1.17.1",
+        "pg-boss": "^9.0.3",
         "react": "18.2.0",
         "remeda": "^1.27.1",
         "stripe": "^12.7.0",

packages/lib/package.json

@@ -27,18 +27,19 @@
     "@next-auth/prisma-adapter": "1.0.7",
     "@noble/ciphers": "0.4.0",
     "@noble/hashes": "1.3.2",
+    "@node-rs/bcrypt": "^1.10.0",
     "@pdf-lib/fontkit": "^1.1.1",
     "@scure/base": "^1.1.3",
     "@sindresorhus/slugify": "^2.2.1",
     "@upstash/redis": "^1.20.6",
     "@vvo/tzdb": "^6.117.0",
-    "@node-rs/bcrypt": "^1.10.0",
     "luxon": "^3.4.0",
     "nanoid": "^4.0.2",
     "next": "14.0.3",
     "next-auth": "4.24.5",
     "oslo": "^0.17.0",
     "pdf-lib": "^1.17.1",
+    "pg-boss": "^9.0.3",
     "react": "18.2.0",
     "remeda": "^1.27.1",
     "stripe": "^12.7.0",

packages/lib/server-only/document/seal-document.ts

@@ -18,8 +18,8 @@ import { putFile } from '../../universal/upload/put-file';
 import { flattenAnnotations } from '../pdf/flatten-annotations';
 import { insertFieldInPDF } from '../pdf/insert-field-in-pdf';
 import { normalizeSignatureAppearances } from '../pdf/normalize-signature-appearances';
+import { queueJob } from '../queue/job';
 import { triggerWebhook } from '../webhooks/trigger/trigger-webhook';
-import { sendCompletedEmail } from './send-completed-email';
 
 export type SealDocumentOptions = {
   documentId: number;
@@ -150,7 +150,10 @@ export const sealDocument = async ({
   });
 
   if (sendEmail && !isResealing) {
-    await sendCompletedEmail({ documentId, requestMetadata });
+    await queueJob({
+      job: 'send-completed-email',
+      args: { documentId, requestMetadata },
+    });
   }
 
   await triggerWebhook({

packages/lib/server-only/queue/index.ts

@@ -0,0 +1,52 @@
+import type { WorkHandler } from 'pg-boss';
+import PgBoss from 'pg-boss';
+
+import { jobHandlers } from './job';
+
+type QueueState = {
+  isReady: boolean;
+  queue: PgBoss | null;
+};
+
+let initPromise: Promise<PgBoss> | null = null;
+const state: QueueState = {
+  isReady: false,
+  queue: null,
+};
+
+export async function initQueue() {
+  if (state.isReady) {
+    return state.queue as PgBoss;
+  }
+
+  if (initPromise) {
+    return initPromise;
+  }
+
+  initPromise = (async () => {
+    const queue = new PgBoss({
+      connectionString: 'postgres://postgres:password@127.0.0.1:54321/queue',
+
+      schema: 'documenso_queue',
+    });
+
+    try {
+      await queue.start();
+    } catch (error) {
+      console.error('Failed to start queue', error);
+    }
+
+    await Promise.all(
+      Object.entries(jobHandlers).map(async ([job, jobHandler]) => {
+        await queue.work(job, jobHandler as WorkHandler<unknown>);
+      }),
+    );
+
+    state.isReady = true;
+    state.queue = queue;
+
+    return queue;
+  })();
+
+  return initPromise;
+}

packages/lib/server-only/queue/job.ts

@@ -0,0 +1,34 @@
+import type { WorkHandler } from 'pg-boss';
+
+import { initQueue } from '.';
+import {
+  type SendDocumentOptions as SendCompletedDocumentOptions,
+  sendCompletedEmail,
+} from '../document/send-completed-email';
+
+type JobOptions = {
+  'send-completed-email': SendCompletedDocumentOptions;
+};
+
+export const jobHandlers: {
+  [K in keyof JobOptions]: WorkHandler<JobOptions[K]>;
+} = {
+  'send-completed-email': async ({ data }) => {
+    await sendCompletedEmail({
+      documentId: data.documentId,
+      requestMetadata: data.requestMetadata,
+    });
+  },
+};
+
+export const queueJob = async ({
+  job,
+  args,
+}: {
+  job: keyof JobOptions;
+  args?: JobOptions[keyof JobOptions];
+}) => {
+  const queue = await initQueue();
+
+  await queue.send(job, args ?? {});
+};

turbo.json

@@ -93,6 +93,7 @@
     "NEXT_PRIVATE_STRIPE_API_KEY",
     "NEXT_PRIVATE_STRIPE_WEBHOOK_SECRET",
     "NEXT_PRIVATE_GITHUB_TOKEN",
+    "NEXT_RUNTIME",
     "CI",
     "VERCEL",
     "VERCEL_ENV",