diff --git a/.agents/plans/warm-purple-flower-custom-email-domain-sync-and-recovery.md b/.agents/plans/warm-purple-flower-custom-email-domain-sync-and-recovery.md new file mode 100644 index 000000000..c0a566169 --- /dev/null +++ b/.agents/plans/warm-purple-flower-custom-email-domain-sync-and-recovery.md @@ -0,0 +1,263 @@ +--- +date: 2026-02-24 +title: Custom Email Domain Sync And Recovery +--- + +## Problem Statement + +Custom email domains configured via AWS SES can get stuck in a `PENDING` state or fail validation silently. Currently, there is **no automated verification** -- users must manually click "Sync" in the UI to check domain status. If a domain fails to validate, the only option is to delete it and recreate it, which generates new DKIM keys and requires the user to update their DNS records. + +### Current Pain Points + +1. **No background sync** -- Domain verification status is never checked automatically; users must manually click "Sync" +2. **Stuck domains** -- Domains can remain in `PENDING` state indefinitely with no alerting or auto-recovery +3. **Failed recovery requires DNS changes** -- Deleting and recreating a domain generates new keys, forcing the user to update DNS records +4. **No visibility into failure duration** -- There's no tracking of how long a domain has been pending + +## Proposed Solution + +### 1. Hourly Background Sync Job + +Create a new cron job (`internal.sync-email-domains`) that runs every hour to automatically verify all `PENDING` email domains. + +**Job Definition:** `packages/lib/jobs/definitions/internal/sync-email-domains.ts` +**Job Handler:** `packages/lib/jobs/definitions/internal/sync-email-domains.handler.ts` + +**Pattern:** Follow the existing `cleanup-rate-limits` cron job pattern: + +- `cron: '0 * * * *'` (every hour, on the hour) +- Empty `z.object({})` schema (no payload needed) +- Register in `packages/lib/jobs/client.ts` + +**Handler Logic:** + +1. Query all `EmailDomain` records with `status: 'PENDING'` +2. For each domain, call `verifyEmailDomain(emailDomainId)` which: + - Calls AWS SES `GetEmailIdentityCommand` to check current verification status + - Updates DB status to `ACTIVE` if verified, keeps `PENDING` otherwise +3. Log results via `io.logger` (how many checked, how many transitioned to ACTIVE) +4. Process domains in batches to avoid overwhelming SES API rate limits +5. Add error handling per-domain so one failure doesn't stop the entire sweep + +### 2. Schema Changes -- Track Pending Duration + +Add a `lastVerifiedAt` column to the `EmailDomain` model to track when verification was last attempted, enabling "stale domain" detection. + +**File:** `packages/prisma/schema.prisma` + +```prisma +model EmailDomain { + // ... existing fields ... + lastVerifiedAt DateTime? // Last time verification was checked against SES +} +``` + +**Migration:** Create a new Prisma migration for this column addition. + +**Updates needed:** + +- `verify-email-domain.ts` -- Update `lastVerifiedAt` when verification is checked +- The sync job handler -- Use `lastVerifiedAt` to avoid re-checking domains that were just verified + +### 3. Domain Re-registration (Recovery) -- Delete & Recreate in SES Without Changing Keys + +Add a new "Re-register" action that deletes the SES identity and recreates it using the **same** DKIM key pair stored in the database, so the user's DNS records remain valid. + +#### 3a. New Service Function + +**File:** `packages/ee/server-only/lib/reregister-email-domain.ts` + +```typescript +export const reregisterEmailDomain = async (options: { emailDomainId: string }) => { + // 1. Fetch the EmailDomain record (including encrypted privateKey) + // 2. Decrypt the private key using DOCUMENSO_ENCRYPTION_KEY + // 3. Call DeleteEmailIdentityCommand on SES (ignore NotFoundException) + // 4. Call CreateEmailIdentityCommand with BYODKIM using the SAME selector + private key + // 5. Update EmailDomain status back to PENDING, update lastVerifiedAt + // 6. Return the updated domain +}; +``` + +Key points: + +- Uses the existing encrypted `privateKey` from the DB -- no new key generation +- Uses the existing `selector` -- DNS records stay the same +- Deletes first, then recreates -- handles cases where SES state is corrupted +- Resets status to `PENDING` since verification will need to re-occur +- Uses `verifyDomainWithDKIM()` from `create-email-domain.ts` (may need to extract/export this helper) + +#### 3b. Admin TRPC Routes (Find, Get, Re-register) + +All email domain admin routes use `adminProcedure` -- requires system-level `Role.ADMIN`. + +**Find (list) route:** +**File:** `packages/trpc/server/admin-router/find-email-domains.ts` +**Types:** `packages/trpc/server/admin-router/find-email-domains.types.ts` + +- Query route: `admin.emailDomain.find` +- Input: `{ query?: string, page?: number, perPage?: number, status?: EmailDomainStatus }` +- Extends `ZFindSearchParamsSchema` with optional `status` filter +- Returns standard `ZFindResultResponse` with email domain data including: id, domain, status, selector, createdAt, lastVerifiedAt, organisation name, email count +- Prisma query filters by domain name (LIKE search on `query`), optional status, joins organisation for name, counts emails + +**Get (detail) route:** +**File:** `packages/trpc/server/admin-router/get-email-domain.ts` +**Types:** `packages/trpc/server/admin-router/get-email-domain.types.ts` + +- Query route: `admin.emailDomain.get` +- Input: `{ emailDomainId: string }` +- Returns full email domain detail: all fields (except privateKey), organisation info, list of associated emails, DNS records (generated from publicKey + selector) +- Omits `privateKey` from response + +**Re-register (mutation) route:** +**File:** `packages/trpc/server/admin-router/reregister-email-domain.ts` +**Types:** `packages/trpc/server/admin-router/reregister-email-domain.types.ts` + +- Mutation route: `admin.emailDomain.reregister` +- Input: `{ emailDomainId: string }` +- Calls `reregisterEmailDomain()` +- Rationale: Re-registration is a recovery/operational action that deletes and recreates an SES identity. This is a privileged operation that should only be performed by platform operators, not self-service by org admins. + +#### 3c. Register in Admin Router + +**File:** `packages/trpc/server/admin-router/router.ts` + +Add a new `emailDomain` namespace to the admin router: + +```typescript +emailDomain: { + find: findEmailDomainsRoute, + get: getEmailDomainRoute, + reregister: reregisterEmailDomainRoute, +}, +``` + +#### 3d. Admin Panel UI -- Email Domains Section + +**List page:** `apps/remix/app/routes/_authenticated+/admin+/email-domains._index.tsx` + +- New admin panel page at `/admin/email-domains` +- Follow the existing admin documents list pattern (client-side TRPC data fetching) +- Search input (debounced) filtering by domain name +- Status filter dropdown (All / Pending / Active) +- DataTable with columns: Domain, Organisation, Status (badge), Email Count, Created, Last Verified, Actions +- Actions dropdown per row: View details, Re-register +- Pagination via `DataTablePagination` + +**Detail page:** `apps/remix/app/routes/_authenticated+/admin+/email-domains.$id.tsx` + +- Shows full domain details: domain, selector, status, organisation, created date, last verified date +- Shows DNS records (DKIM + SPF) with copy buttons (reuse `organisation-email-domain-records-dialog` pattern) +- Table of associated organisation emails +- "Re-register" button with confirmation dialog explaining the action (SES identity will be deleted and recreated with the same keys) +- "Verify Now" button to manually trigger a verification check +- Shows how long the domain has been pending (using `lastVerifiedAt` or `createdAt`) + +**Navigation:** Add menu item to admin sidebar in `_layout.tsx`: + +```tsx + +``` + +**Table component:** `apps/remix/app/components/tables/admin-email-domains-table.tsx` (optional -- can be inline in the route file like the documents page) + +#### 3e. Automatic Re-registration in Sync Job (Optional Enhancement) + +In the hourly sync job, after checking verification status, if a domain has been `PENDING` for more than 48 hours: + +- Automatically call `reregisterEmailDomain()` to attempt recovery +- Log the auto-recovery attempt +- This provides a self-healing mechanism without user intervention + +## Implementation Plan + +### Phase 1: Background Sync Job (Core) + +1. Create `sync-email-domains.ts` job definition with hourly cron +2. Create `sync-email-domains.handler.ts` with batch verification logic +3. Register job in `packages/lib/jobs/client.ts` +4. Add error handling and logging + +### Phase 2: Schema Enhancement + +5. Add `lastVerifiedAt` column to `EmailDomain` model +6. Create Prisma migration +7. Update `verifyEmailDomain()` to set `lastVerifiedAt` on each check +8. Update sync job to use `lastVerifiedAt` for intelligent scheduling + +### Phase 3: Admin Email Domains Panel + +9. Create `find-email-domains` admin TRPC route + types (list/search with pagination and status filter) +10. Create `get-email-domain` admin TRPC route + types (detail view with org info, emails, DNS records) +11. Register find + get routes in admin router under `emailDomain` namespace +12. Create admin list page (`admin+/email-domains._index.tsx`) with search, status filter, DataTable +13. Create admin detail page (`admin+/email-domains.$id.tsx`) with domain info, emails table, DNS records +14. Add "Email Domains" menu item to admin sidebar (`_layout.tsx`) + +### Phase 4: Re-registration Feature + +15. Extract `verifyDomainWithDKIM()` as a shared helper (if not already exported) +16. Create `reregisterEmailDomain()` service function +17. Create `reregister-email-domain` admin TRPC mutation route + types +18. Register reregister route in admin router under `emailDomain.reregister` +19. Add "Re-register" button + confirmation dialog on admin detail page + +### Phase 5: Auto-Recovery (Optional) + +20. Add 48-hour stale detection logic to sync job +21. Auto-trigger re-registration for stale domains +22. Add logging/notifications for auto-recovery events + +## Files to Create/Modify + +### New Files + +- `packages/lib/jobs/definitions/internal/sync-email-domains.ts` +- `packages/lib/jobs/definitions/internal/sync-email-domains.handler.ts` +- `packages/ee/server-only/lib/reregister-email-domain.ts` +- `packages/trpc/server/admin-router/find-email-domains.ts` +- `packages/trpc/server/admin-router/find-email-domains.types.ts` +- `packages/trpc/server/admin-router/get-email-domain.ts` +- `packages/trpc/server/admin-router/get-email-domain.types.ts` +- `packages/trpc/server/admin-router/reregister-email-domain.ts` +- `packages/trpc/server/admin-router/reregister-email-domain.types.ts` +- `apps/remix/app/routes/_authenticated+/admin+/email-domains._index.tsx` +- `apps/remix/app/routes/_authenticated+/admin+/email-domains.$id.tsx` + +### Modified Files + +- `packages/prisma/schema.prisma` -- Add `lastVerifiedAt` field +- `packages/lib/jobs/client.ts` -- Register new sync job +- `packages/ee/server-only/lib/verify-email-domain.ts` -- Update `lastVerifiedAt` +- `packages/ee/server-only/lib/create-email-domain.ts` -- Export `verifyDomainWithDKIM` helper +- `packages/trpc/server/admin-router/router.ts` -- Add `emailDomain.{find, get, reregister}` routes +- `apps/remix/app/routes/_authenticated+/admin+/_layout.tsx` -- Add "Email Domains" nav item to sidebar +- New Prisma migration file + +## Technical Considerations + +1. **SES API Rate Limits** -- AWS SES has rate limits on `GetEmailIdentityCommand`. The sync job should process domains in batches with small delays between calls (e.g., 5-10 per batch with 1s delay). + +2. **Concurrency** -- The local job provider has deterministic deduplication via SHA-256 IDs, so multiple app instances won't run the same cron tick twice. + +3. **Error Isolation** -- Each domain verification in the sync job should be wrapped in try/catch so one failing domain doesn't prevent others from being checked. + +4. **Re-registration Safety** -- The re-register function should be idempotent. Deleting a non-existent SES identity should be handled gracefully (already done in `deleteEmailDomain`). + +5. **Private Key Security** -- The private key is encrypted at rest and should only be decrypted transiently during re-registration. It should never be logged or exposed in API responses. + +6. **Feature Gating** -- The sync job should only process domains belonging to organisations with active `emailDomains` claim flags. This prevents processing domains for orgs that have downgraded. + +7. **Observability** -- Add structured logging to the sync job so operations teams can monitor domain verification health across all tenants. diff --git a/apps/remix/app/routes/_authenticated+/admin+/_layout.tsx b/apps/remix/app/routes/_authenticated+/admin+/_layout.tsx index 8757372b1..43fe8feda 100644 --- a/apps/remix/app/routes/_authenticated+/admin+/_layout.tsx +++ b/apps/remix/app/routes/_authenticated+/admin+/_layout.tsx @@ -3,6 +3,7 @@ import { BarChart3, Building2Icon, FileStack, + MailIcon, Settings, Trophy, Users, @@ -122,6 +123,20 @@ export default function AdminLayout({ loaderData }: Route.ComponentProps) { + + + + + + + + Re-register Email Domain + + + + + This will delete the existing SES identity for{' '} + {emailDomain.domain} and recreate it using the same DKIM keys. + The user will not need to update their DNS records. The domain status will be + reset to Pending. + + + + + + + Cancel + + + reregisterDomain({ emailDomainId: emailDomain.id })} + > + Re-register + + + + + + +
+ +

+ DNS Records +

+ +
+ {dnsRecords.map((record, index) => ( +
+
+
+ {record.type} Record +
+ + +
+ +
+
+ + Name:{' '} + + {record.name} +
+
+ + Value:{' '} + + {record.value} +
+
+
+ ))} +
+ +
+ +

+ Emails ({emailDomain.emails.length}) +

+ +
+ {emailDomain.emails.length > 0 ? ( + {}} + /> + ) : ( +

+ No emails configured for this domain. +

+ )} +
+ + ); +} diff --git a/apps/remix/app/routes/_authenticated+/admin+/email-domains._index.tsx b/apps/remix/app/routes/_authenticated+/admin+/email-domains._index.tsx new file mode 100644 index 000000000..669747e6d --- /dev/null +++ b/apps/remix/app/routes/_authenticated+/admin+/email-domains._index.tsx @@ -0,0 +1,207 @@ +import { useMemo, useState } from 'react'; + +import { msg } from '@lingui/core/macro'; +import { useLingui } from '@lingui/react'; +import { Trans } from '@lingui/react/macro'; +import { EmailDomainStatus } from '@prisma/client'; +import { CheckCircle2Icon, ClockIcon, Loader } from 'lucide-react'; +import { Link, useSearchParams } from 'react-router'; +import { match } from 'ts-pattern'; + +import { useDebouncedValue } from '@documenso/lib/client-only/hooks/use-debounced-value'; +import { useUpdateSearchParams } from '@documenso/lib/client-only/hooks/use-update-search-params'; +import { trpc } from '@documenso/trpc/react'; +import { Badge } from '@documenso/ui/primitives/badge'; +import { Button } from '@documenso/ui/primitives/button'; +import type { DataTableColumnDef } from '@documenso/ui/primitives/data-table'; +import { DataTable } from '@documenso/ui/primitives/data-table'; +import { DataTablePagination } from '@documenso/ui/primitives/data-table-pagination'; +import { Input } from '@documenso/ui/primitives/input'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@documenso/ui/primitives/select'; + +export default function AdminEmailDomainsPage() { + const { _, i18n } = useLingui(); + + const [searchParams] = useSearchParams(); + const updateSearchParams = useUpdateSearchParams(); + + const [term, setTerm] = useState(() => searchParams?.get?.('term') ?? ''); + const debouncedTerm = useDebouncedValue(term, 500); + + const page = searchParams?.get?.('page') ? Number(searchParams.get('page')) : undefined; + const perPage = searchParams?.get?.('perPage') ? Number(searchParams.get('perPage')) : undefined; + const statusParam = searchParams?.get?.('status') ?? 'ALL'; + + const statusFilter = + statusParam === 'PENDING' || statusParam === 'ACTIVE' ? statusParam : undefined; + + const { data: findEmailDomainsData, isPending: isFindEmailDomainsLoading } = + trpc.admin.emailDomain.find.useQuery( + { + query: debouncedTerm, + page: page || 1, + perPage: perPage || 20, + status: statusFilter, + }, + { + placeholderData: (previousData) => previousData, + }, + ); + + const results = findEmailDomainsData ?? { + data: [], + perPage: 20, + currentPage: 1, + totalPages: 1, + }; + + const columns = useMemo(() => { + return [ + { + header: _(msg`Domain`), + accessorKey: 'domain', + cell: ({ row }) => ( + + {row.original.domain} + + ), + }, + { + header: _(msg`Organisation`), + accessorKey: 'organisation', + cell: ({ row }) => ( + + {row.original.organisation.name} + + ), + }, + { + header: _(msg`Status`), + accessorKey: 'status', + cell: ({ row }) => + match(row.original.status) + .with(EmailDomainStatus.ACTIVE, () => ( + + + Active + + )) + .with(EmailDomainStatus.PENDING, () => ( + + + Pending + + )) + .exhaustive(), + }, + { + header: _(msg`Emails`), + accessorKey: '_count', + cell: ({ row }) => row.original._count.emails, + }, + { + header: _(msg`Created`), + accessorKey: 'createdAt', + cell: ({ row }) => i18n.date(row.original.createdAt), + }, + { + header: _(msg`Last Verified`), + accessorKey: 'lastVerifiedAt', + cell: ({ row }) => + row.original.lastVerifiedAt ? i18n.date(row.original.lastVerifiedAt) : '-', + }, + { + header: _(msg`Actions`), + cell: ({ row }) => ( + + ), + }, + ] satisfies DataTableColumnDef<(typeof results)['data'][number]>[]; + }, []); + + const onPaginationChange = (newPage: number, newPerPage: number) => { + updateSearchParams({ + page: newPage, + perPage: newPerPage, + }); + }; + + const onStatusChange = (value: string) => { + updateSearchParams({ + status: value === 'ALL' ? undefined : value, + page: 1, + }); + }; + + return ( +
+

+ Email Domains +

+ +
+
+ setTerm(e.target.value)} + /> + + +
+ +
+ + {(table) => } + + + {isFindEmailDomainsLoading && ( +
+ +
+ )} +
+
+
+ ); +} diff --git a/packages/ee/server-only/lib/create-email-domain.ts b/packages/ee/server-only/lib/create-email-domain.ts index b50a55965..061a71cc1 100644 --- a/packages/ee/server-only/lib/create-email-domain.ts +++ b/packages/ee/server-only/lib/create-email-domain.ts @@ -142,6 +142,7 @@ export const createEmailDomain = async ({ domain, organisationId }: CreateEmailD publicKey: true, createdAt: true, updatedAt: true, + lastVerifiedAt: true, emails: true, }, }); diff --git a/packages/ee/server-only/lib/reregister-email-domain.ts b/packages/ee/server-only/lib/reregister-email-domain.ts new file mode 100644 index 000000000..1b9960c6b --- /dev/null +++ b/packages/ee/server-only/lib/reregister-email-domain.ts @@ -0,0 +1,93 @@ +import { DeleteEmailIdentityCommand } from '@aws-sdk/client-sesv2'; +import { EmailDomainStatus } from '@prisma/client'; + +import { DOCUMENSO_ENCRYPTION_KEY } from '@documenso/lib/constants/crypto'; +import { AppError, AppErrorCode } from '@documenso/lib/errors/app-error'; +import { symmetricDecrypt } from '@documenso/lib/universal/crypto'; +import { prisma } from '@documenso/prisma'; + +import { getSesClient, verifyDomainWithDKIM } from './create-email-domain'; + +type ReregisterEmailDomainOptions = { + emailDomainId: string; +}; + +/** + * Re-register an email domain in SES using the same DKIM key pair. + * + * This deletes the existing SES identity and recreates it with the same + * selector and private key, so the user does not need to update their DNS records. + * + * Permission is assumed to be checked in the caller. + */ +export const reregisterEmailDomain = async ({ emailDomainId }: ReregisterEmailDomainOptions) => { + const encryptionKey = DOCUMENSO_ENCRYPTION_KEY; + + if (!encryptionKey) { + throw new Error('Missing DOCUMENSO_ENCRYPTION_KEY'); + } + + const emailDomain = await prisma.emailDomain.findUnique({ + where: { + id: emailDomainId, + }, + }); + + if (!emailDomain) { + throw new AppError(AppErrorCode.NOT_FOUND, { + message: 'Email domain not found', + }); + } + + const sesClient = getSesClient(); + + // Delete the existing SES identity, ignoring if it no longer exists. + await sesClient + .send( + new DeleteEmailIdentityCommand({ + EmailIdentity: emailDomain.domain, + }), + ) + .catch((err) => { + if (err.name === 'NotFoundException') { + return; + } + + throw err; + }); + + // Decrypt the stored private key. + const decryptedPrivateKeyBytes = symmetricDecrypt({ + key: encryptionKey, + data: emailDomain.privateKey, + }); + + const decryptedPrivateKey = new TextDecoder().decode(decryptedPrivateKeyBytes); + + // The selector field in the DB is the full record name (e.g. "documenso-orgid._domainkey.example.com"). + // We need to extract just the selector part (before "._domainkey."). + const selectorParts = emailDomain.selector.split('._domainkey.'); + const selector = selectorParts[0]; + + if (!selector) { + throw new AppError(AppErrorCode.UNKNOWN_ERROR, { + message: 'Could not extract selector from email domain record', + }); + } + + // Recreate the SES identity with the same DKIM key pair. + await verifyDomainWithDKIM(emailDomain.domain, selector, decryptedPrivateKey); + + // Reset status to PENDING and update lastVerifiedAt. + const updatedEmailDomain = await prisma.emailDomain.update({ + where: { + id: emailDomainId, + }, + data: { + status: EmailDomainStatus.PENDING, + lastVerifiedAt: new Date(), + }, + }); + + return updatedEmailDomain; +}; diff --git a/packages/ee/server-only/lib/verify-email-domain.ts b/packages/ee/server-only/lib/verify-email-domain.ts index 0c898016e..bf82a78da 100644 --- a/packages/ee/server-only/lib/verify-email-domain.ts +++ b/packages/ee/server-only/lib/verify-email-domain.ts @@ -35,6 +35,7 @@ export const verifyEmailDomain = async (emailDomainId: string) => { }, data: { status: isVerified ? EmailDomainStatus.ACTIVE : EmailDomainStatus.PENDING, + lastVerifiedAt: new Date(), }, }); diff --git a/packages/lib/jobs/client.ts b/packages/lib/jobs/client.ts index 0471e2018..af6241320 100644 --- a/packages/lib/jobs/client.ts +++ b/packages/lib/jobs/client.ts @@ -16,6 +16,7 @@ import { EXECUTE_WEBHOOK_JOB_DEFINITION } from './definitions/internal/execute-w import { EXPIRE_RECIPIENTS_SWEEP_JOB_DEFINITION } from './definitions/internal/expire-recipients-sweep'; import { PROCESS_RECIPIENT_EXPIRED_JOB_DEFINITION } from './definitions/internal/process-recipient-expired'; import { SEAL_DOCUMENT_JOB_DEFINITION } from './definitions/internal/seal-document'; +import { SYNC_EMAIL_DOMAINS_JOB_DEFINITION } from './definitions/internal/sync-email-domains'; /** * The `as const` assertion is load bearing as it provides the correct level of type inference for @@ -39,6 +40,7 @@ export const jobsClient = new JobClient([ EXPIRE_RECIPIENTS_SWEEP_JOB_DEFINITION, PROCESS_RECIPIENT_EXPIRED_JOB_DEFINITION, CLEANUP_RATE_LIMITS_JOB_DEFINITION, + SYNC_EMAIL_DOMAINS_JOB_DEFINITION, ] as const); export const jobs = jobsClient; diff --git a/packages/lib/jobs/client/inngest.ts b/packages/lib/jobs/client/inngest.ts index 8b565342b..702fc1de7 100644 --- a/packages/lib/jobs/client/inngest.ts +++ b/packages/lib/jobs/client/inngest.ts @@ -26,6 +26,7 @@ export class InngestJobProvider extends BaseJobProvider { const client = new InngestClient({ id: env('NEXT_PRIVATE_INNGEST_APP_ID') || 'documenso-app', eventKey: env('INNGEST_EVENT_KEY') || env('NEXT_PRIVATE_INNGEST_EVENT_KEY'), + logger: console, }); this._instance = new InngestJobProvider({ client }); @@ -90,7 +91,10 @@ export class InngestJobProvider extends BaseJobProvider { return { wait: step.sleep, logger: { - ...ctx.logger, + info: ctx.logger.info, + debug: ctx.logger.debug, + error: ctx.logger.error, + warn: ctx.logger.warn, log: ctx.logger.info, }, runTask: async (cacheKey, callback) => { diff --git a/packages/lib/jobs/definitions/internal/sync-email-domains.handler.ts b/packages/lib/jobs/definitions/internal/sync-email-domains.handler.ts new file mode 100644 index 000000000..8d580a010 --- /dev/null +++ b/packages/lib/jobs/definitions/internal/sync-email-domains.handler.ts @@ -0,0 +1,86 @@ +import { DateTime } from 'luxon'; + +import { reregisterEmailDomain } from '@documenso/ee/server-only/lib/reregister-email-domain'; +import { verifyEmailDomain } from '@documenso/ee/server-only/lib/verify-email-domain'; +import { prisma } from '@documenso/prisma'; + +import type { JobRunIO } from '../../client/_internal/job'; +import type { TSyncEmailDomainsJobDefinition } from './sync-email-domains'; + +const BATCH_SIZE = 10; +const AUTO_REREGISTER_AFTER_HOURS = 48; + +export const run = async ({ io }: { payload: TSyncEmailDomainsJobDefinition; io: JobRunIO }) => { + const pendingDomains = await prisma.emailDomain.findMany({ + where: { + status: 'PENDING', + }, + select: { + id: true, + domain: true, + createdAt: true, + lastVerifiedAt: true, + }, + orderBy: { + lastVerifiedAt: { sort: 'asc', nulls: 'first' }, + }, + }); + + if (pendingDomains.length === 0) { + io.logger.info('No pending email domains to sync'); + return; + } + + io.logger.info(`Found ${pendingDomains.length} pending email domains to sync`); + + let verifiedCount = 0; + let reregisteredCount = 0; + let errorCount = 0; + + const reregisterCutoff = DateTime.now().minus({ hours: AUTO_REREGISTER_AFTER_HOURS }).toJSDate(); + + for (let i = 0; i < pendingDomains.length; i += BATCH_SIZE) { + const batch = pendingDomains.slice(i, i + BATCH_SIZE); + + const results = await Promise.allSettled( + batch.map(async (domain) => { + const shouldReregister = domain.createdAt < reregisterCutoff; + + if (shouldReregister) { + io.logger.info( + `Domain "${domain.domain}" has been pending since ${domain.createdAt.toISOString()}, attempting re-registration`, + ); + + await reregisterEmailDomain({ emailDomainId: domain.id }); + return 'reregistered' as const; + } + + const { isVerified } = await verifyEmailDomain(domain.id); + + return isVerified ? ('verified' as const) : ('pending' as const); + }), + ); + + for (const result of results) { + if (result.status === 'rejected') { + errorCount++; + io.logger.error(`Failed to process email domain: ${String(result.reason)}`); + } else if (result.value === 'verified') { + verifiedCount++; + } else if (result.value === 'reregistered') { + reregisteredCount++; + } + } + + // Small delay between batches to respect SES API rate limits. + if (i + BATCH_SIZE < pendingDomains.length) { + await new Promise((resolve) => { + setTimeout(resolve, 1000); + }); + } + } + + io.logger.info( + `Sync complete: ${verifiedCount} verified, ${reregisteredCount} re-registered, ${errorCount} errors out of ${pendingDomains.length} pending domains`, + ); +}; diff --git a/packages/lib/jobs/definitions/internal/sync-email-domains.ts b/packages/lib/jobs/definitions/internal/sync-email-domains.ts new file mode 100644 index 000000000..f20f3b18d --- /dev/null +++ b/packages/lib/jobs/definitions/internal/sync-email-domains.ts @@ -0,0 +1,30 @@ +import { z } from 'zod'; + +import { type JobDefinition } from '../../client/_internal/job'; + +const SYNC_EMAIL_DOMAINS_JOB_DEFINITION_ID = 'internal.sync-email-domains'; + +const SYNC_EMAIL_DOMAINS_JOB_DEFINITION_SCHEMA = z.object({}); + +export type TSyncEmailDomainsJobDefinition = z.infer< + typeof SYNC_EMAIL_DOMAINS_JOB_DEFINITION_SCHEMA +>; + +export const SYNC_EMAIL_DOMAINS_JOB_DEFINITION = { + id: SYNC_EMAIL_DOMAINS_JOB_DEFINITION_ID, + name: 'Sync Email Domains', + version: '1.0.0', + trigger: { + name: SYNC_EMAIL_DOMAINS_JOB_DEFINITION_ID, + schema: SYNC_EMAIL_DOMAINS_JOB_DEFINITION_SCHEMA, + cron: '0 * * * *', // Every hour, on the hour. + }, + handler: async ({ payload, io }) => { + const handler = await import('./sync-email-domains.handler'); + + await handler.run({ payload, io }); + }, +} as const satisfies JobDefinition< + typeof SYNC_EMAIL_DOMAINS_JOB_DEFINITION_ID, + TSyncEmailDomainsJobDefinition +>; diff --git a/packages/lib/types/email-domain.ts b/packages/lib/types/email-domain.ts index 2dbf79b2a..df5944bfc 100644 --- a/packages/lib/types/email-domain.ts +++ b/packages/lib/types/email-domain.ts @@ -18,6 +18,7 @@ export const ZEmailDomainSchema = EmailDomainSchema.pick({ publicKey: true, createdAt: true, updatedAt: true, + lastVerifiedAt: true, }).extend({ emails: ZOrganisationEmailLiteSchema.array(), }); @@ -35,6 +36,7 @@ export const ZEmailDomainManySchema = EmailDomainSchema.pick({ selector: true, createdAt: true, updatedAt: true, + lastVerifiedAt: true, }); export type TEmailDomainMany = z.infer; diff --git a/packages/prisma/migrations/20260224000000_add_email_domain_last_verified_at/migration.sql b/packages/prisma/migrations/20260224000000_add_email_domain_last_verified_at/migration.sql new file mode 100644 index 000000000..6d2dc6073 --- /dev/null +++ b/packages/prisma/migrations/20260224000000_add_email_domain_last_verified_at/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "EmailDomain" ADD COLUMN "lastVerifiedAt" TIMESTAMP(3); diff --git a/packages/prisma/schema.prisma b/packages/prisma/schema.prisma index 63a9f6a0e..467be8d2e 100644 --- a/packages/prisma/schema.prisma +++ b/packages/prisma/schema.prisma @@ -1037,10 +1037,11 @@ model EmailDomain { status EmailDomainStatus @default(PENDING) - selector String @unique - domain String @unique - publicKey String - privateKey String + selector String @unique + domain String @unique + publicKey String + privateKey String + lastVerifiedAt DateTime? organisationId String organisation Organisation @relation(fields: [organisationId], references: [id], onDelete: Cascade) diff --git a/packages/trpc/server/admin-router/find-email-domains.ts b/packages/trpc/server/admin-router/find-email-domains.ts new file mode 100644 index 000000000..5caa8d58e --- /dev/null +++ b/packages/trpc/server/admin-router/find-email-domains.ts @@ -0,0 +1,101 @@ +import { Prisma } from '@prisma/client'; + +import type { FindResultResponse } from '@documenso/lib/types/search-params'; +import { prisma } from '@documenso/prisma'; + +import { adminProcedure } from '../trpc'; +import { + ZFindEmailDomainsRequestSchema, + ZFindEmailDomainsResponseSchema, +} from './find-email-domains.types'; + +export const findEmailDomainsRoute = adminProcedure + .input(ZFindEmailDomainsRequestSchema) + .output(ZFindEmailDomainsResponseSchema) + .query(async ({ input }) => { + const { query, page, perPage, status } = input; + + return await findEmailDomains({ query, page, perPage, status }); + }); + +type FindEmailDomainsOptions = { + query?: string; + page?: number; + perPage?: number; + status?: 'PENDING' | 'ACTIVE'; +}; + +const findEmailDomains = async ({ + query, + page = 1, + perPage = 20, + status, +}: FindEmailDomainsOptions) => { + const whereClause: Prisma.EmailDomainWhereInput = {}; + + if (query) { + whereClause.OR = [ + { + domain: { + contains: query, + mode: Prisma.QueryMode.insensitive, + }, + }, + { + organisation: { + name: { + contains: query, + mode: Prisma.QueryMode.insensitive, + }, + }, + }, + ]; + } + + if (status) { + whereClause.status = status; + } + + const [data, count] = await Promise.all([ + prisma.emailDomain.findMany({ + where: whereClause, + skip: Math.max(page - 1, 0) * perPage, + take: perPage, + orderBy: { + createdAt: 'desc', + }, + select: { + id: true, + domain: true, + status: true, + selector: true, + createdAt: true, + updatedAt: true, + lastVerifiedAt: true, + organisation: { + select: { + id: true, + name: true, + url: true, + }, + }, + _count: { + select: { + emails: true, + }, + }, + }, + }), + prisma.emailDomain.count({ + where: whereClause, + }), + ]); + + return { + data, + count, + currentPage: Math.max(page, 1), + perPage, + totalPages: Math.ceil(count / perPage), + } satisfies FindResultResponse; +}; diff --git a/packages/trpc/server/admin-router/find-email-domains.types.ts b/packages/trpc/server/admin-router/find-email-domains.types.ts new file mode 100644 index 000000000..c4118436e --- /dev/null +++ b/packages/trpc/server/admin-router/find-email-domains.types.ts @@ -0,0 +1,36 @@ +import { z } from 'zod'; + +import { ZFindResultResponse, ZFindSearchParamsSchema } from '@documenso/lib/types/search-params'; +import EmailDomainStatusSchema from '@documenso/prisma/generated/zod/inputTypeSchemas/EmailDomainStatusSchema'; +import EmailDomainSchema from '@documenso/prisma/generated/zod/modelSchema/EmailDomainSchema'; +import OrganisationSchema from '@documenso/prisma/generated/zod/modelSchema/OrganisationSchema'; + +export const ZFindEmailDomainsRequestSchema = ZFindSearchParamsSchema.extend({ + status: EmailDomainStatusSchema.optional(), +}); + +export const ZFindEmailDomainsResponseSchema = ZFindResultResponse.extend({ + data: EmailDomainSchema.pick({ + id: true, + domain: true, + status: true, + selector: true, + createdAt: true, + updatedAt: true, + lastVerifiedAt: true, + }) + .extend({ + organisation: OrganisationSchema.pick({ + id: true, + name: true, + url: true, + }), + _count: z.object({ + emails: z.number(), + }), + }) + .array(), +}); + +export type TFindEmailDomainsRequest = z.infer; +export type TFindEmailDomainsResponse = z.infer; diff --git a/packages/trpc/server/admin-router/get-email-domain.ts b/packages/trpc/server/admin-router/get-email-domain.ts new file mode 100644 index 000000000..b4acf4f69 --- /dev/null +++ b/packages/trpc/server/admin-router/get-email-domain.ts @@ -0,0 +1,42 @@ +import { AppError, AppErrorCode } from '@documenso/lib/errors/app-error'; +import { prisma } from '@documenso/prisma'; + +import { adminProcedure } from '../trpc'; +import { + ZGetEmailDomainRequestSchema, + ZGetEmailDomainResponseSchema, +} from './get-email-domain.types'; + +export const getEmailDomainRoute = adminProcedure + .input(ZGetEmailDomainRequestSchema) + .output(ZGetEmailDomainResponseSchema) + .query(async ({ input }) => { + const { emailDomainId } = input; + + const emailDomain = await prisma.emailDomain.findUnique({ + where: { + id: emailDomainId, + }, + omit: { + privateKey: true, + }, + include: { + organisation: { + select: { + id: true, + name: true, + url: true, + }, + }, + emails: true, + }, + }); + + if (!emailDomain) { + throw new AppError(AppErrorCode.NOT_FOUND, { + message: 'Email domain not found', + }); + } + + return emailDomain; + }); diff --git a/packages/trpc/server/admin-router/get-email-domain.types.ts b/packages/trpc/server/admin-router/get-email-domain.types.ts new file mode 100644 index 000000000..fdfc00810 --- /dev/null +++ b/packages/trpc/server/admin-router/get-email-domain.types.ts @@ -0,0 +1,30 @@ +import { z } from 'zod'; + +import { ZOrganisationEmailLiteSchema } from '@documenso/lib/types/organisation-email'; +import EmailDomainSchema from '@documenso/prisma/generated/zod/modelSchema/EmailDomainSchema'; +import OrganisationSchema from '@documenso/prisma/generated/zod/modelSchema/OrganisationSchema'; + +export const ZGetEmailDomainRequestSchema = z.object({ + emailDomainId: z.string(), +}); + +export const ZGetEmailDomainResponseSchema = EmailDomainSchema.pick({ + id: true, + domain: true, + status: true, + selector: true, + publicKey: true, + createdAt: true, + updatedAt: true, + lastVerifiedAt: true, +}).extend({ + organisation: OrganisationSchema.pick({ + id: true, + name: true, + url: true, + }), + emails: ZOrganisationEmailLiteSchema.array(), +}); + +export type TGetEmailDomainRequest = z.infer; +export type TGetEmailDomainResponse = z.infer; diff --git a/packages/trpc/server/admin-router/reregister-email-domain.ts b/packages/trpc/server/admin-router/reregister-email-domain.ts new file mode 100644 index 000000000..d334c66e3 --- /dev/null +++ b/packages/trpc/server/admin-router/reregister-email-domain.ts @@ -0,0 +1,22 @@ +import { reregisterEmailDomain } from '@documenso/ee/server-only/lib/reregister-email-domain'; + +import { adminProcedure } from '../trpc'; +import { + ZReregisterEmailDomainRequestSchema, + ZReregisterEmailDomainResponseSchema, +} from './reregister-email-domain.types'; + +export const reregisterEmailDomainRoute = adminProcedure + .input(ZReregisterEmailDomainRequestSchema) + .output(ZReregisterEmailDomainResponseSchema) + .mutation(async ({ input, ctx }) => { + const { emailDomainId } = input; + + ctx.logger.info({ + input: { + emailDomainId, + }, + }); + + await reregisterEmailDomain({ emailDomainId }); + }); diff --git a/packages/trpc/server/admin-router/reregister-email-domain.types.ts b/packages/trpc/server/admin-router/reregister-email-domain.types.ts new file mode 100644 index 000000000..de7d222f4 --- /dev/null +++ b/packages/trpc/server/admin-router/reregister-email-domain.types.ts @@ -0,0 +1,10 @@ +import { z } from 'zod'; + +export const ZReregisterEmailDomainRequestSchema = z.object({ + emailDomainId: z.string(), +}); + +export const ZReregisterEmailDomainResponseSchema = z.void(); + +export type TReregisterEmailDomainRequest = z.infer; +export type TReregisterEmailDomainResponse = z.infer; diff --git a/packages/trpc/server/admin-router/router.ts b/packages/trpc/server/admin-router/router.ts index 17783072d..dd472705f 100644 --- a/packages/trpc/server/admin-router/router.ts +++ b/packages/trpc/server/admin-router/router.ts @@ -11,11 +11,14 @@ import { findAdminOrganisationsRoute } from './find-admin-organisations'; import { findDocumentAuditLogsRoute } from './find-document-audit-logs'; import { findDocumentJobsRoute } from './find-document-jobs'; import { findDocumentsRoute } from './find-documents'; +import { findEmailDomainsRoute } from './find-email-domains'; import { findSubscriptionClaimsRoute } from './find-subscription-claims'; import { findUserTeamsRoute } from './find-user-teams'; import { getAdminOrganisationRoute } from './get-admin-organisation'; +import { getEmailDomainRoute } from './get-email-domain'; import { getUserRoute } from './get-user'; import { promoteMemberToOwnerRoute } from './promote-member-to-owner'; +import { reregisterEmailDomainRoute } from './reregister-email-domain'; import { resealDocumentRoute } from './reseal-document'; import { resetTwoFactorRoute } from './reset-two-factor-authentication'; import { resyncLicenseRoute } from './resync-license'; @@ -68,5 +71,10 @@ export const adminRouter = router({ recipient: { update: updateRecipientRoute, }, + emailDomain: { + find: findEmailDomainsRoute, + get: getEmailDomainRoute, + reregister: reregisterEmailDomainRoute, + }, updateSiteSetting: updateSiteSettingRoute, }); diff --git a/packages/trpc/server/enterprise-router/find-organisation-email-domain.ts b/packages/trpc/server/enterprise-router/find-organisation-email-domain.ts index 6901efb76..e8694fc38 100644 --- a/packages/trpc/server/enterprise-router/find-organisation-email-domain.ts +++ b/packages/trpc/server/enterprise-router/find-organisation-email-domain.ts @@ -95,6 +95,7 @@ export const findOrganisationEmailDomains = async ({ selector: true, createdAt: true, updatedAt: true, + lastVerifiedAt: true, _count: { select: { emails: true,