Merge branch 'main' into confluence-importer

apps/server/src/integrations/environment/environment.service.ts

@@ -122,6 +122,26 @@ export class EnvironmentService {
     return this.configService.get<string>('AWS_S3_URL');
   }
 
+  getAzureStorageAccountName(): string {
+    return this.configService.get<string>('AZURE_STORAGE_ACCOUNT_NAME');
+  }
+
+  getAzureStorageContainer(): string {
+    return this.configService.get<string>('AZURE_STORAGE_CONTAINER');
+  }
+
+  getAzureStorageAccountKey(): string {
+    return this.configService.get<string>('AZURE_STORAGE_ACCOUNT_KEY');
+  }
+
+  getAzureStorageEndpoint(): string {
+    return this.configService.get<string>('AZURE_STORAGE_ENDPOINT');
+  }
+
+  getAzureStorageUrl(): string {
+    return this.configService.get<string>('AZURE_STORAGE_URL');
+  }
+
   getMailDriver(): string {
     return this.configService.get<string>('MAIL_DRIVER', 'log');
   }
@@ -332,4 +352,19 @@ export class EnvironmentService {
       .toLowerCase();
     return disabled === 'true';
   }
+
+  isIframeEmbedAllowed(): boolean {
+    const allowed = this.configService
+      .get<string>('IFRAME_EMBED_ALLOWED', 'false')
+      .toLowerCase();
+    return allowed === 'true';
+  }
+
+  getIframeAllowedOrigins(): string[] {
+    const raw = this.configService.get<string>('IFRAME_ALLOWED_ORIGINS', '');
+    return raw
+      .split(',')
+      .map((o) => o.trim())
+      .filter(Boolean);
+  }
 }

apps/server/src/integrations/environment/environment.validation.ts

@@ -49,7 +49,7 @@ export class EnvironmentVariables {
   MAIL_DRIVER: string;
 
   @IsOptional()
-  @IsIn(['local', 's3'])
+  @IsIn(['local', 's3', 'azure'])
   STORAGE_DRIVER: string;
 
   @IsOptional()

apps/server/src/integrations/import/processors/file-task.processor.ts

@@ -82,8 +82,11 @@ export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy {
 
   @OnWorkerEvent('failed')
   async onFailed(job: Job) {
+    const fileTaskId = job.data?.fileTaskId;
     this.logger.error(
-      `Error processing ${job.name} job. File Task ID: ${job.data?.fileTaskId}. Reason: ${job.failedReason}`,
+      fileTaskId
+        ? `Error processing ${job.name} job. File Task ID: ${fileTaskId}. Reason: ${job.failedReason}`
+        : `Error processing ${job.name} job. Reason: ${job.failedReason}`,
     );
 
     if (job.name === QueueJob.IMPORT_TASK) {
@@ -97,8 +100,11 @@ export class FileTaskProcessor extends WorkerHost implements OnModuleDestroy {
 
   @OnWorkerEvent('completed')
   async onCompleted(job: Job) {
+    const fileTaskId = job.data?.fileTaskId;
     this.logger.log(
-      `Completed ${job.name} job for File task ID ${job.data?.fileTaskId}`,
+      fileTaskId
+        ? `Completed ${job.name} job for File task ID ${fileTaskId}`
+        : `Completed ${job.name} job`,
     );
 
     if (job.name === QueueJob.IMPORT_TASK) {

apps/server/src/integrations/queue/constants/queue.constants.ts

@@ -9,6 +9,7 @@ export enum QueueName {
   HISTORY_QUEUE = '{history-queue}',
   NOTIFICATION_QUEUE = '{notification-queue}',
   AUDIT_QUEUE = '{audit-queue}',
+  BASE_QUEUE = '{base-queue}',
 }
 
 export enum QueueJob {
@@ -84,4 +85,8 @@ export enum QueueJob {
 
   PDF_EXPORT_TASK = 'pdf-export-task',
   PDF_EXPORT_CLEANUP = 'pdf-export-cleanup',
+
+  BASE_TYPE_CONVERSION = 'base-type-conversion',
+  BASE_CELL_GC = 'base-cell-gc',
+  BASE_FORMULA_RECOMPUTE = 'base-formula-recompute',
 }

apps/server/src/integrations/queue/constants/queue.interface.ts

@@ -113,3 +113,47 @@ export interface IApprovalRejectedNotificationJob {
   requestedById: string;
   comment?: string;
 }
+
+export interface IBaseTypeConversionJob {
+  pageId: string;
+  propertyId: string;
+  workspaceId: string;
+  fromType: string;
+  toType: string;
+  // Snapshots taken at enqueue time so the job stays correct even if the
+  // property's current typeOptions drift while the job waits in the queue.
+  fromTypeOptions: unknown;
+  toTypeOptions: unknown;
+  // When true, the job nulls the cell values for that property instead of
+  // attempting a value conversion. Used for any conversion where the new
+  // type has no meaningful representation of the old value (e.g. involving
+  // a system type).
+  clearMode: boolean;
+  // Staging identity: guards redelivery and failure cleanup against a
+  // same-type re-stage made after this job was enqueued.
+  pendingToken: string;
+  actorId?: string;
+}
+
+export interface IBaseCellGcJob {
+  pageId: string;
+  propertyId: string;
+  workspaceId: string;
+}
+
+export interface IBaseFormulaRecomputeJob {
+  pageId: string;
+  workspaceId: string;
+  propertyIds: string[]; // formula properties to recompute
+  reason:
+    | 'formula_created'
+    | 'formula_edited'
+    | 'dep_type_changed'
+    | 'dep_deleted'
+    | 'bulk_import'
+    | 'manual';
+  actorId?: string | null;
+  // When set, scope recompute to these row IDs instead of the whole base.
+  // Used by the bulk-write path (> FORMULA_INLINE_ROW_THRESHOLD).
+  rowIds?: string[];
+}

apps/server/src/integrations/queue/queue.module.ts

@@ -92,6 +92,14 @@ import { GeneralQueueProcessor } from './processors/general-queue.processor';
         attempts: 3,
       },
     }),
+    BullModule.registerQueue({
+      name: QueueName.BASE_QUEUE,
+      defaultJobOptions: {
+        attempts: 2,
+        removeOnComplete: { count: 200 },
+        removeOnFail: { count: 100 },
+      },
+    }),
   ],
   exports: [BullModule],
   providers: [GeneralQueueProcessor],

apps/server/src/integrations/storage/drivers/azure.driver.ts

@@ -0,0 +1,192 @@
+import { Readable } from 'stream';
+import {
+  AzureStorageConfig,
+  StorageDriver,
+  StorageOption,
+} from '../interfaces';
+import {
+  BlobSASPermissions,
+  BlobServiceClient,
+  BlockBlobClient,
+  ContainerClient,
+  generateBlobSASQueryParameters,
+  SASProtocol,
+  StorageSharedKeyCredential,
+} from '@azure/storage-blob';
+import { Logger } from '@nestjs/common';
+import { getMimeType } from '../../../common/helpers';
+
+export class AzureDriver implements StorageDriver {
+  private readonly config: AzureStorageConfig;
+  private readonly blobServiceClient: BlobServiceClient;
+  private readonly containerClient: ContainerClient;
+  private readonly sharedKeyCredential: StorageSharedKeyCredential;
+  private readonly accountUrl: string;
+
+  constructor(config: AzureStorageConfig) {
+    this.config = config;
+
+    if (!config.accountName) {
+      throw new Error('AzureDriver: accountName is required');
+    }
+    if (!config.container) {
+      throw new Error('AzureDriver: container is required');
+    }
+    if (!config.accountKey) {
+      throw new Error('AzureDriver: accountKey is required');
+    }
+
+    this.accountUrl =
+      config.endpoint ??
+      `https://${config.accountName}.blob.core.windows.net`;
+
+    this.sharedKeyCredential = new StorageSharedKeyCredential(
+      config.accountName,
+      config.accountKey,
+    );
+
+    this.blobServiceClient = this.createBlobServiceClient();
+    this.containerClient = this.blobServiceClient.getContainerClient(
+      config.container,
+    );
+  }
+
+  private blockBlob(filePath: string): BlockBlobClient {
+    return this.containerClient.getBlockBlobClient(filePath);
+  }
+
+  async upload(filePath: string, file: Buffer | Readable): Promise<void> {
+    const stream: Readable = Buffer.isBuffer(file) ? Readable.from(file) : file;
+    await this.uploadStream(filePath, stream);
+  }
+
+  async uploadStream(
+    filePath: string,
+    file: Readable,
+    options?: { recreateClient?: boolean },
+  ): Promise<void> {
+    const clientToUse = options?.recreateClient
+      ? this.createBlobServiceClient()
+          .getContainerClient(this.config.container)
+          .getBlockBlobClient(filePath)
+      : this.blockBlob(filePath);
+
+    try {
+      const contentType = getMimeType(filePath);
+      await clientToUse.uploadStream(file, undefined, undefined, {
+        blobHTTPHeaders: { blobContentType: contentType },
+      });
+    } catch (err) {
+      Logger.error(err);
+      throw new Error(`Failed to upload file: ${(err as Error).message}`);
+    }
+  }
+
+  async copy(fromFilePath: string, toFilePath: string): Promise<void> {
+    try {
+      if (!(await this.exists(fromFilePath))) {
+        return;
+      }
+      const sourceUrl = await this.getSignedUrl(fromFilePath, 60);
+      const dest = this.blockBlob(toFilePath);
+      await dest.syncCopyFromURL(sourceUrl);
+    } catch (err) {
+      throw new Error(`Failed to copy file: ${(err as Error).message}`);
+    }
+  }
+
+  async read(filePath: string): Promise<Buffer> {
+    try {
+      return await this.blockBlob(filePath).downloadToBuffer();
+    } catch (err) {
+      throw new Error(
+        `Failed to read file from Azure: ${(err as Error).message}`,
+      );
+    }
+  }
+
+  async readStream(filePath: string): Promise<Readable> {
+    try {
+      const response = await this.blockBlob(filePath).download();
+      return response.readableStreamBody as Readable;
+    } catch (err) {
+      throw new Error(
+        `Failed to read file from Azure: ${(err as Error).message}`,
+      );
+    }
+  }
+
+  async readRangeStream(
+    filePath: string,
+    range: { start: number; end: number },
+  ): Promise<Readable> {
+    try {
+      const count = range.end - range.start + 1;
+      const response = await this.blockBlob(filePath).download(
+        range.start,
+        count,
+      );
+      return response.readableStreamBody as Readable;
+    } catch (err) {
+      throw new Error(
+        `Failed to read file from Azure: ${(err as Error).message}`,
+      );
+    }
+  }
+
+  async exists(filePath: string): Promise<boolean> {
+    try {
+      return await this.blockBlob(filePath).exists();
+    } catch (err) {
+      throw new Error(
+        `Failed to check existence in Azure: ${(err as Error).message}`,
+      );
+    }
+  }
+
+  getUrl(filePath: string): string {
+    const base = this.config.baseUrl ?? this.accountUrl;
+    return `${base}/${this.config.container}/${filePath}`;
+  }
+
+  async getSignedUrl(filePath: string, expiresIn: number): Promise<string> {
+    const expiresOn = new Date(Date.now() + expiresIn * 1000);
+    const sas = generateBlobSASQueryParameters(
+      {
+        containerName: this.config.container,
+        blobName: filePath,
+        permissions: BlobSASPermissions.parse('r'),
+        expiresOn,
+        protocol: SASProtocol.HttpsAndHttp,
+      },
+      this.sharedKeyCredential,
+    ).toString();
+    return `${this.accountUrl}/${this.config.container}/${filePath}?${sas}`;
+  }
+
+  async delete(filePath: string): Promise<void> {
+    try {
+      await this.blockBlob(filePath).delete();
+    } catch (err) {
+      throw new Error(
+        `Error deleting file ${filePath} from Azure: ${(err as Error).message}`,
+      );
+    }
+  }
+
+  getDriver(): BlobServiceClient {
+    return this.blobServiceClient;
+  }
+
+  getDriverName(): string {
+    return StorageOption.AZURE;
+  }
+
+  getConfig(): Record<string, any> {
+    return this.config;
+  }
+
+  private createBlobServiceClient(): BlobServiceClient {
+    return new BlobServiceClient(this.accountUrl, this.sharedKeyCredential);
+  }
+}

apps/server/src/integrations/storage/drivers/index.ts

@@ -1,2 +1,3 @@
 export { LocalDriver } from './local.driver';
 export { S3Driver } from './s3.driver';
+export { AzureDriver } from './azure.driver';

apps/server/src/integrations/storage/interfaces/storage.interface.ts

@@ -3,11 +3,13 @@ import { S3ClientConfig } from '@aws-sdk/client-s3';
 export enum StorageOption {
   LOCAL = 'local',
   S3 = 's3',
+  AZURE = 'azure',
 }
 
 export type StorageConfig =
   | { driver: StorageOption.LOCAL; config: LocalStorageConfig }
-  | { driver: StorageOption.S3; config: S3StorageConfig };
+  | { driver: StorageOption.S3; config: S3StorageConfig }
+  | { driver: StorageOption.AZURE; config: AzureStorageConfig };
 
 export interface LocalStorageConfig {
   storagePath: string;
@@ -20,6 +22,14 @@ export interface S3StorageConfig
   baseUrl?: string; // Optional CDN URL for assets
 }
 
+export interface AzureStorageConfig {
+  accountName: string;
+  container: string;
+  accountKey: string;
+  endpoint?: string;
+  baseUrl?: string;
+}
+
 export interface StorageOptions {
   disk: StorageConfig;
 }

apps/server/src/integrations/storage/providers/storage.provider.ts

@@ -4,13 +4,14 @@ import {
 } from '../constants/storage.constants';
 import { EnvironmentService } from '../../environment/environment.service';
 import {
+  AzureStorageConfig,
   LocalStorageConfig,
   S3StorageConfig,
   StorageConfig,
   StorageDriver,
   StorageOption,
 } from '../interfaces';
-import { LocalDriver, S3Driver } from '../drivers';
+import { AzureDriver, LocalDriver, S3Driver } from '../drivers';
 import * as process from 'node:process';
 import { LOCAL_STORAGE_PATH } from '../../../common/helpers';
 import path from 'path';
@@ -21,6 +22,8 @@ function createStorageDriver(disk: StorageConfig): StorageDriver {
       return new LocalDriver(disk.config as LocalStorageConfig);
     case StorageOption.S3:
       return new S3Driver(disk.config as S3StorageConfig);
+    case StorageOption.AZURE:
+      return new AzureDriver(disk.config as AzureStorageConfig);
     default:
       throw new Error(`Unknown storage driver`);
   }
@@ -70,6 +73,18 @@ export const storageDriverConfigProvider = {
 
         return s3Config; }
 
+      case StorageOption.AZURE:
+        return {
+          driver,
+          config: {
+            accountName: environmentService.getAzureStorageAccountName(),
+            container: environmentService.getAzureStorageContainer(),
+            accountKey: environmentService.getAzureStorageAccountKey(),
+            endpoint: environmentService.getAzureStorageEndpoint() || undefined,
+            baseUrl: environmentService.getAzureStorageUrl() || undefined,
+          },
+        };
+
       default:
         throw new Error(`Unknown storage driver: ${driver}`);
     }