feat(API): page content update and retrieval (#1937)

* feat: page content update and retrieval output

* import module

* refactor naming
* support prepend

* rename contentOperation -> operation

* dry

* add yjs utils

apps/server/src/collaboration/collaboration.handler.ts

@@ -1,5 +1,12 @@
 import { Injectable, Logger } from '@nestjs/common';
 import { Hocuspocus, Document } from '@hocuspocus/server';
+import { TiptapTransformer } from '@hocuspocus/transformer';
+import {
+  prosemirrorNodeToYElement,
+  tiptapExtensions,
+} from './collaboration.util';
+import * as Y from 'yjs';
+import { User } from '@docmost/db/types/entity.types';
 
 export type CollabEventHandlers = ReturnType<
   CollaborationHandler['getHandlers']
@@ -20,6 +27,44 @@ export class CollaborationHandler {
         //   const fragment = doc.getXmlFragment('default');
         //});
       },
+      updatePageContent: async (
+        documentName: string,
+        payload: {
+          prosemirrorJson: any;
+          operation: string;
+          user: User;
+        },
+      ) => {
+        const { prosemirrorJson, operation, user } = payload;
+        this.logger.debug('Updating page content via yjs', documentName);
+        await this.withYdocConnection(
+          hocuspocus,
+          documentName,
+          { user },
+          (doc) => {
+            const fragment = doc.getXmlFragment('default');
+
+            if (operation === 'replace') {
+              if (fragment.length > 0) {
+                fragment.delete(0, fragment.length);
+              }
+
+              const newDoc = TiptapTransformer.toYdoc(
+                prosemirrorJson,
+                'default',
+                tiptapExtensions,
+              );
+              Y.applyUpdate(doc, Y.encodeStateAsUpdate(newDoc));
+            } else {
+              const newContent = prosemirrorJson.content || [];
+              const yElements = newContent.map(prosemirrorNodeToYElement);
+              const position =
+                operation === 'prepend' ? 0 : fragment.length;
+              fragment.insert(position, yElements);
+            }
+          },
+        );
+      },
     };
   }
 

apps/server/src/collaboration/collaboration.module.ts

@@ -1,4 +1,10 @@
-import { Logger, Module, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
+import {
+  Global,
+  Logger,
+  Module,
+  OnModuleDestroy,
+  OnModuleInit,
+} from '@nestjs/common';
 import { AuthenticationExtension } from './extensions/authentication.extension';
 import { PersistenceExtension } from './extensions/persistence.extension';
 import { CollaborationGateway } from './collaboration.gateway';

apps/server/src/collaboration/collaboration.util.ts

@@ -34,6 +34,7 @@ import {
   Highlight,
   UniqueID,
   addUniqueIdsToDoc,
+  htmlToMarkdown,
 } from '@docmost/editor-ext';
 import { generateText, getSchema, JSONContent } from '@tiptap/core';
 import { generateHTML, generateJSON } from '../common/helpers/prosemirror/html';
@@ -42,6 +43,7 @@ import { generateHTML, generateJSON } from '../common/helpers/prosemirror/html';
 // see:https://github.com/ueberdosis/tiptap/issues/4089
 //import { generateJSON } from '@tiptap/html';
 import { Node, Schema } from '@tiptap/pm/model';
+import * as Y from 'yjs';
 import { Logger } from '@nestjs/common';
 
 export const tiptapExtensions = [
@@ -161,3 +163,37 @@ function stripUnknownNodes(
 
   return json;
 }
+
+export function prosemirrorNodeToYElement(node: any): Y.XmlElement | Y.XmlText {
+  if (node.type === 'text') {
+    const ytext = new Y.XmlText();
+    ytext.insert(0, node.text || '');
+    if (node.marks?.length > 0) {
+      const attrs: Record<string, any> = {};
+      for (const mark of node.marks) {
+        attrs[mark.type] = mark.attrs || true;
+      }
+      ytext.format(0, node.text?.length || 0, attrs);
+    }
+    return ytext;
+  }
+
+  const element = new Y.XmlElement(node.type);
+  if (node.attrs) {
+    for (const [key, value] of Object.entries(node.attrs)) {
+      if (value !== null && value !== undefined) {
+        element.setAttribute(key, value as any);
+      }
+    }
+  }
+  if (node.content?.length > 0) {
+    const children = node.content.map(prosemirrorNodeToYElement);
+    element.insert(0, children);
+  }
+  return element;
+}
+
+export function jsonToMarkdown(tiptapJson: any): string {
+  const html = jsonToHtml(tiptapJson);
+  return htmlToMarkdown(html);
+}

apps/server/src/collaboration/extensions/persistence.extension.ts

@@ -181,7 +181,8 @@ export class PersistenceExtension implements Extension {
 
   async onChange(data: onChangePayload) {
     const documentName = data.documentName;
-    const userId = data.context?.user.id;
+    const userId = data.context?.user?.id;
+
     if (!userId) return;
 
     if (!this.contributors.has(documentName)) {

apps/server/src/collaboration/yjs.util.ts

@@ -0,0 +1,177 @@
+import {
+  initProseMirrorDoc,
+  relativePositionToAbsolutePosition,
+} from 'y-prosemirror';
+import * as Y from 'yjs';
+import { Document } from '@hocuspocus/server';
+import { getSchema } from '@tiptap/core';
+import { tiptapExtensions } from './collaboration.util';
+
+export type YjsSelection = {
+  anchor: any;
+  head: any;
+};
+
+export function setYjsMark(
+  doc: Document,
+  fragment: Y.XmlFragment,
+  yjsSelection: YjsSelection,
+  markName: string,
+  markAttributes: Record<string, any>,
+) {
+  const schema = getSchema(tiptapExtensions);
+  const { mapping } = initProseMirrorDoc(fragment, schema);
+
+  // Convert JSON positions to Y.js RelativePosition objects
+  const anchorRelPos = Y.createRelativePositionFromJSON(yjsSelection.anchor);
+  const headRelPos = Y.createRelativePositionFromJSON(yjsSelection.head);
+
+  const anchor = relativePositionToAbsolutePosition(
+    doc,
+    fragment,
+    anchorRelPos,
+    mapping,
+  );
+  const head = relativePositionToAbsolutePosition(
+    doc,
+    fragment,
+    headRelPos,
+    mapping,
+  );
+
+  if (anchor === null || head === null) {
+    throw new Error(
+      'Could not resolve Y.js relative positions to absolute positions',
+    );
+  }
+
+  const from = Math.min(anchor, head);
+  const to = Math.max(anchor, head);
+
+  // Apply mark directly to Y.js XmlText nodes
+  // This bypasses updateYFragment which has compatibility issues
+  applyMarkToYFragment(fragment, from, to, markName, markAttributes);
+}
+
+function applyMarkToYFragment(
+  fragment: Y.XmlFragment,
+  from: number,
+  to: number,
+  markName: string,
+  markAttributes: Record<string, any>,
+) {
+  let pos = 0;
+
+  const processItem = (item: any): boolean => {
+    if (pos >= to) return false;
+
+    if (item instanceof Y.XmlText) {
+      const textLength = item.length;
+      const itemEnd = pos + textLength;
+
+      if (itemEnd > from && pos < to) {
+        const formatFrom = Math.max(0, from - pos);
+        const formatTo = Math.min(textLength, to - pos);
+        const formatLength = formatTo - formatFrom;
+
+        if (formatLength > 0) {
+          item.format(formatFrom, formatLength, { [markName]: markAttributes });
+        }
+      }
+      pos = itemEnd;
+    } else if (item instanceof Y.XmlElement) {
+      pos++; // Opening tag
+      for (let i = 0; i < item.length; i++) {
+        if (!processItem(item.get(i))) return false;
+      }
+      pos++; // Closing tag
+    }
+    return true;
+  };
+
+  for (let i = 0; i < fragment.length; i++) {
+    if (!processItem(fragment.get(i))) break;
+  }
+}
+
+/**
+ * Removes a mark from all text in the fragment that has the specified attribute value.
+ * Useful for deleting comments by commentId.
+ */
+export function removeYjsMarkByAttribute(
+  fragment: Y.XmlFragment,
+  markName: string,
+  attributeName: string,
+  attributeValue: string,
+) {
+  const processItem = (item: any) => {
+    if (item instanceof Y.XmlText) {
+      // Get all formatting deltas to find ranges with this mark
+      const deltas = item.toDelta();
+      let offset = 0;
+
+      for (const delta of deltas) {
+        const length = delta.insert?.length ?? 0;
+        const attributes = delta.attributes ?? {};
+        const markAttr = attributes[markName];
+
+        if (markAttr && markAttr[attributeName] === attributeValue) {
+          // Remove the mark by setting it to null
+          item.format(offset, length, { [markName]: null });
+        }
+        offset += length;
+      }
+    } else if (item instanceof Y.XmlElement) {
+      for (let i = 0; i < item.length; i++) {
+        processItem(item.get(i));
+      }
+    }
+  };
+
+  for (let i = 0; i < fragment.length; i++) {
+    processItem(fragment.get(i));
+  }
+}
+
+/**
+ * Updates a mark's attributes for all text that has the specified attribute value.
+ * Useful for resolving/unresolving comments by commentId.
+ */
+export function updateYjsMarkAttribute(
+  fragment: Y.XmlFragment,
+  markName: string,
+  findByAttribute: { name: string; value: string },
+  newAttributes: Record<string, any>,
+) {
+  const processItem = (item: any) => {
+    if (item instanceof Y.XmlText) {
+      const deltas = item.toDelta();
+      let offset = 0;
+
+      for (const delta of deltas) {
+        const length = delta.insert?.length ?? 0;
+        const attributes = delta.attributes ?? {};
+        const markAttr = attributes[markName];
+
+        if (
+          markAttr &&
+          markAttr[findByAttribute.name] === findByAttribute.value
+        ) {
+          // Update the mark with new attributes (merge with existing)
+          item.format(offset, length, {
+            [markName]: { ...markAttr, ...newAttributes },
+          });
+        }
+        offset += length;
+      }
+    } else if (item instanceof Y.XmlElement) {
+      for (let i = 0; i < item.length; i++) {
+        processItem(item.get(i));
+      }
+    }
+  };
+
+  for (let i = 0; i < fragment.length; i++) {
+    processItem(fragment.get(i));
+  }
+}