diff --git a/apps/server/package.json b/apps/server/package.json index 80aa52a1..6d16b795 100644 --- a/apps/server/package.json +++ b/apps/server/package.json @@ -37,6 +37,7 @@ "@fastify/cookie": "^11.0.2", "@fastify/multipart": "^9.0.3", "@fastify/static": "^8.2.0", + "@langchain/textsplitters": "^0.1.0", "@nestjs/bullmq": "^11.0.2", "@nestjs/common": "^11.1.3", "@nestjs/config": "^4.0.2", @@ -78,6 +79,7 @@ "passport-jwt": "^4.0.1", "pg": "^8.16.0", "pg-tsquery": "^8.4.2", + "pgvector": "^0.2.1", "postmark": "^4.0.5", "react": "^18.3.1", "reflect-metadata": "^0.2.2", diff --git a/apps/server/src/collaboration/extensions/persistence.extension.ts b/apps/server/src/collaboration/extensions/persistence.extension.ts index 88284fd2..6c3f25ec 100644 --- a/apps/server/src/collaboration/extensions/persistence.extension.ts +++ b/apps/server/src/collaboration/extensions/persistence.extension.ts @@ -35,6 +35,7 @@ export class PersistenceExtension implements Extension { @InjectKysely() private readonly db: KyselyDB, private eventEmitter: EventEmitter2, @InjectQueue(QueueName.GENERAL_QUEUE) private generalQueue: Queue, + @InjectQueue(QueueName.AI_QUEUE) private aiQueue: Queue, ) {} async onLoadDocument(data: onLoadDocumentPayload) { @@ -181,10 +182,22 @@ export class PersistenceExtension implements Extension { } this.contributors.get(documentName).add(userId); + + console.log('embedd me') + const pageId = getPageId(documentName); + + await this.aiQueue.add(QueueJob.GENERATE_PAGE_EMBEDDINGS, { + pageId: pageId, + }); } async afterUnloadDocument(data: afterUnloadDocumentPayload) { const documentName = data.documentName; + const pageId = getPageId(documentName); + this.contributors.delete(documentName); + + // should only queue embed after unload// should delay so we dont embed always + } } diff --git a/apps/server/src/database/repos/page/page.repo.ts b/apps/server/src/database/repos/page/page.repo.ts index c814240a..26cb4745 100644 --- a/apps/server/src/database/repos/page/page.repo.ts +++ b/apps/server/src/database/repos/page/page.repo.ts @@ -63,6 +63,7 @@ export class PageRepo { pageId: string, opts?: { includeContent?: boolean; + includeText?: boolean; includeYdoc?: boolean; includeSpace?: boolean; includeCreator?: boolean; @@ -79,6 +80,7 @@ export class PageRepo { .selectFrom('pages') .select(this.baseFields) .$if(opts?.includeContent, (qb) => qb.select('content')) + .$if(opts?.includeText, (qb) => qb.select('textContent')) .$if(opts?.includeYdoc, (qb) => qb.select('ydoc')) .$if(opts?.includeHasChildren, (qb) => qb.select((eb) => this.withHasChildren(eb)), diff --git a/apps/server/src/database/types/db.interface.ts b/apps/server/src/database/types/db.interface.ts new file mode 100644 index 00000000..2ae1872f --- /dev/null +++ b/apps/server/src/database/types/db.interface.ts @@ -0,0 +1,45 @@ +import { + Attachments, + AuthAccounts, + AuthProviders, + Backlinks, + Billing, + Comments, + FileTasks, + Groups, + GroupUsers, + PageHistory, + Pages, + Shares, + SpaceMembers, + Spaces, + UserMfa, + Users, + UserTokens, + WorkspaceInvitations, + Workspaces, +} from '@docmost/db/types/db'; +import { Embeddings } from '@docmost/db/types/embeddings.types'; + +export interface DbInterface { + attachments: Attachments; + authAccounts: AuthAccounts; + authProviders: AuthProviders; + backlinks: Backlinks; + billing: Billing; + comments: Comments; + fileTasks: FileTasks; + groups: Groups; + groupUsers: GroupUsers; + pageEmbeddings: Embeddings; + pageHistory: PageHistory; + pages: Pages; + shares: Shares; + spaceMembers: SpaceMembers; + spaces: Spaces; + userMfa: UserMfa; + users: Users; + userTokens: UserTokens; + workspaceInvitations: WorkspaceInvitations; + workspaces: Workspaces; +} diff --git a/apps/server/src/database/types/embeddings.types.ts b/apps/server/src/database/types/embeddings.types.ts new file mode 100644 index 00000000..417594ae --- /dev/null +++ b/apps/server/src/database/types/embeddings.types.ts @@ -0,0 +1,17 @@ +import { Json, Timestamp, Generated } from '@docmost/db/types/db'; + +// page_embeddings type +export interface Embeddings { + id: Generated; + pageId: string; + spaceId: string; + workspaceId: string; + embedding: number[] | Buffer | string; + chunkIndex: Generated; + chunkStart: Generated; + chunkLength: Generated; + metadata: Generated; + createdAt: Generated; + updatedAt: Generated; + deletedAt: Timestamp | null; +} diff --git a/apps/server/src/database/types/entity.types.ts b/apps/server/src/database/types/entity.types.ts index b23fa775..0f358519 100644 --- a/apps/server/src/database/types/entity.types.ts +++ b/apps/server/src/database/types/entity.types.ts @@ -20,6 +20,7 @@ import { FileTasks, UserMfa as _UserMFA, } from './db'; +import { Embeddings } from '@docmost/db/types/embeddings.types'; // Workspace export type Workspace = Selectable; @@ -119,3 +120,8 @@ export type UpdatableFileTask = Updateable>; export type UserMFA = Selectable<_UserMFA>; export type InsertableUserMFA = Insertable<_UserMFA>; export type UpdatableUserMFA = Updateable>; + +// Page Embedding +export type PageEmbedding = Selectable; +export type InsertablePageEmbedding = Insertable; +export type UpdatablePageEmbedding = Updateable>; diff --git a/apps/server/src/database/types/kysely.types.ts b/apps/server/src/database/types/kysely.types.ts index 39dae715..d1bf8adc 100644 --- a/apps/server/src/database/types/kysely.types.ts +++ b/apps/server/src/database/types/kysely.types.ts @@ -1,5 +1,5 @@ -import { DB } from './db'; import { Kysely, Transaction } from 'kysely'; +import { DbInterface } from '@docmost/db/types/db.interface'; -export type KyselyDB = Kysely; -export type KyselyTransaction = Transaction; +export type KyselyDB = Kysely; +export type KyselyTransaction = Transaction; diff --git a/apps/server/src/ee b/apps/server/src/ee index 4100345c..d027cfe6 160000 --- a/apps/server/src/ee +++ b/apps/server/src/ee @@ -1 +1 @@ -Subproject commit 4100345c189026bfc3ba7d04edbef5ae7d28d91c +Subproject commit d027cfe622aeeea163f40df8d269aac01d221be5 diff --git a/apps/server/src/integrations/queue/constants/queue.constants.ts b/apps/server/src/integrations/queue/constants/queue.constants.ts index 150c098e..c345e956 100644 --- a/apps/server/src/integrations/queue/constants/queue.constants.ts +++ b/apps/server/src/integrations/queue/constants/queue.constants.ts @@ -4,6 +4,7 @@ export enum QueueName { GENERAL_QUEUE = '{general-queue}', BILLING_QUEUE = '{billing-queue}', FILE_TASK_QUEUE = '{file-task-queue}', + AI_QUEUE = '{ai-queue}', } export enum QueueJob { @@ -23,4 +24,7 @@ export enum QueueJob { IMPORT_TASK = 'import-task', EXPORT_TASK = 'export-task', + + GENERATE_PAGE_EMBEDDINGS = 'generate-page-embeddings', + DELETE_PAGE_EMBEDDINGS = 'delete-page-embeddings', } diff --git a/apps/server/src/integrations/queue/queue.module.ts b/apps/server/src/integrations/queue/queue.module.ts index 81aa0a5f..39c5a034 100644 --- a/apps/server/src/integrations/queue/queue.module.ts +++ b/apps/server/src/integrations/queue/queue.module.ts @@ -57,6 +57,14 @@ import { BacklinksProcessor } from './processors/backlinks.processor'; attempts: 1, }, }), + BullModule.registerQueue({ + name: QueueName.AI_QUEUE, + defaultJobOptions: { + removeOnComplete: true, + removeOnFail: true, + attempts: 1, + }, + }), ], exports: [BullModule], providers: [BacklinksProcessor], diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6e405f70..5777c3f7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -444,6 +444,9 @@ importers: '@fastify/static': specifier: ^8.2.0 version: 8.2.0 + '@langchain/textsplitters': + specifier: ^0.1.0 + version: 0.1.0(@langchain/core@0.3.72(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.2)(zod@3.25.56))) '@nestjs/bullmq': specifier: ^11.0.2 version: 11.0.2(@nestjs/common@11.1.3(class-transformer@0.5.1)(class-validator@0.14.1)(reflect-metadata@0.2.2)(rxjs@7.8.2))(@nestjs/core@11.1.3)(bullmq@5.53.2) @@ -567,6 +570,9 @@ importers: pg-tsquery: specifier: ^8.4.2 version: 8.4.2 + pgvector: + specifier: ^0.2.1 + version: 0.2.1 postmark: specifier: ^4.0.5 version: 4.0.5 @@ -1710,6 +1716,9 @@ packages: '@casl/ability': ^3.0.0 || ^4.0.0 || ^5.1.0 || ^6.0.0 react: ^16.0.0 || ^17.0.0 || ^18.0.0 + '@cfworker/json-schema@4.1.1': + resolution: {integrity: sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==} + '@chevrotain/cst-dts-gen@11.0.3': resolution: {integrity: sha512-BvIKpRLeS/8UbfxXxgC33xOumsacaeCKAjAeLyOn7Pcp95HiRbrpl14S+9vaZLolnbssPIUuiUd8IvgkRyt6NQ==} @@ -2513,6 +2522,16 @@ packages: '@keyv/serialize@1.0.3': resolution: {integrity: sha512-qnEovoOp5Np2JDGonIDL6Ayihw0RhnRh6vxPuHo4RDn1UOzwEo4AeIfpL6UGIrsceWrCMiVPgwRjbHu4vYFc3g==} + '@langchain/core@0.3.72': + resolution: {integrity: sha512-WsGWVZYnlKffj2eEfDocPNiaTRoxyYiLSQdQ7oxZvxGZBqo/90vpjbC33UGK1uPNBM4kT+pkdaol/MnvKUh8TQ==} + engines: {node: '>=18'} + + '@langchain/textsplitters@0.1.0': + resolution: {integrity: sha512-djI4uw9rlkAb5iMhtLED+xJebDdAG935AdP4eRTB02R7OB/act55Bj9wsskhZsvuyQRpO4O1wQOp85s6T6GWmw==} + engines: {node: '>=18'} + peerDependencies: + '@langchain/core': '>=0.2.21 <0.4.0' + '@lifeomic/attempt@3.0.3': resolution: {integrity: sha512-GlM2AbzrErd/TmLL3E8hAHmb5Q7VhDJp35vIbyPVA5Rz55LZuRr8pwL3qrwwkVNo05gMX1J44gURKb4MHQZo7w==} @@ -4391,6 +4410,9 @@ packages: '@types/react@18.3.12': resolution: {integrity: sha512-D2wOSq/d6Agt28q7rSI3jhU7G6aiuzljDGZ2hTZHIkrTLUI+AF3WMeKkEZ9nN2fkBAlcktT6vcZjDFiIhMYEQw==} + '@types/retry@0.12.0': + resolution: {integrity: sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==} + '@types/send@0.17.4': resolution: {integrity: sha512-x2EM6TJOybec7c52BX0ZspPodMsQUd5L6PRwOunVyVUhXiBSKf3AezDL8Dgvgt5o0UfKNfuA0eMLr2wLT4AiBA==} @@ -5232,6 +5254,9 @@ packages: console-control-strings@1.1.0: resolution: {integrity: sha512-ty/fTekppD2fIwRvnZAVdeOiGd1c7YXEixbgJTNzqcxJWKQnjJ/V1bNEEE6hygpM3WjwHFUVK6HTjWSzV4a8sQ==} + console-table-printer@2.14.6: + resolution: {integrity: sha512-MCBl5HNVaFuuHW6FGbL/4fB7N/ormCy+tQ+sxTrF6QtSbSNETvPuOVbkJBhzDgYhvjWGrTma4eYJa37ZuoQsPw==} + content-disposition@0.5.4: resolution: {integrity: sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==} engines: {node: '>= 0.6'} @@ -5954,6 +5979,9 @@ packages: eventemitter2@6.4.9: resolution: {integrity: sha512-JEPTiaOt9f04oa6NOkc4aH+nVp5I3wEjpHbIPqfgCdD5v5bUzy7xQqwcVO2aDQgOWhI28da57HksMrzK9HlRxg==} + eventemitter3@4.0.7: + resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==} + events@3.3.0: resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} engines: {node: '>=0.8.x'} @@ -6856,6 +6884,9 @@ packages: resolution: {integrity: sha512-cEiJEAEoIbWfCZYKWhVwFuvPX1gETRYPw6LlaTKoxD3s2AkXzkCjnp6h0V77ozyqj0jakteJ4YqDJT830+lVGw==} engines: {node: '>=14'} + js-tiktoken@1.0.21: + resolution: {integrity: sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==} + js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} @@ -7017,6 +7048,23 @@ packages: resolution: {integrity: sha512-QJv/h939gDpvT+9SiLVlY7tZC3xB2qK57v0J04Sh9wpMb6MP1q8gB21L3WIo8T5P1MSMg3Ep14L7KkDCFG3y4w==} engines: {node: '>=16.0.0'} + langsmith@0.3.61: + resolution: {integrity: sha512-b7Cpfj3xpWQO41G3xXeG6uzPzBcWfkEo5cK62WOcTqsKCchN2i42z7q45QQrbU6mdLwXp6pjRfnvr7wFu+Y5iQ==} + peerDependencies: + '@opentelemetry/api': '*' + '@opentelemetry/exporter-trace-otlp-proto': '*' + '@opentelemetry/sdk-trace-base': '*' + openai: '*' + peerDependenciesMeta: + '@opentelemetry/api': + optional: true + '@opentelemetry/exporter-trace-otlp-proto': + optional: true + '@opentelemetry/sdk-trace-base': + optional: true + openai: + optional: true + layout-base@1.0.2: resolution: {integrity: sha512-8h2oVEZNktL4BH2JCOI90iD1yXwL6iNW7KcCKT2QZgQJR2vbqDsldCTPRU9NifTCqHZci57XvQQ15YTu+sTYPg==} @@ -7444,6 +7492,10 @@ packages: multimath@2.0.0: resolution: {integrity: sha512-toRx66cAMJ+Ccz7pMIg38xSIrtnbozk0dchXezwQDMgQmbGpfxjtv68H+L00iFL8hxDaVjrmwAFSb3I6bg8Q2g==} + mustache@4.2.0: + resolution: {integrity: sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==} + hasBin: true + mute-stream@2.0.0: resolution: {integrity: sha512-WWdIxpyjEn+FhQJQQv9aQAYlHoNVdzIzUySNV1gHUPDSdZJ3yZn7pAAbQcV7B56Mvu881q9FZV+0Vx2xC44VWA==} engines: {node: ^18.17.0 || >=20.5.0} @@ -7698,6 +7750,10 @@ packages: otpauth@9.4.0: resolution: {integrity: sha512-fHIfzIG5RqCkK9cmV8WU+dPQr9/ebR5QOwGZn2JAr1RQF+lmAuLL2YdtdqvmBjNmgJlYk3KZ4a0XokaEhg1Jsw==} + p-finally@1.0.0: + resolution: {integrity: sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==} + engines: {node: '>=4'} + p-limit@2.3.0: resolution: {integrity: sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==} engines: {node: '>=6'} @@ -7722,6 +7778,18 @@ packages: resolution: {integrity: sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw==} engines: {node: '>=6'} + p-queue@6.6.2: + resolution: {integrity: sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==} + engines: {node: '>=8'} + + p-retry@4.6.2: + resolution: {integrity: sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==} + engines: {node: '>=8'} + + p-timeout@3.2.0: + resolution: {integrity: sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==} + engines: {node: '>=8'} + p-try@2.2.0: resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==} engines: {node: '>=6'} @@ -7886,6 +7954,10 @@ packages: pgpass@1.0.5: resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} + pgvector@0.2.1: + resolution: {integrity: sha512-nKaQY9wtuiidwLMdVIce1O3kL0d+FxrigCVzsShnoqzOSaWWWOvuctb/sYwlai5cTwwzRSNa+a/NtN2kVZGNJw==} + engines: {node: '>= 18'} + pica@7.1.1: resolution: {integrity: sha512-WY73tMvNzXWEld2LicT9Y260L43isrZ85tPuqRyvtkljSDLmnNFQmZICt4xUJMVulmcc6L9O7jbBrtx3DOz/YQ==} @@ -8504,6 +8576,10 @@ packages: resolution: {integrity: sha512-I1XxrZSQ+oErkRR4jYbAyEEu2I0avBvvMM5JN+6EBprOGRCs63ENqZ3vjavq8fBw2+62G5LF5XelKwuJpcvcxw==} engines: {node: '>=10'} + retry@0.13.1: + resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} + engines: {node: '>= 4'} + reusify@1.0.4: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} @@ -8684,6 +8760,9 @@ packages: resolution: {integrity: sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==} engines: {node: '>=14'} + simple-wcswidth@1.1.2: + resolution: {integrity: sha512-j7piyCjAeTDSjzTSQ7DokZtMNwNlEAyxqSZeCS+CXH7fJ4jx3FuJ/mTW3mE+6JLs4VJBbcll0Kjn+KXI5t21Iw==} + sisteransi@1.0.5: resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==} @@ -9292,6 +9371,10 @@ packages: resolution: {integrity: sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==} engines: {node: '>= 0.4.0'} + uuid@10.0.0: + resolution: {integrity: sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==} + hasBin: true + uuid@11.1.0: resolution: {integrity: sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==} hasBin: true @@ -9678,6 +9761,11 @@ packages: resolution: {integrity: sha512-dtZ0aQSFyZmoJS0m06/xBN1SazUBPL5HpzlAcs/KcRW0rzadYw12deQBjeMhGKMMeGEp7bA9vmikMLaO4exBcg==} engines: {node: '>=14.13.1'} + zod-to-json-schema@3.24.6: + resolution: {integrity: sha512-h/z3PKvcTcTetyjl1fkj79MHNEjm+HpD6NXheWjzOekY7kV+lwDYnHw+ivHkijnCSMz1yJaWBD9vu/Fcmk+vEg==} + peerDependencies: + zod: ^3.24.1 + zod@3.25.56: resolution: {integrity: sha512-rd6eEF3BTNvQnR2e2wwolfTmUTnp70aUTqr0oaGbHifzC3BKJsoV+Gat8vxUMR1hwOKBs6El+qWehrHbCpW6SQ==} @@ -11459,6 +11547,8 @@ snapshots: '@casl/ability': 6.7.2 react: 18.3.1 + '@cfworker/json-schema@4.1.1': {} + '@chevrotain/cst-dts-gen@11.0.3': dependencies: '@chevrotain/gast': 11.0.3 @@ -12325,6 +12415,31 @@ snapshots: dependencies: buffer: 6.0.3 + '@langchain/core@0.3.72(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.2)(zod@3.25.56))': + dependencies: + '@cfworker/json-schema': 4.1.1 + ansi-styles: 5.2.0 + camelcase: 6.3.0 + decamelize: 1.2.0 + js-tiktoken: 1.0.21 + langsmith: 0.3.61(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.2)(zod@3.25.56)) + mustache: 4.2.0 + p-queue: 6.6.2 + p-retry: 4.6.2 + uuid: 10.0.0 + zod: 3.25.56 + zod-to-json-schema: 3.24.6(zod@3.25.56) + transitivePeerDependencies: + - '@opentelemetry/api' + - '@opentelemetry/exporter-trace-otlp-proto' + - '@opentelemetry/sdk-trace-base' + - openai + + '@langchain/textsplitters@0.1.0(@langchain/core@0.3.72(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.2)(zod@3.25.56)))': + dependencies: + '@langchain/core': 0.3.72(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.2)(zod@3.25.56)) + js-tiktoken: 1.0.21 + '@lifeomic/attempt@3.0.3': {} '@lukeed/csprng@1.1.0': {} @@ -14293,6 +14408,8 @@ snapshots: '@types/prop-types': 15.7.11 csstype: 3.1.3 + '@types/retry@0.12.0': {} + '@types/send@0.17.4': dependencies: '@types/mime': 1.3.5 @@ -15327,6 +15444,10 @@ snapshots: console-control-strings@1.1.0: {} + console-table-printer@2.14.6: + dependencies: + simple-wcswidth: 1.1.2 + content-disposition@0.5.4: dependencies: safe-buffer: 5.2.1 @@ -16234,6 +16355,8 @@ snapshots: eventemitter2@6.4.9: {} + eventemitter3@4.0.7: {} + events@3.3.0: {} execa@5.1.1: @@ -17371,6 +17494,10 @@ snapshots: js-cookie@3.0.5: {} + js-tiktoken@1.0.21: + dependencies: + base64-js: 1.5.1 + js-tokens@4.0.0: {} js-yaml@3.14.1: @@ -17532,6 +17659,19 @@ snapshots: vscode-languageserver-textdocument: 1.0.12 vscode-uri: 3.0.8 + langsmith@0.3.61(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.2)(zod@3.25.56)): + dependencies: + '@types/uuid': 10.0.0 + chalk: 4.1.2 + console-table-printer: 2.14.6 + p-queue: 6.6.2 + p-retry: 4.6.2 + semver: 7.7.2 + uuid: 10.0.0 + optionalDependencies: + '@opentelemetry/api': 1.9.0 + openai: 5.12.2(ws@8.18.2)(zod@3.25.56) + layout-base@1.0.2: {} layout-base@2.0.1: {} @@ -18049,6 +18189,8 @@ snapshots: glur: 1.1.2 object-assign: 4.1.1 + mustache@4.2.0: {} + mute-stream@2.0.0: {} nanoid@3.3.11: {} @@ -18331,6 +18473,8 @@ snapshots: dependencies: '@noble/hashes': 1.7.1 + p-finally@1.0.0: {} + p-limit@2.3.0: dependencies: p-try: 2.2.0 @@ -18353,6 +18497,20 @@ snapshots: p-map@2.1.0: {} + p-queue@6.6.2: + dependencies: + eventemitter3: 4.0.7 + p-timeout: 3.2.0 + + p-retry@4.6.2: + dependencies: + '@types/retry': 0.12.0 + retry: 0.13.1 + + p-timeout@3.2.0: + dependencies: + p-finally: 1.0.0 + p-try@2.2.0: {} package-json-from-dist@1.0.0: {} @@ -18511,6 +18669,8 @@ snapshots: dependencies: split2: 4.2.0 + pgvector@0.2.1: {} + pica@7.1.1: dependencies: glur: 1.1.2 @@ -19175,6 +19335,8 @@ snapshots: ret@0.5.0: {} + retry@0.13.1: {} + reusify@1.0.4: {} rfdc@1.3.1: {} @@ -19379,6 +19541,8 @@ snapshots: signal-exit@4.1.0: {} + simple-wcswidth@1.1.2: {} + sisteransi@1.0.5: {} slash@3.0.0: {} @@ -20026,6 +20190,8 @@ snapshots: utils-merge@1.0.1: {} + uuid@10.0.0: {} + uuid@11.1.0: {} uuid@9.0.1: {} @@ -20375,6 +20541,10 @@ snapshots: css-what: 6.1.0 entities: 5.0.0 + zod-to-json-schema@3.24.6(zod@3.25.56): + dependencies: + zod: 3.25.56 + zod@3.25.56: {} zustand@4.5.6(@types/react@18.3.12)(react@18.3.1):