From 2cb2f0e4e8c065416b406e481edad5041148168b Mon Sep 17 00:00:00 2001 From: Roberto Musso Date: Tue, 24 Feb 2026 21:34:48 +0100 Subject: [PATCH] feat: integrate vectordb for note embeddings - Added `vectordb` as a dependency in `package.json`. - Implemented `embedText` function in `src/main/ai/embeddings.ts` to handle text embeddings using GitHub Copilot OAuth token or OpenAI token. - Created `vectordb.ts` for managing LanceDB connection and embedding notes with upsert strategy. - Updated `index.ts` to initialize vector database and migrate existing notes on app ready. - Modified `router/index.ts` to fire-and-forget embedding calls on note creation and updates. - Enhanced `progress.txt` with detailed implementation notes and learnings regarding the integration. --- package-lock.json | 530 ++++++++++++++++++++++++++++++++++++-- package.json | 1 + prd.json | 4 +- progress.txt | 26 ++ src/main/ai/embeddings.ts | 73 ++++++ src/main/db/vectordb.ts | 113 ++++++++ src/main/index.ts | 5 + src/main/router/index.ts | 24 +- vite.main.config.mts | 1 + 9 files changed, 750 insertions(+), 27 deletions(-) create mode 100644 src/main/ai/embeddings.ts create mode 100644 src/main/db/vectordb.ts diff --git a/package-lock.json b/package-lock.json index e48411f..25651dd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -43,6 +43,7 @@ "remark-gfm": "^4.0.1", "tailwind-merge": "^3.5.0", "tw-animate-css": "^1.4.0", + "vectordb": "^0.21.2", "zod": "^4.3.6" }, "devDependencies": { @@ -75,6 +76,30 @@ "vite": "^5.4.21" } }, + "node_modules/@75lb/deep-merge": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@75lb/deep-merge/-/deep-merge-1.1.2.tgz", + "integrity": "sha512-08K9ou5VNbheZFxM5tDWoqjA3ImC50DiuuJ2tj1yEPRfkp8lLLg6XAaJ4On+a0yAXor/8ay5gHnAIshRM44Kpw==", + "license": "MIT", + "peer": true, + "dependencies": { + "lodash": "^4.17.21", + "typical": "^7.1.1" + }, + "engines": { + "node": ">=12.17" + } + }, + "node_modules/@75lb/deep-merge/node_modules/typical": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/typical/-/typical-7.3.0.tgz", + "integrity": "sha512-ya4mg/30vm+DOWfBg4YK3j2WD6TWtRkCbasOJr40CseYENzCUby/7rIvXA99JGsQHeNxLbnXdyLLxKSv3tauFw==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=12.17" + } + }, "node_modules/@antfu/ni": { "version": "25.0.0", "resolved": "https://registry.npmjs.org/@antfu/ni/-/ni-25.0.0.tgz", @@ -117,6 +142,39 @@ } } }, + "node_modules/@apache-arrow/ts": { + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/@apache-arrow/ts/-/ts-14.0.2.tgz", + "integrity": "sha512-CtwAvLkK0CZv7xsYeCo91ml6PvlfzAmAJZkRYuz2GNBwfYufj5SVi0iuSMwIMkcU/szVwvLdzORSLa5PlF/2ug==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@types/command-line-args": "5.2.0", + "@types/command-line-usage": "5.0.2", + "@types/node": "20.3.0", + "@types/pad-left": "2.1.1", + "command-line-args": "5.2.1", + "command-line-usage": "7.0.1", + "flatbuffers": "23.5.26", + "json-bignum": "^0.0.3", + "pad-left": "^2.1.0", + "tslib": "^2.5.3" + } + }, + "node_modules/@apache-arrow/ts/node_modules/@types/node": { + "version": "20.3.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.0.tgz", + "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==", + "license": "MIT", + "peer": true + }, + "node_modules/@apache-arrow/ts/node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD", + "peer": true + }, "node_modules/@babel/code-frame": { "version": "7.29.0", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz", @@ -3771,6 +3829,72 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@lancedb/vectordb-darwin-arm64": { + "version": "0.21.2", + "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-arm64/-/vectordb-darwin-arm64-0.21.2.tgz", + "integrity": "sha512-NAQnIKLw9K33KMODNXBEW0qC8/safWzZtqbVC7j1GcE7PSk0Uc6x7w5nrH5gvleZggjaxY9jaRVTqmtg7PNmqw==", + "cpu": [ + "arm64" + ], + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@lancedb/vectordb-darwin-x64": { + "version": "0.21.2", + "resolved": "https://registry.npmjs.org/@lancedb/vectordb-darwin-x64/-/vectordb-darwin-x64-0.21.2.tgz", + "integrity": "sha512-PudbltlbRiXvBf/bkAaDPL8+RqcI4TG69u00rQHxwkhH7PgPYRTUjfzfaQfiDXZuLXuZHQq703RyoHOqzsHN0Q==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@lancedb/vectordb-linux-arm64-gnu": { + "version": "0.21.2", + "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-arm64-gnu/-/vectordb-linux-arm64-gnu-0.21.2.tgz", + "integrity": "sha512-3lJ8lootlwLmhqabCdg0DKftv0Ujep6NTWAoLWK/6VQe2IgHmu/ZPRNQkOSZ5tnYlmRyDiMDMB2tlAzo45sV8Q==", + "cpu": [ + "arm64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@lancedb/vectordb-linux-x64-gnu": { + "version": "0.21.2", + "resolved": "https://registry.npmjs.org/@lancedb/vectordb-linux-x64-gnu/-/vectordb-linux-x64-gnu-0.21.2.tgz", + "integrity": "sha512-5I2drMOIyRODlAHPsipQBTrRRgcOZ45N5GsuhqcKnz3Tg8GAdc1MQKyK3BrdJzKHLPdRtIyRJ6QTLB3wZvDsQQ==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@lancedb/vectordb-win32-x64-msvc": { + "version": "0.21.2", + "resolved": "https://registry.npmjs.org/@lancedb/vectordb-win32-x64-msvc/-/vectordb-win32-x64-msvc-0.21.2.tgz", + "integrity": "sha512-gjpFukq0NTQSRpWPNIpq4XFtaudjSNBT6DMsagC61D2nx9ZLEdSAdU0wdkeluQwhoMvNnXEPdP9HxDSFUXk+Ww==", + "cpu": [ + "x64" + ], + "license": "Apache-2.0", + "optional": true, + "os": [ + "win32" + ] + }, "node_modules/@langchain/anthropic": { "version": "1.3.19", "resolved": "https://registry.npmjs.org/@langchain/anthropic/-/anthropic-1.3.19.tgz", @@ -4657,6 +4781,12 @@ "@tybys/wasm-util": "^0.10.0" } }, + "node_modules/@neon-rs/load": { + "version": "0.0.74", + "resolved": "https://registry.npmjs.org/@neon-rs/load/-/load-0.0.74.tgz", + "integrity": "sha512-/cPZD907UNz55yrc/ud4wDgQKtU1TvkD9jeqZWG6J4IMmZkp6zgjkQcKA8UvpkZlcpPHvc8J17sGzLFbP/LUYg==", + "license": "MIT" + }, "node_modules/@noble/ciphers": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/@noble/ciphers/-/ciphers-1.3.0.tgz", @@ -7640,6 +7770,20 @@ "@types/responselike": "^1.0.0" } }, + "node_modules/@types/command-line-args": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.0.tgz", + "integrity": "sha512-UuKzKpJJ/Ief6ufIaIzr3A/0XnluX7RvFgwkV89Yzvm77wCh1kFaFmqN8XEnGcN62EuHdedQjEMb8mYxFLGPyA==", + "license": "MIT", + "peer": true + }, + "node_modules/@types/command-line-usage": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/@types/command-line-usage/-/command-line-usage-5.0.2.tgz", + "integrity": "sha512-n7RlEEJ+4x4TS7ZQddTmNSxP+zziEG0TNsMfiRIxcIVXt71ENJ9ojeXmGO3wPoTdn7pJcU2xc3CJYMktNT6DPg==", + "license": "MIT", + "peer": true + }, "node_modules/@types/debug": { "version": "4.1.12", "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", @@ -7799,6 +7943,13 @@ "undici-types": "~7.18.0" } }, + "node_modules/@types/pad-left": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/@types/pad-left/-/pad-left-2.1.1.tgz", + "integrity": "sha512-Xd22WCRBydkGSApl5Bw0PhAOHKSVjNL3E3AwzKaps96IMraPqy5BvZIsBVK6JLwdybUzjHnuWVwpDd0JjTfHXA==", + "license": "MIT", + "peer": true + }, "node_modules/@types/react": { "version": "19.2.14", "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", @@ -8909,7 +9060,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, "license": "MIT", "dependencies": { "color-convert": "^2.0.1" @@ -8945,6 +9095,42 @@ "node": ">= 8" } }, + "node_modules/apache-arrow": { + "version": "14.0.2", + "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-14.0.2.tgz", + "integrity": "sha512-EBO2xJN36/XoY81nhLcwCJgFwkboDZeyNQ+OPsG7bCoQjc2BT0aTyH/MR6SrL+LirSNz+cYqjGRlupMMlP1aEg==", + "license": "Apache-2.0", + "peer": true, + "dependencies": { + "@types/command-line-args": "5.2.0", + "@types/command-line-usage": "5.0.2", + "@types/node": "20.3.0", + "@types/pad-left": "2.1.1", + "command-line-args": "5.2.1", + "command-line-usage": "7.0.1", + "flatbuffers": "23.5.26", + "json-bignum": "^0.0.3", + "pad-left": "^2.1.0", + "tslib": "^2.5.3" + }, + "bin": { + "arrow2csv": "bin/arrow2csv.js" + } + }, + "node_modules/apache-arrow/node_modules/@types/node": { + "version": "20.3.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.3.0.tgz", + "integrity": "sha512-cumHmIAf6On83X7yP+LrsEyUOf/YlociZelmpRYaGFydoaPdxdt80MAbu6vWerQT2COCp2nPvHdsbD7tHn/YlQ==", + "license": "MIT", + "peer": true + }, + "node_modules/apache-arrow/node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD", + "peer": true + }, "node_modules/argparse": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", @@ -8970,6 +9156,16 @@ "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", "license": "0BSD" }, + "node_modules/array-back": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-3.1.0.tgz", + "integrity": "sha512-TkuxA4UCOvxuDK6NZYXCalszEzj+TLszyASooky+i742l9TqsOdYCMJJupxRic61hwquNtppB3hgcuq9SVSH1Q==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=6" + } + }, "node_modules/array-buffer-byte-length": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/array-buffer-byte-length/-/array-buffer-byte-length-1.0.2.tgz", @@ -9132,6 +9328,12 @@ "node": ">= 0.4" } }, + "node_modules/asynckit": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "license": "MIT" + }, "node_modules/at-least-node": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz", @@ -9177,6 +9379,17 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/axios": { + "version": "1.13.5", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.5.tgz", + "integrity": "sha512-cz4ur7Vb0xS4/KUN0tPWe44eqxrIu31me+fbang3ijiNscE129POzipJJA6zniq2C/Z6sJCjMimjS8Lc/GAs8Q==", + "license": "MIT", + "dependencies": { + "follow-redirects": "^1.15.11", + "form-data": "^4.0.5", + "proxy-from-env": "^1.1.0" + } + }, "node_modules/babel-dead-code-elimination": { "version": "1.0.12", "resolved": "https://registry.npmjs.org/babel-dead-code-elimination/-/babel-dead-code-elimination-1.0.12.tgz", @@ -9596,7 +9809,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", - "dev": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0", @@ -9680,7 +9892,6 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, "license": "MIT", "dependencies": { "ansi-styles": "^4.1.0", @@ -9693,6 +9904,22 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, + "node_modules/chalk-template": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/chalk-template/-/chalk-template-0.4.0.tgz", + "integrity": "sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==", + "license": "MIT", + "peer": true, + "dependencies": { + "chalk": "^4.1.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/chalk-template?sponsor=1" + } + }, "node_modules/character-entities": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", @@ -9999,7 +10226,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "license": "MIT", "dependencies": { "color-name": "~1.1.4" @@ -10012,7 +10238,6 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true, "license": "MIT" }, "node_modules/colorette": { @@ -10022,6 +10247,18 @@ "dev": true, "license": "MIT" }, + "node_modules/combined-stream": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "license": "MIT", + "dependencies": { + "delayed-stream": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/comma-separated-tokens": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz", @@ -10032,6 +10269,58 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/command-line-args": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-5.2.1.tgz", + "integrity": "sha512-H4UfQhZyakIjC74I9d34fGYDwk3XpSr17QhEd0Q3I9Xq1CETHo4Hcuo87WyWHpAF1aSLjLRf5lD9ZGX2qStUvg==", + "license": "MIT", + "peer": true, + "dependencies": { + "array-back": "^3.1.0", + "find-replace": "^3.0.0", + "lodash.camelcase": "^4.3.0", + "typical": "^4.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/command-line-usage": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-7.0.1.tgz", + "integrity": "sha512-NCyznE//MuTjwi3y84QVUGEOT+P5oto1e1Pk/jFPVdPPfsG03qpTIl3yw6etR+v73d0lXsoojRpvbru2sqePxQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "array-back": "^6.2.2", + "chalk-template": "^0.4.0", + "table-layout": "^3.0.0", + "typical": "^7.1.1" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/command-line-usage/node_modules/array-back": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.2.tgz", + "integrity": "sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=12.17" + } + }, + "node_modules/command-line-usage/node_modules/typical": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/typical/-/typical-7.3.0.tgz", + "integrity": "sha512-ya4mg/30vm+DOWfBg4YK3j2WD6TWtRkCbasOJr40CseYENzCUby/7rIvXA99JGsQHeNxLbnXdyLLxKSv3tauFw==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=12.17" + } + }, "node_modules/commander": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz", @@ -10627,6 +10916,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/delayed-stream": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "license": "MIT", + "engines": { + "node": ">=0.4.0" + } + }, "node_modules/depd": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", @@ -11395,7 +11693,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", - "dev": true, "license": "MIT", "dependencies": { "call-bind-apply-helpers": "^1.0.1", @@ -12220,7 +12517,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -12230,7 +12526,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -12247,7 +12542,6 @@ "version": "1.1.1", "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", - "dev": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0" @@ -12260,7 +12554,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", - "dev": true, "license": "MIT", "dependencies": { "es-errors": "^1.3.0", @@ -13340,6 +13633,19 @@ "url": "https://opencollective.com/express" } }, + "node_modules/find-replace": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/find-replace/-/find-replace-3.0.0.tgz", + "integrity": "sha512-6Tb2myMioCAgv5kfvP5/PkZZ/ntTpVK39fHY7WkWBgvbeE+VHd/tZuZ4mrC+bxh4cfOZeYKVPaJIZtZXV7GNCQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "array-back": "^3.0.1" + }, + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/find-up": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", @@ -13372,6 +13678,13 @@ "node": "^10.12.0 || >=12.0.0" } }, + "node_modules/flatbuffers": { + "version": "23.5.26", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-23.5.26.tgz", + "integrity": "sha512-vE+SI9vrJDwi1oETtTIFldC/o9GsVKRM+s6EL0nQgxXlYV1Vc4Tk30hj4xGICftInKQKj1F3up2n8UbIVobISQ==", + "license": "SEE LICENSE IN LICENSE", + "peer": true + }, "node_modules/flatted": { "version": "3.3.3", "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.3.3.tgz", @@ -13393,6 +13706,26 @@ "node": ">= 12" } }, + "node_modules/follow-redirects": { + "version": "1.15.11", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", + "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, "node_modules/for-each": { "version": "0.3.5", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.5.tgz", @@ -13409,6 +13742,22 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/form-data": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", + "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", + "license": "MIT", + "dependencies": { + "asynckit": "^0.4.0", + "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", + "mime-types": "^2.1.12" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/formdata-polyfill": { "version": "4.0.10", "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", @@ -13534,7 +13883,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", - "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" @@ -13671,7 +14019,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", - "dev": true, "license": "MIT", "dependencies": { "call-bind-apply-helpers": "^1.0.2", @@ -13751,7 +14098,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", - "dev": true, "license": "MIT", "dependencies": { "dunder-proto": "^1.0.1", @@ -13949,7 +14295,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -14024,7 +14369,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -14063,7 +14407,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -14076,7 +14419,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "dev": true, "license": "MIT", "dependencies": { "has-symbols": "^1.0.3" @@ -14092,7 +14434,6 @@ "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", - "dev": true, "license": "MIT", "dependencies": { "function-bind": "^1.1.2" @@ -15214,6 +15555,15 @@ "node": ">=6" } }, + "node_modules/json-bignum": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/json-bignum/-/json-bignum-0.0.3.tgz", + "integrity": "sha512-2WHyXj3OfHSgNyuzDbSxI1w2jgw5gkWSWhS7Qg4bWXx1nLk3jnbwfUeS0PSba3IzpTUWdHxBieELUzXRjQB2zg==", + "peer": true, + "engines": { + "node": ">=0.8" + } + }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -15745,7 +16095,6 @@ "version": "4.17.23", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.23.tgz", "integrity": "sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==", - "dev": true, "license": "MIT" }, "node_modules/lodash-es": { @@ -15754,6 +16103,13 @@ "integrity": "sha512-kVI48u3PZr38HdYz98UmfPnXl2DXrpdctLrFLCd3kOx1xUkOmpFPx7gCWWM5MPkL/fD8zb+Ph0QzjGFs4+hHWg==", "license": "MIT" }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "license": "MIT", + "peer": true + }, "node_modules/lodash.get": { "version": "4.4.2", "resolved": "https://registry.npmjs.org/lodash.get/-/lodash.get-4.4.2.tgz", @@ -15952,7 +16308,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.4" @@ -16929,7 +17284,6 @@ "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.6" @@ -16939,7 +17293,6 @@ "version": "2.1.35", "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", - "dev": true, "license": "MIT", "dependencies": { "mime-db": "1.52.0" @@ -18070,6 +18423,19 @@ "dev": true, "license": "MIT" }, + "node_modules/pad-left": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/pad-left/-/pad-left-2.1.0.tgz", + "integrity": "sha512-HJxs9K9AztdIQIAIa/OIazRAUW/L6B9hbQDxO4X07roW3eo9XqZc2ur9bn1StH9CnbbI9EgvejHQX7CBpCF1QA==", + "license": "MIT", + "peer": true, + "dependencies": { + "repeat-string": "^1.5.4" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -18822,6 +19188,12 @@ "node": ">= 0.10" } }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "license": "MIT" + }, "node_modules/pump": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.3.tgz", @@ -19656,6 +20028,16 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/repeat-string": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/repeat-string/-/repeat-string-1.6.1.tgz", + "integrity": "sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=0.10" + } + }, "node_modules/require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", @@ -21189,6 +21571,16 @@ "node": ">= 0.4" } }, + "node_modules/stream-read-all": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/stream-read-all/-/stream-read-all-3.0.1.tgz", + "integrity": "sha512-EWZT9XOceBPlVJRrYcykW8jyRSZYbkb/0ZK36uLEmoWVO5gxBOnntNTseNzfREsqxqdfEGQrD8SXQ3QWbBmq8A==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=10" + } + }, "node_modules/strict-event-emitter": { "version": "0.5.1", "resolved": "https://registry.npmjs.org/strict-event-emitter/-/strict-event-emitter-0.5.1.tgz", @@ -21476,7 +21868,6 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, "license": "MIT", "dependencies": { "has-flag": "^4.0.0" @@ -21498,6 +21889,48 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/table-layout": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/table-layout/-/table-layout-3.0.2.tgz", + "integrity": "sha512-rpyNZYRw+/C+dYkcQ3Pr+rLxW4CfHpXjPDnG7lYhdRoUcZTUt+KEsX+94RGp/aVp/MQU35JCITv2T/beY4m+hw==", + "license": "MIT", + "peer": true, + "dependencies": { + "@75lb/deep-merge": "^1.1.1", + "array-back": "^6.2.2", + "command-line-args": "^5.2.1", + "command-line-usage": "^7.0.0", + "stream-read-all": "^3.0.1", + "typical": "^7.1.1", + "wordwrapjs": "^5.1.0" + }, + "bin": { + "table-layout": "bin/cli.js" + }, + "engines": { + "node": ">=12.17" + } + }, + "node_modules/table-layout/node_modules/array-back": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.2.tgz", + "integrity": "sha512-gUAZ7HPyb4SJczXAMUXMGAvI976JoK3qEx9v1FTmeYuJj0IBiaKttG1ydtGKdkfqWkIkouke7nG8ufGy77+Cvw==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=12.17" + } + }, + "node_modules/table-layout/node_modules/typical": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/typical/-/typical-7.3.0.tgz", + "integrity": "sha512-ya4mg/30vm+DOWfBg4YK3j2WD6TWtRkCbasOJr40CseYENzCUby/7rIvXA99JGsQHeNxLbnXdyLLxKSv3tauFw==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=12.17" + } + }, "node_modules/tagged-tag": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/tagged-tag/-/tagged-tag-1.0.0.tgz", @@ -22620,6 +23053,16 @@ "node": ">=14.17" } }, + "node_modules/typical": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/typical/-/typical-4.0.0.tgz", + "integrity": "sha512-VAH4IvQ7BDFYglMd7BPRDfLgxZZX4O4TFcRDA6EN5X7erNJJq+McIEp8np9aVtxrCJ6qx4GTYVfOWNjcqwZgRw==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=8" + } + }, "node_modules/unbox-primitive": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/unbox-primitive/-/unbox-primitive-1.1.0.tgz", @@ -23049,6 +23492,37 @@ "node": ">= 0.8" } }, + "node_modules/vectordb": { + "version": "0.21.2", + "resolved": "https://registry.npmjs.org/vectordb/-/vectordb-0.21.2.tgz", + "integrity": "sha512-5tiwUq0jDtfIpcr7NY+kNCTecHCzSq0AqQtMzJphH7z6H6gfrw9t5/Aoy5/QnS0uAWIgqvCbE5qneQOFGxE+Og==", + "cpu": [ + "x64", + "arm64" + ], + "deprecated": "Use @lancedb/lancedb instead.", + "license": "Apache-2.0", + "os": [ + "darwin", + "linux", + "win32" + ], + "dependencies": { + "@neon-rs/load": "^0.0.74", + "axios": "^1.4.0" + }, + "optionalDependencies": { + "@lancedb/vectordb-darwin-arm64": "0.21.2", + "@lancedb/vectordb-darwin-x64": "0.21.2", + "@lancedb/vectordb-linux-arm64-gnu": "0.21.2", + "@lancedb/vectordb-linux-x64-gnu": "0.21.2", + "@lancedb/vectordb-win32-x64-msvc": "0.21.2" + }, + "peerDependencies": { + "@apache-arrow/ts": "^14.0.2", + "apache-arrow": "^14.0.2" + } + }, "node_modules/vfile": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/vfile/-/vfile-6.0.3.tgz", @@ -23405,6 +23879,16 @@ "node": ">=0.10.0" } }, + "node_modules/wordwrapjs": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-5.1.1.tgz", + "integrity": "sha512-0yweIbkINJodk27gX9LBGMzyQdBDan3s/dEAiwBOj+Mf0PPyWL6/rikalkv8EeD0E8jm4o5RXEOrFTP3NXbhJg==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=12.17" + } + }, "node_modules/wrap-ansi": { "version": "8.1.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", diff --git a/package.json b/package.json index 323ce14..fbb9895 100644 --- a/package.json +++ b/package.json @@ -79,6 +79,7 @@ "remark-gfm": "^4.0.1", "tailwind-merge": "^3.5.0", "tw-animate-css": "^1.4.0", + "vectordb": "^0.21.2", "zod": "^4.3.6" } } diff --git a/prd.json b/prd.json index 18185e4..9ba5929 100644 --- a/prd.json +++ b/prd.json @@ -401,8 +401,8 @@ "Typecheck passes" ], "priority": 22, - "passes": false, - "notes": "" + "passes": true, + "notes": "Completed: vectordb npm package installed. src/main/ai/embeddings.ts reads GitHub Copilot OAuth token from ~/.copilot/config.json (falls back to stored OpenAI token) and calls OpenAIEmbeddings with Copilot base URL or standard OpenAI. src/main/db/vectordb.ts: initVectorDb() singleton, upsertNoteEmbedding() with delete-then-add strategy (table created on first upsert from schema inferred record), migrateNotesIfNeeded() embeds all SQLite notes sequentially on first launch. notes.create and notes.update made async with fire-and-forget .catch() embed calls; update re-fetches full note from SQLite to embed current title+content. initVectorDb().then(migrateNotesIfNeeded) chained in app.ready after initAI(). vectordb externalized in vite.main.config.mts. Typecheck passes (tsc --noEmit clean). Note: @github/copilot-sdk has no embeddings API; embeddings use @langchain/openai's OpenAIEmbeddings pointed at https://api.githubcopilot.com with the CLI OAuth token." }, { "id": "US-023", diff --git a/progress.txt b/progress.txt index fa8daef..a6a6ddd 100644 --- a/progress.txt +++ b/progress.txt @@ -493,3 +493,29 @@ - Short-circuit optimizations that skip LLM nodes are only safe for providers where the SDK has no internal state to initialize (OpenAI, Anthropic) - If you want to restore the short-circuit as an OpenAI/Anthropic optimization, gate it: `if (TOOL_CALLING_PROVIDERS.has(getActiveProviderName()) && state.chatContext.type === 'project')` --- + +## [2026-02-24] - US-022 +- What was implemented: + - Installed `vectordb` (LanceDB Node.js binding v0.21.2) as a project dependency + - Created `src/main/ai/embeddings.ts`: reads GitHub Copilot OAuth token from `~/.copilot/config.json` (via `copilot_tokens` map), falls back to stored OpenAI token via `getToken('openai')`. Uses `@langchain/openai` `OpenAIEmbeddings` with `baseURL: 'https://api.githubcopilot.com'` for Copilot path, or standard OpenAI API for fallback. Exposes `embedText(text): Promise` + - Created `src/main/db/vectordb.ts`: LanceDB singleton (`initVectorDb()` / `getConn()`), `upsertNoteEmbedding(noteId, projectId, content)` with delete-then-add upsert strategy (first call auto-creates table with schema inferred from first record), `migrateNotesIfNeeded()` that checks table existence on startup and bulk-embeds all SQLite notes sequentially with per-note error isolation + - Modified `src/main/router/index.ts`: imported `upsertNoteEmbedding`, made `notes.create` and `notes.update` async, added fire-and-forget embedding calls with `.catch(console.error)` in both handlers; `notes.update` re-fetches the full note from SQLite after the write to embed current title+content + - Modified `src/main/index.ts`: imported `initVectorDb` + `migrateNotesIfNeeded`, added `initVectorDb().then(() => migrateNotesIfNeeded()).catch(...)` chain in `app.on('ready')` + - Modified `vite.main.config.mts`: added `'vectordb'` to the `external` array so ViteRollup doesn't try to bundle the NAPI-RS binary +- Files changed: + - `package.json` (vectordb dependency added) + - `vite.main.config.mts` + - `src/main/index.ts` + - `src/main/router/index.ts` + - `src/main/ai/embeddings.ts` (new) + - `src/main/db/vectordb.ts` (new) +- **Learnings for future iterations:** + - `@github/copilot-sdk` has **no embeddings API** — it is a pure chat/session SDK. The `CopilotClient` type definitions contain zero mention of "embedding". Do not assume any LLM provider SDK supports embeddings + - GitHub Copilot CLI stores OAuth tokens in `~/.copilot/config.json` under `copilot_tokens["{host}:{login}"]`. The token format is `gho_*` (GitHub OAuth). These tokens work with the GitHub Copilot REST API (`https://api.githubcopilot.com`) which is OpenAI-compatible — including embeddings + - `@langchain/openai`'s `OpenAIEmbeddings` accepts a `configuration.baseURL` option that makes it work against any OpenAI-compatible endpoint + - `vectordb` (v0.21.2): deprecated but functional. The new package name is `@lancedb/lancedb`. `vectordb` requires at least one data record for `createTable()` (cannot create an empty table — schema is inferred from the first record). Use delete-then-add for upsert since there's no native upsert API at this version + - When using dynamic `import('@langchain/openai')`, TypeScript cannot infer the exact return type of `embedDocuments()` — it resolves to `{}` instead of `number[][]`. Fix: cast explicitly `as number[][]` + - tRPC mutation handlers support both sync and async functions transparently — making a mutation `async` does not break the renderer-side interface + - `notes.update` allows partial field updates (title or content can be omitted). Always re-fetch the full note from SQLite after the update write to get the correct combined text for embedding + - `vectordb`'s `table.delete(where)` accepts a raw SQL WHERE clause string. UUID v4 IDs are safe to interpolate directly (only `[0-9a-f-]` characters) +--- diff --git a/src/main/ai/embeddings.ts b/src/main/ai/embeddings.ts new file mode 100644 index 0000000..a94566b --- /dev/null +++ b/src/main/ai/embeddings.ts @@ -0,0 +1,73 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { getToken } from './token'; + +interface CopilotConfig { + copilot_tokens?: Record; +} + +/** + * Read the GitHub Copilot OAuth token from the CLI config file. + * Stored at ~/.copilot/config.json under copilot_tokens["{host}:{login}"]. + * Returns the first available token, or null if unavailable. + */ +function readCopilotToken(): string | null { + try { + const raw = fs.readFileSync( + path.join(os.homedir(), '.copilot', 'config.json'), + 'utf-8', + ); + const cfg = JSON.parse(raw) as CopilotConfig; + const vals = Object.values(cfg.copilot_tokens ?? {}); + return vals[0] ?? null; + } catch { + return null; + } +} + +/** + * Embed a single text string using the best available credentials. + * + * Priority: + * 1. GitHub Copilot CLI token → OpenAI-compatible embeddings endpoint at + * https://api.githubcopilot.com + * 2. Stored OpenAI token → standard OpenAI embeddings API + * + * Throws if no credentials are available or the API call fails. + * Callers must .catch() this and handle the error without rejecting + * the surrounding tRPC mutation. + */ +export async function embedText(text: string): Promise { + const { OpenAIEmbeddings } = await import('@langchain/openai'); + + const copilotToken = readCopilotToken(); + + let embeddingsInstance; + if (copilotToken) { + embeddingsInstance = new OpenAIEmbeddings({ + apiKey: copilotToken, + model: 'text-embedding-3-small', + configuration: { baseURL: 'https://api.githubcopilot.com' }, + }); + } else { + const openaiToken = await getToken('openai'); + if (!openaiToken) { + throw new Error( + '[Embeddings] No credentials available. Authenticate with Copilot CLI or add an OpenAI token in Settings.', + ); + } + embeddingsInstance = new OpenAIEmbeddings({ + apiKey: openaiToken, + model: 'text-embedding-3-small', + }); + } + + // embedDocuments returns number[][] — cast explicitly to satisfy strict TS + const results = (await embeddingsInstance.embedDocuments([text])) as number[][]; + const vector = results[0] as number[] | undefined; + if (!vector || vector.length === 0) { + throw new Error('[Embeddings] Empty vector returned from embedding API'); + } + return vector; +} diff --git a/src/main/db/vectordb.ts b/src/main/db/vectordb.ts new file mode 100644 index 0000000..4b483d3 --- /dev/null +++ b/src/main/db/vectordb.ts @@ -0,0 +1,113 @@ +import * as lancedb from 'vectordb'; +import { app } from 'electron'; +import path from 'node:path'; +import { getDb } from './index'; +import { notes } from './schema'; +import { embedText } from '../ai/embeddings'; + +interface NoteRecord { + id: string; + /** Empty string when the note has no project (Arrow string fields don't cleanly handle null) */ + projectId: string; + content: string; + vector: number[]; +} + +let conn: lancedb.Connection | null = null; + +/** + * Initialize the LanceDB connection. Must be called before any other + * function in this module. Vector data is stored at userData/vectors/. + */ +export async function initVectorDb(): Promise { + const vectorPath = path.join(app.getPath('userData'), 'vectors'); + conn = await lancedb.connect(vectorPath); + console.log('[VectorDB] Connected at:', vectorPath); +} + +function getConn(): lancedb.Connection { + if (!conn) throw new Error('[VectorDB] Not initialized. Call initVectorDb() first.'); + return conn; +} + +/** + * Embed note content and upsert the record into the LanceDB 'notes' table. + * + * Upsert strategy: delete-then-add. + * table.delete(where) is a no-op when no rows match, so this is safe for + * both first-time inserts and subsequent updates. + * + * On the very first call when the table does not yet exist, createTable + * infers the Arrow schema from the initial record. + * + * Throws on error — callers fire-and-forget via .catch(). + */ +export async function upsertNoteEmbedding( + noteId: string, + projectId: string | null, + content: string, +): Promise { + const c = getConn(); + const vector = await embedText(content); + + const record: NoteRecord = { + id: noteId, + projectId: projectId ?? '', + content, + vector, + }; + + const tableNames = await c.tableNames(); + + if (!tableNames.includes('notes')) { + // First embedding: createTable infers the Arrow schema from this record. + // The vector dimension (1536 for text-embedding-3-small) is baked in here. + await c.createTable('notes', [record]); + console.log('[VectorDB] Created notes table'); + return; + } + + const table = await c.openTable('notes'); + // Note IDs are UUID v4 — only [0-9a-f-] chars, no SQL injection risk. + await table.delete(`id = '${noteId}'`); + await table.add([record]); +} + +/** + * On first startup, check if the LanceDB 'notes' table exists. + * If not, embed all existing SQLite notes and populate LanceDB. + * + * Per-note errors are caught and logged; a single failure does not + * abort the remaining notes. + */ +export async function migrateNotesIfNeeded(): Promise { + const c = getConn(); + const tableNames = await c.tableNames(); + + if (tableNames.includes('notes')) { + console.log('[VectorDB] Notes table exists, skipping migration'); + return; + } + + const allNotes = getDb().select().from(notes).all(); + + if (allNotes.length === 0) { + console.log('[VectorDB] No existing notes to migrate'); + return; + } + + console.log(`[VectorDB] Migrating ${allNotes.length} notes...`); + let successCount = 0; + + for (const note of allNotes) { + try { + const embeddingText = `${note.title}\n\n${note.content}`; + await upsertNoteEmbedding(note.id, note.projectId ?? null, embeddingText); + successCount++; + } catch (err) { + console.error(`[VectorDB] Failed to embed note ${note.id} during migration:`, err); + } + } + + console.log(`[VectorDB] Migration complete: ${successCount}/${allNotes.length} notes embedded`); +} diff --git a/src/main/index.ts b/src/main/index.ts index cd13c2f..9b567fc 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -5,6 +5,7 @@ import { initDb } from './db'; import { appRouter } from './router'; import { createIPCHandler } from './ipc'; import { initAI } from './ai/provider'; +import { initVectorDb, migrateNotesIfNeeded } from './db/vectordb'; // Import to trigger provider registration before initAI() runs import './ai/copilot'; @@ -54,6 +55,10 @@ app.on('ready', () => { createIPCHandler({ router: appRouter, windows: [win] }); // AI init is best-effort — never block window creation initAI().catch((err) => console.error('[AI] Init failed:', err)); + // Vector DB init + migration is best-effort — runs after window is shown + initVectorDb() + .then(() => migrateNotesIfNeeded()) + .catch((err) => console.error('[VectorDB] Init or migration failed:', err)); }); // Quit when all windows are closed, except on macOS. There, it's common diff --git a/src/main/router/index.ts b/src/main/router/index.ts index 96bfb33..b187b81 100644 --- a/src/main/router/index.ts +++ b/src/main/router/index.ts @@ -7,6 +7,7 @@ import { clients, projects, tasks, checkpoints, notes, taskComments } from '../d import { getStore } from '../store'; import { saveTokenAndInit, hasActiveToken } from '../ai/provider'; import { orchestrate } from '../ai/orchestrator'; +import { upsertNoteEmbedding } from '../db/vectordb'; import type { TRPCContext } from '../ipc'; const t = initTRPC.context().create(); @@ -406,7 +407,7 @@ const notesRouter = router({ create: publicProcedure .input(z.object({ title: z.string(), content: z.string(), projectId: z.string().optional() })) - .mutation(({ input }) => { + .mutation(async ({ input }) => { const id = crypto.randomUUID(); const now = Date.now(); getDb().insert(notes).values({ @@ -417,18 +418,37 @@ const notesRouter = router({ createdAt: now, updatedAt: now, }).run(); + // Fire-and-forget: embed the note. Errors are logged, never thrown. + upsertNoteEmbedding(id, input.projectId ?? null, `${input.title}\n\n${input.content}`) + .catch((err) => console.error('[VectorDB] Failed to embed note on create:', err)); return { id }; }), update: publicProcedure .input(z.object({ id: z.string(), title: z.string().optional(), content: z.string().optional() })) - .mutation(({ input }) => { + .mutation(async ({ input }) => { const set: Partial<{ title: string; content: string; updatedAt: number }> = {}; if (input.title !== undefined) set.title = input.title; if (input.content !== undefined) set.content = input.content; // Always update updatedAt set.updatedAt = Date.now(); getDb().update(notes).set(set).where(eq(notes.id, input.id)).run(); + + // Re-embed if searchable text fields changed. + // Re-fetch from SQLite so the embedding reflects the full current note + // (the update may have changed only one of title or content). + if (input.title !== undefined || input.content !== undefined) { + const updated = getDb() + .select({ id: notes.id, projectId: notes.projectId, title: notes.title, content: notes.content }) + .from(notes) + .where(eq(notes.id, input.id)) + .all()[0]; + if (updated) { + upsertNoteEmbedding(updated.id, updated.projectId ?? null, `${updated.title}\n\n${updated.content}`) + .catch((err) => console.error('[VectorDB] Failed to embed note on update:', err)); + } + } + return null; }), diff --git a/vite.main.config.mts b/vite.main.config.mts index 4a336f6..c8ffe97 100644 --- a/vite.main.config.mts +++ b/vite.main.config.mts @@ -17,6 +17,7 @@ export default defineConfig({ '@langchain/anthropic', '@langchain/langgraph-checkpoint', '@langchain/langgraph-sdk', + 'vectordb', ], output: { entryFileNames: 'main.js',