diff --git a/typescript-testing-examples/manual-experiments/deno.json b/typescript-testing-examples/manual-experiments/deno.json new file mode 100644 index 00000000..19d085b7 --- /dev/null +++ b/typescript-testing-examples/manual-experiments/deno.json @@ -0,0 +1,10 @@ +{ + "imports": { + "dotenv/config": "npm:/dotenv/config", + "langsmith": "npm:/langsmith@0.1.14", + "langsmith/": "npm:/langsmith@0.1.14/", + "uuid": "npm:/uuid", + "openai": "https://deno.land/x/openai@v4.26.1/mod.ts", + "openai/": "https://deno.land/x/openai@v4.26.1/" + } +} diff --git a/typescript-testing-examples/manual-experiments/deno.lock b/typescript-testing-examples/manual-experiments/deno.lock new file mode 100644 index 00000000..fabdaf52 --- /dev/null +++ b/typescript-testing-examples/manual-experiments/deno.lock @@ -0,0 +1,475 @@ +{ + "version": "3", + "packages": { + "specifiers": { + "npm:dotenv": "npm:dotenv@16.4.4", + "npm:langchain@0.1.19-rc.2": "npm:langchain@0.1.19-rc.2_zod@3.22.4", + "npm:langsmith@0.1.3": "npm:langsmith@0.1.3", + "npm:openai": "npm:openai@4.28.0", + "npm:wikipedia": "npm:wikipedia@2.1.2" + }, + "npm": { + "@anthropic-ai/sdk@0.9.1": { + "integrity": "sha512-wa1meQ2WSfoY8Uor3EdrJq0jTiZJoKoSii2ZVWRY1oN4Tlr5s59pADg9T79FTbPe1/se5c3pBeZgJL63wmuoBA==", + "dependencies": { + "@types/node": "@types/node@18.19.17", + "@types/node-fetch": "@types/node-fetch@2.6.11", + "abort-controller": "abort-controller@3.0.0", + "agentkeepalive": "agentkeepalive@4.5.0", + "digest-fetch": "digest-fetch@1.3.0", + "form-data-encoder": "form-data-encoder@1.7.2", + "formdata-node": "formdata-node@4.4.1", + "node-fetch": "node-fetch@2.7.0", + "web-streams-polyfill": "web-streams-polyfill@3.3.3" + } + }, + "@langchain/community@0.0.30": { + "integrity": "sha512-duYJtC8wN8FeIBkijGmDcOPpcK6dPU3Uwamh9K3ASuM97AUI2gEzZJeVtUPQiWFRPyDOAlPfNtrqxVUy2kmp9g==", + "dependencies": { + "@langchain/core": "@langchain/core@0.1.30_zod@3.22.4", + "@langchain/openai": "@langchain/openai@0.0.14_zod@3.22.4", + "flat": "flat@5.0.2", + "langsmith": "langsmith@0.1.3", + "uuid": "uuid@9.0.1", + "zod": "zod@3.22.4" + } + }, + "@langchain/core@0.1.30_zod@3.22.4": { + "integrity": "sha512-3oqEKgwe7U/efieZrCC6BLwOHm4YPj27mRNBZLaB5BwPh3a7gXIevxEnbjN3o5j9kJqc5acG7nn35h7Wkrf2Ag==", + "dependencies": { + "ansi-styles": "ansi-styles@5.2.0", + "camelcase": "camelcase@6.3.0", + "decamelize": "decamelize@1.2.0", + "js-tiktoken": "js-tiktoken@1.0.10", + "langsmith": "langsmith@0.1.3", + "ml-distance": "ml-distance@4.0.1", + "p-queue": "p-queue@6.6.2", + "p-retry": "p-retry@4.6.2", + "uuid": "uuid@9.0.1", + "zod": "zod@3.22.4", + "zod-to-json-schema": "zod-to-json-schema@3.22.4_zod@3.22.4" + } + }, + "@langchain/openai@0.0.14_zod@3.22.4": { + "integrity": "sha512-co6nRylPrLGY/C3JYxhHt6cxLq07P086O7K3QaZH7SFFErIN9wSzJonpvhZR07DEUq6eK6wKgh2ORxA/NcjSRQ==", + "dependencies": { + "@langchain/core": "@langchain/core@0.1.30_zod@3.22.4", + "js-tiktoken": "js-tiktoken@1.0.10", + "openai": "openai@4.28.0", + "zod": "zod@3.22.4", + "zod-to-json-schema": "zod-to-json-schema@3.22.4_zod@3.22.4" + } + }, + "@types/node-fetch@2.6.11": { + "integrity": "sha512-24xFj9R5+rfQJLRyM56qh+wnVSYhyXC2tkoBndtY0U+vubqNsYXGjufB2nn8Q6gt0LrARwL6UBtMCSVCwl4B1g==", + "dependencies": { + "@types/node": "@types/node@18.16.19", + "form-data": "form-data@4.0.0" + } + }, + "@types/node@18.16.19": { + "integrity": "sha512-IXl7o+R9iti9eBW4Wg2hx1xQDig183jj7YLn8F7udNceyfkbn1ZxmzZXuak20gR40D7pIkIY1kYGx5VIGbaHKA==", + "dependencies": {} + }, + "@types/node@18.19.17": { + "integrity": "sha512-SzyGKgwPzuWp2SHhlpXKzCX0pIOfcI4V2eF37nNBJOhwlegQ83omtVQ1XxZpDE06V/d6AQvfQdPfnw0tRC//Ng==", + "dependencies": { + "undici-types": "undici-types@5.26.5" + } + }, + "@types/retry@0.12.0": { + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", + "dependencies": {} + }, + "@types/uuid@9.0.8": { + "integrity": "sha512-jg+97EGIcY9AGHJJRaaPVgetKDsrTgbRjQ5Msgjh/DQKEFl0DtyRr/VCOyD1T2R1MNeWPK/u7JoGhlDZnKBAfA==", + "dependencies": {} + }, + "abort-controller@3.0.0": { + "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "dependencies": { + "event-target-shim": "event-target-shim@5.0.1" + } + }, + "agentkeepalive@4.5.0": { + "integrity": "sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==", + "dependencies": { + "humanize-ms": "humanize-ms@1.2.1" + } + }, + "ansi-styles@5.2.0": { + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dependencies": {} + }, + "argparse@2.0.1": { + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dependencies": {} + }, + "asynckit@0.4.0": { + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", + "dependencies": {} + }, + "axios@1.6.7": { + "integrity": "sha512-/hDJGff6/c7u0hDkvkGxR/oy6CbCs8ziCsC7SqmhjfozqiJGc8Z11wrv9z9lYfY4K8l+H9TpjcMDX0xOZmx+RA==", + "dependencies": { + "follow-redirects": "follow-redirects@1.15.5", + "form-data": "form-data@4.0.0", + "proxy-from-env": "proxy-from-env@1.1.0" + } + }, + "base-64@0.1.0": { + "integrity": "sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==", + "dependencies": {} + }, + "base64-js@1.5.1": { + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "dependencies": {} + }, + "binary-extensions@2.2.0": { + "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", + "dependencies": {} + }, + "binary-search@1.3.6": { + "integrity": "sha512-nbE1WxOTTrUWIfsfZ4aHGYu5DOuNkbxGokjV6Z2kxfJK3uaAb8zNK1muzOeipoLHZjInT4Br88BHpzevc681xA==", + "dependencies": {} + }, + "camelcase@4.1.0": { + "integrity": "sha512-FxAv7HpHrXbh3aPo4o2qxHay2lkLY3x5Mw3KeE4KQE8ysVfziWeRZDwcjauvwBSGEC/nXUPzZy8zeh4HokqOnw==", + "dependencies": {} + }, + "camelcase@6.3.0": { + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "dependencies": {} + }, + "charenc@0.0.2": { + "integrity": "sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==", + "dependencies": {} + }, + "combined-stream@1.0.8": { + "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", + "dependencies": { + "delayed-stream": "delayed-stream@1.0.0" + } + }, + "commander@10.0.1": { + "integrity": "sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug==", + "dependencies": {} + }, + "crypt@0.0.2": { + "integrity": "sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==", + "dependencies": {} + }, + "decamelize@1.2.0": { + "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", + "dependencies": {} + }, + "delayed-stream@1.0.0": { + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", + "dependencies": {} + }, + "digest-fetch@1.3.0": { + "integrity": "sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==", + "dependencies": { + "base-64": "base-64@0.1.0", + "md5": "md5@2.3.0" + } + }, + "dotenv@16.4.4": { + "integrity": "sha512-XvPXc8XAQThSjAbY6cQ/9PcBXmFoWuw1sQ3b8HqUCR6ziGXjkTi//kB9SWa2UwqlgdAIuRqAa/9hVljzPehbYg==", + "dependencies": {} + }, + "event-target-shim@5.0.1": { + "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", + "dependencies": {} + }, + "eventemitter3@4.0.7": { + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "dependencies": {} + }, + "expr-eval@2.0.2": { + "integrity": "sha512-4EMSHGOPSwAfBiibw3ndnP0AvjDWLsMvGOvWEZ2F96IGk0bIVdjQisOHxReSkE13mHcfbuCiXw+G4y0zv6N8Eg==", + "dependencies": {} + }, + "flat@5.0.2": { + "integrity": "sha512-b6suED+5/3rTpUBdG1gupIl8MPFCAMA0QXwmljLhvCUKcUvdE4gWky9zpuGCcXHOsz4J9wPGNWq6OKpmIzz3hQ==", + "dependencies": {} + }, + "follow-redirects@1.15.5": { + "integrity": "sha512-vSFWUON1B+yAw1VN4xMfxgn5fTUiaOzAJCKBwIIgT/+7CuGy9+r+5gITvP62j3RmaD5Ph65UaERdOSRGUzZtgw==", + "dependencies": {} + }, + "form-data-encoder@1.7.2": { + "integrity": "sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==", + "dependencies": {} + }, + "form-data@4.0.0": { + "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "dependencies": { + "asynckit": "asynckit@0.4.0", + "combined-stream": "combined-stream@1.0.8", + "mime-types": "mime-types@2.1.35" + } + }, + "formdata-node@4.4.1": { + "integrity": "sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==", + "dependencies": { + "node-domexception": "node-domexception@1.0.0", + "web-streams-polyfill": "web-streams-polyfill@4.0.0-beta.3" + } + }, + "humanize-ms@1.2.1": { + "integrity": "sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==", + "dependencies": { + "ms": "ms@2.1.3" + } + }, + "infobox-parser@3.6.4": { + "integrity": "sha512-d2lTlxKZX7WsYxk9/UPt51nkmZv5tbC75SSw4hfHqZ3LpRAn6ug0oru9xI2X+S78va3aUAze3xl/UqMuwLmJUw==", + "dependencies": { + "camelcase": "camelcase@4.1.0" + } + }, + "is-any-array@2.0.1": { + "integrity": "sha512-UtilS7hLRu++wb/WBAw9bNuP1Eg04Ivn1vERJck8zJthEvXCBEBpGR/33u/xLKWEQf95803oalHrVDptcAvFdQ==", + "dependencies": {} + }, + "is-buffer@1.1.6": { + "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==", + "dependencies": {} + }, + "js-tiktoken@1.0.10": { + "integrity": "sha512-ZoSxbGjvGyMT13x6ACo9ebhDha/0FHdKA+OsQcMOWcm1Zs7r90Rhk5lhERLzji+3rA7EKpXCgwXcM5fF3DMpdA==", + "dependencies": { + "base64-js": "base64-js@1.5.1" + } + }, + "js-yaml@4.1.0": { + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dependencies": { + "argparse": "argparse@2.0.1" + } + }, + "jsonpointer@5.0.1": { + "integrity": "sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==", + "dependencies": {} + }, + "langchain@0.1.19-rc.2_zod@3.22.4": { + "integrity": "sha512-2b0w5VM+VYH7GmIm1DldDLyowE8asE0jZya0AXRJgEgxjux+7L/4tb6HXRJIByiGBgmFVGcxoBvJaC71GM3NPw==", + "dependencies": { + "@anthropic-ai/sdk": "@anthropic-ai/sdk@0.9.1", + "@langchain/community": "@langchain/community@0.0.30", + "@langchain/core": "@langchain/core@0.1.30_zod@3.22.4", + "@langchain/openai": "@langchain/openai@0.0.14_zod@3.22.4", + "binary-extensions": "binary-extensions@2.2.0", + "expr-eval": "expr-eval@2.0.2", + "js-tiktoken": "js-tiktoken@1.0.10", + "js-yaml": "js-yaml@4.1.0", + "jsonpointer": "jsonpointer@5.0.1", + "langchainhub": "langchainhub@0.0.8", + "langsmith": "langsmith@0.1.3", + "ml-distance": "ml-distance@4.0.1", + "openapi-types": "openapi-types@12.1.3", + "p-retry": "p-retry@4.6.2", + "uuid": "uuid@9.0.1", + "yaml": "yaml@2.3.4", + "zod": "zod@3.22.4", + "zod-to-json-schema": "zod-to-json-schema@3.22.4_zod@3.22.4" + } + }, + "langchainhub@0.0.8": { + "integrity": "sha512-Woyb8YDHgqqTOZvWIbm2CaFDGfZ4NTSyXV687AG4vXEfoNo7cGQp7nhl7wL3ehenKWmNEmcxCLgOZzW8jE6lOQ==", + "dependencies": {} + }, + "langsmith@0.1.3": { + "integrity": "sha512-kQMS3QySeU0Qt9A71d9trUXbeKn33HfxpRc7hRjSB967zcdTAngh66NcqYqBflD3nOL4FK6LKmvfb3vbNDEoPg==", + "dependencies": { + "@types/uuid": "@types/uuid@9.0.8", + "commander": "commander@10.0.1", + "p-queue": "p-queue@6.6.2", + "p-retry": "p-retry@4.6.2", + "uuid": "uuid@9.0.1" + } + }, + "md5@2.3.0": { + "integrity": "sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==", + "dependencies": { + "charenc": "charenc@0.0.2", + "crypt": "crypt@0.0.2", + "is-buffer": "is-buffer@1.1.6" + } + }, + "mime-db@1.52.0": { + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "dependencies": {} + }, + "mime-types@2.1.35": { + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "dependencies": { + "mime-db": "mime-db@1.52.0" + } + }, + "ml-array-mean@1.1.6": { + "integrity": "sha512-MIdf7Zc8HznwIisyiJGRH9tRigg3Yf4FldW8DxKxpCCv/g5CafTw0RRu51nojVEOXuCQC7DRVVu5c7XXO/5joQ==", + "dependencies": { + "ml-array-sum": "ml-array-sum@1.1.6" + } + }, + "ml-array-sum@1.1.6": { + "integrity": "sha512-29mAh2GwH7ZmiRnup4UyibQZB9+ZLyMShvt4cH4eTK+cL2oEMIZFnSyB3SS8MlsTh6q/w/yh48KmqLxmovN4Dw==", + "dependencies": { + "is-any-array": "is-any-array@2.0.1" + } + }, + "ml-distance-euclidean@2.0.0": { + "integrity": "sha512-yC9/2o8QF0A3m/0IXqCTXCzz2pNEzvmcE/9HFKOZGnTjatvBbsn4lWYJkxENkA4Ug2fnYl7PXQxnPi21sgMy/Q==", + "dependencies": {} + }, + "ml-distance@4.0.1": { + "integrity": "sha512-feZ5ziXs01zhyFUUUeZV5hwc0f5JW0Sh0ckU1koZe/wdVkJdGxcP06KNQuF0WBTj8FttQUzcvQcpcrOp/XrlEw==", + "dependencies": { + "ml-array-mean": "ml-array-mean@1.1.6", + "ml-distance-euclidean": "ml-distance-euclidean@2.0.0", + "ml-tree-similarity": "ml-tree-similarity@1.0.0" + } + }, + "ml-tree-similarity@1.0.0": { + "integrity": "sha512-XJUyYqjSuUQkNQHMscr6tcjldsOoAekxADTplt40QKfwW6nd++1wHWV9AArl0Zvw/TIHgNaZZNvr8QGvE8wLRg==", + "dependencies": { + "binary-search": "binary-search@1.3.6", + "num-sort": "num-sort@2.1.0" + } + }, + "ms@2.1.3": { + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dependencies": {} + }, + "node-domexception@1.0.0": { + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "dependencies": {} + }, + "node-fetch@2.7.0": { + "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==", + "dependencies": { + "whatwg-url": "whatwg-url@5.0.0" + } + }, + "num-sort@2.1.0": { + "integrity": "sha512-1MQz1Ed8z2yckoBeSfkQHHO9K1yDRxxtotKSJ9yvcTUUxSvfvzEq5GwBrjjHEpMlq/k5gvXdmJ1SbYxWtpNoVg==", + "dependencies": {} + }, + "openai@4.28.0": { + "integrity": "sha512-JM8fhcpmpGN0vrUwGquYIzdcEQHtFuom6sRCbbCM6CfzZXNuRk33G7KfeRAIfnaCxSpzrP5iHtwJzIm6biUZ2Q==", + "dependencies": { + "@types/node": "@types/node@18.19.17", + "@types/node-fetch": "@types/node-fetch@2.6.11", + "abort-controller": "abort-controller@3.0.0", + "agentkeepalive": "agentkeepalive@4.5.0", + "digest-fetch": "digest-fetch@1.3.0", + "form-data-encoder": "form-data-encoder@1.7.2", + "formdata-node": "formdata-node@4.4.1", + "node-fetch": "node-fetch@2.7.0", + "web-streams-polyfill": "web-streams-polyfill@3.3.3" + } + }, + "openapi-types@12.1.3": { + "integrity": "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==", + "dependencies": {} + }, + "p-finally@1.0.0": { + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "dependencies": {} + }, + "p-queue@6.6.2": { + "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", + "dependencies": { + "eventemitter3": "eventemitter3@4.0.7", + "p-timeout": "p-timeout@3.2.0" + } + }, + "p-retry@4.6.2": { + "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", + "dependencies": { + "@types/retry": "@types/retry@0.12.0", + "retry": "retry@0.13.1" + } + }, + "p-timeout@3.2.0": { + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "dependencies": { + "p-finally": "p-finally@1.0.0" + } + }, + "proxy-from-env@1.1.0": { + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "dependencies": {} + }, + "retry@0.13.1": { + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "dependencies": {} + }, + "tr46@0.0.3": { + "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==", + "dependencies": {} + }, + "undici-types@5.26.5": { + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dependencies": {} + }, + "uuid@9.0.1": { + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "dependencies": {} + }, + "web-streams-polyfill@3.3.3": { + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "dependencies": {} + }, + "web-streams-polyfill@4.0.0-beta.3": { + "integrity": "sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==", + "dependencies": {} + }, + "webidl-conversions@3.0.1": { + "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==", + "dependencies": {} + }, + "whatwg-url@5.0.0": { + "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", + "dependencies": { + "tr46": "tr46@0.0.3", + "webidl-conversions": "webidl-conversions@3.0.1" + } + }, + "wikipedia@2.1.2": { + "integrity": "sha512-RAYaMpXC9/E873RaSEtlEa8dXK4e0p5k98GKOd210MtkE5emm6fcnwD+N6ZA4cuffjDWagvhaQKtp/mGp2BOVQ==", + "dependencies": { + "axios": "axios@1.6.7", + "infobox-parser": "infobox-parser@3.6.4" + } + }, + "yaml@2.3.4": { + "integrity": "sha512-8aAvwVUSHpfEqTQ4w/KMlf3HcRdt50E5ODIQJBw1fQ5RL34xabzxtUlzTXVqc4rkZsPbvrXKWnABCD7kWSmocA==", + "dependencies": {} + }, + "zod-to-json-schema@3.22.4_zod@3.22.4": { + "integrity": "sha512-2Ed5dJ+n/O3cU383xSY28cuVi0BCQhF8nYqWU5paEpl7fVdqdAmiLdqLyfblbNdfOFwFfi/mqU4O1pwc60iBhQ==", + "dependencies": { + "zod": "zod@3.22.4" + } + }, + "zod@3.22.4": { + "integrity": "sha512-iC+8Io04lddc+mVqQ9AZ7OQ2MrUKGN+oIQyq1vemgt46jwCwLfhq7/pwnBnNXXXZb8VTVLKwp9EDkx+ryxIWmg==", + "dependencies": {} + } + } + }, + "remote": { + "https://deno.land/x/p_limit@v1.0.0/mod.ts": "0348ca21c2cce293f6e8b42a96bb235a07550eba8e52c6713365ec247a0b2d8c", + "https://deno.land/x/yocto_queue@v0.1.3/mod.ts": "f63406731f43af2038c2a9158446d020d7a6f203ba5e1d9a6c18b1247ee0fb63" + }, + "workspace": { + "dependencies": [ + "npm:dotenv", + "npm:langchain@0.1.19-rc.2", + "npm:langsmith@0.1.3" + ] + } +} diff --git a/typescript-testing-examples/manual-experiments/manual-experimentation-langsmith.ipynb b/typescript-testing-examples/manual-experiments/manual-experimentation-langsmith.ipynb new file mode 100644 index 00000000..0353cb33 --- /dev/null +++ b/typescript-testing-examples/manual-experiments/manual-experimentation-langsmith.ipynb @@ -0,0 +1,342 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "42b8fc9e-1346-4c7a-be2e-f88f092f5fd5", + "metadata": {}, + "source": [ + "# Log manual experiments\n", + "\n", + "Here, we'll do a very basic walkthrough of how you can log individual predictions as LangSmith `experiments`. This is useful if you already have some evaluation flow set up but you still want to take advantage of LangSmith's experiment tracking functionality.\n", + "\n", + "First, create a LangSmith account and API Key, then create an .env file with values for the following variables in the same directory as this notebook:\n", + "\n", + "```\n", + "LANGCHAIN_API_KEY=\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "945ce28f-aeae-41ee-88f3-f86af63ae609", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Module: null prototype] { default: {} }" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import \"dotenv/config\"; // Load env vars from .env file" + ] + }, + { + "cell_type": "markdown", + "id": "984f1fbf-c171-4059-9e7d-12ced72fe150", + "metadata": {}, + "source": [ + "## Create Dataset\n", + "\n", + "First, create a dataset from the inputs (and optional reference outputs) we are evaluating over. These examples let us compare predictions from different models or systems on similar data points." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "ebb84a49-88d1-4920-96f4-063a04afd8a7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{\n", + " inputs: { input: \u001b[32m\"Foo 2\"\u001b[39m },\n", + " outputs: { output: \u001b[32m\"Bar 2\"\u001b[39m },\n", + " dataset_id: \u001b[32m\"893dd7d2-8ba6-4a47-9ebb-6fc133aaeba1\"\u001b[39m,\n", + " source_run_id: \u001b[1mnull\u001b[22m,\n", + " metadata: \u001b[1mnull\u001b[22m,\n", + " created_at: \u001b[32m\"2024-04-17T22:56:51.675000+00:00\"\u001b[39m,\n", + " id: \u001b[32m\"3a3b8945-156e-4a35-8e2f-12f7619a77ce\"\u001b[39m,\n", + " name: \u001b[32m\"\"\u001b[39m,\n", + " modified_at: \u001b[32m\"2024-04-17T22:56:51.675000+00:00\"\u001b[39m\n", + "}" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import { Client } from \"langsmith\";\n", + "\n", + "const client = new Client();\n", + "\n", + "const datasetName = `My-Dataset-${new Date().toISOString()}`;\n", + "const dataset = await client.createDataset(datasetName);\n", + "await client.createExample({\"input\": \"Foo\"}, {\"output\": \"Bar\"}, {datasetId: dataset.id})\n", + "await client.createExample({\"input\": \"Foo 2\"}, {\"output\": \"Bar 2\"}, {datasetId: dataset.id})" + ] + }, + { + "cell_type": "markdown", + "id": "b84a02fa-5aac-47e4-a683-05f2d2408cf8", + "metadata": {}, + "source": [ + "## AddPrediction helper\n", + "\n", + "Next, define an `addPrediction` helper function. This does 2 things:\n", + "1. Creates an experiment, if it doesn't already exist.\n", + "2. Creates a `Run` to represent your prediction on this data point.\n", + "\n", + "It returns the runId you can use for logging feedback." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "9435fc39-727a-4bd8-bc94-690a9317b5a0", + "metadata": {}, + "outputs": [], + "source": [ + "import { v4 as uuidv4 } from \"uuid\";\n", + "\n", + "interface addPredictionProps {\n", + " experimentName: string;\n", + " name?: string;\n", + " runId?: string;\n", + " referenceExampleId: string;\n", + " inputs?: unknown;\n", + "}\n", + "\n", + "async function addPrediction(client: Client, prediction: unknown, props: addPredictionProps): Promise {\n", + " const { experimentName, runId, referenceExampleId, inputs, name } = props;\n", + "\n", + " const example = await client.readExample(referenceExampleId);\n", + "\n", + " await client.createProject({ projectName: experimentName, referenceDatasetId: example.dataset_id, upsert: true });\n", + " const runId_ = runId ?? uuidv4();\n", + " await client.createRun({\n", + " name: name ?? \"Tested\" ,\n", + " id: runId_,\n", + " inputs: inputs ? {input: inputs}: undefined,\n", + " outputs: {output: prediction}, \n", + " run_type: 'chain', \n", + " reference_example_id: referenceExampleId,\n", + " start_time: Date.now(),\n", + " end_time: Date.now(),\n", + " project_name: experimentName,\n", + " });\n", + " return runId_;\n", + "}\n" + ] + }, + { + "cell_type": "markdown", + "id": "fd5df87a-8879-44bb-ad2c-bf4083ebaf99", + "metadata": {}, + "source": [ + "## Add your first prediction\n", + "\n", + "To log a prediction, at minimum we need:\n", + "- The predicted value\n", + "- The dataset example we were predicting for\n", + "- The experiment name to associate this prediction with" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "d3357ac7-d1d7-4c70-8995-effeb19c5f73", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[\n", + " \u001b[32m\"3a3b8945-156e-4a35-8e2f-12f7619a77ce\"\u001b[39m,\n", + " \u001b[32m\"8164992a-8db6-495e-bd23-bef7769629a9\"\u001b[39m\n", + "]\n" + ] + } + ], + "source": [ + "const exampleIds = [];\n", + "for await (const example of client.listExamples({datasetName})) {\n", + " exampleIds.push(example.id)\n", + "}\n", + "console.log(exampleIds)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "4e7efb70-68f0-443f-8de7-15820fd82509", + "metadata": {}, + "outputs": [], + "source": [ + "const predictionOne = \"Foo\"; \n", + "const experimentName = \"MyExperiment\"\n", + "\n", + "const runId = await addPrediction(client, predictionOne, {referenceExampleId: exampleIds[0], experimentName})" + ] + }, + { + "cell_type": "markdown", + "id": "577de06f-e72a-422d-80eb-139a5c22688a", + "metadata": {}, + "source": [ + "## Log Feedback\n", + "\n", + "Now you can log any type of feedback metrics for this prediction. For instance, you can score this with continuous values:" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "8b09c584-8a13-4be3-80d3-183b941088d2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{\n", + " id: \u001b[32m\"a619d236-a28f-444a-a127-aea3498b0808\"\u001b[39m,\n", + " run_id: \u001b[32m\"5abc59ee-0e42-4f5f-b600-7f8e76cb4794\"\u001b[39m,\n", + " key: \u001b[32m\"correctness\"\u001b[39m,\n", + " score: \u001b[33m1\u001b[39m,\n", + " value: \u001b[90mundefined\u001b[39m,\n", + " correction: \u001b[90mundefined\u001b[39m,\n", + " comment: \u001b[32m\"This looks impeccable.\"\u001b[39m,\n", + " feedback_source: { type: \u001b[32m\"api\"\u001b[39m, metadata: {} },\n", + " feedbackConfig: \u001b[90mundefined\u001b[39m\n", + "}" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "await client.createFeedback(runId, \"correctness\", {score: 1, comment: \"This looks impeccable.\"})" + ] + }, + { + "cell_type": "markdown", + "id": "e88feee2-a3d3-4363-b50d-c6490e172aa3", + "metadata": {}, + "source": [ + "You can also log unstructured notes without a score." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "afc9db6e-c0d3-44fc-9ed5-833de18c8808", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{\n", + " id: \u001b[32m\"2acb6755-6c35-4d24-83c8-9bbb79e85674\"\u001b[39m,\n", + " run_id: \u001b[32m\"5abc59ee-0e42-4f5f-b600-7f8e76cb4794\"\u001b[39m,\n", + " key: \u001b[32m\"note\"\u001b[39m,\n", + " score: \u001b[90mundefined\u001b[39m,\n", + " value: \u001b[90mundefined\u001b[39m,\n", + " correction: \u001b[90mundefined\u001b[39m,\n", + " comment: \u001b[32m\"I think I could do better though. Not gonna leave a score here.\"\u001b[39m,\n", + " feedback_source: { type: \u001b[32m\"api\"\u001b[39m, metadata: {} },\n", + " feedbackConfig: \u001b[90mundefined\u001b[39m\n", + "}" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "await client.createFeedback(runId, \"note\", { comment: \"I think I could do better though. Not gonna leave a score here.\"})" + ] + }, + { + "cell_type": "markdown", + "id": "b857026d-abfc-4969-8fb5-479758054283", + "metadata": {}, + "source": [ + "## Add more predictions\n", + "\n", + "Continue in a similar way, logging predictions for each example in the dataset." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "b99762c7-1dbf-4e7d-bda1-830b2b6a45e0", + "metadata": {}, + "outputs": [], + "source": [ + "const predictionTwo = \"Bar\";\n", + "const runId2 = await addPrediction(client, predictionTwo, {referenceExampleId: exampleIds[1], experimentName})" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "27c57250-07ac-4c90-94af-8a0d749a8939", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{\n", + " id: \u001b[32m\"aa8f1d3f-ff14-47b5-b974-48da07ff2faa\"\u001b[39m,\n", + " run_id: \u001b[32m\"5abc59ee-0e42-4f5f-b600-7f8e76cb4794\"\u001b[39m,\n", + " key: \u001b[32m\"Correctness\"\u001b[39m,\n", + " score: \u001b[33m0\u001b[39m,\n", + " value: \u001b[90mundefined\u001b[39m,\n", + " correction: \u001b[90mundefined\u001b[39m,\n", + " comment: \u001b[32m\"Completely wrong.\"\u001b[39m,\n", + " feedback_source: { type: \u001b[32m\"api\"\u001b[39m, metadata: {} },\n", + " feedbackConfig: \u001b[90mundefined\u001b[39m\n", + "}" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "await client.createFeedback(runId, \"Correctness\", {score: 0, comment: \"Completely wrong.\"})" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Deno", + "language": "typescript", + "name": "deno" + }, + "language_info": { + "file_extension": ".ts", + "mimetype": "text/x.typescript", + "name": "typescript", + "nb_converter": "script", + "pygments_lexer": "typescript", + "version": "5.3.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}