diff --git a/poetry.lock b/poetry.lock index 49e2144..7cbf1db 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1597,6 +1597,104 @@ files = [ {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, ] +[[package]] +name = "ijson" +version = "3.2.3" +description = "Iterative JSON parser with standard Python iterator interfaces" +optional = false +python-versions = "*" +files = [ + {file = "ijson-3.2.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0a4ae076bf97b0430e4e16c9cb635a6b773904aec45ed8dcbc9b17211b8569ba"}, + {file = "ijson-3.2.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cfced0a6ec85916eb8c8e22415b7267ae118eaff2a860c42d2cc1261711d0d31"}, + {file = "ijson-3.2.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0b9d1141cfd1e6d6643aa0b4876730d0d28371815ce846d2e4e84a2d4f471cf3"}, + {file = "ijson-3.2.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9e0a27db6454edd6013d40a956d008361aac5bff375a9c04ab11fc8c214250b5"}, + {file = "ijson-3.2.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c0d526ccb335c3c13063c273637d8611f32970603dfb182177b232d01f14c23"}, + {file = "ijson-3.2.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:545a30b3659df2a3481593d30d60491d1594bc8005f99600e1bba647bb44cbb5"}, + {file = "ijson-3.2.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:9680e37a10fedb3eab24a4a7e749d8a73f26f1a4c901430e7aa81b5da15f7307"}, + {file = "ijson-3.2.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:2a80c0bb1053055d1599e44dc1396f713e8b3407000e6390add72d49633ff3bb"}, + {file = "ijson-3.2.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f05ed49f434ce396ddcf99e9fd98245328e99f991283850c309f5e3182211a79"}, + {file = "ijson-3.2.3-cp310-cp310-win32.whl", hash = "sha256:b4eb2304573c9fdf448d3fa4a4fdcb727b93002b5c5c56c14a5ffbbc39f64ae4"}, + {file = "ijson-3.2.3-cp310-cp310-win_amd64.whl", hash = "sha256:923131f5153c70936e8bd2dd9dcfcff43c67a3d1c789e9c96724747423c173eb"}, + {file = "ijson-3.2.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:904f77dd3d87736ff668884fe5197a184748eb0c3e302ded61706501d0327465"}, + {file = "ijson-3.2.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0974444c1f416e19de1e9f567a4560890095e71e81623c509feff642114c1e53"}, + {file = "ijson-3.2.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c1a4b8eb69b6d7b4e94170aa991efad75ba156b05f0de2a6cd84f991def12ff9"}, + {file = "ijson-3.2.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d052417fd7ce2221114f8d3b58f05a83c1a2b6b99cafe0b86ac9ed5e2fc889df"}, + {file = "ijson-3.2.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b8064a85ec1b0beda7dd028e887f7112670d574db606f68006c72dd0bb0e0e2"}, + {file = "ijson-3.2.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eaac293853f1342a8d2a45ac1f723c860f700860e7743fb97f7b76356df883a8"}, + {file = "ijson-3.2.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6c32c18a934c1dc8917455b0ce478fd7a26c50c364bd52c5a4fb0fc6bb516af7"}, + {file = "ijson-3.2.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:713a919e0220ac44dab12b5fed74f9130f3480e55e90f9d80f58de129ea24f83"}, + {file = "ijson-3.2.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:4a3a6a2fbbe7550ffe52d151cf76065e6b89cfb3e9d0463e49a7e322a25d0426"}, + {file = "ijson-3.2.3-cp311-cp311-win32.whl", hash = "sha256:6a4db2f7fb9acfb855c9ae1aae602e4648dd1f88804a0d5cfb78c3639bcf156c"}, + {file = "ijson-3.2.3-cp311-cp311-win_amd64.whl", hash = "sha256:ccd6be56335cbb845f3d3021b1766299c056c70c4c9165fb2fbe2d62258bae3f"}, + {file = "ijson-3.2.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:055b71bbc37af5c3c5861afe789e15211d2d3d06ac51ee5a647adf4def19c0ea"}, + {file = "ijson-3.2.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c075a547de32f265a5dd139ab2035900fef6653951628862e5cdce0d101af557"}, + {file = "ijson-3.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:457f8a5fc559478ac6b06b6d37ebacb4811f8c5156e997f0d87d708b0d8ab2ae"}, + {file = "ijson-3.2.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9788f0c915351f41f0e69ec2618b81ebfcf9f13d9d67c6d404c7f5afda3e4afb"}, + {file = "ijson-3.2.3-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fa234ab7a6a33ed51494d9d2197fb96296f9217ecae57f5551a55589091e7853"}, + {file = "ijson-3.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdd0dc5da4f9dc6d12ab6e8e0c57d8b41d3c8f9ceed31a99dae7b2baf9ea769a"}, + {file = "ijson-3.2.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:c6beb80df19713e39e68dc5c337b5c76d36ccf69c30b79034634e5e4c14d6904"}, + {file = "ijson-3.2.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:a2973ce57afb142d96f35a14e9cfec08308ef178a2c76b8b5e1e98f3960438bf"}, + {file = "ijson-3.2.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:105c314fd624e81ed20f925271ec506523b8dd236589ab6c0208b8707d652a0e"}, + {file = "ijson-3.2.3-cp312-cp312-win32.whl", hash = "sha256:ac44781de5e901ce8339352bb5594fcb3b94ced315a34dbe840b4cff3450e23b"}, + {file = "ijson-3.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:0567e8c833825b119e74e10a7c29761dc65fcd155f5d4cb10f9d3b8916ef9912"}, + {file = "ijson-3.2.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:eeb286639649fb6bed37997a5e30eefcacddac79476d24128348ec890b2a0ccb"}, + {file = "ijson-3.2.3-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:396338a655fb9af4ac59dd09c189885b51fa0eefc84d35408662031023c110d1"}, + {file = "ijson-3.2.3-cp36-cp36m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0e0243d166d11a2a47c17c7e885debf3b19ed136be2af1f5d1c34212850236ac"}, + {file = "ijson-3.2.3-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85afdb3f3a5d0011584d4fa8e6dccc5936be51c27e84cd2882fe904ca3bd04c5"}, + {file = "ijson-3.2.3-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:4fc35d569eff3afa76bfecf533f818ecb9390105be257f3f83c03204661ace70"}, + {file = "ijson-3.2.3-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:455d7d3b7a6aacfb8ab1ebcaf697eedf5be66e044eac32508fccdc633d995f0e"}, + {file = "ijson-3.2.3-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:c63f3d57dbbac56cead05b12b81e8e1e259f14ce7f233a8cbe7fa0996733b628"}, + {file = "ijson-3.2.3-cp36-cp36m-win32.whl", hash = "sha256:a4d7fe3629de3ecb088bff6dfe25f77be3e8261ed53d5e244717e266f8544305"}, + {file = "ijson-3.2.3-cp36-cp36m-win_amd64.whl", hash = "sha256:96190d59f015b5a2af388a98446e411f58ecc6a93934e036daa75f75d02386a0"}, + {file = "ijson-3.2.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:35194e0b8a2bda12b4096e2e792efa5d4801a0abb950c48ade351d479cd22ba5"}, + {file = "ijson-3.2.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1053fb5f0b010ee76ca515e6af36b50d26c1728ad46be12f1f147a835341083"}, + {file = "ijson-3.2.3-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:211124cff9d9d139dd0dfced356f1472860352c055d2481459038b8205d7d742"}, + {file = "ijson-3.2.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:92dc4d48e9f6a271292d6079e9fcdce33c83d1acf11e6e12696fb05c5889fe74"}, + {file = "ijson-3.2.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3dcc33ee56f92a77f48776014ddb47af67c33dda361e84371153c4f1ed4434e1"}, + {file = "ijson-3.2.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:98c6799925a5d1988da4cd68879b8eeab52c6e029acc45e03abb7921a4715c4b"}, + {file = "ijson-3.2.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:4252e48c95cd8ceefc2caade310559ab61c37d82dfa045928ed05328eb5b5f65"}, + {file = "ijson-3.2.3-cp37-cp37m-win32.whl", hash = "sha256:644f4f03349ff2731fd515afd1c91b9e439e90c9f8c28292251834154edbffca"}, + {file = "ijson-3.2.3-cp37-cp37m-win_amd64.whl", hash = "sha256:ba33c764afa9ecef62801ba7ac0319268a7526f50f7601370d9f8f04e77fc02b"}, + {file = "ijson-3.2.3-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:4b2ec8c2a3f1742cbd5f36b65e192028e541b5fd8c7fd97c1fc0ca6c427c704a"}, + {file = "ijson-3.2.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7dc357da4b4ebd8903e77dbcc3ce0555ee29ebe0747c3c7f56adda423df8ec89"}, + {file = "ijson-3.2.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bcc51c84bb220ac330122468fe526a7777faa6464e3b04c15b476761beea424f"}, + {file = "ijson-3.2.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f8d54b624629f9903005c58d9321a036c72f5c212701bbb93d1a520ecd15e370"}, + {file = "ijson-3.2.3-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6ea7c7e3ec44742e867c72fd750c6a1e35b112f88a917615332c4476e718d40"}, + {file = "ijson-3.2.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:916acdc5e504f8b66c3e287ada5d4b39a3275fc1f2013c4b05d1ab9933671a6c"}, + {file = "ijson-3.2.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:81815b4184b85ce124bfc4c446d5f5e5e643fc119771c5916f035220ada29974"}, + {file = "ijson-3.2.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:b49fd5fe1cd9c1c8caf6c59f82b08117dd6bea2ec45b641594e25948f48f4169"}, + {file = "ijson-3.2.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:86b3c91fdcb8ffb30556c9669930f02b7642de58ca2987845b04f0d7fe46d9a8"}, + {file = "ijson-3.2.3-cp38-cp38-win32.whl", hash = "sha256:a729b0c8fb935481afe3cf7e0dadd0da3a69cc7f145dbab8502e2f1e01d85a7c"}, + {file = "ijson-3.2.3-cp38-cp38-win_amd64.whl", hash = "sha256:d34e049992d8a46922f96483e96b32ac4c9cffd01a5c33a928e70a283710cd58"}, + {file = "ijson-3.2.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:9c2a12dcdb6fa28f333bf10b3a0f80ec70bc45280d8435be7e19696fab2bc706"}, + {file = "ijson-3.2.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1844c5b57da21466f255a0aeddf89049e730d7f3dfc4d750f0e65c36e6a61a7c"}, + {file = "ijson-3.2.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2ec3e5ff2515f1c40ef6a94983158e172f004cd643b9e4b5302017139b6c96e4"}, + {file = "ijson-3.2.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:46bafb1b9959872a1f946f8dd9c6f1a30a970fc05b7bfae8579da3f1f988e598"}, + {file = "ijson-3.2.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ab4db9fee0138b60e31b3c02fff8a4c28d7b152040553b6a91b60354aebd4b02"}, + {file = "ijson-3.2.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f4bc87e69d1997c6a55fff5ee2af878720801ff6ab1fb3b7f94adda050651e37"}, + {file = "ijson-3.2.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:e9fd906f0c38e9f0bfd5365e1bed98d649f506721f76bb1a9baa5d7374f26f19"}, + {file = "ijson-3.2.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:e84d27d1acb60d9102728d06b9650e5b7e5cb0631bd6e3dfadba8fb6a80d6c2f"}, + {file = "ijson-3.2.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2cc04fc0a22bb945cd179f614845c8b5106c0b3939ee0d84ce67c7a61ac1a936"}, + {file = "ijson-3.2.3-cp39-cp39-win32.whl", hash = "sha256:e641814793a037175f7ec1b717ebb68f26d89d82cfd66f36e588f32d7e488d5f"}, + {file = "ijson-3.2.3-cp39-cp39-win_amd64.whl", hash = "sha256:6bd3e7e91d031f1e8cea7ce53f704ab74e61e505e8072467e092172422728b22"}, + {file = "ijson-3.2.3-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:06f9707da06a19b01013f8c65bf67db523662a9b4a4ff027e946e66c261f17f0"}, + {file = "ijson-3.2.3-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be8495f7c13fa1f622a2c6b64e79ac63965b89caf664cc4e701c335c652d15f2"}, + {file = "ijson-3.2.3-pp37-pypy37_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7596b42f38c3dcf9d434dddd50f46aeb28e96f891444c2b4b1266304a19a2c09"}, + {file = "ijson-3.2.3-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fbac4e9609a1086bbad075beb2ceec486a3b138604e12d2059a33ce2cba93051"}, + {file = "ijson-3.2.3-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:db2d6341f9cb538253e7fe23311d59252f124f47165221d3c06a7ed667ecd595"}, + {file = "ijson-3.2.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:fa8b98be298efbb2588f883f9953113d8a0023ab39abe77fe734b71b46b1220a"}, + {file = "ijson-3.2.3-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:674e585361c702fad050ab4c153fd168dc30f5980ef42b64400bc84d194e662d"}, + {file = "ijson-3.2.3-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fd12e42b9cb9c0166559a3ffa276b4f9fc9d5b4c304e5a13668642d34b48b634"}, + {file = "ijson-3.2.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d31e0d771d82def80cd4663a66de277c3b44ba82cd48f630526b52f74663c639"}, + {file = "ijson-3.2.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:7ce4c70c23521179d6da842bb9bc2e36bb9fad1e0187e35423ff0f282890c9ca"}, + {file = "ijson-3.2.3-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:39f551a6fbeed4433c85269c7c8778e2aaea2501d7ebcb65b38f556030642c17"}, + {file = "ijson-3.2.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3b14d322fec0de7af16f3ef920bf282f0dd747200b69e0b9628117f381b7775b"}, + {file = "ijson-3.2.3-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7851a341429b12d4527ca507097c959659baf5106c7074d15c17c387719ffbcd"}, + {file = "ijson-3.2.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db3bf1b42191b5cc9b6441552fdcb3b583594cb6b19e90d1578b7cbcf80d0fae"}, + {file = "ijson-3.2.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:6f662dc44362a53af3084d3765bb01cd7b4734d1f484a6095cad4cb0cbfe5374"}, + {file = "ijson-3.2.3.tar.gz", hash = "sha256:10294e9bf89cb713da05bc4790bdff616610432db561964827074898e174f917"}, +] + [[package]] name = "importlib-metadata" version = "7.0.1" @@ -5445,4 +5543,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.10,<4.0" -content-hash = "06f06416910d0f882c2e755ad286c7e6cea974948962e2afd34286d8aa1b5209" +content-hash = "f88434bbdcaecd0c87291ea47bb9d4e22c734687c0f76b287eb0500a8dfa5999" diff --git a/pyproject.toml b/pyproject.toml index 99f5266..209f7ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] # https://python-poetry.org/docs/pyproject/ name = "dspygen" -version = "2024.3.14.2" +version = "2024.3.17" description = "A Ruby on Rails style framework for the DSPy (Demonstrate, Search, Predict) project for Language Models like GPT, BERT, and LLama." authors = ["Sean Chatman "] readme = "README.md" @@ -43,6 +43,7 @@ paho-mqtt = "^2.0.0" psutil = "^5.9.8" st-pages = "^0.4.5" pykka = "^4.0.2" +ijson = "^3.2.3" [tool.poetry.group.test.dependencies] # https://python-poetry.org/docs/master/managing-dependencies/ coverage = { extras = ["toml"], version = ">=7.2.5" } diff --git a/src/dspygen/dsl/dsl_pipeline_executor.py b/src/dspygen/dsl/dsl_pipeline_executor.py index db15d2b..798ac14 100644 --- a/src/dspygen/dsl/dsl_pipeline_executor.py +++ b/src/dspygen/dsl/dsl_pipeline_executor.py @@ -10,6 +10,7 @@ from dspygen.dsl.utils.dsl_language_model_utils import _get_language_model_instance from dspygen.dsl.dsl_pydantic_models import PipelineDSLModel, LanguageModelConfig from dspygen.dsl.utils.dsl_module_utils import _get_module_instance +from dspygen.dsl.utils.dsl_retrieval_model_utils import _get_retrieval_model_instance from dspygen.dsl.utils.dsl_signature_utils import _create_signature_from_model from dspygen.typetemp.functional import render @@ -45,8 +46,8 @@ def _execute_step(pipeline, step): Execute a step in a pipeline. Creates the LM, renders the args using Jinja2, runs the module, and updates the context. """ - if not pipeline.models: - pipeline.models = [LanguageModelConfig(label="default", name="OpenAI", args={})] + if not pipeline.lm_models: + pipeline.lm_models = [LanguageModelConfig(label="default", name="OpenAI", args={})] rendered_args = {arg: render(str(value), **pipeline.context) for arg, value in step.args.items()} @@ -54,7 +55,9 @@ def _execute_step(pipeline, step): lm_inst = _get_language_model_instance(pipeline, step) - with dspy.context(lm=lm_inst): + rm_inst = _get_retrieval_model_instance(pipeline, step) + + with dspy.context(lm=lm_inst, rm=rm_inst): module_output = module_inst.forward(**rendered_args) pipeline.context[step.module] = module_output @@ -90,8 +93,8 @@ async def run_pipeline(request: PipelineRequest): def main(): - # context = execute_pipeline('examples/blog_pipeline.yaml') - context = execute_pipeline('/Users/candacechatman/dev/dspygen/pipeline.yaml', {"news": "$12,500 Retainer Contract"}) + context = execute_pipeline('/Users/candacechatman/dev/dspygen/src/dspygen/dsl/examples/example_pipeline.yaml') + # context = execute_pipeline('/Users/candacechatman/dev/dspygen/pipeline.yaml', {"news": "$12,500 Retainer Contract"}) # context = execute_pipeline('examples/example_pipeline.yaml') print(context) diff --git a/src/dspygen/dsl/dsl_pydantic_models.py b/src/dspygen/dsl/dsl_pydantic_models.py index 8c0a516..3c430b4 100644 --- a/src/dspygen/dsl/dsl_pydantic_models.py +++ b/src/dspygen/dsl/dsl_pydantic_models.py @@ -1,82 +1,110 @@ -from dspygen.utils.yaml_tools import YAMLMixin -from pydantic import BaseModel, Extra -from typing import List, Dict, Optional, Union +from typing import Optional -class InputFieldModel(BaseModel): - name: str - desc: Optional[str] = None +from pydantic import BaseModel, Field +from dspygen.utils.yaml_tools import YAMLMixin -class OutputFieldModel(BaseModel): - name: str - prefix: Optional[str] = "" - desc: Optional[str] = "" +class InputFieldModel(BaseModel): + """Defines an input field for a DSPy Signature.""" + name: str = Field( + ..., + description="The key used to access and pass the input within the Signature.", + ) + prefix: str = Field( + "", + description="Optional additional context or labeling for the input field.", + ) + desc: str = Field( + ..., + description="Description of the input field's purpose or the nature of content it should contain.", + ) +class OutputFieldModel(BaseModel): + """Defines an output field for a DSPy Signature.""" + + name: str = Field( + ..., + description="The key used to access and pass the input within the Signature.", + ) + prefix: str = Field( + "", + description="Optional additional context or labeling for the output field.", + ) + desc: str = Field( + ..., + description="Description of the output field's purpose or the nature of content it should contain.", + ) + + +# Define SignatureDSLModel for capturing signature details class SignatureDSLModel(BaseModel): - name: str - docstring: Optional[str] = None - inputs: List[InputFieldModel] = [] - outputs: List[OutputFieldModel] = [] + name: str = Field(..., description="The unique name identifying the Signature.") + docstring: str = Field(..., description="Documentation of the Signature's purpose.") + inputs: list[InputFieldModel] = Field(default=[], description="List of input fields required by the Signature.") + outputs: list[OutputFieldModel] = Field(default=[], description="List of output fields produced by the Signature.") + class ArgumentModel(BaseModel): name: str - value: Union[str, Dict[str, str]] = "" # Allow for both direct values and references + value: str | dict = "" # Allow for both direct values and references +# Define ModuleDSLModel for capturing module details class ModuleDSLModel(BaseModel): - name: str = "" - signature: str = "" - assertions: List[str] = [] - suggestions: List[str] = [] - predictor: Optional[str] = "Predict" - args: List[ArgumentModel] = [] + name: str = Field(..., description="Name of the module. Used for referencing within the pipeline.") + signature: str = Field(..., description="Name of the signature associated with this module.") + predictor: Optional[str] = Field("Predict", description="Type of predictor to be used with this module.") + args: list[ArgumentModel] = Field(default=[], + description="List of arguments to be passed to the module during its execution.") +# Define PipelineStepModel for pipeline steps class PipelineStepModel(BaseModel): - module: str - assertions: List[str] = [] - suggestions: List[str] = [] - signature: str = "" - model: str = "default" - args: dict = {} # Support nested argument structures + module: str = Field(..., description="Name of the module to be executed in this step of the pipeline.") + signature: str = Field(default="", + description="Signature associated with this step. Optional if defined within the module.") + lm_model: str = Field(default="default", description="Identifier of the language model to be used in this step.") + rm_model: str = Field(default="default", description="Identifier of the retrieval model to be used in this step.") + args: dict = Field(default={}, description="Arguments for the module in this step.") class LanguageModelConfig(BaseModel): - label: str - name: str + label: str = Field(default="default", description="Used for referencing in the Modules") + name: str = Field(default="OpenAI", description="The class name of the dspy language model to use") args: dict = {} -class AssertionModel(BaseModel): - label: str - logic: str - message: str - - -class SuggestionModel(BaseModel): - label: str - assertion: str - message: str +class RetrievalModelConfig(BaseModel): + label: str = Field(..., description="Used for referencing in the Modules") + name: str = Field(..., description="The class name of the dspy retrieval model to use") + args: dict = {} class PipelineConfigModel(BaseModel): - global_signatures: Dict[str, SignatureDSLModel] = [] - current_step: PipelineStepModel = None + global_signatures: dict = Field(default={}, + description="A dictionary of global signatures available in the pipeline.") + current_step: Optional[PipelineStepModel] = Field(None, + description="The current step being executed in the pipeline.") class Config: - extra = Extra.allow + extra = "allow" class PipelineDSLModel(BaseModel, YAMLMixin): - assertions: List[AssertionModel] = [] - suggestions: List[SuggestionModel] = [] - models: List[LanguageModelConfig] = [] - signatures: List[SignatureDSLModel] = [] - modules: List[ModuleDSLModel] = [] - steps: List[PipelineStepModel] = [] - context: dict = {} - config: PipelineConfigModel = PipelineConfigModel() - + lm_models: list[LanguageModelConfig] = Field(default=[], + description="list of language model configurations used in the pipeline.") + rm_models: list[RetrievalModelConfig] = Field(default=[], + description="list of retrieval model configurations used in the pipeline.") + signatures: list[SignatureDSLModel] = Field(default=[], + description="list of signatures defined for use in the pipeline.") + modules: list[ModuleDSLModel] = Field(default=[], + description="list of modules defined for execution in the pipeline.") + steps: list[PipelineStepModel] = Field(default=[], + description="Sequential steps to be executed in the pipeline.") + context: dict = Field(default={}, + description="A context dictionary for storing global values accessible across the pipeline.") + config: PipelineConfigModel = Field(default_factory=PipelineConfigModel, + description="Configuration settings for the pipeline execution.") diff --git a/src/dspygen/dsl/examples/blog_pipeline.yaml b/src/dspygen/dsl/examples/blog_pipeline.yaml index 1249cfb..ab81fd4 100644 --- a/src/dspygen/dsl/examples/blog_pipeline.yaml +++ b/src/dspygen/dsl/examples/blog_pipeline.yaml @@ -1,10 +1,4 @@ -assertions: - - label: "AssertLen" - logic: "len(result) == length" - message: "The length of the {{ result }} should be {{ length}} but was {{ result|length }}" - steps: - module: "dspygen.modules.blog_module.BlogModule" - args: - subject: "Buckminster Fuller's book Critical Path off the charts!" + subject: "{{ subject }}" diff --git a/src/dspygen/dsl/examples/example_pipeline.yaml b/src/dspygen/dsl/examples/example_pipeline.yaml index 30280c8..dadd1d1 100644 --- a/src/dspygen/dsl/examples/example_pipeline.yaml +++ b/src/dspygen/dsl/examples/example_pipeline.yaml @@ -1,4 +1,4 @@ -models: +lm_models: - label: "default" name: "OpenAI" args: @@ -44,7 +44,7 @@ modules: predictor: "Predict" args: - name: "raw_data" - value: "{{ user_input_data }}" + value: "{{ user_input }}" - name: "data_format" value: "JSON" @@ -57,13 +57,13 @@ modules: steps: - module: "DataProcessorModule" - model: "default" + lm_model: "default" args: raw_data: "id, name, age\n1, John, 25\n2, Jane, 30" data_format: "YAML" - module: "ReportGeneratorModule" - model: "fast" + lm_model: "fast" args: processed_data: "{{ processed_data }}" report_template: "templates/standard_report.html" diff --git a/src/dspygen/dsl/utils/dsl_language_model_utils.py b/src/dspygen/dsl/utils/dsl_language_model_utils.py index 701810c..069093f 100644 --- a/src/dspygen/dsl/utils/dsl_language_model_utils.py +++ b/src/dspygen/dsl/utils/dsl_language_model_utils.py @@ -1,13 +1,15 @@ import dspy +from dspygen.dsl.dsl_pydantic_models import PipelineDSLModel -def _get_language_model_instance(pipeline, step): + +def _get_language_model_instance(pipeline: PipelineDSLModel, step): """ Get the language model instance for a given step from the top level definition. """ - lm_label = step.model + lm_label = step.lm_model # Find the lm class within the dspy module. Need to import the class dynamically from the dspy module - lm_config = next((m for m in pipeline.models if m.label == lm_label), None) + lm_config = next((m for m in pipeline.lm_models if m.label == lm_label), None) lm_class = getattr(dspy, lm_config.name) lm_inst = lm_class(**lm_config.args) return lm_inst \ No newline at end of file diff --git a/src/dspygen/dsl/utils/dsl_retrieval_model_utils.py b/src/dspygen/dsl/utils/dsl_retrieval_model_utils.py new file mode 100644 index 0000000..b53717b --- /dev/null +++ b/src/dspygen/dsl/utils/dsl_retrieval_model_utils.py @@ -0,0 +1,17 @@ +import dspy + +from dspygen.dsl.dsl_pydantic_models import PipelineStepModel, PipelineDSLModel + + +def _get_retrieval_model_instance(pipeline: PipelineDSLModel, step: PipelineStepModel): + """ + Get the retrieval model instance for a given step from the top level definition. + """ + rm_label = step.rm_model + # Find the rm class within the dspy module. Need to import the class dynamically from the dspy module + rm_config = next((m for m in pipeline.rm_models if m.label == rm_label), None) + + if rm_config and hasattr(dspy, rm_config.name): + rm_class = getattr(dspy, rm_config.name) + rm_inst = rm_class(**rm_config.args) + return rm_inst diff --git a/src/dspygen/experiments/chatgpt_conversation_parser.py b/src/dspygen/experiments/chatgpt_conversation_parser.py new file mode 100644 index 0000000..0a3c2a1 --- /dev/null +++ b/src/dspygen/experiments/chatgpt_conversation_parser.py @@ -0,0 +1,89 @@ +import ijson + + +from typing import Optional + +from pydantic import BaseModel, ValidationError + + +class Author(BaseModel): + role: str + name: Optional[str] = None + metadata: dict + + +class ContentPart(BaseModel): + content_type: str + parts: list[str] | None + + +class Message(BaseModel): + id: str + author: Author + content: ContentPart + status: str + metadata: dict + + +class Data(BaseModel): + id: str + message: Message | None + parent: str | None + children: list[str] + + +class Conversation(BaseModel): + title: str + mapping: dict + + +# Function to process each conversation chunk +def process_conversations_chunk(chunk): + # Define Pydantic models as before + + # Process each conversation in the chunk + for chunked in chunk: + try: + conversation = Conversation(**chunked) + # Do whatever processing you need with the conversation + # print(f"Title: {conversation.title}") + for key in conversation.mapping: + data = Data(**conversation.mapping[key]) + if data.message and data.message.author.role == "assistant": + for part in data.message.content.parts: + if "ContractSign" in part: + print(part) + # encoding = tiktoken.encoding_for_model("text-embedding-ada-002") + # print(len(encoding.encode(part))) + # print(part) + except ValidationError: + # print(e) + pass + + +def main(): + # Define the path to your large JSON file + json_file_path = "/Users/candacechatman/dev/dspygen/data/conversations.json" + + # Open the JSON file for streaming + with open(json_file_path, "rb") as json_file: + conversations_generator = ijson.items( + json_file, "item" + ) # Assumes each conversation is a separate JSON object + + # Process the conversations in chunks (adjust the chunk size as needed) + chunk_size = 10 # Define your desired chunk size + chunk = [] + for conversation in conversations_generator: + chunk.append(conversation) + if len(chunk) >= chunk_size: + process_conversations_chunk(chunk) + chunk = [] + + # Process any remaining conversations in the last chunk + if chunk: + process_conversations_chunk(chunk) + + +if __name__ == "__main__": + main() diff --git a/src/dspygen/experiments/wizard.py b/src/dspygen/experiments/wizard.py new file mode 100644 index 0000000..8136457 --- /dev/null +++ b/src/dspygen/experiments/wizard.py @@ -0,0 +1,69 @@ +import typer +from typing import Optional +from pydantic import BaseModel, ValidationError +import yaml +import os +import json + +from dspygen.dsl.dsl_pydantic_models import PipelineConfigModel + +app = typer.Typer() + + +def load_user_history() -> dict: + """ + Simulate loading user's history to predict configurations. + """ + # Placeholder for loading and analyzing user's history + return {} + + +def generate_optimized_configuration(user_history: dict) -> PipelineConfigModel: + """ + Generate an optimized pipeline configuration based on the user's history. + """ + # Placeholder for AI logic to generate optimized configs + return PipelineConfigModel() + + +def save_configuration(config: PipelineConfigModel, filename: str): + """ + Save the generated pipeline configuration to a YAML file. + """ + with open(filename, 'w') as file: + yaml.dump(json.loads(config.json()), file) + + +@app.command() +def create_pipeline(filename: Optional[str] = typer.Option(None, "--filename", "-f", + help="Filename to save the pipeline configuration.")): + """ + Main command to initiate the pipeline configuration generation process. + """ + user_history = load_user_history() + optimized_config = generate_optimized_configuration(user_history) + + # Proactively offer the optimized configuration with an option for details or modifications + typer.echo("Based on your history, an optimized pipeline configuration has been generated.") + action = typer.prompt("Do you want to proceed with the optimized configuration? (Y/n/details)", default="Y") + + if action.lower() == 'n': + typer.echo("Exiting... You can start over and customize your configuration.") + raise typer.Exit() + elif action.lower() == 'details': + typer.echo(optimized_config.json(indent=2)) + modify = typer.prompt("Do you want to modify this configuration? (Y/n)", default="n") + if modify.lower() == 'y': + # Placeholder for modification logic + typer.echo("Modifying configurations... (This part is yet to be implemented)") + + # Confirm and save the configuration + if not filename: + filename = typer.prompt("Enter a filename to save your configuration (default: optimized_pipeline.yaml): ", + default="optimized_pipeline.yaml") + save_configuration(optimized_config, filename) + typer.echo(f"Configuration saved to {filename}. Your optimized pipeline is ready!") + + +if __name__ == "__main__": + app() diff --git a/test_pipeline.http b/test_pipeline.http index 57e7e83..7dbc86b 100644 --- a/test_pipeline.http +++ b/test_pipeline.http @@ -1,4 +1,4 @@ -POST http://127.0.0.1:8888/execute_pipeline/ +POST http://127.0.0.1:8000/execute_pipeline/ Content-Type: application/json {