From 8f39f87553ccebdc770f97e031893294576a0449 Mon Sep 17 00:00:00 2001 From: Matt Stark Date: Thu, 14 Nov 2024 12:22:19 +1100 Subject: [PATCH] Minimize the set of tokio features that need to be enabled. --- watchman/cli/Cargo.lock | 1081 +++++++++++++++ .../pywatchman/CMakeLists.txt | 15 + .../pywatchman/__init__.py | 1180 +++++++++++++++++ .../pywatchman/bser.c | 633 +++++++++ .../pywatchman/bser.h | 67 + .../pywatchman/bsermodule.c | 649 +++++++++ .../pywatchman/capabilities.py | 52 + .../pywatchman/encoding.py | 31 + .../pywatchman/load.py | 82 ++ .../pywatchman/pybser.py | 513 +++++++ .../pywatchman/windows.py | 305 +++++ .../python/build/scripts-3.11/watchman-make | 307 +++++ .../watchman-replicate-subscription | 397 ++++++ .../python/build/scripts-3.11/watchman-wait | 272 ++++ watchman/rust/watchman_client/Cargo.toml | 6 +- 15 files changed, 5588 insertions(+), 2 deletions(-) create mode 100644 watchman/cli/Cargo.lock create mode 100644 watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/CMakeLists.txt create mode 100644 watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/__init__.py create mode 100644 watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/bser.c create mode 100644 watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/bser.h create mode 100644 watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/bsermodule.c create mode 100644 watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/capabilities.py create mode 100644 watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/encoding.py create mode 100644 watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/load.py create mode 100644 watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/pybser.py create mode 100644 watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/windows.py create mode 100755 watchman/python/build/scripts-3.11/watchman-make create mode 100755 watchman/python/build/scripts-3.11/watchman-replicate-subscription create mode 100755 watchman/python/build/scripts-3.11/watchman-wait diff --git a/watchman/cli/Cargo.lock b/watchman/cli/Cargo.lock new file mode 100644 index 000000000000..dfc850387965 --- /dev/null +++ b/watchman/cli/Cargo.lock @@ -0,0 +1,1081 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "ansi_term" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" +dependencies = [ + "winapi", +] + +[[package]] +name = "anyhow" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775" + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi 0.1.19", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "backtrace" +version = "0.3.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d82cb332cdfaed17ae235a638438ac4d4839913cc2af585c3c6746e8f8bee1a" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ac0150caa2ae65ca5bd83f25c7de183dea78d4d366469f148435e2acfbad0da" +dependencies = [ + "serde", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "ansi_term", + "atty", + "bitflags 1.3.2", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "crossbeam" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-queue" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + +[[package]] +name = "duct" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ab5718d1224b63252cd0c6f74f6480f9ffeb117438a2e0f5cf6d9a4798929c" +dependencies = [ + "libc", + "once_cell", + "os_pipe", + "shared_child", +] + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + +[[package]] +name = "futures" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures 0.1.31", + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "jwalk" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dbcda57db8b6dc067e589628b7348639014e793d9e8137d8cf215e8b133a0bd" +dependencies = [ + "crossbeam", + "rayon", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.162" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18d287de67fe55fd7e1581fe933d965a5a9477b38e949cfa9f8574ef01506398" + +[[package]] +name = "lock_api" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80e04d1dcff3aae0704555fe5fee3bcfaf3d1fdf8a7e521d5b9d2b42acb52cec" +dependencies = [ + "hermit-abi 0.3.9", + "libc", + "wasi", + "windows-sys 0.52.0", +] + +[[package]] +name = "nix" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f346ff70e7dbfd675fe90590b92d59ef2de15a8779ae305ebcbfd3f0caf59be4" +dependencies = [ + "autocfg", + "bitflags 1.3.2", + "cfg-if", + "libc", + "memoffset", + "pin-utils", +] + +[[package]] +name = "ntapi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4" +dependencies = [ + "winapi", +] + +[[package]] +name = "object" +version = "0.36.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "os_pipe" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ffd2b0a5634335b135d5728d84c5e0fd726954b87111f7506a61c502280d982" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915a1e146535de9163f3987b8944ed8cf49a18bb0056bcebcdcece385cece4ff" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn 1.0.109", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f139b0662de085916d1fb67d2b4169d1addddda1919e696f3252b740b629986e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "redox_syscall" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +dependencies = [ + "bitflags 2.6.0", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.215" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_bser" +version = "0.4.0" +dependencies = [ + "anyhow", + "byteorder", + "bytes", + "serde", + "serde_bytes", + "thiserror", +] + +[[package]] +name = "serde_bytes" +version = "0.11.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "387cc504cb06bb40a96c8e04e951fe01854cf6bc921053c954e4a606d9675c6a" +dependencies = [ + "serde", +] + +[[package]] +name = "serde_derive" +version = "1.0.215" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "serde_json" +version = "1.0.132" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "shared_child" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09fa9338aed9a1df411814a5b2252f7cd206c55ae9bf2fa763f8de84603aa60c" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "socket2" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "structopt" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10" +dependencies = [ + "clap", + "lazy_static", + "structopt-derive", +] + +[[package]] +name = "structopt-derive" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sysinfo" +version = "0.30.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a5b4ddaee55fb2bea2bf0e5000747e5f5c0de765e5a5ff87f4cd106439f4bb3" +dependencies = [ + "cfg-if", + "core-foundation-sys", + "libc", + "ntapi", + "once_cell", + "rayon", + "windows", +] + +[[package]] +name = "tabular" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9a2882c514780a1973df90de9d68adcd8871bacc9a6331c3f28e6d2ff91a3d1" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "tokio" +version = "1.41.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cfb5bee7a6a52939ca9224d6ac897bb669134078daa8735560897f69de4d33" +dependencies = [ + "backtrace", + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "tracing", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-macros" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] + +[[package]] +name = "tokio-util" +version = "0.6.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507" +dependencies = [ + "bytes", + "futures-core", + "futures-io", + "futures-sink", + "log", + "pin-project-lite", + "slab", + "tokio", +] + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +dependencies = [ + "once_cell", +] + +[[package]] +name = "unicode-ident" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" + +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "watchman_client" +version = "0.9.0" +dependencies = [ + "anyhow", + "bytes", + "futures 0.3.31", + "maplit", + "serde", + "serde_bser", + "thiserror", + "tokio", + "tokio-util", + "winapi", +] + +[[package]] +name = "watchmanctl" +version = "0.1.0" +dependencies = [ + "ahash", + "anyhow", + "duct", + "jwalk", + "nix", + "serde", + "serde_json", + "structopt", + "sysinfo", + "tabular", + "tokio", + "watchman_client", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" +dependencies = [ + "windows-core", + "windows-targets", +] + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.87", +] diff --git a/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/CMakeLists.txt b/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/CMakeLists.txt new file mode 100644 index 000000000000..4ff79c6436bd --- /dev/null +++ b/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/CMakeLists.txt @@ -0,0 +1,15 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +add_fb_python_library(pywatchman + SOURCES + __init__.py + capabilities.py + encoding.py + load.py + pybser.py + windows.py + NAMESPACE pywatchman +) diff --git a/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/__init__.py b/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/__init__.py new file mode 100644 index 000000000000..6c57a2be0e11 --- /dev/null +++ b/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/__init__.py @@ -0,0 +1,1180 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + + +import inspect +import math +import os +import socket +import subprocess +import sys +import time +import typing + +from . import capabilities, encoding + + +# Sometimes it's really hard to get Python extensions to compile, +# so fall back to a pure Python implementation. +try: + from . import bser + + # Demandimport causes modules to be loaded lazily. Force the load now + # so that we can fall back on pybser if bser doesn't exist + bser.pdu_info +except ImportError: + from . import pybser as bser + + +bser: typing.Any + + +if os.name == "nt": + import ctypes + from ctypes import wintypes + + from .windows import ( + CancelIoEx, + CloseHandle, + CreateEvent, + CreateFile, + ERROR_IO_PENDING, + FILE_FLAG_OVERLAPPED, + FORMAT_MESSAGE_ALLOCATE_BUFFER, + FORMAT_MESSAGE_FROM_SYSTEM, + FORMAT_MESSAGE_IGNORE_INSERTS, + FormatMessage, + GENERIC_READ, + GENERIC_WRITE, + GetLastError, + GetOverlappedResult, + GetOverlappedResultEx, + INVALID_HANDLE_VALUE, + LocalFree, + OPEN_EXISTING, + OVERLAPPED, + ReadFile, + SetLastError, + WAIT_FAILED, + WAIT_IO_COMPLETION, + WAIT_OBJECT_0, + WAIT_TIMEOUT, + WaitForSingleObjectEx, + WindowsSocketException, + WindowsSocketHandle, + WriteFile, + ) + +# 2 bytes marker, 1 byte int size, 8 bytes int64 value +sniff_len = 13 + +# This is a helper for debugging the client. +_debugging = False +if _debugging: + + def log(fmt, *args): + print( + "[%s] %s" + % (time.strftime("%a, %d %b %Y %H:%M:%S", time.gmtime()), fmt % args[:]), + file=sys.stderr, + ) + +else: + + def log(fmt, *args): + pass + + +def _win32_strerror(err): + """expand a win32 error code into a human readable message""" + + # FormatMessage will allocate memory and assign it here + buf = ctypes.c_char_p() + FormatMessage( + FORMAT_MESSAGE_FROM_SYSTEM + | FORMAT_MESSAGE_ALLOCATE_BUFFER + | FORMAT_MESSAGE_IGNORE_INSERTS, + None, + err, + 0, + buf, + 0, + None, + ) + try: + return buf.value + finally: + LocalFree(buf) + + +class WatchmanError(Exception): + def __init__(self, msg=None, cmd=None): + self.msg = msg + self.cmd = cmd + + def setCommand(self, cmd): + self.cmd = cmd + + def __str__(self): + if self.cmd: + return "%s, while executing %s" % (self.msg, self.cmd) + return self.msg + + +class BSERv1Unsupported(WatchmanError): + pass + + +class UseAfterFork(WatchmanError): + pass + + +class WatchmanEnvironmentError(WatchmanError): + def __init__(self, msg, errno, errmsg, cmd=None): + super(WatchmanEnvironmentError, self).__init__( + "{0}: errno={1} errmsg={2}".format(msg, errno, errmsg), cmd + ) + + +class SocketConnectError(WatchmanError): + def __init__(self, sockpath, exc): + super(SocketConnectError, self).__init__( + "unable to connect to %s: %s" % (sockpath, exc) + ) + self.sockpath = sockpath + self.exc = exc + + +class SocketTimeout(WatchmanError): + """A specialized exception raised for socket timeouts during communication to/from watchman. + This makes it easier to implement non-blocking loops as callers can easily distinguish + between a routine timeout and an actual error condition. + + Note that catching WatchmanError will also catch this as it is a super-class, so backwards + compatibility in exception handling is preserved. + """ + + +class CommandError(WatchmanError): + """error returned by watchman + + self.msg is the message returned by watchman. + """ + + def __init__(self, msg, cmd=None): + super(CommandError, self).__init__("watchman command error: %s" % (msg,), cmd) + + +def is_named_pipe_path(path: str) -> bool: + """Returns True if path is a watchman named pipe path""" + return path.startswith("\\\\.\\pipe\\watchman") + + +class SockPath: + """Describes how to connect to watchman""" + + unix_domain = None + named_pipe = None + tcp_address = None + + def __init__( + self, unix_domain=None, named_pipe=None, sockpath=None, tcp_address=None + ): + if named_pipe is None and sockpath is not None and is_named_pipe_path(sockpath): + named_pipe = sockpath + + if ( + unix_domain is None + and sockpath is not None + and not is_named_pipe_path(sockpath) + ): + unix_domain = sockpath + + self.unix_domain = unix_domain + self.named_pipe = named_pipe + self.tcp_address = tcp_address + + def legacy_sockpath(self): + """Returns a sockpath suitable for passing to the watchman + CLI --sockname parameter""" + log("legacy_sockpath called: %r", self) + if os.name == "nt": + return self.named_pipe + return self.unix_domain + + +class Transport: + """communication transport to the watchman server""" + + buf = None + + def close(self): + """tear it down""" + raise NotImplementedError() + + def readBytes(self, size): + """read size bytes""" + raise NotImplementedError() + + def write(self, buf): + """write some data""" + raise NotImplementedError() + + def setTimeout(self, value): + pass + + def readLine(self): + """read a line + Maintains its own buffer, callers of the transport should not mix + calls to readBytes and readLine. + """ + if self.buf is None: + self.buf = [] + + # Buffer may already have a line if we've received unilateral + # response(s) from the server + if len(self.buf) == 1 and b"\n" in self.buf[0]: + (line, b) = self.buf[0].split(b"\n", 1) + self.buf = [b] + return line + + while True: + b = self.readBytes(4096) + if b"\n" in b: + result = b"".join(self.buf) + (line, b) = b.split(b"\n", 1) + self.buf = [b] + return result + line + self.buf.append(b) + + +class Codec: + """communication encoding for the watchman server""" + + transport = None + + def __init__(self, transport): + self.transport = transport + + def receive(self): + raise NotImplementedError() + + def send(self, *args): + raise NotImplementedError() + + def setTimeout(self, value): + self.transport.setTimeout(value) + + +class SocketTransport(Transport): + """abstract socket transport""" + + sock = None + timeout = None + + def __init__(self): + pass + + def close(self): + if self.sock: + self.sock.close() + self.sock = None + + def setTimeout(self, value): + self.timeout = value + self.sock.settimeout(self.timeout) + + def readBytes(self, size): + try: + buf = [self.sock.recv(size)] + if not buf[0]: + raise WatchmanError("empty watchman response") + return buf[0] + except socket.timeout: + raise SocketTimeout("timed out waiting for response") + + def write(self, data): + try: + log("write %r", data) + self.sock.sendall(data) + except socket.timeout: + raise SocketTimeout("timed out sending query command") + + +class UnixSocketTransport(SocketTransport): + """local unix domain socket transport""" + + def __init__(self, sockpath, timeout): + super(UnixSocketTransport, self).__init__() + self.sockpath = sockpath + self.timeout = timeout + + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + sock.settimeout(self.timeout) + sock.connect(self.sockpath.unix_domain) + self.sock = sock + except socket.error as e: + sock.close() + raise SocketConnectError(self.sockpath.unix_domain, e) + + +class WindowsUnixSocketTransport(SocketTransport): + """local unix domain socket transport on Windows""" + + sock = None + timeout = None + + def __init__(self, sockpath, timeout): + super(WindowsUnixSocketTransport, self).__init__() + self.sockpath = sockpath + self.timeout = timeout + + sock = None + try: + sock = WindowsSocketHandle() + sock.settimeout(self.timeout) + sock.connect(self.sockpath.unix_domain) + self.sock = sock + except WindowsSocketException as e: + if sock is not None: + sock.close() + raise SocketConnectError(self.sockpath.unix_domain, e) + + +class TcpSocketTransport(SocketTransport): + """TCP socket transport""" + + def __init__(self, sockpath, timeout): + super(TcpSocketTransport, self).__init__() + self.sockpath = sockpath + self.timeout = timeout + + try: + # Note that address resolution does not respect 'timeout' + # which applies only to socket traffic. + results = socket.getaddrinfo( + self.sockpath.tcp_address[0], + self.sockpath.tcp_address[1], + 0, + socket.IPPROTO_TCP, + ) + (family, type, proto, canonname, sockaddr) = results[0] + except Exception as e: + raise SocketConnectError( + "Error resolving address: %r" % (self.sockpath.tcp_address,), e + ) + + sock = socket.socket(family, socket.SOCK_STREAM) + try: + sock.settimeout(self.timeout) + sock.connect(sockaddr) + self.sock = sock + except socket.error as e: + sock.close() + raise SocketConnectError(str(self.sockpath.tcp_address), e) + + +def _get_overlapped_result_ex_impl(pipe, olap, nbytes, millis, alertable): + """Windows 7 and earlier does not support GetOverlappedResultEx. The + alternative is to use GetOverlappedResult and wait for read or write + operation to complete. This is done be using CreateEvent and + WaitForSingleObjectEx. CreateEvent, WaitForSingleObjectEx + and GetOverlappedResult are all part of Windows API since WindowsXP. + This is the exact same implementation that can be found in the watchman + source code (see get_overlapped_result_ex_impl in stream_win.c). This + way, maintenance should be simplified. + """ + log("Preparing to wait for maximum %dms", millis) + if millis != 0: + waitReturnCode = WaitForSingleObjectEx(olap.hEvent, millis, alertable) + if waitReturnCode == WAIT_OBJECT_0: + # Event is signaled, overlapped IO operation result should be available. + pass + elif waitReturnCode == WAIT_IO_COMPLETION: + # WaitForSingleObjectEx returnes because the system added an I/O completion + # routine or an asynchronous procedure call (APC) to the thread queue. + SetLastError(WAIT_IO_COMPLETION) + pass + elif waitReturnCode == WAIT_TIMEOUT: + # We reached the maximum allowed wait time, the IO operation failed + # to complete in timely fashion. + SetLastError(WAIT_TIMEOUT) + return False + elif waitReturnCode == WAIT_FAILED: + # something went wrong calling WaitForSingleObjectEx + err = GetLastError() + log("WaitForSingleObjectEx failed: %s", _win32_strerror(err)) + return False + else: + # unexpected situation deserving investigation. + err = GetLastError() + log("Unexpected error: %s", _win32_strerror(err)) + return False + + return GetOverlappedResult(pipe, olap, nbytes, False) + + +class WindowsNamedPipeTransport(Transport): + """connect to a named pipe""" + + def __init__(self, sockpath, timeout): + self.sockpath = sockpath + self.timeout = int(math.ceil(timeout * 1000)) + self._iobuf = None + + path = os.fsencode(self.sockpath.named_pipe) + + log("CreateFile %r", path) + + self.pipe = CreateFile( + path, + GENERIC_READ | GENERIC_WRITE, + 0, + None, + OPEN_EXISTING, + FILE_FLAG_OVERLAPPED, + None, + ) + + err = GetLastError() + if self.pipe == INVALID_HANDLE_VALUE or self.pipe == 0: + self.pipe = None + raise SocketConnectError( + self.sockpath.named_pipe, self._make_win_err("", err) + ) + + # event for the overlapped I/O operations + self._waitable = CreateEvent(None, True, False, None) + err = GetLastError() + if self._waitable is None: + self._raise_win_err("CreateEvent failed", err) + + self._get_overlapped_result_ex = GetOverlappedResultEx + if ( + os.getenv("WATCHMAN_WIN7_COMPAT") == "1" + or self._get_overlapped_result_ex is None + ): + self._get_overlapped_result_ex = _get_overlapped_result_ex_impl + + def _raise_win_err(self, msg, err): + raise self._make_win_err(msg, err) + + def _make_win_err(self, msg, err): + return IOError("%s win32 error code: %d %s" % (msg, err, _win32_strerror(err))) + + def close(self): + if self.pipe: + log("Closing pipe") + CloseHandle(self.pipe) + self.pipe = None + + if self._waitable is not None: + # We release the handle for the event + CloseHandle(self._waitable) + self._waitable = None + + def setTimeout(self, value): + # convert to milliseconds + self.timeout = int(value * 1000) + + def readBytes(self, size): + """A read can block for an unbounded amount of time, even if the + kernel reports that the pipe handle is signalled, so we need to + always perform our reads asynchronously + """ + + # try to satisfy the read from any buffered data + if self._iobuf: + if size >= len(self._iobuf): + res = self._iobuf + self.buf = None + return res + res = self._iobuf[:size] + self._iobuf = self._iobuf[size:] + return res + + # We need to initiate a read + buf = ctypes.create_string_buffer(size) + olap = OVERLAPPED() + olap.hEvent = self._waitable + + log("made read buff of size %d", size) + + # ReadFile docs warn against sending in the nread parameter for async + # operations, so we always collect it via GetOverlappedResultEx + immediate = ReadFile(self.pipe, buf, size, None, olap) + + if not immediate: + err = GetLastError() + if err != ERROR_IO_PENDING: + self._raise_win_err("failed to read %d bytes" % size, err) + + nread = wintypes.DWORD() + if not self._get_overlapped_result_ex( + self.pipe, olap, nread, 0 if immediate else self.timeout, True + ): + err = GetLastError() + CancelIoEx(self.pipe, olap) + + if err == WAIT_TIMEOUT: + log("GetOverlappedResultEx timedout") + raise SocketTimeout( + "timed out after waiting %dms for read" % self.timeout + ) + + log("GetOverlappedResultEx reports error %d", err) + self._raise_win_err("error while waiting for read", err) + + nread = nread.value + if nread == 0: + # Docs say that named pipes return 0 byte when the other end did + # a zero byte write. Since we don't ever do that, the only + # other way this shows up is if the client has gotten in a weird + # state, so let's bail out + CancelIoEx(self.pipe, olap) + raise IOError("Async read yielded 0 bytes; unpossible!") + + # Holds precisely the bytes that we read from the prior request + buf = buf[:nread] + + returned_size = min(nread, size) + if returned_size == nread: + return buf + + # keep any left-overs around for a later read to consume + self._iobuf = buf[returned_size:] + return buf[:returned_size] + + def write(self, data): + olap = OVERLAPPED() + olap.hEvent = self._waitable + + immediate = WriteFile(self.pipe, ctypes.c_char_p(data), len(data), None, olap) + + if not immediate: + err = GetLastError() + if err != ERROR_IO_PENDING: + self._raise_win_err( + "failed to write %d bytes to handle %r" % (len(data), self.pipe), + err, + ) + + # Obtain results, waiting if needed + nwrote = wintypes.DWORD() + if self._get_overlapped_result_ex( + self.pipe, olap, nwrote, 0 if immediate else self.timeout, True + ): + log("made write of %d bytes", nwrote.value) + return nwrote.value + + err = GetLastError() + + # It's potentially unsafe to allow the write to continue after + # we unwind, so let's make a best effort to avoid that happening + CancelIoEx(self.pipe, olap) + + if err == WAIT_TIMEOUT: + raise SocketTimeout("timed out after waiting %dms for write" % self.timeout) + self._raise_win_err( + "error while waiting for write of %d bytes" % len(data), err + ) + + +def _default_binpath(binpath=None) -> str: + if binpath: + return binpath + # The test harness sets WATCHMAN_BINARY to the binary under test, + # so we use that by default, otherwise, allow resolving watchman + # from the users PATH. + return os.environ.get("WATCHMAN_BINARY", "watchman") + + +class CLIProcessTransport(Transport): + """open a pipe to the cli to talk to the service + This intended to be used only in the test harness! + + The CLI is an oddball because we only support JSON input + and cannot send multiple commands through the same instance, + so we spawn a new process for each command. + + We disable server spawning for this implementation, again, because + it is intended to be used only in our test harness. You really + should not need to use the CLI transport for anything real. + + While the CLI can output in BSER, our Transport interface doesn't + support telling this instance that it should do so. That effectively + limits this implementation to JSON input and output only at this time. + + It is the responsibility of the caller to set the send and + receive codecs appropriately. + """ + + proc = None + closed = True + + def __init__(self, sockpath, timeout, binpath=None): + self.sockpath = sockpath + self.timeout = timeout + self.binpath = _default_binpath(binpath) + + def close(self): + if self.proc: + if self.proc.pid is not None: + self.proc.kill() + self.proc.stdin.close() + self.proc.stdout.close() + self.proc.wait() + self.proc = None + + def _connect(self): + if self.proc: + return self.proc + args = [ + self.binpath, + "--unix-listener-path={0}".format(self.sockpath.unix_domain), + "--named-pipe-path={0}".format(self.sockpath.named_pipe), + "--logfile=/BOGUS", + "--statefile=/BOGUS", + "--no-spawn", + "--no-local", + "--no-pretty", + "-j", + ] + log("starting with %r", args) + self.proc = subprocess.Popen( + args, stdin=subprocess.PIPE, stdout=subprocess.PIPE + ) + return self.proc + + def readBytes(self, size): + self._connect() + res = self.proc.stdout.read(size) + log("CLI read %r", repr(res)) + if not res: + raise WatchmanError("EOF on CLI process transport") + return res + + def write(self, data): + if self.closed: + self.close() + self.closed = False + self._connect() + log("CLI write %r", data) + res = self.proc.stdin.write(data) + self.proc.stdin.close() + self.closed = True + return res + + +class BserCodec(Codec): + """use the BSER encoding. This is the default, preferred codec""" + + def __init__(self, transport, value_encoding, value_errors): + super(BserCodec, self).__init__(transport) + self._value_encoding = value_encoding + self._value_errors = value_errors + + def _loads(self, response): + return bser.loads( + response, + value_encoding=self._value_encoding, + value_errors=self._value_errors, + ) + + def receive(self): + buf = [self.transport.readBytes(sniff_len)] + if not buf[0]: + raise WatchmanError("empty watchman response") + + _1, _2, elen = bser.pdu_info(buf[0]) + + rlen = len(buf[0]) + while elen > rlen: + buf.append(self.transport.readBytes(elen - rlen)) + rlen += len(buf[-1]) + + response = b"".join(buf) + try: + res = self._loads(response) + return res + except ValueError as e: + raise WatchmanError("watchman response decode error: %s" % e) + + def send(self, *args): + cmd = bser.dumps(*args) # Defaults to BSER v1 + self.transport.write(cmd) + + +class ImmutableBserCodec(BserCodec): + """use the BSER encoding, decoding values using the newer + immutable object support""" + + def _loads(self, response): + return bser.loads( + response, + False, + value_encoding=self._value_encoding, + value_errors=self._value_errors, + ) + + +class Bser2WithFallbackCodec(BserCodec): + """use BSER v2 encoding""" + + def __init__(self, transport, value_encoding, value_errors): + super(Bser2WithFallbackCodec, self).__init__( + transport, value_encoding, value_errors + ) + bserv2_key = "required" + + self.send(["version", {bserv2_key: ["bser-v2"]}]) + + capabilities = self.receive() + + if "error" in capabilities: + raise BSERv1Unsupported( + "The watchman server version does not support Python 3. Please " + "upgrade your watchman server." + ) + + if capabilities["capabilities"]["bser-v2"]: + self.bser_version = 2 + self.bser_capabilities = 0 + else: + self.bser_version = 1 + self.bser_capabilities = 0 + + def receive(self): + buf = [self.transport.readBytes(sniff_len)] + if not buf[0]: + raise WatchmanError("empty watchman response") + + recv_bser_version, recv_bser_capabilities, elen = bser.pdu_info(buf[0]) + + if hasattr(self, "bser_version"): + # Readjust BSER version and capabilities if necessary + self.bser_version = max(self.bser_version, recv_bser_version) + self.capabilities = self.bser_capabilities & recv_bser_capabilities + + rlen = len(buf[0]) + while elen > rlen: + buf.append(self.transport.readBytes(elen - rlen)) + rlen += len(buf[-1]) + + response = b"".join(buf) + try: + res = self._loads(response) + return res + except ValueError as e: + raise WatchmanError("watchman response decode error: %s" % e) + + def send(self, *args): + if hasattr(self, "bser_version"): + cmd = bser.dumps( + *args, version=self.bser_version, capabilities=self.bser_capabilities + ) + else: + cmd = bser.dumps(*args) + self.transport.write(cmd) + + +class ImmutableBser2Codec(Bser2WithFallbackCodec, ImmutableBserCodec): + """use the BSER encoding, decoding values using the newer + immutable object support""" + + pass + + +class JsonCodec(Codec): + """Use json codec. This is here primarily for testing purposes""" + + json = None + + def __init__(self, transport): + super(JsonCodec, self).__init__(transport) + # optional dep on json, only if JsonCodec is used + import json + + self.json = json + + def receive(self): + line = self.transport.readLine() + try: + # In Python 3, json.loads is a transformation from Unicode string to + # objects possibly containing Unicode strings. We typically expect + # the JSON blob to be ASCII-only with non-ASCII characters escaped, + # but it's possible we might get non-ASCII bytes that are valid + # UTF-8. + line = line.decode("utf-8") + return self.json.loads(line) + except Exception as e: + print(e, line) + raise + + def send(self, *args): + cmd = self.json.dumps(*args) + # In Python 3, json.dumps is a transformation from objects possibly + # containing Unicode strings to Unicode string. Even with (the default) + # ensure_ascii=True, dumps returns a Unicode string. + cmd = cmd.encode("ascii") + self.transport.write(cmd + b"\n") + + +class client: + """Handles the communication with the watchman service""" + + sockpath = None + transport = None + sendCodec = None + recvCodec = None + sendConn = None + recvConn = None + subs = {} # Keyed by subscription name + sub_by_root = {} # Keyed by root, then by subscription name + logs = [] # When log level is raised + unilateral = ["log", "subscription"] + tport = None + useImmutableBser = None + pid = None + + def __init__( + self, + sockpath=None, + tcpAddress=None, + timeout=1.0, + transport=None, + sendEncoding=None, + recvEncoding=None, + useImmutableBser=False, + # use False for these two because None has a special + # meaning + valueEncoding=False, + valueErrors=False, + binpath=None, + ): + if sockpath is not None and not isinstance(sockpath, SockPath): + sockpath = SockPath(sockpath=sockpath, tcp_address=tcpAddress) + self.sockpath = sockpath + self.timeout = timeout + self.useImmutableBser = useImmutableBser + self.binpath = _default_binpath(binpath) + + if inspect.isclass(transport) and issubclass(transport, Transport): + self.transport = transport + else: + log( + "figure out transport. param=%r, env=%r", + transport, + os.getenv("WATCHMAN_TRANSPORT"), + ) + transport = transport or os.getenv("WATCHMAN_TRANSPORT") or "local" + if self.transport == "tcp" and tcpAddress is None: + raise WatchmanError( + "Constructor requires argument tcpAddress when 'tcp' " + "transport protocol is used" + ) + if (transport == "namedpipe") or (transport == "local" and os.name == "nt"): + self.transport = WindowsNamedPipeTransport + elif transport == "unix" and os.name == "nt": + self.transport = WindowsUnixSocketTransport + elif transport == "local" or transport == "unix": + self.transport = UnixSocketTransport + elif transport == "cli": + self.transport = CLIProcessTransport + if sendEncoding is None: + sendEncoding = "json" + if recvEncoding is None: + recvEncoding = sendEncoding + elif transport == "tcp": + self.transport = TcpSocketTransport + else: + raise WatchmanError("invalid transport %s" % transport) + + sendEncoding = str(sendEncoding or os.getenv("WATCHMAN_ENCODING") or "bser") + recvEncoding = str(recvEncoding or os.getenv("WATCHMAN_ENCODING") or "bser") + + self.recvCodec = self._parseEncoding(recvEncoding) + self.sendCodec = self._parseEncoding(sendEncoding) + + # We want to act like the native OS methods as much as possible. This + # means returning bytestrings on Python 2 by default and Unicode + # strings on Python 3. However we take an optional argument that lets + # users override this. + if valueEncoding is False: + self.valueEncoding = encoding.get_local_encoding() + self.valueErrors = encoding.default_local_errors + else: + self.valueEncoding = valueEncoding + if valueErrors is False: + self.valueErrors = encoding.default_local_errors + else: + self.valueErrors = valueErrors + + def _makeBSERCodec(self, codec): + def make_codec(transport): + return codec(transport, self.valueEncoding, self.valueErrors) + + return make_codec + + def _parseEncoding(self, enc): + if enc == "bser": + if self.useImmutableBser: + return self._makeBSERCodec(ImmutableBser2Codec) + return self._makeBSERCodec(Bser2WithFallbackCodec) + elif enc == "bser-v1": + raise BSERv1Unsupported( + "Python 3 does not support the BSER v1 encoding: specify " + '"bser" or omit the sendEncoding and recvEncoding ' + "arguments" + ) + elif enc == "json": + return JsonCodec + else: + raise WatchmanError("invalid encoding %s" % enc) + + def _hasprop(self, result, name): + if self.useImmutableBser: + return hasattr(result, name) + return name in result + + def _resolvesockname(self): + # if invoked via a trigger, watchman will set this env var; we + # should use it unless explicitly set otherwise + path = os.getenv("WATCHMAN_SOCK") + if path: + return SockPath(sockpath=path) + + cmd = [self.binpath, "--output-encoding=bser", "get-sockname"] + try: + args = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE) # noqa: C408 + + if os.name == "nt": + # if invoked via an application with graphical user interface, + # this call will cause a brief command window pop-up. + # Using the flag STARTF_USESHOWWINDOW to avoid this behavior. + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + args["startupinfo"] = startupinfo + + p = subprocess.Popen(cmd, **args) + + except OSError as e: + raise WatchmanError('"watchman" executable not in PATH (%s)', e) + + stdout, stderr = p.communicate() + exitcode = p.poll() + + if exitcode: + raise WatchmanError("watchman exited with code %d" % exitcode) + + result = bser.loads(stdout) + if "error" in result: + raise WatchmanError("get-sockname error: %s" % result["error"]) + + def get_path_result(name): + value = result.get(name, None) + if value is None: + return None + return value.decode(sys.getfilesystemencoding(), errors="surrogateescape") + + # sockname is always present + sockpath = get_path_result("sockname") + assert sockpath is not None + + return SockPath( + # unix_domain and named_pipe are reported by newer versions + # of the server and may not be present + unix_domain=get_path_result("unix_domain"), + named_pipe=get_path_result("named_pipe"), + # sockname is always present + sockpath=sockpath, + ) + + def _connect(self): + """establish transport connection""" + + if self.recvConn: + if self.pid != os.getpid(): + raise UseAfterFork( + "do not re-use a connection after fork; open a new client instead" + ) + return + + if self.sockpath is None: + self.sockpath = self._resolvesockname() + + kwargs = {} + if self.transport == CLIProcessTransport: + kwargs["binpath"] = self.binpath + + self.tport = self.transport(self.sockpath, self.timeout, **kwargs) + self.sendConn = self.sendCodec(self.tport) + self.recvConn = self.recvCodec(self.tport) + self.pid = os.getpid() + + def __del__(self): + self.close() + + def __enter__(self): + self._connect() + return self + + def __exit__(self, exc_type, exc_value, exc_traceback): + self.close() + + def close(self): + if self.tport: + self.tport.close() + self.tport = None + self.recvConn = None + self.sendConn = None + + def receive(self): + """receive the next PDU from the watchman service + + If the client has activated subscriptions or logs then + this PDU may be a unilateral PDU sent by the service to + inform the client of a log event or subscription change. + + It may also simply be the response portion of a request + initiated by query. + + There are clients in production that subscribe and call + this in a loop to retrieve all subscription responses, + so care should be taken when making changes here. + """ + + self._connect() + result = self.recvConn.receive() + if self._hasprop(result, "error"): + raise CommandError(result["error"]) + + if self._hasprop(result, "log"): + self.logs.append(result["log"]) + + if self._hasprop(result, "subscription"): + sub = result["subscription"] + if not (sub in self.subs): + self.subs[sub] = [] + self.subs[sub].append(result) + + # also accumulate in {root,sub} keyed store + root = os.path.normpath(os.path.normcase(result["root"])) + if not root in self.sub_by_root: + self.sub_by_root[root] = {} + if not sub in self.sub_by_root[root]: + self.sub_by_root[root][sub] = [] + self.sub_by_root[root][sub].append(result) + + return result + + def isUnilateralResponse(self, res): + if "unilateral" in res and res["unilateral"]: + return True + # Fall back to checking for known unilateral responses + for k in self.unilateral: + if k in res: + return True + return False + + def getLog(self, remove=True): + """Retrieve buffered log data + + If remove is true the data will be removed from the buffer. + Otherwise it will be left in the buffer + """ + res = self.logs + if remove: + self.logs = [] + return res + + def getSubscription(self, name, remove=True, root=None): + """Retrieve the data associated with a named subscription + + If remove is True (the default), the subscription data is removed + from the buffer. Otherwise the data is returned but left in + the buffer. + + Returns None if there is no data associated with `name` + + If root is not None, then only return the subscription + data that matches both root and name. When used in this way, + remove processing impacts both the unscoped and scoped stores + for the subscription data. + """ + if root is not None: + root = os.path.normpath(os.path.normcase(root)) + if root not in self.sub_by_root: + return None + if name not in self.sub_by_root[root]: + return None + sub = self.sub_by_root[root][name] + if remove: + del self.sub_by_root[root][name] + # don't let this grow unbounded + if name in self.subs: + del self.subs[name] + return sub + + if name not in self.subs: + return None + sub = self.subs[name] + if remove: + del self.subs[name] + return sub + + def query(self, *args): + """Send a query to the watchman service and return the response + + This call will block until the response is returned. + If any unilateral responses are sent by the service in between + the request-response they will be buffered up in the client object + and NOT returned via this method. + """ + + log("calling client.query") + self._connect() + try: + self.sendConn.send(args) + + res = self.receive() + while self.isUnilateralResponse(res): + res = self.receive() + + return res + except EnvironmentError as ee: + # When we can depend on Python 3, we can use PEP 3134 + # exception chaining here. + raise WatchmanEnvironmentError( + "I/O error communicating with watchman daemon", + ee.errno, + ee.strerror, + args, + ) + except WatchmanError as ex: + ex.setCommand(args) + raise + + def capabilityCheck(self, optional=None, required=None): + """Perform a server capability check""" + opts = {"optional": optional or [], "required": required or []} + res = self.query("version", opts) + + if not self._hasprop(res, "capabilities"): + # Server doesn't support capabilities, so we need to + # synthesize the results based on the version + capabilities.synthesize(res, opts) + if "error" in res: + raise CommandError(res["error"]) + + return res + + def listCapabilities(self): + return self.query("list-capabilities", {})["capabilities"] + + def setTimeout(self, value): + self.recvConn.setTimeout(value) + self.sendConn.setTimeout(value) diff --git a/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/bser.c b/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/bser.c new file mode 100644 index 000000000000..0aa166a48c76 --- /dev/null +++ b/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/bser.c @@ -0,0 +1,633 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "bser.h" + +static Py_ssize_t bserobj_tuple_length(PyObject* o) { + bserObject* obj = (bserObject*)o; + + return PySequence_Length(obj->keys); +} + +static PyObject* bserobj_tuple_item(PyObject* o, Py_ssize_t i) { + bserObject* obj = (bserObject*)o; + + return PySequence_GetItem(obj->values, i); +} + +// clang-format off +static PySequenceMethods bserobj_sq = { + bserobj_tuple_length, /* sq_length */ + 0, /* sq_concat */ + 0, /* sq_repeat */ + bserobj_tuple_item, /* sq_item */ + 0, /* sq_ass_item */ + 0, /* sq_contains */ + 0, /* sq_inplace_concat */ + 0 /* sq_inplace_repeat */ +}; +// clang-format on + +static void bserobj_dealloc(PyObject* o) { + bserObject* obj = (bserObject*)o; + + Py_CLEAR(obj->keys); + Py_CLEAR(obj->values); + PyObject_Del(o); +} + +static PyObject* bserobj_getattrro(PyObject* o, PyObject* name) { + bserObject* obj = (bserObject*)o; + Py_ssize_t i, n; + PyObject* name_bytes = NULL; + PyObject* key_bytes = NULL; + PyObject* ret = NULL; + const char* namestr; + const char* keystr; + + if (PyIndex_Check(name)) { + i = PyNumber_AsSsize_t(name, PyExc_IndexError); + if (i == -1 && PyErr_Occurred()) { + goto bail; + } + ret = PySequence_GetItem(obj->values, i); + goto bail; + } + + // We can be passed in Unicode objects here -- we don't support anything other + // than UTF-8 for keys. + if (PyUnicode_Check(name)) { + name_bytes = PyUnicode_AsUTF8String(name); + if (name_bytes == NULL) { + goto bail; + } + namestr = PyBytes_AsString(name_bytes); + } else { + namestr = PyBytes_AsString(name); + } + + if (namestr == NULL) { + goto bail; + } + // hack^Wfeature to allow mercurial to use "st_size" to reference "size" + if (!strncmp(namestr, "st_", 3)) { + namestr += 3; + } + + n = PyTuple_GET_SIZE(obj->keys); + for (i = 0; i < n; i++) { + PyObject* key = PyTuple_GET_ITEM(obj->keys, i); + + if (PyUnicode_Check(key)) { + key_bytes = PyUnicode_AsUTF8String(key); + if (key_bytes == NULL) { + goto bail; + } + keystr = PyBytes_AsString(key_bytes); + } else { + keystr = PyBytes_AsString(key); + } + + if (keystr == NULL) { + goto bail; + } + + if (!strcmp(keystr, namestr)) { + ret = PySequence_GetItem(obj->values, i); + goto bail; + } + Py_XDECREF(key_bytes); + key_bytes = NULL; + } + + PyErr_Format( + PyExc_AttributeError, "bserobject has no attribute '%.400s'", namestr); +bail: + Py_XDECREF(name_bytes); + Py_XDECREF(key_bytes); + return ret; +} + +// clang-format off +static PyMappingMethods bserobj_map = { + bserobj_tuple_length, /* mp_length */ + bserobj_getattrro, /* mp_subscript */ + 0 /* mp_ass_subscript */ +}; + +PyTypeObject bserObjectType = { + PyVarObject_HEAD_INIT(NULL, 0) + "bserobj_tuple", /* tp_name */ + sizeof(bserObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + bserobj_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + &bserobj_sq, /* tp_as_sequence */ + &bserobj_map, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + bserobj_getattrro, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + "bserobj tuple", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ +}; +// clang-format on + +int bunser_int(const char** ptr, const char* end, int64_t* val) { + int needed; + const char* buf = *ptr; + int8_t i8; + int16_t i16; + int32_t i32; + int64_t i64; + + if (buf >= end) { + PyErr_SetString(PyExc_ValueError, "input buffer to small for int encoding"); + return 0; + } + + switch (buf[0]) { + case BSER_INT8: + needed = 2; + break; + case BSER_INT16: + needed = 3; + break; + case BSER_INT32: + needed = 5; + break; + case BSER_INT64: + needed = 9; + break; + default: + PyErr_Format( + PyExc_ValueError, "invalid bser int encoding 0x%02x", buf[0]); + return 0; + } + if (end - buf < needed) { + PyErr_SetString(PyExc_ValueError, "input buffer to small for int encoding"); + return 0; + } + *ptr = buf + needed; + switch (buf[0]) { + case BSER_INT8: + memcpy(&i8, buf + 1, sizeof(i8)); + *val = i8; + return 1; + case BSER_INT16: + memcpy(&i16, buf + 1, sizeof(i16)); + *val = i16; + return 1; + case BSER_INT32: + memcpy(&i32, buf + 1, sizeof(i32)); + *val = i32; + return 1; + case BSER_INT64: + memcpy(&i64, buf + 1, sizeof(i64)); + *val = i64; + return 1; + default: + return 0; + } +} + +static int bunser_bytestring( + const char** ptr, + const char* end, + const char** start, + int64_t* len) { + const char* buf = *ptr; + + // skip string marker + buf++; + if (!bunser_int(&buf, end, len)) { + return 0; + } + + if (buf + *len > end) { + PyErr_Format(PyExc_ValueError, "invalid string length in bser data"); + return 0; + } + + *ptr = buf + *len; + *start = buf; + return 1; +} + +static PyObject* +bunser_array(const char** ptr, const char* end, const unser_ctx_t* ctx) { + const char* buf = *ptr; + int64_t nitems, i; + int mutable = ctx->is_mutable; + PyObject* res; + + assert(buf < end); + + // skip array header + buf++; + if (!bunser_int(&buf, end, &nitems)) { + return 0; + } + *ptr = buf; + + if (nitems > UINT32_MAX) { + PyErr_Format(PyExc_ValueError, "too many items for python array"); + return NULL; + } + + if (nitems > end - buf) { + // BSER guarantees each value will consume at least one byte of the input. + PyErr_Format(PyExc_ValueError, "document too short for array's size"); + return NULL; + } + + if (mutable) { + res = PyList_New((Py_ssize_t)nitems); + } else { + res = PyTuple_New((Py_ssize_t)nitems); + } + + if (!res) { + return NULL; + } + + for (i = 0; i < nitems; i++) { + PyObject* ele = bser_loads_recursive(ptr, end, ctx); + + if (!ele) { + Py_DECREF(res); + return NULL; + } + + if (mutable) { + PyList_SET_ITEM(res, i, ele); + } else { + PyTuple_SET_ITEM(res, i, ele); + } + // DECREF(ele) not required as SET_ITEM steals the ref + } + + return res; +} + +static PyObject* +bunser_object(const char** ptr, const char* end, const unser_ctx_t* ctx) { + const char* buf = *ptr; + int64_t nitems, i; + int mutable = ctx->is_mutable; + PyObject* res; + bserObject* obj; + + // skip array header + buf++; + if (!bunser_int(&buf, end, &nitems)) { + return 0; + } + *ptr = buf; + + if (nitems > UINT32_MAX) { + PyErr_Format(PyExc_ValueError, "object too big"); + return NULL; + } + + if (2 * nitems > end - buf) { + // Each key-value pair in the input will be at least two bytes long. This + // check ensures we only pre-allocate an amount of memory for the key and + // value tuples proportional to the length of the input. + PyErr_Format(PyExc_ValueError, "document too short for object's size"); + return NULL; + } + + if (mutable) { + res = PyDict_New(); + } else { + obj = PyObject_New(bserObject, &bserObjectType); + obj->keys = PyTuple_New((Py_ssize_t)nitems); + obj->values = PyTuple_New((Py_ssize_t)nitems); + res = (PyObject*)obj; + } + + for (i = 0; i < nitems; i++) { + const char* keystr; + int64_t keylen; + PyObject* key; + PyObject* ele; + + if (!bunser_bytestring(ptr, end, &keystr, &keylen)) { + Py_DECREF(res); + return NULL; + } + + if (keylen > LONG_MAX) { + PyErr_Format(PyExc_ValueError, "string too big for python"); + Py_DECREF(res); + return NULL; + } + + if (mutable) { + // This will interpret the key as UTF-8. + key = PyUnicode_FromStringAndSize(keystr, (Py_ssize_t)keylen); + } else { + // For immutable objects we'll manage key lookups, so we can avoid going + // through the Unicode APIs. This avoids a potentially expensive and + // definitely unnecessary conversion to UTF-16 and back for Python 2. + // TODO: On Python 3 the Unicode APIs are smarter: we might be able to use + // Unicode keys there without an appreciable performance loss. + key = PyBytes_FromStringAndSize(keystr, (Py_ssize_t)keylen); + } + + if (!key) { + Py_DECREF(res); + return NULL; + } + + ele = bser_loads_recursive(ptr, end, ctx); + + if (!ele) { + Py_DECREF(key); + Py_DECREF(res); + return NULL; + } + + if (mutable) { + PyDict_SetItem(res, key, ele); + Py_DECREF(key); + Py_DECREF(ele); + } else { + /* PyTuple_SET_ITEM steals ele, key */ + PyTuple_SET_ITEM(obj->values, i, ele); + PyTuple_SET_ITEM(obj->keys, i, key); + } + } + + return res; +} + +static PyObject* +bunser_template(const char** ptr, const char* end, const unser_ctx_t* ctx) { + const char* buf = *ptr; + int64_t nitems, i; + int mutable = ctx->is_mutable; + PyObject* arrval; + PyObject* keys; + Py_ssize_t numkeys, keyidx; + unser_ctx_t keys_ctx = {0}; + if (mutable) { + keys_ctx.is_mutable = 1; + // Decode keys as UTF-8 in this case. + keys_ctx.value_encoding = "utf-8"; + keys_ctx.value_errors = "strict"; + } else { + // Treat keys as bytestrings in this case -- we'll do Unicode conversions at + // lookup time. + } + + if (buf + 1 >= end) { + PyErr_SetString( + PyExc_ValueError, "input buffer to small for template encoding"); + return 0; + } + + if (buf[1] != BSER_ARRAY) { + PyErr_Format(PyExc_ValueError, "Expect ARRAY to follow TEMPLATE"); + return NULL; + } + + // skip header + buf++; + *ptr = buf; + + // Load template keys. + // For keys we don't want to do any decoding right now. + keys = bunser_array(ptr, end, &keys_ctx); + if (!keys) { + return NULL; + } + + numkeys = PySequence_Length(keys); + if (numkeys == 0) { + PyErr_Format(PyExc_ValueError, "Expected non-empty ARRAY in TEMPLATE"); + return NULL; + } + + // Load number of array elements + if (!bunser_int(ptr, end, &nitems)) { + Py_DECREF(keys); + return 0; + } + + if (nitems > UINT32_MAX) { + PyErr_Format(PyExc_ValueError, "Too many items for python"); + Py_DECREF(keys); + return NULL; + } + + arrval = PyList_New(0); + if (!arrval) { + Py_DECREF(keys); + return NULL; + } + + for (i = 0; i < nitems; i++) { + PyObject* dict = NULL; + bserObject* obj = NULL; + + if (mutable) { + dict = PyDict_New(); + } else { + obj = PyObject_New(bserObject, &bserObjectType); + if (obj) { + obj->keys = keys; + Py_INCREF(obj->keys); + obj->values = PyTuple_New(numkeys); + } + dict = (PyObject*)obj; + } + if (!dict) { + fail: + Py_DECREF(keys); + Py_DECREF(arrval); + return NULL; + } + + for (keyidx = 0; keyidx < numkeys; keyidx++) { + PyObject* key; + PyObject* ele; + + if (*ptr >= end) { + PyErr_SetString(PyExc_ValueError, "input buffer too small"); + return 0; + } + + if (**ptr == BSER_SKIP) { + *ptr = *ptr + 1; + ele = Py_None; + Py_INCREF(ele); + } else { + ele = bser_loads_recursive(ptr, end, ctx); + } + + if (!ele) { + goto fail; + } + + if (mutable) { + key = PyList_GET_ITEM(keys, keyidx); + PyDict_SetItem(dict, key, ele); + Py_DECREF(ele); + } else { + PyTuple_SET_ITEM(obj->values, keyidx, ele); + // DECREF(ele) not required as SET_ITEM steals the ref + } + } + + int error = PyList_Append(arrval, dict); + Py_DECREF(dict); + if (error != 0) { + goto fail; + } + } + + Py_DECREF(keys); + + return arrval; +} + +PyObject* bser_loads_recursive( + const char** ptr, + const char* end, + const unser_ctx_t* ctx) { + const char* buf = *ptr; + + if (buf >= end) { + PyErr_SetString(PyExc_ValueError, "input buffer too small"); + return 0; + } + + switch (buf[0]) { + case BSER_INT8: + case BSER_INT16: + case BSER_INT32: + case BSER_INT64: { + int64_t ival; + if (!bunser_int(ptr, end, &ival)) { + return NULL; + } +// Python 3 has one integer type. +#if PY_MAJOR_VERSION >= 3 + return PyLong_FromLongLong(ival); +#else + if (ival < LONG_MIN || ival > LONG_MAX) { + return PyLong_FromLongLong(ival); + } + return PyInt_FromSsize_t(Py_SAFE_DOWNCAST(ival, int64_t, Py_ssize_t)); +#endif // PY_MAJOR_VERSION >= 3 + } + + case BSER_REAL: { + if (buf + 1 + sizeof(double) > end) { + PyErr_SetString( + PyExc_ValueError, "input buffer too small for real encoding"); + return 0; + } + double dval; + memcpy(&dval, buf + 1, sizeof(dval)); + *ptr = buf + 1 + sizeof(double); + return PyFloat_FromDouble(dval); + } + + case BSER_TRUE: + *ptr = buf + 1; + Py_INCREF(Py_True); + return Py_True; + + case BSER_FALSE: + *ptr = buf + 1; + Py_INCREF(Py_False); + return Py_False; + + case BSER_NULL: + *ptr = buf + 1; + Py_INCREF(Py_None); + return Py_None; + + case BSER_BYTESTRING: { + const char* start; + int64_t len; + + if (!bunser_bytestring(ptr, end, &start, &len)) { + return NULL; + } + + if (len > LONG_MAX) { + PyErr_Format(PyExc_ValueError, "string too long for python"); + return NULL; + } + + if (ctx->value_encoding != NULL) { + return PyUnicode_Decode( + start, (long)len, ctx->value_encoding, ctx->value_errors); + } else { + return PyBytes_FromStringAndSize(start, (long)len); + } + } + + case BSER_UTF8STRING: { + const char* start; + int64_t len; + + if (!bunser_bytestring(ptr, end, &start, &len)) { + return NULL; + } + + if (len > LONG_MAX) { + PyErr_Format(PyExc_ValueError, "string too long for python"); + return NULL; + } + + return PyUnicode_Decode(start, (long)len, "utf-8", "strict"); + } + + case BSER_ARRAY: + return bunser_array(ptr, end, ctx); + + case BSER_OBJECT: + return bunser_object(ptr, end, ctx); + + case BSER_TEMPLATE: + return bunser_template(ptr, end, ctx); + + default: + PyErr_Format(PyExc_ValueError, "unhandled bser opcode 0x%02x", buf[0]); + } + + return NULL; +} diff --git a/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/bser.h b/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/bser.h new file mode 100644 index 000000000000..d29f14371bc8 --- /dev/null +++ b/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/bser.h @@ -0,0 +1,67 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#pragma once + +#define PY_SSIZE_T_CLEAN +#include // @manual=fbsource//third-party/python:python + +#ifdef __cplusplus +extern "C" { +#endif + +#define BSER_ARRAY 0x00 +#define BSER_OBJECT 0x01 +#define BSER_BYTESTRING 0x02 +#define BSER_INT8 0x03 +#define BSER_INT16 0x04 +#define BSER_INT32 0x05 +#define BSER_INT64 0x06 +#define BSER_REAL 0x07 +#define BSER_TRUE 0x08 +#define BSER_FALSE 0x09 +#define BSER_NULL 0x0a +#define BSER_TEMPLATE 0x0b +#define BSER_SKIP 0x0c +#define BSER_UTF8STRING 0x0d + +// An immutable object representation of BSER_OBJECT. +// Rather than build a hash table, key -> value are obtained +// by walking the list of keys to determine the offset into +// the values array. The assumption is that the number of +// array elements will be typically small (~6 for the top +// level query result and typically 3-5 for the file entries) +// so that the time overhead for this is small compared to +// using a proper hash table. Even with this simplistic +// approach, this is still faster for the mercurial use case +// as it helps to eliminate creating N other objects to +// represent the stat information in the hgwatchman extension +// clang-format off +typedef struct { + PyObject_HEAD + PyObject *keys; // tuple of field names + PyObject *values; // tuple of values +} bserObject; + +extern PyTypeObject bserObjectType; + +typedef struct loads_ctx { + int is_mutable; + const char* value_encoding; + const char* value_errors; + uint32_t bser_version; + uint32_t bser_capabilities; +} unser_ctx_t; + +int bunser_int(const char** ptr, const char* end, int64_t* val); + +PyObject* +bser_loads_recursive(const char** ptr, const char* end, const unser_ctx_t* ctx); + +#ifdef __cplusplus +} +#endif diff --git a/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/bsermodule.c b/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/bsermodule.c new file mode 100644 index 000000000000..ec93ff905073 --- /dev/null +++ b/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/bsermodule.c @@ -0,0 +1,649 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#define PY_SSIZE_T_CLEAN +#include // @manual=fbsource//third-party/python:python +#include // @manual=fbsource//third-party/python:python +#ifdef _MSC_VER +#define inline __inline +#if _MSC_VER >= 1800 +#include +#else +// The compiler associated with Python 2.7 on Windows doesn't ship +// with stdint.h, so define the small subset that we use here. +typedef __int8 int8_t; +typedef __int16 int16_t; +typedef __int32 int32_t; +typedef __int64 int64_t; +typedef unsigned __int8 uint8_t; +typedef unsigned __int16 uint16_t; +typedef unsigned __int32 uint32_t; +typedef unsigned __int64 uint64_t; +#define UINT32_MAX 4294967295U +#endif +#endif + +#include "bser.h" + +// clang-format off +/* Return the smallest size int that can store the value */ +#define INT_SIZE(x) (((x) == ((int8_t)x)) ? 1 : \ + ((x) == ((int16_t)x)) ? 2 : \ + ((x) == ((int32_t)x)) ? 4 : 8) + +// clang-format on + +static const char bser_true = BSER_TRUE; +static const char bser_false = BSER_FALSE; +static const char bser_null = BSER_NULL; +static const char bser_bytestring_hdr = BSER_BYTESTRING; +static const char bser_array_hdr = BSER_ARRAY; +static const char bser_object_hdr = BSER_OBJECT; + +static inline uint32_t next_power_2(uint32_t n) { + n |= (n >> 16); + n |= (n >> 8); + n |= (n >> 4); + n |= (n >> 2); + n |= (n >> 1); + return n + 1; +} + +// A buffer we use for building up the serialized result +struct bser_buffer { + char* buf; + uint32_t wpos; + uint32_t allocd; + uint32_t bser_version; + uint32_t capabilities; +}; +typedef struct bser_buffer bser_t; + +static int bser_append(bser_t* bser, const char* data, uint32_t len) { + if (bser->wpos >= UINT32_MAX - len) { + // 4 GiB overflow + errno = ENOMEM; + return 0; + } + + uint32_t newlen = next_power_2(bser->wpos + len); + if (newlen == 0) { + // We wrapped around - can't store 4G in allocd, so give up. This limits + // total output to 2 GiB. + errno = ENOMEM; + return 0; + } + + if (newlen > bser->allocd) { + char* nbuf = realloc(bser->buf, newlen); + if (!nbuf) { + return 0; + } + + bser->buf = nbuf; + bser->allocd = newlen; + } + + memcpy(bser->buf + bser->wpos, data, len); + bser->wpos += len; + return 1; +} + +static int bser_init(bser_t* bser, uint32_t version, uint32_t capabilities) { + bser->allocd = 8192; + bser->wpos = 0; + bser->buf = malloc(bser->allocd); + bser->bser_version = version; + bser->capabilities = capabilities; + if (!bser->buf) { + return 0; + } + +// Leave room for the serialization header, which includes +// our overall length. To make things simpler, we'll use an +// int32 for the header +#define EMPTY_HEADER "\x00\x01\x05\x00\x00\x00\x00" + +// Version 2 also carries an integer indicating the capabilities. The +// capabilities integer comes before the PDU size. +#define EMPTY_HEADER_V2 "\x00\x02\x00\x00\x00\x00\x05\x00\x00\x00\x00" + if (version == 2) { + bser_append(bser, EMPTY_HEADER_V2, sizeof(EMPTY_HEADER_V2) - 1); + } else { + bser_append(bser, EMPTY_HEADER, sizeof(EMPTY_HEADER) - 1); + } + + return 1; +} + +static void bser_dtor(bser_t* bser) { + free(bser->buf); + bser->buf = NULL; +} + +static int bser_long(bser_t* bser, int64_t val) { + int8_t i8; + int16_t i16; + int32_t i32; + int64_t i64; + char sz; + int size = INT_SIZE(val); + char* iptr; + + switch (size) { + case 1: + sz = BSER_INT8; + i8 = (int8_t)val; + iptr = (char*)&i8; + break; + case 2: + sz = BSER_INT16; + i16 = (int16_t)val; + iptr = (char*)&i16; + break; + case 4: + sz = BSER_INT32; + i32 = (int32_t)val; + iptr = (char*)&i32; + break; + case 8: + sz = BSER_INT64; + i64 = (int64_t)val; + iptr = (char*)&i64; + break; + default: + PyErr_SetString(PyExc_RuntimeError, "Cannot represent this long value!?"); + return 0; + } + + if (!bser_append(bser, &sz, sizeof(sz))) { + return 0; + } + + return bser_append(bser, iptr, size); +} + +static int bser_bytestring(bser_t* bser, PyObject* sval) { + char* buf = NULL; + Py_ssize_t len; + int res; + PyObject* utf = NULL; + + if (PyUnicode_Check(sval)) { + utf = PyUnicode_AsEncodedString(sval, "utf-8", "ignore"); + sval = utf; + } + + res = PyBytes_AsStringAndSize(sval, &buf, &len); + if (res == -1) { + res = 0; + goto out; + } + + if (!bser_append(bser, &bser_bytestring_hdr, sizeof(bser_bytestring_hdr))) { + res = 0; + goto out; + } + + if (!bser_long(bser, len)) { + res = 0; + goto out; + } + + if (len > UINT32_MAX) { + PyErr_Format(PyExc_ValueError, "string too big"); + res = 0; + goto out; + } + + res = bser_append(bser, buf, (uint32_t)len); + +out: + if (utf) { + Py_DECREF(utf); + } + + return res; +} + +static int bser_recursive(bser_t* bser, PyObject* val) { + if (PyBool_Check(val)) { + if (val == Py_True) { + return bser_append(bser, &bser_true, sizeof(bser_true)); + } + return bser_append(bser, &bser_false, sizeof(bser_false)); + } + + if (val == Py_None) { + return bser_append(bser, &bser_null, sizeof(bser_null)); + } + +// Python 3 has one integer type. +#if PY_MAJOR_VERSION < 3 + if (PyInt_Check(val)) { + return bser_long(bser, PyInt_AS_LONG(val)); + } +#endif // PY_MAJOR_VERSION < 3 + + if (PyLong_Check(val)) { + return bser_long(bser, PyLong_AsLongLong(val)); + } + + if (PyBytes_Check(val) || PyUnicode_Check(val)) { + return bser_bytestring(bser, val); + } + + if (PyFloat_Check(val)) { + double dval = PyFloat_AS_DOUBLE(val); + char sz = BSER_REAL; + + if (!bser_append(bser, &sz, sizeof(sz))) { + return 0; + } + + return bser_append(bser, (char*)&dval, sizeof(dval)); + } + + if (PyList_Check(val)) { + Py_ssize_t i, len = PyList_GET_SIZE(val); + + if (!bser_append(bser, &bser_array_hdr, sizeof(bser_array_hdr))) { + return 0; + } + + if (!bser_long(bser, len)) { + return 0; + } + + for (i = 0; i < len; i++) { + PyObject* ele = PyList_GET_ITEM(val, i); + + if (!bser_recursive(bser, ele)) { + return 0; + } + } + + return 1; + } + + if (PyTuple_Check(val)) { + Py_ssize_t i, len = PyTuple_GET_SIZE(val); + + if (!bser_append(bser, &bser_array_hdr, sizeof(bser_array_hdr))) { + return 0; + } + + if (!bser_long(bser, len)) { + return 0; + } + + for (i = 0; i < len; i++) { + PyObject* ele = PyTuple_GET_ITEM(val, i); + + if (!bser_recursive(bser, ele)) { + return 0; + } + } + + return 1; + } + + if (PyMapping_Check(val)) { + Py_ssize_t len = PyMapping_Length(val); + Py_ssize_t pos = 0; + PyObject *key, *ele; + + if (!bser_append(bser, &bser_object_hdr, sizeof(bser_object_hdr))) { + return 0; + } + + if (!bser_long(bser, len)) { + return 0; + } + + while (PyDict_Next(val, &pos, &key, &ele)) { + if (!bser_bytestring(bser, key)) { + return 0; + } + if (!bser_recursive(bser, ele)) { + return 0; + } + } + + return 1; + } + + PyErr_SetString(PyExc_ValueError, "Unsupported value type"); + return 0; +} + +static PyObject* bser_dumps(PyObject* self, PyObject* args, PyObject* kw) { + PyObject *val = NULL, *res; + bser_t bser; + uint32_t len, bser_version = 1, bser_capabilities = 0; + + (void)self; + + static char* kw_list[] = {"val", "version", "capabilities", NULL}; + + if (!PyArg_ParseTupleAndKeywords( + args, + kw, + "O|ii:dumps", + kw_list, + &val, + &bser_version, + &bser_capabilities)) { + return NULL; + } + + if (!bser_init(&bser, bser_version, bser_capabilities)) { + return PyErr_NoMemory(); + } + + if (!bser_recursive(&bser, val)) { + bser_dtor(&bser); + if (errno == ENOMEM) { + return PyErr_NoMemory(); + } + // otherwise, we've already set the error to something reasonable + return NULL; + } + + // Now fill in the overall length + if (bser_version == 1) { + len = bser.wpos - (sizeof(EMPTY_HEADER) - 1); + memcpy(bser.buf + 3, &len, sizeof(len)); + } else { + len = bser.wpos - (sizeof(EMPTY_HEADER_V2) - 1); + // The BSER capabilities block comes before the PDU length + memcpy(bser.buf + 2, &bser_capabilities, sizeof(bser_capabilities)); + memcpy(bser.buf + 7, &len, sizeof(len)); + } + + res = PyBytes_FromStringAndSize(bser.buf, bser.wpos); + bser_dtor(&bser); + + return res; +} + +static int _pdu_info_helper( + const char* data, + const char* end, + uint32_t* bser_version_out, + uint32_t* bser_capabilities_out, + int64_t* expected_len_out, + off_t* position_out) { + uint32_t bser_version; + uint32_t bser_capabilities = 0; + int64_t expected_len; + + const char* start; + start = data; + // Validate the header and length + if (memcmp(data, EMPTY_HEADER, 2) == 0) { + bser_version = 1; + } else if (memcmp(data, EMPTY_HEADER_V2, 2) == 0) { + bser_version = 2; + } else { + PyErr_SetString(PyExc_ValueError, "invalid bser header"); + return 0; + } + + data += 2; + + if (bser_version == 2) { + // Expect an integer telling us what capabilities are supported by the + // remote server (currently unused). + if (!memcpy(&bser_capabilities, &data, sizeof(bser_capabilities))) { + return 0; + } + data += sizeof(bser_capabilities); + } + + // Expect an integer telling us how big the rest of the data + // should be + if (!bunser_int(&data, end, &expected_len)) { + return 0; + } + + *bser_version_out = bser_version; + *bser_capabilities_out = (uint32_t)bser_capabilities; + *expected_len_out = expected_len; + *position_out = (off_t)(data - start); + return 1; +} + +// This function parses the PDU header and provides info about the packet +// Returns false if unsuccessful +static int pdu_info_helper( + PyObject* self, + PyObject* args, + uint32_t* bser_version_out, + uint32_t* bser_capabilities_out, + int64_t* total_len_out) { + const char* start = NULL; + const char* data = NULL; + Py_ssize_t datalen = 0; + const char* end; + int64_t expected_len; + off_t position; + + (void)self; + + if (!PyArg_ParseTuple(args, "s#", &start, &datalen)) { + return 0; + } + data = start; + end = data + datalen; + + if (!_pdu_info_helper( + data, + end, + bser_version_out, + bser_capabilities_out, + &expected_len, + &position)) { + return 0; + } + *total_len_out = (int64_t)(expected_len + position); + return 1; +} + +// Expected use case is to read a packet from the socket and then call +// bser.pdu_info on the packet. It returns the BSER version, BSER capabilities, +// and the total length of the entire response that the peer is sending, +// including the bytes already received. This allows the client to compute the +// data size it needs to read before it can decode the data. +static PyObject* bser_pdu_info(PyObject* self, PyObject* args) { + uint32_t version, capabilities; + int64_t total_len; + if (!pdu_info_helper(self, args, &version, &capabilities, &total_len)) { + return NULL; + } + return Py_BuildValue("kkL", version, capabilities, total_len); +} + +static PyObject* bser_pdu_len(PyObject* self, PyObject* args) { + uint32_t version, capabilities; + int64_t total_len; + if (!pdu_info_helper(self, args, &version, &capabilities, &total_len)) { + return NULL; + } + return Py_BuildValue("L", total_len); +} + +static PyObject* bser_loads(PyObject* self, PyObject* args, PyObject* kw) { + const char* data = NULL; + Py_ssize_t datalen = 0; + const char* start; + const char* end; + int64_t expected_len; + off_t position; + PyObject* mutable_obj = NULL; + const char* value_encoding = NULL; + const char* value_errors = NULL; + unser_ctx_t ctx = {1, 0}; + + static char* kw_list[] = { + "buf", "mutable", "value_encoding", "value_errors", NULL}; + + (void)self; + + if (!PyArg_ParseTupleAndKeywords( + args, + kw, + "s#|Ozz:loads", + kw_list, + &start, + &datalen, + &mutable_obj, + &value_encoding, + &value_errors)) { + return NULL; + } + + if (mutable_obj) { + ctx.is_mutable = PyObject_IsTrue(mutable_obj) > 0 ? 1 : 0; + } + ctx.value_encoding = value_encoding; + if (value_encoding == NULL) { + ctx.value_errors = NULL; + } else if (value_errors == NULL) { + ctx.value_errors = "strict"; + } else { + ctx.value_errors = value_errors; + } + data = start; + end = data + datalen; + + if (!_pdu_info_helper( + data, + end, + &ctx.bser_version, + &ctx.bser_capabilities, + &expected_len, + &position)) { + return NULL; + } + + data = start + position; + // Verify + if (expected_len + data != end) { + PyErr_SetString(PyExc_ValueError, "bser data len != header len"); + return NULL; + } + + return bser_loads_recursive(&data, end, &ctx); +} + +static PyObject* bser_load(PyObject* self, PyObject* args, PyObject* kw) { + PyObject* load; + PyObject* load_method; + PyObject* string; + PyObject* load_method_args; + PyObject* load_method_kwargs; + PyObject* fp = NULL; + PyObject* mutable_obj = NULL; + PyObject* value_encoding = NULL; + PyObject* value_errors = NULL; + + static char* kw_list[] = { + "fp", "mutable", "value_encoding", "value_errors", NULL}; + + (void)self; + + if (!PyArg_ParseTupleAndKeywords( + args, + kw, + "O|OOO:load", + kw_list, + &fp, + &mutable_obj, + &value_encoding, + &value_errors)) { + return NULL; + } + + load = PyImport_ImportModule("pywatchman.load"); + if (load == NULL) { + return NULL; + } + load_method = PyObject_GetAttrString(load, "load"); + if (load_method == NULL) { + return NULL; + } + // Mandatory method arguments + load_method_args = Py_BuildValue("(O)", fp); + if (load_method_args == NULL) { + return NULL; + } + // Optional method arguments + load_method_kwargs = PyDict_New(); + if (load_method_kwargs == NULL) { + return NULL; + } + if (mutable_obj) { + PyDict_SetItemString(load_method_kwargs, "mutable", mutable_obj); + } + if (value_encoding) { + PyDict_SetItemString(load_method_kwargs, "value_encoding", value_encoding); + } + if (value_errors) { + PyDict_SetItemString(load_method_kwargs, "value_errors", value_errors); + } + string = PyObject_Call(load_method, load_method_args, load_method_kwargs); + Py_DECREF(load_method_kwargs); + Py_DECREF(load_method_args); + Py_DECREF(load_method); + Py_DECREF(load); + return string; +} + +// clang-format off +static PyMethodDef bser_methods[] = { + {"loads", (PyCFunction)bser_loads, METH_VARARGS | METH_KEYWORDS, + "Deserialize string."}, + {"load", (PyCFunction)bser_load, METH_VARARGS | METH_KEYWORDS, + "Deserialize a file object"}, + {"pdu_info", (PyCFunction)bser_pdu_info, METH_VARARGS, + "Extract PDU information."}, + {"pdu_len", (PyCFunction)bser_pdu_len, METH_VARARGS, + "Extract total PDU length."}, + {"dumps", (PyCFunction)bser_dumps, METH_VARARGS | METH_KEYWORDS, + "Serialize string."}, + {NULL, NULL, 0, NULL} +}; + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef bser_module = { + PyModuleDef_HEAD_INIT, + "bser", + "Efficient encoding and decoding of BSER.", + -1, + bser_methods +}; +// clang-format on + +PyMODINIT_FUNC PyInit_bser(void) { + PyObject* mod; + + mod = PyModule_Create(&bser_module); + PyType_Ready(&bserObjectType); + + return mod; +} +#else + +PyMODINIT_FUNC initbser(void) { + (void)Py_InitModule("bser", bser_methods); + PyType_Ready(&bserObjectType); +} +#endif // PY_MAJOR_VERSION >= 3 + +/* vim:ts=2:sw=2:et: + */ diff --git a/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/capabilities.py b/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/capabilities.py new file mode 100644 index 000000000000..3fbf8c291686 --- /dev/null +++ b/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/capabilities.py @@ -0,0 +1,52 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + + +import re + + +def parse_version(vstr) -> int: + res = 0 + for n in vstr.split("."): + res = res * 1000 + res = res + int(n) + return res + + +cap_versions = { + "cmd-watch-del-all": "3.1.1", + "cmd-watch-project": "3.1", + "relative_root": "3.3", + "term-dirname": "3.1", + "term-idirname": "3.1", + "wildmatch": "3.7", +} + + +def check(version, name: str): + if name in cap_versions: + return version >= parse_version(cap_versions[name]) + return False + + +def synthesize(vers, opts): + """Synthesize a capability enabled version response + This is a very limited emulation for relatively recent feature sets + """ + parsed_version = parse_version(vers["version"]) + vers["capabilities"] = {} + for name in opts["optional"]: + vers["capabilities"][name] = check(parsed_version, name) + failed = False # noqa: F841 T25377293 Grandfathered in + for name in opts["required"]: + have = check(parsed_version, name) + vers["capabilities"][name] = have + if not have: + vers["error"] = ( + "client required capabilities [" + + name + + "] not supported by this server" + ) + return vers diff --git a/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/encoding.py b/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/encoding.py new file mode 100644 index 000000000000..629929a1e105 --- /dev/null +++ b/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/encoding.py @@ -0,0 +1,31 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + + +import sys + + +"""Module to deal with filename encoding on the local system, as returned by +Watchman.""" + + +default_local_errors = "surrogateescape" + + +def get_local_encoding() -> str: + if sys.platform == "win32": + # Watchman always returns UTF-8 encoded strings on Windows. + return "utf-8" + # On the Python 3 versions we support, sys.getfilesystemencoding never + # returns None. + return sys.getfilesystemencoding() + + +def encode_local(s): + return s.encode(get_local_encoding(), default_local_errors) + + +def decode_local(bs): + return bs.decode(get_local_encoding(), default_local_errors) diff --git a/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/load.py b/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/load.py new file mode 100644 index 000000000000..babb8d4115f0 --- /dev/null +++ b/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/load.py @@ -0,0 +1,82 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + + +import ctypes + + +try: + from . import bser +except ImportError: + from . import pybser as bser + + +EMPTY_HEADER = b"\x00\x01\x05\x00\x00\x00\x00" + + +def _read_bytes(fp, buf): + """Read bytes from a file-like object + + @param fp: File-like object that implements read(int) + @type fp: file + + @param buf: Buffer to read into + @type buf: bytes + + @return: buf + """ + + # Do the first read without resizing the input buffer + offset = 0 + remaining = len(buf) + while remaining > 0: + l = fp.readinto((ctypes.c_char * remaining).from_buffer(buf, offset)) + if l is None or l == 0: + return offset + offset += l + remaining -= l + return offset + + +def load(fp, mutable: bool = True, value_encoding=None, value_errors=None): + """Deserialize a BSER-encoded blob. + + @param fp: The file-object to deserialize. + @type file: + + @param mutable: Whether to return mutable results. + @type mutable: bool + + @param value_encoding: Optional codec to use to decode values. If + unspecified or None, return values as bytestrings. + @type value_encoding: str + + @param value_errors: Optional error handler for codec. 'strict' by default. + The other most common argument is 'surrogateescape' on + Python 3. If value_encoding is None, this is ignored. + @type value_errors: str + """ + buf = ctypes.create_string_buffer(8192) + SNIFF_BUFFER_SIZE = len(EMPTY_HEADER) + header = (ctypes.c_char * SNIFF_BUFFER_SIZE).from_buffer(buf) + read_len = _read_bytes(fp, header) + if read_len < len(header): + return None + + total_len = bser.pdu_len(buf) + if total_len > len(buf): + ctypes.resize(buf, total_len) + + body = (ctypes.c_char * (total_len - len(header))).from_buffer(buf, len(header)) + read_len = _read_bytes(fp, body) + if read_len < len(body): + raise RuntimeError("bser data ended early") + + return bser.loads( + (ctypes.c_char * total_len).from_buffer(buf, 0), + mutable, + value_encoding, + value_errors, + ) diff --git a/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/pybser.py b/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/pybser.py new file mode 100644 index 000000000000..db3a95ecd82e --- /dev/null +++ b/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/pybser.py @@ -0,0 +1,513 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + + +import binascii +import collections.abc as collections_abc +import ctypes +import struct +import sys + + +BSER_ARRAY = b"\x00" +BSER_OBJECT = b"\x01" +BSER_BYTESTRING = b"\x02" +BSER_INT8 = b"\x03" +BSER_INT16 = b"\x04" +BSER_INT32 = b"\x05" +BSER_INT64 = b"\x06" +BSER_REAL = b"\x07" +BSER_TRUE = b"\x08" +BSER_FALSE = b"\x09" +BSER_NULL = b"\x0a" +BSER_TEMPLATE = b"\x0b" +BSER_SKIP = b"\x0c" +BSER_UTF8STRING = b"\x0d" + +STRING_TYPES = (str, bytes) +unicode = str + + +def tobytes(i): + return str(i).encode("ascii") + + +long = int + +# Leave room for the serialization header, which includes +# our overall length. To make things simpler, we'll use an +# int32 for the header +EMPTY_HEADER = b"\x00\x01\x05\x00\x00\x00\x00" +EMPTY_HEADER_V2 = b"\x00\x02\x00\x00\x00\x00\x05\x00\x00\x00\x00" + + +def _int_size(x) -> int: + """Return the smallest size int that can store the value""" + if -0x80 <= x <= 0x7F: + return 1 + elif -0x8000 <= x <= 0x7FFF: + return 2 + elif -0x80000000 <= x <= 0x7FFFFFFF: + return 4 + elif long(-0x8000000000000000) <= x <= long(0x7FFFFFFFFFFFFFFF): + return 8 + else: + raise RuntimeError("Cannot represent value: " + str(x)) + + +def _buf_pos(buf, pos) -> bytes: + ret = buf[pos] + # Normalize the return type to bytes + if not isinstance(ret, bytes): + ret = bytes((ret,)) + return ret + + +class _bser_buffer: + def __init__(self, version): + self.bser_version = version + self.buf = ctypes.create_string_buffer(8192) + if self.bser_version == 1: + struct.pack_into( + tobytes(len(EMPTY_HEADER)) + b"s", self.buf, 0, EMPTY_HEADER + ) + self.wpos = len(EMPTY_HEADER) + else: + assert self.bser_version == 2 + struct.pack_into( + tobytes(len(EMPTY_HEADER_V2)) + b"s", self.buf, 0, EMPTY_HEADER_V2 + ) + self.wpos = len(EMPTY_HEADER_V2) + + def ensure_size(self, size): + buf = self.buf + old_size = ctypes.sizeof(buf) + new_size = old_size + while new_size - self.wpos < size: + new_size *= 2 + if old_size != new_size: + ctypes.resize(buf, new_size) + + def append_long(self, val): + size = _int_size(val) + to_write = size + 1 + self.ensure_size(to_write) + if size == 1: + struct.pack_into(b"=cb", self.buf, self.wpos, BSER_INT8, val) + elif size == 2: + struct.pack_into(b"=ch", self.buf, self.wpos, BSER_INT16, val) + elif size == 4: + struct.pack_into(b"=ci", self.buf, self.wpos, BSER_INT32, val) + elif size == 8: + struct.pack_into(b"=cq", self.buf, self.wpos, BSER_INT64, val) + else: + raise RuntimeError("Cannot represent this long value") + self.wpos += to_write + + def append_string(self, s): + if isinstance(s, unicode): + s = s.encode("utf-8") + s_len = len(s) + size = _int_size(s_len) + to_write = 2 + size + s_len + self.ensure_size(to_write) + if size == 1: + struct.pack_into( + b"=ccb" + tobytes(s_len) + b"s", + self.buf, + self.wpos, + BSER_BYTESTRING, + BSER_INT8, + s_len, + s, + ) + elif size == 2: + struct.pack_into( + b"=cch" + tobytes(s_len) + b"s", + self.buf, + self.wpos, + BSER_BYTESTRING, + BSER_INT16, + s_len, + s, + ) + elif size == 4: + struct.pack_into( + b"=cci" + tobytes(s_len) + b"s", + self.buf, + self.wpos, + BSER_BYTESTRING, + BSER_INT32, + s_len, + s, + ) + elif size == 8: + struct.pack_into( + b"=ccq" + tobytes(s_len) + b"s", + self.buf, + self.wpos, + BSER_BYTESTRING, + BSER_INT64, + s_len, + s, + ) + else: + raise RuntimeError("Cannot represent this string value") + self.wpos += to_write + + def append_recursive(self, val): + if isinstance(val, bool): + needed = 1 + self.ensure_size(needed) + if val: + to_encode = BSER_TRUE + else: + to_encode = BSER_FALSE + struct.pack_into(b"=c", self.buf, self.wpos, to_encode) + self.wpos += needed + elif val is None: + needed = 1 + self.ensure_size(needed) + struct.pack_into(b"=c", self.buf, self.wpos, BSER_NULL) + self.wpos += needed + elif isinstance(val, (int, long)): + self.append_long(val) + elif isinstance(val, STRING_TYPES): + self.append_string(val) + elif isinstance(val, float): + needed = 9 + self.ensure_size(needed) + struct.pack_into(b"=cd", self.buf, self.wpos, BSER_REAL, val) + self.wpos += needed + elif isinstance(val, collections_abc.Mapping) and isinstance( + val, collections_abc.Sized + ): + val_len = len(val) + size = _int_size(val_len) + needed = 2 + size + self.ensure_size(needed) + if size == 1: + struct.pack_into( + b"=ccb", self.buf, self.wpos, BSER_OBJECT, BSER_INT8, val_len + ) + elif size == 2: + struct.pack_into( + b"=cch", self.buf, self.wpos, BSER_OBJECT, BSER_INT16, val_len + ) + elif size == 4: + struct.pack_into( + b"=cci", self.buf, self.wpos, BSER_OBJECT, BSER_INT32, val_len + ) + elif size == 8: + struct.pack_into( + b"=ccq", self.buf, self.wpos, BSER_OBJECT, BSER_INT64, val_len + ) + else: + raise RuntimeError("Cannot represent this mapping value") + self.wpos += needed + iteritems = val.items() + for k, v in iteritems: + self.append_string(k) + self.append_recursive(v) + elif isinstance(val, collections_abc.Iterable) and isinstance( + val, collections_abc.Sized + ): + val_len = len(val) + size = _int_size(val_len) + needed = 2 + size + self.ensure_size(needed) + if size == 1: + struct.pack_into( + b"=ccb", self.buf, self.wpos, BSER_ARRAY, BSER_INT8, val_len + ) + elif size == 2: + struct.pack_into( + b"=cch", self.buf, self.wpos, BSER_ARRAY, BSER_INT16, val_len + ) + elif size == 4: + struct.pack_into( + b"=cci", self.buf, self.wpos, BSER_ARRAY, BSER_INT32, val_len + ) + elif size == 8: + struct.pack_into( + b"=ccq", self.buf, self.wpos, BSER_ARRAY, BSER_INT64, val_len + ) + else: + raise RuntimeError("Cannot represent this sequence value") + self.wpos += needed + for v in val: + self.append_recursive(v) + else: + raise RuntimeError("Cannot represent unknown value type") + + +def dumps(obj, version: int = 1, capabilities: int = 0): + bser_buf = _bser_buffer(version=version) + bser_buf.append_recursive(obj) + # Now fill in the overall length + if version == 1: + obj_len = bser_buf.wpos - len(EMPTY_HEADER) + try: + struct.pack_into(b"=i", bser_buf.buf, 3, obj_len) + except struct.error: + # The C implementation treats overflow as MemoryError. Do the same here. + raise MemoryError + else: + obj_len = bser_buf.wpos - len(EMPTY_HEADER_V2) + struct.pack_into(b"=i", bser_buf.buf, 2, capabilities) + struct.pack_into(b"=i", bser_buf.buf, 7, obj_len) + return bser_buf.buf.raw[: bser_buf.wpos] + + +# This is a quack-alike with the bserObjectType in bser.c +# It provides by getattr accessors and getitem for both index +# and name. +class _BunserDict: + __slots__ = ("_keys", "_values") + + def __init__(self, keys, values): + self._keys = keys + self._values = values + + def __getattr__(self, name): + return self.__getitem__(name) + + def __getitem__(self, key): + if isinstance(key, (int, long)): + return self._values[key] + elif key.startswith("st_"): + # hack^Wfeature to allow mercurial to use "st_size" to + # reference "size" + key = key[3:] + try: + return self._values[self._keys.index(key)] + except ValueError: + raise KeyError("_BunserDict has no key %s" % key) + + def __len__(self): + return len(self._keys) + + +class Bunser: + def __init__(self, mutable=True, value_encoding=None, value_errors=None): + self.mutable = mutable + self.value_encoding = value_encoding + + if value_encoding is None: + self.value_errors = None + elif value_errors is None: + self.value_errors = "strict" + else: + self.value_errors = value_errors + + @staticmethod + def unser_int(buf, pos): + try: + int_type = _buf_pos(buf, pos) + except IndexError: + raise ValueError("Invalid bser int encoding, pos out of range") + if int_type == BSER_INT8: + needed = 2 + fmt = b"=b" + elif int_type == BSER_INT16: + needed = 3 + fmt = b"=h" + elif int_type == BSER_INT32: + needed = 5 + fmt = b"=i" + elif int_type == BSER_INT64: + needed = 9 + fmt = b"=q" + else: + raise ValueError( + "Invalid bser int encoding 0x%s at position %s" + % (binascii.hexlify(int_type).decode("ascii"), pos) + ) + int_val = struct.unpack_from(fmt, buf, pos + 1)[0] + return (int_val, pos + needed) + + def unser_utf8_string(self, buf, pos): + str_len, pos = self.unser_int(buf, pos + 1) + str_val = struct.unpack_from(tobytes(str_len) + b"s", buf, pos)[0] + return (str_val.decode("utf-8"), pos + str_len) + + def unser_bytestring(self, buf, pos): + str_len, pos = self.unser_int(buf, pos + 1) + str_val = struct.unpack_from(tobytes(str_len) + b"s", buf, pos)[0] + if self.value_encoding is not None: + str_val = str_val.decode(self.value_encoding, self.value_errors) + # str_len stays the same because that's the length in bytes + return (str_val, pos + str_len) + + def unser_array(self, buf, pos): + arr_len, pos = self.unser_int(buf, pos + 1) + arr = [] + for _ in range(arr_len): + arr_item, pos = self.loads_recursive(buf, pos) + arr.append(arr_item) + + if not self.mutable: + arr = tuple(arr) + + return arr, pos + + def unser_object(self, buf, pos): + obj_len, pos = self.unser_int(buf, pos + 1) + if self.mutable: + obj = {} + else: + keys = [] + vals = [] + + for _ in range(obj_len): + key, pos = self.unser_utf8_string(buf, pos) + val, pos = self.loads_recursive(buf, pos) + if self.mutable: + obj[key] = val + else: + keys.append(key) + vals.append(val) + + if not self.mutable: + obj = _BunserDict(keys, vals) + + return obj, pos + + def unser_template(self, buf, pos): + val_type = _buf_pos(buf, pos + 1) + if val_type != BSER_ARRAY: + raise RuntimeError("Expect ARRAY to follow TEMPLATE") + # force UTF-8 on keys + keys_bunser = Bunser(mutable=self.mutable, value_encoding="utf-8") + keys, pos = keys_bunser.unser_array(buf, pos + 1) + nitems, pos = self.unser_int(buf, pos) + arr = [] + for _ in range(nitems): + if self.mutable: + obj = {} + else: + vals = [] + + for keyidx in range(len(keys)): + if _buf_pos(buf, pos) == BSER_SKIP: + pos += 1 + ele = None + else: + ele, pos = self.loads_recursive(buf, pos) + + if self.mutable: + key = keys[keyidx] + obj[key] = ele + else: + vals.append(ele) + + if not self.mutable: + obj = _BunserDict(keys, vals) + + arr.append(obj) + return arr, pos + + def loads_recursive(self, buf, pos): + val_type = _buf_pos(buf, pos) + if ( + val_type == BSER_INT8 + or val_type == BSER_INT16 + or val_type == BSER_INT32 + or val_type == BSER_INT64 + ): + return self.unser_int(buf, pos) + elif val_type == BSER_REAL: + val = struct.unpack_from(b"=d", buf, pos + 1)[0] + return (val, pos + 9) + elif val_type == BSER_TRUE: + return (True, pos + 1) + elif val_type == BSER_FALSE: + return (False, pos + 1) + elif val_type == BSER_NULL: + return (None, pos + 1) + elif val_type == BSER_BYTESTRING: + return self.unser_bytestring(buf, pos) + elif val_type == BSER_UTF8STRING: + return self.unser_utf8_string(buf, pos) + elif val_type == BSER_ARRAY: + return self.unser_array(buf, pos) + elif val_type == BSER_OBJECT: + return self.unser_object(buf, pos) + elif val_type == BSER_TEMPLATE: + return self.unser_template(buf, pos) + else: + raise ValueError( + "unhandled bser opcode 0x%s" + % binascii.hexlify(val_type).decode("ascii") + ) + + +def _pdu_info_helper(buf): + bser_version = -1 + if buf[0:2] == EMPTY_HEADER[0:2]: + bser_version = 1 + bser_capabilities = 0 + expected_len, pos2 = Bunser.unser_int(buf, 2) + elif buf[0:2] == EMPTY_HEADER_V2[0:2]: + if len(buf) < 8: + raise ValueError("Invalid BSER header") + bser_version = 2 + bser_capabilities = struct.unpack_from("I", buf, 2)[0] + expected_len, pos2 = Bunser.unser_int(buf, 6) + else: + raise ValueError("Invalid BSER header") + + return bser_version, bser_capabilities, expected_len, pos2 + + +def pdu_info(buf): + info = _pdu_info_helper(buf) + return info[0], info[1], info[2] + info[3] + + +def pdu_len(buf): + info = _pdu_info_helper(buf) + return info[2] + info[3] + + +def loads(buf, mutable: bool = True, value_encoding=None, value_errors=None): + """Deserialize a BSER-encoded blob. + + @param buf: The buffer to deserialize. + @type buf: bytes + + @param mutable: Whether to return mutable results. + @type mutable: bool + + @param value_encoding: Optional codec to use to decode values. If + unspecified or None, return values as bytestrings. + @type value_encoding: str + + @param value_errors: Optional error handler for codec. 'strict' by default. + The other most common argument is 'surrogateescape' on + Python 3. If value_encoding is None, this is ignored. + @type value_errors: str + """ + + info = _pdu_info_helper(buf) + expected_len = info[2] + pos = info[3] + + if len(buf) != expected_len + pos: + raise ValueError( + "bser data len %d != header len %d" % (expected_len + pos, len(buf)) + ) + + bunser = Bunser( + mutable=mutable, value_encoding=value_encoding, value_errors=value_errors + ) + + return bunser.loads_recursive(buf, pos)[0] + + +def load(fp, mutable: bool = True, value_encoding=None, value_errors=None): + from . import load + + return load.load(fp, mutable, value_encoding, value_errors) diff --git a/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/windows.py b/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/windows.py new file mode 100644 index 000000000000..80e53d289e5f --- /dev/null +++ b/watchman/python/build/lib.linux-x86_64-cpython-311/pywatchman/windows.py @@ -0,0 +1,305 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + + +import ctypes +import ctypes.wintypes +import os +import socket + + +# pyre-ignore +wintypes = ctypes.wintypes +GENERIC_READ = 0x80000000 +GENERIC_WRITE = 0x40000000 +FILE_FLAG_OVERLAPPED = 0x40000000 +OPEN_EXISTING = 3 +INVALID_HANDLE_VALUE = ctypes.c_void_p(-1).value +FORMAT_MESSAGE_FROM_SYSTEM = 0x00001000 +FORMAT_MESSAGE_ALLOCATE_BUFFER = 0x00000100 +FORMAT_MESSAGE_IGNORE_INSERTS = 0x00000200 +WAIT_FAILED = 0xFFFFFFFF +WAIT_TIMEOUT = 0x00000102 +WAIT_OBJECT_0 = 0x00000000 +WAIT_IO_COMPLETION = 0x000000C0 +INFINITE = 0xFFFFFFFF +SO_SNDTIMEO = 0x1005 +SO_RCVTIMEO = 0x1006 +SOL_SOCKET = 0xFFFF +WSAETIMEDOUT = 10060 + +# Overlapped I/O operation is in progress. (997) +ERROR_IO_PENDING = 0x000003E5 + +# The pointer size follows the architecture +# We use WPARAM since this type is already conditionally defined +ULONG_PTR = ctypes.wintypes.WPARAM + + +class OVERLAPPED(ctypes.Structure): + _fields_ = [ + ("Internal", ULONG_PTR), + ("InternalHigh", ULONG_PTR), + ("Offset", wintypes.DWORD), + ("OffsetHigh", wintypes.DWORD), + ("hEvent", wintypes.HANDLE), + ] + + def __init__(self): + self.Internal = 0 + self.InternalHigh = 0 + self.Offset = 0 + self.OffsetHigh = 0 + self.hEvent = 0 + + +LPDWORD = ctypes.POINTER(wintypes.DWORD) + +# pyre-ignore +windll = ctypes.windll + +CreateFile = windll.kernel32.CreateFileA +CreateFile.argtypes = [ + wintypes.LPSTR, + wintypes.DWORD, + wintypes.DWORD, + wintypes.LPVOID, + wintypes.DWORD, + wintypes.DWORD, + wintypes.HANDLE, +] +CreateFile.restype = wintypes.HANDLE + +CloseHandle = windll.kernel32.CloseHandle +CloseHandle.argtypes = [wintypes.HANDLE] +CloseHandle.restype = wintypes.BOOL + +ReadFile = windll.kernel32.ReadFile +ReadFile.argtypes = [ + wintypes.HANDLE, + wintypes.LPVOID, + wintypes.DWORD, + LPDWORD, + ctypes.POINTER(OVERLAPPED), +] +ReadFile.restype = wintypes.BOOL + +WriteFile = windll.kernel32.WriteFile +WriteFile.argtypes = [ + wintypes.HANDLE, + wintypes.LPVOID, + wintypes.DWORD, + LPDWORD, + ctypes.POINTER(OVERLAPPED), +] +WriteFile.restype = wintypes.BOOL + +GetLastError = windll.kernel32.GetLastError +GetLastError.argtypes = [] +GetLastError.restype = wintypes.DWORD + +SetLastError = windll.kernel32.SetLastError +SetLastError.argtypes = [wintypes.DWORD] +SetLastError.restype = None + +FormatMessage = windll.kernel32.FormatMessageA +FormatMessage.argtypes = [ + wintypes.DWORD, + wintypes.LPVOID, + wintypes.DWORD, + wintypes.DWORD, + ctypes.POINTER(wintypes.LPSTR), + wintypes.DWORD, + wintypes.LPVOID, +] +FormatMessage.restype = wintypes.DWORD + +LocalFree = windll.kernel32.LocalFree + +GetOverlappedResult = windll.kernel32.GetOverlappedResult +GetOverlappedResult.argtypes = [ + wintypes.HANDLE, + ctypes.POINTER(OVERLAPPED), + LPDWORD, + wintypes.BOOL, +] +GetOverlappedResult.restype = wintypes.BOOL + +GetOverlappedResultEx = getattr(windll.kernel32, "GetOverlappedResultEx", None) +if GetOverlappedResultEx is not None: + GetOverlappedResultEx.argtypes = [ + wintypes.HANDLE, + ctypes.POINTER(OVERLAPPED), + LPDWORD, + wintypes.DWORD, + wintypes.BOOL, + ] + GetOverlappedResultEx.restype = wintypes.BOOL + +WaitForSingleObjectEx = windll.kernel32.WaitForSingleObjectEx +WaitForSingleObjectEx.argtypes = [wintypes.HANDLE, wintypes.DWORD, wintypes.BOOL] +WaitForSingleObjectEx.restype = wintypes.DWORD + +CreateEvent = windll.kernel32.CreateEventA +CreateEvent.argtypes = [LPDWORD, wintypes.BOOL, wintypes.BOOL, wintypes.LPSTR] +CreateEvent.restype = wintypes.HANDLE + +# Windows Vista is the minimum supported client for CancelIoEx. +CancelIoEx = windll.kernel32.CancelIoEx +CancelIoEx.argtypes = [wintypes.HANDLE, ctypes.POINTER(OVERLAPPED)] +CancelIoEx.restype = wintypes.BOOL + +WinSocket = windll.ws2_32.socket +WinSocket.argtypes = [ctypes.c_int, ctypes.c_int, ctypes.c_int] +WinSocket.restype = ctypes.wintypes.HANDLE + +WinConnect = windll.ws2_32.connect +WinConnect.argtypes = [ctypes.wintypes.HANDLE, ctypes.c_void_p, ctypes.c_int] +WinConnect.restype = ctypes.c_int + +WinSend = windll.ws2_32.send +WinSend.argtypes = [ctypes.wintypes.HANDLE, ctypes.c_char_p, ctypes.c_int, ctypes.c_int] +WinSend.restype = ctypes.c_int + +WinRecv = windll.ws2_32.recv +WinRecv.argtypes = [ctypes.wintypes.HANDLE, ctypes.c_char_p, ctypes.c_int, ctypes.c_int] +WinRecv.restype = ctypes.c_int + +closesocket = windll.ws2_32.closesocket +closesocket.argtypes = [ctypes.wintypes.HANDLE] +closesocket.restype = ctypes.c_int + +WinSetIntSockOpt = windll.ws2_32.setsockopt +WinSetIntSockOpt.argtypes = [ + ctypes.wintypes.HANDLE, + ctypes.c_int, + ctypes.c_int, + wintypes.LPDWORD, + ctypes.c_int, +] +WinSetIntSockOpt.restype = ctypes.c_int + +WSAGetLastError = windll.ws2_32.WSAGetLastError +WSAGetLastError.argtypes = [] + +WSADESCRIPTION_LEN = 256 + 1 +WSASYS_STATUS_LEN = 128 + 1 + + +class WSAData64(ctypes.Structure): + _fields_ = [ + ("wVersion", ctypes.c_ushort), + ("wHighVersion", ctypes.c_ushort), + ("iMaxSockets", ctypes.c_ushort), + ("iMaxUdpDg", ctypes.c_ushort), + ("lpVendorInfo", ctypes.c_char_p), + ("szDescription", ctypes.c_ushort * WSADESCRIPTION_LEN), + ("szSystemStatus", ctypes.c_ushort * WSASYS_STATUS_LEN), + ] + + +WSAStartup = windll.ws2_32.WSAStartup +WSAStartup.argtypes = [ctypes.wintypes.WORD, ctypes.POINTER(WSAData64)] +WSAStartup.restype = ctypes.c_int + + +class SOCKADDR_UN(ctypes.Structure): + _fields_ = [("sun_family", ctypes.c_ushort), ("sun_path", ctypes.c_char * 108)] + + +class WindowsSocketException(Exception): + def __init__(self, code: int) -> None: + super(WindowsSocketException, self).__init__( + "Windows Socket Error: {}".format(code) + ) + + +class WindowsSocketHandle: + AF_UNIX = 1 + SOCK_STREAM = 1 + + fd: int = -1 + address: str = "" + + @staticmethod + def _checkReturnCode(retcode): + if retcode == -1: + errcode = WSAGetLastError() + if errcode == WSAETIMEDOUT: + raise socket.timeout() + raise WindowsSocketException(errcode) + + def __init__(self): + wsa_data = WSAData64() + # ctypes.c_ushort(514) = MAKE_WORD(2,2) which is for the winsock + # library version 2.2 + errcode = WSAStartup(ctypes.c_ushort(514), ctypes.pointer(wsa_data)) + if errcode != 0: + raise WindowsSocketException(errcode) + + fd = WinSocket(self.AF_UNIX, self.SOCK_STREAM, 0) + self._checkReturnCode(fd) + self.fd = fd + + def fileno(self) -> int: + return self.fd + + def settimeout(self, timeout: int) -> None: + timeout = wintypes.DWORD(0 if timeout is None else int(timeout * 1000)) + retcode = WinSetIntSockOpt( + self.fd, + SOL_SOCKET, + SO_RCVTIMEO, + ctypes.byref(timeout), + ctypes.sizeof(timeout), + ) + self._checkReturnCode(retcode) + retcode = WinSetIntSockOpt( + self.fd, + SOL_SOCKET, + SO_SNDTIMEO, + ctypes.byref(timeout), + ctypes.sizeof(timeout), + ) + self._checkReturnCode(retcode) + return None + + def connect(self, address: str) -> None: + # pyre-ignore + address = os.fsencode(os.path.normpath(address)) + addr = SOCKADDR_UN(sun_family=self.AF_UNIX, sun_path=address) + self._checkReturnCode( + WinConnect(self.fd, ctypes.pointer(addr), ctypes.sizeof(addr)) + ) + self.address = address + + def send(self, buff: bytes) -> int: + retcode = WinSend(self.fd, buff, len(buff), 0) + self._checkReturnCode(retcode) + return retcode + + def sendall(self, buff: bytes) -> None: + while len(buff) > 0: + x = self.send(buff) + if x > 0: + buff = buff[x:] + else: + break + return None + + def recv(self, size: int) -> bytes: + buff = ctypes.create_string_buffer(size) + retsize = WinRecv(self.fd, buff, size, 0) + self._checkReturnCode(retsize) + return buff.raw[0:retsize] + + def getpeername(self) -> str: + return self.address + + def getsockname(self) -> str: + return self.address + + def close(self) -> int: + return closesocket(self.fd) diff --git a/watchman/python/build/scripts-3.11/watchman-make b/watchman/python/build/scripts-3.11/watchman-make new file mode 100755 index 000000000000..16240667bb1e --- /dev/null +++ b/watchman/python/build/scripts-3.11/watchman-make @@ -0,0 +1,307 @@ +#!/usr/bin/python3 +import argparse +import os +import subprocess +import sys + +import pywatchman + + +STRING_TYPES = (str, bytes) + + +def patterns_to_terms(pats): + # convert a list of globs into the equivalent watchman expression term + if pats is None or len(pats) == 0: + return ["true"] + terms = ["anyof"] + for p in pats: + terms.append(["match", p, "wholename", {"includedotfiles": True}]) + return terms + + +class Target(object): + """Base Class for a Target + + We track the patterns that we consider to be the dependencies for + this target and establish a subscription for them. + + When we receive notifications for that subscription, we know that + we should execute the command. + """ + + def __init__(self, name, patterns, cmd): + self.name = name + self.patterns = patterns + self.cmd = cmd + self.triggered = False + + def start(self, client, root): + query = {"expression": patterns_to_terms(self.patterns), "fields": ["name"]} + watch = client.query("watch-project", root) + if "warning" in watch: + print("WARNING: ", watch["warning"], file=sys.stderr) + root_dir = watch["watch"] + if "relative_path" in watch: + query["relative_root"] = watch["relative_path"] + + # get the initial clock value so that we only get updates + query["since"] = client.query("clock", root_dir)["clock"] + + print( + "# Changes to files matching %s will execute `%s`" + % (" ".join(self.patterns), self.cmd), + file=sys.stderr, + ) + sub = client.query("subscribe", root_dir, self.name, query) + + def consumeEvents(self, client): + data = client.getSubscription(self.name) + if data is None: + return + for item in data: + # We only want to trigger if files matched; + # updates without a files list are metadata + # such as state-enter/leave notices so we skip them + if "files" in item: + self.triggered = True + if "canceled" in item: + raise RuntimeError("Watch was cancelled") + + def execute(self): + if not self.triggered: + return + self.triggered = False + print("# Execute: `%s`" % self.cmd, file=sys.stderr) + subprocess.call(self.cmd, shell=True) + + +class MakefileTarget(Target): + """Represents a Makefile target that we'd like to build.""" + + def __init__(self, name, make, targets, patterns): + self.make = make + self.targets = targets + cmd = "%s %s" % (self.make, " ".join(self.targets)) + super(MakefileTarget, self).__init__(name, patterns, cmd) + + def __repr__(self): + return "{make=%r targets=%r pat=%r}" % (self.make, self.targets, self.patterns) + + +class RunTarget(Target): + """Represents a script that we'd like to run.""" + + def __init__(self, name, runfile, patterns): + self.runfile = runfile + super(RunTarget, self).__init__(name, patterns, self.runfile) + + def __repr__(self): + return "{runfile=%r pat=%r}" % (self.runfile, self.patterns) + + +class DefineTarget(argparse.Action): + """argument parser helper to manage defining MakefileTarget instances.""" + + def __init__(self, option_strings, dest, **kwargs): + super(DefineTarget, self).__init__(option_strings, dest, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + targets = getattr(namespace, self.dest) + if targets is None: + targets = [] + setattr(namespace, self.dest, targets) + + if isinstance(values, STRING_TYPES): + values = [values] + + if namespace.pattern is None or len(namespace.pattern) == 0: + print("no patterns were specified for target %s" % values, file=sys.stderr) + sys.exit(1) + + target = MakefileTarget( + "target_%d" % len(targets), namespace.make, values, namespace.pattern + ) + targets.append(target) + + # Clear out patterns between targets + namespace.pattern = None + + +parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description=""" +watchman-make waits for changes to files and then invokes a build tool +(by default, `make`) or provided script to process those changes. +It uses the watchman service to efficiently watch the appropriate files. + +Events are consolidated and settled before they are dispatched to your build +tool, so that it won't start executing until after the files have stopped +changing. + +You can tell watchman-make about one or more build targets and dependencies +for those targets or provide a script to run. +watchman-make will then trigger the build for the given targets or +run the provided script as changes are detected. + +""", +) +parser.add_argument( + "-t", + "--target", + nargs="+", + type=str, + action=DefineTarget, + help=""" +Specify a list of target(s) to pass to the make tool. The --make and +--pattern options that precede --target are used to define the trigger +condition. +""", +) +parser.add_argument( + "-s", + "--settle", + type=float, + default=0.2, + help="How long to wait to allow changes to settle before invoking targets", +) +parser.add_argument( + "--make", + type=str, + default="make", + help=""" +The name of the make tool to use for the next --target. The default is `make`. +You may include additional arguments; you are not limited to just the +path to a tool or script. +""", +) +parser.add_argument( + "-p", + "--pattern", + type=str, + nargs="+", + help=""" +Define filename matching patterns that will be used to trigger the next +--target definition. + +The pattern syntax is wildmatch style; globbing with recursive matching +via '**'. + +--pattern is reset to empty after each --target argument. +""", +) +parser.add_argument( + "--root", + type=str, + default=".", + help=""" +Define the root of the project. The default is to use the PWD. +All patterns are considered to be relative to this root, and the build +tool is executed with this location set as its PWD. +""", +) +parser.add_argument( + "-r", + "--run", + type=str, + help=""" +The script that should be run when changes are detected +""", +) +parser.add_argument( + "--connect-timeout", + type=float, + default=600, + help=""" +Initial watchman client connection timeout. It should be sufficiently large to +prevent timeouts when watchman is busy (eg. performing a crawl). The default +value is 600 seconds. +""", +) +args = parser.parse_args() + +if args.target is None and args.run is None: + print("# No run script or targets were specified, nothing to do.", file=sys.stderr) + sys.exit(1) + +if args.target is None: + args.target = [] + if args.run is not None: + args.target.append(RunTarget("RunTarget", args.run, args.pattern)) + + +def check_root(desired_root): + try: + root = os.path.abspath(desired_root) + os.chdir(root) + return root + except Exception as ex: + print( + "--root=%s: specified path is invalid: %s" % (desired_root, ex), + file=sys.stderr, + ) + sys.exit(1) + + +targets = {} +client = pywatchman.client(timeout=args.connect_timeout) +try: + client.capabilityCheck(required=["cmd-watch-project", "wildmatch"]) + root = check_root(args.root) + print("# Relative to %s" % root, file=sys.stderr) + for t in args.target: + t.start(client, root) + targets[t.name] = t + +except pywatchman.CommandError as ex: + print("watchman:", str(ex), file=sys.stderr) + sys.exit(1) + +print("# waiting for changes", file=sys.stderr) +while True: + try: + # Wait for changes to start to occur. We're happy to wait + # quite some time for this + client.setTimeout(600) + + result = client.receive() + for _, t in targets.items(): + t.consumeEvents(client) + + # Now we wait for events to settle + client.setTimeout(args.settle) + settled = False + while not settled: + try: + result = client.receive() + for _, t in targets.items(): + t.consumeEvents(client) + except pywatchman.SocketTimeout as ex: + # Our short settle timeout hit, so we're now settled + settled = True + break + + # Now we can work on executing the targets + for _, t in targets.items(): + t.execute() + + # Print this at the bottom of the loop rather than the top + # because we may timeout every so often and it looks weird + # to keep printing 'waiting for changes' each time we do. + print("# waiting for changes", file=sys.stderr) + + except pywatchman.SocketTimeout as ex: + # Let's check to see if we're still functional + try: + vers = client.query("version") + except Exception as ex: + print("watchman:", str(ex), file=sys.stderr) + sys.exit(1) + + except pywatchman.WatchmanError as ex: + print("watchman:", str(ex), file=sys.stderr) + sys.exit(1) + + except KeyboardInterrupt: + # suppress ugly stack trace when they Ctrl-C + break diff --git a/watchman/python/build/scripts-3.11/watchman-replicate-subscription b/watchman/python/build/scripts-3.11/watchman-replicate-subscription new file mode 100755 index 000000000000..368de565020f --- /dev/null +++ b/watchman/python/build/scripts-3.11/watchman-replicate-subscription @@ -0,0 +1,397 @@ +#!/usr/bin/python3 +import argparse +import json +import os +import sys +import time + +import pywatchman + + +def fieldlist(s): + # helper for splitting a list of fields by comma + return s.split(",") + + +parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description=""" +watchman-replicate-subscription can replicate an existing watchman +subscription. It queries watchman for a list of subscriptions, identifies the +source subscription (i.e., the subscription to replicate) and subscribes to watchman +using the same query. + +Integrators can use this client to validate the watchman notifications their +client is receiving to localize anomalous behavior. + +The source subscription is identified using any combination of the 'name', +'pid', and 'client' arguments. The provided combination must uniquely identify +a subscription. Source subscription details for a watched root can be +retrieved by running the command 'watchman-replicate-subscription PATH --list'. + +By default, the replicated subscription will take the source subscription +name and prepend the substring 'replicate: ' to it. The 'qname' option can be +used to specify the replicated subscription name. + +The subscription can stop after a configurable number of events are observed. +The default is a single event. You may also remove the limit and allow it to +execute continuously. + +watchman-replicate-subscription will print one event per line. The event +information is determined by the fields in the identified subscription, with +each field separated by a space (or your choice of --separator). + +Subscription state-enter and state-leave PDUs will be interleaved with other +events. Known subscription PDUs (currently only those generated by the +mercurial fsmonitor extension) will be enclosed in square brackets. All others will be +output in JSON format. + +Events are consolidated and settled by the watchman server before they are +dispatched to watchman-replicate-subscription. + +Exit Status: + +The following exit status codes can be used to determine what caused +watchman-wait to exit: + +0 After successfully waiting for event(s) or listing matching subscriptions +1 In case of a runtime error of some kind +2 The -t/--timeout option was used and that amount of time passed + before an event was received +3 Execution was interrupted (Ctrl-C) + +""", +) +parser.add_argument( + "path", + type=str, + help=""" +The path to a watched root whose subscription we'd like to replicate. The list +of watched roots can be retrieved by running 'watchman watch-list'. +""", +) +parser.add_argument( + "-s", + "--separator", + type=str, + default=" ", + help="String to use as field separator for event output.", +) +parser.add_argument( + "-q", + "--qname", + type=str, + default=None, + help=""" +The replicated subscription name. The default will be the source subscription +with the string 'replicate: ' prepended to it. +""", +) +parser.add_argument( + "-n", + "--name", + type=str, + default=None, + help=""" +The name of the subscription to replicate. +""", +) +parser.add_argument( + "-c", + "--client", + type=str, + default=None, + help=""" +The client id of the subscription to replicate. +""", +) +parser.add_argument( + "-p", + "--pid", + type=str, + default=None, + help=""" +The process id of the subscription to replicate. +""", +) +parser.add_argument( + "-m", + "--max-events", + type=int, + default=1, + help=""" +Set the maximum number of events that will be processed. When the limit +is reached, watchman-replicate-subscription exit. The default is 1. Setting +the limit to 0 removes the limit, causing watchman-wait to execute indefinitely. +""", +) +parser.add_argument( + "-t", + "--timeout", + type=float, + default=0, + help=""" +Exit if no events trigger within the specified timeout. If timeout is +zero (the default) then keep running indefinitely. +""", +) +parser.add_argument( + "--connect-timeout", + type=float, + default=100, + help=""" +Initial watchman client connection timeout. It should be sufficiently large to +prevent timeouts when watchman is busy (eg. performing a crawl). The default +value is 100 seconds. +""", +) +parser.add_argument( + "-l", + "--list", + action="store_true", + help=""" +Print the matching subscription list and exit. +""", +) +parser.add_argument( + "-f", + "--full", + action="store_true", + help=""" +Use with '--list' to print complete subscription information, including the +query. +""", +) +parser.add_argument( + "--state-only", + action="store_true", + help=""" +Print only the subscription state-enter and state-leave PDUs. +""", +) +args = parser.parse_args() + +# Running total of individual file events we've seen +total_events = 0 + + +class Subscription(object): + root = None # Watched root + name = None # Our name for this subscription + path = None + + def __init__(self, path, name, query): + self.name = name + self.query = query + self.path = os.path.abspath(path) + if not os.path.exists(self.path): + print("path %s (%s) does not exist." % (path, self.path), file=sys.stderr) + sys.exit(1) + + def __repr__(self): + return "Subscription(path=%s, name=%s, query=%s)" % ( + self.path, + self.name, + json.dumps(self.query), + ) + + def start(self, client): + watch = client.query("watch-project", self.path) + if "warning" in watch: + print("WARNING: ", watch["warning"], file=sys.stderr) + self.root = watch["watch"] + + # get the initial clock value so that we only get updates + self.query["since"] = client.query("clock", self.root)["clock"] + sub = client.query("subscribe", self.root, self.name, self.query) + + def formatField(self, fname, val): + return str(val) + + def formatFsmonitorSubPdu(self, sub): + out = [] + keys = ["state-enter", "state-leave"] + for key in keys: + if key in sub: + out.append(key) + out.append(sub[key]) + keys = ["abandoned"] + for key in keys: + if key in sub: + out.append(key) + if "metadata" in sub: + metadata = sub["metadata"] + keys = ["status", "rev", "partial", "distance"] + for key in keys: + if key in keys: + out.append(str(metadata[key])) + return "[ %s ]" % (" ".join(out)) + + def formatSubPdu(self, sub): + # If fsmonitor metadata fields present, pretty-print + if "metadata" in sub and all( + key in sub["metadata"] for key in ["status", "rev", "partial", "distance"] + ): + return self.formatFsmonitorSubPdu(sub) + return json.dumps(sub, indent=2) + + def emit(self, client): + global total_events + data = client.getSubscription(self.name) + if data is None: + return False + for dat in data: + if any(key in dat.keys() for key in ["state-enter", "state-leave"]): + print(self.formatSubPdu(dat)) + sys.stdout.flush() + total_events = total_events + 1 + if args.max_events > 0 and total_events >= args.max_events: + sys.exit(0) + if args.state_only: + continue + for f in dat.get("files", []): + out = [] + if len(repFields) == 1: + # When only 1 field is specified, the result is a + # list of just the values + out.append(self.formatField(repFields[0], f)) + else: + # Otherwise it is a list of objects + for fname, val in f.items(): + out.append(self.formatField(fname, val)) + print(args.separator.join(out)) + sys.stdout.flush() + total_events = total_events + 1 + if args.max_events > 0 and total_events >= args.max_events: + sys.exit(0) + return True + + +def getSubIdInfo(subName, subClient, subPid): + return "(name='%s', pid='%s' client='%s')" % ( + "Any" if subName is None else subName, + "Any" if subPid is None else subPid, + "Any" if subClient is None else subClient, + ) + + +def getSubInfo(sub, keys=None): + rslt = {} + if keys is None: + keys = ["name", "pid", "client", "query"] + subInfo = sub["info"] if "info" in sub else {} + for key in keys: + if key in subInfo: + rslt[key] = subInfo[key] + else: + rslt[key] = "" + keyWarn = true + return rslt + + +path = args.path +subName = args.name +subPid = args.pid +subClient = args.client +client = pywatchman.client(timeout=args.connect_timeout) + +# We use debug-get-subscriptions to get subscription information. Note: +# the debug commands are not stable/supported, so we are susceptible to breakage +# if they change. Do not copy this approach without understanding this risk. +subs = client.query("debug-get-subscriptions", path) +matchSubs = [] +for sub in subs["subscribers"]: + info = sub["info"] + if ( + (subName is None or sub["info"]["name"] == subName) + and (subPid is None or str(sub["info"]["pid"]) == subPid) + and (subClient is None or str(sub["info"]["client"]) == subClient) + ): + matchSubs.append(sub) + +if args.list: + if len(matchSubs) == 0: + print("No matching subscriptions") + for sub in matchSubs: + keys = None if args.full else ["name", "pid", "client"] + print(json.dumps(getSubInfo(sub, keys=keys), indent=2), file=sys.stdout) + sys.exit(0) + +if len(matchSubs) == 0: + print( + "Error, no matching subscriptions:\n" + "\tcriteria: %s\n" + "\tpath: %s\n" + "To get a list of subscriptions for a watched root, use:\n" + "\twatchman-replicate-subscription PATH --list" + % (getSubIdInfo(subName, subClient, subPid), path), + file=sys.stderr, + ) + sys.exit(1) + +if len(matchSubs) > 1: + print( + "Error, found multiple matching subscriptions:\n" + "\tcriteria: %s\n" + "\tpath: %s\n" + "Use the '--name', '--client' and '--pid' options to identify a subscription.\n" + "To get a list of subscriptions for a watched root, use:\n" + "\twatchman-replicate-subscription PATH --list" + % (getSubIdInfo(subName, subClient, subPid), path), + file=sys.stderr, + ) + sys.exit(1) + +matchSub = matchSubs[0] +repQuery = matchSubs[0]["info"]["query"] + +# Subscriptions with no fields specified, will get watchman default +# and will describe themselves in the response. If they specified +# just a single field then we'd like to know what it is because we +# won't know the name from the `files` list so we make an attempt +# to capture the field definition here and use it only in the case +# that it is a single field name, +repFields = matchSubs[0]["info"]["query"].get("fields", []) +repName = args.qname if args.qname else "replicate: " + matchSub["info"]["name"] + +repSub = Subscription(name=repName, path=path, query=repQuery) +print("%s" % (repSub)) + +deadline = None +if args.timeout > 0: + deadline = time.time() + args.timeout + +try: + repSub.start(client) + +except pywatchman.CommandError as ex: + print("watchman:", ex.msg, file=sys.stderr) + sys.exit(1) + +while deadline is None or time.time() < deadline: + try: + if deadline is not None: + client.setTimeout(deadline - time.time()) + # wait for a unilateral response + result = client.receive() + + # in theory we can parse just the result variable here, but + # the client object will accumulate all subscription results + # over time, so we ask it to remove and return those values + # for each of the subscriptions + repSub.emit(client) + + except pywatchman.SocketTimeout as ex: + if deadline is not None and time.time() >= deadline: + sys.exit(2) + + # Let's check to see if we're still functional + try: + vers = client.query("version") + except Exception as ex: + print("watchman:", str(ex), file=sys.stderr) + sys.exit(1) + + except KeyboardInterrupt: + # suppress ugly stack trace when they Ctrl-C + sys.exit(3) diff --git a/watchman/python/build/scripts-3.11/watchman-wait b/watchman/python/build/scripts-3.11/watchman-wait new file mode 100755 index 000000000000..d10bca1fcf42 --- /dev/null +++ b/watchman/python/build/scripts-3.11/watchman-wait @@ -0,0 +1,272 @@ +#!/usr/bin/python3 +import argparse +import os +import sys +import time + +import pywatchman + + +def fieldlist(s): + # helper for splitting a list of fields by comma + return s.split(",") + + +parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description=""" +watchman-wait waits for changes to files. It uses the watchman service to +efficiently and recursively watch your specified list of paths. + +It is suitable for waiting for changes to files from shell scripts. + +It can stop after a configurable number of events are observed. The default +is a single event. You may also remove the limit and allow it to execute +continuously. + +watchman-wait will print one event per line. The event information includes +your specified list of fields, with each field separated by a space (or your +choice of --separator). + +Events are consolidated and settled by the watchman server before they are +dispatched to watchman-wait. + +Exit Status: + +The following exit status codes can be used to determine what caused +watchman-wait to exit: + +0 After successfully waiting for event(s) +1 In case of a runtime error of some kind +2 The -t/--timeout option was used and that amount of time passed + before an event was received +3 Execution was interrupted (Ctrl-C) + +""", +) +parser.add_argument("path", type=str, nargs="+", help="path(s) to watch") +parser.add_argument( + "--relative", + type=str, + default=".", + help="print paths relative to this dir (default=PWD)", +) +parser.add_argument( + "--fields", + type=fieldlist, + default=["name"], + help=""" +Comma separated list of file information fields to return. +The default is just the name. For a list of possible fields, see: +https://facebook.github.io/watchman/docs/cmd/query.html#available-fields +""", +) +parser.add_argument( + "-s", + "--separator", + type=str, + default=" ", + help="String to use as field separator for event output.", +) +parser.add_argument( + "-0", + "--null", + action="store_true", + help=""" +Use a NUL byte as a field separator, takes precedence over --separator. +""", +) +parser.add_argument( + "-m", + "--max-events", + type=int, + default=1, + help=""" +Set the maximum number of events that will be processed. When the limit +is reached, watchman-wait will exit. The default is 1. Setting the +limit to 0 removes the limit, causing watchman-wait to execute indefinitely. +""", +) +parser.add_argument( + "-p", + "--pattern", + type=str, + nargs="+", + help=""" +Only emit paths that match this list of patterns. Patterns are +applied by the watchman server and are matched against the root-relative +paths. + +You will almost certainly want to use quotes around your pattern list +so that your shell doesn't interpret the pattern. + +The pattern syntax is wildmatch style; globbing with recursive matching +via '**'. +""", +) +parser.add_argument( + "-t", + "--timeout", + type=float, + default=0, + help=""" +Exit if no events trigger within the specified timeout. If timeout is +zero (the default) then keep running indefinitely. +""", +) +parser.add_argument( + "--connect-timeout", + type=float, + default=100, + help=""" +Initial watchman client connection timeout. It should be sufficiently large to +prevent timeouts when watchman is busy (eg. performing a crawl). The default +value is 100 seconds. +""", +) +args = parser.parse_args() +if args.null: + args.separator = "\0" + + +# We parse the list of paths into a set of subscriptions +subscriptions = {} + +# Running total of individual file events we've seen +total_events = 0 + + +class Subscription(object): + root = None # Watched root + relpath = None # Offset to dir of interest + name = None # Our name for this subscription + path = None + + def __init__(self, path): + if path in subscriptions: + raise ValueError("path %s already specified" % path) + self.name = path + self.path = os.path.abspath(path) + if not os.path.exists(self.path): + print( + """path %s (%s) does not exist. +Perhaps you should use the --pattern option?""" + % (path, self.path), + file=sys.stderr, + ) + sys.exit(1) + subscriptions[self.name] = self + + def __repr__(self): + return "Subscription(root=%s, rel=%s, name=%s)" % ( + self.root, + self.relpath, + self.name, + ) + + def start(self, client): + dir_to_watch = self.path + if args.pattern: + expr = ["anyof"] + for p in args.pattern: + expr.append(["match", p, "wholename", {"includedotfiles": True}]) + else: + expr = ["true"] + if not os.path.isdir(self.path): + # Need to watch its parent + dir_to_watch = os.path.dirname(self.path) + expr = ["name", os.path.basename(self.path)] + + query = {"expression": expr, "fields": args.fields} + watch = client.query("watch-project", dir_to_watch) + if "warning" in watch: + print("WARNING: ", watch["warning"], file=sys.stderr) + + self.root = watch["watch"] + if "relative_path" in watch: + query["relative_root"] = watch["relative_path"] + + # get the initial clock value so that we only get updates + query["since"] = client.query("clock", self.root)["clock"] + + sub = client.query("subscribe", self.root, self.name, query) + + def formatField(self, fname, val): + if fname == "name": + # Respect the --relative path printing option + return os.path.relpath(os.path.join(self.name, val), args.relative) + # otherwise just make sure it's a string so that we can join it + return str(val) + + def emit(self, client): + global total_events + data = client.getSubscription(self.name) + if data is None: + return False + for dat in data: + for f in dat.get("files", []): + out = [] + if len(args.fields) == 1: + # When only 1 field is specified, the result is a + # list of just the values + out.append(self.formatField(args.fields[0], f)) + else: + # Otherwise it is a list of objects + for fname in args.fields: + out.append(self.formatField(fname, f[fname])) + print(args.separator.join(out)) + sys.stdout.flush() + total_events = total_events + 1 + if args.max_events > 0 and total_events >= args.max_events: + sys.exit(0) + return True + + +# Translate paths into subscriptions +for path in args.path: + sub = Subscription(path) + +# and start up the client + subscriptions +client = pywatchman.client(timeout=args.connect_timeout) + +deadline = None +if args.timeout > 0: + deadline = time.time() + args.timeout + +try: + client.capabilityCheck(required=["term-dirname", "cmd-watch-project", "wildmatch"]) + for _, sub in subscriptions.items(): + sub.start(client) + +except pywatchman.CommandError as ex: + print("watchman:", ex.msg, file=sys.stderr) + sys.exit(1) + +while deadline is None or time.time() < deadline: + try: + if deadline is not None: + client.setTimeout(deadline - time.time()) + # wait for a unilateral response + result = client.receive() + + # in theory we can parse just the result variable here, but + # the client object will accumulate all subscription results + # over time, so we ask it to remove and return those values + # for each of the subscriptions + for _, sub in subscriptions.items(): + sub.emit(client) + + except pywatchman.SocketTimeout as ex: + if deadline is not None and time.time() >= deadline: + sys.exit(2) + + # Let's check to see if we're still functional + try: + vers = client.query("version") + except Exception as ex: + print("watchman:", str(ex), file=sys.stderr) + sys.exit(1) + + except KeyboardInterrupt: + # suppress ugly stack trace when they Ctrl-C + sys.exit(3) diff --git a/watchman/rust/watchman_client/Cargo.toml b/watchman/rust/watchman_client/Cargo.toml index 0278749c8f3e..4b45979b2925 100644 --- a/watchman/rust/watchman_client/Cargo.toml +++ b/watchman/rust/watchman_client/Cargo.toml @@ -17,14 +17,16 @@ maplit = "1.0" serde = { version = "1.0.126", features = ["derive", "rc"] } serde_bser = { version = "0.4", path = "../serde_bser" } thiserror = "1.0" -tokio = { version = "1.7.1", features = ["full", "test-util"] } -tokio-util = { version = "0.6", features = ["full"] } +tokio = { version = "1.7.1", features = ["io-util", "net", "rt", "process"] } +tokio-util = { version = "0.6", features = ["codec"] } [target.'cfg(windows)'.dependencies] winapi = { version = "0.3", features = ["fileapi", "handleapi", "winbase", "winuser"] } [dev-dependencies] clap = { version = "4.5.7", features = ["derive"] } +tokio = { version = "1.7.1", features = ["io-util", "net", "macros", "process", "rt-multi-thread", "test-util"] } +tokio-util = { version = "0.6", features = ["codec", "io"] } [lints.rust] unexpected_cfgs = { level = "warn", check-cfg = ["cfg(fbcode_build)"] }