From b5b6a8a56c5a32547ce97cb8bfbcf497a90d1a2f Mon Sep 17 00:00:00 2001 From: Yotaro Kubo Date: Thu, 21 Sep 2023 15:49:17 +0900 Subject: [PATCH] Optimize bit serialization by combination of several methods. - custom ByteVec struct is now used as a BitSink - add aligned byte addition API for BitSink - CRC computation buffer is now thread-locally pre-allocated --- Cargo.lock | 2 +- flacenc-bin/Cargo.lock | 553 +++++++++++++++++++++++++++++++++++++++- flacenc-bin/Cargo.toml | 1 - flacenc-bin/src/main.rs | 14 +- report/report.md | 14 +- src/bitsink.rs | 171 ++++++++++++- src/component.rs | 102 ++++---- src/test_helper.rs | 9 +- 8 files changed, 798 insertions(+), 68 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e9f9a73..e49a509 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -305,7 +305,7 @@ checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" [[package]] name = "flacenc" -version = "0.1.0" +version = "0.1.1" dependencies = [ "bitvec", "claxon", diff --git a/flacenc-bin/Cargo.lock b/flacenc-bin/Cargo.lock index fd4415b..3603672 100644 --- a/flacenc-bin/Cargo.lock +++ b/flacenc-bin/Cargo.lock @@ -2,6 +2,33 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "ahash" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "version_check", +] + [[package]] name = "approx" version = "0.5.1" @@ -11,6 +38,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + [[package]] name = "atomic-polyfill" version = "0.1.11" @@ -26,7 +59,7 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ - "hermit-abi", + "hermit-abi 0.1.19", "libc", "winapi", ] @@ -37,12 +70,33 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "bitflags" version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" + [[package]] name = "bitvec" version = "1.0.1" @@ -67,6 +121,21 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +[[package]] +name = "cc" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + [[package]] name = "clap" version = "3.2.25" @@ -74,10 +143,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" dependencies = [ "atty", - "bitflags", + "bitflags 1.3.2", "clap_derive", "clap_lex", - "indexmap", + "indexmap 1.9.3", "once_cell", "strsim", "termcolor", @@ -106,6 +175,15 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "cpp_demangle" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8227005286ec39567949b33df9896bcadfa6051bccca2488129f108ca23119" +dependencies = [ + "cfg-if", +] + [[package]] name = "crc" version = "2.1.0" @@ -127,9 +205,63 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7059fff8937831a9ae6f0fe4d658ffabf58f2ca96aa9dec1c889f936f705f216" +[[package]] +name = "debugid" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef552e6f588e446098f6ba40d89ac146c8c7b64aade83c051ee00bb5d2bc18d" +dependencies = [ + "uuid", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "fastrand" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" + +[[package]] +name = "findshlibs" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40b9e59cd0f7e0806cca4be089683ecb6434e602038df21fe6bf6711b2f07f64" +dependencies = [ + "cc", + "lazy_static", + "libc", + "winapi", +] + [[package]] name = "flacenc" -version = "0.1.0" +version = "0.1.1" dependencies = [ "bitvec", "crc", @@ -146,10 +278,10 @@ dependencies = [ name = "flacenc-bin" version = "0.1.1" dependencies = [ - "bitvec", "clap", "flacenc", "hound", + "pprof", "toml", ] @@ -159,6 +291,23 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +[[package]] +name = "getrandom" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" + [[package]] name = "hash32" version = "0.2.1" @@ -174,6 +323,12 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" + [[package]] name = "heapless" version = "0.7.16" @@ -202,6 +357,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7" + [[package]] name = "hound" version = "3.5.0" @@ -215,15 +376,72 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", +] + +[[package]] +name = "inferno" +version = "0.11.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c50453ec3a6555fad17b1cd1a80d16af5bc7cb35094f64e429fd46549018c6a3" +dependencies = [ + "ahash", + "indexmap 2.0.0", + "is-terminal", + "itoa", + "log", + "num-format", + "once_cell", + "quick-xml", + "rgb", + "str_stack", +] + +[[package]] +name = "is-terminal" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +dependencies = [ + "hermit-abi 0.3.3", + "rustix", + "windows-sys", ] +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + [[package]] name = "libc" version = "0.2.148" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" +[[package]] +name = "linux-raw-sys" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a9bad9f94746442c783ca431b22403b519cd7fbeed0533fdd6328b2f2212128" + [[package]] name = "lock_api" version = "0.4.10" @@ -234,6 +452,12 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + [[package]] name = "matrixmultiply" version = "0.3.7" @@ -250,6 +474,30 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" +[[package]] +name = "memchr" +version = "2.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" + +[[package]] +name = "memmap2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +dependencies = [ + "libc", +] + +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + [[package]] name = "nalgebra" version = "0.31.4" @@ -277,6 +525,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "nix" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b" +dependencies = [ + "bitflags 1.3.2", + "cfg-if", + "libc", +] + [[package]] name = "num-complex" version = "0.4.4" @@ -286,6 +545,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-format" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" +dependencies = [ + "arrayvec", + "itoa", +] + [[package]] name = "num-integer" version = "0.1.45" @@ -316,6 +585,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "object" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.18.0" @@ -328,12 +606,58 @@ version = "6.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d5d9eb14b174ee9aa2ef96dc2b94637a2d4b6e7cb873c7e171f0c20c6cf3eac" +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + [[package]] name = "paste" version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" +[[package]] +name = "pprof" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978385d59daf9269189d052ca8a84c1acfd0715c0599a5d5188d4acc078ca46a" +dependencies = [ + "backtrace", + "cfg-if", + "findshlibs", + "inferno", + "libc", + "log", + "nix", + "once_cell", + "parking_lot", + "protobuf", + "protobuf-codegen-pure", + "smallvec", + "symbolic-demangle", + "tempfile", + "thiserror", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -367,6 +691,40 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "protobuf" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" + +[[package]] +name = "protobuf-codegen" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "033460afb75cf755fcfc16dfaed20b86468082a2ea24e05ac35ab4a099a017d6" +dependencies = [ + "protobuf", +] + +[[package]] +name = "protobuf-codegen-pure" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a29399fc94bcd3eeaa951c715f7bea69409b2445356b00519740bcd6ddd865" +dependencies = [ + "protobuf", + "protobuf-codegen", +] + +[[package]] +name = "quick-xml" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f50b1c63b38611e7d4d7f68b82d3ad0cc71a2ad2e7f61fc10f1328d917c93cd" +dependencies = [ + "memchr", +] + [[package]] name = "quote" version = "1.0.33" @@ -388,6 +746,30 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "rgb" +version = "0.8.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20ec2d3e3fc7a92ced357df9cebd5a10b6fb2aa1ee797bf7e9ce2f17dffc8f59" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustc_version" version = "0.4.0" @@ -397,6 +779,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "0.38.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "747c788e9ce8e92b12cd485c49ddf90723550b654b32508f979b71a7b1ecda4f" +dependencies = [ + "bitflags 2.4.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + [[package]] name = "safe_arch" version = "0.7.1" @@ -457,6 +852,12 @@ dependencies = [ "wide", ] +[[package]] +name = "smallvec" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" + [[package]] name = "spin" version = "0.9.8" @@ -472,12 +873,41 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "str_stack" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb" + [[package]] name = "strsim" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "symbolic-common" +version = "12.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e0e9bc48b3852f36a84f8d0da275d50cb3c2b88b59b9ec35fdd8b7fa239e37d" +dependencies = [ + "debugid", + "memmap2", + "stable_deref_trait", + "uuid", +] + +[[package]] +name = "symbolic-demangle" +version = "12.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "691e53bdc0702aba3a5abc2cffff89346fcbd4050748883c7e2f714b33a69045" +dependencies = [ + "cpp_demangle", + "rustc-demangle", + "symbolic-common", +] + [[package]] name = "syn" version = "1.0.109" @@ -506,6 +936,19 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tempfile" +version = "3.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" +dependencies = [ + "cfg-if", + "fastrand", + "redox_syscall", + "rustix", + "windows-sys", +] + [[package]] name = "termcolor" version = "1.3.0" @@ -521,6 +964,26 @@ version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" +[[package]] +name = "thiserror" +version = "1.0.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.37", +] + [[package]] name = "toml" version = "0.5.11" @@ -542,12 +1005,24 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "uuid" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79daa5ed5740825c40b389c5e50312b9c86df53fccd33f281df655642b43869d" + [[package]] name = "version_check" version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + [[package]] name = "wide" version = "0.7.11" @@ -589,6 +1064,72 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + [[package]] name = "wyz" version = "0.5.1" diff --git a/flacenc-bin/Cargo.toml b/flacenc-bin/Cargo.toml index b6f38c7..659c0dd 100644 --- a/flacenc-bin/Cargo.toml +++ b/flacenc-bin/Cargo.toml @@ -21,7 +21,6 @@ path = "src/main.rs" [dependencies] -bitvec = "1.0.0" clap = { version = "3.1.8", features = ["derive"] } flacenc = { version = "0.1.1", path = ".." } hound = "3.5.0" diff --git a/flacenc-bin/src/main.rs b/flacenc-bin/src/main.rs index ff8989f..e31de9b 100644 --- a/flacenc-bin/src/main.rs +++ b/flacenc-bin/src/main.rs @@ -51,8 +51,6 @@ use std::io::BufWriter; use std::io::Write; use std::path::Path; -use bitvec::prelude::BitVec; -use bitvec::prelude::Msb0; use clap::Parser; #[cfg(feature = "pprof")] use pprof::protos::Message; @@ -91,12 +89,16 @@ struct Args { #[allow(clippy::expect_used)] fn write_stream(stream: &Stream, file: &mut F) { eprintln!("{} bits to be written", stream.count_bits()); - let mut bv: BitVec = BitVec::with_capacity(stream.count_bits()); + // let mut bv: BitVec = BitVec::with_capacity(stream.count_bits()); + // stream.write(&mut bv).expect("Bitstream formatting failed."); + // let mut writer = BufWriter::new(file); + // writer + // .write_all(bv.as_raw_slice()) + // .expect("File write failed."); + let mut bv = flacenc::bitsink::ByteVec::new(); stream.write(&mut bv).expect("Bitstream formatting failed."); let mut writer = BufWriter::new(file); - writer - .write_all(bv.as_raw_slice()) - .expect("File write failed."); + writer.write_all(&bv.bytes).expect(""); } /// Collect iterator of `Result`s, and returns values or the first error. diff --git a/report/report.md b/report/report.md index 64d40fe..6158445 100644 --- a/report/report.md +++ b/report/report.md @@ -21,14 +21,14 @@ Sources used: wikimedia.i_love_you_california, wikimedia.winter_kiss, wikimedia. ### Average compression speed (inverse RTF) - Reference - - opt8lax: 259.8460879213409 - - opt8: 257.5709619935005 - - opt5: 492.01272615592 + - opt8lax: 257.7849467896516 + - opt8: 251.1527880915968 + - opt5: 501.03630516078925 - Ours - - default: 37.37252844587295 - - dmse: 37.371418342367136 - - bsbs: 36.8924604184309 - - mae: 37.171911348439224 + - default: 52.7637956617831 + - dmse: 52.35116821103436 + - bsbs: 52.11728073283382 + - mae: 52.33861337278744 diff --git a/src/bitsink.rs b/src/bitsink.rs index c095413..7f82670 100644 --- a/src/bitsink.rs +++ b/src/bitsink.rs @@ -25,8 +25,22 @@ use bitvec::view::BitView; pub trait BitSink: Sized { fn write_bitslice(&mut self, other: &BitSlice); + /// Puts zeros to `BitSink` until the length aligns to the byte boundaries. + /// + /// # Returns + /// + /// The number of zeros put. fn align_to_byte(&mut self) -> usize; + /// Writes bytes after alignment, and returns padded bits. + fn write_bytes_aligned(&mut self, bytes: &[u8]) -> usize { + let ret = self.align_to_byte(); + for b in bytes { + self.write_lsbs(*b, 8); + } + ret + } + // Type signature may change. Don't override. #[inline] fn write_lsbs>(&mut self, val: T, nbits: usize) { @@ -35,7 +49,8 @@ pub trait BitSink: Sized { } #[inline] - fn write_msbs(&mut self, val: T, nbits: usize) { + fn write_msbs>(&mut self, val: T, nbits: usize) { + let val: u64 = val.into(); self.write_bitslice(&val.view_bits::()[0..nbits]); } @@ -56,10 +71,12 @@ where T2: BitStore, O2: BitOrder, { + #[inline] fn write_bitslice(&mut self, other: &BitSlice) { self.extend_from_bitslice(other); } + #[inline] fn align_to_byte(&mut self) -> usize { let npad = 8 - self.len() % 8; if npad == 8 { @@ -95,6 +112,132 @@ impl<'a, L: BitSink, R: BitSink> BitSink for Tee<'a, L, R> { } } +pub struct ByteVec { + pub bytes: Vec, + pub bitlength: usize, +} + +impl Default for ByteVec { + fn default() -> Self { + Self::new() + } +} + +impl ByteVec { + pub fn new() -> Self { + Self { + bytes: vec![], + bitlength: 0usize, + } + } + + pub fn with_capacity(capacity: usize) -> Self { + Self { + bytes: Vec::with_capacity(capacity / 8 + 1), + bitlength: 0usize, + } + } + + pub fn clear(&mut self) { + self.bytes.clear(); + self.bitlength = 0; + } + + pub fn reserve(&mut self, capacity: usize) { + self.bytes.reserve(capacity / 8 + 1); + } + + #[inline] + pub fn rem_bits(&self) -> usize { + let r = self.bitlength % 8; + if r == 0 { + 0 + } else { + 8 - r + } + } + + pub fn to_debug_bitstring(&self) -> String { + let mut ret = String::new(); + for b in &self.bytes { + ret.push_str(&format!("{b:08b}_")); + } + ret.pop(); + ret + } + + #[inline] + fn write_u64_msbs(&mut self, val: u64, nbits: usize) { + let mut val: u64 = val; + let mut nbits = nbits; + let nbitlength = self.bitlength + nbits; + let r = self.rem_bits(); + + if r != 0 { + let b: u8 = ((val >> (64 - r)) & ((1 << r) - 1)) as u8; + let tail = self.bytes.len() - 1; + self.bytes[tail] |= b; + val <<= r; + nbits = if nbits > r { nbits - r } else { 0 }; + } + while nbits >= 8 { + let b: u8 = (val >> (64 - 8) & 0xFFu64) as u8; + self.bytes.push(b); + val <<= 8; + nbits -= 8; + } + if nbits > 0 { + let b: u8 = ((val >> (64 - nbits)) << (8 - nbits)) as u8; + self.bytes.push(b); + } + self.bitlength = nbitlength; + } +} + +impl BitSink for ByteVec { + #[inline] + fn write_bitslice(&mut self, other: &BitSlice) { + let mut bv: BitVec = BitVec::with_capacity(other.len()); + bv.extend_from_bitslice(other); + let mut size = bv.len(); + for elem in bv.as_raw_slice() { + self.write_msbs(*elem, std::cmp::min(64, size)); + size = if size > 64 { size - 64 } else { 0 }; + } + } + + fn align_to_byte(&mut self) -> usize { + let r = self.rem_bits(); + self.bitlength += r; + r + } + + #[inline] + fn write_bytes_aligned(&mut self, bytes: &[u8]) -> usize { + let ret = self.align_to_byte(); + self.align_to_byte(); + self.bytes.extend_from_slice(bytes); + ret + } + + #[inline] + fn write_msbs>(&mut self, val: T, nbits: usize) { + if nbits == 0 { + return; + } + let initial_shift = 64 - (std::mem::size_of::() * 8); + self.write_u64_msbs(val.into() << initial_shift, nbits); + } + + #[inline] + fn write_lsbs>(&mut self, val: T, nbits: usize) { + if nbits == 0 { + return; + } + self.write_u64_msbs(val.into() << (64 - nbits), nbits); + } +} + #[cfg(test)] mod tests { use super::*; @@ -137,4 +280,30 @@ mod tests { sink.write_twoc(-7, 4); assert_eq!(sink, bits![1, 0, 0, 1]); } + + #[test] + fn bytevec_write_msb() { + let mut bv = ByteVec::new(); + bv.write_msbs(0xFFu8, 3); + bv.write_msbs(0x0u64, 12); + bv.write_msbs(0xFFFF_FFFFu32, 9); + bv.write_msbs(0x0u16, 8); + assert_eq!( + bv.to_debug_bitstring(), + "11100000_00000001_11111111_00000000" + ); + } + + #[test] + fn bytevec_write_lsb() { + let mut bv = ByteVec::new(); + bv.write_lsbs(0xFFu8, 3); + bv.write_lsbs(0x0u64, 12); + bv.write_lsbs(0xFFFF_FFFFu32, 9); + bv.write_lsbs(0x0u16, 8); + assert_eq!( + bv.to_debug_bitstring(), + "11100000_00000001_11111111_00000000" + ); + } } diff --git a/src/component.rs b/src/component.rs index de92586..1ed7eea 100644 --- a/src/component.rs +++ b/src/component.rs @@ -14,6 +14,7 @@ //! Components to be written in the output file. +use std::cell::RefCell; use std::cmp::max; use std::cmp::min; @@ -24,6 +25,7 @@ use bitvec::prelude::Msb0; use bitvec::view::BitView; use super::bitsink::BitSink; +use super::bitsink::ByteVec; use super::constant::MAX_CHANNELS; use super::constant::MAX_LPC_ORDER; use super::error::EncodeError; @@ -248,11 +250,9 @@ impl BitRepr for MetadataBlock { fn write(&self, dest: &mut S) -> Result<(), EncodeError> { let block_type = self.block_type as u32 + if self.is_last { 0x80 } else { 0x00 }; dest.write_lsbs(block_type, 8); - let mut data_buf: BitVec = BitVec::with_capacity(self.data.count_bits()); - self.data.write(&mut data_buf)?; - let data_size: u32 = (data_buf.len() / 8) as u32; + let data_size: u32 = (self.data.count_bits() / 8) as u32; dest.write_lsbs(data_size, 24); - dest.write_bitslice(&data_buf); + self.data.write(dest)?; Ok(()) } } @@ -435,6 +435,10 @@ impl Frame { } } +thread_local! { + static FRAME_CRC_BUFFER: RefCell = RefCell::new(ByteVec::new()); +} + impl BitRepr for Frame { fn count_bits(&self) -> usize { let header = self.header.count_bits(); @@ -446,17 +450,21 @@ impl BitRepr for Frame { } fn write(&self, dest: &mut S) -> Result<(), EncodeError> { - let mut frame_buffer: BitVec = BitVec::with_capacity(self.count_bits()); - - self.header.write(&mut frame_buffer)?; - for sub in &self.subframes { - sub.write(&mut frame_buffer)?; - } - frame_buffer.align_to_byte(); + FRAME_CRC_BUFFER.with(|frame_buffer| { + let frame_buffer: &mut ByteVec = &mut frame_buffer.borrow_mut(); + frame_buffer.clear(); + frame_buffer.reserve(self.count_bits()); + + self.header.write(frame_buffer)?; + for sub in &self.subframes { + sub.write(frame_buffer)?; + } + frame_buffer.align_to_byte(); - dest.write_bitslice(&frame_buffer); - dest.write(crc::Crc::::new(&CRC_16_FLAC).checksum(frame_buffer.as_raw_slice())); - Ok(()) + dest.write_bytes_aligned(&frame_buffer.bytes); + dest.write(crc::Crc::::new(&CRC_16_FLAC).checksum(&frame_buffer.bytes)); + Ok(()) + }) } } @@ -514,13 +522,13 @@ impl BitRepr for ChannelAssignment { dest.write_lsbs(ch - 1, 4); } Self::LeftSide => { - dest.write_bitslice(bits![1, 0, 0, 0]); + dest.write_lsbs(0x8u64, 4); } Self::RightSide => { - dest.write_bitslice(bits![1, 0, 0, 1]); + dest.write_lsbs(0x9u64, 4); } Self::MidSide => { - dest.write_bitslice(bits![1, 0, 1, 0]); + dest.write_lsbs(0xAu64, 4); } } Ok(()) @@ -588,6 +596,10 @@ impl FrameHeader { } } +thread_local! { + static HEADER_CRC_BUFFER: RefCell = RefCell::new(ByteVec::new()); +} + impl BitRepr for FrameHeader { fn count_bits(&self) -> usize { let mut ret = 40; @@ -602,35 +614,37 @@ impl BitRepr for FrameHeader { } fn write(&self, dest: &mut S) -> Result<(), EncodeError> { - let mut header_buffer: BitVec = BitVec::with_capacity(128 * 8); - { - let dest = &mut header_buffer; - // sync-code + reserved 1-bit + variable-block indicator - let header_word = 0xFFF8u16 + u16::from(self.variable_block_size); - // ^ `from` converts true to 1 and false to 0. - dest.write_lsbs(header_word, 16); - - let (head, foot, footsize) = block_size_spec(self.block_size); - dest.write_lsbs(head, 4); - // sample rate must be declared in header, not here. - dest.write_bitslice(bits![0, 0, 0, 0]); - self.channel_assignment.write(dest)?; - - dest.write_lsbs(self.sample_size, 3); - dest.write_bitslice(bits![0]); // reserved - - if self.variable_block_size { - encode_to_utf8like(self.start_sample_number, dest)?; - } else { - encode_to_utf8like(u64::from(self.frame_number), dest)?; + HEADER_CRC_BUFFER.with(|header_buffer| { + { + let dest: &mut ByteVec = &mut header_buffer.borrow_mut(); + dest.clear(); + dest.reserve(self.count_bits()); + + // sync-code + reserved 1-bit + variable-block indicator + let header_word = 0xFFF8u16 + u16::from(self.variable_block_size); + // ^ `from` converts true to 1 and false to 0. + dest.write_lsbs(header_word, 16); + + let (head, foot, footsize) = block_size_spec(self.block_size); + // head + 4-bit sample rate specifier. + dest.write_lsbs(head << 4, 8); + self.channel_assignment.write(dest)?; + + // sample size specifier + 1-bit reserved (zero) + dest.write_lsbs(self.sample_size << 1, 4); + + if self.variable_block_size { + encode_to_utf8like(self.start_sample_number, dest)?; + } else { + encode_to_utf8like(u64::from(self.frame_number), dest)?; + } + dest.write_lsbs(foot, footsize); } - dest.write_lsbs(foot, footsize); - } - - dest.write_bitslice(&header_buffer); - dest.write(crc::Crc::::new(&CRC_8_FLAC).checksum(header_buffer.as_raw_slice())); - Ok(()) + dest.write_bytes_aligned(&header_buffer.borrow().bytes); + dest.write(crc::Crc::::new(&CRC_8_FLAC).checksum(&header_buffer.borrow().bytes)); + Ok(()) + }) } } diff --git a/src/test_helper.rs b/src/test_helper.rs index c61b73c..f2d0999 100644 --- a/src/test_helper.rs +++ b/src/test_helper.rs @@ -16,13 +16,12 @@ use std::collections::BTreeMap; use std::f32::consts::PI; use std::io::Write; -use bitvec::prelude::BitVec; -use bitvec::prelude::Msb0; use once_cell::sync::Lazy; use rand::distributions::Distribution; use rand::distributions::Uniform; use tempfile::NamedTempFile; +use super::bitsink::ByteVec; use super::component::BitRepr; use super::component::Stream; use super::source::PreloadedSignal; @@ -155,10 +154,16 @@ where let stream = encoder(src.clone()); let mut file = NamedTempFile::new().expect("Failed to create temp file."); + /* let mut bv: BitVec = BitVec::with_capacity(stream.count_bits()); stream.write(&mut bv).expect("Bitstream formatting failed."); file.write_all(bv.as_raw_slice()) .expect("File write failed."); + */ + + let mut bv: ByteVec = ByteVec::with_capacity(stream.count_bits()); + stream.write(&mut bv).expect("Bitstream formatting failed."); + file.write_all(&bv.bytes).expect("File write failed."); let flac_path = file.into_temp_path();