diff --git a/go.mod b/go.mod index 36f9edb..a17f3b2 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/browserutils/kooky go 1.18 require ( - github.com/Velocidex/ordereddict v0.0.0-20230909174157-2aa49cc5d11d github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89 github.com/go-ini/ini v1.67.0 github.com/go-sqlite/sqlite3 v0.0.0-20180313105335-53dd8e640ee7 @@ -14,13 +13,9 @@ require ( golang.org/x/net v0.25.0 golang.org/x/sys v0.20.0 golang.org/x/text v0.15.0 - www.velocidex.com/golang/go-ese v0.2.0 ) require ( - github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a // indirect - github.com/Velocidex/yaml/v2 v2.2.8 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gonuts/binary v0.2.0 // indirect ) diff --git a/go.sum b/go.sum index f55be11..26a8098 100644 --- a/go.sum +++ b/go.sum @@ -1,20 +1,5 @@ -github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a h1:AeXPUzhU0yhID/v5JJEIkjaE85ASe+Vh4Kuv1RSLL+4= -github.com/Velocidex/json v0.0.0-20220224052537-92f3c0326e5a/go.mod h1:ukJBuruT9b24pdgZwWDvOaCYHeS03B7oQPCUWh25bwM= -github.com/Velocidex/ordereddict v0.0.0-20220107075049-3dbe58412844/go.mod h1:Y5Tfx5SKGOzkulpqfonrdILSPIuNg+GqKE/DhVJgnpg= -github.com/Velocidex/ordereddict v0.0.0-20230909174157-2aa49cc5d11d h1:fn372EqKyazBxYUP5HPpBi3jId4MXuppEypEALGfvEk= -github.com/Velocidex/ordereddict v0.0.0-20230909174157-2aa49cc5d11d/go.mod h1:+MqO5UMBemyFSm+yRXslbpFTwPUDhFHUf7HPV92twg4= -github.com/Velocidex/yaml/v2 v2.2.8 h1:GUrSy4SBJ6RjGt43k6MeBKtw2z/27gh4A3hfFmFY3No= -github.com/Velocidex/yaml/v2 v2.2.8/go.mod h1:PlXIg/Pxmoja48C1vMHo7C5pauAZvLq/UEPOQ3DsjS4= -github.com/alecthomas/assert v1.0.0/go.mod h1:va/d2JC+M7F6s+80kl/R3G7FUiW6JzUO+hPhLyJ36ZY= -github.com/alecthomas/colour v0.1.0/go.mod h1:QO9JBoKquHd+jz9nshCh40fOfO+JzsoXy8qTHF68zU0= -github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8= -github.com/alecthomas/repr v0.1.1 h1:87P60cSmareLAxMc4Hro0r2RBY4ROm0dYwkJNpS4pPs= -github.com/alecthomas/repr v0.1.1/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89 h1:2pkAuIM8OF1fy4ToFpMnI4oE+VeUNRbGrpSLKshK0oQ= github.com/bobesa/go-domain-util v0.0.0-20190911083921-4033b5f7dd89/go.mod h1:/09nEjna1UMoasyyQDhOrIn8hi2v2kiJglPWed1idck= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= @@ -27,28 +12,11 @@ github.com/gonuts/binary v0.2.0 h1:caITwMWAoQWlL0RNvv2lTU/AHqAJlVuu6nZmNgfbKW4= github.com/gonuts/binary v0.2.0/go.mod h1:kM+CtBrCGDSKdv8WXTuCUsw+loiy8f/QEI8YCCC0M/E= github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6 h1:IsMZxCuZqKuao2vNdfD82fjjgPLfyHLpR41Z88viRWs= github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeWNIJaW+O5xpRQbPp0Ybqu1vJd/pm7s2F473HRrkw= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/sebdah/goldie v1.0.0/go.mod h1:jXP4hmWywNEwZzhMuv2ccnqTSFpuq8iyQhtQdkkZBH4= -github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/zalando/go-keyring v0.2.4 h1:wi2xxTqdiwMKbM6TWwi+uJCG/Tum2UV0jqaQhCa9/68= github.com/zalando/go-keyring v0.2.4/go.mod h1:HL4k+OXQfJUWaMnqyuSOc0drfGPX2b51Du6K+MRgZMk= @@ -57,22 +25,9 @@ golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -www.velocidex.com/golang/go-ese v0.2.0 h1:8/hzEMupfqEF0oMi1/EzsMN1xLN0GBFcB3GqxqRnb9s= -www.velocidex.com/golang/go-ese v0.2.0/go.mod h1:6fC9T6UGLbM7icuA0ugomU5HbFC5XA5I30zlWtZT8YE= diff --git a/internal/eseparser/LICENSE b/internal/eseparser/LICENSE new file mode 100644 index 0000000..269653d --- /dev/null +++ b/internal/eseparser/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2018 velocidex + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/internal/eseparser/README.md b/internal/eseparser/README.md new file mode 100644 index 0000000..25cc1e9 --- /dev/null +++ b/internal/eseparser/README.md @@ -0,0 +1,172 @@ +# A Go based ESE parser. + +The Extensible Storage Engine (ESE) Database File is commonly used +within Windows to store various application specific information. It +is the Microsoft analogue to sqlite - so just like sqlite is used to +store chrome history, ESE is used to store Internet Explorer history. + +In essence it is a flat file database. This project is a library to +help read such a file. The following description is a high lieve +account of the main feautures of the file format and how to access +these using the library. + +## File format overview + +The file consists of pages. The page size can vary but it is specified +in the file header. + +The file may contain multiple objects (tables) stored within +pages. The pages form a B tree where data is stored in the actual leaf +pages. As the file grows the tree canbe extended by inserting pages +into it. + +Data is stored inside each page in a `Tag`. Tags are just a series of +data blobs stored in each page. + +The WalkPages() function can be used to produce all the tags starting +at a root page. The function walks the B+ tree automatically and +parses out each tag. The callback is called for each tag - if the +callback returns an error the walk is stopped and the error is relayed +to the WalkPages() caller. + +## The page + +Each page contains a series of tags. You can see help about a specific +page using the `page` command: + +``` +$ eseparser page WebCacheV01.dat 4 +Page 4: struct PageHeader @ 0x28000: + LastModified: { + struct DBTime @ 0x28008: + Hours: 0xfbf4 + Min: 0x0 + Sec: 0x0 + } + PreviousPageNumber: 0x0 + NextPageNumber: 0x0 + FatherPage: 0x2 + AvailableDataSize: 0x7f3b + AvailableDataOffset: 0x5d + AvailablePageTag: 0x6 + Flags: 108549 (Parent,Root) + +Tag 0 @ 0x2fffc offset 0x0 length 0x10 +Tag 1 @ 0x2fff8 offset 0x16 length 0x13 +Tag 2 @ 0x2fff4 offset 0x29 length 0xe +Tag 3 @ 0x2fff0 offset 0x37 length 0x13 +Tag 4 @ 0x2ffec offset 0x4a length 0x13 +Tag 5 @ 0x2ffe8 offset 0x10 length 0x6 +struct ESENT_ROOT_HEADER @ 0x0: + InitialNumberOfPages: 0x14 + ParentFDP: 0x1 + ExtentSpace: Multiple (1) + SpaceTreePageNumber: 0x5 + +struct ESENT_BRANCH_ENTRY @ 0x0: + LocalPageKeySize: 0xd + ChildPageNumber: 0xd +struct ESENT_BRANCH_ENTRY @ 0x0: + LocalPageKeySize: 0x8 + ChildPageNumber: 0xe +struct ESENT_BRANCH_ENTRY @ 0x0: + LocalPageKeySize: 0xd + ChildPageNumber: 0x13 +struct ESENT_BRANCH_ENTRY @ 0x0: + LocalPageKeySize: 0xd + ChildPageNumber: 0x14 +struct ESENT_BRANCH_ENTRY @ 0x0: + LocalPageKeySize: 0x0 + ChildPageNumber: 0x16 +``` + +The example above shows a root page (4) containing 5 branch nodes. + +## The catalog + +The ESE file contains a catalog starting from page 4. The catalog +defines all the tables, their columns and types stat are stored in the +database. + +You can see the catalong by runing the `catalog` command: + +``` +$ eseparser catalog /shared/WebCacheV01.dat +[MSysObjects] (FDP 0x4): + Columns + 0 ObjidTable Signed long + 1 Type Signed short + 2 Id Signed long + 3 ColtypOrPgnoFDP Signed long + 4 SpaceUsage Signed long + 5 Flags Signed long + 6 PagesOrLocale Signed long + 7 RootFlag Boolean + 8 RecordOffset Signed short +``` + +The first table in the catalog called `MSysObjects` is really a +database table containing a description of all the tables in the file. + +## Tables + +Ultimately the ESE format is a database storage engine and it stores +rows in tables. Each table is stored inside the B tree rooted by the +DFP ID shown in the catalog. Each row is stored inside a tag (inside +one of the pages within the tree). + +There are three types of columns: + +- Fixed size (e.g. integers) have a known size. +- Variable size (e.g. Strings) have a variable size. +- Tagged data - these columns are often null and therefore may not be + present. The database stores these with their column ID as a map. + +Therefore within the tag for each column, there are three distinct +storage areas. You can see how each record is parsed using the --debug flag: + +``` +$ eseparser dump WebCacheV01.dat MSysObjects --debug +Walking page 4 +Got 6 values for page 4 +Walking page 13 +Got 404 values for page 13 +Processing row in Tag @ 491512 0xd (0x37)([]uint8) (len=55 cap=55) { + 00000000 07 00 06 00 01 7f 80 00 00 02 08 80 20 00 02 00 |............ ...| + 00000010 00 00 01 00 02 00 00 00 04 00 00 00 50 00 00 00 |............P...| + 00000020 00 00 00 c0 14 00 00 00 ff 00 0b 00 4d 53 79 73 |............MSys| + 00000030 4f 62 6a 65 63 74 73 |Objects| +} +struct ESENT_LEAF_ENTRY @ 0x2: + CommonPageKeySize: 0x7 + LocalPageKeySize: 0x6 + +struct ESENT_DATA_DEFINITION_HEADER @ 0xa: + LastFixedType: 0x8 + LastVariableDataType: 0x80 + VariableSizeOffset: 0x20 + +Column ObjidTable Identifier 1 Type Signed long +Consume 0x4 bytes of FIXED space from 0xe +Column Type Identifier 2 Type Signed short +Consume 0x2 bytes of FIXED space from 0x12 +Column Id Identifier 3 Type Signed long +... +{"ObjidTable":2,"Type":1,"Id":2,"ColtypOrPgnoFDP":4,"SpaceUsage":80,"Flags":-1073741824,"PagesOrLocale":20,"RootFlag":true,"Name":"MSysObjects"} +``` + +If you just want to dump out the columns omit the `--debug` flag: +``` +$ ./eseparser dump /shared/WebCacheV01.dat HstsEntryEx_46 +{"EntryId":1,"MinimizedRDomainHash":0,"MinimizedRDomainLength":8,"IncludeSubdomains":177,"Expires":9223372036854775807,"LastTimeUsed":9223372036854775807,"RDomain":":version"} +{"EntryId":2,"MinimizedRDomainHash":1536723792475384667,"MinimizedRDomainLength":10,"IncludeSubdomains":1,"Expires":132508317752329580,"LastTimeUsed":132192957752329580,"RDomain":"com.office.www"} +``` + + + +References: + * https://github.com/SecureAuthCorp/impacket.git + * https://blogs.technet.microsoft.com/askds/2017/12/04/ese-deep-dive-part-1-the-anatomy-of-an-ese-database/ + * https://docs.microsoft.com/en-us/previous-versions/windows/it-pro/windows-server-2012-R2-and-2012/hh875546(v=ws.11)?redirectedfrom=MSDN + * http://hh.diva-portal.org/smash/get/diva2:635743/FULLTEXT02.pdf + * https://github.com/libyal/libesedb/blob/master/documentation/Extensible%20Storage%20Engine%20(ESE)%20Database%20File%20(EDB)%20format.asciidoc diff --git a/internal/eseparser/catalog.go b/internal/eseparser/catalog.go new file mode 100644 index 0000000..9efff11 --- /dev/null +++ b/internal/eseparser/catalog.go @@ -0,0 +1,756 @@ +// Parser based on https://github.com/SecureAuthCorp/impacket.git + +package eseparser + +import ( + "encoding/hex" + "errors" + "fmt" + "math" + "sort" + "time" + + "github.com/browserutils/kooky/internal/eseparser/ordereddict" +) + +const ( + CATALOG_PAGE_NUMBER = 4 + + // https://github.com/microsoft/Extensible-Storage-Engine/blob/933dc839b5a97b9a5b3e04824bdd456daf75a57d/dev/ese/src/inc/node.hxx#L226 + fNDCompressed = 4 << 13 +) + +// Store a simple struct of column spec for speed. +type ColumnSpec struct { + FDPId uint32 + Name string + Identifier uint32 + Type string + Flags uint32 + SpaceUsage int64 +} + +type Table struct { + ctx *ESEContext + Header *CATALOG_TYPE_TABLE + FatherDataPageNumber uint32 + Name string + Columns []*ColumnSpec + Indexes *ordereddict.Dict + LongValueLookup LongValueLookup +} + +// The tag contains a single row. +// 00000000 09 00 7f 80 00 00 00 00 00 00 01 06 7f 2d 00 01 |.............-..| +// 00000010 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 08 |................| +// 00000020 00 00 00 b1 00 00 00 ff ff ff ff ff ff ff 7f ff |................| +// 00000030 ff ff ff ff ff ff 7f c0 00 01 04 00 01 3a 00 76 |.............:.v| +// 00000040 00 65 00 72 00 73 00 69 00 6f 00 6e 00 00 00 |.e.r.s.i.o.n...| + +// The key consumes the first 11 bytes: +// struct ESENT_LEAF_ENTRY @ 0x0: +// CommonPageKeySize: 0x0 +// LocalPageKeySize: 0x9 + +// Followed by a data definition header: +//struct ESENT_DATA_DEFINITION_HEADER @ 0xb: +// LastFixedType: 0x6 +// LastVariableDataType: 0x7f +// VariableSizeOffset: 0x2d + +// Column IDs below LastFixedSize will be stored in the fixed size +// portion. Column id below LastVariableDataType will be stored in the +// variable data section and higher than LastVariableDataType will be +// tagged. + +// The fixed section starts immediately after the +// ESENT_DATA_DEFINITION_HEADER (offset 0xb + 4 = 0xf) + +// Then the following columns consume their types: +// Column EntryId Identifier 1 Type Long long +// Column MinimizedRDomainHash Identifier 2 Type Long long +// Column MinimizedRDomainLength Identifier 3 Type Unsigned long +// Column IncludeSubdomains Identifier 4 Type Unsigned long +// Column Expires Identifier 5 Type Long long +// Column LastTimeUsed Identifier 6 Type Long long + +// In the above example we have no variable sized columns, so we go +// straight to the tagged values: + +// Then the tagged values are consumed +// Column RDomain Identifier 256 Type Long Text + +func (self *Table) tagToRecord(value *Value, header *PageHeader) *ordereddict.Dict { + result := ordereddict.NewDict() + + var taggedItems map[uint32][]byte + + reader := value.Reader() + + tag := NewESENT_LEAF_ENTRY(self.ctx, value) + dd_header := self.ctx.Profile.ESENT_DATA_DEFINITION_HEADER(reader, tag.EntryData()) + + // Start to parse immediately after the dd_header + offset := dd_header.Offset + int64(dd_header.Size()) + + if Debug { + fmt.Println(dd_header.DebugString()) + } + + prevItemLen := int64(0) + variableSizeOffset := dd_header.Offset + int64(dd_header.VariableSizeOffset()) + variableDataBytesProcessed := int64(dd_header.LastVariableDataType()-127) * 2 + last_fixed_type := uint32(dd_header.LastFixedType()) + last_variable_data_type := uint32(dd_header.LastVariableDataType()) + + // Iterate over the column definitions and decode each + // identifier according to where it comes from. + for _, column := range self.Columns { + if Debug { + fmt.Printf("Column %v Identifier %v Type %v\n", column.Name, + column.Identifier, column.Type) + } + + // Column is stored in the fixed section. + if column.Identifier <= last_fixed_type { + switch column.Type { + case "Boolean": + if column.SpaceUsage == 1 { + result.Set(column.Name, ParseUint8(reader, offset) > 0) + } + + case "Signed byte": + if column.SpaceUsage == 1 { + result.Set(column.Name, ParseUint8(reader, offset)) + } + + case "Signed short": + if column.SpaceUsage == 2 { + result.Set(column.Name, ParseInt16(reader, offset)) + } + + case "Unsigned short": + if column.SpaceUsage == 2 { + result.Set(column.Name, ParseUint16(reader, offset)) + } + + case "Signed long": + if column.SpaceUsage == 4 { + result.Set(column.Name, ParseInt32(reader, offset)) + } + + case "Unsigned long": + if column.SpaceUsage == 4 { + result.Set(column.Name, ParseUint32(reader, offset)) + } + + case "Single precision FP": + if column.SpaceUsage == 4 { + result.Set(column.Name, math.Float32frombits( + ParseUint32(reader, offset))) + } + + case "Double precision FP": + if column.SpaceUsage == 8 { + result.Set(column.Name, math.Float64frombits( + ParseUint64(reader, offset))) + } + + case "DateTime": + if column.SpaceUsage == 8 { + switch column.Flags { + case 1: + // A more modern way of encoding + result.Set(column.Name, WinFileTime64(reader, offset)) + + case 0: + // Some hair brained time serialization method + // https://docs.microsoft.com/en-us/windows/win32/extensible-storage-engine/jet-coltyp + + value_int := ParseUint64(reader, offset) + days_since_1900 := math.Float64frombits(value_int) + + // In python time.mktime((1900,1,1,0,0,0,0,365,0)) + + // From https://docs.microsoft.com/en-us/windows/win32/api/oleauto/nf-oleauto-varianttimetosystemtime + // A variant time is stored as an 8-byte real + // value (double), representing a date between + // January 1, 100 and December 31, 9999, + // inclusive. The value 2.0 represents January + // 1, 1900; 3.0 represents January 2, 1900, + // and so on. Adding 1 to the value increments + // the date by a day. The fractional part of + // the value represents the time of + // day. Therefore, 2.5 represents noon on + // January 1, 1900; 3.25 represents 6:00 + // A.M. on January 2, 1900, and so + // on. Negative numbers represent the dates + // prior to December 30, 1899. + result.Set(column.Name, + time.Unix(int64( + days_since_1900*24*60*60)+ + + // Number of Sec between 1900 and 1970 + -2208988800- + + // Jan 1 1900 is actually value of 2 + // days so correct for it here. + 2*24*60*60, 0).UTC()) + + default: + // We have no idea + result.Set(column.Name, ParseUint64(reader, offset)) + } + } + + case "Long Text", "Text": + if column.SpaceUsage < 2000 { + data := make([]byte, column.SpaceUsage) + n, err := reader.ReadAt(data, offset) + + if err == nil { + + // Flags can be given as the first char or in the + // column definition. + result.Set(column.Name, ParseLongText(data[:n], column.Flags)) + } + } + + case "Long long", "Currency": + if column.SpaceUsage == 8 { + result.Set(column.Name, ParseUint64(reader, offset)) + } + + case "GUID": + if column.SpaceUsage == 16 { + result.Set(column.Name, + self.Header.Profile.GUID(reader, offset).AsString()) + } + + case "Binary": + if column.SpaceUsage < 1024 { + data := make([]byte, column.SpaceUsage) + n, err := reader.ReadAt(data, offset) + if err == nil { + result.Set(column.Name, data[:n]) + } + } + default: + fmt.Printf("Can not handle Column %v fixed data %v\n", + column.Name, column) + } + + if Debug { + fmt.Printf("Consumed %#x bytes of FIXED space from %#x\n", + column.SpaceUsage, offset) + } + + // Move our offset along + offset += column.SpaceUsage + + // Identifier is stored in the variable section + } else if 127 < column.Identifier && + column.Identifier <= last_variable_data_type { + + // Variable data type + index := int64(column.Identifier) - 127 - 1 + itemLen := int64(ParseUint16(reader, variableSizeOffset+index*2)) + + if itemLen&0x8000 > 0 { + // Empty Item + itemLen = prevItemLen + result.Set(column.Name, nil) + + } else { + switch column.Type { + case "Binary": + result.Set(column.Name, hex.EncodeToString([]byte( + ParseString(reader, + variableSizeOffset+variableDataBytesProcessed, + itemLen-prevItemLen)))) + + case "Text": + result.Set(column.Name, ParseText(reader, + variableSizeOffset+variableDataBytesProcessed, + itemLen-prevItemLen, column.Flags)) + + default: + fmt.Printf("Can not handle Column %v variable data %v\n", + column.Name, column) + } + } + + if Debug { + fmt.Printf("Consumed %#x bytes of VARIABLE space from %#x\n", + itemLen-prevItemLen, variableDataBytesProcessed) + } + + variableDataBytesProcessed += itemLen - prevItemLen + prevItemLen = itemLen + + // Tagged values + } else if column.Identifier > 255 { + if taggedItems == nil { + if Debug { + fmt.Printf("Slice is %#x-%#x %x\n", + variableDataBytesProcessed+variableSizeOffset, + value.BufferSize, + getValueSlice(value, uint64(variableDataBytesProcessed+ + variableSizeOffset), uint64(value.BufferSize))) + } + taggedItems = ParseTaggedValues( + self.ctx, getValueSlice(value, + uint64(variableDataBytesProcessed+variableSizeOffset), + uint64(value.BufferSize))) + } + + buf, pres := taggedItems[column.Identifier] + if pres { + reader := &BufferReaderAt{buf} + switch column.Type { + case "Binary": + result.Set(column.Name, hex.EncodeToString(buf)) + + case "Long Binary": + // If the buf is key size (4 or 8 bytes) then we + // can look it up in the LV cache. Otherwise it is + // stored literally. + if len(buf) == 4 || len(buf) == 8 { + data, pres := self.LongValueLookup.GetLid(buf) + if pres { + buf = data + } + } + + result.Set(column.Name, hex.EncodeToString(buf)) + + case "Long Text": + // If the buf is key size (4 or 8 bytes) then we + // can look it up in the LV cache. Otherwise it is + // stored literally. + if len(buf) == 4 || len(buf) == 8 { + data, pres := self.LongValueLookup.GetLid(buf) + if pres { + buf = data + } + } + + // Flags can be given as the first char or in the + // column definition. + result.Set(column.Name, ParseLongText(buf, column.Flags)) + + case "Boolean": + if column.SpaceUsage == 1 { + result.Set(column.Name, ParseUint8(reader, 0) > 0) + } + + case "Signed byte": + if column.SpaceUsage == 1 { + result.Set(column.Name, ParseUint8(reader, 0)) + } + + case "Signed short": + if column.SpaceUsage == 2 { + result.Set(column.Name, ParseInt16(reader, 0)) + } + + case "Unsigned short": + if column.SpaceUsage == 2 { + result.Set(column.Name, ParseUint16(reader, 0)) + } + + case "Signed long": + if column.SpaceUsage == 4 { + result.Set(column.Name, ParseInt32(reader, 0)) + } + + case "Unsigned long": + if column.SpaceUsage == 4 { + result.Set(column.Name, ParseUint32(reader, 0)) + } + + case "Single precision FP": + if column.SpaceUsage == 4 { + result.Set(column.Name, math.Float32frombits( + ParseUint32(reader, 0))) + } + + case "Double precision FP": + if column.SpaceUsage == 8 { + result.Set(column.Name, math.Float64frombits( + ParseUint64(reader, 0))) + } + + case "DateTime": + if column.SpaceUsage == 8 { + switch column.Flags { + case 1: + // A more modern way of encoding + result.Set(column.Name, WinFileTime64(reader, 0)) + + case 0: + // Some hair brained time serialization method + // https://docs.microsoft.com/en-us/windows/win32/extensible-storage-engine/jet-coltyp + + value_int := ParseUint64(reader, 0) + days_since_1900 := math.Float64frombits(value_int) + + // In python time.mktime((1900,1,1,0,0,0,0,365,0)) + result.Set(column.Name, + time.Unix(int64(days_since_1900*24*60*60)+ + -2208988800, 0).UTC()) + + default: + // We have no idea + result.Set(column.Name, ParseUint64(reader, 0)) + } + } + + case "Long long", "Currency": + if column.SpaceUsage == 8 { + result.Set(column.Name, ParseUint64(reader, 0)) + } + + case "GUID": + if column.SpaceUsage == 16 { + result.Set(column.Name, + self.Header.Profile.GUID(reader, 0).AsString()) + } + + default: + if Debug { + fmt.Printf("Can not handle Column %v tagged data %v\n", + column.Name, column) + } + } + } + } + } + + return result +} + +func (self *RecordTag) FlagSkip() uint64 { + return 1 +} + +func getValueSlice(value *Value, start, end uint64) []byte { + if end < start { + return nil + } + + length := end - start + if length > 1*1024*1024 { + return nil + } + + buffer := make([]byte, length) + value.reader.ReadAt(buffer, value.BufferOffset+int64(start)) + + return buffer +} + +// working slice to reassemble data +type tagBuffer struct { + identifier uint32 + start, length uint64 + flags uint64 +} + +/* +Tagged values are used to store sparse values. + +They consist of an array of RecordTag, each RecordTag has an +Identifier and an offset to the start of its data. The length of the +data in each record is determine by the start of the next record. + +Example: + +00000050 00 01 0c 40 a4 01 21 00 a5 01 23 00 01 6c 00 61 |...@..!...#..l.a| +00000060 00 62 00 5c 00 64 00 63 00 2d 00 31 00 24 00 00 |.b.\.d.c.-.1.$..| +00000070 00 3d 00 f9 00 |.=...| + +Slice is 0x50-0x75 00010c40a4012100a5012300016c00610062005c00640063002d003100240000003d00f900 +Consumed 0x15 bytes of TAGGED space from 0xc to 0x21 for tag 0x100 +Consumed 0x2 bytes of TAGGED space from 0x21 to 0x23 for tag 0x1a4 +Consumed 0x2 bytes of TAGGED space from 0x23 to 0x25 for tag 0x1a5 +*/ +func ParseTaggedValues(ctx *ESEContext, buffer []byte) map[uint32][]byte { + result := make(map[uint32][]byte) + + if len(buffer) < 2 { + return result + } + + reader := &BufferReaderAt{buffer} + first_record := ctx.Profile.RecordTag(reader, 0) + tags := []tagBuffer{} + + // Tags go from 0 to the start of the first tag's data + for offset := int64(0); offset < int64(first_record.DataOffset()); offset += 4 { + record_tag := ctx.Profile.RecordTag(reader, offset) + if Debug { + fmt.Printf("RecordTag %v\n", record_tag.DebugString()) + } + tags = append(tags, tagBuffer{ + identifier: uint32(record_tag.Identifier()), + start: record_tag.DataOffset(), + flags: record_tag.Flags(), + }) + } + + // Now build a map from identifier to buffer. + for idx, tag := range tags { + // The last tag goes until the end of the buffer + end := uint64(len(buffer)) + start := tag.start + if idx < len(tags)-1 { + end = tags[idx+1].start + } + + if tag.flags > 0 { + start += 1 + } + + if start > uint64(len(buffer)) { + start = uint64(len(buffer)) + } + + if end > uint64(len(buffer)) { + end = uint64(len(buffer)) + } + + if end < start { + end = start + } + + result[tag.identifier] = buffer[start:end] + if Debug { + fmt.Printf("Consumed %#x bytes of TAGGED space from %#x to %#x for tag %#x\n", + end-start, start, end, tag.identifier) + } + } + return result +} + +// DumpTable extracts all rows in the named table and passes them into +// the callback. The callback may cancel the walk at any time by +// returning an error which is passed to our caller. +func (self *Catalog) DumpTable(name string, cb func(row *ordereddict.Dict) error) error { + table_any, pres := self.Tables.Get(name) + if !pres { + return errors.New("Table not found") + } + + table := table_any.(*Table) + err := WalkPages(self.ctx, int64(table.FatherDataPageNumber), + func(header *PageHeader, id int64, value *Value) error { + // Each tag stores a single row - all the + // columns in the row are encoded in this tag. + row := table.tagToRecord(value, header) + if len(row.Keys()) == 0 { + return nil + } + return cb(row) + }) + if err != nil { + return err + } + return nil +} + +// Catalog represents the database's catalog. +type Catalog struct { + ctx *ESEContext + + Tables *ordereddict.Dict + + currentTable *Table +} + +func parseItemName(dd_header *ESENT_DATA_DEFINITION_HEADER) string { + last_variable_data_type := int64(dd_header.LastVariableDataType()) + numEntries := last_variable_data_type + + if last_variable_data_type > 127 { + numEntries = last_variable_data_type - 127 + } + + itemLen := ParseUint16(dd_header.Reader, + dd_header.Offset+int64(dd_header.VariableSizeOffset())) + + return ParseString(dd_header.Reader, + dd_header.Offset+int64(dd_header.VariableSizeOffset())+ + 2*numEntries, int64(itemLen)) +} + +// Walking over each LINE in the catalog tree, we parse the data +// definitions. +func (self *Catalog) __addItem(header *PageHeader, id int64, value *Value) error { + leaf_entry := NewESENT_LEAF_ENTRY(self.ctx, value) + dd_header := self.ctx.Profile.ESENT_DATA_DEFINITION_HEADER( + leaf_entry.Reader, leaf_entry.EntryData()) + + itemName := parseItemName(dd_header) + + // Catalog follows the dd header + catalog := self.ctx.Profile.ESENT_CATALOG_DATA_DEFINITION_ENTRY(dd_header.Reader, + dd_header.Offset+int64(dd_header.Size())) + + switch catalog.Type().Name { + case "CATALOG_TYPE_TABLE": + table := &Table{ + ctx: self.ctx, + Header: catalog.Table(), + Name: itemName, + FatherDataPageNumber: catalog.Table().FatherDataPageNumber(), + Indexes: ordereddict.NewDict(), + LongValueLookup: NewLongValueLookup(), + } + self.currentTable = table + self.Tables.Set(itemName, table) + + case "CATALOG_TYPE_COLUMN": + if self.currentTable == nil { + return errors.New("Internal Error: No existing table when adding column") + } + column := catalog.Column() + + self.currentTable.Columns = append(self.currentTable.Columns, &ColumnSpec{ + Name: itemName, + FDPId: catalog.FDPId(), + Identifier: catalog.Identifier(), + Type: column.ColumnType().Name, + Flags: column.ColumnFlags(), + SpaceUsage: int64(column.SpaceUsage()), + }) + + case "CATALOG_TYPE_INDEX": + if self.currentTable == nil { + return errors.New("Internal Error: No existing table when adding index") + } + + self.currentTable.Indexes.Set(itemName, catalog) + + case "CATALOG_TYPE_LONG_VALUE": + if Debug { + fmt.Printf("Catalog name %v for table %v\n", itemName, self.currentTable.Name) + } + lv := catalog.LongValue() + + WalkPages(self.ctx, int64(lv.FatherDataPageNumber()), + func(header *PageHeader, id int64, value *Value) error { + // Ignore tags that are too small to contain a key + if value.BufferSize < 8 { + return nil + } + + lv := self.ctx.Profile.LVKEY_BUFFER(value.reader, value.BufferOffset) + key := lv.ParseKey(self.ctx, header, value) + + long_value := &LongValue{ + Value: value, + header: header, + Key: key, + } + + self.currentTable.LongValueLookup[key.Key()] = long_value + + if Debug { + size := int(value.Tag._ValueSize()) + if size > 100 { + size = 100 + } + buffer := make([]byte, size) + value.Reader().ReadAt(buffer, 0) + + lv_buffer := long_value.Buffer() + if len(lv_buffer) > 100 { + lv_buffer = lv_buffer[:100] + } + + fmt.Printf("------\nPage header %v\nID %v Tag %v\nPageID %v Flags %v\nKey %v \nLVBuffer %02x\nBuffer %02x \nTagLookup %v\n", + DebugPageHeader(self.ctx, header), id, + DebugTag(self.ctx, value.Tag, header), + value.PageID, + value.Flags, + long_value.Key.DebugString(), + lv_buffer, buffer, + len(self.currentTable.LongValueLookup)) + } + return nil + }) + } + + return nil +} + +type DumpOptions struct { + LongValueTables bool + Indexes bool + Tables bool +} + +func (self *Catalog) Dump(options DumpOptions) string { + result := "" + + for _, name := range self.Tables.Keys() { + table_any, _ := self.Tables.Get(name) + table := table_any.(*Table) + + space := " " + result += fmt.Sprintf("[%v] (FDP %#x):\n%sColumns\n", table.Name, + table.FatherDataPageNumber, space) + for idx, column := range table.Columns { + result += fmt.Sprintf("%s%s%-5d%-30v%-15vFlags %v\n", space, space, idx, + column.Name, column.Type, column.Flags) + } + + if options.Indexes { + result += fmt.Sprintf("%sIndexes\n", space) + for _, index := range table.Indexes.Keys() { + result += fmt.Sprintf("%s%s%v:\n", space, space, index) + } + result += "\n" + } + + if options.LongValueTables && len(table.LongValueLookup) > 0 { + result += fmt.Sprintf("%sLongValues\n", space) + values := []*LongValue{} + for _, lv := range table.LongValueLookup { + values = append(values, lv) + } + + sort.Slice(values, func(i, j int) bool { + return values[i].Key.Key() < values[j].Key.Key() + }) + + for _, lv := range values { + buffer := lv.Buffer() + size := len(buffer) + if size > 100 { + buffer = buffer[:100] + } + result += fmt.Sprintf("%s%s%02x: \"%02x\"\n", + space, space, + lv.Key.Key(), + //lv.Key.DebugString(), + buffer) + } + result += "\n" + } + + } + + return result +} + +func ReadCatalog(ctx *ESEContext) (*Catalog, error) { + result := &Catalog{ctx: ctx, Tables: ordereddict.NewDict()} + + err := WalkPages(ctx, CATALOG_PAGE_NUMBER, result.__addItem) + if err != nil { + return nil, err + } + return result, nil +} diff --git a/internal/eseparser/compression.go b/internal/eseparser/compression.go new file mode 100644 index 0000000..0f4bb8a --- /dev/null +++ b/internal/eseparser/compression.go @@ -0,0 +1,111 @@ +package eseparser + +import ( + "encoding/binary" + "fmt" + "io" + "strings" +) + +const ( + // Flags on the record header + COMPRESSION = 0x02 + INLINE_STRING = 0x01 + INLINE_STRING_2 = 0x08 + LZMA_COMPRESSION = 0x18 // Not supported +) + +func Decompress7BitCompression(buf []byte) string { + result := make([]byte, 0, (len(buf)+5)*8/7) + + value_16bit := uint16(0) + bit_index := 0 + + for i := 1; i < len(buf); i++ { + slice := buf[i : i+1] + if i+1 < len(buf) { + slice = append(slice, buf[i+1]) + } + + for len(slice) < 2 { + slice = append(slice, 0) + } + + value_16bit |= binary.LittleEndian.Uint16(slice) << bit_index + result = append(result, byte(value_16bit&0x7f)) + + value_16bit >>= 7 + bit_index++ + + if bit_index == 7 { + result = append(result, byte(value_16bit&0x7f)) + value_16bit >>= 7 + bit_index = 0 + } + } + + return strings.Split(string(result), "\x00")[0] +} + +func ParseLongText(buf []byte, flag uint32) string { + if len(buf) < 2 { + return "" + } + + // fmt.Printf("Column Flags %v\n", flag) + leading_byte := buf[0] + if leading_byte != 0 && leading_byte != 1 && leading_byte != 8 && + leading_byte != 3 && leading_byte != 0x18 { + return strings.Split( + UTF16BytesToUTF8(buf, binary.LittleEndian), "\x00")[0] + + } + // fmt.Printf("Inline Flags %v\n", flag) + + // Lzxpress compression - not supported right now. + if leading_byte == 0x18 { + fmt.Printf("LZXPRESS compression not supported currently\n") + return strings.Split(string(buf), "\x00")[0] + } + + // The following is either 7 bit compressed or utf16 encoded. Its + // hard to figure out which it is though because there is no + // consistency in the flags. We do our best to guess!! + var result string + if len(buf) >= 3 && buf[2] == 0 { + // Probably UTF16 encoded + result = UTF16BytesToUTF8(buf[1:], binary.LittleEndian) + + } else { + // Probably 7bit compressed + result = Decompress7BitCompression(buf[1:]) + } + + //fmt.Printf("returned %v\n", result) + return strings.Split(result, "\x00")[0] +} + +func ParseText(reader io.ReaderAt, offset int64, len int64, flags uint32) string { + if len < 0 { + return "" + + } + if len > 1024*10 { + len = 1024 * 10 + } + + data := make([]byte, len) + n, err := reader.ReadAt(data, offset) + if err != nil { + return "" + } + data = data[:n] + + var str string + if flags == 1 { + str = string(data[:n]) + } else { + str = UTF16BytesToUTF8(data, binary.LittleEndian) + } + return strings.Split(str, "\x00")[0] +} diff --git a/internal/eseparser/context.go b/internal/eseparser/context.go new file mode 100644 index 0000000..9f4bf62 --- /dev/null +++ b/internal/eseparser/context.go @@ -0,0 +1,51 @@ +package eseparser + +import ( + "errors" + "fmt" + "io" +) + +type ESEContext struct { + Reader io.ReaderAt + Profile *ESEProfile + PageSize int64 + Header *FileHeader + Version uint32 + Revision uint32 +} + +func NewESEContext(reader io.ReaderAt) (*ESEContext, error) { + result := &ESEContext{ + Profile: NewESEProfile(), + Reader: reader, + } + + result.Header = result.Profile.FileHeader(reader, 0) + if result.Header.Magic() != 0x89abcdef { + return nil, errors.New(fmt.Sprintf( + "Unsupported ESE file: Magic is %x should be 0x89abcdef", + result.Header.Magic())) + } + + result.PageSize = int64(result.Header.PageSize()) + switch result.PageSize { + case 0x1000, 0x2000, 0x4000, 0x8000: + default: + return nil, errors.New(fmt.Sprintf( + "Unsupported page size %x", result.PageSize)) + } + + result.Version = result.Header.FormatVersion() + result.Revision = result.Header.FormatRevision() + return result, nil +} + +func (self *ESEContext) GetPage(id int64) *PageHeader { + // First file page is file header, second page is backup of file + // header. + return &PageHeader{ + PageHeader_: self.Profile.PageHeader_( + self.Reader, (id+1)*self.PageSize), + } +} diff --git a/internal/eseparser/conversion.spec.yaml b/internal/eseparser/conversion.spec.yaml new file mode 100644 index 0000000..d1d40e7 --- /dev/null +++ b/internal/eseparser/conversion.spec.yaml @@ -0,0 +1,43 @@ +Module: parser +Profile: ESEProfile +Filename: ese_profile.json +GenerateDebugString: true +Structs: + # https://github.com/microsoft/Extensible-Storage-Engine/blob/933dc839b5a97b9a5b3e04824bdd456daf75a57d/dev/ese/src/inc/daedef.hxx#L2695 + - FileHeader + - DBTime + - JET_LOGTIME + - JET_SIGNATURE + + # https://github.com/microsoft/Extensible-Storage-Engine/blob/933dc839b5a97b9a5b3e04824bdd456daf75a57d/dev/ese/src/inc/cpage.hxx#L887 + - PageHeader_ + - Tag + - ESENT_ROOT_HEADER + - ESENT_BRANCH_HEADER + - ESENT_SPACE_TREE_HEADER + - ESENT_LEAF_HEADER + - ESENT_SPACE_TREE_ENTRY + - ESENT_INDEX_ENTRY + - ESENT_LEAF_ENTRY + - ESENT_BRANCH_ENTRY + - CATALOG_TYPE_TABLE + - CATALOG_TYPE_TABLE + - CATALOG_TYPE_COLUMN + - CATALOG_TYPE_INDEX + - CATALOG_TYPE_LONG_VALUE + - ESENT_DATA_DEFINITION_HEADER + - ESENT_CATALOG_DATA_DEFINITION_ENTRY + - RecordTag + - GUID + - LongValueHeader + + # https://github.com/microsoft/Extensible-Storage-Engine/blob/933dc839b5a97b9a5b3e04824bdd456daf75a57d/dev/ese/src/inc/lv.hxx#L37 + - LVKEY64 + - LVKEY32 + - LVKEY_BUFFER + + # This one is here to include some utility functions we might need. + - Misc + + # some helpers + - CompressedKey diff --git a/internal/eseparser/debug.go b/internal/eseparser/debug.go new file mode 100644 index 0000000..af2b561 --- /dev/null +++ b/internal/eseparser/debug.go @@ -0,0 +1,25 @@ +package eseparser + +import "fmt" + +var ( + // General purpose debug statements. + Debug = false + + // Debugging during walk + DebugWalk = false +) + +func DlvDebug() { + +} + +func DebugPageHeader(ctx *ESEContext, page *PageHeader) string { + return page.DebugString() + fmt.Sprintf(" EndOffset: %#x \n", page.EndOffset(ctx)) +} + +func DebugTag(ctx *ESEContext, tag *Tag, page *PageHeader) string { + return tag.DebugString() + + fmt.Sprintf(" ValueOffsetInPage: %#x \n", + tag.ValueOffsetInPage(ctx, page)) +} diff --git a/internal/eseparser/ese_gen.go b/internal/eseparser/ese_gen.go new file mode 100644 index 0000000..d8db5e1 --- /dev/null +++ b/internal/eseparser/ese_gen.go @@ -0,0 +1,1347 @@ +package eseparser + +// Autogenerated code from ese_profile.json. Do not edit. + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "sort" + "strings" + "unicode/utf16" + "unicode/utf8" +) + +func indent(text string) string { + result := []string{} + lines := strings.Split(text, "\n") + for _, line := range lines { + result = append(result, " "+line) + } + return strings.Join(result, "\n") +} + +type ESEProfile struct { + Off_CATALOG_TYPE_COLUMN_ColumnType int64 + Off_CATALOG_TYPE_COLUMN_SpaceUsage int64 + Off_CATALOG_TYPE_COLUMN_ColumnFlags int64 + Off_CATALOG_TYPE_COLUMN_CodePage int64 + Off_CATALOG_TYPE_INDEX_FatherDataPageNumber int64 + Off_CATALOG_TYPE_INDEX_SpaceUsage int64 + Off_CATALOG_TYPE_INDEX_IndexFlags int64 + Off_CATALOG_TYPE_INDEX_Locale int64 + Off_CATALOG_TYPE_LONG_VALUE_FatherDataPageNumber int64 + Off_CATALOG_TYPE_LONG_VALUE_SpaceUsage int64 + Off_CATALOG_TYPE_LONG_VALUE_LVFlags int64 + Off_CATALOG_TYPE_LONG_VALUE_InitialNumberOfPages int64 + Off_CATALOG_TYPE_TABLE_FatherDataPageNumber int64 + Off_CATALOG_TYPE_TABLE_SpaceUsage int64 + Off_DBTime_Hours int64 + Off_DBTime_Min int64 + Off_DBTime_Sec int64 + Off_ESENT_BRANCH_ENTRY_LocalPageKeySize int64 + Off_ESENT_BRANCH_HEADER_CommonPageKey int64 + Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_FDPId int64 + Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Type int64 + Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Identifier int64 + Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Column int64 + Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Table int64 + Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Index int64 + Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_LongValue int64 + Off_ESENT_DATA_DEFINITION_HEADER_LastFixedType int64 + Off_ESENT_DATA_DEFINITION_HEADER_LastVariableDataType int64 + Off_ESENT_DATA_DEFINITION_HEADER_VariableSizeOffset int64 + Off_ESENT_INDEX_ENTRY_RecordPageKey int64 + Off_ESENT_LEAF_ENTRY_CommonPageKeySize int64 + Off_ESENT_LEAF_ENTRY_LocalPageKeySize int64 + Off_ESENT_LEAF_HEADER_CommonPageKey int64 + Off_ESENT_ROOT_HEADER_InitialNumberOfPages int64 + Off_ESENT_ROOT_HEADER_ParentFDP int64 + Off_ESENT_ROOT_HEADER_ExtentSpace int64 + Off_ESENT_ROOT_HEADER_SpaceTreePageNumber int64 + Off_ESENT_SPACE_TREE_ENTRY_PageKeySize int64 + Off_ESENT_SPACE_TREE_ENTRY_LastPageNumber int64 + Off_ESENT_SPACE_TREE_ENTRY_NumberOfPages int64 + Off_FileHeader_Magic int64 + Off_FileHeader_FormatVersion int64 + Off_FileHeader_FormatRevision int64 + Off_FileHeader_FileType int64 + Off_FileHeader_DataBaseTime int64 + Off_FileHeader_Signature int64 + Off_FileHeader_PageSize int64 + Off_GUID_Data1 int64 + Off_GUID_Data2 int64 + Off_GUID_Data3 int64 + Off_GUID_Data4 int64 + Off_JET_LOGTIME_Sec int64 + Off_JET_LOGTIME_Min int64 + Off_JET_LOGTIME_Hours int64 + Off_JET_LOGTIME_Days int64 + Off_JET_LOGTIME_Month int64 + Off_JET_LOGTIME_Year int64 + Off_JET_SIGNATURE_Creation int64 + Off_JET_SIGNATURE_CreatorMachine int64 + Off_LVKEY32_Lid int64 + Off_LVKEY32_SegmentOffset int64 + Off_LVKEY64_Lid int64 + Off_LVKEY64_SegmentOffset int64 + Off_LVKEY_BUFFER_PrefixLength int64 + Off_LVKEY_BUFFER_SuffixLength int64 + Off_LVKEY_BUFFER_KeyBuffer int64 + Off_Misc_Misc int64 + Off_Misc_Misc2 int64 + Off_Misc_Misc3 int64 + Off_Misc_Misc5 int64 + Off_Misc_Misc4 int64 + Off_PageHeader__LastModified int64 + Off_PageHeader__PreviousPageNumber int64 + Off_PageHeader__NextPageNumber int64 + Off_PageHeader__FatherPage int64 + Off_PageHeader__AvailableDataSize int64 + Off_PageHeader__AvailableDataOffset int64 + Off_PageHeader__AvailablePageTag int64 + Off_PageHeader__Flags int64 + Off_RecordTag_Identifier int64 + Off_RecordTag_DataOffset int64 + Off_RecordTag_Flags int64 + Off_Tag__ValueSize int64 + Off_Tag__ValueOffset int64 + Off_Tag_Flags_ int64 + Off_Tag_Flags int64 +} + +func NewESEProfile() *ESEProfile { + // Specific offsets can be tweaked to cater for slight version mismatches. + self := &ESEProfile{0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 0, 2, 4, 0, 0, 0, 4, 6, 10, 10, 10, 10, 0, 1, 2, 0, -2, 0, 0, 0, 4, 8, 12, 0, 0, 0, 4, 8, 232, 12, 16, 24, 236, 0, 4, 6, 8, 0, 1, 2, 3, 4, 5, 4, 12, 0, 4, 0, 8, 0, 2, 4, 0, 0, 0, 0, 0, 8, 16, 20, 24, 28, 32, 34, 36, 0, 2, 2, 0, 2, 2, 2} + return self +} + +func (self *ESEProfile) CATALOG_TYPE_COLUMN(reader io.ReaderAt, offset int64) *CATALOG_TYPE_COLUMN { + return &CATALOG_TYPE_COLUMN{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) CATALOG_TYPE_INDEX(reader io.ReaderAt, offset int64) *CATALOG_TYPE_INDEX { + return &CATALOG_TYPE_INDEX{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) CATALOG_TYPE_LONG_VALUE(reader io.ReaderAt, offset int64) *CATALOG_TYPE_LONG_VALUE { + return &CATALOG_TYPE_LONG_VALUE{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) CATALOG_TYPE_TABLE(reader io.ReaderAt, offset int64) *CATALOG_TYPE_TABLE { + return &CATALOG_TYPE_TABLE{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) DBTime(reader io.ReaderAt, offset int64) *DBTime { + return &DBTime{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) ESENT_BRANCH_ENTRY(reader io.ReaderAt, offset int64) *ESENT_BRANCH_ENTRY { + return &ESENT_BRANCH_ENTRY{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) ESENT_BRANCH_HEADER(reader io.ReaderAt, offset int64) *ESENT_BRANCH_HEADER { + return &ESENT_BRANCH_HEADER{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) ESENT_CATALOG_DATA_DEFINITION_ENTRY(reader io.ReaderAt, offset int64) *ESENT_CATALOG_DATA_DEFINITION_ENTRY { + return &ESENT_CATALOG_DATA_DEFINITION_ENTRY{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) ESENT_DATA_DEFINITION_HEADER(reader io.ReaderAt, offset int64) *ESENT_DATA_DEFINITION_HEADER { + return &ESENT_DATA_DEFINITION_HEADER{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) ESENT_INDEX_ENTRY(reader io.ReaderAt, offset int64) *ESENT_INDEX_ENTRY { + return &ESENT_INDEX_ENTRY{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) ESENT_LEAF_ENTRY(reader io.ReaderAt, offset int64) *ESENT_LEAF_ENTRY { + return &ESENT_LEAF_ENTRY{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) ESENT_LEAF_HEADER(reader io.ReaderAt, offset int64) *ESENT_LEAF_HEADER { + return &ESENT_LEAF_HEADER{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) ESENT_ROOT_HEADER(reader io.ReaderAt, offset int64) *ESENT_ROOT_HEADER { + return &ESENT_ROOT_HEADER{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) ESENT_SPACE_TREE_ENTRY(reader io.ReaderAt, offset int64) *ESENT_SPACE_TREE_ENTRY { + return &ESENT_SPACE_TREE_ENTRY{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) ESENT_SPACE_TREE_HEADER(reader io.ReaderAt, offset int64) *ESENT_SPACE_TREE_HEADER { + return &ESENT_SPACE_TREE_HEADER{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) FileHeader(reader io.ReaderAt, offset int64) *FileHeader { + return &FileHeader{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) GUID(reader io.ReaderAt, offset int64) *GUID { + return &GUID{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) JET_LOGTIME(reader io.ReaderAt, offset int64) *JET_LOGTIME { + return &JET_LOGTIME{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) JET_SIGNATURE(reader io.ReaderAt, offset int64) *JET_SIGNATURE { + return &JET_SIGNATURE{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) LVKEY32(reader io.ReaderAt, offset int64) *LVKEY32 { + return &LVKEY32{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) LVKEY64(reader io.ReaderAt, offset int64) *LVKEY64 { + return &LVKEY64{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) LVKEY_BUFFER(reader io.ReaderAt, offset int64) *LVKEY_BUFFER { + return &LVKEY_BUFFER{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) Misc(reader io.ReaderAt, offset int64) *Misc { + return &Misc{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) PageHeader_(reader io.ReaderAt, offset int64) *PageHeader_ { + return &PageHeader_{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) RecordTag(reader io.ReaderAt, offset int64) *RecordTag { + return &RecordTag{Reader: reader, Offset: offset, Profile: self} +} + +func (self *ESEProfile) Tag(reader io.ReaderAt, offset int64) *Tag { + return &Tag{Reader: reader, Offset: offset, Profile: self} +} + +type CATALOG_TYPE_COLUMN struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *CATALOG_TYPE_COLUMN) Size() int { + return 0 +} + +func (self *CATALOG_TYPE_COLUMN) ColumnType() *Enumeration { + value := ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_COLUMN_ColumnType+self.Offset) + name := "Unknown" + switch value { + + case 0: + name = "NULL" + + case 1: + name = "Boolean" + + case 2: + name = "Signed byte" + + case 3: + name = "Signed short" + + case 4: + name = "Signed long" + + case 5: + name = "Currency" + + case 6: + name = "Single precision FP" + + case 7: + name = "Double precision FP" + + case 8: + name = "DateTime" + + case 9: + name = "Binary" + + case 10: + name = "Text" + + case 11: + name = "Long Binary" + + case 12: + name = "Long Text" + + case 13: + name = "Obsolete" + + case 14: + name = "Unsigned long" + + case 15: + name = "Long long" + + case 16: + name = "GUID" + + case 17: + name = "Unsigned short" + + case 18: + name = "Max" + } + return &Enumeration{Value: uint64(value), Name: name} +} + +func (self *CATALOG_TYPE_COLUMN) SpaceUsage() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_COLUMN_SpaceUsage+self.Offset) +} + +func (self *CATALOG_TYPE_COLUMN) ColumnFlags() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_COLUMN_ColumnFlags+self.Offset) +} + +func (self *CATALOG_TYPE_COLUMN) CodePage() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_COLUMN_CodePage+self.Offset) +} +func (self *CATALOG_TYPE_COLUMN) DebugString() string { + result := fmt.Sprintf("struct CATALOG_TYPE_COLUMN @ %#x:\n", self.Offset) + result += fmt.Sprintf(" ColumnType: %v\n", self.ColumnType().DebugString()) + result += fmt.Sprintf(" SpaceUsage: %#0x\n", self.SpaceUsage()) + result += fmt.Sprintf(" ColumnFlags: %#0x\n", self.ColumnFlags()) + result += fmt.Sprintf(" CodePage: %#0x\n", self.CodePage()) + return result +} + +type CATALOG_TYPE_INDEX struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *CATALOG_TYPE_INDEX) Size() int { + return 0 +} + +func (self *CATALOG_TYPE_INDEX) FatherDataPageNumber() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_INDEX_FatherDataPageNumber+self.Offset) +} + +func (self *CATALOG_TYPE_INDEX) SpaceUsage() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_INDEX_SpaceUsage+self.Offset) +} + +func (self *CATALOG_TYPE_INDEX) IndexFlags() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_INDEX_IndexFlags+self.Offset) +} + +func (self *CATALOG_TYPE_INDEX) Locale() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_INDEX_Locale+self.Offset) +} +func (self *CATALOG_TYPE_INDEX) DebugString() string { + result := fmt.Sprintf("struct CATALOG_TYPE_INDEX @ %#x:\n", self.Offset) + result += fmt.Sprintf(" FatherDataPageNumber: %#0x\n", self.FatherDataPageNumber()) + result += fmt.Sprintf(" SpaceUsage: %#0x\n", self.SpaceUsage()) + result += fmt.Sprintf(" IndexFlags: %#0x\n", self.IndexFlags()) + result += fmt.Sprintf(" Locale: %#0x\n", self.Locale()) + return result +} + +type CATALOG_TYPE_LONG_VALUE struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *CATALOG_TYPE_LONG_VALUE) Size() int { + return 0 +} + +func (self *CATALOG_TYPE_LONG_VALUE) FatherDataPageNumber() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_LONG_VALUE_FatherDataPageNumber+self.Offset) +} + +func (self *CATALOG_TYPE_LONG_VALUE) SpaceUsage() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_LONG_VALUE_SpaceUsage+self.Offset) +} + +func (self *CATALOG_TYPE_LONG_VALUE) LVFlags() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_LONG_VALUE_LVFlags+self.Offset) +} + +func (self *CATALOG_TYPE_LONG_VALUE) InitialNumberOfPages() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_LONG_VALUE_InitialNumberOfPages+self.Offset) +} +func (self *CATALOG_TYPE_LONG_VALUE) DebugString() string { + result := fmt.Sprintf("struct CATALOG_TYPE_LONG_VALUE @ %#x:\n", self.Offset) + result += fmt.Sprintf(" FatherDataPageNumber: %#0x\n", self.FatherDataPageNumber()) + result += fmt.Sprintf(" SpaceUsage: %#0x\n", self.SpaceUsage()) + result += fmt.Sprintf(" LVFlags: %#0x\n", self.LVFlags()) + result += fmt.Sprintf(" InitialNumberOfPages: %#0x\n", self.InitialNumberOfPages()) + return result +} + +type CATALOG_TYPE_TABLE struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *CATALOG_TYPE_TABLE) Size() int { + return 0 +} + +func (self *CATALOG_TYPE_TABLE) FatherDataPageNumber() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_TABLE_FatherDataPageNumber+self.Offset) +} + +func (self *CATALOG_TYPE_TABLE) SpaceUsage() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_CATALOG_TYPE_TABLE_SpaceUsage+self.Offset) +} +func (self *CATALOG_TYPE_TABLE) DebugString() string { + result := fmt.Sprintf("struct CATALOG_TYPE_TABLE @ %#x:\n", self.Offset) + result += fmt.Sprintf(" FatherDataPageNumber: %#0x\n", self.FatherDataPageNumber()) + result += fmt.Sprintf(" SpaceUsage: %#0x\n", self.SpaceUsage()) + return result +} + +type DBTime struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *DBTime) Size() int { + return 8 +} + +func (self *DBTime) Hours() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_DBTime_Hours+self.Offset) +} + +func (self *DBTime) Min() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_DBTime_Min+self.Offset) +} + +func (self *DBTime) Sec() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_DBTime_Sec+self.Offset) +} +func (self *DBTime) DebugString() string { + result := fmt.Sprintf("struct DBTime @ %#x:\n", self.Offset) + result += fmt.Sprintf(" Hours: %#0x\n", self.Hours()) + result += fmt.Sprintf(" Min: %#0x\n", self.Min()) + result += fmt.Sprintf(" Sec: %#0x\n", self.Sec()) + return result +} + +type ESENT_BRANCH_ENTRY struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *ESENT_BRANCH_ENTRY) Size() int { + return 16 +} + +func (self *ESENT_BRANCH_ENTRY) LocalPageKeySize() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_ESENT_BRANCH_ENTRY_LocalPageKeySize+self.Offset) +} +func (self *ESENT_BRANCH_ENTRY) DebugString() string { + result := fmt.Sprintf("struct ESENT_BRANCH_ENTRY @ %#x:\n", self.Offset) + result += fmt.Sprintf(" LocalPageKeySize: %#0x\n", self.LocalPageKeySize()) + return result +} + +type ESENT_BRANCH_HEADER struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *ESENT_BRANCH_HEADER) Size() int { + return 16 +} + +func (self *ESENT_BRANCH_HEADER) CommonPageKey() string { + return ParseTerminatedString(self.Reader, self.Profile.Off_ESENT_BRANCH_HEADER_CommonPageKey+self.Offset) +} +func (self *ESENT_BRANCH_HEADER) DebugString() string { + result := fmt.Sprintf("struct ESENT_BRANCH_HEADER @ %#x:\n", self.Offset) + result += fmt.Sprintf(" CommonPageKey: %v\n", string(self.CommonPageKey())) + return result +} + +type ESENT_CATALOG_DATA_DEFINITION_ENTRY struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *ESENT_CATALOG_DATA_DEFINITION_ENTRY) Size() int { + return 0 +} + +func (self *ESENT_CATALOG_DATA_DEFINITION_ENTRY) FDPId() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_FDPId+self.Offset) +} + +func (self *ESENT_CATALOG_DATA_DEFINITION_ENTRY) Type() *Enumeration { + value := ParseUint16(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Type+self.Offset) + name := "Unknown" + switch value { + + case 1: + name = "CATALOG_TYPE_TABLE" + + case 2: + name = "CATALOG_TYPE_COLUMN" + + case 3: + name = "CATALOG_TYPE_INDEX" + + case 4: + name = "CATALOG_TYPE_LONG_VALUE" + } + return &Enumeration{Value: uint64(value), Name: name} +} + +func (self *ESENT_CATALOG_DATA_DEFINITION_ENTRY) Identifier() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Identifier+self.Offset) +} + +func (self *ESENT_CATALOG_DATA_DEFINITION_ENTRY) Column() *CATALOG_TYPE_COLUMN { + return self.Profile.CATALOG_TYPE_COLUMN(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Column+self.Offset) +} + +func (self *ESENT_CATALOG_DATA_DEFINITION_ENTRY) Table() *CATALOG_TYPE_TABLE { + return self.Profile.CATALOG_TYPE_TABLE(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Table+self.Offset) +} + +func (self *ESENT_CATALOG_DATA_DEFINITION_ENTRY) Index() *CATALOG_TYPE_INDEX { + return self.Profile.CATALOG_TYPE_INDEX(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_Index+self.Offset) +} + +func (self *ESENT_CATALOG_DATA_DEFINITION_ENTRY) LongValue() *CATALOG_TYPE_LONG_VALUE { + return self.Profile.CATALOG_TYPE_LONG_VALUE(self.Reader, self.Profile.Off_ESENT_CATALOG_DATA_DEFINITION_ENTRY_LongValue+self.Offset) +} +func (self *ESENT_CATALOG_DATA_DEFINITION_ENTRY) DebugString() string { + result := fmt.Sprintf("struct ESENT_CATALOG_DATA_DEFINITION_ENTRY @ %#x:\n", self.Offset) + result += fmt.Sprintf(" FDPId: %#0x\n", self.FDPId()) + result += fmt.Sprintf(" Type: %v\n", self.Type().DebugString()) + result += fmt.Sprintf(" Identifier: %#0x\n", self.Identifier()) + result += fmt.Sprintf(" Column: {\n%v}\n", indent(self.Column().DebugString())) + result += fmt.Sprintf(" Table: {\n%v}\n", indent(self.Table().DebugString())) + result += fmt.Sprintf(" Index: {\n%v}\n", indent(self.Index().DebugString())) + result += fmt.Sprintf(" LongValue: {\n%v}\n", indent(self.LongValue().DebugString())) + return result +} + +type ESENT_DATA_DEFINITION_HEADER struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *ESENT_DATA_DEFINITION_HEADER) Size() int { + return 4 +} + +func (self *ESENT_DATA_DEFINITION_HEADER) LastFixedType() byte { + return ParseUint8(self.Reader, self.Profile.Off_ESENT_DATA_DEFINITION_HEADER_LastFixedType+self.Offset) +} + +func (self *ESENT_DATA_DEFINITION_HEADER) LastVariableDataType() byte { + return ParseUint8(self.Reader, self.Profile.Off_ESENT_DATA_DEFINITION_HEADER_LastVariableDataType+self.Offset) +} + +func (self *ESENT_DATA_DEFINITION_HEADER) VariableSizeOffset() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_ESENT_DATA_DEFINITION_HEADER_VariableSizeOffset+self.Offset) +} +func (self *ESENT_DATA_DEFINITION_HEADER) DebugString() string { + result := fmt.Sprintf("struct ESENT_DATA_DEFINITION_HEADER @ %#x:\n", self.Offset) + result += fmt.Sprintf(" LastFixedType: %#0x\n", self.LastFixedType()) + result += fmt.Sprintf(" LastVariableDataType: %#0x\n", self.LastVariableDataType()) + result += fmt.Sprintf(" VariableSizeOffset: %#0x\n", self.VariableSizeOffset()) + return result +} + +type ESENT_INDEX_ENTRY struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *ESENT_INDEX_ENTRY) Size() int { + return 16 +} + +func (self *ESENT_INDEX_ENTRY) RecordPageKey() string { + return ParseTerminatedString(self.Reader, self.Profile.Off_ESENT_INDEX_ENTRY_RecordPageKey+self.Offset) +} +func (self *ESENT_INDEX_ENTRY) DebugString() string { + result := fmt.Sprintf("struct ESENT_INDEX_ENTRY @ %#x:\n", self.Offset) + result += fmt.Sprintf(" RecordPageKey: %v\n", string(self.RecordPageKey())) + return result +} + +type ESENT_LEAF_ENTRY struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *ESENT_LEAF_ENTRY) Size() int { + return 16 +} + +func (self *ESENT_LEAF_ENTRY) CommonPageKeySize() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_ESENT_LEAF_ENTRY_CommonPageKeySize+self.Offset) +} + +func (self *ESENT_LEAF_ENTRY) LocalPageKeySize() uint64 { + value := ParseUint16(self.Reader, self.Profile.Off_ESENT_LEAF_ENTRY_LocalPageKeySize+self.Offset) + return (uint64(value) & 0x1fff) >> 0x0 +} +func (self *ESENT_LEAF_ENTRY) DebugString() string { + result := fmt.Sprintf("struct ESENT_LEAF_ENTRY @ %#x:\n", self.Offset) + result += fmt.Sprintf(" CommonPageKeySize: %#0x\n", self.CommonPageKeySize()) + result += fmt.Sprintf(" LocalPageKeySize: %#0x\n", self.LocalPageKeySize()) + return result +} + +type ESENT_LEAF_HEADER struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *ESENT_LEAF_HEADER) Size() int { + return 16 +} + +func (self *ESENT_LEAF_HEADER) CommonPageKey() string { + return ParseTerminatedString(self.Reader, self.Profile.Off_ESENT_LEAF_HEADER_CommonPageKey+self.Offset) +} +func (self *ESENT_LEAF_HEADER) DebugString() string { + result := fmt.Sprintf("struct ESENT_LEAF_HEADER @ %#x:\n", self.Offset) + result += fmt.Sprintf(" CommonPageKey: %v\n", string(self.CommonPageKey())) + return result +} + +type ESENT_ROOT_HEADER struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *ESENT_ROOT_HEADER) Size() int { + return 16 +} + +func (self *ESENT_ROOT_HEADER) InitialNumberOfPages() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_ESENT_ROOT_HEADER_InitialNumberOfPages+self.Offset) +} + +func (self *ESENT_ROOT_HEADER) ParentFDP() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_ESENT_ROOT_HEADER_ParentFDP+self.Offset) +} + +func (self *ESENT_ROOT_HEADER) ExtentSpace() *Enumeration { + value := ParseUint32(self.Reader, self.Profile.Off_ESENT_ROOT_HEADER_ExtentSpace+self.Offset) + name := "Unknown" + switch value { + + case 0: + name = "Single" + + case 1: + name = "Multiple" + } + return &Enumeration{Value: uint64(value), Name: name} +} + +func (self *ESENT_ROOT_HEADER) SpaceTreePageNumber() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_ESENT_ROOT_HEADER_SpaceTreePageNumber+self.Offset) +} +func (self *ESENT_ROOT_HEADER) DebugString() string { + result := fmt.Sprintf("struct ESENT_ROOT_HEADER @ %#x:\n", self.Offset) + result += fmt.Sprintf(" InitialNumberOfPages: %#0x\n", self.InitialNumberOfPages()) + result += fmt.Sprintf(" ParentFDP: %#0x\n", self.ParentFDP()) + result += fmt.Sprintf(" ExtentSpace: %v\n", self.ExtentSpace().DebugString()) + result += fmt.Sprintf(" SpaceTreePageNumber: %#0x\n", self.SpaceTreePageNumber()) + return result +} + +type ESENT_SPACE_TREE_ENTRY struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *ESENT_SPACE_TREE_ENTRY) Size() int { + return 16 +} + +func (self *ESENT_SPACE_TREE_ENTRY) PageKeySize() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_ESENT_SPACE_TREE_ENTRY_PageKeySize+self.Offset) +} + +func (self *ESENT_SPACE_TREE_ENTRY) LastPageNumber() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_ESENT_SPACE_TREE_ENTRY_LastPageNumber+self.Offset) +} + +func (self *ESENT_SPACE_TREE_ENTRY) NumberOfPages() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_ESENT_SPACE_TREE_ENTRY_NumberOfPages+self.Offset) +} +func (self *ESENT_SPACE_TREE_ENTRY) DebugString() string { + result := fmt.Sprintf("struct ESENT_SPACE_TREE_ENTRY @ %#x:\n", self.Offset) + result += fmt.Sprintf(" PageKeySize: %#0x\n", self.PageKeySize()) + result += fmt.Sprintf(" LastPageNumber: %#0x\n", self.LastPageNumber()) + result += fmt.Sprintf(" NumberOfPages: %#0x\n", self.NumberOfPages()) + return result +} + +type ESENT_SPACE_TREE_HEADER struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *ESENT_SPACE_TREE_HEADER) Size() int { + return 16 +} +func (self *ESENT_SPACE_TREE_HEADER) DebugString() string { + result := fmt.Sprintf("struct ESENT_SPACE_TREE_HEADER @ %#x:\n", self.Offset) + return result +} + +type FileHeader struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *FileHeader) Size() int { + return 0 +} + +func (self *FileHeader) Magic() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_FileHeader_Magic+self.Offset) +} + +func (self *FileHeader) FormatVersion() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_FileHeader_FormatVersion+self.Offset) +} + +func (self *FileHeader) FormatRevision() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_FileHeader_FormatRevision+self.Offset) +} + +func (self *FileHeader) FileType() *Enumeration { + value := ParseUint32(self.Reader, self.Profile.Off_FileHeader_FileType+self.Offset) + name := "Unknown" + switch value { + + case 0: + name = "Database" + + case 1: + name = "StreamingFile" + } + return &Enumeration{Value: uint64(value), Name: name} +} + +func (self *FileHeader) DataBaseTime() *DBTime { + return self.Profile.DBTime(self.Reader, self.Profile.Off_FileHeader_DataBaseTime+self.Offset) +} + +func (self *FileHeader) Signature() *JET_SIGNATURE { + return self.Profile.JET_SIGNATURE(self.Reader, self.Profile.Off_FileHeader_Signature+self.Offset) +} + +func (self *FileHeader) PageSize() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_FileHeader_PageSize+self.Offset) +} +func (self *FileHeader) DebugString() string { + result := fmt.Sprintf("struct FileHeader @ %#x:\n", self.Offset) + result += fmt.Sprintf(" Magic: %#0x\n", self.Magic()) + result += fmt.Sprintf(" FormatVersion: %#0x\n", self.FormatVersion()) + result += fmt.Sprintf(" FormatRevision: %#0x\n", self.FormatRevision()) + result += fmt.Sprintf(" FileType: %v\n", self.FileType().DebugString()) + result += fmt.Sprintf(" DataBaseTime: {\n%v}\n", indent(self.DataBaseTime().DebugString())) + result += fmt.Sprintf(" Signature: {\n%v}\n", indent(self.Signature().DebugString())) + result += fmt.Sprintf(" PageSize: %#0x\n", self.PageSize()) + return result +} + +type GUID struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *GUID) Size() int { + return 16 +} + +func (self *GUID) Data1() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_GUID_Data1+self.Offset) +} + +func (self *GUID) Data2() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_GUID_Data2+self.Offset) +} + +func (self *GUID) Data3() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_GUID_Data3+self.Offset) +} + +func (self *GUID) Data4() []byte { + return ParseArray_byte(self.Profile, self.Reader, self.Profile.Off_GUID_Data4+self.Offset, 8) +} +func (self *GUID) DebugString() string { + result := fmt.Sprintf("struct GUID @ %#x:\n", self.Offset) + result += fmt.Sprintf(" Data1: %#0x\n", self.Data1()) + result += fmt.Sprintf(" Data2: %#0x\n", self.Data2()) + result += fmt.Sprintf(" Data3: %#0x\n", self.Data3()) + return result +} + +type JET_LOGTIME struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *JET_LOGTIME) Size() int { + return 8 +} + +func (self *JET_LOGTIME) Sec() byte { + return ParseUint8(self.Reader, self.Profile.Off_JET_LOGTIME_Sec+self.Offset) +} + +func (self *JET_LOGTIME) Min() byte { + return ParseUint8(self.Reader, self.Profile.Off_JET_LOGTIME_Min+self.Offset) +} + +func (self *JET_LOGTIME) Hours() byte { + return ParseUint8(self.Reader, self.Profile.Off_JET_LOGTIME_Hours+self.Offset) +} + +func (self *JET_LOGTIME) Days() byte { + return ParseUint8(self.Reader, self.Profile.Off_JET_LOGTIME_Days+self.Offset) +} + +func (self *JET_LOGTIME) Month() byte { + return ParseUint8(self.Reader, self.Profile.Off_JET_LOGTIME_Month+self.Offset) +} + +func (self *JET_LOGTIME) Year() byte { + return ParseUint8(self.Reader, self.Profile.Off_JET_LOGTIME_Year+self.Offset) +} +func (self *JET_LOGTIME) DebugString() string { + result := fmt.Sprintf("struct JET_LOGTIME @ %#x:\n", self.Offset) + result += fmt.Sprintf(" Sec: %#0x\n", self.Sec()) + result += fmt.Sprintf(" Min: %#0x\n", self.Min()) + result += fmt.Sprintf(" Hours: %#0x\n", self.Hours()) + result += fmt.Sprintf(" Days: %#0x\n", self.Days()) + result += fmt.Sprintf(" Month: %#0x\n", self.Month()) + result += fmt.Sprintf(" Year: %#0x\n", self.Year()) + return result +} + +type JET_SIGNATURE struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *JET_SIGNATURE) Size() int { + return 28 +} + +func (self *JET_SIGNATURE) Creation() *JET_LOGTIME { + return self.Profile.JET_LOGTIME(self.Reader, self.Profile.Off_JET_SIGNATURE_Creation+self.Offset) +} + +func (self *JET_SIGNATURE) CreatorMachine() string { + return ParseTerminatedString(self.Reader, self.Profile.Off_JET_SIGNATURE_CreatorMachine+self.Offset) +} +func (self *JET_SIGNATURE) DebugString() string { + result := fmt.Sprintf("struct JET_SIGNATURE @ %#x:\n", self.Offset) + result += fmt.Sprintf(" Creation: {\n%v}\n", indent(self.Creation().DebugString())) + result += fmt.Sprintf(" CreatorMachine: %v\n", string(self.CreatorMachine())) + return result +} + +type LVKEY32 struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *LVKEY32) Size() int { + return 8 +} + +func (self *LVKEY32) Lid() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_LVKEY32_Lid+self.Offset) +} + +func (self *LVKEY32) SegmentOffset() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_LVKEY32_SegmentOffset+self.Offset) +} +func (self *LVKEY32) DebugString() string { + result := fmt.Sprintf("struct LVKEY32 @ %#x:\n", self.Offset) + result += fmt.Sprintf(" Lid: %#0x\n", self.Lid()) + result += fmt.Sprintf(" SegmentOffset: %#0x\n", self.SegmentOffset()) + return result +} + +type LVKEY64 struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *LVKEY64) Size() int { + return 12 +} + +func (self *LVKEY64) Lid() uint64 { + return ParseUint64(self.Reader, self.Profile.Off_LVKEY64_Lid+self.Offset) +} + +func (self *LVKEY64) SegmentOffset() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_LVKEY64_SegmentOffset+self.Offset) +} +func (self *LVKEY64) DebugString() string { + result := fmt.Sprintf("struct LVKEY64 @ %#x:\n", self.Offset) + result += fmt.Sprintf(" Lid: %#0x\n", self.Lid()) + result += fmt.Sprintf(" SegmentOffset: %#0x\n", self.SegmentOffset()) + return result +} + +type LVKEY_BUFFER struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *LVKEY_BUFFER) Size() int { + return 0 +} + +func (self *LVKEY_BUFFER) PrefixLength() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_LVKEY_BUFFER_PrefixLength+self.Offset) +} + +func (self *LVKEY_BUFFER) SuffixLength() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_LVKEY_BUFFER_SuffixLength+self.Offset) +} + +func (self *LVKEY_BUFFER) KeyBuffer() string { + return ParseString(self.Reader, self.Profile.Off_LVKEY_BUFFER_KeyBuffer+self.Offset, 12) +} +func (self *LVKEY_BUFFER) DebugString() string { + result := fmt.Sprintf("struct LVKEY_BUFFER @ %#x:\n", self.Offset) + result += fmt.Sprintf(" PrefixLength: %#0x\n", self.PrefixLength()) + result += fmt.Sprintf(" SuffixLength: %#0x\n", self.SuffixLength()) + result += fmt.Sprintf(" KeyBuffer: %v\n", string(self.KeyBuffer())) + return result +} + +type Misc struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *Misc) Size() int { + return 0 +} + +func (self *Misc) Misc() int32 { + return ParseInt32(self.Reader, self.Profile.Off_Misc_Misc+self.Offset) +} + +func (self *Misc) Misc2() int16 { + return ParseInt16(self.Reader, self.Profile.Off_Misc_Misc2+self.Offset) +} + +func (self *Misc) Misc3() int64 { + return ParseInt64(self.Reader, self.Profile.Off_Misc_Misc3+self.Offset) +} + +func (self *Misc) Misc5() uint64 { + return ParseUint64(self.Reader, self.Profile.Off_Misc_Misc5+self.Offset) +} + +func (self *Misc) Misc4() string { + return ParseTerminatedUTF16String(self.Reader, self.Profile.Off_Misc_Misc4+self.Offset) +} +func (self *Misc) DebugString() string { + result := fmt.Sprintf("struct Misc @ %#x:\n", self.Offset) + result += fmt.Sprintf(" Misc: %#0x\n", self.Misc()) + result += fmt.Sprintf(" Misc2: %#0x\n", self.Misc2()) + result += fmt.Sprintf(" Misc3: %#0x\n", self.Misc3()) + result += fmt.Sprintf(" Misc5: %#0x\n", self.Misc5()) + result += fmt.Sprintf(" Misc4: %v\n", string(self.Misc4())) + return result +} + +type PageHeader_ struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *PageHeader_) Size() int { + return 0 +} + +func (self *PageHeader_) LastModified() *DBTime { + return self.Profile.DBTime(self.Reader, self.Profile.Off_PageHeader__LastModified+self.Offset) +} + +func (self *PageHeader_) PreviousPageNumber() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_PageHeader__PreviousPageNumber+self.Offset) +} + +func (self *PageHeader_) NextPageNumber() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_PageHeader__NextPageNumber+self.Offset) +} + +func (self *PageHeader_) FatherPage() uint32 { + return ParseUint32(self.Reader, self.Profile.Off_PageHeader__FatherPage+self.Offset) +} + +func (self *PageHeader_) AvailableDataSize() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_PageHeader__AvailableDataSize+self.Offset) +} + +func (self *PageHeader_) AvailableDataOffset() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_PageHeader__AvailableDataOffset+self.Offset) +} + +func (self *PageHeader_) AvailablePageTag() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_PageHeader__AvailablePageTag+self.Offset) +} + +func (self *PageHeader_) Flags() *Flags { + value := ParseUint32(self.Reader, self.Profile.Off_PageHeader__Flags+self.Offset) + names := make(map[string]bool) + + if value&1 != 0 { + names["Root"] = true + } + + if value&2 != 0 { + names["Leaf"] = true + } + + if value&4 != 0 { + names["Parent"] = true + } + + if value&8 != 0 { + names["Empty"] = true + } + + if value&32 != 0 { + names["SpaceTree"] = true + } + + if value&64 != 0 { + names["Index"] = true + } + + if value&128 != 0 { + names["Long"] = true + } + + return &Flags{Value: uint64(value), Names: names} +} + +func (self *PageHeader_) DebugString() string { + result := fmt.Sprintf("struct PageHeader_ @ %#x:\n", self.Offset) + result += fmt.Sprintf(" LastModified: {\n%v}\n", indent(self.LastModified().DebugString())) + result += fmt.Sprintf(" PreviousPageNumber: %#0x\n", self.PreviousPageNumber()) + result += fmt.Sprintf(" NextPageNumber: %#0x\n", self.NextPageNumber()) + result += fmt.Sprintf(" FatherPage: %#0x\n", self.FatherPage()) + result += fmt.Sprintf(" AvailableDataSize: %#0x\n", self.AvailableDataSize()) + result += fmt.Sprintf(" AvailableDataOffset: %#0x\n", self.AvailableDataOffset()) + result += fmt.Sprintf(" AvailablePageTag: %#0x\n", self.AvailablePageTag()) + result += fmt.Sprintf(" Flags: %v\n", self.Flags().DebugString()) + return result +} + +type RecordTag struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *RecordTag) Size() int { + return 4 +} + +func (self *RecordTag) Identifier() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_RecordTag_Identifier+self.Offset) +} + +func (self *RecordTag) DataOffset() uint64 { + value := ParseUint16(self.Reader, self.Profile.Off_RecordTag_DataOffset+self.Offset) + return (uint64(value) & 0x1fff) >> 0x0 +} + +func (self *RecordTag) Flags() uint64 { + value := ParseUint16(self.Reader, self.Profile.Off_RecordTag_Flags+self.Offset) + return (uint64(value) & 0xffff) >> 0xe +} +func (self *RecordTag) DebugString() string { + result := fmt.Sprintf("struct RecordTag @ %#x:\n", self.Offset) + result += fmt.Sprintf(" Identifier: %#0x\n", self.Identifier()) + result += fmt.Sprintf(" DataOffset: %#0x\n", self.DataOffset()) + result += fmt.Sprintf(" Flags: %#0x\n", self.Flags()) + return result +} + +type Tag struct { + Reader io.ReaderAt + Offset int64 + Profile *ESEProfile +} + +func (self *Tag) Size() int { + return 4 +} + +func (self *Tag) _ValueSize() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_Tag__ValueSize+self.Offset) +} + +func (self *Tag) _ValueOffset() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_Tag__ValueOffset+self.Offset) +} + +func (self *Tag) Flags_() uint16 { + return ParseUint16(self.Reader, self.Profile.Off_Tag_Flags_+self.Offset) +} + +func (self *Tag) Flags() *Flags { + value := ParseUint16(self.Reader, self.Profile.Off_Tag_Flags+self.Offset) + names := make(map[string]bool) + + if value&8192 != 0 { + names["fNDVersion"] = true + } + + if value&16384 != 0 { + names["fNDDeleted"] = true + } + + if value&32768 != 0 { + names["fNDCompressed"] = true + } + + return &Flags{Value: uint64(value), Names: names} +} + +func (self *Tag) DebugString() string { + result := fmt.Sprintf("struct Tag @ %#x:\n", self.Offset) + result += fmt.Sprintf(" _ValueSize: %#0x\n", self._ValueSize()) + result += fmt.Sprintf(" _ValueOffset: %#0x\n", self._ValueOffset()) + result += fmt.Sprintf(" Flags_: %#0x\n", self.Flags_()) + result += fmt.Sprintf(" Flags: %v\n", self.Flags().DebugString()) + return result +} + +type Enumeration struct { + Value uint64 + Name string +} + +func (self Enumeration) DebugString() string { + return fmt.Sprintf("%s (%d)", self.Name, self.Value) +} + +type Flags struct { + Value uint64 + Names map[string]bool +} + +func (self Flags) DebugString() string { + names := []string{} + for k, _ := range self.Names { + names = append(names, k) + } + + sort.Strings(names) + + return fmt.Sprintf("%d (%s)", self.Value, strings.Join(names, ",")) +} + +func (self Flags) IsSet(flag string) bool { + result, _ := self.Names[flag] + return result +} + +func (self Flags) Values() []string { + result := make([]string, 0, len(self.Names)) + for k, _ := range self.Names { + result = append(result, k) + } + return result +} + +func ParseArray_byte(profile *ESEProfile, reader io.ReaderAt, offset int64, count int) []byte { + result := make([]byte, 0, count) + for i := 0; i < count; i++ { + value := ParseUint8(reader, offset) + result = append(result, value) + offset += int64(1) + } + return result +} + +func ParseInt16(reader io.ReaderAt, offset int64) int16 { + data := make([]byte, 2) + _, err := reader.ReadAt(data, offset) + if err != nil { + return 0 + } + return int16(binary.LittleEndian.Uint16(data)) +} + +func ParseInt32(reader io.ReaderAt, offset int64) int32 { + data := make([]byte, 4) + _, err := reader.ReadAt(data, offset) + if err != nil { + return 0 + } + return int32(binary.LittleEndian.Uint32(data)) +} + +func ParseInt64(reader io.ReaderAt, offset int64) int64 { + data := make([]byte, 8) + _, err := reader.ReadAt(data, offset) + if err != nil { + return 0 + } + return int64(binary.LittleEndian.Uint64(data)) +} + +func ParseUint16(reader io.ReaderAt, offset int64) uint16 { + data := make([]byte, 2) + _, err := reader.ReadAt(data, offset) + if err != nil { + return 0 + } + return binary.LittleEndian.Uint16(data) +} + +func ParseUint32(reader io.ReaderAt, offset int64) uint32 { + data := make([]byte, 4) + _, err := reader.ReadAt(data, offset) + if err != nil { + return 0 + } + return binary.LittleEndian.Uint32(data) +} + +func ParseUint64(reader io.ReaderAt, offset int64) uint64 { + data := make([]byte, 8) + _, err := reader.ReadAt(data, offset) + if err != nil { + return 0 + } + return binary.LittleEndian.Uint64(data) +} + +func ParseUint8(reader io.ReaderAt, offset int64) byte { + result := make([]byte, 1) + _, err := reader.ReadAt(result, offset) + if err != nil { + return 0 + } + return result[0] +} + +func ParseTerminatedString(reader io.ReaderAt, offset int64) string { + data := make([]byte, 1024) + n, err := reader.ReadAt(data, offset) + if err != nil && err != io.EOF { + return "" + } + idx := bytes.Index(data[:n], []byte{0}) + if idx < 0 { + idx = n + } + return string(data[0:idx]) +} + +func ParseString(reader io.ReaderAt, offset int64, length int64) string { + data := make([]byte, length) + n, err := reader.ReadAt(data, offset) + if err != nil && err != io.EOF { + return "" + } + return string(data[:n]) +} + +func ParseTerminatedUTF16String(reader io.ReaderAt, offset int64) string { + data := make([]byte, 1024) + n, err := reader.ReadAt(data, offset) + if err != nil && err != io.EOF { + return "" + } + + idx := bytes.Index(data[:n], []byte{0, 0}) + if idx < 0 { + idx = n - 1 + } + if idx%2 != 0 { + idx += 1 + } + return UTF16BytesToUTF8(data[0:idx], binary.LittleEndian) +} + +func ParseUTF16String(reader io.ReaderAt, offset int64, length int64) string { + data := make([]byte, length) + n, err := reader.ReadAt(data, offset) + if err != nil && err != io.EOF { + return "" + } + return UTF16BytesToUTF8(data[:n], binary.LittleEndian) +} + +func UTF16BytesToUTF8(b []byte, o binary.ByteOrder) string { + if len(b) < 2 { + return "" + } + + if b[0] == 0xff && b[1] == 0xfe { + o = binary.BigEndian + b = b[2:] + } else if b[0] == 0xfe && b[1] == 0xff { + o = binary.LittleEndian + b = b[2:] + } + + utf := make([]uint16, (len(b)+(2-1))/2) + + for i := 0; i+(2-1) < len(b); i += 2 { + utf[i/2] = o.Uint16(b[i:]) + } + if len(b)/2 < len(utf) { + utf[len(utf)-1] = utf8.RuneError + } + + return string(utf16.Decode(utf)) +} diff --git a/internal/eseparser/ese_profile.json b/internal/eseparser/ese_profile.json new file mode 100644 index 0000000..319c04a --- /dev/null +++ b/internal/eseparser/ese_profile.json @@ -0,0 +1,244 @@ +{ + "FileHeader": [0, { + "Magic": [4, ["unsigned long"]], + "FormatVersion": [8, ["unsigned long"]], + "FormatRevision": [232, ["unsigned long"]], + "FileType": [12, ["Enumeration", { + "target": "unsigned long", + "choices": { + "0": "Database", + "1": "StreamingFile" + } + }]], + "DataBaseTime": [16, ["DBTime"]], + "Signature": [24, ["JET_SIGNATURE"]], + "PageSize": [236, ["unsigned long"]] + }], + "DBTime": [8, { + "Hours": [0, ["unsigned short"]], + "Min": [2, ["unsigned short"]], + "Sec": [4, ["unsigned short"]] + }], + "JET_SIGNATURE": [28, { + "Creation": [4, ["JET_LOGTIME"]], + "CreatorMachine": [12, ["String"]] + }], + "JET_LOGTIME": [8, { + "Sec": [0, ["unsigned char"]], + "Min": [1, ["unsigned char"]], + "Hours": [2, ["unsigned char"]], + "Days": [3, ["unsigned char"]], + "Month": [4, ["unsigned char"]], + "Year": [5, ["unsigned char"]] + }], + "PageHeader_": [0, { + "LastModified": [8, ["DBTime"]], + "PreviousPageNumber": [16, ["unsigned long"]], + "NextPageNumber": [20, ["unsigned long"]], + "FatherPage": [24, ["unsigned long"]], + "AvailableDataSize": [28, ["unsigned short"]], + "AvailableDataOffset": [32, ["unsigned short"]], + "AvailablePageTag": [34, ["unsigned short"]], + "Flags": [36, ["Flags", { + "target": "unsigned long", + "maskmap": { + "Root": 1, + "Leaf": 2, + "Parent": 4, + "Empty": 8, + "SpaceTree": 32, + "Index": 64, + "Long": 128 + } + }]] + }], + "Tag": [4, { + "_ValueSize": [0, ["unsigned short"]], + "_ValueOffset": [2, ["unsigned short"]], + "Flags_": [2, ["unsigned short"]], + "Flags": [2, ["Flags", { + "target": "unsigned short", + "maskmap": { + "fNDVersion": 8192, + "fNDDeleted": 16384, + "fNDCompressed": 32768 + } + }]] + }], + "ESENT_ROOT_HEADER": [16, { + "InitialNumberOfPages": [0, ["unsigned long"]], + "ParentFDP": [4, ["unsigned long"]], + "ExtentSpace": [8, ["Enumeration", { + "target": "unsigned long", + "choices": { + "0": "Single", + "1": "Multiple" + } + }]], + "SpaceTreePageNumber": [12, ["unsigned long"]] + }], + + "ESENT_BRANCH_HEADER": [16, { + "CommonPageKey": [0, ["String"]] + }], + + "ESENT_SPACE_TREE_HEADER": [16, { + }], + + "ESENT_LEAF_HEADER": [16, { + "CommonPageKey": [0, ["String"]] + }], + + "ESENT_SPACE_TREE_ENTRY": [16, { + "PageKeySize": [0, ["unsigned short"]], + "LastPageNumber": [0, ["unsigned long"]], + "NumberOfPages": [0, ["unsigned long"]] + }], + + "ESENT_INDEX_ENTRY": [16, { + "RecordPageKey": [0, ["String"]] + }], + + "ESENT_LEAF_ENTRY": [16, { + "CommonPageKeySize": [-2, ["unsigned short"]], + "LocalPageKeySize": [0, ["BitField", { + "target": "unsigned short", + "start_bit": 0, + "end_bit": 13 + }]] + }], + + "ESENT_BRANCH_ENTRY": [16, { + "LocalPageKeySize": [0, ["unsigned short"]] + }], + + "CATALOG_TYPE_TABLE": [0, { + "FatherDataPageNumber": [0, ["unsigned long"]], + "SpaceUsage": [4, ["unsigned long"]], + "Flags": [8, ["unsigned long"]], + "InitialNumberOfPages": [12, ["unsigned long"]] + }], + + "CATALOG_TYPE_TABLE": [0, { + "FatherDataPageNumber": [0, ["unsigned long"]], + "SpaceUsage": [4, ["unsigned long"]] + }], + + "CATALOG_TYPE_COLUMN": [0, { + "ColumnType": [0, ["Enumeration", { + "target": "unsigned long", + "choices": { + "0" : "NULL", + "1" : "Boolean", + "2" : "Signed byte", + "3" : "Signed short", + "4" : "Signed long", + "5" : "Currency", + "6" : "Single precision FP", + "7" : "Double precision FP", + "8" : "DateTime", + "9" : "Binary", + "10" : "Text", + "11" : "Long Binary", + "12" : "Long Text", + "13" : "Obsolete", + "14" : "Unsigned long", + "15" : "Long long", + "16" : "GUID", + "17" : "Unsigned short", + "18" : "Max" + } + }]], + "SpaceUsage": [4, ["unsigned long"]], + "ColumnFlags": [8, ["unsigned long"]], + "CodePage": [12, ["unsigned long"]] + }], + + "CATALOG_TYPE_INDEX": [0, { + "FatherDataPageNumber": [0, ["unsigned long"]], + "SpaceUsage": [4, ["unsigned long"]], + "IndexFlags": [8, ["unsigned long"]], + "Locale": [12, ["unsigned long"]] + }], + + "CATALOG_TYPE_LONG_VALUE": [0, { + "FatherDataPageNumber": [0, ["unsigned long"]], + "SpaceUsage": [4, ["unsigned long"]], + "LVFlags": [8, ["unsigned long"]], + "InitialNumberOfPages": [12, ["unsigned long"]] + }], + + "ESENT_DATA_DEFINITION_HEADER": [4, { + "LastFixedType": [0, ["unsigned char"]], + "LastVariableDataType": [1, ["unsigned char"]], + "VariableSizeOffset": [2, ["unsigned short"]] + }], + + "ESENT_CATALOG_DATA_DEFINITION_ENTRY": [0, { + "FDPId": [0, ["unsigned long"]], + "Type": [4, ["Enumeration", { + "target": "unsigned short", + "choices": { + "1": "CATALOG_TYPE_TABLE", + "2": "CATALOG_TYPE_COLUMN", + "3": "CATALOG_TYPE_INDEX", + "4": "CATALOG_TYPE_LONG_VALUE" + } + }]], + "Identifier": [6, ["unsigned long"]], + "Column": [10, ["CATALOG_TYPE_COLUMN"]], + "Table": [10, ["CATALOG_TYPE_TABLE"]], + "Index": [10, ["CATALOG_TYPE_INDEX"]], + "LongValue": [10, ["CATALOG_TYPE_LONG_VALUE"]] + }], + + "RecordTag": [4, { + "Identifier": [0, ["unsigned short"]], + "DataOffset": [2, ["BitField", { + "target": "unsigned short", + "start_bit": 0, + "end_bit": 13 + }]], + "Flags": [2, ["BitField", { + "target": "unsigned short", + "start_bit": 14, + "end_bit": 16 + }]] + }], + + "Misc": [0, { + "Misc": [0, ["long"]], + "Misc2": [0, ["short"]], + "Misc3": [0, ["long long"]], + "Misc5": [0, ["unsigned long long"]], + "Misc4": [0, ["UnicodeString"]] + }], + + "GUID": [16, { + "Data1": [0, ["unsigned long", {}]], + "Data2": [4, ["unsigned short", {}]], + "Data3": [6, ["unsigned short", {}]], + "Data4": [8, ["Array", { + "count": 8, + "target": "unsigned char" + }]] + }], + + "LVKEY64": [12, { + "Lid": [0, ["unsigned long long"]], + "SegmentOffset": [8, ["unsigned long"]] + }], + + "LVKEY32": [8, { + "Lid": [0, ["unsigned long"]], + "SegmentOffset": [4, ["unsigned long"]] + }], + + "LVKEY_BUFFER": [0, { + "PrefixLength": [0, ["unsigned short"]], + "SuffixLength": [2, ["unsigned short"]], + "KeyBuffer": [4, ["String", { + "length": 12 + }]] + }] +} diff --git a/internal/eseparser/guid.go b/internal/eseparser/guid.go new file mode 100644 index 0000000..03479ae --- /dev/null +++ b/internal/eseparser/guid.go @@ -0,0 +1,12 @@ +package eseparser + +import "fmt" + +func (self GUID) AsString() string { + data4 := self.Data4() + return fmt.Sprintf( + "{%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x}", self.Data1(), + self.Data2(), self.Data3(), + data4[0], data4[1], data4[2], data4[3], + data4[4], data4[5], data4[6], data4[7]) +} diff --git a/internal/eseparser/long_values.go b/internal/eseparser/long_values.go new file mode 100644 index 0000000..7fa9636 --- /dev/null +++ b/internal/eseparser/long_values.go @@ -0,0 +1,130 @@ +package eseparser + +import ( + "fmt" + "io" +) + +type LongValue struct { + Value *Value + header *PageHeader + + // Calculated key for this long value object. + Key Key +} + +func (self *LongValue) Buffer() []byte { + start := int64(self.Key.EndOffset()) + result := make([]byte, self.Value.BufferSize-start) + self.Value.Reader().ReadAt(result, start) + return result +} + +func (self *LongValue) Reader() io.ReaderAt { + start := int64(self.Key.EndOffset()) + return NewOffsetReader( + self.Value.Reader(), start, self.Value.BufferSize-start) +} + +type LongValueLookup map[string]*LongValue + +// This can potentially return a lot of data +func (self LongValueLookup) GetLid(lid []byte) ([]byte, bool) { + if len(lid) != 4 { + return nil, false + } + + // Swap byte order between Lid and key + swapped := []byte{lid[3], lid[2], lid[1], lid[0]} + + // For now we only get the first segment + key := Key{ + prefix: swapped, + suffix: make([]byte, 4), + } + + // Try to find segments + value, pres := self[key.Key()] + if pres { + return value.Buffer(), true + } + return nil, false +} + +func NewLongValueLookup() LongValueLookup { + return make(LongValueLookup) +} + +type Key struct { + prefix []byte + suffix []byte + + end_offset uint64 +} + +func (self *Key) Key() string { + return string(self.prefix) + string(self.suffix) +} + +func (self *Key) DebugString() string { + result := "" + if len(self.prefix) > 0 { + result += fmt.Sprintf("prefix %02x ", self.prefix) + } + + if len(self.suffix) > 0 { + result += fmt.Sprintf("suffix %02x ", self.suffix) + } + return result + fmt.Sprintf(" key %02x ", self.Key()) +} + +func (self *Key) EndOffset() uint64 { + return self.end_offset +} + +func (self *LVKEY_BUFFER) ParseKey(ctx *ESEContext, header *PageHeader, value *Value) (key Key) { + key.end_offset = uint64(ctx.Profile.Off_LVKEY_BUFFER_KeyBuffer) + + prefix_lenth := uint64(self.PrefixLength()) + if prefix_lenth > 8 { + prefix_lenth = 8 + } + + suffix_length := uint64(self.SuffixLength()) + if suffix_length > 8-prefix_lenth { + suffix_length = 8 - prefix_lenth + } + + // Compressed keys + if value.Tag.Flags_()&fNDCompressed > 0 { + external_value := header.ExternalValueBytes(ctx) + if prefix_lenth > uint64(len(external_value)) { + prefix_lenth = uint64(len(external_value)) + } + + for i := uint64(0); i < prefix_lenth; i++ { + key.prefix = append(key.prefix, external_value[i]) + } + key_buffer := self.KeyBuffer() + if suffix_length > 8 { + suffix_length = 8 + } + + key.suffix = []byte(key_buffer[:suffix_length]) + key.end_offset += suffix_length + + } else { + // The key is not compressed - we read both prefix + // and suffix from the actual key + key_buffer := []byte(self.KeyBuffer()) + for uint64(len(key_buffer)) < prefix_lenth+suffix_length { + key_buffer = append(key_buffer, 0) + } + + key.prefix = key_buffer[:prefix_lenth] + key.suffix = key_buffer[prefix_lenth : prefix_lenth+suffix_length] + key.end_offset += suffix_length + prefix_lenth + } + + return key +} diff --git a/internal/eseparser/ordereddict/LICENSE b/internal/eseparser/ordereddict/LICENSE new file mode 100644 index 0000000..dfadb57 --- /dev/null +++ b/internal/eseparser/ordereddict/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2019 velocidex + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/internal/eseparser/ordereddict/README.txt b/internal/eseparser/ordereddict/README.txt new file mode 100644 index 0000000..a4e211a --- /dev/null +++ b/internal/eseparser/ordereddict/README.txt @@ -0,0 +1,14 @@ +# Ordered Dict implementation for go. + +There are a number of OrderedDict implementations out there and this +is one is less capable than a more generic implementation: + +- We do not support key deletion +- Replacing a key is O(n) with the number of keys + +Therefore this implementation is only suitable for dicts with few keys +that are not generally replaced or deleted. + +The main benefit of this implementation is that it maintains key order +when serializing/unserializing from JSON and it is very memory +efficient. diff --git a/internal/eseparser/ordereddict/ordereddict.go b/internal/eseparser/ordereddict/ordereddict.go new file mode 100644 index 0000000..7fb34fc --- /dev/null +++ b/internal/eseparser/ordereddict/ordereddict.go @@ -0,0 +1,265 @@ +package ordereddict + +import ( + "reflect" + "strings" + "sync" +) + +// A concrete implementation of a row - similar to Python's +// OrderedDict. Main difference is that delete is not implemented - +// we just preserve the order of insertions. +type Dict struct { + sync.Mutex + + store map[string]interface{} + keys []string + case_map map[string]string + + default_value interface{} +} + +func NewDict() *Dict { + return &Dict{ + store: make(map[string]interface{}), + } +} + +func (self *Dict) IsCaseInsensitive() bool { + self.Lock() + defer self.Unlock() + + return self.case_map != nil +} + +func (self *Dict) MergeFrom(other *Dict) { + for _, key := range other.keys { + value, pres := other.Get(key) + if pres { + self.Set(key, value) + } + } +} + +func (self *Dict) SetDefault(value interface{}) *Dict { + self.Lock() + defer self.Unlock() + + self.default_value = value + return self +} + +func (self *Dict) GetDefault() interface{} { + self.Lock() + defer self.Unlock() + + return self.default_value +} + +func (self *Dict) SetCaseInsensitive() *Dict { + self.Lock() + defer self.Unlock() + + self.case_map = make(map[string]string) + return self +} + +func remove(s []string, r string) []string { + for i, v := range s { + if v == r { + return append(s[:i], s[i+1:]...) + } + } + return s +} + +// Very inefficient but ok for occasional use. +func (self *Dict) Delete(key string) { + new_keys := make([]string, 0, len(self.keys)) + for _, old_key := range self.keys { + if key != old_key { + new_keys = append(new_keys, old_key) + } + } + self.keys = new_keys + delete(self.store, key) +} + +// Like Set() but does not effect the order. +func (self *Dict) Update(key string, value interface{}) *Dict { + self.Lock() + defer self.Unlock() + + _, pres := self.store[key] + if pres { + self.store[key] = value + } else { + self.set(key, value) + } + + return self +} + +func (self *Dict) Set(key string, value interface{}) *Dict { + self.Lock() + defer self.Unlock() + + return self.set(key, value) +} + +func (self *Dict) set(key string, value interface{}) *Dict { + // O(n) but for our use case this is faster since Dicts are + // typically small and we rarely overwrite a key. + _, pres := self.store[key] + if pres { + self.keys = append(remove(self.keys, key), key) + } else { + self.keys = append(self.keys, key) + } + + if self.store == nil { + self.store = make(map[string]interface{}) + } + + self.store[key] = value + + if self.case_map != nil { + self.case_map[strings.ToLower(key)] = key + } + + return self +} + +func (self *Dict) Len() int { + self.Lock() + defer self.Unlock() + + return len(self.store) +} + +func (self *Dict) Get(key string) (interface{}, bool) { + self.Lock() + defer self.Unlock() + + if self.case_map != nil { + real_key, pres := self.case_map[strings.ToLower(key)] + if pres { + key = real_key + } + } + + val, ok := self.store[key] + if !ok && self.default_value != nil { + return self.default_value, false + } + + return val, ok +} + +func (self *Dict) GetString(key string) (string, bool) { + v, pres := self.Get(key) + if pres { + v_str, ok := to_string(v) + if ok { + return v_str, true + } + } + return "", false +} + +func (self *Dict) GetBool(key string) (bool, bool) { + v, pres := self.Get(key) + if pres { + v_bool, ok := v.(bool) + if ok { + return v_bool, true + } + } + return false, false +} + +func to_string(x interface{}) (string, bool) { + switch t := x.(type) { + case string: + return t, true + case *string: + return *t, true + case []byte: + return string(t), true + default: + return "", false + } +} + +func (self *Dict) GetStrings(key string) ([]string, bool) { + v, pres := self.Get(key) + if pres && v != nil { + slice := reflect.ValueOf(v) + if slice.Type().Kind() == reflect.Slice { + result := []string{} + for i := 0; i < slice.Len(); i++ { + value := slice.Index(i).Interface() + item, ok := to_string(value) + if ok { + result = append(result, item) + } + } + return result, true + } + } + return nil, false +} + +func (self *Dict) GetInt64(key string) (int64, bool) { + value, pres := self.Get(key) + if pres { + switch t := value.(type) { + case int: + return int64(t), true + case int8: + return int64(t), true + case int16: + return int64(t), true + case int32: + return int64(t), true + case int64: + return int64(t), true + case uint8: + return int64(t), true + case uint16: + return int64(t), true + case uint32: + return int64(t), true + case uint64: + return int64(t), true + case float32: + return int64(t), true + case float64: + return int64(t), true + } + } + return 0, false +} + +func (self *Dict) Keys() []string { + self.Lock() + defer self.Unlock() + + return self.keys[:] +} + +func (self *Dict) ToDict() *map[string]interface{} { + self.Lock() + defer self.Unlock() + + result := make(map[string]interface{}) + + for _, key := range self.keys { + value, pres := self.store[key] + if pres { + result[key] = value + } + } + + return &result +} diff --git a/internal/eseparser/ordereddict/utils.go b/internal/eseparser/ordereddict/utils.go new file mode 100644 index 0000000..6f0987d --- /dev/null +++ b/internal/eseparser/ordereddict/utils.go @@ -0,0 +1,75 @@ +package ordereddict + +import "strings" + +func GetString(event_map *Dict, members string) (string, bool) { + value, pres := GetAny(event_map, members) + if pres { + switch t := value.(type) { + case string: + return t, true + case *string: + return *t, true + } + } + + return "", false +} + +func GetMap(event_map *Dict, members string) (*Dict, bool) { + value, pres := GetAny(event_map, members) + if pres { + switch t := value.(type) { + case *Dict: + return t, true + } + } + return nil, false +} + +func GetAny(event_map *Dict, members string) (interface{}, bool) { + var value interface{} = event_map + var pres bool + + for _, member := range strings.Split(members, ".") { + if event_map == nil { + return nil, false + } + + value, pres = event_map.Get(member) + if !pres { + return nil, false + } + event_map, pres = value.(*Dict) + } + + return value, true +} + +func GetInt(event_map *Dict, members string) (int, bool) { + value, pres := GetAny(event_map, members) + if pres { + switch t := value.(type) { + case int: + return t, true + case uint8: + return int(t), true + case uint16: + return int(t), true + case uint32: + return int(t), true + case uint64: + return int(t), true + case int8: + return int(t), true + case int16: + return int(t), true + case int32: + return int(t), true + case int64: + return int(t), true + } + } + + return 0, false +} diff --git a/internal/eseparser/pages.go b/internal/eseparser/pages.go new file mode 100644 index 0000000..8148118 --- /dev/null +++ b/internal/eseparser/pages.go @@ -0,0 +1,338 @@ +package eseparser + +import ( + "fmt" + "io" +) + +const ( + TAG_COMMON = 4 +) + +// TODO: This is called LINE in the MS code. It represents a single +// node in the page. Depending on the page type it needs to be further +// parsed. +type Value struct { + Tag *Tag + PageID int64 + Flags uint64 + + reader io.ReaderAt + BufferOffset int64 + BufferSize int64 +} + +func (self *Value) GetBuffer() []byte { + result := make([]byte, self.BufferSize) + self.reader.ReadAt(result, self.BufferOffset) + return result +} + +func (self *Value) Reader() io.ReaderAt { + return NewOffsetReader(self.reader, + self.BufferOffset, self.BufferSize) +} + +func NewReaderValue(ctx *ESEContext, tag *Tag, PageID int64, + reader io.ReaderAt, start, length int64) *Value { + result := &Value{Tag: tag, PageID: PageID, reader: reader, + BufferOffset: start, BufferSize: length} + if ctx.Version == 0x620 && ctx.Revision >= 17 && + ctx.PageSize > 8192 && length > 0 { + + buffer := make([]byte, 4) + reader.ReadAt(buffer, start) + + result.Flags = uint64(buffer[1] >> 5) + buffer[1] &= 0x1f + } else { + result.Flags = uint64(tag._ValueOffset()) >> 13 + } + return result +} + +func (self *Tag) valueOffset(ctx *ESEContext) uint16 { + if ctx.Version == 0x620 && ctx.Revision >= 17 && ctx.PageSize > 8192 { + return self._ValueOffset() & 0x7FFF + } + return self._ValueOffset() & 0x1FFF +} + +func (self *Tag) ValueOffsetInPage(ctx *ESEContext, page *PageHeader) int64 { + return int64(self.valueOffset(ctx)) + page.EndOffset(ctx) +} + +func (self *Tag) FFlags() uint16 { + // CPAGE::TAG::FFlags + // https://github.com/microsoft/Extensible-Storage-Engine/blob/933dc839b5a97b9a5b3e04824bdd456daf75a57d/dev/ese/src/ese/cpage.cxx#1212 + return (self._ValueOffset() & 0x1fff) >> 13 +} + +func (self *Tag) ValueSize(ctx *ESEContext) uint16 { + if ctx.Version == 0x620 && ctx.Revision >= 17 && + !IsSmallPage(ctx.PageSize) { + return self._ValueSize() & 0x7FFF + } + return self._ValueSize() & 0x1FFF +} + +func GetPageValues(ctx *ESEContext, header *PageHeader, id int64) []*Value { + result := []*Value{} + + // Tags are written from the end of the page. Sizeof(Tag) = 4 + offset := ctx.PageSize + header.Offset - 4 + + // Skip the external value tag because it is fetched using a + // dedicated call to PageHeader.ExternalValue() + offset -= 4 + + for tag_count := header.AvailablePageTag() - 1; tag_count > 0; tag_count-- { + tag := ctx.Profile.Tag(ctx.Reader, offset) + + result = append(result, NewReaderValue( + ctx, tag, id, ctx.Reader, + tag.ValueOffsetInPage(ctx, header), + int64(tag.ValueSize(ctx)))) + offset -= 4 + } + + if DebugWalk { + fmt.Printf("Got %v values for page %v\n", len(result), id) + } + return result +} + +func GetRoot(ctx *ESEContext, value *Value) *ESENT_ROOT_HEADER { + return ctx.Profile.ESENT_ROOT_HEADER(value.Reader(), 0) +} + +func GetBranch(ctx *ESEContext, value *Value) *ESENT_BRANCH_HEADER { + return ctx.Profile.ESENT_BRANCH_HEADER(value.Reader(), 0) +} + +type PageHeader struct { + *PageHeader_ + + // The value pointed to by tag 0 + external_value_bytes []byte +} + +func (self *PageHeader) ExternalValueBytes(ctx *ESEContext) []byte { + if self.external_value_bytes != nil { + return self.external_value_bytes + } + + self.external_value_bytes = self.ExternalValue(ctx).GetBuffer() + return self.external_value_bytes +} + +// The External value is the zero'th tag +func (self *PageHeader) ExternalValue(ctx *ESEContext) *Value { + offset := ctx.PageSize + self.Offset - 4 + tag := self.Profile.Tag(self.Reader, offset) + + return NewReaderValue( + ctx, tag, 0, ctx.Reader, + tag.ValueOffsetInPage(ctx, self), + int64(tag.ValueSize(ctx))) +} + +func (self *PageHeader) IsBranch() bool { + return !self.Flags().IsSet("Leaf") +} + +func (self *PageHeader) IsLeaf() bool { + return self.Flags().IsSet("Leaf") +} + +func (self *PageHeader) EndOffset(ctx *ESEContext) int64 { + size := int64(40) + + // The header is larger when the pagesize is bigger (PGHDR2 vs + // PGHDR) + // https://github.com/microsoft/Extensible-Storage-Engine/blob/933dc839b5a97b9a5b3e04824bdd456daf75a57d/dev/ese/src/inc/cpage.hxx#L885 + if !IsSmallPage(ctx.PageSize) { + size = 80 + } + return self.Offset + size +} + +func DumpPage(ctx *ESEContext, id int64) { + header := ctx.GetPage(id) + fmt.Printf("Page %v: %v\n", id, header.DebugString()) + + // Show the tags + values := GetPageValues(ctx, header, id) + if len(values) == 0 { + return + } + + for i, value := range values { + fmt.Printf("Tag %v @ %#x offset %#x length %#x\n", + i, value.Tag.Offset, + value.Tag.ValueOffsetInPage(ctx, header), + value.Tag.ValueSize(ctx)) + } + + flags := header.Flags() + + if flags.IsSet("Root") { + GetRoot(ctx, values[0]).Dump() + + // Branch header + } else if header.IsBranch() { + GetBranch(ctx, values[0]).Dump() + + // SpaceTree header + } else if flags.IsSet("SpaceTree") { + ctx.Profile.ESENT_SPACE_TREE_HEADER( + ctx.Reader, values[0].BufferOffset).Dump() + + // Leaf header + } else if header.IsLeaf() { + NewESENT_LEAF_ENTRY(ctx, values[0]).Dump() + } + + for _, value := range values[1:] { + if header.IsBranch() { + NewESENT_BRANCH_ENTRY(ctx, value).Dump() + } else if header.IsLeaf() { + if flags.IsSet("SpaceTree") { + ctx.Profile.ESENT_SPACE_TREE_ENTRY(value.Reader(), 0).Dump() + } else if flags.IsSet("Index") { + ctx.Profile.ESENT_INDEX_ENTRY(value.Reader(), 0).Dump() + } else if flags.IsSet("Long") { + // TODO + } else { + NewESENT_LEAF_ENTRY(ctx, value).Dump() + } + } + } +} + +func (self *ESENT_ROOT_HEADER) Dump() { + fmt.Println(self.DebugString()) +} + +func (self *ESENT_SPACE_TREE_ENTRY) Dump() { + fmt.Println(self.DebugString()) +} +func (self *ESENT_INDEX_ENTRY) Dump() { + fmt.Println(self.DebugString()) +} + +// NewESENT_LEAF_ENTRY creates a new ESENT_LEAF_ENTRY +// object. Depending on the Tag flags, there may be present a +// CommonPageKeySize field before the struct. This constructor then +// positions the struct appropriately. +func NewESENT_LEAF_ENTRY(ctx *ESEContext, value *Value) *ESENT_LEAF_ENTRY { + if value.Flags&TAG_COMMON > 0 { + // Skip the common header + return ctx.Profile.ESENT_LEAF_ENTRY(value.Reader(), 2) + } + return ctx.Profile.ESENT_LEAF_ENTRY(value.Reader(), 0) +} + +func (self *ESENT_LEAF_ENTRY) Dump() { + fmt.Println(self.DebugString()) +} + +func (self *ESENT_LEAF_ENTRY) EntryData() int64 { + // Tag includes Local Page Key - skip it and the common page key + return self.Offset + 2 + int64(self.LocalPageKeySize()) +} + +func (self *ESENT_BRANCH_HEADER) Dump() { + fmt.Println(self.DebugString()) +} + +// NewESENT_BRANCH_ENTRY creates a new ESENT_BRANCH_ENTRY +// object. Depending on the Tag flags, there may be present a +// CommonPageKeySize field before the struct. This construstor then +// positions the struct appropriately. +func NewESENT_BRANCH_ENTRY(ctx *ESEContext, value *Value) *ESENT_BRANCH_ENTRY { + if value.Flags&TAG_COMMON > 0 { + // Skip the common header + return ctx.Profile.ESENT_BRANCH_ENTRY(value.Reader(), 2) + } + return ctx.Profile.ESENT_BRANCH_ENTRY(value.Reader(), 0) +} + +func (self *ESENT_BRANCH_ENTRY) Dump() { + fmt.Printf("%s", self.DebugString()) + fmt.Printf(" ChildPageNumber: %#x\n", self.ChildPageNumber()) +} + +func (self *ESENT_BRANCH_ENTRY) ChildPageNumber() int64 { + return int64(ParseUint32(self.Reader, self.Offset+2+ + int64(self.LocalPageKeySize()))) +} + +func (self *ESENT_SPACE_TREE_HEADER) Dump() { + fmt.Println(self.DebugString()) +} + +func (self *ESENT_LEAF_HEADER) Dump() { + fmt.Println(self.DebugString()) +} + +// WalkPages walks the b tree starting with the page id specified and +// extracts all tagged values into the callback. The callback may +// return an error which will cause WalkPages to stop and relay that +// error to our caller. +func WalkPages(ctx *ESEContext, + id int64, + cb func(header *PageHeader, page_id int64, value *Value) error) error { + seen := make(map[int64]bool) + + return _walkPages(ctx, id, seen, cb) +} + +func _walkPages(ctx *ESEContext, + id int64, seen map[int64]bool, + cb func(header *PageHeader, page_id int64, value *Value) error) error { + + _, pres := seen[id] + if id <= 0 || pres { + return nil + } + seen[id] = true + + header := ctx.GetPage(id) + values := GetPageValues(ctx, header, id) + if DebugWalk { + fmt.Printf("Walking page %v %v\n", id, header.DebugString()) + } + + // No more records. + if len(values) == 0 { + return nil + } + + for _, value := range values { + if header.IsLeaf() { + // Allow the callback to return early (e.g. in case of + // cancellation) + err := cb(header, id, value) + if err != nil { + return err + } + } else if header.IsBranch() { + // Walk the branch + branch := NewESENT_BRANCH_ENTRY(ctx, value) + err := _walkPages(ctx, branch.ChildPageNumber(), seen, cb) + if err != nil { + return err + } + } + } + + if header.NextPageNumber() > 0 { + err := _walkPages(ctx, int64(header.NextPageNumber()), seen, cb) + if err != nil { + return err + } + } + + return nil +} diff --git a/internal/eseparser/reader.go b/internal/eseparser/reader.go new file mode 100644 index 0000000..477edc0 --- /dev/null +++ b/internal/eseparser/reader.go @@ -0,0 +1,53 @@ +package eseparser + +import "io" + +type BufferReaderAt struct { + buffer []byte +} + +func (self *BufferReaderAt) ReadAt(buf []byte, offset int64) (int, error) { + to_read := int64(len(buf)) + if offset < 0 { + to_read += offset + offset = 0 + } + + if offset+to_read > int64(len(self.buffer)) { + to_read = int64(len(self.buffer)) - offset + } + + if to_read < 0 { + return 0, nil + } + + n := copy(buf, self.buffer[offset:offset+to_read]) + + return n, nil +} + +type OffsetReader struct { + reader io.ReaderAt + offset int64 + length int64 +} + +func (self OffsetReader) ReadAt(buff []byte, off int64) (int, error) { + to_read := int64(len(buff)) + if off+to_read > self.length { + to_read = self.length - off + } + + if to_read < 0 { + return 0, nil + } + return self.reader.ReadAt(buff, off+self.offset) +} + +func NewOffsetReader(reader io.ReaderAt, offset, size int64) io.ReaderAt { + return &OffsetReader{ + reader: reader, + offset: offset, + length: offset + size, + } +} diff --git a/internal/eseparser/utils.go b/internal/eseparser/utils.go new file mode 100644 index 0000000..b34101f --- /dev/null +++ b/internal/eseparser/utils.go @@ -0,0 +1,15 @@ +package eseparser + +import ( + "io" + "time" +) + +func WinFileTime64(reader io.ReaderAt, offset int64) time.Time { + value := ParseInt64(reader, offset) + return time.Unix((value/10000000)-11644473600, 0).UTC() +} + +func IsSmallPage(page_size int64) bool { + return page_size <= 1024*8 +} diff --git a/internal/ie/cookiestore.go b/internal/ie/cookiestore.go index 3faa0d4..5b6caad 100644 --- a/internal/ie/cookiestore.go +++ b/internal/ie/cookiestore.go @@ -7,9 +7,8 @@ import ( "github.com/browserutils/kooky" "github.com/browserutils/kooky/internal/cookies" + "github.com/browserutils/kooky/internal/eseparser" "github.com/browserutils/kooky/internal/utils" - - "www.velocidex.com/golang/go-ese/parser" ) type CookieStore struct { @@ -44,7 +43,7 @@ var _ cookies.CookieStore = (*IECacheCookieStore)(nil) type ESECookieStore struct { cookies.DefaultCookieStore - ESECatalog *parser.Catalog + ESECatalog *eseparser.Catalog } var _ cookies.CookieStore = (*ESECookieStore)(nil) @@ -71,12 +70,12 @@ func (s *ESECookieStore) Open() error { return err } - ese_ctx, err := parser.NewESEContext(s.File) + ese_ctx, err := eseparser.NewESEContext(s.File) if err != nil { return err } - catalog, err := parser.ReadCatalog(ese_ctx) + catalog, err := eseparser.ReadCatalog(ese_ctx) if err != nil { return err } diff --git a/internal/ie/ese.go b/internal/ie/ese.go index 51a8bf2..d249f03 100644 --- a/internal/ie/ese.go +++ b/internal/ie/ese.go @@ -9,8 +9,8 @@ import ( "github.com/browserutils/kooky" "github.com/browserutils/kooky/internal/timex" - "github.com/Velocidex/ordereddict" - "www.velocidex.com/golang/go-ese/parser" + "github.com/browserutils/kooky/internal/eseparser" + "github.com/browserutils/kooky/internal/eseparser/ordereddict" ) func (s *ESECookieStore) ReadCookies(filters ...kooky.Filter) ([]*kooky.Cookie, error) { @@ -212,13 +212,13 @@ type webCacheContainer struct { directory string // Directory } -func getEdgeCookieDirectories(catalog *parser.Catalog) ([]webCacheContainer, error) { +func getEdgeCookieDirectories(catalog *eseparser.Catalog) ([]webCacheContainer, error) { var cookiesContainers []webCacheContainer cbContainers := func(row *ordereddict.Dict) error { var name, directory string if n, ok := row.GetString(`Name`); ok { - name = strings.TrimRight(parser.UTF16BytesToUTF8([]byte(n), binary.LittleEndian), "\x00") + name = strings.TrimRight(eseparser.UTF16BytesToUTF8([]byte(n), binary.LittleEndian), "\x00") } else { return nil }