Skip to content

Commit

Permalink
improved pdz11 xformat
Browse files Browse the repository at this point in the history
  • Loading branch information
@fligt
fligt committed Jul 3, 2024
1 parent d3a7147 commit 2b50ba6
Show file tree
Hide file tree
Showing 2 changed files with 181 additions and 55 deletions.
220 changes: 171 additions & 49 deletions notebooks/40_pdz-archeology.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -165,10 +165,14 @@
"\n",
"```python\n",
"PDZ11_STRUCT_DICT = {\n",
" 'pdz11_8698_bytes' : {'xformat': '2s-4s-h-154s-f-192s-Z-148s', \n",
" 'param_keys': ['pdz-version', '??', 'NumberOfChannels', '??', 'XrayVoltageInkV', '??', 'PhotonCounts', '??']}, \n",
" 'pdz11_4454_bytes' : {'xformat': '2s-4s-h-154s-f-28s-z-148s', \n",
" 'param_keys': ['pdz-version', '??', 'NumberOfChannels', '??', 'XrayVoltageInkV', '??', 'PhotonCounts', '??']}\n",
" 'pdz11_8698_bytes' : {'xformat': '2s-4s-h-34s-d-d-86s-i-i-10s-f-f-188s-Z-148s', \n",
" 'param_keys': ['pdz-version', '??', 'NumberOfChannels', '??', 'LiveTimeInSeconds', \n",
" 'eVPerChannel', '??', 'RawCounts', 'ValidCounts', '??', 'XrayVoltageInkV', \n",
" 'XrayFilamentCurrentInMicroAmps', '??', 'PhotonCounts (2048 channels)', '??']}, \n",
" 'pdz11_4454_bytes' : {'xformat': '2s-4s-h-34s-d-d-86s-i-i-10s-f-f-24s-z-148s', \n",
" 'param_keys': ['pdz-version', '??', 'NumberOfChannels', '??', 'LiveTimeInSeconds', \n",
" 'eVPerChannel', '??', 'RawCounts', 'ValidCounts', '??', 'XrayVoltageInkV', \n",
" 'XrayFilamentCurrentInMicroAmps', '??', 'PhotonCounts (1024 channels)', '??']} \n",
"}\n",
"```\n"
]
Expand All @@ -178,7 +182,7 @@
"id": "4c529835-44fe-4f6d-a1f3-3fea390b6173",
"metadata": {},
"source": [
"So far I came across two types of pdz11 files with file sizes of 8698 or 4454 bytes. The difference in file size is due to the different number of detector channels (1024 versus 2048) for different instruments. The extended format string `xformat` that is required to parse these files is slightly different "
"So far I came across two types of pdz11 files with file sizes of 8698 or 4454 bytes. The difference in file size is due to the different number of detector channels (1024 versus 2048) for different instruments. The extended format string `xformat` that is required to parse these files is slightly different. Given the exact pdz file type we can now also parse legacy pdz files as shown below. "
]
},
{
Expand All @@ -191,7 +195,9 @@
"name": "stdout",
"output_type": "stream",
"text": [
"file: 'ANALYZE_EMP-10.pdz' --> pdz type: 'pdz11_8698_bytes' --> xformat: '2s-4s-h-154s-f-192s-Z-148s'\n"
"file: 'ANALYZE_EMP-10.pdz'\n",
"--> pdz type: 'pdz11_8698_bytes'\n",
"--> xformat string: '2s-4s-h-34s-d-d-86s-i-i-10s-f-f-188s-Z-148s'\n"
]
}
],
Expand All @@ -204,15 +210,7 @@
"\n",
"xformat = PDZ11_STRUCT_DICT[pdz_type]['xformat']\n",
"param_keys = PDZ11_STRUCT_DICT[pdz_type]['param_keys']\n",
"print(f\"file: '{pdz_file}' --> pdz type: '{pdz_type}' --> xformat: '{xformat}'\")"
]
},
{
"cell_type": "markdown",
"id": "b68e3ef4-08c6-46a9-83ef-132679c1ee51",
"metadata": {},
"source": [
"Given the exact pdz file type we can now also parse legacy pdz files. "
"print(f\"file: '{pdz_file}'\\n--> pdz type: '{pdz_type}'\\n--> xformat string: '{xformat}'\")"
]
},
{
Expand Down Expand Up @@ -269,21 +267,56 @@
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>60.73299</td>\n",
" <td>LiveTimeInSeconds</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>20.010274</td>\n",
" <td>eVPerChannel</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>b'\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00...</td>\n",
" <td>??</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>131792896</td>\n",
" <td>RawCounts</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>131084</td>\n",
" <td>ValidCounts</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>b'\\r\\x00\\x02\\x00\\x0b\\x002\\x00\\x00\\x00'</td>\n",
" <td>??</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>40.0</td>\n",
" <td>XrayVoltageInkV</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>b'ff&amp;A\\xa9\\xc7\\x00\\xc9\\xfe\\x01\\x01\\x01\\xfe\\xbe...</td>\n",
" <th>11</th>\n",
" <td>10.4</td>\n",
" <td>XrayFilamentCurrentInMicroAmps</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>b'\\xa9\\xc7\\x00\\xc9\\xfe\\x01\\x01\\x01\\xfe\\xbe\\x98...</td>\n",
" <td>??</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <th>13</th>\n",
" <td>[1869, 1524, 889, 469, 150, 42, 9, 2, 2, 1, 1,...</td>\n",
" <td>PhotonCounts</td>\n",
" <td>PhotonCounts (2048 channels)</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <th>14</th>\n",
" <td>b'\\x04\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x03\\x14\\x00...</td>\n",
" <td>??</td>\n",
" </tr>\n",
Expand All @@ -292,15 +325,39 @@
"</div>"
],
"text/plain": [
" values param_keys\n",
"0 b'\\x01\\x01' pdz-version\n",
"1 b'\\x17\\x00\\x00\\x00' ??\n",
"2 2048 NumberOfChannels\n",
"3 b'\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00... ??\n",
"4 40.0 XrayVoltageInkV\n",
"5 b'ff&A\\xa9\\xc7\\x00\\xc9\\xfe\\x01\\x01\\x01\\xfe\\xbe... ??\n",
"6 [1869, 1524, 889, 469, 150, 42, 9, 2, 2, 1, 1,... PhotonCounts\n",
"7 b'\\x04\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x03\\x14\\x00... ??"
" values \\\n",
"0 b'\\x01\\x01' \n",
"1 b'\\x17\\x00\\x00\\x00' \n",
"2 2048 \n",
"3 b'\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00... \n",
"4 60.73299 \n",
"5 20.010274 \n",
"6 b'\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00... \n",
"7 131792896 \n",
"8 131084 \n",
"9 b'\\r\\x00\\x02\\x00\\x0b\\x002\\x00\\x00\\x00' \n",
"10 40.0 \n",
"11 10.4 \n",
"12 b'\\xa9\\xc7\\x00\\xc9\\xfe\\x01\\x01\\x01\\xfe\\xbe\\x98... \n",
"13 [1869, 1524, 889, 469, 150, 42, 9, 2, 2, 1, 1,... \n",
"14 b'\\x04\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x03\\x14\\x00... \n",
"\n",
" param_keys \n",
"0 pdz-version \n",
"1 ?? \n",
"2 NumberOfChannels \n",
"3 ?? \n",
"4 LiveTimeInSeconds \n",
"5 eVPerChannel \n",
"6 ?? \n",
"7 RawCounts \n",
"8 ValidCounts \n",
"9 ?? \n",
"10 XrayVoltageInkV \n",
"11 XrayFilamentCurrentInMicroAmps \n",
"12 ?? \n",
"13 PhotonCounts (2048 channels) \n",
"14 ?? "
]
},
"metadata": {},
Expand All @@ -321,7 +378,9 @@
"name": "stdout",
"output_type": "stream",
"text": [
"file: '3-brown left edge sample spot.pdz' --> pdz type: 'pdz11_4454_bytes' --> xformat: '2s-4s-h-154s-f-28s-z-148s'\n"
"file: '3-brown left edge sample spot.pdz'\n",
"--> pdz type: 'pdz11_4454_bytes'\n",
"--> xformat string: '2s-4s-h-34s-d-d-86s-i-i-10s-f-f-24s-z-148s'\n"
]
}
],
Expand All @@ -334,7 +393,7 @@
"\n",
"xformat = PDZ11_STRUCT_DICT[pdz_type]['xformat']\n",
"param_keys = PDZ11_STRUCT_DICT[pdz_type]['param_keys']\n",
"print(f\"file: '{pdz_file}' --> pdz type: '{pdz_type}' --> xformat: '{xformat}'\")"
"print(f\"file: '{pdz_file}'\\n--> pdz type: '{pdz_type}'\\n--> xformat string: '{xformat}'\")"
]
},
{
Expand Down Expand Up @@ -391,21 +450,56 @@
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>60.76</td>\n",
" <td>LiveTimeInSeconds</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>40.0</td>\n",
" <td>eVPerChannel</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>b'\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00...</td>\n",
" <td>??</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>1001652224</td>\n",
" <td>RawCounts</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>519897088</td>\n",
" <td>ValidCounts</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>b'\\x00\\x00\\xc4w\\r\\x000Y\\x07\\x00'</td>\n",
" <td>??</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>40.0</td>\n",
" <td>XrayVoltageInkV</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>b'\\xcd\\xcc\\x0c@\\xb9\\xa5\\x01\\x10\\xfe\\x00\\x00\\xf...</td>\n",
" <th>11</th>\n",
" <td>2.2</td>\n",
" <td>XrayFilamentCurrentInMicroAmps</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>b'\\xb9\\xa5\\x01\\x10\\xfe\\x00\\x00\\xff\\x00\\x00\\x00...</td>\n",
" <td>??</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <th>13</th>\n",
" <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
" <td>PhotonCounts</td>\n",
" <td>PhotonCounts (1024 channels)</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <th>14</th>\n",
" <td>b'\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x01\\x00\\x00...</td>\n",
" <td>??</td>\n",
" </tr>\n",
Expand All @@ -414,15 +508,39 @@
"</div>"
],
"text/plain": [
" values param_keys\n",
"0 b'\\x01\\x01' pdz-version\n",
"1 b'\\x14\\x00\\x00\\x00' ??\n",
"2 1024 NumberOfChannels\n",
"3 b'\\xc8\\x00\\xe6\\x00\\xc8\\x00\\xca\\x00\\xd6\\x00\\xd8... ??\n",
"4 40.0 XrayVoltageInkV\n",
"5 b'\\xcd\\xcc\\x0c@\\xb9\\xa5\\x01\\x10\\xfe\\x00\\x00\\xf... ??\n",
"6 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ... PhotonCounts\n",
"7 b'\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x01\\x00\\x00... ??"
" values \\\n",
"0 b'\\x01\\x01' \n",
"1 b'\\x14\\x00\\x00\\x00' \n",
"2 1024 \n",
"3 b'\\xc8\\x00\\xe6\\x00\\xc8\\x00\\xca\\x00\\xd6\\x00\\xd8... \n",
"4 60.76 \n",
"5 40.0 \n",
"6 b'\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00... \n",
"7 1001652224 \n",
"8 519897088 \n",
"9 b'\\x00\\x00\\xc4w\\r\\x000Y\\x07\\x00' \n",
"10 40.0 \n",
"11 2.2 \n",
"12 b'\\xb9\\xa5\\x01\\x10\\xfe\\x00\\x00\\xff\\x00\\x00\\x00... \n",
"13 [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ... \n",
"14 b'\\x00\\x00\\x00\\x00\\x02\\x00\\x00\\x00\\x01\\x00\\x00... \n",
"\n",
" param_keys \n",
"0 pdz-version \n",
"1 ?? \n",
"2 NumberOfChannels \n",
"3 ?? \n",
"4 LiveTimeInSeconds \n",
"5 eVPerChannel \n",
"6 ?? \n",
"7 RawCounts \n",
"8 ValidCounts \n",
"9 ?? \n",
"10 XrayVoltageInkV \n",
"11 XrayFilamentCurrentInMicroAmps \n",
"12 ?? \n",
"13 PhotonCounts (1024 channels) \n",
"14 ?? "
]
},
"metadata": {},
Expand Down Expand Up @@ -464,10 +582,14 @@
"#|export \n",
"\n",
"PDZ11_STRUCT_DICT = {\n",
" 'pdz11_8698_bytes' : {'xformat': '2s-4s-h-154s-f-192s-Z-148s', \n",
" 'param_keys': ['pdz-version', '??', 'NumberOfChannels', '??', 'XrayVoltageInkV', '??', 'PhotonCounts', '??']}, \n",
" 'pdz11_4454_bytes' : {'xformat': '2s-4s-h-154s-f-28s-z-148s', \n",
" 'param_keys': ['pdz-version', '??', 'NumberOfChannels', '??', 'XrayVoltageInkV', '??', 'PhotonCounts', '??']}\n",
" 'pdz11_8698_bytes' : {'xformat': '2s-4s-h-34s-d-d-86s-i-i-10s-f-f-188s-Z-148s', \n",
" 'param_keys': ['pdz-version', '??', 'NumberOfChannels', '??', 'LiveTimeInSeconds', \n",
" 'eVPerChannel', '??', 'RawCounts', 'ValidCounts', '??', 'XrayVoltageInkV', \n",
" 'XrayFilamentCurrentInMicroAmps', '??', 'PhotonCounts (2048 channels)', '??']}, \n",
" 'pdz11_4454_bytes' : {'xformat': '2s-4s-h-34s-d-d-86s-i-i-10s-f-f-24s-z-148s', \n",
" 'param_keys': ['pdz-version', '??', 'NumberOfChannels', '??', 'LiveTimeInSeconds', \n",
" 'eVPerChannel', '??', 'RawCounts', 'ValidCounts', '??', 'XrayVoltageInkV', \n",
" 'XrayFilamentCurrentInMicroAmps', '??', 'PhotonCounts (1024 channels)', '??']} \n",
"}\n",
"\n",
"\n",
Expand Down
16 changes: 10 additions & 6 deletions read_pdz/legacy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,20 @@
# %% auto 0
__all__ = ['PDZ11_STRUCT_DICT', 'check_pdz_type']

# %% ../notebooks/40_pdz-archeology.ipynb 17
# %% ../notebooks/40_pdz-archeology.ipynb 16
from . import file_to_bytes, get_blocks, multiparse, parse
import struct

# %% ../notebooks/40_pdz-archeology.ipynb 18
# %% ../notebooks/40_pdz-archeology.ipynb 17
PDZ11_STRUCT_DICT = {
'pdz11_8698_bytes' : {'xformat': '2s-4s-h-154s-f-192s-Z-148s',
'param_keys': ['pdz-version', '??', 'NumberOfChannels', '??', 'XrayVoltageInkV', '??', 'PhotonCounts', '??']},
'pdz11_4454_bytes' : {'xformat': '2s-4s-h-154s-f-28s-z-148s',
'param_keys': ['pdz-version', '??', 'NumberOfChannels', '??', 'XrayVoltageInkV', '??', 'PhotonCounts', '??']}
'pdz11_8698_bytes' : {'xformat': '2s-4s-h-34s-d-d-86s-i-i-10s-f-f-188s-Z-148s',
'param_keys': ['pdz-version', '??', 'NumberOfChannels', '??', 'LiveTimeInSeconds',
'eVPerChannel', '??', 'RawCounts', 'ValidCounts', '??', 'XrayVoltageInkV',
'XrayFilamentCurrentInMicroAmps', '??', 'PhotonCounts (2048 channels)', '??']},
'pdz11_4454_bytes' : {'xformat': '2s-4s-h-34s-d-d-86s-i-i-10s-f-f-24s-z-148s',
'param_keys': ['pdz-version', '??', 'NumberOfChannels', '??', 'LiveTimeInSeconds',
'eVPerChannel', '??', 'RawCounts', 'ValidCounts', '??', 'XrayVoltageInkV',
'XrayFilamentCurrentInMicroAmps', '??', 'PhotonCounts (1024 channels)', '??']}
}


Expand Down

0 comments on commit 2b50ba6

Please sign in to comment.