pianoroll · broadwell · Jul 11, 2024 · Jul 9, 2024 · Jul 10, 2024 · Jul 11, 2024
diff --git a/.github/workflows/update-data.yml b/.github/workflows/update-data.yml
@@ -2,14 +2,14 @@ name: update-data
 
 on:
   push:
-    branches: [ main ]
+    branches: [main]
   pull_request:
-    branches: [ main ]
+    branches: [main]
 
   workflow_dispatch:
-  
+
   #schedule:
-  #  - cron: "0 0 1 * *" # Runs at 00:00 UTC on the first of every month 
+  #  - cron: "0 0 1 * *" # Runs at 00:00 UTC on the first of every month
 
 jobs:
   build:
@@ -23,33 +23,33 @@ jobs:
         uses: actions/setup-python@v3
         with:
           python-version: 3.9
-        
+
       - name: install pipenv
         run: |
           pip install --user pipenv
           USER_BASE=`python -m site --user-base`
           export PATH=$USER_BASE/bin:$PATH
-        
+
       - name: install Python dependencies
         run: pipenv install
-      
+
       - name: clone roll-wrangler repo
         run: |
           git clone https://github.com/pianoroll/roll-wrangler.git
-        
+
       - name: move roll-wrangler files to expected locations
         run: |
           mv ./roll-wrangler/midi/note/*.mid midi/note/.
           mv ./roll-wrangler/midi/exp/*.mid midi/exp/.
           mv ./roll-wrangler/txt/*.txt input/txt/.
-        
+
       - name: run the data generation script
         run: pipenv run python build-pianolatron-files.py
-        
+
       - name: commit the new/updated output files to the repo
         uses: EndBug/[email protected]
         with:
-          add: '["output/json/*.json","output/midi/*.mid","midi/exp/*.mid","midi/note/*.mid","output/catalog.json"]'
-          message: 'Automatic update'
+          add: '["output/json/*.json","output/midi/exp/*.mod","output/midi/note/*.mid","midi/exp/*.mid","midi/note/*.mid","output/catalog.json"]'
+          message: "Automatic update"
           author_name: GitHub Actions
           author_email: [email protected]
diff --git a/build-pianolatron-files.py b/build-pianolatron-files.py
@@ -31,6 +31,7 @@
     "Welte-Mignon red roll (T-100)": "welte-red",
     "Welte-Mignon red roll (T-100).": "welte-red",
     "Welte-Mignon red roll (T-100)..": "welte-red",
+    "Scale: 88n": "88-note",
     "Scale: 88n.": "88-note",
     "Scale: 65n.": "65-note",
     "88n": "88-note",
@@ -120,7 +121,13 @@ def get_value_by_xpaths(xpaths):
 
     if roll_type == "NA" or type_note == "standard":
         for note in xml_tree.xpath("(x:note)", namespaces=NS):
-            if note is not None and note.text in ROLL_TYPES:
+            if (
+                note is not None
+                and note.text in ROLL_TYPES
+                # Most rolls of any type are marked as "88n", so don't let this
+                # setting overwrite a more specific roll type note.
+                and (ROLL_TYPES[note.text] != "88-note" or roll_type == "NA")
+            ):
                 roll_type = ROLL_TYPES[note.text]
 
     metadata = {
@@ -211,13 +218,13 @@ def get_value_by_xpaths(xpaths):
     # Derive the value for the IIIF info.json file URL, which is eventually
     # used to display the roll image in a viewer such as OpenSeadragon
     image_id = re.search(
-        r"^.*?<label>(?:display image|jp2|Image 1)<\/label>.*?<file id=\"([^\.]*)\.jp2",
+        r"^.*?<label>(?:display image|jp2|[Ii]mage 1)<\/label>.*?<file id=\"([^\.]*)\.jp2",
         xml_data,
         re.MULTILINE | re.DOTALL,
     ).group(1)
-    metadata[
-        "image_url"
-    ] = f"https://stacks.stanford.edu/image/iiif/{image_id.split('_')[0]}/{image_id}/info.json"
+    metadata["image_url"] = (
+        f"https://stacks.stanford.edu/image/iiif/{image_id.split('_')[0]}/{image_id}/info.json"
+    )
 
     return metadata
 
@@ -249,7 +256,7 @@ def merge_midi_velocities(roll_data, hole_data, druid, roll_type):
     output file for use when highlighting the note holes in the roll when it is
     displayed in the Pianolatron app."""
 
-    midi_filepath = Path(f"output/midi/{druid}.mid")
+    midi_filepath = Path(f"output/midi/exp/{druid}.mid")
 
     if not midi_filepath.exists():
         logging.info(
@@ -342,7 +349,17 @@ def get_hole_report_data(druid, analysis_source_dir):
                 if key in roll_keys:
                     roll_data[key] = value.replace("px", "").strip()
 
-        while (line := _fh.readline()) and line != "@@END: HOLES\n":
+        # Out-of-spec holes are marked as "BAD" in a special section of the
+        # @ATON .txt hole data file, but are still interpreted as control or
+        # note holes when generating the MIDI file for the roll. So it seems
+        # best to include their data in the output JSON file so that they'll be
+        # highlighted properly in the player.
+        # NOTE that this parsing procedure assumes the BADHOLES section always
+        # follows the end of the HOLES section, and that the hole data file
+        # always has a BADHOLES section even when no bad holes have been
+        # detected on the roll.
+
+        while (line := _fh.readline()) and line != "@@END: BADHOLES\n":
             if line == "@@BEGIN: HOLE\n":
                 hole = {}
             if match := re.match(r"^@([^@\s]+):\s+(.*)", line):
@@ -355,8 +372,10 @@ def get_hole_report_data(druid, analysis_source_dir):
                     assert hole["NOTE_ATTACK"] == hole["ORIGIN_ROW"]
                     del hole["NOTE_ATTACK"]
                     if hole["ORIGIN_ROW"] >= hole["OFF_TIME"]:
-                        logging.info(f"WARNING: invalid note duration: {hole}")
-                    hole_data.append(hole)
+                        # logging.info(f"WARNING: invalid note duration: {hole}")
+                        dropped_holes += 1
+                    else:
+                        hole_data.append(hole)
                 else:
                     assert "OFF_TIME" not in hole
                     dropped_holes += 1
@@ -586,15 +605,10 @@ def main():
         action="store_true",
         help="Always download XML files, overwriting files in input/xml/",
     )
-    argparser.add_argument(
-        "--use-exp-midi",
-        action="store_true",
-        help="Use expressionized MIDI for output .mid files (default is to use note MIDI)",
-    )
     argparser.add_argument(
         "--midi-source-dir",
         default=MIDI_DIR,
-        help="Folder containg note (DIR/note/DRUID_note.mid) or expressionized (DIR/exp/DRUID_exp.mid) MIDI files",
+        help="Folder containg note (DIR/note/DRUID_note.mid) and expression (DIR/exp/DRUID_exp.mid) MIDI files",
     )
     argparser.add_argument(
         "--analysis-source-dir",
@@ -636,24 +650,17 @@ def main():
             logging.info(f"Unable to get metadata for DRUID {druid}, skipping")
             continue
 
-        if (
-            not args.use_exp_midi
-            and Path(f"{args.midi_source_dir}/note/{druid}_note.mid").exists()
-        ) or (
-            args.use_exp_midi
-            and not Path(f"{args.midi_source_dir}/exp/{druid}_exp.mid").exists()
-        ):
-            copy(
-                Path(f"{args.midi_source_dir}/note/{druid}_note.mid"),
-                Path(f"output/midi/{druid}.mid"),
-            )
-            note_midi = MidiFile(Path(f"output/midi/{druid}.mid"))
-            metadata["NOTE_MIDI_TPQ"] = note_midi.ticks_per_beat
-        elif Path(f"{args.midi_source_dir}/exp/{druid}_exp.mid").exists():
-            copy(
-                Path(f"{args.midi_source_dir}/exp/{druid}_exp.mid"),
-                Path(f"output/midi/{druid}.mid"),
-            )
+        copy(
+            Path(f"{args.midi_source_dir}/note/{druid}_note.mid"),
+            Path(f"output/midi/note/{druid}.mid"),
+        )
+        note_midi = MidiFile(Path(f"output/midi/note/{druid}.mid"))
+        metadata["NOTE_MIDI_TPQ"] = note_midi.ticks_per_beat
+
+        copy(
+            Path(f"{args.midi_source_dir}/exp/{druid}_exp.mid"),
+            Path(f"output/midi/exp/{druid}.mid"),
+        )
 
         if WRITE_TEMPO_MAPS:
             metadata["tempoMap"] = build_tempo_map_from_midi(druid)

diff --git a/input/druids/gen2_88_note_druids.txt b/input/druids/gen2_88_note_druids.txt
@@ -0,0 +1,17 @@
+bv720gq2633
+bx696fq3697
+cn498vz4750
+ff220ct4404
+fz108ts9393
+gp583km0927
+kh288bx3828
+ky144fd6229
+nr538vy9112
+ns724rz6542
+ny190nz5753
+ph633tr8751
+tn472jy2817
+wb438xf5756
+wt920yb3768
+xn815zk7379
+xn861tf7013
diff --git a/midi/exp/bh573cc4821_exp.mid b/midi/exp/bh573cc4821_exp.mid
diff --git a/midi/exp/bp752qs6053_exp.mid b/midi/exp/bp752qs6053_exp.mid
diff --git a/midi/exp/br324td1538_exp.mid b/midi/exp/br324td1538_exp.mid
diff --git a/midi/exp/br351xc9456_exp.mid b/midi/exp/br351xc9456_exp.mid
diff --git a/midi/exp/bv720gq2633_exp.mid b/midi/exp/bv720gq2633_exp.mid
diff --git a/midi/exp/bx696fq3697_exp.mid b/midi/exp/bx696fq3697_exp.mid
diff --git a/midi/exp/ch984dr4932_exp.mid b/midi/exp/ch984dr4932_exp.mid
diff --git a/midi/exp/cn498vz4750_exp.mid b/midi/exp/cn498vz4750_exp.mid
diff --git a/midi/exp/cw746gb6753_exp.mid b/midi/exp/cw746gb6753_exp.mid
diff --git a/midi/exp/cy287wz7683_exp.mid b/midi/exp/cy287wz7683_exp.mid
diff --git a/midi/exp/dk418xy4378_exp.mid b/midi/exp/dk418xy4378_exp.mid
diff --git a/midi/exp/dv468zh8208_exp.mid b/midi/exp/dv468zh8208_exp.mid
diff --git a/midi/exp/ff220ct4404_exp.mid b/midi/exp/ff220ct4404_exp.mid
diff --git a/midi/exp/fz108ts9393_exp.mid b/midi/exp/fz108ts9393_exp.mid
diff --git a/midi/exp/gn803sk7089_exp.mid b/midi/exp/gn803sk7089_exp.mid
diff --git a/midi/exp/gp583km0927_exp.mid b/midi/exp/gp583km0927_exp.mid
diff --git a/midi/exp/gv741wj6162_exp.mid b/midi/exp/gv741wj6162_exp.mid
diff --git a/midi/exp/hb523vs3190_exp.mid b/midi/exp/hb523vs3190_exp.mid
diff --git a/midi/exp/hh361xy4933_exp.mid b/midi/exp/hh361xy4933_exp.mid
diff --git a/midi/exp/hj502vv5162_exp.mid b/midi/exp/hj502vv5162_exp.mid
diff --git a/midi/exp/ht999gf1829_exp.mid b/midi/exp/ht999gf1829_exp.mid
diff --git a/midi/exp/jg489yw0942_exp.mid b/midi/exp/jg489yw0942_exp.mid
diff --git a/midi/exp/jp202jq5812_exp.mid b/midi/exp/jp202jq5812_exp.mid
diff --git a/midi/exp/jv485mx5227_exp.mid b/midi/exp/jv485mx5227_exp.mid
diff --git a/midi/exp/jy320zw1102_exp.mid b/midi/exp/jy320zw1102_exp.mid
diff --git a/midi/exp/kh288bx3828_exp.mid b/midi/exp/kh288bx3828_exp.mid
diff --git a/midi/exp/km927sd2616_exp.mid b/midi/exp/km927sd2616_exp.mid
diff --git a/midi/exp/ky144fd6229_exp.mid b/midi/exp/ky144fd6229_exp.mid
diff --git a/midi/exp/kz454tt7354_exp.mid b/midi/exp/kz454tt7354_exp.mid
diff --git a/midi/exp/ms604gp1206_exp.mid b/midi/exp/ms604gp1206_exp.mid
diff --git a/midi/exp/mw139zj5450_exp.mid b/midi/exp/mw139zj5450_exp.mid
diff --git a/midi/exp/nr538vy9112_exp.mid b/midi/exp/nr538vy9112_exp.mid
diff --git a/midi/exp/ns724rz6542_exp.mid b/midi/exp/ns724rz6542_exp.mid
diff --git a/midi/exp/nw222vm1898_exp.mid b/midi/exp/nw222vm1898_exp.mid
diff --git a/midi/exp/ny190nz5753_exp.mid b/midi/exp/ny190nz5753_exp.mid
diff --git a/midi/exp/ny606dc0992_exp.mid b/midi/exp/ny606dc0992_exp.mid
diff --git a/midi/exp/ph633tr8751_exp.mid b/midi/exp/ph633tr8751_exp.mid
diff --git a/midi/exp/ps866sb0637_exp.mid b/midi/exp/ps866sb0637_exp.mid
diff --git a/midi/exp/pz594dj8436_exp.mid b/midi/exp/pz594dj8436_exp.mid
diff --git a/midi/exp/qd454gb9111_exp.mid b/midi/exp/qd454gb9111_exp.mid
diff --git a/midi/exp/qf152xr6941_exp.mid b/midi/exp/qf152xr6941_exp.mid
diff --git a/midi/exp/qh464nj1973_exp.mid b/midi/exp/qh464nj1973_exp.mid
diff --git a/midi/exp/qj765hb3428_exp.mid b/midi/exp/qj765hb3428_exp.mid
diff --git a/midi/exp/rh357hg0211_exp.mid b/midi/exp/rh357hg0211_exp.mid
diff --git a/midi/exp/rs892mq5364_exp.mid b/midi/exp/rs892mq5364_exp.mid
diff --git a/midi/exp/rx870zt5437_exp.mid b/midi/exp/rx870zt5437_exp.mid
diff --git a/midi/exp/sb388kg1678_exp.mid b/midi/exp/sb388kg1678_exp.mid
diff --git a/midi/exp/sm344xv2220_exp.mid b/midi/exp/sm344xv2220_exp.mid
diff --git a/midi/exp/sz948zd1422_exp.mid b/midi/exp/sz948zd1422_exp.mid
diff --git a/midi/exp/tc493dq4498_exp.mid b/midi/exp/tc493dq4498_exp.mid
diff --git a/midi/exp/tn412mq0486_exp.mid b/midi/exp/tn412mq0486_exp.mid
diff --git a/midi/exp/tn472jy2817_exp.mid b/midi/exp/tn472jy2817_exp.mid
diff --git a/midi/exp/tp613xb9088_exp.mid b/midi/exp/tp613xb9088_exp.mid
diff --git a/midi/exp/vc908xz3554_exp.mid b/midi/exp/vc908xz3554_exp.mid
diff --git a/midi/exp/vc943nk4921_exp.mid b/midi/exp/vc943nk4921_exp.mid
diff --git a/midi/exp/vk572hg6636_exp.mid b/midi/exp/vk572hg6636_exp.mid
diff --git a/midi/exp/vq862qw5039_exp.mid b/midi/exp/vq862qw5039_exp.mid
diff --git a/midi/exp/vr247rd6766_exp.mid b/midi/exp/vr247rd6766_exp.mid
diff --git a/midi/exp/vz386xg9451_exp.mid b/midi/exp/vz386xg9451_exp.mid
diff --git a/midi/exp/wb438xf5756_exp.mid b/midi/exp/wb438xf5756_exp.mid
diff --git a/midi/exp/wq696jd1149_exp.mid b/midi/exp/wq696jd1149_exp.mid
diff --git a/midi/exp/wt920yb3768_exp.mid b/midi/exp/wt920yb3768_exp.mid
diff --git a/midi/exp/wx138dx1211_exp.mid b/midi/exp/wx138dx1211_exp.mid
diff --git a/midi/exp/xn196fs1128_exp.mid b/midi/exp/xn196fs1128_exp.mid
diff --git a/midi/exp/xn815zk7379_exp.mid b/midi/exp/xn815zk7379_exp.mid
diff --git a/midi/exp/xn861tf7013_exp.mid b/midi/exp/xn861tf7013_exp.mid
diff --git a/midi/exp/xy736dn5214_exp.mid b/midi/exp/xy736dn5214_exp.mid
diff --git a/midi/exp/xz373zg4809_exp.mid b/midi/exp/xz373zg4809_exp.mid
diff --git a/midi/exp/zk199jz1564_exp.mid b/midi/exp/zk199jz1564_exp.mid
diff --git a/midi/exp/zm791gz4425_exp.mid b/midi/exp/zm791gz4425_exp.mid
diff --git a/midi/exp/zx785pg9042_exp.mid b/midi/exp/zx785pg9042_exp.mid
diff --git a/midi/exp/zy618kz1670_exp.mid b/midi/exp/zy618kz1670_exp.mid
diff --git a/midi/note/bp752qs6053_note.mid b/midi/note/bp752qs6053_note.mid
diff --git a/midi/note/br351xc9456_note.mid b/midi/note/br351xc9456_note.mid
diff --git a/midi/note/bv720gq2633_note.mid b/midi/note/bv720gq2633_note.mid
diff --git a/midi/note/bx696fq3697_note.mid b/midi/note/bx696fq3697_note.mid
diff --git a/midi/note/ch984dr4932_note.mid b/midi/note/ch984dr4932_note.mid
diff --git a/midi/note/cn498vz4750_note.mid b/midi/note/cn498vz4750_note.mid
diff --git a/midi/note/cw746gb6753_note.mid b/midi/note/cw746gb6753_note.mid
diff --git a/midi/note/cy287wz7683_note.mid b/midi/note/cy287wz7683_note.mid
diff --git a/midi/note/dk418xy4378_note.mid b/midi/note/dk418xy4378_note.mid
diff --git a/midi/note/dv468zh8208_note.mid b/midi/note/dv468zh8208_note.mid
diff --git a/midi/note/ff220ct4404_note.mid b/midi/note/ff220ct4404_note.mid
diff --git a/midi/note/fz108ts9393_note.mid b/midi/note/fz108ts9393_note.mid
diff --git a/midi/note/gn803sk7089_note.mid b/midi/note/gn803sk7089_note.mid
diff --git a/midi/note/gp583km0927_note.mid b/midi/note/gp583km0927_note.mid
diff --git a/midi/note/gv741wj6162_note.mid b/midi/note/gv741wj6162_note.mid
diff --git a/midi/note/hb523vs3190_note.mid b/midi/note/hb523vs3190_note.mid
diff --git a/midi/note/ht999gf1829_note.mid b/midi/note/ht999gf1829_note.mid
diff --git a/midi/note/jg489yw0942_note.mid b/midi/note/jg489yw0942_note.mid
diff --git a/midi/note/jp202jq5812_note.mid b/midi/note/jp202jq5812_note.mid
diff --git a/midi/note/jv485mx5227_note.mid b/midi/note/jv485mx5227_note.mid
diff --git a/midi/note/jy320zw1102_note.mid b/midi/note/jy320zw1102_note.mid
diff --git a/midi/note/kh288bx3828_note.mid b/midi/note/kh288bx3828_note.mid
diff --git a/midi/note/km927sd2616_note.mid b/midi/note/km927sd2616_note.mid
diff --git a/midi/note/ky144fd6229_note.mid b/midi/note/ky144fd6229_note.mid
diff --git a/midi/note/kz454tt7354_note.mid b/midi/note/kz454tt7354_note.mid
diff --git a/midi/note/ms604gp1206_note.mid b/midi/note/ms604gp1206_note.mid
diff --git a/midi/note/mw139zj5450_note.mid b/midi/note/mw139zj5450_note.mid
diff --git a/midi/note/nr538vy9112_note.mid b/midi/note/nr538vy9112_note.mid
diff --git a/midi/note/ns724rz6542_note.mid b/midi/note/ns724rz6542_note.mid
diff --git a/midi/note/nw222vm1898_note.mid b/midi/note/nw222vm1898_note.mid
diff --git a/midi/note/ny190nz5753_note.mid b/midi/note/ny190nz5753_note.mid
diff --git a/midi/note/ph633tr8751_note.mid b/midi/note/ph633tr8751_note.mid
diff --git a/midi/note/ps866sb0637_note.mid b/midi/note/ps866sb0637_note.mid
diff --git a/midi/note/pz594dj8436_note.mid b/midi/note/pz594dj8436_note.mid
diff --git a/midi/note/qd454gb9111_note.mid b/midi/note/qd454gb9111_note.mid
diff --git a/midi/note/qf152xr6941_note.mid b/midi/note/qf152xr6941_note.mid
diff --git a/midi/note/qh464nj1973_note.mid b/midi/note/qh464nj1973_note.mid
diff --git a/midi/note/qj765hb3428_note.mid b/midi/note/qj765hb3428_note.mid
diff --git a/midi/note/rh357hg0211_note.mid b/midi/note/rh357hg0211_note.mid
diff --git a/midi/note/rs892mq5364_note.mid b/midi/note/rs892mq5364_note.mid
diff --git a/midi/note/rx870zt5437_note.mid b/midi/note/rx870zt5437_note.mid
diff --git a/midi/note/sb388kg1678_note.mid b/midi/note/sb388kg1678_note.mid
diff --git a/midi/note/sm344xv2220_note.mid b/midi/note/sm344xv2220_note.mid
diff --git a/midi/note/sz948zd1422_note.mid b/midi/note/sz948zd1422_note.mid
diff --git a/midi/note/tc493dq4498_note.mid b/midi/note/tc493dq4498_note.mid
diff --git a/midi/note/tn412mq0486_note.mid b/midi/note/tn412mq0486_note.mid
diff --git a/midi/note/tn472jy2817_note.mid b/midi/note/tn472jy2817_note.mid
diff --git a/midi/note/tp613xb9088_note.mid b/midi/note/tp613xb9088_note.mid
diff --git a/midi/note/vc908xz3554_note.mid b/midi/note/vc908xz3554_note.mid
diff --git a/midi/note/vc943nk4921_note.mid b/midi/note/vc943nk4921_note.mid
diff --git a/midi/note/vk572hg6636_note.mid b/midi/note/vk572hg6636_note.mid
diff --git a/midi/note/vq862qw5039_note.mid b/midi/note/vq862qw5039_note.mid
diff --git a/midi/note/vr247rd6766_note.mid b/midi/note/vr247rd6766_note.mid
diff --git a/midi/note/vz386xg9451_note.mid b/midi/note/vz386xg9451_note.mid
diff --git a/midi/note/wb438xf5756_note.mid b/midi/note/wb438xf5756_note.mid
diff --git a/midi/note/wq696jd1149_note.mid b/midi/note/wq696jd1149_note.mid
diff --git a/midi/note/wt920yb3768_note.mid b/midi/note/wt920yb3768_note.mid
diff --git a/midi/note/wx138dx1211_note.mid b/midi/note/wx138dx1211_note.mid
diff --git a/midi/note/xn196fs1128_note.mid b/midi/note/xn196fs1128_note.mid
diff --git a/midi/note/xn815zk7379_note.mid b/midi/note/xn815zk7379_note.mid
diff --git a/midi/note/xn861tf7013_note.mid b/midi/note/xn861tf7013_note.mid
diff --git a/midi/note/xy736dn5214_note.mid b/midi/note/xy736dn5214_note.mid
diff --git a/midi/note/xz373zg4809_note.mid b/midi/note/xz373zg4809_note.mid
diff --git a/midi/note/zk199jz1564_note.mid b/midi/note/zk199jz1564_note.mid
diff --git a/midi/note/zm791gz4425_note.mid b/midi/note/zm791gz4425_note.mid
diff --git a/midi/note/zx785pg9042_note.mid b/midi/note/zx785pg9042_note.mid
diff --git a/midi/note/zy618kz1670_note.mid b/midi/note/zy618kz1670_note.mid