use VLQ for delta encoding

ipsilon · May 9, 2024 · 9f0701e · 9f0701e
1 parent 18b5518
commit 9f0701e
Show file tree

Hide file tree

Showing 2 changed files with 411 additions and 406 deletions.
diff --git a/analysis/code_analysis.py b/analysis/code_analysis.py
@@ -143,18 +143,22 @@ def __init__(self, name: str, width: int):
 
     def encode_entry(self, delta: int, value: int) -> list[int]:
         assert 0 <= value <= self.VALUE_MAX
-        skip_only_max = self.SKIP_ONLY - 1
         ops = []
 
-        # Generate skips if needed.
-        while delta > self.VALUE_SKIP_MAX:
-            d = min(delta - self.SKIP_BIAS, skip_only_max)
-            assert 0 <= d <= skip_only_max
-            ops.append(self.SKIP_ONLY | d)
-            delta -= d + self.SKIP_BIAS
+        SKIP_MOD = self.VALUE_SKIP_MAX + 1
+        value_skip = delta % SKIP_MOD
+        ext_skip = delta // SKIP_MOD
 
-        assert 0 <= delta <= self.VALUE_SKIP_MAX
-        encoded = delta * self.VALUE_MOD + value
+        if ext_skip > 0:
+            parts = []
+            while ext_skip > 0:  # TODO: unit test missing for count > 1
+                parts.append(self.SKIP_ONLY | ext_skip % self.SKIP_ONLY)
+                ext_skip = ext_skip // self.SKIP_ONLY
+            assert len(ops) == 0
+            ops = list(reversed(parts))
+
+        assert 0 <= value_skip <= self.VALUE_SKIP_MAX
+        encoded = value_skip * self.VALUE_MOD + value
         assert encoded.bit_length() <= self.WIDTH
         ops.append(encoded)
         return ops
@@ -170,20 +174,21 @@ def encode(self, chunks: dict[int, int]) -> tuple[list[int], int]:
         return ops, self.WIDTH * len(ops)
 
     def decode(self, ops: list[int]) -> dict[int, int]:
-        m = dict()
+        SKIP_MOD = self.VALUE_SKIP_MAX + 1
+        m = {}
         i = 0
+        running_skip = 0
         for op in ops:
             if op & self.SKIP_ONLY:
-                delta = (op ^ self.SKIP_ONLY) + self.SKIP_BIAS
-                value = None
-            else:
-                delta = op // self.VALUE_MOD
-                value = op % self.VALUE_MOD
+                running_skip = running_skip * self.SKIP_ONLY + (op ^ self.SKIP_ONLY)
+                continue
+
+            value_skip = op // self.VALUE_MOD
+            value = op % self.VALUE_MOD
+            delta = running_skip * SKIP_MOD + value_skip
             i += delta
-            print(f"{delta:+4}")
-            if value is not None:
-                m[i] = value
-                print(f"{i:4}: {value}")
+            m[i] = value
+            running_skip = 0  # FIXME: unit test missing
         return m
 
 
@@ -243,8 +248,8 @@ def test_scheme_skip_entry_0():
     chunks = {0: 1, 130: 3}
     ops, _ = sch.encode(chunks)
     assert ops[0] == 0 * sch.VALUE_MOD + 1
-    assert ops[1] == sch.SKIP_ONLY + 127
-    assert ops[2] == 0 * sch.VALUE_MOD + 3
+    # assert ops[1] == sch.SKIP_ONLY + 127
+    # assert ops[2] == 0 * sch.VALUE_MOD + 3
 
     assert sch.decode(ops) == chunks
 
@@ -255,8 +260,8 @@ def test_scheme_skip_entry_2():
     chunks = {0: 1, 132: 3}
     ops, _ = sch.encode(chunks)
     assert ops[0] == 0 * sch.VALUE_MOD + 1
-    assert ops[1] == sch.SKIP_ONLY + 127
-    assert ops[2] == 2 * sch.VALUE_MOD + 3
+    # assert ops[1] == sch.SKIP_ONLY + 127
+    # assert ops[2] == 2 * sch.VALUE_MOD + 3
 
     assert sch.decode(ops) == chunks
 
@@ -266,8 +271,8 @@ def test_scheme_skip_entry_minimal():
 
     chunks = {3: 1}
     ops, _ = sch.encode(chunks)
-    assert ops[0] == sch.SKIP_ONLY + 0
-    assert ops[1] == 0 * sch.VALUE_MOD + 1
+    # assert ops[0] == sch.SKIP_ONLY + 0
+    # assert ops[1] == 0 * sch.VALUE_MOD + 1
 
     assert sch.decode(ops) == chunks
 
@@ -277,10 +282,10 @@ def test_scheme_double_skip_entry():
 
     chunks = {0: 1, 262: 3}
     ops, _ = sch.encode(chunks)
-    assert ops[0] == 0 * sch.VALUE_MOD + 1
-    assert ops[1] == sch.SKIP_ONLY + 130 - sch.SKIP_BIAS
-    assert ops[2] == sch.SKIP_ONLY + 130 - sch.SKIP_BIAS
-    assert ops[3] == 2 * sch.VALUE_MOD + 3
+    # assert ops[0] == 0 * sch.VALUE_MOD + 1
+    # assert ops[1] == sch.SKIP_ONLY + 130 - sch.SKIP_BIAS
+    # assert ops[2] == sch.SKIP_ONLY + 130 - sch.SKIP_BIAS
+    # assert ops[3] == 2 * sch.VALUE_MOD + 3
 
     assert sch.decode(ops) == chunks
 
@@ -290,8 +295,8 @@ def test_scheme_skip_entry_first_0():
 
     chunks = {130: 3}
     ops, _ = sch.encode(chunks)
-    assert ops[0] == sch.SKIP_ONLY + 127
-    assert ops[1] == 0 * sch.VALUE_MOD + 3
+    # assert ops[0] == sch.SKIP_ONLY + 127
+    # assert ops[1] == 0 * sch.VALUE_MOD + 3
 
     assert sch.decode(ops) == chunks
 
@@ -301,8 +306,8 @@ def test_scheme_skip_entry_first_1():
 
     chunks = {131: 3}
     ops, _ = sch.encode(chunks)
-    assert ops[0] == sch.SKIP_ONLY + 127
-    assert ops[1] == 1 * sch.VALUE_MOD + 3
+    # assert ops[0] == sch.SKIP_ONLY + 127
+    # assert ops[1] == 1 * sch.VALUE_MOD + 3
 
     assert sch.decode(ops) == chunks
 
@@ -312,8 +317,8 @@ def test_scheme_skip_entry_first_2():
 
     chunks = {132: 3}
     ops, _ = sch.encode(chunks)
-    assert ops[0] == sch.SKIP_ONLY + 127
-    assert ops[1] == 2 * sch.VALUE_MOD + 3
+    # assert ops[0] == sch.SKIP_ONLY + 127
+    # assert ops[1] == 2 * sch.VALUE_MOD + 3
 
     assert sch.decode(ops) == chunks
 
@@ -323,9 +328,9 @@ def test_scheme_double_skip_entry_first():
 
     chunks = {262: 3}
     ops, _ = sch.encode(chunks)
-    assert ops[0] == sch.SKIP_ONLY + 130 - sch.SKIP_BIAS
-    assert ops[1] == sch.SKIP_ONLY + 130 - sch.SKIP_BIAS
-    assert ops[2] == 2 * sch.VALUE_MOD + 3
+    # assert ops[0] == sch.SKIP_ONLY + 130 - sch.SKIP_BIAS
+    # assert ops[1] == sch.SKIP_ONLY + 130 - sch.SKIP_BIAS
+    # assert ops[2] == 2 * sch.VALUE_MOD + 3
 
     assert sch.decode(ops) == chunks