From 5bfa5ebe7d22d54293d99dbc694097ad3721f87b Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Tue, 16 Apr 2024 17:14:41 +0200
Subject: [PATCH 01/66] WIP adds automatic offset tracking for streams

---
 src/lavinmq/client/channel/stream_consumer.cr |  8 +-
 src/lavinmq/mfile.cr                          |  9 +++
 src/lavinmq/queue/stream_queue.cr             |  4 +
 .../queue/stream_queue_message_store.cr       | 80 ++++++++++++++++++-
 4 files changed, 98 insertions(+), 3 deletions(-)

diff --git a/src/lavinmq/client/channel/stream_consumer.cr b/src/lavinmq/client/channel/stream_consumer.cr
index 33a7a04ad..9aba5c96e 100644
--- a/src/lavinmq/client/channel/stream_consumer.cr
+++ b/src/lavinmq/client/channel/stream_consumer.cr
@@ -13,7 +13,8 @@ module LavinMQ
         def initialize(@channel : Client::Channel, @queue : StreamQueue, frame : AMQP::Frame::Basic::Consume)
           validate_preconditions(frame)
           offset = frame.arguments["x-stream-offset"]?
-          @offset, @segment, @pos = stream_queue.find_offset(offset)
+          @tag = frame.consumer_tag
+          @offset, @segment, @pos = stream_queue.find_offset(offset, @tag)
           super
         end
 
@@ -82,6 +83,11 @@ module LavinMQ
           @queue.as(StreamQueue)
         end
 
+        def ack(sp)
+          stream_queue.save_offset_by_consumer_tag(@tag, @offset) # TODO only if no x-stream-offset?
+          super
+        end
+
         def reject(sp, requeue : Bool)
           super
           if requeue
diff --git a/src/lavinmq/mfile.cr b/src/lavinmq/mfile.cr
index 394bd5c1e..a926a7246 100644
--- a/src/lavinmq/mfile.cr
+++ b/src/lavinmq/mfile.cr
@@ -180,6 +180,15 @@ class MFile < IO
     @size = new_size
   end
 
+  def write_at(pos : Int64, slice : Bytes) : Nil
+    raise ClosedError.new if @closed
+    end_pos = pos + slice.size
+    @size = end_pos if end_pos > @size
+    raise IO::EOFError.new if end_pos > @capacity
+    slice.copy_to(buffer + pos, slice.size)
+  end
+
+
   def read(slice : Bytes)
     pos = @pos
     new_pos = pos + slice.size
diff --git a/src/lavinmq/queue/stream_queue.cr b/src/lavinmq/queue/stream_queue.cr
index 766087aa2..2c583e902 100644
--- a/src/lavinmq/queue/stream_queue.cr
+++ b/src/lavinmq/queue/stream_queue.cr
@@ -74,6 +74,10 @@ module LavinMQ
       end
     end
 
+    def save_offset_by_consumer_tag(consumer_tag : String, offset : Int64) : Nil
+      stream_queue_msg_store.save_offset_by_consumer_tag(consumer_tag, offset)
+    end
+
     # yield the next message in the ready queue
     # returns true if a message was deliviered, false otherwise
     # if we encouncer an unrecoverable ReadError, close queue
diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index 2754cc643..dca53101f 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -9,10 +9,17 @@ module LavinMQ
       property max_age : Time::Span | Time::MonthSpan | Nil
       getter last_offset : Int64
       @segment_last_ts = Hash(UInt32, Int64).new(0i64) # used for max-age
+      @consumer_offsets_hash : Hash(String, Int64) # used for consumer offsets
+      @consumer_offsets : MFile
+      @consumer_offset_path : String
 
       def initialize(@queue_data_dir : String, @replicator : Clustering::Replicator?)
         super
+
         @last_offset = get_last_offset
+        @consumer_offset_path = File.join(@data_dir, "consumer_offsets")
+        @consumer_offsets = MFile.new(@consumer_offset_path, 5000) # TODO: size?
+        @consumer_offsets_hash = consumer_offsets_hash
         drop_overflow
       end
 
@@ -32,13 +39,17 @@ module LavinMQ
       # Used once when a consumer is started
       # Populates `segment` and `position` by iterating through segments
       # until `offset` is found
-      def find_offset(offset) : Tuple(Int64, UInt32, UInt32)
+      def find_offset(offset, tag = nil) : Tuple(Int64, UInt32, UInt32)
         raise ClosedError.new if @closed
         case offset
-        when "first", nil then offset_at(@segments.first_key, 4u32)
+        when "first" then offset_at(@segments.first_key, 4u32)
         when "last"       then offset_at(@segments.last_key, 4u32)
         when "next"       then last_offset_seg_pos
         when Time         then find_offset_in_segments(offset)
+        when nil
+          consumer_last_offset = last_offset_by_consumer_tag(tag) || 0
+          find_offset_in_segments(consumer_last_offset)
+          # offset_at(@segments.first_key, 4u32) # TODO does this need to be handled?
         when Int
           if offset > @last_offset
             last_offset_seg_pos
@@ -88,6 +99,71 @@ module LavinMQ
         {msg_offset, segment, pos}
       end
 
+      def last_offset_by_consumer_tag(consumer_tag)
+        begin
+          if @consumer_offsets_hash[consumer_tag]
+            pos = @consumer_offsets_hash[consumer_tag]
+            tx = @consumer_offsets.to_slice(pos, 8)
+            return IO::ByteFormat::SystemEndian.decode(Int64, tx)
+          end
+        rescue KeyError
+        end
+      end
+
+      private def consumer_offsets_hash
+        hash = Hash(String, Int64).new
+        slice = @consumer_offsets.to_slice
+        more_to_read = true
+        i = 0_i64
+        ctag_start = 0
+        while more_to_read && slice.size > 0
+          if slice[i] == 32
+            ctag = String.new(slice[ctag_start..i-1])
+            pos = i+1
+            hash[ctag] = pos
+            ctag_start = pos + 8
+          end
+          more_to_read = false if (i += 1) == slice.size-1
+        end
+        hash
+      end
+
+      def save_offset_by_consumer_tag(consumer_tag, new_offset)
+        pos = 0_i64
+        begin
+          if pos = @consumer_offsets_hash[consumer_tag]
+            buf = uninitialized UInt8[8]
+            IO::ByteFormat::LittleEndian.encode(new_offset.as(Int64), buf.to_slice)
+            @consumer_offsets.write_at(pos, buf.to_slice)
+          end
+        rescue KeyError
+          write_new_ctag_to_file(consumer_tag, new_offset)
+        end
+      end
+
+      def write_new_ctag_to_file(consumer_tag, new_offset)
+        # pos = @consumer_offsets.size + slice.size
+        # @consumer_offsets.write(slice + buf.to_slice)
+        # TODO this should work?
+        # instead of having to manually find the position
+        highest_pos = 0_i64
+        @consumer_offsets_hash.each do |key, value|
+          if value > highest_pos
+            highest_pos = value
+          end
+        end
+        pos = highest_pos
+        pos += 8 unless pos == 0
+        slice = "#{consumer_tag} ".to_slice
+        @consumer_offsets.write_at(pos, slice)
+        pos += slice.size
+        buf = uninitialized UInt8[8]
+        IO::ByteFormat::LittleEndian.encode(new_offset.as(Int64), buf.to_slice)
+        @consumer_offsets.write_at(pos, buf.to_slice)
+        @consumer_offsets_hash[consumer_tag] = pos
+        pos
+      end
+
       def shift?(consumer : Client::Channel::StreamConsumer) : Envelope?
         raise ClosedError.new if @closed
 

From 5c2b46b21faa9870e0d144e5950a5aa6d49a4e4a Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Tue, 16 Apr 2024 17:14:46 +0200
Subject: [PATCH 02/66] specs

---
 spec/stream_queue_spec.cr | 74 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 74 insertions(+)

diff --git a/spec/stream_queue_spec.cr b/spec/stream_queue_spec.cr
index cb4ad7eb0..affd108ff 100644
--- a/spec/stream_queue_spec.cr
+++ b/spec/stream_queue_spec.cr
@@ -1,5 +1,6 @@
 require "./spec_helper"
 require "./../src/lavinmq/queue"
+#require "./../src/lavinmq/queue/stream_queue_message_store"
 
 describe LavinMQ::StreamQueue do
   stream_queue_args = LavinMQ::AMQP::Table.new({"x-queue-type": "stream"})
@@ -203,4 +204,77 @@ describe LavinMQ::StreamQueue do
       end
     end
   end
+
+  describe "Automatic consumer offset tracking" do
+    args = {"x-queue-type": "stream"}
+
+    it "resumes from last offset on reconnect" do
+      queue_name = Random::Secure.hex
+      consumer_tag = "ctag-1"
+      offset = 10
+
+      # publish 10 messages
+      with_channel do |ch|
+        q = ch.queue(queue_name, args: AMQP::Client::Arguments.new(args))
+        10.times { |i| q.publish "m#{i}" }
+      end
+
+      # get 10 messages, offset should be tracked and saved
+      with_channel do |ch|
+        ch.prefetch 1
+        q = ch.queue(queue_name, args: AMQP::Client::Arguments.new(args))
+        msgs = Channel(AMQP::Client::DeliverMessage).new
+        q.subscribe(no_ack: false, tag: consumer_tag) do |msg|
+          msgs.send msg
+          msg.ack
+        end
+        offset.times do
+          msgs.receive
+        end
+      end
+      sleep 1.second
+
+      # consume again, should start from last offset automatically
+      msg_offset = 0
+      with_channel do |ch|
+        ch.prefetch 1
+        q = ch.queue(queue_name, args: AMQP::Client::Arguments.new(args))
+        msgs = Channel(AMQP::Client::DeliverMessage).new
+        q.publish "m10"
+        q.subscribe(no_ack: false, tag: consumer_tag) do |msg|
+          msgs.send msg
+        end
+        msg = msgs.receive
+        msg.body_io.to_s.should eq "m10"
+        msg_offset = msg.properties.headers.not_nil!["x-stream-offset"].as(Int64)
+      end
+      msg_offset.should eq 11
+    end
+
+    it "reads offset file on init" do
+      queue_name = Random::Secure.hex
+      vhost = Server.vhosts["/"]
+      offsets = [84_i64, Random.rand(Int64), Random.rand(Int64), Random.rand(Int64)]
+      tag_prefix = "ctag-"
+      with_channel do |ch|
+        ch.prefetch 1
+        q = ch.queue(queue_name, args: AMQP::Client::Arguments.new(args))
+        q.publish "a"
+      end
+      data_dir = File.join(vhost.data_dir, Digest::SHA1.hexdigest queue_name)
+      msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
+      offsets.each_with_index do |offset, i|
+        msg_store.save_offset_by_consumer_tag(tag_prefix + i.to_s, offset)
+      end
+      msg_store.close
+
+      sleep 0.1
+
+      msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
+      offsets.each_with_index do |offset, i|
+        msg_store.last_offset_by_consumer_tag(tag_prefix + i.to_s).should eq offset
+      end
+      msg_store.close
+    end
+  end
 end

From 0e50b86d0b0ce42204778e2aad0f9c367e8d623e Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 17 Apr 2024 11:40:13 +0200
Subject: [PATCH 03/66] add specs for edge cases. refactor specs so they are
 more readable

---
 spec/stream_queue_spec.cr                     | 110 +++++++++++-------
 src/lavinmq/client/channel/stream_consumer.cr |   8 +-
 2 files changed, 72 insertions(+), 46 deletions(-)

diff --git a/spec/stream_queue_spec.cr b/spec/stream_queue_spec.cr
index affd108ff..6162d89a9 100644
--- a/spec/stream_queue_spec.cr
+++ b/spec/stream_queue_spec.cr
@@ -2,6 +2,30 @@ require "./spec_helper"
 require "./../src/lavinmq/queue"
 #require "./../src/lavinmq/queue/stream_queue_message_store"
 
+module StreamQueueSpecHelpers
+  def self.publish(queue_name, nr_of_messages)
+    args = {"x-queue-type": "stream"}
+    with_channel do |ch|
+      q = ch.queue(queue_name, args: AMQP::Client::Arguments.new(args))
+      nr_of_messages.times { |i| q.publish "m#{i}" }
+    end
+  end
+
+  def self.consume_one(queue_name, c_tag, c_args = AMQP::Client::Arguments.new())
+    args = {"x-queue-type": "stream"}
+    with_channel do |ch|
+      ch.prefetch 1
+      q = ch.queue(queue_name, args: AMQP::Client::Arguments.new(args))
+      msgs = Channel(AMQP::Client::DeliverMessage).new
+      q.subscribe(no_ack: false, tag: c_tag, args: c_args) do |msg|
+        msgs.send msg
+        msg.ack
+      end
+      msgs.receive
+    end
+  end
+end
+
 describe LavinMQ::StreamQueue do
   stream_queue_args = LavinMQ::AMQP::Table.new({"x-queue-type": "stream"})
 
@@ -206,68 +230,34 @@ describe LavinMQ::StreamQueue do
   end
 
   describe "Automatic consumer offset tracking" do
-    args = {"x-queue-type": "stream"}
-
     it "resumes from last offset on reconnect" do
       queue_name = Random::Secure.hex
-      consumer_tag = "ctag-1"
-      offset = 10
+      consumer_tag = Random::Secure.hex
+      offset = 3
 
-      # publish 10 messages
-      with_channel do |ch|
-        q = ch.queue(queue_name, args: AMQP::Client::Arguments.new(args))
-        10.times { |i| q.publish "m#{i}" }
-      end
+      StreamQueueSpecHelpers.publish(queue_name, offset+1)
 
-      # get 10 messages, offset should be tracked and saved
-      with_channel do |ch|
-        ch.prefetch 1
-        q = ch.queue(queue_name, args: AMQP::Client::Arguments.new(args))
-        msgs = Channel(AMQP::Client::DeliverMessage).new
-        q.subscribe(no_ack: false, tag: consumer_tag) do |msg|
-          msgs.send msg
-          msg.ack
-        end
-        offset.times do
-          msgs.receive
-        end
-      end
-      sleep 1.second
+      offset.times { StreamQueueSpecHelpers.consume_one(queue_name, consumer_tag) }
+      sleep 0.1
 
       # consume again, should start from last offset automatically
-      msg_offset = 0
-      with_channel do |ch|
-        ch.prefetch 1
-        q = ch.queue(queue_name, args: AMQP::Client::Arguments.new(args))
-        msgs = Channel(AMQP::Client::DeliverMessage).new
-        q.publish "m10"
-        q.subscribe(no_ack: false, tag: consumer_tag) do |msg|
-          msgs.send msg
-        end
-        msg = msgs.receive
-        msg.body_io.to_s.should eq "m10"
-        msg_offset = msg.properties.headers.not_nil!["x-stream-offset"].as(Int64)
-      end
-      msg_offset.should eq 11
+      msg = StreamQueueSpecHelpers.consume_one(queue_name, consumer_tag)
+      msg.properties.headers.not_nil!["x-stream-offset"].as(Int64).should eq offset+1
     end
 
-    it "reads offset file on init" do
+    it "reads offsets from file on init" do
       queue_name = Random::Secure.hex
       vhost = Server.vhosts["/"]
       offsets = [84_i64, Random.rand(Int64), Random.rand(Int64), Random.rand(Int64)]
       tag_prefix = "ctag-"
-      with_channel do |ch|
-        ch.prefetch 1
-        q = ch.queue(queue_name, args: AMQP::Client::Arguments.new(args))
-        q.publish "a"
-      end
+      StreamQueueSpecHelpers.publish(queue_name, 1)
+
       data_dir = File.join(vhost.data_dir, Digest::SHA1.hexdigest queue_name)
       msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
       offsets.each_with_index do |offset, i|
         msg_store.save_offset_by_consumer_tag(tag_prefix + i.to_s, offset)
       end
       msg_store.close
-
       sleep 0.1
 
       msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
@@ -276,5 +266,37 @@ describe LavinMQ::StreamQueue do
       end
       msg_store.close
     end
+
+    it "does not track offset if x-stream-offset is set" do
+      queue_name = Random::Secure.hex
+      consumer_tag = Random::Secure.hex
+      c_args = AMQP::Client::Arguments.new({"x-stream-offset": 0})
+
+      StreamQueueSpecHelpers.publish(queue_name, 2)
+      msg = StreamQueueSpecHelpers.consume_one(queue_name, consumer_tag, c_args)
+      msg.properties.headers.not_nil!["x-stream-offset"].as(Int64).should eq 1
+      sleep 0.1
+
+      # should consume the same message again since tracking was not saved from last consume
+      msg_2 = StreamQueueSpecHelpers.consume_one(queue_name, consumer_tag)
+      msg_2.properties.headers.not_nil!["x-stream-offset"].as(Int64).should eq 1
+    end
+
+    it "should not use saved offset if x-stream-offset is set" do
+      queue_name = Random::Secure.hex
+      consumer_tag = Random::Secure.hex
+      c_args = AMQP::Client::Arguments.new({"x-stream-offset": 0})
+
+      StreamQueueSpecHelpers.publish(queue_name, 2)
+
+      # get message without x-stream-offset, tracks offset
+      msg = StreamQueueSpecHelpers.consume_one(queue_name, consumer_tag)
+      msg.properties.headers.not_nil!["x-stream-offset"].as(Int64).should eq 1
+      sleep 0.1
+
+      # consume with x-stream-offset set, should consume the same message again
+      msg_2 = StreamQueueSpecHelpers.consume_one(queue_name, consumer_tag, c_args)
+      msg_2.properties.headers.not_nil!["x-stream-offset"].as(Int64).should eq 1
+    end
   end
 end
diff --git a/src/lavinmq/client/channel/stream_consumer.cr b/src/lavinmq/client/channel/stream_consumer.cr
index 9aba5c96e..2c05b1e3e 100644
--- a/src/lavinmq/client/channel/stream_consumer.cr
+++ b/src/lavinmq/client/channel/stream_consumer.cr
@@ -9,6 +9,7 @@ module LavinMQ
         property segment : UInt32
         property pos : UInt32
         getter requeued = Deque(SegmentPosition).new
+        @track_offset = false
 
         def initialize(@channel : Client::Channel, @queue : StreamQueue, frame : AMQP::Frame::Basic::Consume)
           validate_preconditions(frame)
@@ -35,7 +36,10 @@ module LavinMQ
             raise Error::PreconditionFailed.new("x-priority not supported on stream queues")
           end
           case frame.arguments["x-stream-offset"]?
-          when Nil, Int, Time, "first", "next", "last"
+          when Nil
+            @track_offset = true
+          when Int, Time, "first", "next", "last"
+            @track_offset = false
           else raise Error::PreconditionFailed.new("x-stream-offset must be an integer, a timestamp, 'first', 'next' or 'last'")
           end
         end
@@ -84,7 +88,7 @@ module LavinMQ
         end
 
         def ack(sp)
-          stream_queue.save_offset_by_consumer_tag(@tag, @offset) # TODO only if no x-stream-offset?
+          stream_queue.save_offset_by_consumer_tag(@tag, @offset) if @track_offset
           super
         end
 

From da8962fda542b44a55eadb4a79120dc7e8ea7e53 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 17 Apr 2024 12:00:46 +0200
Subject: [PATCH 04/66] format

---
 spec/stream_queue_spec.cr                       |  7 +++----
 src/lavinmq/mfile.cr                            |  1 -
 src/lavinmq/queue/stream_queue_message_store.cr | 14 +++++++-------
 3 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/spec/stream_queue_spec.cr b/spec/stream_queue_spec.cr
index 6162d89a9..5d18257c4 100644
--- a/spec/stream_queue_spec.cr
+++ b/spec/stream_queue_spec.cr
@@ -1,6 +1,5 @@
 require "./spec_helper"
 require "./../src/lavinmq/queue"
-#require "./../src/lavinmq/queue/stream_queue_message_store"
 
 module StreamQueueSpecHelpers
   def self.publish(queue_name, nr_of_messages)
@@ -11,7 +10,7 @@ module StreamQueueSpecHelpers
     end
   end
 
-  def self.consume_one(queue_name, c_tag, c_args = AMQP::Client::Arguments.new())
+  def self.consume_one(queue_name, c_tag, c_args = AMQP::Client::Arguments.new)
     args = {"x-queue-type": "stream"}
     with_channel do |ch|
       ch.prefetch 1
@@ -235,14 +234,14 @@ describe LavinMQ::StreamQueue do
       consumer_tag = Random::Secure.hex
       offset = 3
 
-      StreamQueueSpecHelpers.publish(queue_name, offset+1)
+      StreamQueueSpecHelpers.publish(queue_name, offset + 1)
 
       offset.times { StreamQueueSpecHelpers.consume_one(queue_name, consumer_tag) }
       sleep 0.1
 
       # consume again, should start from last offset automatically
       msg = StreamQueueSpecHelpers.consume_one(queue_name, consumer_tag)
-      msg.properties.headers.not_nil!["x-stream-offset"].as(Int64).should eq offset+1
+      msg.properties.headers.not_nil!["x-stream-offset"].as(Int64).should eq offset + 1
     end
 
     it "reads offsets from file on init" do
diff --git a/src/lavinmq/mfile.cr b/src/lavinmq/mfile.cr
index a926a7246..dd6111432 100644
--- a/src/lavinmq/mfile.cr
+++ b/src/lavinmq/mfile.cr
@@ -188,7 +188,6 @@ class MFile < IO
     slice.copy_to(buffer + pos, slice.size)
   end
 
-
   def read(slice : Bytes)
     pos = @pos
     new_pos = pos + slice.size
diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index dca53101f..9b30f09f3 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -9,7 +9,7 @@ module LavinMQ
       property max_age : Time::Span | Time::MonthSpan | Nil
       getter last_offset : Int64
       @segment_last_ts = Hash(UInt32, Int64).new(0i64) # used for max-age
-      @consumer_offsets_hash : Hash(String, Int64) # used for consumer offsets
+      @consumer_offsets_hash : Hash(String, Int64)     # used for consumer offsets
       @consumer_offsets : MFile
       @consumer_offset_path : String
 
@@ -43,9 +43,9 @@ module LavinMQ
         raise ClosedError.new if @closed
         case offset
         when "first" then offset_at(@segments.first_key, 4u32)
-        when "last"       then offset_at(@segments.last_key, 4u32)
-        when "next"       then last_offset_seg_pos
-        when Time         then find_offset_in_segments(offset)
+        when "last"  then offset_at(@segments.last_key, 4u32)
+        when "next"  then last_offset_seg_pos
+        when Time    then find_offset_in_segments(offset)
         when nil
           consumer_last_offset = last_offset_by_consumer_tag(tag) || 0
           find_offset_in_segments(consumer_last_offset)
@@ -118,12 +118,12 @@ module LavinMQ
         ctag_start = 0
         while more_to_read && slice.size > 0
           if slice[i] == 32
-            ctag = String.new(slice[ctag_start..i-1])
-            pos = i+1
+            ctag = String.new(slice[ctag_start..i - 1])
+            pos = i + 1
             hash[ctag] = pos
             ctag_start = pos + 8
           end
-          more_to_read = false if (i += 1) == slice.size-1
+          more_to_read = false if (i += 1) == slice.size - 1
         end
         hash
       end

From c6e649df67d07893201afd952f3c7466ba4c9939 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 17 Apr 2024 15:03:38 +0200
Subject: [PATCH 05/66] refactor, dont init consumer tracking stuff unless
 needed. resize file on init to facilitate simple write.

---
 .../queue/stream_queue_message_store.cr       | 63 ++++++++-----------
 1 file changed, 26 insertions(+), 37 deletions(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index 9b30f09f3..d9b9be1c1 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -9,17 +9,12 @@ module LavinMQ
       property max_age : Time::Span | Time::MonthSpan | Nil
       getter last_offset : Int64
       @segment_last_ts = Hash(UInt32, Int64).new(0i64) # used for max-age
-      @consumer_offsets_hash : Hash(String, Int64)     # used for consumer offsets
-      @consumer_offsets : MFile
-      @consumer_offset_path : String
+      @consumer_offset_positions : Hash(String, Int64)?    # used for consumer offsets
+      @consumer_offsets : MFile?
 
       def initialize(@queue_data_dir : String, @replicator : Clustering::Replicator?)
         super
-
         @last_offset = get_last_offset
-        @consumer_offset_path = File.join(@data_dir, "consumer_offsets")
-        @consumer_offsets = MFile.new(@consumer_offset_path, 5000) # TODO: size?
-        @consumer_offsets_hash = consumer_offsets_hash
         drop_overflow
       end
 
@@ -101,40 +96,48 @@ module LavinMQ
 
       def last_offset_by_consumer_tag(consumer_tag)
         begin
-          if @consumer_offsets_hash[consumer_tag]
-            pos = @consumer_offsets_hash[consumer_tag]
-            tx = @consumer_offsets.to_slice(pos, 8)
+          if consumer_offset_positions[consumer_tag]
+            pos = consumer_offset_positions[consumer_tag]
+            tx = consumer_offsets.to_slice(pos, 8)
             return IO::ByteFormat::SystemEndian.decode(Int64, tx)
           end
         rescue KeyError
         end
       end
 
-      private def consumer_offsets_hash
-        hash = Hash(String, Int64).new
-        slice = @consumer_offsets.to_slice
+      def consumer_offsets : MFile
+        return @consumer_offsets.not_nil! if @consumer_offsets
+        path = File.join(@data_dir, "consumer_offsets")
+        @consumer_offsets = MFile.new(path, 5000) # TODO: size?
+      end
+
+      private def consumer_offset_positions
+        return @consumer_offset_positions.not_nil! if @consumer_offset_positions
+        positions = Hash(String, Int64).new
+        slice = consumer_offsets.to_slice
         more_to_read = true
         i = 0_i64
         ctag_start = 0
-        while more_to_read && slice.size > 0
-          if slice[i] == 32
+
+        slice.each_with_index do |byte, i|
+          if byte == 32 # if space
             ctag = String.new(slice[ctag_start..i - 1])
             pos = i + 1
-            hash[ctag] = pos
+            positions[ctag] = pos
             ctag_start = pos + 8
           end
-          more_to_read = false if (i += 1) == slice.size - 1
         end
-        hash
+        consumer_offsets.resize(ctag_start) # resize mfile to remove any empty bytes
+        @consumer_offset_positions = positions
       end
 
       def save_offset_by_consumer_tag(consumer_tag, new_offset)
         pos = 0_i64
         begin
-          if pos = @consumer_offsets_hash[consumer_tag]
+          if pos = consumer_offset_positions[consumer_tag]
             buf = uninitialized UInt8[8]
             IO::ByteFormat::LittleEndian.encode(new_offset.as(Int64), buf.to_slice)
-            @consumer_offsets.write_at(pos, buf.to_slice)
+            consumer_offsets.write_at(pos, buf.to_slice)
           end
         rescue KeyError
           write_new_ctag_to_file(consumer_tag, new_offset)
@@ -142,26 +145,12 @@ module LavinMQ
       end
 
       def write_new_ctag_to_file(consumer_tag, new_offset)
-        # pos = @consumer_offsets.size + slice.size
-        # @consumer_offsets.write(slice + buf.to_slice)
-        # TODO this should work?
-        # instead of having to manually find the position
-        highest_pos = 0_i64
-        @consumer_offsets_hash.each do |key, value|
-          if value > highest_pos
-            highest_pos = value
-          end
-        end
-        pos = highest_pos
-        pos += 8 unless pos == 0
         slice = "#{consumer_tag} ".to_slice
-        @consumer_offsets.write_at(pos, slice)
-        pos += slice.size
         buf = uninitialized UInt8[8]
         IO::ByteFormat::LittleEndian.encode(new_offset.as(Int64), buf.to_slice)
-        @consumer_offsets.write_at(pos, buf.to_slice)
-        @consumer_offsets_hash[consumer_tag] = pos
-        pos
+        pos = consumer_offsets.size + slice.size
+        consumer_offsets.write(slice + buf.to_slice)
+        consumer_offset_positions[consumer_tag] = pos
       end
 
       def shift?(consumer : Client::Channel::StreamConsumer) : Envelope?

From f3a3090374c659299727189abcb20cc9a3c585dc Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 17 Apr 2024 15:04:12 +0200
Subject: [PATCH 06/66] add spec that checks that only one entry is saved per
 consumer tag

---
 spec/stream_queue_spec.cr | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/spec/stream_queue_spec.cr b/spec/stream_queue_spec.cr
index 5d18257c4..040f9bf93 100644
--- a/spec/stream_queue_spec.cr
+++ b/spec/stream_queue_spec.cr
@@ -266,6 +266,28 @@ describe LavinMQ::StreamQueue do
       msg_store.close
     end
 
+    it "only saves one entry per consumer tag" do
+      queue_name = Random::Secure.hex
+      vhost = Server.vhosts["/"]
+      offsets = [84_i64, Random.rand(Int64), Random.rand(Int64), Random.rand(Int64)]
+      consumer_tag = "ctag-1"
+      StreamQueueSpecHelpers.publish(queue_name, 1)
+
+      data_dir = File.join(vhost.data_dir, Digest::SHA1.hexdigest queue_name)
+      msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
+      offsets.each do |offset|
+        msg_store.save_offset_by_consumer_tag(consumer_tag, offset)
+      end
+      msg_store.close
+      sleep 0.1
+
+      msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
+      msg_store.last_offset_by_consumer_tag(consumer_tag).should eq offsets.last
+      msg_store.consumer_offsets.size.should eq 15
+
+      msg_store.close
+    end
+
     it "does not track offset if x-stream-offset is set" do
       queue_name = Random::Secure.hex
       consumer_tag = Random::Secure.hex

From 69ab9b4e52590cd33eaa4aceea77d28d6a640dda Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 17 Apr 2024 15:04:43 +0200
Subject: [PATCH 07/66] format

---
 src/lavinmq/queue/stream_queue_message_store.cr | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index d9b9be1c1..9a5aba006 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -8,8 +8,8 @@ module LavinMQ
       property max_length_bytes : Int64?
       property max_age : Time::Span | Time::MonthSpan | Nil
       getter last_offset : Int64
-      @segment_last_ts = Hash(UInt32, Int64).new(0i64) # used for max-age
-      @consumer_offset_positions : Hash(String, Int64)?    # used for consumer offsets
+      @segment_last_ts = Hash(UInt32, Int64).new(0i64)  # used for max-age
+      @consumer_offset_positions : Hash(String, Int64)? # used for consumer offsets
       @consumer_offsets : MFile?
 
       def initialize(@queue_data_dir : String, @replicator : Clustering::Replicator?)

From a3c7a26393c27c3a4915ce21d46ce23bacec10da Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 17 Apr 2024 16:40:16 +0200
Subject: [PATCH 08/66] lint

---
 src/lavinmq/queue/stream_queue_message_store.cr | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index 9a5aba006..dca37f79f 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -95,14 +95,12 @@ module LavinMQ
       end
 
       def last_offset_by_consumer_tag(consumer_tag)
-        begin
-          if consumer_offset_positions[consumer_tag]
-            pos = consumer_offset_positions[consumer_tag]
-            tx = consumer_offsets.to_slice(pos, 8)
-            return IO::ByteFormat::SystemEndian.decode(Int64, tx)
-          end
-        rescue KeyError
+        if consumer_offset_positions[consumer_tag]
+          pos = consumer_offset_positions[consumer_tag]
+          tx = consumer_offsets.to_slice(pos, 8)
+          return IO::ByteFormat::SystemEndian.decode(Int64, tx)
         end
+      rescue KeyError
       end
 
       def consumer_offsets : MFile

From 27a702889cfe01b36099a174199989457dccfe0c Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Mon, 22 Apr 2024 11:40:34 +0200
Subject: [PATCH 09/66] add function to remove consumer tags from file

---
 .../queue/stream_queue_message_store.cr       | 20 ++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index dca37f79f..e3f6c6698 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -104,7 +104,7 @@ module LavinMQ
       end
 
       def consumer_offsets : MFile
-        return @consumer_offsets.not_nil! if @consumer_offsets
+        return @consumer_offsets.not_nil! if @consumer_offsets && !@consumer_offsets.not_nil!.closed?
         path = File.join(@data_dir, "consumer_offsets")
         @consumer_offsets = MFile.new(path, 5000) # TODO: size?
       end
@@ -151,6 +151,24 @@ module LavinMQ
         consumer_offset_positions[consumer_tag] = pos
       end
 
+      def remove_consumer_tag_from_file(consumer_tag)
+        @consumer_offset_positions = consumer_offset_positions.reject! { |k, v| k == consumer_tag }
+
+        offsets_to_save = Hash(String, Int64).new
+        consumer_offset_positions.each do |ctag, pos|
+          offset = last_offset_by_consumer_tag(ctag)
+          next unless offset
+          offsets_to_save[ctag] = offset
+        end
+
+        consumer_offsets.close
+        consumer_offsets.delete
+        offsets_to_save.each do |ctag, offset|
+          write_new_ctag_to_file(ctag, offset)
+        end
+
+      end
+
       def shift?(consumer : Client::Channel::StreamConsumer) : Envelope?
         raise ClosedError.new if @closed
 

From d4ea06f85f84f0cce8edaf203b8ba8891c435193 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Mon, 22 Apr 2024 11:40:45 +0200
Subject: [PATCH 10/66] spec for removing consumer tags

---
 spec/stream_queue_spec.cr | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/spec/stream_queue_spec.cr b/spec/stream_queue_spec.cr
index 040f9bf93..18cfb24c3 100644
--- a/spec/stream_queue_spec.cr
+++ b/spec/stream_queue_spec.cr
@@ -319,5 +319,27 @@ describe LavinMQ::StreamQueue do
       msg_2 = StreamQueueSpecHelpers.consume_one(queue_name, consumer_tag, c_args)
       msg_2.properties.headers.not_nil!["x-stream-offset"].as(Int64).should eq 1
     end
+
+    it "removes offset" do
+      queue_name = Random::Secure.hex
+      vhost = Server.vhosts["/"]
+      offsets = [84_i64, 10_i64]
+      tag_prefix = "ctag-"
+      StreamQueueSpecHelpers.publish(queue_name, 1)
+
+      data_dir = File.join(vhost.data_dir, Digest::SHA1.hexdigest queue_name)
+      msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
+      offsets.each_with_index do |offset, i|
+        msg_store.save_offset_by_consumer_tag(tag_prefix + i.to_s, offset)
+      end
+      sleep 0.1
+      msg_store.remove_consumer_tag_from_file(tag_prefix + 1.to_s)
+      msg_store.close
+      sleep 0.1
+
+      msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
+      msg_store.last_offset_by_consumer_tag(tag_prefix + 1.to_s).should eq nil
+      msg_store.close
+    end
   end
 end

From 7af298bf9aae2f4066e9ba236414fbb0cc422877 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Mon, 22 Apr 2024 12:29:37 +0200
Subject: [PATCH 11/66] save length of ctag in file, dont use space as
 deliminator

---
 .../queue/stream_queue_message_store.cr       | 42 +++++++++++--------
 1 file changed, 25 insertions(+), 17 deletions(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index e3f6c6698..3b42ca343 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -113,19 +113,20 @@ module LavinMQ
         return @consumer_offset_positions.not_nil! if @consumer_offset_positions
         positions = Hash(String, Int64).new
         slice = consumer_offsets.to_slice
-        more_to_read = true
-        i = 0_i64
-        ctag_start = 0
-
-        slice.each_with_index do |byte, i|
-          if byte == 32 # if space
-            ctag = String.new(slice[ctag_start..i - 1])
-            pos = i + 1
-            positions[ctag] = pos
-            ctag_start = pos + 8
-          end
+        return positions if slice.size.zero?
+        pos = 0
+
+        loop do
+          ctag_length = IO::ByteFormat::LittleEndian.decode(UInt8, slice[pos, pos+1])
+          break if ctag_length == 0
+          pos += 1
+          ctag = String.new(slice[pos..pos+ctag_length-1])
+          pos += ctag_length
+          positions[ctag] = pos
+          pos += 8
+          break if pos >= slice.size
         end
-        consumer_offsets.resize(ctag_start) # resize mfile to remove any empty bytes
+        consumer_offsets.resize(pos) # resize mfile to remove any empty bytes
         @consumer_offset_positions = positions
       end
 
@@ -142,12 +143,19 @@ module LavinMQ
         end
       end
 
+      # should we write a null byte after each offset?
       def write_new_ctag_to_file(consumer_tag, new_offset)
-        slice = "#{consumer_tag} ".to_slice
-        buf = uninitialized UInt8[8]
-        IO::ByteFormat::LittleEndian.encode(new_offset.as(Int64), buf.to_slice)
-        pos = consumer_offsets.size + slice.size
-        consumer_offsets.write(slice + buf.to_slice)
+        slice = consumer_tag.to_slice
+        consumer_tag_length = slice.size.to_u8
+        pos = consumer_offsets.size + slice.size + 1
+
+        length_buffer = uninitialized UInt8[1]
+        IO::ByteFormat::LittleEndian.encode(consumer_tag_length, length_buffer.to_slice)
+
+        offset_buffer = uninitialized UInt8[8]
+        IO::ByteFormat::LittleEndian.encode(new_offset.as(Int64), offset_buffer.to_slice)
+
+        consumer_offsets.write(length_buffer.to_slice + slice + offset_buffer.to_slice)
         consumer_offset_positions[consumer_tag] = pos
       end
 

From aa11afdc8f800998546e24fb0acb9af59b805729 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Mon, 22 Apr 2024 12:30:13 +0200
Subject: [PATCH 12/66] format

---
 src/lavinmq/queue/stream_queue_message_store.cr | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index 3b42ca343..3e56edfb4 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -117,10 +117,10 @@ module LavinMQ
         pos = 0
 
         loop do
-          ctag_length = IO::ByteFormat::LittleEndian.decode(UInt8, slice[pos, pos+1])
+          ctag_length = IO::ByteFormat::LittleEndian.decode(UInt8, slice[pos, pos + 1])
           break if ctag_length == 0
           pos += 1
-          ctag = String.new(slice[pos..pos+ctag_length-1])
+          ctag = String.new(slice[pos..pos + ctag_length - 1])
           pos += ctag_length
           positions[ctag] = pos
           pos += 8
@@ -174,7 +174,6 @@ module LavinMQ
         offsets_to_save.each do |ctag, offset|
           write_new_ctag_to_file(ctag, offset)
         end
-
       end
 
       def shift?(consumer : Client::Channel::StreamConsumer) : Envelope?

From 10fd5d0fabda66f1e271677691080c17e2906e61 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Mon, 22 Apr 2024 12:37:17 +0200
Subject: [PATCH 13/66] lint

---
 src/lavinmq/queue/stream_queue_message_store.cr | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index 3e56edfb4..322bd35ac 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -160,10 +160,10 @@ module LavinMQ
       end
 
       def remove_consumer_tag_from_file(consumer_tag)
-        @consumer_offset_positions = consumer_offset_positions.reject! { |k, v| k == consumer_tag }
+        @consumer_offset_positions = consumer_offset_positions.reject! { |k, _v| k == consumer_tag }
 
         offsets_to_save = Hash(String, Int64).new
-        consumer_offset_positions.each do |ctag, pos|
+        consumer_offset_positions.each do |ctag, _p|
           offset = last_offset_by_consumer_tag(ctag)
           next unless offset
           offsets_to_save[ctag] = offset

From 3aa0ddf11a0b3c94d8717d38c17c551b9d184aac Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Tue, 23 Apr 2024 12:02:14 +0200
Subject: [PATCH 14/66] remove methods, use instance variables directly. add
 cleanup function

---
 .../queue/stream_queue_message_store.cr       | 66 ++++++++++++-------
 1 file changed, 42 insertions(+), 24 deletions(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index 322bd35ac..90d4f5588 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -9,12 +9,15 @@ module LavinMQ
       property max_age : Time::Span | Time::MonthSpan | Nil
       getter last_offset : Int64
       @segment_last_ts = Hash(UInt32, Int64).new(0i64)  # used for max-age
-      @consumer_offset_positions : Hash(String, Int64)? # used for consumer offsets
-      @consumer_offsets : MFile?
+      @consumer_offset_positions = Hash(String, Int64).new # used for consumer offsets
+      @consumer_offsets : MFile
 
       def initialize(@queue_data_dir : String, @replicator : Clustering::Replicator?)
         super
         @last_offset = get_last_offset
+        path = File.join(@data_dir, "consumer_offsets")
+        @consumer_offsets = MFile.new(path, 5000) # TODO: size?
+        @consumer_offset_positions = consumer_offset_positions
         drop_overflow
       end
 
@@ -95,24 +98,16 @@ module LavinMQ
       end
 
       def last_offset_by_consumer_tag(consumer_tag)
-        if consumer_offset_positions[consumer_tag]
-          pos = consumer_offset_positions[consumer_tag]
-          tx = consumer_offsets.to_slice(pos, 8)
+        if pos = @consumer_offset_positions[consumer_tag]
+          tx = @consumer_offsets.to_slice(pos, 8)
           return IO::ByteFormat::SystemEndian.decode(Int64, tx)
         end
       rescue KeyError
       end
 
-      def consumer_offsets : MFile
-        return @consumer_offsets.not_nil! if @consumer_offsets && !@consumer_offsets.not_nil!.closed?
-        path = File.join(@data_dir, "consumer_offsets")
-        @consumer_offsets = MFile.new(path, 5000) # TODO: size?
-      end
-
       private def consumer_offset_positions
-        return @consumer_offset_positions.not_nil! if @consumer_offset_positions
         positions = Hash(String, Int64).new
-        slice = consumer_offsets.to_slice
+        slice = @consumer_offsets.to_slice
         return positions if slice.size.zero?
         pos = 0
 
@@ -126,17 +121,17 @@ module LavinMQ
           pos += 8
           break if pos >= slice.size
         end
-        consumer_offsets.resize(pos) # resize mfile to remove any empty bytes
-        @consumer_offset_positions = positions
+        @consumer_offsets.resize(pos) # resize mfile to remove any empty bytes
+        positions
       end
 
       def save_offset_by_consumer_tag(consumer_tag, new_offset)
         pos = 0_i64
         begin
-          if pos = consumer_offset_positions[consumer_tag]
+          if pos = @consumer_offset_positions[consumer_tag]
             buf = uninitialized UInt8[8]
             IO::ByteFormat::LittleEndian.encode(new_offset.as(Int64), buf.to_slice)
-            consumer_offsets.write_at(pos, buf.to_slice)
+            @consumer_offsets.write_at(pos, buf.to_slice)
           end
         rescue KeyError
           write_new_ctag_to_file(consumer_tag, new_offset)
@@ -147,7 +142,7 @@ module LavinMQ
       def write_new_ctag_to_file(consumer_tag, new_offset)
         slice = consumer_tag.to_slice
         consumer_tag_length = slice.size.to_u8
-        pos = consumer_offsets.size + slice.size + 1
+        pos = @consumer_offsets.size + slice.size + 1
 
         length_buffer = uninitialized UInt8[1]
         IO::ByteFormat::LittleEndian.encode(consumer_tag_length, length_buffer.to_slice)
@@ -155,27 +150,50 @@ module LavinMQ
         offset_buffer = uninitialized UInt8[8]
         IO::ByteFormat::LittleEndian.encode(new_offset.as(Int64), offset_buffer.to_slice)
 
-        consumer_offsets.write(length_buffer.to_slice + slice + offset_buffer.to_slice)
-        consumer_offset_positions[consumer_tag] = pos
+        @consumer_offsets.write(length_buffer.to_slice + slice + offset_buffer.to_slice)
+        @consumer_offset_positions[consumer_tag] = pos
+      end
+
+      def cleanup_consumer_offsets
+        offsets_to_save = Hash(String, Int64).new
+        lowest_offset_in_stream, _seg, _pos = offset_at(@segments.first_key, 4u32) # handle
+        @consumer_offset_positions.each do |ctag, pos|
+          offset = last_offset_by_consumer_tag(ctag).not_nil!
+          next if offset < lowest_offset_in_stream
+          # Other scenarios to remove?
+          offsets_to_save[ctag] = offset
+        end
+
+        delete_and_reopen_offsets_file
+        @consumer_offset_positions = Hash(String, Int64).new
+        offsets_to_save.each do |ctag, offset|
+          write_new_ctag_to_file(ctag, offset)
+        end
       end
 
       def remove_consumer_tag_from_file(consumer_tag)
-        @consumer_offset_positions = consumer_offset_positions.reject! { |k, _v| k == consumer_tag }
+        @consumer_offset_positions = @consumer_offset_positions.reject! { |k, _v| k == consumer_tag }
 
         offsets_to_save = Hash(String, Int64).new
-        consumer_offset_positions.each do |ctag, _p|
+        @consumer_offset_positions.each do |ctag, _p|
           offset = last_offset_by_consumer_tag(ctag)
           next unless offset
           offsets_to_save[ctag] = offset
         end
 
-        consumer_offsets.close
-        consumer_offsets.delete
+        delete_and_reopen_offsets_file
         offsets_to_save.each do |ctag, offset|
           write_new_ctag_to_file(ctag, offset)
         end
       end
 
+      def delete_and_reopen_offsets_file
+        @consumer_offsets.close
+        @consumer_offsets.delete
+        path = File.join(@data_dir, "consumer_offsets")
+        @consumer_offsets = MFile.new(path, 5000) # TODO: size?
+      end
+
       def shift?(consumer : Client::Channel::StreamConsumer) : Envelope?
         raise ClosedError.new if @closed
 

From 9877191dc0512108d33b5a73dad757a6721e978e Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Tue, 23 Apr 2024 12:02:31 +0200
Subject: [PATCH 15/66] add spec for cleanup

---
 spec/stream_queue_spec.cr | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/spec/stream_queue_spec.cr b/spec/stream_queue_spec.cr
index 18cfb24c3..c72c07ee6 100644
--- a/spec/stream_queue_spec.cr
+++ b/spec/stream_queue_spec.cr
@@ -283,7 +283,7 @@ describe LavinMQ::StreamQueue do
 
       msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
       msg_store.last_offset_by_consumer_tag(consumer_tag).should eq offsets.last
-      msg_store.consumer_offsets.size.should eq 15
+      msg_store.@consumer_offsets.size.should eq 15
 
       msg_store.close
     end
@@ -339,6 +339,30 @@ describe LavinMQ::StreamQueue do
 
       msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
       msg_store.last_offset_by_consumer_tag(tag_prefix + 1.to_s).should eq nil
+      msg_store.last_offset_by_consumer_tag(tag_prefix + 0.to_s).should eq offsets[0]
+      msg_store.close
+    end
+
+    it "cleanup_consumer_offsets removes outdated offset" do
+      queue_name = Random::Secure.hex
+      vhost = Server.vhosts["/"]
+      offsets = [84_i64, -10_i64]
+      tag_prefix = "ctag-"
+      StreamQueueSpecHelpers.publish(queue_name, 1)
+
+      data_dir = File.join(vhost.data_dir, Digest::SHA1.hexdigest queue_name)
+      msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
+      offsets.each_with_index do |offset, i|
+        msg_store.save_offset_by_consumer_tag(tag_prefix + i.to_s, offset)
+      end
+      sleep 0.1
+      msg_store.cleanup_consumer_offsets
+      msg_store.close
+      sleep 0.1
+
+      msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
+      msg_store.last_offset_by_consumer_tag(tag_prefix + 1.to_s).should eq nil
+      msg_store.last_offset_by_consumer_tag(tag_prefix + 0.to_s).should eq offsets[0]
       msg_store.close
     end
   end

From 1c3f6b57fbf35bab8a30950af524920bcaaa167f Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor.erlingsson@gmail.com>
Date: Wed, 24 Apr 2024 01:32:08 +0200
Subject: [PATCH 16/66] raise before updating instance variable
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Carl Hörberg <carl@84codes.com>
---
 src/lavinmq/mfile.cr | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lavinmq/mfile.cr b/src/lavinmq/mfile.cr
index dd6111432..01029781d 100644
--- a/src/lavinmq/mfile.cr
+++ b/src/lavinmq/mfile.cr
@@ -183,8 +183,8 @@ class MFile < IO
   def write_at(pos : Int64, slice : Bytes) : Nil
     raise ClosedError.new if @closed
     end_pos = pos + slice.size
-    @size = end_pos if end_pos > @size
     raise IO::EOFError.new if end_pos > @capacity
+    @size = end_pos if end_pos > @size
     slice.copy_to(buffer + pos, slice.size)
   end
 

From 1c4124a83be4c1d2ea3b264fac5fcd4705858c14 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor.erlingsson@gmail.com>
Date: Wed, 24 Apr 2024 01:34:25 +0200
Subject: [PATCH 17/66] refactor
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Carl Hörberg <carl@84codes.com>
---
 src/lavinmq/queue/stream_queue_message_store.cr | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index 90d4f5588..5388b3f3a 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -98,11 +98,10 @@ module LavinMQ
       end
 
       def last_offset_by_consumer_tag(consumer_tag)
-        if pos = @consumer_offset_positions[consumer_tag]
+        if pos = @consumer_offset_positions[consumer_tag]?
           tx = @consumer_offsets.to_slice(pos, 8)
           return IO::ByteFormat::SystemEndian.decode(Int64, tx)
         end
-      rescue KeyError
       end
 
       private def consumer_offset_positions

From e2d457cb586732e7f91a1997ce9c7a06a56cfd10 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor.erlingsson@gmail.com>
Date: Wed, 24 Apr 2024 01:35:36 +0200
Subject: [PATCH 18/66] remove comment
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Carl Hörberg <carl@84codes.com>
---
 src/lavinmq/queue/stream_queue_message_store.cr | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index 5388b3f3a..31a46e636 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -137,7 +137,6 @@ module LavinMQ
         end
       end
 
-      # should we write a null byte after each offset?
       def write_new_ctag_to_file(consumer_tag, new_offset)
         slice = consumer_tag.to_slice
         consumer_tag_length = slice.size.to_u8

From a616f9b521a6e9757e9068c51dabc7c060fd071f Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor.erlingsson@gmail.com>
Date: Wed, 24 Apr 2024 01:36:09 +0200
Subject: [PATCH 19/66] refactor
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Carl Hörberg <carl@84codes.com>
---
 src/lavinmq/queue/stream_queue_message_store.cr | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index 31a46e636..b8afac2d5 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -127,7 +127,7 @@ module LavinMQ
       def save_offset_by_consumer_tag(consumer_tag, new_offset)
         pos = 0_i64
         begin
-          if pos = @consumer_offset_positions[consumer_tag]
+          if pos = @consumer_offset_positions[consumer_tag]?
             buf = uninitialized UInt8[8]
             IO::ByteFormat::LittleEndian.encode(new_offset.as(Int64), buf.to_slice)
             @consumer_offsets.write_at(pos, buf.to_slice)

From 04c9eff33cff09b8c252d411320538f008fdfb79 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor.erlingsson@gmail.com>
Date: Wed, 24 Apr 2024 01:36:21 +0200
Subject: [PATCH 20/66] refactor
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Carl Hörberg <carl@84codes.com>
---
 src/lavinmq/queue/stream_queue_message_store.cr | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index b8afac2d5..6ce2f3aca 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -131,9 +131,9 @@ module LavinMQ
             buf = uninitialized UInt8[8]
             IO::ByteFormat::LittleEndian.encode(new_offset.as(Int64), buf.to_slice)
             @consumer_offsets.write_at(pos, buf.to_slice)
+          else
+            write_new_ctag_to_file(consumer_tag, new_offset)
           end
-        rescue KeyError
-          write_new_ctag_to_file(consumer_tag, new_offset)
         end
       end
 

From 2f5e77e1feb8fa5de0f59c28d3cdfc14793daf03 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 24 Apr 2024 01:38:39 +0200
Subject: [PATCH 21/66] format

---
 src/lavinmq/queue/stream_queue_message_store.cr | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index 6ce2f3aca..46b585e59 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -8,7 +8,7 @@ module LavinMQ
       property max_length_bytes : Int64?
       property max_age : Time::Span | Time::MonthSpan | Nil
       getter last_offset : Int64
-      @segment_last_ts = Hash(UInt32, Int64).new(0i64)  # used for max-age
+      @segment_last_ts = Hash(UInt32, Int64).new(0i64)     # used for max-age
       @consumer_offset_positions = Hash(String, Int64).new # used for consumer offsets
       @consumer_offsets : MFile
 

From 5a4100874fa6c88d26e89df99241fbefea2d9645 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 24 Apr 2024 10:08:30 +0200
Subject: [PATCH 22/66] set size to 32768 for offsets file, change path name

---
 src/lavinmq/queue/stream_queue_message_store.cr | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index 46b585e59..1e4e1f1f6 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -11,12 +11,14 @@ module LavinMQ
       @segment_last_ts = Hash(UInt32, Int64).new(0i64)     # used for max-age
       @consumer_offset_positions = Hash(String, Int64).new # used for consumer offsets
       @consumer_offsets : MFile
+      @consumer_offset_path : String
+      @consumer_offset_capacity = 32_768
 
       def initialize(@queue_data_dir : String, @replicator : Clustering::Replicator?)
         super
         @last_offset = get_last_offset
-        path = File.join(@data_dir, "consumer_offsets")
-        @consumer_offsets = MFile.new(path, 5000) # TODO: size?
+        @consumer_offset_path  = File.join(@data_dir, "consumer_offsets")
+        @consumer_offsets = MFile.new(@consumer_offset_path, @consumer_offset_capacity)
         @consumer_offset_positions = consumer_offset_positions
         drop_overflow
       end
@@ -188,8 +190,7 @@ module LavinMQ
       def delete_and_reopen_offsets_file
         @consumer_offsets.close
         @consumer_offsets.delete
-        path = File.join(@data_dir, "consumer_offsets")
-        @consumer_offsets = MFile.new(path, 5000) # TODO: size?
+        @consumer_offsets = MFile.new(@consumer_offset_path, @consumer_offset_capacity)
       end
 
       def shift?(consumer : Client::Channel::StreamConsumer) : Envelope?

From 9e5b7ba0f0adb532b0d3cb5ff9ac0a9ab7bba840 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 24 Apr 2024 10:09:00 +0200
Subject: [PATCH 23/66] remove comment

---
 src/lavinmq/queue/stream_queue_message_store.cr | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index 1e4e1f1f6..f3c07f239 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -49,7 +49,6 @@ module LavinMQ
         when nil
           consumer_last_offset = last_offset_by_consumer_tag(tag) || 0
           find_offset_in_segments(consumer_last_offset)
-          # offset_at(@segments.first_key, 4u32) # TODO does this need to be handled?
         when Int
           if offset > @last_offset
             last_offset_seg_pos

From 940704bd11087a43ddf84dfc440e2df66ce1c18f Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 24 Apr 2024 10:22:18 +0200
Subject: [PATCH 24/66] refactor restore_consumer_offset_positions

---
 .../queue/stream_queue_message_store.cr       | 25 ++++++++-----------
 1 file changed, 11 insertions(+), 14 deletions(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index f3c07f239..4ce2eac69 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -19,7 +19,7 @@ module LavinMQ
         @last_offset = get_last_offset
         @consumer_offset_path  = File.join(@data_dir, "consumer_offsets")
         @consumer_offsets = MFile.new(@consumer_offset_path, @consumer_offset_capacity)
-        @consumer_offset_positions = consumer_offset_positions
+        @consumer_offset_positions = restore_consumer_offset_positions
         drop_overflow
       end
 
@@ -105,23 +105,20 @@ module LavinMQ
         end
       end
 
-      private def consumer_offset_positions
+      private def restore_consumer_offset_positions
         positions = Hash(String, Int64).new
-        slice = @consumer_offsets.to_slice
-        return positions if slice.size.zero?
-        pos = 0
+        return positions if @consumer_offsets.size.zero?
 
         loop do
-          ctag_length = IO::ByteFormat::LittleEndian.decode(UInt8, slice[pos, pos + 1])
-          break if ctag_length == 0
-          pos += 1
-          ctag = String.new(slice[pos..pos + ctag_length - 1])
-          pos += ctag_length
-          positions[ctag] = pos
-          pos += 8
-          break if pos >= slice.size
+          ctag_length = @consumer_offsets.read_byte
+          break if !ctag_length || ctag_length.zero?
+          ctag = @consumer_offsets.read_string(ctag_length)
+          positions[ctag] = @consumer_offsets.pos
+          @consumer_offsets.skip(8)
+        rescue IO::EOFError
+          break
         end
-        @consumer_offsets.resize(pos) # resize mfile to remove any empty bytes
+        @consumer_offsets.resize(@consumer_offsets.pos) # resize mfile to remove any empty bytes
         positions
       end
 

From 2183b0698213de2cbee5d6c7205b963e495bff5a Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 24 Apr 2024 10:22:36 +0200
Subject: [PATCH 25/66] remove unused var

---
 src/lavinmq/queue/stream_queue_message_store.cr | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index 4ce2eac69..ebf5230d4 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -123,7 +123,6 @@ module LavinMQ
       end
 
       def save_offset_by_consumer_tag(consumer_tag, new_offset)
-        pos = 0_i64
         begin
           if pos = @consumer_offset_positions[consumer_tag]?
             buf = uninitialized UInt8[8]

From 37aa4fead980bde00ab3ed9ce74e3f252428c2f3 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 24 Apr 2024 10:24:19 +0200
Subject: [PATCH 26/66] save_offset_by_consumer_tag -> update_consumer_offset

---
 spec/stream_queue_spec.cr                       | 8 ++++----
 src/lavinmq/client/channel/stream_consumer.cr   | 2 +-
 src/lavinmq/queue/stream_queue.cr               | 4 ++--
 src/lavinmq/queue/stream_queue_message_store.cr | 2 +-
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/spec/stream_queue_spec.cr b/spec/stream_queue_spec.cr
index c72c07ee6..e8d061448 100644
--- a/spec/stream_queue_spec.cr
+++ b/spec/stream_queue_spec.cr
@@ -254,7 +254,7 @@ describe LavinMQ::StreamQueue do
       data_dir = File.join(vhost.data_dir, Digest::SHA1.hexdigest queue_name)
       msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
       offsets.each_with_index do |offset, i|
-        msg_store.save_offset_by_consumer_tag(tag_prefix + i.to_s, offset)
+        msg_store.update_consumer_offset(tag_prefix + i.to_s, offset)
       end
       msg_store.close
       sleep 0.1
@@ -276,7 +276,7 @@ describe LavinMQ::StreamQueue do
       data_dir = File.join(vhost.data_dir, Digest::SHA1.hexdigest queue_name)
       msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
       offsets.each do |offset|
-        msg_store.save_offset_by_consumer_tag(consumer_tag, offset)
+        msg_store.update_consumer_offset(consumer_tag, offset)
       end
       msg_store.close
       sleep 0.1
@@ -330,7 +330,7 @@ describe LavinMQ::StreamQueue do
       data_dir = File.join(vhost.data_dir, Digest::SHA1.hexdigest queue_name)
       msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
       offsets.each_with_index do |offset, i|
-        msg_store.save_offset_by_consumer_tag(tag_prefix + i.to_s, offset)
+        msg_store.update_consumer_offset(tag_prefix + i.to_s, offset)
       end
       sleep 0.1
       msg_store.remove_consumer_tag_from_file(tag_prefix + 1.to_s)
@@ -353,7 +353,7 @@ describe LavinMQ::StreamQueue do
       data_dir = File.join(vhost.data_dir, Digest::SHA1.hexdigest queue_name)
       msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
       offsets.each_with_index do |offset, i|
-        msg_store.save_offset_by_consumer_tag(tag_prefix + i.to_s, offset)
+        msg_store.update_consumer_offset(tag_prefix + i.to_s, offset)
       end
       sleep 0.1
       msg_store.cleanup_consumer_offsets
diff --git a/src/lavinmq/client/channel/stream_consumer.cr b/src/lavinmq/client/channel/stream_consumer.cr
index 2c05b1e3e..da235f61a 100644
--- a/src/lavinmq/client/channel/stream_consumer.cr
+++ b/src/lavinmq/client/channel/stream_consumer.cr
@@ -88,7 +88,7 @@ module LavinMQ
         end
 
         def ack(sp)
-          stream_queue.save_offset_by_consumer_tag(@tag, @offset) if @track_offset
+          stream_queue.update_consumer_offset(@tag, @offset) if @track_offset
           super
         end
 
diff --git a/src/lavinmq/queue/stream_queue.cr b/src/lavinmq/queue/stream_queue.cr
index 2c583e902..7a991ccc3 100644
--- a/src/lavinmq/queue/stream_queue.cr
+++ b/src/lavinmq/queue/stream_queue.cr
@@ -74,8 +74,8 @@ module LavinMQ
       end
     end
 
-    def save_offset_by_consumer_tag(consumer_tag : String, offset : Int64) : Nil
-      stream_queue_msg_store.save_offset_by_consumer_tag(consumer_tag, offset)
+    def update_consumer_offset(consumer_tag : String, offset : Int64) : Nil
+      stream_queue_msg_store.update_consumer_offset(consumer_tag, offset)
     end
 
     # yield the next message in the ready queue
diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index ebf5230d4..ccf57ae12 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -122,7 +122,7 @@ module LavinMQ
         positions
       end
 
-      def save_offset_by_consumer_tag(consumer_tag, new_offset)
+      def update_consumer_offset(consumer_tag : String, new_offset : Int64)
         begin
           if pos = @consumer_offset_positions[consumer_tag]?
             buf = uninitialized UInt8[8]

From 088d46db9cbc46c6bb0c6b7090ad9f0f9587fd34 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 24 Apr 2024 10:25:15 +0200
Subject: [PATCH 27/66] write_new_ctag_to_file -> store_consumer_offset

---
 src/lavinmq/queue/stream_queue_message_store.cr | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index ccf57ae12..b9efb48d4 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -129,12 +129,12 @@ module LavinMQ
             IO::ByteFormat::LittleEndian.encode(new_offset.as(Int64), buf.to_slice)
             @consumer_offsets.write_at(pos, buf.to_slice)
           else
-            write_new_ctag_to_file(consumer_tag, new_offset)
+            store_consumer_offset(consumer_tag, new_offset)
           end
         end
       end
 
-      def write_new_ctag_to_file(consumer_tag, new_offset)
+      def store_consumer_offset(consumer_tag, new_offset)
         slice = consumer_tag.to_slice
         consumer_tag_length = slice.size.to_u8
         pos = @consumer_offsets.size + slice.size + 1
@@ -162,7 +162,7 @@ module LavinMQ
         delete_and_reopen_offsets_file
         @consumer_offset_positions = Hash(String, Int64).new
         offsets_to_save.each do |ctag, offset|
-          write_new_ctag_to_file(ctag, offset)
+          store_consumer_offset(ctag, offset)
         end
       end
 
@@ -178,7 +178,7 @@ module LavinMQ
 
         delete_and_reopen_offsets_file
         offsets_to_save.each do |ctag, offset|
-          write_new_ctag_to_file(ctag, offset)
+          store_consumer_offset(ctag, offset)
         end
       end
 

From 397a7f80c69ba2d11b124055577dcf10ecaab5b2 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 24 Apr 2024 10:26:37 +0200
Subject: [PATCH 28/66] refactor update_consumer_offset

---
 src/lavinmq/queue/stream_queue_message_store.cr | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index b9efb48d4..64a1f522f 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -125,9 +125,8 @@ module LavinMQ
       def update_consumer_offset(consumer_tag : String, new_offset : Int64)
         begin
           if pos = @consumer_offset_positions[consumer_tag]?
-            buf = uninitialized UInt8[8]
-            IO::ByteFormat::LittleEndian.encode(new_offset.as(Int64), buf.to_slice)
-            @consumer_offsets.write_at(pos, buf.to_slice)
+            @consumer_offsets.pos = pos
+            @consumer_offsets.write_bytes new_offset
           else
             store_consumer_offset(consumer_tag, new_offset)
           end

From 1e5616b661b85ea83e48d4e809043f2f9add478f Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 24 Apr 2024 11:11:29 +0200
Subject: [PATCH 29/66] add option to get a writeable slice from mfile

---
 src/lavinmq/mfile.cr | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lavinmq/mfile.cr b/src/lavinmq/mfile.cr
index 01029781d..5d90cebb0 100644
--- a/src/lavinmq/mfile.cr
+++ b/src/lavinmq/mfile.cr
@@ -237,9 +237,9 @@ class MFile < IO
     Bytes.new(buffer, @size, read_only: true)
   end
 
-  def to_slice(pos, size)
+  def to_slice(pos, size, read_only = true)
     raise IO::EOFError.new if pos + size > @size
-    Bytes.new(buffer + pos, size, read_only: true)
+    Bytes.new(buffer + pos, size, read_only: read_only)
   end
 
   def advise(advice : Advice, addr = buffer, offset = 0, length = @capacity) : Nil

From 3bc3a42c5b76faf41d6d657442396c976cbb8cfd Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 24 Apr 2024 11:11:58 +0200
Subject: [PATCH 30/66] encode directly into mmap. fix bug with pos/size

---
 src/lavinmq/queue/stream_queue_message_store.cr | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index 64a1f522f..ef4cd7d72 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -118,15 +118,15 @@ module LavinMQ
         rescue IO::EOFError
           break
         end
-        @consumer_offsets.resize(@consumer_offsets.pos) # resize mfile to remove any empty bytes
+        @consumer_offsets.pos = 0 if @consumer_offsets.pos == 1
+        @consumer_offsets.resize(@consumer_offsets.pos)
         positions
       end
 
       def update_consumer_offset(consumer_tag : String, new_offset : Int64)
         begin
           if pos = @consumer_offset_positions[consumer_tag]?
-            @consumer_offsets.pos = pos
-            @consumer_offsets.write_bytes new_offset
+            IO::ByteFormat::SystemEndian.encode(new_offset, @consumer_offsets.to_slice(pos, 8, false))
           else
             store_consumer_offset(consumer_tag, new_offset)
           end

From 86fa2ba99df8f5ed72e44e006b3c82946d74b30b Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 24 Apr 2024 11:12:06 +0200
Subject: [PATCH 31/66] update spec

---
 spec/stream_queue_spec.cr | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spec/stream_queue_spec.cr b/spec/stream_queue_spec.cr
index e8d061448..a8c288fb1 100644
--- a/spec/stream_queue_spec.cr
+++ b/spec/stream_queue_spec.cr
@@ -283,7 +283,7 @@ describe LavinMQ::StreamQueue do
 
       msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
       msg_store.last_offset_by_consumer_tag(consumer_tag).should eq offsets.last
-      msg_store.@consumer_offsets.size.should eq 15
+      msg_store.@consumer_offsets.size.should eq 16
 
       msg_store.close
     end

From ee1dbbbd84c283df990d9e749f7f3e5433adccad Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 24 Apr 2024 11:16:09 +0200
Subject: [PATCH 32/66] more strict with types

---
 src/lavinmq/queue/stream_queue_message_store.cr | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index ef4cd7d72..6ebc77e83 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -98,14 +98,14 @@ module LavinMQ
         {msg_offset, segment, pos}
       end
 
-      def last_offset_by_consumer_tag(consumer_tag)
+      def last_offset_by_consumer_tag(consumer_tag : String)
         if pos = @consumer_offset_positions[consumer_tag]?
           tx = @consumer_offsets.to_slice(pos, 8)
           return IO::ByteFormat::SystemEndian.decode(Int64, tx)
         end
       end
 
-      private def restore_consumer_offset_positions
+      private def restore_consumer_offset_positions : Hash(String, Int64)
         positions = Hash(String, Int64).new
         return positions if @consumer_offsets.size.zero?
 
@@ -133,7 +133,7 @@ module LavinMQ
         end
       end
 
-      def store_consumer_offset(consumer_tag, new_offset)
+      def store_consumer_offset(consumer_tag : String, new_offset : Int64)
         slice = consumer_tag.to_slice
         consumer_tag_length = slice.size.to_u8
         pos = @consumer_offsets.size + slice.size + 1
@@ -165,7 +165,7 @@ module LavinMQ
         end
       end
 
-      def remove_consumer_tag_from_file(consumer_tag)
+      def remove_consumer_tag_from_file(consumer_tag : String)
         @consumer_offset_positions = @consumer_offset_positions.reject! { |k, _v| k == consumer_tag }
 
         offsets_to_save = Hash(String, Int64).new

From 3658c4d5ba57fb4e404d698cc81a59023e85ac01 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 24 Apr 2024 11:16:55 +0200
Subject: [PATCH 33/66] format

---
 src/lavinmq/queue/stream_queue_message_store.cr | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index 6ebc77e83..cd554e12b 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -17,7 +17,7 @@ module LavinMQ
       def initialize(@queue_data_dir : String, @replicator : Clustering::Replicator?)
         super
         @last_offset = get_last_offset
-        @consumer_offset_path  = File.join(@data_dir, "consumer_offsets")
+        @consumer_offset_path = File.join(@data_dir, "consumer_offsets")
         @consumer_offsets = MFile.new(@consumer_offset_path, @consumer_offset_capacity)
         @consumer_offset_positions = restore_consumer_offset_positions
         drop_overflow

From 0586bc8d6ccb0e3b7247f71a6456d60610c25da2 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 24 Apr 2024 11:37:06 +0200
Subject: [PATCH 34/66] lint

---
 src/lavinmq/queue/stream_queue_message_store.cr | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index cd554e12b..ff427f602 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -124,12 +124,10 @@ module LavinMQ
       end
 
       def update_consumer_offset(consumer_tag : String, new_offset : Int64)
-        begin
-          if pos = @consumer_offset_positions[consumer_tag]?
-            IO::ByteFormat::SystemEndian.encode(new_offset, @consumer_offsets.to_slice(pos, 8, false))
-          else
-            store_consumer_offset(consumer_tag, new_offset)
-          end
+        if pos = @consumer_offset_positions[consumer_tag]?
+          IO::ByteFormat::SystemEndian.encode(new_offset, @consumer_offsets.to_slice(pos, 8, false))
+        else
+          store_consumer_offset(consumer_tag, new_offset)
         end
       end
 

From 46670ea990f365ef39823ada9ee5592f6277b923 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 24 Apr 2024 11:39:26 +0200
Subject: [PATCH 35/66] lint

---
 src/lavinmq/queue/stream_queue_message_store.cr | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index ff427f602..36b5d7134 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -148,11 +148,10 @@ module LavinMQ
 
       def cleanup_consumer_offsets
         offsets_to_save = Hash(String, Int64).new
-        lowest_offset_in_stream, _seg, _pos = offset_at(@segments.first_key, 4u32) # handle
-        @consumer_offset_positions.each do |ctag, pos|
-          offset = last_offset_by_consumer_tag(ctag).not_nil!
-          next if offset < lowest_offset_in_stream
-          # Other scenarios to remove?
+        lowest_offset_in_stream, _seg, _pos = offset_at(@segments.first_key, 4u32)
+        @consumer_offset_positions.each do |ctag, _pos|
+          offset = last_offset_by_consumer_tag(ctag)
+          next if !offset || offset < lowest_offset_in_stream
           offsets_to_save[ctag] = offset
         end
 

From d83d2f373cdeb6a9b221ee70ad49c7795030f562 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Thu, 16 May 2024 13:59:41 +0200
Subject: [PATCH 36/66] String#bytesize
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Carl Hörberg <carl@84codes.com>
---
 src/lavinmq/queue/stream_queue_message_store.cr | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index 36b5d7134..e8b1fb2b0 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -133,7 +133,7 @@ module LavinMQ
 
       def store_consumer_offset(consumer_tag : String, new_offset : Int64)
         slice = consumer_tag.to_slice
-        consumer_tag_length = slice.size.to_u8
+        consumer_tag_length = slice.bytesize.to_u8
         pos = @consumer_offsets.size + slice.size + 1
 
         length_buffer = uninitialized UInt8[1]

From 8afb057c3842cde0f7de35972520fed14e8731b3 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Thu, 16 May 2024 14:27:49 +0200
Subject: [PATCH 37/66] refactor store_consumer_offset and
 restore_consumer_offset_positions

---
 .../queue/stream_queue_message_store.cr       | 20 +++++--------------
 1 file changed, 5 insertions(+), 15 deletions(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index e8b1fb2b0..f6eb5f374 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -110,9 +110,8 @@ module LavinMQ
         return positions if @consumer_offsets.size.zero?
 
         loop do
-          ctag_length = @consumer_offsets.read_byte
-          break if !ctag_length || ctag_length.zero?
-          ctag = @consumer_offsets.read_string(ctag_length)
+          ctag = AMQ::Protocol::ShortString.from_io(@consumer_offsets)
+          break if ctag.empty?
           positions[ctag] = @consumer_offsets.pos
           @consumer_offsets.skip(8)
         rescue IO::EOFError
@@ -132,18 +131,9 @@ module LavinMQ
       end
 
       def store_consumer_offset(consumer_tag : String, new_offset : Int64)
-        slice = consumer_tag.to_slice
-        consumer_tag_length = slice.bytesize.to_u8
-        pos = @consumer_offsets.size + slice.size + 1
-
-        length_buffer = uninitialized UInt8[1]
-        IO::ByteFormat::LittleEndian.encode(consumer_tag_length, length_buffer.to_slice)
-
-        offset_buffer = uninitialized UInt8[8]
-        IO::ByteFormat::LittleEndian.encode(new_offset.as(Int64), offset_buffer.to_slice)
-
-        @consumer_offsets.write(length_buffer.to_slice + slice + offset_buffer.to_slice)
-        @consumer_offset_positions[consumer_tag] = pos
+        @consumer_offsets.write_bytes AMQ::Protocol::ShortString.new(consumer_tag)
+        @consumer_offset_positions[consumer_tag] = @consumer_offsets.size
+        @consumer_offsets.write_bytes new_offset
       end
 
       def cleanup_consumer_offsets

From 0f86d6818abd6afc15d0abec5ceb9164d64d6f68 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Thu, 16 May 2024 14:40:10 +0200
Subject: [PATCH 38/66] refactor cleanup_consumer_offsets

---
 src/lavinmq/queue/stream_queue_message_store.cr | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index f6eb5f374..c6c07cc17 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -140,9 +140,9 @@ module LavinMQ
         offsets_to_save = Hash(String, Int64).new
         lowest_offset_in_stream, _seg, _pos = offset_at(@segments.first_key, 4u32)
         @consumer_offset_positions.each do |ctag, _pos|
-          offset = last_offset_by_consumer_tag(ctag)
-          next if !offset || offset < lowest_offset_in_stream
-          offsets_to_save[ctag] = offset
+          if offset = last_offset_by_consumer_tag(ctag)
+            offsets_to_save[ctag] = offset if offset > lowest_offset_in_stream
+          end
         end
 
         delete_and_reopen_offsets_file

From d5108455d7b4596971df0eb14e889c701f5a9873 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Thu, 16 May 2024 14:49:16 +0200
Subject: [PATCH 39/66] replace offset file instead of delete

---
 .../queue/stream_queue_message_store.cr       | 24 +++++++++++--------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index c6c07cc17..f72e9c35e 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -145,10 +145,11 @@ module LavinMQ
           end
         end
 
-        delete_and_reopen_offsets_file
         @consumer_offset_positions = Hash(String, Int64).new
-        offsets_to_save.each do |ctag, offset|
-          store_consumer_offset(ctag, offset)
+        replace_offsets_file do
+          offsets_to_save.each do |ctag, offset|
+            store_consumer_offset(ctag, offset)
+          end
         end
       end
 
@@ -162,16 +163,19 @@ module LavinMQ
           offsets_to_save[ctag] = offset
         end
 
-        delete_and_reopen_offsets_file
-        offsets_to_save.each do |ctag, offset|
-          store_consumer_offset(ctag, offset)
+        replace_offsets_file do
+          offsets_to_save.each do |ctag, offset|
+            store_consumer_offset(ctag, offset)
+          end
         end
       end
 
-      def delete_and_reopen_offsets_file
-        @consumer_offsets.close
-        @consumer_offsets.delete
-        @consumer_offsets = MFile.new(@consumer_offset_path, @consumer_offset_capacity)
+      def replace_offsets_file(&)
+        old_consumer_offsets = @consumer_offsets
+        @consumer_offsets = MFile.new("#{@consumer_offset_path}.tmp", @consumer_offset_capacity)
+        yield # fill the new file with correct data in this block
+        File.rename "#{@consumer_offset_path}.tmp", @consumer_offset_path
+        old_consumer_offsets.close
       end
 
       def shift?(consumer : Client::Channel::StreamConsumer) : Envelope?

From 5d97755e84019f70f775cc6e6e64bacddf6b4d8f Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Thu, 16 May 2024 15:03:49 +0200
Subject: [PATCH 40/66] lint

---
 spec/stream_queue_spec.cr | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/spec/stream_queue_spec.cr b/spec/stream_queue_spec.cr
index a8c288fb1..460aca38f 100644
--- a/spec/stream_queue_spec.cr
+++ b/spec/stream_queue_spec.cr
@@ -23,6 +23,13 @@ module StreamQueueSpecHelpers
       msgs.receive
     end
   end
+
+  def self.offset_from_headers(headers)
+    if headers
+      headers["x-stream-offset"].as(Int64)
+    else fail("No headers found")
+    end
+  end
 end
 
 describe LavinMQ::StreamQueue do
@@ -241,7 +248,7 @@ describe LavinMQ::StreamQueue do
 
       # consume again, should start from last offset automatically
       msg = StreamQueueSpecHelpers.consume_one(queue_name, consumer_tag)
-      msg.properties.headers.not_nil!["x-stream-offset"].as(Int64).should eq offset + 1
+      StreamQueueSpecHelpers.offset_from_headers(msg.properties.headers).should eq offset + 1
     end
 
     it "reads offsets from file on init" do
@@ -295,12 +302,12 @@ describe LavinMQ::StreamQueue do
 
       StreamQueueSpecHelpers.publish(queue_name, 2)
       msg = StreamQueueSpecHelpers.consume_one(queue_name, consumer_tag, c_args)
-      msg.properties.headers.not_nil!["x-stream-offset"].as(Int64).should eq 1
+      StreamQueueSpecHelpers.offset_from_headers(msg.properties.headers).should eq 1
       sleep 0.1
 
       # should consume the same message again since tracking was not saved from last consume
       msg_2 = StreamQueueSpecHelpers.consume_one(queue_name, consumer_tag)
-      msg_2.properties.headers.not_nil!["x-stream-offset"].as(Int64).should eq 1
+      StreamQueueSpecHelpers.offset_from_headers(msg_2.properties.headers).should eq 1
     end
 
     it "should not use saved offset if x-stream-offset is set" do
@@ -312,12 +319,12 @@ describe LavinMQ::StreamQueue do
 
       # get message without x-stream-offset, tracks offset
       msg = StreamQueueSpecHelpers.consume_one(queue_name, consumer_tag)
-      msg.properties.headers.not_nil!["x-stream-offset"].as(Int64).should eq 1
+      StreamQueueSpecHelpers.offset_from_headers(msg.properties.headers).should eq 1
       sleep 0.1
 
       # consume with x-stream-offset set, should consume the same message again
       msg_2 = StreamQueueSpecHelpers.consume_one(queue_name, consumer_tag, c_args)
-      msg_2.properties.headers.not_nil!["x-stream-offset"].as(Int64).should eq 1
+      StreamQueueSpecHelpers.offset_from_headers(msg_2.properties.headers).should eq 1
     end
 
     it "removes offset" do

From 511ac3232fe95095df866939ec2620eda7b5dbe8 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Thu, 16 May 2024 15:15:00 +0200
Subject: [PATCH 41/66] lint

---
 spec/stream_queue_spec.cr | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/spec/stream_queue_spec.cr b/spec/stream_queue_spec.cr
index 460aca38f..740ba5ef6 100644
--- a/spec/stream_queue_spec.cr
+++ b/spec/stream_queue_spec.cr
@@ -27,7 +27,8 @@ module StreamQueueSpecHelpers
   def self.offset_from_headers(headers)
     if headers
       headers["x-stream-offset"].as(Int64)
-    else fail("No headers found")
+    else
+      fail("No headers found")
     end
   end
 end

From 4eb1a0cea2d89b661042a2f12bd9e6815a0692db Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Thu, 23 May 2024 16:38:14 +0200
Subject: [PATCH 42/66] dont track offsets when consumer tag is generated

---
 spec/stream_queue_spec.cr                     | 23 +++++++++++++++++++
 src/lavinmq/client/channel/stream_consumer.cr |  4 ++--
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/spec/stream_queue_spec.cr b/spec/stream_queue_spec.cr
index 740ba5ef6..8beee44ae 100644
--- a/spec/stream_queue_spec.cr
+++ b/spec/stream_queue_spec.cr
@@ -373,5 +373,28 @@ describe LavinMQ::StreamQueue do
       msg_store.last_offset_by_consumer_tag(tag_prefix + 0.to_s).should eq offsets[0]
       msg_store.close
     end
+
+    it "does not track offset if c-tag is auto-generated" do
+      queue_name = Random::Secure.hex
+      StreamQueueSpecHelpers.publish(queue_name, 1)
+      args = {"x-queue-type": "stream"}
+      c_tag = ""
+      with_channel do |ch|
+        ch.prefetch 1
+        q = ch.queue(queue_name, args: AMQP::Client::Arguments.new(args))
+        msgs = Channel(AMQP::Client::DeliverMessage).new
+        c_tag = q.subscribe(no_ack: false) do |msg|
+          msgs.send msg
+          msg.ack
+        end
+        msg = msgs.receive
+      end
+
+      sleep 0.1
+      vhost = Server.vhosts["/"]
+      data_dir = File.join(vhost.data_dir, Digest::SHA1.hexdigest queue_name)
+      msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
+      msg_store.last_offset_by_consumer_tag(c_tag).should eq nil
+    end
   end
 end
diff --git a/src/lavinmq/client/channel/stream_consumer.cr b/src/lavinmq/client/channel/stream_consumer.cr
index da235f61a..0a44a2a9b 100644
--- a/src/lavinmq/client/channel/stream_consumer.cr
+++ b/src/lavinmq/client/channel/stream_consumer.cr
@@ -12,9 +12,9 @@ module LavinMQ
         @track_offset = false
 
         def initialize(@channel : Client::Channel, @queue : StreamQueue, frame : AMQP::Frame::Basic::Consume)
+          @tag = frame.consumer_tag
           validate_preconditions(frame)
           offset = frame.arguments["x-stream-offset"]?
-          @tag = frame.consumer_tag
           @offset, @segment, @pos = stream_queue.find_offset(offset, @tag)
           super
         end
@@ -37,7 +37,7 @@ module LavinMQ
           end
           case frame.arguments["x-stream-offset"]?
           when Nil
-            @track_offset = true
+            @track_offset = true unless @tag.starts_with?("amq.ctag-")
           when Int, Time, "first", "next", "last"
             @track_offset = false
           else raise Error::PreconditionFailed.new("x-stream-offset must be an integer, a timestamp, 'first', 'next' or 'last'")

From e008ec16b11f25e411e53e8f048607f4677e07cc Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Thu, 23 May 2024 16:44:56 +0200
Subject: [PATCH 43/66] remove unused code

---
 spec/stream_queue_spec.cr                     | 23 -------------------
 .../queue/stream_queue_message_store.cr       | 17 --------------
 2 files changed, 40 deletions(-)

diff --git a/spec/stream_queue_spec.cr b/spec/stream_queue_spec.cr
index 8beee44ae..673ef46cc 100644
--- a/spec/stream_queue_spec.cr
+++ b/spec/stream_queue_spec.cr
@@ -328,29 +328,6 @@ describe LavinMQ::StreamQueue do
       StreamQueueSpecHelpers.offset_from_headers(msg_2.properties.headers).should eq 1
     end
 
-    it "removes offset" do
-      queue_name = Random::Secure.hex
-      vhost = Server.vhosts["/"]
-      offsets = [84_i64, 10_i64]
-      tag_prefix = "ctag-"
-      StreamQueueSpecHelpers.publish(queue_name, 1)
-
-      data_dir = File.join(vhost.data_dir, Digest::SHA1.hexdigest queue_name)
-      msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
-      offsets.each_with_index do |offset, i|
-        msg_store.update_consumer_offset(tag_prefix + i.to_s, offset)
-      end
-      sleep 0.1
-      msg_store.remove_consumer_tag_from_file(tag_prefix + 1.to_s)
-      msg_store.close
-      sleep 0.1
-
-      msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
-      msg_store.last_offset_by_consumer_tag(tag_prefix + 1.to_s).should eq nil
-      msg_store.last_offset_by_consumer_tag(tag_prefix + 0.to_s).should eq offsets[0]
-      msg_store.close
-    end
-
     it "cleanup_consumer_offsets removes outdated offset" do
       queue_name = Random::Secure.hex
       vhost = Server.vhosts["/"]
diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index f72e9c35e..63c821166 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -153,23 +153,6 @@ module LavinMQ
         end
       end
 
-      def remove_consumer_tag_from_file(consumer_tag : String)
-        @consumer_offset_positions = @consumer_offset_positions.reject! { |k, _v| k == consumer_tag }
-
-        offsets_to_save = Hash(String, Int64).new
-        @consumer_offset_positions.each do |ctag, _p|
-          offset = last_offset_by_consumer_tag(ctag)
-          next unless offset
-          offsets_to_save[ctag] = offset
-        end
-
-        replace_offsets_file do
-          offsets_to_save.each do |ctag, offset|
-            store_consumer_offset(ctag, offset)
-          end
-        end
-      end
-
       def replace_offsets_file(&)
         old_consumer_offsets = @consumer_offsets
         @consumer_offsets = MFile.new("#{@consumer_offset_path}.tmp", @consumer_offset_capacity)

From 3d186ce98318b5e24fc01bd08da503e12bec59db Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Tue, 28 May 2024 16:42:26 +0200
Subject: [PATCH 44/66] cleanup consumer offsets when dropping overflow

---
 spec/stream_queue_spec.cr                     | 47 +++++++++++++++++--
 .../queue/stream_queue_message_store.cr       |  5 +-
 2 files changed, 48 insertions(+), 4 deletions(-)

diff --git a/spec/stream_queue_spec.cr b/spec/stream_queue_spec.cr
index 673ef46cc..44d00a266 100644
--- a/spec/stream_queue_spec.cr
+++ b/spec/stream_queue_spec.cr
@@ -255,7 +255,7 @@ describe LavinMQ::StreamQueue do
     it "reads offsets from file on init" do
       queue_name = Random::Secure.hex
       vhost = Server.vhosts["/"]
-      offsets = [84_i64, Random.rand(Int64), Random.rand(Int64), Random.rand(Int64)]
+      offsets = [84_i64, 24_i64, 1_i64, 100_i64, 42_i64]
       tag_prefix = "ctag-"
       StreamQueueSpecHelpers.publish(queue_name, 1)
 
@@ -277,7 +277,7 @@ describe LavinMQ::StreamQueue do
     it "only saves one entry per consumer tag" do
       queue_name = Random::Secure.hex
       vhost = Server.vhosts["/"]
-      offsets = [84_i64, Random.rand(Int64), Random.rand(Int64), Random.rand(Int64)]
+      offsets = [84_i64, 24_i64, 1_i64, 100_i64, 42_i64]
       consumer_tag = "ctag-1"
       StreamQueueSpecHelpers.publish(queue_name, 1)
 
@@ -291,7 +291,7 @@ describe LavinMQ::StreamQueue do
 
       msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
       msg_store.last_offset_by_consumer_tag(consumer_tag).should eq offsets.last
-      msg_store.@consumer_offsets.size.should eq 16
+      msg_store.@consumer_offsets.size.should eq 15
 
       msg_store.close
     end
@@ -351,6 +351,47 @@ describe LavinMQ::StreamQueue do
       msg_store.close
     end
 
+    it "runs cleanup when removing segment" do
+      consumer_tag = "ctag-1"
+      offset = -1
+      vhost = Server.vhosts["/"]
+      queue_name = Random::Secure.hex
+      args = {"x-queue-type": "stream", "x-max-length": 2}
+      body_size = 256 * 1024
+      data = Bytes.new(body_size)
+      data_dir = File.join(vhost.data_dir, Digest::SHA1.hexdigest queue_name)
+
+      with_channel do |ch|
+        q = ch.queue(queue_name, args: AMQP::Client::Arguments.new(args))
+        q.publish_confirm data
+      end
+      sleep 0.1
+
+      with_channel do |ch|
+        ch.prefetch 1
+        q = ch.queue(queue_name, args: AMQP::Client::Arguments.new(args))
+        msgs = Channel(AMQP::Client::DeliverMessage).new
+        q.subscribe(no_ack: false, tag: consumer_tag) do |msg|
+          msgs.send msg
+          msg.ack
+        end
+        msgs.receive
+      end
+
+      sleep 0.1
+      msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
+      msg_store.last_offset_by_consumer_tag(consumer_tag).should eq 2
+
+      with_channel do |ch|
+        q = ch.queue(queue_name, args: AMQP::Client::Arguments.new(args))
+        4.times { q.publish_confirm data }
+      end
+      sleep 0.1
+      msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
+      msg_store.last_offset_by_consumer_tag(consumer_tag).should eq nil
+
+    end
+
     it "does not track offset if c-tag is auto-generated" do
       queue_name = Random::Secure.hex
       StreamQueueSpecHelpers.publish(queue_name, 1)
diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index 63c821166..d35592a7b 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -137,11 +137,13 @@ module LavinMQ
       end
 
       def cleanup_consumer_offsets
+        return if @consumer_offsets.size.zero?
+
         offsets_to_save = Hash(String, Int64).new
         lowest_offset_in_stream, _seg, _pos = offset_at(@segments.first_key, 4u32)
         @consumer_offset_positions.each do |ctag, _pos|
           if offset = last_offset_by_consumer_tag(ctag)
-            offsets_to_save[ctag] = offset if offset > lowest_offset_in_stream
+            offsets_to_save[ctag] = offset if offset >= lowest_offset_in_stream
           end
         end
 
@@ -241,6 +243,7 @@ module LavinMQ
             Time.unix_ms(last_ts) < min_ts
           end
         end
+        cleanup_consumer_offsets
       end
 
       private def drop_segments_while(& : UInt32 -> Bool)

From 17fed1716acbbdd782ff7404f2a39e03e96dc297 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Tue, 28 May 2024 16:43:32 +0200
Subject: [PATCH 45/66] format

---
 spec/stream_queue_spec.cr | 1 -
 1 file changed, 1 deletion(-)

diff --git a/spec/stream_queue_spec.cr b/spec/stream_queue_spec.cr
index 44d00a266..2943adfbb 100644
--- a/spec/stream_queue_spec.cr
+++ b/spec/stream_queue_spec.cr
@@ -389,7 +389,6 @@ describe LavinMQ::StreamQueue do
       sleep 0.1
       msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
       msg_store.last_offset_by_consumer_tag(consumer_tag).should eq nil
-
     end
 
     it "does not track offset if c-tag is auto-generated" do

From e8e64860e14c2e47bb6e061bfe6694b9473f8d05 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Tue, 28 May 2024 16:44:56 +0200
Subject: [PATCH 46/66] lint

---
 spec/stream_queue_spec.cr | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/spec/stream_queue_spec.cr b/spec/stream_queue_spec.cr
index 2943adfbb..5c1ef0bc6 100644
--- a/spec/stream_queue_spec.cr
+++ b/spec/stream_queue_spec.cr
@@ -353,7 +353,6 @@ describe LavinMQ::StreamQueue do
 
     it "runs cleanup when removing segment" do
       consumer_tag = "ctag-1"
-      offset = -1
       vhost = Server.vhosts["/"]
       queue_name = Random::Secure.hex
       args = {"x-queue-type": "stream", "x-max-length": 2}
@@ -404,7 +403,7 @@ describe LavinMQ::StreamQueue do
           msgs.send msg
           msg.ack
         end
-        msg = msgs.receive
+        msgs.receive
       end
 
       sleep 0.1

From 0c9856a957c52a90b364fa6251f9ad84e64da0b9 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Thu, 30 May 2024 15:22:56 +0200
Subject: [PATCH 47/66] handle large messages causing first segment to be empty

---
 src/lavinmq/queue/stream_queue_message_store.cr | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index d35592a7b..4675df530 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -59,12 +59,15 @@ module LavinMQ
         end
       end
 
-      private def offset_at(seg, pos) : Tuple(Int64, UInt32, UInt32)
+      private def offset_at(seg, pos, retried = false) : Tuple(Int64, UInt32, UInt32)
         return {@last_offset, seg, pos} if @size.zero?
         mfile = @segments[seg]
         msg = BytesMessage.from_bytes(mfile.to_slice + pos)
         offset = offset_from_headers(msg.properties.headers)
         {offset, seg, pos}
+      rescue ex : IndexError # first segment can be empty if message size >= segment size
+        return offset_at(seg + 1, pos, true) unless retried
+        raise ex
       end
 
       private def last_offset_seg_pos

From 5b70c04725b235ada27972b9409624ec7831f8ac Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Thu, 30 May 2024 15:23:12 +0200
Subject: [PATCH 48/66] cleanup spec

---
 spec/stream_queue_spec.cr | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/spec/stream_queue_spec.cr b/spec/stream_queue_spec.cr
index 5c1ef0bc6..01367448a 100644
--- a/spec/stream_queue_spec.cr
+++ b/spec/stream_queue_spec.cr
@@ -355,16 +355,14 @@ describe LavinMQ::StreamQueue do
       consumer_tag = "ctag-1"
       vhost = Server.vhosts["/"]
       queue_name = Random::Secure.hex
-      args = {"x-queue-type": "stream", "x-max-length": 2}
-      body_size = 256 * 1024
-      data = Bytes.new(body_size)
+      args = {"x-queue-type": "stream", "x-max-length": 1}
+      msg_body = Bytes.new(LavinMQ::Config.instance.segment_size)
       data_dir = File.join(vhost.data_dir, Digest::SHA1.hexdigest queue_name)
 
       with_channel do |ch|
         q = ch.queue(queue_name, args: AMQP::Client::Arguments.new(args))
-        q.publish_confirm data
+        q.publish_confirm msg_body
       end
-      sleep 0.1
 
       with_channel do |ch|
         ch.prefetch 1
@@ -377,15 +375,14 @@ describe LavinMQ::StreamQueue do
         msgs.receive
       end
 
-      sleep 0.1
       msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
       msg_store.last_offset_by_consumer_tag(consumer_tag).should eq 2
 
       with_channel do |ch|
         q = ch.queue(queue_name, args: AMQP::Client::Arguments.new(args))
-        4.times { q.publish_confirm data }
+        2.times { q.publish_confirm msg_body }
       end
-      sleep 0.1
+
       msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
       msg_store.last_offset_by_consumer_tag(consumer_tag).should eq nil
     end

From 6b30bf077afb1582a5b6a06c9357671dbc1ddd30 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Tue, 4 Jun 2024 15:38:19 +0200
Subject: [PATCH 49/66] add option to use broker tracking when x-stream-offset
 is set by using x-stream-use-automatic-offset

---
 src/lavinmq/client/channel/stream_consumer.cr   | 4 ++--
 src/lavinmq/queue/stream_queue_message_store.cr | 7 ++++++-
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/lavinmq/client/channel/stream_consumer.cr b/src/lavinmq/client/channel/stream_consumer.cr
index 0a44a2a9b..55fbaf864 100644
--- a/src/lavinmq/client/channel/stream_consumer.cr
+++ b/src/lavinmq/client/channel/stream_consumer.cr
@@ -15,7 +15,7 @@ module LavinMQ
           @tag = frame.consumer_tag
           validate_preconditions(frame)
           offset = frame.arguments["x-stream-offset"]?
-          @offset, @segment, @pos = stream_queue.find_offset(offset, @tag)
+          @offset, @segment, @pos = stream_queue.find_offset(offset, @tag, @track_offset)
           super
         end
 
@@ -39,7 +39,7 @@ module LavinMQ
           when Nil
             @track_offset = true unless @tag.starts_with?("amq.ctag-")
           when Int, Time, "first", "next", "last"
-            @track_offset = false
+            @track_offset = true if frame.arguments["x-stream-use-automatic-offset"]?
           else raise Error::PreconditionFailed.new("x-stream-offset must be an integer, a timestamp, 'first', 'next' or 'last'")
           end
         end
diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index 4675df530..1d44b5b30 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -39,8 +39,13 @@ module LavinMQ
       # Used once when a consumer is started
       # Populates `segment` and `position` by iterating through segments
       # until `offset` is found
-      def find_offset(offset, tag = nil) : Tuple(Int64, UInt32, UInt32)
+      def find_offset(offset, tag = nil, track_offset = false) : Tuple(Int64, UInt32, UInt32)
         raise ClosedError.new if @closed
+        if track_offset
+          consumer_last_offset = last_offset_by_consumer_tag(tag)
+          return find_offset_in_segments(consumer_last_offset) if consumer_last_offset
+        end
+
         case offset
         when "first" then offset_at(@segments.first_key, 4u32)
         when "last"  then offset_at(@segments.last_key, 4u32)

From ca6112f0b2dd3a379006b8058d38ae973aeb60f9 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Tue, 4 Jun 2024 15:38:30 +0200
Subject: [PATCH 50/66] add spec

---
 spec/stream_queue_spec.cr | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/spec/stream_queue_spec.cr b/spec/stream_queue_spec.cr
index 01367448a..01aec6637 100644
--- a/spec/stream_queue_spec.cr
+++ b/spec/stream_queue_spec.cr
@@ -328,6 +328,23 @@ describe LavinMQ::StreamQueue do
       StreamQueueSpecHelpers.offset_from_headers(msg_2.properties.headers).should eq 1
     end
 
+    it "should use saved offset if x-stream-offset & x-stream-use-automatic-offset is set" do
+      queue_name = Random::Secure.hex
+      consumer_tag = Random::Secure.hex
+      c_args = AMQP::Client::Arguments.new({"x-stream-offset": 0, "x-stream-use-automatic-offset": true})
+
+      StreamQueueSpecHelpers.publish(queue_name, 2)
+
+      # get message without x-stream-offset, tracks offset
+      msg = StreamQueueSpecHelpers.consume_one(queue_name, consumer_tag, c_args)
+      StreamQueueSpecHelpers.offset_from_headers(msg.properties.headers).should eq 1
+      sleep 0.1
+
+      # consume with x-stream-offset set, should consume the same message again
+      msg_2 = StreamQueueSpecHelpers.consume_one(queue_name, consumer_tag, c_args)
+      StreamQueueSpecHelpers.offset_from_headers(msg_2.properties.headers).should eq 2
+    end
+
     it "cleanup_consumer_offsets removes outdated offset" do
       queue_name = Random::Secure.hex
       vhost = Server.vhosts["/"]

From 728f2f75e86193c52c9399c5e636a8b05fd0353d Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Mon, 17 Jun 2024 10:36:29 +0200
Subject: [PATCH 51/66] no need to truncate mfile, it's being deleted
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Carl Hörberg <carl@84codes.com>
---
 src/lavinmq/queue/stream_queue_message_store.cr | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index 1d44b5b30..b629a62ab 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -168,7 +168,7 @@ module LavinMQ
         @consumer_offsets = MFile.new("#{@consumer_offset_path}.tmp", @consumer_offset_capacity)
         yield # fill the new file with correct data in this block
         File.rename "#{@consumer_offset_path}.tmp", @consumer_offset_path
-        old_consumer_offsets.close
+        old_consumer_offsets.close(truncate_to_size: false)
       end
 
       def shift?(consumer : Client::Channel::StreamConsumer) : Envelope?

From 91519e148b78191f483667cd7b2ffbd95b961c18 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Mon, 17 Jun 2024 10:38:29 +0200
Subject: [PATCH 52/66] x-stream-use-automatic-offset ->
 x-stream-automatic-offset-tracking

---
 spec/stream_queue_spec.cr                     | 4 ++--
 src/lavinmq/client/channel/stream_consumer.cr | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/spec/stream_queue_spec.cr b/spec/stream_queue_spec.cr
index 01aec6637..8a9cddc77 100644
--- a/spec/stream_queue_spec.cr
+++ b/spec/stream_queue_spec.cr
@@ -328,10 +328,10 @@ describe LavinMQ::StreamQueue do
       StreamQueueSpecHelpers.offset_from_headers(msg_2.properties.headers).should eq 1
     end
 
-    it "should use saved offset if x-stream-offset & x-stream-use-automatic-offset is set" do
+    it "should use saved offset if x-stream-offset & x-stream-automatic-offset-tracking is set" do
       queue_name = Random::Secure.hex
       consumer_tag = Random::Secure.hex
-      c_args = AMQP::Client::Arguments.new({"x-stream-offset": 0, "x-stream-use-automatic-offset": true})
+      c_args = AMQP::Client::Arguments.new({"x-stream-offset": 0, "x-stream-automatic-offset-tracking": true})
 
       StreamQueueSpecHelpers.publish(queue_name, 2)
 
diff --git a/src/lavinmq/client/channel/stream_consumer.cr b/src/lavinmq/client/channel/stream_consumer.cr
index 55fbaf864..57680887d 100644
--- a/src/lavinmq/client/channel/stream_consumer.cr
+++ b/src/lavinmq/client/channel/stream_consumer.cr
@@ -39,7 +39,7 @@ module LavinMQ
           when Nil
             @track_offset = true unless @tag.starts_with?("amq.ctag-")
           when Int, Time, "first", "next", "last"
-            @track_offset = true if frame.arguments["x-stream-use-automatic-offset"]?
+            @track_offset = true if frame.arguments["x-stream-automatic-offset-tracking"]?
           else raise Error::PreconditionFailed.new("x-stream-offset must be an integer, a timestamp, 'first', 'next' or 'last'")
           end
         end

From fb8228b1998eed59eaff146d86fe6ac8d67c66f6 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Mon, 17 Jun 2024 10:51:24 +0200
Subject: [PATCH 53/66] implement rename in mfile

---
 src/lavinmq/mfile.cr                            | 5 +++++
 src/lavinmq/queue/stream_queue_message_store.cr | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/lavinmq/mfile.cr b/src/lavinmq/mfile.cr
index 5d90cebb0..7cd2f8c8e 100644
--- a/src/lavinmq/mfile.cr
+++ b/src/lavinmq/mfile.cr
@@ -269,4 +269,9 @@ class MFile < IO
     raise IO::Error.from_errno("pread") if cnt == -1
     cnt
   end
+
+  def rename(new_path : String) : Nil
+    File.rename @path, new_path
+    @path = new_path
+  end
 end
diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index b629a62ab..3cda17cf3 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -167,7 +167,7 @@ module LavinMQ
         old_consumer_offsets = @consumer_offsets
         @consumer_offsets = MFile.new("#{@consumer_offset_path}.tmp", @consumer_offset_capacity)
         yield # fill the new file with correct data in this block
-        File.rename "#{@consumer_offset_path}.tmp", @consumer_offset_path
+        @consumer_offsets.rename(@consumer_offset_path)
         old_consumer_offsets.close(truncate_to_size: false)
       end
 

From c988961576a57a4fcaded53ef91f40b244e9478d Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Mon, 17 Jun 2024 14:33:53 +0200
Subject: [PATCH 54/66] expand consumer offsets file if full

---
 spec/stream_queue_spec.cr                     | 25 +++++++++++++++++++
 .../queue/stream_queue_message_store.cr       | 12 +++++++++
 2 files changed, 37 insertions(+)

diff --git a/spec/stream_queue_spec.cr b/spec/stream_queue_spec.cr
index 8a9cddc77..9a5fdb0d0 100644
--- a/spec/stream_queue_spec.cr
+++ b/spec/stream_queue_spec.cr
@@ -426,5 +426,30 @@ describe LavinMQ::StreamQueue do
       msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
       msg_store.last_offset_by_consumer_tag(c_tag).should eq nil
     end
+
+    it "expands consumer offset file when needed" do
+      queue_name = Random::Secure.hex
+      vhost = Server.vhosts["/"]
+      consumer_tag_prefix = "ctag-"
+      StreamQueueSpecHelpers.publish(queue_name, 1)
+
+      data_dir = File.join(vhost.data_dir, Digest::SHA1.hexdigest queue_name)
+      msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
+      2000.times do |i|
+        next if i == 0
+        msg_store.update_consumer_offset("#{consumer_tag_prefix}#{i}", i)
+      end
+      msg_store.close
+
+      msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
+      msg_store.@consumer_offsets.size.should eq 34_875
+
+      2000.times do |i|
+        next if i == 0
+        msg_store.last_offset_by_consumer_tag("#{consumer_tag_prefix}#{i}").should eq i
+      end
+
+      msg_store.close
+    end
   end
 end
diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index 3cda17cf3..1d651832c 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -139,11 +139,23 @@ module LavinMQ
       end
 
       def store_consumer_offset(consumer_tag : String, new_offset : Int64)
+        expand_consumer_offset_file if consumer_offset_file_full?(consumer_tag)
         @consumer_offsets.write_bytes AMQ::Protocol::ShortString.new(consumer_tag)
         @consumer_offset_positions[consumer_tag] = @consumer_offsets.size
         @consumer_offsets.write_bytes new_offset
       end
 
+      def consumer_offset_file_full?(consumer_tag)
+        (@consumer_offsets.size + consumer_tag.bytesize + 8) >= @consumer_offsets.capacity
+      end
+
+      def expand_consumer_offset_file
+        pos = @consumer_offsets.size
+        @consumer_offset_capacity += 32_768
+        @consumer_offsets = MFile.new(@consumer_offset_path, @consumer_offset_capacity)
+        @consumer_offsets.resize(pos)
+      end
+
       def cleanup_consumer_offsets
         return if @consumer_offsets.size.zero?
 

From 69cd07a7ee9497a4718c4de1af795693fe0a1928 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Mon, 17 Jun 2024 14:34:13 +0200
Subject: [PATCH 55/66] remove unused code

---
 src/lavinmq/mfile.cr | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/src/lavinmq/mfile.cr b/src/lavinmq/mfile.cr
index 7cd2f8c8e..b103d654d 100644
--- a/src/lavinmq/mfile.cr
+++ b/src/lavinmq/mfile.cr
@@ -180,14 +180,6 @@ class MFile < IO
     @size = new_size
   end
 
-  def write_at(pos : Int64, slice : Bytes) : Nil
-    raise ClosedError.new if @closed
-    end_pos = pos + slice.size
-    raise IO::EOFError.new if end_pos > @capacity
-    @size = end_pos if end_pos > @size
-    slice.copy_to(buffer + pos, slice.size)
-  end
-
   def read(slice : Bytes)
     pos = @pos
     new_pos = pos + slice.size

From 6dbbc1ecc350e8ecdce8278cf985eee65e2a335e Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Mon, 17 Jun 2024 14:36:20 +0200
Subject: [PATCH 56/66] use LittleEndian
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Carl Hörberg <carl@84codes.com>
---
 src/lavinmq/queue/stream_queue_message_store.cr | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index 1d651832c..c9eab7b9f 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -109,7 +109,7 @@ module LavinMQ
       def last_offset_by_consumer_tag(consumer_tag : String)
         if pos = @consumer_offset_positions[consumer_tag]?
           tx = @consumer_offsets.to_slice(pos, 8)
-          return IO::ByteFormat::SystemEndian.decode(Int64, tx)
+          return IO::ByteFormat::LittleEndian.decode(Int64, tx)
         end
       end
 

From 749f154fda5bffe41e42f57ecd44587698fc93cf Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 19 Jun 2024 14:40:20 +0200
Subject: [PATCH 57/66] remove instance variables

---
 src/lavinmq/queue/stream_queue_message_store.cr | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index c9eab7b9f..0e77799d1 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -11,14 +11,11 @@ module LavinMQ
       @segment_last_ts = Hash(UInt32, Int64).new(0i64)     # used for max-age
       @consumer_offset_positions = Hash(String, Int64).new # used for consumer offsets
       @consumer_offsets : MFile
-      @consumer_offset_path : String
-      @consumer_offset_capacity = 32_768
 
       def initialize(@queue_data_dir : String, @replicator : Clustering::Replicator?)
         super
         @last_offset = get_last_offset
-        @consumer_offset_path = File.join(@data_dir, "consumer_offsets")
-        @consumer_offsets = MFile.new(@consumer_offset_path, @consumer_offset_capacity)
+        @consumer_offsets = MFile.new(File.join(@data_dir, "consumer_offsets"), 32 * 1024)
         @consumer_offset_positions = restore_consumer_offset_positions
         drop_overflow
       end
@@ -151,8 +148,7 @@ module LavinMQ
 
       def expand_consumer_offset_file
         pos = @consumer_offsets.size
-        @consumer_offset_capacity += 32_768
-        @consumer_offsets = MFile.new(@consumer_offset_path, @consumer_offset_capacity)
+        @consumer_offsets = MFile.new(@consumer_offsets.path, @consumer_offsets.capacity + 32 * 1024)
         @consumer_offsets.resize(pos)
       end
 
@@ -177,9 +173,9 @@ module LavinMQ
 
       def replace_offsets_file(&)
         old_consumer_offsets = @consumer_offsets
-        @consumer_offsets = MFile.new("#{@consumer_offset_path}.tmp", @consumer_offset_capacity)
+        @consumer_offsets = MFile.new("#{@consumer_offsets.path}.tmp", 32 * 1024)
         yield # fill the new file with correct data in this block
-        @consumer_offsets.rename(@consumer_offset_path)
+        @consumer_offsets.rename(@consumer_offsets.path.sub(".tmp",""))
         old_consumer_offsets.close(truncate_to_size: false)
       end
 

From c7819f37638c815baaecf4c7d027c56c99289e50 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 19 Jun 2024 14:42:41 +0200
Subject: [PATCH 58/66] use old_consumer_offsets.path

---
 src/lavinmq/queue/stream_queue_message_store.cr | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index 0e77799d1..8dad4810d 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -173,9 +173,9 @@ module LavinMQ
 
       def replace_offsets_file(&)
         old_consumer_offsets = @consumer_offsets
-        @consumer_offsets = MFile.new("#{@consumer_offsets.path}.tmp", 32 * 1024)
+        @consumer_offsets = MFile.new("#{old_consumer_offsets.path}.tmp", 32 * 1024)
         yield # fill the new file with correct data in this block
-        @consumer_offsets.rename(@consumer_offsets.path.sub(".tmp",""))
+        @consumer_offsets.rename(old_consumer_offsets.path)
         old_consumer_offsets.close(truncate_to_size: false)
       end
 

From 78eacb71e382b13f007c16ab95e2487dc65c7d11 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 19 Jun 2024 15:17:18 +0200
Subject: [PATCH 59/66] start reading at pos=4 after IndexError in offset_at

---
 src/lavinmq/queue/stream_queue_message_store.cr | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index 8dad4810d..884c2bc4c 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -68,7 +68,7 @@ module LavinMQ
         offset = offset_from_headers(msg.properties.headers)
         {offset, seg, pos}
       rescue ex : IndexError # first segment can be empty if message size >= segment size
-        return offset_at(seg + 1, pos, true) unless retried
+        return offset_at(seg + 1, 4_u32, true) unless retried
         raise ex
       end
 

From 3c2f97accc223e4ab9a48d580cc4f08a365df512 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 19 Jun 2024 15:35:23 +0200
Subject: [PATCH 60/66] use queue_data_dir

---
 src/lavinmq/queue/stream_queue_message_store.cr | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index 884c2bc4c..3edfbbf3e 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -15,7 +15,7 @@ module LavinMQ
       def initialize(@queue_data_dir : String, @replicator : Clustering::Replicator?)
         super
         @last_offset = get_last_offset
-        @consumer_offsets = MFile.new(File.join(@data_dir, "consumer_offsets"), 32 * 1024)
+        @consumer_offsets = MFile.new(File.join(@queue_data_dir, "consumer_offsets"), 32 * 1024)
         @consumer_offset_positions = restore_consumer_offset_positions
         drop_overflow
       end

From a74d4a645d03abbf7a3b10f907b099b3d8179a15 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 19 Jun 2024 15:38:55 +0200
Subject: [PATCH 61/66] update specs to start amqp servers where needed

---
 spec/stream_queue_spec.cr | 282 ++++++++++++++++++++------------------
 1 file changed, 150 insertions(+), 132 deletions(-)

diff --git a/spec/stream_queue_spec.cr b/spec/stream_queue_spec.cr
index 9a5fdb0d0..4829f4429 100644
--- a/spec/stream_queue_spec.cr
+++ b/spec/stream_queue_spec.cr
@@ -2,17 +2,17 @@ require "./spec_helper"
 require "./../src/lavinmq/queue"
 
 module StreamQueueSpecHelpers
-  def self.publish(queue_name, nr_of_messages)
+  def self.publish(s, queue_name, nr_of_messages)
     args = {"x-queue-type": "stream"}
-    with_channel do |ch|
+    with_channel(s) do |ch|
       q = ch.queue(queue_name, args: AMQP::Client::Arguments.new(args))
       nr_of_messages.times { |i| q.publish "m#{i}" }
     end
   end
 
-  def self.consume_one(queue_name, c_tag, c_args = AMQP::Client::Arguments.new)
+  def self.consume_one(s, queue_name, c_tag, c_args = AMQP::Client::Arguments.new)
     args = {"x-queue-type": "stream"}
-    with_channel do |ch|
+    with_channel(s) do |ch|
       ch.prefetch 1
       q = ch.queue(queue_name, args: AMQP::Client::Arguments.new(args))
       msgs = Channel(AMQP::Client::DeliverMessage).new
@@ -242,58 +242,63 @@ describe LavinMQ::StreamQueue do
       consumer_tag = Random::Secure.hex
       offset = 3
 
-      StreamQueueSpecHelpers.publish(queue_name, offset + 1)
-
-      offset.times { StreamQueueSpecHelpers.consume_one(queue_name, consumer_tag) }
-      sleep 0.1
+      with_amqp_server do |s|
+        StreamQueueSpecHelpers.publish(s, queue_name, offset + 1)
+        offset.times { StreamQueueSpecHelpers.consume_one(s, queue_name, consumer_tag) }
+        sleep 0.1
 
-      # consume again, should start from last offset automatically
-      msg = StreamQueueSpecHelpers.consume_one(queue_name, consumer_tag)
-      StreamQueueSpecHelpers.offset_from_headers(msg.properties.headers).should eq offset + 1
+        # consume again, should start from last offset automatically
+        msg = StreamQueueSpecHelpers.consume_one(s, queue_name, consumer_tag)
+        StreamQueueSpecHelpers.offset_from_headers(msg.properties.headers).should eq offset + 1
+      end
     end
 
     it "reads offsets from file on init" do
       queue_name = Random::Secure.hex
-      vhost = Server.vhosts["/"]
       offsets = [84_i64, 24_i64, 1_i64, 100_i64, 42_i64]
       tag_prefix = "ctag-"
-      StreamQueueSpecHelpers.publish(queue_name, 1)
 
-      data_dir = File.join(vhost.data_dir, Digest::SHA1.hexdigest queue_name)
-      msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
-      offsets.each_with_index do |offset, i|
-        msg_store.update_consumer_offset(tag_prefix + i.to_s, offset)
-      end
-      msg_store.close
-      sleep 0.1
+      with_amqp_server do |s|
+        vhost = s.vhosts["/"]
+        StreamQueueSpecHelpers.publish(s, queue_name, 1)
+
+        data_dir = File.join(vhost.data_dir, Digest::SHA1.hexdigest queue_name)
+        msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
+        offsets.each_with_index do |offset, i|
+          msg_store.update_consumer_offset(tag_prefix + i.to_s, offset)
+        end
+        msg_store.close
+        sleep 0.1
 
-      msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
-      offsets.each_with_index do |offset, i|
-        msg_store.last_offset_by_consumer_tag(tag_prefix + i.to_s).should eq offset
+        msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
+        offsets.each_with_index do |offset, i|
+          msg_store.last_offset_by_consumer_tag(tag_prefix + i.to_s).should eq offset
+        end
+        msg_store.close
       end
-      msg_store.close
     end
 
     it "only saves one entry per consumer tag" do
       queue_name = Random::Secure.hex
-      vhost = Server.vhosts["/"]
       offsets = [84_i64, 24_i64, 1_i64, 100_i64, 42_i64]
       consumer_tag = "ctag-1"
-      StreamQueueSpecHelpers.publish(queue_name, 1)
+      with_amqp_server do |s|
+        StreamQueueSpecHelpers.publish(s, queue_name, 1)
 
-      data_dir = File.join(vhost.data_dir, Digest::SHA1.hexdigest queue_name)
-      msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
-      offsets.each do |offset|
-        msg_store.update_consumer_offset(consumer_tag, offset)
-      end
-      msg_store.close
-      sleep 0.1
+        data_dir = File.join(s.vhosts["/"].data_dir, Digest::SHA1.hexdigest queue_name)
+        msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
+        offsets.each do |offset|
+          msg_store.update_consumer_offset(consumer_tag, offset)
+        end
+        msg_store.close
+        sleep 0.1
 
-      msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
-      msg_store.last_offset_by_consumer_tag(consumer_tag).should eq offsets.last
-      msg_store.@consumer_offsets.size.should eq 15
+        msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
+        msg_store.last_offset_by_consumer_tag(consumer_tag).should eq offsets.last
+        msg_store.@consumer_offsets.size.should eq 15
 
-      msg_store.close
+        msg_store.close
+      end
     end
 
     it "does not track offset if x-stream-offset is set" do
@@ -301,14 +306,16 @@ describe LavinMQ::StreamQueue do
       consumer_tag = Random::Secure.hex
       c_args = AMQP::Client::Arguments.new({"x-stream-offset": 0})
 
-      StreamQueueSpecHelpers.publish(queue_name, 2)
-      msg = StreamQueueSpecHelpers.consume_one(queue_name, consumer_tag, c_args)
-      StreamQueueSpecHelpers.offset_from_headers(msg.properties.headers).should eq 1
-      sleep 0.1
-
-      # should consume the same message again since tracking was not saved from last consume
-      msg_2 = StreamQueueSpecHelpers.consume_one(queue_name, consumer_tag)
-      StreamQueueSpecHelpers.offset_from_headers(msg_2.properties.headers).should eq 1
+      with_amqp_server do |s|
+        StreamQueueSpecHelpers.publish(s, queue_name, 2)
+        msg = StreamQueueSpecHelpers.consume_one(s, queue_name, consumer_tag, c_args)
+        StreamQueueSpecHelpers.offset_from_headers(msg.properties.headers).should eq 1
+        sleep 0.1
+
+        # should consume the same message again since tracking was not saved from last consume
+        msg_2 = StreamQueueSpecHelpers.consume_one(s, queue_name, consumer_tag)
+        StreamQueueSpecHelpers.offset_from_headers(msg_2.properties.headers).should eq 1
+      end
     end
 
     it "should not use saved offset if x-stream-offset is set" do
@@ -316,16 +323,18 @@ describe LavinMQ::StreamQueue do
       consumer_tag = Random::Secure.hex
       c_args = AMQP::Client::Arguments.new({"x-stream-offset": 0})
 
-      StreamQueueSpecHelpers.publish(queue_name, 2)
+      with_amqp_server do |s|
+        StreamQueueSpecHelpers.publish(s, queue_name, 2)
 
-      # get message without x-stream-offset, tracks offset
-      msg = StreamQueueSpecHelpers.consume_one(queue_name, consumer_tag)
-      StreamQueueSpecHelpers.offset_from_headers(msg.properties.headers).should eq 1
-      sleep 0.1
+        # get message without x-stream-offset, tracks offset
+        msg = StreamQueueSpecHelpers.consume_one(s, queue_name, consumer_tag)
+        StreamQueueSpecHelpers.offset_from_headers(msg.properties.headers).should eq 1
+        sleep 0.1
 
-      # consume with x-stream-offset set, should consume the same message again
-      msg_2 = StreamQueueSpecHelpers.consume_one(queue_name, consumer_tag, c_args)
-      StreamQueueSpecHelpers.offset_from_headers(msg_2.properties.headers).should eq 1
+        # consume with x-stream-offset set, should consume the same message again
+        msg_2 = StreamQueueSpecHelpers.consume_one(s, queue_name, consumer_tag, c_args)
+        StreamQueueSpecHelpers.offset_from_headers(msg_2.properties.headers).should eq 1
+      end
     end
 
     it "should use saved offset if x-stream-offset & x-stream-automatic-offset-tracking is set" do
@@ -333,123 +342,132 @@ describe LavinMQ::StreamQueue do
       consumer_tag = Random::Secure.hex
       c_args = AMQP::Client::Arguments.new({"x-stream-offset": 0, "x-stream-automatic-offset-tracking": true})
 
-      StreamQueueSpecHelpers.publish(queue_name, 2)
+      with_amqp_server do |s|
+        StreamQueueSpecHelpers.publish(s, queue_name, 2)
 
-      # get message without x-stream-offset, tracks offset
-      msg = StreamQueueSpecHelpers.consume_one(queue_name, consumer_tag, c_args)
-      StreamQueueSpecHelpers.offset_from_headers(msg.properties.headers).should eq 1
-      sleep 0.1
+        # get message without x-stream-offset, tracks offset
+        msg = StreamQueueSpecHelpers.consume_one(s, queue_name, consumer_tag, c_args)
+        StreamQueueSpecHelpers.offset_from_headers(msg.properties.headers).should eq 1
+        sleep 0.1
 
-      # consume with x-stream-offset set, should consume the same message again
-      msg_2 = StreamQueueSpecHelpers.consume_one(queue_name, consumer_tag, c_args)
-      StreamQueueSpecHelpers.offset_from_headers(msg_2.properties.headers).should eq 2
+        # consume with x-stream-offset set, should consume the same message again
+        msg_2 = StreamQueueSpecHelpers.consume_one(s, queue_name, consumer_tag, c_args)
+        StreamQueueSpecHelpers.offset_from_headers(msg_2.properties.headers).should eq 2
+      end
     end
 
     it "cleanup_consumer_offsets removes outdated offset" do
       queue_name = Random::Secure.hex
-      vhost = Server.vhosts["/"]
       offsets = [84_i64, -10_i64]
       tag_prefix = "ctag-"
-      StreamQueueSpecHelpers.publish(queue_name, 1)
 
-      data_dir = File.join(vhost.data_dir, Digest::SHA1.hexdigest queue_name)
-      msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
-      offsets.each_with_index do |offset, i|
-        msg_store.update_consumer_offset(tag_prefix + i.to_s, offset)
+      with_amqp_server do |s|
+        StreamQueueSpecHelpers.publish(s, queue_name, 1)
+
+        data_dir = File.join(s.vhosts["/"].data_dir, Digest::SHA1.hexdigest queue_name)
+        msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
+        offsets.each_with_index do |offset, i|
+          msg_store.update_consumer_offset(tag_prefix + i.to_s, offset)
+        end
+        sleep 0.1
+        msg_store.cleanup_consumer_offsets
+        msg_store.close
+        sleep 0.1
+
+        msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
+        msg_store.last_offset_by_consumer_tag(tag_prefix + 1.to_s).should eq nil
+        msg_store.last_offset_by_consumer_tag(tag_prefix + 0.to_s).should eq offsets[0]
+        msg_store.close
       end
-      sleep 0.1
-      msg_store.cleanup_consumer_offsets
-      msg_store.close
-      sleep 0.1
-
-      msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
-      msg_store.last_offset_by_consumer_tag(tag_prefix + 1.to_s).should eq nil
-      msg_store.last_offset_by_consumer_tag(tag_prefix + 0.to_s).should eq offsets[0]
-      msg_store.close
     end
 
     it "runs cleanup when removing segment" do
       consumer_tag = "ctag-1"
-      vhost = Server.vhosts["/"]
       queue_name = Random::Secure.hex
       args = {"x-queue-type": "stream", "x-max-length": 1}
       msg_body = Bytes.new(LavinMQ::Config.instance.segment_size)
-      data_dir = File.join(vhost.data_dir, Digest::SHA1.hexdigest queue_name)
 
-      with_channel do |ch|
-        q = ch.queue(queue_name, args: AMQP::Client::Arguments.new(args))
-        q.publish_confirm msg_body
-      end
+      with_amqp_server do |s|
+        data_dir = File.join(s.vhosts["/"].data_dir, Digest::SHA1.hexdigest queue_name)
 
-      with_channel do |ch|
-        ch.prefetch 1
-        q = ch.queue(queue_name, args: AMQP::Client::Arguments.new(args))
-        msgs = Channel(AMQP::Client::DeliverMessage).new
-        q.subscribe(no_ack: false, tag: consumer_tag) do |msg|
-          msgs.send msg
-          msg.ack
+        with_channel(s) do |ch|
+          q = ch.queue(queue_name, args: AMQP::Client::Arguments.new(args))
+          q.publish_confirm msg_body
+        end
+
+        with_channel(s) do |ch|
+          ch.prefetch 1
+          q = ch.queue(queue_name, args: AMQP::Client::Arguments.new(args))
+          msgs = Channel(AMQP::Client::DeliverMessage).new
+          q.subscribe(no_ack: false, tag: consumer_tag) do |msg|
+            msgs.send msg
+            msg.ack
+          end
+          msgs.receive
         end
-        msgs.receive
-      end
 
-      msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
-      msg_store.last_offset_by_consumer_tag(consumer_tag).should eq 2
+        msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
+        msg_store.last_offset_by_consumer_tag(consumer_tag).should eq 2
 
-      with_channel do |ch|
-        q = ch.queue(queue_name, args: AMQP::Client::Arguments.new(args))
-        2.times { q.publish_confirm msg_body }
-      end
+        with_channel(s) do |ch|
+          q = ch.queue(queue_name, args: AMQP::Client::Arguments.new(args))
+          2.times { q.publish_confirm msg_body }
+        end
 
-      msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
-      msg_store.last_offset_by_consumer_tag(consumer_tag).should eq nil
+        msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
+        msg_store.last_offset_by_consumer_tag(consumer_tag).should eq nil
+      end
     end
 
     it "does not track offset if c-tag is auto-generated" do
       queue_name = Random::Secure.hex
-      StreamQueueSpecHelpers.publish(queue_name, 1)
-      args = {"x-queue-type": "stream"}
-      c_tag = ""
-      with_channel do |ch|
-        ch.prefetch 1
-        q = ch.queue(queue_name, args: AMQP::Client::Arguments.new(args))
-        msgs = Channel(AMQP::Client::DeliverMessage).new
-        c_tag = q.subscribe(no_ack: false) do |msg|
-          msgs.send msg
-          msg.ack
+
+      with_amqp_server do |s|
+        StreamQueueSpecHelpers.publish(s, queue_name, 1)
+        args = {"x-queue-type": "stream"}
+        c_tag = ""
+        with_channel(s) do |ch|
+          ch.prefetch 1
+          q = ch.queue(queue_name, args: AMQP::Client::Arguments.new(args))
+          msgs = Channel(AMQP::Client::DeliverMessage).new
+          c_tag = q.subscribe(no_ack: false) do |msg|
+            msgs.send msg
+            msg.ack
+          end
+          msgs.receive
         end
-        msgs.receive
-      end
 
-      sleep 0.1
-      vhost = Server.vhosts["/"]
-      data_dir = File.join(vhost.data_dir, Digest::SHA1.hexdigest queue_name)
-      msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
-      msg_store.last_offset_by_consumer_tag(c_tag).should eq nil
+        sleep 0.1
+        data_dir = File.join(s.vhosts["/"].data_dir, Digest::SHA1.hexdigest queue_name)
+        msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
+        msg_store.last_offset_by_consumer_tag(c_tag).should eq nil
+      end
     end
 
     it "expands consumer offset file when needed" do
       queue_name = Random::Secure.hex
-      vhost = Server.vhosts["/"]
       consumer_tag_prefix = "ctag-"
-      StreamQueueSpecHelpers.publish(queue_name, 1)
+      with_amqp_server do |s|
+        StreamQueueSpecHelpers.publish(s, queue_name, 1)
 
-      data_dir = File.join(vhost.data_dir, Digest::SHA1.hexdigest queue_name)
-      msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
-      2000.times do |i|
-        next if i == 0
-        msg_store.update_consumer_offset("#{consumer_tag_prefix}#{i}", i)
-      end
-      msg_store.close
+        data_dir = File.join(s.vhosts["/"].data_dir, Digest::SHA1.hexdigest queue_name)
+        msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
+        2000.times do |i|
+          next if i == 0
+          msg_store.update_consumer_offset("#{consumer_tag_prefix}#{i}", i)
+        end
+        msg_store.close
 
-      msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
-      msg_store.@consumer_offsets.size.should eq 34_875
+        msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
+        msg_store.@consumer_offsets.size.should eq 34_875
 
-      2000.times do |i|
-        next if i == 0
-        msg_store.last_offset_by_consumer_tag("#{consumer_tag_prefix}#{i}").should eq i
-      end
+        2000.times do |i|
+          next if i == 0
+          msg_store.last_offset_by_consumer_tag("#{consumer_tag_prefix}#{i}").should eq i
+        end
 
-      msg_store.close
+        msg_store.close
+      end
     end
   end
 end

From 8c21f06e1ad29b16a60b8198e3e2779bc2946ef8 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 19 Jun 2024 18:42:53 +0200
Subject: [PATCH 62/66] ameba:disable Metrics/CyclomaticComplexity for
 find_offset

---
 src/lavinmq/queue/stream_queue_message_store.cr | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index 3edfbbf3e..d4cbd37ae 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -36,6 +36,7 @@ module LavinMQ
       # Used once when a consumer is started
       # Populates `segment` and `position` by iterating through segments
       # until `offset` is found
+      # ameba:disable Metrics/CyclomaticComplexity
       def find_offset(offset, tag = nil, track_offset = false) : Tuple(Int64, UInt32, UInt32)
         raise ClosedError.new if @closed
         if track_offset

From 64120953af1dc5813e501d8963854321271cef35 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Fri, 21 Jun 2024 00:34:53 +0200
Subject: [PATCH 63/66] include tag size prefix byte in
 consumer_offset_file_full?
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Carl Hörberg <carl@84codes.com>
---
 src/lavinmq/queue/stream_queue_message_store.cr | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index d4cbd37ae..dac02fe2e 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -144,7 +144,7 @@ module LavinMQ
       end
 
       def consumer_offset_file_full?(consumer_tag)
-        (@consumer_offsets.size + consumer_tag.bytesize + 8) >= @consumer_offsets.capacity
+        (@consumer_offsets.size + 1 + consumer_tag.bytesize + 8) >= @consumer_offsets.capacity
       end
 
       def expand_consumer_offset_file

From 173377f7247f44912cd2f22e24bfe255e056b777 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Wed, 26 Jun 2024 13:06:48 +0200
Subject: [PATCH 64/66] only append consumer offsets file, compact when full,
 expand if still full. Use config.instance.segment_size

---
 spec/stream_queue_spec.cr                     | 52 ++++++++++++++++---
 src/lavinmq/client/channel/stream_consumer.cr |  2 +-
 src/lavinmq/queue/stream_queue.cr             |  4 +-
 .../queue/stream_queue_message_store.cr       | 16 ++----
 4 files changed, 53 insertions(+), 21 deletions(-)

diff --git a/spec/stream_queue_spec.cr b/spec/stream_queue_spec.cr
index 4829f4429..864a68c59 100644
--- a/spec/stream_queue_spec.cr
+++ b/spec/stream_queue_spec.cr
@@ -265,7 +265,7 @@ describe LavinMQ::StreamQueue do
         data_dir = File.join(vhost.data_dir, Digest::SHA1.hexdigest queue_name)
         msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
         offsets.each_with_index do |offset, i|
-          msg_store.update_consumer_offset(tag_prefix + i.to_s, offset)
+          msg_store.store_consumer_offset(tag_prefix + i.to_s, offset)
         end
         msg_store.close
         sleep 0.1
@@ -278,7 +278,7 @@ describe LavinMQ::StreamQueue do
       end
     end
 
-    it "only saves one entry per consumer tag" do
+    it "appends consumer tag file" do
       queue_name = Random::Secure.hex
       offsets = [84_i64, 24_i64, 1_i64, 100_i64, 42_i64]
       consumer_tag = "ctag-1"
@@ -288,15 +288,53 @@ describe LavinMQ::StreamQueue do
         data_dir = File.join(s.vhosts["/"].data_dir, Digest::SHA1.hexdigest queue_name)
         msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
         offsets.each do |offset|
-          msg_store.update_consumer_offset(consumer_tag, offset)
+          msg_store.store_consumer_offset(consumer_tag, offset)
+        end
+        bytesize = consumer_tag.bytesize + 1 + 8
+        msg_store.@consumer_offsets.size.should eq bytesize*5
+        msg_store.last_offset_by_consumer_tag(consumer_tag).should eq offsets.last
+        msg_store.close
+      end
+    end
+
+    it "compacts consumer tag file on restart" do
+      queue_name = Random::Secure.hex
+      offsets = [84_i64, 24_i64, 1_i64, 100_i64, 42_i64]
+      consumer_tag = "ctag-1"
+      with_amqp_server do |s|
+        StreamQueueSpecHelpers.publish(s, queue_name, 1)
+
+        data_dir = File.join(s.vhosts["/"].data_dir, Digest::SHA1.hexdigest queue_name)
+        msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
+        offsets.each do |offset|
+          msg_store.store_consumer_offset(consumer_tag, offset)
         end
         msg_store.close
-        sleep 0.1
 
         msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
         msg_store.last_offset_by_consumer_tag(consumer_tag).should eq offsets.last
-        msg_store.@consumer_offsets.size.should eq 15
+        bytesize = consumer_tag.bytesize + 1 + 8
+        msg_store.@consumer_offsets.size.should eq bytesize
+        msg_store.close
+      end
+    end
 
+    it "compacts consumer tag file when full" do
+      queue_name = Random::Secure.hex
+      offsets = [84_i64, 24_i64, 1_i64, 100_i64, 42_i64]
+      consumer_tag = Random::Secure.hex(32)
+      with_amqp_server do |s|
+        StreamQueueSpecHelpers.publish(s, queue_name, 1)
+        data_dir = File.join(s.vhosts["/"].data_dir, Digest::SHA1.hexdigest queue_name)
+        msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
+        bytesize = consumer_tag.bytesize + 1 + 8
+
+        offsets = (LavinMQ::Config.instance.segment_size / bytesize).to_i32 + 1
+        offsets.times do |i|
+          msg_store.store_consumer_offset(consumer_tag, i)
+        end
+        msg_store.last_offset_by_consumer_tag(consumer_tag).should eq offsets - 1
+        msg_store.@consumer_offsets.size.should eq bytesize*2
         msg_store.close
       end
     end
@@ -367,7 +405,7 @@ describe LavinMQ::StreamQueue do
         data_dir = File.join(s.vhosts["/"].data_dir, Digest::SHA1.hexdigest queue_name)
         msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
         offsets.each_with_index do |offset, i|
-          msg_store.update_consumer_offset(tag_prefix + i.to_s, offset)
+          msg_store.store_consumer_offset(tag_prefix + i.to_s, offset)
         end
         sleep 0.1
         msg_store.cleanup_consumer_offsets
@@ -454,7 +492,7 @@ describe LavinMQ::StreamQueue do
         msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
         2000.times do |i|
           next if i == 0
-          msg_store.update_consumer_offset("#{consumer_tag_prefix}#{i}", i)
+          msg_store.store_consumer_offset("#{consumer_tag_prefix}#{i}", i)
         end
         msg_store.close
 
diff --git a/src/lavinmq/client/channel/stream_consumer.cr b/src/lavinmq/client/channel/stream_consumer.cr
index 57680887d..2dfc4a5ed 100644
--- a/src/lavinmq/client/channel/stream_consumer.cr
+++ b/src/lavinmq/client/channel/stream_consumer.cr
@@ -88,7 +88,7 @@ module LavinMQ
         end
 
         def ack(sp)
-          stream_queue.update_consumer_offset(@tag, @offset) if @track_offset
+          stream_queue.store_consumer_offset(@tag, @offset) if @track_offset
           super
         end
 
diff --git a/src/lavinmq/queue/stream_queue.cr b/src/lavinmq/queue/stream_queue.cr
index 7a991ccc3..972441008 100644
--- a/src/lavinmq/queue/stream_queue.cr
+++ b/src/lavinmq/queue/stream_queue.cr
@@ -74,8 +74,8 @@ module LavinMQ
       end
     end
 
-    def update_consumer_offset(consumer_tag : String, offset : Int64) : Nil
-      stream_queue_msg_store.update_consumer_offset(consumer_tag, offset)
+    def store_consumer_offset(consumer_tag : String, offset : Int64) : Nil
+      stream_queue_msg_store.store_consumer_offset(consumer_tag, offset)
     end
 
     # yield the next message in the ready queue
diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index dac02fe2e..7e3214694 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -15,7 +15,7 @@ module LavinMQ
       def initialize(@queue_data_dir : String, @replicator : Clustering::Replicator?)
         super
         @last_offset = get_last_offset
-        @consumer_offsets = MFile.new(File.join(@queue_data_dir, "consumer_offsets"), 32 * 1024)
+        @consumer_offsets = MFile.new(File.join(@queue_data_dir, "consumer_offsets"), Config.instance.segment_size)
         @consumer_offset_positions = restore_consumer_offset_positions
         drop_overflow
       end
@@ -128,19 +128,13 @@ module LavinMQ
         positions
       end
 
-      def update_consumer_offset(consumer_tag : String, new_offset : Int64)
-        if pos = @consumer_offset_positions[consumer_tag]?
-          IO::ByteFormat::SystemEndian.encode(new_offset, @consumer_offsets.to_slice(pos, 8, false))
-        else
-          store_consumer_offset(consumer_tag, new_offset)
-        end
-      end
-
       def store_consumer_offset(consumer_tag : String, new_offset : Int64)
+        cleanup_consumer_offsets if consumer_offset_file_full?(consumer_tag)
         expand_consumer_offset_file if consumer_offset_file_full?(consumer_tag)
         @consumer_offsets.write_bytes AMQ::Protocol::ShortString.new(consumer_tag)
         @consumer_offset_positions[consumer_tag] = @consumer_offsets.size
         @consumer_offsets.write_bytes new_offset
+        # replicate
       end
 
       def consumer_offset_file_full?(consumer_tag)
@@ -149,7 +143,7 @@ module LavinMQ
 
       def expand_consumer_offset_file
         pos = @consumer_offsets.size
-        @consumer_offsets = MFile.new(@consumer_offsets.path, @consumer_offsets.capacity + 32 * 1024)
+        @consumer_offsets = MFile.new(@consumer_offsets.path, @consumer_offsets.capacity + Config.instance.segment_size)
         @consumer_offsets.resize(pos)
       end
 
@@ -174,7 +168,7 @@ module LavinMQ
 
       def replace_offsets_file(&)
         old_consumer_offsets = @consumer_offsets
-        @consumer_offsets = MFile.new("#{old_consumer_offsets.path}.tmp", 32 * 1024)
+        @consumer_offsets = MFile.new("#{old_consumer_offsets.path}.tmp", Config.instance.segment_size)
         yield # fill the new file with correct data in this block
         @consumer_offsets.rename(old_consumer_offsets.path)
         old_consumer_offsets.close(truncate_to_size: false)

From 834bb330b2fed0bfc87099b44b223442ce4c3d9f Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Thu, 27 Jun 2024 14:41:47 +0200
Subject: [PATCH 65/66] set capacity of consumer offsets file to 1000*current
 size when compacting

---
 spec/stream_queue_spec.cr                     | 27 +++++++++----------
 .../queue/stream_queue_message_store.cr       | 10 +++----
 2 files changed, 17 insertions(+), 20 deletions(-)

diff --git a/spec/stream_queue_spec.cr b/spec/stream_queue_spec.cr
index 864a68c59..5c627f36a 100644
--- a/spec/stream_queue_spec.cr
+++ b/spec/stream_queue_spec.cr
@@ -484,27 +484,24 @@ describe LavinMQ::StreamQueue do
 
     it "expands consumer offset file when needed" do
       queue_name = Random::Secure.hex
-      consumer_tag_prefix = "ctag-"
+      consumer_tag_prefix = Random::Secure.hex(32)
       with_amqp_server do |s|
         StreamQueueSpecHelpers.publish(s, queue_name, 1)
-
         data_dir = File.join(s.vhosts["/"].data_dir, Digest::SHA1.hexdigest queue_name)
         msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
-        2000.times do |i|
-          next if i == 0
-          msg_store.store_consumer_offset("#{consumer_tag_prefix}#{i}", i)
+        one_offset_bytesize = "#{consumer_tag_prefix}#{1000}".bytesize + 1 + 8
+        offsets = (LavinMQ::Config.instance.segment_size / one_offset_bytesize).to_i32 + 1
+        bytesize = 0
+        offsets.times do |i|
+          consumer_tag = "#{consumer_tag_prefix}#{i + 1000}"
+          msg_store.store_consumer_offset(consumer_tag, i + 1000)
+          bytesize += consumer_tag.bytesize + 1 + 8
         end
-        msg_store.close
-
-        msg_store = LavinMQ::StreamQueue::StreamQueueMessageStore.new(data_dir, nil)
-        msg_store.@consumer_offsets.size.should eq 34_875
-
-        2000.times do |i|
-          next if i == 0
-          msg_store.last_offset_by_consumer_tag("#{consumer_tag_prefix}#{i}").should eq i
+        msg_store.@consumer_offsets.size.should eq bytesize
+        msg_store.@consumer_offsets.size.should be > LavinMQ::Config.instance.segment_size
+        offsets.times do |i|
+          msg_store.last_offset_by_consumer_tag("#{consumer_tag_prefix}#{i + 1000}").should eq i + 1000
         end
-
-        msg_store.close
       end
     end
   end
diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index 7e3214694..a9e15fe30 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -130,7 +130,6 @@ module LavinMQ
 
       def store_consumer_offset(consumer_tag : String, new_offset : Int64)
         cleanup_consumer_offsets if consumer_offset_file_full?(consumer_tag)
-        expand_consumer_offset_file if consumer_offset_file_full?(consumer_tag)
         @consumer_offsets.write_bytes AMQ::Protocol::ShortString.new(consumer_tag)
         @consumer_offset_positions[consumer_tag] = @consumer_offsets.size
         @consumer_offsets.write_bytes new_offset
@@ -152,23 +151,24 @@ module LavinMQ
 
         offsets_to_save = Hash(String, Int64).new
         lowest_offset_in_stream, _seg, _pos = offset_at(@segments.first_key, 4u32)
+        capacity = 0
         @consumer_offset_positions.each do |ctag, _pos|
           if offset = last_offset_by_consumer_tag(ctag)
             offsets_to_save[ctag] = offset if offset >= lowest_offset_in_stream
+            capacity += ctag.bytesize + 1 + 8
           end
         end
-
         @consumer_offset_positions = Hash(String, Int64).new
-        replace_offsets_file do
+        replace_offsets_file(capacity * 1000) do
           offsets_to_save.each do |ctag, offset|
             store_consumer_offset(ctag, offset)
           end
         end
       end
 
-      def replace_offsets_file(&)
+      def replace_offsets_file(capacity : Int, &)
         old_consumer_offsets = @consumer_offsets
-        @consumer_offsets = MFile.new("#{old_consumer_offsets.path}.tmp", Config.instance.segment_size)
+        @consumer_offsets = MFile.new("#{old_consumer_offsets.path}.tmp", capacity)
         yield # fill the new file with correct data in this block
         @consumer_offsets.rename(old_consumer_offsets.path)
         old_consumer_offsets.close(truncate_to_size: false)

From 221f2483eb39f879db0b25220400be68960dc043 Mon Sep 17 00:00:00 2001
From: Viktor Erlingsson <viktor@84codes.com>
Date: Thu, 27 Jun 2024 15:48:11 +0200
Subject: [PATCH 66/66] replicate consumer offsets file. remove unused code

---
 src/lavinmq/queue/stream_queue_message_store.cr | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/lavinmq/queue/stream_queue_message_store.cr b/src/lavinmq/queue/stream_queue_message_store.cr
index a9e15fe30..2f75f5c51 100644
--- a/src/lavinmq/queue/stream_queue_message_store.cr
+++ b/src/lavinmq/queue/stream_queue_message_store.cr
@@ -16,6 +16,7 @@ module LavinMQ
         super
         @last_offset = get_last_offset
         @consumer_offsets = MFile.new(File.join(@queue_data_dir, "consumer_offsets"), Config.instance.segment_size)
+        @replicator.try &.register_file @consumer_offsets
         @consumer_offset_positions = restore_consumer_offset_positions
         drop_overflow
       end
@@ -133,19 +134,13 @@ module LavinMQ
         @consumer_offsets.write_bytes AMQ::Protocol::ShortString.new(consumer_tag)
         @consumer_offset_positions[consumer_tag] = @consumer_offsets.size
         @consumer_offsets.write_bytes new_offset
-        # replicate
+        @replicator.try &.append(@consumer_offsets.path, (@consumer_offsets.size - consumer_tag.bytesize - 1 - 8).to_i32)
       end
 
       def consumer_offset_file_full?(consumer_tag)
         (@consumer_offsets.size + 1 + consumer_tag.bytesize + 8) >= @consumer_offsets.capacity
       end
 
-      def expand_consumer_offset_file
-        pos = @consumer_offsets.size
-        @consumer_offsets = MFile.new(@consumer_offsets.path, @consumer_offsets.capacity + Config.instance.segment_size)
-        @consumer_offsets.resize(pos)
-      end
-
       def cleanup_consumer_offsets
         return if @consumer_offsets.size.zero?
 
@@ -172,6 +167,7 @@ module LavinMQ
         yield # fill the new file with correct data in this block
         @consumer_offsets.rename(old_consumer_offsets.path)
         old_consumer_offsets.close(truncate_to_size: false)
+        @replicator.try &.replace_file @consumer_offsets.path
       end
 
       def shift?(consumer : Client::Channel::StreamConsumer) : Envelope?