Text editor: Add support for using free space when string database ru…

…ns out of room

dsvedit/text_editor_dialog.rb

@@ -105,7 +105,10 @@ def save_current_text
     reload_all_text_lists()
   rescue Text::TextEncodeError, Encoding::UndefinedConversionError => e
     Qt::MessageBox.warning(self, "Error encoding text", e.message)
-  rescue TextDatabase::StringDatabaseTooLargeError => e
-    Qt::MessageBox.warning(self, "Not enough free space", "Not enough room to fit all expanded strings")
+  rescue FreeSpaceManager::FreeSpaceFindError => e
+    Qt::MessageBox.warning(self,
+      "Failed to find free space",
+      "Failed to find free space to put the text.\n\n#{NO_FREE_SPACE_MESSAGE}"
+    )
   end
 end

dsvlib/free_space_manager.rb

@@ -103,14 +103,24 @@ def write_free_space_to_text_file(base_directory=@filesystem_directory)
   def free_unused_space(ram_address, length)
     return if length <= 0
 
+    actual_length = length
+    path, offset = convert_ram_address_to_path_and_offset(ram_address)
+    file = files_by_path[path]
+    if offset + length > file[:size]
+      diff = (offset + length) - file[:size]
+      if diff < 4
+        # If the free space goes 1-3 bytes past the end of the file, it's probably just something caused by rounding the free spaces up to the nearest 4 bytes, so simply decrease the length to avoid the error.
+        actual_length -= diff
+      end
+    end
+
     if !@changes_in_current_free_space_batch.nil?
       data = read(ram_address, length)
       @changes_in_current_free_space_batch << [:free_unused_space, [ram_address, data]]
     end
 
-    path, offset = convert_ram_address_to_path_and_offset(ram_address)
     @free_spaces << {path: path, offset: offset, length: length}
-    write_by_file(path, offset, "\0"*length, freeing_space: true)
+    write_by_file(path, offset, "\0"*actual_length, freeing_space: true)
     merge_overlapping_free_spaces()
 
     round_free_spaces()
@@ -208,6 +218,11 @@ def round_free_spaces
     end
   end
 
+  def automatically_remove_nonzero_free_spaces_for_overlay(overlay_id = nil)
+    files_to_check = get_files_to_check_for_overlay(overlay_id)
+    automatically_remove_nonzero_free_spaces(files_to_check)
+  end
+
   def automatically_remove_nonzero_free_spaces(files_to_check)
     # This is an extra safeguard to make absolutely sure no nonzero data is treated as free space.
     # (We also count FF as zero data, since HoD's free space is all FFs by default.)
@@ -253,11 +268,7 @@ def automatically_remove_nonzero_free_spaces(files_to_check)
     end
   end
 
-  def get_free_space(length_needed, overlay_id = nil)
-    if length_needed <= 0
-      raise FreeSpaceFindError.new("Invalid free space length to find: #{length_needed}")
-    end
-
+  def get_files_to_check_for_overlay(overlay_id = nil)
     files_to_check = []
 
     if SYSTEM == :nds
@@ -273,7 +284,19 @@ def get_free_space(length_needed, overlay_id = nil)
       files_to_check << "/rom.gba"
     end
 
-    automatically_remove_nonzero_free_spaces(files_to_check)
+    return files_to_check
+  end
+
+  def get_free_space(length_needed, overlay_id = nil, remove_nonzero_spaces = true)
+    if length_needed <= 0
+      raise FreeSpaceFindError.new("Invalid free space length to find: #{length_needed}")
+    end
+
+    files_to_check = get_files_to_check_for_overlay(overlay_id)
+
+    if remove_nonzero_spaces
+      automatically_remove_nonzero_free_spaces(files_to_check)
+    end
 
     free_spaces_sorted = @free_spaces.sort_by{|free_space| free_space[:length]}
 

dsvlib/nds_file_system.rb

@@ -472,11 +472,9 @@ def has_uncommitted_changes?
     !@uncommitted_files.empty?
   end
 
-  def expand_file_and_get_end_of_file_ram_address(ram_address, length_to_expand_by)
-    file_path, offset_in_file = convert_ram_address_to_path_and_offset(ram_address)
-    file = @currently_loaded_files.values.find{|file| file[:file_path] == file_path}
-
-    return expand_file_and_get_end(file, length_to_expand_by)
+  def expand_overlay_and_get_end(overlay_id, length_to_expand_by)
+    overlay = @overlays[overlay_id]
+    return expand_file_and_get_end(overlay, length_to_expand_by)
   end
 
   def expand_file_and_get_end(file, length_to_expand_by)

dsvlib/text.rb

@@ -7,7 +7,8 @@ class TextEncodeError < StandardError ; end
 
   attr_reader :text_id,
               :text_ram_pointer,
-              :fs
+              :fs,
+              :original_encoded_string_length
   attr_accessor :string_ram_pointer,
                 :decoded_string,
                 :encoded_string
@@ -37,6 +38,7 @@ def read_from_rom
       @encoded_string = fs.read_until_end_marker(string_ram_pointer+2, [0xEA]) # Skip the first 2 bytes which are always 01 00.
     end
 
+    @original_encoded_string_length = @encoded_string.length
     @decoded_string = decode_string(@encoded_string)
   end
 
@@ -52,6 +54,7 @@ def write_to_rom
       data = [1].pack("v") + encoded_string + [0xEA].pack("C")
     end
     fs.write(string_ram_pointer, data)
+    @original_encoded_string_length = @encoded_string.length
   end
 
   def overlay_id

dsvlib/text_database.rb

@@ -1,7 +1,5 @@
 
 class TextDatabase
-  class StringDatabaseTooLargeError < StandardError ; end
-
   attr_reader :fs,
               :text_list
 
@@ -26,39 +24,65 @@ def write_to_rom
       header_footer_length = 3
     end
 
+    # Clear all text that is currently using free space beforehand.
+    text_list.each do |text|
+      if text.string_ram_pointer < STRING_DATABASE_START_OFFSET || text.string_ram_pointer >= STRING_DATABASE_ALLOWABLE_END_OFFSET
+        # This text isn't currently in the original string database. It's in free space.
+        # Therefore, we tell the free space manager to clear it, so this space can be used again.
+        string_length = text.original_encoded_string_length + header_footer_length
+        if string_length % 4 != 0
+          # Because the free space manager pads the free spaces it gives up to 4 bytes, we have to clear up to the padded length to fully clear each of these.
+          string_length = ((string_length / 4) * 4) + 4
+        end
+        fs.free_unused_space(text.string_ram_pointer, string_length)
+      end
+    end
+
     overlays = TEXT_REGIONS_OVERLAYS.values.uniq
     overlays.each do |overlay|
       fs.load_overlay(overlay) if overlay
 
+      # Remove nonzero free spaces just once for each overlay, instead of once for each string.
+      fs.automatically_remove_nonzero_free_spaces_for_overlay(overlay)
+
       text_list_for_overlay = text_list.select{|text| text.overlay_id == overlay}
 
       next_string_ram_pointer = STRING_DATABASE_START_OFFSET
       writing_to_end_of_file = false
+      using_free_space_manager = false
       text_list_for_overlay.each do |text|
         if next_string_ram_pointer + text.encoded_string.length + header_footer_length >= STRING_DATABASE_ALLOWABLE_END_OFFSET
-          # Writing strings past this point would result in something being overwritten, so raise an error.
-          raise StringDatabaseTooLargeError.new
+          # Writing strings past this point would result in something being overwritten, so start using the free space manager instead.
+          using_free_space_manager = true
         end
 
         region_name = TEXT_REGIONS.find{|name, range| range.include?(text.text_id)}[0]
 
-        if !writing_to_end_of_file && GAME == "ooe" && next_string_ram_pointer + text.encoded_string.length + header_footer_length >= STRING_DATABASE_ORIGINAL_END_OFFSET
-          # Reached the end of where strings were in the original game, but in OoE we can expand the file.
-          writing_to_end_of_file = true
-        end
-        if writing_to_end_of_file
-          next_string_ram_pointer = fs.expand_file_and_get_end_of_file_ram_address(text.string_ram_pointer, text.encoded_string.length + header_footer_length)
-        end
-
-        # System strings and AoS strings must be aligned to the nearest 4 bytes or they won't be displayed.
-        region_name = TEXT_REGIONS.find{|name, range| range.include?(text.text_id)}[0]
-        if GAME == "aos" || (region_name == "System" && next_string_ram_pointer % 4 != 0)
-          next_string_ram_pointer = ((next_string_ram_pointer / 4) * 4) + 4
+        if using_free_space_manager
+          next_string_ram_pointer = fs.get_free_space(text.encoded_string.length + header_footer_length, overlay, remove_nonzero_spaces = false)
+
+          # Write null bytes to where the string will take up so the free space manager doesn't consider this space free.
+          string_length = text.encoded_string.length + header_footer_length
+          fs.write(next_string_ram_pointer, "\0"*string_length)
+        else
+          if !writing_to_end_of_file && GAME == "ooe" && next_string_ram_pointer + text.encoded_string.length + header_footer_length >= STRING_DATABASE_ORIGINAL_END_OFFSET
+            # Reached the end of where strings were in the original game, but in OoE we can expand the file.
+            writing_to_end_of_file = true
+          end
+          if writing_to_end_of_file
+            next_string_ram_pointer = fs.expand_overlay_and_get_end(overlay, text.encoded_string.length + header_footer_length)
+          end
+
+          # System strings and AoS strings must be aligned to the nearest 4 bytes or they won't be displayed.
+          region_name = TEXT_REGIONS.find{|name, range| range.include?(text.text_id)}[0]
+          if GAME == "aos" || (region_name == "System" && next_string_ram_pointer % 4 != 0)
+            next_string_ram_pointer = ((next_string_ram_pointer / 4) * 4) + 4
+          end
         end
 
         text.string_ram_pointer = next_string_ram_pointer
 
-        if !writing_to_end_of_file
+        if !writing_to_end_of_file && !using_free_space_manager
           next_string_ram_pointer += text.encoded_string.length + header_footer_length
         end
       end