Add support for reading SDI from InnoDB

bin/innodb_space

@@ -679,6 +679,16 @@ def space_inodes_detail(space)
   end
 end
 
+def space_sdi_construct_json(space)
+  space.sdi.each_object.map do |object|
+    { type: object.dd_object_type, id: object.id, object: object.data }
+  end
+end
+
+def space_sdi_json_dump(space)
+  puts JSON.pretty_generate(space_sdi_construct_json(space))
+end
+
 def page_account(innodb_system, space, page_number)
   puts "Accounting for page #{page_number}:"
 
@@ -1403,6 +1413,10 @@ The following modes are supported:
   space-inodes-detail
     Iterate through all inodes, printing a detailed report of each FSEG.
 
+  space-sdi-json-dump
+    Dump the contents of any SDI (serialized dictionary information) from a
+    space, in JSON format, similar to ibd2sdi.
+
   index-recurse
     Recurse an index, starting at the root (which must be provided in the first
     --page/-p argument), printing the node pages, node pointers (links), leaf
@@ -1767,6 +1781,8 @@ when "space-inodes-summary"
   space_inodes_summary(space)
 when "space-inodes-detail"
   space_inodes_detail(space)
+when "space-sdi-json-dump"
+  space_sdi_json_dump(space)
 when "index-recurse"
   index_recurse(index)
 when "index-record-offsets"

lib/innodb.rb

@@ -22,15 +22,25 @@ def self.debug=(value)
 require "innodb/version"
 require "innodb/stats"
 require "innodb/checksum"
+require "innodb/mysql_type"
 require "innodb/record_describer"
 require "innodb/data_dictionary"
+require "innodb/sdi"
+require "innodb/sdi/sdi_object"
+require "innodb/sdi/table"
+require "innodb/sdi/table_column"
+require "innodb/sdi/table_index"
+require "innodb/sdi/table_index_element"
+require "innodb/sdi/tablespace"
 require "innodb/page"
 require "innodb/page/blob"
 require "innodb/page/fsp_hdr_xdes"
 require "innodb/page/ibuf_bitmap"
 require "innodb/page/inode"
 require "innodb/page/index"
 require "innodb/page/trx_sys"
+require "innodb/page/sdi"
+require "innodb/page/sdi_blob"
 require "innodb/page/sys"
 require "innodb/page/undo_log"
 require "innodb/record"

lib/innodb/data_dictionary.rb

@@ -4,12 +4,6 @@
 # tables, columns, and indexes.
 module Innodb
   class DataDictionary
-    MysqlType = Struct.new(
-      :value,
-      :type,
-      keyword_init: true
-    )
-
     # rubocop:disable Layout/ExtraSpacing
 
     # A record describer for SYS_TABLES clustered records.
@@ -78,45 +72,6 @@ class SysFieldsPrimary < Innodb::RecordDescriber
       SYS_FIELDS: { PRIMARY: SysFieldsPrimary }.freeze,
     }.freeze
 
-    # A hash of MySQL's internal type system to the stored
-    # values for those types, and the 'external' SQL type.
-    # rubocop:disable Layout/HashAlignment
-    # rubocop:disable Layout/CommentIndentation
-    MYSQL_TYPE = {
-    # DECIMAL:      MysqlType.new(value: 0,   type: :DECIMAL),
-      TINY:         MysqlType.new(value: 1,   type: :TINYINT),
-      SHORT:        MysqlType.new(value: 2,   type: :SMALLINT),
-      LONG:         MysqlType.new(value: 3,   type: :INT),
-      FLOAT:        MysqlType.new(value: 4,   type: :FLOAT),
-      DOUBLE:       MysqlType.new(value: 5,   type: :DOUBLE),
-    # NULL:         MysqlType.new(value: 6,   type: nil),
-      TIMESTAMP:    MysqlType.new(value: 7,   type: :TIMESTAMP),
-      LONGLONG:     MysqlType.new(value: 8,   type: :BIGINT),
-      INT24:        MysqlType.new(value: 9,   type: :MEDIUMINT),
-    # DATE:         MysqlType.new(value: 10,  type: :DATE),
-      TIME:         MysqlType.new(value: 11,  type: :TIME),
-      DATETIME:     MysqlType.new(value: 12,  type: :DATETIME),
-      YEAR:         MysqlType.new(value: 13,  type: :YEAR),
-      NEWDATE:      MysqlType.new(value: 14,  type: :DATE),
-      VARCHAR:      MysqlType.new(value: 15,  type: :VARCHAR),
-      BIT:          MysqlType.new(value: 16,  type: :BIT),
-      NEWDECIMAL:   MysqlType.new(value: 246, type: :CHAR),
-    # ENUM:         MysqlType.new(value: 247, type: :ENUM),
-    # SET:          MysqlType.new(value: 248, type: :SET),
-      TINY_BLOB:    MysqlType.new(value: 249, type: :TINYBLOB),
-      MEDIUM_BLOB:  MysqlType.new(value: 250, type: :MEDIUMBLOB),
-      LONG_BLOB:    MysqlType.new(value: 251, type: :LONGBLOB),
-      BLOB:         MysqlType.new(value: 252, type: :BLOB),
-    # VAR_STRING:   MysqlType.new(value: 253, type: :VARCHAR),
-      STRING:       MysqlType.new(value: 254, type: :CHAR),
-      GEOMETRY:     MysqlType.new(value: 255, type: :GEOMETRY),
-    }.freeze
-    # rubocop:enable Layout/CommentIndentation
-    # rubocop:enable Layout/HashAlignment
-
-    # A hash of MYSQL_TYPE keys by value :value key.
-    MYSQL_TYPE_BY_VALUE = MYSQL_TYPE.transform_values(&:value).invert.freeze
-
     # A hash of InnoDB's internal type system to the values
     # stored for each type.
     COLUMN_MTYPE = {
@@ -173,8 +128,8 @@ class SysFieldsPrimary < Innodb::RecordDescriber
     # the MySQL-to-InnoDB interface regarding types.
     def self.mtype_prtype_to_type_string(mtype, prtype, len, prec)
       mysql_type = prtype & COLUMN_PRTYPE_MYSQL_TYPE_MASK
-      internal_type = MYSQL_TYPE_BY_VALUE[mysql_type]
-      external_type = MYSQL_TYPE[internal_type].type
+      internal_type = Innodb::MysqlType.by_mysql_field_type(mysql_type)
+      external_type = internal_type.handle_as
 
       case external_type
       when :VARCHAR

lib/innodb/data_type.rb

@@ -344,6 +344,20 @@ def value(data)
       end
     end
 
+    class EnumType
+      attr_reader :name
+
+      def initialize(base_type, modifiers, properties)
+        @width = 1
+        @name = Innodb::DataType.make_name(base_type, modifiers, properties)
+      end
+
+      def value(data)
+        nbits = @width * 8
+        BinData.const_get("Int%dbe" % nbits).read(data) ^ (-1 << (nbits - 1))
+      end
+    end
+
     #
     # Data types for InnoDB system columns.
     #
@@ -433,6 +447,7 @@ def value(data)
       TIMESTAMP: TimestampType,
       TRX_ID: TransactionIdType,
       ROLL_PTR: RollPointerType,
+      ENUM: EnumType,
     }.freeze
 
     def self.make_name(base_type, modifiers, properties)

lib/innodb/field.rb

@@ -115,8 +115,8 @@ def read_extern(cursor)
 
     # Parse a data type definition and extract the base type and any modifiers.
     def parse_type_definition(type_string)
-      matches = /^([a-zA-Z0-9_]+)(\(([0-9, ]+)\))?$/.match(type_string)
-      return unless matches
+      matches = /^([a-zA-Z0-9_]+)(\((.+)\))?(\s+unsigned)?$/.match(type_string)
+      raise "Unparseable type #{type_string}" unless matches
 
       base_type = matches[1].upcase.to_sym
       return [base_type, []] unless matches[3]

lib/innodb/mysql_type.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Innodb
+  class MysqlType
+    attr_reader :mysql_field_type_value
+    attr_reader :sdi_column_type_value
+    attr_reader :type
+    attr_reader :handle_as
+
+    def initialize(type:, mysql_field_type_value:, sdi_column_type_value:, handle_as: nil)
+      @mysql_field_type_value = mysql_field_type_value
+      @sdi_column_type_value = sdi_column_type_value
+      @type = type
+      @handle_as = handle_as || type
+    end
+
+    # A hash of MySQL's internal type system to the stored
+    # values for those types, and the 'external' SQL type.
+    TYPES = [
+      new(type: :DECIMAL, mysql_field_type_value: 0, sdi_column_type_value: 1),
+      new(type: :TINYINT, mysql_field_type_value: 1, sdi_column_type_value: 2),
+      new(type: :SMALLINT, mysql_field_type_value: 2, sdi_column_type_value: 3),
+      new(type: :INT, mysql_field_type_value: 3, sdi_column_type_value: 4),
+      new(type: :FLOAT, mysql_field_type_value: 4, sdi_column_type_value: 5),
+      new(type: :DOUBLE, mysql_field_type_value: 5, sdi_column_type_value: 6),
+      new(type: :TYPE_NULL, mysql_field_type_value: 6, sdi_column_type_value: 7),
+      new(type: :TIMESTAMP, mysql_field_type_value: 7, sdi_column_type_value: 8),
+      new(type: :BIGINT, mysql_field_type_value: 8, sdi_column_type_value: 9),
+      new(type: :MEDIUMINT, mysql_field_type_value: 9, sdi_column_type_value: 10),
+      new(type: :DATE, mysql_field_type_value: 10, sdi_column_type_value: 11),
+      new(type: :TIME, mysql_field_type_value: 11, sdi_column_type_value: 12),
+      new(type: :DATETIME, mysql_field_type_value: 12, sdi_column_type_value: 13),
+      new(type: :YEAR, mysql_field_type_value: 13, sdi_column_type_value: 14),
+      new(type: :DATE, mysql_field_type_value: 14, sdi_column_type_value: 15),
+      new(type: :VARCHAR, mysql_field_type_value: 15, sdi_column_type_value: 16),
+      new(type: :BIT, mysql_field_type_value: 16, sdi_column_type_value: 17),
+      new(type: :TIMESTAMP2, mysql_field_type_value: 17, sdi_column_type_value: 18),
+      new(type: :DATETIME2, mysql_field_type_value: 18, sdi_column_type_value: 19),
+      new(type: :TIME2, mysql_field_type_value: 19, sdi_column_type_value: 20),
+      new(type: :NEWDECIMAL, mysql_field_type_value: 246, sdi_column_type_value: 21, handle_as: :CHAR),
+      new(type: :ENUM, mysql_field_type_value: 247, sdi_column_type_value: 22),
+      new(type: :SET, mysql_field_type_value: 248, sdi_column_type_value: 23),
+      new(type: :TINYBLOB, mysql_field_type_value: 249, sdi_column_type_value: 24),
+      new(type: :MEDIUMBLOB, mysql_field_type_value: 250, sdi_column_type_value: 25),
+      new(type: :LONGBLOB, mysql_field_type_value: 251, sdi_column_type_value: 26),
+      new(type: :BLOB, mysql_field_type_value: 252, sdi_column_type_value: 27),
+      new(type: :VARCHAR, mysql_field_type_value: 253, sdi_column_type_value: 28),
+      new(type: :CHAR, mysql_field_type_value: 254, sdi_column_type_value: 29),
+      new(type: :GEOMETRY, mysql_field_type_value: 255, sdi_column_type_value: 30),
+      new(type: :JSON, mysql_field_type_value: 245, sdi_column_type_value: 31),
+    ].freeze
+
+    # A hash of types by mysql_field_type_value.
+    TYPES_BY_MYSQL_FIELD_TYPE_VALUE = Innodb::MysqlType::TYPES.to_h { |t| [t.mysql_field_type_value, t] }.freeze
+
+    # A hash of types by sdi_column_type_value.
+    TYPES_BY_SDI_COLUMN_TYPE_VALUE = Innodb::MysqlType::TYPES.to_h { |t| [t.sdi_column_type_value, t] }.freeze
+
+    def self.by_mysql_field_type(value)
+      TYPES_BY_MYSQL_FIELD_TYPE_VALUE[value]
+    end
+
+    def self.by_sdi_column_type(value)
+      TYPES_BY_SDI_COLUMN_TYPE_VALUE[value]
+    end
+  end
+end

lib/innodb/page/sdi.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Innodb
+  class Page
+    # SDI (Serialized Dictionary Information) pages are actually INDEX pages and store data dictionary
+    # information in an InnoDB index structure in the typical way. However they use a fixed definition
+    # for the (unnamed except for in-memory as "SDI_<space_id>") index, since there would, logically,
+    # be nowhere else to store the definition of this index.
+    class Sdi < Index
+      specialization_for :SDI
+
+      # Every SDI index has the same structure, equivalent to the following SQL:
+      #
+      #   CREATE TABLE `SDI_<space_id>` (
+      #     `type` INT UNSIGNED NOT NULL,
+      #     `id` BIGINT UNSIGNED NOT NULL,
+      #     `uncompressed_len` INT UNSIGNED NOT NULL,
+      #     `compressed_len` INT UNSIGNED NOT NULL,
+      #     `data` LONGBLOB NOT NULL,
+      #     PRIMARY KEY (`type`, `id`)
+      #   )
+      #
+      class RecordDescriber < Innodb::RecordDescriber
+        type :clustered
+        key "type",             :INT,    :UNSIGNED,  :NOT_NULL
+        key "id",               :BIGINT, :UNSIGNED,  :NOT_NULL
+        row "uncompressed_len", :INT,    :UNSIGNED,  :NOT_NULL
+        row "compressed_len",   :INT,    :UNSIGNED,  :NOT_NULL
+        row "data",             :BLOB,   :NOT_NULL
+      end
+
+      def make_record_describer
+        RecordDescriber.new
+      end
+    end
+  end
+end

lib/innodb/page/sdi_blob.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Innodb
+  class Page
+    # SDI (Serialized Dictionary Information) BLOB pages are actually BLOB pages with a different page
+    # type number but otherwise the same structure.
+    class SdiBlob < Blob
+      specialization_for :SDI_BLOB
+    end
+  end
+end

lib/innodb/sdi.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Innodb
+  class Sdi
+    # A hash of page types to specialized classes to handle them. Normally
+    # subclasses will register themselves in this list.
+    @specialized_classes = {}
+
+    class << self
+      attr_reader :specialized_classes
+
+      def register_specialization(id, specialized_class)
+        @specialized_classes[id] = specialized_class
+      end
+    end
+
+    attr_reader :space
+
+    def initialize(space)
+      @space = space
+    end
+
+    def sdi_header
+      @sdi_header ||= space.page(0).sdi_header
+    end
+
+    def version
+      sdi_header[:version]
+    end
+
+    def root_page_number
+      sdi_header[:root_page_number]
+    end
+
+    def valid?
+      root_page_number != 0
+    end
+
+    def index
+      return unless valid?
+
+      space.index(root_page_number)
+    end
+
+    def each_object
+      return unless valid?
+      return enum_for(:each_object) unless block_given?
+
+      index.each_record do |record|
+        yield SdiObject.from_record(record)
+      end
+
+      nil
+    end
+
+    def each_table
+      return enum_for(:each_table) unless block_given?
+
+      each_object { |o| yield o if o.is_a?(Table) }
+
+      nil
+    end
+
+    def each_tablespace
+      return enum_for(:each_tablespace) unless block_given?
+
+      each_object { |o| yield o if o.is_a?(Tablespace) }
+
+      nil
+    end
+  end
+end