From f236a5e1c2810c9b08f763bc14a4bab29d34704e Mon Sep 17 00:00:00 2001 From: Jeremy Cole Date: Sat, 9 Nov 2024 22:47:59 -0800 Subject: [PATCH] Refactor DataDictionary; support SYS_* and SDI --- .rubocop.yml | 2 + bin/innodb_space | 171 ++--- lib/innodb.rb | 4 +- lib/innodb/data_dictionary.rb | 610 +----------------- lib/innodb/data_dictionary/column.rb | 17 + lib/innodb/data_dictionary/columns.rb | 14 + lib/innodb/data_dictionary/index.rb | 44 ++ .../data_dictionary/index_column_reference.rb | 23 + .../index_column_references.rb | 14 + lib/innodb/data_dictionary/indexes.rb | 14 + lib/innodb/data_dictionary/object_store.rb | 42 ++ lib/innodb/data_dictionary/table.rb | 28 + lib/innodb/data_dictionary/tables.rb | 14 + lib/innodb/data_dictionary/tablespace.rb | 15 + lib/innodb/data_dictionary/tablespaces.rb | 14 + lib/innodb/data_type.rb | 2 + lib/innodb/page/index.rb | 8 +- lib/innodb/page/sys_data_dictionary_header.rb | 18 +- lib/innodb/record_describer.rb | 17 - lib/innodb/sdi.rb | 28 +- lib/innodb/sdi/sdi_object.rb | 4 +- lib/innodb/sdi/table.rb | 8 - lib/innodb/sdi/table_column.rb | 8 +- lib/innodb/sdi/table_index.rb | 27 +- lib/innodb/sdi/table_index_element.rb | 7 + lib/innodb/sdi_data_dictionary.rb | 81 +++ lib/innodb/space.rb | 19 +- lib/innodb/sys_data_dictionary.rb | 310 +++++++++ lib/innodb/system.rb | 141 ++-- spec/data/{ => orphan}/ib_logfile0 | Bin spec/data/{ => orphan}/ib_logfile1 | Bin spec/data/{ => orphan}/ibdata1 | Bin spec/innodb/data_dictionary_spec.rb | 207 ------ spec/innodb/log_block_spec.rb | 2 +- spec/innodb/log_group_spec.rb | 4 +- spec/innodb/log_reader_spec.rb | 2 +- spec/innodb/log_record_spec.rb | 4 +- spec/innodb/log_spec.rb | 26 +- spec/innodb/page/inode_spec.rb | 4 +- spec/innodb/page/trx_sys_spec.rb | 2 +- spec/innodb/page_spec.rb | 10 +- spec/innodb/space_spec.rb | 8 +- spec/innodb/sys_data_dictionary_spec.rb | 27 + spec/innodb/system_spec.rb | 102 +-- spec/innodb/xdes_spec.rb | 2 +- 45 files changed, 905 insertions(+), 1199 deletions(-) create mode 100644 lib/innodb/data_dictionary/column.rb create mode 100644 lib/innodb/data_dictionary/columns.rb create mode 100644 lib/innodb/data_dictionary/index.rb create mode 100644 lib/innodb/data_dictionary/index_column_reference.rb create mode 100644 lib/innodb/data_dictionary/index_column_references.rb create mode 100644 lib/innodb/data_dictionary/indexes.rb create mode 100644 lib/innodb/data_dictionary/object_store.rb create mode 100644 lib/innodb/data_dictionary/table.rb create mode 100644 lib/innodb/data_dictionary/tables.rb create mode 100644 lib/innodb/data_dictionary/tablespace.rb create mode 100644 lib/innodb/data_dictionary/tablespaces.rb create mode 100644 lib/innodb/sdi_data_dictionary.rb create mode 100644 lib/innodb/sys_data_dictionary.rb rename spec/data/{ => orphan}/ib_logfile0 (100%) rename spec/data/{ => orphan}/ib_logfile1 (100%) rename spec/data/{ => orphan}/ibdata1 (100%) delete mode 100644 spec/innodb/data_dictionary_spec.rb create mode 100644 spec/innodb/sys_data_dictionary_spec.rb diff --git a/.rubocop.yml b/.rubocop.yml index 2278ce0a..f0fa685f 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -40,6 +40,8 @@ Metrics/AbcSize: Enabled: false Metrics/CyclomaticComplexity: Enabled: false +Metrics/ParameterLists: + Max: 10 Metrics/PerceivedComplexity: Enabled: false Style/SymbolArray: diff --git a/bin/innodb_space b/bin/innodb_space index 61f289ec..edcd107e 100755 --- a/bin/innodb_space +++ b/bin/innodb_space @@ -174,124 +174,70 @@ end # Print a summary of all spaces in the InnoDB system. def system_spaces(innodb_system) - puts "%-32s%-12s%-12s" % %w[ + puts "%-64s%-12s" % %w[ name pages - indexes ] - print_space_information = lambda do |name, space| - puts "%-32s%-12i%-12i" % [ - name, + innodb_system&.each_space do |space| + puts "%-64s%-12i" % [ + space.name.gsub("#{innodb_system.data_directory}/", ""), space.pages, - space.each_index.to_a.size, ] end - print_space_information.call("(system)", innodb_system.system_space) - - innodb_system.each_table_name do |table_name| - space = innodb_system.space_by_table_name(table_name) - next unless space - - print_space_information.call(table_name, space) - end - innodb_system.each_orphan do |table_name| - puts "%-43s (orphan/tmp)" % table_name + puts "%-64s%-12s" % [table_name, "(orphan/tmp)"] end end -# Print the contents of the SYS_TABLES data dictionary table. +# Print the contents of the table list from the data dictionary. def data_dictionary_tables(innodb_system) - puts "%-32s%-12s%-12s%-12s%-12s%-12s%-15s%-12s" % %w[ + puts "%-64s%-12s%-12s" % %w[ name - id - n_cols - type - mix_id - mix_len - cluster_name - space + columns + indexes ] - innodb_system.data_dictionary.each_table do |record| - puts "%-32s%-12i%-12i%-12i%-12i%-12i%-15s%-12i" % [ - record["NAME"], - record["ID"], - record["N_COLS"], - record["TYPE"], - record["MIX_ID"], - record["MIX_LEN"], - record["CLUSTER_NAME"], - record["SPACE"], + innodb_system.data_dictionary.tables.each do |table| + puts "%-64s%-12i%-12i" % [ + table.name, + table.columns.count, + table.indexes.count, ] end end -# Print the contents of the SYS_COLUMNS data dictionary table. +# Print the contents of the column list from the data dictionary. def data_dictionary_columns(innodb_system) - puts "%-12s%-6s%-32s%-12s%-12s%-6s%-6s" % %w[ - table_id - pos + puts "%-64s%-32s%-32s" % %w[ + table name - mtype - prtype - len - prec + description ] - innodb_system.data_dictionary.each_column do |record| - puts "%-12i%-6i%-32s%-12i%-12i%-6i%-6i" % [ - record["TABLE_ID"], - record["POS"], - record["NAME"], - record["MTYPE"], - record["PRTYPE"], - record["LEN"], - record["PREC"], + innodb_system.data_dictionary.columns.each do |column| + puts "%-64s%-32s%-32s" % [ + column.table.name, + column.name, + column.description, ] end end -# Print the contents of the SYS_INDEXES data dictionary table. +# Print the contents of the index list from the data dictionary. def data_dictionary_indexes(innodb_system) - puts "%-12s%-12s%-32s%-10s%-6s%-12s%-12s" % %w[ - table_id - id + puts "%-64s%-32s%-32s" % %w[ + table name - n_fields - type - space - page_no + columns ] - innodb_system.data_dictionary.each_index do |record| - puts "%-12i%-12i%-32s%-10i%-6i%-12i%-12i" % [ - record["TABLE_ID"], - record["ID"], - record["NAME"], - record["N_FIELDS"], - record["TYPE"], - record["SPACE"], - record["PAGE_NO"], - ] - end -end - -# Print the contents of the SYS_FIELDS data dictionary table. -def data_dictionary_fields(innodb_system) - puts "%-12s%-12s%-32s" % %w[ - index_id - pos - col_name - ] - - innodb_system.data_dictionary.each_field do |record| - puts "%-12i%-12i%-32s" % [ - record["INDEX_ID"], - record["POS"], - record["COL_NAME"], + innodb_system.data_dictionary.indexes.each do |index| + puts "%-64s%-32s%-32s" % [ + index.table.name, + index.name, + index.column_references.each.map(&:name), ] end end @@ -411,7 +357,7 @@ def space_indexes(innodb_system, space) index.each_fseg do |fseg_name, fseg| puts "%-12i%-32s%-12i%-12s%-12i%-12i%-12i%-12s" % [ index.id, - innodb_system ? innodb_system.index_name_by_id(index.id) : "", + innodb_system.data_dictionary.indexes.find(innodb_index_id: index.id)&.name, index.root.offset, fseg_name, fseg.fseg_id, @@ -440,11 +386,13 @@ def space_index_pages_free_plot(space, start_page) end end - image_name = space.name.sub(".ibd", "").gsub(/[^a-zA-Z0-9_]/, "_").sub(/\A_+/, "") + image_name = space.name + image_name = image_name.sub(%r{^#{space.innodb_system.data_directory}/}, "") if space.innodb_system + image_name = image_name.sub(".ibd", "").gsub(/[^a-zA-Z0-9_]/, "_").sub(/\A_+/, "") image_file = "#{image_name}_free.png" # Aim for one horizontal pixel per extent, but min 1k and max 10k width. - (space.pages / space.pages_per_extent).clamp(1_000, 10_000) + image_width = (space.pages / space.pages_per_extent).clamp(1_000, 10_000) Gnuplot.open do |gp| Gnuplot::Plot.new(gp) do |plot| @@ -493,8 +441,8 @@ def space_extents_illustrate_page_status(space, entry, count_by_identifier, iden unless identifiers[identifier] identifiers[identifier] = page.ibuf_index? ? "Insert Buffer Index" : "Index #{page.index_id}" if space.innodb_system - table, index = space.innodb_system.table_and_index_name_by_id(page.index_id) - identifiers[identifier] += " (%s.%s)" % [table, index] if table && index + dd_index = space.innodb_system.data_dictionary.indexes.find(innodb_index_id: page.index_id) + identifiers[identifier] += " (%s.%s)" % [dd_index.table.name, dd_index.name] if dd_index end end end @@ -686,6 +634,8 @@ def space_sdi_construct_json(space) end def space_sdi_json_dump(space) + raise "Space does not have SDI data; is it older than MySQL 8.0?" unless space.sdi.valid? + puts JSON.pretty_generate(space_sdi_construct_json(space)) end @@ -751,8 +701,8 @@ def page_account(innodb_system, space, page_number) puts " Fseg is in #{fseg_name} fseg of index #{index.id}." puts " Index root is page #{index.root.offset}." if innodb_system - table_name, index_name = innodb_system.table_and_index_name_by_id(index.id) - puts " Index is #{table_name}.#{index_name}." if table_name && index_name + dd_index = innodb_system.data_dictionary.indexes.find(innodb_index_id: index.id) + puts " Index is #{dd_index.table.name}.#{dd_index.name}." if dd_index end end end @@ -1233,7 +1183,7 @@ def undo_history_summary(innodb_system) history_list.each do |history| history.each_undo_record do |undo| - table_name = innodb_system.table_name_by_id(undo.table_id) + table_name = innodb_system.data_dictionary.tables(innodb_table_id: undo.table_id) puts "%-8s%-8s%-14s%-20s%s" % [ undo.page, undo.offset, @@ -1340,16 +1290,13 @@ The following modes are supported: Print a summary of all spaces in the system. data-dictionary-tables - Print all records in the SYS_TABLES data dictionary table. + Print the contents of the table list from the data dictionary. data-dictionary-columns - Print all records in the SYS_COLUMNS data dictionary table. + Print the contents of the column list from the data dictionary. data-dictionary-indexes - Print all records in the SYS_INDEXES data dictionary table. - - data-dictionary-fields - Print all records in the SYS_FIELDS data dictionary table. + Print the contents of the index list from the data dictionary. space-summary Summarize all pages within a tablespace. A starting page number can be @@ -1437,20 +1384,20 @@ The following modes are supported: index-fseg-internal-lists index-fseg-leaf-lists - Print a summary of all lists in an index file segment. Index root page must - be provided with --page/-p. + Print a summary of all lists in an index file segment. Index must be specified + by name with --index-name/-I or by root page number with --page/-p. index-fseg-internal-list-iterate index-fseg-leaf-list-iterate Iterate the file segment list (whose name is provided in the first --list/-L - argument) for internal or leaf pages for a given index (whose root page - is provided in the first --page/-p argument). The lists used for each + argument) for internal or leaf pages for a given index (specified by name with + --index-name/-I or by root page number with --page/-p). The lists used for each index are 'full', 'not_full', and 'free'. index-fseg-internal-frag-pages index-fseg-leaf-frag-pages - Print a summary of all fragment pages in an index file segment. Index root - page must be provided with --page/-p. + Print a summary of all fragment pages in an index file segment. Index must be + specified by name with --index-name/-I or by root page number with --page/-p. page-dump Dump the contents of a page, using the Ruby pp ('pretty-print') module. @@ -1631,14 +1578,8 @@ if @options.system_space_file end if innodb_system && @options.table_name - table_tablespace = innodb_system.space_by_table_name(@options.table_name) - if table_tablespace - space = table_tablespace - elsif @options.system_space_tables - space = innodb_system.system_space - else - raise "Tablespace file not found and --system-space-tables (-x) is not enabled" - end + space = innodb_system.space_by_table_name(@options.table_name) + raise "Couldn't load space for table #{@options.table_name}" unless space elsif @options.space_file space = Innodb::Space.new(@options.space_file) else @@ -1749,8 +1690,6 @@ when "data-dictionary-columns" data_dictionary_columns(innodb_system) when "data-dictionary-indexes" data_dictionary_indexes(innodb_system) -when "data-dictionary-fields" - data_dictionary_fields(innodb_system) when "space-summary" space_summary(space, @options.page || 0) when "space-index-pages-summary" diff --git a/lib/innodb.rb b/lib/innodb.rb index 22294548..99dbcc84 100644 --- a/lib/innodb.rb +++ b/lib/innodb.rb @@ -24,7 +24,7 @@ def self.debug=(value) require "innodb/checksum" require "innodb/mysql_type" require "innodb/record_describer" -require "innodb/data_dictionary" +require "innodb/sys_data_dictionary" require "innodb/sdi" require "innodb/sdi/sdi_object" require "innodb/sdi/table" @@ -32,6 +32,8 @@ def self.debug=(value) require "innodb/sdi/table_index" require "innodb/sdi/table_index_element" require "innodb/sdi/tablespace" +require "innodb/sdi_data_dictionary" +require "innodb/data_dictionary" require "innodb/page" require "innodb/page/blob" require "innodb/page/fsp_hdr_xdes" diff --git a/lib/innodb/data_dictionary.rb b/lib/innodb/data_dictionary.rb index 168c53a9..b401f0ae 100644 --- a/lib/innodb/data_dictionary.rb +++ b/lib/innodb/data_dictionary.rb @@ -1,593 +1,47 @@ # frozen_string_literal: true -# A class representing InnoDB's data dictionary, which contains metadata about -# tables, columns, and indexes. +require "innodb/data_dictionary/tablespaces" +require "innodb/data_dictionary/tables" + module Innodb class DataDictionary - # rubocop:disable Layout/ExtraSpacing - - # A record describer for SYS_TABLES clustered records. - class SysTablesPrimary < Innodb::RecordDescriber - type :clustered - key "NAME", "VARCHAR(100)", :NOT_NULL - row "ID", :BIGINT, :UNSIGNED, :NOT_NULL - row "N_COLS", :INT, :UNSIGNED, :NOT_NULL - row "TYPE", :INT, :UNSIGNED, :NOT_NULL - row "MIX_ID", :BIGINT, :UNSIGNED, :NOT_NULL - row "MIX_LEN", :INT, :UNSIGNED, :NOT_NULL - row "CLUSTER_NAME", "VARCHAR(100)", :NOT_NULL - row "SPACE", :INT, :UNSIGNED, :NOT_NULL - end - - # A record describer for SYS_TABLES secondary key on ID. - class SysTablesId < Innodb::RecordDescriber - type :secondary - key "ID", :BIGINT, :UNSIGNED, :NOT_NULL - row "NAME", "VARCHAR(100)", :NOT_NULL - end - - # A record describer for SYS_COLUMNS clustered records. - class SysColumnsPrimary < Innodb::RecordDescriber - type :clustered - key "TABLE_ID", :BIGINT, :UNSIGNED, :NOT_NULL - key "POS", :INT, :UNSIGNED, :NOT_NULL - row "NAME", "VARCHAR(100)", :NOT_NULL - row "MTYPE", :INT, :UNSIGNED, :NOT_NULL - row "PRTYPE", :INT, :UNSIGNED, :NOT_NULL - row "LEN", :INT, :UNSIGNED, :NOT_NULL - row "PREC", :INT, :UNSIGNED, :NOT_NULL - end - - # A record describer for SYS_INDEXES clustered records. - class SysIndexesPrimary < Innodb::RecordDescriber - type :clustered - key "TABLE_ID", :BIGINT, :UNSIGNED, :NOT_NULL - key "ID", :BIGINT, :UNSIGNED, :NOT_NULL - row "NAME", "VARCHAR(100)", :NOT_NULL - row "N_FIELDS", :INT, :UNSIGNED, :NOT_NULL - row "TYPE", :INT, :UNSIGNED, :NOT_NULL - row "SPACE", :INT, :UNSIGNED, :NOT_NULL - row "PAGE_NO", :INT, :UNSIGNED, :NOT_NULL - end - - # A record describer for SYS_FIELDS clustered records. - class SysFieldsPrimary < Innodb::RecordDescriber - type :clustered - key "INDEX_ID", :BIGINT, :UNSIGNED, :NOT_NULL - key "POS", :INT, :UNSIGNED, :NOT_NULL - row "COL_NAME", "VARCHAR(100)", :NOT_NULL - end - - # rubocop:enable Layout/ExtraSpacing - - # A hash of hashes of table name and index name to describer - # class. - DATA_DICTIONARY_RECORD_DESCRIBERS = { - SYS_TABLES: { - PRIMARY: SysTablesPrimary, - ID: SysTablesId, - }.freeze, - SYS_COLUMNS: { PRIMARY: SysColumnsPrimary }.freeze, - SYS_INDEXES: { PRIMARY: SysIndexesPrimary }.freeze, - SYS_FIELDS: { PRIMARY: SysFieldsPrimary }.freeze, - }.freeze - - # A hash of InnoDB's internal type system to the values - # stored for each type. - COLUMN_MTYPE = { - VARCHAR: 1, - CHAR: 2, - FIXBINARY: 3, - BINARY: 4, - BLOB: 5, - INT: 6, - SYS_CHILD: 7, - SYS: 8, - FLOAT: 9, - DOUBLE: 10, - DECIMAL: 11, - VARMYSQL: 12, - MYSQL: 13, - }.freeze - - # A hash of COLUMN_MTYPE keys by value. - COLUMN_MTYPE_BY_VALUE = COLUMN_MTYPE.invert.freeze - - # A hash of InnoDB 'precise type' bitwise flags. - COLUMN_PRTYPE_FLAG = { - NOT_NULL: 256, - UNSIGNED: 512, - BINARY: 1024, - LONG_TRUE_VARCHAR: 4096, - }.freeze - - # A hash of COLUMN_PRTYPE keys by value. - COLUMN_PRTYPE_FLAG_BY_VALUE = COLUMN_PRTYPE_FLAG.invert.freeze - - # The bitmask to extract the MySQL internal type - # from the InnoDB 'precise type'. - COLUMN_PRTYPE_MYSQL_TYPE_MASK = 0xFF - - # A hash of InnoDB's index type flags. - INDEX_TYPE_FLAG = { - CLUSTERED: 1, - UNIQUE: 2, - UNIVERSAL: 4, - IBUF: 8, - CORRUPT: 16, - FTS: 32, - }.freeze - - # A hash of INDEX_TYPE_FLAG keys by value. - INDEX_TYPE_FLAG_BY_VALUE = INDEX_TYPE_FLAG.invert.freeze - - # Return the 'external' SQL type string (such as 'VARCHAR' or - # 'INT') given the stored mtype and prtype from the InnoDB - # data dictionary. Note that not all types are extractable - # into fully defined SQL types due to the lossy nature of - # the MySQL-to-InnoDB interface regarding types. - def self.mtype_prtype_to_type_string(mtype, prtype, len, prec) - mysql_type = prtype & COLUMN_PRTYPE_MYSQL_TYPE_MASK - internal_type = Innodb::MysqlType.by_mysql_field_type(mysql_type) - external_type = internal_type.handle_as - - case external_type - when :VARCHAR - # One-argument: length. - "%s(%i)" % [external_type, len] - when :FLOAT, :DOUBLE - # Two-argument: length and precision. - "%s(%i,%i)" % [external_type, len, prec] - when :CHAR - if COLUMN_MTYPE_BY_VALUE[mtype] == :MYSQL - # When the mtype is :MYSQL, the column is actually - # stored as VARCHAR despite being a CHAR. This is - # done for CHAR columns having multi-byte character - # sets in order to limit size. Note that such data - # are still space-padded to at least len. - "VARCHAR(%i)" % [len] - else - "CHAR(%i)" % [len] + attr_reader :tablespaces + attr_reader :tables + attr_reader :indexes + attr_reader :columns + + def initialize + @tablespaces = Tablespaces.new + @tables = Tables.new + @indexes = Indexes.new + @columns = Columns.new + end + + def inspect + format("#<%s: %i tablespaces, %i tables, %i indexes, %i columns>", + self.class.name, + tablespaces.count, + tables.count, + indexes.count, + columns.count) + end + + def refresh + tables.each do |table| + table.indexes.each do |index| + indexes.add(index) end - when :DECIMAL - # The DECIMAL type is designated as DECIMAL(M,D) - # however the M and D definitions are not stored - # in the InnoDB data dictionary. We need to define - # the column as something which will extract the - # raw bytes in order to read the column, but we - # can't figure out the right decimal type. The - # len stored here is actually the on-disk storage - # size. - "CHAR(%i)" % [len] - else - external_type - end - end - - # Return a full data type given an mtype and prtype, such - # as ['VARCHAR(10)', :NOT_NULL] or [:INT, :UNSIGNED]. - def self.mtype_prtype_to_data_type(mtype, prtype, len, prec) - type = mtype_prtype_to_type_string(mtype, prtype, len, prec) - raise "Unsupported type (mtype #{mtype}, prtype #{prtype})" unless type - - data_type = [type] - data_type << :NOT_NULL if prtype & COLUMN_PRTYPE_FLAG[:NOT_NULL] != 0 - data_type << :UNSIGNED if prtype & COLUMN_PRTYPE_FLAG[:UNSIGNED] != 0 - - data_type - end - - attr_reader :system_space - - def initialize(system_space) - @system_space = system_space - end - - # A helper method to reach inside the system space and retrieve - # the data dictionary index locations from the data dictionary - # header. - def data_dictionary_indexes - system_space.data_dictionary_page.data_dictionary_header[:indexes] - end - - # Check if the data dictionary indexes are all available. - def found? - data_dictionary_indexes.values.map(&:values).flatten.none?(&:zero?) - end - - def data_dictionary_index_ids - raise "Data Dictionary not found; is the MySQL version supported?" unless found? - - return @data_dictionary_index_ids if @data_dictionary_index_ids - - # TODO: This could probably be done a lot more Ruby-like. - @data_dictionary_index_ids = {} - data_dictionary_indexes.each do |table, indexes| - indexes.each do |index, root_page_number| - root_page = system_space.page(root_page_number) - next unless root_page - - @data_dictionary_index_ids[root_page.index_id] = { - table: table, - index: index, - } - end - end - - @data_dictionary_index_ids - end - - def data_dictionary_table?(table_name) - DATA_DICTIONARY_RECORD_DESCRIBERS.include?(table_name.to_sym) - end - - def data_dictionary_index?(table_name, index_name) - return false unless data_dictionary_table?(table_name) - - DATA_DICTIONARY_RECORD_DESCRIBERS[table_name.to_sym].include?(index_name.to_sym) - end - - def data_dictionary_index_describer(table_name, index_name) - return unless data_dictionary_index?(table_name, index_name) - - DATA_DICTIONARY_RECORD_DESCRIBERS[table_name.to_sym][index_name.to_sym].new - end - - # Return an Innodb::Index object initialized to the - # internal data dictionary index with an appropriate - # record describer so that records can be recursed. - def data_dictionary_index(table_name, index_name) - raise "Data Dictionary not found; is the MySQL version supported?" unless found? - - table_entry = data_dictionary_indexes[table_name] - raise "Unknown data dictionary table #{table_name}" unless table_entry - - index_root_page = table_entry[index_name] - raise "Unknown data dictionary index #{table_name}.#{index_name}" unless index_root_page - - # If we have a record describer for this index, load it. - record_describer = data_dictionary_index_describer(table_name, index_name) - - system_space.index(index_root_page, record_describer) - end - - # Iterate through all data dictionary indexes, yielding the - # table name, index name, and root page number. - def each_data_dictionary_index_root_page_number - return enum_for(:each_data_dictionary_index_root_page_number) unless block_given? - - data_dictionary_indexes.each do |table_name, indexes| - indexes.each do |index_name, root_page_number| - yield table_name, index_name, root_page_number - end - end - - nil - end - - # Iterate through all data dictionary indexes, yielding the table - # name, index name, and the index itself as an Innodb::Index. - def each_data_dictionary_index - return enum_for(:each_data_dictionary_index) unless block_given? - - data_dictionary_indexes.each do |table_name, indexes| - indexes.each_key do |index_name| - yield table_name, index_name, - data_dictionary_index(table_name, index_name) - end - end - - nil - end - - # Iterate through records from a data dictionary index yielding each record - # as a Innodb::Record object. - def each_record_from_data_dictionary_index(table, index, &block) - return enum_for(:each_record_from_data_dictionary_index, table, index) unless block_given? - - data_dictionary_index(table, index).each_record(&block) - - nil - end - - # Iterate through the records in the SYS_TABLES data dictionary table. - def each_table - return enum_for(:each_table) unless block_given? - - each_record_from_data_dictionary_index(:SYS_TABLES, :PRIMARY) do |record| - yield record.fields - end - - nil - end - - # Iterate through the records in the SYS_COLUMNS data dictionary table. - def each_column - return enum_for(:each_column) unless block_given? - - each_record_from_data_dictionary_index(:SYS_COLUMNS, :PRIMARY) do |record| - yield record.fields - end - - nil - end - - # Iterate through the records in the SYS_INDEXES dictionary table. - def each_index - return enum_for(:each_index) unless block_given? - - each_record_from_data_dictionary_index(:SYS_INDEXES, :PRIMARY) do |record| - yield record.fields - end - - nil - end - - # Iterate through the records in the SYS_FIELDS data dictionary table. - def each_field - return enum_for(:each_field) unless block_given? - - each_record_from_data_dictionary_index(:SYS_FIELDS, :PRIMARY) do |record| - yield record.fields - end - - nil - end - - # A helper to iterate the method provided and return the first record - # where the record's field matches the provided value. - def object_by_field(method, field, value) - send(method).select { |o| o[field] == value }.first - end - - # A helper to iterate the method provided and return the first record - # where the record's fields f1 and f2 match the provided values v1 and v2. - def object_by_two_fields(method, field1, value1, field2, value2) - send(method).select { |o| o[field1] == value1 && o[field2] == value2 }.first - end - - # Lookup a table by table ID. - def table_by_id(table_id) - object_by_field(:each_table, "ID", table_id) - end - - # Lookup a table by table name. - def table_by_name(table_name) - object_by_field(:each_table, "NAME", table_name) - end - - # Lookup a table by space ID. - def table_by_space_id(space_id) - object_by_field(:each_table, "SPACE", space_id) - end - - # Lookup a column by table name and column name. - def column_by_name(table_name, column_name) - table = table_by_name(table_name) - return unless table - - object_by_two_fields(:each_column, "TABLE_ID", table["ID"], "NAME", column_name) - end - - # Lookup an index by index ID. - def index_by_id(index_id) - object_by_field(:each_index, "ID", index_id) - end - - # Lookup an index by table name and index name. - def index_by_name(table_name, index_name) - table = table_by_name(table_name) - return unless table - - object_by_two_fields(:each_index, "TABLE_ID", table["ID"], "NAME", index_name) - end - - # Iterate through indexes by space ID. - def each_index_by_space_id(space_id) - return enum_for(:each_index_by_space_id, space_id) unless block_given? - - each_index do |record| - yield record if record["SPACE"] == space_id - end - - nil - end - - # Iterate through all indexes in a table by table ID. - def each_index_by_table_id(table_id) - return enum_for(:each_index_by_table_id, table_id) unless block_given? - - each_index do |record| - yield record if record["TABLE_ID"] == table_id - end - nil - end - - # Iterate through all indexes in a table by table name. - def each_index_by_table_name(table_name, &block) - return enum_for(:each_index_by_table_name, table_name) unless block_given? - - table = table_by_name(table_name) - raise "Table #{table_name} not found" unless table - - each_index_by_table_id(table["ID"], &block) - - nil - end - - # Iterate through all fields in an index by index ID. - def each_field_by_index_id(index_id) - return enum_for(:each_field_by_index_id, index_id) unless block_given? - - each_field do |record| - yield record if record["INDEX_ID"] == index_id - end - - nil - end - - # Iterate through all fields in an index by index name. - def each_field_by_index_name(table_name, index_name, &block) - return enum_for(:each_field_by_index_name, table_name, index_name) unless block_given? - - index = index_by_name(table_name, index_name) - raise "Index #{index_name} for table #{table_name} not found" unless index - - each_field_by_index_id(index["ID"], &block) - - nil - end - - # Iterate through all columns in a table by table ID. - def each_column_by_table_id(table_id) - return enum_for(:each_column_by_table_id, table_id) unless block_given? - - each_column do |record| - yield record if record["TABLE_ID"] == table_id - end - - nil - end - - # Iterate through all columns in a table by table name. - def each_column_by_table_name(table_name, &block) - return enum_for(:each_column_by_table_name, table_name) unless block_given? - raise "Table #{table_name} not found" unless (table = table_by_name(table_name)) - - each_column_by_table_id(table["ID"], &block) - - nil - end - - # Iterate through all columns in an index by table name and index name. - def each_column_in_index_by_name(table_name, index_name) - return enum_for(:each_column_in_index_by_name, table_name, index_name) unless block_given? - - each_field_by_index_name(table_name, index_name) do |record| - yield column_by_name(table_name, record["COL_NAME"]) - end - - nil - end - - # Iterate through all columns not in an index by table name and index name. - # This is useful when building index descriptions for secondary indexes. - def each_column_not_in_index_by_name(table_name, index_name) - return enum_for(:each_column_not_in_index_by_name, table_name, index_name) unless block_given? - - columns_in_index = {} - each_column_in_index_by_name(table_name, index_name) do |record| - columns_in_index[record["NAME"]] = 1 - end - - each_column_by_table_name(table_name) do |record| - yield record unless columns_in_index.include?(record["NAME"]) - end - - nil - end - - # Return the name of the clustered index (usually 'PRIMARY', but not always) - # for a given table name. - def clustered_index_name_by_table_name(table_name) - table_record = table_by_name(table_name) - raise "Table #{table_name} not found" unless table_record - - index_record = object_by_two_fields(:each_index, "TABLE_ID", table_record["ID"], "TYPE", 3) - index_record["NAME"] if index_record - end - - # Produce a Innodb::RecordDescriber-compatible column description - # given a type (:key, :row) and data dictionary SYS_COLUMNS record. - def _make_column_description(type, record) - { - type: type, - name: record["NAME"], - description: self.class.mtype_prtype_to_data_type( - record["MTYPE"], - record["PRTYPE"], - record["LEN"], - record["PREC"] - ), - } - end - - # Iterate through Innodb::RecordDescriber-compatible column descriptions - # for a given index by table name and index name. - def each_column_description_by_index_name(table_name, index_name) - return enum_for(:each_column_description_by_index_name, table_name, index_name) unless block_given? - - unless (index = index_by_name(table_name, index_name)) - raise "Index #{index_name} for table #{table_name} not found" - end - - columns_in_index = {} - each_column_in_index_by_name(table_name, index_name) do |record| - columns_in_index[record["NAME"]] = 1 - yield _make_column_description(:key, record) - end - - if (index["TYPE"] & INDEX_TYPE_FLAG[:CLUSTERED]).zero? - clustered_index_name = clustered_index_name_by_table_name(table_name) - - each_column_in_index_by_name(table_name, clustered_index_name) do |record| - yield _make_column_description(:row, record) - end - else - each_column_by_table_name(table_name) do |record| - yield _make_column_description(:row, record) unless columns_in_index.include?(record["NAME"]) + table.columns.each do |column| + columns.add(column) end end nil end - # Return an Innodb::RecordDescriber object describing records for a given - # index by table name and index name. - def record_describer_by_index_name(table_name, index_name) - return data_dictionary_index_describer(table_name, index_name) if data_dictionary_index?(table_name, index_name) - - unless (index = index_by_name(table_name, index_name)) - raise "Index #{index_name} for table #{table_name} not found" - end - - describer = Innodb::RecordDescriber.new - - if (index["TYPE"] & INDEX_TYPE_FLAG[:CLUSTERED]).zero? - describer.type :secondary - else - describer.type :clustered - end - - each_column_description_by_index_name(table_name, index_name) do |column| - case column[:type] - when :key - describer.key column[:name], *column[:description] - when :row - describer.row column[:name], *column[:description] - end - end - - describer - end - - # Return an Innodb::RecordDescriber object describing the records - # in a given index by index ID. - def record_describer_by_index_id(index_id) - if (dd_index = data_dictionary_index_ids[index_id]) - return data_dictionary_index_describer(dd_index[:table], dd_index[:index]) - end - - unless (index = index_by_id(index_id)) - raise "Index #{index_id} not found" - end - - unless (table = table_by_id(index["TABLE_ID"])) - raise "Table #{INDEX['TABLE_ID']} not found" - end - - record_describer_by_index_name(table["NAME"], index["NAME"]) + def populated? + tablespaces.any? || tables.any? end end end diff --git a/lib/innodb/data_dictionary/column.rb b/lib/innodb/data_dictionary/column.rb new file mode 100644 index 00000000..8439811f --- /dev/null +++ b/lib/innodb/data_dictionary/column.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Innodb + class DataDictionary + class Column + attr_reader :name + attr_reader :description + attr_reader :table + + def initialize(name:, description:, table:) + @name = name + @description = description + @table = table + end + end + end +end diff --git a/lib/innodb/data_dictionary/columns.rb b/lib/innodb/data_dictionary/columns.rb new file mode 100644 index 00000000..b9b154e2 --- /dev/null +++ b/lib/innodb/data_dictionary/columns.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require "innodb/data_dictionary/object_store" +require "innodb/data_dictionary/column" + +module Innodb + class DataDictionary + class Columns < ObjectStore + def initialize + super(allowed_type: Column) + end + end + end +end diff --git a/lib/innodb/data_dictionary/index.rb b/lib/innodb/data_dictionary/index.rb new file mode 100644 index 00000000..6200728a --- /dev/null +++ b/lib/innodb/data_dictionary/index.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require "innodb/data_dictionary/index_column_references" + +module Innodb + class DataDictionary + class Index + attr_reader :name + attr_reader :type + attr_reader :tablespace + attr_reader :root_page_number + attr_reader :table + attr_reader :innodb_index_id + attr_reader :column_references + + def initialize(name:, type:, table:, tablespace:, root_page_number:, innodb_index_id: nil) + @name = name + @type = type + @table = table + @tablespace = tablespace + @root_page_number = root_page_number + @innodb_index_id = innodb_index_id + @column_references = IndexColumnReferences.new + end + + def record_describer + describer = Innodb::RecordDescriber.new + + describer.type(type) + + column_references.each do |column_reference| + case column_reference.usage + when :key + describer.key(column_reference.column.name, *column_reference.column.description) + when :row + describer.row(column_reference.column.name, *column_reference.column.description) + end + end + + describer + end + end + end +end diff --git a/lib/innodb/data_dictionary/index_column_reference.rb b/lib/innodb/data_dictionary/index_column_reference.rb new file mode 100644 index 00000000..3d4e84cc --- /dev/null +++ b/lib/innodb/data_dictionary/index_column_reference.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require "forwardable" + +module Innodb + class DataDictionary + class IndexColumnReference + extend Forwardable + + attr_reader :column + attr_reader :usage + attr_reader :index + + def_delegators :column, :id, :name, :description + + def initialize(column:, usage:, index:) + @column = column + @usage = usage + @index = index + end + end + end +end diff --git a/lib/innodb/data_dictionary/index_column_references.rb b/lib/innodb/data_dictionary/index_column_references.rb new file mode 100644 index 00000000..28393b53 --- /dev/null +++ b/lib/innodb/data_dictionary/index_column_references.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require "innodb/data_dictionary/object_store" +require "innodb/data_dictionary/index_column_reference" + +module Innodb + class DataDictionary + class IndexColumnReferences < ObjectStore + def initialize + super(allowed_type: IndexColumnReference) + end + end + end +end diff --git a/lib/innodb/data_dictionary/indexes.rb b/lib/innodb/data_dictionary/indexes.rb new file mode 100644 index 00000000..f71b023b --- /dev/null +++ b/lib/innodb/data_dictionary/indexes.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require "innodb/data_dictionary/object_store" +require "innodb/data_dictionary/index" + +module Innodb + class DataDictionary + class Indexes < ObjectStore + def initialize + super(allowed_type: Index) + end + end + end +end diff --git a/lib/innodb/data_dictionary/object_store.rb b/lib/innodb/data_dictionary/object_store.rb new file mode 100644 index 00000000..7d381dad --- /dev/null +++ b/lib/innodb/data_dictionary/object_store.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require "forwardable" + +module Innodb + class DataDictionary + class ObjectStore + extend Forwardable + def_delegators :@objects, :[], :first, :each, :empty?, :any?, :count + + class ObjectTypeError < RuntimeError; end + + attr_reader :allowed_type + attr_reader :objects + + def initialize(allowed_type: Object) + @allowed_type = allowed_type + @objects = [] + end + + def add(new_object) + raise ObjectTypeError unless new_object.is_a?(@allowed_type) + + @objects.push(new_object) + + new_object + end + + def make(**attributes) + add(@allowed_type.new(**attributes)) + end + + def by(**attributes) + @objects.select { |o| attributes.all? { |k, v| o.send(k) == v } } + end + + def find(**attributes) + by(**attributes).first + end + end + end +end diff --git a/lib/innodb/data_dictionary/table.rb b/lib/innodb/data_dictionary/table.rb new file mode 100644 index 00000000..7cc8cc2e --- /dev/null +++ b/lib/innodb/data_dictionary/table.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +require "innodb/data_dictionary/columns" +require "innodb/data_dictionary/indexes" + +module Innodb + class DataDictionary + class Table + attr_reader :name + attr_reader :tablespace + attr_reader :innodb_table_id + attr_reader :columns + attr_reader :indexes + + def initialize(name:, tablespace: nil, innodb_table_id: nil) + @name = name + @tablespace = tablespace + @innodb_table_id = innodb_table_id + @columns = Columns.new + @indexes = Indexes.new + end + + def clustered_index + indexes.find(type: :clustered) + end + end + end +end diff --git a/lib/innodb/data_dictionary/tables.rb b/lib/innodb/data_dictionary/tables.rb new file mode 100644 index 00000000..4d314303 --- /dev/null +++ b/lib/innodb/data_dictionary/tables.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require "innodb/data_dictionary/object_store" +require "innodb/data_dictionary/table" + +module Innodb + class DataDictionary + class Tables < ObjectStore + def initialize + super(allowed_type: Table) + end + end + end +end diff --git a/lib/innodb/data_dictionary/tablespace.rb b/lib/innodb/data_dictionary/tablespace.rb new file mode 100644 index 00000000..137b3596 --- /dev/null +++ b/lib/innodb/data_dictionary/tablespace.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Innodb + class DataDictionary + class Tablespace + attr_reader :name + attr_reader :innodb_space_id + + def initialize(name:, innodb_space_id:) + @name = name + @innodb_space_id = innodb_space_id + end + end + end +end diff --git a/lib/innodb/data_dictionary/tablespaces.rb b/lib/innodb/data_dictionary/tablespaces.rb new file mode 100644 index 00000000..64df7a73 --- /dev/null +++ b/lib/innodb/data_dictionary/tablespaces.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require "innodb/data_dictionary/object_store" +require "innodb/data_dictionary/tablespace" + +module Innodb + class DataDictionary + class Tablespaces < ObjectStore + def initialize + super(allowed_type: Tablespace) + end + end + end +end diff --git a/lib/innodb/data_type.rb b/lib/innodb/data_type.rb index 49d98fd7..bfe9e1f3 100644 --- a/lib/innodb/data_type.rb +++ b/lib/innodb/data_type.rb @@ -346,6 +346,7 @@ def value(data) class EnumType attr_reader :name + attr_reader :width def initialize(base_type, modifiers, properties) @width = 1 @@ -448,6 +449,7 @@ def value(data) TRX_ID: TransactionIdType, ROLL_PTR: RollPointerType, ENUM: EnumType, + JSON: BlobType, }.freeze def self.make_name(base_type, modifiers, properties) diff --git a/lib/innodb/page/index.rb b/lib/innodb/page/index.rb index 61d954b0..c7a43253 100644 --- a/lib/innodb/page/index.rb +++ b/lib/innodb/page/index.rb @@ -546,8 +546,12 @@ def supremum end def make_record_describer - if space&.innodb_system&.data_dictionary&.found? && index_id && !ibuf_index? - @record_describer = space.innodb_system.data_dictionary.record_describer_by_index_id(index_id) + if space.innodb_system.data_dictionary && index_id && !ibuf_index? + @record_describer = space.innodb_system + .data_dictionary + .indexes + .find(innodb_index_id: index_id) + .record_describer elsif space @record_describer = space.record_describer end diff --git a/lib/innodb/page/sys_data_dictionary_header.rb b/lib/innodb/page/sys_data_dictionary_header.rb index 9ae7c46d..1fe0601d 100644 --- a/lib/innodb/page/sys_data_dictionary_header.rb +++ b/lib/innodb/page/sys_data_dictionary_header.rb @@ -36,25 +36,25 @@ def data_dictionary_header unused_mix_id_low: c.name("unused_mix_id_low") { c.read_uint32 }, indexes: c.name("indexes") do { - SYS_TABLES: c.name("SYS_TABLES") do + "SYS_TABLES" => c.name("SYS_TABLES") do { - PRIMARY: c.name("PRIMARY") { c.read_uint32 }, - ID: c.name("ID") { c.read_uint32 }, + "PRIMARY" => c.name("PRIMARY") { c.read_uint32 }, + "ID" => c.name("ID") { c.read_uint32 }, } end, - SYS_COLUMNS: c.name("SYS_COLUMNS") do + "SYS_COLUMNS" => c.name("SYS_COLUMNS") do { - PRIMARY: c.name("PRIMARY") { c.read_uint32 }, + "PRIMARY" => c.name("PRIMARY") { c.read_uint32 }, } end, - SYS_INDEXES: c.name("SYS_INDEXES") do + "SYS_INDEXES" => c.name("SYS_INDEXES") do { - PRIMARY: c.name("PRIMARY") { c.read_uint32 }, + "PRIMARY" => c.name("PRIMARY") { c.read_uint32 }, } end, - SYS_FIELDS: c.name("SYS_FIELDS") do + "SYS_FIELDS" => c.name("SYS_FIELDS") do { - PRIMARY: c.name("PRIMARY") { c.read_uint32 }, + "PRIMARY" => c.name("PRIMARY") { c.read_uint32 }, } end, } diff --git a/lib/innodb/record_describer.rb b/lib/innodb/record_describer.rb index fea8d8d7..acbfa4d2 100644 --- a/lib/innodb/record_describer.rb +++ b/lib/innodb/record_describer.rb @@ -116,22 +116,5 @@ def field_names end names end - - def generate_class(name = "Describer_#{object_id}") - str = "class #{name}\n".dup - str << format(" type %s\n", description[:type].inspect) - %i[key row].each do |group| - description[group].each do |item| - str << format( - " %s %s, %s\n", - group, - item[:name].inspect, - item[:type].map(&:inspect).join(", ") - ) - end - end - str << "end\n" - str - end end end diff --git a/lib/innodb/sdi.rb b/lib/innodb/sdi.rb index 4003d536..25f7a9a0 100644 --- a/lib/innodb/sdi.rb +++ b/lib/innodb/sdi.rb @@ -12,6 +12,18 @@ class << self def register_specialization(id, specialized_class) @specialized_classes[id] = specialized_class end + + def parse_semicolon_value_list(data) + data&.split(";").to_h { |x| x.split("=") } + end + + def parse_se_private_data(data) + parse_semicolon_value_list(data) + end + + def parse_options(data) + parse_semicolon_value_list(data) + end end attr_reader :space @@ -53,20 +65,12 @@ def each_object nil end - def each_table - return enum_for(:each_table) unless block_given? - - each_object { |o| yield o if o.is_a?(Table) } - - nil + def tables + each_object.select { |o| o.is_a?(Table) } end - def each_tablespace - return enum_for(:each_tablespace) unless block_given? - - each_object { |o| yield o if o.is_a?(Tablespace) } - - nil + def tablespaces + each_object.select { |o| o.is_a?(Tablespace) } end end end diff --git a/lib/innodb/sdi/sdi_object.rb b/lib/innodb/sdi/sdi_object.rb index c1a4a28e..da70681b 100644 --- a/lib/innodb/sdi/sdi_object.rb +++ b/lib/innodb/sdi/sdi_object.rb @@ -61,11 +61,11 @@ def name end def options - data["options"]&.split(";").to_h { |x| x.split("=") } + Innodb::Sdi.parse_options(data["options"]) end def se_private_data - data["se_private_data"]&.split(";").to_h { |x| x.split("=") } + Innodb::Sdi.parse_se_private_data(dd_object["se_private_data"]) end end end diff --git a/lib/innodb/sdi/table.rb b/lib/innodb/sdi/table.rb index 89720ea7..e9c068d2 100644 --- a/lib/innodb/sdi/table.rb +++ b/lib/innodb/sdi/table.rb @@ -21,14 +21,6 @@ def space_id indexes.first.space_id end - def each_column(&block) - return enum_for(:each_column) unless block_given? - - dd_object["columns"].each(&block) - - nil - end - def find_index_by_name(name) indexes.find { |index| index.name == name } end diff --git a/lib/innodb/sdi/table_column.rb b/lib/innodb/sdi/table_column.rb index 3c1c7ac1..c20595ce 100644 --- a/lib/innodb/sdi/table_column.rb +++ b/lib/innodb/sdi/table_column.rb @@ -70,14 +70,14 @@ def system? def description [ - data["column_type_utf8"], - unsigned? ? :unsigned : nil, - nullable? ? nil : :not_null, + data["column_type_utf8"].sub(/ unsigned$/, ""), + unsigned? ? :UNSIGNED : nil, + nullable? ? nil : :NOT_NULL, ].compact end def se_private_data - data["se_private_data"]&.split(";").to_h { |x| x.split("=") } + Innodb::Sdi.parse_se_private_data(data["se_private_data"]) end def table_id diff --git a/lib/innodb/sdi/table_index.rb b/lib/innodb/sdi/table_index.rb index 8b27019c..7cfbc8c2 100644 --- a/lib/innodb/sdi/table_index.rb +++ b/lib/innodb/sdi/table_index.rb @@ -50,10 +50,10 @@ def elements end def se_private_data - data["se_private_data"]&.split(";").to_h { |x| x.split("=") } + Innodb::Sdi.parse_se_private_data(data["se_private_data"]) end - def index_id + def innodb_index_id se_private_data["id"].to_i end @@ -61,11 +61,11 @@ def root_page_number se_private_data["root"].to_i end - def space_id + def innodb_space_id se_private_data["space_id"].to_i end - def table_id + def innodb_table_id se_private_data["table_id"].to_i end @@ -74,7 +74,7 @@ def trx_id end def options - data["options"]&.split(";").to_h { |x| x.split("=") } + Innodb::Sdi.parse_options(data["options"]) end def type @@ -86,22 +86,7 @@ def primary? end def clustered? - table.clustered_index.index_id == index_id - end - - def record_describer - describer = Innodb::RecordDescriber.new - describer.type(clustered? ? :clustered : :secondary) - - elements.each do |element| - if element.key? - describer.key(element.column.name, *element.column.description) - elsif element.row? - describer.row(element.column.name, *element.column.description) - end - end - - describer + table.clustered_index.innodb_index_id == innodb_index_id end end end diff --git a/lib/innodb/sdi/table_index_element.rb b/lib/innodb/sdi/table_index_element.rb index 5c4b37c9..9bf9b085 100644 --- a/lib/innodb/sdi/table_index_element.rb +++ b/lib/innodb/sdi/table_index_element.rb @@ -39,6 +39,13 @@ def row? hidden? end + def type + return :sys if %w[DB_TRX_ID DB_ROLL_PTR].include?(column.name) + return :key if key? + + :row + end + def column_opx data["column_opx"] end diff --git a/lib/innodb/sdi_data_dictionary.rb b/lib/innodb/sdi_data_dictionary.rb new file mode 100644 index 00000000..2cb2aaa8 --- /dev/null +++ b/lib/innodb/sdi_data_dictionary.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +# A class representing MySQL's SDI-based data dictionary (used in MySQL +# versions starting in MySQL 8.0), which contains metadata about tables, +# columns, and indexes distributed in BLOBs of JSON stored in each InnoDB +# tablespace file. +module Innodb + class SdiDataDictionary + extend Forwardable + + attr_reader :innodb_system + + def_delegator :innodb_system, :data_dictionary + + def initialize(innodb_system) + @innodb_system = innodb_system + end + + def populate_data_dictionary_using_space_sdi(space) + sdi_tablespace = space.sdi.tablespaces.first + return unless sdi_tablespace + + innodb_space_id = sdi_tablespace.se_private_data["id"].to_i + new_tablespace = data_dictionary.tablespaces.make(name: sdi_tablespace.name, innodb_space_id: innodb_space_id) + + space.sdi.tables.each do |table| + new_table = data_dictionary.tables.make(name: table.name, tablespace: new_tablespace) + + table.columns.each do |column| + next if %w[DB_TRX_ID DB_ROLL_PTR].include?(column.name) + + new_table.columns.make(name: column.name, description: column.description, table: new_table) + end + + table.indexes.each do |index| + new_index = new_table.indexes.make( + name: index.name, + type: index.clustered? ? :clustered : :secondary, + table: new_table, + tablespace: new_tablespace, + root_page_number: index.root_page_number, + innodb_index_id: index.innodb_index_id + ) + + db_trx_id = Innodb::DataDictionary::Column.new( + name: "DB_TRX_ID", + description: %i[DB_TRX_ID], + table: new_table + ) + db_roll_ptr = Innodb::DataDictionary::Column.new( + name: "DB_ROLL_PTR", + description: %i[DB_ROLL_PTR], + table: new_table + ) + + index.elements.each do |element| + case element.column.name + when "DB_TRX_ID" + new_index.column_references.make(column: db_trx_id, usage: :sys, index: new_index) + when "DB_ROLL_PTR" + new_index.column_references.make(column: db_roll_ptr, usage: :sys, index: new_index) + else + new_index.column_references.make(column: new_table.columns.find(name: element.column.name), + usage: element.type, index: new_index) + end + end + end + end + + nil + end + + def populate_data_dictionary + data_dictionary.tablespaces.make(name: "innodb_system", innodb_space_id: 0) + + innodb_system.each_space do |space| + populate_data_dictionary_using_space_sdi(space) + end + end + end +end diff --git a/lib/innodb/space.rb b/lib/innodb/space.rb index 8d99a782..d6ac35fb 100644 --- a/lib/innodb/space.rb +++ b/lib/innodb/space.rb @@ -48,16 +48,13 @@ def initialize(filename, offset) end def name - prefix = "" - prefix = "#{File.basename(File.dirname(file.path))}/" if File.extname(file.path) == ".ibd" - - prefix + File.basename(file.path) + file.path end end # Open a space file, optionally providing the page size to use. Pages # that aren't 16 KiB may not be supported well. - def initialize(filenames) + def initialize(filenames, innodb_system: nil) filenames = [filenames] unless filenames.is_a?(Array) @data_files = [] @@ -73,7 +70,7 @@ def initialize(filenames) @compressed = fsp_flags.compressed @pages = (@size / @page_size) - @innodb_system = nil + @innodb_system = innodb_system @record_describer = nil end @@ -334,10 +331,14 @@ def index(root_page_number, record_describer = nil) def each_index_root_page_number return enum_for(:each_index_root_page_number) unless block_given? - if innodb_system + if innodb_system&.data_dictionary&.populated? # Retrieve the index root page numbers from the data dictionary. - innodb_system.data_dictionary.each_index_by_space_id(space_id) do |record| - yield record["PAGE_NO"] + # TODO: An efficient way to handle this? + tablespace = innodb_system.data_dictionary.tablespaces.find(innodb_space_id: space_id) + innodb_system.data_dictionary.tables.each do |table| + table.indexes.by(tablespace: tablespace).each do |index| + yield index.root_page_number + end end else # Guess that the index root pages will be present starting at page 3, diff --git a/lib/innodb/sys_data_dictionary.rb b/lib/innodb/sys_data_dictionary.rb new file mode 100644 index 00000000..c61d4a1e --- /dev/null +++ b/lib/innodb/sys_data_dictionary.rb @@ -0,0 +1,310 @@ +# frozen_string_literal: true + +require "forwardable" + +# A class representing InnoDB's SYS_* data dictionary (used in MySQL +# versions prior to MySQL 8.0), which contains metadata about tables, +# columns, and indexes in internal InnoDB tables named SYS_*. +module Innodb + class SysDataDictionary + # rubocop:disable Layout/ExtraSpacing + SYSTEM_TABLES = [ + { + name: "SYS_TABLES", + columns: [ + { name: "NAME", description: ["VARCHAR(100)", :NOT_NULL] }, + { name: "ID", description: %i[BIGINT UNSIGNED NOT_NULL] }, + { name: "N_COLS", description: %i[INT UNSIGNED NOT_NULL] }, + { name: "TYPE", description: %i[INT UNSIGNED NOT_NULL] }, + { name: "MIX_ID", description: %i[BIGINT UNSIGNED NOT_NULL] }, + { name: "MIX_LEN", description: %i[INT UNSIGNED NOT_NULL] }, + { name: "CLUSTER_NAME", description: ["VARCHAR(100)", :NOT_NULL] }, + { name: "SPACE", description: %i[INT UNSIGNED NOT_NULL] }, + ], + indexes: [ + { name: "PRIMARY", type: :clustered, column_names: ["NAME"] }, + { name: "ID", type: :secondary, column_names: ["ID"] }, + ], + }, + { + name: "SYS_COLUMNS", + columns: [ + { name: "TABLE_ID", description: %i[BIGINT UNSIGNED NOT_NULL] }, + { name: "POS", description: %i[INT UNSIGNED NOT_NULL] }, + { name: "NAME", description: ["VARCHAR(100)", :NOT_NULL] }, + { name: "MTYPE", description: %i[INT UNSIGNED NOT_NULL] }, + { name: "PRTYPE", description: %i[INT UNSIGNED NOT_NULL] }, + { name: "LEN", description: %i[INT UNSIGNED NOT_NULL] }, + { name: "PREC", description: %i[INT UNSIGNED NOT_NULL] }, + ], + indexes: [ + { name: "PRIMARY", type: :clustered, column_names: %w[TABLE_ID POS] }, + ], + }, + { + name: "SYS_INDEXES", + columns: [ + { name: "TABLE_ID", description: %i[BIGINT UNSIGNED NOT_NULL] }, + { name: "ID", description: %i[BIGINT UNSIGNED NOT_NULL] }, + { name: "NAME", description: ["VARCHAR(100)", :NOT_NULL] }, + { name: "N_FIELDS", description: %i[INT UNSIGNED NOT_NULL] }, + { name: "TYPE", description: %i[INT UNSIGNED NOT_NULL] }, + { name: "SPACE", description: %i[INT UNSIGNED NOT_NULL] }, + { name: "PAGE_NO", description: %i[INT UNSIGNED NOT_NULL] }, + ], + indexes: [ + { name: "PRIMARY", type: :clustered, column_names: %w[TABLE_ID ID] }, + ], + }, + { + name: "SYS_FIELDS", + columns: [ + { name: "INDEX_ID", description: %i[BIGINT UNSIGNED NOT_NULL] }, + { name: "POS", description: %i[INT UNSIGNED NOT_NULL] }, + { name: "COL_NAME", description: ["VARCHAR(100)", :NOT_NULL] }, + ], + indexes: [ + { name: "PRIMARY", type: :clustered, column_names: %w[INDEX_ID POS] }, + ], + }, + ].freeze + # rubocop:enable Layout/ExtraSpacing + + # A hash of InnoDB's internal type system to the values + # stored for each type. + COLUMN_MTYPE = { + VARCHAR: 1, + CHAR: 2, + FIXBINARY: 3, + BINARY: 4, + BLOB: 5, + INT: 6, + SYS_CHILD: 7, + SYS: 8, + FLOAT: 9, + DOUBLE: 10, + DECIMAL: 11, + VARMYSQL: 12, + MYSQL: 13, + }.freeze + + # A hash of COLUMN_MTYPE keys by value. + COLUMN_MTYPE_BY_VALUE = COLUMN_MTYPE.invert.freeze + + # A hash of InnoDB 'precise type' bitwise flags. + COLUMN_PRTYPE_FLAG = { + NOT_NULL: 256, + UNSIGNED: 512, + BINARY: 1024, + LONG_TRUE_VARCHAR: 4096, + }.freeze + + # A hash of COLUMN_PRTYPE keys by value. + COLUMN_PRTYPE_FLAG_BY_VALUE = COLUMN_PRTYPE_FLAG.invert.freeze + + # The bitmask to extract the MySQL internal type + # from the InnoDB 'precise type'. + COLUMN_PRTYPE_MYSQL_TYPE_MASK = 0xFF + + # A hash of InnoDB's index type flags. + INDEX_TYPE_FLAG = { + CLUSTERED: 1, + UNIQUE: 2, + UNIVERSAL: 4, + IBUF: 8, + CORRUPT: 16, + FTS: 32, + }.freeze + + # A hash of INDEX_TYPE_FLAG keys by value. + INDEX_TYPE_FLAG_BY_VALUE = INDEX_TYPE_FLAG.invert.freeze + + # Return the 'external' SQL type string (such as 'VARCHAR' or + # 'INT') given the stored mtype and prtype from the InnoDB + # data dictionary. Note that not all types are extractable + # into fully defined SQL types due to the lossy nature of + # the MySQL-to-InnoDB interface regarding types. + def self.mtype_prtype_to_type_string(mtype, prtype, len, prec) + mysql_type = prtype & COLUMN_PRTYPE_MYSQL_TYPE_MASK + internal_type = Innodb::MysqlType.by_mysql_field_type(mysql_type) + external_type = internal_type.handle_as + + case external_type + when :VARCHAR + # One-argument: length. + "%s(%i)" % [external_type, len] + when :FLOAT, :DOUBLE + # Two-argument: length and precision. + "%s(%i,%i)" % [external_type, len, prec] + when :CHAR + if COLUMN_MTYPE_BY_VALUE[mtype] == :MYSQL + # When the mtype is :MYSQL, the column is actually + # stored as VARCHAR despite being a CHAR. This is + # done for CHAR columns having multi-byte character + # sets in order to limit size. Note that such data + # are still space-padded to at least len. + "VARCHAR(%i)" % [len] + else + "CHAR(%i)" % [len] + end + when :DECIMAL + # The DECIMAL type is designated as DECIMAL(M,D) + # however the M and D definitions are not stored + # in the InnoDB data dictionary. We need to define + # the column as something which will extract the + # raw bytes in order to read the column, but we + # can't figure out the right decimal type. The + # len stored here is actually the on-disk storage + # size. + "CHAR(%i)" % [len] + else + external_type + end + end + + # Return a full data type given an mtype and prtype, such + # as ['VARCHAR(10)', :NOT_NULL] or [:INT, :UNSIGNED]. + def self.mtype_prtype_to_data_type(mtype, prtype, len, prec) + type = mtype_prtype_to_type_string(mtype, prtype, len, prec) + raise "Unsupported type (mtype #{mtype}, prtype #{prtype})" unless type + + data_type = [type] + data_type << :NOT_NULL if prtype & COLUMN_PRTYPE_FLAG[:NOT_NULL] != 0 + data_type << :UNSIGNED if prtype & COLUMN_PRTYPE_FLAG[:UNSIGNED] != 0 + + data_type + end + + extend Forwardable + + attr_reader :innodb_system + + def_delegator :innodb_system, :data_dictionary + + def initialize(innodb_system) + @innodb_system = innodb_system + end + + private + + # A helper method to reach inside the system space and retrieve + # the data dictionary index locations from the data dictionary + # header. + def _data_dictionary_indexes + innodb_system.system_space.data_dictionary_page.data_dictionary_header[:indexes] + end + + def _populate_index_with_system_and_non_key_columns(new_table, new_index) + if new_index.type == :clustered + db_trx_id = Innodb::DataDictionary::Column.new(name: "DB_TRX_ID", description: %i[DB_TRX_ID], table: new_table) + new_index.column_references.make(column: db_trx_id, usage: :sys, index: new_index) + db_roll_ptr = Innodb::DataDictionary::Column.new(name: "DB_ROLL_PTR", description: %i[DB_ROLL_PTR], + table: new_table) + new_index.column_references.make(column: db_roll_ptr, usage: :sys, index: new_index) + + new_table.columns.each do |column| + unless new_index.column_references.find(name: column.name) + new_index.column_references.make(column: column, usage: :row, + index: new_index) + end + end + else + clustered_index = new_table.indexes.find(type: :clustered) + clustered_index.column_references.each do |column| + new_index.column_references.make(column: column, usage: :row, index: new_index) unless column.usage == :sys + end + end + end + + public + + def populate_data_dictionary_with_system_table_definitions + system_tablespace = data_dictionary.tablespaces.make(name: "innodb_system", innodb_space_id: 0) + + SYSTEM_TABLES.each do |table| + new_table = data_dictionary.tables.make(name: table[:name], tablespace: system_tablespace) + + table[:columns].each do |column| + new_table.columns.make(name: column[:name], description: column[:description], table: new_table) + end + + table[:indexes].each do |index| + new_index = new_table.indexes.make( + name: index[:name], + type: index[:type], + table: new_table, + tablespace: system_tablespace, + root_page_number: _data_dictionary_indexes[table[:name]][index[:name]] + ) + index[:column_names].each do |column_name| + new_index.column_references.make( + column: new_table.columns.find(name: column_name), + usage: :key, + index: new_index + ) + end + _populate_index_with_system_and_non_key_columns(new_table, new_index) + end + end + + nil + end + + def populate_data_dictionary_from_system_tables + # Read the entire contents of all tables for efficiency sake, since we'll need to do many sub-iterations + # below and don't want to re-parse the records every time. + sys_tables = innodb_system.index_by_name("SYS_TABLES", "PRIMARY").each_record.map(&:fields) + sys_columns = innodb_system.index_by_name("SYS_COLUMNS", "PRIMARY").each_record.map(&:fields) + sys_indexes = innodb_system.index_by_name("SYS_INDEXES", "PRIMARY").each_record.map(&:fields) + sys_fields = innodb_system.index_by_name("SYS_FIELDS", "PRIMARY").each_record.map(&:fields) + + sys_tables.each do |table_record| + tablespace = data_dictionary.tablespaces.find(innodb_space_id: table_record["SPACE"]) + tablespace ||= data_dictionary.tablespaces.make(name: table_record["NAME"], + innodb_space_id: table_record["SPACE"]) + + new_table = data_dictionary.tables.make(name: table_record["NAME"], tablespace: tablespace, + innodb_table_id: table_record["ID"]) + + sys_columns.select { |r| r["TABLE_ID"] == table_record["ID"] }.each do |column_record| + description = self.class.mtype_prtype_to_data_type( + column_record["MTYPE"], + column_record["PRTYPE"], + column_record["LEN"], + column_record["PREC"] + ) + new_table.columns.make(name: column_record["NAME"], description: description, table: new_table) + end + + sys_indexes.select { |r| r["TABLE_ID"] == table_record["ID"] }.each do |index_record| + raise "Different tablespace between table and index" unless table_record["SPACE"] == index_record["SPACE"] + + type = index_record["TYPE"] & INDEX_TYPE_FLAG[:CLUSTERED] ? :clustered : :secondary + new_index = new_table.indexes.make( + name: index_record["NAME"], + type: type, + table: new_table, + tablespace: tablespace, + root_page_number: index_record["PAGE_NO"], + innodb_index_id: index_record["ID"] + ) + sys_fields.select { |r| r["INDEX_ID"] == index_record["ID"] }.each do |field_record| + new_index.column_references.make(column: new_table.columns.find(name: field_record["COL_NAME"]), + usage: :key, index: new_index) + end + + _populate_index_with_system_and_non_key_columns(new_table, new_index) + end + end + + nil + end + + def populate_data_dictionary + populate_data_dictionary_with_system_table_definitions + populate_data_dictionary_from_system_tables + + nil + end + end +end diff --git a/lib/innodb/system.rb b/lib/innodb/system.rb index b28296e4..6039da85 100644 --- a/lib/innodb/system.rb +++ b/lib/innodb/system.rb @@ -19,7 +19,12 @@ class System # The space ID of the system space, always 0. SYSTEM_SPACE_ID = 0 + # The space ID of the mysql.ibd space, always 4294967294 (2**32-2). + MYSQL_SPACE_ID = 4_294_967_294 + def initialize(arg, data_directory: nil) + @data_dictionary = Innodb::DataDictionary.new + if arg.is_a?(Array) && arg.size > 1 data_filenames = arg else @@ -40,7 +45,24 @@ def initialize(arg, data_directory: nil) add_space_file(data_filenames) - @data_dictionary = Innodb::DataDictionary.new(system_space) + add_mysql_space_file + add_all_ibd_files + + @internal_data_dictionary = if system_space.page(0).prev > 80_000 # ugh + Innodb::SdiDataDictionary.new(self) + else + Innodb::SysDataDictionary.new(self) + end + @internal_data_dictionary.populate_data_dictionary + data_dictionary.refresh + + data_dictionary.tables.each do |table| + add_table(table.name) unless spaces[table.tablespace.innodb_space_id] + end + end + + def data_directory + config[:data_directory] end # A helper to get the system space. @@ -48,18 +70,21 @@ def system_space spaces[SYSTEM_SPACE_ID] end + def mysql_space + spaces[MYSQL_SPACE_ID] + end + # Add an already-constructed Innodb::Space object. def add_space(space) raise "Object was not an Innodb::Space" unless space.is_a?(Innodb::Space) - spaces[space.space_id.to_i] = space + spaces[space.space_id] = space end # Add a space by filename. def add_space_file(space_filenames) - space = Innodb::Space.new(space_filenames) - space.innodb_system = self - add_space(space) + space = Innodb::Space.new(space_filenames, innodb_system: self) + add_space(space) unless spaces[space.space_id] end # Add an orphaned space. @@ -70,7 +95,8 @@ def add_space_orphan(space_file) # Add a space by table name, constructing an appropriate filename # from the provided table name. def add_table(table_name) - space_file = "%s/%s.ibd" % [config[:data_directory], table_name] + space_file = File.join(config[:data_directory], format("%s.ibd", table_name)) + if File.exist?(space_file) add_space_file(space_file) else @@ -83,122 +109,75 @@ def add_table(table_name) def space(space_id) return spaces[space_id] if spaces[space_id] - unless (table_record = data_dictionary.table_by_space_id(space_id)) + unless (table = data_dictionary.tables.find(innodb_space_id: space_id)) raise "Table with space ID #{space_id} not found" end - add_table(table_record["NAME"]) + add_table(table.name) spaces[space_id] end - # Return an Innodb::Space object by table name. def space_by_table_name(table_name) - unless (table_record = data_dictionary.table_by_name(table_name)) - raise "Table #{table_name} not found" - end - - return if table_record["SPACE"].zero? + space_id = data_dictionary.tables.find(name: table_name)&.tablespace&.innodb_space_id - space(table_record["SPACE"]) + spaces[space_id] if space_id end - # Iterate through all table names. - def each_table_name - return enum_for(:each_table_name) unless block_given? - - data_dictionary.each_table do |record| - yield record["NAME"] - end - - nil + def add_mysql_space_file + mysql_ibd = File.join(data_directory, "mysql.ibd") + add_space_file(mysql_ibd) if File.exist?(mysql_ibd) end - # Iterate throught all orphaned spaces. - def each_orphan(&block) - return enum_for(:each_orphan) unless block_given? + # Iterate through all table names. + def each_ibd_file_name(&block) + return enum_for(:each_ibd_file_name) unless block_given? - orphans.each(&block) + Dir.glob(File.join(data_directory, "**/*.ibd")) + .map { |f| f.sub(File.join(data_directory, "/"), "") }.each(&block) nil end - # Iterate through all column names by table name. - def each_column_name_by_table_name(table_name) - return enum_for(:each_column_name_by_table_name, table_name) unless block_given? - - data_dictionary.each_column_by_table_name(table_name) do |record| - yield record["NAME"] + def add_all_ibd_files + each_ibd_file_name do |file_name| + add_space_file(File.join(data_directory, file_name)) end nil end - # Iterate through all index names by table name. - def each_index_name_by_table_name(table_name) - return enum_for(:each_index_name_by_table_name, table_name) unless block_given? + def each_space(&block) + return enum_for(:each_space) unless block_given? - data_dictionary.each_index_by_table_name(table_name) do |record| - yield record["NAME"] - end + spaces.each_value(&block) nil end - # Iterate through all field names in a given index by table name - # and index name. - def each_index_field_name_by_index_name(table_name, index_name) - return enum_for(:each_index_field_name_by_index_name, table_name, index_name) unless block_given? + # Iterate throught all orphaned spaces. + def each_orphan(&block) + return enum_for(:each_orphan) unless block_given? - data_dictionary.each_field_by_index_name(table_name, index_name) do |record| - yield record["COL_NAME"] - end + orphans.each(&block) nil end - # Return the table name given a table ID. - def table_name_by_id(table_id) - data_dictionary.table_by_id(table_id).fetch("NAME", nil) - end - - # Return the index name given an index ID. - def index_name_by_id(index_id) - data_dictionary.index_by_id(index_id).fetch("NAME", nil) - end - - # Return the clustered index name given a table name. - def clustered_index_by_table_name(table_name) - data_dictionary.clustered_index_name_by_table_name(table_name) - end - - # Return an array of the table name and index name given an index ID. - def table_and_index_name_by_id(index_id) - if (dd_index = data_dictionary.data_dictionary_index_ids[index_id]) - # This is a data dictionary index, which won't be found in the data - # dictionary itself. - [dd_index[:table], dd_index[:index]] - elsif (index_record = data_dictionary.index_by_id(index_id)) - # This is a system or user index. - [table_name_by_id(index_record["TABLE_ID"]), index_record["NAME"]] - end - end - # Return an Innodb::Index object given a table name and index name. def index_by_name(table_name, index_name) - index_record = data_dictionary.index_by_name(table_name, index_name) + table = data_dictionary.tables.find(name: table_name) + index = table.indexes.find(name: index_name) - index_space = space(index_record["SPACE"]) - describer = data_dictionary.record_describer_by_index_name(table_name, index_name) - index_space.index(index_record["PAGE_NO"], describer) + space(index.tablespace.innodb_space_id).index(index.root_page_number, index.record_describer) end # Return the clustered index given a table ID. def clustered_index_by_table_id(table_id) - table_name = table_name_by_id(table_id) - return unless table_name + table = data_dictionary.tables.find(innodb_table_id: table_id) + return unless table - index_by_name(table_name, clustered_index_by_table_name(table_name)) + index_by_name(table.name, table.clustered_index.name) end def history diff --git a/spec/data/ib_logfile0 b/spec/data/orphan/ib_logfile0 similarity index 100% rename from spec/data/ib_logfile0 rename to spec/data/orphan/ib_logfile0 diff --git a/spec/data/ib_logfile1 b/spec/data/orphan/ib_logfile1 similarity index 100% rename from spec/data/ib_logfile1 rename to spec/data/orphan/ib_logfile1 diff --git a/spec/data/ibdata1 b/spec/data/orphan/ibdata1 similarity index 100% rename from spec/data/ibdata1 rename to spec/data/orphan/ibdata1 diff --git a/spec/innodb/data_dictionary_spec.rb b/spec/innodb/data_dictionary_spec.rb deleted file mode 100644 index cfd81930..00000000 --- a/spec/innodb/data_dictionary_spec.rb +++ /dev/null @@ -1,207 +0,0 @@ -# frozen_string_literal: true - -require "spec_helper" - -describe Innodb::DataDictionary do - before :all do - @system = Innodb::System.new("spec/data/sakila/compact/ibdata1") - @dict = @system.data_dictionary - end - - describe "#mtype_prtype_to_type_string" do - it "produces the correct type string or symbol" do - type = Innodb::DataDictionary.mtype_prtype_to_type_string(6, 1794, 2, 0) - type.should eql :SMALLINT - end - end - - describe "#mtype_prtype_to_data_type" do - it "produces the correct type array" do - type = Innodb::DataDictionary.mtype_prtype_to_data_type(6, 1794, 2, 0) - type.should be_an_instance_of Array - type.should eql %i[SMALLINT NOT_NULL UNSIGNED] - end - end - - describe "#data_dictionary_indexes" do - it "is a Hash" do - @dict.data_dictionary_indexes.should be_an_instance_of Hash - end - - it "contains Hashes" do - key = @dict.data_dictionary_indexes.keys.first - @dict.data_dictionary_indexes[key].should be_an_instance_of Hash - end - end - - describe "#data_dictionary_index" do - it "returns Innodb::Index objects" do - index = @dict.data_dictionary_index(:SYS_TABLES, :PRIMARY) - index.should be_an_instance_of Innodb::Index - end - end - - describe "#each_data_dictionary_index_root_page_number" do - it "is an enumerator" do - is_enumerator?(@dict.each_data_dictionary_index_root_page_number).should be_truthy - end - end - - describe "#each_data_dictionary_index" do - it "is an enumerator" do - is_enumerator?(@dict.each_data_dictionary_index).should be_truthy - end - end - - describe "#each_record_from_data_dictionary_index" do - it "is an enumerator" do - is_enumerator?(@dict.each_record_from_data_dictionary_index(:SYS_TABLES, :PRIMARY)).should be_truthy - end - end - - describe "#each_table" do - it "is an enumerator" do - is_enumerator?(@dict.each_table).should be_truthy - end - end - - describe "#each_column" do - it "is an enumerator" do - is_enumerator?(@dict.each_column).should be_truthy - end - end - - describe "#each_index" do - it "is an enumerator" do - is_enumerator?(@dict.each_index).should be_truthy - end - end - - describe "#each_field" do - it "is an enumerator" do - is_enumerator?(@dict.each_field).should be_truthy - end - end - - describe "#table_by_id" do - it "finds the correct table" do - table = @dict.table_by_id(19) - table.should be_an_instance_of Hash - table["NAME"].should eql "sakila/film" - end - end - - describe "#table_by_name" do - it "finds the correct table" do - table = @dict.table_by_name("sakila/film") - table.should be_an_instance_of Hash - table["NAME"].should eql "sakila/film" - end - end - - describe "#table_by_space_id" do - it "finds the correct table" do - table = @dict.table_by_space_id(7) - table.should be_an_instance_of Hash - table["NAME"].should eql "sakila/film" - end - end - - describe "#column_by_name" do - it "finds the correct column" do - column = @dict.column_by_name("sakila/film", "film_id") - column.should be_an_instance_of Hash - column["NAME"].should eql "film_id" - end - end - - describe "#index_by_id" do - it "finds the correct index" do - index = @dict.index_by_id(27) - index.should be_an_instance_of Hash - index["NAME"].should eql "PRIMARY" - end - end - - describe "#index_by_name" do - it "finds the correct index" do - index = @dict.index_by_name("sakila/film", "PRIMARY") - index.should be_an_instance_of Hash - index["NAME"].should eql "PRIMARY" - end - end - - describe "#each_index_by_space_id" do - it "is an enumerator" do - is_enumerator?(@dict.each_index_by_space_id(7)).should be_truthy - end - end - - describe "#each_index_by_table_id" do - it "is an enumerator" do - is_enumerator?(@dict.each_index_by_table_id(19)).should be_truthy - end - end - - describe "#each_index_by_table_name" do - it "is an enumerator" do - is_enumerator?(@dict.each_index_by_table_name("sakila/film")).should be_truthy - end - end - - describe "#each_field_by_index_id" do - it "is an enumerator" do - is_enumerator?(@dict.each_field_by_index_id(27)).should be_truthy - end - end - - describe "#each_field_by_index_name" do - it "is an enumerator" do - is_enumerator?(@dict.each_field_by_index_name("sakila/film", "PRIMARY")).should be_truthy - end - end - - describe "#each_column_by_table_id" do - it "is an enumerator" do - is_enumerator?(@dict.each_column_by_table_id(19)).should be_truthy - end - end - - describe "#each_column_by_table_name" do - it "is an enumerator" do - is_enumerator?(@dict.each_column_by_table_name("sakila/film")).should be_truthy - end - end - - describe "#each_column_in_index_by_name" do - it "is an enumerator" do - is_enumerator?(@dict.each_column_in_index_by_name("sakila/film", "PRIMARY")).should be_truthy - end - end - - describe "#each_column_not_in_index_by_name" do - it "is an enumerator" do - is_enumerator?(@dict.each_column_not_in_index_by_name("sakila/film", "PRIMARY")).should be_truthy - end - end - - describe "#each_column_description_by_index_name" do - it "is an enumerator" do - is_enumerator?(@dict.each_column_description_by_index_name("sakila/film", "PRIMARY")).should be_truthy - end - end - - describe "#record_describer_by_index_name" do - it "returns an Innodb::RecordDescriber" do - desc = @dict.record_describer_by_index_name("sakila/film", "PRIMARY") - desc.should be_an_instance_of Innodb::RecordDescriber - end - end - - describe "#record_describer_by_index_id" do - it "returns an Innodb::RecordDescriber" do - desc = @dict.record_describer_by_index_id(27) - desc.should be_an_instance_of Innodb::RecordDescriber - end - end -end diff --git a/spec/innodb/log_block_spec.rb b/spec/innodb/log_block_spec.rb index bf54c022..79830ac7 100644 --- a/spec/innodb/log_block_spec.rb +++ b/spec/innodb/log_block_spec.rb @@ -4,7 +4,7 @@ describe Innodb::LogBlock do before :all do - @log = Innodb::Log.new("spec/data/ib_logfile0") + @log = Innodb::Log.new("spec/data/sakila/compact/ib_logfile0") @block = @log.block(0) end diff --git a/spec/innodb/log_group_spec.rb b/spec/innodb/log_group_spec.rb index d70f3be6..9e0cba1a 100644 --- a/spec/innodb/log_group_spec.rb +++ b/spec/innodb/log_group_spec.rb @@ -5,8 +5,8 @@ describe Innodb::LogGroup do before :all do @log_files = %w[ - spec/data/ib_logfile0 - spec/data/ib_logfile1 + spec/data/sakila/compact/ib_logfile0 + spec/data/sakila/compact/ib_logfile1 ] @log_file_size = 5_242_880 @log_group = Innodb::LogGroup.new(@log_files) diff --git a/spec/innodb/log_reader_spec.rb b/spec/innodb/log_reader_spec.rb index cf1be186..40b044f7 100644 --- a/spec/innodb/log_reader_spec.rb +++ b/spec/innodb/log_reader_spec.rb @@ -4,7 +4,7 @@ describe Innodb::LogReader do before :all do - log_files = ["spec/data/ib_logfile0", "spec/data/ib_logfile1"] + log_files = ["spec/data/sakila/compact/ib_logfile0", "spec/data/sakila/compact/ib_logfile1"] @group = Innodb::LogGroup.new(log_files) @reader = @group.reader end diff --git a/spec/innodb/log_record_spec.rb b/spec/innodb/log_record_spec.rb index c75dded0..1e4475cc 100644 --- a/spec/innodb/log_record_spec.rb +++ b/spec/innodb/log_record_spec.rb @@ -5,8 +5,8 @@ describe Innodb::LogRecord do before :all do log_files = %w[ - spec/data/ib_logfile0 - spec/data/ib_logfile1 + spec/data/sakila/compact/ib_logfile0 + spec/data/sakila/compact/ib_logfile1 ] @group = Innodb::LogGroup.new(log_files) end diff --git a/spec/innodb/log_spec.rb b/spec/innodb/log_spec.rb index b75fb0c2..62cb3796 100644 --- a/spec/innodb/log_spec.rb +++ b/spec/innodb/log_spec.rb @@ -4,7 +4,7 @@ describe Innodb::Log do before :all do - @log = Innodb::Log.new("spec/data/ib_logfile0") + @log = Innodb::Log.new("spec/data/sakila/compact/ib_logfile0") end describe "#new" do @@ -70,27 +70,27 @@ it "has a correct checkpoint_1" do c = @log.checkpoint.checkpoint_1 - c.number.should eql 10 - c.lsn.should eql 1_603_732 - c.lsn_offset.should eql 1_597_588 + c.number.should eql 14 + c.lsn.should eql 8_400_260 + c.lsn_offset.should eql 8_396_164 c.buffer_size.should eql 1_048_576 c.archived_lsn.should eql 18_446_744_073_709_551_615 - c.checksum_1.should eql 654_771_786 - c.checksum_2.should eql 1_113_429_956 - c.fsp_free_limit.should eql 5 + c.checksum_1.should eql 2_424_759_900 + c.checksum_2.should eql 3_026_016_186 + c.fsp_free_limit.should eql 9 c.fsp_magic.should eql 1_441_231_243 # LOG_CHECKPOINT_FSP_MAGIC_N_VAL end it "has a correct checkpoint_2" do c = @log.checkpoint.checkpoint_2 - c.number.should eql 11 - c.lsn.should eql 1_603_732 - c.lsn_offset.should eql 1_597_588 + c.number.should eql 15 + c.lsn.should eql 8_400_260 + c.lsn_offset.should eql 8_396_164 c.buffer_size.should eql 1_048_576 c.archived_lsn.should eql 18_446_744_073_709_551_615 - c.checksum_1.should eql 843_938_123 - c.checksum_2.should eql 674_570_893 - c.fsp_free_limit.should eql 5 + c.checksum_1.should eql 125_194_589 + c.checksum_2.should eql 1_139_538_825 + c.fsp_free_limit.should eql 9 c.fsp_magic.should eql 1_441_231_243 # LOG_CHECKPOINT_FSP_MAGIC_N_VAL end end diff --git a/spec/innodb/page/inode_spec.rb b/spec/innodb/page/inode_spec.rb index 7b963c35..35f26455 100644 --- a/spec/innodb/page/inode_spec.rb +++ b/spec/innodb/page/inode_spec.rb @@ -4,7 +4,7 @@ describe Innodb::Page::Inode do before :all do - @space = Innodb::Space.new("spec/data/ibdata1") + @space = Innodb::Space.new("spec/data/sakila/compact/ibdata1") @page = @space.page(2) end @@ -32,7 +32,7 @@ it "has the right keys and values" do @page.list_entry.size.should eql 2 @page.list_entry[:prev].should eql nil - @page.list_entry[:next].should eql nil + @page.list_entry[:next].should eql Innodb::Page::Address.new(page: 243, offset: 38) end it "has helper functions" do diff --git a/spec/innodb/page/trx_sys_spec.rb b/spec/innodb/page/trx_sys_spec.rb index 72b7928f..46a142a7 100644 --- a/spec/innodb/page/trx_sys_spec.rb +++ b/spec/innodb/page/trx_sys_spec.rb @@ -4,7 +4,7 @@ describe Innodb::Page::TrxSys do before :all do - @space = Innodb::Space.new("spec/data/ibdata1") + @space = Innodb::Space.new("spec/data/sakila/compact/ibdata1") @page = @space.page(5) end diff --git a/spec/innodb/page_spec.rb b/spec/innodb/page_spec.rb index a84f832a..6b03a46f 100644 --- a/spec/innodb/page_spec.rb +++ b/spec/innodb/page_spec.rb @@ -4,7 +4,7 @@ describe Innodb::Page do before :all do - @space = Innodb::Space.new("spec/data/ibdata1") + @space = Innodb::Space.new("spec/data/sakila/compact/ibdata1") @page_data = @space.page_data(0) @page = @space.page(0) end @@ -108,13 +108,13 @@ it "has the right keys and values" do @page.fil_header.size.should eql 8 - @page.fil_header[:checksum].should eql 2_067_631_406 + @page.fil_header[:checksum].should eql 3_774_490_636 @page.fil_header[:offset].should eql 0 @page.fil_header[:prev].should eql 0 @page.fil_header[:next].should eql 0 - @page.fil_header[:lsn].should eql 1_601_269 + @page.fil_header[:lsn].should eql 8_400_049 @page.fil_header[:type].should eql :FSP_HDR - @page.fil_header[:flush_lsn].should eql 1_603_732 + @page.fil_header[:flush_lsn].should eql 8_400_260 @page.fil_header[:space_id].should eql 0 end @@ -130,7 +130,7 @@ describe "#checksum_innodb" do it "calculates the right checksum" do - @page.checksum_innodb.should eql 2_067_631_406 + @page.checksum_innodb.should eql 3_774_490_636 @page.corrupt?.should eql false end end diff --git a/spec/innodb/space_spec.rb b/spec/innodb/space_spec.rb index fbcf5c87..479b6465 100644 --- a/spec/innodb/space_spec.rb +++ b/spec/innodb/space_spec.rb @@ -4,8 +4,8 @@ describe Innodb::Space do before :all do - @space = Innodb::Space.new("spec/data/ibdata1") - @space_ibd = Innodb::Space.new("spec/data/t_empty.ibd") + @space = Innodb::Space.new("spec/data/sakila/compact/ibdata1") + @space_ibd = Innodb::Space.new("spec/data/sakila/compact/sakila/film.ibd") end describe "DEFAULT_PAGE_SIZE" do @@ -134,7 +134,7 @@ end it "iterates through indexes" do - @space_ibd.each_index.to_a.size.should eql 1 + @space_ibd.each_index.to_a.size.should eql 4 end it "yields an Innodb::Index" do @@ -148,7 +148,7 @@ end it "iterates through pages" do - @space_ibd.each_page.to_a.size.should eql 6 + @space_ibd.each_page.to_a.size.should eql 21 end it "yields an Array of [page_number, page]" do diff --git a/spec/innodb/sys_data_dictionary_spec.rb b/spec/innodb/sys_data_dictionary_spec.rb new file mode 100644 index 00000000..aa4bd837 --- /dev/null +++ b/spec/innodb/sys_data_dictionary_spec.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require "spec_helper" + +describe Innodb::SysDataDictionary do + before :all do + @system = Innodb::System.new("spec/data/sakila/compact/ibdata1") + @dict = @system.data_dictionary + end + + describe "#mtype_prtype_to_type_string" do + it "produces the correct type string or symbol" do + type = Innodb::SysDataDictionary.mtype_prtype_to_type_string(6, 1794, 2, 0) + type.should eql :SMALLINT + end + end + + describe "#mtype_prtype_to_data_type" do + it "produces the correct type array" do + type = Innodb::SysDataDictionary.mtype_prtype_to_data_type(6, 1794, 2, 0) + type.should be_an_instance_of Array + type.should eql %i[SMALLINT NOT_NULL UNSIGNED] + end + end + + # TODO: Write new tests for SysDataDictionary +end diff --git a/spec/innodb/system_spec.rb b/spec/innodb/system_spec.rb index 0b84f1a0..2da5928a 100644 --- a/spec/innodb/system_spec.rb +++ b/spec/innodb/system_spec.rb @@ -87,105 +87,6 @@ end end - describe "#each_table_name" do - it "is an enumerator" do - is_enumerator?(@system.each_table_name).should be_truthy - end - - it "iterates all tables in the system" do - expected = %w[ - SYS_FOREIGN - SYS_FOREIGN_COLS - sakila/actor - sakila/address - sakila/category - sakila/city - sakila/country - sakila/customer - sakila/film - sakila/film_actor - sakila/film_category - sakila/inventory - sakila/language - sakila/payment - sakila/rental - sakila/staff - sakila/store - ] - - actual = @system.each_table_name.to_a - result = actual.map { |n| expected.include?(n) }.uniq - result.should eql [true] - end - end - - describe "#each_column_name_by_table_name" do - it "is an enumerator" do - is_enumerator?(@system.each_column_name_by_table_name("sakila/film")).should be_truthy - end - - it "iterates all columns in the table" do - expected = %w[ - film_id - title - description - release_year - language_id - original_language_id - rental_duration - rental_rate - length - replacement_cost - rating - special_features - last_update - ] - - actual = @system.each_column_name_by_table_name("sakila/film").to_a - result = actual.map { |n| expected.include?(n) }.uniq - result.should eql [true] - end - end - - describe "#each_index_name_by_table_name" do - it "is an enumerator" do - is_enumerator?(@system.each_index_name_by_table_name("sakila/film")).should be_truthy - end - - it "iterates all indexes in the table" do - expected = %w[ - PRIMARY - idx_title - idx_fk_language_id - idx_fk_original_language_id - ] - - actual = @system.each_index_name_by_table_name("sakila/film").to_a - result = actual.map { |n| expected.include?(n) }.uniq - result.should eql [true] - end - end - - describe "#table_name_by_id" do - it "returns the correct table name" do - @system.table_name_by_id(19).should eql "sakila/film" - end - end - - describe "#index_name_by_id" do - it "returns the correct index name" do - @system.index_name_by_id(27).should eql "PRIMARY" - end - end - - describe "#table_and_index_name_by_id" do - it "returns the correct table and index name" do - table, index = @system.table_and_index_name_by_id(27) - table.should eql "sakila/film" - index.should eql "PRIMARY" - end - end - describe "#index_by_name" do it "returns an Innodb::Index object" do index = @system.index_by_name("sakila/film", "PRIMARY") @@ -195,7 +96,8 @@ describe "#each_orphan" do before :all do - @system = Innodb::System.new("spec/data/ibdata1") + # Tablespace has a missing tablespace file for test/t_empty. + @system = Innodb::System.new("spec/data/orphan/ibdata1") end it "has an orphan space" do diff --git a/spec/innodb/xdes_spec.rb b/spec/innodb/xdes_spec.rb index 6305992e..c0bcb203 100644 --- a/spec/innodb/xdes_spec.rb +++ b/spec/innodb/xdes_spec.rb @@ -4,7 +4,7 @@ describe Innodb::Xdes do before :all do - @space = Innodb::Space.new("spec/data/ibdata1") + @space = Innodb::Space.new("spec/data/sakila/compact/ibdata1") @page = @space.page(0) @cursor = @page.cursor(@page.pos_xdes_array) @xdes0 = Innodb::Xdes.new(@page, @cursor)