Merge remote-tracking branch 'zweitag/master' into foreign-key-constr…

…aints

VERSION

@@ -1 +1 @@
-0.3.1
+0.3.2

docker-compose.yml

@@ -0,0 +1,9 @@
+version: "3.3"
+
+services:
+  app:
+    image: ruby:2.3-onbuild
+    command: gem build yaml_db
+    working_dir: /usr/src/app
+    volumes:
+      - .:/usr/src/app

lib/serialization_helper.rb

@@ -93,6 +93,7 @@ def self.tables
 
 
     def self.load_table(table, data)
+      return if table == 'ar_internal_metadata'
       column_names = data['columns']
       load_records(table, column_names, data['records'])
       reset_pk_sequence!(table)
@@ -162,28 +163,52 @@ def self.unhash_records(records, keys)
     end
 
     def self.convert_booleans(records, columns)
-      records.each do |record|
+      records.map do |record|
         columns.each do |column|
           next if is_boolean(record[column])
           record[column] = convert_boolean(record[column])
         end
+        record
+      end
+    end
+
+    def self.convert_jsons(records, columns)
+      records.map do |record|
+        columns.each do |column|
+          next if is_json(record[column])
+          record[column] = convert_json(record[column])
+        end
+        record
       end
-      records
     end
 
     def self.convert_boolean(value)
       ['t', '1', true, 1].include?(value)
     end
 
+    def self.convert_json(value)
+      return nil if value.nil?
+      JSON.parse(value)
+    end
+
     def self.boolean_columns(table)
       columns = ActiveRecord::Base.connection.columns(table).reject { |c| silence_warnings { c.type != :boolean } }
       columns.map { |c| c.name }
     end
 
+    def self.json_columns(table)
+      columns = ActiveRecord::Base.connection.columns(table).select { |c| c.sql_type == 'json' }
+      columns.map { |c| c.name }
+    end
+
     def self.is_boolean(value)
       value.kind_of?(TrueClass) or value.kind_of?(FalseClass)
     end
 
+    def self.is_json(value)
+      value.kind_of?(Hash) or value.kind_of?(Array)
+    end
+
     def self.quote_table(table)
       ActiveRecord::Base.connection.quote_table_name(table)
     end
@@ -228,12 +253,14 @@ def self.each_table_page(table, records_per_page=1000)
       pages = (total_count.to_f / records_per_page).ceil - 1
       id = table_column_names(table).first
       boolean_columns = SerializationHelper::Utils.boolean_columns(table)
+      json_columns = SerializationHelper::Utils.json_columns(table)
       quoted_table_name = SerializationHelper::Utils.quote_table(table)
 
       (0..pages).to_a.each_with_index do |page, index|
         query = Arel::Table.new(table).order(id).skip(records_per_page*page).take(records_per_page).project(Arel.sql('*'))
         records = ActiveRecord::Base.connection.select_all(query)
         records = SerializationHelper::Utils.convert_booleans(records, boolean_columns)
+        records = SerializationHelper::Utils.convert_jsons(records, json_columns)
 
         page_types = []
         page_types << :first if index == 0

spec/csv_dump_spec.rb

@@ -6,7 +6,7 @@
 		silence_warnings { ActiveRecord::Base = mock('ActiveRecord::Base', :null_object => true) }
 		ActiveRecord::Base.stub(:connection).and_return(stub('connection').as_null_object)
 		ActiveRecord::Base.connection.stub!(:tables).and_return([ 'mytable', 'schema_info', 'schema_migrations' ])
-		ActiveRecord::Base.connection.stub!(:columns).with('mytable').and_return([ mock('a',:name => 'a', :type => :string), mock('b', :name => 'b', :type => :string) ])
+		ActiveRecord::Base.connection.stub!(:columns).with('mytable').and_return([ mock('a', name: 'a', type: :string, sql_type: 'text'), mock('b', name: 'b', type: :string, sql_type: 'text') ])
 		ActiveRecord::Base.connection.stub!(:select_one).and_return({"count"=>"2"})
 		ActiveRecord::Base.connection.stub!(:select_all).and_return([ { 'a' => 1, 'b' => 2 }, { 'a' => 3, 'b' => 4 } ])
 	end

spec/json_dump_spec.rb

@@ -7,7 +7,7 @@
 		silence_warnings { ActiveRecord::Base = mock('ActiveRecord::Base', :null_object => true) }
 		ActiveRecord::Base.stub(:connection).and_return(stub('connection').as_null_object)
 		ActiveRecord::Base.connection.stub!(:tables).and_return([ 'mytable', 'schema_info', 'schema_migrations' ])
-		ActiveRecord::Base.connection.stub!(:columns).with('mytable').and_return([ mock('a',:name => 'a', :type => :string), mock('b', :name => 'b', :type => :string) ])
+		ActiveRecord::Base.connection.stub!(:columns).with('mytable').and_return([ mock('a', name: 'a', type: :string, sql_type: 'text'), mock('b', name: 'b', type: :string, sql_type: 'text') ])
 		ActiveRecord::Base.connection.stub!(:select_one).and_return({"count"=>"2"})
 		ActiveRecord::Base.connection.stub!(:select_all).and_return([ { 'a' => 1, 'b' => 2 }, { 'a' => 3, 'b' => 4 } ])
 	end
@@ -18,7 +18,7 @@
 	end
 
   it "should dump a valid json document with correct data" do
-		ActiveRecord::Base.connection.stub!(:columns).and_return([ mock('a',:name => 'a', :type => :string), mock('b', :name => 'b', :type => :string) ])
+		ActiveRecord::Base.connection.stub!(:columns).and_return([ mock('a', name: 'a', type: :string, sql_type: 'text'), mock('b', name: 'b', type: :string, sql_type: 'text') ])
 
     JsonDb::Dump.dump(@io)
     @io.rewind
@@ -62,11 +62,20 @@
   end
 
   it 'should dump datetime objects using custom Ruby serialization' do
-		ActiveRecord::Base.connection.stub!(:columns).with('mytable').and_return([ mock('datetime',:name => 'datetime', :type => :datetime)])
+		ActiveRecord::Base.connection.stub!(:columns).with('mytable').and_return([ mock('datetime', name: 'datetime', type: :datetime, sql_type: 'datetime')])
 		ActiveRecord::Base.connection.stub!(:select_one).and_return({"count"=>"1"})
 		ActiveRecord::Base.connection.stub!(:select_all).and_return([ { 'datetime' => DateTime.new(2014, 1, 1, 12, 20, 00) } ])
 		JsonDb::Dump.dump_table_records(@io, 'mytable')
 		@io.rewind
 		@io.read.should == '"records": [ [{"json_class":"DateTime","y":2014,"m":1,"d":1,"H":12,"M":20,"S":0,"of":"0/1","sg":2299161.0}] ]'
   end
+
+  it 'should correctly serialize json columns' do
+    ActiveRecord::Base.connection.stub!(:columns).with('mytable').and_return([ mock('json', name: 'json', type: :json, sql_type: 'json')])
+    ActiveRecord::Base.connection.stub!(:select_one).and_return({"count"=>"1"})
+    ActiveRecord::Base.connection.stub!(:select_all).and_return([ { 'json' => '[{"a":1},{"b":2}]' } ])
+    JsonDb::Dump.dump_table_records(@io, 'mytable')
+    @io.rewind
+    @io.read.should == '"records": [ [[{"a":1},{"b":2}]] ]'
+  end
 end

spec/serialization_helper_dump_spec.rb

@@ -2,64 +2,71 @@
 
 describe SerializationHelper::Dump do
 
-	before do
-		silence_warnings { ActiveRecord::Base = mock('ActiveRecord::Base', :null_object => true) }
-		ActiveRecord::Base.stub(:connection).and_return(stub('connection').as_null_object)
-		ActiveRecord::Base.connection.stub!(:tables).and_return([ 'mytable', 'schema_info', 'schema_migrations' ])
-		ActiveRecord::Base.connection.stub!(:columns).with('mytable').and_return([ mock('a', :name => 'a', :type => :string), mock('b', :name => 'b', :type => :string) ])
-		ActiveRecord::Base.connection.stub!(:select_one).and_return({"count"=>"2"})
-		ActiveRecord::Base.connection.stub!(:select_all).and_return([ { 'a' => 1, 'b' => 2 }, { 'a' => 3, 'b' => 4 } ])
-		SerializationHelper::Utils.stub!(:quote_table).with('mytable').and_return('mytable')
-	end
+  before do
+    silence_warnings { ActiveRecord::Base = mock('ActiveRecord::Base', :null_object => true) }
+    ActiveRecord::Base.stub(:connection).and_return(stub('connection').as_null_object)
+    ActiveRecord::Base.connection.stub!(:tables).and_return([ 'mytable', 'schema_info', 'schema_migrations' ])
+    ActiveRecord::Base.connection.stub!(:columns).with('mytable').and_return([
+      mock('a', name: 'a', type: :string, sql_type: 'text'),
+      mock('b', name: 'b', type: :string, sql_type: 'text'),
+      mock('json', name: 'json', type: :json, sql_type: 'json')
+    ])
+    ActiveRecord::Base.connection.stub!(:select_one).and_return({"count"=>"2"})
+    ActiveRecord::Base.connection.stub!(:select_all).and_return([
+      { 'a' => 1, 'b' => 2, 'json' => '[{"c": 3}]' },
+      { 'a' => 3, 'b' => 4, 'json' => nil }
+    ])
+    SerializationHelper::Utils.stub!(:quote_table).with('mytable').and_return('mytable')
+  end
 
-	before(:each) do   
-	  File.stub!(:new).with('dump.yml', 'w').and_return(StringIO.new)
-	  @io = StringIO.new
-	end
+  before(:each) do
+    File.stub!(:new).with('dump.yml', 'w').and_return(StringIO.new)
+    @io = StringIO.new
+  end
 
-	it "should return a list of column names" do
-		SerializationHelper::Dump.table_column_names('mytable').should == [ 'a', 'b' ]
-	end
+  it "should return a list of column names" do
+    SerializationHelper::Dump.table_column_names('mytable').should == [ 'a', 'b', 'json' ]
+  end
 
-	it "should return a list of tables including the rails schema table" do
-		SerializationHelper::Dump.tables.should == ['mytable', 'schema_info', 'schema_migrations']
-	end
+  it "should return a list of tables including the rails schema table" do
+    SerializationHelper::Dump.tables.should == ['mytable', 'schema_info', 'schema_migrations']
+  end
 
-	it "should return the total number of records in a table" do
-		SerializationHelper::Dump.table_record_count('mytable').should == 2
-	end
+  it "should return the total number of records in a table" do
+    SerializationHelper::Dump.table_record_count('mytable').should == 2
+  end
 
-	it "should return all records from the database and return them when there is only 1 page" do
-		SerializationHelper::Dump.each_table_page('mytable') do |records|
-			records.should == [ { 'a' => 1, 'b' => 2 }, { 'a' => 3, 'b' => 4 } ]
-		end
-	end
-
-	it "should paginate records from the database and return them" do
-		ActiveRecord::Base.connection.stub!(:select_all).and_return([ { 'a' => 1, 'b' => 2 } ], [ { 'a' => 3, 'b' => 4 } ])
-
-		records = [ ]
-		SerializationHelper::Dump.each_table_page('mytable', 1) do |page|
-			page.size.should == 1
-			records.concat(page)
-		end
-
-		records.should == [ { 'a' => 1, 'b' => 2 }, { 'a' => 3, 'b' => 4 } ]
-	end
+  it "should return all records from the database and return them when there is only 1 page" do
+    SerializationHelper::Dump.each_table_page('mytable') do |records|
+      records.should == [ { 'a' => 1, 'b' => 2, 'json' => [{'c' => 3}] }, { 'a' => 3, 'b' => 4, 'json' => nil } ]
+    end
+  end
 
-	it "should dump a table's contents to yaml" do
-		SerializationHelper::Dump.should_receive(:dump_table_columns)
-		SerializationHelper::Dump.should_receive(:dump_table_records)
-		SerializationHelper::Dump.dump_table(@io, 'mytable')
-	end
+  it "should paginate records from the database and return them" do
+    ActiveRecord::Base.connection.stub!(:select_all).and_return(
+      [ { 'a' => 1, 'b' => 2, 'json' => '[{"c": 3}]' } ],
+      [ { 'a' => 3, 'b' => 4, 'json' => nil } ]
+    )
 
-	it "should not dump a table's contents when the record count is zero" do
-		SerializationHelper::Dump.stub!(:table_record_count).with('mytable').and_return(0)
-		SerializationHelper::Dump.should_not_receive(:dump_table_columns)
-		SerializationHelper::Dump.should_not_receive(:dump_table_records)
-		SerializationHelper::Dump.dump_table(@io, 'mytable')
+    records = [ ]
+    SerializationHelper::Dump.each_table_page('mytable', 1) do |page|
+      page.size.should == 1
+      records.concat(page)
     end
 
+    records.should == [ { 'a' => 1, 'b' => 2, 'json' => [{'c' => 3}] }, { 'a' => 3, 'b' => 4, 'json' => nil } ]
+  end
 
+  it "should dump a table's contents to yaml" do
+    SerializationHelper::Dump.should_receive(:dump_table_columns)
+    SerializationHelper::Dump.should_receive(:dump_table_records)
+    SerializationHelper::Dump.dump_table(@io, 'mytable')
+  end
 
+  it "should not dump a table's contents when the record count is zero" do
+    SerializationHelper::Dump.stub!(:table_record_count).with('mytable').and_return(0)
+    SerializationHelper::Dump.should_not_receive(:dump_table_columns)
+    SerializationHelper::Dump.should_not_receive(:dump_table_records)
+    SerializationHelper::Dump.dump_table(@io, 'mytable')
+  end
 end

spec/yaml_dump_spec.rb

@@ -6,7 +6,7 @@
 		silence_warnings { ActiveRecord::Base = mock('ActiveRecord::Base', :null_object => true) }
 		ActiveRecord::Base.stub(:connection).and_return(stub('connection').as_null_object)
 		ActiveRecord::Base.connection.stub!(:tables).and_return([ 'mytable', 'schema_info', 'schema_migrations' ])
-		ActiveRecord::Base.connection.stub!(:columns).with('mytable').and_return([ mock('a',:name => 'a', :type => :string), mock('b', :name => 'b', :type => :string) ])
+		ActiveRecord::Base.connection.stub!(:columns).with('mytable').and_return([ mock('a', name: 'a', type: :string, sql_type: 'text'), mock('b', name: 'b', type: :string, sql_type: 'text') ])
 		ActiveRecord::Base.connection.stub!(:select_one).and_return({"count"=>"2"})
 		ActiveRecord::Base.connection.stub!(:select_all).and_return([ { 'a' => 1, 'b' => 2 }, { 'a' => 3, 'b' => 4 } ])
 		YamlDb::Utils.stub!(:quote_table).with('mytable').and_return('mytable')

yaml_db.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "yaml_db_with_schema_tables"
-  s.version = "0.3.1"
+  s.version = "0.3.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Adam Wiggins", "Orion Henry", "Martin Honermeyer"]
-  s.date = "2014-01-30"
+  s.date = "2017-12-04"
   s.description = "\nYamlDb is a database-independent format for dumping and restoring data.  It complements the the database-independent schema format found in db/schema.rb.  The data is saved into db/data.yml.\nThis can be used as a replacement for mysqldump or pg_dump, but only for the databases typically used by Rails apps.  Users, permissions, schemas, triggers, and other advanced database features are not supported - by design.\nAny database that has an ActiveRecord adapter should work\n"
   s.email = "[email protected]"
   s.extra_rdoc_files = [