From a2282a0276f78a15ada0f42120e36833794860bd Mon Sep 17 00:00:00 2001 From: James Smith Date: Sat, 17 Apr 2021 00:20:05 +0100 Subject: [PATCH 01/12] use decribed_class in model specs so that when we change names, there's less to do --- spec/models/package_processor_spec.rb | 12 +++--- spec/models/package_spec.rb | 34 +++++++-------- spec/models/schema_processor_spec.rb | 8 ++-- spec/models/summary_spec.rb | 44 +++++++++---------- spec/models/validation_spec.rb | 62 +++++++++++++-------------- 5 files changed, 80 insertions(+), 80 deletions(-) diff --git a/spec/models/package_processor_spec.rb b/spec/models/package_processor_spec.rb index dcf680bc..98156add 100644 --- a/spec/models/package_processor_spec.rb +++ b/spec/models/package_processor_spec.rb @@ -12,7 +12,7 @@ it "creates a package from a url" do mock_file("http://example.com/test.csv", 'csvs/valid.csv') - processor = PackageProcessor.new({ + processor = described_class.new({ urls: ['http://example.com/test.csv'] }, @package.id) processor.process @@ -21,7 +21,7 @@ end it "creates a package from a data url" do - processor = PackageProcessor.new({ + processor = described_class.new({ files_data: create_data_uri('csvs/valid.csv') }, @package.id) processor.process @@ -30,7 +30,7 @@ end it "creates a package from an uploaded file" do - processor = PackageProcessor.new({ + processor = described_class.new({ file_ids: [ mock_upload('valid.csv') ] @@ -41,7 +41,7 @@ end it "creates a package from a vanilla file upload" do - processor = PackageProcessor.new({ + processor = described_class.new({ files: [ mock_uploaded_file('csvs/valid.csv') ] @@ -53,7 +53,7 @@ it "creates a package from a zipped url" do mock_file("http://example.com/valid.zip", 'csvs/valid.zip') - processor = PackageProcessor.new({ + processor = described_class.new({ urls: ['http://example.com/valid.zip'] }, @package.id) processor.process @@ -66,7 +66,7 @@ StoredChunk.save('chunked_file', i.to_s, i) end - processor = PackageProcessor.new({ + processor = described_class.new({ file_ids: [ 'chunked_file,10' ] diff --git a/spec/models/package_spec.rb b/spec/models/package_spec.rb index 1768b704..801ddd84 100644 --- a/spec/models/package_spec.rb +++ b/spec/models/package_spec.rb @@ -12,7 +12,7 @@ it "creates a single validation" do mock_file("http://example.org/valid.csv", 'csvs/valid.csv') - package = Package.new + package = described_class.new package = package.create_package(['http://example.org/valid.csv']) expect(package.validations.length).to eq(1) end @@ -33,13 +33,13 @@ end it "creates multiple validations" do - package = Package.new + package = described_class.new package = package.create_package(@urls) expect(package.validations.length).to eq(4) end it "sets the right type" do - package = Package.new + package = described_class.new package = package.create_package(@urls) expect(package.type).to eq("urls") end @@ -49,7 +49,7 @@ mock_file(schema_url, 'schemas/valid.json', 'application/javascript') schema = Csvlint::Schema.load_from_uri(schema_url) - package = Package.new + package = described_class.new package = package.create_package(@urls, schema_url, schema) package.validations.each do |validation| @@ -72,13 +72,13 @@ end it "creates multiple validations" do - package = Package.new + package = described_class.new package = package.create_package(@files) expect(package.validations.length).to eq(4) end it "sets the right type" do - package = Package.new + package = described_class.new package = package.create_package(@files) expect(package.type).to eq("files") end @@ -88,7 +88,7 @@ mock_file(schema_url, 'schemas/valid.json', 'application/javascript') schema = Csvlint::Schema.load_from_uri(schema_url) - package = Package.new + package = described_class.new package = package.create_package(@files, schema_url, schema) package.validations.each do |validation| @@ -107,7 +107,7 @@ mock_file(url, 'datapackages/single-datapackage.json', 'application/javascript') mock_file("http://example.org/valid.csv", 'csvs/valid.csv') - package = Package.new + package = described_class.new package = package.create_package([url]) dataset = DataKitten::Dataset.new(access_url: url) package_dataset = Marshal.load(package.dataset) @@ -127,7 +127,7 @@ mock_file("http://example.org/valid.csv", 'csvs/valid.csv') mock_file("http://example.org/valid2.csv", 'csvs/valid.csv') - package = Package.new + package = described_class.new package = package.create_package([url]) expect(package.validations.length).to eq(2) @@ -138,7 +138,7 @@ mock_file(url, 'datapackages/datapackage-with-schema.json', 'application/javascript') mock_file("http://example.org/all_constraints.csv", 'csvs/all_constraints.csv') - package = Package.new + package = described_class.new package = package.create_package([url]) result = Marshal.load package.validations.first.result @@ -160,7 +160,7 @@ mock_file(url, 'datapackages/non-csv-datapackage.json', 'application/javascript') mock_file("http://example.org/some-json.json", 'datapackages/non-csv-datapackage.json') - package = Package.new + package = described_class.new package = package.create_package([url]) expect(package).to eq(nil) @@ -172,7 +172,7 @@ mock_file("http://example.org/some-json.json", 'csvs/valid.csv') mock_file("http://example.org/valid.csv", 'csvs/valid.csv') - package = Package.new + package = described_class.new package = package.create_package([url]) expect(package.validations.length).to eq(1) @@ -186,7 +186,7 @@ files = [ mock_uploaded_file('datapackages/single-datapackage.json') ] mock_file("http://example.org/valid.csv", 'csvs/valid.csv') - package = Package.new + package = described_class.new package = package.create_package(files) expect(package.validations.length).to eq(1) @@ -196,7 +196,7 @@ files = [ mock_uploaded_file('datapackages/local-and-remote-datapackage.json') ] mock_file("http://example.org/valid.csv", 'csvs/valid.csv') - package = Package.new + package = described_class.new package = package.create_package(files) expect(package.validations.length).to eq(1) @@ -216,7 +216,7 @@ it "creates a validation for a CKAN package with a single CSV", :vcr do url = 'http://data.gov.uk/dataset/uk-open-access-non-vosa-sites' - package = Package.new + package = described_class.new package = package.create_package([url]) dataset = DataKitten::Dataset.new(access_url: url) package_dataset = Marshal.load(package.dataset) @@ -233,7 +233,7 @@ it "creates multiple validations for a datapackage with multiple CSVs", :vcr do url = 'http://data.gov.uk/dataset/uk-civil-service-high-earners' - package = Package.new + package = described_class.new package = package.create_package([url]) expect(package.validations.length).to eq(4) @@ -242,7 +242,7 @@ it "returns nil if there are no CSVs", :vcr do url = 'http://data.gov.uk/dataset/ratio-of-median-house-price-to-median-earnings' - package = Package.new + package = described_class.new package = package.create_package([url]) expect(package).to eq(nil) end diff --git a/spec/models/schema_processor_spec.rb b/spec/models/schema_processor_spec.rb index c9851f08..aa6e3e53 100644 --- a/spec/models/schema_processor_spec.rb +++ b/spec/models/schema_processor_spec.rb @@ -11,7 +11,7 @@ schema_url = "http://example.org/schema.json" mock_file(schema_url, @schema, 'application/javascript') - schema = SchemaProcessor.new(url: schema_url).schema + schema = described_class.new(url: schema_url).schema expect(schema.fields.count).to eq(3) expect(schema.fields[0].name).to eq("FirstName") @@ -21,7 +21,7 @@ it "reads a schema from a data URI" do schema_data = create_data_uri(@schema, 'application/json') - schema = SchemaProcessor.new(data: schema_data).schema + schema = described_class.new(data: schema_data).schema expect(schema.fields.count).to eq(3) expect(schema.fields[0].name).to eq("FirstName") @@ -31,7 +31,7 @@ it "reads a schema from a file" do file = mock_uploaded_file(@schema, 'application/json') - schema = SchemaProcessor.new(file: file).schema + schema = described_class.new(file: file).schema expect(schema.fields.count).to eq(3) expect(schema.fields[0].name).to eq("FirstName") @@ -42,7 +42,7 @@ context "with datapackage" do before(:each) do @file = mock_uploaded_file('datapackages/datapackage-with-schema.json', 'application/json') - @schema = SchemaProcessor.new(file: @file) + @schema = described_class.new(file: @file) end it "detects a datapackage" do diff --git a/spec/models/summary_spec.rb b/spec/models/summary_spec.rb index 26a21bc7..4f5201e7 100644 --- a/spec/models/summary_spec.rb +++ b/spec/models/summary_spec.rb @@ -1,21 +1,21 @@ require "spec_helper" describe Summary, type: :model do - + context "when generating summary" do before(:each) do mock_file("http://example.com/test.csv", 'csvs/valid.csv') validation = Validation.create_validation("http://example.com/test.csv") - + mock_file("http://example.com/test2.csv", 'csvs/errors.csv') validation = Validation.create_validation("http://example.com/test2.csv") - + mock_file("http://www.example.com/test3.csv", 'csvs/warnings.csv') validation = Validation.create_validation("http://www.example.com/test3.csv") - - @summary = Summary.generate + + @summary = described_class.generate end - + it "should record number of valid sources" do expect(@summary.states["valid"]).to eq(1) end @@ -23,48 +23,48 @@ it "should record number of invalid sources" do expect(@summary.states["invalid"]).to eq(1) end - + it "should record number of not found sources" do expect(@summary.states["not_found"]).to eq(0) - end + end it "should record number of sources with warnings" do expect(@summary.states["warnings"]).to eq(1) - end - + end + it "should record number of unique sources" do expect(@summary.sources).to eq(3) end - + it "should record number of sources per host" do expect(@summary.hosts.keys.length).to eq(1) expect(@summary.hosts["example\uff0ecom"]).to eq(3) - end - + end + it "should record occurences of errors" do expect(@summary.level_summary.errors_breakdown[:ragged_rows]).to eq(1) expect(@summary.level_summary.warnings_breakdown[:check_options]).to eq(1) end - + it "should count occurences across population of sources" do mock_file("http://example.com/test4.csv", 'csvs/multiple_errors.csv') - validation = Validation.create_validation("http://example.com/test4.csv") - @summary = Summary.generate + validation = Validation.create_validation("http://example.com/test4.csv") + @summary = described_class.generate expect(@summary.sources).to eq(4) expect(@summary.states["invalid"]).to eq(2) expect(@summary.level_summary.errors_breakdown[:ragged_rows]).to eq(2) expect(@summary.level_summary.warnings_breakdown[:check_options]).to eq(1) end - + it "should record categories of problem" do mock_file("http://example.com/test4.csv", 'csvs/multiple_errors.csv') validation = Validation.create_validation("http://example.com/test4.csv") - @summary = Summary.generate - expect(@summary.sources).to eq(4) + @summary = described_class.generate + expect(@summary.sources).to eq(4) expect(@summary.level_summary.errors_breakdown[:ragged_rows]).to eq(2) expect(@summary.category_summary.structure_breakdown[:ragged_rows]).to eq(2) end - + end - -end \ No newline at end of file + +end diff --git a/spec/models/validation_spec.rb b/spec/models/validation_spec.rb index 8c21395a..00e8eb52 100644 --- a/spec/models/validation_spec.rb +++ b/spec/models/validation_spec.rb @@ -6,21 +6,21 @@ describe '#expiry_fields' do it "should assign a TTL field to any validation formed from an uploaded CSV file" do @file = mock_uploaded_file('csvs/valid.csv') - validation = Validation.create_validation(@file) + validation = described_class.create_validation(@file) expect(validation.expirable_created_at).not_to eq(nil) end it "should assign the TTL field as a Mongoid index" do @file = mock_uploaded_file('csvs/valid.csv') - validation = Validation.create_validation(@file) + validation = described_class.create_validation(@file) # retrieve the mongo collection associated with above created file, ensure that the expirable field is present expect(validation.collection.indexes.get(expirable_created_at: 1).present?).to eq(true) end it "should have an expiry value of 24 hours" do @file = mock_uploaded_file('csvs/valid.csv') - validation = Validation.create_validation(@file) + validation = described_class.create_validation(@file) # retrieve the mongo collection associated with above created file, ensure that the expirable field is set to 24 hours validation.collection.indexes.get(expirable_created_at: 1).select{|k,v| k=="expireAfterSeconds"}.has_value?(24.hours.to_i) end @@ -29,13 +29,13 @@ it "should not assign a TTL field to any validation formed from a hyperlinked CSV file" do mock_file("http://example.com/test.csv", 'csvs/valid.csv') - validation = Validation.create_validation("http://example.com/test.csv") + validation = described_class.create_validation("http://example.com/test.csv") expect(validation.expirable_created_at).to eq(nil) end it "should recheck validations after two hours" do mock_file("http://example.com/test.csv", 'csvs/valid.csv') - validation = Validation.create_validation("http://example.com/test.csv") + validation = described_class.create_validation("http://example.com/test.csv") expect(validation.state).to eq("valid") mock_file("http://example.com/test.csv", 'csvs/errors.csv') Timecop.freeze(DateTime.now + 2.hours) do @@ -47,50 +47,50 @@ it "should not requeue a validation for images if revalidate is set to false" do mock_file("http://example.com/test.csv", 'csvs/valid.csv') - validation = Validation.create_validation("http://example.com/test.csv") - expect_any_instance_of(Validation).not_to receive(:delay) - Validation.fetch_validation(validation.id, "png", false) + validation = described_class.create_validation("http://example.com/test.csv") + expect_any_instance_of(described_class).not_to receive(:delay) + described_class.fetch_validation(validation.id, "png", false) end it "should not requeue a validation for images if revalidate is set to false as a string" do mock_file("http://example.com/test.csv", 'csvs/valid.csv') - validation = Validation.create_validation("http://example.com/test.csv") - expect_any_instance_of(Validation).not_to receive(:delay) - Validation.fetch_validation(validation.id, "png", "false") + validation = described_class.create_validation("http://example.com/test.csv") + expect_any_instance_of(described_class).not_to receive(:delay) + described_class.fetch_validation(validation.id, "png", "false") end it "should not revalidate for html if revalidate is set to false" do mock_file("http://example.com/test.csv", 'csvs/valid.csv') - validation = Validation.create_validation("http://example.com/test.csv") - expect_any_instance_of(Validation).not_to receive(:check_validation) - Validation.fetch_validation(validation.id, "html", false) + validation = described_class.create_validation("http://example.com/test.csv") + expect_any_instance_of(described_class).not_to receive(:check_validation) + described_class.fetch_validation(validation.id, "html", false) end it "should not revalidate for html if revalidate is set to false as a string" do mock_file("http://example.com/test.csv", 'csvs/valid.csv') - validation = Validation.create_validation("http://example.com/test.csv") - expect_any_instance_of(Validation).not_to receive(:check_validation) - Validation.fetch_validation(validation.id, "html", "false") + validation = described_class.create_validation("http://example.com/test.csv") + expect_any_instance_of(described_class).not_to receive(:check_validation) + described_class.fetch_validation(validation.id, "html", "false") end it "should only create one validation per url" do mock_file("http://example.com/test.csv", 'csvs/valid.csv') - validation = Validation.create_validation("http://example.com/test.csv") - expect(Validation.count).to eq(1) + validation = described_class.create_validation("http://example.com/test.csv") + expect(described_class.count).to eq(1) - validation = Validation.create_validation("http://example.com/test.csv") - expect(Validation.count).to eq(1) + validation = described_class.create_validation("http://example.com/test.csv") + expect(described_class.count).to eq(1) end it "parse options should be created with validation" do @file = mock_uploaded_file('csvs/valid.csv') - validation = Validation.create_validation(@file) + validation = described_class.create_validation(@file) expect(validation.parse_options).not_to eq(nil) end it "should generate parse options for older validations" do @file = mock_uploaded_file('csvs/valid.csv') - validation = Validation.create_validation(@file) + validation = described_class.create_validation(@file) validation.parse_options = nil validation.save expect(validation.parse_options).not_to eq(nil) @@ -99,15 +99,15 @@ it "should clean up old validations without urls" do @file = mock_uploaded_file('csvs/valid.csv') - 5.times { Validation.create_validation(@file) } + 5.times { described_class.create_validation(@file) } Timecop.freeze(25.hours.from_now) - 7.times { Validation.create_validation(@file) } + 7.times { described_class.create_validation(@file) } - Validation.clean_up(24) + described_class.clean_up(24) - expect(Validation.count).to eq(7) + expect(described_class.count).to eq(7) Timecop.return end @@ -116,18 +116,18 @@ @file = mock_uploaded_file('csvs/valid.csv') mock_file('http://example.com/test.csv', 'csvs/valid.csv') - 5.times { Validation.create_validation(@file) } + 5.times { described_class.create_validation(@file) } 2.times do |i| url = "http://example.com/test#{i}.csv" mock_file(url, 'csvs/valid.csv') - Validation.create_validation(url) + described_class.create_validation(url) end Timecop.freeze(25.hours.from_now) - Validation.clean_up(24) + described_class.clean_up(24) - expect(Validation.count).to eq(2) + expect(described_class.count).to eq(2) Timecop.return end From 9cdf87160299e4ba8808f199d813cc8bc2493dbd Mon Sep 17 00:00:00 2001 From: James Smith Date: Sat, 17 Apr 2021 00:31:03 +0100 Subject: [PATCH 02/12] Move mongoDB models to legacy namespace and create delegators --- app/models/legacy/package.rb | 138 +++++++++++++++++ app/models/legacy/schema.rb | 14 ++ app/models/legacy/summary.rb | 76 ++++++++++ app/models/legacy/validation.rb | 258 ++++++++++++++++++++++++++++++++ app/models/package.rb | 137 +---------------- app/models/schema.rb | 14 +- app/models/summary.rb | 75 +--------- app/models/validation.rb | 257 +------------------------------ 8 files changed, 490 insertions(+), 479 deletions(-) create mode 100644 app/models/legacy/package.rb create mode 100644 app/models/legacy/schema.rb create mode 100644 app/models/legacy/summary.rb create mode 100644 app/models/legacy/validation.rb diff --git a/app/models/legacy/package.rb b/app/models/legacy/package.rb new file mode 100644 index 00000000..f2ba7bb6 --- /dev/null +++ b/app/models/legacy/package.rb @@ -0,0 +1,138 @@ +require 'package_processor' +require 'lite_kitten/lite_kitten' + +class LocalDataset < DataKitten::Dataset + extend DataKitten::PublishingFormats::Datapackage + + def origin + :local + end + + def publishing_format + # A 'package' can be a DataPackage, or it can be any collection of multiple CSVs, + # and/or collection of CSVs and their schema metadata + # ?? is this used in conjunction with package_helper + :datapackage + end +end + +class RemoteDataset < DataKitten::Dataset + + def initialize(url) + @access_url = url + detect_publishing_format + end + + def detect_publishing_format + [ + DataKitten::PublishingFormats::Datapackage, + DataKitten::PublishingFormats::CKAN + ].each do |format| + if format.supported?(self) + extend format + break + end + end + end +end + +class Legacy::Package + include Mongoid::Document + store_in collection: "packages" + include Mongoid::Timestamps + + field :url, type: String + field :dataset, type: String + field :type, type: String + + has_many :validations + + def parse_package(dataset, validations) + attributes = { + :url => dataset.origin == :local ? nil : dataset.access_url, + :dataset => Marshal.dump(dataset), + :validations => validations, + :type => dataset.publishing_format + } + + return attributes + end + + def self.process(id, params) + PackageProcessor.new(params, id).process + end + + def create_package(sources, schema_url = nil, schema = nil) + return nil if sources.count == 0 + + if sources.count == 1 && possible_package?(sources.first) + dataset = create_dataset(sources.first) + return create_datapackage(dataset) unless dataset.nil? + end + + update_attributes({ type: set_type(sources) }) + + sources.each do |source| + validations << Validation.create_validation(source, schema_url, schema) + end + + save + self + end + + def create_dataset(source) + if source.respond_to?(:body) + dataset = LocalDataset.new(access_url: source.string_io) + else + dataset = RemoteDataset.new(source) + end + return nil unless [:ckan, :datapackage].include? dataset.publishing_format + dataset + end + + def create_datapackage(dataset) + validations = create_validations(dataset) + + return nil if validations.count == 0 + + update_attributes( parse_package(dataset, validations) ) + save + self + end + + def create_validations(dataset) + validations = [] + dataset.distributions.each do |distribution| + if can_validate?(distribution) + validations << Validation.create_validation(distribution.access_url, nil, create_schema(distribution) ) + end + end + validations + end + + def possible_package?(source) + source.class == String || local_package?( source ) + end + + def local_package?(source) + source.respond_to?(:string_io) && source.filename =~ /datapackage\.json/ + end + + def set_type(sources) + return "files" if sources.first.respond_to?(:tempfile) + return "urls" if sources.first.class == String + end + + def can_validate?(distribution) + return false unless distribution.format.extension == :csv + return distribution.access_url && distribution.access_url =~ /^http(s?)/ + end + + def create_schema(distribution) + unless distribution.schema.nil? + schema = Csvlint::Schema.from_json_table(nil, distribution.schema) + end + return schema + end + +end diff --git a/app/models/legacy/schema.rb b/app/models/legacy/schema.rb new file mode 100644 index 00000000..1c31b266 --- /dev/null +++ b/app/models/legacy/schema.rb @@ -0,0 +1,14 @@ +class Legacy::Schema + include Mongoid::Document + store_in collection: "schemas" + include Mongoid::Timestamps + + field :url, type: String + + has_many :validations + + def to_param + id.to_s + end + +end diff --git a/app/models/legacy/summary.rb b/app/models/legacy/summary.rb new file mode 100644 index 00000000..88bf9718 --- /dev/null +++ b/app/models/legacy/summary.rb @@ -0,0 +1,76 @@ +class LevelSummary + include Mongoid::Document + embedded_in :summary + field :errors_breakdown, type: Hash + field :warnings_breakdown, type: Hash + field :info_messages_breakdown, type: Hash +end + +class CategorySummary + include Mongoid::Document + embedded_in :summary + field :structure_breakdown, type: Hash + field :schema_breakdown, type: Hash + field :context_breakdown, type: Hash +end + +class Legacy::Summary + include Mongoid::Document + store_in collection: "summaries" + include Mongoid::Timestamps + + field :sources, type: Integer + field :states, type: Hash + field :hosts, type: Hash + + embeds_one :level_summary + embeds_one :category_summary + + def self.generate + summary = Summary.create + + validations = Validation.where(:url.ne => nil).order_by(:created_at.desc) + # retrieve validations from Mongo Datastore, ordered in reverse by date created + + summary.sources = validations.length + summary.states = {"warnings"=>0, "valid"=>0, "not_found"=>0, "invalid"=>0} + summary.hosts = Hash.new 0 + summary.create_level_summary( errors_breakdown: Hash.new(0), warnings_breakdown: Hash.new(0), info_messages_breakdown: Hash.new(0) ) + summary.create_category_summary( structure_breakdown: Hash.new(0), schema_breakdown: Hash.new(0), context_breakdown: Hash.new(0) ) + + validations.each do |validation| + summary.states[validation.state] += 1 + host = source_host(validation.url) + summary.hosts[host] += 1 unless host.nil? + validator = validation.validator + messages = [] + [:errors, :warnings, :info_messages].each do |level| + unless validator.send(level).nil? + messages = messages + validator.send(level) + validator.send(level).uniq { |m| m.type }.each do |msg| + summary.level_summary.send("#{level}_breakdown".to_sym)[ msg.type ] += 1 + end + end + end + [:structure, :schema, :context].each do |category| + messages.reject {|m| m.category != category }.uniq { |m| m.type }.each do |msg| + summary.category_summary.send("#{category}_breakdown".to_sym)[ msg.type ] += 1 + end + end + end + summary.save + summary + end + + private + + def self.source_host(url) + host = URI.parse(url.to_s).host + return if host.nil? + host.downcase! + host = host.start_with?('www.') ? host[4..-1] : host + #TODO better option? + host.gsub(".", "\uff0e") + end + +end diff --git a/app/models/legacy/validation.rb b/app/models/legacy/validation.rb new file mode 100644 index 00000000..7bc5403b --- /dev/null +++ b/app/models/legacy/validation.rb @@ -0,0 +1,258 @@ +class Legacy::Validation + include Mongoid::Document + store_in collection: "validations" + include Mongoid::Timestamps + + field :filename, type: String + field :url, type: String + field :state, type: String + field :result, type: String + field :csv_id, type: String + field :parse_options, type: Hash + field :expirable_created_at, type: Time + + index :created_at => 1 + index({expirable_created_at: 1}, {expire_after_seconds: 24.hours}) + # invoke the mongo time-to-live feature which will automatically expire entries + # - this index is only enabled for a subset of validations, which are validations uploaded as file + + belongs_to :schema + accepts_nested_attributes_for :schema + + belongs_to :package + + def self.validate(io, schema_url = nil, schema = nil, dialect = nil, expiry) + if io.respond_to?(:tempfile) + # uncertain what state triggers the above + # byebug + filename = io.original_filename + csv = File.new(io.tempfile) + io = File.new(io.tempfile) + elsif io.respond_to?(:body) + filename = io.key + io = StringIO.new(io.body) + end + + # Validate + validator = Csvlint::Validator.new( io, (dialect || {}), schema && schema.fields.empty? ? nil : schema ) + # ternary evaluation above follows the following format:: condition ? if_true : if_false + check_schema(validator, schema) unless schema.blank? + # in prior versions this method only executed on schema_url.nil, a condition that caused some schema uploads to pass + # when they should have failed + check_dialect(validator, dialect) unless dialect.blank? + # assign state, used in later evaluation by partials in validation > views + state = "valid" + state = "warnings" unless validator.warnings.empty? + state = "invalid" unless validator.errors.empty? + state = "not_found" unless validator.errors.select { |e| e.type == :not_found }.empty? + + if io.class == String + # It's a url! + url = io + filename = File.basename(URI.parse(url).path) + csv_id = nil + else + # It's a file! + url = nil + validator.remove_instance_variable(:@source) + validator.remove_instance_variable(:@source_url) if validator.instance_variable_defined?(:@source_url) + end + + # Don't save the data + validator.remove_instance_variable(:@data) rescue nil + # Don't save the lambda either + validator.remove_instance_variable(:@lambda) rescue nil + + # Headers are set as a Typhoeus::Response::Header, but this has a proc, so we cast as a hash + # TODO: We really need to stop dumping the whole object here + validator.instance_variable_set("@headers", {}.merge(validator.headers)) + + attributes = { + :url => url, + :filename => filename, + :state => state, + :result => Marshal.dump(validator).force_encoding("UTF-8"), + :parse_options => Validation.generate_options(validator.dialect) + } + + attributes[:expirable_created_at] = Time.now if expiry.eql?(true) + # enable the expirable index, initialise it with current time + + attributes[:csv_id] = csv_id if csv_id.present? + # do not override csv_id if already part of validation + + if schema_url.present? + # Find matching schema if possible and retrieve + schema = Schema.where(url: schema_url).first + attributes[:schema] = schema || { :url => schema_url } + end + # byebug + attributes + + end # end of validate method + + + def self.fetch_validation(id, format, revalidate = nil) + # returns a mongo database record + v = self.find(id) + + unless revalidate.to_s == "false" + if ["png", "svg"].include?(format) + # suspect the above functions tied to the use of badges as hyperlinks to valid schemas & csvs + Validation.delay.check_validation(v.id) + else + v.check_validation + end + end + v + end + + def self.check_schema(validator, schema) + + # @param validator = CSVlint Validator Object + # @param schema = schema file obtained at initialisation of the Validation object + + if schema.nil? + validator.errors.prepend( + Csvlint::ErrorMessage.new(:invalid_schema, :schema, nil, nil, nil, nil) + ) + elsif schema.description.eql?("malformed") + # this conditional is tied to a cludge evaluation in lines 93 - 97 of PackageController + # and are earmarked for future change + validator.errors.prepend( + Csvlint::ErrorMessage.new(:malformed_schema, :schema, nil, nil, nil, nil) + # "JSON schema provided has some structural errors" + ) + elsif schema.fields.empty? + # catch a rare case of an empty json upload, i.e. {} within a .JSON file + validator.errors.prepend( + Csvlint::ErrorMessage.new(:empty_schema, :schema, nil, nil, nil, nil) + ) + end + + end + + def self.check_dialect(validator, dialect) + # byebug + if dialect != standard_dialect + validator.warnings.prepend( + Csvlint::ErrorMessage.new(:non_standard_dialect, :dialect, nil, nil, nil, nil) + ) + end + end + + def self.standard_dialect + { + "header" => true, + "delimiter" => ",", + "skipInitialSpace" => true, + "lineTerminator" => :auto, + "quoteChar" => '"' + } + end + + def self.create_validation(io, schema_url = nil, schema = nil) + # this method instantate the Object then calls its validate method. Below conditional discriminates between URL CSV + # and uploaded CSV. Uploaded CSVs = do not retain + # this method invokes the validate method below rather than self.validate + # returns validation object + if io.class == String + validation = Validation.find_or_initialize_by(url: io) + expiry = false + else + validation = Validation.create + expiry = true + end + validation.validate(io, schema_url, schema, expiry) + # expiry is set to true or false based on inferring that uploaded file meets the do not retain criteria + validation + end + + def validate(io, schema_url = nil, schema = nil, expiry) + validation = Validation.validate(io, schema_url, schema, nil, expiry) + self.update_attributes(validation) + # update_attributes is a method from Mongoid + end + + def update_validation(dialect = nil, expiry=nil) + loaded_schema = schema ? Csvlint::Schema.load_from_uri(schema.url) : nil + io = self.url.nil? ? StoredCSV.fetch(self.filename) : self.url + validation = Validation.validate(io, schema.try(:url), loaded_schema, dialect, expiry) + self.update_attributes(validation) + # update mongoDB record + self + end + + def csv + # method that retrieves stored entire CSV file from mongoDB + if self.url + csv = open(self.url).read + else + # above line means this method triggers only when user opts to revalidate their CSV with suggested prompts + csv = StoredCSV.fetch(self.filename).body + end + + if csv + file = Tempfile.new('csv') + File.open(file, "w") do |f| + f.write csv.encode("UTF-8", :invalid => :replace, :undef => :replace, :replace => "?") + end + file + end + end + + def parse_options + if self.read_attribute(:parse_options).nil? + self.parse_options = Validation.generate_options(self.validator.dialect) + save + end + self.read_attribute(:parse_options) + end + + def self.check_validation(id) + Validation.find(id).check_validation + end + + def check_validation + # this method should only be called against URL listed validations i.e. 'added to list of recent validations' + unless url.blank? + begin + RestClient.head(url, if_modified_since: updated_at.rfc2822 ) if updated_at + update_validation(validator.dialect) if updated_at <= 2.hours.ago + rescue RestClient::NotModified + nil + rescue + update_attributes(state: "not_found") + end + end + end + + def validator + Marshal.load(self.result) + end + + # Empty method? Intended functionality? + def badge + + end + + def self.clean_up(hours) + delete_validations Validation.where(:created_at.lte => hours.hours.ago, :url => nil) + rescue => e + Airbrake.notify(e) if ENV['CSVLINT_AIRBRAKE_KEY'] # Exit cleanly, but still notify airbrake + end + + def self.delete_validations(validations) + validations.each { |v| v.delete } + end + + def self.generate_options(dialect) + dialect ||= {} + { + col_sep: dialect["delimiter"], + row_sep: dialect["lineTerminator"], + quote_char: dialect["quoteChar"], + } + end + +end diff --git a/app/models/package.rb b/app/models/package.rb index 60e480af..3ec35679 100644 --- a/app/models/package.rb +++ b/app/models/package.rb @@ -1,137 +1,2 @@ -require 'package_processor' -require 'lite_kitten/lite_kitten' - -class LocalDataset < DataKitten::Dataset - extend DataKitten::PublishingFormats::Datapackage - - def origin - :local - end - - def publishing_format - # A 'package' can be a DataPackage, or it can be any collection of multiple CSVs, - # and/or collection of CSVs and their schema metadata - # ?? is this used in conjunction with package_helper - :datapackage - end -end - -class RemoteDataset < DataKitten::Dataset - - def initialize(url) - @access_url = url - detect_publishing_format - end - - def detect_publishing_format - [ - DataKitten::PublishingFormats::Datapackage, - DataKitten::PublishingFormats::CKAN - ].each do |format| - if format.supported?(self) - extend format - break - end - end - end -end - -class Package - include Mongoid::Document - include Mongoid::Timestamps - - field :url, type: String - field :dataset, type: String - field :type, type: String - - has_many :validations - - def parse_package(dataset, validations) - attributes = { - :url => dataset.origin == :local ? nil : dataset.access_url, - :dataset => Marshal.dump(dataset), - :validations => validations, - :type => dataset.publishing_format - } - - return attributes - end - - def self.process(id, params) - PackageProcessor.new(params, id).process - end - - def create_package(sources, schema_url = nil, schema = nil) - return nil if sources.count == 0 - - if sources.count == 1 && possible_package?(sources.first) - dataset = create_dataset(sources.first) - return create_datapackage(dataset) unless dataset.nil? - end - - update_attributes({ type: set_type(sources) }) - - sources.each do |source| - validations << Validation.create_validation(source, schema_url, schema) - end - - save - self - end - - def create_dataset(source) - if source.respond_to?(:body) - dataset = LocalDataset.new(access_url: source.string_io) - else - dataset = RemoteDataset.new(source) - end - return nil unless [:ckan, :datapackage].include? dataset.publishing_format - dataset - end - - def create_datapackage(dataset) - validations = create_validations(dataset) - - return nil if validations.count == 0 - - update_attributes( parse_package(dataset, validations) ) - save - self - end - - def create_validations(dataset) - validations = [] - dataset.distributions.each do |distribution| - if can_validate?(distribution) - validations << Validation.create_validation(distribution.access_url, nil, create_schema(distribution) ) - end - end - validations - end - - def possible_package?(source) - source.class == String || local_package?( source ) - end - - def local_package?(source) - source.respond_to?(:string_io) && source.filename =~ /datapackage\.json/ - end - - def set_type(sources) - return "files" if sources.first.respond_to?(:tempfile) - return "urls" if sources.first.class == String - end - - def can_validate?(distribution) - return false unless distribution.format.extension == :csv - return distribution.access_url && distribution.access_url =~ /^http(s?)/ - end - - def create_schema(distribution) - unless distribution.schema.nil? - schema = Csvlint::Schema.from_json_table(nil, distribution.schema) - end - return schema - end - +class Package < Legacy::Package end diff --git a/app/models/schema.rb b/app/models/schema.rb index 87d11d2b..a9ffbd4b 100644 --- a/app/models/schema.rb +++ b/app/models/schema.rb @@ -1,14 +1,2 @@ -class Schema - include Mongoid::Document - include Mongoid::Timestamps - - field :url, type: String - - has_many :validations - - def to_param - id.to_s - end - +class Schema < Legacy::Schema end - \ No newline at end of file diff --git a/app/models/summary.rb b/app/models/summary.rb index 07e3686f..0f71f683 100644 --- a/app/models/summary.rb +++ b/app/models/summary.rb @@ -1,75 +1,2 @@ -class LevelSummary - include Mongoid::Document - embedded_in :summary - field :errors_breakdown, type: Hash - field :warnings_breakdown, type: Hash - field :info_messages_breakdown, type: Hash -end - -class CategorySummary - include Mongoid::Document - embedded_in :summary - field :structure_breakdown, type: Hash - field :schema_breakdown, type: Hash - field :context_breakdown, type: Hash -end - -class Summary - include Mongoid::Document - include Mongoid::Timestamps - - field :sources, type: Integer - field :states, type: Hash - field :hosts, type: Hash - - embeds_one :level_summary - embeds_one :category_summary - - def self.generate - summary = Summary.create - - validations = Validation.where(:url.ne => nil).order_by(:created_at.desc) - # retrieve validations from Mongo Datastore, ordered in reverse by date created - - summary.sources = validations.length - summary.states = {"warnings"=>0, "valid"=>0, "not_found"=>0, "invalid"=>0} - summary.hosts = Hash.new 0 - summary.create_level_summary( errors_breakdown: Hash.new(0), warnings_breakdown: Hash.new(0), info_messages_breakdown: Hash.new(0) ) - summary.create_category_summary( structure_breakdown: Hash.new(0), schema_breakdown: Hash.new(0), context_breakdown: Hash.new(0) ) - - validations.each do |validation| - summary.states[validation.state] += 1 - host = source_host(validation.url) - summary.hosts[host] += 1 unless host.nil? - validator = validation.validator - messages = [] - [:errors, :warnings, :info_messages].each do |level| - unless validator.send(level).nil? - messages = messages + validator.send(level) - validator.send(level).uniq { |m| m.type }.each do |msg| - summary.level_summary.send("#{level}_breakdown".to_sym)[ msg.type ] += 1 - end - end - end - [:structure, :schema, :context].each do |category| - messages.reject {|m| m.category != category }.uniq { |m| m.type }.each do |msg| - summary.category_summary.send("#{category}_breakdown".to_sym)[ msg.type ] += 1 - end - end - end - summary.save - summary - end - - private - - def self.source_host(url) - host = URI.parse(url.to_s).host - return if host.nil? - host.downcase! - host = host.start_with?('www.') ? host[4..-1] : host - #TODO better option? - host.gsub(".", "\uff0e") - end - +class Summary < Legacy::Summary end diff --git a/app/models/validation.rb b/app/models/validation.rb index 6baaa3b4..90ed17b5 100644 --- a/app/models/validation.rb +++ b/app/models/validation.rb @@ -1,257 +1,2 @@ -class Validation - include Mongoid::Document - include Mongoid::Timestamps - - field :filename, type: String - field :url, type: String - field :state, type: String - field :result, type: String - field :csv_id, type: String - field :parse_options, type: Hash - field :expirable_created_at, type: Time - - index :created_at => 1 - index({expirable_created_at: 1}, {expire_after_seconds: 24.hours}) - # invoke the mongo time-to-live feature which will automatically expire entries - # - this index is only enabled for a subset of validations, which are validations uploaded as file - - belongs_to :schema - accepts_nested_attributes_for :schema - - belongs_to :package - - def self.validate(io, schema_url = nil, schema = nil, dialect = nil, expiry) - if io.respond_to?(:tempfile) - # uncertain what state triggers the above - # byebug - filename = io.original_filename - csv = File.new(io.tempfile) - io = File.new(io.tempfile) - elsif io.respond_to?(:body) - filename = io.key - io = StringIO.new(io.body) - end - - # Validate - validator = Csvlint::Validator.new( io, (dialect || {}), schema && schema.fields.empty? ? nil : schema ) - # ternary evaluation above follows the following format:: condition ? if_true : if_false - check_schema(validator, schema) unless schema.blank? - # in prior versions this method only executed on schema_url.nil, a condition that caused some schema uploads to pass - # when they should have failed - check_dialect(validator, dialect) unless dialect.blank? - # assign state, used in later evaluation by partials in validation > views - state = "valid" - state = "warnings" unless validator.warnings.empty? - state = "invalid" unless validator.errors.empty? - state = "not_found" unless validator.errors.select { |e| e.type == :not_found }.empty? - - if io.class == String - # It's a url! - url = io - filename = File.basename(URI.parse(url).path) - csv_id = nil - else - # It's a file! - url = nil - validator.remove_instance_variable(:@source) - validator.remove_instance_variable(:@source_url) if validator.instance_variable_defined?(:@source_url) - end - - # Don't save the data - validator.remove_instance_variable(:@data) rescue nil - # Don't save the lambda either - validator.remove_instance_variable(:@lambda) rescue nil - - # Headers are set as a Typhoeus::Response::Header, but this has a proc, so we cast as a hash - # TODO: We really need to stop dumping the whole object here - validator.instance_variable_set("@headers", {}.merge(validator.headers)) - - attributes = { - :url => url, - :filename => filename, - :state => state, - :result => Marshal.dump(validator).force_encoding("UTF-8"), - :parse_options => Validation.generate_options(validator.dialect) - } - - attributes[:expirable_created_at] = Time.now if expiry.eql?(true) - # enable the expirable index, initialise it with current time - - attributes[:csv_id] = csv_id if csv_id.present? - # do not override csv_id if already part of validation - - if schema_url.present? - # Find matching schema if possible and retrieve - schema = Schema.where(url: schema_url).first - attributes[:schema] = schema || { :url => schema_url } - end - # byebug - attributes - - end # end of validate method - - - def self.fetch_validation(id, format, revalidate = nil) - # returns a mongo database record - v = self.find(id) - - unless revalidate.to_s == "false" - if ["png", "svg"].include?(format) - # suspect the above functions tied to the use of badges as hyperlinks to valid schemas & csvs - Validation.delay.check_validation(v.id) - else - v.check_validation - end - end - v - end - - def self.check_schema(validator, schema) - - # @param validator = CSVlint Validator Object - # @param schema = schema file obtained at initialisation of the Validation object - - if schema.nil? - validator.errors.prepend( - Csvlint::ErrorMessage.new(:invalid_schema, :schema, nil, nil, nil, nil) - ) - elsif schema.description.eql?("malformed") - # this conditional is tied to a cludge evaluation in lines 93 - 97 of PackageController - # and are earmarked for future change - validator.errors.prepend( - Csvlint::ErrorMessage.new(:malformed_schema, :schema, nil, nil, nil, nil) - # "JSON schema provided has some structural errors" - ) - elsif schema.fields.empty? - # catch a rare case of an empty json upload, i.e. {} within a .JSON file - validator.errors.prepend( - Csvlint::ErrorMessage.new(:empty_schema, :schema, nil, nil, nil, nil) - ) - end - - end - - def self.check_dialect(validator, dialect) - # byebug - if dialect != standard_dialect - validator.warnings.prepend( - Csvlint::ErrorMessage.new(:non_standard_dialect, :dialect, nil, nil, nil, nil) - ) - end - end - - def self.standard_dialect - { - "header" => true, - "delimiter" => ",", - "skipInitialSpace" => true, - "lineTerminator" => :auto, - "quoteChar" => '"' - } - end - - def self.create_validation(io, schema_url = nil, schema = nil) - # this method instantate the Object then calls its validate method. Below conditional discriminates between URL CSV - # and uploaded CSV. Uploaded CSVs = do not retain - # this method invokes the validate method below rather than self.validate - # returns validation object - if io.class == String - validation = Validation.find_or_initialize_by(url: io) - expiry = false - else - validation = Validation.create - expiry = true - end - validation.validate(io, schema_url, schema, expiry) - # expiry is set to true or false based on inferring that uploaded file meets the do not retain criteria - validation - end - - def validate(io, schema_url = nil, schema = nil, expiry) - validation = Validation.validate(io, schema_url, schema, nil, expiry) - self.update_attributes(validation) - # update_attributes is a method from Mongoid - end - - def update_validation(dialect = nil, expiry=nil) - loaded_schema = schema ? Csvlint::Schema.load_from_uri(schema.url) : nil - io = self.url.nil? ? StoredCSV.fetch(self.filename) : self.url - validation = Validation.validate(io, schema.try(:url), loaded_schema, dialect, expiry) - self.update_attributes(validation) - # update mongoDB record - self - end - - def csv - # method that retrieves stored entire CSV file from mongoDB - if self.url - csv = open(self.url).read - else - # above line means this method triggers only when user opts to revalidate their CSV with suggested prompts - csv = StoredCSV.fetch(self.filename).body - end - - if csv - file = Tempfile.new('csv') - File.open(file, "w") do |f| - f.write csv.encode("UTF-8", :invalid => :replace, :undef => :replace, :replace => "?") - end - file - end - end - - def parse_options - if self.read_attribute(:parse_options).nil? - self.parse_options = Validation.generate_options(self.validator.dialect) - save - end - self.read_attribute(:parse_options) - end - - def self.check_validation(id) - Validation.find(id).check_validation - end - - def check_validation - # this method should only be called against URL listed validations i.e. 'added to list of recent validations' - unless url.blank? - begin - RestClient.head(url, if_modified_since: updated_at.rfc2822 ) if updated_at - update_validation(validator.dialect) if updated_at <= 2.hours.ago - rescue RestClient::NotModified - nil - rescue - update_attributes(state: "not_found") - end - end - end - - def validator - Marshal.load(self.result) - end - - # Empty method? Intended functionality? - def badge - - end - - def self.clean_up(hours) - delete_validations Validation.where(:created_at.lte => hours.hours.ago, :url => nil) - rescue => e - Airbrake.notify(e) if ENV['CSVLINT_AIRBRAKE_KEY'] # Exit cleanly, but still notify airbrake - end - - def self.delete_validations(validations) - validations.each { |v| v.delete } - end - - def self.generate_options(dialect) - dialect ||= {} - { - col_sep: dialect["delimiter"], - row_sep: dialect["lineTerminator"], - quote_char: dialect["quoteChar"], - } - end - +class Validation < Legacy::Validation end From bbb35e2ce65e81afbf7cdf9f10e9b35a39a70509 Mon Sep 17 00:00:00 2001 From: James Smith Date: Sat, 17 Apr 2021 19:12:05 +0100 Subject: [PATCH 03/12] Change everything to use legacy models explicitly --- app/controllers/application_controller.rb | 4 +- app/controllers/package_controller.rb | 8 +-- app/controllers/schemas_controller.rb | 6 +-- app/controllers/summary_controller.rb | 10 ++-- app/controllers/validation_controller.rb | 6 +-- app/models/legacy.rb | 5 ++ app/models/legacy/package.rb | 6 +-- app/models/legacy/schema.rb | 2 +- app/models/legacy/summary.rb | 4 +- app/models/legacy/validation.rb | 26 +++++----- app/models/package.rb | 2 - app/models/schema.rb | 2 - app/models/summary.rb | 2 - app/models/validation.rb | 2 - app/views/summary/index.json.jbuilder | 13 +++-- features/step_definitions/api_steps.rb | 2 +- .../background_validation_steps.rb | 6 +-- features/step_definitions/by_url_steps.rb | 4 +- .../step_definitions/datapackages_steps.rb | 12 ++--- .../multiple_validations_steps.rb | 8 +-- features/step_definitions/summary_steps.rb | 3 +- features/step_definitions/validation_steps.rb | 22 ++++---- features/step_definitions/web_steps.rb | 2 +- features/support/hooks.rb | 4 +- lib/package_processor.rb | 2 +- ...n_for_a_CKAN_package_with_a_single_CSV.yml | 0 ...s_for_a_datapackage_with_multiple_CSVs.yml | 0 .../returns_nil_if_there_are_no_CSVs.yml | 0 spec/controllers/package_controller_spec.rb | 26 +++++----- .../controllers/validation_controller_spec.rb | 50 +++++++++---------- spec/factories/schemas.rb | 2 +- spec/factories/validations.rb | 2 +- spec/models/package_processor_spec.rb | 2 +- spec/models/package_spec.rb | 2 +- spec/models/summary_spec.rb | 12 ++--- spec/models/validation_spec.rb | 2 +- spec/spec_helper.rb | 4 +- .../validation/validate.html.erb_spec.rb | 4 +- 38 files changed, 132 insertions(+), 137 deletions(-) create mode 100644 app/models/legacy.rb delete mode 100644 app/models/package.rb delete mode 100644 app/models/schema.rb delete mode 100644 app/models/summary.rb delete mode 100644 app/models/validation.rb rename spec/cassettes/{Package => Legacy_Package}/with_a_CKAN_URL/creates_a_validation_for_a_CKAN_package_with_a_single_CSV.yml (100%) rename spec/cassettes/{Package => Legacy_Package}/with_a_CKAN_URL/creates_multiple_validations_for_a_datapackage_with_multiple_CSVs.yml (100%) rename spec/cassettes/{Package => Legacy_Package}/with_a_CKAN_URL/returns_nil_if_there_are_no_CSVs.yml (100%) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index e13dff20..3979a2a1 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -8,10 +8,10 @@ def standard_csv_options def index if params[:uri] - validator = Validation.where(:url => params[:uri]).first + validator = Legacy::Validation.where(:url => params[:uri]).first if validator.nil? - Validation.delay.create_validation(params[:uri]) + Legacy::Validation.delay.create_validation(params[:uri]) respond_to do |wants| wants.html { render status: 202 } wants.json { render status: 202 } diff --git a/app/controllers/package_controller.rb b/app/controllers/package_controller.rb index dd5e8686..e19e51fb 100644 --- a/app/controllers/package_controller.rb +++ b/app/controllers/package_controller.rb @@ -6,11 +6,11 @@ class PackageController < ApplicationController before_filter(:only => [:show]) { alternate_formats [:json] } def create - @package = Package.create + @package = Legacy::Package.create if params[:format] == "json" - Package.delay.process(@package.id, params) + Legacy::Package.delay.process(@package.id, params) else - Package.process(@package.id, params) + Legacy::Package.process(@package.id, params) if @package.validations.count == 1 redirect_to validation_path(@package.validations.first) else @@ -20,7 +20,7 @@ def create end def show - @package = Package.find( params[:id] ) + @package = Legacy::Package.find( params[:id] ) if @package.validations.count == 1 && params[:format] != "json" redirect_to validation_path(@package.validations.first) diff --git a/app/controllers/schemas_controller.rb b/app/controllers/schemas_controller.rb index 3e5a8372..279f8255 100644 --- a/app/controllers/schemas_controller.rb +++ b/app/controllers/schemas_controller.rb @@ -6,17 +6,17 @@ class SchemasController < ApplicationController def index if params[:uri] - schema = Schema.where(url: params[:uri]).first + schema = Legacy::Schema.where(url: params[:uri]).first render status: 404 and return if schema.nil? redirect_to schema, status: 303 else - schemas = Schema.all + schemas = Legacy::Schema.all @schemas = Kaminari.paginate_array(schemas).page(params[:page]) end end def show - @db_schema = Schema.where(id: params[:id]).first + @db_schema = Legacy::Schema.where(id: params[:id]).first @schema = Csvlint::Schema.load_from_uri(@db_schema.url) respond_to do |wants| wants.html diff --git a/app/controllers/summary_controller.rb b/app/controllers/summary_controller.rb index 15326ca4..7512a142 100644 --- a/app/controllers/summary_controller.rb +++ b/app/controllers/summary_controller.rb @@ -1,9 +1,9 @@ class SummaryController < ApplicationController - + before_filter(:only => [:index]) { alternate_formats [:json] } - + def index - @summary = Summary.last + @summary = Legacy::Summary.last end - -end \ No newline at end of file + +end diff --git a/app/controllers/validation_controller.rb b/app/controllers/validation_controller.rb index 5bd53020..a8da619e 100644 --- a/app/controllers/validation_controller.rb +++ b/app/controllers/validation_controller.rb @@ -6,13 +6,13 @@ class ValidationController < ApplicationController before_filter(:only => [:show]) { alternate_formats [:json] } def show - @validation = Validation.fetch_validation(params[:id], params[:format], params[:revalidate]) + @validation = Legacy::Validation.fetch_validation(params[:id], params[:format], params[:revalidate]) raise ActionController::RoutingError.new('Not Found') if @validation.state.nil? # @result stores all the validation errors, warnings and information messages @result = @validation.validator # byebug - @dialect = @result.dialect || Validation.standard_dialect + @dialect = @result.dialect || Legacy::Validation.standard_dialect # Responses respond_to do |wants| wants.html @@ -27,7 +27,7 @@ def update # this method is triggered when user revalidates a schema dialect = build_dialect(params) # build a fresh dialect for comparing a file against - v = Validation.find(params[:id]) + v = Legacy::Validation.find(params[:id]) if v.has_attribute?(:expirable_created_at) # has expirable_created_at value v.update_validation(dialect, expiry=true) diff --git a/app/models/legacy.rb b/app/models/legacy.rb new file mode 100644 index 00000000..971fd714 --- /dev/null +++ b/app/models/legacy.rb @@ -0,0 +1,5 @@ +module Legacy + def self.use_relative_model_naming? + true + end +end diff --git a/app/models/legacy/package.rb b/app/models/legacy/package.rb index f2ba7bb6..985deb48 100644 --- a/app/models/legacy/package.rb +++ b/app/models/legacy/package.rb @@ -45,7 +45,7 @@ class Legacy::Package field :dataset, type: String field :type, type: String - has_many :validations + has_many :validations, class_name: 'Legacy::Validation' def parse_package(dataset, validations) attributes = { @@ -73,7 +73,7 @@ def create_package(sources, schema_url = nil, schema = nil) update_attributes({ type: set_type(sources) }) sources.each do |source| - validations << Validation.create_validation(source, schema_url, schema) + validations << Legacy::Validation.create_validation(source, schema_url, schema) end save @@ -104,7 +104,7 @@ def create_validations(dataset) validations = [] dataset.distributions.each do |distribution| if can_validate?(distribution) - validations << Validation.create_validation(distribution.access_url, nil, create_schema(distribution) ) + validations << Legacy::Validation.create_validation(distribution.access_url, nil, create_schema(distribution) ) end end validations diff --git a/app/models/legacy/schema.rb b/app/models/legacy/schema.rb index 1c31b266..448ebdf7 100644 --- a/app/models/legacy/schema.rb +++ b/app/models/legacy/schema.rb @@ -5,7 +5,7 @@ class Legacy::Schema field :url, type: String - has_many :validations + has_many :validations, class_name: 'Legacy::Validation' def to_param id.to_s diff --git a/app/models/legacy/summary.rb b/app/models/legacy/summary.rb index 88bf9718..33db914d 100644 --- a/app/models/legacy/summary.rb +++ b/app/models/legacy/summary.rb @@ -27,9 +27,9 @@ class Legacy::Summary embeds_one :category_summary def self.generate - summary = Summary.create + summary = Legacy::Summary.create - validations = Validation.where(:url.ne => nil).order_by(:created_at.desc) + validations = Legacy::Validation.where(:url.ne => nil).order_by(:created_at.desc) # retrieve validations from Mongo Datastore, ordered in reverse by date created summary.sources = validations.length diff --git a/app/models/legacy/validation.rb b/app/models/legacy/validation.rb index 7bc5403b..1e18094f 100644 --- a/app/models/legacy/validation.rb +++ b/app/models/legacy/validation.rb @@ -16,10 +16,10 @@ class Legacy::Validation # invoke the mongo time-to-live feature which will automatically expire entries # - this index is only enabled for a subset of validations, which are validations uploaded as file - belongs_to :schema - accepts_nested_attributes_for :schema + belongs_to :schema, class_name: 'Legacy::Schema' + accepts_nested_attributes_for :schema, class_name: 'Legacy::Schema' - belongs_to :package + belongs_to :package, class_name: 'Legacy::Package' def self.validate(io, schema_url = nil, schema = nil, dialect = nil, expiry) if io.respond_to?(:tempfile) @@ -72,7 +72,7 @@ def self.validate(io, schema_url = nil, schema = nil, dialect = nil, expiry) :filename => filename, :state => state, :result => Marshal.dump(validator).force_encoding("UTF-8"), - :parse_options => Validation.generate_options(validator.dialect) + :parse_options => Legacy::Validation.generate_options(validator.dialect) } attributes[:expirable_created_at] = Time.now if expiry.eql?(true) @@ -83,7 +83,7 @@ def self.validate(io, schema_url = nil, schema = nil, dialect = nil, expiry) if schema_url.present? # Find matching schema if possible and retrieve - schema = Schema.where(url: schema_url).first + schema = Legacy::Schema.where(url: schema_url).first attributes[:schema] = schema || { :url => schema_url } end # byebug @@ -99,7 +99,7 @@ def self.fetch_validation(id, format, revalidate = nil) unless revalidate.to_s == "false" if ["png", "svg"].include?(format) # suspect the above functions tied to the use of badges as hyperlinks to valid schemas & csvs - Validation.delay.check_validation(v.id) + Legacy::Validation.delay.check_validation(v.id) else v.check_validation end @@ -157,10 +157,10 @@ def self.create_validation(io, schema_url = nil, schema = nil) # this method invokes the validate method below rather than self.validate # returns validation object if io.class == String - validation = Validation.find_or_initialize_by(url: io) + validation = Legacy::Validation.find_or_initialize_by(url: io) expiry = false else - validation = Validation.create + validation = Legacy::Validation.create expiry = true end validation.validate(io, schema_url, schema, expiry) @@ -169,7 +169,7 @@ def self.create_validation(io, schema_url = nil, schema = nil) end def validate(io, schema_url = nil, schema = nil, expiry) - validation = Validation.validate(io, schema_url, schema, nil, expiry) + validation = Legacy::Validation.validate(io, schema_url, schema, nil, expiry) self.update_attributes(validation) # update_attributes is a method from Mongoid end @@ -177,7 +177,7 @@ def validate(io, schema_url = nil, schema = nil, expiry) def update_validation(dialect = nil, expiry=nil) loaded_schema = schema ? Csvlint::Schema.load_from_uri(schema.url) : nil io = self.url.nil? ? StoredCSV.fetch(self.filename) : self.url - validation = Validation.validate(io, schema.try(:url), loaded_schema, dialect, expiry) + validation = Legacy::Validation.validate(io, schema.try(:url), loaded_schema, dialect, expiry) self.update_attributes(validation) # update mongoDB record self @@ -203,14 +203,14 @@ def csv def parse_options if self.read_attribute(:parse_options).nil? - self.parse_options = Validation.generate_options(self.validator.dialect) + self.parse_options = Legacy::Validation.generate_options(self.validator.dialect) save end self.read_attribute(:parse_options) end def self.check_validation(id) - Validation.find(id).check_validation + Legacy::Validation.find(id).check_validation end def check_validation @@ -237,7 +237,7 @@ def badge end def self.clean_up(hours) - delete_validations Validation.where(:created_at.lte => hours.hours.ago, :url => nil) + delete_validations Legacy::Validation.where(:created_at.lte => hours.hours.ago, :url => nil) rescue => e Airbrake.notify(e) if ENV['CSVLINT_AIRBRAKE_KEY'] # Exit cleanly, but still notify airbrake end diff --git a/app/models/package.rb b/app/models/package.rb deleted file mode 100644 index 3ec35679..00000000 --- a/app/models/package.rb +++ /dev/null @@ -1,2 +0,0 @@ -class Package < Legacy::Package -end diff --git a/app/models/schema.rb b/app/models/schema.rb deleted file mode 100644 index a9ffbd4b..00000000 --- a/app/models/schema.rb +++ /dev/null @@ -1,2 +0,0 @@ -class Schema < Legacy::Schema -end diff --git a/app/models/summary.rb b/app/models/summary.rb deleted file mode 100644 index 0f71f683..00000000 --- a/app/models/summary.rb +++ /dev/null @@ -1,2 +0,0 @@ -class Summary < Legacy::Summary -end diff --git a/app/models/validation.rb b/app/models/validation.rb deleted file mode 100644 index 90ed17b5..00000000 --- a/app/models/validation.rb +++ /dev/null @@ -1,2 +0,0 @@ -class Validation < Legacy::Validation -end diff --git a/app/views/summary/index.json.jbuilder b/app/views/summary/index.json.jbuilder index 0070ba09..dccb34e5 100644 --- a/app/views/summary/index.json.jbuilder +++ b/app/views/summary/index.json.jbuilder @@ -1,8 +1,8 @@ json.created @summary.created_at json.sources @summary.sources json.date_range do - json.from Validation.first.created_at - json.to Validation.last.created_at + json.from Legacy::Validation.first.created_at + json.to Legacy::Validation.last.created_at end json.results do @summary.states.each do |state, count| @@ -15,21 +15,20 @@ json.hosts do end end json.severity do - [:errors, :warnings, :info_messages].each do |level| + [:errors, :warnings, :info_messages].each do |level| json.set! level do @summary.level_summary.send("#{level}_breakdown").sort_by{|k,v| v}.reverse.each_with_index do |(type,count), index| - json.set! type, count + json.set! type, count end end end end json.category do - [:structure, :schema, :context].each do |category| + [:structure, :schema, :context].each do |category| json.set! category do @summary.category_summary.send("#{category}_breakdown").sort_by{|k,v| v}.reverse.each_with_index do |(type,count), index| - json.set! type, count + json.set! type, count end end end end - diff --git a/features/step_definitions/api_steps.rb b/features/step_definitions/api_steps.rb index db07d887..4e64f1dc 100644 --- a/features/step_definitions/api_steps.rb +++ b/features/step_definitions/api_steps.rb @@ -17,7 +17,7 @@ Given(/^I have a package with the following URLs:$/) do |table| urls = table.raw.map! { |url| url[0] } - @package = Package.create + @package = Legacy::Package.create @package.create_package(urls) @package.save end diff --git a/features/step_definitions/background_validation_steps.rb b/features/step_definitions/background_validation_steps.rb index f2aacab9..7d4fbbde 100644 --- a/features/step_definitions/background_validation_steps.rb +++ b/features/step_definitions/background_validation_steps.rb @@ -1,7 +1,7 @@ require 'package_processor' Then(/^my CSV should be placed in a background job$/) do - Package.should_receive(:delay).and_call_original + Legacy::Package.should_receive(:delay).and_call_original end When(/^I wait for the ajax request to finish$/) do @@ -18,7 +18,7 @@ When(/^I wait for the package to be created$/) do patiently do - Package.first.should_not be_nil + Legacy::Package.first.should_not be_nil end end @@ -30,6 +30,6 @@ Then(/^I should be redirected to my validation results$/) do patiently do - current_path.should == validation_path(Validation.first) + current_path.should == validation_path(Legacy::Validation.first) end end diff --git a/features/step_definitions/by_url_steps.rb b/features/step_definitions/by_url_steps.rb index 6f3df7ee..fcedfb73 100644 --- a/features/step_definitions/by_url_steps.rb +++ b/features/step_definitions/by_url_steps.rb @@ -1,11 +1,11 @@ Given(/^I have already validated the URL "(.*?)"$/) do |url| @url = url - @validation = Validation.create_validation(@url) + @validation = Legacy::Validation.create_validation(@url) end Given(/^I have not already validated the URL "(.*?)"$/) do |url| @url = url - validation = Validation.where(:url => @url).first + validation = Legacy::Validation.where(:url => @url).first expect(validation).to eq(nil) end diff --git a/features/step_definitions/datapackages_steps.rb b/features/step_definitions/datapackages_steps.rb index 4a729791..eeb2af6c 100644 --- a/features/step_definitions/datapackages_steps.rb +++ b/features/step_definitions/datapackages_steps.rb @@ -1,20 +1,20 @@ Then(/^the url "(.*?)" should be persisted in the database$/) do |url| - Validation.count.should == 1 - Validation.first.url.should == url + Legacy::Validation.count.should == 1 + Legacy::Validation.first.url.should == url filename = File.basename(URI.parse(url).path) - Validation.first.filename.should == filename + Legacy::Validation.first.filename.should == filename end Then(/^I should be redirected to my package page$/) do patiently do - current_path.should == package_path(Package.first) + current_path.should == package_path(Legacy::Package.first) end end Then(/^my datapackage should be persisited in the database$/) do - Package.count.should == 1 + Legacy::Package.count.should == 1 end When(/^I click on the first report link$/) do click_link("View report") -end \ No newline at end of file +end diff --git a/features/step_definitions/multiple_validations_steps.rb b/features/step_definitions/multiple_validations_steps.rb index 4f1fa770..9fe8531e 100644 --- a/features/step_definitions/multiple_validations_steps.rb +++ b/features/step_definitions/multiple_validations_steps.rb @@ -1,8 +1,8 @@ When(/^I enter the following urls:$/) do |table| count = 0 - + urls = table.raw.map { |url| url.first } - + urls.each do |url| fill_in "url_#{count}", with: url page.find(".btn-clone").click @@ -11,11 +11,11 @@ end Then(/^the package validations should have the correct schema$/) do - package = Package.first + package = Legacy::Package.first package.validations.each do |validation| result = Marshal.load validation.result result.schema.fields[0].name.should == "FirstName" result.schema.fields[1].name.should == "LastName" result.schema.fields[2].name.should == "Insult" end -end \ No newline at end of file +end diff --git a/features/step_definitions/summary_steps.rb b/features/step_definitions/summary_steps.rb index 080c7154..ee23e035 100644 --- a/features/step_definitions/summary_steps.rb +++ b/features/step_definitions/summary_steps.rb @@ -1,5 +1,5 @@ Given(/^that a Summary has been generated$/) do - @summary = Summary.generate + @summary = Legacy::Summary.generate @summary.save end @@ -8,4 +8,3 @@ And I send a GET request to "/statistics" } end - diff --git a/features/step_definitions/validation_steps.rb b/features/step_definitions/validation_steps.rb index bc6002bb..4afbd899 100644 --- a/features/step_definitions/validation_steps.rb +++ b/features/step_definitions/validation_steps.rb @@ -19,38 +19,38 @@ Then(/^my file should be persisted in the database$/) do - Validation.count.should == 1 - Validation.first.filename.should =~ /#{File.basename(@file)}/ + Legacy::Validation.count.should == 1 + Legacy::Validation.first.filename.should =~ /#{File.basename(@file)}/ end Then(/^"(.*?)" should be persisted in the database$/) do |filename| - Validation.count.should == 1 - Validation.first.filename.should =~ /#{filename}/ + Legacy::Validation.count.should == 1 + Legacy::Validation.first.filename.should =~ /#{filename}/ end Then(/^my file should not be saved in the database$/) do - Validation.count.should == 0 + Legacy::Validation.count.should == 0 end Then(/^my url should be persisted in the database$/) do - Validation.count.should == 1 - Validation.first.url.should == @url + Legacy::Validation.count.should == 1 + Legacy::Validation.first.url.should == @url filename = File.basename(URI.parse(@url).path) - Validation.first.filename.should == filename + Legacy::Validation.first.filename.should == filename end Then(/^the database record should have a "(.*?)" of the type "(.*?)"$/) do |category, type| - result = Marshal.load(Validation.first.result) + result = Marshal.load(Legacy::Validation.first.result) result.send(category.pluralize).first.type.should == type.to_sym end Then(/^the validation should be updated$/) do - Validation.any_instance.should_receive(:update_attributes).once + Legacy::Validation.any_instance.should_receive(:update_attributes).once end Then(/^the validation should not be updated$/) do - Validation.any_instance.should_not_receive(:update_attributes) + Legacy::Validation.any_instance.should_not_receive(:update_attributes) end Then(/^I should be given the option to revalidate using a different dialect$/) do diff --git a/features/step_definitions/web_steps.rb b/features/step_definitions/web_steps.rb index 35e07920..bf86e924 100644 --- a/features/step_definitions/web_steps.rb +++ b/features/step_definitions/web_steps.rb @@ -62,5 +62,5 @@ end When(/^I access my page of validation results$/) do - visit('/package/'+ Package.first.id) + visit('/package/'+ Legacy::Package.first.id) end diff --git a/features/support/hooks.rb b/features/support/hooks.rb index 8bcb20be..2743d5d9 100644 --- a/features/support/hooks.rb +++ b/features/support/hooks.rb @@ -4,8 +4,8 @@ Before ('@data_expiry') do # Mongoid only supports a truncation strategy, which means that any indices created will persist - the following code # flushes the indices on the Validation model - Validation.remove_indexes - Validation.create_indexes + Legacy::Validation.remove_indexes + Legacy::Validation.create_indexes end After('@rackmock') do diff --git a/lib/package_processor.rb b/lib/package_processor.rb index 8038cfa0..8d110f72 100644 --- a/lib/package_processor.rb +++ b/lib/package_processor.rb @@ -35,7 +35,7 @@ def process end def package - Package.find(@package_id) + Legacy::Package.find(@package_id) end def create_package diff --git a/spec/cassettes/Package/with_a_CKAN_URL/creates_a_validation_for_a_CKAN_package_with_a_single_CSV.yml b/spec/cassettes/Legacy_Package/with_a_CKAN_URL/creates_a_validation_for_a_CKAN_package_with_a_single_CSV.yml similarity index 100% rename from spec/cassettes/Package/with_a_CKAN_URL/creates_a_validation_for_a_CKAN_package_with_a_single_CSV.yml rename to spec/cassettes/Legacy_Package/with_a_CKAN_URL/creates_a_validation_for_a_CKAN_package_with_a_single_CSV.yml diff --git a/spec/cassettes/Package/with_a_CKAN_URL/creates_multiple_validations_for_a_datapackage_with_multiple_CSVs.yml b/spec/cassettes/Legacy_Package/with_a_CKAN_URL/creates_multiple_validations_for_a_datapackage_with_multiple_CSVs.yml similarity index 100% rename from spec/cassettes/Package/with_a_CKAN_URL/creates_multiple_validations_for_a_datapackage_with_multiple_CSVs.yml rename to spec/cassettes/Legacy_Package/with_a_CKAN_URL/creates_multiple_validations_for_a_datapackage_with_multiple_CSVs.yml diff --git a/spec/cassettes/Package/with_a_CKAN_URL/returns_nil_if_there_are_no_CSVs.yml b/spec/cassettes/Legacy_Package/with_a_CKAN_URL/returns_nil_if_there_are_no_CSVs.yml similarity index 100% rename from spec/cassettes/Package/with_a_CKAN_URL/returns_nil_if_there_are_no_CSVs.yml rename to spec/cassettes/Legacy_Package/with_a_CKAN_URL/returns_nil_if_there_are_no_CSVs.yml diff --git a/spec/controllers/package_controller_spec.rb b/spec/controllers/package_controller_spec.rb index aac43c27..7f66ffe4 100644 --- a/spec/controllers/package_controller_spec.rb +++ b/spec/controllers/package_controller_spec.rb @@ -38,7 +38,7 @@ mock_file("http://example.com/test.csv", 'csvs/valid.csv') post 'create', urls: ['http://example.com/test.csv'] expect(response).to be_redirect - validation = Validation.first + validation = Legacy::Validation.first expect(response.location).to eq(validation_url(validation)) end @@ -52,7 +52,7 @@ 'http://example.com/test3.csv' ] expect(response).to be_redirect - package = Package.first + package = Legacy::Package.first expect(package.validations.count).to eq(3) expect(response.location).to eq(package_url(package)) end @@ -64,7 +64,7 @@ mock_upload('valid.csv') ] expect(response).to be_redirect - package = Package.first + package = Legacy::Package.first expect(package.validations.count).to eq(3) expect(response.location).to eq(package_url(package)) end @@ -77,7 +77,7 @@ 'http://example.com/multiple_files.zip', ] expect(response).to be_redirect - package = Package.first + package = Legacy::Package.first expect(package.validations.count).to eq(4) expect(response.location).to eq(package_url(package)) end @@ -88,7 +88,7 @@ mock_upload('multiple_files.zip'), ] expect(response).to be_redirect - package = Package.first + package = Legacy::Package.first expect(package.validations.count).to eq(4) expect(response.location).to eq(package_url(package)) end @@ -99,7 +99,7 @@ ] expect(response).to be_redirect - package = Package.first + package = Legacy::Package.first expect(package.validations.count).to eq(1) expect(response.location).to eq(validation_url(package.validations.first)) end @@ -113,7 +113,7 @@ ] expect(response).to be_redirect - package = Package.first + package = Legacy::Package.first expect(package.validations.count).to eq(4) expect(response.location).to eq(package_url(package)) end @@ -123,7 +123,7 @@ create_data_uri('csvs/valid.zip', 'application/zip'), ] expect(response).to be_redirect - package = Package.first + package = Legacy::Package.first expect(package.validations.count).to eq(1) expect(response.location).to eq(validation_url(package.validations.first)) end @@ -134,7 +134,7 @@ create_data_uri('csvs/multiple_files.zip', 'application/zip'), ] expect(response).to be_redirect - package = Package.first + package = Legacy::Package.first expect(package.validations.count).to eq(4) expect(response.location).to eq(package_url(package)) end @@ -148,7 +148,7 @@ schema_data: create_data_uri('schemas/all_constraints.json', 'application/json') # above accurately emulates how the file upload works by a user expect(response).to be_redirect - package = Package.first + package = Legacy::Package.first validation = package.validations.first validator = validation.validator expect(response.location).to eq(validation_url(validation)) @@ -175,7 +175,7 @@ mock_file("http://example.com/test.csv", 'csvs/valid.csv') post 'create', urls: ['http://example.com/test.csv'] expect(response).to be_redirect - validation = Marshal.load(Validation.first.result) + validation = Marshal.load(Legacy::Validation.first.result) expect(validation.warnings).to be_empty expect(validation.errors).to be_empty end @@ -184,7 +184,7 @@ mock_file("http://example.com/test.csv", 'csvs/warnings.csv') post 'create', urls: ['http://example.com/test.csv'] expect(response).to be_redirect - validation = Marshal.load(Validation.first.result) + validation = Marshal.load(Legacy::Validation.first.result) expect(validation.warnings).not_to be_empty expect(validation.errors).to be_empty end @@ -193,7 +193,7 @@ mock_file("http://example.com/test.csv", 'csvs/errors.csv') post 'create', urls: ['http://example.com/test.csv'] expect(response).to be_redirect - validation = Marshal.load(Validation.first.result) + validation = Marshal.load(Legacy::Validation.first.result) expect(validation.errors).not_to be_empty end diff --git a/spec/controllers/validation_controller_spec.rb b/spec/controllers/validation_controller_spec.rb index 646dcc98..46926b23 100644 --- a/spec/controllers/validation_controller_spec.rb +++ b/spec/controllers/validation_controller_spec.rb @@ -15,17 +15,17 @@ it "updates a CSV sucessfully" do mock_file("http://example.com/test.csv", 'csvs/valid.csv') - Validation.create_validation('http://example.com/test.csv') - put 'update', id: Validation.first.id + Legacy::Validation.create_validation('http://example.com/test.csv') + put 'update', id: Legacy::Validation.first.id expect(response).to be_redirect end it "updates a CSV with a new schema sucessfully" do mock_file("http://example.com/revalidate.csv", 'csvs/revalidate.csv') - Validation.create_validation('http://example.com/revalidate.csv') + Legacy::Validation.create_validation('http://example.com/revalidate.csv') params = { - :id => Validation.first.id, + :id => Legacy::Validation.first.id, :header => "true", :delimiter => ";", :skip_initial_space => "true", @@ -35,7 +35,7 @@ put 'update', params - validator = Marshal.load Validation.first.result + validator = Marshal.load Legacy::Validation.first.result expect(validator.warnings.select { |warning| warning.type == :check_options }.count).to eq(0) expect(response).to be_redirect @@ -43,9 +43,9 @@ it 'purges the cache when updating' do mock_file("http://example.com/test.csv", 'csvs/valid.csv') - Validation.create_validation('http://example.com/test.csv') - expect(@connection).to receive(:zone_file_purge).with('csvlint.io', validation_url(Validation.first)) - put 'update', id: Validation.first.id + Legacy::Validation.create_validation('http://example.com/test.csv') + expect(@connection).to receive(:zone_file_purge).with('csvlint.io', validation_url(Legacy::Validation.first)) + put 'update', id: Legacy::Validation.first.id end end @@ -54,8 +54,8 @@ it "returns valid image for a good CSV" do mock_file("http://example.com/test.csv", 'csvs/valid.csv') - Validation.create_validation('http://example.com/test.csv') - validation = Validation.first + Legacy::Validation.create_validation('http://example.com/test.csv') + validation = Legacy::Validation.first get 'show', id: validation.id, format: :png expect(response).to be_success expect(response.body.length).to eq(1588) @@ -63,8 +63,8 @@ it "returns invalid image for a CSV with errors" do mock_file("http://example.com/test.csv", 'csvs/errors.csv') - Validation.create_validation('http://example.com/test.csv') - validation = Validation.first + Legacy::Validation.create_validation('http://example.com/test.csv') + validation = Legacy::Validation.first get 'show', id: validation.id, format: :png expect(response).to be_success expect(response.body.length).to eq(1760) @@ -72,8 +72,8 @@ it "returns warning image for a CSV with warnings" do mock_file("http://example.com/test.csv", 'csvs/warnings.csv') - Validation.create_validation('http://example.com/test.csv') - validation = Validation.first + Legacy::Validation.create_validation('http://example.com/test.csv') + validation = Legacy::Validation.first get 'show', id: validation.id, format: :png expect(response).to be_success expect(response.body.length).to eq(2099) @@ -85,8 +85,8 @@ it "returns valid image for a good CSV" do mock_file("http://example.com/test.csv", 'csvs/valid.csv') - Validation.create_validation('http://example.com/test.csv') - validation = Validation.first + Legacy::Validation.create_validation('http://example.com/test.csv') + validation = Legacy::Validation.first get 'show', id: validation.id, format: :svg expect(response).to be_success expect(response.body).to include(">valid<") @@ -94,8 +94,8 @@ it "returns invalid image for a CSV with errors" do mock_file("http://example.com/test.csv", 'csvs/errors.csv') - Validation.create_validation('http://example.com/test.csv') - validation = Validation.first + Legacy::Validation.create_validation('http://example.com/test.csv') + validation = Legacy::Validation.first get 'show', id: validation.id, format: :svg expect(response).to be_success expect(response.body).to include(">invalid<") @@ -103,8 +103,8 @@ it "returns warning image for a CSV with warnings" do mock_file("http://example.com/test.csv", 'csvs/warnings.csv') - Validation.create_validation('http://example.com/test.csv') - validation = Validation.first + Legacy::Validation.create_validation('http://example.com/test.csv') + validation = Legacy::Validation.first get 'show', id: validation.id, format: :svg expect(response).to be_success expect(response.body).to include(">warnings<") @@ -112,8 +112,8 @@ it "queues another check when the image is loaded" do mock_file("http://example.com/test.csv", 'csvs/valid.csv') - Validation.create_validation('http://example.com/test.csv') - validation = Validation.first + Legacy::Validation.create_validation('http://example.com/test.csv') + validation = Legacy::Validation.first expect { get 'show', id: validation.id, format: :svg }.to change(Sidekiq::Extensions::DelayedClass.jobs, :size).by(1) @@ -121,9 +121,9 @@ it "doesn't queue another check when the image is loaded if revalidate is false" do mock_file("http://example.com/test.csv", 'csvs/valid.csv') - Validation.create_validation('http://example.com/test.csv') - validation = Validation.first - expect(Validation).to_not receive(:delay) + Legacy::Validation.create_validation('http://example.com/test.csv') + validation = Legacy::Validation.first + expect(Legacy::Validation).to_not receive(:delay) expect { get 'show', id: validation.id, format: :svg, revalidate: false }.to change(Sidekiq::Extensions::DelayedClass.jobs, :size).by(0) diff --git a/spec/factories/schemas.rb b/spec/factories/schemas.rb index 595b2601..b0b53b2c 100644 --- a/spec/factories/schemas.rb +++ b/spec/factories/schemas.rb @@ -1,7 +1,7 @@ # Read about factories at https://github.com/thoughtbot/factory_girl FactoryBot.define do - factory :schema do + factory :schema, class: 'legacy/schema' do url { Faker::Internet.url } end end diff --git a/spec/factories/validations.rb b/spec/factories/validations.rb index f2487fa3..1eea2d51 100644 --- a/spec/factories/validations.rb +++ b/spec/factories/validations.rb @@ -1,5 +1,5 @@ FactoryBot.define do - factory :validation do + factory :validation, class: 'legacy/validation' do url { Faker::Internet.url } end end diff --git a/spec/models/package_processor_spec.rb b/spec/models/package_processor_spec.rb index 98156add..4211ae85 100644 --- a/spec/models/package_processor_spec.rb +++ b/spec/models/package_processor_spec.rb @@ -4,7 +4,7 @@ describe PackageProcessor do before(:each) do - @package = Package.create + @package = Legacy::Package.create mock_client = double(Pusher::Channel) allow(Pusher).to receive(:[]) { mock_client } allow(mock_client).to receive(:trigger) diff --git a/spec/models/package_spec.rb b/spec/models/package_spec.rb index 801ddd84..cff74d64 100644 --- a/spec/models/package_spec.rb +++ b/spec/models/package_spec.rb @@ -1,6 +1,6 @@ require "spec_helper" -describe Package, type: :model do +describe Legacy::Package, type: :model do include ActionDispatch::TestProcess diff --git a/spec/models/summary_spec.rb b/spec/models/summary_spec.rb index 4f5201e7..d13f38f1 100644 --- a/spec/models/summary_spec.rb +++ b/spec/models/summary_spec.rb @@ -1,17 +1,17 @@ require "spec_helper" -describe Summary, type: :model do +describe Legacy::Summary, type: :model do context "when generating summary" do before(:each) do mock_file("http://example.com/test.csv", 'csvs/valid.csv') - validation = Validation.create_validation("http://example.com/test.csv") + validation = Legacy::Validation.create_validation("http://example.com/test.csv") mock_file("http://example.com/test2.csv", 'csvs/errors.csv') - validation = Validation.create_validation("http://example.com/test2.csv") + validation = Legacy::Validation.create_validation("http://example.com/test2.csv") mock_file("http://www.example.com/test3.csv", 'csvs/warnings.csv') - validation = Validation.create_validation("http://www.example.com/test3.csv") + validation = Legacy::Validation.create_validation("http://www.example.com/test3.csv") @summary = described_class.generate end @@ -48,7 +48,7 @@ it "should count occurences across population of sources" do mock_file("http://example.com/test4.csv", 'csvs/multiple_errors.csv') - validation = Validation.create_validation("http://example.com/test4.csv") + validation = Legacy::Validation.create_validation("http://example.com/test4.csv") @summary = described_class.generate expect(@summary.sources).to eq(4) expect(@summary.states["invalid"]).to eq(2) @@ -58,7 +58,7 @@ it "should record categories of problem" do mock_file("http://example.com/test4.csv", 'csvs/multiple_errors.csv') - validation = Validation.create_validation("http://example.com/test4.csv") + validation = Legacy::Validation.create_validation("http://example.com/test4.csv") @summary = described_class.generate expect(@summary.sources).to eq(4) expect(@summary.level_summary.errors_breakdown[:ragged_rows]).to eq(2) diff --git a/spec/models/validation_spec.rb b/spec/models/validation_spec.rb index 00e8eb52..c4d3cf4a 100644 --- a/spec/models/validation_spec.rb +++ b/spec/models/validation_spec.rb @@ -1,7 +1,7 @@ require "spec_helper" # require "byebug" -describe Validation, type: :model do +describe Legacy::Validation, type: :model do describe '#expiry_fields' do it "should assign a TTL field to any validation formed from an uploaded CSV file" do diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 685c66d7..19f96441 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -49,7 +49,7 @@ #config.treat_symbols_as_metadata_keys_with_true_values = true config.before(:all) do - Validation.create_indexes + Legacy::Validation.create_indexes Fog.mock! end @@ -85,7 +85,7 @@ end config.after(:all) do - Validation.remove_indexes + Legacy::Validation.remove_indexes end diff --git a/spec/views/validation/validate.html.erb_spec.rb b/spec/views/validation/validate.html.erb_spec.rb index 7902de9b..ca912bc5 100644 --- a/spec/views/validation/validate.html.erb_spec.rb +++ b/spec/views/validation/validate.html.erb_spec.rb @@ -22,7 +22,7 @@ allow(validator).to receive(:line_breaks) { "\n" } allow(validator).to receive(:schema) { nil } - @validation = Validation.create(result: validator) + @validation = Legacy::Validation.create(result: validator) render :partial => "validation/message", :locals => { :message => message, :validator => validator } @@ -31,4 +31,4 @@ end -end \ No newline at end of file +end From a10f99b88c345f23484d9bd2d7f329946fc1371c Mon Sep 17 00:00:00 2001 From: James Smith Date: Thu, 22 Apr 2021 16:49:52 +0100 Subject: [PATCH 04/12] remove gridfs gem, not used in ages --- Gemfile | 1 - Gemfile.lock | 4 ---- 2 files changed, 5 deletions(-) diff --git a/Gemfile b/Gemfile index 18df136a..92313980 100644 --- a/Gemfile +++ b/Gemfile @@ -77,7 +77,6 @@ gem 'rack-google-analytics', '~> 1.2' gem 'mongo', '~> 2.1', git: 'https://github.com/Data-Liberation-Front/mongo-ruby-driver.git', ref: "212-with-authsource-fix" gem 'mongoid', '~> 5.1' gem 'kaminari-mongoid', '~> 1.0' -gem 'mongoid-grid_fs', '~> 2.4' gem 'kaminari', '~> 1.2' gem 'bootstrap-kaminari-views', '~> 0.0' gem 'data_kitten', git: 'https://github.com/Data-Liberation-Front/data_kitten.git', ref: "e343510bd15e3329c1f2fab35035e248195348be", require: false diff --git a/Gemfile.lock b/Gemfile.lock index de23b2de..1d4126d0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -504,9 +504,6 @@ GEM mongo (~> 2.1) origin (~> 2.2) tzinfo (>= 0.3.37) - mongoid-grid_fs (2.4.0) - mime-types (>= 1.0, < 4.0) - mongoid (>= 3.0, < 8.0) multi_json (1.15.0) multi_test (0.1.2) multipart-post (2.1.1) @@ -823,7 +820,6 @@ DEPENDENCIES kaminari-mongoid (~> 1.0) mongo (~> 2.1)! mongoid (~> 5.1) - mongoid-grid_fs (~> 2.4) nokogiri (~> 1.10) poltergeist (~> 1.6) pry (~> 0.14) From bb944fdaf10878c110f018dbbed7e8a55d44184d Mon Sep 17 00:00:00 2001 From: James Smith Date: Thu, 22 Apr 2021 17:19:24 +0100 Subject: [PATCH 05/12] move legacy tests to subdir --- spec/models/{ => legacy}/package_spec.rb | 0 spec/models/{ => legacy}/summary_spec.rb | 0 spec/models/{ => legacy}/validation_spec.rb | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename spec/models/{ => legacy}/package_spec.rb (100%) rename spec/models/{ => legacy}/summary_spec.rb (100%) rename spec/models/{ => legacy}/validation_spec.rb (100%) diff --git a/spec/models/package_spec.rb b/spec/models/legacy/package_spec.rb similarity index 100% rename from spec/models/package_spec.rb rename to spec/models/legacy/package_spec.rb diff --git a/spec/models/summary_spec.rb b/spec/models/legacy/summary_spec.rb similarity index 100% rename from spec/models/summary_spec.rb rename to spec/models/legacy/summary_spec.rb diff --git a/spec/models/validation_spec.rb b/spec/models/legacy/validation_spec.rb similarity index 100% rename from spec/models/validation_spec.rb rename to spec/models/legacy/validation_spec.rb From 9ae28082376ac069e3c487a38366ae5d86b8382a Mon Sep 17 00:00:00 2001 From: James Smith Date: Thu, 22 Apr 2021 17:36:44 +0100 Subject: [PATCH 06/12] install pg gem and fix sqlite3 gem to 1.3 for Rails 4 --- Gemfile | 4 ++-- Gemfile.lock | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Gemfile b/Gemfile index 18df136a..4ff65019 100644 --- a/Gemfile +++ b/Gemfile @@ -34,6 +34,7 @@ group :production do gem 'thin', '~> 1.8' gem 'rails_12factor', '~> 0.0' gem 'puma', '~> 5.2' + gem 'pg', '~> 1.2' end group :development, :test do @@ -53,8 +54,7 @@ group :development, :test do gem 'poltergeist', '~> 1.6' gem 'vcr', '~> 6.0' gem 'cucumber-api-steps', require: false, git: 'https://github.com/Data-Liberation-Front/cucumber-api-steps.git', branch: 'feature-test-content-type' - # Use sqlite3 as the database for Active Record - gem 'sqlite3', '~> 1.4' + gem 'sqlite3', '~> 1.3.0' end group :development do diff --git a/Gemfile.lock b/Gemfile.lock index de23b2de..67c3d179 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -525,6 +525,7 @@ GEM open_uri_redirections (0.2.1) optimist (3.0.1) origin (2.3.1) + pg (1.2.3) poltergeist (1.6.0) capybara (~> 2.1) cliver (~> 0.3.1) @@ -729,7 +730,7 @@ GEM actionpack (>= 4.0) activesupport (>= 4.0) sprockets (>= 3.0.0) - sqlite3 (1.4.2) + sqlite3 (1.3.13) sxp (1.1.0) rdf (~> 3.1) sync (0.5.0) @@ -825,6 +826,7 @@ DEPENDENCIES mongoid (~> 5.1) mongoid-grid_fs (~> 2.4) nokogiri (~> 1.10) + pg (~> 1.2) poltergeist (~> 1.6) pry (~> 0.14) puma (~> 5.2) @@ -843,7 +845,7 @@ DEPENDENCIES sidekiq (~> 4.2) simplecov (~> 0.16) spring (~> 2.1) - sqlite3 (~> 1.4) + sqlite3 (~> 1.3.0) therubyracer (~> 0.12) thin (~> 1.8) timecop (~> 0.9) From 8f4f3f97f3c16738ec4ca94d7b4bb95f6c6156dc Mon Sep 17 00:00:00 2001 From: James Smith Date: Thu, 22 Apr 2021 23:26:30 +0100 Subject: [PATCH 07/12] re-include AR railtie had to specify particular options for DatabaseCleaner too or it got confused. --- config/application.rb | 6 +++++- features/support/env.rb | 17 +---------------- spec/spec_helper.rb | 4 ++-- 3 files changed, 8 insertions(+), 19 deletions(-) diff --git a/config/application.rb b/config/application.rb index 2156cf17..995e8c75 100644 --- a/config/application.rb +++ b/config/application.rb @@ -1,7 +1,7 @@ require File.expand_path('../boot', __FILE__) # Pick the frameworks you want: -#require "active_record/railtie" +require "active_record/railtie" require "action_controller/railtie" require "action_mailer/railtie" require "sprockets/railtie" @@ -30,6 +30,10 @@ class Application < Rails::Application config.middleware.use("Rack::GoogleAnalytics", :tracker => ENV['CSVLINT_ANALYTICS_KEY']) end + # config.generators do |g| + # g.orm :active_record + # end + config.middleware.use Rack::Cors do allow do origins '*' diff --git a/features/support/env.rb b/features/support/env.rb index 6591c4e4..de4b3f9b 100644 --- a/features/support/env.rb +++ b/features/support/env.rb @@ -45,26 +45,11 @@ # Remove/comment out the lines below if your app doesn't have a database. # For some databases (like MongoDB and CouchDB) you may need to use :truncation instead. begin - DatabaseCleaner.strategy = :truncation + DatabaseCleaner[:mongoid].strategy = :truncation rescue NameError raise "You need to add database_cleaner to your Gemfile (in the :test group) if you wish to use it." end -# You may also want to configure DatabaseCleaner to use different strategies for certain features and scenarios. -# See the DatabaseCleaner documentation for details. Example: -# -# Before('@no-txn,@selenium,@culerity,@celerity,@javascript') do -# # { :except => [:widgets] } may not do what you expect here -# # as Cucumber::Rails::Database.javascript_strategy overrides -# # this setting. -# DatabaseCleaner.strategy = :truncation -# end -# -# Before('~@no-txn', '~@selenium', '~@culerity', '~@celerity', '~@javascript') do -# DatabaseCleaner.strategy = :transaction -# end -# - Capybara.register_driver :poltergeist do |app| Capybara::Poltergeist::Driver.new(app, {debug: false}) end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 19f96441..4fa50611 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -21,7 +21,7 @@ ENV['AWS_SECRET_ACCESS_KEY'] = 'fakesecret' ENV['AWS_BUCKET_NAME'] = 'buckethead' -DatabaseCleaner.strategy = :truncation +DatabaseCleaner[:mongoid].strategy = :truncation # Requires supporting ruby files with custom matchers and macros, etc, # in spec/support/ and its subdirectories. @@ -81,7 +81,7 @@ config.order = "random" config.after(:each) do - DatabaseCleaner.clean + DatabaseCleaner[:mongoid].clean end config.after(:all) do From 6221a89989754cfd50fef7e12bea4a90bed54c8a Mon Sep 17 00:00:00 2001 From: James Smith Date: Thu, 22 Apr 2021 23:29:19 +0100 Subject: [PATCH 08/12] run database_cleaner on activerecord as well as mongoid --- features/support/env.rb | 1 + spec/spec_helper.rb | 2 ++ 2 files changed, 3 insertions(+) diff --git a/features/support/env.rb b/features/support/env.rb index de4b3f9b..30b745ce 100644 --- a/features/support/env.rb +++ b/features/support/env.rb @@ -46,6 +46,7 @@ # For some databases (like MongoDB and CouchDB) you may need to use :truncation instead. begin DatabaseCleaner[:mongoid].strategy = :truncation + DatabaseCleaner[:active_record].strategy = :truncation rescue NameError raise "You need to add database_cleaner to your Gemfile (in the :test group) if you wish to use it." end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 4fa50611..929d86ab 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -22,6 +22,7 @@ ENV['AWS_BUCKET_NAME'] = 'buckethead' DatabaseCleaner[:mongoid].strategy = :truncation +DatabaseCleaner[:active_record].strategy = :truncation # Requires supporting ruby files with custom matchers and macros, etc, # in spec/support/ and its subdirectories. @@ -82,6 +83,7 @@ config.after(:each) do DatabaseCleaner[:mongoid].clean + DatabaseCleaner[:active_record].clean end config.after(:all) do From 1e48f664530b40f34f7fe99f4ba04fcfb7bfa2a6 Mon Sep 17 00:00:00 2001 From: James Smith Date: Thu, 22 Apr 2021 23:30:02 +0100 Subject: [PATCH 09/12] change rails model generator to create AR models This code can probably go away once mongoid is removed and stops changing the default. --- config/application.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/config/application.rb b/config/application.rb index 995e8c75..1422c4e4 100644 --- a/config/application.rb +++ b/config/application.rb @@ -30,9 +30,9 @@ class Application < Rails::Application config.middleware.use("Rack::GoogleAnalytics", :tracker => ENV['CSVLINT_ANALYTICS_KEY']) end - # config.generators do |g| - # g.orm :active_record - # end + config.generators do |g| + g.orm :active_record + end config.middleware.use Rack::Cors do allow do From 1ee4ce45f086b0a22bdbca1f5cec354dc3f4a30a Mon Sep 17 00:00:00 2001 From: James Smith Date: Thu, 22 Apr 2021 23:45:18 +0100 Subject: [PATCH 10/12] move legacy factories aside so we can make new ones --- features/step_definitions/fixture_steps.rb | 4 ++-- spec/controllers/application_controller_spec.rb | 2 +- spec/controllers/schemas_controller_spec.rb | 2 +- spec/factories/{validations.rb => legacy_schemas.rb} | 2 +- spec/factories/legacy_validations.rb | 5 +++++ spec/factories/schemas.rb | 7 ------- 6 files changed, 10 insertions(+), 12 deletions(-) rename spec/factories/{validations.rb => legacy_schemas.rb} (54%) create mode 100644 spec/factories/legacy_validations.rb delete mode 100644 spec/factories/schemas.rb diff --git a/features/step_definitions/fixture_steps.rb b/features/step_definitions/fixture_steps.rb index 7f538394..5f768f7b 100644 --- a/features/step_definitions/fixture_steps.rb +++ b/features/step_definitions/fixture_steps.rb @@ -20,7 +20,7 @@ Given(/^there are (\d+) schemas in the database$/) do |num| num.to_i.times do |n| - FactoryBot.create :schema + FactoryBot.create :legacy_schema end end @@ -45,7 +45,7 @@ end Given(/^"(.*?)" has been previously used for validation$/) do |url| - FactoryBot.create :schema, url: url + FactoryBot.create :legacy_schema, url: url end Given(/^the data exceeds the amount the proxy can handle/) do diff --git a/spec/controllers/application_controller_spec.rb b/spec/controllers/application_controller_spec.rb index af6c0224..e6d4df90 100644 --- a/spec/controllers/application_controller_spec.rb +++ b/spec/controllers/application_controller_spec.rb @@ -9,7 +9,7 @@ end it "returns 303 redirect if validation is in DB" do - validation = FactoryBot.create :validation, url: "http://data.com/data.csv" + validation = FactoryBot.create :legacy_validation, url: "http://data.com/data.csv" get 'index', uri: "http://data.com/data.csv" expect(response).to be_redirect expect(response.code).to eq("303") diff --git a/spec/controllers/schemas_controller_spec.rb b/spec/controllers/schemas_controller_spec.rb index 4a8b3b69..d58d9ff8 100644 --- a/spec/controllers/schemas_controller_spec.rb +++ b/spec/controllers/schemas_controller_spec.rb @@ -12,7 +12,7 @@ describe "GET 'index' with uri arg" do it "returns 303 redirect if schema is in DB" do - schema = FactoryBot.create :schema, url: "http://data.com/schema.json" + schema = FactoryBot.create :legacy_schema, url: "http://data.com/schema.json" get 'index', uri: "http://data.com/schema.json" expect(response).to be_redirect expect(response.code).to eq("303") diff --git a/spec/factories/validations.rb b/spec/factories/legacy_schemas.rb similarity index 54% rename from spec/factories/validations.rb rename to spec/factories/legacy_schemas.rb index 1eea2d51..257d6072 100644 --- a/spec/factories/validations.rb +++ b/spec/factories/legacy_schemas.rb @@ -1,5 +1,5 @@ FactoryBot.define do - factory :validation, class: 'legacy/validation' do + factory :legacy_schema, class: 'legacy/schema' do url { Faker::Internet.url } end end diff --git a/spec/factories/legacy_validations.rb b/spec/factories/legacy_validations.rb new file mode 100644 index 00000000..80517548 --- /dev/null +++ b/spec/factories/legacy_validations.rb @@ -0,0 +1,5 @@ +FactoryBot.define do + factory :legacy_validation, class: 'legacy/validation' do + url { Faker::Internet.url } + end +end diff --git a/spec/factories/schemas.rb b/spec/factories/schemas.rb deleted file mode 100644 index b0b53b2c..00000000 --- a/spec/factories/schemas.rb +++ /dev/null @@ -1,7 +0,0 @@ -# Read about factories at https://github.com/thoughtbot/factory_girl - -FactoryBot.define do - factory :schema, class: 'legacy/schema' do - url { Faker::Internet.url } - end -end From 88e28f3d7e54e15d8647af8e61fa05f94ee30461 Mon Sep 17 00:00:00 2001 From: James Smith Date: Fri, 23 Apr 2021 00:24:04 +0100 Subject: [PATCH 11/12] Move pg to default gem group --- Gemfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile b/Gemfile index b8588988..ba7b5c5c 100644 --- a/Gemfile +++ b/Gemfile @@ -29,12 +29,12 @@ gem 'jbuilder', '~> 2.9' gem 'sdoc', '~> 2.1', group: :doc gem 'foreman', '~> 0.87' +gem 'pg', '~> 1.2' group :production do gem 'thin', '~> 1.8' gem 'rails_12factor', '~> 0.0' gem 'puma', '~> 5.2' - gem 'pg', '~> 1.2' end group :development, :test do From 779f7c7c968c8a39979bc797ffbcd46411898071 Mon Sep 17 00:00:00 2001 From: James Smith Date: Fri, 23 Apr 2021 00:37:38 +0100 Subject: [PATCH 12/12] use pg < 1.0 for Rails 4 See https://github.com/rails/rails/issues/31673 --- Gemfile | 2 +- Gemfile.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index ba7b5c5c..cf29aa5d 100644 --- a/Gemfile +++ b/Gemfile @@ -29,7 +29,7 @@ gem 'jbuilder', '~> 2.9' gem 'sdoc', '~> 2.1', group: :doc gem 'foreman', '~> 0.87' -gem 'pg', '~> 1.2' +gem 'pg', '~> 0.18' group :production do gem 'thin', '~> 1.8' diff --git a/Gemfile.lock b/Gemfile.lock index d189a413..08b15555 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -522,7 +522,7 @@ GEM open_uri_redirections (0.2.1) optimist (3.0.1) origin (2.3.1) - pg (1.2.3) + pg (0.21.0) poltergeist (1.6.0) capybara (~> 2.1) cliver (~> 0.3.1) @@ -822,7 +822,7 @@ DEPENDENCIES mongo (~> 2.1)! mongoid (~> 5.1) nokogiri (~> 1.10) - pg (~> 1.2) + pg (~> 0.18) poltergeist (~> 1.6) pry (~> 0.14) puma (~> 5.2)