From 5fc0e4df1a0f93256b4420d2f02b778af7f59ced Mon Sep 17 00:00:00 2001 From: Tilo Sloboda Date: Sun, 7 Jan 2024 15:10:12 -0800 Subject: [PATCH 1/2] v1.10.1 fix issue #268 --- CONTRIBUTORS.md | 1 + lib/smarter_csv/smarter_csv.rb | 2 +- lib/smarter_csv/version.rb | 2 +- spec/smarter_csv/file_encoding_spec.rb | 64 ++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 2 deletions(-) create mode 100644 spec/smarter_csv/file_encoding_spec.rb diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 6c648028..89669533 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -51,3 +51,4 @@ A Big Thank you to everyone who filed issues, sent comments, and who contributed * [Rahul Chaudhary](https://github.com/rahulch95) * [Alessandro Fazzi](https://github.com/pioneerskies) * [JP Camara](https://github.com/jpcamara) + * [Hiro Watari](https://github.com/hirowatari) diff --git a/lib/smarter_csv/smarter_csv.rb b/lib/smarter_csv/smarter_csv.rb index c0fcb459..f7420a62 100644 --- a/lib/smarter_csv/smarter_csv.rb +++ b/lib/smarter_csv/smarter_csv.rb @@ -22,7 +22,7 @@ def SmarterCSV.process(input, given_options = {}, &block) # rubocop:disable Lint begin fh = input.respond_to?(:readline) ? input : File.open(input, "r:#{options[:file_encoding]}") - if @enforce_utf8 && (fh.respond_to?(:external_encoding) && fh.external_encoding != Encoding.find('UTF-8') || fh.respond_to?(:encoding) && fh.encoding != Encoding.find('UTF-8')) + if (options[:force_utf8] || options[:file_encoding] =~ /utf-8/i) && (fh.respond_to?(:external_encoding) && fh.external_encoding != Encoding.find('UTF-8') || fh.respond_to?(:encoding) && fh.encoding != Encoding.find('UTF-8')) puts 'WARNING: you are trying to process UTF-8 input, but did not open the input with "b:utf-8" option. See README file "NOTES about File Encodings".' end diff --git a/lib/smarter_csv/version.rb b/lib/smarter_csv/version.rb index ea86e60d..b3512e7a 100644 --- a/lib/smarter_csv/version.rb +++ b/lib/smarter_csv/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SmarterCSV - VERSION = "1.10.0" + VERSION = "1.10.1" end diff --git a/spec/smarter_csv/file_encoding_spec.rb b/spec/smarter_csv/file_encoding_spec.rb new file mode 100644 index 00000000..dce39fff --- /dev/null +++ b/spec/smarter_csv/file_encoding_spec.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe SmarterCSV do + describe 'encoding warning message' do + let(:file_path) { 'path/to/csvfile.csv' } + let(:file_content) { "some,content\nwith,lines" } + let(:file_double) { StringIO.new(file_content) } + + before do + allow(File).to receive(:open).with(file_path, anything).and_return(file_double) + end + + context 'with force_utf8 option and non-UTF-8 file encoding' do + let(:options) { { force_utf8: true } } + + before do + allow(file_double).to receive(:external_encoding).and_return(Encoding.find('ISO-8859-1')) + end + + it 'prints a warning about UTF-8 processing' do + expect { described_class.process(file_path, options) }.to output(/WARNING: you are trying to process UTF-8 input/).to_stdout + end + end + + context 'with utf-8 file_encoding option and non-UTF-8 file encoding' do + let(:options) { { file_encoding: 'utf-8' } } + + before do + allow(file_double).to receive(:external_encoding).and_return(Encoding.find('ISO-8859-1')) + end + + it 'prints a warning about UTF-8 processing' do + expect { described_class.process(file_path, options) }.to output(/WARNING: you are trying to process UTF-8 input/).to_stdout + end + end + + context 'with non-matching file_encoding option and non-UTF-8 file encoding' do + let(:options) { { file_encoding: 'other-encoding' } } + + before do + allow(file_double).to receive(:external_encoding).and_return(Encoding.find('ISO-8859-1')) + end + + it 'does not print a warning about UTF-8 processing' do + expect { described_class.process(file_path, options) }.not_to output(/WARNING: you are trying to process UTF-8 input/).to_stdout + end + end + + context 'with force_utf8 option and UTF-8 file encoding' do + let(:options) { { force_utf8: true } } + + before do + allow(file_double).to receive(:external_encoding).and_return(Encoding.find('UTF-8')) + end + + it 'does not print a warning about UTF-8 processing' do + expect { described_class.process(file_path, options) }.not_to output(/WARNING: you are trying to process UTF-8 input/).to_stdout + end + end + end +end + From 4a2ef3e9269142b2f97fe618cbab4cf550a95693 Mon Sep 17 00:00:00 2001 From: Tilo Sloboda Date: Sun, 7 Jan 2024 15:13:33 -0800 Subject: [PATCH 2/2] changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 85981362..464b0484 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ # SmarterCSV 1.x Change Log +## 1.10.1 (2024-01-07) + * fix incorrect warning about UTF-8 (issue #268, thanks hirowatari) + ## 1.10.0 (2023-12-31) ⚡ BREAKING ⚡ * BREAKING CHANGES: