From cadff25dfabd1f59e7d894292437277aa8cd1a26 Mon Sep 17 00:00:00 2001 From: Bernardo Amorim Date: Fri, 12 May 2017 14:58:29 -0300 Subject: [PATCH 01/13] Write minimal PreParser --- lib/kuniri/language/language.rb | 11 ++++++++++- lib/kuniri/language/ruby/ruby_syntax.rb | 2 ++ lib/kuniri/preparser/preparser.rb | 18 ++++++++++++++++++ lib/kuniri/preparser/ruby.rb | 23 +++++++++++++++++++++++ 4 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 lib/kuniri/preparser/preparser.rb create mode 100644 lib/kuniri/preparser/ruby.rb diff --git a/lib/kuniri/language/language.rb b/lib/kuniri/language/language.rb index 65eb34a..a2b4952 100644 --- a/lib/kuniri/language/language.rb +++ b/lib/kuniri/language/language.rb @@ -4,6 +4,8 @@ # This source code is licensed under the GNU lesser general public license, # Version 3. See the file COPYING for more details +require 'fileutils' + require_relative '../state_machine/OO_structured_fsm/attribute_state' require_relative '../state_machine/OO_structured_fsm/class_state' require_relative '../state_machine/OO_structured_fsm/constructor_state' @@ -113,8 +115,15 @@ def set_source(pSource) # method, work like a hook for give more flexibility to implements any # needed steps. def analyse_source(fileElement, source) - analyse_first_step(fileElement, source) + temp_path = temp_file_path(source) + @preParser.pre_parse(source, temp_path) # should be instantiated on child classes + analyse_first_step(fileElement, File.open(temp_path)) analyse_second_step + FileUtils.rm(temp_path) + end + + def temp_file_path(source) + return source.path + ".tmp" end def analyse_first_step(_fileElement, _source) diff --git a/lib/kuniri/language/ruby/ruby_syntax.rb b/lib/kuniri/language/ruby/ruby_syntax.rb index 7e6abc1..61f4683 100644 --- a/lib/kuniri/language/ruby/ruby_syntax.rb +++ b/lib/kuniri/language/ruby/ruby_syntax.rb @@ -23,6 +23,7 @@ require_relative 'method_ruby' require_relative 'aggregation_ruby' require_relative '../metadata' +require_relative '../../../kuniri/preparser/ruby.rb' module Languages @@ -30,6 +31,7 @@ module Languages class RubySyntax < Languages::Language def initialize super + @preParser = RubyPreParser.new @externRequirementHandler = Languages::Ruby::ExternRequirementRuby.new @variableHandler = Languages::Ruby::VariableGlobalRuby.new @functionHandler = Languages::Ruby::FunctionBehaviorRuby.new diff --git a/lib/kuniri/preparser/preparser.rb b/lib/kuniri/preparser/preparser.rb new file mode 100644 index 0000000..c99ef6a --- /dev/null +++ b/lib/kuniri/preparser/preparser.rb @@ -0,0 +1,18 @@ +class PreParser + def pre_parse(source, out_path) + pre_parsed_lines = [] + for line in source + pre_parsed_line = remove_comments(line) + pre_parsed_line.slice!("\n") + pre_parsed_lines.push(pre_parsed_line) unless (pre_parsed_line.empty?) + end + out_file = File.open(out_path, "w") + for pre_parsed_line in pre_parsed_lines + out_file.write(pre_parsed_lines) + end + end + + def remove_comments(line) + raise NotImplementedError + end +end diff --git a/lib/kuniri/preparser/ruby.rb b/lib/kuniri/preparser/ruby.rb new file mode 100644 index 0000000..1b9750e --- /dev/null +++ b/lib/kuniri/preparser/ruby.rb @@ -0,0 +1,23 @@ +require_relative 'preparser.rb' + +class RubyPreParser < PreParser + def initialize + @on_multiline_comment = false + end + def remove_comments(line) + if line =~ /^=begin(.*?)/ + @on_multiline_comment = true + end + if @on_multiline_comment + pre_parsed_line = '' + else + comment_regex = /(?[^#]*)#.*/ + match = line.match(comment_regex) + pre_parsed_line = if not match.nil? then match["non_comm"] else line end + end + if line =~ /^=end/ + @on_multiline_comment = false + end + return pre_parsed_line + end +end From d9d4c13d0489927ce5f36da91f7c3864389dcb3d Mon Sep 17 00:00:00 2001 From: Bernardo Amorim Date: Fri, 12 May 2017 15:35:05 -0300 Subject: [PATCH 02/13] Raise rubocop's maximum allowed conditional branch size --- .rubocop.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.rubocop.yml b/.rubocop.yml index 4182958..60b1d16 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -45,7 +45,7 @@ Metrics/LineLength: # Offense count: 86 Metrics/AbcSize: - Max: 22 + Max: 25 # Offense count: 32 # Configuration parameters: CountComments. From 1d66e3bb28cc4a9d60f2f82db8a32d8c47d1d0f8 Mon Sep 17 00:00:00 2001 From: Bernardo Amorim Date: Wed, 24 May 2017 14:28:13 -0300 Subject: [PATCH 03/13] Change comment and switch from double quotes to single ones Signed-off-by: Bruno Sesso Signed-off-by: Bernardo Amorim --- lib/kuniri/language/language.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/kuniri/language/language.rb b/lib/kuniri/language/language.rb index a2b4952..2e70814 100644 --- a/lib/kuniri/language/language.rb +++ b/lib/kuniri/language/language.rb @@ -116,14 +116,14 @@ def set_source(pSource) # needed steps. def analyse_source(fileElement, source) temp_path = temp_file_path(source) - @preParser.pre_parse(source, temp_path) # should be instantiated on child classes + @preParser.pre_parse(source, temp_path) # instantiated on child classes analyse_first_step(fileElement, File.open(temp_path)) analyse_second_step FileUtils.rm(temp_path) end def temp_file_path(source) - return source.path + ".tmp" + return source.path + '.tmp' end def analyse_first_step(_fileElement, _source) From 73819ca60e49fc47afaaf8c089a177dada923691 Mon Sep 17 00:00:00 2001 From: Bernardo Amorim Date: Wed, 24 May 2017 14:32:28 -0300 Subject: [PATCH 04/13] Implement single-line and multiple-line pre-parsing Signed-off-by: Bruno Sesso Signed-off-by: Bernardo Amorim --- lib/kuniri/preparser/preparser.rb | 45 +++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/lib/kuniri/preparser/preparser.rb b/lib/kuniri/preparser/preparser.rb index c99ef6a..17208ff 100644 --- a/lib/kuniri/preparser/preparser.rb +++ b/lib/kuniri/preparser/preparser.rb @@ -1,18 +1,47 @@ class PreParser def pre_parse(source, out_path) pre_parsed_lines = [] - for line in source - pre_parsed_line = remove_comments(line) - pre_parsed_line.slice!("\n") + source.each do |line| + pre_parsed_line = pre_parse_line(line) pre_parsed_lines.push(pre_parsed_line) unless (pre_parsed_line.empty?) end - out_file = File.open(out_path, "w") - for pre_parsed_line in pre_parsed_lines - out_file.write(pre_parsed_lines) - end + pre_parse_multiple_lines(pre_parsed_lines) + out_file = File.open(out_path, 'w') + out_file.write(pre_parsed_lines.join("\n")) + out_file.close + end + + def pre_parse_line(pre_parsed_line) + pre_parsed_line = remove_whitespaces(pre_parsed_line) + pre_parsed_line = remove_newline(pre_parsed_line) + return pre_parsed_line + end + + def pre_parse_multiple_lines(lines) + lines = zip_multiple_lines_command(lines) + lines = split_multiple_command_line(lines) + return lines + end + + def split_multiple_command_line(_lines) + raise NotImplementedError end - def remove_comments(line) + def zip_multiple_lines_command(_lines) raise NotImplementedError end + + def remove_whitespaces(line) + return line.strip + end + + def remove_comments(_line) + raise NotImplementedError + end + + def remove_newline(line) + no_newline = line + no_newline.slice("\n") + return no_newline + end end From 0cac9733d896c88c9a70df1aa85a1849229539d4 Mon Sep 17 00:00:00 2001 From: Bernardo Amorim Date: Wed, 24 May 2017 14:35:18 -0300 Subject: [PATCH 05/13] Comply file to rubocop Signed-off-by: Bruno Sesso Signed-off-by: Bernardo Amorim --- lib/kuniri/preparser/ruby.rb | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/kuniri/preparser/ruby.rb b/lib/kuniri/preparser/ruby.rb index 1b9750e..6d8283c 100644 --- a/lib/kuniri/preparser/ruby.rb +++ b/lib/kuniri/preparser/ruby.rb @@ -4,20 +4,21 @@ class RubyPreParser < PreParser def initialize @on_multiline_comment = false end + def remove_comments(line) - if line =~ /^=begin(.*?)/ - @on_multiline_comment = true - end + @on_multiline_comment = true if line =~ /^=begin(.*?)/ if @on_multiline_comment pre_parsed_line = '' else comment_regex = /(?[^#]*)#.*/ match = line.match(comment_regex) - pre_parsed_line = if not match.nil? then match["non_comm"] else line end - end - if line =~ /^=end/ - @on_multiline_comment = false + pre_parsed_line = match.nil? ? line : match['non_comm'] end + @on_multiline_comment = false if line =~ /^=end/ return pre_parsed_line end + + def zip_multiple_lines_command(line) end + + def split_multiple_command_line(line) end end From 3c95cd92bf062dd4747c2e988556538f814a1daf Mon Sep 17 00:00:00 2001 From: Bernardo Amorim Date: Wed, 24 May 2017 15:00:37 -0300 Subject: [PATCH 06/13] Comply file to rubocop Signed-off-by: Bruno Sesso Signed-off-by: Bernardo Amorim --- lib/kuniri/language/language.rb | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/kuniri/language/language.rb b/lib/kuniri/language/language.rb index 2e70814..cd6178b 100644 --- a/lib/kuniri/language/language.rb +++ b/lib/kuniri/language/language.rb @@ -115,11 +115,10 @@ def set_source(pSource) # method, work like a hook for give more flexibility to implements any # needed steps. def analyse_source(fileElement, source) - temp_path = temp_file_path(source) - @preParser.pre_parse(source, temp_path) # instantiated on child classes - analyse_first_step(fileElement, File.open(temp_path)) + @preParser.pre_parse(source, temp_file_path(source)) + analyse_first_step(fileElement, File.open(temp_file_path(source))) analyse_second_step - FileUtils.rm(temp_path) + FileUtils.rm(temp_file_path(source)) end def temp_file_path(source) @@ -223,8 +222,7 @@ def less_nested # Verify if is nested or not def nested? - return true if @countNestedCondLoop.positive? - return false + return countNestedCondLoop.positive? end # Reset nested structure From c7096fabb996839f6539f0f804afe28780852009 Mon Sep 17 00:00:00 2001 From: Bernardo Amorim Date: Wed, 24 May 2017 16:01:39 -0300 Subject: [PATCH 07/13] Implement multiple-line statement zipping Signed-off-by: Bruno Sesso Signed-off-by: Bernardo Amorim --- lib/kuniri/preparser/ruby.rb | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/lib/kuniri/preparser/ruby.rb b/lib/kuniri/preparser/ruby.rb index 6d8283c..1b06630 100644 --- a/lib/kuniri/preparser/ruby.rb +++ b/lib/kuniri/preparser/ruby.rb @@ -1,7 +1,12 @@ require_relative 'preparser.rb' + + class RubyPreParser < PreParser + + def initialize + @LINE_CONTINUATION_CHARS = ['.', ','] @on_multiline_comment = false end @@ -18,7 +23,26 @@ def remove_comments(line) return pre_parsed_line end - def zip_multiple_lines_command(line) end + def zip_multiple_lines_command(lines) + zipped_lines = [] + curr_line = "" + append_next = true + for line in lines + if append_next + curr_line << line + else + zipped_lines.push(curr_line) + curr_line = line + end + if @LINE_CONTINUATION_CHARS.include?(line[-1]) + append_next = true + else + append_next = false + end + end + zipped_lines.push(curr_line) + return zipped_lines + end def split_multiple_command_line(line) end end From 5a0476997916677d49a2ccfc06c3d52f7799411a Mon Sep 17 00:00:00 2001 From: Bernardo Amorim Date: Thu, 25 May 2017 10:11:48 -0300 Subject: [PATCH 08/13] Switch from function with side-effects to a value-returning one Signed-off-by: Bruno Sesso Signed-off-by: Bernardo Amorim --- lib/kuniri/preparser/preparser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/kuniri/preparser/preparser.rb b/lib/kuniri/preparser/preparser.rb index 17208ff..f355bb4 100644 --- a/lib/kuniri/preparser/preparser.rb +++ b/lib/kuniri/preparser/preparser.rb @@ -5,7 +5,7 @@ def pre_parse(source, out_path) pre_parsed_line = pre_parse_line(line) pre_parsed_lines.push(pre_parsed_line) unless (pre_parsed_line.empty?) end - pre_parse_multiple_lines(pre_parsed_lines) + pre_parsed_lines = pre_parse_multiple_lines(pre_parsed_lines) out_file = File.open(out_path, 'w') out_file.write(pre_parsed_lines.join("\n")) out_file.close From 49755eba87127ccce46b1bd93ca43f18028b62f7 Mon Sep 17 00:00:00 2001 From: Bernardo Amorim Date: Thu, 25 May 2017 10:14:41 -0300 Subject: [PATCH 09/13] Implement prototype of 'split_multiple_command_line' Signed-off-by: Bruno Sesso Signed-off-by: Bernardo Amorim --- lib/kuniri/preparser/ruby.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/kuniri/preparser/ruby.rb b/lib/kuniri/preparser/ruby.rb index 1b06630..d497443 100644 --- a/lib/kuniri/preparser/ruby.rb +++ b/lib/kuniri/preparser/ruby.rb @@ -44,5 +44,7 @@ def zip_multiple_lines_command(lines) return zipped_lines end - def split_multiple_command_line(line) end + def split_multiple_command_line(line) + return line + end end From 304164760c83cccf1829bee81c814c90f329bf55 Mon Sep 17 00:00:00 2001 From: Bernardo Amorim Date: Thu, 25 May 2017 11:27:29 -0300 Subject: [PATCH 10/13] Adapt tests to Kuniri with preparser Signed-off-by: Bruno Sesso Signed-off-by: Bernardo Amorim Signed-off-by: Lucas Kanashiro Signed-off-by: Giuliano Belinassi --- spec/language/ruby/ruby_syntax_spec.rb | 4 ++-- .../comment/simple_multiple_line_comment_class.rb | 2 +- .../comment/simple_multiple_line_comment_global.rb | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/spec/language/ruby/ruby_syntax_spec.rb b/spec/language/ruby/ruby_syntax_spec.rb index 5f433ef..28c115e 100644 --- a/spec/language/ruby/ruby_syntax_spec.rb +++ b/spec/language/ruby/ruby_syntax_spec.rb @@ -541,7 +541,7 @@ def openFile(file) @syntax.analyse_source(instances[0], instances[1]) expect(@syntax.fileElements[0].comments) - .to eq('Comment 1: Multiple line.') + .to eq('Comment 1: Multiple line') expect(@syntax.fileElements[0].global_functions[0].comments) .to eq('Global Function number one multiple line') expect(@syntax.fileElements[0].global_variables[0].comments) @@ -557,7 +557,7 @@ def openFile(file) @syntax.analyse_source(instances[0], instances[1]) expect(@syntax.fileElements[0].classes[0].comments) - .to eq('This is the first class of this file.') + .to eq('This is the first class of this file') expect(@syntax.fileElements[0].classes[0].constructors[0].comments) .to eq('Constructor initialize') expect(@syntax.fileElements[0].classes[0].methods[0].comments) diff --git a/spec/samples/rubySyntaxParts/comment/simple_multiple_line_comment_class.rb b/spec/samples/rubySyntaxParts/comment/simple_multiple_line_comment_class.rb index bb77738..de142dc 100644 --- a/spec/samples/rubySyntaxParts/comment/simple_multiple_line_comment_class.rb +++ b/spec/samples/rubySyntaxParts/comment/simple_multiple_line_comment_class.rb @@ -1,7 +1,7 @@ =begin This is the first class -of this file. +of this file =end class Xpto diff --git a/spec/samples/rubySyntaxParts/comment/simple_multiple_line_comment_global.rb b/spec/samples/rubySyntaxParts/comment/simple_multiple_line_comment_global.rb index edfa012..b8a3f67 100644 --- a/spec/samples/rubySyntaxParts/comment/simple_multiple_line_comment_global.rb +++ b/spec/samples/rubySyntaxParts/comment/simple_multiple_line_comment_global.rb @@ -1,6 +1,6 @@ =begin Comment 1: -Multiple line. +Multiple line =end require_relative 'simple_multiple_line_comment_class.rb' From 9afdec0fb3664a1a0c1b596b2fb13e41fddc868e Mon Sep 17 00:00:00 2001 From: Bernardo Amorim Date: Thu, 25 May 2017 11:33:18 -0300 Subject: [PATCH 11/13] Write tests for RubyPreParser Signed-off-by: Bruno Sesso Signed-off-by: Bernardo Amorim Signed-off-by: Lucas Kanashiro Signed-off-by: Giuliano Belinassi --- spec/preparser/ruby_spec.rb | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 spec/preparser/ruby_spec.rb diff --git a/spec/preparser/ruby_spec.rb b/spec/preparser/ruby_spec.rb new file mode 100644 index 0000000..e6570d7 --- /dev/null +++ b/spec/preparser/ruby_spec.rb @@ -0,0 +1,30 @@ +require_relative '../spec_helper' + + +RSpec.describe Object do + + before :all do + @pre_parser = RubyPreParser.new + end + + + context "zip multiple lines" do + it "Multiple-line function declaration" do + multi_line_function = ["def foo(a,","b,","c)"] + zipped_function = @pre_parser.zip_multiple_lines_command(multi_line_function) + expect(zipped_function).to eq ["def foo(a,b,c)"] + end + it "Multiple-line method chaining" do + multi_line_chain = ["foo.", "a.", "b"] + zipped_chain = @pre_parser.zip_multiple_lines_command(multi_line_chain) + expect(zipped_chain).to eq ["foo.a.b"] + end + it "Multiple-line function with one new line per parameter" do + multi_line_function = ["def foo(", "a,","b,","c)"] + zipped_function = @pre_parser.zip_multiple_lines_command(multi_line_function) + expect(zipped_function).to eq ["def foo(a,b,c)"] + end + end + +end + From d272467f183e06eb9f40f51bdb2701d9cf0291e7 Mon Sep 17 00:00:00 2001 From: Bernardo Amorim Date: Thu, 25 May 2017 11:34:25 -0300 Subject: [PATCH 12/13] Rename multiple-command-line-splitting function Signed-off-by: Bruno Sesso Signed-off-by: Bernardo Amorim Signed-off-by: Lucas Kanashiro Signed-off-by: Giuliano Belinassi --- lib/kuniri/preparser/preparser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/kuniri/preparser/preparser.rb b/lib/kuniri/preparser/preparser.rb index f355bb4..87c4f3c 100644 --- a/lib/kuniri/preparser/preparser.rb +++ b/lib/kuniri/preparser/preparser.rb @@ -19,7 +19,7 @@ def pre_parse_line(pre_parsed_line) def pre_parse_multiple_lines(lines) lines = zip_multiple_lines_command(lines) - lines = split_multiple_command_line(lines) + lines = split_multiple_command_lines(lines) return lines end From 47f375b88c3e0fb70f99911f947df9d272910de6 Mon Sep 17 00:00:00 2001 From: Bernardo Amorim Date: Thu, 25 May 2017 11:35:58 -0300 Subject: [PATCH 13/13] Comply file to rubocop and add parenthesis as line-zipping char Signed-off-by: Bruno Sesso Signed-off-by: Bernardo Amorim Signed-off-by: Lucas Kanashiro Signed-off-by: Giuliano Belinassi --- lib/kuniri/preparser/ruby.rb | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/lib/kuniri/preparser/ruby.rb b/lib/kuniri/preparser/ruby.rb index d497443..3e961e6 100644 --- a/lib/kuniri/preparser/ruby.rb +++ b/lib/kuniri/preparser/ruby.rb @@ -1,12 +1,9 @@ require_relative 'preparser.rb' - - class RubyPreParser < PreParser - def initialize - @LINE_CONTINUATION_CHARS = ['.', ','] + @LINE_CONTINUATION_CHARS = ['.', ',', '('] @on_multiline_comment = false end @@ -25,26 +22,22 @@ def remove_comments(line) def zip_multiple_lines_command(lines) zipped_lines = [] - curr_line = "" + curr_line = '' append_next = true - for line in lines + lines.each do |line| if append_next curr_line << line else zipped_lines.push(curr_line) curr_line = line end - if @LINE_CONTINUATION_CHARS.include?(line[-1]) - append_next = true - else - append_next = false - end + append_next = @LINE_CONTINUATION_CHARS.include?(line[-1]) end zipped_lines.push(curr_line) return zipped_lines end - def split_multiple_command_line(line) - return line + def split_multiple_command_lines(lines) + return lines end end