From df51d238601cc73f94578af15430718fec29994a Mon Sep 17 00:00:00 2001 From: Fanon Jupkwo Date: Fri, 12 May 2023 18:52:32 +0100 Subject: [PATCH 1/8] starting to implement encoding --- .../core/internal/ConverterDefault.java | 17 +- .../api/rest/ConvertController.java | 6 +- .../test/api/JsGeneratorApiTest.java | 549 +++++++++++++++++- .../jsgenerator/test/core/ConverterTest.java | 101 ++-- 4 files changed, 609 insertions(+), 64 deletions(-) diff --git a/jsgenerator-core/src/main/java/com/osscameroon/jsgenerator/core/internal/ConverterDefault.java b/jsgenerator-core/src/main/java/com/osscameroon/jsgenerator/core/internal/ConverterDefault.java index a9806a1d..f0bfe3ba 100644 --- a/jsgenerator-core/src/main/java/com/osscameroon/jsgenerator/core/internal/ConverterDefault.java +++ b/jsgenerator-core/src/main/java/com/osscameroon/jsgenerator/core/internal/ConverterDefault.java @@ -23,8 +23,16 @@ import static java.lang.String.format; import static java.lang.String.join; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.jsoup.parser.Parser.xmlParser; +//TODO: Think about the user will use this library , if needed provide 4 explicit methods for these 4 cases +// code html to code js +// code html to file js +// file html to code js +// file html to file js + + public class ConverterDefault implements Converter { private static final List BOOLEAN_ATTRIBUTES = List.of("allowfullscreen", "async", "autofocus", "autoplay", "checked", "controls", "default", "defer", "disabled", "formnovalidate", "ismap", "itemscope", @@ -57,16 +65,19 @@ public void convert(InputStream inputStream, OutputStream outputStream, Configur final var scanner = new Scanner(inputStream); while (scanner.hasNext()) { - stringBuilder.append(scanner.nextLine()); + stringBuilder.append(new String(scanner.nextLine().getBytes(UTF_8), UTF_8)); } final var content = stringBuilder.toString(); + //Encoding in UTF8 then decoding in UTF8 + var contentWithUTF8Charset = content; + // NOTE: There is nothing to do - if (content.isBlank()) return; + if (contentWithUTF8Charset.isBlank()) return; final var variableNameStrategy = configuration.getVariableNameStrategy(); - final var document = Jsoup.parse(content, xmlParser()); + final var document = Jsoup.parse(contentWithUTF8Charset, xmlParser()); final var writer = new OutputStreamWriter(outputStream); final var selector = configuration.getTargetElementSelector(); diff --git a/jsgenerator-slim-api/src/main/java/com/osscameroon/jsgenerator/api/rest/ConvertController.java b/jsgenerator-slim-api/src/main/java/com/osscameroon/jsgenerator/api/rest/ConvertController.java index 1178825a..c0ebd5c1 100644 --- a/jsgenerator-slim-api/src/main/java/com/osscameroon/jsgenerator/api/rest/ConvertController.java +++ b/jsgenerator-slim-api/src/main/java/com/osscameroon/jsgenerator/api/rest/ConvertController.java @@ -24,6 +24,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; import java.util.Optional; @@ -49,7 +50,7 @@ public class ConvertController { private final OutputStreamResolver pathOutputStreamResolver; private final Converter converter; - //TODO: Make sure all these 4 case are taken into account + //TODO: Make sure all these 4 cases are taken into account // code html to code js OK // code html to file js // file html to code js @@ -67,6 +68,7 @@ public Reply> convertAction(@RequestBody @Valid .map(content -> convert( configuration, new ByteArrayOutputStream(), + //convertInlineContentWithCopyrightCharacterWithComment works after doing this, why ? It used to work without this ? What happened ? new ByteArrayInputStream(content.getBytes(UTF_8)))) .map(content -> { final var filename = inlineOutputStreamResolver.resolve(options.getPattern(), Map.of( @@ -111,7 +113,7 @@ public MultiValueMap convertAction(@RequestPart("optio return new Output(filename, content); }) .forEach(output -> - map.add(output.getFilename(), new ByteArrayResource(output.getContent().getBytes(UTF_8)))); + map.add(output.getFilename(), new ByteArrayResource(output.getContent().getBytes()))); return map; } diff --git a/jsgenerator-test/jsgenerator-test-api/src/test/java/com/osscameroon/jsgenerator/test/api/JsGeneratorApiTest.java b/jsgenerator-test/jsgenerator-test-api/src/test/java/com/osscameroon/jsgenerator/test/api/JsGeneratorApiTest.java index f3c235b6..06e671ab 100644 --- a/jsgenerator-test/jsgenerator-test-api/src/test/java/com/osscameroon/jsgenerator/test/api/JsGeneratorApiTest.java +++ b/jsgenerator-test/jsgenerator-test-api/src/test/java/com/osscameroon/jsgenerator/test/api/JsGeneratorApiTest.java @@ -222,6 +222,7 @@ void convertInlineContent(final VariableDeclaration variableDeclaration, final b } + @ParameterizedTest @MethodSource("provideVariableDeclarationsAndQuerySelectorAddedAndCommentConversionModeActivated") void convertInlineContentWithComment(final VariableDeclaration variableDeclaration, final boolean querySelectorAdded,final boolean commentConversionModeActivated) throws Exception { @@ -362,7 +363,6 @@ void convertInlineContentWithComment(final VariableDeclaration variableDeclarati } } - @ParameterizedTest @MethodSource("provideVariableDeclarationsAndQuerySelectorAdded") void convertUploadedFilesContent(final VariableDeclaration variableDeclaration, final boolean querySelectorAdded) throws Exception { @@ -380,7 +380,7 @@ void convertUploadedFilesContent(final VariableDeclaration variableDeclaration, "extension", ".%s".formatted(extension), "querySelectorAdded", true, "commentConversionModeActivated", true - )).getBytes(UTF_8))) + )).getBytes())) .file(new MockMultipartFile( "files", SAMPLE_INPUT.getFilename(), MULTIPART_FORM_DATA_VALUE, SAMPLE_INPUT.getInputStream())) .file(new MockMultipartFile( @@ -427,10 +427,549 @@ void convertUploadedFilesContent(final VariableDeclaration variableDeclaration, } + } + + /* + * We got an issue related to the encoding-decoding process of copyright character ©, the test method named "convertUploadedFilesContentWithComment" was failing. + * https://github.com/osscameroon/js-generator/issues/238 + * Finally this test helped us to understand that there was no issue when the output is just code but issue occurred when output was a js file. + * */ + //@Disabled + @ParameterizedTest + @MethodSource("provideVariableDeclarationsAndQuerySelectorAddedAndCommentConversionModeActivated") + void convertInlineContentWithCopyrightCharacterWithComment(final VariableDeclaration variableDeclaration, final boolean querySelectorAdded, final boolean commentConversionModeActivated) throws Exception { + final var keyword = keyword(variableDeclaration); + final var extension = randomUUID().toString(); + final var prefix = randomUUID().toString(); + final var content = randomUUID().toString(); + final var input = """ + + + + + + Sample + + + +
+ +
+

Main

+

This is the main content.

+ +
+ +
+ + +"""; + + if (querySelectorAdded) { + if(commentConversionModeActivated){ + + mockMvc.perform(post(ConvertController.MAPPING) + .header(CONTENT_TYPE, APPLICATION_JSON) + .content(objectMapper.writeValueAsString(of( + "contents", List.of(input), + "pattern", "%s.{{ index }}{{ extension }}".formatted(prefix), + "variableDeclaration", variableDeclaration, + "extension", ".%s".formatted(extension), + "querySelectorAdded", true, + "commentConversionModeActivated", true + )))) + .andExpectAll( + status().isOk(), + header().string(CONTENT_TYPE, APPLICATION_JSON_VALUE), + jsonPath("$.status").value("SUCCESS"), + jsonPath("$.content").isArray(), + jsonPath("$.content.length()").value(1), + jsonPath("$.content.[0].filename").value("%s.0.%s".formatted(prefix, extension)), + jsonPath("$.content.[0].content").value(new Match(new String[]{ + "%s targetElement_000 = document.querySelector(`:root > body`);".formatted(keyword), + "%s html_000 = document.createElement('html');".formatted(keyword), + "%s text_000 = document.createTextNode(` `);".formatted(keyword), + "html_000.appendChild(text_000);", + "%s head_000 = document.createElement('head');".formatted(keyword), + "%s text_001 = document.createTextNode(` `);".formatted(keyword), + "head_000.appendChild(text_001);", + "%s meta_000 = document.createElement('meta');".formatted(keyword), + "meta_000.setAttribute(`charset`, `utf-8`);", + "head_000.appendChild(meta_000);", + "%s text_002 = document.createTextNode(` `);".formatted(keyword), + "head_000.appendChild(text_002);", + "%s title_000 = document.createElement('title');".formatted(keyword), + "%s text_003 = document.createTextNode(`Sample`);".formatted(keyword), + "title_000.appendChild(text_003);", + "head_000.appendChild(title_000);", + "%s text_004 = document.createTextNode(` `);".formatted(keyword), + "head_000.appendChild(text_004);", + "%s link_000 = document.createElement('link');".formatted(keyword), + "link_000.setAttribute(`rel`, `stylesheet`);", + "link_000.setAttribute(`href`, ``);", + "head_000.appendChild(link_000);", + "%s text_005 = document.createTextNode(` `);".formatted(keyword), + "head_000.appendChild(text_005);", + "html_000.appendChild(head_000);", + "%s text_006 = document.createTextNode(` `);".formatted(keyword), + "html_000.appendChild(text_006);", + "%s body_000 = document.createElement('body');".formatted(keyword), + "%s text_007 = document.createTextNode(` `);".formatted(keyword), + "body_000.appendChild(text_007);", + "%s div_000 = document.createElement('div');".formatted(keyword), + "div_000.setAttribute(`id`, `container`);", + "%s text_008 = document.createTextNode(` `);".formatted(keyword), + "div_000.appendChild(text_008);", + "%s div_001 = document.createElement('div');".formatted(keyword), + "div_001.setAttribute(`id`, `header`);", + "%s text_009 = document.createTextNode(` `);".formatted(keyword), + "div_001.appendChild(text_009);", + "%s comment_000 = document.createComment(` Sample H1 `);".formatted(keyword), + "div_001.appendChild(comment_000);", + "%s text_010 = document.createTextNode(` `);".formatted(keyword), + "div_001.appendChild(text_010);", + "%s h1_000 = document.createElement('h1');".formatted(keyword), + "%s text_011 = document.createTextNode(`Sample`);".formatted(keyword), + "h1_000.appendChild(text_011);", + "div_001.appendChild(h1_000);", + "%s text_012 = document.createTextNode(` `);".formatted(keyword), + "div_001.appendChild(text_012);", + "%s img_000 = document.createElement('img');".formatted(keyword), + "img_000.setAttribute(`src`, `kanye.jpg`);", + "img_000.setAttribute(`alt`, `kanye`);", + "div_001.appendChild(img_000);", + "%s text_013 = document.createTextNode(` `);".formatted(keyword), + "div_001.appendChild(text_013);", + "div_000.appendChild(div_001);", + "%s text_014 = document.createTextNode(` `);".formatted(keyword), + "div_000.appendChild(text_014);", + "%s div_002 = document.createElement('div');".formatted(keyword), + "div_002.setAttribute(`id`, `main`);", + "%s text_015 = document.createTextNode(` `);".formatted(keyword), + "div_002.appendChild(text_015);", + "%s h2_000 = document.createElement('h2');".formatted(keyword), + "%s text_016 = document.createTextNode(`Main`);".formatted(keyword), + "h2_000.appendChild(text_016);", + "div_002.appendChild(h2_000);", + "%s text_017 = document.createTextNode(` `);".formatted(keyword), + "div_002.appendChild(text_017);", + "%s p_000 = document.createElement('p');".formatted(keyword), + "%s text_018 = document.createTextNode(`This is the main content.`);".formatted(keyword), + "p_000.appendChild(text_018);", + "div_002.appendChild(p_000);", + "%s text_019 = document.createTextNode(` `);".formatted(keyword), + "div_002.appendChild(text_019);", + "%s img_001 = document.createElement('img');".formatted(keyword), + "img_001.setAttribute(`src`, ``);", + "img_001.setAttribute(`alt`, ``);", + "div_002.appendChild(img_001);", + "%s text_020 = document.createTextNode(` `);".formatted(keyword), + "div_002.appendChild(text_020);", + "div_000.appendChild(div_002);", + "%s text_021 = document.createTextNode(` `);".formatted(keyword), + "div_000.appendChild(text_021);", + "%s div_003 = document.createElement('div');".formatted(keyword), + "div_003.setAttribute(`id`, `footer`);", + "%s text_022 = document.createTextNode(` `);".formatted(keyword), + "div_003.appendChild(text_022);", + "%s comment_001 = document.createComment(` Copyright `);".formatted(keyword), + "div_003.appendChild(comment_001);", + "%s text_023 = document.createTextNode(` `);".formatted(keyword), + "div_003.appendChild(text_023);", + "%s p_001 = document.createElement('p');".formatted(keyword), + "%s text_024 = document.createTextNode(`Copyright © 2019`);".formatted(keyword), + "p_001.appendChild(text_024);", + "div_003.appendChild(p_001);", + "%s text_025 = document.createTextNode(` `);".formatted(keyword), + "div_003.appendChild(text_025);", + "div_000.appendChild(div_003);", + "%s text_026 = document.createTextNode(` `);".formatted(keyword), + "div_000.appendChild(text_026);", + "body_000.appendChild(div_000);", + "%s text_027 = document.createTextNode(` `);".formatted(keyword), + "body_000.appendChild(text_027);", + "html_000.appendChild(body_000);", + "targetElement_000.appendChild(html_000);" + }))); + }else{ + mockMvc.perform(post(ConvertController.MAPPING) + .header(CONTENT_TYPE, APPLICATION_JSON) + .content(objectMapper.writeValueAsString(of( + "contents", List.of(input), + "pattern", "%s.{{ index }}{{ extension }}".formatted(prefix), + "variableDeclaration", variableDeclaration, + "extension", ".%s".formatted(extension), + "querySelectorAdded", true, + "commentConversionModeActivated", false + )))) + .andExpectAll( + status().isOk(), + header().string(CONTENT_TYPE, APPLICATION_JSON_VALUE), + jsonPath("$.status").value("SUCCESS"), + jsonPath("$.content").isArray(), + jsonPath("$.content.length()").value(1), + jsonPath("$.content.[0].filename").value("%s.0.%s".formatted(prefix, extension)), + jsonPath("$.content.[0].content").value(new Match(new String[]{ + "%s targetElement_000 = document.querySelector(`:root > body`);".formatted(keyword), + "%s html_000 = document.createElement('html');".formatted(keyword), + "%s text_000 = document.createTextNode(` `);".formatted(keyword), + "html_000.appendChild(text_000);", + "%s head_000 = document.createElement('head');".formatted(keyword), + "%s text_001 = document.createTextNode(` `);".formatted(keyword), + "head_000.appendChild(text_001);", + "%s meta_000 = document.createElement('meta');".formatted(keyword), + "meta_000.setAttribute(`charset`, `utf-8`);", + "head_000.appendChild(meta_000);", + "%s text_002 = document.createTextNode(` `);".formatted(keyword), + "head_000.appendChild(text_002);", + "%s title_000 = document.createElement('title');".formatted(keyword), + "%s text_003 = document.createTextNode(`Sample`);".formatted(keyword), + "title_000.appendChild(text_003);", + "head_000.appendChild(title_000);", + "%s text_004 = document.createTextNode(` `);".formatted(keyword), + "head_000.appendChild(text_004);", + "%s link_000 = document.createElement('link');".formatted(keyword), + "link_000.setAttribute(`rel`, `stylesheet`);", + "link_000.setAttribute(`href`, ``);", + "head_000.appendChild(link_000);", + "%s text_005 = document.createTextNode(` `);".formatted(keyword), + "head_000.appendChild(text_005);", + "html_000.appendChild(head_000);", + "%s text_006 = document.createTextNode(` `);".formatted(keyword), + "html_000.appendChild(text_006);", + "%s body_000 = document.createElement('body');".formatted(keyword), + "%s text_007 = document.createTextNode(` `);".formatted(keyword), + "body_000.appendChild(text_007);", + "%s div_000 = document.createElement('div');".formatted(keyword), + "div_000.setAttribute(`id`, `container`);", + "%s text_008 = document.createTextNode(` `);".formatted(keyword), + "div_000.appendChild(text_008);", + "%s div_001 = document.createElement('div');".formatted(keyword), + "div_001.setAttribute(`id`, `header`);", + "%s text_009 = document.createTextNode(` `);".formatted(keyword), + "div_001.appendChild(text_009);", + "%s text_010 = document.createTextNode(` `);".formatted(keyword), + "div_001.appendChild(text_010);", + "%s h1_000 = document.createElement('h1');".formatted(keyword), + "%s text_011 = document.createTextNode(`Sample`);".formatted(keyword), + "h1_000.appendChild(text_011);", + "div_001.appendChild(h1_000);", + "%s text_012 = document.createTextNode(` `);".formatted(keyword), + "div_001.appendChild(text_012);", + "%s img_000 = document.createElement('img');".formatted(keyword), + "img_000.setAttribute(`src`, `kanye.jpg`);", + "img_000.setAttribute(`alt`, `kanye`);", + "div_001.appendChild(img_000);", + "%s text_013 = document.createTextNode(` `);".formatted(keyword), + "div_001.appendChild(text_013);", + "div_000.appendChild(div_001);", + "%s text_014 = document.createTextNode(` `);".formatted(keyword), + "div_000.appendChild(text_014);", + "%s div_002 = document.createElement('div');".formatted(keyword), + "div_002.setAttribute(`id`, `main`);", + "%s text_015 = document.createTextNode(` `);".formatted(keyword), + "div_002.appendChild(text_015);", + "%s h2_000 = document.createElement('h2');".formatted(keyword), + "%s text_016 = document.createTextNode(`Main`);".formatted(keyword), + "h2_000.appendChild(text_016);", + "div_002.appendChild(h2_000);", + "%s text_017 = document.createTextNode(` `);".formatted(keyword), + "div_002.appendChild(text_017);", + "%s p_000 = document.createElement('p');".formatted(keyword), + "%s text_018 = document.createTextNode(`This is the main content.`);".formatted(keyword), + "p_000.appendChild(text_018);", + "div_002.appendChild(p_000);", + "%s text_019 = document.createTextNode(` `);".formatted(keyword), + "div_002.appendChild(text_019);", + "%s img_001 = document.createElement('img');".formatted(keyword), + "img_001.setAttribute(`src`, ``);", + "img_001.setAttribute(`alt`, ``);", + "div_002.appendChild(img_001);", + "%s text_020 = document.createTextNode(` `);".formatted(keyword), + "div_002.appendChild(text_020);", + "div_000.appendChild(div_002);", + "%s text_021 = document.createTextNode(` `);".formatted(keyword), + "div_000.appendChild(text_021);", + "%s div_003 = document.createElement('div');".formatted(keyword), + "div_003.setAttribute(`id`, `footer`);", + "%s text_022 = document.createTextNode(` `);".formatted(keyword), + "div_003.appendChild(text_022);", + "%s text_023 = document.createTextNode(` `);".formatted(keyword), + "div_003.appendChild(text_023);", + "%s p_001 = document.createElement('p');".formatted(keyword), + "%s text_024 = document.createTextNode(`Copyright © 2019`);".formatted(keyword), + "p_001.appendChild(text_024);", + "div_003.appendChild(p_001);", + "%s text_025 = document.createTextNode(` `);".formatted(keyword), + "div_003.appendChild(text_025);", + "div_000.appendChild(div_003);", + "%s text_026 = document.createTextNode(` `);".formatted(keyword), + "div_000.appendChild(text_026);", + "body_000.appendChild(div_000);", + "%s text_027 = document.createTextNode(` `);".formatted(keyword), + "body_000.appendChild(text_027);", + "html_000.appendChild(body_000);", + "targetElement_000.appendChild(html_000);" + }))); + } + + } else { + + if(commentConversionModeActivated){ + mockMvc.perform(post(ConvertController.MAPPING) + .header(CONTENT_TYPE, APPLICATION_JSON) + .content(objectMapper.writeValueAsString(of( + "contents", List.of(input), + "pattern", "%s.{{ index }}{{ extension }}".formatted(prefix), + "variableDeclaration", variableDeclaration, + "extension", ".%s".formatted(extension), + "querySelectorAdded", false, + "commentConversionModeActivated", true + )))) + .andExpectAll( + status().isOk(), + header().string(CONTENT_TYPE, APPLICATION_JSON_VALUE), + jsonPath("$.status").value("SUCCESS"), + jsonPath("$.content").isArray(), + jsonPath("$.content.length()").value(1), + jsonPath("$.content.[0].filename").value("%s.0.%s".formatted(prefix, extension)), + jsonPath("$.content.[0].content").value(new Match(new String[]{ + "%s html_000 = document.createElement('html');".formatted(keyword), + "%s text_000 = document.createTextNode(` `);".formatted(keyword), + "html_000.appendChild(text_000);", + "%s head_000 = document.createElement('head');".formatted(keyword), + "%s text_001 = document.createTextNode(` `);".formatted(keyword), + "head_000.appendChild(text_001);", + "%s meta_000 = document.createElement('meta');".formatted(keyword), + "meta_000.setAttribute(`charset`, `utf-8`);", + "head_000.appendChild(meta_000);", + "%s text_002 = document.createTextNode(` `);".formatted(keyword), + "head_000.appendChild(text_002);", + "%s title_000 = document.createElement('title');".formatted(keyword), + "%s text_003 = document.createTextNode(`Sample`);".formatted(keyword), + "title_000.appendChild(text_003);", + "head_000.appendChild(title_000);", + "%s text_004 = document.createTextNode(` `);".formatted(keyword), + "head_000.appendChild(text_004);", + "%s link_000 = document.createElement('link');".formatted(keyword), + "link_000.setAttribute(`rel`, `stylesheet`);", + "link_000.setAttribute(`href`, ``);", + "head_000.appendChild(link_000);", + "%s text_005 = document.createTextNode(` `);".formatted(keyword), + "head_000.appendChild(text_005);", + "html_000.appendChild(head_000);", + "%s text_006 = document.createTextNode(` `);".formatted(keyword), + "html_000.appendChild(text_006);", + "%s body_000 = document.createElement('body');".formatted(keyword), + "%s text_007 = document.createTextNode(` `);".formatted(keyword), + "body_000.appendChild(text_007);", + "%s div_000 = document.createElement('div');".formatted(keyword), + "div_000.setAttribute(`id`, `container`);", + "%s text_008 = document.createTextNode(` `);".formatted(keyword), + "div_000.appendChild(text_008);", + "%s div_001 = document.createElement('div');".formatted(keyword), + "div_001.setAttribute(`id`, `header`);", + "%s text_009 = document.createTextNode(` `);".formatted(keyword), + "div_001.appendChild(text_009);", + "%s comment_000 = document.createComment(` Sample H1 `);".formatted(keyword), + "div_001.appendChild(comment_000);", + "%s text_010 = document.createTextNode(` `);".formatted(keyword), + "div_001.appendChild(text_010);", + "%s h1_000 = document.createElement('h1');".formatted(keyword), + "%s text_011 = document.createTextNode(`Sample`);".formatted(keyword), + "h1_000.appendChild(text_011);", + "div_001.appendChild(h1_000);", + "%s text_012 = document.createTextNode(` `);".formatted(keyword), + "div_001.appendChild(text_012);", + "%s img_000 = document.createElement('img');".formatted(keyword), + "img_000.setAttribute(`src`, `kanye.jpg`);", + "img_000.setAttribute(`alt`, `kanye`);", + "div_001.appendChild(img_000);", + "%s text_013 = document.createTextNode(` `);".formatted(keyword), + "div_001.appendChild(text_013);", + "div_000.appendChild(div_001);", + "%s text_014 = document.createTextNode(` `);".formatted(keyword), + "div_000.appendChild(text_014);", + "%s div_002 = document.createElement('div');".formatted(keyword), + "div_002.setAttribute(`id`, `main`);", + "%s text_015 = document.createTextNode(` `);".formatted(keyword), + "div_002.appendChild(text_015);", + "%s h2_000 = document.createElement('h2');".formatted(keyword), + "%s text_016 = document.createTextNode(`Main`);".formatted(keyword), + "h2_000.appendChild(text_016);", + "div_002.appendChild(h2_000);", + "%s text_017 = document.createTextNode(` `);".formatted(keyword), + "div_002.appendChild(text_017);", + "%s p_000 = document.createElement('p');".formatted(keyword), + "%s text_018 = document.createTextNode(`This is the main content.`);".formatted(keyword), + "p_000.appendChild(text_018);", + "div_002.appendChild(p_000);", + "%s text_019 = document.createTextNode(` `);".formatted(keyword), + "div_002.appendChild(text_019);", + "%s img_001 = document.createElement('img');".formatted(keyword), + "img_001.setAttribute(`src`, ``);", + "img_001.setAttribute(`alt`, ``);", + "div_002.appendChild(img_001);", + "%s text_020 = document.createTextNode(` `);".formatted(keyword), + "div_002.appendChild(text_020);", + "div_000.appendChild(div_002);", + "%s text_021 = document.createTextNode(` `);".formatted(keyword), + "div_000.appendChild(text_021);", + "%s div_003 = document.createElement('div');".formatted(keyword), + "div_003.setAttribute(`id`, `footer`);", + "%s text_022 = document.createTextNode(` `);".formatted(keyword), + "div_003.appendChild(text_022);", + "%s comment_001 = document.createComment(` Copyright `);".formatted(keyword), + "div_003.appendChild(comment_001);", + "%s text_023 = document.createTextNode(` `);".formatted(keyword), + "div_003.appendChild(text_023);", + "%s p_001 = document.createElement('p');".formatted(keyword), + "%s text_024 = document.createTextNode(`Copyright © 2019`);".formatted(keyword), + "p_001.appendChild(text_024);", + "div_003.appendChild(p_001);", + "%s text_025 = document.createTextNode(` `);".formatted(keyword), + "div_003.appendChild(text_025);", + "div_000.appendChild(div_003);", + "%s text_026 = document.createTextNode(` `);".formatted(keyword), + "div_000.appendChild(text_026);", + "body_000.appendChild(div_000);", + "%s text_027 = document.createTextNode(` `);".formatted(keyword), + "body_000.appendChild(text_027);", + "html_000.appendChild(body_000);" + }))); + + }else{ + + mockMvc.perform(post(ConvertController.MAPPING) + .header(CONTENT_TYPE, APPLICATION_JSON) + .content(objectMapper.writeValueAsString(of( + "contents", List.of(input), + "pattern", "%s.{{ index }}{{ extension }}".formatted(prefix), + "variableDeclaration", variableDeclaration, + "extension", ".%s".formatted(extension), + "querySelectorAdded", false, + "commentConversionModeActivated", false + )))) + .andExpectAll( + status().isOk(), + header().string(CONTENT_TYPE, APPLICATION_JSON_VALUE), + jsonPath("$.status").value("SUCCESS"), + jsonPath("$.content").isArray(), + jsonPath("$.content.length()").value(1), + jsonPath("$.content.[0].filename").value("%s.0.%s".formatted(prefix, extension)), + jsonPath("$.content.[0].content").value(new Match(new String[]{ + "%s html_000 = document.createElement('html');".formatted(keyword), + "%s text_000 = document.createTextNode(` `);".formatted(keyword), + "html_000.appendChild(text_000);", + "%s head_000 = document.createElement('head');".formatted(keyword), + "%s text_001 = document.createTextNode(` `);".formatted(keyword), + "head_000.appendChild(text_001);", + "%s meta_000 = document.createElement('meta');".formatted(keyword), + "meta_000.setAttribute(`charset`, `utf-8`);", + "head_000.appendChild(meta_000);", + "%s text_002 = document.createTextNode(` `);".formatted(keyword), + "head_000.appendChild(text_002);", + "%s title_000 = document.createElement('title');".formatted(keyword), + "%s text_003 = document.createTextNode(`Sample`);".formatted(keyword), + "title_000.appendChild(text_003);", + "head_000.appendChild(title_000);", + "%s text_004 = document.createTextNode(` `);".formatted(keyword), + "head_000.appendChild(text_004);", + "%s link_000 = document.createElement('link');".formatted(keyword), + "link_000.setAttribute(`rel`, `stylesheet`);", + "link_000.setAttribute(`href`, ``);", + "head_000.appendChild(link_000);", + "%s text_005 = document.createTextNode(` `);".formatted(keyword), + "head_000.appendChild(text_005);", + "html_000.appendChild(head_000);", + "%s text_006 = document.createTextNode(` `);".formatted(keyword), + "html_000.appendChild(text_006);", + "%s body_000 = document.createElement('body');".formatted(keyword), + "%s text_007 = document.createTextNode(` `);".formatted(keyword), + "body_000.appendChild(text_007);", + "%s div_000 = document.createElement('div');".formatted(keyword), + "div_000.setAttribute(`id`, `container`);", + "%s text_008 = document.createTextNode(` `);".formatted(keyword), + "div_000.appendChild(text_008);", + "%s div_001 = document.createElement('div');".formatted(keyword), + "div_001.setAttribute(`id`, `header`);", + "%s text_009 = document.createTextNode(` `);".formatted(keyword), + "div_001.appendChild(text_009);", + "%s text_010 = document.createTextNode(` `);".formatted(keyword), + "div_001.appendChild(text_010);", + "%s h1_000 = document.createElement('h1');".formatted(keyword), + "%s text_011 = document.createTextNode(`Sample`);".formatted(keyword), + "h1_000.appendChild(text_011);", + "div_001.appendChild(h1_000);", + "%s text_012 = document.createTextNode(` `);".formatted(keyword), + "div_001.appendChild(text_012);", + "%s img_000 = document.createElement('img');".formatted(keyword), + "img_000.setAttribute(`src`, `kanye.jpg`);", + "img_000.setAttribute(`alt`, `kanye`);", + "div_001.appendChild(img_000);", + "%s text_013 = document.createTextNode(` `);".formatted(keyword), + "div_001.appendChild(text_013);", + "div_000.appendChild(div_001);", + "%s text_014 = document.createTextNode(` `);".formatted(keyword), + "div_000.appendChild(text_014);", + "%s div_002 = document.createElement('div');".formatted(keyword), + "div_002.setAttribute(`id`, `main`);", + "%s text_015 = document.createTextNode(` `);".formatted(keyword), + "div_002.appendChild(text_015);", + "%s h2_000 = document.createElement('h2');".formatted(keyword), + "%s text_016 = document.createTextNode(`Main`);".formatted(keyword), + "h2_000.appendChild(text_016);", + "div_002.appendChild(h2_000);", + "%s text_017 = document.createTextNode(` `);".formatted(keyword), + "div_002.appendChild(text_017);", + "%s p_000 = document.createElement('p');".formatted(keyword), + "%s text_018 = document.createTextNode(`This is the main content.`);".formatted(keyword), + "p_000.appendChild(text_018);", + "div_002.appendChild(p_000);", + "%s text_019 = document.createTextNode(` `);".formatted(keyword), + "div_002.appendChild(text_019);", + "%s img_001 = document.createElement('img');".formatted(keyword), + "img_001.setAttribute(`src`, ``);", + "img_001.setAttribute(`alt`, ``);", + "div_002.appendChild(img_001);", + "%s text_020 = document.createTextNode(` `);".formatted(keyword), + "div_002.appendChild(text_020);", + "div_000.appendChild(div_002);", + "%s text_021 = document.createTextNode(` `);".formatted(keyword), + "div_000.appendChild(text_021);", + "%s div_003 = document.createElement('div');".formatted(keyword), + "div_003.setAttribute(`id`, `footer`);", + "%s text_022 = document.createTextNode(` `);".formatted(keyword), + "div_003.appendChild(text_022);", + "%s text_023 = document.createTextNode(` `);".formatted(keyword), + "div_003.appendChild(text_023);", + "%s p_001 = document.createElement('p');".formatted(keyword), + "%s text_024 = document.createTextNode(`Copyright © 2019`);".formatted(keyword), + "p_001.appendChild(text_024);", + "div_003.appendChild(p_001);", + "%s text_025 = document.createTextNode(` `);".formatted(keyword), + "div_003.appendChild(text_025);", + "div_000.appendChild(div_003);", + "%s text_026 = document.createTextNode(` `);".formatted(keyword), + "div_000.appendChild(text_026);", + "body_000.appendChild(div_000);", + "%s text_027 = document.createTextNode(` `);".formatted(keyword), + "body_000.appendChild(text_027);", + "html_000.appendChild(body_000);" + }))); + + } + } } - @Disabled("Encoding Issue here with '©', we will solve that") @ParameterizedTest @MethodSource("provideVariableDeclarationsAndQuerySelectorAddedAndCommentConversionModeActivated") void convertUploadedFilesContentWithComment(final VariableDeclaration variableDeclaration, final boolean querySelectorAdded, final boolean commentConversionModeActivated) throws Exception { @@ -450,7 +989,7 @@ void convertUploadedFilesContentWithComment(final VariableDeclaration variableDe "extension", ".%s".formatted(extension), "querySelectorAdded", true, "commentConversionModeActivated", true - )).getBytes(UTF_8))) + )).getBytes())) .file(new MockMultipartFile( "files", SAMPLE_INPUT_WITH_COMMENT.getFilename(), MULTIPART_FORM_DATA_VALUE, SAMPLE_INPUT_WITH_COMMENT.getInputStream())) .file(new MockMultipartFile( @@ -538,7 +1077,7 @@ void convertUploadedFilesContentWithComment(final VariableDeclaration variableDe "extension", ".%s".formatted(extension), "querySelectorAdded", false, "commentConversionModeActivated", false - )).getBytes(UTF_8))) + )).getBytes())) .file(new MockMultipartFile( "files", SAMPLE_INPUT_WITH_COMMENT.getFilename(), MULTIPART_FORM_DATA_VALUE, SAMPLE_INPUT_WITH_COMMENT.getInputStream())) .file(new MockMultipartFile( diff --git a/jsgenerator-test/jsgenerator-test-core/src/test/java/com/osscameroon/jsgenerator/test/core/ConverterTest.java b/jsgenerator-test/jsgenerator-test-core/src/test/java/com/osscameroon/jsgenerator/test/core/ConverterTest.java index a179682b..6f8a32f5 100644 --- a/jsgenerator-test/jsgenerator-test-core/src/test/java/com/osscameroon/jsgenerator/test/core/ConverterTest.java +++ b/jsgenerator-test/jsgenerator-test-core/src/test/java/com/osscameroon/jsgenerator/test/core/ConverterTest.java @@ -5,7 +5,6 @@ import com.osscameroon.jsgenerator.core.VariableDeclaration; import lombok.NonNull; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -19,7 +18,6 @@ import java.io.OutputStream; import java.util.stream.Stream; -import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.UUID.randomUUID; import static org.assertj.core.api.Assertions.assertThat; import static org.slf4j.LoggerFactory.getLogger; @@ -33,6 +31,8 @@ * To do so, just copy and paste the result from the method "printConverted(String[])". * * */ + +// @ExtendWith(MockitoExtension.class) class ConverterTest { private static final Logger logger = getLogger(ConverterTest.class); @@ -167,7 +167,7 @@ public void before() { * * * */ - //TODO: Remove this method and put that in comparison issue + //TODO: keep this method and use it in comparison issue @ParameterizedTest @MethodSource("provideVariableDeclarationsAndQuerySelectorAdded") void comparisonBetweenJsGeneratorAndOtherConverters(final VariableDeclaration variableDeclaration, final boolean querySelectorAdded) throws IOException { @@ -184,7 +184,7 @@ void comparisonBetweenJsGeneratorAndOtherConverters(final VariableDeclaration va """, new Configuration(variableDeclaration, querySelectorAdded)); - printConverted(converted); + loggingOutput(converted); if (querySelectorAdded) { @@ -272,7 +272,7 @@ void comparisonBetweenJsGeneratorAndOtherConvertersWithComment(final VariableDec """, new Configuration(variableDeclaration, querySelectorAdded, commentConversionModeActivated)); - printConverted(converted); + loggingOutput(converted); if (querySelectorAdded) { if(commentConversionModeActivated){ @@ -443,7 +443,7 @@ void issue145WithCustomTagJavaScriptIdentifiers(final VariableDeclaration variab Web Component""", new Configuration(variableDeclaration, querySelectorAdded)); - printConverted(converted); + loggingOutput(converted); if (querySelectorAdded) { assertThat(converted).containsExactly( @@ -478,7 +478,7 @@ void issue145WithCustomTagJavaScriptIdentifiersWithComment(final VariableDeclara Web Component""", new Configuration(variableDeclaration, querySelectorAdded, commentConversionModeActivated)); - printConverted(converted); + loggingOutput(converted); if (querySelectorAdded) { if (commentConversionModeActivated){ @@ -546,7 +546,7 @@ void issue41WithSelfClosingTags(final VariableDeclaration variableDeclaration, f """, new Configuration(variableDeclaration, querySelectorAdded, commentConversionModeActivated)); - printConverted(converted); + loggingOutput(converted); if (querySelectorAdded) { if(commentConversionModeActivated){ @@ -642,7 +642,7 @@ void produceValidCodeWhenGivenCDATA(VariableDeclaration variableDeclaration, fin final var configuration = new Configuration(variableDeclaration, querySelectorAdded); final var converted = convert(" & ]]>".formatted(token), configuration); - printConverted(converted); + loggingOutput(converted); if (querySelectorAdded) { assertThat(converted).containsExactly( @@ -660,7 +660,7 @@ void produceValidCodeWhenGivenPlainText(VariableDeclaration variableDeclaration, final var configuration = new Configuration(variableDeclaration, querySelectorAdded); final var converted = convert(token, configuration); - printConverted(converted); + loggingOutput(converted); if (querySelectorAdded) { assertThat(converted).containsExactly( @@ -678,7 +678,7 @@ void produceValidCodeWhenGivenPlainTextWithComment(VariableDeclaration variableD final var configuration = new Configuration(variableDeclaration, querySelectorAdded, commentConversionModeActivated); final var converted = convert(" %s".formatted(token,token), configuration); - printConverted(converted); + loggingOutput(converted); if (querySelectorAdded) { if(commentConversionModeActivated){ @@ -711,7 +711,7 @@ void produceValidCodeWhenGivenComment(VariableDeclaration variableDeclaration, f final var configuration = new Configuration(variableDeclaration, querySelectorAdded,commentConversionModeActivated); final var converted = convert("".formatted(token), configuration); - printConverted(converted); + loggingOutput(converted); if (querySelectorAdded) { if(commentConversionModeActivated){ @@ -740,7 +740,7 @@ void produceValidCodeWhenGivenScript(VariableDeclaration variableDeclaration, fi final var configuration = new Configuration(variableDeclaration, querySelectorAdded); final var converted = convert("".formatted(token), configuration); - printConverted(converted); + loggingOutput(converted); if (querySelectorAdded) { assertThat(converted).containsExactly( @@ -779,7 +779,7 @@ void produceValidCodeWhenGivenTagWithAttributes(VariableDeclaration variableDecl var configuration = new Configuration(variableDeclaration, true); var converted = convert(input, configuration); - printConverted(converted); + loggingOutput(converted); assertThat(converted).containsExactly( "%s targetElement_000 = document.querySelector(`:root > body`);".formatted(keyword), "%s div_000 = document.createElement('div');".formatted(keyword), @@ -790,7 +790,7 @@ void produceValidCodeWhenGivenTagWithAttributes(VariableDeclaration variableDecl configuration = new Configuration(variableDeclaration, true); converted = convert(input, configuration); - printConverted(converted); + loggingOutput(converted); assertThat(converted).containsExactly( "%s targetElement_000 = document.querySelector(`:root > body`);".formatted(keyword), "%s details_000 = document.createElement('details');".formatted(keyword), @@ -801,7 +801,7 @@ void produceValidCodeWhenGivenTagWithAttributes(VariableDeclaration variableDecl configuration = new Configuration(variableDeclaration, true); converted = convert(input, configuration); - printConverted(converted); + loggingOutput(converted); assertThat(converted).containsExactly( "%s targetElement_000 = document.querySelector(`:root > body`);".formatted(keyword), "%s p_000 = document.createElement('p');".formatted(keyword), @@ -812,7 +812,7 @@ void produceValidCodeWhenGivenTagWithAttributes(VariableDeclaration variableDecl configuration = new Configuration(variableDeclaration, true); converted = convert(input, configuration); - printConverted(converted); + loggingOutput(converted); assertThat(converted).containsExactly( "%s targetElement_000 = document.querySelector(`:root > body`);".formatted(keyword), "%s p_000 = document.createElement('p');".formatted(keyword), @@ -825,7 +825,7 @@ void produceValidCodeWhenGivenTagWithAttributes(VariableDeclaration variableDecl var configuration = new Configuration(variableDeclaration, false); var converted = convert(input, configuration); - printConverted(converted); + loggingOutput(converted); assertThat(converted).containsExactly( "%s div_000 = document.createElement('div');".formatted(keyword), "div_000.setAttribute(`id`, `id-value`);"); @@ -834,7 +834,7 @@ void produceValidCodeWhenGivenTagWithAttributes(VariableDeclaration variableDecl configuration = new Configuration(variableDeclaration, false); converted = convert(input, configuration); - printConverted(converted); + loggingOutput(converted); assertThat(converted).containsExactly( "%s details_000 = document.createElement('details');".formatted(keyword), "details_000.setAttribute(`open`, `true`);"); @@ -843,7 +843,7 @@ void produceValidCodeWhenGivenTagWithAttributes(VariableDeclaration variableDecl configuration = new Configuration(variableDeclaration, false); converted = convert(input, configuration); - printConverted(converted); + loggingOutput(converted); assertThat(converted).containsExactly( "%s p_000 = document.createElement('p');".formatted(keyword), "p_000.setAttribute(`class`, ``);"); @@ -852,7 +852,7 @@ void produceValidCodeWhenGivenTagWithAttributes(VariableDeclaration variableDecl configuration = new Configuration(variableDeclaration, false); converted = convert(input, configuration); - printConverted(converted); + loggingOutput(converted); assertThat(converted).containsExactly( "%s p_000 = document.createElement('p');".formatted(keyword), "p_000.setAttribute(`class`, `class-value`);"); @@ -872,7 +872,7 @@ void produceValidCodeWhenGivenTagWithAttributesWithComment(VariableDeclaration v if(commentConversionModeActivated){ - printConverted(converted); + loggingOutput(converted); assertThat(converted).containsExactly( "%s targetElement_000 = document.querySelector(`:root > body`);".formatted(keyword), "%s comment_000 = document.createComment(` Div with id `);".formatted(keyword), @@ -883,7 +883,7 @@ void produceValidCodeWhenGivenTagWithAttributesWithComment(VariableDeclaration v }else{ - printConverted(converted); + loggingOutput(converted); assertThat(converted).containsExactly( "%s targetElement_000 = document.querySelector(`:root > body`);".formatted(keyword), "%s div_000 = document.createElement('div');".formatted(keyword), @@ -898,7 +898,7 @@ void produceValidCodeWhenGivenTagWithAttributesWithComment(VariableDeclaration v if(commentConversionModeActivated){ - printConverted(converted); + loggingOutput(converted); assertThat(converted).containsExactly( "%s targetElement_000 = document.querySelector(`:root > body`);".formatted(keyword), "%s comment_000 = document.createComment(` Details tag with open attribute `);".formatted(keyword), @@ -909,7 +909,7 @@ void produceValidCodeWhenGivenTagWithAttributesWithComment(VariableDeclaration v }else{ - printConverted(converted); + loggingOutput(converted); assertThat(converted).containsExactly( "%s targetElement_000 = document.querySelector(`:root > body`);".formatted(keyword), "%s details_000 = document.createElement('details');".formatted(keyword), @@ -924,7 +924,7 @@ void produceValidCodeWhenGivenTagWithAttributesWithComment(VariableDeclaration v if(commentConversionModeActivated){ - printConverted(converted); + loggingOutput(converted); assertThat(converted).containsExactly( "%s targetElement_000 = document.querySelector(`:root > body`);".formatted(keyword), "%s comment_000 = document.createComment(` Paragraph tag with empty class attribute `);".formatted(keyword), @@ -935,7 +935,7 @@ void produceValidCodeWhenGivenTagWithAttributesWithComment(VariableDeclaration v }else{ - printConverted(converted); + loggingOutput(converted); assertThat(converted).containsExactly( "%s targetElement_000 = document.querySelector(`:root > body`);".formatted(keyword), "%s p_000 = document.createElement('p');".formatted(keyword), @@ -949,7 +949,7 @@ void produceValidCodeWhenGivenTagWithAttributesWithComment(VariableDeclaration v if(commentConversionModeActivated){ - printConverted(converted); + loggingOutput(converted); assertThat(converted).containsExactly( "%s targetElement_000 = document.querySelector(`:root > body`);".formatted(keyword), "%s comment_000 = document.createComment(` Paragraph tag with class attribute `);".formatted(keyword), @@ -960,7 +960,7 @@ void produceValidCodeWhenGivenTagWithAttributesWithComment(VariableDeclaration v }else{ - printConverted(converted); + loggingOutput(converted); assertThat(converted).containsExactly( "%s targetElement_000 = document.querySelector(`:root > body`);".formatted(keyword), "%s p_000 = document.createElement('p');".formatted(keyword), @@ -975,14 +975,14 @@ void produceValidCodeWhenGivenTagWithAttributesWithComment(VariableDeclaration v var converted = convert(input, configuration); if(commentConversionModeActivated){ - printConverted(converted); + loggingOutput(converted); assertThat(converted).containsExactly( "%s comment_000 = document.createComment(` Div with id `);".formatted(keyword), "%s div_000 = document.createElement('div');".formatted(keyword), "div_000.setAttribute(`id`, `id-value`);"); }else{ - printConverted(converted); + loggingOutput(converted); assertThat(converted).containsExactly( "%s div_000 = document.createElement('div');".formatted(keyword), "div_000.setAttribute(`id`, `id-value`);"); @@ -994,7 +994,7 @@ void produceValidCodeWhenGivenTagWithAttributesWithComment(VariableDeclaration v if(commentConversionModeActivated){ - printConverted(converted); + loggingOutput(converted); assertThat(converted).containsExactly( "%s comment_000 = document.createComment(` Details tag with open attribute `);".formatted(keyword), "%s details_000 = document.createElement('details');".formatted(keyword), @@ -1002,7 +1002,7 @@ void produceValidCodeWhenGivenTagWithAttributesWithComment(VariableDeclaration v }else{ - printConverted(converted); + loggingOutput(converted); assertThat(converted).containsExactly( "%s details_000 = document.createElement('details');".formatted(keyword), "details_000.setAttribute(`open`, `true`);"); @@ -1015,7 +1015,7 @@ void produceValidCodeWhenGivenTagWithAttributesWithComment(VariableDeclaration v if(commentConversionModeActivated){ - printConverted(converted); + loggingOutput(converted); assertThat(converted).containsExactly( "%s comment_000 = document.createComment(` Paragraph tag with empty class attribute `);".formatted(keyword), "%s p_000 = document.createElement('p');".formatted(keyword), @@ -1023,7 +1023,7 @@ void produceValidCodeWhenGivenTagWithAttributesWithComment(VariableDeclaration v }else{ - printConverted(converted); + loggingOutput(converted); assertThat(converted).containsExactly( "%s p_000 = document.createElement('p');".formatted(keyword), "p_000.setAttribute(`class`, ``);"); @@ -1035,7 +1035,7 @@ void produceValidCodeWhenGivenTagWithAttributesWithComment(VariableDeclaration v if(commentConversionModeActivated){ - printConverted(converted); + loggingOutput(converted); assertThat(converted).containsExactly( "%s comment_000 = document.createComment(` Paragraph tag with class attribute `);".formatted(keyword), "%s p_000 = document.createElement('p');".formatted(keyword), @@ -1043,7 +1043,7 @@ void produceValidCodeWhenGivenTagWithAttributesWithComment(VariableDeclaration v }else{ - printConverted(converted); + loggingOutput(converted); assertThat(converted).containsExactly( "%s p_000 = document.createElement('p');".formatted(keyword), "p_000.setAttribute(`class`, `class-value`);"); @@ -1061,7 +1061,7 @@ void produceValidCodeWhenGivenMultipleNodeAtRoot(VariableDeclaration variableDec final var configuration = new Configuration(variableDeclaration, querySelectorAdded, commentConversionModeActivated); final var converted = convert("
".formatted(token), configuration); - printConverted(converted); + loggingOutput(converted); if (querySelectorAdded) { if(commentConversionModeActivated){ @@ -1096,7 +1096,7 @@ void produceValidCodeWithIncrementVariableNameWhenGivenMultipleNodeWithSameTagNa final var configuration = new Configuration(variableDeclaration, querySelectorAdded); final var converted = convert("
", configuration); - printConverted(converted); + loggingOutput(converted); if (querySelectorAdded) { @@ -1122,7 +1122,7 @@ void produceValidCodeWhenGivenNestedNodes(VariableDeclaration variableDeclaratio final var configuration = new Configuration(variableDeclaration, querySelectorAdded); final var converted = convert("
A ...

Well, case!

", configuration); - printConverted(converted); + loggingOutput(converted); if (querySelectorAdded) { assertThat(converted).containsExactly( @@ -1170,9 +1170,9 @@ void produceValidCodeWhenGivenPathToAFile(VariableDeclaration variableDeclaratio final var input = getClass().getClassLoader() .getResourceAsStream("htmlFilesInput/sample.html") .readAllBytes(); - final var converted = convert(new String(input, UTF_8), configuration); + final var converted = convert(new String(input), configuration); - printConverted(converted); + loggingOutput(converted); // TODO: This doesn't make sense and deserve an issue to fix: @@ -1392,9 +1392,9 @@ void produceValidCodeWhenGivenPathToAFileWithComment(VariableDeclaration variabl final var input = getClass().getClassLoader() .getResourceAsStream("htmlFilesInput/sampleWithComment.html") .readAllBytes(); - final var converted = convert(new String(input, UTF_8), configuration); + final var converted = convert(new String(input), configuration); - printConverted(converted); + loggingOutput(converted); // TODO: This doesn't make sense and deserve an issue to fix: @@ -1616,10 +1616,8 @@ void produceValidCodeWhenGivenPathToAFileWithComment(VariableDeclaration variabl "html_000.appendChild(body_000);", "targetElement_000.appendChild(html_000);"); - } - } else { if(commentConversionModeActivated){ @@ -1728,7 +1726,6 @@ void produceValidCodeWhenGivenPathToAFileWithComment(VariableDeclaration variabl "body_000.appendChild(text_027);", "html_000.appendChild(body_000);"); - }else{ assertThat(converted).containsExactly("%s html_000 = document.createElement('html');".formatted(keyword), @@ -1836,7 +1833,7 @@ void produceValidCodeWhenGivenPathToAFileWithComment(VariableDeclaration variabl } - + //Think about moving this method to convertdefault /** * A helper method to work with language-native String and array of data structures. * @@ -1845,7 +1842,7 @@ void produceValidCodeWhenGivenPathToAFileWithComment(VariableDeclaration variabl * @return Lines of output JS code */ private String[] convert(@NonNull String input, Configuration configuration) throws IOException { - final var inputStream = new ByteArrayInputStream(input.getBytes(UTF_8)); + final var inputStream = new ByteArrayInputStream(input.getBytes()); final var outputStream = new ByteArrayOutputStream(); converter.convert(inputStream, outputStream, configuration); @@ -1866,18 +1863,14 @@ private String keyword(final VariableDeclaration variableDeclaration) { return variableDeclaration.name().toLowerCase(); } - private void printConverted(String[] s) { - - + private void loggingOutput(String[] s) { String convertedString = String.join("\n", s); - String space = """ ------------------------------------------------------------------- """; - logger.info(space + convertedString + space); } } \ No newline at end of file From cfd3c16e5f5264090848e8be619f82481f309467 Mon Sep 17 00:00:00 2001 From: Fanon Jupkwo Date: Sun, 9 Jul 2023 04:46:38 +0100 Subject: [PATCH 2/8] 238 trying to handle non ascii characters --- .../jsgenerator/core/Converter.java | 30 ++++ .../core/internal/ConverterDefault.java | 10 +- .../api/rest/ConvertController.java | 3 +- .../osscameroon/jsgenerator/cli/Command.java | 2 + .../test/api/JsGeneratorApiTest.java | 15 +- .../test/resources/htmlFilesInput/sample.html | 4 +- .../htmlFilesInput/sampleWithComment.html | 2 +- .../commentConversionModeActivated/sample.js | 2 +- .../sample.js | 2 +- .../querySelectorAdded/sample.js | 4 +- .../commentConversionModeActivated/sample.js | 2 +- .../sample.js | 2 +- .../querySelectorNotAdded/sample.js | 4 +- .../jsgenerator/test/core/ConverterTest.java | 136 ++++++++++-------- .../test/resources/htmlFilesInput/sample.html | 2 +- .../htmlFilesInput/sampleWithComment.html | 2 +- 16 files changed, 132 insertions(+), 90 deletions(-) diff --git a/jsgenerator-core/src/main/java/com/osscameroon/jsgenerator/core/Converter.java b/jsgenerator-core/src/main/java/com/osscameroon/jsgenerator/core/Converter.java index bcc10711..ada9b7f5 100644 --- a/jsgenerator-core/src/main/java/com/osscameroon/jsgenerator/core/Converter.java +++ b/jsgenerator-core/src/main/java/com/osscameroon/jsgenerator/core/Converter.java @@ -3,16 +3,46 @@ import com.osscameroon.jsgenerator.core.internal.ConverterDefault; import lombok.NonNull; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import static java.nio.charset.StandardCharsets.UTF_8; + @FunctionalInterface public interface Converter { default void convert(@NonNull final InputStream inputStream, @NonNull final OutputStream outputStream) throws IOException { convert(inputStream, outputStream, new Configuration()); } + /** + * A helper method to work with language-native String and array of data structures. + * + * @param input The input HTML string + * @param configuration The object related to variable declaration (let, const or var) and query selector + * @return Lines of output JS code + */ + default String[] convert(@NonNull String input, Configuration configuration) throws IOException{ + final var inputStream = new ByteArrayInputStream(input.getBytes(UTF_8)); + final var outputStream = new ByteArrayOutputStream(); + + convert(inputStream, outputStream, configuration); + + return outputAsStrippedLines(outputStream); + } + + private String[] outputAsStrippedLines(ByteArrayOutputStream outputStream) { + return outputStream + .toString(UTF_8) + .lines() + .map(String::strip) + .filter(line -> !line.isEmpty()) + .toArray(String[]::new); + } + + void convert(@NonNull final InputStream inputStream, @NonNull final OutputStream outputStream, @NonNull Configuration configuration) throws IOException; static Converter of() { diff --git a/jsgenerator-core/src/main/java/com/osscameroon/jsgenerator/core/internal/ConverterDefault.java b/jsgenerator-core/src/main/java/com/osscameroon/jsgenerator/core/internal/ConverterDefault.java index f0bfe3ba..85648ff0 100644 --- a/jsgenerator-core/src/main/java/com/osscameroon/jsgenerator/core/internal/ConverterDefault.java +++ b/jsgenerator-core/src/main/java/com/osscameroon/jsgenerator/core/internal/ConverterDefault.java @@ -23,7 +23,6 @@ import static java.lang.String.format; import static java.lang.String.join; -import static java.nio.charset.StandardCharsets.UTF_8; import static org.jsoup.parser.Parser.xmlParser; //TODO: Think about the user will use this library , if needed provide 4 explicit methods for these 4 cases @@ -65,19 +64,16 @@ public void convert(InputStream inputStream, OutputStream outputStream, Configur final var scanner = new Scanner(inputStream); while (scanner.hasNext()) { - stringBuilder.append(new String(scanner.nextLine().getBytes(UTF_8), UTF_8)); + stringBuilder.append(scanner.nextLine()); } final var content = stringBuilder.toString(); - //Encoding in UTF8 then decoding in UTF8 - var contentWithUTF8Charset = content; - // NOTE: There is nothing to do - if (contentWithUTF8Charset.isBlank()) return; + if (content.isBlank()) return; final var variableNameStrategy = configuration.getVariableNameStrategy(); - final var document = Jsoup.parse(contentWithUTF8Charset, xmlParser()); + final var document = Jsoup.parse(content, xmlParser()); final var writer = new OutputStreamWriter(outputStream); final var selector = configuration.getTargetElementSelector(); diff --git a/jsgenerator-slim-api/src/main/java/com/osscameroon/jsgenerator/api/rest/ConvertController.java b/jsgenerator-slim-api/src/main/java/com/osscameroon/jsgenerator/api/rest/ConvertController.java index c0ebd5c1..b516d5eb 100644 --- a/jsgenerator-slim-api/src/main/java/com/osscameroon/jsgenerator/api/rest/ConvertController.java +++ b/jsgenerator-slim-api/src/main/java/com/osscameroon/jsgenerator/api/rest/ConvertController.java @@ -68,7 +68,7 @@ public Reply> convertAction(@RequestBody @Valid .map(content -> convert( configuration, new ByteArrayOutputStream(), - //convertInlineContentWithCopyrightCharacterWithComment works after doing this, why ? It used to work without this ? What happened ? + //convertInlineContentWithCopyrightCharacterWithComment works after doing this, why ? What happened ? new ByteArrayInputStream(content.getBytes(UTF_8)))) .map(content -> { final var filename = inlineOutputStreamResolver.resolve(options.getPattern(), Map.of( @@ -124,7 +124,6 @@ private String convert(Configuration configuration, ByteArrayOutputStream output } catch (IOException exception) { throw new RuntimeException(exception); } - return outputStream.toString(UTF_8); } } \ No newline at end of file diff --git a/jsgenerator-slim-cli/src/main/java/com/osscameroon/jsgenerator/cli/Command.java b/jsgenerator-slim-cli/src/main/java/com/osscameroon/jsgenerator/cli/Command.java index 50cffd1d..6408ecab 100644 --- a/jsgenerator-slim-cli/src/main/java/com/osscameroon/jsgenerator/cli/Command.java +++ b/jsgenerator-slim-cli/src/main/java/com/osscameroon/jsgenerator/cli/Command.java @@ -17,6 +17,8 @@ public interface Command extends Callable { boolean isQuerySelectorAdded(); + boolean isCommentConversionModeActivated(); + List getInlineContents(); Converter getConverter(); diff --git a/jsgenerator-test/jsgenerator-test-api/src/test/java/com/osscameroon/jsgenerator/test/api/JsGeneratorApiTest.java b/jsgenerator-test/jsgenerator-test-api/src/test/java/com/osscameroon/jsgenerator/test/api/JsGeneratorApiTest.java index 06e671ab..e96ffe58 100644 --- a/jsgenerator-test/jsgenerator-test-api/src/test/java/com/osscameroon/jsgenerator/test/api/JsGeneratorApiTest.java +++ b/jsgenerator-test/jsgenerator-test-api/src/test/java/com/osscameroon/jsgenerator/test/api/JsGeneratorApiTest.java @@ -7,7 +7,6 @@ import com.osscameroon.jsgenerator.core.VariableDeclaration; import org.hamcrest.CustomMatcher; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -434,14 +433,12 @@ void convertUploadedFilesContent(final VariableDeclaration variableDeclaration, * https://github.com/osscameroon/js-generator/issues/238 * Finally this test helped us to understand that there was no issue when the output is just code but issue occurred when output was a js file. * */ - //@Disabled @ParameterizedTest @MethodSource("provideVariableDeclarationsAndQuerySelectorAddedAndCommentConversionModeActivated") - void convertInlineContentWithCopyrightCharacterWithComment(final VariableDeclaration variableDeclaration, final boolean querySelectorAdded, final boolean commentConversionModeActivated) throws Exception { + void convertInlineContentWithNonASCIICharactersWithComment(final VariableDeclaration variableDeclaration, final boolean querySelectorAdded, final boolean commentConversionModeActivated) throws Exception { final var keyword = keyword(variableDeclaration); final var extension = randomUUID().toString(); final var prefix = randomUUID().toString(); - final var content = randomUUID().toString(); final var input = """ @@ -465,7 +462,7 @@ void convertInlineContentWithCopyrightCharacterWithComment(final VariableDeclara @@ -584,7 +581,7 @@ void convertInlineContentWithCopyrightCharacterWithComment(final VariableDeclara "%s text_023 = document.createTextNode(` `);".formatted(keyword), "div_003.appendChild(text_023);", "%s p_001 = document.createElement('p');".formatted(keyword), - "%s text_024 = document.createTextNode(`Copyright © 2019`);".formatted(keyword), + "%s text_024 = document.createTextNode(`Ã – string çöntäining nön äsçii çhäräçtérs couldn't Copyright © 2019`);".formatted(keyword), "p_001.appendChild(text_024);", "div_003.appendChild(p_001);", "%s text_025 = document.createTextNode(` `);".formatted(keyword), @@ -705,7 +702,7 @@ void convertInlineContentWithCopyrightCharacterWithComment(final VariableDeclara "%s text_023 = document.createTextNode(` `);".formatted(keyword), "div_003.appendChild(text_023);", "%s p_001 = document.createElement('p');".formatted(keyword), - "%s text_024 = document.createTextNode(`Copyright © 2019`);".formatted(keyword), + "%s text_024 = document.createTextNode(`Ã – string çöntäining nön äsçii çhäräçtérs couldn't Copyright © 2019`);".formatted(keyword), "p_001.appendChild(text_024);", "div_003.appendChild(p_001);", "%s text_025 = document.createTextNode(` `);".formatted(keyword), @@ -832,7 +829,7 @@ void convertInlineContentWithCopyrightCharacterWithComment(final VariableDeclara "%s text_023 = document.createTextNode(` `);".formatted(keyword), "div_003.appendChild(text_023);", "%s p_001 = document.createElement('p');".formatted(keyword), - "%s text_024 = document.createTextNode(`Copyright © 2019`);".formatted(keyword), + "%s text_024 = document.createTextNode(`Ã – string çöntäining nön äsçii çhäräçtérs couldn't Copyright © 2019`);".formatted(keyword), "p_001.appendChild(text_024);", "div_003.appendChild(p_001);", "%s text_025 = document.createTextNode(` `);".formatted(keyword), @@ -952,7 +949,7 @@ void convertInlineContentWithCopyrightCharacterWithComment(final VariableDeclara "%s text_023 = document.createTextNode(` `);".formatted(keyword), "div_003.appendChild(text_023);", "%s p_001 = document.createElement('p');".formatted(keyword), - "%s text_024 = document.createTextNode(`Copyright © 2019`);".formatted(keyword), + "%s text_024 = document.createTextNode(`Ã – string çöntäining nön äsçii çhäräçtérs couldn't Copyright © 2019`);".formatted(keyword), "p_001.appendChild(text_024);", "div_003.appendChild(p_001);", "%s text_025 = document.createTextNode(` `);".formatted(keyword), diff --git a/jsgenerator-test/jsgenerator-test-api/src/test/resources/htmlFilesInput/sample.html b/jsgenerator-test/jsgenerator-test-api/src/test/resources/htmlFilesInput/sample.html index 459b4962..5523713e 100644 --- a/jsgenerator-test/jsgenerator-test-api/src/test/resources/htmlFilesInput/sample.html +++ b/jsgenerator-test/jsgenerator-test-api/src/test/resources/htmlFilesInput/sample.html @@ -14,11 +14,11 @@

Sample

Main

-

This is the main content.

+

This is the main content. Ã – string çöntäining nön äsçii çhäräçtérs couldn't

diff --git a/jsgenerator-test/jsgenerator-test-api/src/test/resources/htmlFilesInput/sampleWithComment.html b/jsgenerator-test/jsgenerator-test-api/src/test/resources/htmlFilesInput/sampleWithComment.html index 74273c01..ba5ab964 100644 --- a/jsgenerator-test/jsgenerator-test-api/src/test/resources/htmlFilesInput/sampleWithComment.html +++ b/jsgenerator-test/jsgenerator-test-api/src/test/resources/htmlFilesInput/sampleWithComment.html @@ -15,7 +15,7 @@

Sample

Main

-

This is the main content.

+

This is the main content. Ã – string çöntäining nön äsçii çhäräçtérs couldn't

diff --git a/jsgenerator-test/jsgenerator-test-core/src/test/resources/htmlFilesInput/sampleWithComment.html b/jsgenerator-test/jsgenerator-test-core/src/test/resources/htmlFilesInput/sampleWithComment.html index 48ee7ecb..bd5e2386 100644 --- a/jsgenerator-test/jsgenerator-test-core/src/test/resources/htmlFilesInput/sampleWithComment.html +++ b/jsgenerator-test/jsgenerator-test-core/src/test/resources/htmlFilesInput/sampleWithComment.html @@ -20,7 +20,7 @@

Main

From d4d5ee07c3fcb46915bfdf8d3f6f711a180ad2df Mon Sep 17 00:00:00 2001 From: Fanon Jupkwo Date: Thu, 3 Aug 2023 22:48:36 +0100 Subject: [PATCH 3/8] =?UTF-8?q?238=20trying=20to=20handle=20"=E2=80=93"=20?= =?UTF-8?q?characters?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../jsgenerator/core/Converter.java | 14 +++ .../core/internal/ConverterDefault.java | 102 ++++++++++++++++++ .../api/rest/ConvertController.java | 31 ++++-- .../src/main/resources/application.yaml | 6 ++ .../test/api/JsGeneratorApiTest.java | 9 +- .../jsgenerator/test/core/ConverterTest.java | 4 +- 6 files changed, 151 insertions(+), 15 deletions(-) diff --git a/jsgenerator-core/src/main/java/com/osscameroon/jsgenerator/core/Converter.java b/jsgenerator-core/src/main/java/com/osscameroon/jsgenerator/core/Converter.java index ada9b7f5..0253053f 100644 --- a/jsgenerator-core/src/main/java/com/osscameroon/jsgenerator/core/Converter.java +++ b/jsgenerator-core/src/main/java/com/osscameroon/jsgenerator/core/Converter.java @@ -8,6 +8,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import static java.nio.charset.StandardCharsets.UTF_8; @@ -42,6 +43,19 @@ private String[] outputAsStrippedLines(ByteArrayOutputStream outputStream) { .toArray(String[]::new); } + default String[] convert(@NonNull byte[] input, Configuration configuration) throws IOException{ + return convert(new String(input, UTF_8),configuration); + } + + default String convert(Configuration configuration, ByteArrayOutputStream outputStream, ByteArrayInputStream inputStream) { + try { + convert(inputStream, outputStream, configuration); + } catch (IOException exception) { + throw new RuntimeException(exception); + } + return outputStream.toString(UTF_8); + } + void convert(@NonNull final InputStream inputStream, @NonNull final OutputStream outputStream, @NonNull Configuration configuration) throws IOException; diff --git a/jsgenerator-core/src/main/java/com/osscameroon/jsgenerator/core/internal/ConverterDefault.java b/jsgenerator-core/src/main/java/com/osscameroon/jsgenerator/core/internal/ConverterDefault.java index 85648ff0..9f74598b 100644 --- a/jsgenerator-core/src/main/java/com/osscameroon/jsgenerator/core/internal/ConverterDefault.java +++ b/jsgenerator-core/src/main/java/com/osscameroon/jsgenerator/core/internal/ConverterDefault.java @@ -11,11 +11,22 @@ import org.jsoup.nodes.Node; import org.jsoup.nodes.TextNode; +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.Writer; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.text.Normalizer; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -23,6 +34,7 @@ import static java.lang.String.format; import static java.lang.String.join; +import static java.nio.charset.StandardCharsets.UTF_8; import static org.jsoup.parser.Parser.xmlParser; //TODO: Think about the user will use this library , if needed provide 4 explicit methods for these 4 cases @@ -72,9 +84,12 @@ public void convert(InputStream inputStream, OutputStream outputStream, Configur // NOTE: There is nothing to do if (content.isBlank()) return; + //String normalisedContent = nfdNormalized(content); + final var variableNameStrategy = configuration.getVariableNameStrategy(); final var document = Jsoup.parse(content, xmlParser()); final var writer = new OutputStreamWriter(outputStream); + //final var writer = new OutputStreamWriter(outputStream, StandardCharsets.UTF_8); final var selector = configuration.getTargetElementSelector(); @@ -323,4 +338,91 @@ private void visitScriptNode(Writer writer, Element element, String variable, private String resolveDeclarationKeyWord(VariableDeclaration variableDeclaration) { return variableDeclaration.name().toLowerCase(); } + + private String nfdNormalized(String txt) { + if (!Normalizer.isNormalized(txt, Normalizer.Form.NFD)) { + return Normalizer.normalize(txt, Normalizer.Form.NFD); + } + return txt; + } + + private String decodeTextWithUTF8(String input) throws IOException { + return + new BufferedReader( + new InputStreamReader( + new ByteArrayInputStream(input.getBytes()), + UTF_8)) + .readLine(); + } + + private String a (String text) throws CharacterCodingException { + // Text to decode with unknown encoding + //String text = "你好, こんにちは, 안녕하세요, مرحبًا"; + + // Charset for encoding and decoding (UTF-8) + Charset charset = StandardCharsets.UTF_8; + + // Create a CharsetDecoder for decoding + CharsetDecoder decoder = charset.newDecoder(); + decoder.onMalformedInput(CodingErrorAction.REPLACE); + decoder.onUnmappableCharacter(CodingErrorAction.REPLACE); + + // Encode the text into bytes using UTF-8 + ByteBuffer encodedBytes = charset.encode(text); + + // Reset the decoder and decode the bytes to characters + decoder.reset(); + CharBuffer decodedChars = decoder.decode(encodedBytes); + + // Get the decoded text + String decodedText = decodedChars.toString(); + + return decodedText; + } + + private String b (String text) throws CharacterCodingException { + // Text to decode with unknown encoding + //String text = "你好, こんにちは, 안녕하세요, مرحبًا"; + + // Normalize the text using NFKC normalization + String normalizedText = Normalizer.normalize(text, Normalizer.Form.NFD); + + // Decode the normalized text with UTF-8 + byte[] bytes = normalizedText.getBytes(StandardCharsets.UTF_8); + String decodedText = new String(bytes, StandardCharsets.UTF_8); + + //System.out.println("Original Text: " + text); + //System.out.println("Decoded Text: " + decodedText); + + return decodedText; + } + + private String c (String text) throws CharacterCodingException { + // Text to decode with unknown encoding + //String text = "你好, こんにちは, 안녕하세요, مرحبًا"; + + // Normalize the text using NFKC normalization + String normalizedText = Normalizer.normalize(text, Normalizer.Form.NFD); + + // Charset for encoding and decoding (UTF-8) + Charset charset = StandardCharsets.UTF_8; + + // Create a CharsetDecoder for decoding + CharsetDecoder decoder = charset.newDecoder(); + decoder.onMalformedInput(CodingErrorAction.REPLACE); + decoder.onUnmappableCharacter(CodingErrorAction.REPLACE); + + // Encode the text into bytes using UTF-8 + ByteBuffer encodedBytes = charset.encode(normalizedText); + + // Reset the decoder and decode the bytes to characters + decoder.reset(); + CharBuffer decodedChars = decoder.decode(encodedBytes); + + // Get the decoded text + String decodedText = decodedChars.toString(); + + return decodedText; + } + } \ No newline at end of file diff --git a/jsgenerator-slim-api/src/main/java/com/osscameroon/jsgenerator/api/rest/ConvertController.java b/jsgenerator-slim-api/src/main/java/com/osscameroon/jsgenerator/api/rest/ConvertController.java index b516d5eb..aad07895 100644 --- a/jsgenerator-slim-api/src/main/java/com/osscameroon/jsgenerator/api/rest/ConvertController.java +++ b/jsgenerator-slim-api/src/main/java/com/osscameroon/jsgenerator/api/rest/ConvertController.java @@ -3,13 +3,13 @@ import com.osscameroon.jsgenerator.api.domain.InlineOptions; import com.osscameroon.jsgenerator.api.domain.MultipartOptions; import com.osscameroon.jsgenerator.api.domain.Output; -import com.osscameroon.jsgenerator.core.Configuration; import com.osscameroon.jsgenerator.core.Converter; import com.osscameroon.jsgenerator.core.OutputStreamResolver; import lombok.RequiredArgsConstructor; import org.slf4j.Logger; import org.springframework.core.io.AbstractResource; import org.springframework.core.io.ByteArrayResource; +import org.springframework.http.MediaType; import org.springframework.util.LinkedMultiValueMap; import org.springframework.util.MultiValueMap; import org.springframework.web.bind.annotation.PostMapping; @@ -25,6 +25,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; +import java.text.Normalizer; import java.util.List; import java.util.Map; import java.util.Optional; @@ -65,7 +66,7 @@ public Reply> convertAction(@RequestBody @Valid final var configuration = options.toConfiguration(); return Reply.ofSuccesses(options.getContents().stream() - .map(content -> convert( + .map(content -> converter.convert( configuration, new ByteArrayOutputStream(), //convertInlineContentWithCopyrightCharacterWithComment works after doing this, why ? What happened ? @@ -93,7 +94,7 @@ public MultiValueMap convertAction(@RequestPart("optio multipartFiles.stream().map(multipartFile -> { try { - return convert( + return converter.convert( command.toConfiguration(), new ByteArrayOutputStream(), new ByteArrayInputStream(multipartFile.getBytes())); @@ -118,12 +119,22 @@ public MultiValueMap convertAction(@RequestPart("optio return map; } - private String convert(Configuration configuration, ByteArrayOutputStream outputStream, ByteArrayInputStream inputStream) { - try { - converter.convert(inputStream, outputStream, configuration); - } catch (IOException exception) { - throw new RuntimeException(exception); - } - return outputStream.toString(UTF_8); + private byte[] encodingAndDecodingInUTF8(MultipartFile file) throws IOException { + + // Encode the MultipartFile as UTF-8 bytes + byte[] encodedBytes = file.getBytes(); + String encodedText = new String(encodedBytes,StandardCharsets.UTF_8); + + // Decode the UTF-8 bytes back to MultipartFile + byte[] decodedBytes = encodedText.getBytes(StandardCharsets.UTF_8); + +/* + String content = new String (multipartFile.getBytes()); + + byte[] encodedBytes = content.getBytes(UTF_8); + + return new String(encodedBytes, UTF_8).getBytes();*/ + + return decodedBytes; } } \ No newline at end of file diff --git a/jsgenerator-slim-api/src/main/resources/application.yaml b/jsgenerator-slim-api/src/main/resources/application.yaml index af045141..6d150bbd 100644 --- a/jsgenerator-slim-api/src/main/resources/application.yaml +++ b/jsgenerator-slim-api/src/main/resources/application.yaml @@ -20,3 +20,9 @@ springdoc: path: / use-root-path: false disable-swagger-default-url: true +server: + servlet: + encoding: + charset: UTF-8 + enabled: true + force: true \ No newline at end of file diff --git a/jsgenerator-test/jsgenerator-test-api/src/test/java/com/osscameroon/jsgenerator/test/api/JsGeneratorApiTest.java b/jsgenerator-test/jsgenerator-test-api/src/test/java/com/osscameroon/jsgenerator/test/api/JsGeneratorApiTest.java index e96ffe58..f90a4ac9 100644 --- a/jsgenerator-test/jsgenerator-test-api/src/test/java/com/osscameroon/jsgenerator/test/api/JsGeneratorApiTest.java +++ b/jsgenerator-test/jsgenerator-test-api/src/test/java/com/osscameroon/jsgenerator/test/api/JsGeneratorApiTest.java @@ -24,6 +24,8 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.text.Normalizer; import java.util.List; import java.util.stream.Stream; @@ -379,11 +381,12 @@ void convertUploadedFilesContent(final VariableDeclaration variableDeclaration, "extension", ".%s".formatted(extension), "querySelectorAdded", true, "commentConversionModeActivated", true - )).getBytes())) + )).getBytes(UTF_8))) .file(new MockMultipartFile( "files", SAMPLE_INPUT.getFilename(), MULTIPART_FORM_DATA_VALUE, SAMPLE_INPUT.getInputStream())) .file(new MockMultipartFile( - "files", SAMPLE_INPUT.getFilename(), MULTIPART_FORM_DATA_VALUE, SAMPLE_INPUT.getInputStream()))) + "files", SAMPLE_INPUT.getFilename(), MULTIPART_FORM_DATA_VALUE, SAMPLE_INPUT.getInputStream())) + .characterEncoding(UTF_8)) .andExpectAll( status().isOk(), withMultipart().size(2), @@ -411,7 +414,7 @@ void convertUploadedFilesContent(final VariableDeclaration variableDeclaration, .file(new MockMultipartFile( "files", SAMPLE_INPUT.getFilename(), MULTIPART_FORM_DATA_VALUE, SAMPLE_INPUT.getInputStream())) .file(new MockMultipartFile( - "files", SAMPLE_INPUT.getFilename(), MULTIPART_FORM_DATA_VALUE, SAMPLE_INPUT.getInputStream()))) + "files", SAMPLE_INPUT.getFilename(), MULTIPART_FORM_DATA_VALUE, SAMPLE_INPUT.getInputStream())).characterEncoding(UTF_8)) .andExpectAll( status().isOk(), withMultipart().size(2), diff --git a/jsgenerator-test/jsgenerator-test-core/src/test/java/com/osscameroon/jsgenerator/test/core/ConverterTest.java b/jsgenerator-test/jsgenerator-test-core/src/test/java/com/osscameroon/jsgenerator/test/core/ConverterTest.java index 6f6e204c..bcabb350 100644 --- a/jsgenerator-test/jsgenerator-test-core/src/test/java/com/osscameroon/jsgenerator/test/core/ConverterTest.java +++ b/jsgenerator-test/jsgenerator-test-core/src/test/java/com/osscameroon/jsgenerator/test/core/ConverterTest.java @@ -1174,7 +1174,7 @@ void produceValidCodeWhenGivenPathToAFile(VariableDeclaration variableDeclaratio final var input = getClass().getClassLoader() .getResourceAsStream("htmlFilesInput/sample.html") .readAllBytes(); - final var converted = converter.convert(new String(input, UTF_8), configuration); + final var converted = converter.convert(input, configuration); loggingOutput(converted); @@ -1396,7 +1396,7 @@ void produceValidCodeWhenGivenPathToAFileWithComment(VariableDeclaration variabl final var input = getClass().getClassLoader() .getResourceAsStream("htmlFilesInput/sampleWithComment.html") .readAllBytes(); - final var converted = converter.convert(new String(input, UTF_8), configuration); + final var converted = converter.convert(input, configuration); loggingOutput(converted); From 1cb211e7f44365f76aa7d5327691bb71e7812cd6 Mon Sep 17 00:00:00 2001 From: Fanon Jupkwo Date: Thu, 3 Aug 2023 23:19:13 +0100 Subject: [PATCH 4/8] 238 adding maven "clean" goal --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 3ffbdabe..e57ee61b 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -31,7 +31,7 @@ jobs: # -B,--batch-mode Run in non-interactive (batch) mode (disables output color) # To learn more about options: https://maven.apache.org/ref/3.6.3/maven-embedder/cli.html run: | - mvn package -B -e -X + mvn clean package -B -e -X mvn site -B -e -X --projects 'jsgenerator-core' env: MAVEN_SITE_GITHUB_OAUTH_TOKEN: ${{ secrets.MAVEN_SITE_GITHUB_OAUTH_TOKEN }} From 850a90778301444cdc74d419b9ddda7896ca9143 Mon Sep 17 00:00:00 2001 From: Fanon Jupkwo Date: Thu, 3 Aug 2023 23:31:25 +0100 Subject: [PATCH 5/8] 238 delete cache --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index e57ee61b..9239f058 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -24,7 +24,7 @@ jobs: java-version: 17 # Given the fact that this is a multimodule project, build process will take long time so we activate caching # To know more: https://maven.apache.org/extensions/maven-build-cache-extension/cache.html - cache: 'maven' + #cache: 'maven' - name: Build with Maven #To see the full stack trace of the errors, re-run Maven with the -e switch. #Re-run Maven using the -X switch to enable full debug logging. From 0f1c94f0814f9c399c227f14c4808dc424e5f83c Mon Sep 17 00:00:00 2001 From: Fanon Jupkwo Date: Thu, 3 Aug 2023 23:44:48 +0100 Subject: [PATCH 6/8] 238 using test goal --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 9239f058..02c8209f 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -31,7 +31,7 @@ jobs: # -B,--batch-mode Run in non-interactive (batch) mode (disables output color) # To learn more about options: https://maven.apache.org/ref/3.6.3/maven-embedder/cli.html run: | - mvn clean package -B -e -X + mvn clean test -B -e -X mvn site -B -e -X --projects 'jsgenerator-core' env: MAVEN_SITE_GITHUB_OAUTH_TOKEN: ${{ secrets.MAVEN_SITE_GITHUB_OAUTH_TOKEN }} From 6266b035af30b72ec54d08a7ccfc947b5571c729 Mon Sep 17 00:00:00 2001 From: Fanon Jupkwo Date: Thu, 3 Aug 2023 23:55:52 +0100 Subject: [PATCH 7/8] 238 making test failed --- .../osscameroon/jsgenerator/test/api/JsGeneratorApiTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jsgenerator-test/jsgenerator-test-api/src/test/java/com/osscameroon/jsgenerator/test/api/JsGeneratorApiTest.java b/jsgenerator-test/jsgenerator-test-api/src/test/java/com/osscameroon/jsgenerator/test/api/JsGeneratorApiTest.java index f90a4ac9..8384a4b3 100644 --- a/jsgenerator-test/jsgenerator-test-api/src/test/java/com/osscameroon/jsgenerator/test/api/JsGeneratorApiTest.java +++ b/jsgenerator-test/jsgenerator-test-api/src/test/java/com/osscameroon/jsgenerator/test/api/JsGeneratorApiTest.java @@ -963,7 +963,7 @@ void convertInlineContentWithNonASCIICharactersWithComment(final VariableDeclara "body_000.appendChild(div_000);", "%s text_027 = document.createTextNode(` `);".formatted(keyword), "body_000.appendChild(text_027);", - "html_000.appendChild(body_000);" + "html_000.appendChild(body_100);" }))); } From d08adc2808dfc54973275038f3a75e6177a6620e Mon Sep 17 00:00:00 2001 From: Fanon Jupkwo Date: Mon, 4 Sep 2023 23:12:40 +0100 Subject: [PATCH 8/8] 238 Correct failing test by replacing body_100 with body_000 --- .../osscameroon/jsgenerator/test/api/JsGeneratorApiTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jsgenerator-test/jsgenerator-test-api/src/test/java/com/osscameroon/jsgenerator/test/api/JsGeneratorApiTest.java b/jsgenerator-test/jsgenerator-test-api/src/test/java/com/osscameroon/jsgenerator/test/api/JsGeneratorApiTest.java index 8384a4b3..f90a4ac9 100644 --- a/jsgenerator-test/jsgenerator-test-api/src/test/java/com/osscameroon/jsgenerator/test/api/JsGeneratorApiTest.java +++ b/jsgenerator-test/jsgenerator-test-api/src/test/java/com/osscameroon/jsgenerator/test/api/JsGeneratorApiTest.java @@ -963,7 +963,7 @@ void convertInlineContentWithNonASCIICharactersWithComment(final VariableDeclara "body_000.appendChild(div_000);", "%s text_027 = document.createTextNode(` `);".formatted(keyword), "body_000.appendChild(text_027);", - "html_000.appendChild(body_100);" + "html_000.appendChild(body_000);" }))); }