diff --git a/checkstyle/checkstyle.xml b/checkstyle/checkstyle.xml index 3b5825a3be..f487bbd489 100644 --- a/checkstyle/checkstyle.xml +++ b/checkstyle/checkstyle.xml @@ -100,6 +100,8 @@ { @@ -45,7 +45,7 @@ public void onFetchPage(@Nonnull final Downloader downloader) json = JsonParser.object().from(getDownloader().postWithContentTypeJson( FEATURED_API_URL, Collections.emptyMap(), - "{\"platform\":\"\",\"version\":0}".getBytes(StandardCharsets.UTF_8)) + "{\"platform\":\"\",\"version\":0}".getBytes(UTF_8)) .responseBody()); } catch (final JsonParserException e) { throw new ParsingException("Could not parse Bandcamp featured API response", e); diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java index d7c523bcf3..3bc2425c48 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java @@ -18,12 +18,13 @@ import org.schabi.newpipe.extractor.utils.Utils; import java.io.IOException; -import java.nio.charset.StandardCharsets; +import java.io.UnsupportedEncodingException; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY; import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty; +import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; import javax.annotation.Nonnull; @@ -130,11 +131,11 @@ public InfoItemsPage getPage(final Page page) } } else { try { - json = JsonParser.object().from(new String(page.getBody(), StandardCharsets.UTF_8)); + json = JsonParser.object().from(new String(page.getBody(), UTF_8)); isReply = true; total = json.getArray(CHILDREN).size(); collectRepliesFrom(collector, json); - } catch (final JsonParserException e) { + } catch (final JsonParserException | UnsupportedEncodingException e) { throw new ParsingException( "Could not parse json data for nested comments info", e); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java index 57a0444906..25793bebcf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsInfoItemExtractor.java @@ -17,13 +17,14 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; -import java.nio.charset.StandardCharsets; +import java.io.UnsupportedEncodingException; import java.util.List; import java.util.Objects; import static org.schabi.newpipe.extractor.services.peertube.extractors.PeertubeCommentsExtractor.CHILDREN; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.getAvatarsFromOwnerAccountOrVideoChannelObject; import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.parseDateFrom; +import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; public class PeertubeCommentsInfoItemExtractor implements CommentsInfoItemExtractor { @Nonnull @@ -130,8 +131,13 @@ public Page getReplies() throws ParsingException { // is also structured like a JsonObject. final JsonObject pageContent = new JsonObject(); pageContent.put(CHILDREN, children); - return new Page(repliesUrl, threadId, - JsonWriter.string(pageContent).getBytes(StandardCharsets.UTF_8)); + try { + return new Page(repliesUrl, threadId, + JsonWriter.string(pageContent).getBytes(UTF_8)); + } catch (final UnsupportedEncodingException e) { + throw new ParsingException( + "Could not parse data", e); + } } return new Page(repliesUrl, threadId); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ProtoBuilder.java b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ProtoBuilder.java index f0e223a9ca..05c89cf59c 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ProtoBuilder.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/utils/ProtoBuilder.java @@ -3,8 +3,8 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.util.Base64; +import java.io.UnsupportedEncodingException; +import static org.schabi.newpipe.extractor.utils.Utils.UTF_8; public class ProtoBuilder { ByteArrayOutputStream byteBuffer; @@ -18,8 +18,12 @@ public byte[] toBytes() { } public String toUrlencodedBase64() { - final String b64 = Base64.getUrlEncoder().encodeToString(toBytes()); - return URLEncoder.encode(b64, StandardCharsets.UTF_8); + try { + final String b64 = encodeUrl(toBytes()); + return URLEncoder.encode(b64, UTF_8); + } catch (final IOException e) { + throw new RuntimeException(e); + } } private void writeVarint(final long val) { @@ -56,8 +60,12 @@ public void varint(final int field, final long val) { } public void string(final int field, final String string) { - final byte[] strBts = string.getBytes(StandardCharsets.UTF_8); - bytes(field, strBts); + try { + final byte[] strBts = string.getBytes(UTF_8); + bytes(field, strBts); + } catch (final IOException e) { + throw new RuntimeException(e); + } } public void bytes(final int field, final byte[] bytes) { @@ -69,4 +77,59 @@ public void bytes(final int field, final byte[] bytes) { throw new RuntimeException(e); } } + + private static final byte[] MAP = new byte[] { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', + 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', + 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', + '5', '6', '7', '8', '9', '+', '/' + }; + + private static final byte[] URL_MAP = new byte[] { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', + 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', + 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', + '5', '6', '7', '8', '9', '-', '_' + }; + + private static String encode(final byte[] in) { + return encode(in, MAP); + } + + private static String encodeUrl(final byte[] in) { + return encode(in, URL_MAP); + } + + private static String encode(final byte[] in, final byte[] map) { + final int length = (in.length + 2) / 3 * 4; + final byte[] out = new byte[length]; + int index = 0; + final int end = in.length - in.length % 3; + for (int i = 0; i < end; i += 3) { + out[index++] = map[(in[i] & 0xff) >> 2]; + out[index++] = map[((in[i] & 0x03) << 4) | ((in[i + 1] & 0xff) >> 4)]; + out[index++] = map[((in[i + 1] & 0x0f) << 2) | ((in[i + 2] & 0xff) >> 6)]; + out[index++] = map[(in[i + 2] & 0x3f)]; + } + switch (in.length % 3) { + case 1: + out[index++] = map[(in[end] & 0xff) >> 2]; + out[index++] = map[(in[end] & 0x03) << 4]; + out[index++] = '='; + out[index++] = '='; + break; + case 2: + out[index++] = map[(in[end] & 0xff) >> 2]; + out[index++] = map[((in[end] & 0x03) << 4) | ((in[end + 1] & 0xff) >> 4)]; + out[index++] = map[((in[end + 1] & 0x0f) << 2)]; + out[index++] = '='; + break; + } + try { + return new String(out, "US-ASCII"); + } catch (final UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + }