Use Charset decoding when reading packet content in `ByteReadChanne…

…l.toCharIterator`
msasikanth · Jun 4, 2024 · f4256b6 · f4256b6
1 parent 434871d
commit f4256b6
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 6 deletions.
diff --git a/...etwork/src/commonMain/kotlin/dev/sasikanth/rss/reader/core/network/fetcher/FeedFetcher.kt b/...etwork/src/commonMain/kotlin/dev/sasikanth/rss/reader/core/network/fetcher/FeedFetcher.kt
@@ -32,6 +32,7 @@ import io.ktor.http.HttpStatusCode
 import io.ktor.http.URLBuilder
 import io.ktor.http.URLProtocol
 import io.ktor.http.Url
+import io.ktor.http.charset
 import io.ktor.http.contentType
 import me.tatarka.inject.annotations.Inject
 
@@ -97,7 +98,8 @@ class FeedFetcher(private val httpClient: HttpClient, private val feedParser: Fe
       }
     } else {
       val content = response.bodyAsChannel()
-      val feedPayload = feedParser.parse(content = content, feedUrl = url)
+      val feedPayload =
+        feedParser.parse(content = content, charset = response.charset(), feedUrl = url)
 
       FeedFetchResult.Success(feedPayload)
     }

diff --git a/.../network/src/commonMain/kotlin/dev/sasikanth/rss/reader/core/network/parser/FeedParser.kt b/.../network/src/commonMain/kotlin/dev/sasikanth/rss/reader/core/network/parser/FeedParser.kt
@@ -24,6 +24,9 @@ import io.ktor.http.URLBuilder
 import io.ktor.http.URLProtocol
 import io.ktor.http.set
 import io.ktor.utils.io.ByteReadChannel
+import io.ktor.utils.io.charsets.Charset
+import io.ktor.utils.io.charsets.Charsets
+import io.ktor.utils.io.charsets.decode
 import kotlin.coroutines.CoroutineContext
 import kotlin.coroutines.EmptyCoroutineContext
 import kotlinx.coroutines.runBlocking
@@ -36,10 +39,15 @@ import org.kobjects.ktxml.mini.MiniXmlPullParser
 @AppScope
 class FeedParser(private val dispatchersProvider: DispatchersProvider) {
 
-  suspend fun parse(content: ByteReadChannel, feedUrl: String): FeedPayload {
+  suspend fun parse(
+    content: ByteReadChannel,
+    feedUrl: String,
+    charset: Charset?,
+  ): FeedPayload {
     return try {
       withContext(dispatchersProvider.io) {
-        val parser = MiniXmlPullParser(source = content.toCharIterator())
+        val parser =
+          MiniXmlPullParser(source = content.toCharIterator(charset = charset ?: Charsets.UTF_8))
 
         parser.nextTag()
 
@@ -134,6 +142,7 @@ class FeedParser(private val dispatchersProvider: DispatchersProvider) {
 }
 
 private fun ByteReadChannel.toCharIterator(
+  charset: Charset,
   context: CoroutineContext = EmptyCoroutineContext
 ): CharIterator {
   return object : CharIterator() {
@@ -148,7 +157,9 @@ private fun ByteReadChannel.toCharIterator(
       if (this@toCharIterator.isClosedForRead) return false
 
       val packet = runBlocking(context) { this@toCharIterator.readRemaining(DEFAULT_BUFFER_SIZE) }
-      currentBuffer = packet.readText().toCharArray()
+      val decoder = charset.newDecoder()
+
+      currentBuffer = decoder.decode(packet).toCharArray()
       packet.release()
       currentIndex = 0
       return currentBuffer.isNotEmpty()

diff --git a/shared/src/commonTest/kotlin/dev/sasikanth/rss/reader/FeedParserTest.kt b/shared/src/commonTest/kotlin/dev/sasikanth/rss/reader/FeedParserTest.kt
@@ -19,6 +19,7 @@ import dev.sasikanth.rss.reader.core.model.remote.FeedPayload
 import dev.sasikanth.rss.reader.core.model.remote.PostPayload
 import dev.sasikanth.rss.reader.core.network.parser.FeedParser
 import io.ktor.utils.io.ByteReadChannel
+import io.ktor.utils.io.charsets.Charsets
 import io.ktor.utils.io.core.toByteArray
 import kotlin.test.Test
 import kotlin.test.assertEquals
@@ -113,7 +114,7 @@ class FeedParserTest {
 
     // when
     val content = ByteReadChannel(rssXmlContent.toByteArray())
-    val payload = feedParser.parse(content, feedUrl)
+    val payload = feedParser.parse(content, feedUrl, Charsets.UTF_8)
 
     // then
     assertEquals(expectedFeedPayload, payload)
@@ -190,7 +191,7 @@ class FeedParserTest {
 
     // when
     val content = ByteReadChannel(atomXmlContent.toByteArray())
-    val payload = feedParser.parse(content, feedUrl)
+    val payload = feedParser.parse(content, feedUrl, Charsets.UTF_8)
 
     // then
     assertEquals(expectedFeedPayload, payload)