diff --git a/rssparser/src/androidMain/kotlin/com/prof18/rssparser/internal/XmlPullParser+.kt b/rssparser/src/androidMain/kotlin/com/prof18/rssparser/internal/XmlPullParser+.kt
index 32be7ca0..045cbf06 100644
--- a/rssparser/src/androidMain/kotlin/com/prof18/rssparser/internal/XmlPullParser+.kt
+++ b/rssparser/src/androidMain/kotlin/com/prof18/rssparser/internal/XmlPullParser+.kt
@@ -4,9 +4,16 @@ import org.xmlpull.v1.XmlPullParser
import org.xmlpull.v1.XmlPullParserException
internal fun XmlPullParser.nextTrimmedText(): String? {
+ val startDepth = this.depth
return try {
this.nextText()?.trim()
- } catch (_: XmlPullParserException) {
+ } catch (exception: XmlPullParserException) {
+ // Only a child tag is recovered here: nextText() stopped on it, one level deeper. An error
+ // in the text itself (a bare `&`, an HTML entity like `ä`) leaves the depth unchanged
+ // and has to reach the caller, which parses the feed again with the XML escaped. Recovering
+ // from it would drop the text before the error and read on past the closing tag.
+ if (this.depth <= startDepth) throw exception
+
// Handle malformed HTML content (e.g., mismatched tag cases like ...) or HTML tags in content
// When nextText() throws, there are nested tags, so we manually parse and collect all text
// Note: nextText() may have consumed some text before throwing, so we start fresh from current position
diff --git a/rssparser/src/commonMain/kotlin/com/prof18/rssparser/RssParser.kt b/rssparser/src/commonMain/kotlin/com/prof18/rssparser/RssParser.kt
index 02fb6e01..98170728 100644
--- a/rssparser/src/commonMain/kotlin/com/prof18/rssparser/RssParser.kt
+++ b/rssparser/src/commonMain/kotlin/com/prof18/rssparser/RssParser.kt
@@ -1,6 +1,7 @@
package com.prof18.rssparser
import com.prof18.rssparser.exception.RssParsingException
+import com.prof18.rssparser.internal.HtmlNamedEntities
import com.prof18.rssparser.internal.XmlFetcher
import com.prof18.rssparser.internal.XmlParser
import com.prof18.rssparser.model.RssChannel
@@ -59,12 +60,15 @@ public class RssParser internal constructor(
/**
* Escapes invalid XML entities in the input string and fixes common XML issues.
+ * - Replaces HTML named entities (`Ä`, ` `) with numeric character references
* - Escapes standalone ampersands that aren't part of valid entity references
* - Fixes self-closing or unclosed tags that should be properly closed
* - Fixes duplicate closing tags with content between them
*/
internal fun escapeInvalidXmlEntities(xml: String): String {
- return xml
+ // HTML named entities (Ä, ) are unknown to XML: turn them into numeric
+ // references first, so the ampersand escaping below doesn't make them literal text
+ return HtmlNamedEntities.toNumericReferences(xml)
// Fix standalone ampersands in URLs and text
.replace(STANDALONE_AMPERSAND_REGEX, "&")
// Fix duplicate closing tags with content between them
diff --git a/rssparser/src/commonMain/kotlin/com/prof18/rssparser/internal/HtmlNamedEntities.kt b/rssparser/src/commonMain/kotlin/com/prof18/rssparser/internal/HtmlNamedEntities.kt
new file mode 100644
index 00000000..e4e2de54
--- /dev/null
+++ b/rssparser/src/commonMain/kotlin/com/prof18/rssparser/internal/HtmlNamedEntities.kt
@@ -0,0 +1,81 @@
+package com.prof18.rssparser.internal
+
+/**
+ * The HTML 4 named character references, which XML doesn't know about. Feeds often copy them
+ * straight from their HTML (`Ä`, ` `, `—`), and XML parsers reject them.
+ */
+internal object HtmlNamedEntities {
+
+ private val NAMED_ENTITY_REGEX = Regex("&([A-Za-z][A-Za-z0-9]{1,7});")
+
+ /**
+ * Replaces every known HTML named entity with the numeric character reference XML accepts,
+ * so `Ä` becomes `Ä`. The five XML entities and unknown names are left untouched.
+ */
+ fun toNumericReferences(xml: String): String =
+ NAMED_ENTITY_REGEX.replace(xml) { match ->
+ val codePoint = CODE_POINTS[match.groupValues[1]]
+ if (codePoint == null) match.value else "$codePoint;"
+ }
+
+ // Latin-1, U+00A0 to U+00FF, in code point order.
+ private val LATIN_1 = listOf(
+ "nbsp", "iexcl", "cent", "pound", "curren", "yen", "brvbar", "sect",
+ "uml", "copy", "ordf", "laquo", "not", "shy", "reg", "macr",
+ "deg", "plusmn", "sup2", "sup3", "acute", "micro", "para", "middot",
+ "cedil", "sup1", "ordm", "raquo", "frac14", "frac12", "frac34", "iquest",
+ "Agrave", "Aacute", "Acirc", "Atilde", "Auml", "Aring", "AElig", "Ccedil",
+ "Egrave", "Eacute", "Ecirc", "Euml", "Igrave", "Iacute", "Icirc", "Iuml",
+ "ETH", "Ntilde", "Ograve", "Oacute", "Ocirc", "Otilde", "Ouml", "times",
+ "Oslash", "Ugrave", "Uacute", "Ucirc", "Uuml", "Yacute", "THORN", "szlig",
+ "agrave", "aacute", "acirc", "atilde", "auml", "aring", "aelig", "ccedil",
+ "egrave", "eacute", "ecirc", "euml", "igrave", "iacute", "icirc", "iuml",
+ "eth", "ntilde", "ograve", "oacute", "ocirc", "otilde", "ouml", "divide",
+ "oslash", "ugrave", "uacute", "ucirc", "uuml", "yacute", "thorn", "yuml",
+ )
+
+ // Greek capitals start at U+0391 and lowercase at U+03B1; U+03A2 has no capital.
+ private val GREEK_UPPER = listOf(
+ "Alpha", "Beta", "Gamma", "Delta", "Epsilon", "Zeta", "Eta", "Theta", "Iota",
+ "Kappa", "Lambda", "Mu", "Nu", "Xi", "Omicron", "Pi", "Rho", null, "Sigma",
+ "Tau", "Upsilon", "Phi", "Chi", "Psi", "Omega",
+ )
+ private val GREEK_LOWER = listOf(
+ "alpha", "beta", "gamma", "delta", "epsilon", "zeta", "eta", "theta", "iota",
+ "kappa", "lambda", "mu", "nu", "xi", "omicron", "pi", "rho", "sigmaf", "sigma",
+ "tau", "upsilon", "phi", "chi", "psi", "omega",
+ )
+
+ private val OTHERS = mapOf(
+ "OElig" to 338, "oelig" to 339, "Scaron" to 352, "scaron" to 353, "Yuml" to 376,
+ "fnof" to 402, "circ" to 710, "tilde" to 732,
+ "thetasym" to 977, "upsih" to 978, "piv" to 982,
+ "ensp" to 8194, "emsp" to 8195, "thinsp" to 8201, "zwnj" to 8204, "zwj" to 8205,
+ "lrm" to 8206, "rlm" to 8207, "ndash" to 8211, "mdash" to 8212, "lsquo" to 8216,
+ "rsquo" to 8217, "sbquo" to 8218, "ldquo" to 8220, "rdquo" to 8221, "bdquo" to 8222,
+ "dagger" to 8224, "Dagger" to 8225, "bull" to 8226, "hellip" to 8230,
+ "permil" to 8240, "prime" to 8242, "Prime" to 8243, "lsaquo" to 8249,
+ "rsaquo" to 8250, "oline" to 8254, "frasl" to 8260, "euro" to 8364,
+ "image" to 8465, "weierp" to 8472, "real" to 8476, "trade" to 8482, "alefsym" to 8501,
+ "larr" to 8592, "uarr" to 8593, "rarr" to 8594, "darr" to 8595, "harr" to 8596,
+ "crarr" to 8629, "lArr" to 8656, "uArr" to 8657, "rArr" to 8658, "dArr" to 8659,
+ "hArr" to 8660, "forall" to 8704, "part" to 8706, "exist" to 8707, "empty" to 8709,
+ "nabla" to 8711, "isin" to 8712, "notin" to 8713, "ni" to 8715, "prod" to 8719,
+ "sum" to 8721, "minus" to 8722, "lowast" to 8727, "radic" to 8730, "prop" to 8733,
+ "infin" to 8734, "ang" to 8736, "and" to 8743, "or" to 8744, "cap" to 8745,
+ "cup" to 8746, "int" to 8747, "there4" to 8756, "sim" to 8764, "cong" to 8773,
+ "asymp" to 8776, "ne" to 8800, "equiv" to 8801, "le" to 8804, "ge" to 8805,
+ "sub" to 8834, "sup" to 8835, "nsub" to 8836, "sube" to 8838, "supe" to 8839,
+ "oplus" to 8853, "otimes" to 8855, "perp" to 8869, "sdot" to 8901,
+ "lceil" to 8968, "rceil" to 8969, "lfloor" to 8970, "rfloor" to 8971,
+ "lang" to 9001, "rang" to 9002, "loz" to 9674,
+ "spades" to 9824, "clubs" to 9827, "hearts" to 9829, "diams" to 9830,
+ )
+
+ private val CODE_POINTS: Map = buildMap {
+ LATIN_1.forEachIndexed { index, name -> put(name, 0xA0 + index) }
+ GREEK_UPPER.forEachIndexed { index, name -> if (name != null) put(name, 0x391 + index) }
+ GREEK_LOWER.forEachIndexed { index, name -> put(name, 0x3B1 + index) }
+ putAll(OTHERS)
+ }
+}
diff --git a/rssparser/src/commonTest/kotlin/com/prof18/rssparser/EscapeInvalidXmlEntitiesTest.kt b/rssparser/src/commonTest/kotlin/com/prof18/rssparser/EscapeInvalidXmlEntitiesTest.kt
index 6f804e46..ae182ae2 100644
--- a/rssparser/src/commonTest/kotlin/com/prof18/rssparser/EscapeInvalidXmlEntitiesTest.kt
+++ b/rssparser/src/commonTest/kotlin/com/prof18/rssparser/EscapeInvalidXmlEntitiesTest.kt
@@ -50,6 +50,73 @@ class EscapeInvalidXmlEntitiesTest : XmlParserTestExecutor() {
assertEquals(xml, rssParser().escapeInvalidXmlEntities(xml))
}
+ @Test
+ fun htmlNamedEntitiesAreTurnedIntoNumericReferences() = runTest {
+ val xml = "Myrskylä ÄÖ café — 5€ …"
+
+ val escaped = rssParser().escapeInvalidXmlEntities(xml)
+
+ assertEquals(
+ "Myrskylä ÄÖ café — 5€ …",
+ escaped,
+ )
+ }
+
+ @Test
+ fun unknownNamedEntitiesAreEscapedAsText() = runTest {
+ val xml = "¬anentity; &AUML;"
+
+ val escaped = rssParser().escapeInvalidXmlEntities(xml)
+
+ assertEquals("¬anentity; &AUML;", escaped)
+ }
+
+ @Test
+ fun aFeedWithHtmlNamedEntitiesIsParsedWithTheRightCharacters() = runTest {
+ val xml = """
+
+
+
+ Myrskylä
+ https://myrskyla.fi
+ Kunta
+
+ Äänestys – tulokset
+ https://myrskyla.fi/a?x=1&y=2
+ Café & ravintola
+
+
+
+ """.trimIndent()
+
+ val channel = rssParser().parse(xml)
+
+ assertEquals("Myrskylä", channel.title)
+ val item = channel.items.single()
+ assertEquals("Äänestys – tulokset", item.title)
+ assertEquals("https://myrskyla.fi/a?x=1&y=2", item.link)
+ assertEquals("Café & ravintola", item.description)
+ }
+
+ @Test
+ fun aFeedWithABareAmpersandIsParsedOnEveryPlatform() = runTest {
+ val xml = """
+
+
+
+ Tom & Jerry
+ First
+ Second
+
+
+ """.trimIndent()
+
+ val channel = rssParser().parse(xml)
+
+ assertEquals("Tom & Jerry", channel.title)
+ assertEquals(listOf("First", "Second"), channel.items.map { it.title })
+ }
+
private fun rssParser(): RssParser = RssParser(
xmlFetcher = object : XmlFetcher {
override suspend fun fetchXml(url: String): ParserInput =
diff --git a/rssparser/src/webMain/kotlin/com/prof18/rssparser/internal/WebXmlParser.kt b/rssparser/src/webMain/kotlin/com/prof18/rssparser/internal/WebXmlParser.kt
index 84c832a7..33938899 100644
--- a/rssparser/src/webMain/kotlin/com/prof18/rssparser/internal/WebXmlParser.kt
+++ b/rssparser/src/webMain/kotlin/com/prof18/rssparser/internal/WebXmlParser.kt
@@ -13,7 +13,9 @@ import kotlinx.serialization.decodeFromString
import kotlinx.serialization.modules.SerializersModule
import kotlinx.serialization.modules.polymorphic
import kotlinx.serialization.modules.subclass
+import nl.adaptivity.xmlutil.EventType
import nl.adaptivity.xmlutil.serialization.XML
+import nl.adaptivity.xmlutil.xmlStreaming
internal class WebXmlParser(
private val dispatcher: CoroutineDispatcher,
@@ -38,7 +40,7 @@ internal class WebXmlParser(
override suspend fun parseXML(input: ParserInput): RssChannel = withContext(dispatcher) {
val feedEntity = try {
- xml.decodeFromString(input.data)
+ decodeFeed(input.data)
} catch (_: Throwable) {
// Fallback to handle XHTML/HTML content and mismatched HTML tags in content/description tags
val cleanedData = input.data
@@ -62,7 +64,7 @@ internal class WebXmlParser(
"$cleanedContent"
}
try {
- xml.decodeFromString(cleanedData)
+ decodeFeed(cleanedData)
} catch (exception: Exception) {
throw RssParsingException(
message = "Something went wrong when parsing the feed. Please check if the XML is valid",
@@ -73,6 +75,37 @@ internal class WebXmlParser(
return@withContext feedEntity.toRssChannel(input.baseUrl)
}
+ /**
+ * The browser's DOMParser doesn't throw on malformed XML (a bare `&`, an HTML entity like
+ * `ä`): it stops at the error and puts a `parsererror` element in the document. Decoding
+ * that document gives an empty or truncated feed, so the error is turned into an exception here,
+ * which lets the caller retry with the escaped XML like on the other platforms.
+ */
+ private fun decodeFeed(data: String): FeedEntity {
+ if (hasParserError(data)) {
+ throw IllegalArgumentException("The XML is not well formed")
+ }
+ return xml.decodeFromString(data)
+ }
+
+ private fun hasParserError(data: String): Boolean {
+ val reader = xmlStreaming.newReader(data)
+ try {
+ while (reader.hasNext()) {
+ if (reader.next() == EventType.START_ELEMENT && reader.localName == PARSER_ERROR_TAG) {
+ return true
+ }
+ }
+ return false
+ } finally {
+ reader.close()
+ }
+ }
+
+ private companion object {
+ private const val PARSER_ERROR_TAG = "parsererror"
+ }
+
override fun generateParserInputFromString(rawRssFeed: String): ParserInput =
ParserInput(
data = rawRssFeed,