Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,16 @@ import org.xmlpull.v1.XmlPullParser
import org.xmlpull.v1.XmlPullParserException

internal fun XmlPullParser.nextTrimmedText(): String? {
val startDepth = this.depth
return try {
this.nextText()?.trim()
} catch (_: XmlPullParserException) {
} catch (exception: XmlPullParserException) {
// Only a child tag is recovered here: nextText() stopped on it, one level deeper. An error
// in the text itself (a bare `&`, an HTML entity like `ä`) leaves the depth unchanged
// and has to reach the caller, which parses the feed again with the XML escaped. Recovering
// from it would drop the text before the error and read on past the closing tag.
if (this.depth <= startDepth) throw exception

// Handle malformed HTML content (e.g., mismatched tag cases like <em>...</EM>) or HTML tags in content
// When nextText() throws, there are nested tags, so we manually parse and collect all text
// Note: nextText() may have consumed some text before throwing, so we start fresh from current position
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package com.prof18.rssparser

import com.prof18.rssparser.exception.RssParsingException
import com.prof18.rssparser.internal.HtmlNamedEntities
import com.prof18.rssparser.internal.XmlFetcher
import com.prof18.rssparser.internal.XmlParser
import com.prof18.rssparser.model.RssChannel
Expand Down Expand Up @@ -59,12 +60,15 @@ public class RssParser internal constructor(

/**
* Escapes invalid XML entities in the input string and fixes common XML issues.
* - Replaces HTML named entities (`&Auml;`, `&nbsp;`) with numeric character references
* - Escapes standalone ampersands that aren't part of valid entity references
* - Fixes self-closing or unclosed tags that should be properly closed
* - Fixes duplicate closing tags with content between them
*/
internal fun escapeInvalidXmlEntities(xml: String): String {
return xml
// HTML named entities (&Auml;, &nbsp;) are unknown to XML: turn them into numeric
// references first, so the ampersand escaping below doesn't make them literal text
return HtmlNamedEntities.toNumericReferences(xml)
// Fix standalone ampersands in URLs and text
.replace(STANDALONE_AMPERSAND_REGEX, "&amp;")
// Fix duplicate closing tags with content between them
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
package com.prof18.rssparser.internal

/**
* The HTML 4 named character references, which XML doesn't know about. Feeds often copy them
* straight from their HTML (`&Auml;`, `&nbsp;`, `&mdash;`), and XML parsers reject them.
*/
internal object HtmlNamedEntities {

private val NAMED_ENTITY_REGEX = Regex("&([A-Za-z][A-Za-z0-9]{1,7});")

/**
* Replaces every known HTML named entity with the numeric character reference XML accepts,
* so `&Auml;` becomes `&#196;`. The five XML entities and unknown names are left untouched.
*/
fun toNumericReferences(xml: String): String =
NAMED_ENTITY_REGEX.replace(xml) { match ->
val codePoint = CODE_POINTS[match.groupValues[1]]
if (codePoint == null) match.value else "&#$codePoint;"
}

// Latin-1, U+00A0 to U+00FF, in code point order.
private val LATIN_1 = listOf(
"nbsp", "iexcl", "cent", "pound", "curren", "yen", "brvbar", "sect",
"uml", "copy", "ordf", "laquo", "not", "shy", "reg", "macr",
"deg", "plusmn", "sup2", "sup3", "acute", "micro", "para", "middot",
"cedil", "sup1", "ordm", "raquo", "frac14", "frac12", "frac34", "iquest",
"Agrave", "Aacute", "Acirc", "Atilde", "Auml", "Aring", "AElig", "Ccedil",
"Egrave", "Eacute", "Ecirc", "Euml", "Igrave", "Iacute", "Icirc", "Iuml",
"ETH", "Ntilde", "Ograve", "Oacute", "Ocirc", "Otilde", "Ouml", "times",
"Oslash", "Ugrave", "Uacute", "Ucirc", "Uuml", "Yacute", "THORN", "szlig",
"agrave", "aacute", "acirc", "atilde", "auml", "aring", "aelig", "ccedil",
"egrave", "eacute", "ecirc", "euml", "igrave", "iacute", "icirc", "iuml",
"eth", "ntilde", "ograve", "oacute", "ocirc", "otilde", "ouml", "divide",
"oslash", "ugrave", "uacute", "ucirc", "uuml", "yacute", "thorn", "yuml",
)

// Greek capitals start at U+0391 and lowercase at U+03B1; U+03A2 has no capital.
private val GREEK_UPPER = listOf(
"Alpha", "Beta", "Gamma", "Delta", "Epsilon", "Zeta", "Eta", "Theta", "Iota",
"Kappa", "Lambda", "Mu", "Nu", "Xi", "Omicron", "Pi", "Rho", null, "Sigma",
"Tau", "Upsilon", "Phi", "Chi", "Psi", "Omega",
)
private val GREEK_LOWER = listOf(
"alpha", "beta", "gamma", "delta", "epsilon", "zeta", "eta", "theta", "iota",
"kappa", "lambda", "mu", "nu", "xi", "omicron", "pi", "rho", "sigmaf", "sigma",
"tau", "upsilon", "phi", "chi", "psi", "omega",
)

private val OTHERS = mapOf(
"OElig" to 338, "oelig" to 339, "Scaron" to 352, "scaron" to 353, "Yuml" to 376,
"fnof" to 402, "circ" to 710, "tilde" to 732,
"thetasym" to 977, "upsih" to 978, "piv" to 982,
"ensp" to 8194, "emsp" to 8195, "thinsp" to 8201, "zwnj" to 8204, "zwj" to 8205,
"lrm" to 8206, "rlm" to 8207, "ndash" to 8211, "mdash" to 8212, "lsquo" to 8216,
"rsquo" to 8217, "sbquo" to 8218, "ldquo" to 8220, "rdquo" to 8221, "bdquo" to 8222,
"dagger" to 8224, "Dagger" to 8225, "bull" to 8226, "hellip" to 8230,
"permil" to 8240, "prime" to 8242, "Prime" to 8243, "lsaquo" to 8249,
"rsaquo" to 8250, "oline" to 8254, "frasl" to 8260, "euro" to 8364,
"image" to 8465, "weierp" to 8472, "real" to 8476, "trade" to 8482, "alefsym" to 8501,
"larr" to 8592, "uarr" to 8593, "rarr" to 8594, "darr" to 8595, "harr" to 8596,
"crarr" to 8629, "lArr" to 8656, "uArr" to 8657, "rArr" to 8658, "dArr" to 8659,
"hArr" to 8660, "forall" to 8704, "part" to 8706, "exist" to 8707, "empty" to 8709,
"nabla" to 8711, "isin" to 8712, "notin" to 8713, "ni" to 8715, "prod" to 8719,
"sum" to 8721, "minus" to 8722, "lowast" to 8727, "radic" to 8730, "prop" to 8733,
"infin" to 8734, "ang" to 8736, "and" to 8743, "or" to 8744, "cap" to 8745,
"cup" to 8746, "int" to 8747, "there4" to 8756, "sim" to 8764, "cong" to 8773,
"asymp" to 8776, "ne" to 8800, "equiv" to 8801, "le" to 8804, "ge" to 8805,
"sub" to 8834, "sup" to 8835, "nsub" to 8836, "sube" to 8838, "supe" to 8839,
"oplus" to 8853, "otimes" to 8855, "perp" to 8869, "sdot" to 8901,
"lceil" to 8968, "rceil" to 8969, "lfloor" to 8970, "rfloor" to 8971,
"lang" to 9001, "rang" to 9002, "loz" to 9674,
"spades" to 9824, "clubs" to 9827, "hearts" to 9829, "diams" to 9830,
)

private val CODE_POINTS: Map<String, Int> = buildMap {
LATIN_1.forEachIndexed { index, name -> put(name, 0xA0 + index) }
GREEK_UPPER.forEachIndexed { index, name -> if (name != null) put(name, 0x391 + index) }
GREEK_LOWER.forEachIndexed { index, name -> put(name, 0x3B1 + index) }
putAll(OTHERS)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,73 @@ class EscapeInvalidXmlEntitiesTest : XmlParserTestExecutor() {
assertEquals(xml, rssParser().escapeInvalidXmlEntities(xml))
}

@Test
fun htmlNamedEntitiesAreTurnedIntoNumericReferences() = runTest {
val xml = "<title>Myrskyl&auml; &Auml;&Ouml; caf&eacute;&nbsp;&mdash; 5&euro; &hellip;</title>"

val escaped = rssParser().escapeInvalidXmlEntities(xml)

assertEquals(
"<title>Myrskyl&#228; &#196;&#214; caf&#233;&#160;&#8212; 5&#8364; &#8230;</title>",
escaped,
)
}

@Test
fun unknownNamedEntitiesAreEscapedAsText() = runTest {
val xml = "<title>&notanentity; &AUML;</title>"

val escaped = rssParser().escapeInvalidXmlEntities(xml)

assertEquals("<title>&amp;notanentity; &amp;AUML;</title>", escaped)
}

@Test
fun aFeedWithHtmlNamedEntitiesIsParsedWithTheRightCharacters() = runTest {
val xml = """
<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0">
<channel>
<title>Myrskyl&auml;</title>
<link>https://myrskyla.fi</link>
<description>Kunta</description>
<item>
<title>&Auml;&auml;nestys &ndash; tulokset</title>
<link>https://myrskyla.fi/a?x=1&y=2</link>
<description>Caf&eacute; &amp; ravintola</description>
</item>
</channel>
</rss>
""".trimIndent()

val channel = rssParser().parse(xml)

assertEquals("Myrskylä", channel.title)
val item = channel.items.single()
assertEquals("Äänestys – tulokset", item.title)
assertEquals("https://myrskyla.fi/a?x=1&y=2", item.link)
assertEquals("Café & ravintola", item.description)
}

@Test
fun aFeedWithABareAmpersandIsParsedOnEveryPlatform() = runTest {
val xml = """
<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0">
<channel>
<title>Tom & Jerry</title>
<item><title>First</title></item>
<item><title>Second</title></item>
</channel>
</rss>
""".trimIndent()

val channel = rssParser().parse(xml)

assertEquals("Tom & Jerry", channel.title)
assertEquals(listOf("First", "Second"), channel.items.map { it.title })
}

private fun rssParser(): RssParser = RssParser(
xmlFetcher = object : XmlFetcher {
override suspend fun fetchXml(url: String): ParserInput =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,9 @@ import kotlinx.serialization.decodeFromString
import kotlinx.serialization.modules.SerializersModule
import kotlinx.serialization.modules.polymorphic
import kotlinx.serialization.modules.subclass
import nl.adaptivity.xmlutil.EventType
import nl.adaptivity.xmlutil.serialization.XML
import nl.adaptivity.xmlutil.xmlStreaming

internal class WebXmlParser(
private val dispatcher: CoroutineDispatcher,
Expand All @@ -38,7 +40,7 @@ internal class WebXmlParser(

override suspend fun parseXML(input: ParserInput): RssChannel = withContext(dispatcher) {
val feedEntity = try {
xml.decodeFromString<FeedEntity>(input.data)
decodeFeed(input.data)
} catch (_: Throwable) {
// Fallback to handle XHTML/HTML content and mismatched HTML tags in content/description tags
val cleanedData = input.data
Expand All @@ -62,7 +64,7 @@ internal class WebXmlParser(
"<description>$cleanedContent</description>"
}
try {
xml.decodeFromString<FeedEntity>(cleanedData)
decodeFeed(cleanedData)
} catch (exception: Exception) {
throw RssParsingException(
message = "Something went wrong when parsing the feed. Please check if the XML is valid",
Expand All @@ -73,6 +75,37 @@ internal class WebXmlParser(
return@withContext feedEntity.toRssChannel(input.baseUrl)
}

/**
* The browser's DOMParser doesn't throw on malformed XML (a bare `&`, an HTML entity like
* `&auml;`): it stops at the error and puts a `parsererror` element in the document. Decoding
* that document gives an empty or truncated feed, so the error is turned into an exception here,
* which lets the caller retry with the escaped XML like on the other platforms.
*/
private fun decodeFeed(data: String): FeedEntity {
if (hasParserError(data)) {
throw IllegalArgumentException("The XML is not well formed")
}
return xml.decodeFromString<FeedEntity>(data)
}

private fun hasParserError(data: String): Boolean {
val reader = xmlStreaming.newReader(data)
try {
while (reader.hasNext()) {
if (reader.next() == EventType.START_ELEMENT && reader.localName == PARSER_ERROR_TAG) {
return true
}
}
return false
} finally {
reader.close()
}
}

private companion object {
private const val PARSER_ERROR_TAG = "parsererror"
}

override fun generateParserInputFromString(rawRssFeed: String): ParserInput =
ParserInput(
data = rawRssFeed,
Expand Down
Loading