Removed title and descritpion extraction from HTML

docs/ai/ai_actions/ai_actions.md

@@ -18,7 +18,7 @@ You can also extend it to perform other tasks or support additional AI services.
 "ai/ai_actions/configure_ai_actions",
 ("content_management/taxonomy/taxonomy#taxonomy-suggestions", "Taxonomy suggestions", "Learn how to use AI to suggest tags and categories"),
 ("permissions/policies#ai-actions", "Policies", "Learn about the available AI Actions policies"),
-("https://doc.ibexa.co/projects/userguide/en/5.0/ai_actions/work_with_ai_actions/"),
+("https://doc.ibexa.co/projects/userguide/en/5.0/ai_actions/work_with_ai_actions/", "Work with AI Actions", "Create new AI actions or modify existing ones to work faster and increase creativity."),
 ], columns=2) =]]
 
 ## Development

docs/commerce/shopping_list/shopping_list.md

@@ -21,7 +21,7 @@ A user can have several shopping lists, including a default one named "My Wishli
 [[= cards([
 "commerce/shopping_list/shopping_list_design",
 "commerce/shopping_list/shopping_list_api",
-("api/php_api/php_api_reference/namespaces/ibexa-contracts-shoppinglist.html", "PHP API Reference", "<code>Ibexa\Contracts\ShoppingList</code>"),
+("api/php_api/php_api_reference/namespaces/ibexa-contracts-shoppinglist.html", "PHP API Reference", "<code>Ibexa\\Contracts\\ShoppingList</code>"),
 "api/event_reference/shopping_list_events/",
 "search/shopping_list_search_reference/shopping_list_criteria/",
 "search/shopping_list_search_reference/shopping_list_sort_clauses/",

docs/content_management/collaborative_editing/collaborative_editing.md

@@ -21,7 +21,7 @@ This feature also introduces new dashboard tabs for managing shared drafts and j
 "content_management/collaborative_editing/collaborative_editing_guide",
 "content_management/collaborative_editing/configure_collaborative_editing",
 ("permissions/policies#collaborative-editing", "Policies", "Learn about the available Collaborative editing policies"),
-("https://doc.ibexa.co/projects/userguide/en/5.0/content_management/collaborative_editing/"),
+("https://doc.ibexa.co/projects/userguide/en/5.0/content_management/collaborative_editing/", "Collaborative editing", "Learn about Collaborative editing feature and its capabilities."),
 ], columns=2) =]]
 
 ## Development

docs/discounts/discounts.md

@@ -19,7 +19,7 @@ You can also extend the feature, for example, by creating custom pricing rules,
 "discounts/discounts_guide",
 "discounts/configure_discounts",
 ("permissions/policies#discounts", "Policies", "Learn about the available Discounts policies"),
-("https://doc.ibexa.co/projects/userguide/en/5.0/commerce/discounts/work_with_discounts/"),
+("https://doc.ibexa.co/projects/userguide/en/5.0/commerce/discounts/work_with_discounts/", "Work with Discounts", "Create and edit discounts, toggle discount status."),
 ], columns=2) =]]
 
 ## Development

docs/product_catalog/quable/quable.md

@@ -16,7 +16,7 @@
 
 [[= cards([
     "product_catalog/quable/quable_guide",
-    ("https://doc.ibexa.co/projects/userguide/en/5.0/product_catalog/quable_pim_integration/"),
+    ("https://doc.ibexa.co/projects/userguide/en/5.0/product_catalog/quable_pim_integration/", "Quable PIM integration", "Quable PIM integration allows you to use products managed in Quable as the source of product data in Ibexa DXP."),
     ("https://www.quable.com/en", "Quable - PIM solution for product data management", "Manage your product data and accelerate sales with Quable. Discover the new PIM platform that revolutionizes the product experience"),
     ("https://docs.quable.com/", "Quable resources", "Find all Quable PIM, DAM, and Portal resources: user guides, training content, product documentation, technical documentation, and the PIM API for developers."),
 ]) =]]

main.py

@@ -1,8 +1,6 @@
 import os
 import pprint
 import re
-import urllib.error
-import urllib.request
 from mkdocs.structure.pages import Page
 from mkdocs.utils import meta
 from typing import List
@@ -72,7 +70,6 @@ def include_code(file_path, start_line=1, end_line=None, indent_level=0, remove_
     @env.macro
     def cards(pages, columns=1, style="cards", force_version=False):
         current_page = env.variables.page
-        absolute_url = current_page.abs_url
         canonical = current_page.canonical_url
         url_parts = re.search(r"^(https?)://([^/]+)/([^/]+)/([^/]+)/", canonical)
         (scheme, site, language, version) = url_parts.groups()
@@ -104,63 +101,31 @@ def cards(pages, columns=1, style="cards", force_version=False):
             if hash:
                 hash = '#' + hash
 
-            if re.search(r"^https?://", path):
-                html = True
-                try:
-                    content = urllib.request.urlopen(path).read().decode('utf-8')
-                except urllib.error.URLError:
-                    content = ""
-            elif re.search(".html$", path):
-                html = True
-                content = open("docs/%s" % path, "r").read()
-                page = _absolute_page_url(scheme, site, language, version, page)
+            is_external = path.startswith(("http://", "https://"))
+            if is_external or path.endswith(".html"):
+                if not (custom_title and custom_description):
+                    raise ValueError(
+                        "cards(): card %r must supply a title and a description - only "
+                        "Markdown pages carry front matter to read them from." % path
+                    )
+                href = page if is_external else _absolute_page_url(
+                    scheme, site, language, version, page
+                )
+                title = custom_title
+                description = custom_description
             else:
-                html = False
                 path = path.rstrip('/')
                 content = open("docs/%s.md" % path, "r").read()
-                page = _absolute_page_url(scheme, site, language, version, path, hash)
-
-            if html:
-                match = re.search("<meta property=\"og:title\" content=\"(.*)\"", content, re.MULTILINE)
-                if match:
-                    title = match.groups()[0]
-                else:
-                    match = re.search("<title>(.*)</title>", content, re.MULTILINE)
-                    if match:
-                        title = match.groups()[0]
-                    else:
-                        title = ""
-                match = re.search("<meta property=\"og:description\" content=\"(.*)\"", content, re.MULTILINE)
-                if match:
-                    description = match.groups()[0]
-                else:
-                    match = re.search("<meta name=\"description\" content=\"(.*)\"", content, re.MULTILINE)
-                    if match:
-                        description = match.groups()[0]
-                    else:
-                        description = ""
-                href = page
-                title = custom_title if custom_title else title
-                title = title.replace("(Ibexa Documentation)", "").strip()
-                description = custom_description if custom_description else description
-            else:
                 match = re.search("^# (.*)", content, re.MULTILINE)
-                if match:
-                    header = match.groups()[0]
-                else:
-                    header = ""
-                default_meta = {
-                    "title": header,
-                    "short": "",
-                    "description": ""
-                }
                 current_meta = {
-                    **default_meta,
-                    **meta.get_data(content)[1]
+                    "title": match.groups()[0] if match else "",
+                    "short": "",
+                    "description": "",
+                    **meta.get_data(content)[1],
                 }
-                href = page
-                title = custom_title if custom_title else current_meta['short'] or current_meta['title']
-                description = custom_description if custom_description else current_meta['description'] or "&nbsp;"
+                href = _absolute_page_url(scheme, site, language, version, path, hash)
+                title = custom_title or current_meta['short'] or current_meta['title']
+                description = custom_description or current_meta['description'] or "&nbsp;"
                 title = resolve_variables(title, var_start, var_end, variables)
                 description = resolve_variables(description, var_start, var_end, variables)
 

tests/python/test_preprocess.py

@@ -1,4 +1,5 @@
 import mdformat
+import pytest
 from bs4 import BeautifulSoup
 from mkdocs_llmstxt._internal.plugin import _converter
 
@@ -91,6 +92,20 @@ def test_cards_become_link_list():
     assert "- [Page title](https://example.com/page/): Page description" in to_markdown(html)
 
 
+def test_card_with_empty_title_raises():
+    # An empty title used to produce <a href="..."></a>, which markdownify drops
+    # entirely - the link and its list item disappeared from the generated
+    # Markdown with nothing left to detect. Fail loudly instead.
+    html = (
+        '<div class="cards two-in-row"><div class="card-wrapper">'
+        '<a class="card" href="https://example.com/page/">'
+        '<p class="title"></p><p class="description"></p>'
+        "</a></div></div>"
+    )
+    with pytest.raises(ValueError, match="empty title"):
+        to_markdown(html)
+
+
 def test_inline_pill_becomes_parenthetical():
     html = '<p>Feature<span class="pill--inline pill--experience"></span> is available.</p>'
     assert "Feature (Experience) is available." in to_markdown(html)

tools/llms_txt/llmstxt_preprocess.py

@@ -371,12 +371,18 @@ def _process_cards(soup: Soup) -> None:
             title_elem = link.find("p", class_="title")
             description_elem = link.find("p", class_="description")
 
-            if not title_elem:
-                continue
-
-            title = title_elem.get_text(strip=True)
+            title = title_elem.get_text(strip=True) if title_elem else ""
             description = description_elem.get_text(strip=True) if description_elem else ""
 
+            # A titleless card would become <a href="..."></a>, which markdownify
+            # converts to the empty string - deleting the link and its whole list
+            # item without leaving a trace. Fail instead of shrinking the output.
+            if not title:
+                raise ValueError(
+                    "card has an empty title, refusing to emit a link that would be "
+                    "silently dropped: %s" % (href or "<no href>")
+                )
+
             li = soup.new_tag("li")
             link_tag = soup.new_tag("a", href=href)
             link_tag.string = title