diff --git a/packages/markitdown/src/markitdown/_markitdown.py b/packages/markitdown/src/markitdown/_markitdown.py index 8e2fe6a37c..e1f4901b30 100644 --- a/packages/markitdown/src/markitdown/_markitdown.py +++ b/packages/markitdown/src/markitdown/_markitdown.py @@ -42,6 +42,7 @@ DocumentIntelligenceConverter, ContentUnderstandingConverter, CsvConverter, + YamlConverter, ) from ._base_converter import DocumentConverter, DocumentConverterResult @@ -250,6 +251,7 @@ def enable_builtins(self, **kwargs) -> None: self.register_converter(OutlookMsgConverter()) self.register_converter(EpubConverter()) self.register_converter(CsvConverter()) + self.register_converter(YamlConverter()) # Register Document Intelligence converter at the top of the stack if endpoint is provided docintel_endpoint = kwargs.get("docintel_endpoint") diff --git a/packages/markitdown/src/markitdown/converters/__init__.py b/packages/markitdown/src/markitdown/converters/__init__.py index 77f8b1acd1..7d810cd107 100644 --- a/packages/markitdown/src/markitdown/converters/__init__.py +++ b/packages/markitdown/src/markitdown/converters/__init__.py @@ -27,6 +27,7 @@ ) from ._epub_converter import EpubConverter from ._csv_converter import CsvConverter +from ._yaml_converter import YamlConverter __all__ = [ "PlainTextConverter", @@ -51,4 +52,5 @@ "ContentUnderstandingFileType", "EpubConverter", "CsvConverter", + "YamlConverter", ] diff --git a/packages/markitdown/src/markitdown/converters/_yaml_converter.py b/packages/markitdown/src/markitdown/converters/_yaml_converter.py new file mode 100644 index 0000000000..5b20c1eb52 --- /dev/null +++ b/packages/markitdown/src/markitdown/converters/_yaml_converter.py @@ -0,0 +1,131 @@ +import io +from typing import BinaryIO, Any + +import yaml +from .._base_converter import DocumentConverter, DocumentConverterResult +from .._stream_info import StreamInfo + +ACCEPTED_FILE_EXTENSIONS = [".yaml", ".yml"] + + +class YamlConverter(DocumentConverter): + """Converts YAML files to Markdown.""" + + def __init__(self): + super().__init__() + + def accepts( + self, + file_stream: BinaryIO, + stream_info: StreamInfo, + **kwargs: Any, + ) -> bool: + extension = (stream_info.extension or "").lower() + return extension in ACCEPTED_FILE_EXTENSIONS + + def convert( + self, + file_stream: BinaryIO, + stream_info: StreamInfo, + **kwargs: Any, + ) -> DocumentConverterResult: + content = file_stream.read().decode("utf-8") + + try: + docs = list(yaml.safe_load_all(content)) + except yaml.YAMLError: + return DocumentConverterResult(markdown="") + + if len(docs) == 1: + markdown = self._render_value(docs[0]) + else: + parts = [] + for i, doc in enumerate(docs, 1): + parts.append(f"---\n\n**Document {i}:**\n\n{self._render_value(doc)}") + markdown = "\n\n".join(parts) + + return DocumentConverterResult(markdown=markdown.strip()) + + def _render_value(self, value: Any) -> str: + if value is None: + return "null" + if isinstance(value, bool): + return str(value).lower() + if isinstance(value, (int, float)): + return str(value) + if isinstance(value, str): + return value + if isinstance(value, list): + return self._render_list(value) + if isinstance(value, dict): + return self._render_dict(value) + return str(value) + + def _render_dict(self, data: dict) -> str: + if not data: + return "" + + if self._is_table_data(data): + return self._render_table(data) + + lines = [] + for key, value in data.items(): + if isinstance(value, (dict, list)): + lines.append(f"**{key}:**") + lines.append(self._render_value(value)) + else: + lines.append(f"**{key}:** {self._render_value(value)}") + return "\n\n".join(lines) + + def _render_list(self, items: list) -> str: + if not items: + return "" + + if items and isinstance(items[0], dict): + if self._is_uniform_dicts(items): + return self._render_list_table(items) + + lines = [] + for item in items: + lines.append(f"- {self._render_value(item)}") + return "\n".join(lines) + + def _is_table_data(self, data: Any) -> bool: + if isinstance(data, dict): + values = list(data.values()) + if values and isinstance(values[0], list): + if values[0] and isinstance(values[0][0], dict): + return True + return False + + def _is_uniform_dicts(self, items: list) -> bool: + if not items or not isinstance(items[0], dict): + return False + keys = set(items[0].keys()) + return len(keys) > 0 and all(isinstance(i, dict) and set(i.keys()) == keys for i in items) + + def _render_table(self, data: dict) -> str: + headers = list(data.keys()) + rows = list(zip(*[data[h] for h in headers])) + + header_line = "| " + " | ".join(str(h) for h in headers) + " |" + separator = "| " + " | ".join("---" for _ in headers) + " |" + data_lines = [] + for row in rows: + data_lines.append("| " + " | ".join(str(v) for v in row) + " |") + + return "\n".join([header_line, separator] + data_lines) + + def _render_list_table(self, items: list) -> str: + if not items: + return "" + + headers = list(items[0].keys()) + header_line = "| " + " | ".join(str(h) for h in headers) + " |" + separator = "| " + " | ".join("---" for _ in headers) + " |" + data_lines = [] + for item in items: + row = [str(item.get(h, "")) for h in headers] + data_lines.append("| " + " | ".join(row) + " |") + + return "\n".join([header_line, separator] + data_lines) diff --git a/packages/markitdown/tests/test_yaml_converter_328df3.py b/packages/markitdown/tests/test_yaml_converter_328df3.py new file mode 100644 index 0000000000..c3a3ae6152 --- /dev/null +++ b/packages/markitdown/tests/test_yaml_converter_328df3.py @@ -0,0 +1,242 @@ +"""Tests for the YAML converter.""" + +import io +import pytest +from markitdown import MarkItDown + + +@pytest.fixture +def md(): + return MarkItDown() + + +class TestYamlConverterBasic: + """Basic YAML conversion tests.""" + + def test_simple_key_value(self, md): + """Simple key-value pairs should convert to labeled blocks.""" + yaml_content = b"name: John\nage: 30\ncity: New York" + result = md.convert_stream( + io.BytesIO(yaml_content), file_extension=".yaml" + ) + md_text = result.markdown + assert "**name:** John" in md_text + assert "**age:** 30" in md_text + assert "**city:** New York" in md_text + + def test_nested_mapping(self, md): + """Nested mappings should use bold labels with proper hierarchy.""" + yaml_content = b"person:\n name: Alice\n address:\n city: Boston\n zip: \"02101\"" + result = md.convert_stream( + io.BytesIO(yaml_content), file_extension=".yaml" + ) + md_text = result.markdown + assert "**person:**" in md_text + assert "**name:** Alice" in md_text + assert "**city:** Boston" in md_text + assert "**zip:** 02101" in md_text + + def test_sequence_of_mappings_table(self, md): + """Sequence of mappings with uniform keys must render as a Markdown table with pipes and separators.""" + yaml_content = ( + b"- name: Bob\n age: 25\n" + b"- name: Carol\n age: 28\n" + b"- name: Dave\n age: 35" + ) + result = md.convert_stream( + io.BytesIO(yaml_content), file_extension=".yaml" + ) + md_text = result.markdown + assert "|" in md_text, "Table must use pipe characters" + lines = [l.strip() for l in md_text.strip().split("\n") if l.strip()] + table_lines = [l for l in lines if l.startswith("|")] + assert len(table_lines) >= 3, "Table must have header + separator + data rows" + assert "---" in table_lines[1], "Second row must be separator" + header_cells = [c.strip().lower() for c in table_lines[0].split("|") if c.strip()] + assert "name" in header_cells + assert "age" in header_cells + for name in ["bob", "carol", "dave"]: + assert name in md_text.lower() + + def test_yaml_extension(self, md): + """Both .yaml and .yml extensions should work.""" + yaml_content = b"key: value" + for ext in [".yaml", ".yml"]: + result = md.convert_stream(io.BytesIO(yaml_content), file_extension=ext) + assert result.markdown.strip() != "" + + def test_yml_extension(self, md): + """YML extension should produce same output as YAML.""" + yaml_content = b"key: value" + r1 = md.convert_stream(io.BytesIO(yaml_content), file_extension=".yaml") + r2 = md.convert_stream(io.BytesIO(yaml_content), file_extension=".yml") + assert r1.markdown.strip() == r2.markdown.strip() + + +class TestYamlConverterEdgeCases: + """Edge case tests for YAML converter.""" + + def test_empty_yaml(self, md): + """Empty YAML should not crash and should return a result.""" + yaml_content = b"" + result = md.convert_stream( + io.BytesIO(yaml_content), file_extension=".yaml" + ) + assert result is not None + assert isinstance(result.markdown, str) + + def test_scalar_value(self, md): + """A single scalar string value should be in the output.""" + yaml_content = b'"just a string"' + result = md.convert_stream( + io.BytesIO(yaml_content), file_extension=".yaml" + ) + assert "just a string" in result.markdown + + def test_deeply_nested(self, md): + """Deeply nested structures should render all levels.""" + yaml_content = b"a:\n b:\n c:\n d:\n e: deep_value" + result = md.convert_stream( + io.BytesIO(yaml_content), file_extension=".yaml" + ) + assert "deep_value" in result.markdown.lower() + assert "**a:**" in result.markdown + + def test_special_characters(self, md): + """YAML with special characters should be handled.""" + yaml_content = b"message: \"Hello World!\"\npath: /usr/local/bin" + result = md.convert_stream( + io.BytesIO(yaml_content), file_extension=".yaml" + ) + assert "hello world" in result.markdown.lower() + + def test_multiline_string(self, md): + """Multiline YAML strings should be preserved.""" + yaml_content = b"description: |\n This is line one.\n This is line two.\n This is line three." + result = md.convert_stream( + io.BytesIO(yaml_content), file_extension=".yaml" + ) + assert "line one" in result.markdown.lower() + assert "line two" in result.markdown.lower() + assert "line three" in result.markdown.lower() + + def test_list_of_scalars(self, md): + """A list of scalar values should use bullet points.""" + yaml_content = b"- apple\n- banana\n- cherry" + result = md.convert_stream( + io.BytesIO(yaml_content), file_extension=".yaml" + ) + md_text = result.markdown + assert "- " in md_text, "Lists should use bullet points" + assert "apple" in md_text.lower() + assert "banana" in md_text.lower() + assert "cherry" in md_text.lower() + + def test_mixed_types_in_list(self, md): + """Lists with mixed types should not crash.""" + yaml_content = b"- 42\n- hello\n- true\n- 3.14" + result = md.convert_stream( + io.BytesIO(yaml_content), file_extension=".yaml" + ) + assert "42" in result.markdown + assert "hello" in result.markdown.lower() + + def test_null_values(self, md): + """YAML null values should be rendered as 'null'.""" + yaml_content = b"name: test\nvalue: null" + result = md.convert_stream( + io.BytesIO(yaml_content), file_extension=".yaml" + ) + assert "**name:** test" in result.markdown + assert "**value:** null" in result.markdown + + def test_boolean_values(self, md): + """YAML booleans should be rendered as lowercase.""" + yaml_content = b"enabled: true\ndisabled: false" + result = md.convert_stream( + io.BytesIO(yaml_content), file_extension=".yaml" + ) + assert "**enabled:** true" in result.markdown + assert "**disabled:** false" in result.markdown + + def test_numeric_types(self, md): + """YAML numeric types should be rendered as strings.""" + yaml_content = b"integer: 42\nfloat: 3.14\nnegative: -10" + result = md.convert_stream( + io.BytesIO(yaml_content), file_extension=".yaml" + ) + assert "**integer:** 42" in result.markdown + assert "**float:** 3.14" in result.markdown + assert "**negative:** -10" in result.markdown + + def test_multiple_documents(self, md): + """Multiple YAML documents separated by --- should be handled in order.""" + yaml_content = b"first: one\n---\nsecond: two" + result = md.convert_stream( + io.BytesIO(yaml_content), file_extension=".yaml" + ) + md_lower = result.markdown.lower() + first_pos = md_lower.find("first") + second_pos = md_lower.find("second") + assert first_pos != -1, "Should contain 'first'" + assert second_pos != -1, "Should contain 'second'" + assert first_pos < second_pos, "Documents should appear in order" + + def test_anchor_and_alias(self, md): + """YAML anchors and aliases should be resolved.""" + yaml_content = b"defaults: &defaults\n adapter: postgres\n host: localhost\ndevelopment:\n <<: *defaults" + result = md.convert_stream( + io.BytesIO(yaml_content), file_extension=".yaml" + ) + assert "postgres" in result.markdown.lower() + assert "localhost" in result.markdown.lower() + + def test_complex_key(self, md): + """YAML with complex (quoted) keys should be handled.""" + yaml_content = b"\"quoted key\": value\n'another key': other" + result = md.convert_stream( + io.BytesIO(yaml_content), file_extension=".yaml" + ) + assert "value" in result.markdown.lower() + assert "other" in result.markdown.lower() + + def test_flow_mapping(self, md): + """YAML flow mappings {key: value} should be handled.""" + yaml_content = b"inline: {name: test, value: 42}" + result = md.convert_stream( + io.BytesIO(yaml_content), file_extension=".yaml" + ) + assert "test" in result.markdown.lower() + assert "42" in result.markdown + + def test_flow_sequence(self, md): + """YAML flow sequences [1, 2, 3] should be handled.""" + yaml_content = b"items: [apple, banana, cherry]" + result = md.convert_stream( + io.BytesIO(yaml_content), file_extension=".yaml" + ) + assert "apple" in result.markdown.lower() + assert "banana" in result.markdown.lower() + assert "cherry" in result.markdown.lower() + + def test_mixed_table_and_non_table(self, md): + """A list of dicts with different keys should use bullets, not a table.""" + yaml_content = ( + b"- name: Bob\n age: 25\n" + b"- city: Boston\n zip: 02101" + ) + result = md.convert_stream( + io.BytesIO(yaml_content), file_extension=".yaml" + ) + md_text = result.markdown + table_lines = [l for l in md_text.strip().split("\n") if l.strip().startswith("|")] + assert len(table_lines) < 3, "Non-uniform dicts should not render as table" + + def test_multiline_folded_string(self, md): + """YAML folded multiline strings should be preserved.""" + yaml_content = b"description: >\n This is a long\n description that\n spans multiple lines" + result = md.convert_stream( + io.BytesIO(yaml_content), file_extension=".yaml" + ) + assert "long" in result.markdown.lower() + assert "description" in result.markdown.lower() diff --git a/test.sh b/test.sh new file mode 100644 index 0000000000..6820317deb --- /dev/null +++ b/test.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Test runner for the YAML converter tests. +# +# Usage: +# ./test.sh [--output_path ] +# +# base run the existing repository tests; these must pass both before and +# after the solution is applied. +# new run the new tests; these fail before the solution and pass after it. +set -uo pipefail + +cd /app + +OUTPUT_PATH="" +if [ "${1:-}" = "--output_path" ]; then + OUTPUT_PATH="$2" + shift 2 +fi + +MODE="${1:-new}" + +TEST_LOG="$(mktemp)" +STATUS=0 + +run_tests() { + regex="$1" + shift + if [ -n "$regex" ]; then + python -m pytest "$@" -k "$regex" -v 2>&1 | tee -a "$TEST_LOG" + else + python -m pytest "$@" -v 2>&1 | tee -a "$TEST_LOG" + fi + status=${PIPESTATUS[0]} + if [ "$status" -ne 0 ]; then + STATUS=$status + fi +} + +case "$MODE" in + base) + if [ -n "$OUTPUT_PATH" ]; then + python -m pytest packages/markitdown/tests/test_module_misc.py packages/markitdown/tests/test_html_converter.py -v --junitxml="$OUTPUT_PATH" -k "not test_markitdown_remote and not test_speech_transcription" 2>&1 | tee -a "$TEST_LOG" + status=${PIPESTATUS[0]} + if [ "$status" -ne 0 ]; then + STATUS=$status + fi + else + run_tests "" packages/markitdown/tests/test_module_misc.py packages/markitdown/tests/test_html_converter.py -k "not test_markitdown_remote and not test_speech_transcription" + fi + ;; + new) + if [ -n "$OUTPUT_PATH" ]; then + python -m pytest packages/markitdown/tests/test_yaml_converter_328df3.py -v --junitxml="$OUTPUT_PATH" 2>&1 | tee -a "$TEST_LOG" + status=${PIPESTATUS[0]} + if [ "$status" -ne 0 ]; then + STATUS=$status + fi + else + run_tests "" packages/markitdown/tests/test_yaml_converter_328df3.py + fi + ;; + *) + echo "unknown mode: $MODE (expected base or new)" >&2 + exit 2 + ;; +esac + +exit "$STATUS"