diff --git a/packages/markitdown/src/markitdown/converters/_markdownify.py b/packages/markitdown/src/markitdown/converters/_markdownify.py index 19e8a2984..d08a1bcd1 100644 --- a/packages/markitdown/src/markitdown/converters/_markdownify.py +++ b/packages/markitdown/src/markitdown/converters/_markdownify.py @@ -2,7 +2,24 @@ import markdownify from typing import Any, Optional -from urllib.parse import quote, unquote, urlparse, urlunparse +from urllib.parse import quote, urlparse, urlunparse + + +_PERCENT_ENCODED_OCTET = re.compile(r"%[0-9A-Fa-f]{2}") + + +def _quote_path_preserving_percent_encoded_octets(path: str) -> str: + """Quote a URL path while preserving existing %HH byte encodings.""" + parts: list[str] = [] + last_end = 0 + + for match in _PERCENT_ENCODED_OCTET.finditer(path): + parts.append(quote(path[last_end : match.start()])) + parts.append(match.group(0)) + last_end = match.end() + + parts.append(quote(path[last_end:])) + return "".join(parts) class _CustomMarkdownify(markdownify.MarkdownConverter): @@ -60,7 +77,13 @@ def convert_a( parsed_url = urlparse(href) # type: ignore if parsed_url.scheme and parsed_url.scheme.lower() not in ["http", "https", "file"]: # type: ignore return "%s%s%s" % (prefix, text, suffix) - href = urlunparse(parsed_url._replace(path=quote(unquote(parsed_url.path)))) # type: ignore + href = urlunparse( + parsed_url._replace( + path=_quote_path_preserving_percent_encoded_octets( + parsed_url.path + ) + ) + ) # type: ignore except ValueError: # It's not clear if this ever gets thrown return "%s%s%s" % (prefix, text, suffix) diff --git a/packages/markitdown/tests/test_html_converter.py b/packages/markitdown/tests/test_html_converter.py new file mode 100644 index 000000000..e3e62b51a --- /dev/null +++ b/packages/markitdown/tests/test_html_converter.py @@ -0,0 +1,65 @@ +import io + +from markitdown import MarkItDown + + +def _convert_html(html: str) -> str: + result = MarkItDown().convert_stream( + io.BytesIO(html.encode("utf-8")), + file_extension=".html", + ) + return result.markdown + + +def test_preserves_non_utf8_percent_encoded_href_path() -> None: + href = "https://abc.com/hist/" "%a5%c8%a5%c3%a5%d7%a5%da%a1%bc%a5%b8" + html = f'example' + + markdown = _convert_html(html) + + assert f"[example]({href})" in markdown + assert "%EF%BF%BD" not in markdown + + +def test_html_href_still_quotes_raw_unicode_and_spaces() -> None: + href = "https://example.com/a path/日本語" + expected_href = "https://example.com/a%20path/" "%E6%97%A5%E6%9C%AC%E8%AA%9E" + + markdown = _convert_html(f'example') + + assert f"[example]({expected_href})" in markdown + + +def test_html_href_quotes_literal_percent_sign() -> None: + href = "https://example.com/100% complete" + expected_href = "https://example.com/100%25%20complete" + + markdown = _convert_html(f'example') + + assert f"[example]({expected_href})" in markdown + + +def test_html_href_quotes_malformed_percent_escape() -> None: + href = "https://example.com/items/%ZZ/%2F" + expected_href = "https://example.com/items/%25ZZ/%2F" + + markdown = _convert_html(f'example') + + assert f"[example]({expected_href})" in markdown + + +def test_html_href_preserves_encoded_slash() -> None: + href = "https://example.com/items/a%2Fb" + + markdown = _convert_html(f'example') + + assert f"[example]({href})" in markdown + + +def test_html_href_does_not_quote_query_or_fragment() -> None: + href = "https://example.com/a path?query=a b%20c#fragment with spaces" + expected_href = "https://example.com/a%20path?query=a b%20c#fragment with spaces" + + markdown = _convert_html(f'example') + + assert f"[example]({expected_href})" in markdown