Create our own HTML to Markdown converter

discord_rss_bot/custom_filters.py

@@ -1,7 +1,6 @@
 import urllib.parse
 from functools import lru_cache
 
-import html2text
 from reader import Entry, Reader
 
 from discord_rss_bot.filter.blacklist import has_black_tags, should_be_skipped
@@ -54,14 +53,3 @@ def entry_is_blacklisted(entry_to_check: Entry) -> bool:
 
     """
     return bool(has_black_tags(reader, entry_to_check.feed) and should_be_skipped(reader, entry_to_check))
-
-
-@lru_cache()
-def convert_to_md(thing: str) -> str:
-    """Discord does not support tables so we need to remove them from the markdown."""
-    text_maker: html2text.HTML2Text = html2text.HTML2Text()
-
-    # Ignore tables
-    text_maker.ignore_tables = True
-
-    return text_maker.handle(thing) if thing else ""

discord_rss_bot/custom_message.py

@@ -1,32 +1,43 @@
-import re
+from bs4 import BeautifulSoup
-from functools import lru_cache
-
 from reader import Entry, Feed, Reader, TagNotFoundError
 
-from discord_rss_bot.custom_filters import convert_to_md
+from discord_rss_bot.markdown import convert_html_to_md
 from discord_rss_bot.settings import get_reader
 
 
-def get_images_from_entry(entry: Entry, summary: bool = False) -> list[str]:
+def get_images_from_entry(entry: Entry):
     """Get images from a entry.
 
     Args:
         entry: The entry to get the images from.
-        summary: Whether to get the images from the summary or the content.
 
     Returns:
         Returns a list of images.
     """
-    # This regex will match any markdown image that follows the format of ![alt text](image url).
-    image_regex = r"!\[(.*)\]\((.*)\)"
 
-    if summary:
+    def return_image(found_images):
-        return re.findall(image_regex, convert_to_md(entry.summary)) if entry.summary else []
+        soup: BeautifulSoup = BeautifulSoup(found_images, "html.parser")
+        images = soup.find_all("img")
+        for image in images:
+            image_src = image["src"] or ""
+            image_alt: str = "Link to image"
+            if image.get("alt"):
+                image_alt = image.get("alt")
+            return [(image_src, image_alt)]
 
-    return re.findall(image_regex, convert_to_md(entry.content[0].value)) if entry.content else []
+    images = []
+    # Get the images from the summary with beautiful soup
+    if entry.summary:
+        images = return_image(entry.summary)
+
+    # Get the images from the content with beautiful soup
+    if entry.content:
+        images = return_image(entry.content[0].value)
+
+    # No images found
+    return images
 
 
-@lru_cache()
 def try_to_replace(custom_message: str, template: str, replace_with: str) -> str:
     """Try to replace a tag in custom_message.
 
@@ -45,19 +56,6 @@ def try_to_replace(custom_message: str, template: str, replace_with: str) -> str
         return custom_message
 
 
-@lru_cache()
-def remove_image_tags(message: str) -> str:
-    """Remove image tags from message.
-
-    Args:
-        message: The message to remove the tags from.
-
-    Returns:
-        Returns the message with the image tags removed.
-    """
-    return re.sub(r"!\[(.*)\]\((.*)\)", "", message)
-
-
 def replace_tags(feed: Feed, entry: Entry) -> str:
     """Replace tags in custom_message.
 
@@ -75,17 +73,15 @@ def replace_tags(feed: Feed, entry: Entry) -> str:
     content = ""
     if entry.summary:
         summary: str = entry.summary
-        summary = convert_to_md(summary)
+        summary = convert_html_to_md(summary)
-        summary = remove_image_tags(message=summary)
 
     if entry.content:
         for content_item in entry.content:
             content: str = content_item.value
-            content = convert_to_md(content)
+            content = convert_html_to_md(content)
-            content = remove_image_tags(message=content)
 
     if images := get_images_from_entry(entry=entry):
-        first_image: str = images[0][1]
+        first_image: str = images[0][0]
     else:
         first_image = ""
 
@@ -123,10 +119,7 @@ def replace_tags(feed: Feed, entry: Entry) -> str:
         for template, replace_with in replacement.items():
             custom_message = try_to_replace(custom_message, template, replace_with)
 
-    # Replace \\n with newlines.
+    return custom_message.replace("\\n", "\n")
-    custom_message_with_newlines = custom_message.replace("\\n", "\n")
-
-    return custom_message_with_newlines
 
 
 def get_custom_message(custom_reader: Reader, feed: Feed) -> str:

discord_rss_bot/main.py

@@ -12,11 +12,12 @@ from reader import Entry, EntryCounts, EntrySearchCounts, EntrySearchResult, Fee
 from starlette.responses import RedirectResponse
 
 from discord_rss_bot import settings
-from discord_rss_bot.custom_filters import convert_to_md, encode_url, entry_is_blacklisted, entry_is_whitelisted
+from discord_rss_bot.custom_filters import encode_url, entry_is_blacklisted, entry_is_whitelisted
-from discord_rss_bot.custom_message import get_custom_message, get_images_from_entry, remove_image_tags
+from discord_rss_bot.custom_message import get_custom_message, get_images_from_entry, replace_tags
 from discord_rss_bot.feeds import get_entry_from_id, send_entry_to_discord, send_to_discord
 from discord_rss_bot.filter.blacklist import get_blacklist_content, get_blacklist_summary, get_blacklist_title
 from discord_rss_bot.filter.whitelist import get_whitelist_content, get_whitelist_summary, get_whitelist_title
+from discord_rss_bot.markdown import convert_html_to_md
 from discord_rss_bot.search import create_html_for_search_results
 from discord_rss_bot.settings import default_custom_message, get_reader, list_webhooks
 
@@ -30,8 +31,7 @@ reader: Reader = get_reader()
 templates.env.filters["encode_url"] = encode_url
 templates.env.filters["entry_is_whitelisted"] = entry_is_whitelisted
 templates.env.filters["entry_is_blacklisted"] = entry_is_blacklisted
-templates.env.filters["discord_markdown"] = convert_to_md
+templates.env.filters["discord_markdown"] = convert_html_to_md
-templates.env.filters["remove_image_tags"] = remove_image_tags
 
 
 @app.post("/add_webhook")
@@ -429,19 +429,13 @@ def create_html_for_feed(entries: Iterable[Entry]) -> str:
         first_image = ""
         first_image_text = ""
         if images := get_images_from_entry(entry=entry):
-            first_image: str = images[0][1]
+            first_image: str = images[0][0]
-            first_image_text: str = images[0][0]
+            first_image_text: str = images[0][1]
 
         # Get the text from the entry.
+        text = replace_tags(entry.feed, entry)
+        if not text:
             text = "<div class='text-muted'>No content available.</div>"
-        if entry.summary:
-            summary: str = convert_to_md(entry.summary)
-            summary = remove_image_tags(message=summary)
-            text: str = f"<div class='text-muted'>{summary}</div>"
-        elif entry.content:
-            content: str = convert_to_md(entry.content[0].value)
-            content = remove_image_tags(message=content)
-            text = f"<div class='text-muted'>{content}</div>"
 
         published = ""
         if entry.published:

discord_rss_bot/markdown.py

@@ -0,0 +1,68 @@
+from functools import lru_cache
+
+from bs4 import BeautifulSoup
+
+
+@lru_cache(maxsize=2048)
+def convert_html_to_md(html: str) -> str:
+    """Convert HTML to Markdown.
+
+    Discord supports:
+    - Bold with **text**
+    - Italic with *text*
+    - Blockquote with >>> text
+    - Code with `text`
+        - Fence code with ```text```
+    - Links with [text](url)
+    - Syntax highlighting with ```language
+    - Strikethrough with ~~text~~
+    """
+    soup: BeautifulSoup = BeautifulSoup(html, features="lxml")
+
+    # Bold
+    for bold in soup.find_all("b") + soup.find_all("strong"):
+        bold.replace_with(f"**{bold.text}**")
+
+    # Italic
+    for italic in soup.find_all("i") + soup.find_all("em"):
+        italic.replace_with(f"*{italic.text}*")
+
+    # Blockquote
+    for blockquote in soup.find_all("blockquote") + soup.find_all("q"):
+        blockquote.replace_with(f">>> {blockquote.text}")
+
+    # Code
+    for code in soup.find_all("code") + soup.find_all("pre"):
+        code.replace_with(f"`{code.text}`")
+
+    # Links
+    for link in soup.find_all("a") + soup.find_all("link"):
+        link_text = link.text or link.get("href") or "Link"
+        link.replace_with(f"[{link_text}]({link.get('href')})")
+
+    # Strikethrough
+    for strikethrough in soup.find_all("s") + soup.find_all("del") + soup.find_all("strike"):
+        strikethrough.replace_with(f"~~{strikethrough.text}~~")
+
+    # <br> tags
+    for br in soup.find_all("br"):
+        br.replace_with("\n")
+
+    # Remove all other tags
+    for tag in soup.find_all(True):
+        tag.replace_with(tag.text)
+
+    # If the text ends with a newline, remove it
+    # return soup.text[:-1] if soup.text.endswith("\n") else soup.text
+    return soup.text
+
+
+# Test the function
+if __name__ == "__main__":
+    html: str = """
+    <p><b>bold</b> <i>italic</i> <a href="https://example.com">link</a> <code>code</code> <s>strikethrough</s></p>
+    <blockquote>blockquote</blockquote>
+    <pre><code>pre code</code></pre>
+    <strong>strong</strong>
+    """
+    print(convert_html_to_md(html))

discord_rss_bot/templates/custom.html

@@ -119,7 +119,7 @@
                                         {% raw %}
                                             {{entry_content}}
                                         {% endraw %}
-                                    </code>{{entry.content[0].value|discord_markdown|remove_image_tags}}
+                                    </code>{{entry.content[0].value|discord_markdown}}
                                 </li>
                                 <li>
                                     <code>
@@ -175,7 +175,7 @@
                                         {% raw %}
                                             {{entry_summary}}
                                         {% endraw %}
-                                    </code>{{entry.summary|discord_markdown|remove_image_tags}}
+                                    </code>{{entry.summary|discord_markdown}}
                                 </li>
                                 <li>
                                     <code>
@@ -222,7 +222,7 @@
                                     <code>
                                         <pre>
 {{feed.title -}}
-{{- entry.content[0].value|discord_markdown|remove_image_tags -}}
+{{- entry.content[0].value|discord_markdown -}}
                             </pre>
                                     </code>
                                 </li>

poetry.lock

@@ -309,18 +309,6 @@ files = [
     {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"},
 ]
 
-[[package]]
-name = "html2text"
-version = "2020.1.16"
-description = "Turn HTML into equivalent Markdown-structured text."
-category = "main"
-optional = false
-python-versions = ">=3.5"
-files = [
-    {file = "html2text-2020.1.16-py3-none-any.whl", hash = "sha256:c7c629882da0cf377d66f073329ccf34a12ed2adf0169b9285ae4e63ef54c82b"},
-    {file = "html2text-2020.1.16.tar.gz", hash = "sha256:e296318e16b059ddb97f7a8a1d6a5c1d7af4544049a01e261731d2d5cc277bbb"},
-]
-
 [[package]]
 name = "httpcore"
 version = "0.16.3"
@@ -494,6 +482,99 @@ win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""}
 [package.extras]
 dev = ["Sphinx (>=4.1.1)", "black (>=19.10b0)", "colorama (>=0.3.4)", "docutils (==0.16)", "flake8 (>=3.7.7)", "isort (>=5.1.1)", "pytest (>=4.6.2)", "pytest-cov (>=2.7.1)", "sphinx-autobuild (>=0.7.1)", "sphinx-rtd-theme (>=0.4.3)", "tox (>=3.9.0)"]
 
+[[package]]
+name = "lxml"
+version = "4.9.2"
+description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API."
+category = "main"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, != 3.4.*"
+files = [
+    {file = "lxml-4.9.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:76cf573e5a365e790396a5cc2b909812633409306c6531a6877c59061e42c4f2"},
+    {file = "lxml-4.9.2-cp27-cp27m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:b1f42b6921d0e81b1bcb5e395bc091a70f41c4d4e55ba99c6da2b31626c44892"},
+    {file = "lxml-4.9.2-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:9f102706d0ca011de571de32c3247c6476b55bb6bc65a20f682f000b07a4852a"},
+    {file = "lxml-4.9.2-cp27-cp27m-win32.whl", hash = "sha256:8d0b4612b66ff5d62d03bcaa043bb018f74dfea51184e53f067e6fdcba4bd8de"},
+    {file = "lxml-4.9.2-cp27-cp27m-win_amd64.whl", hash = "sha256:4c8f293f14abc8fd3e8e01c5bd86e6ed0b6ef71936ded5bf10fe7a5efefbaca3"},
+    {file = "lxml-4.9.2-cp27-cp27mu-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2899456259589aa38bfb018c364d6ae7b53c5c22d8e27d0ec7609c2a1ff78b50"},
+    {file = "lxml-4.9.2-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6749649eecd6a9871cae297bffa4ee76f90b4504a2a2ab528d9ebe912b101975"},
+    {file = "lxml-4.9.2-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:a08cff61517ee26cb56f1e949cca38caabe9ea9fbb4b1e10a805dc39844b7d5c"},
+    {file = "lxml-4.9.2-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:85cabf64adec449132e55616e7ca3e1000ab449d1d0f9d7f83146ed5bdcb6d8a"},
+    {file = "lxml-4.9.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:8340225bd5e7a701c0fa98284c849c9b9fc9238abf53a0ebd90900f25d39a4e4"},
+    {file = "lxml-4.9.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:1ab8f1f932e8f82355e75dda5413a57612c6ea448069d4fb2e217e9a4bed13d4"},
+    {file = "lxml-4.9.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:699a9af7dffaf67deeae27b2112aa06b41c370d5e7633e0ee0aea2e0b6c211f7"},
+    {file = "lxml-4.9.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b9cc34af337a97d470040f99ba4282f6e6bac88407d021688a5d585e44a23184"},
+    {file = "lxml-4.9.2-cp310-cp310-win32.whl", hash = "sha256:d02a5399126a53492415d4906ab0ad0375a5456cc05c3fc0fc4ca11771745cda"},
+    {file = "lxml-4.9.2-cp310-cp310-win_amd64.whl", hash = "sha256:a38486985ca49cfa574a507e7a2215c0c780fd1778bb6290c21193b7211702ab"},
+    {file = "lxml-4.9.2-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:c83203addf554215463b59f6399835201999b5e48019dc17f182ed5ad87205c9"},
+    {file = "lxml-4.9.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:2a87fa548561d2f4643c99cd13131acb607ddabb70682dcf1dff5f71f781a4bf"},
+    {file = "lxml-4.9.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:d6b430a9938a5a5d85fc107d852262ddcd48602c120e3dbb02137c83d212b380"},
+    {file = "lxml-4.9.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:3efea981d956a6f7173b4659849f55081867cf897e719f57383698af6f618a92"},
+    {file = "lxml-4.9.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:df0623dcf9668ad0445e0558a21211d4e9a149ea8f5666917c8eeec515f0a6d1"},
+    {file = "lxml-4.9.2-cp311-cp311-win32.whl", hash = "sha256:da248f93f0418a9e9d94b0080d7ebc407a9a5e6d0b57bb30db9b5cc28de1ad33"},
+    {file = "lxml-4.9.2-cp311-cp311-win_amd64.whl", hash = "sha256:3818b8e2c4b5148567e1b09ce739006acfaa44ce3156f8cbbc11062994b8e8dd"},
+    {file = "lxml-4.9.2-cp35-cp35m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ca989b91cf3a3ba28930a9fc1e9aeafc2a395448641df1f387a2d394638943b0"},
+    {file = "lxml-4.9.2-cp35-cp35m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:822068f85e12a6e292803e112ab876bc03ed1f03dddb80154c395f891ca6b31e"},
+    {file = "lxml-4.9.2-cp35-cp35m-win32.whl", hash = "sha256:be7292c55101e22f2a3d4d8913944cbea71eea90792bf914add27454a13905df"},
+    {file = "lxml-4.9.2-cp35-cp35m-win_amd64.whl", hash = "sha256:998c7c41910666d2976928c38ea96a70d1aa43be6fe502f21a651e17483a43c5"},
+    {file = "lxml-4.9.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:b26a29f0b7fc6f0897f043ca366142d2b609dc60756ee6e4e90b5f762c6adc53"},
+    {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:ab323679b8b3030000f2be63e22cdeea5b47ee0abd2d6a1dc0c8103ddaa56cd7"},
+    {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:689bb688a1db722485e4610a503e3e9210dcc20c520b45ac8f7533c837be76fe"},
+    {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:f49e52d174375a7def9915c9f06ec4e569d235ad428f70751765f48d5926678c"},
+    {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:36c3c175d34652a35475a73762b545f4527aec044910a651d2bf50de9c3352b1"},
+    {file = "lxml-4.9.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a35f8b7fa99f90dd2f5dc5a9fa12332642f087a7641289ca6c40d6e1a2637d8e"},
+    {file = "lxml-4.9.2-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:58bfa3aa19ca4c0f28c5dde0ff56c520fbac6f0daf4fac66ed4c8d2fb7f22e74"},
+    {file = "lxml-4.9.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:bc718cd47b765e790eecb74d044cc8d37d58562f6c314ee9484df26276d36a38"},
+    {file = "lxml-4.9.2-cp36-cp36m-win32.whl", hash = "sha256:d5bf6545cd27aaa8a13033ce56354ed9e25ab0e4ac3b5392b763d8d04b08e0c5"},
+    {file = "lxml-4.9.2-cp36-cp36m-win_amd64.whl", hash = "sha256:3ab9fa9d6dc2a7f29d7affdf3edebf6ece6fb28a6d80b14c3b2fb9d39b9322c3"},
+    {file = "lxml-4.9.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:05ca3f6abf5cf78fe053da9b1166e062ade3fa5d4f92b4ed688127ea7d7b1d03"},
+    {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:a5da296eb617d18e497bcf0a5c528f5d3b18dadb3619fbdadf4ed2356ef8d941"},
+    {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:04876580c050a8c5341d706dd464ff04fd597095cc8c023252566a8826505726"},
+    {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:c9ec3eaf616d67db0764b3bb983962b4f385a1f08304fd30c7283954e6a7869b"},
+    {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2a29ba94d065945944016b6b74e538bdb1751a1db6ffb80c9d3c2e40d6fa9894"},
+    {file = "lxml-4.9.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a82d05da00a58b8e4c0008edbc8a4b6ec5a4bc1e2ee0fb6ed157cf634ed7fa45"},
+    {file = "lxml-4.9.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:223f4232855ade399bd409331e6ca70fb5578efef22cf4069a6090acc0f53c0e"},
+    {file = "lxml-4.9.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d17bc7c2ccf49c478c5bdd447594e82692c74222698cfc9b5daae7ae7e90743b"},
+    {file = "lxml-4.9.2-cp37-cp37m-win32.whl", hash = "sha256:b64d891da92e232c36976c80ed7ebb383e3f148489796d8d31a5b6a677825efe"},
+    {file = "lxml-4.9.2-cp37-cp37m-win_amd64.whl", hash = "sha256:a0a336d6d3e8b234a3aae3c674873d8f0e720b76bc1d9416866c41cd9500ffb9"},
+    {file = "lxml-4.9.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:da4dd7c9c50c059aba52b3524f84d7de956f7fef88f0bafcf4ad7dde94a064e8"},
+    {file = "lxml-4.9.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:821b7f59b99551c69c85a6039c65b75f5683bdc63270fec660f75da67469ca24"},
+    {file = "lxml-4.9.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:e5168986b90a8d1f2f9dc1b841467c74221bd752537b99761a93d2d981e04889"},
+    {file = "lxml-4.9.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:8e20cb5a47247e383cf4ff523205060991021233ebd6f924bca927fcf25cf86f"},
+    {file = "lxml-4.9.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:13598ecfbd2e86ea7ae45ec28a2a54fb87ee9b9fdb0f6d343297d8e548392c03"},
+    {file = "lxml-4.9.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:880bbbcbe2fca64e2f4d8e04db47bcdf504936fa2b33933efd945e1b429bea8c"},
+    {file = "lxml-4.9.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:7d2278d59425777cfcb19735018d897ca8303abe67cc735f9f97177ceff8027f"},
+    {file = "lxml-4.9.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5344a43228767f53a9df6e5b253f8cdca7dfc7b7aeae52551958192f56d98457"},
+    {file = "lxml-4.9.2-cp38-cp38-win32.whl", hash = "sha256:925073b2fe14ab9b87e73f9a5fde6ce6392da430f3004d8b72cc86f746f5163b"},
+    {file = "lxml-4.9.2-cp38-cp38-win_amd64.whl", hash = "sha256:9b22c5c66f67ae00c0199f6055705bc3eb3fcb08d03d2ec4059a2b1b25ed48d7"},
+    {file = "lxml-4.9.2-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:5f50a1c177e2fa3ee0667a5ab79fdc6b23086bc8b589d90b93b4bd17eb0e64d1"},
+    {file = "lxml-4.9.2-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:090c6543d3696cbe15b4ac6e175e576bcc3f1ccfbba970061b7300b0c15a2140"},
+    {file = "lxml-4.9.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:63da2ccc0857c311d764e7d3d90f429c252e83b52d1f8f1d1fe55be26827d1f4"},
+    {file = "lxml-4.9.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:5b4545b8a40478183ac06c073e81a5ce4cf01bf1734962577cf2bb569a5b3bbf"},
+    {file = "lxml-4.9.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2e430cd2824f05f2d4f687701144556646bae8f249fd60aa1e4c768ba7018947"},
+    {file = "lxml-4.9.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6804daeb7ef69e7b36f76caddb85cccd63d0c56dedb47555d2fc969e2af6a1a5"},
+    {file = "lxml-4.9.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a6e441a86553c310258aca15d1c05903aaf4965b23f3bc2d55f200804e005ee5"},
+    {file = "lxml-4.9.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ca34efc80a29351897e18888c71c6aca4a359247c87e0b1c7ada14f0ab0c0fb2"},
+    {file = "lxml-4.9.2-cp39-cp39-win32.whl", hash = "sha256:6b418afe5df18233fc6b6093deb82a32895b6bb0b1155c2cdb05203f583053f1"},
+    {file = "lxml-4.9.2-cp39-cp39-win_amd64.whl", hash = "sha256:f1496ea22ca2c830cbcbd473de8f114a320da308438ae65abad6bab7867fe38f"},
+    {file = "lxml-4.9.2-pp37-pypy37_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:b264171e3143d842ded311b7dccd46ff9ef34247129ff5bf5066123c55c2431c"},
+    {file = "lxml-4.9.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:0dc313ef231edf866912e9d8f5a042ddab56c752619e92dfd3a2c277e6a7299a"},
+    {file = "lxml-4.9.2-pp38-pypy38_pp73-macosx_10_15_x86_64.whl", hash = "sha256:16efd54337136e8cd72fb9485c368d91d77a47ee2d42b057564aae201257d419"},
+    {file = "lxml-4.9.2-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:0f2b1e0d79180f344ff9f321327b005ca043a50ece8713de61d1cb383fb8ac05"},
+    {file = "lxml-4.9.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:7b770ed79542ed52c519119473898198761d78beb24b107acf3ad65deae61f1f"},
+    {file = "lxml-4.9.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:efa29c2fe6b4fdd32e8ef81c1528506895eca86e1d8c4657fda04c9b3786ddf9"},
+    {file = "lxml-4.9.2-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7e91ee82f4199af8c43d8158024cbdff3d931df350252288f0d4ce656df7f3b5"},
+    {file = "lxml-4.9.2-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_24_i686.whl", hash = "sha256:b23e19989c355ca854276178a0463951a653309fb8e57ce674497f2d9f208746"},
+    {file = "lxml-4.9.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_24_x86_64.whl", hash = "sha256:01d36c05f4afb8f7c20fd9ed5badca32a2029b93b1750f571ccc0b142531caf7"},
+    {file = "lxml-4.9.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7b515674acfdcadb0eb5d00d8a709868173acece5cb0be3dd165950cbfdf5409"},
+    {file = "lxml-4.9.2.tar.gz", hash = "sha256:2455cfaeb7ac70338b3257f41e21f0724f4b5b0c0e7702da67ee6c3640835b67"},
+]
+
+[package.extras]
+cssselect = ["cssselect (>=0.7)"]
+html5 = ["html5lib"]
+htmlsoup = ["BeautifulSoup4"]
+source = ["Cython (>=0.29.7)"]
+
 [[package]]
 name = "markupsafe"
 version = "2.1.2"
@@ -1214,4 +1295,4 @@ dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.10"
-content-hash = "9bb3d8c3451aa0fd56afac7c8e4100dc9734fb921cb5f35290e34a0f1d387dfd"
+content-hash = "500d8f046eeb2848745c27d0cb194d72a678b6951f0e908dc3e9eca66243954b"

pyproject.toml

@@ -17,7 +17,8 @@ python-multipart = "^0.0.5"
 python-dotenv = "^0.21.0"
 tomlkit = "^0.11.6"
 loguru = "^0.6.0"
-html2text = "^2020.1.16"
+beautifulsoup4 = "^4.11.1"
+lxml = "^4.9.2"
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.2.0"