from __future__ import annotations
import logging
import re
from html import unescape
logger = logging.getLogger(__name__)
try:
from bs4 import BeautifulSoup
HAS_BS4 = True
except ImportError:
HAS_BS4 = False
logger.warning("beautifulsoup4 not installed — Help Scout HTML parsing will use fallback regex")
_EMAIL_QUOTE_PATTERNS = [
re.compile(r"^On\s+.*wrote:\s*$", re.IGNORECASE | re.MULTILINE),
re.compile(r"^>{1,}\s?", re.MULTILINE),
re.compile(r"^-{2,}\s*Original Message\s*-{2,}", re.IGNORECASE),
re.compile(r"^From:\s+.*$", re.IGNORECASE | re.MULTILINE),
re.compile(r"^Sent:\s+.*$", re.IGNORECASE | re.MULTILINE),
re.compile(r"^To:\s+.*$", re.IGNORECASE | re.MULTILINE),
re.compile(r"^Subject:\s+.*$", re.IGNORECASE | re.MULTILINE),
re.compile(r"^Date:\s+.*$", re.IGNORECASE | re.MULTILINE),
]
def html_to_text(html_body: str) -> str:
if not html_body:
return ""
if HAS_BS4:
return _bs4_html_to_text(html_body)
return _regex_html_to_text(html_body)
def _bs4_html_to_text(html_body: str) -> str:
soup = BeautifulSoup(html_body, "html.parser")
for tag in soup(["style", "script", "head"]):
tag.decompose()
for a_tag in soup.find_all("a", href=True):
text = a_tag.get_text(strip=True)
url = a_tag["href"]
if text and url:
a_tag.replace_with(f"[{text}]({url})")
for br in soup.find_all("br"):
br.replace_with("\n")
for p in soup.find_all("p"):
p.insert_before("\n")
p.insert_after("\n")
p.unwrap()
for tag in soup.find_all(["strong", "b"]):
if tag.string:
tag.string.replace_with(f"**{tag.string}**")
tag.unwrap()
for tag in soup.find_all(["em", "i"]):
if tag.string:
tag.string.replace_with(f"*{tag.string}*")
tag.unwrap()
for li in soup.find_all("li"):
li.insert_before("- ")
li.insert_after("\n")
li.unwrap()
text = soup.get_text(separator="\n")
text = unescape(text)
text = strip_email_quote(text)
text = re.sub(r"\n{3,}", "\n\n", text)
return text.strip()
def _regex_html_to_text(html_body: str) -> str:
import html
text = html_body
for tag in ("style", "script", "head"):
text = re.sub(
f"<{tag}[^>]*>.*?{tag}>",
"",
text,
flags=re.DOTALL | re.IGNORECASE,
)
text = re.sub(
r']*href=["\']([^"\']*)["\'][^>]*>(.*?)',
r"[\2](\1)",
text,
flags=re.DOTALL,
)
text = re.sub(r"
", "\n", text, flags=re.IGNORECASE)
text = re.sub(r"
]*>", "\n", text, flags=re.IGNORECASE) text = re.sub(r"
", "\n", text, flags=re.IGNORECASE) text = re.sub( r"<(?:strong|b)[^>]*>(.*?)(?:strong|b)>", r"**\1**", text, flags=re.DOTALL, ) text = re.sub(r"<(?:em|i)[^>]*>(.*?)(?:em|i)>", r"*\1*", text, flags=re.DOTALL) text = re.sub(r"