from __future__ import annotations import logging import re from html import unescape logger = logging.getLogger(__name__) try: from bs4 import BeautifulSoup HAS_BS4 = True except ImportError: HAS_BS4 = False logger.warning("beautifulsoup4 not installed — Help Scout HTML parsing will use fallback regex") _EMAIL_QUOTE_PATTERNS = [ re.compile(r"^On\s+.*wrote:\s*$", re.IGNORECASE | re.MULTILINE), re.compile(r"^>{1,}\s?", re.MULTILINE), re.compile(r"^-{2,}\s*Original Message\s*-{2,}", re.IGNORECASE), re.compile(r"^From:\s+.*$", re.IGNORECASE | re.MULTILINE), re.compile(r"^Sent:\s+.*$", re.IGNORECASE | re.MULTILINE), re.compile(r"^To:\s+.*$", re.IGNORECASE | re.MULTILINE), re.compile(r"^Subject:\s+.*$", re.IGNORECASE | re.MULTILINE), re.compile(r"^Date:\s+.*$", re.IGNORECASE | re.MULTILINE), ] def html_to_text(html_body: str) -> str: if not html_body: return "" if HAS_BS4: return _bs4_html_to_text(html_body) return _regex_html_to_text(html_body) def _bs4_html_to_text(html_body: str) -> str: soup = BeautifulSoup(html_body, "html.parser") for tag in soup(["style", "script", "head"]): tag.decompose() for a_tag in soup.find_all("a", href=True): text = a_tag.get_text(strip=True) url = a_tag["href"] if text and url: a_tag.replace_with(f"[{text}]({url})") for br in soup.find_all("br"): br.replace_with("\n") for p in soup.find_all("p"): p.insert_before("\n") p.insert_after("\n") p.unwrap() for tag in soup.find_all(["strong", "b"]): if tag.string: tag.string.replace_with(f"**{tag.string}**") tag.unwrap() for tag in soup.find_all(["em", "i"]): if tag.string: tag.string.replace_with(f"*{tag.string}*") tag.unwrap() for li in soup.find_all("li"): li.insert_before("- ") li.insert_after("\n") li.unwrap() text = soup.get_text(separator="\n") text = unescape(text) text = strip_email_quote(text) text = re.sub(r"\n{3,}", "\n\n", text) return text.strip() def _regex_html_to_text(html_body: str) -> str: import html text = html_body for tag in ("style", "script", "head"): text = re.sub( f"<{tag}[^>]*>.*?", "", text, flags=re.DOTALL | re.IGNORECASE, ) text = re.sub( r']*href=["\']([^"\']*)["\'][^>]*>(.*?)', r"[\2](\1)", text, flags=re.DOTALL, ) text = re.sub(r"", "\n", text, flags=re.IGNORECASE) text = re.sub(r"]*>", "\n", text, flags=re.IGNORECASE) text = re.sub(r"

", "\n", text, flags=re.IGNORECASE) text = re.sub( r"<(?:strong|b)[^>]*>(.*?)", r"**\1**", text, flags=re.DOTALL, ) text = re.sub(r"<(?:em|i)[^>]*>(.*?)", r"*\1*", text, flags=re.DOTALL) text = re.sub(r"]*>(.*?)", r"- \1\n", text, flags=re.DOTALL) text = re.sub(r"<[^>]+>", "", text) text = html.unescape(text) text = strip_email_quote(text) text = re.sub(r"\n{3,}", "\n\n", text) text = re.sub(r"[ \t]{2,}", " ", text) return text.strip() def strip_email_quote(text: str) -> str: lines = text.split("\n") cleaned_lines = [] for line in lines: is_quote = False for pattern in _EMAIL_QUOTE_PATTERNS: if pattern.match(line.strip()): is_quote = True break if not is_quote: cleaned_lines.append(line) else: break return "\n".join(cleaned_lines) def extract_plain_text(text: str, max_length: int = 500) -> str: plain = re.sub(r"[#*_>\-\[\]\(\)]", "", text) plain = re.sub(r"\s+", " ", plain).strip() if len(plain) > max_length: plain = plain[: max_length - 3] + "..." return plain