from __future__ import annotations def sanitize_gfm_text(text: str) -> str: return text.replace("\x00", "") def chunk_gfm_markdown(text: str, limit: int = 65000) -> list[str]: if len(text) <= limit: return [text] chunks: list[str] = [] in_code_block = False current = "" for line in text.split("\n"): stripped = line.strip() if stripped.startswith("```"): in_code_block = not in_code_block candidate = current + line + "\n" if len(candidate) > limit and not in_code_block: chunks.append(current.rstrip()) current = line + "\n" else: current = candidate if current.strip(): chunks.append(current.rstrip()) return chunks if chunks else [text]