#!/usr/bin/env python3
"""
Tiny wrapper for Salesforce documentation extraction.

Behavior:
- If the URL is on help.salesforce.com, automatically route to the dedicated
  Help extractor with shadow DOM heuristics.
- Otherwise, use a lightweight browser-rendered extractor for official
  Salesforce-owned documentation sites such as developer.salesforce.com,
  architect.salesforce.com, admin.salesforce.com, lightningdesignsystem.com,
  and other supported official documentation hosts.

Examples:
  python3 skills/fetching-salesforce-docs/scripts/extract_salesforce_doc.py \
    --url "https://help.salesforce.com/s/articleView?id=service.miaw_security.htm&type=5" \
    --pretty

  python3 skills/fetching-salesforce-docs/scripts/extract_salesforce_doc.py \
    --url "https://developer.salesforce.com/docs/platform/lwc/guide/use-message-channel-intro.html" \
    --pretty

  python3 skills/fetching-salesforce-docs/scripts/extract_salesforce_doc.py \
    --url "https://architect.salesforce.com/well-architected/overview" \
    --stealth --pretty
"""

from __future__ import annotations

import argparse
import json
import re
from typing import Any, Dict
from urllib.parse import unquote, urlparse

from runtime_bootstrap import maybe_reexec_in_sf_docs_runtime

maybe_reexec_in_sf_docs_runtime(__file__)

from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
from playwright.sync_api import sync_playwright

try:
    from playwright_stealth import Stealth
except ImportError:
    Stealth = None

try:
    from playwright_stealth import stealth_sync
except ImportError:
    stealth_sync = None

from extract_help_salesforce import extract as extract_help_salesforce


USER_AGENT = (
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
    "AppleWebKit/537.36 (KHTML, like Gecko) "
    "Chrome/122.0.0.0 Safari/537.36"
)

STRONG_SHELL_TOKENS = [
    "loading",
    "sorry to interrupt",
    "css error",
    "enable javascript",
    "we looked high and low",
    "couldn't find that page",
    "404 error",
]

WEAK_SHELL_TOKENS = [
    "sign in",
    "cookie preferences",
]

OFFICIAL_DOC_EXACT_HOSTS = {
    "salesforce.com",
    "lightningdesignsystem.com",
}

OFFICIAL_DOC_SUFFIXES = (
    ".salesforce.com",
    ".lightningdesignsystem.com",
)

DEVELOPER_DOCUMENT_ENDPOINT = "/docs/get_document/"
DEVELOPER_DOCUMENT_CONTENT_ENDPOINT = "/docs/get_document_content/"


def normalize_text(text: str) -> str:
    text = text.replace("\u00a0", " ").replace("\r", "")
    text = re.sub(r"\n{3,}", "\n\n", text)
    text = re.sub(r"[ \t]+", " ", text)
    return text.strip()


def normalize_document_id(document_id: str | None) -> str | None:
    if not document_id:
        return None
    return re.sub(r"\.html?$", "", unquote(document_id), flags=re.IGNORECASE).lower()


def requested_document_id(page_url: str) -> str | None:
    path_parts = [part for part in urlparse(page_url).path.split("/") if part]
    if (
        len(path_parts) >= 4
        and path_parts[0] == "docs"
        and path_parts[1].startswith("atlas.")
    ):
        return normalize_document_id(path_parts[3])
    return None


def response_document_id(response_url: str, payload: Dict[str, Any]) -> str | None:
    payload_id = normalize_document_id(payload.get("content_document_id"))
    if payload_id:
        return payload_id

    path_parts = [part for part in urlparse(response_url).path.split("/") if part]
    try:
        endpoint_index = path_parts.index("get_document_content")
    except ValueError:
        return None

    content_id_index = endpoint_index + 2
    if content_id_index >= len(path_parts):
        return None
    return normalize_document_id(path_parts[content_id_index])


def extract_developer_document_payload(page, responses, page_url: str) -> Dict[str, Any] | None:
    """Extract an Atlas article from the JSON payload used by Salesforce's doc component."""
    parsed_page_url = urlparse(page_url)
    link_base_url = f"{parsed_page_url.scheme}://{parsed_page_url.netloc}/docs/"
    target_document_id = requested_document_id(page_url)

    for response in reversed(responses):
        if response.status != 200:
            continue

        try:
            payload = response.json()
        except Exception:
            continue

        if not isinstance(payload, dict):
            continue

        payload_document_id = response_document_id(response.url, payload)
        if target_document_id and payload_document_id != target_document_id:
            continue

        content = payload.get("content")
        if not isinstance(content, str) or not content.strip():
            continue

        try:
            parsed = page.evaluate(
                r"""
                ({ html, baseUrl }) => {
                  const documentFragment = new DOMParser().parseFromString(html, 'text/html');
                  const skippedTags = new Set(['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEMPLATE', 'SVG']);
                  const blockTags = new Set([
                    'ADDRESS', 'ARTICLE', 'ASIDE', 'BLOCKQUOTE', 'DIV', 'DL', 'DT', 'DD',
                    'FIELDSET', 'FIGCAPTION', 'FIGURE', 'FOOTER', 'FORM', 'H1', 'H2', 'H3',
                    'H4', 'H5', 'H6', 'HEADER', 'HR', 'LI', 'MAIN', 'NAV', 'OL', 'P', 'PRE',
                    'SECTION', 'TABLE', 'TBODY', 'THEAD', 'TFOOT', 'TR', 'UL'
                  ]);
                  const chunks = [];

                  function walk(node) {
                    if (node.nodeType === 3) {
                      chunks.push(node.nodeValue || '');
                      return;
                    }
                    if (node.nodeType !== 1 || skippedTags.has(node.tagName)) return;

                    if (node.tagName === 'BR') {
                      chunks.push('\n');
                      return;
                    }

                    const isBlock = blockTags.has(node.tagName);
                    if (isBlock) chunks.push('\n');
                    for (const child of node.childNodes) walk(child);
                    if (node.tagName === 'TD' || node.tagName === 'TH') chunks.push('\t');
                    if (isBlock) chunks.push('\n');
                  }

                  walk(documentFragment.body);

                  const links = [];
                  const seen = new Set();
                  for (const anchor of documentFragment.querySelectorAll('a[href]')) {
                    const href = anchor.getAttribute('href') || '';
                    if (!href || href.startsWith('javascript:') || href.startsWith('mailto:')) continue;
                    try {
                      const absoluteUrl = new URL(href, baseUrl).href;
                      if (!seen.has(absoluteUrl)) {
                        seen.add(absoluteUrl);
                        links.push(absoluteUrl);
                      }
                    } catch (error) {
                      // Ignore malformed links while preserving the article text.
                    }
                  }

                  return { text: chunks.join(''), links };
                }
                """,
                {"html": content, "baseUrl": link_base_url},
            )
        except Exception:
            continue

        text = normalize_text(re.sub(r"[ \t]*\n[ \t]*", "\n", parsed.get("text", "")))
        if not text:
            continue

        return {
            "title": payload.get("title") or payload.get("doc_title"),
            "text": text,
            "links": parsed.get("links", []),
            "responseUrl": response.url,
            "responsePath": (
                DEVELOPER_DOCUMENT_CONTENT_ENDPOINT
                if DEVELOPER_DOCUMENT_CONTENT_ENDPOINT in response.url
                else DEVELOPER_DOCUMENT_ENDPOINT
            ),
        }

    return None


def looks_like_shell(title: str, text: str) -> bool:
    haystack = f"{title}\n{text}".lower()
    if any(token in haystack for token in STRONG_SHELL_TOKENS):
        return True
    if any(token in haystack for token in WEAK_SHELL_TOKENS):
        return len(text.strip()) < 600
    return False


def apply_stealth(page) -> bool:
    if stealth_sync is not None:
        try:
            stealth_sync(page)
            return True
        except Exception:
            pass
    if Stealth is not None:
        try:
            Stealth().apply_stealth_sync(page)
            return True
        except Exception:
            return False
    return False


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="Extract official Salesforce documentation from a URL")
    parser.add_argument("--url", required=True, help="Official Salesforce doc URL")
    parser.add_argument("--timeout", type=int, default=60, help="Timeout in seconds (default: 60)")
    parser.add_argument("--stealth", action="store_true", help="Best-effort stealth mode for bot-sensitive pages")
    parser.add_argument("--pretty", action="store_true", help="Pretty-print JSON")
    return parser.parse_args()


def is_official_salesforce_host(host: str) -> bool:
    host = (host or "").lower()
    return host in OFFICIAL_DOC_EXACT_HOSTS or any(host.endswith(suffix) for suffix in OFFICIAL_DOC_SUFFIXES)


def route_kind(url: str) -> str:
    host = (urlparse(url).hostname or "").lower()
    if host.endswith("help.salesforce.com"):
        return "help"
    if is_official_salesforce_host(host):
        return "official"
    raise SystemExit(f"Unsupported host for fetching-salesforce-docs extractor: {host or url}")


def extract_official_salesforce_doc(url: str, timeout_seconds: int, use_stealth: bool = False) -> Dict[str, Any]:
    timeout_ms = timeout_seconds * 1000
    host = (urlparse(url).hostname or "").lower()

    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True)
        page = browser.new_page(user_agent=USER_AGENT, viewport={"width": 1440, "height": 1400})
        stealth_used = apply_stealth(page) if use_stealth else False
        developer_document_responses = []

        def capture_developer_document(response) -> None:
            is_document_response = (
                DEVELOPER_DOCUMENT_ENDPOINT in response.url
                or DEVELOPER_DOCUMENT_CONTENT_ENDPOINT in response.url
            )
            if host == "developer.salesforce.com" and is_document_response:
                developer_document_responses.append(response)

        page.on("response", capture_developer_document)

        try:
            response = page.goto(url, wait_until="domcontentloaded", timeout=timeout_ms)
            http_status = response.status if response else None
            page.wait_for_timeout(1500)
            try:
                page.wait_for_load_state("networkidle", timeout=min(timeout_ms, 15000))
            except PlaywrightTimeoutError:
                pass

            # Wait for content: check both light DOM and shadow DOM content
            is_developer = "developer.salesforce.com" in host
            wait_js = r"""
                    () => {
                      // Light DOM check (works for most official sites)
                      const el = document.querySelector('main, article, [role="main"]');
                      const lightText = (el?.innerText || el?.textContent || '').trim();
                      if (lightText.length > 200) return true;

                      // Shadow DOM check (needed for developer.salesforce.com)
                      function findDeepText(root, depth) {
                        if (depth > 5 || !root) return '';
                        let best = '';
                        const els = root.querySelectorAll ? root.querySelectorAll('*') : [];
                        for (const child of els) {
                          if (child.shadowRoot) {
                            // Check direct children of shadow root for text
                            for (const sc of child.shadowRoot.children) {
                              const t = (sc.innerText || '').trim();
                              if (t.length > best.length) best = t;
                            }
                            const deeper = findDeepText(child.shadowRoot, depth + 1);
                            if (deeper.length > best.length) best = deeper;
                          }
                        }
                        return best;
                      }
                      const shadowText = findDeepText(document, 0);
                      return shadowText.length > 200;
                    }
                    """
            try:
                page.wait_for_function(
                    wait_js,
                    timeout=min(timeout_ms, 20000 if is_developer else 15000),
                )
            except PlaywrightTimeoutError:
                pass
            page.wait_for_timeout(500)

            payload = page.evaluate(
                r"""
                (host) => {
                  function normalize(text) {
                    return String(text || '')
                      .replace(/\u00a0/g, ' ')
                      .replace(/\r/g, '')
                      .replace(/\n{3,}/g, '\n\n')
                      .trim();
                  }

                  function isVisible(el) {
                    if (!el || !el.getBoundingClientRect) return false;
                    const rect = el.getBoundingClientRect();
                    const style = window.getComputedStyle(el);
                    return rect.width > 0 && rect.height > 0 && style.visibility !== 'hidden' && style.display !== 'none';
                  }

                  function allRoots() {
                    const roots = [document];
                    const queue = [document];
                    while (queue.length) {
                      const current = queue.shift();
                      if (!current || !current.querySelectorAll) continue;
                      const elements = current.querySelectorAll('*');
                      for (const el of elements) {
                        if (el.shadowRoot) {
                          roots.push(el.shadowRoot);
                          queue.push(el.shadowRoot);
                        }
                      }
                    }
                    return roots;
                  }

                  function deepQueryAll(selector) {
                    const results = [];
                    const seen = new Set();
                    for (const root of allRoots()) {
                      if (!root.querySelectorAll) continue;
                      for (const el of root.querySelectorAll(selector)) {
                        if (!seen.has(el)) {
                          seen.add(el);
                          results.push(el);
                        }
                      }
                    }
                    return results;
                  }

                  function collectLinks(scope) {
                    const urls = new Set();
                    const nodes = scope && scope.querySelectorAll ? scope.querySelectorAll('a[href]') : [];
                    for (const a of nodes) {
                      const href = a.href || a.getAttribute('href') || '';
                      if (!href) continue;
                      if (href.startsWith('javascript:') || href.startsWith('mailto:')) continue;
                      urls.add(href);
                    }
                    return Array.from(urls);
                  }

                  // --- Noise removal: strip cookie/consent overlays before scoring ---
                  document.querySelectorAll(
                    '#onetrust-banner-sdk, #onetrust-consent-sdk, .onetrust-consent-sdk, ' +
                    '[class*="onetrust-pc"], #onetrust-pc-sdk, ' +
                    '[id*="cookie-consent"], [class*="cookie-banner"], ' +
                    '.evidon-consent-button, #evidon-banner'
                  ).forEach(function(el) { el.remove(); });

                  const title = document.title || normalize(document.querySelector('title')?.innerText || 'Untitled');
                  const childLinks = new Set();
                  for (const root of allRoots()) {
                    for (const link of collectLinks(root)) childLinks.add(link);
                  }

                  const selectorConfigs = [
                    { selector: 'article', strategy: 'article', base: 260 },
                    { selector: 'main', strategy: 'main', base: 220 },
                    { selector: '[role="main"]', strategy: 'role-main', base: 220 },
                    { selector: '.slds-text-longform', strategy: 'longform', base: 200 },
                    { selector: '.markdown-content', strategy: 'markdown-content', base: 190 },
                    { selector: '.content-body', strategy: 'content-body', base: 180 },
                    { selector: '.article-body', strategy: 'article-body', base: 180 },
                    { selector: '.article-content', strategy: 'article-content', base: 180 },
                    { selector: '.post-content', strategy: 'post-content', base: 170 },
                    { selector: '.main-content', strategy: 'main-content', base: 170 },
                    { selector: '.tds-content', strategy: 'tds-content', base: 165 },
                    { selector: '.siteforceContentArea .content', strategy: 'siteforce-content', base: 160 },
                    { selector: 'doc-content-layout', strategy: 'legacy-doc-layout', base: 150 },
                    { selector: 'doc-xml-content', strategy: 'legacy-doc-xml', base: 145 },
                    { selector: 'doc-amf-reference .markdown-content', strategy: 'legacy-amf-markdown', base: 150 },
                    { selector: 'main .content, article .content', strategy: 'nested-content', base: 140 },
                  ];

                  // --- Host-aware selector overrides for developer.salesforce.com ---
                  if (host.includes('developer.salesforce.com')) {
                    selectorConfigs.unshift(
                      { selector: 'doc-content', strategy: 'dev-doc-content', base: 295 },
                      { selector: '#maincontent', strategy: 'dev-maincontent', base: 290 },
                      { selector: '.doc-body', strategy: 'dev-doc-body', base: 285 },
                      { selector: '#topic-content', strategy: 'dev-topic-content', base: 285 },
                      { selector: '.topicContent', strategy: 'dev-topic', base: 280 },
                      { selector: '[class*="docContent"]', strategy: 'dev-doc-content-class', base: 275 },
                      { selector: 'doc-xml-content .content', strategy: 'dev-xml-content', base: 270 },
                    );
                  }

                  // Extract text from an element, including shadow root content
                  function extractText(el) {
                    let text = (el.innerText || el.textContent || '').trim();
                    // If element is a shadow host with no slotted light DOM text,
                    // gather text from its shadow root's children
                    if (text.length < 200 && el.shadowRoot) {
                      let shadowText = '';
                      for (const child of el.shadowRoot.children) {
                        shadowText += (child.innerText || child.textContent || '') + '\n';
                      }
                      if (shadowText.trim().length > text.length) {
                        text = shadowText.trim();
                      }
                    }
                    return text;
                  }

                  const candidates = [];
                  for (const cfg of selectorConfigs) {
                    const nodes = deepQueryAll(cfg.selector);
                    for (const node of nodes) {
                      if (!isVisible(node)) continue;
                      const text = normalize(extractText(node));
                      if (text.length < 200) continue;
                      let score = cfg.base + Math.min(text.length, 5000) / 30;
                      const lowered = text.toLowerCase();
                      if (lowered.includes(title.toLowerCase())) score += 50;
                      if (lowered.includes('table of contents')) score -= 80;
                      if (lowered.includes('cookie preferences')) score -= 120;
                      if (lowered.includes('sign in')) score -= 120;
                      candidates.push({
                        strategy: cfg.strategy,
                        selector: cfg.selector,
                        score,
                        text,
                        links: collectLinks(node).slice(0, 200),
                      });
                    }
                  }

                  const bodyText = normalize(document.body?.innerText || '');
                  if (bodyText.length >= 200) {
                    candidates.push({
                      strategy: 'body',
                      selector: 'body',
                      score: Math.min(bodyText.length, 5000) / 50,
                      text: bodyText,
                      links: Array.from(childLinks).slice(0, 200),
                    });
                  }

                  candidates.sort((a, b) => b.score - a.score);
                  const best = candidates[0] || null;

                  return {
                    url: window.location.href,
                    title,
                    strategy: best ? best.strategy : 'none',
                    selector: best ? best.selector : null,
                    text: best ? best.text : '',
                    contentLinks: best ? best.links : [],
                    childLinks: Array.from(childLinks).slice(0, 200),
                    candidateCount: candidates.length,
                  };
                }
                """,
                host,
            )

            developer_document = extract_developer_document_payload(
                page,
                developer_document_responses,
                payload.get("url", url),
            )
            if developer_document:
                payload["strategy"] = "developer-doc-api"
                payload["selector"] = developer_document["responsePath"]
                payload["text"] = developer_document["text"]
                payload["contentLinks"] = developer_document["links"]
                payload["childLinks"] = developer_document["links"]
                payload["candidateCount"] = payload.get("candidateCount", 0) + 1
                if not payload.get("title") or payload.get("title") == "Untitled":
                    payload["title"] = developer_document["title"] or "Untitled"

            text = normalize_text(payload.get("text", ""))
            likely_shell = looks_like_shell(payload.get("title", ""), text)
            ok = bool(text) and len(text) >= 300 and not likely_shell

            return {
                "ok": ok,
                "url": payload.get("url", url),
                "httpStatus": http_status,
                "title": payload.get("title") or "Untitled",
                "host": host,
                "hostKind": "official-salesforce",
                "strategy": payload.get("strategy"),
                "selector": payload.get("selector"),
                "likelyShell": likely_shell,
                "stealthRequested": use_stealth,
                "stealthAvailable": stealth_sync is not None or Stealth is not None,
                "stealthUsed": stealth_used,
                "text": text,
                "contentLinks": payload.get("contentLinks", []),
                "childLinks": payload.get("childLinks", []),
                "candidateCount": payload.get("candidateCount", 0),
            }
        finally:
            page.close()
            browser.close()


def main() -> int:
    args = parse_args()
    kind = route_kind(args.url)

    if kind == "help":
        result = extract_help_salesforce(args.url, args.timeout, use_stealth=args.stealth)
        result["routedVia"] = "extract_help_salesforce"
        result.setdefault("hostKind", "help")
    else:
        result = extract_official_salesforce_doc(args.url, args.timeout, use_stealth=args.stealth)
        result["routedVia"] = "generic_official_salesforce_extractor"

    dump = json.dumps(result, indent=2 if args.pretty else None)
    print(dump)
    return 0 if result.get("ok") else 1


if __name__ == "__main__":
    raise SystemExit(main())