from __future__ import annotations

import csv
import base64
import hashlib
import html as html_lib
import json
import re
import ssl
import time
import urllib.parse
import urllib.request
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path

OUT = Path(__file__).resolve().parent
UA = "Mozilla/5.0 (compatible; TankDevResearch/1.0; +https://tankdev.tech/)"
SSL = ssl.create_default_context()

MARKETS = [
    ("United States", "Austin"), ("Canada", "Toronto"), ("United Kingdom", "Manchester"),
    ("Ireland", "Dublin"), ("Germany", "Berlin"), ("France", "Lyon"),
    ("Spain", "Barcelona"), ("Italy", "Milan"), ("Netherlands", "Amsterdam"),
    ("Sweden", "Stockholm"), ("Norway", "Oslo"), ("Poland", "Warsaw"),
    ("Brazil", "Sao Paulo"), ("Mexico", "Monterrey"), ("South Africa", "Cape Town"),
    ("United Arab Emirates", "Dubai"), ("India", "Bangalore"), ("Singapore", "Singapore"),
    ("Australia", "Melbourne"), ("New Zealand", "Auckland"),
]

SECTORS = {
    "dental": {
        "query": "dentist dental clinic {city} {country} official website",
        "terms": ("dentist", "dental", "orthodont", "implant"),
    },
    "legal": {
        "query": "independent law firm lawyers {city} {country} official website",
        "terms": ("law firm", "lawyer", "attorney", "legal", "solicitor", "advocat"),
    },
    "hotel": {
        "query": "independent boutique hotel {city} {country} official website",
        "terms": ("hotel", "rooms", "accommodation", "stay", "suite"),
    },
    "construction": {
        "query": "construction company contractor {city} {country} official website",
        "terms": ("construction", "contractor", "building", "builder", "engineering"),
    },
    "manufacturing": {
        "query": "industrial manufacturer manufacturing company {city} {country} official website",
        "terms": ("manufactur", "factory", "industrial", "products", "production"),
    },
}

EXCLUDED = (
    "duckduckgo.com", "google.", "bing.com", "yahoo.", "facebook.com", "instagram.com",
    "linkedin.com", "youtube.com", "wikipedia.org", "yelp.", "tripadvisor.", "booking.com",
    "expedia.", "clutch.co", "sortlist.", "justia.com", "lawyers.com", "avvo.com",
    "healthgrades.com", "zocdoc.com", "yellowpages", "houzz.", "mapquest.", "reddit.com",
    "indeed.com", "glassdoor.", "pinterest.", "foursquare.", "chamberofcommerce",
)


def fetch(url: str, timeout: int = 18, max_bytes: int = 2_500_000) -> tuple[int, str, str, dict[str, str]]:
    req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "text/html,application/xhtml+xml,*/*;q=0.8"})
    try:
        with urllib.request.urlopen(req, timeout=timeout, context=SSL) as response:
            raw = response.read(max_bytes)
            encoding = response.headers.get_content_charset() or "utf-8"
            return response.status, response.geturl(), raw.decode(encoding, "replace"), dict(response.headers)
    except Exception as exc:
        return 0, url, f"{type(exc).__name__}: {exc}", {}


def result_urls(query: str) -> list[str]:
    url = "https://www.bing.com/search?" + urllib.parse.urlencode({"q": query, "setlang": "en-US", "mkt": "en-US", "cc": "US", "count": "10"})
    status, _, body, _ = fetch(url)
    if status != 200:
        return []
    links = re.findall(r'<li class="b_algo"[\s\S]*?<h2[^>]*>\s*<a[^>]+href="([^"]+)"', body, re.I)
    output = []
    for link in links:
        link = html_lib.unescape(link)
        parsed = urllib.parse.urlparse(link)
        if parsed.netloc.endswith("bing.com") and parsed.path.startswith("/ck/"):
            encoded = urllib.parse.parse_qs(parsed.query).get("u", [""])[0]
            if encoded.startswith("a1"):
                try:
                    payload = encoded[2:] + "=" * (-len(encoded[2:]) % 4)
                    link = base64.urlsafe_b64decode(payload).decode("utf-8", "replace")
                except Exception:
                    continue
        if link.startswith("http"):
            output.append(link)
    return output


def domain(url: str) -> str:
    return urllib.parse.urlparse(url).netloc.lower().removeprefix("www.").split(":")[0]


def visible_text(body: str) -> str:
    body = re.sub(r"<(script|style|svg|noscript)\b[^>]*>.*?</\1>", " ", body, flags=re.I | re.S)
    return re.sub(r"\s+", " ", html_lib.unescape(re.sub(r"<[^>]+>", " ", body))).strip()


def choose_site(sector: str, country: str, city: str, used: set[str]) -> dict[str, object] | None:
    spec = SECTORS[sector]
    query = spec["query"].format(city=city, country=country)
    for rank, candidate in enumerate(result_urls(query)[:12], 1):
        host = domain(candidate)
        if not host or host in used or any(blocked in host for blocked in EXCLUDED):
            continue
        if host.endswith((".gov", ".edu", ".ac.uk")):
            continue
        status, final_url, body, _ = fetch(candidate)
        final_host = domain(final_url)
        if status != 200 or not final_host or any(blocked in final_host for blocked in EXCLUDED):
            continue
        text = visible_text(body[:1_200_000]).lower()
        if not any(term in text for term in spec["terms"]):
            continue
        used.add(final_host)
        return {
            "sector": sector,
            "country": country,
            "city": city,
            "query": query,
            "search_rank": rank,
            "url": f"{urllib.parse.urlparse(final_url).scheme}://{urllib.parse.urlparse(final_url).netloc}/",
            "domain": final_host,
        }
    return None


def has(pattern: str, body: str) -> bool:
    return bool(re.search(pattern, body, re.I | re.S))


def explicitly_blocks_ai_crawler(robots: str) -> bool:
    agents = {"gptbot", "oai-searchbot", "chatgpt-user", "claudebot", "perplexitybot"}
    current_agents: list[str] = []
    groups: list[tuple[list[str], list[str]]] = []
    rules: list[str] = []
    for raw_line in robots.splitlines() + ["user-agent: __end__"]:
        line = raw_line.split("#", 1)[0].strip()
        if not line or ":" not in line:
            continue
        field, value = (part.strip().lower() for part in line.split(":", 1))
        if field == "user-agent":
            if rules:
                groups.append((current_agents, rules))
                current_agents, rules = [], []
            current_agents.append(value)
        elif field in {"allow", "disallow"} and current_agents:
            rules.append(f"{field}:{value}")
    for group_agents, group_rules in groups:
        if agents.intersection(group_agents) and "disallow:/" in group_rules and "allow:/" not in group_rules:
            return True
    return False


def audit_site(sample: dict[str, object]) -> dict[str, object]:
    start = time.perf_counter()
    url = str(sample["url"])
    attempted_urls = [url]
    status, final_url, body, headers = fetch(url, timeout=25)
    if status == 0:
        attempted = urllib.parse.urlsplit(url)
        alternate_host = attempted.netloc[4:] if attempted.netloc.startswith("www.") else f"www.{attempted.netloc}"
        alternate = urllib.parse.urlunsplit((attempted.scheme, alternate_host, attempted.path or "/", attempted.query, ""))
        attempted_urls.append(alternate)
        alt_status, alt_final, alt_body, alt_headers = fetch(alternate, timeout=25)
        if alt_status:
            status, final_url, body, headers = alt_status, alt_final, alt_body, alt_headers
    elapsed_ms = round((time.perf_counter() - start) * 1000)
    body_lower = body.lower()
    text = visible_text(body)
    parsed = urllib.parse.urlparse(final_url)
    origin = f"{parsed.scheme}://{parsed.netloc}"
    robots_status, _, robots, _ = fetch(origin + "/robots.txt", timeout=12, max_bytes=300_000)
    sitemap_status, _, sitemap, _ = fetch(origin + "/sitemap.xml", timeout=12, max_bytes=500_000)
    llms_status, _, llms, _ = fetch(origin + "/llms.txt", timeout=12, max_bytes=300_000)

    title = re.search(r"<title[^>]*>(.*?)</title>", body, re.I | re.S)
    title_text = visible_text(title.group(1)) if title else ""
    description = has(r"<meta[^>]+name=['\"]description['\"][^>]+content=['\"][^'\"]{40,}", body) or has(
        r"<meta[^>]+content=['\"][^'\"]{40,}['\"][^>]+name=['\"]description", body
    )
    canonical = has(r"<link[^>]+rel=['\"][^'\"]*canonical", body)
    lang = has(r"<html[^>]+lang=['\"][^'\"]+", body)
    hreflang = has(r"hreflang=['\"]", body)
    jsonld = has(r"<script[^>]+type=['\"]application/ld\+json", body)
    entity_schema = has(r'"@type"\s*:\s*(?:\[[^\]]*)?["\'](?:Organization|LocalBusiness|Dentist|LegalService|Hotel|LodgingBusiness|ConstructionBusiness|Corporation|ProfessionalService)', body)
    entity_details = sum(bool(has(pattern, body)) for pattern in (r'"name"\s*:', r'"url"\s*:', r'"address"\s*:', r'"telephone"\s*:', r'"sameAs"\s*:'))
    og = has(r"<meta[^>]+property=['\"]og:", body)
    h1 = has(r"<h1\b", body)
    headings = len(re.findall(r"<h[1-3]\b", body, re.I))
    contact = has(r"(?:href=['\"](?:tel:|mailto:)|\+?\d[\d\s().-]{7,}\d)", body)
    about = has(r"href=['\"][^'\"]*(?:about|company|who-we-are|our-story)", body)
    privacy = has(r"href=['\"][^'\"]*(?:privacy|data-protection|datenschutz|privacidad)", body)
    services = has(r"href=['\"][^'\"]*(?:services|practice|treatments|capabilities|products|solutions|projects|rooms)", body)
    faq = has(r"(?:FAQPage|Frequently Asked|FAQs|Common Questions)", body)
    noindex = has(r"<meta[^>]+(?:name=['\"]robots['\"][^>]+content=['\"][^'\"]*noindex|content=['\"][^'\"]*noindex[^>]+name=['\"]robots)", body)
    ai_block = False
    if robots_status == 200:
        ai_block = explicitly_blocks_ai_crawler(robots)
    alt_total = len(re.findall(r"<img\b", body, re.I))
    alt_present = len(re.findall(r"<img\b[^>]+alt=['\"][^'\"]+", body, re.I))
    alt_ratio = round(alt_present / alt_total, 3) if alt_total else None

    checks = {
        "reachable": status == 200,
        "https": parsed.scheme == "https",
        "indexable": status == 200 and not noindex,
        "title": 15 <= len(title_text) <= 70,
        "meta_description": description,
        "canonical": canonical,
        "html_lang": lang,
        "structured_data": jsonld,
        "entity_schema": entity_schema,
        "entity_details_3plus": entity_details >= 3,
        "open_graph": og,
        "single_h1_present": h1,
        "heading_structure": headings >= 3,
        "contact_signal": contact,
        "about_link": about,
        "privacy_link": privacy,
        "service_detail_link": services,
        "faq_signal": faq,
        "robots_available": robots_status == 200,
        "sitemap_available": sitemap_status == 200 and ("<urlset" in sitemap.lower() or "<sitemapindex" in sitemap.lower()),
        "ai_crawlers_not_explicitly_blocked": not ai_block,
    }
    score = round(100 * sum(checks.values()) / len(checks))
    result = {
        **sample,
        "search_rank": None,
        "selection_position_recorded": 0,
        "final_url": final_url,
        "attempted_urls": " | ".join(attempted_urls),
        "http_status": status,
        "fetch_error": body[:300] if status == 0 else "",
        "fetch_ms_observed": elapsed_ms,
        "content_bytes": len(body.encode("utf-8", "ignore")),
        "title_text": title_text,
        "entity_detail_count": entity_details,
        "image_alt_ratio": alt_ratio,
        "llms_txt_present": llms_status == 200 and len(llms.strip()) > 20,
        "score": score,
        **{name: int(value) for name, value in checks.items()},
    }
    return result


def main() -> None:
    sample_path = OUT / "sample.json"
    if sample_path.exists():
        samples = json.loads(sample_path.read_text(encoding="utf-8"))
    else:
        samples = []
        used: set[str] = set()
        for sector in SECTORS:
            for country, city in MARKETS:
                print(f"Selecting {sector}: {city}, {country}", flush=True)
                site = choose_site(sector, country, city, used)
                if site:
                    samples.append(site)
                    sample_path.write_text(json.dumps(samples, ensure_ascii=False, indent=2), encoding="utf-8")
                else:
                    print(f"  NO MATCH: {sector} / {country}", flush=True)
                time.sleep(0.35)

    print(f"Auditing {len(samples)} sites", flush=True)
    results = []
    with ThreadPoolExecutor(max_workers=8) as executor:
        futures = [executor.submit(audit_site, sample) for sample in samples]
        for future in as_completed(futures):
            result = future.result()
            results.append(result)
            print(f"{len(results):03d}/{len(samples)} {result['sector']} {result['country']} {result['score']}", flush=True)

    results.sort(key=lambda row: (str(row["sector"]), str(row["country"])))
    fields = list(results[0])
    with (OUT / "audit_raw.csv").open("w", newline="", encoding="utf-8") as handle:
        writer = csv.DictWriter(handle, fieldnames=fields)
        writer.writeheader()
        writer.writerows(results)

    public_rows = []
    private_fields = {"url", "domain", "final_url", "query", "title_text", "attempted_urls", "fetch_error"}
    for row in results:
        public = {key: value for key, value in row.items() if key not in private_fields}
        public["site_id"] = hashlib.sha256(str(row["domain"]).encode()).hexdigest()[:12]
        public_rows.append(public)
    public_fields = ["site_id"] + [field for field in fields if field not in private_fields]
    with (OUT / "audit_anonymized.csv").open("w", newline="", encoding="utf-8") as handle:
        writer = csv.DictWriter(handle, fieldnames=public_fields, extrasaction="ignore")
        writer.writeheader()
        writer.writerows(public_rows)

    reachable_results = [row for row in results if int(row["reachable"]) == 1]
    summary = {
        "sites": len(results),
        "reachable_sites": len(reachable_results),
        "availability_pct": round(100 * len(reachable_results) / len(results), 1),
        "quality_metrics_denominator": len(reachable_results),
        "mean_score_reachable": round(sum(int(row["score"]) for row in reachable_results) / len(reachable_results), 1),
        "median_score_reachable": sorted(int(row["score"]) for row in reachable_results)[len(reachable_results) // 2],
        "checks": {
            check: {"count": sum(int(row[check]) for row in reachable_results), "pct": round(100 * sum(int(row[check]) for row in reachable_results) / len(reachable_results), 1)}
            for check in [key for key in results[0] if key in {
                "reachable", "https", "indexable", "title", "meta_description", "canonical", "html_lang",
                "structured_data", "entity_schema", "entity_details_3plus", "open_graph", "single_h1_present",
                "heading_structure", "contact_signal", "about_link", "privacy_link", "service_detail_link",
                "faq_signal", "robots_available", "sitemap_available", "ai_crawlers_not_explicitly_blocked",
            }]
        },
        "by_sector": {},
        "by_country": {},
    }
    for group_key in ("sector", "country"):
        target = summary[f"by_{group_key}"]
        for value in sorted({str(row[group_key]) for row in results}):
            group = [row for row in reachable_results if row[group_key] == value]
            target[value] = {
                "n": len(group),
                "mean_score": round(sum(int(row["score"]) for row in group) / len(group), 1) if group else None,
            }
    (OUT / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
    print(json.dumps(summary, ensure_ascii=False, indent=2))


if __name__ == "__main__":
    main()