#!/usr/bin/env python3
"""
Scrape 'The LaRouche Show' for 2002–2006 (best-quality per show).

- 2002–2005: pages often use .asx (stream) and .asf (download). We resolve .asx when needed.
- 2006: pages commonly link .mp3; still scored/handled with the same logic.

Usage:
  python scrape_larouche_2002_2006.py                 # scrape + download all years
  python scrape_larouche_2002_2006.py --skip-download # manifests only
  python scrape_larouche_2002_2006.py --years 2002 2005
  python scrape_larouche_2002_2006.py --out "D:/audio/larouche/2002_2006"
"""

import re
import csv
import time
import argparse
import warnings
from pathlib import Path
from datetime import datetime
from urllib.parse import urlparse, urljoin

import requests
from bs4 import BeautifulSoup, MarkupResemblesLocatorWarning
from tqdm import tqdm

warnings.filterwarnings("ignore", category=MarkupResemblesLocatorWarning)

YEARS_DEFAULT = [2002, 2003, 2004, 2005, 2006]
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"
TIMEOUT = 25
DEFAULT_OUT = "larouche_show/2002_2006"

def get_archive_candidates(year: int):
    """
    Try Schiller mirror first (very consistent), then larouchepub.
    """
    urls = [
        f"https://archive.schillerinstitute.com/calendars/lar_show-{year}.html",
        f"https://larouchepub.com/radio/archive_{year}.html",
    ]
    return urls

def fetch(url, tries=4, timeout=TIMEOUT, stream=False, method="GET"):
    hdrs = {"User-Agent": UA}
    for i in range(tries):
        try:
            if method == "HEAD":
                r = requests.head(url, headers=hdrs, timeout=timeout, allow_redirects=True)
            else:
                r = requests.get(url, headers=hdrs, timeout=timeout, stream=stream, allow_redirects=True)
            r.raise_for_status()
            return r
        except Exception:
            if i == tries - 1:
                raise
            time.sleep(1.2 * (i + 1))

def safe_slug(s: str) -> str:
    s = s.strip().lower()
    s = re.sub(r"[^\w\s-]", "", s)
    s = re.sub(r"[\s-]+", "_", s)
    return s[:120]

def head_size(url: str):
    try:
        r = fetch(url, method="HEAD")
        cl = r.headers.get("Content-Length")
        if cl and cl.isdigit():
            return int(cl)
    except Exception:
        pass
    return None

def resolve_asx(asx_url: str):
    """
    Fetch an .asx playlist and extract candidate media URLs.
    Returns list of dicts: {url, size_bytes (maybe None)}
    """
    try:
        r = fetch(asx_url)
    except Exception:
        return []
    text = r.text

    # Grab HREF=... from <ref> tags (ASX is simple XML-ish)
    hrefs = []
    for m in re.finditer(r'(?i)href\s*=\s*"([^"]+)"', text):
        hrefs.append(m.group(1).strip())

    # De-dup preserve order
    seen = set()
    hrefs = [h for h in hrefs if not (h in seen or seen.add(h))]

    cands = []
    for h in hrefs:
        if h.lower().startswith("mms://"):
            # naive http fallback attempt
            http_try = "http://" + h.split("://", 1)[1]
            size = head_size(http_try)
            if size is not None:
                cands.append({"url": http_try, "size_bytes": size})
            else:
                cands.append({"url": h, "size_bytes": None})
            continue

        size = head_size(h)
        cands.append({"url": h, "size_bytes": size})
    return cands

def score_anchor(label: str, href: str):
    """
    Score links by desirability.
    Highest: explicit Download + High-speed, with .mp3/.asf, then .wma/.wmv, last .asx (until resolved).
    """
    t = (label or "").lower()
    s = 0
    if "download" in t: s += 60
    if "high-speed" in t or "high speed" in t or "high" in t: s += 30
    if "audio and video" in t: s += 20
    if "low-speed" in t or "low speed" in t or "low" in t: s -= 40

    href_l = href.lower()
    if href_l.endswith(".mp3"): s += 90
    elif href_l.endswith(".asf"): s += 70
    elif href_l.endswith(".wma") or href_l.endswith(".wmv"): s += 50
    elif href_l.endswith(".asx"): s += 10

    # Prefer larger size if the label mentions "(xx.xx MB)"
    m = re.search(r"\(([\d.]+)\s*MB\)", label or "", flags=re.I)
    label_mb = float(m.group(1)) if m else None
    return s, label_mb

def pick_best_media(block_html: str, base_url: str):
    """
    From a block, pick the best candidate. If .asx only, resolve and choose the largest http/https media.
    Returns dict:
      {
        origin_url, origin_label, size_label_mb,
        chosen_url, chosen_size, source_type
      }
    """
    soup = BeautifulSoup(block_html, "html.parser")
    anchors = []

    for a in soup.find_all("a"):
        href = a.get("href") or ""
        if not href:
            continue
        href = urljoin(base_url, href)  # handle relative just in case
        label = a.get_text(" ", strip=True)
        score, mb = score_anchor(label, href)
        # consider only plausible media links (.mp3/.asf/.asx/.wma/.wmv)
        if any(href.lower().endswith(ext) for ext in (".mp3", ".asf", ".asx", ".wma", ".wmv")):
            anchors.append((score, mb, label, href))

    if not anchors:
        return None

    # Sort by score, then label MB
    anchors.sort(key=lambda x: (x[0], x[1] or 0.0), reverse=True)
    _, label_mb, label, href = anchors[0]

    ext = href.lower().rsplit(".", 1)[-1]
    source_type = "other"
    if ext in ("mp3", "asf", "wma", "wmv", "asx"):
        source_type = ext

    chosen_url = href
    chosen_size = None

    if source_type == "asx":
        cands = resolve_asx(href)
        if cands:
            # prefer http/https and largest size
            def ckey(c):
                scheme = urlparse(c["url"]).scheme.lower()
                scheme_rank = 1 if scheme in ("http", "https") else 0
                size = c["size_bytes"] or 0
                return (scheme_rank, size)
            best_c = sorted(cands, key=ckey, reverse=True)[0]
            chosen_url = best_c["url"]
            chosen_size = best_c["size_bytes"]
    else:
        # Try HEAD for size if http(s)
        if urlparse(href).scheme.lower() in ("http", "https"):
            chosen_size = head_size(href)

    return {
        "origin_url": href,
        "origin_label": label,
        "size_label_mb": label_mb or "",
        "chosen_url": chosen_url,
        "chosen_size": chosen_size or "",
        "source_type": source_type,
    }

def split_into_date_blocks(html: str, year: int):
    """
    Split the year archive page into per-date blocks on lines like 'Month DD, YYYY:'.
    """
    month_names = "January|February|March|April|May|June|July|August|September|October|November|December"
    date_re = re.compile(rf"({month_names})\s+\d{{1,2}},\s+{year}:", re.I)
    parts = date_re.split(html)

    blocks = []
    for i in range(1, len(parts), 2):
        blocks.append(parts[i] + parts[i+1])
    return blocks

def parse_year(html: str, base_url: str, year: int):
    blocks = split_into_date_blocks(html, year)
    month_names = "January|February|March|April|May|June|July|August|September|October|November|December"
    header_re = re.compile(rf"({month_names})\s+(\d{{1,2}}),\s+{year}:", re.I)

    shows = []
    for b in blocks:
        txt = BeautifulSoup(b, "html.parser").get_text("\n")
        first = txt.strip().split("\n", 1)[0]
        m = header_re.match(first)
        if not m:
            continue
        month, day = m.group(1), m.group(2)
        dt = datetime.strptime(f"{month} {day} {year}", "%B %d %Y")
        ymd = dt.strftime("%Y-%m-%d")
        title = (first.split(":", 1)[1].strip() if ":" in first else "The LaRouche Show") or "The LaRouche Show"

        best = pick_best_media(b, base_url)
        if not best:
            continue

        shows.append({
            "date": ymd,
            "title": title,
            **best
        })

    # De-dup per date: prefer larger chosen_size; if equal/unknown, prefer extension priority mp3>asf>wma>wmv>asx>other
    def ext_priority(u: str):
        e = urlparse(u).path.lower()
        if e.endswith(".mp3"): return 5
        if e.endswith(".asf"): return 4
        if e.endswith(".wma"): return 3
        if e.endswith(".wmv"): return 2
        if e.endswith(".asx"): return 1
        return 0

    dedup = {}
    for s in shows:
        key = s["date"]
        cur = dedup.get(key)
        if cur is None:
            dedup[key] = s
            continue
        cur_size = cur["chosen_size"] if cur["chosen_size"] != "" else -1
        new_size = s["chosen_size"] if s["chosen_size"] != "" else -1
        if new_size > cur_size:
            dedup[key] = s
        elif new_size == cur_size and ext_priority(s["chosen_url"]) > ext_priority(cur["chosen_url"]):
            dedup[key] = s

    return [dedup[k] for k in sorted(dedup.keys())]

def infer_ext_from_url(u: str) -> str:
    p = urlparse(u).path.lower()
    for ext in (".mp3", ".asf", ".wma", ".wmv"):
        if p.endswith(ext):
            return ext
    return ".bin"

def download(url, dest_path: Path):
    with fetch(url, stream=True) as r:
        total = int(r.headers.get("Content-Length", 0))
        with open(dest_path, "wb") as f, tqdm(total=total, unit="B", unit_scale=True, desc=dest_path.name) as p:
            for chunk in r.iter_content(chunk_size=1 << 15):
                if chunk:
                    f.write(chunk)
                    if total:
                        p.update(len(chunk))

def parse_one_year(year: int):
    last_err = None
    for url in get_archive_candidates(year):
        try:
            r = fetch(url)
            return parse_year(r.text, url, year)
        except Exception as e:
            last_err = e
            continue
    # If all sources fail:
    print(f"  FAILED to parse {year}: {last_err}")
    return []

def main():
    ap = argparse.ArgumentParser(description="Scrape 2002–2006 LaRouche Show (best per date; handles .asx/.asf/.mp3).")
    ap.add_argument("--years", nargs="*", type=int, default=YEARS_DEFAULT, help="Years to scrape (default: 2002..2006)")
    ap.add_argument("--out", default=DEFAULT_OUT, help=f"Output root directory (default: {DEFAULT_OUT})")
    ap.add_argument("--skip-download", action="store_true", help="Only build manifests; do not download files.")
    args = ap.parse_args()

    out_root = Path(args.out)
    out_root.mkdir(parents=True, exist_ok=True)

    all_shows = []

    for year in sorted(set(args.years)):
        print(f"Parsing {year} …")
        shows = parse_one_year(year)
        if not shows:
            print(f"  No shows found for {year}.")
            continue

        # Per-year manifest
        ydir = out_root / str(year)
        ydir.mkdir(parents=True, exist_ok=True)
        manifest = ydir / f"urls_{year}.csv"
        with open(manifest, "w", newline="", encoding="utf-8") as f:
            w = csv.writer(f)
            w.writerow(["date","title","origin_label","origin_url","size_label_mb","chosen_url","chosen_size_bytes","source_type","suggested_filename"])
            for s in shows:
                stem = f"{s['date']}_{safe_slug(s['title'])}"
                ext = infer_ext_from_url(s["chosen_url"])
                w.writerow([s["date"], s["title"], s["origin_label"], s["origin_url"], s["size_label_mb"], s["chosen_url"], s["chosen_size"], s["source_type"], stem + ext])
        print(f"  {len(shows)} shows → {manifest}")

        if not args.skip_download:
            for s in shows:
                # Skip pure MMS (requests can't fetch)
                scheme = urlparse(s["chosen_url"]).scheme.lower()
                if scheme == "mms":
                    print(f"  SKIP (mms): {s['date']}  {s['chosen_url']}")
                    continue
                dest = (out_root / str(year)) / f"{s['date']}_{safe_slug(s['title'])}{infer_ext_from_url(s['chosen_url'])}"
                if dest.exists():
                    print(f"  Skip existing: {dest.name}")
                    continue
                try:
                    download(s["chosen_url"], dest)
                except Exception as e:
                    print(f"  FAILED download {s['date']}: {e}")

        all_shows.extend([dict(s, year=year) for s in shows])

    # Combined manifest
    if all_shows:
        combined = out_root / "urls_2002_2006.csv"
        with open(combined, "w", newline="", encoding="utf-8") as f:
            w = csv.writer(f)
            w.writerow(["date","title","year","origin_label","origin_url","size_label_mb","chosen_url","chosen_size_bytes","source_type","relative_filename"])
            for s in sorted(all_shows, key=lambda x: (x["date"], x["year"])):
                rel = Path(str(s["year"])) / f"{s['date']}_{safe_slug(s['title'])}{infer_ext_from_url(s['chosen_url'])}"
                w.writerow([s["date"], s["title"], s["year"], s["origin_label"], s["origin_url"], s["size_label_mb"], s["chosen_url"], s["chosen_size"], s["source_type"], str(rel)])
        print(f"\nCombined manifest: {combined}")
    else:
        print("No shows collected.")

if __name__ == "__main__":
    main()