#!/usr/bin/env python3
"""
Scrape 'The LaRouche Show' archives (2002–2014), keeping only best-quality MP3 per show.

Usage examples:
  python scrape_larouche_show.py                       # scrape & download all years
  python scrape_larouche_show.py --skip-download       # build manifests only
  python scrape_larouche_show.py --years 2006 2007
  python scrape_larouche_show.py --out "D:/audio/larouche"
"""
import warnings
from bs4 import MarkupResemblesLocatorWarning
warnings.filterwarnings("ignore", category=MarkupResemblesLocatorWarning)

import re
import os
import csv
import time
import argparse
from pathlib import Path
from datetime import datetime

import requests
from bs4 import BeautifulSoup
# from slugify import slugify
from slugify import slugify  # works fine with python-slugify
from tqdm import tqdm

# ---- CONFIG ----
YEARS_DEFAULT = list(range(2002, 2015))  # 2002–2014 inclusive
BASE_ARCHIVE_URL = "https://archive.schillerinstitute.com/calendars/lar_show-{year}.html"
DEFAULT_OUT_ROOT = "larouche_show/all"
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"

# ---- HTTP ----
def fetch(url: str, tries: int = 4, timeout: int = 20) -> requests.Response:
    headers = {"User-Agent": UA}
    for i in range(tries):
        try:
            r = requests.get(url, headers=headers, timeout=timeout)
            r.raise_for_status()
            return r
        except Exception:
            if i == tries - 1:
                raise
            time.sleep(1.5 * (i + 1))

# ---- PARSING ----
def parse_archive_for_year(url: str, year: int):
    """
    Returns list of dicts: {date: 'YYYY-MM-DD', title: str, url: mp3_url, year: int}
    Picks only the best-quality MP3 link per show block.
    """
    r = fetch(url)
    html = r.text
    soup = BeautifulSoup(html, "html.parser")

    date_pat_text = rf"[A-Z][a-z]+ \d{{1,2}}, {year}:"
    date_pat = re.compile(rf"^({date_pat_text})")

    # Split raw HTML on each date header to isolate show blocks
    split_pat = re.compile(rf"(<[^>]*?>)?({date_pat_text})", re.M)
    parts = split_pat.split(html)

    blocks = []
    cur = []
    saw_first = False
    for piece in parts:
        if not piece:
            continue
        txt = BeautifulSoup(piece, "html.parser").get_text().strip()
        if date_pat.match(txt):
            if saw_first and cur:
                blocks.append("".join(cur))
                cur = []
            saw_first = True
        if saw_first:
            cur.append(piece)
    if cur:
        blocks.append("".join(cur))

    a_tag_pat = re.compile(r'<a[^>]+href="([^"]+)"[^>]*>(.*?)</a>', re.I | re.S)

    def pick_best_mp3(block_html: str):
        cands = []
        for href, inner in a_tag_pat.findall(block_html):
            text = BeautifulSoup(inner, "html.parser").get_text().strip().lower()
            if href.lower().endswith(".mp3"):
                cands.append((text, href))
        if not cands:
            return None

        def score(t: str, url: str) -> int:
            s = 0
            if "download mp3" in t:
                s += 80
            if "high" in t or "high-speed" in t or "high speed" in t:
                s += 40
            if url.lower().endswith("_hi.mp3"):
                s += 70
            if "low-speed" in t or "low speed" in t or "lo" in t:
                s -= 50
            return s

        cands.sort(key=lambda x: score(*x), reverse=True)
        return cands[0][1]

    shows = []
    for block in blocks:
        text = BeautifulSoup(block, "html.parser").get_text("\n")
        first_line = text.strip().split("\n", 1)[0]
        m = date_pat.match(first_line)
        if not m:
            continue

        date_str_full = m.group(1).rstrip(":")  # e.g. "January 7, 2006"
        desc = first_line[len(m.group(1)) :].strip(" :")

        # Normalize date
        try:
            dt = datetime.strptime(date_str_full, "%B %d, %Y")
        except ValueError:
            cleaned = " ".join(date_str_full.replace("\xa0", " ").split())
            dt = datetime.strptime(cleaned, "%B %d, %Y")
        ymd = dt.strftime("%Y-%m-%d")

        best = pick_best_mp3(block)
        if best:
            shows.append({
                "date": ymd,
                "title": desc if desc else "The LaRouche Show",
                "url": best,
                "year": year
            })

    return shows

# ---- PATHS & DOWNLOAD ----
def year_dir(year: int, out_root: Path) -> Path:
    d = out_root / str(year)
    d.mkdir(parents=True, exist_ok=True)
    return d

def filename_for(show: dict, out_root: Path) -> Path:
    fn = f"{show['date']}_{slugify(show['title'])}.mp3"
    return year_dir(show["year"], out_root) / fn

def download(url: str, dest: Path):
    dest.parent.mkdir(parents=True, exist_ok=True)
    with requests.get(url, headers={"User-Agent": UA}, stream=True, timeout=60) as r:
        r.raise_for_status()
        total = int(r.headers.get("Content-Length", 0))
        with open(dest, "wb") as f, tqdm(total=total, unit="B", unit_scale=True, desc=dest.name) as p:
            for chunk in r.iter_content(chunk_size=1 << 15):
                if chunk:
                    f.write(chunk)
                    if total:
                        p.update(len(chunk))

# ---- MAIN ----
def main():
    ap = argparse.ArgumentParser(description="Scrape 'The LaRouche Show' archives (2002–2014), best-quality MP3s only.")
    ap.add_argument("--years", nargs="*", type=int, default=YEARS_DEFAULT,
                    help="Years to scrape, e.g. --years 2006 2007 2010 (default: 2002..2014)")
    ap.add_argument("--skip-download", action="store_true", help="Only build manifests; do not download audio files.")
    ap.add_argument("--out", default=DEFAULT_OUT_ROOT, help=f"Output root directory (default: {DEFAULT_OUT_ROOT})")
    args = ap.parse_args()

    out_root = Path(args.out)

    all_shows = []

    for year in sorted(set(args.years)):
        url = BASE_ARCHIVE_URL.format(year=year)
        print(f"Parsing {year}: {url}")
        try:
            shows = parse_archive_for_year(url, year)
        except Exception as e:
            print(f"  FAILED to parse {year}: {e}")
            continue

        if not shows:
            print(f"  No shows found for {year}.")
            continue

        # De-dup per date within the year (prefer _hi.mp3)
        dedup = {}
        for s in shows:
            key = s["date"]
            cur = dedup.get(key)
            if (cur is None) or (cur and "_hi.mp3" not in cur["url"].lower() and "_hi.mp3" in s["url"].lower()):
                dedup[key] = s

        final_year = [dedup[k] for k in sorted(dedup.keys())]
        all_shows.extend(final_year)

        # Per-year manifest
        ydir = year_dir(year, out_root)
        manifest_path = ydir / f"urls_{year}.csv"
        with open(manifest_path, "w", newline="", encoding="utf-8") as f:
            w = csv.writer(f)
            w.writerow(["date", "title", "url", "filename"])
            for s in final_year:
                w.writerow([s["date"], s["title"], s["url"], filename_for(s, out_root).name])
        print(f"  {len(final_year)} shows → {manifest_path}")

        if not args.skip_download:
            for s in final_year:
                dest = filename_for(s, out_root)
                if dest.exists():
                    print(f"  Skip existing: {dest.name}")
                    continue
                try:
                    download(s["url"], dest)
                except Exception as e:
                    print(f"  FAILED download {s['date']}: {e}")

    # Combined manifest
    if all_shows:
        out_root.mkdir(parents=True, exist_ok=True)
        combined = out_root / "urls_all_years.csv"
        with open(combined, "w", newline="", encoding="utf-8") as f:
            w = csv.writer(f)
            w.writerow(["date", "title", "url", "filename", "year"])
            for s in sorted(all_shows, key=lambda x: x["date"]):
                rel = filename_for(s, out_root).relative_to(out_root)
                w.writerow([s["date"], s["title"], s["url"], str(rel), s["year"]])
        print(f"\nCombined manifest: {combined}")
    else:
        print("No shows collected across requested years.")

if __name__ == "__main__":
    main()