"""
MegaBlog Crawler
================
1. Pulls active sources from Laravel API
2. Fetches RSS feeds
3. Filters articles: only those published TODAY (or since --since datetime)
4. POSTs each new article to Laravel API (background job handles image + AI rewrite)

Usage:
    python3 crawler.py                          # fetch today's articles from all sources
    python3 crawler.py --continent Africa       # filter by continent
    python3 crawler.py --category Sports        # filter by category
    python3 crawler.py --source-id 12           # single source
    python3 crawler.py --since "2026-04-06"     # fetch from a specific date
    python3 crawler.py --since "2026-04-06 14:00"  # fetch from a specific datetime
    python3 crawler.py --max 20                 # hard cap per source (safety valve)
"""

import argparse
import logging
import re
import time
from datetime import datetime, timezone, date, timedelta
from typing import Optional

import feedparser
import requests
import trafilatura
from bs4 import BeautifulSoup

from laravel_poster import get_sources, post_article, log_crawl

# ── Logging ────────────────────────────────────────────────────────────────────
logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)-8s] %(message)s",
    datefmt="%H:%M:%S",
    handlers=[
        logging.StreamHandler(),
        logging.FileHandler("crawler.log", encoding="utf-8"),
    ],
)
log = logging.getLogger("crawler")

USER_AGENT = "Mozilla/5.0 (compatible; MegaBlogCrawler/2.0)"

TAG_KEYWORDS = {
    "football":     ["football", "soccer", "fifa", "premier league", "champions league", "la liga"],
    "basketball":   ["basketball", "nba", "nba finals"],
    "cricket":      ["cricket", "icc", "test match", "odi", "t20"],
    "tennis":       ["tennis", "wimbledon", "us open", "french open", "atp", "wta"],
    "formula1":     ["formula 1", "f1", "grand prix", "ferrari", "red bull racing"],
    "ai":           ["artificial intelligence", " ai ", "machine learning", "chatgpt", "openai", "llm"],
    "smartphones":  ["smartphone", "iphone", "android", "samsung", "pixel"],
    "startups":     ["startup", "funding", "venture capital", "series a", "ipo"],
    "crypto":       ["bitcoin", "ethereum", "blockchain", "crypto", "web3", "defi"],
    "cybersecurity":["hack", "breach", "malware", "ransomware", "cybersecurity"],
    "movies":       ["movie", "film", "cinema", "box office", "hollywood", "nollywood", "bollywood"],
    "music":        ["music", "album", "song", "concert", "grammy", "billboard"],
    "elections":    ["election", "vote", "ballot", "candidate", "campaign"],
    "war":          ["war", "conflict", "military", "invasion", "ceasefire", "troops"],
    "economy":      ["economy", "inflation", "gdp", "recession", "trade", "tariff"],
    "health":       ["health", "covid", "pandemic", "vaccine", "disease", "who"],
    "climate":      ["climate", "global warming", "carbon", "emissions", "cop"],
}


def extract_tags(title: str, content: str) -> list:
    combined = (title + " " + content).lower()
    return [tag for tag, kws in TAG_KEYWORDS.items() if any(kw in combined for kw in kws)][:6]


def clean_html(raw: str) -> str:
    soup = BeautifulSoup(raw or "", "html.parser")
    return re.sub(r"\s+", " ", soup.get_text(separator=" ")).strip()


def is_truncated(text: str) -> bool:
    """RSS summaries are often cut off with […] or are very short."""
    return len(text) < 400 or any(marker in text for marker in ["[…]", "[...]", "&#8230;", "Read more", "Continue reading"])


def fetch_full_content(url: str) -> Optional[str]:
    """Fetch the full article text from a URL using trafilatura."""
    if not url:
        return None
    try:
        downloaded = trafilatura.fetch_url(url)
        if not downloaded:
            return None
        text = trafilatura.extract(
            downloaded,
            include_comments=False,
            include_tables=False,
            no_fallback=False,
        )
        if text and len(text) > 200:
            return text.strip()
    except Exception:
        pass
    return None


def fetch_og_image(url: str) -> Optional[str]:
    try:
        r = requests.get(url, headers={"User-Agent": USER_AGENT}, timeout=10)
        r.raise_for_status()
        soup = BeautifulSoup(r.text, "html.parser")
        og = soup.find("meta", property="og:image")
        if og:
            val = og.get("content")
            if isinstance(val, str) and val.startswith("http"):
                return val
        for img in soup.find_all("img", src=True):
            src = str(img["src"])
            if src.startswith("http") and not any(x in src for x in ["logo", "icon", "avatar", "1x1", "pixel"]):
                return src
    except Exception:
        pass
    return None


def parse_published(entry) -> Optional[datetime]:
    """Return timezone-aware datetime from feed entry, or None."""
    if getattr(entry, "published_parsed", None):
        return datetime(*entry.published_parsed[:6], tzinfo=timezone.utc)
    return None


def crawl_source(source: dict, since: datetime, max_per_source: int = 100) -> dict:
    name      = source["name"]
    rss_url   = source.get("rss_url")
    source_id = source["id"]

    result = {
        "source_id": source_id, "name": name,
        "found": 0, "saved": 0, "skipped": 0,
        "status": "ok", "error": None,
    }

    if not rss_url:
        result.update({"status": "empty", "error": "No RSS URL"})
        return result

    log.info("━━━ [%s] %s", source.get("continent", "?"), name)

    try:
        r = requests.get(rss_url, headers={"User-Agent": USER_AGENT}, timeout=15)
        r.raise_for_status()
        feed = feedparser.parse(r.content)
    except Exception as exc:
        result.update({"status": "error" if "403" not in str(exc) else "blocked", "error": str(exc)})
        log.error("  Feed error: %s", exc)
        return result

    if not feed.entries:
        result.update({"status": "empty", "error": "No entries in feed"})
        log.warning("  No entries found")
        return result

    # ── Filter by date ────────────────────────────────────────────────────────
    # RSS feeds are newest-first. Stop as soon as we hit an article older than `since`.
    candidates = []
    undated    = []  # articles with no date — always include (can't know)

    for entry in feed.entries:
        pub = parse_published(entry)
        if pub is None:
            undated.append(entry)
            continue
        if pub >= since:
            candidates.append(entry)
        else:
            # Entries are newest-first — everything after this is older
            break

    # Include undated entries only if we have no dated ones yet (avoids spam)
    all_entries = candidates + (undated if not candidates else [])
    all_entries = all_entries[:max_per_source]  # safety cap

    if not all_entries:
        log.info("  No new articles since %s", since.strftime("%Y-%m-%d %H:%M"))
        result["status"] = "empty"
        return result

    log.info("  Feed OK — %d entries since %s (cap=%d)",
             len(all_entries), since.strftime("%Y-%m-%d %H:%M"), max_per_source)
    result["found"] = len(all_entries)

    for entry in all_entries:
        title = clean_html(entry.get("title", "")).strip()
        if not title:
            continue

        link = str(entry.get("link", "") or "")

        raw = (
            entry.get("summary", "")
            or entry.get("description", "")
            or (entry.get("content") or [{}])[0].get("value", "")
        )
        content = clean_html(raw)

        if not content:
            result["skipped"] += 1
            continue

        # Image from feed media fields first (faster than fetching the page)
        image_url = None
        for attr in ("media_content", "media_thumbnail"):
            media = getattr(entry, attr, None)
            if media and isinstance(media, list) and media[0].get("url"):
                image_url = media[0]["url"]
                break

        # Fetch full article if RSS only gave us a truncated excerpt
        if link and is_truncated(content):
            full = fetch_full_content(link)
            if full:
                log.debug("  Full content fetched (%d chars) for: %s", len(full), title[:50])
                content = full
            time.sleep(0.5)

        if not image_url and link:
            image_url = fetch_og_image(link)
            time.sleep(0.3)

        pub = parse_published(entry)
        published = pub.isoformat() if pub else None
        tags = extract_tags(title, content)

        article = {
            "source_id":    source_id,
            "source":       name,
            "title":        title,
            "content":      content,
            "excerpt":      content[:300] + "..." if len(content) > 300 else content,
            "image_url":    image_url,
            "source_url":   link,
            "tags":         tags,
            "published_at": published,
        }

        posted = post_article(article)
        if posted is None:
            result["skipped"] += 1
        else:
            result["saved"] += 1

        time.sleep(0.2)

    log.info("  → saved=%d skipped=%d", result["saved"], result["skipped"])
    return result


def crawl(
    filter_continent=None,
    filter_category=None,
    filter_source_id=None,
    since: datetime = None,
    max_per_source: int = 100,
):
    # Default: articles published since the start of today (UTC)
    if since is None:
        since = datetime.now(timezone.utc).replace(hour=0, minute=0, second=0, microsecond=0)

    log.info("═" * 60)
    log.info("MEGABLOG CRAWLER — %s", datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
    log.info("Fetching articles published since: %s UTC", since.strftime("%Y-%m-%d %H:%M"))
    log.info("═" * 60)

    sources = get_sources()
    if not sources:
        log.error("No sources returned from Laravel. Is the API running?")
        return

    if filter_continent:
        sources = [s for s in sources if s["continent"].lower() == filter_continent.lower()]
    if filter_category:
        sources = [s for s in sources if s["category"].lower() == filter_category.lower()]
    if filter_source_id:
        sources = [s for s in sources if s["id"] == filter_source_id]

    log.info("Running %d sources | since=%s | cap=%d/source",
             len(sources), since.strftime("%Y-%m-%d %H:%M"), max_per_source)

    totals = {"found": 0, "saved": 0, "skipped": 0, "ok": 0, "error": 0}

    for source in sources:
        started  = datetime.now(timezone.utc)
        result   = crawl_source(source, since=since, max_per_source=max_per_source)
        finished = datetime.now(timezone.utc)
        duration = int((finished - started).total_seconds())

        log_crawl(
            source_id   = result["source_id"],
            status      = result["status"],
            found       = result["found"],
            saved       = result["saved"],
            skipped     = result["skipped"],
            started_at  = started.isoformat(),
            finished_at = finished.isoformat(),
            duration    = duration,
            error       = result.get("error"),
        )

        totals["found"]   += result["found"]
        totals["saved"]   += result["saved"]
        totals["skipped"] += result["skipped"]
        totals["ok" if result["status"] == "ok" else "error"] += 1

        time.sleep(0.5)

    log.info("═" * 60)
    log.info("DONE — Sources: %d ok / %d errors", totals["ok"], totals["error"])
    log.info("Articles: %d found / %d queued / %d skipped", totals["found"], totals["saved"], totals["skipped"])
    log.info("═" * 60)


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="MegaBlog Crawler")
    parser.add_argument("--continent",  help="Filter by continent")
    parser.add_argument("--category",   help="Filter by category")
    parser.add_argument("--source-id",  type=int, help="Single source ID")
    parser.add_argument("--since",      help='Fetch articles since this date/time. E.g. "2026-04-06" or "2026-04-06 14:00"')
    parser.add_argument("--max",        type=int, default=100, help="Max articles per source (safety cap, default 100)")
    args = parser.parse_args()

    # Parse --since
    since_dt = None
    if args.since:
        for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M", "%Y-%m-%d"):
            try:
                since_dt = datetime.strptime(args.since, fmt).replace(tzinfo=timezone.utc)
                break
            except ValueError:
                continue
        if since_dt is None:
            log.error("Invalid --since format. Use: YYYY-MM-DD or 'YYYY-MM-DD HH:MM'")
            exit(1)

    crawl(
        filter_continent = args.continent,
        filter_category  = args.category,
        filter_source_id = args.source_id,
        since            = since_dt,
        max_per_source   = args.max,
    )
