from __future__ import annotations

import logging
import re
import threading
import time
from typing import Any, Dict, List
from urllib.parse import urljoin, urlparse

from bs4 import BeautifulSoup

from scrapers.base_scraper import BaseScraper, ScrapeRunResult
from scrapers.common import ProxyPool, ResponseCache, ThreadLocalSessionFactory, detect_challenge_page
from scrapers.exceptions import ScraperAbortError
from scrapers.options import RuntimeOptions

logger = logging.getLogger(__name__)


class Scraper(BaseScraper):
    execution_mode = "api"
    default_request_backend = "playwright"
    store_definitions = (
        {
            "retailer_code": "frankodiszkont-hu",
            "name": "Frankó Diszkont",
            "country_code": "HU",
            "base_url": "https://frankodiszkont.hu",
            "scraper_module": "scrapers.platform.frankodiszkont_scraper",
            "scraper_mapping": {
                "execution_mode": "api",
                "output_schema": "product",
                "pre_run_pool_refresh": "stale",
            },
        },
    )

    def __init__(
        self,
        workers=10,
        proxy_list=None,
        retries=3,
        timeout=120,
        max_proxy_failures=3,
    ):
        super().__init__()
        self.workers = workers
        self.retries = retries
        self.timeout = timeout
        self.max_proxy_failures = max_proxy_failures
        self.base_url = "https://frankodiszkont.hu"
        # Use a full document URL for preflight so browser-backed transports
        # (scrapling_stealth/playwright) can expose cert/proxy issues earlier.
        self.proxy_validation_url = self.base_url
        self.headers = {
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
            "Referer": self.base_url,
        }
        if hasattr(self, "configure_http_transport"):
            self.configure_http_transport(
                headers=self.headers,
                proxy_list=proxy_list,
                max_proxy_failures=max_proxy_failures,
            )
        else:
            self.cache = ResponseCache(ttl=43200)
            self.proxy_pool = ProxyPool(proxy_list, max_failures=max_proxy_failures)
            self.http_proxies_enabled = True
            self.sessions = ThreadLocalSessionFactory(self.headers)
            self._request_rate_lock = threading.Lock()
            self._last_request_at = 0.0

    def run(self, target: Any = None, context: Dict[str, Any] | None = None) -> List[Dict[str, Any]] | ScrapeRunResult:
        opts = RuntimeOptions.from_run_args(target, context)
        return self.collect_products(
            start_urls=opts.start_urls or None,
            limit_categories=opts.limit_categories or opts.max_categories,
            category_filter=opts.category_filter,
            max_products=opts.max_products,
        )

    def collect_products(self, start_urls=None, limit_categories=None, category_filter=None, max_products=None):
        start_urls = start_urls or [self.base_url]

        category_urls = self.collect_category_urls(start_urls)

        if category_filter:
            category_urls = [u for u in category_urls if category_filter in u]
        if limit_categories:
            category_urls = category_urls[:limit_categories]

        if not category_urls:
            raise ScraperAbortError(
                "Frankodiszkont: category discovery returned 0 categories — "
                "likely a challenge page, blocked proxy, or site structure change",
                error_category="selector",
            )

        self.log_startup_status(logger, f"Starting Frankodiszkont crawl - {len(category_urls)} categories")

        self.report_work_unit("categories")
        self.report_work_total(len(category_urls))

        category_results = self.parallel_map(
            self._collect_product_urls_worker,
            category_urls,
            label="Frankó Diszkont categories",
            unit="categories",
        )

        all_products_to_scrape: list[dict[str, Any]] = []
        seen_product_urls = set()
        max_products_int = int(max_products) if max_products else None

        for category_url, result in zip(category_urls, category_results):
            category_products, stop_reason = result if result else ([], "fetch_failed_or_challenge")
            added_count = 0
            duplicate_count = 0
            for product_url, category_name in category_products:
                if product_url in seen_product_urls:
                    duplicate_count += 1
                    continue
                if max_products_int and len(all_products_to_scrape) >= max_products_int:
                    break
                seen_product_urls.add(product_url)
                all_products_to_scrape.append((product_url, category_name))
                added_count += 1
            logger.info(
                "Category scan | url=%s | discovered=%d | added=%d | duplicates=%d | stop_reason=%s",
                category_url,
                len(category_products),
                added_count,
                duplicate_count,
                stop_reason or "",
            )
            if max_products_int and len(all_products_to_scrape) >= max_products_int:
                break

        logger.info("Collected %d unique product URLs to scrape", len(all_products_to_scrape))

        records = self.parallel_map(
            self._parse_product_detail_worker,
            all_products_to_scrape,
            label="Frankó Diszkont products",
            unit="products"
        )

        seen_pids = set()
        unique_records = []
        for record in records:
            pid = record.get("source_product_id") if record else None
            if pid and pid not in seen_pids:
                seen_pids.add(pid)
                unique_records.append(record)
            elif record and not pid:
                unique_records.append(record)

        invalid_records = []
        for record in unique_records:
            missing = [field for field in ("source_product_id", "name", "price", "source_url") if not record.get(field)]
            if missing:
                invalid_records.append({
                    "source_url": record.get("source_url"),
                    "missing": missing,
                    "name": record.get("name"),
                })
        logger.info(
            "Frankó detail results | parsed=%d | unique=%d | invalid_required=%d",
            len(records),
            len(unique_records),
            len(invalid_records),
        )
        if invalid_records:
            logger.warning("Frankó invalid records detected: %d", len(invalid_records))

        self.log_summary(logger, "Frankó Diszkont scrape complete", total=len(unique_records))
        return unique_records

    def collect_category_urls(self, start_urls):
        category_urls = []
        seen = set()
        for start_url in start_urls:
            resp = self._fetch_page_with_challenge_retry(start_url, stage="category_discovery")
            if not resp:
                logger.error("Category discovery fetch failed (no response) for %s", start_url)
                continue
            
            logger.info("Category discovery response | url=%s | status=%d | length=%d", resp.url, resp.status_code, len(resp.text))
            logger.info("Category discovery cache file | url=%s | cache_file=%s", start_url, self._cache_file_for(start_url) or "—")
            if len(resp.text) < 5000:
                logger.info("Response snippet: %s", resp.text[:500].replace("\n", " "))

            soup = BeautifulSoup(resp.text, "html.parser")
            links_found = 0
            discovered_this_page = []
            for a in soup.select("a[href]"):
                links_found += 1
                href = str(a.get("href") or "").strip()
                text = a.get_text(" ", strip=True)
                if not href.startswith(self.base_url):
                    href = urljoin(self.base_url, href)
                if not text or self._is_product_url(href):
                    continue
                if not re.search(r"/[a-z0-9\-]+-\d{1,3}(?:/|\?|$)", href):
                    continue
                if href not in seen:
                    seen.add(href)
                    category_urls.append(href)
                    discovered_this_page.append(href)

            # Wordfence/challenge pages often return 200 but no useful links.
            # Treat that as a soft failure and rotate proxies aggressively.
            if links_found == 0 or not discovered_this_page:
                logger.warning(
                    "Category discovery yielded no category links | url=%s | total_links=%d | retrying with rotated proxies",
                    start_url,
                    links_found,
                )
                reservoir = getattr(self, "proxy_reservoir", None)
                reservoir_size = len(getattr(reservoir, "proxies", []) or [])
                extra_rotate_attempts = max(1, min(15, reservoir_size))
                recovered = False
                for attempt in range(1, extra_rotate_attempts + 1):
                    retry_resp = self.fetch_response_via_backend(
                        start_url,
                        purpose=f"franko category_discovery empty-page rotate #{attempt}",
                        timeout=self.timeout,
                        retries=0,
                        headers=self.headers,
                        use_cache=False,
                    )
                    if not retry_resp:
                        continue
                    retry_soup = BeautifulSoup(retry_resp.text, "html.parser")
                    retry_links_found = 0
                    retry_discovered = []
                    for a in retry_soup.select("a[href]"):
                        retry_links_found += 1
                        href = str(a.get("href") or "").strip()
                        text = a.get_text(" ", strip=True)
                        if not href.startswith(self.base_url):
                            href = urljoin(self.base_url, href)
                        if not text or self._is_product_url(href):
                            continue
                        if not re.search(r"/[a-z0-9\-]+-\d{1,3}(?:/|\?|$)", href):
                            continue
                        if href not in seen:
                            seen.add(href)
                            category_urls.append(href)
                            retry_discovered.append(href)
                    if retry_links_found > 0 and retry_discovered:
                        logger.info(
                            "Category discovery recovered on rotate attempt %d/%d | added=%d",
                            attempt,
                            extra_rotate_attempts,
                            len(retry_discovered),
                        )
                        recovered = True
                        break
                if not recovered:
                    logger.warning(
                        "Category discovery still empty after %d rotate attempts | url=%s",
                        extra_rotate_attempts,
                        start_url,
                    )

            logger.info("Category discovery | url=%s | total_links=%d | categories_found=%d", start_url, links_found, len(category_urls))
        return category_urls

    def collect_product_urls(self, category_url):
        page_url = category_url
        seen = set()
        results = []
        category_total = None
        stop_reason = ""
        while True:
            resp = self._fetch_page_with_challenge_retry(page_url, stage="category_listing")
            if not resp:
                stop_reason = "fetch_failed_or_challenge"
                break
            soup = BeautifulSoup(resp.text, "html.parser")
            if self._is_empty_category_page(soup):
                stop_reason = "empty_category_page"
                break
            category_name = self._extract_category_name(soup, category_url)
            if category_total is None:
                category_total = self._extract_category_total(soup)
            for a in soup.select("a[href]"):
                href = str(a.get("href") or "").strip()
                if not href.startswith(self.base_url):
                    href = urljoin(self.base_url, href)
                if href in seen:
                    continue
                if self._is_product_url(href):
                    seen.add(href)
                    results.append((href, category_name))
                    if category_total and len(results) >= category_total:
                        stop_reason = "reached_category_total"
                        break
            if stop_reason == "reached_category_total":
                break
            next_url = self._next_page_url(soup, resp.url)
            if not next_url:
                stop_reason = "no_next_page"
                break
            page_url = next_url
        return results, stop_reason

    def _extract_category_name(self, soup, fallback_url):
        h1 = soup.select_one("h1")
        if h1:
            return h1.get_text(" ", strip=True)
        return urlparse(fallback_url).path.rsplit("/", 1)[-1]

    def _is_product_url(self, href):
        return (
            bool(re.search(r"/[^/?#]+-\d{13}(?:\?|$)", href)) and "route=" not in href
        )

    def _next_page_url(self, soup, current_url):
        link = soup.select_one("a[rel='next']")
        if link and link.get("href"):
            return urljoin(current_url, str(link.get("href")))
        for a in soup.select("a[href]"):
            label = a.get_text(" ", strip=True)
            if label in {"»", ">", "Következő", "Next"}:
                href = a.get("href")
                if href:
                    return urljoin(current_url, str(href))
        return None

    def _extract_category_total(self, soup):
        text = soup.get_text(" ", strip=True)
        m = re.search(r"Tételek:\s*\d+\s*-\s*\d+\s*/\s*(\d+)", text)
        return int(m.group(1)) if m else None

    def _is_empty_category_page(self, soup):
        text = soup.get_text(" ", strip=True).lower()
        return "nincsenek listázandó termékek ebben a kategóriában" in text

    def _collect_product_urls_worker(self, category_url):
        return self.collect_product_urls(category_url)

    def _parse_product_detail_worker(self, item):
        return self.parse_product_page(item[0], item[1])

    def parse_product_page(self, url, category_name):
        resp = self._fetch_page_with_challenge_retry(url, stage="product_detail")
        if not resp:
            return None

        from scrapers.records import product_record, parse_package, parse_price

        soup = BeautifulSoup(resp.text, "html.parser")
        title = self._text_first(soup, ["h1"]) or self._page_title(soup)
        ean = self._extract_ean(url, soup)
        
        price_val, currency = parse_price(self._extract_price(soup))
        
        packaging_value, packaging_unit = parse_package(title)
        if not packaging_value:
            packaging_value = self._extract_packaging_value(soup)
        if not packaging_unit:
            packaging_unit = self._extract_packaging_unit(soup)

        return product_record(
            source_product_id=ean or self._product_id_from_url(url),
            name=title,
            price=price_val,
            currency=currency,
            image_url=self._extract_image(soup, resp.url),
            category=category_name,
            packaging_value=packaging_value,
            packaging_unit=packaging_unit,
            source_url=url,
            ean_code=ean,
            availability="1" if self._extract_availability(soup) else "0",
            raw=resp.text[:5000],
        )

    def _fetch_page_with_challenge_retry(self, url, *, stage):
        use_cache = stage != "product_detail"
        resp = self.fetch_response_via_backend(
            url,
            purpose=f"franko {stage}",
            timeout=self.timeout,
            retries=self.retries,
            headers=self.headers,
            use_cache=use_cache,
        )
        if not resp:
            # For proxy-heavy runs we want to exhaust more of the leased reservoir,
            # not stop after the backend default retry window.
            reservoir = getattr(self, "proxy_reservoir", None)
            reservoir_size = len(getattr(reservoir, "proxies", []) or [])
            extra_rotate_attempts = max(0, min(12, reservoir_size) - 1)
            for attempt in range(1, extra_rotate_attempts + 1):
                resp = self.fetch_response_via_backend(
                    url,
                    purpose=f"franko {stage} extra rotate #{attempt}",
                    timeout=self.timeout,
                    retries=0,
                    headers=self.headers,
                    use_cache=True,
                )
                if resp:
                    logger.info(
                        "Recovered fetch for %s during %s on extra rotate attempt %d/%d",
                        url,
                        stage,
                        attempt,
                        extra_rotate_attempts,
                    )
                    break
            if not resp:
                return None

        cache_file = self._cache_file_for(url)
        challenge = detect_challenge_page(
            resp.text,
            url=getattr(resp, "url", url),
            status_code=getattr(resp, "status_code", None),
            headers=getattr(resp, "headers", None),
        )
        if not challenge:
            return resp

        logger.warning(
            "Challenge page detected for %s during %s | title=%s | patterns=%s | cache_file=%s | retrying uncached with proxy rotation",
            url,
            stage,
            challenge["title"],
            challenge["matched_patterns"],
            cache_file or "—",
        )

        max_attempts = max(2, int(getattr(self, "max_proxy_failures", 3) or 3))
        if stage == "product_detail":
            max_attempts = max(max_attempts, 5)
        for attempt in range(1, max_attempts + 1):
            resp = self.fetch_response_via_backend(
                url,
                purpose=f"franko {stage} uncached rotate #{attempt}",
                timeout=self.timeout,
                retries=0,
                headers=self.headers,
                use_cache=False,
            )
            if not resp:
                continue

            retry_challenge = detect_challenge_page(
                resp.text,
                url=getattr(resp, "url", url),
                status_code=getattr(resp, "status_code", None),
                headers=getattr(resp, "headers", None),
            )
            if not retry_challenge:
                logger.info(
                    "Challenge cleared for %s during %s on rotate attempt %d/%d",
                    url,
                    stage,
                    attempt,
                    max_attempts,
                )
                return resp

            logger.warning(
                "Challenge persisted for %s during %s on rotate attempt %d/%d | title=%s | patterns=%s",
                url,
                stage,
                attempt,
                max_attempts,
                retry_challenge["title"],
                retry_challenge["matched_patterns"],
            )

        logger.error(
            "Challenge page persisted for %s during %s after %d uncached rotate attempts | cache_file=%s",
            url,
            stage,
            max_attempts,
            cache_file or "—",
        )
        return None

    def _cache_file_for(self, url, params=None):
        cache = getattr(self, "cache", None)
        if not cache or not hasattr(cache, "path_for"):
            return None
        try:
            return cache.path_for(url, params)
        except Exception:
            return None

    def _page_title(self, soup):
        if soup.title:
            return soup.title.get_text(strip=True).replace(" | Frankó Diszkont", "")
        return ""

    def _text_first(self, soup, selectors):
        for selector in selectors:
            node = soup.select_one(selector)
            if node:
                text = node.get_text(" ", strip=True)
                if text:
                    return text
        return ""

    def _extract_price(self, soup):
        for text in soup.stripped_strings:
            text = str(text).replace("\xa0", " ")
            if re.fullmatch(
                r"\d{1,3}(?:[ .]\d{3})*(?:,\d{2})?Ft", text.replace(" ", "")
            ):
                return text.replace("Ft", "").strip()
        price_node = soup.find(string=re.compile(r"\d+\s*Ft"))
        return str(price_node).strip().replace("Ft", "").strip() if price_node else ""

    def _extract_packaging_value(self, soup):
        text = soup.get_text(" ", strip=True)
        m = re.search(r"Kiszerelés\s+([\d.,]+)\s*[A-Za-zÁÉÍÓÖŐÚÜŰ]+", text, re.I)
        return m.group(1).replace(",", ".") if m else ""

    def _extract_packaging_unit(self, soup):
        text = soup.get_text(" ", strip=True)
        m = re.search(r"Kiszerelés\s+[\d.,]+\s*([A-Za-zÁÉÍÓÖŐÚÜŰ]+)", text, re.I)
        return m.group(1).lower() if m else ""

    def _extract_image(self, soup, page_url):
        og = soup.find("meta", attrs={"property": "og:image"})
        if og and og.get("content"):
            return urljoin(page_url, str(og["content"]))
        img = soup.select_one("img")
        return urljoin(page_url, str(img["src"])) if img and img.get("src") else ""

    def _extract_availability(self, soup):
        text = soup.get_text(" ", strip=True).lower()
        if "nincs készleten" in text:
            return False
        if "készleten" in text:
            return True
        return True

    def _product_id_from_url(self, url):
        slug = urlparse(url).path.rstrip("/").rsplit("/", 1)[-1]
        m = re.search(r"(\d{8,14})$", slug)
        return m.group(1) if m else slug

    def _extract_ean(self, url, soup):
        slug = urlparse(url).path.rstrip("/").rsplit("/", 1)[-1]
        m = re.search(r"(\d{13})$", slug)
        if m:
            return m.group(1)
        text = soup.get_text(" ", strip=True)
        m = re.search(r"Cikkszám:\s*(\d{13})", text)
        if m:
            return m.group(1)
        m = re.search(r"(\d{13})", text)
        return m.group(1) if m else ""