from __future__ import annotations

import logging
import re
import threading
import time
from typing import Any, List, Dict, Optional, Mapping, Set, Tuple, Callable
from urllib.parse import urljoin, urlparse


from scrapers.base_scraper import BaseScraper, ScrapeRunResult
from scrapers.common import ProxyPool, ResponseCache, ThreadLocalSessionFactory
from scrapers.exceptions import ScraperAbortError
from scrapers.options import RuntimeOptions
from scrapers.schemas import PARTNER_SCHEMA


logger = logging.getLogger(__name__)

try:
    from playwright.sync_api import sync_playwright
except ImportError:  # pragma: no cover
    sync_playwright: Any = None


class Scraper(BaseScraper):
    default_request_backend = "scrapling"
    # Common feed paths worth probing for Árukereső partners.
    # Keep this list here so future inventory/validation jobs can reuse it.
    common_feed_paths = (
        "/arukereso.xml",
        "/feed/arukereso.xml",
        "/feeds/arukereso.xml",
        "/feed.xml",
        "/arukereso-feed.xml",
        "/xml/arukereso.xml",
    )
    @property
    def user_agent(self):
        from scrapers.user_agents import get_random_user_agent
        return get_random_user_agent()

    @property
    def http_headers(self):
        return {
            "User-Agent": self.user_agent,
            "Accept-Language": "hu-HU,hu;q=0.9,en;q=0.8",
        }

    schema = PARTNER_SCHEMA

    store_definitions = (
        {
            "retailer_code": "arukereso-partners",
            "name": "Árukereső partner directory",
            "country_code": "HU",
            "base_url": "https://www.arukereso.hu/stores/",
            "scraper_module": "scrapers.platform.arukereso_partners_scraper",
            "scraper_mapping": {"execution_mode": "browser", "output_schema": "partner"},
            "validation_url": "https://www.arukereso.hu/stores/",
        },
    )

    def __init__(self, headless: bool = True, max_pages: int = 200):
        super().__init__()
        self.headless = headless
        self.max_pages = max_pages
        self.proxy_validation_url = "https://www.arukereso.hu/"
        if hasattr(self, "configure_http_transport"):
            self.configure_http_transport(headers=self.http_headers)
        else:
            self.cache = ResponseCache(ttl=43200)
            self.proxy_pool = ProxyPool(None, max_failures=3)
            self.http_proxies_enabled = True
            self.sessions = ThreadLocalSessionFactory(self.http_headers)
            self._request_rate_lock = threading.Lock()
            self._last_request_at = 0.0

    def run(self, target: Any, context: Dict[str, Any] | None) -> ScrapeRunResult:
        opts = RuntimeOptions.from_run_args(target, context)
        workers = opts.workers or 5
        
        context = dict(context or {})
        context.setdefault("common_feed_paths", self.common_feed_paths)

        if isinstance(target, dict):
            start_url = target.get("start_url") or "https://www.arukereso.hu/stores/"
            if "max_pages" in target:
                self.max_pages = int(target["max_pages"])
        else:
            start_url = str(
                target or context.get("start_url") or "https://www.arukereso.hu/stores/"
            )

        self.log_startup_status(
            logger,
            f"Starting Árukereső partner scrape: start_url={start_url} max_pages={self.max_pages} workers={workers}",
        )
        
        store_urls = self.collect_store_urls(start_url)
        logger.info("Collected %d store URLs from %s", len(store_urls), start_url)

        if not store_urls:
            raise ScraperAbortError(
                "Árukereső: both HTTP and browser URL collection failed — "
                "site may be blocking or changed structure",
                error_category="network",
            )

        max_items = opts.max_items
        if max_items:
            store_urls = store_urls[:int(max_items)]
            logger.info("Limited to %d store URLs", len(store_urls))

        # Phase 2: Parallel fetch of store details
        records = self.parallel_map(
            self.fetch_store_details,
            store_urls,
            workers=workers,
            label="Árukereső stores",
            unit="stores"
        )
        
        self.log_summary(
            logger,
            "Árukereső partner scrape complete",
            total=len(records),
        )
        return ScrapeRunResult(
            records=records,
            raw_records=records,
            accepted_records=records,
            rejected_records=[],
            telemetry=self.telemetry,
        )

    def collect_store_urls(self, start_url: str) -> list[str]:
        # Always try HTTP first as it is much faster and easier to retry
        urls = self.collect_store_urls_http(start_url)
        if urls:
            logger.info("HTTP collection successful: %d store URLs found", len(urls))
            return urls

        if sync_playwright is None:
            logger.warning("HTTP collection failed and Playwright is not installed.")
            return []

        logger.info("HTTP collection failed or returned no results, falling back to browser")
        
        for attempt in range(1, 4):
            # Ensure reservoir is full BEFORE starting browser, to avoid DB access inside playwright
            if self.proxy_reservoir:
                try:
                    self.proxy_reservoir.refill()
                except Exception:
                    logger.debug("Failed to refill proxy reservoir between attempts", exc_info=True)

            try:
                return self._collect_store_urls_browser(start_url)
            except Exception as exc:
                logger.info("Browser collection attempt %d failed: %s", attempt, exc)
                if attempt < 3:
                    time.sleep(2)
                else:
                    logger.error("All browser collection attempts failed after %d tries", attempt)
        
        return []

    def _collect_store_urls_browser(self, start_url: str) -> list[str]:
        urls: list[str] = []
        seen = set()
        logger.info("Collecting store URLs via browser (max %d pages)", self.max_pages)
        
        self.report_work_unit("pages")
        self.report_work_total(self.max_pages)
        
        # We don't call self.launch_browser here because it might trigger refill if reservoir is empty
        # and we want to be 100% sure no DB access happens inside. 
        # But wait, launch_browser calls _get_playwright_proxy which just gets from _active list.
        # It's mark_failure that calls refill.
        
        from catalog.live_snapshot import safe_db_access
        with safe_db_access():
            with sync_playwright() as p:
                # We pass a specific proxy to avoid automatic refill if possible, 
                # though launch_browser is generally safe if refill is not called.
                browser = self.launch_browser(p)
                page = browser.new_page(user_agent=self.user_agent)
                try:
                    for page_num in range(1, self.max_pages + 1):
                        url = (
                            start_url
                            if page_num == 1
                            else self.with_page_param(start_url, page_num)
                        )
                        logger.info("Loading store listing page %d: %s", page_num, url)
                        try:
                            start = time.perf_counter()
                            # Try domcontentloaded first, but fallback if it seems to hang
                            page.goto(url, wait_until="domcontentloaded", timeout=45000)
                            self.record_request((time.perf_counter() - start) * 1000)
                        except Exception as exc:
                            logger.debug(
                                "Navigation failed on page %d (%s)",
                                page_num,
                                exc,
                            )
                            # If page 1 fails, we raise to trigger a retry with a new browser/proxy
                            if page_num == 1:
                                raise
                            break
                        
                        # Wait a bit for JS to populate if needed
                        page.wait_for_timeout(2000)
                        
                        links = page.eval_on_selector_all(
                            'a[href*="/stores/"]',
                            "els => els.map(a => a.href)",
                        )
                        new_found = 0
                        for link in links or []:
                            normalized = self.normalize_store_url(str(link))
                            if normalized and normalized not in seen:
                                seen.add(normalized)
                                urls.append(normalized)
                                new_found += 1
                        
                        logger.info(
                            "Page %d yielded %d new store URLs (%d total)",
                            page_num,
                            new_found,
                            len(urls),
                        )
                        
                        if new_found == 0:
                            # Check if there is a next page button or if we are truly at the end
                            if page.locator(".next-page, .pagination-next").count() == 0:
                                logger.debug("No next page button and no new URLs on page %d, stopping", page_num)
                                self.report_work_completed(self.max_pages - page_num)
                                break
                            else:
                                logger.debug("Found next page button but no new URLs on page %d, continuing...", page_num)
                        
                        self.report_work_completed(1)
                finally:
                    browser.close()
        
        logger.info("Browser collection complete: %d store URLs found", len(urls))
        return urls

    def collect_store_urls_http(self, start_url: str) -> list[str]:
        urls: list[str] = []
        seen = set()
        logger.info("Collecting store URLs via HTTP (max %d pages)", self.max_pages)

        self.report_work_unit("pages")
        self.report_work_total(self.max_pages)
        for page_num in range(1, self.max_pages + 1):
            url = (
                start_url
                if page_num == 1
                else self.with_page_param(start_url, page_num)
            )
            logger.info("Loading store listing page %d: %s", page_num, url)
            resp = self.fetch_response_via_backend(
                url,
                purpose=f"listing page {page_num}",
                timeout=30,
                headers=self.http_headers,
                scrapling_stealth=False,
            )
            if resp is None:
                logger.debug("HTTP request failed on page %d (listing stop)", page_num)
                break
            status_code = getattr(resp, "status_code", getattr(resp, "status", None))
            if status_code != 200:
                logger.debug("HTTP %d on page %d (listing stop)", status_code, page_num)
                break

            # Use Scrapling Adaptor instead of BeautifulSoup
            page = self.adapt(resp)
            links = page.css('a[href*="/stores/"]::attr(href)').getall()
            new_found = 0
            for link in links:
                if not link:
                    continue
                normalized = self.normalize_store_url(urljoin(url, str(link)))
                if normalized and normalized not in seen:
                    seen.add(normalized)
                    urls.append(normalized)
                    new_found += 1
            logger.info(
                "Page %d yielded %d new store URLs (%d total)",
                page_num,
                new_found,
                len(urls),
            )
            if new_found == 0:
                logger.debug("No new store URLs on page %d, stopping", page_num)
                self.report_work_completed(self.max_pages - page_num)
                break
            self.report_work_completed(1)
        logger.info("HTTP collection complete: %d store URLs found", len(urls))
        return urls

    def fetch_store_details(self, store_url: str) -> dict[str, Any] | None:
        # Use fetch_scrapling for TLS impersonation and adapt() for extraction
        resp = self.fetch_response_via_backend(
            store_url,
            purpose="store detail",
            timeout=30,
            headers=self.http_headers,
            scrapling_stealth=False,
        )
        if not resp:
            logger.debug("Failed to fetch store page: %s", store_url)
            return None

        _status = getattr(resp, "status", getattr(resp, "status_code", "?"))
        _raw = resp.text or ""
        _html = _raw.decode("utf-8", errors="replace") if isinstance(_raw, bytes) else _raw
        _final_url = getattr(resp, "url", store_url)
        _ct = ""
        _headers = getattr(resp, "headers", None)
        if _headers:
            _ct = _headers.get("content-type", _headers.get("Content-Type", ""))
        logger.debug(
            "store detail response | status=%s size=%d ct=%s final_url=%s",
            _status, len(_html), _ct or "-", _final_url,
        )
        if "informacio" not in _html and "Cégnév" not in _html:
            logger.debug(
                "store detail markers absent | status=%s size=%d url=%s — skipping (likely block)",
                _status, len(_html), store_url,
            )
            return None
        logger.debug("store detail httpx hit | status=%s size=%d url=%s", _status, len(_html), store_url)
        page = self.adapt(resp)


        title_node = page.css("title") if page else []
        title = title_node[0].text if title_node else ""
        info_text = self.extract_info_text(page)
        fields = self.parse_info_text(info_text)

        name = fields.get("cegnev") or self.derive_name_from_title(title, store_url)
        domain = self.extract_domain(fields.get("web") or "")
        store_id = self.extract_store_id(store_url)

        return {
            "id": store_id,
            "name": name,
            "url": store_url,
            "domain": domain,
            "cegnev": fields.get("cegnev", ""),
            "cim": fields.get("cim", ""),
            "email": fields.get("email", ""),
            "tel": fields.get("tel", ""),
            "alapitas_eve": fields.get("alapitas_eve", ""),
            "raw": {
                "title": title,
                "info_text": info_text,
                "fields": fields,
            },
        }

    def extract_info_text(self, page: Any) -> str:
        if page is None:
            return ""
        nodes = page.css("#informacio")
        node = nodes[0] if nodes else None
        if node:
            return node.get_all_text(separator="\n").strip()
        return page.get_all_text(separator="\n").strip()

    def parse_info_text(self, text: str) -> dict[str, str]:
        fields: dict[str, str] = {}
        lines = [line.strip() for line in text.splitlines() if line.strip()]
        field_specs = (
            ("cegnev", "Cégnév", False),
            ("alapitas_eve", "Alapítás éve", False),
            ("cim", "Cím", True),
            ("web", "Web", False),
            ("email", "Email", False),
            ("tel", "Tel.", False),
        )
        for key, label, multi_line in field_specs:
            value = self.extract_labeled_value(lines, label, multi_line=multi_line)
            if value:
                fields[key] = value

        found = list(fields.keys())
        missing = [key for key, _, _ in field_specs if key not in fields]
        logger.debug("Parsed info fields — found: %s, missing: %s", found, missing)
        return fields

    def extract_labeled_value(self, lines: list[str], label: str, *, multi_line: bool = False) -> str:
        label_norm = self.normalize_label(label)
        for index, line in enumerate(lines):
            normalized_line = self.normalize_label(line)
            if normalized_line != label_norm and not normalized_line.startswith(label_norm):
                continue

            value = self._value_after_label(line, label)
            if multi_line:
                collected = [value] if value else []
                for following in lines[index + 1 :]:
                    if self._looks_like_label(following):
                        break
                    if following in {",", ".", "-", "–", "—"}:
                        continue
                    collected.append(following)
                combined = " ".join(part for part in collected if part).strip()
                combined = re.sub(r"\s*,\s*", ", ", combined)
                return combined.strip(" ,")

            if value:
                return value.strip(" ,")
            for following in lines[index + 1 :]:
                if self._looks_like_label(following):
                    break
                if following:
                    return following.strip(" ,")
        return ""

    @staticmethod
    def normalize_label(text: str) -> str:
        return re.sub(r"[\W_]+", "", text.casefold())

    @classmethod
    def _looks_like_label(cls, text: str) -> bool:
        normalized = cls.normalize_label(text)
        return normalized in {
            cls.normalize_label("Cégnév"),
            cls.normalize_label("Alapítás éve"),
            cls.normalize_label("Cím"),
            cls.normalize_label("Web"),
            cls.normalize_label("Email"),
            cls.normalize_label("Tel."),
            cls.normalize_label("Facebook"),
            cls.normalize_label("Információ"),
            cls.normalize_label("Vélemények"),
            cls.normalize_label("Üzletek és átvételi pontok"),
            cls.normalize_label("Térkép"),
            cls.normalize_label("Bemutatkozás"),
        }

    @staticmethod
    def _value_after_label(line: str, label: str) -> str:
        pattern = re.compile(
            rf"^{re.escape(label)}\s*:?\s*(.*)$",
            flags=re.IGNORECASE,
        )
        match = pattern.match(line)
        if not match:
            return ""
        return match.group(1).strip()

    def derive_name_from_title(self, title: str, url: str) -> str:
        if title:
            return title.split(" - Árak")[0].strip()
        slug = self.extract_store_slug(url)
        return slug.replace("-", " ").strip().title()

    def extract_domain(self, web: str) -> str:
        if not web:
            return ""
        parsed = urlparse(web if web.startswith("http") else f"https://{web}")
        return parsed.netloc or web

    def extract_store_id(self, url: str) -> str:
        slug = self.extract_store_slug(url)
        return slug or url

    def extract_store_slug(self, url: str) -> str:
        path = urlparse(url).path.rstrip("/")
        return path.split("/")[-1] if path else url

    def normalize_store_url(self, url: str | None) -> str | None:
        if not url:
            return None
        if "/stores/" not in url:
            return None
        return str(url).split("#")[0].rstrip("/") + "/"

    def with_page_param(self, url: str, page_num: int) -> str:
        joiner = "&" if "?" in url else "?"
        return f"{url}{joiner}page={page_num}"

    @classmethod
    def candidate_feed_urls(cls, website_url: str) -> list[str]:
        base = website_url.rstrip("/")
        return [f"{base}{path}" for path in cls.common_feed_paths]