Scraper Source: coopshop_scraper
Read-only source view.
coopshop_scraper.py · 6840 bytes
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 | import logging import threading import time from typing import Any, Dict, List from scrapers.base_scraper import BaseScraper, ScrapeRunResult from scrapers.common import PRODUCT_FIELDNAMES, ProxyPool, ResponseCache, ThreadLocalSessionFactory from scrapers.exceptions import ScraperAbortError from scrapers.options import RuntimeOptions logger = logging.getLogger(__name__) class Scraper(BaseScraper): execution_mode = "api" store_definitions = ( { "retailer_code": "coopshop-hu", "name": "CoopShop Hungary", "country_code": "HU", "base_url": "https://coopshop.hu", "scraper_module": "scrapers.platform.coopshop_scraper", "scraper_mapping": { "execution_mode": "api", "output_schema": "product", }, }, ) def __init__(self, workers=10, proxy_list=None, retries=5, timeout=20, max_proxy_failures=3): super().__init__() self.workers = workers self.retries = retries self.timeout = timeout self.max_proxy_failures = max_proxy_failures self.base_api_url = "https://coopshop.hu/wp-json/wc/store/v1/products" self.proxy_validation_url = "https://coopshop.hu/" self.headers = {} if hasattr(self, "configure_http_transport"): self.configure_http_transport( headers=self.headers, proxy_list=proxy_list, max_proxy_failures=max_proxy_failures, ) else: self.cache = ResponseCache(ttl=43200) self.proxy_pool = ProxyPool(proxy_list, max_failures=max_proxy_failures) self.http_proxies_enabled = True self.sessions = ThreadLocalSessionFactory(self.headers) self._request_rate_lock = threading.Lock() self._last_request_at = 0.0 self.fieldnames = PRODUCT_FIELDNAMES def get_total_pages(self): logger.info("Fetching CoopShop total page count...") resp = self.fetch_response( self.base_api_url, params={"per_page": 100}, retries=self.retries, timeout=self.timeout, retry_sleep=lambda attempt: time.sleep(2 * attempt), ) if resp is None or resp.status_code != 200: logger.error("Failed to fetch total page count after %d attempts: %s", self.retries, self.base_api_url) return None total_pages = int(resp.headers.get("X-WP-TotalPages", 1)) logger.info("CoopShop reports %d product pages.", total_pages) return total_pages def parse_product(self, item): try: from scrapers.records import product_record, parse_package, parse_price, calculate_discount name = item.get('name', '').strip() prices = item.get('prices', {}) price_val, currency = parse_price(prices.get('price')) regular_val, _ = parse_price(prices.get('regular_price')) old_price = "" discount_percent = "" if item.get('on_sale'): old_price = regular_val discount_percent = calculate_discount(price_val, regular_val) image = "" if item.get('images') and len(item['images']) > 0: image = item['images'][0].get('src', '') categories = " > ".join([c.get('name', '') for c in item.get('categories', [])]) packaging_value, packaging_unit = parse_package(name) return product_record( source_product_id=str(item.get('id', '')), name=name, price=price_val, old_price=old_price, currency=currency, discount_percent=discount_percent, image_url=image, category=categories, packaging_value=packaging_value, packaging_unit=packaging_unit, source_url=item.get('permalink', ''), ean_code=item.get('sku', ''), availability="1", # WC Store API returns available products by default raw=item, ) except Exception as e: logger.debug("Product parse skipped for id=%s: %s", item.get("id"), e) return None def scrape_page(self, page): resp = self.fetch_response( self.base_api_url, params={"per_page": 100, "page": page}, retries=self.retries, timeout=self.timeout, retry_sleep=lambda attempt: time.sleep(2 * attempt), ) if resp is None or resp.status_code != 200: logger.debug("CoopShop page fetch failed: page=%d", page) return [] try: items = resp.json() products = [] for item in items: p = self.parse_product(item) if p: products.append(p) return products except Exception as e: logger.error("Error decoding JSON on page %d: %s", page, e) return [] def run(self, target: Any = None, context: Dict[str, Any] | None = None) -> ScrapeRunResult: opts = RuntimeOptions.from_run_args(target, context) records = self.collect_products( limit_categories=opts.limit_categories, category_filter=opts.category_filter, ) return ScrapeRunResult( records=records, raw_records=records, accepted_records=records, rejected_records=[], telemetry=self.telemetry, ) def collect_products(self, limit_categories=None, category_filter=None): self.log_startup_status(logger, "Starting CoopShop crawl") total_pages = self.get_total_pages() if not total_pages: raise ScraperAbortError( "CoopShop: could not determine total pages — API may be unreachable or changed", error_category="network", ) pages = list(range(1, total_pages + 1)) if limit_categories: pages = pages[:limit_categories] all_products = self.parallel_map( self.scrape_page, pages, label="CoopShop pages", unit="pages" ) seen_pids = set() unique_products = [] for product in all_products: pid = product.get("source_product_id") if pid not in seen_pids: seen_pids.add(pid) unique_products.append(product) self.log_summary(logger, f"Crawl complete, found {len(unique_products)} unique products") return unique_products CoopShopScraper = Scraper |