BlocketScraper/config.py at main · kami4ka/BlocketScraper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
"""Configuration settings for Blocket scraper."""

# Base URL for blocket.se
BASE_URL = "https://www.blocket.se"

# ScrapingAnt API settings
SCRAPINGANT_API_URL = "https://api.scrapingant.com/v2/general"

# Request configuration
REQUEST_CONFIG = {
    "browser": True,
    "proxy_country": "DE",  # Germany (Swedish proxy not available)
    "wait_for_selector": "article.sf-search-ad",
}

# Available categories for scraping
CATEGORIES = {
    # Electronics
    "electronics": "category=0.93",
    "phones": "sub_category=1.93.3217",
    "computers": "sub_category=1.93.3215",
    "photo-video": "sub_category=1.93.3904",
    "audio-video": "sub_category=1.93.3906",
    "gaming": "sub_category=1.93.3905",
    "appliances": "sub_category=1.93.3907",

    # Furniture
    "furniture": "category=0.78",
    "tables-chairs": "sub_category=1.78.5196",
    "sofas": "sub_category=1.78.7756",
    "beds": "sub_category=1.78.5197",
    "lamps": "sub_category=1.78.5181",

    # Fashion
    "fashion": "category=0.71",
    "womens-clothing": "sub_category=1.71.3941",
    "mens-clothing": "sub_category=1.71.3950",
    "shoes": "sub_category=1.71.3949",
    "watches": "sub_category=1.71.3945",

    # Sports
    "sports": "category=0.69",
    "bikes": "sub_category=1.69.3963",
    "winter-sports": "sub_category=1.69.3962",
    "golf": "sub_category=1.69.5164",

    # Home & Garden
    "garden": "category=0.67",
    "tools": "sub_category=1.67.5219",

    # Kids
    "kids": "category=0.68",
    "toys": "sub_category=1.68.3912",
    "strollers": "sub_category=1.68.3914",

    # Hobby
    "hobby": "category=0.86",
    "music-instruments": "sub_category=1.86.92",
    "collectibles": "sub_category=1.86.285",
}

# Default categories to scrape
DEFAULT_CATEGORIES = ["phones", "electronics"]

# CSS selectors for parsing
SELECTORS = {
    # Main listing container
    "listing": "article.sf-search-ad",
    # Alternative selector
    "listing_alt": "article",
    # Link inside listing
    "link": "a",
    # Title
    "title": "h3",
    # Image
    "image": "img",
}

# Output settings
OUTPUT_DIR = "output"
DEFAULT_CSV_FILE = "blocket_listings.csv"
DEFAULT_JSON_FILE = "blocket_listings.json"