ThirdwebDocsScraper/run_improved_scraper.sh at main · cobibean/ThirdwebDocsScraper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
#!/bin/bash
# Script to run the improved Thirdweb TypeScript docs scraper

# Set script to exit on error
set -e

# Print colored messages
print_green() {
    echo -e "\033[0;32m$1\033[0m"
}

print_blue() {
    echo -e "\033[0;34m$1\033[0m"
}

print_red() {
    echo -e "\033[0;31m$1\033[0m"
}

# Create and activate virtual environment
print_blue "Setting up virtual environment..."
python3 -m venv venv
source venv/bin/activate

# Install required packages
print_blue "Installing required packages..."
pip install requests beautifulsoup4

# Create output directory
print_blue "Creating output directory..."
mkdir -p thirdweb_typescript_docs

# Run the improved scraper
print_green "Starting the improved Thirdweb TypeScript documentation scraper..."
python improved_scraper.py

# Run the reorganization script
print_green "Organizing documentation into categories..."
python reorganize_docs.py

print_green "Scraping and organization completed!"
print_blue "Documentation has been saved and organized in the thirdweb_typescript_docs directory."
print_blue "Categories: UI Components, React Hooks, Core Functions, Advanced Topics"

# Deactivate virtual environment
deactivate