foreignaidquerying/fetch_real_data.py at main · MaxFritzhand/foreignaidquerying · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
import requests
import json
import time

# REAL DATA FETCHING from World Bank API
# We will fetch "Net bilateral aid flows from DAC donors"
# Indicators: DC.DAC.USAL.CD (US), DC.DAC.DEUL.CD (Germany), etc.
# We will iterate through top donors and fetch their flows to ALL recipients.

DONOR_INDICATORS = {
    "United States": {"indicator": "DC.DAC.USAL.CD", "iso2": "US"},
    "Germany": {"indicator": "DC.DAC.DEUL.CD", "iso2": "DE"},
    "United Kingdom": {"indicator": "DC.DAC.GBRL.CD", "iso2": "GB"},
    "Japan": {"indicator": "DC.DAC.JPNL.CD", "iso2": "JP"},
    "France": {"indicator": "DC.DAC.FRAL.CD", "iso2": "FR"},
    "Canada": {"indicator": "DC.DAC.CANL.CD", "iso2": "CA"},
    "Netherlands": {"indicator": "DC.DAC.NLDL.CD", "iso2": "NL"},
    "Norway": {"indicator": "DC.DAC.NORL.CD", "iso2": "NO"},
    "Australia": {"indicator": "DC.DAC.AUSL.CD", "iso2": "AU"},
    "Switzerland": {"indicator": "DC.DAC.CHEL.CD", "iso2": "CH"},
    "Italy": {"indicator": "DC.DAC.ITAL.CD", "iso2": "IT"},
}

# Mapping of WB Country IDs or Names to clean names if needed.
# The API returns "country": { "id": "AF", "value": "Afghanistan" } so we are good.

def fetch_world_bank_data():
    print("Fetching REAL DATA from World Bank API (Bilateral Flows)...")

    all_records = []
    # We'll fetch 2022, 2023, and attempt 2024
    years = ["2022", "2023", "2024"]

    for year in years:
        print(f"--- Fetching for Year {year} ---")
        for donor_name, info in DONOR_INDICATORS.items():
            indicator = info["indicator"]
            donor_iso = info["iso2"]
            print(f"Fetching flows for {donor_name} ({indicator}) in {year}...")

            url = f"https://api.worldbank.org/v2/country/all/indicator/{indicator}"
            params = {
                "format": "json",
                "date": year,
                "per_page": 500
            }

            try:
                response = requests.get(url, params=params)
                if response.status_code != 200:
                    continue

                data = response.json()
                if len(data) < 2 or not data[1]:
                    continue

                observations = data[1]
                for obs in observations:
                    country_info = obs.get("country", {})
                    recipient = country_info.get("value")
                    recipient_iso = country_info.get("id")
                    amount = obs.get("value")

                    if recipient in ["World", "High income", "Low income", "LMY", "MIC", "HIC", "WLD", "IDA", "IBRD", "Euro area"]: continue
                    if "Sub-Saharan Africa" in recipient or "South Asia" in recipient or "Latin America" in recipient: continue

                    if amount is not None and amount > 0:
                        all_records.append({
                            "id": f"WB-{donor_name}-{recipient_iso}-{year}",
                            "donor": donor_name,
                            "donor_iso2": donor_iso,
                            "recipient": recipient,
                            "recipient_iso2": recipient_iso,
                            "year": int(year),
                            "date": f"{year}-12-31",
                            "amount": float(amount),
                            "sector_category": "Official Development Assistance",
                            "project_name": "Net Bilateral Aid Flow"
                        })

                print(f"  Got {len(observations)} potential records for {donor_name} in {year}")
                time.sleep(0.1)

            except Exception as e:
                print(f"Error fetching {donor_name} for {year}: {e}")

    return all_records

if __name__ == "__main__":
    data = fetch_world_bank_data()
    if len(data) > 0:
        with open("src/data/aid_data.json", "w") as f:
            json.dump(data, f, indent=2)
        print(f"Successfully saved {len(data)} REAL records to src/data/aid_data.json")
    else:
        print("Failed to fetch real data.")