acquire_all_data.py

"""
Comprehensive Data Acquisition Script

This script fetches data for all topics in the Guyana Development Analysis project.
It systematically downloads data from various APIs and saves them to appropriate directories.
"""

import pandas as pd
import numpy as np
import os
import sys
import time
from datetime import datetime
import requests

# Add src to path
sys.path.append('src')

from data_fetchers import (
    fetch_worldbank_data,
    fetch_gbif_species,
    load_or_fetch,
    calculate_cagr
)

# Setup directories
BASE_DIR = os.getcwd()
RAW_DATA_DIR = os.path.join(BASE_DIR, 'data', 'raw')
ECONOMIC_DIR = os.path.join(RAW_DATA_DIR, 'economic')
SOCIAL_DIR = os.path.join(RAW_DATA_DIR, 'social')
RESOURCES_DIR = os.path.join(RAW_DATA_DIR, 'resources')
ENV_DIR = os.path.join(RAW_DATA_DIR, 'environmental')
METADATA_DIR = os.path.join(BASE_DIR, 'data', 'metadata')

# Create all directories
for dir_path in [ECONOMIC_DIR, SOCIAL_DIR, RESOURCES_DIR, ENV_DIR, METADATA_DIR]:
    os.makedirs(dir_path, exist_ok=True)

print("=" * 80)
print("GUYANA DEVELOPMENT ANALYSIS - COMPREHENSIVE DATA ACQUISITION")
print("=" * 80)
print(f"\nStarted: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
print()

# Track data sources for metadata
data_sources = []

# ============================================================================
# PART 1: ECONOMIC INDICATORS
# ============================================================================

print("\n" + "=" * 80)
print("PART 1: ECONOMIC INDICATORS")
print("=" * 80)

# 1.1 GDP and National Accounts
print("\n[1/12] Fetching GDP and National Accounts...")
print("-" * 80)

gdp_indicators = {
    'NY.GDP.MKTP.CD': 'gdp_current_usd',
    'NY.GDP.PCAP.CD': 'gdp_per_capita_current',
    'NY.GDP.MKTP.KD': 'gdp_constant_2015',
    'NY.GDP.PCAP.KD': 'gdp_per_capita_constant',
    'NY.GDP.MKTP.KD.ZG': 'gdp_growth_annual',
    'NE.CON.TOTL.KD': 'final_consumption_constant',
    'NE.GDI.TOTL.KD': 'gross_capital_formation',
    'NE.EXP.GNFS.KD': 'exports_goods_services',
    'NE.IMP.GNFS.KD': 'imports_goods_services'
}

try:
    gdp_df = load_or_fetch(
        os.path.join(ECONOMIC_DIR, 'gdp_national_accounts.csv'),
        fetch_worldbank_data,
        indicators=gdp_indicators,
        country='GUY',
        start_year=1960
    )
    print(f"✓ GDP data: {len(gdp_df)} records ({gdp_df['year'].min()}-{gdp_df['year'].max()})")
    data_sources.append({
        'dataset': 'GDP and National Accounts',
        'source': 'World Bank',
        'url': 'https://data.worldbank.org',
        'date': datetime.now().strftime('%Y-%m-%d'),
        'records': len(gdp_df)
    })
except Exception as e:
    print(f"✗ Error fetching GDP data: {e}")

# 1.2 Trade and Exports
print("\n[2/12] Fetching Trade Data...")
print("-" * 80)

trade_indicators = {
    'NE.EXP.GNFS.ZS': 'exports_percent_gdp',
    'NE.IMP.GNFS.ZS': 'imports_percent_gdp',
    'NE.TRD.GNFS.ZS': 'trade_percent_gdp',
    'TX.VAL.MRCH.CD.WT': 'merchandise_exports_current',
    'TM.VAL.MRCH.CD.WT': 'merchandise_imports_current',
    'BN.CAB.XOKA.CD': 'current_account_balance',
    'BN.CAB.XOKA.GD.ZS': 'current_account_percent_gdp'
}

try:
    trade_df = load_or_fetch(
        os.path.join(ECONOMIC_DIR, 'trade_balance.csv'),
        fetch_worldbank_data,
        indicators=trade_indicators,
        country='GUY',
        start_year=1960
    )
    print(f"✓ Trade data: {len(trade_df)} records")
    data_sources.append({
        'dataset': 'Trade and Balance of Payments',
        'source': 'World Bank',
        'url': 'https://data.worldbank.org',
        'date': datetime.now().strftime('%Y-%m-%d'),
        'records': len(trade_df)
    })
except Exception as e:
    print(f"✗ Error fetching trade data: {e}")

# 1.3 Inflation and Prices
print("\n[3/12] Fetching Inflation and Price Data...")
print("-" * 80)

price_indicators = {
    'FP.CPI.TOTL.ZG': 'inflation_cpi',
    'FP.CPI.TOTL': 'cpi_index',
    'PA.NUS.FCRF': 'exchange_rate_lcu_per_usd',
    'NY.GDP.DEFL.ZS': 'gdp_deflator_change'
}

try:
    price_df = load_or_fetch(
        os.path.join(ECONOMIC_DIR, 'inflation_prices.csv'),
        fetch_worldbank_data,
        indicators=price_indicators,
        country='GUY',
        start_year=1960
    )
    print(f"✓ Inflation data: {len(price_df)} records")
    data_sources.append({
        'dataset': 'Inflation and Prices',
        'source': 'World Bank',
        'url': 'https://data.worldbank.org',
        'date': datetime.now().strftime('%Y-%m-%d'),
        'records': len(price_df)
    })
except Exception as e:
    print(f"✗ Error fetching inflation data: {e}")

# ============================================================================
# PART 2: SOCIAL INDICATORS
# ============================================================================

print("\n" + "=" * 80)
print("PART 2: SOCIAL INDICATORS")
print("=" * 80)

# 2.1 Population and Demographics
print("\n[4/12] Fetching Population Data...")
print("-" * 80)

pop_indicators = {
    'SP.POP.TOTL': 'population_total',
    'SP.POP.GROW': 'population_growth',
    'SP.URB.TOTL.IN.ZS': 'urban_population_percent',
    'SP.POP.DPND': 'age_dependency_ratio',
    'SP.POP.65UP.TO.ZS': 'population_65_plus',
    'SP.POP.0014.TO.ZS': 'population_0_14',
    'SP.DYN.LE00.IN': 'life_expectancy',
    'SP.DYN.TFRT.IN': 'fertility_rate'
}

try:
    pop_df = load_or_fetch(
        os.path.join(SOCIAL_DIR, 'population_demographics.csv'),
        fetch_worldbank_data,
        indicators=pop_indicators,
        country='GUY',
        start_year=1960
    )
    print(f"✓ Population data: {len(pop_df)} records")
    data_sources.append({
        'dataset': 'Population and Demographics',
        'source': 'World Bank',
        'url': 'https://data.worldbank.org',
        'date': datetime.now().strftime('%Y-%m-%d'),
        'records': len(pop_df)
    })
except Exception as e:
    print(f"✗ Error fetching population data: {e}")

# 2.2 Education and Literacy
print("\n[5/12] Fetching Education Data...")
print("-" * 80)

edu_indicators = {
    'SE.ADT.LITR.ZS': 'literacy_rate_adult',
    'SE.ADT.1524.LT.ZS': 'literacy_rate_youth',
    'SE.ADT.LITR.FE.ZS': 'literacy_rate_female',
    'SE.ADT.LITR.MA.ZS': 'literacy_rate_male',
    'SE.PRM.ENRR': 'school_enrollment_primary',
    'SE.SEC.ENRR': 'school_enrollment_secondary',
    'SE.TER.ENRR': 'school_enrollment_tertiary',
    'SE.XPD.TOTL.GD.ZS': 'education_expenditure_percent_gdp'
}

try:
    edu_df = load_or_fetch(
        os.path.join(SOCIAL_DIR, 'education_literacy.csv'),
        fetch_worldbank_data,
        indicators=edu_indicators,
        country='GUY',
        start_year=1960
    )
    print(f"✓ Education data: {len(edu_df)} records")
    data_sources.append({
        'dataset': 'Education and Literacy',
        'source': 'World Bank',
        'url': 'https://data.worldbank.org',
        'date': datetime.now().strftime('%Y-%m-%d'),
        'records': len(edu_df)
    })
except Exception as e:
    print(f"✗ Error fetching education data: {e}")

# 2.3 Health
print("\n[6/12] Fetching Health Data...")
print("-" * 80)

health_indicators = {
    'SH.DYN.MORT': 'infant_mortality_per_1000',
    'SH.DYN.NMRT': 'neonatal_mortality',
    'SH.STA.MMRT': 'maternal_mortality_per_100k',
    'SH.MED.BEDS.ZS': 'hospital_beds_per_1000',
    'SH.XPD.CHEX.GD.ZS': 'health_expenditure_percent_gdp',
    'SH.H2O.SMDW.ZS': 'access_safe_water_percent',
    'SH.STA.BASS.ZS': 'access_sanitation_percent'
}

try:
    health_df = load_or_fetch(
        os.path.join(SOCIAL_DIR, 'health_indicators.csv'),
        fetch_worldbank_data,
        indicators=health_indicators,
        country='GUY',
        start_year=1960
    )
    print(f"✓ Health data: {len(health_df)} records")
    data_sources.append({
        'dataset': 'Health Indicators',
        'source': 'World Bank',
        'url': 'https://data.worldbank.org',
        'date': datetime.now().strftime('%Y-%m-%d'),
        'records': len(health_df)
    })
except Exception as e:
    print(f"✗ Error fetching health data: {e}")

# 2.4 Poverty and Inequality
print("\n[7/12] Fetching Poverty and Inequality Data...")
print("-" * 80)

poverty_indicators = {
    'SI.POV.GINI': 'gini_index',
    'SI.POV.DDAY': 'poverty_headcount_190',
    'SI.POV.LMIC': 'poverty_headcount_320',
    'SI.POV.UMIC': 'poverty_headcount_550',
    'SI.DST.FRST.20': 'income_share_lowest_20',
    'SI.DST.05TH.20': 'income_share_highest_20',
    'SI.DST.10TH.10': 'income_share_top_10'
}

try:
    poverty_df = load_or_fetch(
        os.path.join(SOCIAL_DIR, 'poverty_inequality.csv'),
        fetch_worldbank_data,
        indicators=poverty_indicators,
        country='GUY',
        start_year=1960
    )
    print(f"✓ Poverty data: {len(poverty_df)} records")
    data_sources.append({
        'dataset': 'Poverty and Inequality',
        'source': 'World Bank',
        'url': 'https://data.worldbank.org',
        'date': datetime.now().strftime('%Y-%m-%d'),
        'records': len(poverty_df)
    })
except Exception as e:
    print(f"✗ Error fetching poverty data: {e}")

# 2.5 World Happiness Report
print("\n[8/12] Fetching World Happiness Report Data...")
print("-" * 80)

try:
    # Try to fetch from known URLs
    happiness_urls = [
        "https://happiness-report.s3.amazonaws.com/2024/DataForFigure2.1WHR2024.xls",
        "https://happiness-report.s3.amazonaws.com/2023/DataForFigure2.1WHR2023.xls"
    ]

    happiness_df = None
    for url in happiness_urls:
        try:
            print(f"Trying: {url}")
            df = pd.read_excel(url)
            # Filter for Guyana
            happiness_df = df[df['Country name'] == 'Guyana'].copy()
            if len(happiness_df) > 0:
                happiness_df.to_csv(os.path.join(SOCIAL_DIR, 'happiness_report.csv'), index=False)
                print(f"✓ Happiness data: {len(happiness_df)} records")
                data_sources.append({
                    'dataset': 'World Happiness Report',
                    'source': 'World Happiness Report',
                    'url': url,
                    'date': datetime.now().strftime('%Y-%m-%d'),
                    'records': len(happiness_df)
                })
                break
        except Exception as e:
            continue

    if happiness_df is None or len(happiness_df) == 0:
        print("⚠ Happiness data not available from automatic download")
        print("  Manual download: https://worldhappiness.report/data/")

except Exception as e:
    print(f"⚠ Could not fetch happiness data: {e}")
    print("  Manual download may be required")

# ============================================================================
# PART 3: ENVIRONMENTAL INDICATORS
# ============================================================================

print("\n" + "=" * 80)
print("PART 3: ENVIRONMENTAL INDICATORS")
print("=" * 80)

# 3.1 Environment and Climate
print("\n[9/12] Fetching Environmental Data...")
print("-" * 80)

env_indicators = {
    'AG.LND.FRST.ZS': 'forest_area_percent',
    'AG.LND.FRST.K2': 'forest_area_sqkm',
    'AG.LND.AGRI.ZS': 'agricultural_land_percent',
    'EN.ATM.CO2E.PC': 'co2_emissions_per_capita',
    'EN.ATM.CO2E.KT': 'co2_emissions_total_kt',
    'ER.H2O.FWTL.K3': 'renewable_water_resources',
    'AG.LND.PRCP.MM': 'average_precipitation'
}

try:
    env_df = load_or_fetch(
        os.path.join(ENV_DIR, 'environment_climate.csv'),
        fetch_worldbank_data,
        indicators=env_indicators,
        country='GUY',
        start_year=1960
    )
    print(f"✓ Environmental data: {len(env_df)} records")
    data_sources.append({
        'dataset': 'Environment and Climate',
        'source': 'World Bank',
        'url': 'https://data.worldbank.org',
        'date': datetime.now().strftime('%Y-%m-%d'),
        'records': len(env_df)
    })
except Exception as e:
    print(f"✗ Error fetching environmental data: {e}")

# 3.2 Biodiversity Data (GBIF)
print("\n[10/12] Fetching Biodiversity Data (GBIF)...")
print("-" * 80)

try:
    print("Fetching species occurrence data from GBIF (this may take a while)...")

    # Fetch in batches
    all_records = []
    batch_size = 300
    max_records = 10000

    for offset in range(0, max_records, batch_size):
        try:
            batch = fetch_gbif_species(country='GY', limit=batch_size, offset=offset)
            if len(batch) == 0:
                break
            all_records.append(batch)
            print(f"  Fetched {offset + len(batch)} records...", end='\r')
            time.sleep(1)  # Rate limiting
        except Exception as e:
            print(f"\n  Batch error at offset {offset}: {e}")
            break

    if all_records:
        biodiv_df = pd.concat(all_records, ignore_index=True)
        biodiv_df.to_csv(os.path.join(ENV_DIR, 'gbif_species_occurrences.csv'), index=False)
        print(f"\n✓ Biodiversity data: {len(biodiv_df)} occurrence records")
        data_sources.append({
            'dataset': 'Species Occurrences',
            'source': 'GBIF',
            'url': 'https://www.gbif.org',
            'date': datetime.now().strftime('%Y-%m-%d'),
            'records': len(biodiv_df)
        })
    else:
        print("\n⚠ No biodiversity data fetched")

except Exception as e:
    print(f"\n✗ Error fetching biodiversity data: {e}")

# ============================================================================
# PART 4: RESOURCE PRODUCTION (Manual data sources noted)
# ============================================================================

print("\n" + "=" * 80)
print("PART 4: RESOURCE PRODUCTION DATA")
print("=" * 80)

print("\n[11/12] Resource Production Data...")
print("-" * 80)
print("⚠ Note: Some resource data requires manual download:")
print()
print("GOLD & DIAMOND PRODUCTION:")
print("  Source: USGS Minerals Yearbook")
print("  URL: https://www.usgs.gov/centers/national-minerals-information-center")
print("  Save to: data/raw/resources/usgs_gold_production.csv")
print("  Save to: data/raw/resources/usgs_diamond_production.csv")
print()
print("OIL PRODUCTION:")
print("  Source: U.S. EIA or BP Statistical Review")
print("  URL: https://www.eia.gov/international/data/world")
print("  Save to: data/raw/resources/oil_production.csv")
print("  Note: Guyana oil production started in 2019")
print()

# Create placeholder files with instructions
placeholder_gold = pd.DataFrame({
    'year': [2015, 2020, 2023],
    'production_kg': [None, None, None],
    'note': ['Add USGS data here'] * 3
})
placeholder_gold.to_csv(os.path.join(RESOURCES_DIR, 'PLACEHOLDER_gold_production.csv'), index=False)

placeholder_oil = pd.DataFrame({
    'year': [2019, 2020, 2021, 2022, 2023],
    'production_barrels_per_day': [None, None, None, None, None],
    'note': ['Add EIA data here'] * 5
})
placeholder_oil.to_csv(os.path.join(RESOURCES_DIR, 'PLACEHOLDER_oil_production.csv'), index=False)

print("✓ Created placeholder files with instructions")

# ============================================================================
# PART 5: SATELLITE DATA (Requires GEE setup)
# ============================================================================

print("\n" + "=" * 80)
print("PART 5: SATELLITE IMAGERY DATA")
print("=" * 80)

print("\n[12/12] Satellite Data (Google Earth Engine)...")
print("-" * 80)
print("⚠ Note: Requires Google Cloud Project setup")
print("  See: GEE_SETUP.md for instructions")
print("  Once set up, use src/satellite_utils.py functions")
print()
print("Available datasets:")
print("  - Hansen Global Forest Change (2000-2023)")
print("  - MODIS NDVI (vegetation, 2000-2024)")
print("  - JRC Global Surface Water (1984-2024)")
print("  - Sentinel-2 (high-resolution, 2015-2024)")
print()

# ============================================================================
# SUMMARY AND METADATA
# ============================================================================

print("\n" + "=" * 80)
print("DATA ACQUISITION SUMMARY")
print("=" * 80)

# Create metadata file
metadata_df = pd.DataFrame(data_sources)
metadata_df.to_csv(os.path.join(METADATA_DIR, 'data_sources.csv'), index=False)

print(f"\nTotal datasets acquired: {len(data_sources)}")
print(f"Total records: {metadata_df['records'].sum():,}")
print()

print("Datasets by category:")
economic_count = sum(1 for d in data_sources if 'GDP' in d['dataset'] or 'Trade' in d['dataset'] or 'Inflation' in d['dataset'])
social_count = sum(1 for d in data_sources if any(x in d['dataset'] for x in ['Population', 'Education', 'Health', 'Poverty', 'Happiness']))
env_count = sum(1 for d in data_sources if 'Environment' in d['dataset'] or 'Species' in d['dataset'])

print(f"  Economic: {economic_count}")
print(f"  Social: {social_count}")
print(f"  Environmental: {env_count}")
print()

print("Data saved to:")
print(f"  {ECONOMIC_DIR}/")
print(f"  {SOCIAL_DIR}/")
print(f"  {ENV_DIR}/")
print(f"  {RESOURCES_DIR}/")
print()

print("Metadata saved to:")
print(f"  {os.path.join(METADATA_DIR, 'data_sources.csv')}")
print()

print("=" * 80)
print("DATA ACQUISITION COMPLETE!")
print("=" * 80)
print(f"\nCompleted: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
print()
print("Next steps:")
print("1. Review acquired data in data/raw/ directories")
print("2. Add manual data for resource production (gold, oil)")
print("3. Set up Google Earth Engine for satellite data")
print("4. Run preprocessing script: python preprocess_all_data.py")
print()