Scraper with Supabase Sync #56
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Scraper with Supabase Sync | |
| on: | |
| schedule: | |
| - cron: '0 0 * * *' # Daily at 00:00 UTC | |
| workflow_dispatch: | |
| jobs: | |
| scrape-and-sync: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| environment: production | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Install dependencies | |
| run: | | |
| pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: Run scraper with Supabase sync + contacts | |
| timeout-minutes: 25 | |
| env: | |
| PYTHONPATH: src | |
| SUPABASE_URL: ${{ secrets.SUPABASE_URL }} | |
| SUPABASE_KEY: ${{ secrets.SUPABASE_KEY }} | |
| # Optional overrides (leave empty to use code defaults) | |
| FETCH_TIMEOUT: ${{ vars.FETCH_TIMEOUT || '' }} | |
| ENRICH_TIMEOUT: ${{ vars.ENRICH_TIMEOUT || '' }} | |
| FETCH_MAX_RETRIES: ${{ vars.FETCH_MAX_RETRIES || '' }} | |
| ENRICH_MAX_RETRIES: ${{ vars.ENRICH_MAX_RETRIES || '' }} | |
| FETCH_RETRY_WAIT: ${{ vars.FETCH_RETRY_WAIT || '' }} | |
| ENRICH_RETRY_WAIT: ${{ vars.ENRICH_RETRY_WAIT || '' }} | |
| USE_PROXY: ${{ vars.USE_PROXY || '' }} | |
| PROXY_URL: ${{ secrets.PROXY_URL || vars.PROXY_URL || '' }} | |
| CONTACTS_WORKERS: ${{ vars.CONTACTS_WORKERS || '' }} | |
| CONTACTS_BATCH_SIZE: ${{ vars.CONTACTS_BATCH_SIZE || '' }} | |
| CONTACTS_DAYS_FILTER: ${{ vars.CONTACTS_DAYS_FILTER || '' }} | |
| run: | | |
| CMD="python -m kickstarter_tools.run" | |
| # Only override defaults when a variable is set | |
| if [ -n "$FETCH_TIMEOUT" ]; then CMD="$CMD --fetch-timeout $FETCH_TIMEOUT"; fi | |
| if [ -n "$ENRICH_TIMEOUT" ]; then CMD="$CMD --enrich-timeout $ENRICH_TIMEOUT"; fi | |
| if [ -n "$FETCH_MAX_RETRIES" ]; then CMD="$CMD --fetch-max-retries $FETCH_MAX_RETRIES"; fi | |
| if [ -n "$ENRICH_MAX_RETRIES" ]; then CMD="$CMD --enrich-max-retries $ENRICH_MAX_RETRIES"; fi | |
| if [ -n "$FETCH_RETRY_WAIT" ]; then CMD="$CMD --fetch-retry-wait $FETCH_RETRY_WAIT"; fi | |
| if [ -n "$ENRICH_RETRY_WAIT" ]; then CMD="$CMD --enrich-retry-wait $ENRICH_RETRY_WAIT"; fi | |
| if [ -n "$CONTACTS_WORKERS" ]; then CMD="$CMD --contacts-workers $CONTACTS_WORKERS"; fi | |
| if [ -n "$CONTACTS_BATCH_SIZE" ]; then CMD="$CMD --contacts-batch-size $CONTACTS_BATCH_SIZE"; fi | |
| if [ -n "$CONTACTS_DAYS_FILTER" ]; then CMD="$CMD --days-filter $CONTACTS_DAYS_FILTER"; fi | |
| # Proxy: only force a mode if explicitly set | |
| if [ "$USE_PROXY" = "true" ]; then | |
| CMD="$CMD --use-proxy" | |
| elif [ "$USE_PROXY" = "false" ]; then | |
| CMD="$CMD --disable-proxy" | |
| fi | |
| echo "🚀 Running pipeline: $CMD" | |
| eval $CMD | |
| echo "✅ Kickstarter fetch, Supabase sync, and Firecrawl contacts completed" |