Daily Ecosystem Scan #81
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Daily Ecosystem Scan | |
| on: | |
| schedule: | |
| # Every day at 00:00 UTC (08:00 Taiwan time) | |
| - cron: '0 0 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| scan_limit: | |
| description: 'Number of packages to scan per run' | |
| required: false | |
| default: '50' | |
| skip_crawl: | |
| description: 'Skip registry crawl (use cached registry)' | |
| required: false | |
| default: 'false' | |
| permissions: | |
| contents: write | |
| actions: read | |
| jobs: | |
| scan: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 45 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| cache: npm | |
| - run: npm ci | |
| - name: Build ATR engine | |
| run: npm run build | |
| # Restore previous cumulative results | |
| - name: Restore cumulative scan data | |
| uses: actions/cache@v4 | |
| with: | |
| path: | | |
| data/scan-cumulative.json | |
| mcp-registry.json | |
| key: daily-scan-${{ github.run_number }} | |
| restore-keys: | | |
| daily-scan- | |
| # Step 1: Crawl registries for new skills | |
| - name: Crawl MCP registries | |
| if: ${{ github.event.inputs.skip_crawl != 'true' }} | |
| run: npx tsx scripts/crawl-mcp-registry.ts --limit 10000 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # Step 2: Incremental audit (skip already scanned) | |
| - name: Audit new packages | |
| env: | |
| SCAN_LIMIT: ${{ github.event.inputs.scan_limit || '50' }} | |
| run: | | |
| SKIP_FLAG="" | |
| if [ -f data/scan-cumulative.json ]; then | |
| SKIP_FLAG="--skip-scanned data/scan-cumulative.json" | |
| fi | |
| npx tsx scripts/audit-npm-skills-v2.ts \ | |
| --limit "$SCAN_LIMIT" \ | |
| $SKIP_FLAG \ | |
| --output "data/scan-batch-$(date +%Y%m%d).json" | |
| # Step 3: Merge batch into cumulative (PRIVATE - full details) | |
| - name: Merge scan results | |
| run: npx tsx scripts/merge-scan-results.ts | |
| # Step 4: Export layered outputs | |
| # Layer 1 (PUBLIC): ecosystem-report.csv — package, version, riskLevel, riskScore, toolCount, scannedAt | |
| # Layer 2 (TC ONLY): full findings pushed via API | |
| # Layer 3 (PRIVATE): scan-cumulative.json with genuineThreats, AST, code analysis | |
| - name: Export public ecosystem report | |
| run: npx tsx scripts/export-public-report.ts | |
| - name: Export internal CSV (findings detail) | |
| run: | | |
| npx tsx scripts/export-findings.ts \ | |
| --input data/scan-cumulative.json \ | |
| --output-dir data/exports | |
| # Step 5: Push to Threat Cloud (Layer 2 — detailed findings via API) | |
| - name: Push to Threat Cloud | |
| if: ${{ vars.TC_URL != '' }} | |
| run: | | |
| BATCH_FILE="data/scan-batch-$(date +%Y%m%d).json" | |
| if [ -f "$BATCH_FILE" ]; then | |
| npx tsx scripts/push-to-threat-cloud.ts \ | |
| --input "$BATCH_FILE" \ | |
| --tc-url "${{ vars.TC_URL }}" \ | |
| --auto-propose | |
| fi | |
| # Step 6: Generate Skills Sec posts from flagged findings | |
| - name: Generate Skills Sec content | |
| run: | | |
| BATCH_FILE="data/scan-batch-$(date +%Y%m%d).json" | |
| if [ -f "$BATCH_FILE" ] && [ -f scripts/generate-skillssec-posts.ts ]; then | |
| npx tsx scripts/generate-skillssec-posts.ts --input "$BATCH_FILE" | |
| else | |
| echo "Skipping post generation (batch=$BATCH_FILE exists=$([ -f \"$BATCH_FILE\" ] && echo yes || echo no), script exists=$([ -f scripts/generate-skillssec-posts.ts ] && echo yes || echo no))" | |
| fi | |
| # Step 6.5: Generate clipboard drafts for manual posting | |
| - name: Generate platform drafts | |
| run: | | |
| POSTS_DIR="posts/$(date +%Y-%m-%d)" | |
| if [ -d "$POSTS_DIR" ] && [ -f scripts/publish-posts.ts ]; then | |
| npx tsx scripts/publish-posts.ts --dir "$POSTS_DIR" --drafts-only | |
| else | |
| echo "Skipping draft generation" | |
| fi | |
| # Step 7: Weekly summary (runs on Sundays only via cron) | |
| - name: Generate weekly summary | |
| if: ${{ github.event.schedule != '' }} | |
| run: | | |
| DAY=$(date +%u) | |
| if [ "$DAY" = "7" ] && [ -f data/scan-cumulative.json ] && [ -f scripts/generate-skillssec-posts.ts ]; then | |
| npx tsx scripts/generate-skillssec-posts.ts \ | |
| --input data/scan-cumulative.json --weekly | |
| else | |
| echo "Skipping weekly summary (day=$DAY or script not found)" | |
| fi | |
| # Step 7: Generate daily summary (must run BEFORE commit — writes daily-summary.md) | |
| - name: Generate scan summary | |
| run: | | |
| node -e " | |
| const fs = require('fs'); | |
| const batch = fs.readdirSync('data').filter(f => f.startsWith('scan-batch-')).sort().pop(); | |
| if (!batch) { console.log('No batch file found'); process.exit(0); } | |
| const data = JSON.parse(fs.readFileSync('data/' + batch, 'utf-8')); | |
| const results = data.results || []; | |
| const critical = results.filter(r => r.riskLevel === 'CRITICAL').length; | |
| const high = results.filter(r => r.riskLevel === 'HIGH').length; | |
| const clean = results.filter(r => r.riskLevel === 'CLEAN' || r.riskLevel === 'LOW').length; | |
| const lines = []; | |
| lines.push('## Daily Scan Summary (' + new Date().toISOString().slice(0,10) + ')'); | |
| lines.push('- Scanned: ' + results.length + ' packages'); | |
| lines.push('- CRITICAL: ' + critical); | |
| lines.push('- HIGH: ' + high); | |
| lines.push('- CLEAN/LOW: ' + clean); | |
| if (critical > 0 || high > 0) { | |
| lines.push(''); | |
| lines.push('### Flagged Packages:'); | |
| results.filter(r => r.riskLevel === 'CRITICAL' || r.riskLevel === 'HIGH') | |
| .forEach(r => lines.push('- [' + r.riskLevel + '] ' + r.package + ' (score: ' + r.riskScore + ')')); | |
| } | |
| if (fs.existsSync('data/scan-cumulative.json')) { | |
| const cum = JSON.parse(fs.readFileSync('data/scan-cumulative.json', 'utf-8')); | |
| lines.push(''); | |
| lines.push('### Cumulative:'); | |
| lines.push('- Total scanned: ' + (cum.total || cum.results?.length || 0)); | |
| const s = cum.summary || {}; | |
| Object.entries(s).sort().forEach(([k,v]) => lines.push(' - ' + k + ': ' + v)); | |
| } | |
| const output = lines.join('\n'); | |
| console.log(output); | |
| fs.writeFileSync('data/daily-summary.md', output); | |
| " | |
| # Step 8: Commit public report back to repo (tolerates missing files from upstream failures) | |
| - name: Commit public ecosystem data | |
| run: | | |
| git config user.name "ATR Ecosystem Scanner" | |
| git config user.email "scanner@atr-framework.org" | |
| # Only add files that actually exist so a partial run can still commit what it produced | |
| for f in data/ecosystem-report.csv data/ecosystem-stats.json data/daily-summary.md; do | |
| [ -f "$f" ] && git add "$f" | |
| done | |
| if git diff --cached --quiet; then | |
| echo "No changes to commit" | |
| else | |
| SUMMARY="N/A" | |
| FLAGGED="" | |
| if [ -f data/daily-summary.md ]; then | |
| SUMMARY=$(grep 'Scanned:' data/daily-summary.md | head -1 | sed 's/.*Scanned: //' || echo 'N/A') | |
| FLAGGED=$(grep -E 'CRITICAL|HIGH' data/daily-summary.md | head -2 || true) | |
| fi | |
| git commit -m "chore: daily ecosystem scan $(date +%Y-%m-%d) | |
| Scanned $SUMMARY | |
| $FLAGGED" | |
| git push || echo "Push failed — will retry next run" | |
| fi | |
| # Upload PRIVATE artifacts (full scan data — not in git) | |
| - name: Upload private scan artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: scan-private-${{ github.run_number }} | |
| path: | | |
| data/scan-batch-*.json | |
| data/scan-cumulative.json | |
| data/exports/*.csv | |
| retention-days: 90 | |
| # Upload PUBLIC artifacts (ecosystem report — also committed to git) | |
| - name: Upload public scan artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: scan-public-${{ github.run_number }} | |
| path: | | |
| data/ecosystem-report.csv | |
| data/ecosystem-stats.json | |
| data/daily-summary.md | |
| retention-days: 365 | |
| # Upload generated posts (for review before publishing) | |
| - name: Upload Skills Sec posts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: skillssec-posts-${{ github.run_number }} | |
| path: posts/ | |
| retention-days: 30 | |
| if-no-files-found: ignore | |
| # Cache cumulative data for next run | |
| - name: Save cumulative data | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: | | |
| data/scan-cumulative.json | |
| mcp-registry.json | |
| key: daily-scan-${{ github.run_number }} |