LLMs Token Count #22
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: LLMs Token Count | |
| on: | |
| schedule: | |
| - cron: "0 9 * * 1" # Every Monday at 9am UTC | |
| workflow_dispatch: | |
| inputs: | |
| threshold: | |
| description: "Token threshold for creating an issue" | |
| default: "100000" | |
| env: | |
| TOKEN_THRESHOLD: ${{ inputs.threshold || '100000' }} | |
| jobs: | |
| count-tokens: | |
| name: Count llms-full.txt tokens | |
| runs-on: ubuntu-latest | |
| permissions: | |
| issues: write | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| persist-credentials: false | |
| - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 | |
| - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 | |
| with: | |
| node-version: "22" | |
| cache: "pnpm" | |
| - run: pnpm install --frozen-lockfile | |
| - run: pnpm build | |
| - name: Count tokens | |
| id: count | |
| run: node scripts/count-tokens.ts | |
| - name: Job summary | |
| env: | |
| TOKEN_COUNT: ${{ steps.count.outputs.token_count }} | |
| run: | | |
| COUNT=$TOKEN_COUNT | |
| THRESHOLD=$TOKEN_THRESHOLD | |
| STATUS=$( [ "$COUNT" -gt "$THRESHOLD" ] && echo "Over" || echo "Under" ) | |
| echo "### llms-full.txt token count" >> "$GITHUB_STEP_SUMMARY" | |
| echo "" >> "$GITHUB_STEP_SUMMARY" | |
| echo "| Metric | Value |" >> "$GITHUB_STEP_SUMMARY" | |
| echo "|--------|-------|" >> "$GITHUB_STEP_SUMMARY" | |
| echo "| **Tokens** | $(printf "%'d" $COUNT) |" >> "$GITHUB_STEP_SUMMARY" | |
| echo "| **Threshold** | $(printf "%'d" $THRESHOLD) |" >> "$GITHUB_STEP_SUMMARY" | |
| echo "| **Status** | $STATUS |" >> "$GITHUB_STEP_SUMMARY" | |
| - name: Create or update issue | |
| if: steps.count.outputs.over_threshold == 'true' | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 | |
| env: | |
| TOKEN_COUNT: ${{ steps.count.outputs.token_count }} | |
| TOKEN_THRESHOLD: ${{ env.TOKEN_THRESHOLD }} | |
| with: | |
| script: | | |
| const tokenCount = Number(process.env.TOKEN_COUNT); | |
| const threshold = Number(process.env.TOKEN_THRESHOLD); | |
| const title = `llms-full.txt exceeds ${threshold.toLocaleString()} token threshold`; | |
| const { data: issues } = await github.rest.issues.listForRepo({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| state: 'open', | |
| labels: 'llms-token-count', | |
| }); | |
| const existing = issues.find(i => i.title === title); | |
| if (existing) { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: existing.number, | |
| body: `Updated count: **${tokenCount.toLocaleString()}** tokens (threshold: ${threshold.toLocaleString()})`, | |
| }); | |
| return; | |
| } | |
| await github.rest.issues.create({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| title, | |
| body: [ | |
| `\`llms-full.txt\` has **${tokenCount.toLocaleString()}** tokens (threshold: ${threshold.toLocaleString()}).`, | |
| '', | |
| 'Large context files degrade LLM performance and increase costs. Consider:', | |
| '- Removing redundant or templated content', | |
| '- Splitting into focused `llms.txt` sections', | |
| '- Trimming verbose code examples', | |
| ].join('\n'), | |
| labels: ['llms-token-count'], | |
| }); | |
| - name: Close issue if back under threshold | |
| if: steps.count.outputs.over_threshold == 'false' | |
| uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 | |
| env: | |
| TOKEN_COUNT: ${{ steps.count.outputs.token_count }} | |
| TOKEN_THRESHOLD: ${{ env.TOKEN_THRESHOLD }} | |
| with: | |
| script: | | |
| const tokenCount = Number(process.env.TOKEN_COUNT); | |
| const threshold = Number(process.env.TOKEN_THRESHOLD); | |
| const { data: issues } = await github.rest.issues.listForRepo({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| state: 'open', | |
| labels: 'llms-token-count', | |
| }); | |
| for (const issue of issues) { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: issue.number, | |
| body: `Resolved: **${tokenCount.toLocaleString()}** tokens (under ${threshold.toLocaleString()} threshold). Closing.`, | |
| }); | |
| await github.rest.issues.update({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: issue.number, | |
| state: 'closed', | |
| }); | |
| } |