Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 27 additions & 4 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,10 @@ permissions:

jobs:
tests:
runs-on: ubuntu-latest
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, windows-latest]
test-folders: ["library-tests", "queries-tests"]
steps:
- name: "Checkout"
Expand Down Expand Up @@ -53,8 +54,8 @@ jobs:
- uses: dtolnay/rust-toolchain@4305c38b25d97ef35a8ad1f985ccf2d2242004f2 # stable
if: steps.extractor-changes.outputs.src == 'true'

- name: "Build Extractor"
if: steps.extractor-changes.outputs.src == 'true'
- name: "Build Extractor (Linux/macOS)"
if: steps.extractor-changes.outputs.src == 'true' && matrix.os != 'windows-latest'
env:
GH_TOKEN: ${{ github.token }}
run: |
Expand All @@ -66,12 +67,34 @@ jobs:

gh codeql resolve languages --format=json --search-path ./extractor-pack

- name: "Run Tests"
- name: "Build Extractor (Windows)"
if: steps.extractor-changes.outputs.src == 'true' && matrix.os == 'windows-latest'
env:
GH_TOKEN: ${{ github.token }}
shell: pwsh
run: |
gh extension install github/gh-codeql
gh codeql set-version latest

.\scripts\create-extractor-pack.ps1

gh codeql resolve languages --format=json --search-path .\extractor-pack

- name: "Run Tests (Linux/macOS)"
if: matrix.os != 'windows-latest'
env:
GH_TOKEN: ${{ github.token }}
run: |
./scripts/run-tests.sh "ql/test/${{ matrix.test-folders }}"

- name: "Run Tests (Windows)"
if: matrix.os == 'windows-latest'
env:
GH_TOKEN: ${{ github.token }}
shell: pwsh
run: |
.\scripts\run-tests.ps1 "ql/test/${{ matrix.test-folders }}"

# scanning:
# runs-on: ubuntu-latest
# needs: [tests]
Expand Down
15 changes: 10 additions & 5 deletions .github/workflows/publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,7 @@ jobs:
strategy:
fail-fast: false
matrix:
# TODO: Add windows-latest
os: [ubuntu-latest, macos-latest]
os: [ubuntu-latest, macos-latest, windows-latest]

if: ${{ needs.release-check.outputs.release == 'true' }}
steps:
Expand All @@ -59,14 +58,20 @@ jobs:

- name: "Set up Rust"
uses: dtolnay/rust-toolchain@nightly
if: ${{ matrix.os != 'windows-latest' }}

- name: "Build Extractor"
- name: "Build Extractor (Linux/macOS)"
if: ${{ matrix.os != 'windows-latest' }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: ./scripts/create-extractor-pack.sh

- name: "Build Extractor (Windows)"
if: ${{ matrix.os == 'windows-latest' }}
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
shell: pwsh
run: .\scripts\create-extractor-pack.ps1

- name: "Upload bundle artifact"
uses: actions/upload-artifact@v4
with:
Expand All @@ -85,7 +90,7 @@ jobs:
with:
submodules: true

- name: "Downloadd all artifacts"
- name: "Download all artifacts"
uses: actions/download-artifact@v4
with:
path: "./extractor-pack"
Expand Down
66 changes: 55 additions & 11 deletions scripts/create-extractor-pack.ps1
Original file line number Diff line number Diff line change
@@ -1,14 +1,58 @@
cargo build --release
$ErrorActionPreference = "Stop"

# Set platform
$platform = "win64"

cargo run --release -p ql-generator -- --dbscheme ql/src/ql.dbscheme --library ql/src/codeql_ql/ast/internal/TreeSitter.qll
codeql query format -i ql\src\codeql_ql\ast\internal\TreeSitter.qll
# Check for CodeQL binary
if (Get-Command "codeql" -ErrorAction SilentlyContinue) {
$CODEQL_BINARY = "codeql"
Copy link

Copilot AI Nov 3, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Inconsistent indentation: Line 8 uses tabs while most other lines in the file use spaces. This line should use spaces to match the rest of the file's indentation style.

Suggested change
$CODEQL_BINARY = "codeql"
$CODEQL_BINARY = "codeql"

Copilot uses AI. Check for mistakes.
}
elseif (Get-Command "gh" -ErrorAction SilentlyContinue) {
try {
gh codeql version 2>&1 | Out-Null
$CODEQL_BINARY = "gh codeql"
}
catch {
Copy link

Copilot AI Nov 3, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Inconsistent indentation: Line 15 uses a leading space before 'catch' while other lines use tabs or no leading space. This should be aligned consistently with the surrounding try-catch block.

Suggested change
catch {
catch {

Copilot uses AI. Check for mistakes.
Write-Host "Installing gh-codeql extension..."
gh extension install github/gh-codeql
$CODEQL_BINARY = "gh codeql"
}
}
else {
Write-Error "Neither 'codeql' nor 'gh' command found"
exit 1
Comment on lines +22 to +23
Copy link

Copilot AI Nov 3, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Inconsistent indentation: Lines 22-23 use tabs while most other lines in the file use spaces. These lines should use spaces to match the rest of the file's indentation style.

Suggested change
Write-Error "Neither 'codeql' nor 'gh' command found"
exit 1
Write-Error "Neither 'codeql' nor 'gh' command found"
exit 1

Copilot uses AI. Check for mistakes.
}

Write-Host "Creating extractor pack..."
if (Test-Path -Path extractor-pack) {
rm -Recurse -Force extractor-pack
}
mkdir extractor-pack | Out-Null
cp codeql-extractor.yml, ql\src\ql.dbscheme, ql\src\ql.dbscheme.stats extractor-pack
cp -Recurse tools extractor-pack
mkdir extractor-pack\tools\win64 | Out-Null
cp target\release\ql-extractor.exe extractor-pack\tools\win64\extractor.exe
cp target\release\ql-autobuilder.exe extractor-pack\tools\win64\autobuilder.exe
Remove-Item -Recurse -Force extractor-pack
}
if (Test-Path -Path target) {
Remove-Item -Recurse -Force target
}

Write-Host "Update submodules..."
git submodule update --init --recursive

Write-Host "Building extractor..."
cargo build --release

Write-Host "Generating TreeSitter library..."
cargo run --release --bin codeql-extractor-iac -- generate --dbscheme ql/lib/iac.dbscheme --library ql/lib/codeql/iac/ast/internal/TreeSitter.qll

Write-Host "Formatting generated library..."
if ($CODEQL_BINARY -eq "gh codeql") {
gh codeql query format -i ql/lib/codeql/iac/ast/internal/TreeSitter.qll
}
else {
codeql query format -i ql/lib/codeql/iac/ast/internal/TreeSitter.qll
}

New-Item -ItemType Directory -Path extractor-pack | Out-Null
Copy-Item -Path codeql-extractor.yml, ql/lib/iac.dbscheme, ql/lib/iac.dbscheme.stats -Destination extractor-pack/
Copy-Item -Recurse -Path downgrades, tools -Destination extractor-pack/

New-Item -ItemType Directory -Path "extractor-pack/tools/$platform" -Force | Out-Null
Copy-Item -Path "target/release/codeql-extractor-iac.exe" -Destination "extractor-pack/tools/$platform/extractor.exe"

Write-Host "Extractor pack created successfully!"
26 changes: 26 additions & 0 deletions scripts/install-extractor.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
param(
[string]$ExtractorName = "iac",
[string]$ExtractorLocations = "$env:USERPROFILE\.codeql\extractors"
)

$ErrorActionPreference = "Stop"

Write-Host "Creating extractor directory..."
if (!(Test-Path $ExtractorLocations)) {
New-Item -ItemType Directory -Path $ExtractorLocations -Force | Out-Null
}

Write-Host "Checking latest release..."
gh release list -L 1 -R "advanced-security/codeql-extractor-$ExtractorName"

Write-Host "Downloading extractor pack..."
gh release download `
-R "advanced-security/codeql-extractor-$ExtractorName" `
-D "$ExtractorLocations" `
--clobber `
--pattern 'extractor-*.tar.gz'

Write-Host "Extracting extractor pack..."
tar -zxf "$ExtractorLocations/extractor-$ExtractorName.tar.gz" --directory "$ExtractorLocations"

Write-Host "Installation complete! Extractor installed to: $ExtractorLocations"
57 changes: 57 additions & 0 deletions scripts/run-tests.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
param(
[string]$TestsDir = "ql/test"
)

$ErrorActionPreference = "Stop"

# Check for CodeQL binary
if (Get-Command "codeql" -ErrorAction SilentlyContinue) {
$CODEQL_BINARY = "codeql"
}
elseif (Get-Command "gh" -ErrorAction SilentlyContinue) {
try {
gh codeql version 2>&1 | Out-Null
$CODEQL_BINARY = "gh codeql"
}
catch {
Write-Host "Installing gh-codeql extension..."
gh extension install github/gh-codeql
$CODEQL_BINARY = "gh codeql"
}
}
else {
Write-Error "Neither 'codeql' nor 'gh' command found"
exit 1
}

Write-Host "Installing ql/test pack dependencies..."
if ($CODEQL_BINARY -eq "gh codeql") {
gh codeql pack install ql/test
}
else {
codeql pack install ql/test
}

Write-Host "Running tests in $TestsDir"

if ($CODEQL_BINARY -eq "gh codeql") {
gh codeql test run `
-j 0 `
--check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition `
--search-path ./extractor-pack `
"$TestsDir"
}
else {
codeql test run `
-j 0 `
--check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition `
--search-path ./extractor-pack `
"$TestsDir"
}

if ($LASTEXITCODE -ne 0) {
Write-Error "Tests failed with exit code $LASTEXITCODE"
exit $LASTEXITCODE
}

Write-Host "All tests passed!"
1 change: 1 addition & 0 deletions tools/index-files.cmd
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
@echo off

type NUL && "%CODEQL_EXTRACTOR_IAC_ROOT%\tools\win64\extractor.exe" ^
extract ^
--file-list "%1" ^
--source-archive-dir "%CODEQL_EXTRACTOR_IAC_SOURCE_ARCHIVE_DIR%" ^
--output-dir "%CODEQL_EXTRACTOR_IAC_TRAP_DIR%"
Expand Down
4 changes: 3 additions & 1 deletion tools/qltest.cmd
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@ type NUL && "%CODEQL_DIST%\codeql.exe" database index-files ^
--prune=**/*.testproj ^
--include-extension=.hcl ^
--include-extension=.tf ^
--include-extension=.tfvars ^
--include-extension=.bicep ^
--size-limit=5m ^
--language=hcl ^
--language=iac ^
--working-dir=. ^
"%CODEQL_EXTRACTOR_IAC_WIP_DATABASE%"

Expand Down