Skip to content

Commit b1c137c

Browse files
jsorefjsoref
committed
Autotune memory
Public runners have ~10G of ram available. XL runners have >50G of ram available. It's nice to be able to run tests on public runners. Introduce an action that: * (Linux) frees some memory * (Linux) adds some swap * (Linux and macOS) calculates available memory * (Windows) suggests 10G of available memory
1 parent 31cb308 commit b1c137c

File tree

5 files changed

+113
-4
lines changed

5 files changed

+113
-4
lines changed

‎.github/actions/reasonable-memory/action.yml

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
name: Make more memory available
2+
description: Stops services and adds swap
3+
4+
inputs:
5+
max:
6+
description: Maximum memory to suggest using
7+
default: 50000
8+
9+
outputs:
10+
available:
11+
description: Available memory
12+
value: ${{ steps.memory.outputs.available }}
13+
suggested:
14+
description: Memory suggested for use (constrained by max and available)
15+
value: ${{ steps.memory.outputs.suggested }}
16+
17+
runs:
18+
using: composite
19+
steps:
20+
- name: Stop services
21+
if: env.RUNNER_OS == 'Linux'
22+
shell: bash
23+
run: |
24+
sudo systemctl disable php8.1-fpm mono-xsp4 walinuxagent multipathd walinuxagent chrony cron getty@tty1 networkd-dispatcher rsyslog serial-getty@ttyS0 snapd multipathd.socket snapd.socket
25+
sudo systemctl stop php8.1-fpm mono-xsp4 walinuxagent multipathd walinuxagent chrony cron getty@tty1 networkd-dispatcher rsyslog serial-getty@ttyS0 snapd
26+
sudo killall mono
27+
28+
- name: enable swap
29+
if: env.RUNNER_OS == 'Linux'
30+
shell: bash
31+
run: |
32+
sudo fallocate -l 10G /mnt/big-swapfile
33+
sudo chmod 600 /mnt/big-swapfile
34+
sudo mkswap /mnt/big-swapfile
35+
sudo swapon /mnt/big-swapfile
36+
37+
- name: Report Available Memory (Linux)
38+
id: memory-linux
39+
if: runner.os == 'Linux'
40+
shell: bash
41+
run: |
42+
free |
43+
perl -ne '
44+
next unless /Mem:.*\s(\d+)$/;
45+
my $m = int($1 / 102400)*100;
46+
print "available=$m\n";
47+
' >> "$GITHUB_OUTPUT"
48+
49+
- name: Report Available Memory (macOS)
50+
id: memory-mac
51+
if: runner.os == 'macOS'
52+
shell: bash
53+
run: |
54+
sysctl -a |
55+
perl -e '
56+
my $m;
57+
while (<>) {
58+
next unless /.*memsize.*:\s*(\d+)/;
59+
my $m0 = int($1 >> 20);
60+
$m = $m0 unless ($m && $m > $m0);
61+
}
62+
print "available=$m\n";
63+
' >> "$GITHUB_OUTPUT"
64+
65+
- name: Report Available Memory (Windows)
66+
id: memory-windows
67+
if: runner.os == 'Windows'
68+
shell: bash
69+
run: |
70+
echo "available=10240" >> "$GITHUB_OUTPUT"
71+
72+
- name: Report Memory
73+
id: memory
74+
shell: bash
75+
env:
76+
max: ${{ inputs.max }}
77+
available: ${{ steps.memory-linux.outputs.available || steps.memory-mac.outputs.available || steps.memory-windows.outputs.available }}
78+
run: |
79+
perl -e '
80+
my $available = $ENV{available};
81+
my $max = $ENV{max} || $available;
82+
my $s = ($m < $max ? $available : $max);
83+
print "suggested=$s\n"."available=$available\n";
84+
' >> "$GITHUB_OUTPUT"

‎.github/workflows/csharp-qltest.yml

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,10 @@ on:
55
paths:
66
- "csharp/**"
77
- "shared/**"
8+
- .github/workflows/csharp-qltest.yml
89
- .github/actions/fetch-codeql/action.yml
10+
- .github/actions/cache-query-compilation/action.yml
11+
- .github/actions/reasonable-memory/action.yml
912
- codeql-workspace.yml
1013
branches:
1114
- main
@@ -16,6 +19,8 @@ on:
1619
- "shared/**"
1720
- .github/workflows/csharp-qltest.yml
1821
- .github/actions/fetch-codeql/action.yml
22+
- .github/actions/cache-query-compilation/action.yml
23+
- .github/actions/reasonable-memory/action.yml
1924
- codeql-workspace.yml
2025
branches:
2126
- main
@@ -53,6 +58,9 @@ jobs:
5358
slice: ["1/2", "2/2"]
5459
steps:
5560
- uses: actions/checkout@v4
61+
- name: Allocate memory
62+
id: memory
63+
uses: ./.github/actions/reasonable-memory
5664
- uses: ./csharp/actions/create-extractor-pack
5765
- name: Cache compilation cache
5866
id: query-cache
@@ -61,9 +69,10 @@ jobs:
6169
key: csharp-qltest-${{ matrix.slice }}
6270
- name: Run QL tests
6371
run: |
64-
codeql test run --threads=0 --ram 50000 --slice ${{ matrix.slice }} --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
72+
codeql test run --threads=0 --ram $memory --slice ${{ matrix.slice }} --search-path extractor-pack --check-databases --check-undefined-labels --check-repeated-labels --check-redefined-labels --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
6573
env:
6674
GITHUB_TOKEN: ${{ github.token }}
75+
memory: ${{ steps.memory.outputs.suggested }}
6776
unit-tests:
6877
strategy:
6978
matrix:

‎.github/workflows/ruby-qltest.yml

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,11 @@ on:
55
paths:
66
- "ruby/**"
77
- "shared/**"
8-
- .github/workflows/ruby-build.yml
8+
- .github/workflows/ruby-qltest.yml
99
- .github/actions/fetch-codeql/action.yml
10+
- .github/actions/cache-query-compilation/action.yml
11+
- .github/actions/reasonable-memory/action.yml
12+
- csharp/actions/create-extractor-pack/action.yml
1013
- codeql-workspace.yml
1114
branches:
1215
- main
@@ -17,10 +20,14 @@ on:
1720
- "shared/**"
1821
- .github/workflows/ruby-qltest.yml
1922
- .github/actions/fetch-codeql/action.yml
23+
- .github/actions/cache-query-compilation/action.yml
24+
- .github/actions/reasonable-memory/action.yml
25+
- csharp/actions/create-extractor-pack/action.yml
2026
- codeql-workspace.yml
2127
branches:
2228
- main
2329
- "rc/*"
30+
workflow_dispatch:
2431

2532
env:
2633
CARGO_TERM_COLOR: always
@@ -55,6 +62,9 @@ jobs:
5562
fail-fast: false
5663
steps:
5764
- uses: actions/checkout@v4
65+
- name: Allocate memory
66+
id: memory
67+
uses: ./.github/actions/reasonable-memory
5868
- uses: ./.github/actions/fetch-codeql
5969
- uses: ./ruby/actions/create-extractor-pack
6070
- name: Cache compilation cache
@@ -64,6 +74,7 @@ jobs:
6474
key: ruby-qltest
6575
- name: Run QL tests
6676
run: |
67-
codeql test run --threads=0 --ram 50000 --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
77+
codeql test run --threads=0 --ram $memory --search-path "${{ github.workspace }}/ruby/extractor-pack" --check-databases --check-undefined-labels --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition --consistency-queries ql/consistency-queries ql/test --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"
6878
env:
6979
GITHUB_TOKEN: ${{ github.token }}
80+
memory: ${{ steps.memory.outputs.suggested }}

‎.github/workflows/swift.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ on:
3232
- main
3333
- rc/*
3434
- codeql-cli-*
35+
workflow_dispatch:
3536

3637
jobs:
3738
# not using a matrix as you cannot depend on a specific job in a matrix, and we want to start linux checks

‎swift/actions/run-ql-tests/action.yml

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@ inputs:
88
runs:
99
using: composite
1010
steps:
11+
- name: Allocate memory
12+
id: memory
13+
uses: ./.github/actions/reasonable-memory
1114
- uses: ./swift/actions/share-extractor-pack
1215
- uses: ./.github/actions/fetch-codeql
1316
- id: query-cache
@@ -19,7 +22,7 @@ runs:
1922
run: |
2023
codeql test run \
2124
--threads=0 \
22-
--ram 50000 \
25+
--ram $memory \
2326
--search-path "${{ github.workspace }}/swift/extractor-pack" \
2427
--check-databases \
2528
--check-unused-labels \
@@ -32,3 +35,4 @@ runs:
3235
swift/ql/test
3336
env:
3437
GITHUB_TOKEN: ${{ github.token }}
38+
memory: ${{ steps.memory.outputs.suggested }}

0 commit comments

Comments
 (0)