Skip to content

Commit 0d72435

Browse files
tomato42tomato42
authored
Merge pull request #328 from tlsfuzzer/cosmic-ray
Introduce mutation testing
2 parents 4203e7e + c4c6ff6 commit 0d72435

23 files changed

+775
-76
lines changed

‎.github/workflows/ci.yml

Lines changed: 302 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,10 @@ jobs:
104104
os: ubuntu-latest
105105
python-version: 3.9
106106
tox-env: codechecks
107+
- name: mutation testing
108+
os: ubuntu-latest
109+
python-version: '3.11'
110+
mutation: 'true'
107111
steps:
108112
- uses: actions/checkout@v2
109113
if: ${{ !matrix.container }}
@@ -149,6 +153,11 @@ jobs:
149153
run: |
150154
apt-get update
151155
apt-get install -y git make python-is-python3 python3 curl wget python3-distutils python3-pip
156+
157+
- name: Dependencies for mutation testing
158+
if: ${{ matrix.mutation == 'true' }}
159+
run: |
160+
sudo apt-get install -y sqlite3
152161
- name: workaround git failures with py3.10
153162
run: |
154163
git config --global --add safe.directory /__w/python-ecdsa/python-ecdsa
@@ -256,11 +265,15 @@ jobs:
256265
else
257266
pip install -r build-requirements.txt;
258267
fi
268+
- name: Install mutation testing dependencies
269+
if: ${{ matrix.mutation == 'true' }}
270+
run: |
271+
pip install cosmic-ray
259272
- name: Display installed python package versions
260273
run: pip list
261274
- name: Test native speed
262275
# tox uses pip to install dependenceis, so it breaks on py2.6
263-
if: ${{ !contains(matrix.tox-env, 'gmpy') && matrix.python-version != '2.6'}}
276+
if: ${{ !contains(matrix.tox-env, 'gmpy') && matrix.python-version != '2.6' && ! matrix.mutation && !contains(matrix.tox-env, 'codechecks') }}
264277
run: tox -e speed
265278
- name: Test speed with gmpy
266279
if: ${{ contains(matrix.tox-env, 'gmpyp') }}
@@ -277,6 +290,25 @@ jobs:
277290
- name: Run unit tests
278291
if: ${{ matrix.tox-env }}
279292
run: tox -e ${{ matrix.tox-env }}
293+
- name: Init for mutation testing in PR
294+
if: ${{ matrix.mutation == 'true' && github.event.pull_request }}
295+
run: |
296+
cosmic-ray init cosmic-ray.toml session-vs-master.sqlite
297+
git branch master origin/master
298+
cr-filter-git --config cosmic-ray.toml session-vs-master.sqlite
299+
cr-report session-vs-master.sqlite | tail -n 5
300+
- name: Exec mutation testing for PR
301+
if: ${{ matrix.mutation == 'true' && github.event.pull_request }}
302+
run: |
303+
cosmic-ray exec cosmic-ray.toml session-vs-master.sqlite
304+
- name: Check test coverage for PR
305+
if: ${{ matrix.mutation == 'true' && github.event.pull_request }}
306+
run: |
307+
# remove not-executed results
308+
sqlite3 session-vs-master.sqlite "DELETE from work_results WHERE work_results.worker_outcome = 'SKIPPED'"
309+
cr-report session-vs-master.sqlite | tail -n 5
310+
# check if executed have at most 5% survival rate
311+
cr-rate --fail-over 5 session-vs-master.sqlite
280312
- name: instrumental test coverage on PR
281313
if: ${{ contains(matrix.opt-deps, 'instrumental') && github.event.pull_request }}
282314
env:
@@ -337,3 +369,272 @@ jobs:
337369
COVERALLS_SERVICE_NAME: github
338370
run: |
339371
coveralls --finish
372+
373+
mutation-prepare:
374+
name: Prepare job files for the mutation runners
375+
runs-on: ubuntu-latest
376+
steps:
377+
- uses: actions/checkout@v2
378+
if: ${{ !matrix.container }}
379+
with:
380+
fetch-depth: 50
381+
- name: save session objects
382+
uses: actions/cache@v3
383+
with:
384+
path: |
385+
sessions/
386+
key: sessions-${{ github.sha }}
387+
- name: Install cosmic-ray
388+
run: |
389+
pip3 install cosmic-ray
390+
- name: Install dependencies
391+
run: |
392+
sudo apt-get install -y sqlite3
393+
- name: Display Python version
394+
run: python -c "import sys; print(sys.version)"
395+
- name: Create list of mutations
396+
run: |
397+
cosmic-ray init cosmic-ray.toml session.sqlite
398+
- name: Log number of jobs created
399+
run: |
400+
cr-report session.sqlite | tail -n 3
401+
- name: Split up mutations to workers
402+
run: |
403+
cp session.sqlite session-to_del.sqlite
404+
sqlite3 session-to_del.sqlite "$(cat sql/create_to_del.sql)"
405+
mkdir sessions
406+
for i in $(seq 0 19); do
407+
sed "s/%SHARD%/$i/" < sql/shard-db.sql > shard.sql
408+
cp session-to_del.sqlite session-$i.sqlite
409+
sqlite3 session-$i.sqlite "$(cat shard.sql)"
410+
mv session-$i.sqlite sessions/
411+
done
412+
mutation-execute:
413+
name: Execute mutation testing
414+
needs: mutation-prepare
415+
runs-on: ubuntu-latest
416+
strategy:
417+
fail-fast: false
418+
matrix:
419+
include:
420+
- name: 0
421+
- name: 1
422+
- name: 2
423+
- name: 3
424+
- name: 4
425+
- name: 5
426+
- name: 6
427+
- name: 7
428+
- name: 8
429+
- name: 9
430+
- name: 10
431+
- name: 11
432+
- name: 12
433+
- name: 13
434+
- name: 14
435+
- name: 15
436+
- name: 16
437+
- name: 17
438+
- name: 18
439+
- name: 19
440+
steps:
441+
- uses: actions/checkout@v2
442+
if: ${{ !matrix.container }}
443+
with:
444+
fetch-depth: 1
445+
- name: Session objects
446+
uses: actions/cache@v3
447+
with:
448+
path: |
449+
sessions/
450+
key: sessions-${{ github.sha }}
451+
- name: Session done objects
452+
uses: actions/cache@v3
453+
with:
454+
path: |
455+
sessions-done/session-${{ matrix.name }}-done.sqlite
456+
key: sessions-${{ github.sha }}-${{ matrix.name }}-done
457+
- name: Install gmpy2 dependencies
458+
run: sudo apt-get install -y libmpfr-dev libmpc-dev
459+
- name: Install gmpy2
460+
run: pip install gmpy2
461+
- name: Install build dependencies
462+
run: |
463+
pip install -r build-requirements.txt
464+
pip install cosmic-ray
465+
- name: Run mutation testing
466+
run: |
467+
cp sessions/session-${{ matrix.name }}.sqlite session.sqlite
468+
systemd-run --user --scope -p MemoryMax=2G -p MemoryHigh=2G cosmic-ray exec cosmic-ray.toml session.sqlite &
469+
cosmic_pid=$!
470+
for i in $(seq 1 10); do
471+
echo $i
472+
sleep 60
473+
done
474+
kill $cosmic_pid
475+
mkdir sessions-done/
476+
cp session.sqlite sessions-done/session-${{ matrix.name }}-done.sqlite
477+
- name: Report executed
478+
run: |
479+
cr-report session.sqlite | tail -n 3
480+
mutation-combine:
481+
name: Combine mutation testing results
482+
needs: mutation-execute
483+
runs-on: ubuntu-latest
484+
steps:
485+
- uses: actions/checkout@v2
486+
if: ${{ !matrix.container }}
487+
with:
488+
fetch-depth: 1
489+
- name: Session done 0 objects
490+
uses: actions/cache@v3
491+
with:
492+
path: |
493+
sessions-done/session-0-done.sqlite
494+
key: sessions-${{ github.sha }}-0-done
495+
- name: Session done 1 objects
496+
uses: actions/cache@v3
497+
with:
498+
path: |
499+
sessions-done/session-1-done.sqlite
500+
key: sessions-${{ github.sha }}-1-done
501+
- name: Session done 2 objects
502+
uses: actions/cache@v3
503+
with:
504+
path: |
505+
sessions-done/session-2-done.sqlite
506+
key: sessions-${{ github.sha }}-2-done
507+
- name: Session done 3 objects
508+
uses: actions/cache@v3
509+
with:
510+
path: |
511+
sessions-done/session-3-done.sqlite
512+
key: sessions-${{ github.sha }}-3-done
513+
- name: Session done 4 objects
514+
uses: actions/cache@v3
515+
with:
516+
path: |
517+
sessions-done/session-4-done.sqlite
518+
key: sessions-${{ github.sha }}-4-done
519+
- name: Session done 5 objects
520+
uses: actions/cache@v3
521+
with:
522+
path: |
523+
sessions-done/session-5-done.sqlite
524+
key: sessions-${{ github.sha }}-5-done
525+
- name: Session done 6 objects
526+
uses: actions/cache@v3
527+
with:
528+
path: |
529+
sessions-done/session-6-done.sqlite
530+
key: sessions-${{ github.sha }}-6-done
531+
- name: Session done 7 objects
532+
uses: actions/cache@v3
533+
with:
534+
path: |
535+
sessions-done/session-7-done.sqlite
536+
key: sessions-${{ github.sha }}-7-done
537+
- name: Session done 8 objects
538+
uses: actions/cache@v3
539+
with:
540+
path: |
541+
sessions-done/session-8-done.sqlite
542+
key: sessions-${{ github.sha }}-8-done
543+
- name: Session done 9 objects
544+
uses: actions/cache@v3
545+
with:
546+
path: |
547+
sessions-done/session-9-done.sqlite
548+
key: sessions-${{ github.sha }}-9-done
549+
- name: Session done 10 objects
550+
uses: actions/cache@v3
551+
with:
552+
path: |
553+
sessions-done/session-10-done.sqlite
554+
key: sessions-${{ github.sha }}-10-done
555+
- name: Session done 11 objects
556+
uses: actions/cache@v3
557+
with:
558+
path: |
559+
sessions-done/session-11-done.sqlite
560+
key: sessions-${{ github.sha }}-11-done
561+
- name: Session done 12 objects
562+
uses: actions/cache@v3
563+
with:
564+
path: |
565+
sessions-done/session-12-done.sqlite
566+
key: sessions-${{ github.sha }}-12-done
567+
- name: Session done 13 objects
568+
uses: actions/cache@v3
569+
with:
570+
path: |
571+
sessions-done/session-13-done.sqlite
572+
key: sessions-${{ github.sha }}-13-done
573+
- name: Session done 14 objects
574+
uses: actions/cache@v3
575+
with:
576+
path: |
577+
sessions-done/session-14-done.sqlite
578+
key: sessions-${{ github.sha }}-14-done
579+
- name: Session done 15 objects
580+
uses: actions/cache@v3
581+
with:
582+
path: |
583+
sessions-done/session-15-done.sqlite
584+
key: sessions-${{ github.sha }}-15-done
585+
- name: Session done 16 objects
586+
uses: actions/cache@v3
587+
with:
588+
path: |
589+
sessions-done/session-16-done.sqlite
590+
key: sessions-${{ github.sha }}-16-done
591+
- name: Session done 17 objects
592+
uses: actions/cache@v3
593+
with:
594+
path: |
595+
sessions-done/session-17-done.sqlite
596+
key: sessions-${{ github.sha }}-17-done
597+
- name: Session done 18 objects
598+
uses: actions/cache@v3
599+
with:
600+
path: |
601+
sessions-done/session-18-done.sqlite
602+
key: sessions-${{ github.sha }}-18-done
603+
- name: Session done 19 objects
604+
uses: actions/cache@v3
605+
with:
606+
path: |
607+
sessions-done/session-19-done.sqlite
608+
key: sessions-${{ github.sha }}-19-done
609+
- name: Install cosmic-ray
610+
run: |
611+
pip3 install cosmic-ray
612+
- name: Install dependencies
613+
run: |
614+
sudo apt-get install -y sqlite3
615+
- name: Combine worker results
616+
run: |
617+
cp sessions-done/session-0-done.sqlite session.sqlite
618+
for i in $(seq 1 19); do
619+
cp sessions-done/session-$i-done.sqlite session-to_merge.sqlite && sqlite3 session.sqlite "$(cat sql/combine.sql)" || true
620+
done
621+
- name: Report executed
622+
run: |
623+
cr-report session.sqlite | tail -n 3
624+
- name: Log survival estimate
625+
run: cr-rate --estimate --fail-over 32 --confidence 99.9 session.sqlite || true
626+
- name: Get mutation score
627+
run: |
628+
echo "print(100-$(cr-rate session.sqlite))" > print-score.py
629+
echo "MUT_SCORE=$(python print-score.py)" >> $GITHUB_ENV
630+
- name: Create mutation score badge
631+
uses: schneegans/[email protected]
632+
with:
633+
auth: ${{ secrets.GIST_SECRET }}
634+
gistID: 9b6ca1f3410207fbeca785a178781651
635+
filename: python-ecdsa-mutation-score.json
636+
label: mutation score
637+
message: ${{ env.MUT_SCORE }}%
638+
valColorRange: ${{ env.MUT_SCORE }}
639+
maxColorRange: 100
640+
minColorRange: 0

0 commit comments

Comments
 (0)