sharpetronics/liboqs
1
0
Fork 0
mirror of https://github.com/open-quantum-safe/liboqs.git synced 2025-12-07 00:02:12 -05:00
Code Issues Projects Releases Wiki Activity

release performance regression test (#1152)

Browse Source 
* performance regression test script [skip ci]

* parameterized no-regress testing [skip ci]
This commit is contained in:
Michael Baentsch 2021-12-11 19:51:15 +01:00 committed by GitHub
parent 0a61d5d066
commit ca0cd60a1b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 59 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
scripts/noregress.py Normal file
View File

@ -0,0 +1,18 @@
import json
import sys
import os
cutoffpercent = 15
with open(sys.argv[1], 'r') as json_file:
d1 = json.load(json_file)
with open(sys.argv[2], 'r') as json_file:
d2 = json.load(json_file)
for k in d1.keys(): # algs
if k != "config" and k != "cpuinfo":
if k in d2.keys():
for op in d1[k]:
diff = 100.0*(float(d1[k][op]) - float(d2[k][op]))/float(d1[k][op])
if (abs(diff) > cutoffpercent):
print("Alg: %s, Op: %s, Val1: %s; Val2: %s; diff: %2.2f%%" % (k, op, d1[k][op], d2[k][op], diff))

41
scripts/noregress.sh Executable file
View File

@ -0,0 +1,41 @@
#!/bin/bash
if [ $# -lt 1 ]; then
echo "Usage: $0 <liboqs-release to test regression against> [<cmake opts> [<make cmd>]]. Exiting."
exit -1
fi
if [ $# -lt 3 ]; then
MAKECMD="make -j 2"
else
MAKECMD=$3
fi
# Approach: Check out $1 into tmp folder, build, run speed_kem|sig and compare results
mkdir tmp && cd tmp && git clone --depth 1 --branch $1 https://github.com/open-quantum-safe/liboqs && cd liboqs && mkdir build && cd build && cmake $2 .. && $MAKECMD && ./tests/speed_kem > ../../speed_kem.log && ./tests/speed_sig > ../../speed_sig.log && cd ../../..
if [ $? -ne 0 ]; then
echo "Build and test of baseline $1 failed. Exiting."
exit -1
fi
# transform results into JSON files for simple comparison
cd tmp && git clone --depth 1 https://github.com/open-quantum-safe/profiling.git && python3 profiling/perf/scripts/parse_liboqs_speed.py speed_kem.log && python3 profiling/perf/scripts/parse_liboqs_speed.py speed_sig.log && cd ..
if [ $? -ne 0 ]; then
echo "Failure converting results. Exiting."
exit -1
fi
# obtain current base speed results
rm -rf build && mkdir build && cd build && cmake $2 .. && $MAKECMD && ./tests/speed_kem > speed_kem.log && ./tests/speed_sig > speed_sig.log && python3 ../tmp/profiling/perf/scripts/parse_liboqs_speed.py speed_kem.log && python3 ../tmp/profiling/perf/scripts/parse_liboqs_speed.py speed_sig.log && cd ..
if [ $? -ne 0 ]; then
echo "Failure creating current results. Exiting."
exit -1
fi
# now compare results using old/tmp runs as baseline (for list of algorithms)
python3 scripts/noregress.py tmp/speed_kem.json build/speed_kem.json && python3 scripts/noregress.py tmp/speed_sig.json build/speed_sig.json