mirror of
https://github.com/open-quantum-safe/liboqs.git
synced 2025-10-03 00:02:36 -04:00
Continuous Benchmarking using Github Actions (#2134)
* Added workflows and script for speed beanchmarking Signed-off-by: Pablo Gutiérrez Félix <pablogf@uma.es> * changed branch push to main Signed-off-by: Pablo Gutiérrez Félix <pablogf@uma.es> * Added SPDX-License-Identifer Signed-off-by: Pablo Gutiérrez Félix <pablogf@uma.es> * Fixed github security warnings Signed-off-by: Pablo Gutiérrez <pablogf@uma.es> * Fixed github security warnings 2 Signed-off-by: Pablo Gutiérrez <pablogf@uma.es> * Fixes after commit-to-main tests Signed-off-by: Pablo Gutiérrez <pablogf@uma.es> --------- Signed-off-by: Pablo Gutiérrez Félix <pablogf@uma.es> Signed-off-by: Pablo Gutiérrez <pablogf@uma.es>
This commit is contained in:
Pablo Gutiérrez
GitHub
parent
708b1052d5
commit
d745d35938
10
.github/workflows/commit-to-main.yml
vendored
10
.github/workflows/commit-to-main.yml
vendored
@ -22,3 +22,13 @@ jobs:
|
||||
basic-downstream:
|
||||
uses: ./.github/workflows/downstream-basic.yml
|
||||
secrets: inherit
|
||||
|
||||
call-kem-benchmarking:
|
||||
uses: ./.github/workflows/kem-bench.yml
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
call-sig-benchmarking:
|
||||
uses: ./.github/workflows/sig-bench.yml
|
||||
permissions:
|
||||
contents: write
|
||||
121
.github/workflows/kem-bench.yml
vendored
Normal file
121
.github/workflows/kem-bench.yml
vendored
Normal file
@ -0,0 +1,121 @@
|
||||
name: kem benchmark
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
workflow_call:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
# Checkout repository
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # pin@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Set up dependencies
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y cmake ninja-build gcc g++ python3 python3-pip
|
||||
sudo apt-get install -y python3-cpuinfo
|
||||
|
||||
# Build the speed_kem binary only
|
||||
- name: Build speed_kem binary
|
||||
run: |
|
||||
mkdir -p build
|
||||
cd build
|
||||
cmake -GNinja .. -DBUILD_SHARED_LIBS=OFF
|
||||
ninja speed_kem
|
||||
|
||||
# Copy the parse_liboqs_speed.py script
|
||||
- name: Copy parse_liboqs_speed.py
|
||||
run: |
|
||||
cp scripts/parse_liboqs_speed.py build/tests/
|
||||
|
||||
# Upload the built binary and script as an artifact
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@6027e3dd177782cd8ab9af838c04fd81a07f1d47
|
||||
with:
|
||||
name: built-binary
|
||||
path: build/tests/
|
||||
|
||||
benchmark:
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
strategy:
|
||||
matrix:
|
||||
algorithm: [ # List of available KEMs to perform the benchmarking on
|
||||
"BIKE-L1",
|
||||
"BIKE-L3",
|
||||
"BIKE-L5",
|
||||
"Classic-McEliece-348864",
|
||||
"Classic-McEliece-348864f",
|
||||
"Classic-McEliece-460896",
|
||||
"Classic-McEliece-460896f",
|
||||
"Classic-McEliece-6688128",
|
||||
"Classic-McEliece-6688128f",
|
||||
"Classic-McEliece-6960119",
|
||||
"Classic-McEliece-6960119f",
|
||||
"Classic-McEliece-8192128",
|
||||
"Classic-McEliece-8192128f",
|
||||
"Kyber512",
|
||||
"Kyber768",
|
||||
"Kyber1024",
|
||||
"ML-KEM-512",
|
||||
"ML-KEM-768",
|
||||
"ML-KEM-1024",
|
||||
"sntrup761",
|
||||
"FrodoKEM-640-AES",
|
||||
"FrodoKEM-640-SHAKE",
|
||||
"FrodoKEM-976-AES",
|
||||
"FrodoKEM-976-SHAKE",
|
||||
"FrodoKEM-1344-AES",
|
||||
"FrodoKEM-1344-SHAKE"
|
||||
]
|
||||
max-parallel: 1 # No parallel jobs to not compromise the pull-push operations of the benchmarking actions below
|
||||
|
||||
steps:
|
||||
# Ensure the repository is checked out
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # pin@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Download the built binary and script
|
||||
- name: Download artifacts
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # pin@v4
|
||||
with:
|
||||
name: built-binary
|
||||
path: build/tests/
|
||||
|
||||
# Set execute permissions for the binary
|
||||
- name: Set execute permissions
|
||||
run: chmod +x build/tests/speed_kem
|
||||
|
||||
# Run speed_kem tests for each algorithm
|
||||
- name: Run speed_kem tests
|
||||
run: |
|
||||
cd build/tests
|
||||
./speed_kem "${{matrix.algorithm}}" > ${{matrix.algorithm}}_output.txt
|
||||
python3 parse_liboqs_speed.py ${{matrix.algorithm}}_output.txt --algorithm ${{matrix.algorithm}}
|
||||
|
||||
# Push to GitHub pages using continuous-benchmark
|
||||
- name: Store benchmark result
|
||||
uses: benchmark-action/github-action-benchmark@d48d326b4ca9ba73ca0cd0d59f108f9e02a381c7
|
||||
with:
|
||||
name: ${{matrix.algorithm}}
|
||||
tool: "customSmallerIsBetter"
|
||||
output-file-path: build/tests/${{matrix.algorithm}}_formatted.json
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
auto-push: true
|
||||
comment-on-alert: true
|
||||
summary-always: true
|
||||
alert-threshold: 50%
|
||||
comment-always: true
|
||||
151
.github/workflows/sig-bench.yml
vendored
Normal file
151
.github/workflows/sig-bench.yml
vendored
Normal file
@ -0,0 +1,151 @@
|
||||
name: sig benchmark
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
workflow_call:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
# Checkout repository
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # pin@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Set up dependencies
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y cmake ninja-build gcc g++ python3 python3-pip
|
||||
sudo apt-get install -y python3-cpuinfo
|
||||
|
||||
# Build the speed_sig binary only
|
||||
- name: Build speed_sig binary
|
||||
run: |
|
||||
mkdir -p build
|
||||
cd build
|
||||
cmake -GNinja .. -DBUILD_SHARED_LIBS=OFF
|
||||
ninja speed_sig
|
||||
|
||||
# Copy the parse_liboqs_speed.py script
|
||||
- name: Copy parse_liboqs_speed.py
|
||||
run: |
|
||||
cp scripts/parse_liboqs_speed.py build/tests/
|
||||
|
||||
# Upload the built binary and script as an artifact
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@6027e3dd177782cd8ab9af838c04fd81a07f1d47
|
||||
with:
|
||||
name: built-sig-binary
|
||||
path: build/tests/
|
||||
|
||||
benchmark:
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
strategy:
|
||||
matrix:
|
||||
algorithm: [ # List of available signatures to perform the benchmarking on
|
||||
"Dilithium2",
|
||||
"Dilithium3",
|
||||
"Dilithium5",
|
||||
"ML-DSA-44",
|
||||
"ML-DSA-65",
|
||||
"ML-DSA-87",
|
||||
"Falcon-512",
|
||||
"Falcon-1024",
|
||||
"Falcon-padded-512",
|
||||
"Falcon-padded-1024",
|
||||
"SPHINCS+-SHA2-128f-simple",
|
||||
"SPHINCS+-SHA2-128s-simple",
|
||||
"SPHINCS+-SHA2-192f-simple",
|
||||
"SPHINCS+-SHA2-192s-simple",
|
||||
"SPHINCS+-SHA2-256f-simple",
|
||||
"SPHINCS+-SHA2-256s-simple",
|
||||
"SPHINCS+-SHAKE-128f-simple",
|
||||
"SPHINCS+-SHAKE-128s-simple",
|
||||
"SPHINCS+-SHAKE-192f-simple",
|
||||
"SPHINCS+-SHAKE-192s-simple",
|
||||
"SPHINCS+-SHAKE-256f-simple",
|
||||
"SPHINCS+-SHAKE-256s-simple",
|
||||
"MAYO-1",
|
||||
"MAYO-2",
|
||||
"MAYO-3",
|
||||
"MAYO-5",
|
||||
"cross-rsdp-128-balanced",
|
||||
"cross-rsdp-128-fast",
|
||||
"cross-rsdp-128-small",
|
||||
"cross-rsdp-192-balanced",
|
||||
"cross-rsdp-192-fast",
|
||||
"cross-rsdp-192-small",
|
||||
"cross-rsdp-256-balanced",
|
||||
"cross-rsdp-256-fast",
|
||||
"cross-rsdp-256-small",
|
||||
"cross-rsdpg-128-balanced",
|
||||
"cross-rsdpg-128-fast",
|
||||
"cross-rsdpg-128-small",
|
||||
"cross-rsdpg-192-balanced",
|
||||
"cross-rsdpg-192-fast",
|
||||
"cross-rsdpg-192-small",
|
||||
"cross-rsdpg-256-balanced",
|
||||
"cross-rsdpg-256-fast",
|
||||
"cross-rsdpg-256-small",
|
||||
"OV-Is",
|
||||
"OV-Ip",
|
||||
"OV-III",
|
||||
"OV-V",
|
||||
"OV-Is-pkc",
|
||||
"OV-Ip-pkc",
|
||||
"OV-III-pkc",
|
||||
"OV-V-pkc",
|
||||
"OV-Is-pkc-skc",
|
||||
"OV-Ip-pkc-skc",
|
||||
"OV-III-pkc-skc",
|
||||
"OV-V-pkc-skc"
|
||||
]
|
||||
max-parallel: 1 # No parallel jobs to not compromise the pull-push operations of the benchmarking actions below
|
||||
|
||||
steps:
|
||||
# Ensure the repository is checked out
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # pin@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
# Download the built binary and script
|
||||
- name: Download artifacts
|
||||
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # pin@v4
|
||||
with:
|
||||
name: built-sig-binary
|
||||
path: build/tests/
|
||||
|
||||
# Set execute permissions for the binary
|
||||
- name: Set execute permissions
|
||||
run: chmod +x build/tests/speed_sig
|
||||
|
||||
# Run speed_sig tests for each algorithm
|
||||
- name: Run speed_sig tests
|
||||
run: |
|
||||
cd build/tests
|
||||
./speed_sig "${{matrix.algorithm}}" > ${{matrix.algorithm}}_output.txt
|
||||
python3 parse_liboqs_speed.py ${{matrix.algorithm}}_output.txt --algorithm ${{matrix.algorithm}}
|
||||
|
||||
# Push to GitHub pages using continuous-benchmark
|
||||
- name: Store benchmark result
|
||||
uses: benchmark-action/github-action-benchmark@d48d326b4ca9ba73ca0cd0d59f108f9e02a381c7
|
||||
with:
|
||||
name: ${{matrix.algorithm}}
|
||||
tool: "customSmallerIsBetter"
|
||||
output-file-path: build/tests/${{matrix.algorithm}}_formatted.json
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
auto-push: true
|
||||
comment-on-alert: true
|
||||
summary-always: true
|
||||
alert-threshold: 50%
|
||||
comment-always: true
|
||||
71
scripts/parse_liboqs_speed.py
Normal file
71
scripts/parse_liboqs_speed.py
Normal file
@ -0,0 +1,71 @@
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
import json
|
||||
import re
|
||||
import argparse
|
||||
from enum import Enum
|
||||
|
||||
class State(Enum):
|
||||
starting=0
|
||||
config=1
|
||||
parsing=2
|
||||
|
||||
data=[]
|
||||
|
||||
# Parse command-line arguments
|
||||
parser = argparse.ArgumentParser(description="Parse speed_kem output and extract cycles.")
|
||||
parser.add_argument("logfile", help="Log file to parse")
|
||||
parser.add_argument("--algorithm", help="Algorithm name (e.g., BIKE-L1)", required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
fn = args.logfile
|
||||
alg = args.algorithm
|
||||
state = State.starting
|
||||
|
||||
config = ''
|
||||
|
||||
with open(fn) as fp:
|
||||
while True:
|
||||
line = fp.readline()
|
||||
if not line:
|
||||
break
|
||||
# Remove newlines
|
||||
line = line.rstrip()
|
||||
if state==State.starting:
|
||||
if line.startswith("Configuration info"):
|
||||
state=State.config
|
||||
fp.readline()
|
||||
elif state==State.config:
|
||||
if line=="\n": # Skip forward
|
||||
fp.readline()
|
||||
fp.readline()
|
||||
if line.startswith("-------"):
|
||||
state=State.parsing
|
||||
elif line.startswith("Started at"):
|
||||
fp.readline()
|
||||
elif ":" in line:
|
||||
config = config + line[:line.index(":")] + ": " + line[line.index(":")+1:].lstrip() + " | " # Retrieve build configuration
|
||||
|
||||
elif state==State.parsing:
|
||||
if line.startswith("Ended"): # Finish
|
||||
break
|
||||
else:
|
||||
alg = line[:line.index(" ")]
|
||||
p = re.compile('\S+\s*\|')
|
||||
for i in 0,1,2: # Iterate through the different operations under each algorithm
|
||||
x=p.findall(fp.readline().rstrip())
|
||||
tag = x[0][:x[0].index(" ")] # keygen, encaps, decaps
|
||||
iterations = float(x[1][:x[1].index(" ")]) # Iterations
|
||||
total_t = float(x[2][:x[2].index(" ")]) # Total time
|
||||
mean_t = float(x[3][:x[3].index(" ")]) # Mean time in microseconds
|
||||
cycles = int(x[5][:x[5].index(" ")]) # Cycles
|
||||
val = iterations/total_t # Number of iterations per second
|
||||
|
||||
data.append({"name": alg + " " + tag, "value": cycles, "unit": "cycles", "extra": config})
|
||||
else:
|
||||
print("Unknown state: %s" % (line))
|
||||
|
||||
# Dump data
|
||||
output_file = f"{alg}_formatted.json"
|
||||
with open(output_file, 'w') as outfile:
|
||||
json.dump(data, outfile)
|
||||
Loading…
x
Reference in New Issue
Block a user