Switch KEM KAT checks to Python test suite

.gitignore

@@ -45,7 +45,6 @@ tests/test_sha3
 
 #kat
 tests/kat_kem
-kat_kem_rsp/
 
 # Debug files
 *.dSYM/

.travis/kat-check.sh

@@ -1,28 +0,0 @@
-#!/bin/bash
-
-###
-# Checks that all KATs pass.
-###
-
-set -e
-
-source $(dirname $0)/defs.sh
-
-RET=0
-
-./tests/kat_kem
-scripts/check_kats.sh
-error=$?
-if [ $error -eq 0 ];
-then
-	${PRINT_GREEN}
-	echo "All known answer tests passed.";
-	${PRINT_RESET}
-else
-	${PRINT_RED}
-	echo "Error in known answer tests.";
-	${PRINT_RESET}
-	RET=1
-fi
-
-exit ${RET}

Makefile.am

@@ -105,10 +105,6 @@ test: check
 	tests/test_sha3
 	python3 -m pytest -v
 
-kat: clean-kats check
-	tests/kat_kem
-	scripts/check_kats.sh
-
 links:
 	$(MKDIR_P) include/oqs
 	cp -f src/oqs.h include/oqs
@@ -144,9 +140,6 @@ clean-local:
 clean-tests:
 	rm -f tests/example_kem tests/example_sig tests/speed_kem tests/speed_sig tests/test_kem tests/test_sig tests/test_aes tests/test_sha3
 
-clean-kats:
-	rm -rf tests/kat_kem kat_kem_rsp/
-
 prettyprint:
 	find src tests -name '*.c' -o -name '*.h' | grep -v picnic/external* | grep -v frodokem/external* | grep -v pqclean | xargs $(CLANGFORMAT) -style=file -i
 

README.md

@@ -164,13 +164,17 @@ There are also a variety of test programs built under the `tests` directory:
 
 - `test_kem`: Simple test harness for all enabled key encapsulation mechanisms
 - `test_sig`: Simple test harness for all enabled key signature schemes
-- `kat_kem`: Program that generates known answer test (KAT) values for all enabled key encapsulation mechanisms using the same mechanism as the NIST submission requirements, for checking against submitted KAT values using `scripts/check_kats.sh`
+- `kat_kem`: Program that generates known answer test (KAT) values for all enabled key encapsulation mechanisms using the same mechanism as the NIST submission requirements, for checking against submitted KAT values using `tests/test_kat.py`
 - `speed_kem`: Benchmarking program for key encapsulation mechanisms; see `./speed_kem --help` for usage instructions
 - `speed_sig`: Benchmarking program for signature mechanisms; see `./speed_sig --help` for usage instructions
 - `example_kem`: Minimal runnable example showing the usage of the KEM API
 - `example_sig`: Minimal runnable example showing the usage of the signature API
 - `test_aes`, `test_sha3`: Simple test harnesses for crypto sub-components
 
+A range of tests can be run using
+
+	python3 -m pytest
+
 Building and running on Windows
 -------------------------------
 

VisualStudio/kat-check.bat

@@ -1,13 +0,0 @@
-@echo off
-
-REM compares KAT files for Windows-enabled KEM algs
-for /R src\kem\ %%K in (*.kat) DO (
-    if exist kat_kem_rsp\%%~nxK (
-         FC %%K kat_kem_rsp\%%~nxK >nul 2>nul
-    	 if errorlevel 1 (
-	    echo %%~nxK not matching
-	    exit /b 1
-	 )
-	 )
-)
-

appveyor.yml

@@ -12,6 +12,10 @@ branches:
     - /.*nist.*/
     - /master-new-.*/
 
+init:
+  - set PATH="C:\\Python37";"C:\\Python37\Scripts";%PATH%
+
+
 configuration:
   - Debug
   - DebugDLL
@@ -20,16 +24,16 @@ configuration:
 
 test_script:
 - cmd: >-
+    python -m pip install pytest
+    cd %APPVEYOR_BUILD_FOLDER%
+    python -m pytest -v
+
     %APPVEYOR_BUILD_FOLDER%\VisualStudio\%PLATFORM%\%CONFIGURATION%\test_kem.exe
 
     %APPVEYOR_BUILD_FOLDER%\VisualStudio\%PLATFORM%\%CONFIGURATION%\test_sig.exe
 
     %APPVEYOR_BUILD_FOLDER%\VisualStudio\%PLATFORM%\%CONFIGURATION%\example_kem.exe
 
-    %APPVEYOR_BUILD_FOLDER%\VisualStudio\%PLATFORM%\%CONFIGURATION%\kat_kem.exe
-
-    %APPVEYOR_BUILD_FOLDER%\VisualStudio\kat-check.bat
-
     %APPVEYOR_BUILD_FOLDER%\VisualStudio\%PLATFORM%\%CONFIGURATION%\speed_kem.exe
 
     %APPVEYOR_BUILD_FOLDER%\VisualStudio\%PLATFORM%\%CONFIGURATION%\speed_sig.exe

scripts/check_kats.sh

@@ -1,72 +0,0 @@
-#!/bin/bash
-
-###
-# Checks that all generated KATs match their upstream values.
-###
-
-PRINT_GREEN="tput setaf 2"
-PRINT_RED="tput setaf 1"
-PRINT_RESET="tput sgr 0"
-PRINT_YELLOW="tput setaf 3"
-
-RET=0
-
-ALGS=$(grep -E 'define OQS_(KEM|SIG)_alg_' src/kem/kem.h src/sig/sig.h | grep -v 'default' | grep -v 'sidh' | sed -e 's/^[^"]*"//' | sed -e 's/".*$//' | tr -d '[:blank:]')
-for alg in ${ALGS}; do
-
-	kat=$(find kat_*_rsp -name ${alg}.kat |tr '\n' ' ')
-	if [ -z "${kat}" ];
-	then
-		${PRINT_YELLOW}
-		echo "KAT file not generated for ${alg}"
-		${PRINT_RESET}
-		RET=0
-		continue
-	fi
-
-	origs=$(find src -name ${alg}.kat -o -name ${alg}.*.kat |tr '\n' ' ')
-	if [[ "x${origs}x" == "xx" ]];
-	then
-		${PRINT_RED}
-		echo "No original KAT file found for ${alg}"
-		${PRINT_RESET}
-		RET=1
-		continue
-	fi
-
-	match=0
-	for orig in ${origs}; do
-		diff ${orig} ${kat} > /dev/null 2>&1
-		error=$?
-		if [ $error -eq 0 ]
-		then
-			echo "KAT values match for ${alg} and ${orig}"
-			match=1
-			break
-		elif [ ! $error -eq 1 ]
-		then
-			${PRINT_RED}
-			echo "An error occurred in the diff command"
-			${PRINT_RESET}
-			RET=1
-		fi
-        done
-
-        if [ $match -eq 0 ]
-	then
-		${PRINT_RED}
-		echo "KAT values do not match for ${alg} with any of ${origs}"
-		${PRINT_RESET}
-		RET=1
-	fi
-done
-
-
-if [[ "${RET}" == "0" ]];
-then
-	${PRINT_GREEN}
-	echo "All KAT values matched.";
-	${PRINT_RESET}
-fi
-
-exit ${RET}

tests/helpers.py

@@ -48,3 +48,23 @@ def run_subprocess(command, working_dir='.', env=None, expected_returncode=0, in
     print(stdout.decode('utf-8'))
     assert retcode == expected_returncode, "Got unexpected return code {}".format(retcode)
     return stdout.decode('utf-8')
+
+def enabled_kems_by_name():
+    enabled_symbols = []
+    with open('include/oqs/oqsconfig.h') as fh:
+        for line in fh:
+            if line.startswith("#define OQS_ENABLE_KEM_"):
+                kem_symbol = line.split(' ')[1]
+                kem_symbol = kem_symbol[len("OQS_ENABLE_KEM_"):]
+                enabled_symbols.append(kem_symbol)
+    enabled_names = []
+    with open('include/oqs/kem.h') as fh:
+        for line in fh:
+            if line.startswith("#define OQS_KEM_alg"):
+                kem_symbol = line.split(' ')[1]
+                kem_symbol = kem_symbol[len("OQS_KEM_alg_"):]
+                kem_name = line.split(' ')[2]
+                kem_name = kem_name[1:-2]
+                if kem_symbol in enabled_symbols:
+                    enabled_names.append(kem_name)
+    return enabled_names

tests/kat_kem.c

@@ -40,7 +40,6 @@ OQS_STATUS kem_kat(const char *method_name) {
 	uint8_t *ciphertext = NULL;
 	uint8_t *shared_secret_e = NULL;
 	uint8_t *shared_secret_d = NULL;
-	char filename[200];
 	OQS_STATUS rc, ret = OQS_ERROR;
 	int rv;
 
@@ -60,12 +59,7 @@ OQS_STATUS kem_kat(const char *method_name) {
 	}
 	OQS_randombytes_nist_kat_init(entropy_input, NULL, 256);
 
-	sprintf(filename, "kat_kem_rsp/%s.kat", method_name);
-
-	fh = fopen(filename, "w");
-	if (fh == NULL) {
-		goto err;
-	}
+	fh = stdout;
 
 	fprintf(fh, "count = 0\n");
 	OQS_randombytes(seed, 48);
@@ -123,9 +117,6 @@ algo_not_enabled:
 	ret = OQS_SUCCESS;
 
 cleanup:
-	if (fh != NULL) {
-		fclose(fh);
-	}
 	if (kem != NULL) {
 		OQS_MEM_secure_free(secret_key, kem->length_secret_key);
 		OQS_MEM_secure_free(shared_secret_e, kem->length_shared_secret);
@@ -137,27 +128,25 @@ cleanup:
 	return ret;
 }
 
-int main() {
-
-	int ret = EXIT_SUCCESS;
-	OQS_STATUS rc;
-
-	int status;
-#if defined(_WIN32)
-	status = _mkdir("kat_kem_rsp");
-#else
-	status = mkdir("kat_kem_rsp", S_IRWXU);
-#endif
-	if (!((status == 0) || (errno == EEXIST))) {
+int main(int argc, char **argv) {
+	
+	if (argc != 2) {
+		fprintf(stderr, "Usage: kat_kem algname\n");
+		fprintf(stderr, "  algname: ");
+		for (size_t i = 0; i < OQS_KEM_algs_length; i++) {
+			if (i > 0) {
+				fprintf(stderr, ", ");
+			}
+			fprintf(stderr, "%s", OQS_KEM_alg_identifier(i));
+		}
+		fprintf(stderr, "\n");
 		return EXIT_FAILURE;
 	}
 
-	for (size_t i = 0; i < OQS_KEM_algs_length; i++) {
-		rc = kem_kat(OQS_KEM_alg_identifier(i));
-		if (rc != OQS_SUCCESS) {
-			ret = EXIT_FAILURE;
-		}
+	char *alg_name = argv[1];
+	OQS_STATUS rc = kem_kat(alg_name);
+	if (rc != OQS_SUCCESS) {
+		return EXIT_FAILURE;
 	}
-
-	return ret;
+	return EXIT_SUCCESS;
 }

tests/test_hash.py

@@ -1,11 +1,13 @@
 import hashlib
 import helpers
+import os.path
 import pytest
 
 @pytest.mark.parametrize('msg', ['', 'a', 'abc', '1234567890123456789012345678901678901567890'])
 def test_sha256(msg):
     output = helpers.run_subprocess(
-        ['tests/test_hash', 'sha256'],
+        [os.path.join('.', 'test_hash'), 'sha256'],
+        working_dir = 'tests',
         input = msg.encode(),
     )
     assert(output.rstrip() == hashlib.sha256(msg.encode()).hexdigest())
@@ -13,7 +15,8 @@ def test_sha256(msg):
 @pytest.mark.parametrize('msg', ['', 'a', 'abc', '1234567890123456789012345678901678901567890'])
 def test_sha384(msg):
     output = helpers.run_subprocess(
-        ['tests/test_hash', 'sha384'],
+        [os.path.join('.', 'test_hash'), 'sha384'],
+        working_dir = 'tests',
         input = msg.encode(),
     )
     assert(output.rstrip() == hashlib.sha384(msg.encode()).hexdigest())
@@ -21,7 +24,8 @@ def test_sha384(msg):
 @pytest.mark.parametrize('msg', ['', 'a', 'abc', '1234567890123456789012345678901678901567890'])
 def test_sha512(msg):
     output = helpers.run_subprocess(
-        ['tests/test_hash', 'sha512'],
+        [os.path.join('.', 'test_hash'), 'sha512'],
+        working_dir = 'tests',
         input = msg.encode(),
     )
     assert(output.rstrip() == hashlib.sha512(msg.encode()).hexdigest())

tests/test_kat.py

@@ -0,0 +1,25 @@
+import hashlib
+import helpers
+import os
+import os.path
+import pytest
+
+@pytest.mark.parametrize('kem_name', helpers.enabled_kems_by_name())
+def test_kem(kem_name):
+    if kem_name.startswith('Sidh'): pytest.skip('KATs not available for SIDH')
+    output = helpers.run_subprocess(
+        ['kat_kem', kem_name],
+        working_dir = 'tests'
+    )
+    kats = []
+    for file in os.scandir(os.path.join('tests', 'KATs', 'kem')):
+        filename = file.name
+        if filename.startswith(kem_name + '.') and filename.endswith('.kat'):
+            with open(os.path.join('tests', 'KATs', 'kem', filename), 'r') as myfile:
+                kats.append(myfile.read())
+    assert(output in kats)
+
+if __name__ == "__main__":
+    import sys
+    pytest.main(sys.argv)
+