mirror of
https://github.com/open-quantum-safe/liboqs.git
synced 2025-10-04 00:02:01 -04:00
do full upstream doc generation (#1066)
* do full upstream doc generation * added Common_META merge logic * integrated src and doc upstream copy * 'upstream' information auto-generated * sort 'required_flags' for reproducability
This commit is contained in:
Michael Baentsch
GitHub
parent
75b648e9f6
commit
c0a550f736
@ -25,7 +25,7 @@
|
||||
| Identifier in upstream | Supported architecture(s) | Supported operating system(s) | CPU extension(s) used | No branching-on-secrets claimed? | No branching-on-secrets checked by valgrind? | Large stack usage?‡ |
|
||||
|:------------------------:|:----------------------------|:--------------------------------|:------------------------|:-----------------------------------|:-----------------------------------------------|:----------------------|
|
||||
| ref | All | All | None | True | True | False |
|
||||
| avx2 | x86\_64 | Linux,Darwin | AVX2,POPCNT,BMI2 | True | True | False |
|
||||
| avx2 | x86\_64 | Linux,Darwin | AVX2,BMI2,POPCNT | True | True | False |
|
||||
|
||||
Are implementations chosen based on runtime CPU feature detection? **Yes**.
|
||||
|
||||
@ -36,7 +36,7 @@ Are implementations chosen based on runtime CPU feature detection? **Yes**.
|
||||
| Identifier in upstream | Supported architecture(s) | Supported operating system(s) | CPU extension(s) used | No branching-on-secrets claimed? | No branching-on-secrets checked by valgrind? | Large stack usage? |
|
||||
|:------------------------:|:----------------------------|:--------------------------------|:--------------------------------|:-----------------------------------|:-----------------------------------------------|:---------------------|
|
||||
| ref | All | All | None | True | True | False |
|
||||
| avx2 | x86\_64 | Linux,Darwin | SSE2,POPCNT,BMI2,SSSE3,AVX2,AES | True | True | False |
|
||||
| avx2 | x86\_64 | Linux,Darwin | AES,AVX2,BMI2,POPCNT,SSE2,SSSE3 | True | True | False |
|
||||
|
||||
Are implementations chosen based on runtime CPU feature detection? **Yes**.
|
||||
|
||||
@ -45,7 +45,7 @@ Are implementations chosen based on runtime CPU feature detection? **Yes**.
|
||||
| Identifier in upstream | Supported architecture(s) | Supported operating system(s) | CPU extension(s) used | No branching-on-secrets claimed? | No branching-on-secrets checked by valgrind? | Large stack usage? |
|
||||
|:------------------------:|:----------------------------|:--------------------------------|:------------------------|:-----------------------------------|:-----------------------------------------------|:---------------------|
|
||||
| ref | All | All | None | True | True | False |
|
||||
| avx2 | x86\_64 | Linux,Darwin | AVX2,POPCNT,BMI2 | True | True | False |
|
||||
| avx2 | x86\_64 | Linux,Darwin | AVX2,BMI2,POPCNT | True | True | False |
|
||||
|
||||
Are implementations chosen based on runtime CPU feature detection? **Yes**.
|
||||
|
||||
@ -54,7 +54,7 @@ Are implementations chosen based on runtime CPU feature detection? **Yes**.
|
||||
| Identifier in upstream | Supported architecture(s) | Supported operating system(s) | CPU extension(s) used | No branching-on-secrets claimed? | No branching-on-secrets checked by valgrind? | Large stack usage? |
|
||||
|:------------------------:|:----------------------------|:--------------------------------|:--------------------------------|:-----------------------------------|:-----------------------------------------------|:---------------------|
|
||||
| ref | All | All | None | True | True | False |
|
||||
| avx2 | x86\_64 | Linux,Darwin | SSE2,POPCNT,BMI2,SSSE3,AVX2,AES | True | True | False |
|
||||
| avx2 | x86\_64 | Linux,Darwin | AES,AVX2,BMI2,POPCNT,SSE2,SSSE3 | True | True | False |
|
||||
|
||||
Are implementations chosen based on runtime CPU feature detection? **Yes**.
|
||||
|
||||
@ -63,7 +63,7 @@ Are implementations chosen based on runtime CPU feature detection? **Yes**.
|
||||
| Identifier in upstream | Supported architecture(s) | Supported operating system(s) | CPU extension(s) used | No branching-on-secrets claimed? | No branching-on-secrets checked by valgrind? | Large stack usage? |
|
||||
|:------------------------:|:----------------------------|:--------------------------------|:------------------------|:-----------------------------------|:-----------------------------------------------|:---------------------|
|
||||
| ref | All | All | None | True | True | False |
|
||||
| avx2 | x86\_64 | Linux,Darwin | AVX2,POPCNT,BMI2 | True | True | False |
|
||||
| avx2 | x86\_64 | Linux,Darwin | AVX2,BMI2,POPCNT | True | True | False |
|
||||
|
||||
Are implementations chosen based on runtime CPU feature detection? **Yes**.
|
||||
|
||||
@ -72,7 +72,7 @@ Are implementations chosen based on runtime CPU feature detection? **Yes**.
|
||||
| Identifier in upstream | Supported architecture(s) | Supported operating system(s) | CPU extension(s) used | No branching-on-secrets claimed? | No branching-on-secrets checked by valgrind? | Large stack usage? |
|
||||
|:------------------------:|:----------------------------|:--------------------------------|:--------------------------------|:-----------------------------------|:-----------------------------------------------|:---------------------|
|
||||
| ref | All | All | None | True | True | False |
|
||||
| avx2 | x86\_64 | Linux,Darwin | SSE2,POPCNT,BMI2,SSSE3,AVX2,AES | True | True | False |
|
||||
| avx2 | x86\_64 | Linux,Darwin | AES,AVX2,BMI2,POPCNT,SSE2,SSSE3 | True | True | False |
|
||||
|
||||
Are implementations chosen based on runtime CPU feature detection? **Yes**.
|
||||
|
||||
|
||||
@ -44,8 +44,8 @@ parameter-sets:
|
||||
- Darwin
|
||||
required_flags:
|
||||
- avx2
|
||||
- popcnt
|
||||
- bmi2
|
||||
- popcnt
|
||||
common-crypto:
|
||||
- SHA3: liboqs
|
||||
no-secret-dependent-branching-claimed: true
|
||||
@ -75,12 +75,12 @@ parameter-sets:
|
||||
- Linux
|
||||
- Darwin
|
||||
required_flags:
|
||||
- sse2
|
||||
- popcnt
|
||||
- bmi2
|
||||
- ssse3
|
||||
- avx2
|
||||
- aes
|
||||
- avx2
|
||||
- bmi2
|
||||
- popcnt
|
||||
- sse2
|
||||
- ssse3
|
||||
common-crypto:
|
||||
- AES: pqcrystals-kyber_common_aes
|
||||
- SHA3: liboqs
|
||||
@ -111,8 +111,8 @@ parameter-sets:
|
||||
- Darwin
|
||||
required_flags:
|
||||
- avx2
|
||||
- popcnt
|
||||
- bmi2
|
||||
- popcnt
|
||||
common-crypto:
|
||||
- SHA3: liboqs
|
||||
no-secret-dependent-branching-claimed: true
|
||||
@ -142,12 +142,12 @@ parameter-sets:
|
||||
- Linux
|
||||
- Darwin
|
||||
required_flags:
|
||||
- sse2
|
||||
- popcnt
|
||||
- bmi2
|
||||
- ssse3
|
||||
- avx2
|
||||
- aes
|
||||
- avx2
|
||||
- bmi2
|
||||
- popcnt
|
||||
- sse2
|
||||
- ssse3
|
||||
common-crypto:
|
||||
- AES: pqcrystals-kyber_common_aes
|
||||
- SHA3: liboqs
|
||||
@ -178,8 +178,8 @@ parameter-sets:
|
||||
- Darwin
|
||||
required_flags:
|
||||
- avx2
|
||||
- popcnt
|
||||
- bmi2
|
||||
- popcnt
|
||||
common-crypto:
|
||||
- SHA3: liboqs
|
||||
no-secret-dependent-branching-claimed: true
|
||||
@ -209,12 +209,12 @@ parameter-sets:
|
||||
- Linux
|
||||
- Darwin
|
||||
required_flags:
|
||||
- sse2
|
||||
- popcnt
|
||||
- bmi2
|
||||
- ssse3
|
||||
- avx2
|
||||
- aes
|
||||
- avx2
|
||||
- bmi2
|
||||
- popcnt
|
||||
- sse2
|
||||
- ssse3
|
||||
common-crypto:
|
||||
- AES: pqcrystals-kyber_common_aes
|
||||
- SHA3: liboqs
|
||||
|
||||
@ -54,7 +54,7 @@ Are implementations chosen based on runtime CPU feature detection? **Yes**.
|
||||
| Identifier in upstream | Supported architecture(s) | Supported operating system(s) | CPU extension(s) used | No branching-on-secrets claimed? | No branching-on-secrets checked by valgrind? | Large stack usage? |
|
||||
|:------------------------:|:----------------------------|:--------------------------------|:---------------------------|:-----------------------------------|:-----------------------------------------------|:---------------------|
|
||||
| ref | All | All | None | True | True | False |
|
||||
| avx2 | x86\_64 | Darwin,Linux | SSE2,POPCNT,SSSE3,AVX2,AES | True | True | False |
|
||||
| avx2 | x86\_64 | Darwin,Linux | AES,AVX2,POPCNT,SSE2,SSSE3 | True | True | False |
|
||||
|
||||
Are implementations chosen based on runtime CPU feature detection? **Yes**.
|
||||
|
||||
@ -63,7 +63,7 @@ Are implementations chosen based on runtime CPU feature detection? **Yes**.
|
||||
| Identifier in upstream | Supported architecture(s) | Supported operating system(s) | CPU extension(s) used | No branching-on-secrets claimed? | No branching-on-secrets checked by valgrind? | Large stack usage? |
|
||||
|:------------------------:|:----------------------------|:--------------------------------|:---------------------------|:-----------------------------------|:-----------------------------------------------|:---------------------|
|
||||
| ref | All | All | None | True | True | False |
|
||||
| avx2 | x86\_64 | Darwin,Linux | SSE2,POPCNT,SSSE3,AVX2,AES | True | True | False |
|
||||
| avx2 | x86\_64 | Darwin,Linux | AES,AVX2,POPCNT,SSE2,SSSE3 | True | True | False |
|
||||
|
||||
Are implementations chosen based on runtime CPU feature detection? **Yes**.
|
||||
|
||||
@ -72,7 +72,7 @@ Are implementations chosen based on runtime CPU feature detection? **Yes**.
|
||||
| Identifier in upstream | Supported architecture(s) | Supported operating system(s) | CPU extension(s) used | No branching-on-secrets claimed? | No branching-on-secrets checked by valgrind? | Large stack usage? |
|
||||
|:------------------------:|:----------------------------|:--------------------------------|:---------------------------|:-----------------------------------|:-----------------------------------------------|:---------------------|
|
||||
| ref | All | All | None | True | True | False |
|
||||
| avx2 | x86\_64 | Darwin,Linux | SSE2,POPCNT,SSSE3,AVX2,AES | True | True | False |
|
||||
| avx2 | x86\_64 | Darwin,Linux | AES,AVX2,POPCNT,SSE2,SSSE3 | True | True | False |
|
||||
|
||||
Are implementations chosen based on runtime CPU feature detection? **Yes**.
|
||||
|
||||
|
||||
@ -134,11 +134,11 @@ parameter-sets:
|
||||
- Darwin
|
||||
- Linux
|
||||
required_flags:
|
||||
- sse2
|
||||
- popcnt
|
||||
- ssse3
|
||||
- avx2
|
||||
- aes
|
||||
- avx2
|
||||
- popcnt
|
||||
- sse2
|
||||
- ssse3
|
||||
common-crypto:
|
||||
- AES: pqcrystals
|
||||
- SHA3: liboqs
|
||||
@ -168,11 +168,11 @@ parameter-sets:
|
||||
- Darwin
|
||||
- Linux
|
||||
required_flags:
|
||||
- sse2
|
||||
- popcnt
|
||||
- ssse3
|
||||
- avx2
|
||||
- aes
|
||||
- avx2
|
||||
- popcnt
|
||||
- sse2
|
||||
- ssse3
|
||||
common-crypto:
|
||||
- AES: pqcrystals
|
||||
- SHA3: liboqs
|
||||
@ -202,11 +202,11 @@ parameter-sets:
|
||||
- Darwin
|
||||
- Linux
|
||||
required_flags:
|
||||
- sse2
|
||||
- popcnt
|
||||
- ssse3
|
||||
- avx2
|
||||
- aes
|
||||
- avx2
|
||||
- popcnt
|
||||
- sse2
|
||||
- ssse3
|
||||
common-crypto:
|
||||
- AES: pqcrystals
|
||||
- SHA3: liboqs
|
||||
|
||||
@ -6,7 +6,7 @@
|
||||
- **Auxiliary submitters**: Jean-Philippe Aumasson, Daniel J. Bernstein,, Christoph Dobraunig, Maria Eichlseder, Scott Fluhrer, Stefan-Lukas Gazdag, Panos Kampanakis, Stefan Kölbl, Tanja Lange, Martin M. Lauridsen, Florian Mendel, Ruben Niederhagen, Christian Rechberger, Joost Rijneveld, Peter Schwabe.
|
||||
- **Authors' website**: https://sphincs.org/
|
||||
- **Specification version**: NIST Round 3 submission.
|
||||
- **Implementation source**: https://github.com/PQClean/PQClean/commit/6c1ea921ee4a06a6b50c742ca540bb9b5e51aee2, which takes it from:
|
||||
- **Implementation source**: https://github.com/PQClean/PQClean/commit/6c1ea921ee4a06a6b50c742ca540bb9b5e51aee2 with copy_from_upstream patches, which takes it from:
|
||||
- https://github.com/sphincs/sphincsplus
|
||||
- **Implementation license (SPDX-Identifier)**: CC0-1.0.
|
||||
|
||||
|
||||
@ -24,6 +24,7 @@ nist-round: 3
|
||||
spec-version: NIST Round 3 submission
|
||||
spdx-license-identifier: CC0-1.0
|
||||
upstream: https://github.com/PQClean/PQClean/commit/6c1ea921ee4a06a6b50c742ca540bb9b5e51aee2
|
||||
with copy_from_upstream patches
|
||||
upstream-ancestors:
|
||||
- https://github.com/sphincs/sphincsplus
|
||||
parameter-sets:
|
||||
|
||||
@ -13,6 +13,7 @@ import yaml
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import json
|
||||
import update_upstream_alg_docs
|
||||
|
||||
# kats of all algs
|
||||
kats = {}
|
||||
@ -512,7 +513,7 @@ def copy_from_upstream():
|
||||
if not keepdata:
|
||||
shutil.rmtree('repos')
|
||||
|
||||
print("Remember to update the docs by running update_pqclean_alg_docs.py")
|
||||
update_upstream_alg_docs.do_it(os.environ['LIBOQS_DIR'])
|
||||
|
||||
|
||||
def verify_from_upstream():
|
||||
|
||||
@ -1,190 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
import yaml
|
||||
import inspect
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--liboqs-root", default=os.path.join("..", ".."))
|
||||
parser.add_argument("-w", "--write-changes", dest="write_changes", action='store_true')
|
||||
parser.add_argument("-v", "--verbosity", type=int)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.verbosity:
|
||||
DEBUG = args.verbosity
|
||||
else:
|
||||
DEBUG = 0
|
||||
|
||||
if not args.write_changes:
|
||||
print("--write-changes not set; changes will not be written out.")
|
||||
|
||||
def shell(command, expect=0):
|
||||
subprocess_stdout = None if DEBUG > 0 else subprocess.DEVNULL
|
||||
ret = subprocess.run(command, stdout=subprocess_stdout, stderr=subprocess_stdout)
|
||||
if ret.returncode != expect:
|
||||
raise Exception("'{}' failed with error {}. Expected {}.".format(" ".join(command), ret, expect))
|
||||
|
||||
def load_yaml(filename, encoding='utf-8'):
|
||||
with open(filename, mode='r', encoding=encoding) as fh:
|
||||
return yaml.safe_load(fh.read())
|
||||
|
||||
def store_yaml(filename, contents, encoding='utf-8'):
|
||||
with open(filename, mode='w', encoding=encoding) as fh:
|
||||
yaml.dump(contents, fh, sort_keys=False, allow_unicode=True)
|
||||
|
||||
def fetch_upstream(upstream_info):
|
||||
work_dir_root = os.path.join(args.liboqs_root, 'scripts', 'copy_from_upstream', 'repos')
|
||||
os.makedirs(work_dir_root, exist_ok=True)
|
||||
|
||||
work_dir = os.path.join(work_dir_root, upstream_info['name'])
|
||||
work_dotgit = os.path.join(work_dir, '.git')
|
||||
if not os.path.exists(work_dotgit):
|
||||
shell(['git', 'init', work_dir])
|
||||
shell(['git', '--git-dir', work_dotgit, 'remote', 'add', 'origin', upstream_info['git_url']])
|
||||
shell(['git', '--git-dir', work_dotgit, '--work-tree', work_dir, 'remote', 'set-url', 'origin', upstream_info['git_url']])
|
||||
shell(['git', '--git-dir', work_dotgit, '--work-tree', work_dir, 'fetch', '--depth=1', 'origin', upstream_info['git_commit']])
|
||||
shell(['git', '--git-dir', work_dotgit, '--work-tree', work_dir, 'reset', '--hard', upstream_info['git_commit']])
|
||||
|
||||
return work_dir
|
||||
|
||||
def rhs_if_not_equal(lhs, rhs, not_equal_msg):
|
||||
if lhs != rhs:
|
||||
if DEBUG > 0:
|
||||
caller = inspect.getframeinfo(inspect.stack()[1][0])
|
||||
print("Line {}: Discrepancy in {}: lhs: {}, rhs: {}".format(caller.lineno, not_equal_msg, lhs, rhs))
|
||||
return rhs
|
||||
return lhs
|
||||
|
||||
def update_pqclean_kem_alg_docs(kems, upstream_info, pqclean_root, meta_yaml_path_template):
|
||||
for kem in kems:
|
||||
if kem['upstream_location'] == 'pqclean':
|
||||
oqs_yaml_path = os.path.join(args.liboqs_root, 'docs', 'algorithms', 'kem', '{}.yml'.format(kem['name']))
|
||||
if os.path.isfile(oqs_yaml_path):
|
||||
oqs_yaml = load_yaml(oqs_yaml_path)
|
||||
else:
|
||||
continue
|
||||
|
||||
# We assume that the ordering of "parameter-sets"
|
||||
# in the OQS YAML files matches that of copy_from_upstream.yml
|
||||
for index, scheme in enumerate(kem['schemes']):
|
||||
if DEBUG > 0:
|
||||
print("Examining {}'s META.yml.".format(scheme['pretty_name_full']))
|
||||
pqclean_meta_path = os.path.join(pqclean_root, meta_yaml_path_template.format_map(scheme))
|
||||
pqclean_yaml = load_yaml(pqclean_meta_path)
|
||||
|
||||
oqs_yaml['type'] = rhs_if_not_equal(oqs_yaml['type'], pqclean_yaml['type'], "type")
|
||||
oqs_yaml['principal-submitters'] = rhs_if_not_equal(oqs_yaml['principal-submitters'], pqclean_yaml['principal-submitters'], "principal-submitters")
|
||||
|
||||
upstream_base_url = upstream_info['git_url'][:-len(".git")]
|
||||
oqs_yaml['upstream'] = rhs_if_not_equal(oqs_yaml['upstream'], "{}/commit/{}".format(upstream_base_url, upstream_info['git_commit']), "upstream")
|
||||
|
||||
if 'auxiliary-submitters' in pqclean_yaml:
|
||||
oqs_yaml['auxiliary-submitters'] = rhs_if_not_equal(oqs_yaml['auxiliary-submitters'] if 'auxiliary-submitters' in oqs_yaml else '', pqclean_yaml['auxiliary-submitters'], "auxiliary-submitters")
|
||||
|
||||
oqs_scheme_yaml = oqs_yaml['parameter-sets'][index]
|
||||
|
||||
# TODO: PQClean and liboqs pretty-naming conventions for the
|
||||
# following algorithms are out of sync.
|
||||
if kem['name'] == 'classic_mceliece' or kem['name'] == 'hqc' or kem['name'] == 'ntru' or kem['name'] == 'saber':
|
||||
oqs_scheme_yaml['name'] = rhs_if_not_equal(oqs_scheme_yaml['name'], scheme['pretty_name_full'], "scheme pretty name")
|
||||
else:
|
||||
oqs_scheme_yaml['name'] = rhs_if_not_equal(oqs_scheme_yaml['name'], pqclean_yaml['name'], "scheme pretty name")
|
||||
|
||||
oqs_scheme_yaml['claimed-nist-level'] = rhs_if_not_equal(oqs_scheme_yaml['claimed-nist-level'], pqclean_yaml['claimed-nist-level'], "claimed-nist-level")
|
||||
oqs_scheme_yaml['claimed-security'] = rhs_if_not_equal(oqs_scheme_yaml['claimed-security'], pqclean_yaml['claimed-security'], "claimed-security")
|
||||
oqs_scheme_yaml['length-public-key'] = rhs_if_not_equal(oqs_scheme_yaml['length-public-key'], pqclean_yaml['length-public-key'], "length-public-key")
|
||||
oqs_scheme_yaml['length-ciphertext'] = rhs_if_not_equal(oqs_scheme_yaml['length-ciphertext'], pqclean_yaml['length-ciphertext'], "length-ciphertext")
|
||||
oqs_scheme_yaml['length-secret-key'] = rhs_if_not_equal(oqs_scheme_yaml['length-secret-key'], pqclean_yaml['length-secret-key'], "legnth-secret-key")
|
||||
oqs_scheme_yaml['length-shared-secret'] = rhs_if_not_equal(oqs_scheme_yaml['length-shared-secret'], pqclean_yaml['length-shared-secret'], "length-shared-secret")
|
||||
|
||||
for impl_index, impl in enumerate(oqs_scheme_yaml['implementations']):
|
||||
for pqclean_impl in pqclean_yaml['implementations']:
|
||||
if impl['upstream-id'] == pqclean_impl['name']:
|
||||
break
|
||||
|
||||
if 'supported_platforms' in pqclean_impl:
|
||||
impl['supported-platforms'] = rhs_if_not_equal(impl['supported-platforms'], pqclean_impl['supported_platforms'], "supported-platforms")
|
||||
else:
|
||||
impl['supported-platforms'] = rhs_if_not_equal(impl['supported-platforms'], "all", "supported-platforms")
|
||||
|
||||
oqs_scheme_yaml['implementations'][impl_index] = impl
|
||||
|
||||
oqs_yaml['parameter-sets'][index] = oqs_scheme_yaml
|
||||
|
||||
if args.write_changes:
|
||||
store_yaml(oqs_yaml_path, oqs_yaml)
|
||||
|
||||
def update_pqclean_sig_alg_docs(sigs, upstream_info, pqclean_root, meta_yaml_path_template):
|
||||
for sig in sigs:
|
||||
if sig['upstream_location'] == 'pqclean':
|
||||
oqs_yaml_path = os.path.join(args.liboqs_root, 'docs', 'algorithms', 'sig', '{}.yml'.format(sig['name']))
|
||||
if os.path.isfile(oqs_yaml_path):
|
||||
oqs_yaml = load_yaml(oqs_yaml_path)
|
||||
else:
|
||||
continue
|
||||
|
||||
# We assume that the ordering of "parameter-sets"
|
||||
# in the OQS YAML files matches that of copy_from_upstream.yml
|
||||
for index, scheme in enumerate(sig['schemes']):
|
||||
if DEBUG > 0:
|
||||
print("Examining {}'s META.yml.".format(scheme['pretty_name_full']))
|
||||
pqclean_meta_path = os.path.join(pqclean_root, meta_yaml_path_template.format_map(scheme))
|
||||
pqclean_yaml = load_yaml(pqclean_meta_path)
|
||||
|
||||
oqs_yaml['type'] = rhs_if_not_equal(oqs_yaml['type'], pqclean_yaml['type'], "type")
|
||||
oqs_yaml['principal-submitters'] = rhs_if_not_equal(oqs_yaml['principal-submitters'], pqclean_yaml['principal-submitters'], "principal-submitters")
|
||||
|
||||
upstream_base_url = upstream_info['git_url'][:-len(".git")]
|
||||
oqs_yaml['upstream'] = rhs_if_not_equal(oqs_yaml['upstream'], "{}/commit/{}".format(upstream_base_url, upstream_info['git_commit']), "upstream")
|
||||
|
||||
if 'auxiliary-submitters' in pqclean_yaml:
|
||||
oqs_yaml['auxiliary-submitters'] = rhs_if_not_equal(oqs_yaml['auxiliary-submitters'] if 'auxiliary-submitters' in oqs_yaml else '', pqclean_yaml['auxiliary-submitters'], "auxiliary-submitters")
|
||||
|
||||
oqs_scheme_yaml = oqs_yaml['parameter-sets'][index]
|
||||
|
||||
# TODO: PQClean and liboqs pretty-naming conventions for the
|
||||
# following algorithms are out of sync.
|
||||
if sig['name'] == 'sphincs' or sig['name'] == 'rainbow':
|
||||
oqs_scheme_yaml['name'] = rhs_if_not_equal(oqs_scheme_yaml['name'], scheme['pretty_name_full'], "scheme pretty name")
|
||||
else:
|
||||
oqs_scheme_yaml['name'] = rhs_if_not_equal(oqs_scheme_yaml['name'], pqclean_yaml['name'], "scheme pretty name")
|
||||
|
||||
oqs_scheme_yaml['claimed-nist-level'] = rhs_if_not_equal(oqs_scheme_yaml['claimed-nist-level'], pqclean_yaml['claimed-nist-level'], "claimed-nist-level")
|
||||
oqs_scheme_yaml['claimed-security'] = rhs_if_not_equal(oqs_scheme_yaml['claimed-security'], 'EUF-CMA', "claimed-security")
|
||||
oqs_scheme_yaml['length-public-key'] = rhs_if_not_equal(oqs_scheme_yaml['length-public-key'], pqclean_yaml['length-public-key'], "length-public-key")
|
||||
oqs_scheme_yaml['length-secret-key'] = rhs_if_not_equal(oqs_scheme_yaml['length-secret-key'], pqclean_yaml['length-secret-key'], "legnth-secret-key")
|
||||
oqs_scheme_yaml['length-signature'] = rhs_if_not_equal(oqs_scheme_yaml['length-signature'], pqclean_yaml['length-signature'], "length-signature")
|
||||
|
||||
for impl_index, impl in enumerate(oqs_scheme_yaml['implementations']):
|
||||
for pqclean_impl in pqclean_yaml['implementations']:
|
||||
if impl['upstream-id'] == pqclean_impl['name']:
|
||||
break
|
||||
|
||||
if 'supported_platforms' in pqclean_impl:
|
||||
impl['supported-platforms'] = rhs_if_not_equal(impl['supported-platforms'], pqclean_impl['supported_platforms'], "supported-platforms")
|
||||
else:
|
||||
impl['supported-platforms'] = rhs_if_not_equal(impl['supported-platforms'], "all", "supported-platforms")
|
||||
|
||||
oqs_scheme_yaml['implementations'][impl_index] = impl
|
||||
|
||||
oqs_yaml['parameter-sets'][index] = oqs_scheme_yaml
|
||||
|
||||
if args.write_changes:
|
||||
store_yaml(oqs_yaml_path, oqs_yaml)
|
||||
|
||||
instructions = load_yaml(
|
||||
os.path.join(args.liboqs_root, 'scripts', 'copy_from_upstream', 'copy_from_upstream.yml'),
|
||||
encoding='utf-8')
|
||||
|
||||
for upstream in instructions['upstreams']:
|
||||
if upstream['name'] == 'pqclean':
|
||||
pqclean_info = upstream
|
||||
break
|
||||
pqclean_root = fetch_upstream(pqclean_info)
|
||||
|
||||
update_pqclean_kem_alg_docs(instructions['kems'], pqclean_info, pqclean_root, pqclean_info['kem_meta_path'])
|
||||
update_pqclean_sig_alg_docs(instructions['sigs'], pqclean_info, pqclean_root, pqclean_info['sig_meta_path'])
|
||||
274
scripts/copy_from_upstream/update_upstream_alg_docs.py
Executable file
274
scripts/copy_from_upstream/update_upstream_alg_docs.py
Executable file
@ -0,0 +1,274 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import subprocess
|
||||
import yaml
|
||||
import inspect
|
||||
|
||||
DEBUG = 0
|
||||
|
||||
def shell(command, expect=0):
|
||||
subprocess_stdout = None if DEBUG > 0 else subprocess.DEVNULL
|
||||
ret = subprocess.run(command, stdout=subprocess_stdout, stderr=subprocess_stdout)
|
||||
if ret.returncode != expect:
|
||||
raise Exception("'{}' failed with error {}. Expected {}.".format(" ".join(command), ret, expect))
|
||||
|
||||
def load_yaml(filename, encoding='utf-8'):
|
||||
with open(filename, mode='r', encoding=encoding) as fh:
|
||||
return yaml.safe_load(fh.read())
|
||||
|
||||
def store_yaml(filename, contents, encoding='utf-8'):
|
||||
with open(filename, mode='w', encoding=encoding) as fh:
|
||||
yaml.dump(contents, fh, sort_keys=False, allow_unicode=True)
|
||||
|
||||
def fetch_upstream(liboqs_root, upstream_info):
|
||||
work_dir_root = os.path.join(liboqs_root, 'scripts', 'copy_from_upstream', 'repos')
|
||||
os.makedirs(work_dir_root, exist_ok=True)
|
||||
|
||||
work_dir = os.path.join(work_dir_root, upstream_info['name'])
|
||||
work_dotgit = os.path.join(work_dir, '.git')
|
||||
if not os.path.exists(work_dotgit):
|
||||
shell(['git', 'init', work_dir])
|
||||
shell(['git', '--git-dir', work_dotgit, 'remote', 'add', 'origin', upstream_info['git_url']])
|
||||
shell(['git', '--git-dir', work_dotgit, '--work-tree', work_dir, 'remote', 'set-url', 'origin', upstream_info['git_url']])
|
||||
shell(['git', '--git-dir', work_dotgit, '--work-tree', work_dir, 'fetch', '--depth=1', 'origin', upstream_info['git_commit']])
|
||||
shell(['git', '--git-dir', work_dotgit, '--work-tree', work_dir, 'reset', '--hard', upstream_info['git_commit']])
|
||||
|
||||
return work_dir
|
||||
|
||||
def rhs_if_not_equal(lhs, rhs, not_equal_msg):
|
||||
if lhs != rhs:
|
||||
if DEBUG > 0:
|
||||
caller = inspect.getframeinfo(inspect.stack()[1][0])
|
||||
print("Line {}: Discrepancy in {}: lhs: {}, rhs: {}".format(caller.lineno, not_equal_msg, lhs, rhs))
|
||||
if DEBUG > 1:
|
||||
exit(1)
|
||||
return rhs
|
||||
return lhs
|
||||
|
||||
def get_upstream_info(upstream_list, location):
|
||||
for i in upstream_list:
|
||||
if i['name'] == location:
|
||||
return i
|
||||
print("Error: Cannot find location %s in upstream list" % (location))
|
||||
print(upstream_list)
|
||||
exit(1)
|
||||
|
||||
def get_oqs_yaml(param_list, name):
|
||||
ctr=0
|
||||
for i in param_list:
|
||||
if i['name'] == name:
|
||||
return ctr, i
|
||||
ctr=ctr+1
|
||||
print("Error: Cannot find name %s in param list" % (name))
|
||||
print(param_list)
|
||||
exit(1)
|
||||
|
||||
# Merge documentation contained in liboqs_root/docs/algorithms/kem/kem['name'].yml with upstream information:
|
||||
# Args:
|
||||
# kems: List of kems in copy_from_upstream.yml
|
||||
# upstream_info: Hashtable of upstream information (keyed by upstream source)
|
||||
# incl. entry: 'upstream_root' pointing to local folder containing source code
|
||||
def update_upstream_kem_alg_docs(liboqs_root, kems, upstream_info, write_changes=False):
|
||||
for kem in kems:
|
||||
ui = get_upstream_info(upstream_info, kem['upstream_location'])
|
||||
patches_done=""
|
||||
if 'patches' in ui:
|
||||
for patchfilename in ui['patches']:
|
||||
if kem['name'] in patchfilename:
|
||||
patches_done=" with copy_from_upstream patches"
|
||||
|
||||
upstream_root = ui['upstream_root']
|
||||
meta_yaml_path_template = ui['kem_meta_path']
|
||||
if DEBUG > 1:
|
||||
print("Working on KEM %s using path %s and META file %s" % (kem, upstream_root, meta_yaml_path_template))
|
||||
if True: # for all upstream sources:
|
||||
oqs_yaml_path = os.path.join(liboqs_root, 'docs', 'algorithms', 'kem', '{}.yml'.format(kem['name']))
|
||||
if os.path.isfile(oqs_yaml_path):
|
||||
oqs_yaml = load_yaml(oqs_yaml_path)
|
||||
else:
|
||||
continue
|
||||
|
||||
# We cannot assume that the ordering of "parameter-sets"
|
||||
# in the OQS YAML files matches that of copy_from_upstream.yml
|
||||
# hence use helper function get_oqs_yaml(alg_name)
|
||||
for scheme in kem['schemes']:
|
||||
upstream_meta_path = os.path.join(upstream_root, meta_yaml_path_template.format_map(scheme))
|
||||
if DEBUG > 0:
|
||||
print("Examining {}'s META.yml.".format(scheme['pretty_name_full']))
|
||||
upstream_yaml = load_yaml(upstream_meta_path)
|
||||
|
||||
oqs_yaml['type'] = rhs_if_not_equal(oqs_yaml['type'], upstream_yaml['type'], "type")
|
||||
oqs_yaml['principal-submitters'] = rhs_if_not_equal(oqs_yaml['principal-submitters'], upstream_yaml['principal-submitters'], "principal-submitters")
|
||||
|
||||
upstream_base_url = ui['git_url'][:-len(".git")]
|
||||
# upstream is special: We will take the upstream git commit information
|
||||
# (possibly with added patch comment) as it is what drove the update
|
||||
oqs_yaml['upstream'] = rhs_if_not_equal(oqs_yaml['upstream'], ("{}/commit/{}"+patches_done).format(upstream_base_url, ui['git_commit']), "upstream")
|
||||
|
||||
if 'auxiliary-submitters' in upstream_yaml:
|
||||
oqs_yaml['auxiliary-submitters'] = rhs_if_not_equal(oqs_yaml['auxiliary-submitters'] if 'auxiliary-submitters' in oqs_yaml else '', upstream_yaml['auxiliary-submitters'], "auxiliary-submitters")
|
||||
|
||||
index, oqs_scheme_yaml = get_oqs_yaml(oqs_yaml['parameter-sets'], scheme['pretty_name_full'])
|
||||
|
||||
# TODO: PQClean and liboqs pretty-naming conventions for the
|
||||
# following algorithms are out of sync.
|
||||
if kem['name'] == 'classic_mceliece' or kem['name'] == 'hqc' or kem['name'] == 'ntru' or kem['name'] == 'saber':
|
||||
oqs_scheme_yaml['name'] = rhs_if_not_equal(oqs_scheme_yaml['name'], scheme['pretty_name_full'], "scheme pretty name")
|
||||
else:
|
||||
oqs_scheme_yaml['name'] = rhs_if_not_equal(oqs_scheme_yaml['name'], upstream_yaml['name'], "scheme pretty name")
|
||||
|
||||
oqs_scheme_yaml['claimed-nist-level'] = rhs_if_not_equal(oqs_scheme_yaml['claimed-nist-level'], upstream_yaml['claimed-nist-level'], "claimed-nist-level")
|
||||
oqs_scheme_yaml['claimed-security'] = rhs_if_not_equal(oqs_scheme_yaml['claimed-security'], upstream_yaml['claimed-security'], "claimed-security")
|
||||
oqs_scheme_yaml['length-public-key'] = rhs_if_not_equal(oqs_scheme_yaml['length-public-key'], upstream_yaml['length-public-key'], "length-public-key")
|
||||
oqs_scheme_yaml['length-ciphertext'] = rhs_if_not_equal(oqs_scheme_yaml['length-ciphertext'], upstream_yaml['length-ciphertext'], "length-ciphertext")
|
||||
oqs_scheme_yaml['length-secret-key'] = rhs_if_not_equal(oqs_scheme_yaml['length-secret-key'], upstream_yaml['length-secret-key'], "legnth-secret-key")
|
||||
oqs_scheme_yaml['length-shared-secret'] = rhs_if_not_equal(oqs_scheme_yaml['length-shared-secret'], upstream_yaml['length-shared-secret'], "length-shared-secret")
|
||||
|
||||
for impl_index, impl in enumerate(oqs_scheme_yaml['implementations']):
|
||||
for upstream_impl in upstream_yaml['implementations']:
|
||||
if impl['upstream-id'] == upstream_impl['name']:
|
||||
break
|
||||
|
||||
# Logic to add Common_META.yml components
|
||||
uir = get_upstream_info(upstream_yaml['implementations'], impl['upstream-id'])
|
||||
if (uir != None) and ('common_dep' in uir):
|
||||
upstream_common_path = upstream_meta_path.replace(scheme['pretty_name_full'], "Common")
|
||||
upstream_common_yaml = load_yaml(upstream_common_path)
|
||||
for c in uir['common_dep'].split(' '):
|
||||
ur = get_upstream_info(upstream_common_yaml['commons'], c)
|
||||
if (ur != None) and ('supported_platforms' in ur):
|
||||
if 'required_flags' in ur['supported_platforms'][0].keys():
|
||||
upstream_impl['supported_platforms'][0]['required_flags']=list(set(upstream_impl['supported_platforms'][0]['required_flags']+ur['supported_platforms'][0]['required_flags']))
|
||||
upstream_impl['supported_platforms'][0]['required_flags'].sort()
|
||||
|
||||
if 'supported_platforms' in upstream_impl:
|
||||
impl['supported-platforms'] = rhs_if_not_equal(impl['supported-platforms'], upstream_impl['supported_platforms'], "supported-platforms")
|
||||
else:
|
||||
impl['supported-platforms'] = rhs_if_not_equal(impl['supported-platforms'], "all", "supported-platforms")
|
||||
|
||||
oqs_scheme_yaml['implementations'][impl_index] = impl
|
||||
|
||||
oqs_yaml['parameter-sets'][index] = oqs_scheme_yaml
|
||||
|
||||
if write_changes:
|
||||
store_yaml(oqs_yaml_path, oqs_yaml)
|
||||
|
||||
def update_upstream_sig_alg_docs(liboqs_root, sigs, upstream_info, write_changes=False):
|
||||
for sig in sigs:
|
||||
ui = get_upstream_info(upstream_info, sig['upstream_location'])
|
||||
patches_done=""
|
||||
if 'patches' in ui:
|
||||
for patchfilename in ui['patches']:
|
||||
if sig['name'] in patchfilename:
|
||||
patches_done=" with copy_from_upstream patches"
|
||||
|
||||
upstream_root = ui['upstream_root']
|
||||
meta_yaml_path_template = ui['sig_meta_path']
|
||||
if DEBUG > 1:
|
||||
print("Working on SIG %s using path %s and META file %s" % (sig, upstream_root, meta_yaml_path_template))
|
||||
if True: # for all upstream sources:
|
||||
oqs_yaml_path = os.path.join(liboqs_root, 'docs', 'algorithms', 'sig', '{}.yml'.format(sig['name']))
|
||||
if os.path.isfile(oqs_yaml_path):
|
||||
oqs_yaml = load_yaml(oqs_yaml_path)
|
||||
else:
|
||||
continue
|
||||
|
||||
# We assume that the ordering of "parameter-sets"
|
||||
# in the OQS YAML files matches that of copy_from_upstream.yml
|
||||
for scheme in sig['schemes']:
|
||||
if DEBUG > 0:
|
||||
print("Examining {}'s META.yml.".format(scheme['pretty_name_full']))
|
||||
upstream_meta_path = os.path.join(upstream_root, meta_yaml_path_template.format_map(scheme))
|
||||
upstream_yaml = load_yaml(upstream_meta_path)
|
||||
|
||||
oqs_yaml['type'] = rhs_if_not_equal(oqs_yaml['type'], upstream_yaml['type'], "type")
|
||||
oqs_yaml['principal-submitters'] = rhs_if_not_equal(oqs_yaml['principal-submitters'], upstream_yaml['principal-submitters'], "principal-submitters")
|
||||
|
||||
upstream_base_url = ui['git_url'][:-len(".git")]
|
||||
# upstream is special: We will take the upstream git commit information
|
||||
# (possibly with added patch comment) as it is what drove the update
|
||||
oqs_yaml['upstream'] = rhs_if_not_equal(oqs_yaml['upstream'], ("{}/commit/{}"+patches_done).format(upstream_base_url, ui['git_commit']), "upstream")
|
||||
|
||||
if 'auxiliary-submitters' in upstream_yaml:
|
||||
oqs_yaml['auxiliary-submitters'] = rhs_if_not_equal(oqs_yaml['auxiliary-submitters'] if 'auxiliary-submitters' in oqs_yaml else '', upstream_yaml['auxiliary-submitters'], "auxiliary-submitters")
|
||||
|
||||
index, oqs_scheme_yaml = get_oqs_yaml(oqs_yaml['parameter-sets'], scheme['pretty_name_full'])
|
||||
|
||||
# TODO: PQClean and liboqs pretty-naming conventions for the
|
||||
# following algorithms are out of sync.
|
||||
if sig['name'] == 'sphincs' or sig['name'] == 'rainbow':
|
||||
oqs_scheme_yaml['name'] = rhs_if_not_equal(oqs_scheme_yaml['name'], scheme['pretty_name_full'], "scheme pretty name")
|
||||
else:
|
||||
oqs_scheme_yaml['name'] = rhs_if_not_equal(oqs_scheme_yaml['name'], upstream_yaml['name'], "scheme pretty name")
|
||||
|
||||
oqs_scheme_yaml['claimed-nist-level'] = rhs_if_not_equal(oqs_scheme_yaml['claimed-nist-level'], upstream_yaml['claimed-nist-level'], "claimed-nist-level")
|
||||
oqs_scheme_yaml['claimed-security'] = rhs_if_not_equal(oqs_scheme_yaml['claimed-security'], 'EUF-CMA', "claimed-security")
|
||||
oqs_scheme_yaml['length-public-key'] = rhs_if_not_equal(oqs_scheme_yaml['length-public-key'], upstream_yaml['length-public-key'], "length-public-key")
|
||||
oqs_scheme_yaml['length-secret-key'] = rhs_if_not_equal(oqs_scheme_yaml['length-secret-key'], upstream_yaml['length-secret-key'], "legnth-secret-key")
|
||||
oqs_scheme_yaml['length-signature'] = rhs_if_not_equal(oqs_scheme_yaml['length-signature'], upstream_yaml['length-signature'], "length-signature")
|
||||
|
||||
for impl_index, impl in enumerate(oqs_scheme_yaml['implementations']):
|
||||
for upstream_impl in upstream_yaml['implementations']:
|
||||
if impl['upstream-id'] == upstream_impl['name']:
|
||||
break
|
||||
|
||||
# Logic to add Common_META.yml components
|
||||
uir = get_upstream_info(upstream_yaml['implementations'], impl['upstream-id'])
|
||||
if (uir != None) and ('common_dep' in uir):
|
||||
upstream_common_path = upstream_meta_path.replace(scheme['pretty_name_full'], "Common")
|
||||
upstream_common_yaml = load_yaml(upstream_common_path)
|
||||
for c in uir['common_dep'].split(' '):
|
||||
ur = get_upstream_info(upstream_common_yaml['commons'], c)
|
||||
if (ur != None) and ('supported_platforms' in ur):
|
||||
if 'required_flags' in ur['supported_platforms'][0].keys():
|
||||
upstream_impl['supported_platforms'][0]['required_flags']=list(set(upstream_impl['supported_platforms'][0]['required_flags']+ur['supported_platforms'][0]['required_flags']))
|
||||
upstream_impl['supported_platforms'][0]['required_flags'].sort()
|
||||
|
||||
if 'supported_platforms' in upstream_impl:
|
||||
impl['supported-platforms'] = rhs_if_not_equal(impl['supported-platforms'], upstream_impl['supported_platforms'], "supported-platforms")
|
||||
else:
|
||||
impl['supported-platforms'] = rhs_if_not_equal(impl['supported-platforms'], "all", "supported-platforms")
|
||||
|
||||
oqs_scheme_yaml['implementations'][impl_index] = impl
|
||||
|
||||
oqs_yaml['parameter-sets'][index] = oqs_scheme_yaml
|
||||
|
||||
if write_changes:
|
||||
store_yaml(oqs_yaml_path, oqs_yaml)
|
||||
|
||||
def do_it(liboqs_root):
|
||||
global DEBUG
|
||||
if liboqs_root == None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--liboqs-root", default=os.path.join("..", ".."))
|
||||
parser.add_argument("-w", "--write-changes", dest="write_changes", action='store_true')
|
||||
parser.add_argument("-v", "--verbosity", type=int)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.verbosity:
|
||||
DEBUG = args.verbosity
|
||||
|
||||
liboqs_root = args.liboqs_root
|
||||
write_changes = args.write_changes
|
||||
else:
|
||||
write_changes = True
|
||||
|
||||
if not write_changes:
|
||||
print("--write-changes not set; changes will not be written out.")
|
||||
instructions = load_yaml(
|
||||
os.path.join(liboqs_root, 'scripts', 'copy_from_upstream', 'copy_from_upstream.yml'),
|
||||
encoding='utf-8')
|
||||
|
||||
for upstream in instructions['upstreams']:
|
||||
if 'git_url' in upstream.keys():
|
||||
upstream['upstream_root'] = fetch_upstream(liboqs_root, upstream)
|
||||
|
||||
update_upstream_kem_alg_docs(liboqs_root, instructions['kems'], instructions['upstreams'], write_changes)
|
||||
update_upstream_sig_alg_docs(liboqs_root, instructions['sigs'], instructions['upstreams'], write_changes)
|
||||
|
||||
if __name__ == "__main__":
|
||||
do_it(None)
|
||||
Loading…
x
Reference in New Issue
Block a user