hacked copy_from_slh_dsa_c to extract SLH-DSA schemes, implemented replace_one_fragment so list_standardized_algs.fragment can be rendered with a separate set of instructions

Signed-off-by: Ganyu (Bruce) Xu <g66xu@uwaterloo.ca>

scripts/copy_from_upstream/.CMake/alg_support.cmake/list_standardized_algs.fragment

@@ -5,7 +5,7 @@
         {%- if 'alias_scheme' in scheme -%}KEM_{{ family['name'] }}_{{ scheme['alias_scheme'] }}{%- else -%}KEM_{{ family['name'] }}_{{ scheme['scheme'] }}{%- endif -%};
     {%- endfor -%}
 {%- endfor -%}
-{%- for family in instructions['sigs'] if family['name'] in ['ml_dsa', 'falcon', 'sphincs'] -%}
+{%- for family in instructions['sigs'] if family['name'] in ['ml_dsa', 'falcon', 'slh_dsa'] -%}
     {%- set outer_loop = loop -%}
     {%- for scheme in family['schemes'] -%}
         {%- if 'alias_scheme' in scheme -%}SIG_{{ family['name'] }}_{{ scheme['alias_scheme'] }}{%- else -%}SIG_{{ family['name'] }}_{{ scheme['scheme'] }}{%- endif -%}{%- if not (outer_loop.last and loop.last) -%};{%- endif -%}

scripts/copy_from_upstream/copy_from_slh_dsa_c.py

@@ -337,5 +337,18 @@ def main():
     # apply patches
     apply_patches(slh_patch_dir)
 
+    # NOTE: from [issue 2203](https://github.com/open-quantum-safe/liboqs/issues/2203)
+    # SLH-DSA is not described in copy_from_upstream.yml. It is instead described
+    # here in this separate module. This makes replacing SPHINCS+ with SLH-DSA
+    # in list_standardized_algs.fragment non-trivial because this Jinja template
+    # is rendered from copy_from_upstream.yml.
+    # As a necessary hack, the list of variants (e.g. "pure_sha2_128s") is returned
+    # so that copy_from_upstream.py can use this list to construct a dictionary
+    # that resembles the structure of copy_from_upstream.yml.
+    # In the near future I want to consider refactoring build configuration
+    # management and upstream integration scripts. The status quo is a mess and
+    # will make future integrations all the more difficult.
+    return variants
+
 if __name__ == "__main__":
     main()

scripts/copy_from_upstream/copy_from_upstream.py

@@ -16,6 +16,7 @@ import json
 import platform
 import update_upstream_alg_docs
 import copy_from_slh_dsa_c
+from copy import deepcopy
 
 # kats of all algs
 kats = {}
@@ -97,6 +98,46 @@ def generator_all(filename, instructions):
     contents = jinja2.Template(template).render({'instructions': instructions})
     file_put_contents(filename, contents)
 
+# TODO: consider refactoring replacer by calling replace_one_fragment
+def replace_one_fragment(
+    dst_path: str,
+    template_path: str,
+    instructions: dict,
+    delimiter: str,
+    libjade: bool = False,
+):
+    """Replace a single fragment with a rendered Jinja template
+
+    :param dst_path: path to the rendered file, relative to LIBOQS_DIR
+    :param template_path: path to the Jinja template file, relative to LIBOQS_DIR
+    :param instructions: copy_from_upstream.yml or some patched version
+    :param delimiter: how the identifer for the fragment in the destination file
+    is prefixed
+    """
+    liboqs_dir = os.environ.get("LIBOQS_DIR", None)
+    if not liboqs_dir:
+        raise KeyError("Environment variable LIBOQS_DIR is missing")
+    dst_path = os.path.join(liboqs_dir, dst_path)
+    template_path = os.path.join(liboqs_dir, template_path)
+    with open(template_path, "r") as template_f, open(dst_path, "r") as dst_f:
+        template = template_f.read()
+        dst_content = dst_f.read()
+    identifier, _ = os.path.splitext(os.path.basename(template_path))
+    jade_or_upstream = "LIBJADE" if libjade else "UPSTREAM"
+    identifier_start = f"{delimiter} OQS_COPY_FROM_{jade_or_upstream}_FRAGMENT_{identifier.upper()}_START"
+    identifier_end = f"{delimiter} OQS_COPY_FROM_{jade_or_upstream}_FRAGMENT_{identifier.upper()}_END"
+    preamble = dst_content[: dst_content.find(identifier_start)]
+    postamble = dst_content[dst_content.find(identifier_end) :]
+    dst_content = (
+        preamble
+        + identifier_start
+        + jinja2.Template(template).render(
+            {"instructions": instructions, "non_upstream_kems": non_upstream_kems}
+        )
+        + postamble
+    )
+    with open(dst_path, "w") as f:
+        f.write(dst_content)
 
 def replacer(filename, instructions, delimiter, libjade=False):
     fragments = glob.glob(
@@ -701,14 +742,29 @@ def process_families(instructions, basedir, with_kat, with_generator, with_libja
             )
 
 
-def copy_from_upstream():
+def copy_from_upstream(slh_dsa_inst: dict):
+    """Integrate upstreams implementations and algorithms described in 
+    copy_from_upstream.yml.
+
+    :param slh_dsa_inst: instruction for integrating SLH-DSA, only used for 
+    rendering alg_support.cmake
+    """
     for t in ["kem", "sig"]:
         with open(os.path.join(os.environ['LIBOQS_DIR'], 'tests', 'KATs', t, 'kats.json'), 'r') as fp:
             kats[t] = json.load(fp)
 
     instructions = load_instructions('copy_from_upstream.yml')
+    patched_inst: dict = deepcopy(instructions)
+    patched_inst["sigs"].append(slh_dsa_inst["sigs"][0])
     process_families(instructions, os.environ['LIBOQS_DIR'], True, True)
     replacer('.CMake/alg_support.cmake', instructions, '#####')
+    # NOTE: issue 2203, only for replacing list of standardized algs
+    replace_one_fragment(
+        ".CMake/alg_support.cmake",
+        "scripts/copy_from_upstream/.CMake/alg_support.cmake/list_standardized_algs.fragment",
+        patched_inst,
+        "#####"
+    )
     replacer('CMakeLists.txt', instructions, '#####')
     replacer('src/oqsconfig.h.cmake', instructions, '/////')
     replacer('src/CMakeLists.txt', instructions, '#####')
@@ -839,9 +895,19 @@ non_upstream_kems = count_non_upstream_kems(['bike', 'frodokem', 'ntruprime', 'n
 
 if args.operation == "copy":
     # copy_from_slh_dsa_c will modify slh_dsa.yml before copy_from_upstream modifies md files
-    copy_from_slh_dsa_c.main()
+    slh_dsa_schemes: list[str] = copy_from_slh_dsa_c.main()
+    slh_dsa_instruction = {
+        "sigs": [
+            {
+                "name": "slh_dsa",
+                "schemes": [
+                    {"scheme": scheme} for scheme in slh_dsa_schemes
+                ]
+            }
+        ]
+    }
     os.chdir(os.path.join(os.environ['LIBOQS_DIR'],"scripts","copy_from_upstream"))
-    copy_from_upstream()
+    copy_from_upstream(slh_dsa_instruction)
 elif args.operation == "libjade":
     copy_from_libjade()
 elif args.operation == "verify":