lib: Improve hybrid KEX support

The algorithm was inferred from the key length. That will not always
work as SecP256r1MLKEM768 private key is 2432 bytes, exactly like an
X25519MLKEM768 one.

Raw OAP kex payloads now lead with the algorithm NID in network byte
order, so a peer reads the algorithm from the wire instead of guessing
it from the payload length.

Test if the KEX is hybrid KEM with kex_nid_is_hybrid() based on the
NID range.

The configured algorithm is passed to the raw key loaders. The public
key loader imports the key to validate it, so a corrupt or mismatched
file is reported at load time.

Signed-off-by: Dimitri Staessens <dimitri@ouroboros.rocks>
Signed-off-by: Sander Vrijders <sander@ouroboros.rocks>

2 files changed, 12 insertions, 15 deletions

diff --git a/src/lib/tests/kex_test.c b/src/lib/tests/kex_test.c
index 38835265..d7629f95 100644
--- a/src/lib/tests/kex_test.c
+++ b/src/lib/tests/kex_test.c
@@ -182,17 +182,9 @@ static int test_kex_get_algo_from_pk(const char * algo)
         pk.len  = (size_t) len;
         pk.data = buf;
 
-        /* Use raw decode for hybrid KEMs, DER for others */
-        if (IS_HYBRID_KEM(algo)) {
-                if (kex_get_algo_from_pk_raw(pk, extracted_algo) < 0) {
-                        printf("Failed to extract algo from pk.\n");
-                        goto fail_pkp;
-                }
-        } else {
-                if (kex_get_algo_from_pk_der(pk, extracted_algo) < 0) {
-                        printf("Failed to extract algo from pk.\n");
-                        goto fail_pkp;
-                }
+        if (kex_get_algo_from_pk_der(pk, extracted_algo) < 0) {
+                printf("Failed to extract algo from pk.\n");
+                goto fail_pkp;
         }
 
         /* All algorithms should now return the specific group name */
@@ -221,6 +213,11 @@ static int test_kex_get_algo_from_pk_all(void)
 
         for (i = 0; kex_supported_nids[i] != NID_undef; i++) {
                 const char * algo = kex_nid_to_str(kex_supported_nids[i]);
+
+                /* Raw hybrid PKs are opaque, OAP carries the NID */
+                if (IS_HYBRID_KEM(algo))
+                        continue;
+
                 ret |= test_kex_get_algo_from_pk(algo);
         }
 
diff --git a/src/lib/tests/kex_test_ml_kem.c b/src/lib/tests/kex_test_ml_kem.c
index 1f7d7531..7521c5d5 100644
--- a/src/lib/tests/kex_test_ml_kem.c
+++ b/src/lib/tests/kex_test_ml_kem.c
@@ -220,7 +220,7 @@ static int test_kex_kem(const char * algo)
         pk.data = buf1;
 
         if (IS_HYBRID_KEM(algo))
-                ct_len = kex_kem_encap_raw(pk, buf2, kdf, s1);
+                ct_len = kex_kem_encap_raw(algo, pk, buf2, kdf, s1);
         else
                 ct_len = kex_kem_encap(pk, buf2, kdf, s1);
 
@@ -285,7 +285,7 @@ static int test_kex_kem_corrupted_ciphertext(const char * algo)
         pk.data = buf1;
 
         if (IS_HYBRID_KEM(algo))
-                ct_len = kex_kem_encap_raw(pk, buf2, kdf, s1);
+                ct_len = kex_kem_encap_raw(algo, pk, buf2, kdf, s1);
         else
                 ct_len = kex_kem_encap(pk, buf2, kdf, s1);
 
@@ -360,7 +360,7 @@ static int test_kex_kem_wrong_keypair(const char * algo)
         }
 
         if (IS_HYBRID_KEM(algo))
-                ct_len = kex_kem_encap_raw(pk1, buf3, NID_sha256, s1);
+                ct_len = kex_kem_encap_raw(algo, pk1, buf3, NID_sha256, s1);
         else
                 ct_len = kex_kem_encap(pk1, buf3, NID_sha256, s1);
 
@@ -422,7 +422,7 @@ static int test_kex_kem_truncated_ciphertext(const char * algo)
         pk.data = buf1;
 
         if (IS_HYBRID_KEM(algo))
-                ct_len = kex_kem_encap_raw(pk, buf2, NID_sha256, s1);
+                ct_len = kex_kem_encap_raw(algo, pk, buf2, NID_sha256, s1);
         else
                 ct_len = kex_kem_encap(pk, buf2, NID_sha256, s1);