lib: Improve hybrid KEX support

The algorithm was inferred from the key length. That will not always
work as SecP256r1MLKEM768 private key is 2432 bytes, exactly like an
X25519MLKEM768 one.

Raw OAP kex payloads now lead with the algorithm NID in network byte
order, so a peer reads the algorithm from the wire instead of guessing
it from the payload length.

Test if the KEX is hybrid KEM with kex_nid_is_hybrid() based on the
NID range.

The configured algorithm is passed to the raw key loaders. The public
key loader imports the key to validate it, so a corrupt or mismatched
file is reported at load time.

Signed-off-by: Dimitri Staessens <dimitri@ouroboros.rocks>
Signed-off-by: Sander Vrijders <sander@ouroboros.rocks>

2 files changed, 164 insertions, 214 deletions

diff --git a/src/lib/crypt/openssl.c b/src/lib/crypt/openssl.c
index 7716f27f..8ec3b40f 100644
--- a/src/lib/crypt/openssl.c
+++ b/src/lib/crypt/openssl.c
@@ -721,23 +721,6 @@ static int __openssl_kem_gen_key(const char * algo,
         return -ECRYPT;
 }
 
-/* Determine hybrid KEM algorithm from raw key/ciphertext length */
-static const char * __openssl_hybrid_algo_from_len(size_t len)
-{
-        switch(len) {
-        case X25519MLKEM768_PKSZ:
-                return "X25519MLKEM768";
-        case X25519MLKEM768_CTSZ:
-                return "X25519MLKEM768";
-        case X448MLKEM1024_PKSZ:
-                return "X448MLKEM1024";
-        default:
-                break;
-        }
-
-        return NULL;
-}
-
 static int __openssl_kex_gen_key(const char *  algo,
                                  EVP_PKEY **   kp)
 {
@@ -900,18 +883,19 @@ ssize_t openssl_kem_encap(buffer_t   pk,
         return -ECRYPT;
 }
 
-/* Hybrid KEM encapsulation: raw-encoded public key */
-ssize_t openssl_kem_encap_raw(buffer_t   pk,
-                              uint8_t *  ct,
-                              int        kdf,
-                              uint8_t *  s)
+/* Hybrid KEM encapsulation: NID-tagged raw-encoded public key */
+ssize_t openssl_kem_encap_raw(const char * algo,
+                              buffer_t     pk,
+                              uint8_t *    ct,
+                              int          kdf,
+                              uint8_t *    s)
 {
-        EVP_PKEY *   pub;
-        const char * algo;
-        uint8_t      salt[HKDF_SALT_LEN];
-        buffer_t     salt_b;
-        ssize_t      ret;
+        EVP_PKEY * pub;
+        uint8_t    salt[HKDF_SALT_LEN];
+        buffer_t   salt_b;
+        ssize_t    ret;
 
+        assert(algo != NULL);
         assert(pk.data != NULL);
         assert(ct != NULL);
         assert(s != NULL);
@@ -922,10 +906,6 @@ ssize_t openssl_kem_encap_raw(buffer_t   pk,
         if (derive_salt_from_pk_bytes(pk, salt_b) < 0)
                 goto fail_salt;
 
-        algo = __openssl_hybrid_algo_from_len(pk.len);
-        if (algo == NULL)
-                goto fail_salt;
-
         pub = EVP_PKEY_new_raw_public_key_ex(NULL, algo, NULL,
                                              pk.data, pk.len);
         if (pub == NULL)
@@ -1071,23 +1051,6 @@ int openssl_get_algo_from_pk_der(buffer_t pk,
         return -ECRYPT;
 }
 
-int openssl_get_algo_from_pk_raw(buffer_t pk,
-                                 char *   algo)
-{
-        const char * hybrid_algo;
-
-        assert(pk.data != NULL);
-        assert(algo != NULL);
-
-        hybrid_algo = __openssl_hybrid_algo_from_len(pk.len);
-        if (hybrid_algo == NULL)
-                return -ECRYPT;
-
-        strcpy(algo, hybrid_algo);
-
-        return 0;
-}
-
 int openssl_dhe_derive(EVP_PKEY * pkp,
                        buffer_t   pk,
                        int        kdf,
@@ -1519,14 +1482,16 @@ int openssl_load_pubkey_str(const char * str,
 }
 
 int openssl_load_pubkey_raw_file(const char * path,
+                                 const char * algo,
                                  buffer_t *   buf)
 {
-        FILE *        fp;
-        uint8_t       tmp_buf[CRYPT_KEY_BUFSZ];
-        size_t        bytes_read;
-        const char *  algo;
+        FILE *     fp;
+        uint8_t    tmp_buf[CRYPT_KEY_BUFSZ];
+        size_t     bytes_read;
+        EVP_PKEY * chk;
 
         assert(path != NULL);
+        assert(algo != NULL);
         assert(buf != NULL);
 
         fp = fopen(path, "rb");
@@ -1539,14 +1504,19 @@ int openssl_load_pubkey_raw_file(const char * path,
 
         pthread_cleanup_pop(false);
 
-        if (bytes_read == 0)
+        /* A full buffer means the file was truncated */
+        if (bytes_read == 0 || bytes_read == CRYPT_KEY_BUFSZ)
                 goto fail_read;
 
-        /* Validate that this is a known hybrid KEM format */
-        algo = __openssl_hybrid_algo_from_len(bytes_read);
-        if (algo == NULL)
+        /* Trial import: reject bad keys at load time */
+        chk = EVP_PKEY_new_raw_public_key_ex(NULL, algo, NULL,
+                                             tmp_buf, bytes_read);
+
+        if (chk == NULL)
                 goto fail_read;
 
+        EVP_PKEY_free(chk);
+
         buf->data = malloc(bytes_read);
         if (buf->data == NULL)
                 goto fail_malloc;
@@ -1565,21 +1535,6 @@ int openssl_load_pubkey_raw_file(const char * path,
         return -1;
 }
 
-/* Determine hybrid KEM algorithm from raw private key length */
-static const char * __openssl_hybrid_algo_from_sk_len(size_t len)
-{
-        switch(len) {
-        case X25519MLKEM768_SKSZ:
-                return "X25519MLKEM768";
-        case X448MLKEM1024_SKSZ:
-                return "X448MLKEM1024";
-        default:
-                break;
-        }
-
-        return NULL;
-}
-
 /* Wipe the raw-key staging buffer if a cancel aborts the read. */
 static void __cleanse_key_buf(void * o)
 {
@@ -1587,15 +1542,16 @@ static void __cleanse_key_buf(void * o)
 }
 
 int openssl_load_privkey_raw_file(const char * path,
+                                  const char * algo,
                                   void **      key)
 {
-        FILE *       fp;
-        uint8_t      tmp_buf[CRYPT_KEY_BUFSZ];
-        size_t       bytes_read;
-        const char * algo;
-        EVP_PKEY *   pkey;
+        FILE *     fp;
+        uint8_t    tmp_buf[CRYPT_KEY_BUFSZ];
+        size_t     bytes_read;
+        EVP_PKEY * pkey;
 
         assert(path != NULL);
+        assert(algo != NULL);
         assert(key != NULL);
 
         fp = fopen(path, "rb");
@@ -1613,11 +1569,6 @@ int openssl_load_privkey_raw_file(const char * path,
         if (bytes_read == 0)
                 goto fail_read;
 
-        /* Determine algorithm from key size */
-        algo = __openssl_hybrid_algo_from_sk_len(bytes_read);
-        if (algo == NULL)
-                goto fail_read;
-
         pkey = EVP_PKEY_new_raw_private_key_ex(NULL, algo, NULL,
                                                tmp_buf, bytes_read);
         /* Clear sensitive data from stack */
diff --git a/src/lib/crypt/openssl.h b/src/lib/crypt/openssl.h
index e5cc35f7..6857e39b 100644
--- a/src/lib/crypt/openssl.h
+++ b/src/lib/crypt/openssl.h
@@ -28,72 +28,70 @@
 
 struct ossl_crypt_ctx;
 
-ssize_t openssl_pkp_create(const char * algo,
-                           EVP_PKEY **  pkp,
-                           uint8_t *    pk);
+ssize_t                 openssl_pkp_create(const char * algo,
+                                           EVP_PKEY **  pkp,
+                                           uint8_t *    pk);
 
-void    openssl_pkp_destroy(EVP_PKEY * pkp);
+void                    openssl_pkp_destroy(EVP_PKEY * pkp);
 
-int     openssl_dhe_derive(EVP_PKEY * pkp,
-                           buffer_t   pk,
-                           int        kdf_nid,
-                           uint8_t *  s);
+int                     openssl_dhe_derive(EVP_PKEY * pkp,
+                                           buffer_t   pk,
+                                           int        kdf_nid,
+                                           uint8_t *  s);
 
-ssize_t openssl_kem_encap(buffer_t  pk,
-                          uint8_t * ct,
-                          int       kdf_nid,
-                          uint8_t * s);
+ssize_t                 openssl_kem_encap(buffer_t  pk,
+                                          uint8_t * ct,
+                                          int       kdf_nid,
+                                          uint8_t * s);
 
 /* no X509 DER support yet for DHKEM public keys */
-ssize_t openssl_kem_encap_raw(buffer_t  pk,
-                              uint8_t * ct,
-                              int       kdf_nid,
-                              uint8_t * s);
-
-int     openssl_kem_decap(EVP_PKEY * priv,
-                          buffer_t   ct,
-                          int        kdf_nid,
-                          uint8_t *  s);
-
-int     openssl_get_algo_from_pk_der(buffer_t pk,
-                                     char *   algo);
-
-int     openssl_get_algo_from_pk_raw(buffer_t pk,
-                                     char *   algo);
-
-int     openssl_seal(struct ossl_crypt_ctx * ctx,
-                     const uint8_t *         key,
-                     const uint8_t *         nonce,
-                     buffer_t                aad,
-                     buffer_t                in,
-                     uint8_t *               out,
-                     uint8_t *               tag);
-
-int     openssl_open(struct ossl_crypt_ctx * ctx,
-                     const uint8_t *         key,
-                     const uint8_t *         nonce,
-                     buffer_t                aad,
-                     buffer_t                in,
-                     const uint8_t *         tag,
-                     buffer_t *              out);
-
-int     openssl_oneshot_seal(int             nid,
-                             const uint8_t * key,
-                             const uint8_t * nonce,
-                             buffer_t        aad,
-                             buffer_t        in,
-                             buffer_t *      out);
-
-int     openssl_oneshot_open(int             nid,
-                             const uint8_t * key,
-                             const uint8_t * nonce,
-                             buffer_t        aad,
-                             buffer_t        in,
-                             buffer_t *      out);
-
-int     openssl_hkdf_expand(buffer_t key,
-                            buffer_t info,
-                            buffer_t out);
+ssize_t                 openssl_kem_encap_raw(const char * algo,
+                                              buffer_t     pk,
+                                              uint8_t *    ct,
+                                              int          kdf_nid,
+                                              uint8_t *    s);
+
+int                     openssl_kem_decap(EVP_PKEY * priv,
+                                          buffer_t   ct,
+                                          int        kdf_nid,
+                                          uint8_t *  s);
+
+int                     openssl_get_algo_from_pk_der(buffer_t pk,
+                                                     char *   algo);
+
+int                     openssl_seal(struct ossl_crypt_ctx * ctx,
+                                     const uint8_t *         key,
+                                     const uint8_t *         nonce,
+                                     buffer_t                aad,
+                                     buffer_t                in,
+                                     uint8_t *               out,
+                                     uint8_t *               tag);
+
+int                     openssl_open(struct ossl_crypt_ctx * ctx,
+                                     const uint8_t *         key,
+                                     const uint8_t *         nonce,
+                                     buffer_t                aad,
+                                     buffer_t                in,
+                                     const uint8_t *         tag,
+                                     buffer_t *              out);
+
+int                     openssl_oneshot_seal(int             nid,
+                                             const uint8_t * key,
+                                             const uint8_t * nonce,
+                                             buffer_t        aad,
+                                             buffer_t        in,
+                                             buffer_t *      out);
+
+int                     openssl_oneshot_open(int             nid,
+                                             const uint8_t * key,
+                                             const uint8_t * nonce,
+                                             buffer_t        aad,
+                                             buffer_t        in,
+                                             buffer_t *      out);
+
+int                     openssl_hkdf_expand(buffer_t key,
+                                            buffer_t info,
+                                            buffer_t out);
 
 struct ossl_crypt_ctx * openssl_crypt_create_ctx(struct crypt_sk * sk);
 
@@ -102,112 +100,113 @@ void                    openssl_crypt_destroy_ctx(struct ossl_crypt_ctx * ctx);
 int                     openssl_crypt_get_tagsz(struct ossl_crypt_ctx * ctx);
 
 /* AUTHENTICATION */
+int                     openssl_load_crt_file(const char * path,
+                                              void **      crt);
 
-int     openssl_load_crt_file(const char * path,
-                              void **      crt);
+int                     openssl_load_crt_str(const char * str,
+                                             void **      crt);
 
-int     openssl_load_crt_str(const char * str,
-                             void **      crt);
+int                     openssl_load_crt_der(buffer_t buf,
+                                             void **  crt);
 
-int     openssl_load_crt_der(buffer_t buf,
-                             void **  crt);
+int                     openssl_get_pubkey_crt(void *  crt,
+                                               void ** pk);
 
-int     openssl_get_pubkey_crt(void *  crt,
-                               void ** pk);
+void                    openssl_free_crt(void * crt);
 
-void    openssl_free_crt(void * crt);
+int                     openssl_load_privkey_file(const char * path,
+                                                  void **      key);
 
-int     openssl_load_privkey_file(const char * path,
-                                  void **      key);
+int                     openssl_load_privkey_str(const char * str,
+                                                 void **      key);
 
-int     openssl_load_privkey_str(const char * str,
-                                 void **      key);
+int                     openssl_load_pubkey_file(const char * path,
+                                                 void **      key);
 
-int     openssl_load_pubkey_file(const char * path,
-                                 void **      key);
+int                     openssl_load_pubkey_str(const char * str,
+                                                void **      key);
+int                     openssl_load_pubkey_file_to_der(const char * path,
+                                                        buffer_t *   buf);
+int                     openssl_load_pubkey_raw_file(const char * path,
+                                                     const char * algo,
+                                                     buffer_t *   buf);
 
-int     openssl_load_pubkey_str(const char * str,
-                                void **      key);
-int     openssl_load_pubkey_file_to_der(const char * path,
-                                        buffer_t *   buf);
-int     openssl_load_pubkey_raw_file(const char * path,
-                                     buffer_t *   buf);
-
-int     openssl_load_privkey_raw_file(const char * path,
-                                      void **      key);
+int                     openssl_load_privkey_raw_file(const char * path,
+                                                      const char * algo,
+                                                      void **      key);
 
-int     openssl_cmp_key(const EVP_PKEY * key1,
-                        const EVP_PKEY * key2);
+int                     openssl_cmp_key(const EVP_PKEY * key1,
+                                        const EVP_PKEY * key2);
 
-void    openssl_free_key(EVP_PKEY * key);
+void                    openssl_free_key(EVP_PKEY * key);
 
-int     openssl_check_crt_name(void *       crt,
-                               const char * name);
+int                     openssl_check_crt_name(void *       crt,
+                                               const char * name);
 
-int     openssl_get_crt_name(void * crt,
-                             char * name);
+int                     openssl_get_crt_name(void * crt,
+                                             char * name);
 
-int     openssl_crt_str(const void * crt,
-                        char *       str);
+int                     openssl_crt_str(const void * crt,
+                                        char *       str);
 
-int     openssl_crt_der(const void * crt,
-                        buffer_t *   buf);
+int                     openssl_crt_der(const void * crt,
+                                        buffer_t *   buf);
 
-void *  openssl_auth_create_store(void);
+void *                  openssl_auth_create_store(void);
 
-void    openssl_auth_destroy_store(void * store);
+void                    openssl_auth_destroy_store(void * store);
 
-int     openssl_auth_add_crt_to_store(void * store,
-                                      void * crt);
+int                     openssl_auth_add_crt_to_store(void * store,
+                                                      void * crt);
 
-void *  openssl_auth_create_chain(void);
+void *                  openssl_auth_create_chain(void);
 
-void    openssl_auth_destroy_chain(void * chain);
+void                    openssl_auth_destroy_chain(void * chain);
 
-int     openssl_auth_add_crt_to_chain(void * chain,
-                                      void * crt);
+int                     openssl_auth_add_crt_to_chain(void * chain,
+                                                      void * crt);
 
-int     openssl_verify_crt(void * store,
-                           void * untrusted,
-                           void * crt);
+int                     openssl_verify_crt(void * store,
+                                           void * untrusted,
+                                           void * crt);
 
-int     openssl_verify_crt_pin(void * store,
-                               void * untrusted,
-                               void * crt,
-                               void * pin);
+int                     openssl_verify_crt_pin(void * store,
+                                               void * untrusted,
+                                               void * crt,
+                                               void * pin);
 
-bool    openssl_pk_requires_md(const EVP_PKEY * pk);
+bool                    openssl_pk_requires_md(const EVP_PKEY * pk);
 
-int     openssl_sign(EVP_PKEY * pkp,
-                     int        md_nid,
-                     buffer_t   msg,
-                     buffer_t * sig);
+int                     openssl_sign(EVP_PKEY * pkp,
+                                     int        md_nid,
+                                     buffer_t   msg,
+                                     buffer_t * sig);
 
-int     openssl_verify_sig(EVP_PKEY * pk,
-                           int        md_nid,
-                           buffer_t   msg,
-                           buffer_t   sig);
+int                     openssl_verify_sig(EVP_PKEY * pk,
+                                           int        md_nid,
+                                           buffer_t   msg,
+                                           buffer_t   sig);
 
-ssize_t openssl_md_digest(int        md_nid,
-                          buffer_t   in,
-                          uint8_t *  out);
+ssize_t                 openssl_md_digest(int       md_nid,
+                                          buffer_t  in,
+                                          uint8_t * out);
 
-ssize_t openssl_md_len(int md_nid);
+ssize_t                 openssl_md_len(int md_nid);
 
 /* Secure memory allocation */
-int     openssl_secure_malloc_init(size_t max,
-                                   size_t guard);
+int                     openssl_secure_malloc_init(size_t max,
+                                                   size_t guard);
 
-void    openssl_secure_malloc_fini(void);
+void                    openssl_secure_malloc_fini(void);
 
-void *  openssl_secure_malloc(size_t size);
+void *                  openssl_secure_malloc(size_t size);
 
-void    openssl_secure_free(void * ptr,
-                            size_t size);
+void                    openssl_secure_free(void * ptr,
+                                            size_t size);
 
-void    openssl_secure_clear(void * ptr,
-                             size_t size);
+void                    openssl_secure_clear(void * ptr,
+                                             size_t size);
 
-void    openssl_cleanup(void);
+void                    openssl_cleanup(void);
 
 #endif /* OUROBOROS_LIB_CRYPT_OPENSSL_H */