lib: Add post-quantum cryptography support

This adds initial support for runtime-configurable encryption and
post-quantum Key Encapsulation Mechanisms (KEMs) and authentication
(ML-DSA).

Supported key exchange algorithms:

  ECDH: prime256v1, secp384r1, secp521r1, X25519, X448
  Finite Field DH: ffdhe2048, ffdhe3072, ffdhe4096
  ML-KEM (FIPS 203): ML-KEM-512, ML-KEM-768, ML-KEM-1024
  Hybrid KEMs: X25519MLKEM768, X448MLKEM1024

Supported ciphers:
  AEAD: aes-128-gcm, aes-192-gcm, aes-256-gcm, chacha20-poly1305
  CTR: aes-128-ctr, aes-192-ctr, aes-256-ctr

Supported HKDFs:
  sha256, sha384, sha512, sha3-256, sha3-384, sha3-512,
  blake2b512, blake2s256

Supported Digests for DSA:
  sha256, sha384, sha512, sha3-256, sha3-384, sha3-512,
  blake2b512, blake2s256

PQC support requires OpenSSL 3.4.0+ and is detected automatically via
CMake. A DISABLE_PQC option allows building without PQC even when
available.

KEMs differ from traditional DH in that they require asymmetric roles:
one party encapsulates to the other's public key. This creates a
coordination problem during simultaneous reconnection attempts. The
kem_mode configuration parameter resolves this by pre-assigning roles:

  kem_mode=server  # Server encapsulates (1-RTT, full forward secrecy)
  kem_mode=client  # Client encapsulates (0-RTT, cached server key)

The enc.conf file format supports:

  kex=<algorithm>      # Key exchange algorithm
  cipher=<algorithm>   # Symmetric cipher
  kdf=<KDF>            # Key derivation function
  digest=<digest>      # Digest for DSA
  kem_mode=<mode>      # Server (default) or client
  none                 # Disable encryption

The OAP protocol is extended to negotiate algorithms and exchange KEX
data. All KEX messages are signed using existing authentication
infrastructure for integrity and replay protection.

Tests are split into base and _pqc variants to handle conditional PQC
compilation (kex_test.c/kex_test_pqc.c, oap_test.c/oap_test_pqc.c).

Bumped minimum required OpenSSL version for encryption to 3.0
(required for HKDF API). 1.1.1 is long time EOL.

Signed-off-by: Dimitri Staessens <dimitri@ouroboros.rocks>
Signed-off-by: Sander Vrijders <sander@ouroboros.rocks>

1 files changed, 76 insertions, 23 deletions

diff --git a/src/lib/crypt/openssl.h b/src/lib/crypt/openssl.h
index d4ee73b9..c28d0b4d 100644
--- a/src/lib/crypt/openssl.h
+++ b/src/lib/crypt/openssl.h
@@ -26,28 +26,52 @@
 #ifndef OUROBOROS_LIB_CRYPT_OPENSSL_H
 #define OUROBOROS_LIB_CRYPT_OPENSSL_H
 
-ssize_t openssl_ecdh_pkp_create(void **   pkp,
-                                uint8_t * pk);
+struct ossl_crypt_ctx;
 
-void    openssl_ecdh_pkp_destroy(void * pkp);
+ssize_t openssl_pkp_create(const char * algo,
+                           EVP_PKEY **  pkp,
+                           uint8_t *    pk);
 
-int     openssl_ecdh_derive(void *    pkp,
-                            buffer_t  pk,
-                            uint8_t * s);
+void    openssl_pkp_destroy(EVP_PKEY * pkp);
 
-int     openssl_encrypt(void *     ctx,
-                        uint8_t *  key,
-                        buffer_t   in,
-                        buffer_t * out);
+int     openssl_dhe_derive(EVP_PKEY * pkp,
+                           buffer_t   pk,
+                           int        kdf_nid,
+                           uint8_t *  s);
 
-int     openssl_decrypt(void *     ctx,
-                        uint8_t *  key,
-                        buffer_t   in,
-                        buffer_t * out);
+ssize_t openssl_kem_encap(buffer_t  pk,
+                          uint8_t * ct,
+                          int       kdf_nid,
+                          uint8_t * s);
 
-void *  openssl_crypt_create_ctx(void);
+/* no X509 DER support yet for DHKEM public keys */
+ssize_t openssl_kem_encap_raw(buffer_t  pk,
+                              uint8_t * ct,
+                              int       kdf_nid,
+                              uint8_t * s);
 
-void    openssl_crypt_destroy_ctx(void * ctx);
+int     openssl_kem_decap(EVP_PKEY * priv,
+                          buffer_t   ct,
+                          int        kdf_nid,
+                          uint8_t *  s);
+
+int     openssl_get_algo_from_pk_der(buffer_t pk,
+                                     char *   algo);
+
+int     openssl_get_algo_from_pk_raw(buffer_t pk,
+                                     char *   algo);
+
+int     openssl_encrypt(struct ossl_crypt_ctx * ctx,
+                        buffer_t                in,
+                        buffer_t *              out);
+
+int     openssl_decrypt(struct ossl_crypt_ctx * ctx,
+                        buffer_t                in,
+                        buffer_t *              out);
+
+struct ossl_crypt_ctx * openssl_crypt_create_ctx(struct crypt_sk * sk);
+
+void                    openssl_crypt_destroy_ctx(struct ossl_crypt_ctx * ctx);
 
 /* AUTHENTICATION */
 
@@ -76,15 +100,25 @@ int     openssl_load_pubkey_file(const char * path,
 
 int     openssl_load_pubkey_str(const char * str,
                                 void **      key);
+int     openssl_load_pubkey_file_to_der(const char * path,
+                                        buffer_t *   buf);
+int     openssl_load_pubkey_raw_file(const char * path,
+                                     buffer_t *   buf);
+
+int     openssl_load_privkey_raw_file(const char * path,
+                                      void **      key);
 
-int     openssl_cmp_key(const void * key1,
-                        const void * key2);
+int     openssl_cmp_key(const EVP_PKEY * key1,
+                        const EVP_PKEY * key2);
 
-void    openssl_free_key(void * key);
+void    openssl_free_key(EVP_PKEY * key);
 
 int     openssl_check_crt_name(void *       crt,
                                const char * name);
 
+int     openssl_get_crt_name(void * crt,
+                             char * name);
+
 int     openssl_crt_str(const void * crt,
                         char *       str);
 
@@ -101,12 +135,31 @@ int     openssl_auth_add_crt_to_store(void * store,
 int     openssl_verify_crt(void * store,
                            void * crt);
 
-int     openssl_sign(void *     pkp,
+int     openssl_sign(EVP_PKEY * pkp,
+                     int        md_nid,
                      buffer_t   msg,
                      buffer_t * sig);
 
-int     openssl_verify_sig(void *   pk,
-                           buffer_t msg,
-                           buffer_t sig);
+int     openssl_verify_sig(EVP_PKEY * pk,
+                           int        md_nid,
+                           buffer_t   msg,
+                           buffer_t   sig);
+
+ssize_t openssl_md_digest(int        md_nid,
+                          buffer_t   in,
+                          uint8_t *  out);
+
+ssize_t openssl_md_len(int md_nid);
+
+/* Secure memory allocation */
+int     openssl_secure_malloc_init(size_t max,
+                                   size_t guard);
+
+void    openssl_secure_malloc_fini(void);
+
+void *  openssl_secure_malloc(size_t size);
+
+void    openssl_secure_free(void * ptr,
+                            size_t size);
 
 #endif /* OUROBOROS_LIB_CRYPT_OPENSSL_H */