lib: Add post-quantum cryptography support

This adds initial support for runtime-configurable encryption and
post-quantum Key Encapsulation Mechanisms (KEMs) and authentication
(ML-DSA).

Supported key exchange algorithms:

  ECDH: prime256v1, secp384r1, secp521r1, X25519, X448
  Finite Field DH: ffdhe2048, ffdhe3072, ffdhe4096
  ML-KEM (FIPS 203): ML-KEM-512, ML-KEM-768, ML-KEM-1024
  Hybrid KEMs: X25519MLKEM768, X448MLKEM1024

Supported ciphers:
  AEAD: aes-128-gcm, aes-192-gcm, aes-256-gcm, chacha20-poly1305
  CTR: aes-128-ctr, aes-192-ctr, aes-256-ctr

Supported HKDFs:
  sha256, sha384, sha512, sha3-256, sha3-384, sha3-512,
  blake2b512, blake2s256

Supported Digests for DSA:
  sha256, sha384, sha512, sha3-256, sha3-384, sha3-512,
  blake2b512, blake2s256

PQC support requires OpenSSL 3.4.0+ and is detected automatically via
CMake. A DISABLE_PQC option allows building without PQC even when
available.

KEMs differ from traditional DH in that they require asymmetric roles:
one party encapsulates to the other's public key. This creates a
coordination problem during simultaneous reconnection attempts. The
kem_mode configuration parameter resolves this by pre-assigning roles:

  kem_mode=server  # Server encapsulates (1-RTT, full forward secrecy)
  kem_mode=client  # Client encapsulates (0-RTT, cached server key)

The enc.conf file format supports:

  kex=<algorithm>      # Key exchange algorithm
  cipher=<algorithm>   # Symmetric cipher
  kdf=<KDF>            # Key derivation function
  digest=<digest>      # Digest for DSA
  kem_mode=<mode>      # Server (default) or client
  none                 # Disable encryption

The OAP protocol is extended to negotiate algorithms and exchange KEX
data. All KEX messages are signed using existing authentication
infrastructure for integrity and replay protection.

Tests are split into base and _pqc variants to handle conditional PQC
compilation (kex_test.c/kex_test_pqc.c, oap_test.c/oap_test_pqc.c).

Bumped minimum required OpenSSL version for encryption to 3.0
(required for HKDF API). 1.1.1 is long time EOL.

Signed-off-by: Dimitri Staessens <dimitri@ouroboros.rocks>
Signed-off-by: Sander Vrijders <sander@ouroboros.rocks>

1 files changed, 133 insertions, 0 deletions

diff --git a/src/irmd/oap/internal.h b/src/irmd/oap/internal.h
new file mode 100644
index 00000000..8363e3a2
--- /dev/null
+++ b/src/irmd/oap/internal.h
@@ -0,0 +1,133 @@
+/*
+ * Ouroboros - Copyright (C) 2016 - 2024
+ *
+ * OAP internal definitions
+ *
+ *    Dimitri Staessens <dimitri@ouroboros.rocks>
+ *    Sander Vrijders   <sander@ouroboros.rocks>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., http://www.fsf.org/about/contact/.
+ */
+
+#ifndef OUROBOROS_IRMD_OAP_INTERNAL_H
+#define OUROBOROS_IRMD_OAP_INTERNAL_H
+
+#include <ouroboros/crypt.h>
+#include <ouroboros/list.h>
+#include <ouroboros/name.h>
+#include <ouroboros/pthread.h>
+#include <ouroboros/utils.h>
+
+#include "hdr.h"
+
+#include <stdbool.h>
+#include <stdint.h>
+
+/*
+ * Authentication functions (auth.c)
+ */
+int  oap_check_hdr(const struct oap_hdr * hdr);
+
+int  oap_auth_peer(char *                 name,
+                   const struct oap_hdr * local_hdr,
+                   const struct oap_hdr * peer_hdr);
+
+/*
+ * Key exchange functions (kex.c)
+ */
+int  oap_negotiate_cipher(const struct oap_hdr * peer_hdr,
+                          struct sec_config *    kcfg);
+
+/*
+ * Credential loading (oap.c) - shared between client and server
+ */
+#ifndef OAP_TEST_MODE
+int  load_credentials(const char *                  name,
+                      const struct name_sec_paths * paths,
+                      void **                       pkp,
+                      void **                       crt);
+
+int  load_kex_config(const char *        name,
+                     const char *        path,
+                     struct sec_config * cfg);
+#endif
+
+/*
+ * Server functions (srv.c)
+ */
+#ifndef OAP_TEST_MODE
+int  load_srv_credentials(const struct name_info * info,
+                          void **                  pkp,
+                          void **                  crt);
+
+int  load_srv_kex_config(const struct name_info * info,
+                         struct sec_config *      cfg);
+
+int  load_server_kem_keypair(const char *        name,
+                             struct sec_config * cfg,
+                             void **             pkp);
+#else
+extern int load_srv_credentials(const struct name_info * info,
+                                void **                  pkp,
+                                void **                  crt);
+extern int load_srv_kex_config(const struct name_info * info,
+                               struct sec_config *      cfg);
+extern int load_server_kem_keypair(const char *        name,
+                                   struct sec_config * cfg,
+                                   void **             pkp);
+#endif
+
+int  do_server_kex(const struct name_info * info,
+                   struct oap_hdr *         peer_hdr,
+                   struct sec_config *      kcfg,
+                   buffer_t *               kex,
+                   struct crypt_sk *        sk);
+
+/*
+ * Client functions (cli.c)
+ */
+#ifndef OAP_TEST_MODE
+int  load_cli_credentials(const struct name_info * info,
+                          void **                  pkp,
+                          void **                  crt);
+
+int  load_cli_kex_config(const struct name_info * info,
+                         struct sec_config *      cfg);
+
+int  load_server_kem_pk(const char *        name,
+                        struct sec_config * cfg,
+                        buffer_t *          pk);
+#else
+extern int load_cli_credentials(const struct name_info * info,
+                                void **                  pkp,
+                                void **                  crt);
+extern int load_cli_kex_config(const struct name_info * info,
+                               struct sec_config *      cfg);
+extern int load_server_kem_pk(const char *        name,
+                              struct sec_config * cfg,
+                              buffer_t *          pk);
+#endif
+
+int  oap_client_kex_prepare(struct sec_config * kcfg,
+                            buffer_t            server_pk,
+                            buffer_t *          kex,
+                            uint8_t *           key,
+                            void **             ephemeral_pkp);
+
+int  oap_client_kex_complete(const struct oap_hdr * peer_hdr,
+                             struct sec_config *    kcfg,
+                             void *                 pkp,
+                             uint8_t *              key);
+
+#endif /* OUROBOROS_IRMD_OAP_INTERNAL_H */