fix not signing with mandated hash algorithm when using PQC

src/lib/crypto/dilithium.cpp

@@ -128,4 +128,10 @@ dilithium_hash_allowed(pgp_hash_alg_t hash_alg)
     default:
         return false;
     }
+}
+
+pgp_hash_alg_t
+dilithium_default_hash_alg()
+{
+    return PGP_HASH_SHA3_256;
 }

src/lib/crypto/dilithium.h

@@ -111,4 +111,6 @@ std::pair<pgp_dilithium_public_key_t, pgp_dilithium_private_key_t> dilithium_gen
 
 bool dilithium_hash_allowed(pgp_hash_alg_t hash_alg);
 
+pgp_hash_alg_t dilithium_default_hash_alg();
+
 #endif

src/lib/crypto/sphincsplus.cpp

@@ -383,4 +383,43 @@ sphincsplus_hash_allowed(pgp_pubkey_alg_t        pk_alg,
         break;
     }
     return true;
-}
+}
+
+pgp_hash_alg_t
+sphincsplus_default_hash_alg(pgp_pubkey_alg_t        pk_alg,
+                             sphincsplus_parameter_t sphincsplus_param)
+{
+    switch (sphincsplus_param) {
+    case sphincsplus_simple_128s:
+        FALLTHROUGH_STATEMENT;
+    case sphincsplus_simple_128f:
+        switch (pk_alg) {
+        case PGP_PKA_SPHINCSPLUS_SHA2:
+            return PGP_HASH_SHA256;
+        case PGP_PKA_SPHINCSPLUS_SHAKE:
+            return PGP_HASH_SHA3_256;
+        default:
+            RNP_LOG("invalid parameter given");
+            throw rnp::rnp_exception(RNP_ERROR_BAD_PARAMETERS);
+        }
+    case sphincsplus_simple_192s:
+        FALLTHROUGH_STATEMENT;
+    case sphincsplus_simple_192f:
+        FALLTHROUGH_STATEMENT;
+    case sphincsplus_simple_256s:
+        FALLTHROUGH_STATEMENT;
+    case sphincsplus_simple_256f:
+        switch (pk_alg) {
+        case PGP_PKA_SPHINCSPLUS_SHA2:
+            return PGP_HASH_SHA512;
+        case PGP_PKA_SPHINCSPLUS_SHAKE:
+            return PGP_HASH_SHA3_512;
+        default:
+            RNP_LOG("invalid parameter given");
+            throw rnp::rnp_exception(RNP_ERROR_BAD_PARAMETERS);
+        }
+    default:
+        RNP_LOG("invalid parameter given");
+        throw rnp::rnp_exception(RNP_ERROR_BAD_PARAMETERS);
+    }
+}

src/lib/crypto/sphincsplus.h

@@ -199,4 +199,7 @@ bool sphincsplus_hash_allowed(pgp_pubkey_alg_t        pk_alg,
                               sphincsplus_parameter_t sphincsplus_param,
                               pgp_hash_alg_t          hash_alg);
 
+pgp_hash_alg_t sphincsplus_default_hash_alg(pgp_pubkey_alg_t        pk_alg,
+                                            sphincsplus_parameter_t sphincsplus_param);
+
 #endif

src/lib/pgp-key.cpp

@@ -520,6 +520,28 @@ find_suitable_key(pgp_op_t            op,
 pgp_hash_alg_t
 pgp_hash_adjust_alg_to_key(pgp_hash_alg_t hash, const pgp_key_pkt_t *pubkey)
 {
+#if defined(ENABLE_PQC)
+    switch (pubkey->alg) {
+    case PGP_PKA_SPHINCSPLUS_SHA2:
+        FALLTHROUGH_STATEMENT;
+    case PGP_PKA_SPHINCSPLUS_SHAKE:
+        return sphincsplus_default_hash_alg(pubkey->alg,
+                                            pubkey->material.sphincsplus.pub.param());
+    case PGP_PKA_DILITHIUM3_ED25519:
+        FALLTHROUGH_STATEMENT;
+    case PGP_PKA_DILITHIUM3_P256:
+        FALLTHROUGH_STATEMENT;
+    case PGP_PKA_DILITHIUM5_P384:
+        FALLTHROUGH_STATEMENT;
+    case PGP_PKA_DILITHIUM3_BP256:
+        FALLTHROUGH_STATEMENT;
+    case PGP_PKA_DILITHIUM5_BP384:
+        return dilithium_default_hash_alg();
+    default:
+        break;
+    }
+#endif
+
     if ((pubkey->alg != PGP_PKA_DSA) && (pubkey->alg != PGP_PKA_ECDSA)) {
         return hash;
     }