digest: use EVP_MD_fetch() if available

With the introduction of OpenSSL 3 providers, newly implemented
algorithms do not necessarily have a corresponding NID assigned. To use
such an algorithm, it has to be "fetched" from providers using the new
EVP_*_fetch() functions.

For digest algorithms, we have to use EVP_MD_fetch() instead of the
existing EVP_get_digestbyname(). However, it is not a drop-in
replacement because:

 - EVP_MD_fetch() does not support all algorithm name aliases recognized
   by EVP_get_digestbyname().

 - Both return an EVP_MD, but the one returned by EVP_MD_fetch() is
   sometimes reference counted and the user has to explicitly release
   it with EVP_MD_free().

So, keep using EVP_get_digestbyname() for all OpenSSL versions for now,
and fall back to EVP_MD_fetch() if it fails. In the latter case, prepare
a T_DATA object to manage the fetched EVP_MD's lifetime.

Author: rhenium

ext/openssl/ossl_cipher.c

@@ -264,7 +264,7 @@ ossl_cipher_pkcs5_keyivgen(int argc, VALUE *argv, VALUE self)
 {
     EVP_CIPHER_CTX *ctx;
     const EVP_MD *digest;
-    VALUE vpass, vsalt, viter, vdigest;
+    VALUE vpass, vsalt, viter, vdigest, md_holder;
     unsigned char key[EVP_MAX_KEY_LENGTH], iv[EVP_MAX_IV_LENGTH], *salt = NULL;
     int iter;
 
@@ -279,7 +279,7 @@ ossl_cipher_pkcs5_keyivgen(int argc, VALUE *argv, VALUE self)
     iter = NIL_P(viter) ? 2048 : NUM2INT(viter);
     if (iter <= 0)
 	rb_raise(rb_eArgError, "iterations must be a positive integer");
-    digest = NIL_P(vdigest) ? EVP_md5() : ossl_evp_get_digestbyname(vdigest);
+    digest = NIL_P(vdigest) ? EVP_md5() : ossl_evp_md_fetch(vdigest, &md_holder);
     GetCipher(self, ctx);
     EVP_BytesToKey(EVP_CIPHER_CTX_cipher(ctx), digest, salt,
 		   (unsigned char *)RSTRING_PTR(vpass), RSTRING_LENINT(vpass), iter, key, iv);

ext/openssl/ossl_digest.c

@@ -21,6 +21,7 @@
  */
 static VALUE cDigest;
 static VALUE eDigestError;
+static ID id_md_holder;
 
 static VALUE ossl_digest_alloc(VALUE klass);
 
@@ -38,35 +39,62 @@ static const rb_data_type_t ossl_digest_type = {
     0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED,
 };
 
+#ifdef OSSL_USE_PROVIDER
+static void
+ossl_evp_md_free(void *ptr)
+{
+    // This is safe to call against const EVP_MD * returned by
+    // EVP_get_digestbyname()
+    EVP_MD_free(ptr);
+}
+
+static const rb_data_type_t ossl_evp_md_holder_type = {
+    "OpenSSL/EVP_MD",
+    {
+        .dfree = ossl_evp_md_free,
+    },
+    0, 0, RUBY_TYPED_FREE_IMMEDIATELY | RUBY_TYPED_WB_PROTECTED,
+};
+#endif
+
 /*
  * Public
  */
 const EVP_MD *
-ossl_evp_get_digestbyname(VALUE obj)
+ossl_evp_md_fetch(VALUE obj, volatile VALUE *holder)
 {
-    const EVP_MD *md;
-    ASN1_OBJECT *oid = NULL;
-
-    if (RB_TYPE_P(obj, T_STRING)) {
-    	const char *name = StringValueCStr(obj);
-
-	md = EVP_get_digestbyname(name);
-	if (!md) {
-	    oid = OBJ_txt2obj(name, 0);
-	    md = EVP_get_digestbyobj(oid);
-	    ASN1_OBJECT_free(oid);
-	}
-        if (!md)
-            ossl_raise(rb_eArgError,
-                       "unsupported digest algorithm: %"PRIsVALUE, obj);
-    } else {
+    *holder = Qnil;
+    if (rb_obj_is_kind_of(obj, cDigest)) {
         EVP_MD_CTX *ctx;
-
         GetDigest(obj, ctx);
-
-        md = EVP_MD_CTX_get0_md(ctx);
+        EVP_MD *md = (EVP_MD *)EVP_MD_CTX_get0_md(ctx);
+#ifdef OSSL_USE_PROVIDER
+        *holder = TypedData_Wrap_Struct(0, &ossl_evp_md_holder_type, NULL);
+        if (!EVP_MD_up_ref(md))
+            ossl_raise(eDigestError, "EVP_MD_up_ref");
+        RTYPEDDATA_DATA(*holder) = md;
+#endif
+        return md;
     }
 
+    const char *name = StringValueCStr(obj);
+    EVP_MD *md = (EVP_MD *)EVP_get_digestbyname(name);
+    if (!md) {
+        ASN1_OBJECT *oid = OBJ_txt2obj(name, 0);
+        md = (EVP_MD *)EVP_get_digestbyobj(oid);
+        ASN1_OBJECT_free(oid);
+    }
+#ifdef OSSL_USE_PROVIDER
+    if (!md) {
+        ossl_clear_error();
+        *holder = TypedData_Wrap_Struct(0, &ossl_evp_md_holder_type, NULL);
+        md = EVP_MD_fetch(NULL, name, NULL);
+        RTYPEDDATA_DATA(*holder) = md;
+    }
+#endif
+    if (!md)
+        ossl_raise(rb_eArgError, "unsupported digest algorithm: %"PRIsVALUE,
+                   obj);
     return md;
 }
 
@@ -76,6 +104,9 @@ ossl_digest_new(const EVP_MD *md)
     VALUE ret;
     EVP_MD_CTX *ctx;
 
+    // NOTE: This does not set id_md_holder because this function should
+    // only be called from ossl_engine.c, which will not use any
+    // reference-counted digests.
     ret = ossl_digest_alloc(cDigest);
     ctx = EVP_MD_CTX_new();
     if (!ctx)
@@ -122,10 +153,10 @@ ossl_digest_initialize(int argc, VALUE *argv, VALUE self)
 {
     EVP_MD_CTX *ctx;
     const EVP_MD *md;
-    VALUE type, data;
+    VALUE type, data, md_holder;
 
     rb_scan_args(argc, argv, "11", &type, &data);
-    md = ossl_evp_get_digestbyname(type);
+    md = ossl_evp_md_fetch(type, &md_holder);
     if (!NIL_P(data)) StringValue(data);
 
     TypedData_Get_Struct(self, EVP_MD_CTX, &ossl_digest_type, ctx);
@@ -137,6 +168,7 @@ ossl_digest_initialize(int argc, VALUE *argv, VALUE self)
 
     if (!EVP_DigestInit_ex(ctx, md, NULL))
 	ossl_raise(eDigestError, "Digest initialization failed");
+    rb_ivar_set(self, id_md_holder, md_holder);
 
     if (!NIL_P(data)) return ossl_digest_update(self, data);
     return self;
@@ -443,4 +475,6 @@ Init_ossl_digest(void)
     rb_define_method(cDigest, "block_length", ossl_digest_block_length, 0);
 
     rb_define_method(cDigest, "name", ossl_digest_name, 0);
+
+    id_md_holder = rb_intern_const("EVP_MD_holder");
 }

ext/openssl/ossl_digest.h

@@ -10,7 +10,15 @@
 #if !defined(_OSSL_DIGEST_H_)
 #define _OSSL_DIGEST_H_
 
-const EVP_MD *ossl_evp_get_digestbyname(VALUE);
+/*
+ * Gets EVP_MD from a String or an OpenSSL::Digest instance (discouraged, but
+ * still supported for compatibility). A holder object is created if the EVP_MD
+ * is a "fetched" algorithm.
+ */
+const EVP_MD *ossl_evp_md_fetch(VALUE obj, volatile VALUE *holder);
+/*
+ * This is meant for OpenSSL::Engine#digest. EVP_MD must not be a fetched one.
+ */
 VALUE ossl_digest_new(const EVP_MD *);
 void Init_ossl_digest(void);
 

ext/openssl/ossl_hmac.c

@@ -23,6 +23,7 @@
  */
 static VALUE cHMAC;
 static VALUE eHMACError;
+static ID id_md_holder;
 
 /*
  * Public
@@ -94,16 +95,19 @@ ossl_hmac_initialize(VALUE self, VALUE key, VALUE digest)
 {
     EVP_MD_CTX *ctx;
     EVP_PKEY *pkey;
+    const EVP_MD *md;
+    VALUE md_holder;
 
     GetHMAC(self, ctx);
     StringValue(key);
+    md = ossl_evp_md_fetch(digest, &md_holder);
+    rb_ivar_set(self, id_md_holder, md_holder);
     pkey = EVP_PKEY_new_raw_private_key(EVP_PKEY_HMAC, NULL,
                                         (unsigned char *)RSTRING_PTR(key),
                                         RSTRING_LENINT(key));
     if (!pkey)
         ossl_raise(eHMACError, "EVP_PKEY_new_raw_private_key");
-    if (EVP_DigestSignInit(ctx, NULL, ossl_evp_get_digestbyname(digest),
-                           NULL, pkey) != 1) {
+    if (EVP_DigestSignInit(ctx, NULL, md, NULL, pkey) != 1) {
         EVP_PKEY_free(pkey);
         ossl_raise(eHMACError, "EVP_DigestSignInit");
     }
@@ -300,4 +304,6 @@ Init_ossl_hmac(void)
     rb_define_method(cHMAC, "hexdigest", ossl_hmac_hexdigest, 0);
     rb_define_alias(cHMAC, "inspect", "hexdigest");
     rb_define_alias(cHMAC, "to_s", "hexdigest");
+
+    id_md_holder = rb_intern_const("EVP_MD_holder");
 }

ext/openssl/ossl_kdf.c

@@ -35,7 +35,7 @@ static VALUE mKDF, eKDF;
 static VALUE
 kdf_pbkdf2_hmac(int argc, VALUE *argv, VALUE self)
 {
-    VALUE pass, salt, opts, kwargs[4], str;
+    VALUE pass, salt, opts, kwargs[4], str, md_holder;
     static ID kwargs_ids[4];
     int iters, len;
     const EVP_MD *md;
@@ -53,7 +53,7 @@ kdf_pbkdf2_hmac(int argc, VALUE *argv, VALUE self)
     salt = StringValue(kwargs[0]);
     iters = NUM2INT(kwargs[1]);
     len = NUM2INT(kwargs[2]);
-    md = ossl_evp_get_digestbyname(kwargs[3]);
+    md = ossl_evp_md_fetch(kwargs[3], &md_holder);
 
     str = rb_str_new(0, len);
     if (!PKCS5_PBKDF2_HMAC(RSTRING_PTR(pass), RSTRING_LENINT(pass),
@@ -172,7 +172,7 @@ kdf_scrypt(int argc, VALUE *argv, VALUE self)
 static VALUE
 kdf_hkdf(int argc, VALUE *argv, VALUE self)
 {
-    VALUE ikm, salt, info, opts, kwargs[4], str;
+    VALUE ikm, salt, info, opts, kwargs[4], str, md_holder;
     static ID kwargs_ids[4];
     int saltlen, ikmlen, infolen;
     size_t len;
@@ -197,7 +197,7 @@ kdf_hkdf(int argc, VALUE *argv, VALUE self)
     len = (size_t)NUM2LONG(kwargs[2]);
     if (len > LONG_MAX)
 	rb_raise(rb_eArgError, "length must be non-negative");
-    md = ossl_evp_get_digestbyname(kwargs[3]);
+    md = ossl_evp_md_fetch(kwargs[3], &md_holder);
 
     str = rb_str_new(NULL, (long)len);
     pctx = EVP_PKEY_CTX_new_id(EVP_PKEY_HKDF, NULL);

ext/openssl/ossl_ns_spki.c

@@ -283,13 +283,13 @@ ossl_spki_sign(VALUE self, VALUE key, VALUE digest)
     NETSCAPE_SPKI *spki;
     EVP_PKEY *pkey;
     const EVP_MD *md;
+    VALUE md_holder;
 
     pkey = GetPrivPKeyPtr(key); /* NO NEED TO DUP */
-    md = ossl_evp_get_digestbyname(digest);
+    md = ossl_evp_md_fetch(digest, &md_holder);
     GetSPKI(self, spki);
-    if (!NETSCAPE_SPKI_sign(spki, pkey, md)) {
-	ossl_raise(eSPKIError, NULL);
-    }
+    if (!NETSCAPE_SPKI_sign(spki, pkey, md))
+        ossl_raise(eSPKIError, "NETSCAPE_SPKI_sign");
 
     return self;
 }

ext/openssl/ossl_ocsp.c

@@ -369,7 +369,7 @@ ossl_ocspreq_get_certid(VALUE self)
 static VALUE
 ossl_ocspreq_sign(int argc, VALUE *argv, VALUE self)
 {
-    VALUE signer_cert, signer_key, certs, flags, digest;
+    VALUE signer_cert, signer_key, certs, flags, digest, md_holder;
     OCSP_REQUEST *req;
     X509 *signer;
     EVP_PKEY *key;
@@ -384,18 +384,16 @@ ossl_ocspreq_sign(int argc, VALUE *argv, VALUE self)
     key = GetPrivPKeyPtr(signer_key);
     if (!NIL_P(flags))
 	flg = NUM2INT(flags);
-    if (NIL_P(digest))
-	md = NULL;
-    else
-	md = ossl_evp_get_digestbyname(digest);
+    md = NIL_P(digest) ? NULL : ossl_evp_md_fetch(digest, &md_holder);
     if (NIL_P(certs))
 	flg |= OCSP_NOCERTS;
     else
 	x509s = ossl_x509_ary2sk(certs);
 
     ret = OCSP_request_sign(req, signer, key, md, x509s, flg);
     sk_X509_pop_free(x509s, X509_free);
-    if (!ret) ossl_raise(eOCSPError, NULL);
+    if (!ret)
+        ossl_raise(eOCSPError, "OCSP_request_sign");
 
     return self;
 }
@@ -1000,7 +998,7 @@ ossl_ocspbres_find_response(VALUE self, VALUE target)
 static VALUE
 ossl_ocspbres_sign(int argc, VALUE *argv, VALUE self)
 {
-    VALUE signer_cert, signer_key, certs, flags, digest;
+    VALUE signer_cert, signer_key, certs, flags, digest, md_holder;
     OCSP_BASICRESP *bs;
     X509 *signer;
     EVP_PKEY *key;
@@ -1015,18 +1013,16 @@ ossl_ocspbres_sign(int argc, VALUE *argv, VALUE self)
     key = GetPrivPKeyPtr(signer_key);
     if (!NIL_P(flags))
 	flg = NUM2INT(flags);
-    if (NIL_P(digest))
-	md = NULL;
-    else
-	md = ossl_evp_get_digestbyname(digest);
+    md = NIL_P(digest) ? NULL : ossl_evp_md_fetch(digest, &md_holder);
     if (NIL_P(certs))
 	flg |= OCSP_NOCERTS;
     else
 	x509s = ossl_x509_ary2sk(certs);
 
     ret = OCSP_basic_sign(bs, signer, key, md, x509s, flg);
     sk_X509_pop_free(x509s, X509_free);
-    if (!ret) ossl_raise(eOCSPError, NULL);
+    if (!ret)
+        ossl_raise(eOCSPError, "OCSP_basic_sign");
 
     return self;
 }
@@ -1460,10 +1456,11 @@ ossl_ocspcid_initialize(int argc, VALUE *argv, VALUE self)
     else {
 	X509 *x509s, *x509i;
 	const EVP_MD *md;
+        VALUE md_holder;
 
 	x509s = GetX509CertPtr(subject); /* NO NEED TO DUP */
 	x509i = GetX509CertPtr(issuer); /* NO NEED TO DUP */
-	md = !NIL_P(digest) ? ossl_evp_get_digestbyname(digest) : NULL;
+        md = NIL_P(digest) ? NULL : ossl_evp_md_fetch(digest, &md_holder);
 
 	newid = OCSP_cert_to_id(md, x509s, x509i);
 	if (!newid)

ext/openssl/ossl_pkcs12.c

@@ -271,7 +271,7 @@ static VALUE
 pkcs12_set_mac(int argc, VALUE *argv, VALUE self)
 {
     PKCS12 *p12;
-    VALUE pass, salt, iter, md_name;
+    VALUE pass, salt, iter, md_name, md_holder = Qnil;
     int iter_i = 0;
     const EVP_MD *md_type = NULL;
 
@@ -285,7 +285,7 @@ pkcs12_set_mac(int argc, VALUE *argv, VALUE self)
     if (!NIL_P(iter))
         iter_i = NUM2INT(iter);
     if (!NIL_P(md_name))
-        md_type = ossl_evp_get_digestbyname(md_name);
+        md_type = ossl_evp_md_fetch(md_name, &md_holder);
 
     if (!PKCS12_set_mac(p12, RSTRING_PTR(pass), RSTRING_LENINT(pass),
                         !NIL_P(salt) ? (unsigned char *)RSTRING_PTR(salt) : NULL,

ext/openssl/ossl_pkcs7.c

@@ -68,6 +68,7 @@ static VALUE cPKCS7;
 static VALUE cPKCS7Signer;
 static VALUE cPKCS7Recipient;
 static VALUE ePKCS7Error;
+static ID id_md_holder;
 
 static void
 ossl_pkcs7_free(void *ptr)
@@ -968,14 +969,15 @@ ossl_pkcs7si_initialize(VALUE self, VALUE cert, VALUE key, VALUE digest)
     EVP_PKEY *pkey;
     X509 *x509;
     const EVP_MD *md;
+    VALUE md_holder;
 
     pkey = GetPrivPKeyPtr(key); /* NO NEED TO DUP */
     x509 = GetX509CertPtr(cert); /* NO NEED TO DUP */
-    md = ossl_evp_get_digestbyname(digest);
+    md = ossl_evp_md_fetch(digest, &md_holder);
+    rb_ivar_set(self, id_md_holder, md_holder);
     GetPKCS7si(self, p7si);
-    if (!(PKCS7_SIGNER_INFO_set(p7si, x509, pkey, md))) {
-	ossl_raise(ePKCS7Error, NULL);
-    }
+    if (!(PKCS7_SIGNER_INFO_set(p7si, x509, pkey, md)))
+        ossl_raise(ePKCS7Error, "PKCS7_SIGNER_INFO_set");
 
     return self;
 }
@@ -1161,4 +1163,6 @@ Init_ossl_pkcs7(void)
     DefPKCS7Const(BINARY);
     DefPKCS7Const(NOATTR);
     DefPKCS7Const(NOSMIMECAP);
+
+    id_md_holder = rb_intern_const("EVP_MD_holder");
 }