diff --git a/ext/openssl/ossl_pkey.c b/ext/openssl/ossl_pkey.c
index aba03b15e..db3d98b45 100644
--- a/ext/openssl/ossl_pkey.c
+++ b/ext/openssl/ossl_pkey.c
@@ -441,24 +441,31 @@ add_ec_parameters_to_builder(VALUE key, VALUE value, VALUE arg) {
 
     if(strcmp(OSSL_PKEY_PARAM_GROUP_NAME, key_ptr) == 0) {
         StringValue(value);
-        if(!OSSL_PARAM_BLD_push_utf8_string(params_builder, key_ptr, RSTRING_PTR(value), RSTRING_LENINT(value)))
+        if(!OSSL_PARAM_BLD_push_utf8_string(params_builder, key_ptr, RSTRING_PTR(value), RSTRING_LENINT(value))) {
+            OSSL_PARAM_BLD_free(params_builder);
             ossl_raise(ePKeyError, "OSSL_PARAM_BLD_push_utf8_string");
+        }
         return ST_CONTINUE;
     }
 
     if(strcmp(OSSL_PKEY_PARAM_PUB_KEY, key_ptr) == 0) {
         StringValue(value);
-        if(!OSSL_PARAM_BLD_push_octet_string(params_builder, OSSL_PKEY_PARAM_PUB_KEY, RSTRING_PTR(value), RSTRING_LENINT(value)))
+        if(!OSSL_PARAM_BLD_push_octet_string(params_builder, OSSL_PKEY_PARAM_PUB_KEY, RSTRING_PTR(value), RSTRING_LENINT(value))) {
+            OSSL_PARAM_BLD_free(params_builder);
             ossl_raise(ePKeyError, "OSSL_PARAM_BLD_push_octet_string");
+        }
         return ST_CONTINUE;
     }
 
     if(strcmp(OSSL_PKEY_PARAM_PRIV_KEY, key_ptr) == 0) {
-        if(!OSSL_PARAM_BLD_push_BN(params_builder, OSSL_PKEY_PARAM_PRIV_KEY, GetBNPtr(value)))
+        if(!OSSL_PARAM_BLD_push_BN(params_builder, OSSL_PKEY_PARAM_PRIV_KEY, GetBNPtr(value))) {
+            OSSL_PARAM_BLD_free(params_builder);
             ossl_raise(ePKeyError, "OSSL_PARAM_BLD_push_BN");
+        }
         return ST_CONTINUE;
     }
 
+    OSSL_PARAM_BLD_free(params_builder);
     ossl_raise(ePKeyError, "Unsupported parameter \"%s\"", key_ptr);
 }
 
@@ -524,11 +531,15 @@ add_rsa_parameters_to_builder(VALUE key, VALUE value, VALUE arg) {
        strcmp(OSSL_PKEY_PARAM_RSA_COEFFICIENT7, key_ptr) == 0 ||
        strcmp(OSSL_PKEY_PARAM_RSA_COEFFICIENT8, key_ptr) == 0 ||
        strcmp(OSSL_PKEY_PARAM_RSA_COEFFICIENT9, key_ptr) == 0) {
-        if(!OSSL_PARAM_BLD_push_BN(params_builder, key_ptr, GetBNPtr(value)))
+        if(!OSSL_PARAM_BLD_push_BN(params_builder, key_ptr, GetBNPtr(value))) {
+            OSSL_PARAM_BLD_free(params_builder);
             ossl_raise(ePKeyError, "OSSL_PARAM_BLD_push_BN");
+        }
+
         return ST_CONTINUE;
     }
 
+    OSSL_PARAM_BLD_free(params_builder);
     ossl_raise(ePKeyError, "Unsupported parameter \"%s\"", key_ptr);
 }
 
@@ -571,18 +582,23 @@ add_ffc_parameters_to_builder(VALUE key, VALUE value, VALUE arg) {
        strcmp(OSSL_PKEY_PARAM_FFC_VALIDATE_PQ, key_ptr) == 0 ||
        strcmp(OSSL_PKEY_PARAM_FFC_VALIDATE_G, key_ptr) == 0 ||
        strcmp(OSSL_PKEY_PARAM_FFC_VALIDATE_LEGACY, key_ptr) == 0) {
-        if(!OSSL_PARAM_BLD_push_BN(params_builder, key_ptr, GetBNPtr(value)))
+        if(!OSSL_PARAM_BLD_push_BN(params_builder, key_ptr, GetBNPtr(value))) {
+            OSSL_PARAM_BLD_free(params_builder);
             ossl_raise(ePKeyError, "OSSL_PARAM_BLD_push_BN");
+        }
         return ST_CONTINUE;
     }
 
     if(strcmp(OSSL_PKEY_PARAM_FFC_SEED, key_ptr) == 0) {
         StringValue(value);
-        if(!OSSL_PARAM_BLD_push_octet_string(params_builder, key_ptr, RSTRING_PTR(value), RSTRING_LENINT(value)))
+        if(!OSSL_PARAM_BLD_push_octet_string(params_builder, key_ptr, RSTRING_PTR(value), RSTRING_LENINT(value))) {
+            OSSL_PARAM_BLD_free(params_builder);
             ossl_raise(ePKeyError, "OSSL_PARAM_BLD_push_octet_string");
+        }
         return ST_CONTINUE;
     }
 
+    OSSL_PARAM_BLD_free(params_builder);
     ossl_raise(ePKeyError, "Unsupported parameter \"%s\"", key_ptr);
 }