Fix PEM_write_bio_PrivateKey_traditional() to not output PKCS#8

PEM_write_bio_PrivateKey_traditional() uses i2d_PrivateKey() to do the
actual encoding to DER.  However, i2d_PrivateKey() is a generic
function that will do what it can to produce output according to what
the associated EVP_PKEY_ASN1_METHOD offers.  If that method offers a
function 'old_priv_encode', which is expected to produce the
"traditional" encoded form, then i2d_PrivateKey() uses that.  If not,
i2d_PrivateKey() will go on and used more modern methods, which are
all expected to produce PKCS#8.

To ensure that PEM_write_bio_PrivateKey_traditional() never produces
more modern encoded forms, an extra check that 'old_priv_encode' is
non-NULL is added.  If it is NULL, an error is returned.

Reviewed-by: Tomas Mraz <tmraz@fedoraproject.org>
(Merged from https://github.com/openssl/openssl/pull/12729)

4 files changed, 17 insertions, 2 deletions

diff --git a/crypto/err/openssl.txt b/crypto/err/openssl.txt
index 3ca271beb5..0b5873ebbc 100644
--- a/crypto/err/openssl.txt
+++ b/crypto/err/openssl.txt
@@ -934,6 +934,8 @@ PEM_F_PEM_READ_PRIVATEKEY:124:PEM_read_PrivateKey
 PEM_F_PEM_SIGNFINAL:112:PEM_SignFinal
 PEM_F_PEM_WRITE:113:PEM_write
 PEM_F_PEM_WRITE_BIO:114:PEM_write_bio
+PEM_F_PEM_WRITE_BIO_PRIVATEKEY_TRADITIONAL:147:\
+	PEM_write_bio_PrivateKey_traditional
 PEM_F_PEM_WRITE_PRIVATEKEY:139:PEM_write_PrivateKey
 PEM_F_PEM_X509_INFO_READ:115:PEM_X509_INFO_read
 PEM_F_PEM_X509_INFO_READ_BIO:116:PEM_X509_INFO_read_bio
@@ -2400,6 +2402,7 @@ PEM_R_UNEXPECTED_DEK_IV:130:unexpected dek iv
 PEM_R_UNSUPPORTED_CIPHER:113:unsupported cipher
 PEM_R_UNSUPPORTED_ENCRYPTION:114:unsupported encryption
 PEM_R_UNSUPPORTED_KEY_COMPONENTS:126:unsupported key components
+PEM_R_UNSUPPORTED_PUBLIC_KEY_TYPE:110:unsupported public key type
 PKCS12_R_CANT_PACK_STRUCTURE:100:cant pack structure
 PKCS12_R_CONTENT_TYPE_NOT_DATA:121:content type not data
 PKCS12_R_DECODE_ERROR:101:decode error
diff --git a/crypto/pem/pem_err.c b/crypto/pem/pem_err.c
index f642030aa5..0f3cb02407 100644
--- a/crypto/pem/pem_err.c
+++ b/crypto/pem/pem_err.c
@@ -1,6 +1,6 @@
 /*
  * Generated by util/mkerr.pl DO NOT EDIT
- * Copyright 1995-2018 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the OpenSSL license (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -60,6 +60,8 @@ static const ERR_STRING_DATA PEM_str_functs[] = {
     {ERR_PACK(ERR_LIB_PEM, PEM_F_PEM_SIGNFINAL, 0), "PEM_SignFinal"},
     {ERR_PACK(ERR_LIB_PEM, PEM_F_PEM_WRITE, 0), "PEM_write"},
     {ERR_PACK(ERR_LIB_PEM, PEM_F_PEM_WRITE_BIO, 0), "PEM_write_bio"},
+    {ERR_PACK(ERR_LIB_PEM, PEM_F_PEM_WRITE_BIO_PRIVATEKEY_TRADITIONAL, 0),
+     "PEM_write_bio_PrivateKey_traditional"},
     {ERR_PACK(ERR_LIB_PEM, PEM_F_PEM_WRITE_PRIVATEKEY, 0),
      "PEM_write_PrivateKey"},
     {ERR_PACK(ERR_LIB_PEM, PEM_F_PEM_X509_INFO_READ, 0), "PEM_X509_INFO_read"},
@@ -109,6 +111,8 @@ static const ERR_STRING_DATA PEM_str_reasons[] = {
     "unsupported encryption"},
     {ERR_PACK(ERR_LIB_PEM, 0, PEM_R_UNSUPPORTED_KEY_COMPONENTS),
     "unsupported key components"},
+    {ERR_PACK(ERR_LIB_PEM, 0, PEM_R_UNSUPPORTED_PUBLIC_KEY_TYPE),
+    "unsupported public key type"},
     {0, NULL}
 };
 
diff --git a/crypto/pem/pem_pkey.c b/crypto/pem/pem_pkey.c
index e58cdf4a3e..7bd9aa097e 100644
--- a/crypto/pem/pem_pkey.c
+++ b/crypto/pem/pem_pkey.c
@@ -108,6 +108,12 @@ int PEM_write_bio_PrivateKey_traditional(BIO *bp, EVP_PKEY *x,
                                          pem_password_cb *cb, void *u)
 {
     char pem_str[80];
+
+    if (x->ameth == NULL || x->ameth->old_priv_encode == NULL) {
+        PEMerr(PEM_F_PEM_WRITE_BIO_PRIVATEKEY_TRADITIONAL,
+               PEM_R_UNSUPPORTED_PUBLIC_KEY_TYPE);
+        return 0;
+    }
     BIO_snprintf(pem_str, 80, "%s PRIVATE KEY", x->ameth->pem_str);
     return PEM_ASN1_write_bio((i2d_of_void *)i2d_PrivateKey,
                               pem_str, bp, x, enc, kstr, klen, cb, u);
diff --git a/include/openssl/pemerr.h b/include/openssl/pemerr.h
index 0c45918f3c..4f7e3574b3 100644
--- a/include/openssl/pemerr.h
+++ b/include/openssl/pemerr.h
@@ -1,6 +1,6 @@
 /*
  * Generated by util/mkerr.pl DO NOT EDIT
- * Copyright 1995-2019 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the OpenSSL license (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -61,6 +61,7 @@ int ERR_load_PEM_strings(void);
 # define PEM_F_PEM_SIGNFINAL                              112
 # define PEM_F_PEM_WRITE                                  113
 # define PEM_F_PEM_WRITE_BIO                              114
+# define PEM_F_PEM_WRITE_BIO_PRIVATEKEY_TRADITIONAL       147
 # define PEM_F_PEM_WRITE_PRIVATEKEY                       139
 # define PEM_F_PEM_X509_INFO_READ                         115
 # define PEM_F_PEM_X509_INFO_READ_BIO                     116
@@ -99,5 +100,6 @@ int ERR_load_PEM_strings(void);
 # define PEM_R_UNSUPPORTED_CIPHER                         113
 # define PEM_R_UNSUPPORTED_ENCRYPTION                     114
 # define PEM_R_UNSUPPORTED_KEY_COMPONENTS                 126
+# define PEM_R_UNSUPPORTED_PUBLIC_KEY_TYPE                110
 
 #endif