Extend X509 cert checks and error reporting in v3_{purp,crld}.c and x509_{set,vfy}.c

add various checks for malformedness to static check_chain_extensions() in x509_vfc.c
improve error reporting of X509v3_cache_extensions() in v3_purp.c
add error reporting to x509_init_sig_info() in x509_set.c
improve static setup_dp() and related functions in v3_purp.c and v3_crld.c
add test case for non-conforming cert from https://tools.ietf.org/html/rfc8410#section-10.2

Reviewed-by: Kurt Roeckx <kurt@roeckx.be>
Reviewed-by: Tomas Mraz <tmraz@fedoraproject.org>
(Merged from https://github.com/openssl/openssl/pull/12478)

4 files changed, 103 insertions, 82 deletions

diff --git a/include/crypto/x509.h b/include/crypto/x509.h
index bd8f9ba52d..8c9a288cbc 100644
--- a/include/crypto/x509.h
+++ b/include/crypto/x509.h
@@ -304,9 +304,13 @@ int a2i_ipadd(unsigned char *ipout, const char *ipasc);
 int x509_set1_time(ASN1_TIME **ptm, const ASN1_TIME *tm);
 int x509_print_ex_brief(BIO *bio, X509 *cert, unsigned long neg_cflags);
 int x509v3_cache_extensions(X509 *x);
+int x509_init_sig_info(X509 *x);
+int x509_check_issued_int(X509 *issuer, X509 *subject, OPENSSL_CTX *libctx,
+                          const char *propq);
+
 int x509_set0_libctx(X509 *x, OPENSSL_CTX *libctx, const char *propq);
 int x509_crl_set0_libctx(X509_CRL *x, OPENSSL_CTX *libctx, const char *propq);
-void x509_init_sig_info(X509 *x);
+int x509_init_sig_info(X509 *x);
 int asn1_item_digest_with_libctx(const ASN1_ITEM *it, const EVP_MD *type,
                                  void *data, unsigned char *md,
                                  unsigned int *len, OPENSSL_CTX *libctx,
diff --git a/include/openssl/x509_vfy.h b/include/openssl/x509_vfy.h
index 2d3bd70ae2..ec2021357b 100644
--- a/include/openssl/x509_vfy.h
+++ b/include/openssl/x509_vfy.h
@@ -124,100 +124,110 @@ X509_LOOKUP_ctrl_with_libctx((x), X509_L_LOAD_STORE, (name), 0, NULL,          \
 X509_LOOKUP_ctrl_with_libctx((x), X509_L_ADD_STORE, (name), 0, NULL,           \
                              (libctx), (propq))
 
+# define X509_V_OK                                       0
+# define X509_V_ERR_UNSPECIFIED                          1
+# define X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT            2
+# define X509_V_ERR_UNABLE_TO_GET_CRL                    3
+# define X509_V_ERR_UNABLE_TO_DECRYPT_CERT_SIGNATURE     4
+# define X509_V_ERR_UNABLE_TO_DECRYPT_CRL_SIGNATURE      5
+# define X509_V_ERR_UNABLE_TO_DECODE_ISSUER_PUBLIC_KEY   6
+# define X509_V_ERR_CERT_SIGNATURE_FAILURE               7
+# define X509_V_ERR_CRL_SIGNATURE_FAILURE                8
+# define X509_V_ERR_CERT_NOT_YET_VALID                   9
+# define X509_V_ERR_CERT_HAS_EXPIRED                     10
+# define X509_V_ERR_CRL_NOT_YET_VALID                    11
+# define X509_V_ERR_CRL_HAS_EXPIRED                      12
+# define X509_V_ERR_ERROR_IN_CERT_NOT_BEFORE_FIELD       13
+# define X509_V_ERR_ERROR_IN_CERT_NOT_AFTER_FIELD        14
+# define X509_V_ERR_ERROR_IN_CRL_LAST_UPDATE_FIELD       15
+# define X509_V_ERR_ERROR_IN_CRL_NEXT_UPDATE_FIELD       16
+# define X509_V_ERR_OUT_OF_MEM                           17
+# define X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT          18
+# define X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN            19
+# define X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY    20
+# define X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE      21
+# define X509_V_ERR_CERT_CHAIN_TOO_LONG                  22
+# define X509_V_ERR_CERT_REVOKED                         23
+# define X509_V_ERR_NO_ISSUER_PUBLIC_KEY                 24
+# define X509_V_ERR_PATH_LENGTH_EXCEEDED                 25
+# define X509_V_ERR_INVALID_PURPOSE                      26
+# define X509_V_ERR_CERT_UNTRUSTED                       27
+# define X509_V_ERR_CERT_REJECTED                        28
 
-# define         X509_V_OK                                       0
-# define         X509_V_ERR_UNSPECIFIED                          1
-# define         X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT            2
-# define         X509_V_ERR_UNABLE_TO_GET_CRL                    3
-# define         X509_V_ERR_UNABLE_TO_DECRYPT_CERT_SIGNATURE     4
-# define         X509_V_ERR_UNABLE_TO_DECRYPT_CRL_SIGNATURE      5
-# define         X509_V_ERR_UNABLE_TO_DECODE_ISSUER_PUBLIC_KEY   6
-# define         X509_V_ERR_CERT_SIGNATURE_FAILURE               7
-# define         X509_V_ERR_CRL_SIGNATURE_FAILURE                8
-# define         X509_V_ERR_CERT_NOT_YET_VALID                   9
-# define         X509_V_ERR_CERT_HAS_EXPIRED                     10
-# define         X509_V_ERR_CRL_NOT_YET_VALID                    11
-# define         X509_V_ERR_CRL_HAS_EXPIRED                      12
-# define         X509_V_ERR_ERROR_IN_CERT_NOT_BEFORE_FIELD       13
-# define         X509_V_ERR_ERROR_IN_CERT_NOT_AFTER_FIELD        14
-# define         X509_V_ERR_ERROR_IN_CRL_LAST_UPDATE_FIELD       15
-# define         X509_V_ERR_ERROR_IN_CRL_NEXT_UPDATE_FIELD       16
-# define         X509_V_ERR_OUT_OF_MEM                           17
-# define         X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT          18
-# define         X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN            19
-# define         X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY    20
-# define         X509_V_ERR_UNABLE_TO_VERIFY_LEAF_SIGNATURE      21
-# define         X509_V_ERR_CERT_CHAIN_TOO_LONG                  22
-# define         X509_V_ERR_CERT_REVOKED                         23
-# define         X509_V_ERR_INVALID_CA                           24
-# define         X509_V_ERR_PATH_LENGTH_EXCEEDED                 25
-# define         X509_V_ERR_INVALID_PURPOSE                      26
-# define         X509_V_ERR_CERT_UNTRUSTED                       27
-# define         X509_V_ERR_CERT_REJECTED                        28
 /* These are 'informational' when looking for issuer cert */
-# define         X509_V_ERR_SUBJECT_ISSUER_MISMATCH              29
-# define         X509_V_ERR_AKID_SKID_MISMATCH                   30
-# define         X509_V_ERR_AKID_ISSUER_SERIAL_MISMATCH          31
-# define         X509_V_ERR_KEYUSAGE_NO_CERTSIGN                 32
-# define         X509_V_ERR_UNABLE_TO_GET_CRL_ISSUER             33
-# define         X509_V_ERR_UNHANDLED_CRITICAL_EXTENSION         34
-# define         X509_V_ERR_KEYUSAGE_NO_CRL_SIGN                 35
-# define         X509_V_ERR_UNHANDLED_CRITICAL_CRL_EXTENSION     36
-# define         X509_V_ERR_INVALID_NON_CA                       37
-# define         X509_V_ERR_PROXY_PATH_LENGTH_EXCEEDED           38
-# define         X509_V_ERR_KEYUSAGE_NO_DIGITAL_SIGNATURE        39
-# define         X509_V_ERR_PROXY_CERTIFICATES_NOT_ALLOWED       40
-# define         X509_V_ERR_INVALID_EXTENSION                    41
-# define         X509_V_ERR_INVALID_POLICY_EXTENSION             42
-# define         X509_V_ERR_NO_EXPLICIT_POLICY                   43
-# define         X509_V_ERR_DIFFERENT_CRL_SCOPE                  44
-# define         X509_V_ERR_UNSUPPORTED_EXTENSION_FEATURE        45
-# define         X509_V_ERR_UNNESTED_RESOURCE                    46
-# define         X509_V_ERR_PERMITTED_VIOLATION                  47
-# define         X509_V_ERR_EXCLUDED_VIOLATION                   48
-# define         X509_V_ERR_SUBTREE_MINMAX                       49
+# define X509_V_ERR_SUBJECT_ISSUER_MISMATCH              29
+# define X509_V_ERR_AKID_SKID_MISMATCH                   30
+# define X509_V_ERR_AKID_ISSUER_SERIAL_MISMATCH          31
+# define X509_V_ERR_KEYUSAGE_NO_CERTSIGN                 32
+# define X509_V_ERR_UNABLE_TO_GET_CRL_ISSUER             33
+# define X509_V_ERR_UNHANDLED_CRITICAL_EXTENSION         34
+# define X509_V_ERR_KEYUSAGE_NO_CRL_SIGN                 35
+# define X509_V_ERR_UNHANDLED_CRITICAL_CRL_EXTENSION     36
+# define X509_V_ERR_INVALID_NON_CA                       37
+# define X509_V_ERR_PROXY_PATH_LENGTH_EXCEEDED           38
+# define X509_V_ERR_KEYUSAGE_NO_DIGITAL_SIGNATURE        39
+# define X509_V_ERR_PROXY_CERTIFICATES_NOT_ALLOWED       40
+# define X509_V_ERR_INVALID_EXTENSION                    41
+# define X509_V_ERR_INVALID_POLICY_EXTENSION             42
+# define X509_V_ERR_NO_EXPLICIT_POLICY                   43
+# define X509_V_ERR_DIFFERENT_CRL_SCOPE                  44
+# define X509_V_ERR_UNSUPPORTED_EXTENSION_FEATURE        45
+# define X509_V_ERR_UNNESTED_RESOURCE                    46
+# define X509_V_ERR_PERMITTED_VIOLATION                  47
+# define X509_V_ERR_EXCLUDED_VIOLATION                   48
+# define X509_V_ERR_SUBTREE_MINMAX                       49
 /* The application is not happy */
-# define         X509_V_ERR_APPLICATION_VERIFICATION             50
-# define         X509_V_ERR_UNSUPPORTED_CONSTRAINT_TYPE          51
-# define         X509_V_ERR_UNSUPPORTED_CONSTRAINT_SYNTAX        52
-# define         X509_V_ERR_UNSUPPORTED_NAME_SYNTAX              53
-# define         X509_V_ERR_CRL_PATH_VALIDATION_ERROR            54
+# define X509_V_ERR_APPLICATION_VERIFICATION             50
+# define X509_V_ERR_UNSUPPORTED_CONSTRAINT_TYPE          51
+# define X509_V_ERR_UNSUPPORTED_CONSTRAINT_SYNTAX        52
+# define X509_V_ERR_UNSUPPORTED_NAME_SYNTAX              53
+# define X509_V_ERR_CRL_PATH_VALIDATION_ERROR            54
 /* Another issuer check debug option */
-# define         X509_V_ERR_PATH_LOOP                            55
+# define X509_V_ERR_PATH_LOOP                            55
 /* Suite B mode algorithm violation */
-# define         X509_V_ERR_SUITE_B_INVALID_VERSION              56
-# define         X509_V_ERR_SUITE_B_INVALID_ALGORITHM            57
-# define         X509_V_ERR_SUITE_B_INVALID_CURVE                58
-# define         X509_V_ERR_SUITE_B_INVALID_SIGNATURE_ALGORITHM  59
-# define         X509_V_ERR_SUITE_B_LOS_NOT_ALLOWED              60
-# define         X509_V_ERR_SUITE_B_CANNOT_SIGN_P_384_WITH_P_256 61
+# define X509_V_ERR_SUITE_B_INVALID_VERSION              56
+# define X509_V_ERR_SUITE_B_INVALID_ALGORITHM            57
+# define X509_V_ERR_SUITE_B_INVALID_CURVE                58
+# define X509_V_ERR_SUITE_B_INVALID_SIGNATURE_ALGORITHM  59
+# define X509_V_ERR_SUITE_B_LOS_NOT_ALLOWED              60
+# define X509_V_ERR_SUITE_B_CANNOT_SIGN_P_384_WITH_P_256 61
 /* Host, email and IP check errors */
-# define         X509_V_ERR_HOSTNAME_MISMATCH                    62
-# define         X509_V_ERR_EMAIL_MISMATCH                       63
-# define         X509_V_ERR_IP_ADDRESS_MISMATCH                  64
+# define X509_V_ERR_HOSTNAME_MISMATCH                    62
+# define X509_V_ERR_EMAIL_MISMATCH                       63
+# define X509_V_ERR_IP_ADDRESS_MISMATCH                  64
 /* DANE TLSA errors */
-# define         X509_V_ERR_DANE_NO_MATCH                        65
+# define X509_V_ERR_DANE_NO_MATCH                        65
 /* security level errors */
-# define         X509_V_ERR_EE_KEY_TOO_SMALL                     66
-# define         X509_V_ERR_CA_KEY_TOO_SMALL                     67
-# define         X509_V_ERR_CA_MD_TOO_WEAK                       68
+# define X509_V_ERR_EE_KEY_TOO_SMALL                     66
+# define X509_V_ERR_CA_KEY_TOO_SMALL                     67
+# define X509_V_ERR_CA_MD_TOO_WEAK                       68
 /* Caller error */
-# define         X509_V_ERR_INVALID_CALL                         69
+# define X509_V_ERR_INVALID_CALL                         69
 /* Issuer lookup error */
-# define         X509_V_ERR_STORE_LOOKUP                         70
+# define X509_V_ERR_STORE_LOOKUP                         70
 /* Certificate transparency */
-# define         X509_V_ERR_NO_VALID_SCTS                        71
+# define X509_V_ERR_NO_VALID_SCTS                        71
 
-# define         X509_V_ERR_PROXY_SUBJECT_NAME_VIOLATION         72
+# define X509_V_ERR_PROXY_SUBJECT_NAME_VIOLATION         72
 /* OCSP status errors */
-# define         X509_V_ERR_OCSP_VERIFY_NEEDED                   73  /* Need OCSP verification */
-# define         X509_V_ERR_OCSP_VERIFY_FAILED                   74  /* Couldn't verify cert through OCSP */
-# define         X509_V_ERR_OCSP_CERT_UNKNOWN                    75  /* Certificate wasn't recognized by the OCSP responder */
-
-# define         X509_V_ERR_SIGNATURE_ALGORITHM_MISMATCH         76
-# define         X509_V_ERR_NO_ISSUER_PUBLIC_KEY                 77
-# define         X509_V_ERR_UNSUPPORTED_SIGNATURE_ALGORITHM      78
-
+# define X509_V_ERR_OCSP_VERIFY_NEEDED                   73  /* Need OCSP verification */
+# define X509_V_ERR_OCSP_VERIFY_FAILED                   74  /* Couldn't verify cert through OCSP */
+# define X509_V_ERR_OCSP_CERT_UNKNOWN                    75  /* Certificate wasn't recognized by the OCSP responder */
+
+# define X509_V_ERR_UNSUPPORTED_SIGNATURE_ALGORITHM      76
+# define X509_V_ERR_SIGNATURE_ALGORITHM_MISMATCH         77
+
+/* Errors in case a check in X509_V_FLAG_X509_STRICT mode fails */
+# define X509_V_ERR_SIGNATURE_ALGORITHM_INCONSISTENCY    78
+# define X509_V_ERR_INVALID_CA                           79
+# define X509_V_ERR_PATHLEN_INVALID_FOR_NON_CA           80
+# define X509_V_ERR_PATHLEN_WITHOUT_KU_KEY_CERT_SIGN     81
+# define X509_V_ERR_KU_KEY_CERT_SIGN_INVALID_FOR_NON_CA  82
+# define X509_V_ERR_ISSUER_NAME_EMPTY                    83
+# define X509_V_ERR_SUBJECT_NAME_EMPTY                   84
+# define X509_V_ERR_MISSING_AUTHORITY_KEY_IDENTIFIER     85
+# define X509_V_ERR_MISSING_SUBJECT_KEY_IDENTIFIER       86
+# define X509_V_ERR_EMPTY_SUBJECT_ALT_NAME               87
 
 /* Certificate verify flags */
 
diff --git a/include/openssl/x509err.h b/include/openssl/x509err.h
index 19743b5987..94c5c5b75e 100644
--- a/include/openssl/x509err.h
+++ b/include/openssl/x509err.h
@@ -107,9 +107,12 @@ int ERR_load_X509_strings(void);
 # define X509_R_CERT_ALREADY_IN_HASH_TABLE                101
 # define X509_R_CRL_ALREADY_DELTA                         127
 # define X509_R_CRL_VERIFY_FAILURE                        131
+# define X509_R_ERROR_GETTING_MD_BY_NID                   141
+# define X509_R_ERROR_USING_SIGINF_SET                    142
 # define X509_R_IDP_MISMATCH                              128
 # define X509_R_INVALID_ATTRIBUTES                        138
 # define X509_R_INVALID_DIRECTORY                         113
+# define X509_R_INVALID_DISTPOINT                         143
 # define X509_R_INVALID_FIELD_NAME                        119
 # define X509_R_INVALID_TRUST                             123
 # define X509_R_ISSUER_MISMATCH                           129
@@ -133,6 +136,7 @@ int ERR_load_X509_strings(void);
 # define X509_R_UNKNOWN_KEY_TYPE                          117
 # define X509_R_UNKNOWN_NID                               109
 # define X509_R_UNKNOWN_PURPOSE_ID                        121
+# define X509_R_UNKNOWN_SIGID_ALGS                        144
 # define X509_R_UNKNOWN_TRUST_ID                          120
 # define X509_R_UNSUPPORTED_ALGORITHM                     111
 # define X509_R_WRONG_LOOKUP_TYPE                         112
diff --git a/include/openssl/x509v3err.h b/include/openssl/x509v3err.h
index d7aa5da6ac..b245a63902 100644
--- a/include/openssl/x509v3err.h
+++ b/include/openssl/x509v3err.h
@@ -107,6 +107,7 @@ int ERR_load_X509V3_strings(void);
 # define X509V3_R_DIRNAME_ERROR                           149
 # define X509V3_R_DISTPOINT_ALREADY_SET                   160
 # define X509V3_R_DUPLICATE_ZONE_ID                       133
+# define X509V3_R_EMPTY_KEY_USAGE                         169
 # define X509V3_R_ERROR_CONVERTING_ZONE                   131
 # define X509V3_R_ERROR_CREATING_EXTENSION                144
 # define X509V3_R_ERROR_IN_EXTENSION                      128
@@ -121,6 +122,7 @@ int ERR_load_X509V3_strings(void);
 # define X509V3_R_INVALID_ASNUMBER                        162
 # define X509V3_R_INVALID_ASRANGE                         163
 # define X509V3_R_INVALID_BOOLEAN_STRING                  104
+# define X509V3_R_INVALID_CERTIFICATE                     158
 # define X509V3_R_INVALID_EXTENSION_STRING                105
 # define X509V3_R_INVALID_INHERITANCE                     165
 # define X509V3_R_INVALID_IPADDRESS                       166
@@ -142,6 +144,7 @@ int ERR_load_X509V3_strings(void);
 # define X509V3_R_ISSUER_DECODE_ERROR                     126
 # define X509V3_R_MISSING_VALUE                           124
 # define X509V3_R_NEED_ORGANIZATION_AND_NUMBERS           142
+# define X509V3_R_NEGATIVE_PATHLEN                        168
 # define X509V3_R_NO_CONFIG_DATABASE                      136
 # define X509V3_R_NO_ISSUER_CERTIFICATE                   121
 # define X509V3_R_NO_ISSUER_DETAILS                       127