diff options
Diffstat (limited to 'crypto/bn')
-rw-r--r-- | crypto/bn/bn.h | 18 | ||||
-rw-r--r-- | crypto/bn/bn_mont.c | 2 | ||||
-rw-r--r-- | crypto/bn/bn_nist.c | 64 |
3 files changed, 55 insertions, 29 deletions
diff --git a/crypto/bn/bn.h b/crypto/bn/bn.h index 0a66a2ea0c..259748a417 100644 --- a/crypto/bn/bn.h +++ b/crypto/bn/bn.h @@ -253,6 +253,24 @@ extern "C" { #define BN_HEX_FMT2 "%08X" #endif +/* 2011-02-22 SMS. + * In various places, a size_t variable or a type cast to size_t was + * used to perform integer-only operations on pointers. This failed on + * VMS with 64-bit pointers (CC /POINTER_SIZE = 64) because size_t is + * still only 32 bits. What's needed in these cases is an integer type + * with the same size as a pointer, which size_t is not certain to be. + * The only fix here is VMS-specific. + */ +#if defined(OPENSSL_SYS_VMS) +# if __INITIAL_POINTER_SIZE == 64 +# define PTR_SIZE_INT long long +# else /* __INITIAL_POINTER_SIZE == 64 */ +# define PTR_SIZE_INT int +# endif /* __INITIAL_POINTER_SIZE == 64 [else] */ +#else /* defined(OPENSSL_SYS_VMS) */ +# define PTR_SIZE_INT size_t +#endif /* defined(OPENSSL_SYS_VMS) [else] */ + #define BN_DEFAULT_BITS 1280 #define BN_FLG_MALLOCED 0x01 diff --git a/crypto/bn/bn_mont.c b/crypto/bn/bn_mont.c index 2f7fb843a9..36ac6d8c15 100644 --- a/crypto/bn/bn_mont.c +++ b/crypto/bn/bn_mont.c @@ -279,7 +279,7 @@ static int BN_from_montgomery_word(BIGNUM *ret, BIGNUM *r, BN_MONT_CTX *mont) m1|=m2; /* (al!=ri) */ m1|=(0-(size_t)v); /* (al!=ri || v) */ m1&=~m2; /* (al!=ri || v) && !al>ri */ - nrp=(BN_ULONG *)(((size_t)rp&~m1)|((size_t)ap&m1)); + nrp=(BN_ULONG *)(((PTR_SIZE_INT)rp&~m1)|((PTR_SIZE_INT)ap&m1)); } /* 'i<ri' is chosen to eliminate dependency on input data, even diff --git a/crypto/bn/bn_nist.c b/crypto/bn/bn_nist.c index 97f980a131..4a6c3cc964 100644 --- a/crypto/bn/bn_nist.c +++ b/crypto/bn/bn_nist.c @@ -354,7 +354,7 @@ int BN_nist_mod_192(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, buf[BN_NIST_192_TOP], c_d[BN_NIST_192_TOP], *res; - size_t mask; + PTR_SIZE_INT mask; static const BIGNUM _bignum_nist_p_192_sqr = { (BN_ULONG *)_nist_p_192_sqr, sizeof(_nist_p_192_sqr)/sizeof(_nist_p_192_sqr[0]), @@ -405,9 +405,10 @@ int BN_nist_mod_192(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, * 'tmp=result-modulus; if (!carry || !borrow) result=tmp;' * this is what happens below, but without explicit if:-) a. */ - mask = 0-(size_t)bn_sub_words(c_d,r_d,_nist_p_192[0],BN_NIST_192_TOP); - mask &= 0-(size_t)carry; - res = (BN_ULONG *)(((size_t)c_d&~mask) | ((size_t)r_d&mask)); + mask = 0-(PTR_SIZE_INT)bn_sub_words(c_d,r_d,_nist_p_192[0],BN_NIST_192_TOP); + mask &= 0-(PTR_SIZE_INT)carry; + res = (BN_ULONG *) + (((PTR_SIZE_INT)c_d&~mask) | ((PTR_SIZE_INT)r_d&mask)); nist_cp_bn(r_d, res, BN_NIST_192_TOP); r->top = BN_NIST_192_TOP; bn_correct_top(r); @@ -438,8 +439,8 @@ int BN_nist_mod_224(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, buf[BN_NIST_224_TOP], c_d[BN_NIST_224_TOP], *res; - size_t mask; - union { bn_addsub_f f; size_t p; } u; + PTR_SIZE_INT mask; + union { bn_addsub_f f; PTR_SIZE_INT p; } u; static const BIGNUM _bignum_nist_p_224_sqr = { (BN_ULONG *)_nist_p_224_sqr, sizeof(_nist_p_224_sqr)/sizeof(_nist_p_224_sqr[0]), @@ -510,16 +511,18 @@ int BN_nist_mod_224(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, * to be compared to the modulus and conditionally * adjusted by *subtracting* the latter. */ carry = (int)bn_add_words(r_d,r_d,_nist_p_224[-carry-1],BN_NIST_224_TOP); - mask = 0-(size_t)carry; - u.p = ((size_t)bn_sub_words&mask) | ((size_t)bn_add_words&~mask); + mask = 0-(PTR_SIZE_INT)carry; + u.p = ((PTR_SIZE_INT)bn_sub_words&mask) | + ((PTR_SIZE_INT)bn_add_words&~mask); } else carry = 1; /* otherwise it's effectively same as in BN_nist_mod_192... */ - mask = 0-(size_t)(*u.f)(c_d,r_d,_nist_p_224[0],BN_NIST_224_TOP); - mask &= 0-(size_t)carry; - res = (BN_ULONG *)(((size_t)c_d&~mask) | ((size_t)r_d&mask)); + mask = 0-(PTR_SIZE_INT)(*u.f)(c_d,r_d,_nist_p_224[0],BN_NIST_224_TOP); + mask &= 0-(PTR_SIZE_INT)carry; + res = (BN_ULONG *)(((PTR_SIZE_INT)c_d&~mask) | + ((PTR_SIZE_INT)r_d&mask)); nist_cp_bn(r_d, res, BN_NIST_224_TOP); r->top = BN_NIST_224_TOP; bn_correct_top(r); @@ -549,8 +552,8 @@ int BN_nist_mod_256(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, buf[BN_NIST_256_TOP], c_d[BN_NIST_256_TOP], *res; - size_t mask; - union { bn_addsub_f f; size_t p; } u; + PTR_SIZE_INT mask; + union { bn_addsub_f f; PTR_SIZE_INT p; } u; static const BIGNUM _bignum_nist_p_256_sqr = { (BN_ULONG *)_nist_p_256_sqr, sizeof(_nist_p_256_sqr)/sizeof(_nist_p_256_sqr[0]), @@ -629,15 +632,17 @@ int BN_nist_mod_256(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, else if (carry < 0) { carry = (int)bn_add_words(r_d,r_d,_nist_p_256[-carry-1],BN_NIST_256_TOP); - mask = 0-(size_t)carry; - u.p = ((size_t)bn_sub_words&mask) | ((size_t)bn_add_words&~mask); + mask = 0-(PTR_SIZE_INT)carry; + u.p = ((PTR_SIZE_INT)bn_sub_words&mask) | + ((PTR_SIZE_INT)bn_add_words&~mask); } else carry = 1; - mask = 0-(size_t)(*u.f)(c_d,r_d,_nist_p_256[0],BN_NIST_256_TOP); - mask &= 0-(size_t)carry; - res = (BN_ULONG *)(((size_t)c_d&~mask) | ((size_t)r_d&mask)); + mask = 0-(PTR_SIZE_INT)(*u.f)(c_d,r_d,_nist_p_256[0],BN_NIST_256_TOP); + mask &= 0-(PTR_SIZE_INT)carry; + res = (BN_ULONG *)(((PTR_SIZE_INT)c_d&~mask) | + ((PTR_SIZE_INT)r_d&mask)); nist_cp_bn(r_d, res, BN_NIST_256_TOP); r->top = BN_NIST_256_TOP; bn_correct_top(r); @@ -671,8 +676,8 @@ int BN_nist_mod_384(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, buf[BN_NIST_384_TOP], c_d[BN_NIST_384_TOP], *res; - size_t mask; - union { bn_addsub_f f; size_t p; } u; + PTR_SIZE_INT mask; + union { bn_addsub_f f; PTR_SIZE_INT p; } u; static const BIGNUM _bignum_nist_p_384_sqr = { (BN_ULONG *)_nist_p_384_sqr, sizeof(_nist_p_384_sqr)/sizeof(_nist_p_384_sqr[0]), @@ -754,15 +759,17 @@ int BN_nist_mod_384(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, else if (carry < 0) { carry = (int)bn_add_words(r_d,r_d,_nist_p_384[-carry-1],BN_NIST_384_TOP); - mask = 0-(size_t)carry; - u.p = ((size_t)bn_sub_words&mask) | ((size_t)bn_add_words&~mask); + mask = 0-(PTR_SIZE_INT)carry; + u.p = ((PTR_SIZE_INT)bn_sub_words&mask) | + ((PTR_SIZE_INT)bn_add_words&~mask); } else carry = 1; - mask = 0-(size_t)(*u.f)(c_d,r_d,_nist_p_384[0],BN_NIST_384_TOP); - mask &= 0-(size_t)carry; - res = (BN_ULONG *)(((size_t)c_d&~mask) | ((size_t)r_d&mask)); + mask = 0-(PTR_SIZE_INT)(*u.f)(c_d,r_d,_nist_p_384[0],BN_NIST_384_TOP); + mask &= 0-(PTR_SIZE_INT)carry; + res = (BN_ULONG *)(((PTR_SIZE_INT)c_d&~mask) | + ((PTR_SIZE_INT)r_d&mask)); nist_cp_bn(r_d, res, BN_NIST_384_TOP); r->top = BN_NIST_384_TOP; bn_correct_top(r); @@ -781,7 +788,7 @@ int BN_nist_mod_521(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, BN_ULONG *r_d, *a_d = a->d, t_d[BN_NIST_521_TOP], val,tmp,*res; - size_t mask; + PTR_SIZE_INT mask; static const BIGNUM _bignum_nist_p_521_sqr = { (BN_ULONG *)_nist_p_521_sqr, sizeof(_nist_p_521_sqr)/sizeof(_nist_p_521_sqr[0]), @@ -826,8 +833,9 @@ int BN_nist_mod_521(BIGNUM *r, const BIGNUM *a, const BIGNUM *field, r_d[i] &= BN_NIST_521_TOP_MASK; bn_add_words(r_d,r_d,t_d,BN_NIST_521_TOP); - mask = 0-(size_t)bn_sub_words(t_d,r_d,_nist_p_521,BN_NIST_521_TOP); - res = (BN_ULONG *)(((size_t)t_d&~mask) | ((size_t)r_d&mask)); + mask = 0-(PTR_SIZE_INT)bn_sub_words(t_d,r_d,_nist_p_521,BN_NIST_521_TOP); + res = (BN_ULONG *)(((PTR_SIZE_INT)t_d&~mask) | + ((PTR_SIZE_INT)r_d&mask)); nist_cp_bn(r_d,res,BN_NIST_521_TOP); r->top = BN_NIST_521_TOP; bn_correct_top(r); |