From 67567fc432deb250f2c9e8b922374d38014d0fc6 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 9 May 2016 03:46:54 +0200 Subject: new crypto api, blackbox/AEAD. also adds AES256-GCM. includes: - aes256-ctr-hmac-sha256 (attic/borg legacy, optional aad support) - aes256-gcm (new, optional aad support) uses 96bits for iv, 128bit for auth tag. - header support the caller-provided header will be just copied in front of the rest - this avoids expensive operations (memcpy, garbage collection) in Python. the first bytes in the header may be non-authenticated data if aad_offset > 0. this is to support legacy attic/borg envelope layout, where the type byte is not authenticated. - aad support additional authenticated data - it just contributes to the computed mac, but is not encrypted). the current api assumes that aad starts at some aad_offset inside the given header and extends to the end of it. - iv handling helpers, compute next iv based on amount of processed data - unit tests Note: the changes are intentionally kept isolated / not integrated into the rest of the code, so this has to be done later. --- src/borg/crypto/low_level.pyx | 438 +++++++++++++++++++++++++++++++++--------- src/borg/testsuite/crypto.py | 131 +++++++++++-- 2 files changed, 468 insertions(+), 101 deletions(-) diff --git a/src/borg/crypto/low_level.pyx b/src/borg/crypto/low_level.pyx index a68cd820f..ba4905b75 100644 --- a/src/borg/crypto/low_level.pyx +++ b/src/borg/crypto/low_level.pyx @@ -1,4 +1,49 @@ -"""A thin OpenSSL wrapper""" +"""An AEAD style OpenSSL wrapper + +Note: AES-GCM mode needs OpenSSL >= 1.0.1d due to bug fixes in OpenSSL. + +API: + + encrypt(data, header=b'', aad_offset=0) -> envelope + decrypt(envelope, header_len=0, aad_offset=0) -> data + +Envelope layout: + +|<--------------------------- envelope ------------------------------------------>| +|<------------ header ----------->|<---------- ciphersuite specific ------------->| +|<-- not auth data -->|<-- aad -->|<-- e.g.: S(aad, iv, E(data)), iv, E(data) -->| + +|--- #aad_offset ---->| +|------------- #header_len ------>| + +S means a cryptographic signature function (like HMAC or GMAC). +E means a encryption function (like AES). +iv is the initialization vector / nonce, if needed. + +The split of header into not authenticated data and aad (additional authenticated +data) is done to support the legacy envelope layout as used in attic and early borg +(where the TYPE byte was not authenticated) and avoid unneeded memcpy and string +garbage. + +Newly designed envelope layouts can just authenticate the whole header. + +IV handling: + + iv = ... # just never repeat! + cs = CS(hmac_key, enc_key, iv=iv) + envelope = cs.encrypt(data, header, aad_offset) + iv = cs.next_iv(len(data)) + (repeat) +""" + +# TODO: get rid of small malloc +# as @enkore mentioned on github: +# "Since we do many small-object allocations here it is probably better to use +# PyMem_Alloc/Free instead of malloc/free (PyMem has many optimizations for +# small allocs)." +# +# Small mallocs currently happen if the total input file length is small, so +# the 1 chunk's size is less than what the chunker would produce for big files. import hashlib import hmac @@ -10,6 +55,8 @@ from cpython.bytes cimport PyBytes_FromStringAndSize API_VERSION = '1.1_02' +cdef extern from "openssl/crypto.h": + int CRYPTO_memcmp(const void *a, const void *b, size_t len) cdef extern from "../algorithms/blake2-libselect.h": ctypedef struct blake2b_state: @@ -29,9 +76,12 @@ cdef extern from "openssl/evp.h": pass ctypedef struct ENGINE: pass + const EVP_CIPHER *EVP_aes_256_ctr() - EVP_CIPHER_CTX *EVP_CIPHER_CTX_new() - void EVP_CIPHER_CTX_free(EVP_CIPHER_CTX *a) + const EVP_CIPHER *EVP_aes_256_gcm() + + void EVP_CIPHER_CTX_init(EVP_CIPHER_CTX *a) + void EVP_CIPHER_CTX_cleanup(EVP_CIPHER_CTX *a) int EVP_EncryptInit_ex(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *cipher, ENGINE *impl, const unsigned char *key, const unsigned char *iv) @@ -44,10 +94,27 @@ cdef extern from "openssl/evp.h": int EVP_EncryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl) int EVP_DecryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl) - EVP_MD *EVP_sha256() nogil + int EVP_CIPHER_CTX_ctrl(EVP_CIPHER_CTX *ctx, int type, int arg, void *ptr) + int EVP_CTRL_GCM_GET_TAG + int EVP_CTRL_GCM_SET_TAG + int EVP_CTRL_GCM_SET_IVLEN + const EVP_MD *EVP_sha256() nogil + + EVP_CIPHER_CTX *EVP_CIPHER_CTX_new() + void EVP_CIPHER_CTX_free(EVP_CIPHER_CTX *a) cdef extern from "openssl/hmac.h": + ctypedef struct HMAC_CTX: + pass + + void HMAC_CTX_init(HMAC_CTX *ctx) + void HMAC_CTX_cleanup(HMAC_CTX *ctx) + + int HMAC_Init_ex(HMAC_CTX *ctx, const void *key, int key_len, const EVP_MD *md, ENGINE *impl) + int HMAC_Update(HMAC_CTX *ctx, const unsigned char *data, int len) + int HMAC_Final(HMAC_CTX *ctx, unsigned char *md, unsigned int *len) + unsigned char *HMAC(const EVP_MD *evp_md, const void *key, int key_len, const unsigned char *data, int data_len, @@ -91,114 +158,311 @@ def increment_iv(iv, amount=1): return iv -def num_aes_blocks(int length): +def num_aes_blocks(length): """Return the number of AES blocks required to encrypt/decrypt *length* bytes of data. Note: this is only correct for modes without padding, like AES-CTR. """ return (length + 15) // 16 +class CryptoError(Exception): + """Malfunction in the crypto module.""" + + +class IntegrityError(CryptoError): + """Integrity checks failed. Corrupted or tampered data.""" + + cdef Py_buffer ro_buffer(object data) except *: cdef Py_buffer view PyObject_GetBuffer(data, &view, PyBUF_SIMPLE) return view -cdef class AES: - """A thin wrapper around the OpenSSL EVP cipher API - """ +cdef class AES256_CTR_HMAC_SHA256: + # Layout: HEADER + HMAC 32 + IV 8 + CT (same as attic / borg < 1.2 IF HEADER = TYPE_BYTE, no AAD) + cdef EVP_CIPHER_CTX *ctx - cdef int is_encrypt - cdef unsigned char iv_orig[16] + cdef HMAC_CTX hmac_ctx + cdef unsigned char *mac_key + cdef unsigned char *enc_key + cdef unsigned char iv[16] cdef long long blocks - def __cinit__(self, is_encrypt, key, iv=None): + def __init__(self, mac_key, enc_key, iv=None): + assert isinstance(mac_key, bytes) and len(mac_key) == 32 + assert isinstance(enc_key, bytes) and len(enc_key) == 32 + assert iv is None or isinstance(iv, bytes) and len(iv) == 16 + self.mac_key = mac_key + self.enc_key = enc_key + if iv is not None: + self.set_iv(iv) + + def __cinit__(self, mac_key, enc_key, iv=None): self.ctx = EVP_CIPHER_CTX_new() - self.is_encrypt = is_encrypt - # Set cipher type and mode - cipher_mode = EVP_aes_256_ctr() - if self.is_encrypt: - if not EVP_EncryptInit_ex(self.ctx, cipher_mode, NULL, NULL, NULL): - raise Exception('EVP_EncryptInit_ex failed') - else: # decrypt - if not EVP_DecryptInit_ex(self.ctx, cipher_mode, NULL, NULL, NULL): - raise Exception('EVP_DecryptInit_ex failed') - self.reset(key, iv) + HMAC_CTX_init(&self.hmac_ctx) # XXX def __dealloc__(self): EVP_CIPHER_CTX_free(self.ctx) + HMAC_CTX_cleanup(&self.hmac_ctx) # XXX + + def encrypt(self, data, header=b'', aad_offset=0): + """ + encrypt data, compute mac over aad + iv + cdata, prepend header. + aad_offset is the offset into the header where aad starts. + """ + cdef int ilen = len(data) + cdef int hlen = len(header) + cdef int aoffset = aad_offset + cdef int alen = hlen - aoffset + cdef unsigned char *odata = malloc(hlen + 32 + 8 + ilen + 16) + if not odata: + raise MemoryError + cdef int olen + cdef int offset + cdef Py_buffer idata = ro_buffer(data) + cdef Py_buffer hdata = ro_buffer(header) + try: + offset = 0 + for i in range(hlen): + odata[offset+i] = header[i] + offset += hlen + offset += 32 + self.store_iv(odata+offset, self.iv) + offset += 8 + rc = EVP_EncryptInit_ex(self.ctx, EVP_aes_256_ctr(), NULL, self.enc_key, self.iv) + if not rc: + raise CryptoError('EVP_EncryptInit_ex failed') + rc = EVP_EncryptUpdate(self.ctx, odata+offset, &olen, idata.buf, ilen) + if not rc: + raise CryptoError('EVP_EncryptUpdate failed') + offset += olen + rc = EVP_EncryptFinal_ex(self.ctx, odata+offset, &olen) + if not rc: + raise CryptoError('EVP_EncryptFinal_ex failed') + offset += olen + if not HMAC_Init_ex(&self.hmac_ctx, self.mac_key, 32, EVP_sha256(), NULL): + raise CryptoError('HMAC_Init_ex failed') + if not HMAC_Update(&self.hmac_ctx, hdata.buf+aoffset, alen): + raise CryptoError('HMAC_Update failed') + if not HMAC_Update(&self.hmac_ctx, odata+hlen+32, offset-hlen-32): + raise CryptoError('HMAC_Update failed') + if not HMAC_Final(&self.hmac_ctx, odata+hlen, NULL): + raise CryptoError('HMAC_Final failed') + self.blocks += num_aes_blocks(ilen) + return odata[:offset] + finally: + free(odata) + PyBuffer_Release(&hdata) + PyBuffer_Release(&idata) + + def decrypt(self, envelope, header_len=0, aad_offset=0): + """ + authenticate aad + iv + cdata, decrypt cdata, ignore header bytes up to aad_offset. + """ + cdef int ilen = len(envelope) + cdef int hlen = header_len + cdef int aoffset = aad_offset + cdef int alen = hlen - aoffset + cdef unsigned char *odata = malloc(ilen + 16) + if not odata: + raise MemoryError + cdef int olen + cdef int offset + cdef unsigned char hmac_buf[32] + cdef Py_buffer idata = ro_buffer(envelope) + try: + if not HMAC_Init_ex(&self.hmac_ctx, self.mac_key, 32, EVP_sha256(), NULL): + raise CryptoError('HMAC_Init_ex failed') + if not HMAC_Update(&self.hmac_ctx, idata.buf+aoffset, alen): + raise CryptoError('HMAC_Update failed') + if not HMAC_Update(&self.hmac_ctx, idata.buf+hlen+32, ilen-hlen-32): + raise CryptoError('HMAC_Update failed') + if not HMAC_Final(&self.hmac_ctx, hmac_buf, NULL): + raise CryptoError('HMAC_Final failed') + if CRYPTO_memcmp(hmac_buf, idata.buf+hlen, 32): + raise IntegrityError('HMAC Authentication failed') + iv = self.fetch_iv( idata.buf+hlen+32) + self.set_iv(iv) + if not EVP_DecryptInit_ex(self.ctx, EVP_aes_256_ctr(), NULL, self.enc_key, iv): + raise CryptoError('EVP_DecryptInit_ex failed') + offset = 0 + rc = EVP_DecryptUpdate(self.ctx, odata+offset, &olen, idata.buf+hlen+32+8, ilen-hlen-32-8) + if not rc: + raise CryptoError('EVP_DecryptUpdate failed') + offset += olen + rc = EVP_DecryptFinal_ex(self.ctx, odata+offset, &olen) + if rc <= 0: + raise CryptoError('EVP_DecryptFinal_ex failed') + offset += olen + self.blocks += num_aes_blocks(offset) + return odata[:offset] + finally: + free(odata) + PyBuffer_Release(&idata) + + def set_iv(self, iv): + self.blocks = 0 # how many AES blocks got encrypted with this IV? + for i in range(16): + self.iv[i] = iv[i] + + def next_iv(self): + return increment_iv(self.iv[:16], self.blocks) + + cdef fetch_iv(self, unsigned char * iv_in): + # fetch lower 8 bytes of iv and add upper 8 zero bytes + return b"\0" * 8 + iv_in[0:8] + + cdef store_iv(self, unsigned char * iv_out, unsigned char * iv): + # store only lower 8 bytes, upper 8 bytes are assumed to be 0 + cdef int i + for i in range(8): + iv_out[i] = iv[8+i] + + +cdef class AES256_GCM: + # Layout: HEADER + GMAC 16 + IV 12 + CT - def reset(self, key=None, iv=None): - cdef const unsigned char *key2 = NULL - cdef const unsigned char *iv2 = NULL - if key: - key2 = key - if iv: - iv2 = iv - assert isinstance(iv, bytes) and len(iv) == 16 - for i in range(16): - self.iv_orig[i] = iv[i] - self.blocks = 0 # number of AES blocks encrypted starting with iv_orig - # Initialise key and IV - if self.is_encrypt: - if not EVP_EncryptInit_ex(self.ctx, NULL, NULL, key2, iv2): - raise Exception('EVP_EncryptInit_ex failed') - else: # decrypt - if not EVP_DecryptInit_ex(self.ctx, NULL, NULL, key2, iv2): - raise Exception('EVP_DecryptInit_ex failed') - - @property - def iv(self): - return increment_iv(self.iv_orig[:16], self.blocks) - - def encrypt(self, data): - cdef Py_buffer data_buf = ro_buffer(data) - cdef int inl = len(data) - cdef int ctl = 0 - cdef int outl = 0 - # note: modes that use padding, need up to one extra AES block (16b) - cdef unsigned char *out = malloc(inl+16) - if not out: + cdef EVP_CIPHER_CTX *ctx + cdef unsigned char *enc_key + cdef unsigned char iv[12] + cdef long long blocks + + def __init__(self, mac_key, enc_key, iv=None): + assert mac_key is None + assert isinstance(enc_key, bytes) and len(enc_key) == 32 + assert iv is None or isinstance(iv, bytes) and len(iv) == 12 + self.enc_key = enc_key + if iv is not None: + self.set_iv(iv) + + def __cinit__(self, mac_key, enc_key, iv=None): + self.ctx = EVP_CIPHER_CTX_new() + + def __dealloc__(self): + EVP_CIPHER_CTX_free(self.ctx) + + def encrypt(self, data, header=b'', aad_offset=0): + """ + encrypt data, compute mac over aad + iv + cdata, prepend header. + aad_offset is the offset into the header where aad starts. + """ + cdef int ilen = len(data) + cdef int hlen = len(header) + cdef int aoffset = aad_offset + cdef int alen = hlen - aoffset + cdef unsigned char *odata = malloc(hlen + 16 + 12 + ilen + 16) + if not odata: raise MemoryError + cdef int olen + cdef int offset + cdef Py_buffer idata = ro_buffer(data) + cdef Py_buffer hdata = ro_buffer(header) try: - if not EVP_EncryptUpdate(self.ctx, out, &outl, data_buf.buf, inl): - raise Exception('EVP_EncryptUpdate failed') - ctl = outl - if not EVP_EncryptFinal_ex(self.ctx, out+ctl, &outl): - raise Exception('EVP_EncryptFinal failed') - ctl += outl - self.blocks += num_aes_blocks(ctl) - return out[:ctl] + offset = 0 + for i in range(hlen): + odata[offset+i] = header[i] + offset += hlen + offset += 16 + self.store_iv(odata+offset, self.iv) + rc = EVP_EncryptInit_ex(self.ctx, EVP_aes_256_gcm(), NULL, NULL, NULL) + if not rc: + raise CryptoError('EVP_EncryptInit_ex failed') + if not EVP_CIPHER_CTX_ctrl(self.ctx, EVP_CTRL_GCM_SET_IVLEN, 12, NULL): + raise CryptoError('EVP_CIPHER_CTX_ctrl SET IVLEN failed') + rc = EVP_EncryptInit_ex(self.ctx, NULL, NULL, self.enc_key, self.iv) + if not rc: + raise CryptoError('EVP_EncryptInit_ex failed') + rc = EVP_EncryptUpdate(self.ctx, NULL, &olen, hdata.buf+aoffset, alen) + if not rc: + raise CryptoError('EVP_EncryptUpdate failed') + if not EVP_EncryptUpdate(self.ctx, NULL, &olen, odata+offset, 12): + raise CryptoError('EVP_EncryptUpdate failed') + offset += 12 + rc = EVP_EncryptUpdate(self.ctx, odata+offset, &olen, idata.buf, ilen) + if not rc: + raise CryptoError('EVP_EncryptUpdate failed') + offset += olen + rc = EVP_EncryptFinal_ex(self.ctx, odata+offset, &olen) + if not rc: + raise CryptoError('EVP_EncryptFinal_ex failed') + offset += olen + if not EVP_CIPHER_CTX_ctrl(self.ctx, EVP_CTRL_GCM_GET_TAG, 16, odata+hlen): + raise CryptoError('EVP_CIPHER_CTX_ctrl GET TAG failed') + self.blocks += num_aes_blocks(ilen) + return odata[:offset] finally: - free(out) - PyBuffer_Release(&data_buf) - - def decrypt(self, data): - cdef Py_buffer data_buf = ro_buffer(data) - cdef int inl = len(data) - cdef int ptl = 0 - cdef int outl = 0 - # note: modes that use padding, need up to one extra AES block (16b). - # This is what the openssl docs say. I am not sure this is correct, - # but OTOH it will not cause any harm if our buffer is a little bigger. - cdef unsigned char *out = malloc(inl+16) - if not out: + free(odata) + PyBuffer_Release(&hdata) + PyBuffer_Release(&idata) + + def decrypt(self, envelope, header_len=0, aad_offset=0): + """ + authenticate aad + iv + cdata, decrypt cdata, ignore header bytes up to aad_offset. + """ + cdef int ilen = len(envelope) + cdef int hlen = header_len + cdef int aoffset = aad_offset + cdef int alen = hlen - aoffset + cdef unsigned char *odata = malloc(ilen + 16) + if not odata: raise MemoryError + cdef int olen + cdef int offset + cdef Py_buffer idata = ro_buffer(envelope) try: - if not EVP_DecryptUpdate(self.ctx, out, &outl, data_buf.buf, inl): - raise Exception('EVP_DecryptUpdate failed') - ptl = outl - if EVP_DecryptFinal_ex(self.ctx, out+ptl, &outl) <= 0: - # this error check is very important for modes with padding or - # authentication. for them, a failure here means corrupted data. - # CTR mode does not use padding nor authentication. - raise Exception('EVP_DecryptFinal failed') - ptl += outl - self.blocks += num_aes_blocks(inl) - return out[:ptl] + if not EVP_DecryptInit_ex(self.ctx, EVP_aes_256_gcm(), NULL, NULL, NULL): + raise CryptoError('EVP_DecryptInit_ex failed') + iv = self.fetch_iv( idata.buf+hlen+16) + self.set_iv(iv) + if not EVP_CIPHER_CTX_ctrl(self.ctx, EVP_CTRL_GCM_SET_IVLEN, 12, NULL): + raise CryptoError('EVP_CIPHER_CTX_ctrl SET IVLEN failed') + if not EVP_DecryptInit_ex(self.ctx, NULL, NULL, self.enc_key, iv): + raise CryptoError('EVP_DecryptInit_ex failed') + if not EVP_CIPHER_CTX_ctrl(self.ctx, EVP_CTRL_GCM_SET_TAG, 16, idata.buf+hlen): + raise CryptoError('EVP_CIPHER_CTX_ctrl SET TAG failed') + rc = EVP_DecryptUpdate(self.ctx, NULL, &olen, idata.buf+aoffset, alen) + if not rc: + raise CryptoError('EVP_DecryptUpdate failed') + if not EVP_DecryptUpdate(self.ctx, NULL, &olen, idata.buf+hlen+16, 12): + raise CryptoError('EVP_DecryptUpdate failed') + offset = 0 + rc = EVP_DecryptUpdate(self.ctx, odata+offset, &olen, idata.buf+hlen+16+12, ilen-hlen-16-12) + if not rc: + raise CryptoError('EVP_DecryptUpdate failed') + offset += olen + rc = EVP_DecryptFinal_ex(self.ctx, odata+offset, &olen) + if rc <= 0: + # a failure here means corrupted or tampered tag (mac) or data. + raise IntegrityError('GCM Authentication / EVP_DecryptFinal_ex failed') + offset += olen + self.blocks += num_aes_blocks(offset) + return odata[:offset] finally: - free(out) - PyBuffer_Release(&data_buf) + free(odata) + PyBuffer_Release(&idata) + + def set_iv(self, iv): + self.blocks = 0 # number of AES blocks encrypted with this IV + for i in range(12): + self.iv[i] = iv[i] + + def next_iv(self): + assert self.blocks < 2**32 + # we need 16 bytes for increment_iv: + last_iv = b'\0\0\0\0' + self.iv[:12] + # gcm mode is special: it appends a internal 32bit counter to the 96bit (12 byte) we provide, thus we only + # need to increment the 96bit counter by 1 (and we must not encrypt more than 2^32 AES blocks with same IV): + next_iv = increment_iv(last_iv, 1) + return next_iv[-12:] + + cdef fetch_iv(self, unsigned char * iv_in): + return iv_in[0:12] + + cdef store_iv(self, unsigned char * iv_out, unsigned char * iv): + cdef int i + for i in range(12): + iv_out[i] = iv[i] def hmac_sha256(key, data): @@ -210,7 +474,7 @@ def hmac_sha256(key, data): with nogil: rc = HMAC(EVP_sha256(), key_ptr, key_len, data_buf.buf, data_buf.len, md, NULL) if rc != md: - raise Exception('HMAC(EVP_sha256) failed') + raise CryptoError('HMAC(EVP_sha256) failed') finally: PyBuffer_Release(&data_buf) return PyBytes_FromStringAndSize( &md[0], 32) diff --git a/src/borg/testsuite/crypto.py b/src/borg/testsuite/crypto.py index 6406064d1..4e8c17464 100644 --- a/src/borg/testsuite/crypto.py +++ b/src/borg/testsuite/crypto.py @@ -1,8 +1,9 @@ from binascii import hexlify, unhexlify -from ..crypto.low_level import AES, bytes_to_long, bytes_to_int, long_to_bytes, hmac_sha256, blake2b_256 -from ..crypto.low_level import increment_iv, bytes16_to_int, int_to_bytes16 +from ..crypto.low_level import AES256_CTR_HMAC_SHA256, AES256_GCM, IntegrityError, hmac_sha256, blake2b_256 +from ..crypto.low_level import bytes_to_long, bytes_to_int, long_to_bytes, bytes16_to_int, int_to_bytes16, increment_iv from ..crypto.low_level import hkdf_hmac_sha512 + from . import BaseTestCase # Note: these tests are part of the self test, do not use or import py.test functionality here. @@ -39,21 +40,123 @@ class CryptoTestCase(BaseTestCase): self.assert_equal(increment_iv(iva, 2), ivc) self.assert_equal(increment_iv(iv0, 2**64), ivb) - def test_aes(self): - key = b'X' * 32 + def test_AES256_CTR_HMAC_SHA256(self): + # this tests the layout as in attic / borg < 1.2 (1 type byte, no aad) + mac_key = b'Y' * 32 + enc_key = b'X' * 32 + iv = b'\0' * 16 + data = b'foo' * 10 + header = b'\x42' + # encrypt-then-mac + cs = AES256_CTR_HMAC_SHA256(mac_key, enc_key, iv) + hdr_mac_iv_cdata = cs.encrypt(data, header=header, aad_offset=1) + hdr = hdr_mac_iv_cdata[0:1] + mac = hdr_mac_iv_cdata[1:33] + iv = hdr_mac_iv_cdata[33:41] + cdata = hdr_mac_iv_cdata[41:] + self.assert_equal(hexlify(hdr), b'42') + self.assert_equal(hexlify(mac), b'af90b488b0cc4a8f768fe2d6814fa65aec66b148135e54f7d4d29a27f22f57a8') + self.assert_equal(hexlify(iv), b'0000000000000000') + self.assert_equal(hexlify(cdata), b'c6efb702de12498f34a2c2bbc8149e759996d08bf6dc5c610aefc0c3a466') + self.assert_equal(hexlify(cs.next_iv()), b'00000000000000000000000000000002') + # auth-then-decrypt + cs = AES256_CTR_HMAC_SHA256(mac_key, enc_key) + pdata = cs.decrypt(hdr_mac_iv_cdata, header_len=len(header), aad_offset=1) + self.assert_equal(data, pdata) + self.assert_equal(hexlify(cs.next_iv()), b'00000000000000000000000000000002') + # auth-failure due to corruption (corrupted data) + cs = AES256_CTR_HMAC_SHA256(mac_key, enc_key) + hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:41] + b'\0' + hdr_mac_iv_cdata[42:] + self.assert_raises(IntegrityError, + lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted, header_len=len(header), aad_offset=1)) + + def test_AES256_CTR_HMAC_SHA256_aad(self): + mac_key = b'Y' * 32 + enc_key = b'X' * 32 + iv = b'\0' * 16 data = b'foo' * 10 - # encrypt - aes = AES(is_encrypt=True, key=key) - self.assert_equal(bytes_to_long(aes.iv, 8), 0) - cdata = aes.encrypt(data) + header = b'\x12\x34\x56' + # encrypt-then-mac + cs = AES256_CTR_HMAC_SHA256(mac_key, enc_key, iv) + hdr_mac_iv_cdata = cs.encrypt(data, header=header, aad_offset=1) + hdr = hdr_mac_iv_cdata[0:3] + mac = hdr_mac_iv_cdata[3:35] + iv = hdr_mac_iv_cdata[35:43] + cdata = hdr_mac_iv_cdata[43:] + self.assert_equal(hexlify(hdr), b'123456') + self.assert_equal(hexlify(mac), b'7659a915d9927072ef130258052351a17ef882692893c3850dd798c03d2dd138') + self.assert_equal(hexlify(iv), b'0000000000000000') self.assert_equal(hexlify(cdata), b'c6efb702de12498f34a2c2bbc8149e759996d08bf6dc5c610aefc0c3a466') - self.assert_equal(bytes_to_long(aes.iv, 8), 2) - # decrypt - aes = AES(is_encrypt=False, key=key) - self.assert_equal(bytes_to_long(aes.iv, 8), 0) - pdata = aes.decrypt(cdata) + self.assert_equal(hexlify(cs.next_iv()), b'00000000000000000000000000000002') + # auth-then-decrypt + cs = AES256_CTR_HMAC_SHA256(mac_key, enc_key) + pdata = cs.decrypt(hdr_mac_iv_cdata, header_len=len(header), aad_offset=1) + self.assert_equal(data, pdata) + self.assert_equal(hexlify(cs.next_iv()), b'00000000000000000000000000000002') + # auth-failure due to corruption (corrupted aad) + cs = AES256_CTR_HMAC_SHA256(mac_key, enc_key) + hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:1] + b'\0' + hdr_mac_iv_cdata[2:] + self.assert_raises(IntegrityError, + lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted, header_len=len(header), aad_offset=1)) + + def test_AES_GCM_256_GMAC(self): + # gcm used in legacy-like layout (1 type byte, no aad) + mac_key = None + enc_key = b'X' * 32 + iv = b'\0' * 12 + data = b'foo' * 10 + header = b'\x23' + # encrypt-then-mac + cs = AES256_GCM(mac_key, enc_key, iv) + hdr_mac_iv_cdata = cs.encrypt(data, header=header, aad_offset=1) + hdr = hdr_mac_iv_cdata[0:1] + mac = hdr_mac_iv_cdata[1:17] + iv = hdr_mac_iv_cdata[17:29] + cdata = hdr_mac_iv_cdata[29:] + self.assert_equal(hexlify(hdr), b'23') + self.assert_equal(hexlify(mac), b'66a438843aa41a087d6a7ed1dc1f3c4c') + self.assert_equal(hexlify(iv), b'000000000000000000000000') + self.assert_equal(hexlify(cdata), b'5bbb40be14e4bcbfc75715b77b1242d590d2bf9f7f8a8a910b4469888689') + self.assert_equal(hexlify(cs.next_iv()), b'000000000000000000000001') + # auth-then-decrypt + cs = AES256_GCM(mac_key, enc_key) + pdata = cs.decrypt(hdr_mac_iv_cdata, header_len=len(header), aad_offset=1) + self.assert_equal(data, pdata) + self.assert_equal(hexlify(cs.next_iv()), b'000000000000000000000001') + # auth-failure due to corruption (corrupted data) + cs = AES256_GCM(mac_key, enc_key) + hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:29] + b'\0' + hdr_mac_iv_cdata[30:] + self.assert_raises(IntegrityError, + lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted, header_len=len(header), aad_offset=1)) + + def test_AES_GCM_256_GMAC_aad(self): + mac_key = None + enc_key = b'X' * 32 + iv = b'\0' * 12 + data = b'foo' * 10 + header = b'\x12\x34\x56' + # encrypt-then-mac + cs = AES256_GCM(mac_key, enc_key, iv) + hdr_mac_iv_cdata = cs.encrypt(data, header=header, aad_offset=1) + hdr = hdr_mac_iv_cdata[0:3] + mac = hdr_mac_iv_cdata[3:19] + iv = hdr_mac_iv_cdata[19:31] + cdata = hdr_mac_iv_cdata[31:] + self.assert_equal(hexlify(hdr), b'123456') + self.assert_equal(hexlify(mac), b'4fb0e5b0a0bca57527352cc6240e7cca') + self.assert_equal(hexlify(iv), b'000000000000000000000000') + self.assert_equal(hexlify(cdata), b'5bbb40be14e4bcbfc75715b77b1242d590d2bf9f7f8a8a910b4469888689') + self.assert_equal(hexlify(cs.next_iv()), b'000000000000000000000001') + # auth-then-decrypt + cs = AES256_GCM(mac_key, enc_key) + pdata = cs.decrypt(hdr_mac_iv_cdata, header_len=len(header), aad_offset=1) self.assert_equal(data, pdata) - self.assert_equal(bytes_to_long(aes.iv, 8), 2) + self.assert_equal(hexlify(cs.next_iv()), b'000000000000000000000001') + # auth-failure due to corruption (corrupted aad) + cs = AES256_GCM(mac_key, enc_key) + hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:1] + b'\0' + hdr_mac_iv_cdata[2:] + self.assert_raises(IntegrityError, + lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted, header_len=len(header), aad_offset=1)) def test_hmac_sha256(self): # RFC 4231 test vectors -- cgit v1.2.3 From ee604ab390a8f82312f58ecfef198ab57416ec60 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 25 Jun 2016 22:09:36 +0200 Subject: crypto: use OpenSSL 1.1 HMAC API This breaks it on OpenSSL 1.0.x as there is no HMAC_CTX_new/free() yet. OTOH, this change is consistent with the previous change done for EVP_CIPHER_CTX (which works on 1.0 and 1.1). --- src/borg/crypto/low_level.pyx | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/borg/crypto/low_level.pyx b/src/borg/crypto/low_level.pyx index ba4905b75..5eafc5eba 100644 --- a/src/borg/crypto/low_level.pyx +++ b/src/borg/crypto/low_level.pyx @@ -111,6 +111,9 @@ cdef extern from "openssl/hmac.h": void HMAC_CTX_init(HMAC_CTX *ctx) void HMAC_CTX_cleanup(HMAC_CTX *ctx) + HMAC_CTX *HMAC_CTX_new() + void HMAC_CTX_free(HMAC_CTX *a) + int HMAC_Init_ex(HMAC_CTX *ctx, const void *key, int key_len, const EVP_MD *md, ENGINE *impl) int HMAC_Update(HMAC_CTX *ctx, const unsigned char *data, int len) int HMAC_Final(HMAC_CTX *ctx, unsigned char *md, unsigned int *len) @@ -183,7 +186,7 @@ cdef class AES256_CTR_HMAC_SHA256: # Layout: HEADER + HMAC 32 + IV 8 + CT (same as attic / borg < 1.2 IF HEADER = TYPE_BYTE, no AAD) cdef EVP_CIPHER_CTX *ctx - cdef HMAC_CTX hmac_ctx + cdef HMAC_CTX *hmac_ctx cdef unsigned char *mac_key cdef unsigned char *enc_key cdef unsigned char iv[16] @@ -200,11 +203,11 @@ cdef class AES256_CTR_HMAC_SHA256: def __cinit__(self, mac_key, enc_key, iv=None): self.ctx = EVP_CIPHER_CTX_new() - HMAC_CTX_init(&self.hmac_ctx) # XXX + self.hmac_ctx = HMAC_CTX_new() def __dealloc__(self): EVP_CIPHER_CTX_free(self.ctx) - HMAC_CTX_cleanup(&self.hmac_ctx) # XXX + HMAC_CTX_free(self.hmac_ctx) def encrypt(self, data, header=b'', aad_offset=0): """ @@ -241,13 +244,13 @@ cdef class AES256_CTR_HMAC_SHA256: if not rc: raise CryptoError('EVP_EncryptFinal_ex failed') offset += olen - if not HMAC_Init_ex(&self.hmac_ctx, self.mac_key, 32, EVP_sha256(), NULL): + if not HMAC_Init_ex(self.hmac_ctx, self.mac_key, 32, EVP_sha256(), NULL): raise CryptoError('HMAC_Init_ex failed') - if not HMAC_Update(&self.hmac_ctx, hdata.buf+aoffset, alen): + if not HMAC_Update(self.hmac_ctx, hdata.buf+aoffset, alen): raise CryptoError('HMAC_Update failed') - if not HMAC_Update(&self.hmac_ctx, odata+hlen+32, offset-hlen-32): + if not HMAC_Update(self.hmac_ctx, odata+hlen+32, offset-hlen-32): raise CryptoError('HMAC_Update failed') - if not HMAC_Final(&self.hmac_ctx, odata+hlen, NULL): + if not HMAC_Final(self.hmac_ctx, odata+hlen, NULL): raise CryptoError('HMAC_Final failed') self.blocks += num_aes_blocks(ilen) return odata[:offset] @@ -272,13 +275,13 @@ cdef class AES256_CTR_HMAC_SHA256: cdef unsigned char hmac_buf[32] cdef Py_buffer idata = ro_buffer(envelope) try: - if not HMAC_Init_ex(&self.hmac_ctx, self.mac_key, 32, EVP_sha256(), NULL): + if not HMAC_Init_ex(self.hmac_ctx, self.mac_key, 32, EVP_sha256(), NULL): raise CryptoError('HMAC_Init_ex failed') - if not HMAC_Update(&self.hmac_ctx, idata.buf+aoffset, alen): + if not HMAC_Update(self.hmac_ctx, idata.buf+aoffset, alen): raise CryptoError('HMAC_Update failed') - if not HMAC_Update(&self.hmac_ctx, idata.buf+hlen+32, ilen-hlen-32): + if not HMAC_Update(self.hmac_ctx, idata.buf+hlen+32, ilen-hlen-32): raise CryptoError('HMAC_Update failed') - if not HMAC_Final(&self.hmac_ctx, hmac_buf, NULL): + if not HMAC_Final(self.hmac_ctx, hmac_buf, NULL): raise CryptoError('HMAC_Final failed') if CRYPTO_memcmp(hmac_buf, idata.buf+hlen, 32): raise IntegrityError('HMAC Authentication failed') -- cgit v1.2.3 From 92080f957233d18a7840c475c1d9ad16a507f694 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 14 Jul 2016 22:35:50 +0200 Subject: crypto: add functions missing in openssl 1.0.x --- setup.py | 3 ++- src/borg/crypto/_crypto_helpers.c | 27 +++++++++++++++++++++++++++ src/borg/crypto/_crypto_helpers.h | 11 +++++++++++ src/borg/crypto/low_level.pyx | 8 ++++++++ 4 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 src/borg/crypto/_crypto_helpers.c create mode 100644 src/borg/crypto/_crypto_helpers.h diff --git a/setup.py b/setup.py index dd85db6a1..db47c25c7 100644 --- a/setup.py +++ b/setup.py @@ -52,6 +52,7 @@ from distutils.command.clean import clean compress_source = 'src/borg/compress.pyx' crypto_ll_source = 'src/borg/crypto/low_level.pyx' +crypto_helpers = 'src/borg/crypto/_crypto_helpers.c' chunker_source = 'src/borg/chunker.pyx' hashindex_source = 'src/borg/hashindex.pyx' item_source = 'src/borg/item.pyx' @@ -730,7 +731,7 @@ ext_modules = [] if not on_rtd: ext_modules += [ Extension('borg.compress', [compress_source], libraries=['lz4'], include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros), - Extension('borg.crypto.low_level', [crypto_ll_source], libraries=crypto_libraries, include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros), + Extension('borg.crypto.low_level', [crypto_ll_source, crypto_helpers], libraries=crypto_libraries, include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros), Extension('borg.hashindex', [hashindex_source]), Extension('borg.item', [item_source]), Extension('borg.chunker', [chunker_source]), diff --git a/src/borg/crypto/_crypto_helpers.c b/src/borg/crypto/_crypto_helpers.c new file mode 100644 index 000000000..588e5f1c6 --- /dev/null +++ b/src/borg/crypto/_crypto_helpers.c @@ -0,0 +1,27 @@ +/* add missing HMAC functions, so OpenSSL 1.0.x can be used like 1.1 */ + +#include +#include +#include + +#if OPENSSL_VERSION_NUMBER < 0x10100000L + +HMAC_CTX *HMAC_CTX_new(void) +{ + HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx)); + if (ctx != NULL) { + memset(ctx, 0, sizeof *ctx); + HMAC_CTX_cleanup(ctx); + } + return ctx; +} + +void HMAC_CTX_free(HMAC_CTX *ctx) +{ + if (ctx != NULL) { + HMAC_CTX_cleanup(ctx); + OPENSSL_free(ctx); + } +} + +#endif diff --git a/src/borg/crypto/_crypto_helpers.h b/src/borg/crypto/_crypto_helpers.h new file mode 100644 index 000000000..e26815c6b --- /dev/null +++ b/src/borg/crypto/_crypto_helpers.h @@ -0,0 +1,11 @@ +/* add missing HMAC functions, so OpenSSL 1.0.x can be used like 1.1 */ + +#include +#include + +#if OPENSSL_VERSION_NUMBER < 0x10100000L + +HMAC_CTX *HMAC_CTX_new(void); +void HMAC_CTX_free(HMAC_CTX *ctx); + +#endif diff --git a/src/borg/crypto/low_level.pyx b/src/borg/crypto/low_level.pyx index 5eafc5eba..d854c9313 100644 --- a/src/borg/crypto/low_level.pyx +++ b/src/borg/crypto/low_level.pyx @@ -123,6 +123,14 @@ cdef extern from "openssl/hmac.h": const unsigned char *data, int data_len, unsigned char *md, unsigned int *md_len) nogil +cdef extern from "_crypto_helpers.h": + ctypedef struct HMAC_CTX: + pass + + HMAC_CTX *HMAC_CTX_new() + void HMAC_CTX_free(HMAC_CTX *a) + + import struct _int = struct.Struct('>I') -- cgit v1.2.3 From 15490d520d5eb27b4bd7488b048ba318f21226aa Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Tue, 16 Aug 2016 06:34:52 +0200 Subject: add support for AES-OCB and chacha20-poly1305 also: use AEAD base class --- src/borg/crypto/_crypto_helpers.c | 10 +++- src/borg/crypto/_crypto_helpers.h | 6 ++- src/borg/crypto/low_level.pyx | 52 ++++++++++++++++--- src/borg/testsuite/crypto.py | 102 ++++++++++++++++++++++++++++++++++++-- 4 files changed, 157 insertions(+), 13 deletions(-) diff --git a/src/borg/crypto/_crypto_helpers.c b/src/borg/crypto/_crypto_helpers.c index 588e5f1c6..0a433bb5f 100644 --- a/src/borg/crypto/_crypto_helpers.c +++ b/src/borg/crypto/_crypto_helpers.c @@ -1,4 +1,4 @@ -/* add missing HMAC functions, so OpenSSL 1.0.x can be used like 1.1 */ +/* some helpers, so our code also works with OpenSSL 1.0.x */ #include #include @@ -24,4 +24,12 @@ void HMAC_CTX_free(HMAC_CTX *ctx) } } +const EVP_CIPHER *EVP_aes_256_ocb(void){ /* dummy, so that code compiles */ + return NULL; +} + +const EVP_CIPHER *EVP_chacha20_poly1305(void){ /* dummy, so that code compiles */ + return NULL; +} + #endif diff --git a/src/borg/crypto/_crypto_helpers.h b/src/borg/crypto/_crypto_helpers.h index e26815c6b..bb9afc418 100644 --- a/src/borg/crypto/_crypto_helpers.h +++ b/src/borg/crypto/_crypto_helpers.h @@ -1,11 +1,15 @@ -/* add missing HMAC functions, so OpenSSL 1.0.x can be used like 1.1 */ +/* some helpers, so our code also works with OpenSSL 1.0.x */ #include #include +#include #if OPENSSL_VERSION_NUMBER < 0x10100000L HMAC_CTX *HMAC_CTX_new(void); void HMAC_CTX_free(HMAC_CTX *ctx); +const EVP_CIPHER *EVP_aes_256_ocb(void); /* dummy, so that code compiles */ +const EVP_CIPHER *EVP_chacha20_poly1305(void); /* dummy, so that code compiles */ + #endif diff --git a/src/borg/crypto/low_level.pyx b/src/borg/crypto/low_level.pyx index d854c9313..e4f1c3186 100644 --- a/src/borg/crypto/low_level.pyx +++ b/src/borg/crypto/low_level.pyx @@ -1,7 +1,5 @@ """An AEAD style OpenSSL wrapper -Note: AES-GCM mode needs OpenSSL >= 1.0.1d due to bug fixes in OpenSSL. - API: encrypt(data, header=b'', aad_offset=0) -> envelope @@ -79,6 +77,8 @@ cdef extern from "openssl/evp.h": const EVP_CIPHER *EVP_aes_256_ctr() const EVP_CIPHER *EVP_aes_256_gcm() + const EVP_CIPHER *EVP_aes_256_ocb() + const EVP_CIPHER *EVP_chacha20_poly1305() void EVP_CIPHER_CTX_init(EVP_CIPHER_CTX *a) void EVP_CIPHER_CTX_cleanup(EVP_CIPHER_CTX *a) @@ -124,12 +124,20 @@ cdef extern from "openssl/hmac.h": unsigned char *md, unsigned int *md_len) nogil cdef extern from "_crypto_helpers.h": + long OPENSSL_VERSION_NUMBER + ctypedef struct HMAC_CTX: pass HMAC_CTX *HMAC_CTX_new() void HMAC_CTX_free(HMAC_CTX *a) + const EVP_CIPHER *EVP_aes_256_ocb() # dummy + const EVP_CIPHER *EVP_chacha20_poly1305() # dummy + + +openssl10 = OPENSSL_VERSION_NUMBER < 0x10100000 + import struct @@ -331,9 +339,13 @@ cdef class AES256_CTR_HMAC_SHA256: iv_out[i] = iv[8+i] -cdef class AES256_GCM: - # Layout: HEADER + GMAC 16 + IV 12 + CT +ctypedef const EVP_CIPHER * (* CIPHER)() + +cdef class _AEAD_BASE: + # Layout: HEADER + MAC 16 + IV 12 + CT + + cdef CIPHER cipher cdef EVP_CIPHER_CTX *ctx cdef unsigned char *enc_key cdef unsigned char iv[12] @@ -376,7 +388,7 @@ cdef class AES256_GCM: offset += hlen offset += 16 self.store_iv(odata+offset, self.iv) - rc = EVP_EncryptInit_ex(self.ctx, EVP_aes_256_gcm(), NULL, NULL, NULL) + rc = EVP_EncryptInit_ex(self.ctx, self.cipher(), NULL, NULL, NULL) if not rc: raise CryptoError('EVP_EncryptInit_ex failed') if not EVP_CIPHER_CTX_ctrl(self.ctx, EVP_CTRL_GCM_SET_IVLEN, 12, NULL): @@ -422,7 +434,7 @@ cdef class AES256_GCM: cdef int offset cdef Py_buffer idata = ro_buffer(envelope) try: - if not EVP_DecryptInit_ex(self.ctx, EVP_aes_256_gcm(), NULL, NULL, NULL): + if not EVP_DecryptInit_ex(self.ctx, self.cipher(), NULL, NULL, NULL): raise CryptoError('EVP_DecryptInit_ex failed') iv = self.fetch_iv( idata.buf+hlen+16) self.set_iv(iv) @@ -445,7 +457,7 @@ cdef class AES256_GCM: rc = EVP_DecryptFinal_ex(self.ctx, odata+offset, &olen) if rc <= 0: # a failure here means corrupted or tampered tag (mac) or data. - raise IntegrityError('GCM Authentication / EVP_DecryptFinal_ex failed') + raise IntegrityError('Authentication / EVP_DecryptFinal_ex failed') offset += olen self.blocks += num_aes_blocks(offset) return odata[:offset] @@ -454,7 +466,7 @@ cdef class AES256_GCM: PyBuffer_Release(&idata) def set_iv(self, iv): - self.blocks = 0 # number of AES blocks encrypted with this IV + self.blocks = 0 # number of cipher blocks encrypted with this IV for i in range(12): self.iv[i] = iv[i] @@ -476,6 +488,30 @@ cdef class AES256_GCM: iv_out[i] = iv[i] +cdef class AES256_GCM(_AEAD_BASE): + def __init__(self, mac_key, enc_key, iv=None): + if OPENSSL_VERSION_NUMBER < 0x10001040: + raise ValueError('AES GCM requires OpenSSL >= 1.0.1d. Detected: OpenSSL %08x' % OPENSSL_VERSION_NUMBER) + self.cipher = EVP_aes_256_gcm + super().__init__(mac_key, enc_key, iv=iv) + + +cdef class AES256_OCB(_AEAD_BASE): + def __init__(self, mac_key, enc_key, iv=None): + if OPENSSL_VERSION_NUMBER < 0x10100000: + raise ValueError('AES OCB requires OpenSSL >= 1.1.0. Detected: OpenSSL %08x' % OPENSSL_VERSION_NUMBER) + self.cipher = EVP_aes_256_ocb + super().__init__(mac_key, enc_key, iv=iv) + + +cdef class CHACHA20_POLY1305(_AEAD_BASE): + def __init__(self, mac_key, enc_key, iv=None): + if OPENSSL_VERSION_NUMBER < 0x10100000: + raise ValueError('CHACHA20-POLY1305 requires OpenSSL >= 1.1.0. Detected: OpenSSL %08x' % OPENSSL_VERSION_NUMBER) + self.cipher = EVP_chacha20_poly1305 + super().__init__(mac_key, enc_key, iv=iv) + + def hmac_sha256(key, data): cdef Py_buffer data_buf = ro_buffer(data) cdef const unsigned char *key_ptr = key diff --git a/src/borg/testsuite/crypto.py b/src/borg/testsuite/crypto.py index 4e8c17464..4582cb101 100644 --- a/src/borg/testsuite/crypto.py +++ b/src/borg/testsuite/crypto.py @@ -1,6 +1,7 @@ from binascii import hexlify, unhexlify -from ..crypto.low_level import AES256_CTR_HMAC_SHA256, AES256_GCM, IntegrityError, hmac_sha256, blake2b_256 +from ..crypto.low_level import AES256_CTR_HMAC_SHA256, AES256_GCM, AES256_OCB, CHACHA20_POLY1305, \ + IntegrityError, hmac_sha256, blake2b_256, openssl10 from ..crypto.low_level import bytes_to_long, bytes_to_int, long_to_bytes, bytes16_to_int, int_to_bytes16, increment_iv from ..crypto.low_level import hkdf_hmac_sha512 @@ -99,7 +100,7 @@ class CryptoTestCase(BaseTestCase): self.assert_raises(IntegrityError, lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted, header_len=len(header), aad_offset=1)) - def test_AES_GCM_256_GMAC(self): + def test_AES_GCM_256(self): # gcm used in legacy-like layout (1 type byte, no aad) mac_key = None enc_key = b'X' * 32 @@ -129,7 +130,7 @@ class CryptoTestCase(BaseTestCase): self.assert_raises(IntegrityError, lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted, header_len=len(header), aad_offset=1)) - def test_AES_GCM_256_GMAC_aad(self): + def test_AES_GCM_256_aad(self): mac_key = None enc_key = b'X' * 32 iv = b'\0' * 12 @@ -158,6 +159,101 @@ class CryptoTestCase(BaseTestCase): self.assert_raises(IntegrityError, lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted, header_len=len(header), aad_offset=1)) + def test_AES_OCB_256(self): + if openssl10: # no OCB + return + # ocb used in legacy-like layout (1 type byte, no aad) + mac_key = None + enc_key = b'X' * 32 + iv = b'\0' * 12 + data = b'foo' * 10 + header = b'\x23' + # encrypt-then-mac + cs = AES256_OCB(mac_key, enc_key, iv) + hdr_mac_iv_cdata = cs.encrypt(data, header=header, aad_offset=1) + hdr = hdr_mac_iv_cdata[0:1] + mac = hdr_mac_iv_cdata[1:17] + iv = hdr_mac_iv_cdata[17:29] + cdata = hdr_mac_iv_cdata[29:] + self.assert_equal(hexlify(hdr), b'23') + self.assert_equal(hexlify(mac), b'b6909c23c9aaebd9abbe1ff42097652d') + self.assert_equal(hexlify(iv), b'000000000000000000000000') + self.assert_equal(hexlify(cdata), b'877ce46d2f62dee54699cebc3ba41d9ab613f7c486778c1b3636664b1493') + self.assert_equal(hexlify(cs.next_iv()), b'000000000000000000000001') + # auth-then-decrypt + cs = AES256_OCB(mac_key, enc_key) + pdata = cs.decrypt(hdr_mac_iv_cdata, header_len=len(header), aad_offset=1) + self.assert_equal(data, pdata) + self.assert_equal(hexlify(cs.next_iv()), b'000000000000000000000001') + # auth-failure due to corruption (corrupted data) + cs = AES256_OCB(mac_key, enc_key) + hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:29] + b'\0' + hdr_mac_iv_cdata[30:] + self.assert_raises(IntegrityError, + lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted, header_len=len(header), aad_offset=1)) + + def test_AES_OCB_256_aad(self): + if openssl10: # no OCB + return + mac_key = None + enc_key = b'X' * 32 + iv = b'\0' * 12 + data = b'foo' * 10 + header = b'\x12\x34\x56' + # encrypt-then-mac + cs = AES256_OCB(mac_key, enc_key, iv) + hdr_mac_iv_cdata = cs.encrypt(data, header=header, aad_offset=1) + hdr = hdr_mac_iv_cdata[0:3] + mac = hdr_mac_iv_cdata[3:19] + iv = hdr_mac_iv_cdata[19:31] + cdata = hdr_mac_iv_cdata[31:] + self.assert_equal(hexlify(hdr), b'123456') + self.assert_equal(hexlify(mac), b'f2748c412af1c7ead81863a18c2c1893') + self.assert_equal(hexlify(iv), b'000000000000000000000000') + self.assert_equal(hexlify(cdata), b'877ce46d2f62dee54699cebc3ba41d9ab613f7c486778c1b3636664b1493') + self.assert_equal(hexlify(cs.next_iv()), b'000000000000000000000001') + # auth-then-decrypt + cs = AES256_OCB(mac_key, enc_key) + pdata = cs.decrypt(hdr_mac_iv_cdata, header_len=len(header), aad_offset=1) + self.assert_equal(data, pdata) + self.assert_equal(hexlify(cs.next_iv()), b'000000000000000000000001') + # auth-failure due to corruption (corrupted aad) + cs = AES256_OCB(mac_key, enc_key) + hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:1] + b'\0' + hdr_mac_iv_cdata[2:] + self.assert_raises(IntegrityError, + lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted, header_len=len(header), aad_offset=1)) + + def test_CHACHA20_POLY1305(self): + if openssl10: # no CHACHA20, no POLY1305 + return + # used in legacy-like layout (1 type byte, no aad) + mac_key = None + enc_key = b'X' * 32 + iv = b'\0' * 12 + data = b'foo' * 10 + header = b'\x23' + # encrypt-then-mac + cs = CHACHA20_POLY1305(mac_key, enc_key, iv) + hdr_mac_iv_cdata = cs.encrypt(data, header=header, aad_offset=1) + hdr = hdr_mac_iv_cdata[0:1] + mac = hdr_mac_iv_cdata[1:17] + iv = hdr_mac_iv_cdata[17:29] + cdata = hdr_mac_iv_cdata[29:] + self.assert_equal(hexlify(hdr), b'23') + self.assert_equal(hexlify(mac), b'fd08594796e0706cde1e8b461e3e0555') + self.assert_equal(hexlify(iv), b'000000000000000000000000') + self.assert_equal(hexlify(cdata), b'a093e4b0387526f085d3c40cca84a35230a5c0dd766453b77ba38bcff775') + self.assert_equal(hexlify(cs.next_iv()), b'000000000000000000000001') + # auth-then-decrypt + cs = CHACHA20_POLY1305(mac_key, enc_key) + pdata = cs.decrypt(hdr_mac_iv_cdata, header_len=len(header), aad_offset=1) + self.assert_equal(data, pdata) + self.assert_equal(hexlify(cs.next_iv()), b'000000000000000000000001') + # auth-failure due to corruption (corrupted data) + cs = CHACHA20_POLY1305(mac_key, enc_key) + hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:29] + b'\0' + hdr_mac_iv_cdata[30:] + self.assert_raises(IntegrityError, + lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted, header_len=len(header), aad_offset=1)) + def test_hmac_sha256(self): # RFC 4231 test vectors key = b'\x0b' * 20 -- cgit v1.2.3 From 741ab8ba05615893581fdaf019e665c82a588b98 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 29 Aug 2016 01:53:09 +0200 Subject: use PyMem_Malloc / Free Hopefully it is better dealing with a lot of small-object allocations than malloc/free is. Small allocs happen if the input file is small, so it results only in 1 small chunk. --- src/borg/crypto/low_level.pyx | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/src/borg/crypto/low_level.pyx b/src/borg/crypto/low_level.pyx index e4f1c3186..12321af4b 100644 --- a/src/borg/crypto/low_level.pyx +++ b/src/borg/crypto/low_level.pyx @@ -34,20 +34,11 @@ IV handling: (repeat) """ -# TODO: get rid of small malloc -# as @enkore mentioned on github: -# "Since we do many small-object allocations here it is probably better to use -# PyMem_Alloc/Free instead of malloc/free (PyMem has many optimizations for -# small allocs)." -# -# Small mallocs currently happen if the total input file length is small, so -# the 1 chunk's size is less than what the chunker would produce for big files. - import hashlib import hmac from math import ceil -from libc.stdlib cimport malloc, free +from cpython cimport PyMem_Malloc, PyMem_Free from cpython.buffer cimport PyBUF_SIMPLE, PyObject_GetBuffer, PyBuffer_Release from cpython.bytes cimport PyBytes_FromStringAndSize @@ -234,7 +225,7 @@ cdef class AES256_CTR_HMAC_SHA256: cdef int hlen = len(header) cdef int aoffset = aad_offset cdef int alen = hlen - aoffset - cdef unsigned char *odata = malloc(hlen + 32 + 8 + ilen + 16) + cdef unsigned char *odata = PyMem_Malloc(hlen + 32 + 8 + ilen + 16) if not odata: raise MemoryError cdef int olen @@ -271,7 +262,7 @@ cdef class AES256_CTR_HMAC_SHA256: self.blocks += num_aes_blocks(ilen) return odata[:offset] finally: - free(odata) + PyMem_Free(odata) PyBuffer_Release(&hdata) PyBuffer_Release(&idata) @@ -283,7 +274,7 @@ cdef class AES256_CTR_HMAC_SHA256: cdef int hlen = header_len cdef int aoffset = aad_offset cdef int alen = hlen - aoffset - cdef unsigned char *odata = malloc(ilen + 16) + cdef unsigned char *odata = PyMem_Malloc(ilen + 16) if not odata: raise MemoryError cdef int olen @@ -317,7 +308,7 @@ cdef class AES256_CTR_HMAC_SHA256: self.blocks += num_aes_blocks(offset) return odata[:offset] finally: - free(odata) + PyMem_Free(odata) PyBuffer_Release(&idata) def set_iv(self, iv): @@ -374,7 +365,7 @@ cdef class _AEAD_BASE: cdef int hlen = len(header) cdef int aoffset = aad_offset cdef int alen = hlen - aoffset - cdef unsigned char *odata = malloc(hlen + 16 + 12 + ilen + 16) + cdef unsigned char *odata = PyMem_Malloc(hlen + 16 + 12 + ilen + 16) if not odata: raise MemoryError cdef int olen @@ -415,7 +406,7 @@ cdef class _AEAD_BASE: self.blocks += num_aes_blocks(ilen) return odata[:offset] finally: - free(odata) + PyMem_Free(odata) PyBuffer_Release(&hdata) PyBuffer_Release(&idata) @@ -427,7 +418,7 @@ cdef class _AEAD_BASE: cdef int hlen = header_len cdef int aoffset = aad_offset cdef int alen = hlen - aoffset - cdef unsigned char *odata = malloc(ilen + 16) + cdef unsigned char *odata = PyMem_Malloc(ilen + 16) if not odata: raise MemoryError cdef int olen @@ -462,7 +453,7 @@ cdef class _AEAD_BASE: self.blocks += num_aes_blocks(offset) return odata[:offset] finally: - free(odata) + PyMem_Free(odata) PyBuffer_Release(&idata) def set_iv(self, iv): -- cgit v1.2.3 From d94f64c6d5b6d7de92eabc2fb63d2db5f99a2be1 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Mon, 29 Aug 2016 04:21:53 +0200 Subject: dedup crypto tests for AE/AEAD ciphersuites --- src/borg/testsuite/crypto.py | 222 ++++++++++++++++--------------------------- 1 file changed, 80 insertions(+), 142 deletions(-) diff --git a/src/borg/testsuite/crypto.py b/src/borg/testsuite/crypto.py index 4582cb101..e8eceb236 100644 --- a/src/borg/testsuite/crypto.py +++ b/src/borg/testsuite/crypto.py @@ -100,159 +100,97 @@ class CryptoTestCase(BaseTestCase): self.assert_raises(IntegrityError, lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted, header_len=len(header), aad_offset=1)) - def test_AES_GCM_256(self): - # gcm used in legacy-like layout (1 type byte, no aad) + def test_AE(self): + # used in legacy-like layout (1 type byte, no aad) mac_key = None enc_key = b'X' * 32 iv = b'\0' * 12 data = b'foo' * 10 header = b'\x23' - # encrypt-then-mac - cs = AES256_GCM(mac_key, enc_key, iv) - hdr_mac_iv_cdata = cs.encrypt(data, header=header, aad_offset=1) - hdr = hdr_mac_iv_cdata[0:1] - mac = hdr_mac_iv_cdata[1:17] - iv = hdr_mac_iv_cdata[17:29] - cdata = hdr_mac_iv_cdata[29:] - self.assert_equal(hexlify(hdr), b'23') - self.assert_equal(hexlify(mac), b'66a438843aa41a087d6a7ed1dc1f3c4c') - self.assert_equal(hexlify(iv), b'000000000000000000000000') - self.assert_equal(hexlify(cdata), b'5bbb40be14e4bcbfc75715b77b1242d590d2bf9f7f8a8a910b4469888689') - self.assert_equal(hexlify(cs.next_iv()), b'000000000000000000000001') - # auth-then-decrypt - cs = AES256_GCM(mac_key, enc_key) - pdata = cs.decrypt(hdr_mac_iv_cdata, header_len=len(header), aad_offset=1) - self.assert_equal(data, pdata) - self.assert_equal(hexlify(cs.next_iv()), b'000000000000000000000001') - # auth-failure due to corruption (corrupted data) - cs = AES256_GCM(mac_key, enc_key) - hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:29] + b'\0' + hdr_mac_iv_cdata[30:] - self.assert_raises(IntegrityError, - lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted, header_len=len(header), aad_offset=1)) + tests = [ + # ciphersuite class, exp_mac, exp_cdata + (AES256_GCM, + b'66a438843aa41a087d6a7ed1dc1f3c4c', + b'5bbb40be14e4bcbfc75715b77b1242d590d2bf9f7f8a8a910b4469888689', ) + ] + if not openssl10: + tests += [ + (AES256_OCB, + b'b6909c23c9aaebd9abbe1ff42097652d', + b'877ce46d2f62dee54699cebc3ba41d9ab613f7c486778c1b3636664b1493', ), + (CHACHA20_POLY1305, + b'fd08594796e0706cde1e8b461e3e0555', + b'a093e4b0387526f085d3c40cca84a35230a5c0dd766453b77ba38bcff775', ) + ] + for cs_cls, exp_mac, exp_cdata in tests: + # encrypt/mac + cs = cs_cls(mac_key, enc_key, iv) + hdr_mac_iv_cdata = cs.encrypt(data, header=header, aad_offset=1) + hdr = hdr_mac_iv_cdata[0:1] + mac = hdr_mac_iv_cdata[1:17] + iv = hdr_mac_iv_cdata[17:29] + cdata = hdr_mac_iv_cdata[29:] + self.assert_equal(hexlify(hdr), b'23') + self.assert_equal(hexlify(mac), exp_mac) + self.assert_equal(hexlify(iv), b'000000000000000000000000') + self.assert_equal(hexlify(cdata), exp_cdata) + self.assert_equal(hexlify(cs.next_iv()), b'000000000000000000000001') + # auth/decrypt + cs = cs_cls(mac_key, enc_key) + pdata = cs.decrypt(hdr_mac_iv_cdata, header_len=len(header), aad_offset=1) + self.assert_equal(data, pdata) + self.assert_equal(hexlify(cs.next_iv()), b'000000000000000000000001') + # auth-failure due to corruption (corrupted data) + cs = cs_cls(mac_key, enc_key) + hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:29] + b'\0' + hdr_mac_iv_cdata[30:] + self.assert_raises(IntegrityError, + lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted, header_len=len(header), aad_offset=1)) - def test_AES_GCM_256_aad(self): + def test_AEAD(self): + # test with aad mac_key = None enc_key = b'X' * 32 iv = b'\0' * 12 data = b'foo' * 10 header = b'\x12\x34\x56' - # encrypt-then-mac - cs = AES256_GCM(mac_key, enc_key, iv) - hdr_mac_iv_cdata = cs.encrypt(data, header=header, aad_offset=1) - hdr = hdr_mac_iv_cdata[0:3] - mac = hdr_mac_iv_cdata[3:19] - iv = hdr_mac_iv_cdata[19:31] - cdata = hdr_mac_iv_cdata[31:] - self.assert_equal(hexlify(hdr), b'123456') - self.assert_equal(hexlify(mac), b'4fb0e5b0a0bca57527352cc6240e7cca') - self.assert_equal(hexlify(iv), b'000000000000000000000000') - self.assert_equal(hexlify(cdata), b'5bbb40be14e4bcbfc75715b77b1242d590d2bf9f7f8a8a910b4469888689') - self.assert_equal(hexlify(cs.next_iv()), b'000000000000000000000001') - # auth-then-decrypt - cs = AES256_GCM(mac_key, enc_key) - pdata = cs.decrypt(hdr_mac_iv_cdata, header_len=len(header), aad_offset=1) - self.assert_equal(data, pdata) - self.assert_equal(hexlify(cs.next_iv()), b'000000000000000000000001') - # auth-failure due to corruption (corrupted aad) - cs = AES256_GCM(mac_key, enc_key) - hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:1] + b'\0' + hdr_mac_iv_cdata[2:] - self.assert_raises(IntegrityError, - lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted, header_len=len(header), aad_offset=1)) - - def test_AES_OCB_256(self): - if openssl10: # no OCB - return - # ocb used in legacy-like layout (1 type byte, no aad) - mac_key = None - enc_key = b'X' * 32 - iv = b'\0' * 12 - data = b'foo' * 10 - header = b'\x23' - # encrypt-then-mac - cs = AES256_OCB(mac_key, enc_key, iv) - hdr_mac_iv_cdata = cs.encrypt(data, header=header, aad_offset=1) - hdr = hdr_mac_iv_cdata[0:1] - mac = hdr_mac_iv_cdata[1:17] - iv = hdr_mac_iv_cdata[17:29] - cdata = hdr_mac_iv_cdata[29:] - self.assert_equal(hexlify(hdr), b'23') - self.assert_equal(hexlify(mac), b'b6909c23c9aaebd9abbe1ff42097652d') - self.assert_equal(hexlify(iv), b'000000000000000000000000') - self.assert_equal(hexlify(cdata), b'877ce46d2f62dee54699cebc3ba41d9ab613f7c486778c1b3636664b1493') - self.assert_equal(hexlify(cs.next_iv()), b'000000000000000000000001') - # auth-then-decrypt - cs = AES256_OCB(mac_key, enc_key) - pdata = cs.decrypt(hdr_mac_iv_cdata, header_len=len(header), aad_offset=1) - self.assert_equal(data, pdata) - self.assert_equal(hexlify(cs.next_iv()), b'000000000000000000000001') - # auth-failure due to corruption (corrupted data) - cs = AES256_OCB(mac_key, enc_key) - hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:29] + b'\0' + hdr_mac_iv_cdata[30:] - self.assert_raises(IntegrityError, - lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted, header_len=len(header), aad_offset=1)) - - def test_AES_OCB_256_aad(self): - if openssl10: # no OCB - return - mac_key = None - enc_key = b'X' * 32 - iv = b'\0' * 12 - data = b'foo' * 10 - header = b'\x12\x34\x56' - # encrypt-then-mac - cs = AES256_OCB(mac_key, enc_key, iv) - hdr_mac_iv_cdata = cs.encrypt(data, header=header, aad_offset=1) - hdr = hdr_mac_iv_cdata[0:3] - mac = hdr_mac_iv_cdata[3:19] - iv = hdr_mac_iv_cdata[19:31] - cdata = hdr_mac_iv_cdata[31:] - self.assert_equal(hexlify(hdr), b'123456') - self.assert_equal(hexlify(mac), b'f2748c412af1c7ead81863a18c2c1893') - self.assert_equal(hexlify(iv), b'000000000000000000000000') - self.assert_equal(hexlify(cdata), b'877ce46d2f62dee54699cebc3ba41d9ab613f7c486778c1b3636664b1493') - self.assert_equal(hexlify(cs.next_iv()), b'000000000000000000000001') - # auth-then-decrypt - cs = AES256_OCB(mac_key, enc_key) - pdata = cs.decrypt(hdr_mac_iv_cdata, header_len=len(header), aad_offset=1) - self.assert_equal(data, pdata) - self.assert_equal(hexlify(cs.next_iv()), b'000000000000000000000001') - # auth-failure due to corruption (corrupted aad) - cs = AES256_OCB(mac_key, enc_key) - hdr_mac_iv_cdata_corrupted = hdr_mac_iv_cdata[:1] + b'\0' + hdr_mac_iv_cdata[2:] - self.assert_raises(IntegrityError, - lambda: cs.decrypt(hdr_mac_iv_cdata_corrupted, header_len=len(header), aad_offset=1)) - - def test_CHACHA20_POLY1305(self): - if openssl10: # no CHACHA20, no POLY1305 - return - # used in legacy-like layout (1 type byte, no aad) - mac_key = None - enc_key = b'X' * 32 - iv = b'\0' * 12 - data = b'foo' * 10 - header = b'\x23' - # encrypt-then-mac - cs = CHACHA20_POLY1305(mac_key, enc_key, iv) - hdr_mac_iv_cdata = cs.encrypt(data, header=header, aad_offset=1) - hdr = hdr_mac_iv_cdata[0:1] - mac = hdr_mac_iv_cdata[1:17] - iv = hdr_mac_iv_cdata[17:29] - cdata = hdr_mac_iv_cdata[29:] - self.assert_equal(hexlify(hdr), b'23