Pull request coderanger#27 from kamilbednarz: Add support for encrypted data bag items v2

Author: freimer

chef/__init__.py

@@ -5,6 +5,7 @@
 from chef.api import ChefAPI, autoconfigure
 from chef.client import Client
 from chef.data_bag import DataBag, DataBagItem
+from chef.encrypted_data_bag_item import EncryptedDataBagItem
 from chef.exceptions import ChefError
 from chef.node import Node
 from chef.role import Role

chef/aes.py

@@ -0,0 +1,127 @@
+import os
+
+from ctypes import *
+from rsa import load_crypto_lib, SSLError
+
+_eay = load_crypto_lib()
+
+c_int_p = POINTER(c_int)
+
+# void EVP_CIPHER_CTX_init(EVP_CIPHER_CTX *a);
+EVP_CIPHER_CTX_init = _eay.EVP_CIPHER_CTX_init
+EVP_CIPHER_CTX_init.argtypes = [c_void_p]
+EVP_CIPHER_CTX_init.restype = None
+
+#int EVP_CipherUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out,
+#        int *outl, unsigned char *in, int inl);
+EVP_CipherUpdate = _eay.EVP_CipherUpdate
+EVP_CipherUpdate.argtypes = [c_void_p, c_char_p, c_int_p, c_char_p, c_int]
+EVP_CipherUpdate.restype = c_int
+
+#int EVP_CipherFinal(EVP_CIPHER_CTX *ctx, unsigned char *out,
+#         int *outl);
+EVP_CipherFinal = _eay.EVP_CipherFinal
+EVP_CipherFinal.argtypes = [c_void_p, c_char_p, c_int_p]
+EVP_CipherFinal.restype = c_int
+
+#EVP_CIPHER *EVP_aes_256_cbc(void);
+EVP_aes_256_cbc = _eay.EVP_aes_256_cbc
+EVP_aes_256_cbc.argtypes = []
+EVP_aes_256_cbc.restype = c_void_p
+
+#EVP_MD *EVP_sha1(void);
+EVP_sha1 = _eay.EVP_sha1
+EVP_sha1.argtypes = []
+EVP_sha1.restype = c_void_p
+
+#int EVP_CipherInit(EVP_CIPHER_CTX *ctx, const EVP_CIPHER *type,
+#        unsigned char *key, unsigned char *iv, int enc);
+EVP_CipherInit = _eay.EVP_CipherInit
+EVP_CipherInit.argtypes = [c_void_p, c_void_p, c_char_p, c_char_p, c_int]
+EVP_CipherInit.restype = c_int
+
+#int EVP_CIPHER_CTX_set_padding(EVP_CIPHER_CTX *x, int padding);
+EVP_CIPHER_CTX_set_padding = _eay.EVP_CIPHER_CTX_set_padding
+EVP_CIPHER_CTX_set_padding.argtypes = [c_void_p, c_int]
+EVP_CIPHER_CTX_set_padding.restype = c_int
+
+# Structures required for ctypes
+
+EVP_MAX_IV_LENGTH = 16
+EVP_MAX_BLOCK_LENGTH = 32
+AES_BLOCK_SIZE = 16
+
+class EVP_CIPHER(Structure):
+    _fields_ = [
+            ("nid", c_int),
+            ("block_size", c_int),
+            ("key_len", c_int),
+            ("iv_len", c_int),
+            ("flags", c_ulong),
+            ("init", c_voidp),
+            ("do_cipher", c_voidp),
+            ("cleanup", c_voidp),
+            ("set_asn1_parameters", c_voidp),
+            ("get_asn1_parameters", c_voidp),
+            ("ctrl", c_voidp),
+            ("app_data", c_voidp)
+    ]
+
+class EVP_CIPHER_CTX(Structure):
+    _fields_ = [
+            ("cipher", POINTER(EVP_CIPHER)),
+            ("engine", c_voidp),
+            ("encrypt", c_int),
+            ("buflen", c_int),
+            ("oiv", c_ubyte * EVP_MAX_IV_LENGTH),
+            ("iv", c_ubyte * EVP_MAX_IV_LENGTH),
+            ("buf", c_ubyte * EVP_MAX_BLOCK_LENGTH),
+            ("num", c_int),
+            ("app_data", c_voidp),
+            ("key_len", c_int),
+            ("flags", c_ulong),
+            ("cipher_data", c_voidp),
+            ("final_used", c_int),
+            ("block_mask", c_int),
+            ("final", c_ubyte * EVP_MAX_BLOCK_LENGTH) ]
+
+
+class AES256Cipher(object):
+    def __init__(self, key, iv, salt='12345678'):
+        self.key_data = create_string_buffer(key)
+        self.iv = create_string_buffer(iv)
+        self.encryptor = self.decryptor = None
+        self.salt = create_string_buffer(salt)
+
+        self.encryptor = EVP_CIPHER_CTX()
+        self._init_cipher(byref(self.encryptor), 1)
+
+        self.decryptor = EVP_CIPHER_CTX()
+        self._init_cipher(byref(self.decryptor), 0)
+
+    def _init_cipher(self, ctypes_cipher, crypt_mode):
+        """ crypt_mode parameter is a flag deciding whether the cipher should be
+        used for encryption (1) or decryption (0)  """
+        EVP_CIPHER_CTX_init(ctypes_cipher)
+        EVP_CipherInit(ctypes_cipher, EVP_aes_256_cbc(), self.key_data, self.iv, c_int(crypt_mode))
+        EVP_CIPHER_CTX_set_padding(ctypes_cipher, c_int(1))
+
+    def _process_data(self, ctypes_cipher, data):
+        length = c_int(len(data))
+        buf_length = c_int(length.value + AES_BLOCK_SIZE)
+        buf = create_string_buffer(buf_length.value)
+
+        final_buf = create_string_buffer(AES_BLOCK_SIZE)
+        final_length = c_int(0)
+
+        EVP_CipherUpdate(ctypes_cipher, buf, byref(buf_length), create_string_buffer(data), length)
+        EVP_CipherFinal(ctypes_cipher, final_buf, byref(final_length))
+
+        return string_at(buf, buf_length) + string_at(final_buf, final_length)
+
+
+    def encrypt(self, data):
+        return self._process_data(byref(self.encryptor), data)
+
+    def decrypt(self, data):
+        return self._process_data(byref(self.decryptor), data)

chef/api.py

@@ -56,7 +56,7 @@ class ChefAPI(object):
     env_value_re = re.compile(r'ENV\[(.+)\]')
     ruby_string_re = re.compile(r'^\s*(["\'])(.*?)\1\s*$')
 
-    def __init__(self, url, key, client, version='0.10.8', headers={}, ssl_verify=True):
+    def __init__(self, url, key, client, version='0.10.8', headers={}, secret_file=None, encryption_version=1):
         self.url = url.rstrip('/')
         self.parsed_url = six.moves.urllib.parse.urlparse(self.url)
         if not isinstance(key, Key):
@@ -66,12 +66,19 @@ def __init__(self, url, key, client, version='0.10.8', headers={}, ssl_verify=Tr
         self.key = key
         self.client = client
         self.version = version
+        self.encryption_version = encryption_version
         self.headers = dict((k.lower(), v) for k, v in six.iteritems(headers))
         self.version_parsed = pkg_resources.parse_version(self.version)
         self.platform = self.parsed_url.hostname == 'api.opscode.com'
         self.ssl_verify = ssl_verify
         if not api_stack_value():
             self.set_default()
+        self.encryption_key = None
+        if secret_file is not None:
+            self.secret_file = secret_file
+            if os.path.exists(self.secret_file):
+                self.encryption_key = open(self.secret_file).read().strip()
+
 
     @classmethod
     def from_config_file(cls, path):
@@ -83,7 +90,7 @@ def from_config_file(cls, path):
             # Can't even read the config file
             log.debug('Unable to read config file "%s"', path)
             return
-        url = key_path = client_name = None
+        url = key_path = client_name = encryption_version = None
         ssl_verify = True
         for line in open(path):
             if not line.strip() or line.startswith('#'):
@@ -123,6 +130,9 @@ def _ruby_value(match):
             elif key == 'node_name':
                 log.debug('Found client name: %r', value)
                 client_name = value
+            elif key == 'data_bag_encrypt_version':
+                log.debug('Found data bag encryption version: %r', value)
+                encryption_version = value
             elif key == 'client_key':
                 log.debug('Found key path: %r', value)
                 key_path = value
@@ -158,7 +168,9 @@ def _ruby_value(match):
             return
         if not client_name:
             client_name = socket.getfqdn()
-        return cls(url, key_path, client_name, ssl_verify=ssl_verify)
+        if not encryption_version:
+            encryption_version = 1
+        return cls(url, key_path, client_name, encryption_version=encryption_version)
 
     @staticmethod
     def get_global():

chef/encrypted_data_bag_item.py

@@ -0,0 +1,133 @@
+from chef.exceptions import ChefUnsupportedEncryptionVersionError, ChefDecryptionError
+from chef.aes import AES256Cipher
+from chef.utils import json
+from chef.data_bag import DataBagItem
+
+import os
+import hmac
+import base64
+import chef
+import hashlib
+import itertools
+
+class EncryptedDataBagItem(DataBagItem):
+    SUPPORTED_ENCRYPTION_VERSIONS = (1,2)
+    AES_MODE = 'aes_256_cbc'
+
+    def __getitem__(self, key):
+        if key == 'id':
+            return self.raw_data[key]
+        else:
+            return create_decryptor(self.api.encryption_key, self.raw_data[key]).decrypt()
+
+    def __setitem__(self, key, value):
+        if key == 'id':
+            self.raw_data[key] = value
+        else:
+            self.raw_data[key] = create_encryptor(self.api.encryption_key, value, self.api.encryption_version).to_dict()
+
+def create_encryptor(key, data, version):
+    try:
+        return {
+            1: EncryptorVersion1(key, data),
+            2: EncryptorVersion2(key, data)
+            }[version]
+    except KeyError:
+        raise ChefUnsupportedEncryptionVersionError(version)
+
+class EncryptorVersion1(object):
+    VERSION = 1
+
+    def __init__(self, key, data):
+        self.plain_key = key
+        self.key = hashlib.sha256(key).digest()
+        self.data = data
+        self.iv = os.urandom(8).encode('hex')
+        self.encryptor = AES256Cipher(key=self.key, iv=self.iv)
+        self.encrypted_data = None
+
+    def encrypt(self):
+        if self.encrypted_data is None:
+            data = json.dumps({'json_wrapper': self.data})
+            self.encrypted_data = self.encryptor.encrypt(data)
+        return self.encrypted_data
+
+    def to_dict(self):
+        return {
+            "encrypted_data": base64.standard_b64encode(self.encrypt()),
+            "iv": base64.standard_b64encode(self.iv),
+            "version": self.VERSION,
+            "cipher": "aes-256-cbc"
+            }
+
+class EncryptorVersion2(EncryptorVersion1):
+    VERSION = 2
+
+    def __init__(self, key, data):
+        super(EncryptorVersion2, self).__init__(key, data)
+        self.hmac = None
+
+    def encrypt(self):
+        self.encrypted_data = super(EncryptorVersion2, self).encrypt()
+        self.hmac = (self.hmac if self.hmac is not None else self._generate_hmac())
+        return self.encrypted_data
+
+    def _generate_hmac(self):
+        raw_hmac = hmac.new(self.plain_key, base64.standard_b64encode(self.encrypted_data), hashlib.sha256).digest()
+        return raw_hmac
+
+    def to_dict(self):
+        result = super(EncryptorVersion2, self).to_dict()
+        result['hmac'] = base64.standard_b64encode(self.hmac)
+        return result
+
+def get_decryption_version(data):
+    if data.has_key('version'):
+        if str(data['version']) in map(str, EncryptedDataBagItem.SUPPORTED_ENCRYPTION_VERSIONS):
+            return data['version']
+        else:
+            raise ChefUnsupportedEncryptionVersionError(data['version'])
+    else:
+        return 1
+
+def create_decryptor(key, data):
+    version = get_decryption_version(data)
+    if version == 1:
+        return DecryptorVersion1(key, data['encrypted_data'], data['iv'])
+    elif version == 2:
+        return DecryptorVersion2(key, data['encrypted_data'], data['iv'], data['hmac'])
+
+class DecryptorVersion1(object):
+    def __init__(self, key, data, iv):
+        self.key = hashlib.sha256(key).digest()
+        self.data = base64.standard_b64decode(data)
+        self.iv = base64.standard_b64decode(iv)
+        self.decryptor = AES256Cipher(key=self.key, iv=self.iv)
+
+    def decrypt(self):
+        value = self.decryptor.decrypt(self.data)
+        # After decryption we should get a string with JSON
+        try:
+            value = json.loads(value)
+        except ValueError:
+            raise ChefDecryptionError("Error decrypting data bag value. Most likely the provided key is incorrect")
+        return value['json_wrapper']
+
+class DecryptorVersion2(DecryptorVersion1):
+    def __init__(self, key, data, iv, hmac):
+        super(DecryptorVersion2, self).__init__(key, data, iv)
+        self.hmac = base64.standard_b64decode(hmac)
+        self.encoded_data = data
+
+    def _validate_hmac(self):
+        expected_hmac = hmac.new(self.key, self.encoded_data, hashlib.sha256).digest()
+        valid = len(expected_hmac) ^ len(self.hmac)
+        for expected_char, candidate_char in itertools.izip_longest(expected_hmac, self.hmac):
+            valid |= ord(expected_char) ^ ord(candidate_char)
+        return valid == 0
+
+    def decrypt(self):
+        if self._validate_hmac():
+            return super(DecryptorVersion2, self).decrypt()
+        else:
+            raise ChefDecryptionError("Error decrypting data bag value. HMAC validation failed.")

chef/exceptions.py

@@ -31,6 +31,14 @@ class ChefServerNotFoundError(ChefServerError):
 class ChefAPIVersionError(ChefError):
     """An incompatible API version error"""
 
+class ChefUnsupportedEncryptionVersionError(ChefError):
+    def __init__(self, version):
+        message = "This version of chef does not support encrypted data bag item format version %s" % version
+        return super(ChefError, self).__init__(message)
+
+class ChefDecryptionError(ChefError):
+    """Error decrypting data bag value. Most likely the provided key is incorrect"""
+
 
 class ChefObjectTypeError(ChefError):
     """An invalid object type error"""

chef/tests/configs/encryption.rb

@@ -0,0 +1,5 @@
+chef_server_url 'http://chef:4000'
+client_key '../client.pem'
+node_name "test_1"
+
+data_bag_encrypt_version '2'

chef/tests/encryption_key

@@ -0,0 +1 @@
+1FNnCAvhritIPDerwUYKFNFZ8NaAIWyCXV43hqSVDeGk5pDt253E+RlUHlL7H/3HEFo/gnZWsk9Y5bEyOP7tUQSnT8enCbFqtvyBpiVep+4BYHss2aWBqqsm7aiPXa+BQHagmVHySleU+sFdLcNOASNMLiUB6azk8Xme1Gris8Awavrn/s5vRB7Bsl7xl84nSmu7Lg3C6Vezyye6K4ZmJOA1p0QPSMVGEJC5RkwAmA+W6G5MilBDMdxxN7mxy49WRSFLT35xFQNJOJ+Rvk53FJrhOCmiHkVNumF2MuhIpLsbrqpcdsU5UIxibjd2Dt+yz7/qytCsGSyZkVws09MgAH5icjZYV6DL8Y9CRa39KEyHl5DjHmWiRiuoFTc6oiUa0QAh08X64jz8OvcTWCJD9Fi5PdNkJblDMp9g6vvn/UPTos2s0KjzkLKdRbLrJovCSs52kkhTzfYXOYt4rmi5mQbdtcr2vsXFs+CT68Yfs56RFA2BA/+KLdaNzHFeH/Wl3h/hrciQfpAW62jnttBGr7sMV0pevXQTr2npPWq0fZHWO4gxkrL729najiDPOEeA2TeHV6+h6znZNYvfpNIRPIOMDLG7bdq2+/G7OvuE7u15qHYzWlJpvouhLA55upDK6CK1ONQw14JIK4+s9Dt2gYpV//G7MqnFMsnq3Y9ptt4=

chef/tests/test_aes.py

@@ -0,0 +1,23 @@
+from chef.tests import ChefTestCase, TEST_ROOT
+from chef.aes import AES256Cipher
+from chef.rsa import SSLError
+
+import base64
+import os
+import hashlib
+import json
+
+class AES256CipherTestCase(ChefTestCase):
+    def setUp(self):
+        super(AES256CipherTestCase, self).setUp()
+        key = hashlib.sha256(open(os.path.join(TEST_ROOT, 'encryption_key')).read()).digest()
+        iv = base64.standard_b64decode('GLVikZLxG0SWYnb68Pr8Ag==\n')
+        self.cipher = AES256Cipher(key, iv)
+
+    def test_encrypt(self):
+        encrypted_value = self.cipher.encrypt('{"json_wrapper":"secr3t c0d3"}')
+        self.assertEquals(base64.standard_b64encode(encrypted_value).strip(), "Ym5T8umtSd0wgjDYq1ZDK5dAh6OjgrTxlloGNf2xYhg=")
+
+    def test_decrypt(self):
+        decrypted_value = self.cipher.decrypt(base64.standard_b64decode('Ym5T8umtSd0wgjDYq1ZDK5dAh6OjgrTxlloGNf2xYhg=\n'))
+        self.assertEquals(decrypted_value, '{"json_wrapper":"secr3t c0d3"}')

chef/tests/test_api.py

@@ -27,6 +27,10 @@ def test_env_variables(self):
         finally:
             del os.environ['_PYCHEF_TEST_']
 
+    def test_encryption(self):
+        api = self.load('encryption.rb')
+        self.assertEqual(api.encryption_version, '2')
+
     def test_bad_key_raises(self):
         invalids = [None, '']
         for item in invalids: