From 8634ce87b0a5c1a32a964a5bd4a7c9fad90a8e2e Mon Sep 17 00:00:00 2001 From: Andrej Burger Date: Sat, 10 Dec 2016 19:06:55 +0100 Subject: [PATCH 1/5] Small refactoring --- s3_deploy/deploy.py | 154 ++++++++++++++++++++++++-------------------- 1 file changed, 84 insertions(+), 70 deletions(-) diff --git a/s3_deploy/deploy.py b/s3_deploy/deploy.py index bff496d..5340da7 100644 --- a/s3_deploy/deploy.py +++ b/s3_deploy/deploy.py @@ -112,26 +112,7 @@ def upload_key(obj, path, cache_rules, dry, storage_class=None): content_file.close() -def main(): - logging.basicConfig(level=logging.INFO) - logging.getLogger('boto3').setLevel(logging.WARNING) - - parser = argparse.ArgumentParser( - description='AWS S3 website deployment tool') - parser.add_argument( - '-f', '--force', action='store_true', dest='force', - help='force upload of all files') - parser.add_argument( - '-n', '--dry-run', action='store_true', dest='dry', - help='run without uploading any files') - parser.add_argument( - 'path', help='the .s3_website.yaml configuration file or directory', - default='.', nargs='?') - args = parser.parse_args() - - # Open configuration file - conf, base_path = config.load_config_file(args.path) - +def upload_to_bucket(session, site_dir, conf, dry): bucket_name = conf['s3_bucket'] cache_rules = conf.get('cache_rules', []) if conf.get('s3_reduced_redundancy', False): @@ -144,8 +125,6 @@ def main(): s3 = boto3.resource('s3') bucket = s3.Bucket(bucket_name) - site_dir = os.path.join(base_path, conf['site']) - logger.info('Site: {}'.format(site_dir)) processed_keys = set() @@ -158,21 +137,21 @@ def main(): # Delete keys that have been deleted locally if not os.path.isfile(path): logger.info('Deleting {}...'.format(obj.key)) - if not args.dry: + if not dry: obj.delete() updated_keys.add(obj.key) continue # Skip keys that have not been updated mtime = datetime.fromtimestamp(os.path.getmtime(path), UTC) - if not args.force: + if not conf.get('force', False): if (mtime <= obj.last_modified and obj.storage_class == storage_class): logger.info('Not modified, skipping {}.'.format(obj.key)) continue upload_key( - obj, path, cache_rules, args.dry, storage_class=storage_class) + obj, path, cache_rules, dry, storage_class=storage_class) updated_keys.add(obj.key) for dirpath, dirnames, filenames in os.walk(site_dir): @@ -189,59 +168,94 @@ def main(): logger.info('Creating key {}...'.format(obj.key)) upload_key( - obj, path, cache_rules, args.dry, storage_class=storage_class) + obj, path, cache_rules, dry, storage_class=storage_class) updated_keys.add(key_name) logger.info('Bucket update done.') + + return processed_keys, updated_keys - # Invalidate files in cloudfront distribution - if 'cloudfront_distribution_id' in conf: - logger.info('Connecting to Cloudfront distribution {}...'.format( + +def invalidate_cloudfront(session, processed_keys, updated_keys, conf, dry): + logger.info('Connecting to Cloudfront distribution {}...'.format( conf['cloudfront_distribution_id'])) - index_pattern = None - if 'index_document' in conf: - index_doc = conf['index_document'] - index_pattern = r'(^(?:.*/)?)' + re.escape(index_doc) + '$' - - def path_from_key_name(key_name): - if index_pattern is not None: - m = re.match(index_pattern, key_name) - if m: - return m.group(1) - return key_name - - t = PrefixCoverTree() - for key_name in updated_keys: - t.include(path_from_key_name(key_name)) - for key_name in processed_keys - updated_keys: - t.exclude(path_from_key_name(key_name)) - - paths = [] - for prefix, exact in t.matches(): - path = '/' + prefix + ('' if exact else '*') - logger.info('Preparing to invalidate {}...'.format(path)) - paths.append(path) - - cloudfront = boto3.client('cloudfront') - - if len(paths) > 0: - dist_id = conf['cloudfront_distribution_id'] - if not args.dry: - logger.info('Creating invalidation request...') - response = cloudfront.create_invalidation( - DistributionId=dist_id, - InvalidationBatch=dict( + index_pattern = None + if 'index_document' in conf: + index_doc = conf['index_document'] + index_pattern = r'(^(?:.*/)?)' + re.escape(index_doc) + '$' + + def path_from_key_name(key_name): + if index_pattern is not None: + m = re.match(index_pattern, key_name) + if m: + return m.group(1) + return key_name + + t = PrefixCoverTree() + for key_name in updated_keys: + t.include(path_from_key_name(key_name)) + for key_name in processed_keys - updated_keys: + t.exclude(path_from_key_name(key_name)) + + paths = [] + for prefix, exact in t.matches(): + path = '/' + prefix + ('' if exact else '*') + logger.info('Preparing to invalidate {}...'.format(path)) + paths.append(path) + + cloudfront = boto3.client('cloudfront') + + if len(paths) > 0: + dist_id = conf['cloudfront_distribution_id'] + if not dry: + logger.info('Creating invalidation request...') + response = cloudfront.create_invalidation( + DistributionId=dist_id, + InvalidationBatch=dict( Paths=dict( Quantity=len(paths), - Items=['' + quote_plus(p) + '' - for p in paths] + Items=paths ), CallerReference='s3-deploy-website' ) - ) - invalidation = response['Invalidation'] - logger.info('Invalidation request {} is {}'.format( - invalidation['Id'], invalidation['Status'])) - else: - logger.info('Nothing updated, invalidation skipped.') + ) + invalidation = response['Invalidation'] + logger.info('Invalidation request {} is {}'.format( + invalidation['Id'], invalidation['Status'])) + else: + logger.info('Nothing updated, invalidation skipped.') + + +def main(): + logging.basicConfig(level=logging.INFO) + logging.getLogger('boto3').setLevel(logging.WARNING) + + parser = argparse.ArgumentParser( + description='AWS S3 website deployment tool') + parser.add_argument( + '-f', '--force', action='store_true', dest='force', + help='force upload of all files') + parser.add_argument( + '-n', '--dry-run', action='store_true', dest='dry', + help='run without uploading any files') + parser.add_argument( + 'path', help='the .s3_website.yaml configuration file or directory', + default='.', nargs='?') + args = parser.parse_args() + + # Open configuration file + conf, base_path = config.load_config_file(args.path) + + if args.force: + conf['force'] = True + + processed_keys, updated_keys = upload_to_bucket( + None, + os.path.join(base_path, conf['site']), + conf, + args.dry) + + # Invalidate files in cloudfront distribution + if 'cloudfront_distribution_id' in conf: + invalidate_cloudfront(None, processed_keys, updated_keys, conf, args.dry) From aadb2cd82a08945b8ee7c6455eba76b0c47a9130 Mon Sep 17 00:00:00 2001 From: Andrej Burger Date: Sat, 10 Dec 2016 20:06:17 +0100 Subject: [PATCH 2/5] AWS key and profile settings added --- README.rst | 48 +++++++++++++++++++++++++++++++++++++-------- s3_deploy/deploy.py | 38 ++++++++++++++++++++++++++++++----- 2 files changed, 73 insertions(+), 13 deletions(-) diff --git a/README.rst b/README.rst index 4603e3b..465a13c 100644 --- a/README.rst +++ b/README.rst @@ -24,6 +24,12 @@ The configuration is stored in a YAML file like this: .. code-block:: yaml site: _site + + aws_id: AKIAI6XLRPEQEU6T6ZXA + aws_secret: Ua06LWWJRnAvUZuE4yRY/f6Cyyp9R2pF0PsQnwqd + region: eu-central-1 + aws_profile: deploy + s3_bucket: example.com cloudfront_distribution_id: XXXXXXXXXXX @@ -53,17 +59,23 @@ same directory: Credentials ----------- -AWS credentials can be provided through the environment variables -``AWS_ACCESS_KEY_ID`` and ``AWS_SECRET_ACCESS_KEY``. +AWS credentials can be provided in following ways (in order of priority): -.. code-block:: shell +1. Using the ``--profile PROFILE`` command line option selecting the AWS profile to use. - $ export AWS_ACCESS_KEY_ID=XXXXXX - $ export AWS_SECTER_ACCESS_KEY=XXXXXX - $ s3-deploy-website +2. Using the ``aws_id`` and ``aws_secret`` settings in configuration file. + +3. Using the ``aws_profile`` setting in configuration file to select the AWS profile to use. + +4. Environment variables ``AWS_ACCESS_KEY_ID`` and ``AWS_SECRET_ACCESS_KEY``. + + .. code-block:: shell + + $ export AWS_ACCESS_KEY_ID=XXXXXX + $ export AWS_SECTER_ACCESS_KEY=XXXXXX + $ s3-deploy-website -They can also be provided through the various configuration files that boto_ -reads. +5. They can also be provided through the various configuration files that boto_reads. .. _boto: https://boto.readthedocs.org/en/latest/boto_config_tut.html @@ -73,6 +85,26 @@ Configuration file **site** The directory of the static content to be uploaded (relative to the location of the configuration file (e.g. ``_site`` for Jekyll sites). + +**aws_id** + The AWS access key ID used to connect to the AWS. If ``aws_id`` is + specified, also ``aws_secret`` has to be specified. If either ``aws_id`` + or ``aws_secret`` and also ``aws_profile`` is specified, the specified + access key is used to connect. + +**aws_secret** + The AWS secret access key used to connect to the AWS. If ``aws_secret`` + is specified, also ``aws_id`` has to be specified. If either ``aws_id`` + or ``aws_secret`` and also ``aws_profile`` is specified, the specified + access key is used to connect. + +**region** + The region used to connect to the AWS. + +**aws_profile** + The AWS profile used to connect to the AWS. If ``aws_profile`` and also + either ``aws_id`` or ``aws_secret`` is specified, the specified access + key is used to connect. **s3_bucket** The name of the S3 bucket to upload the files to. You have to allow the diff --git a/s3_deploy/deploy.py b/s3_deploy/deploy.py index 5340da7..ff2075e 100644 --- a/s3_deploy/deploy.py +++ b/s3_deploy/deploy.py @@ -46,6 +46,23 @@ def dst(self, dt): mimetypes.init() +def create_session(conf): + session = None + if conf.get('aws_id', False) or conf.get('aws_secret', False): + if conf.get('aws_id', False) and conf.get('aws_secret', False): + logger.info('Creating AWS session using AWS key id \"{}\" and region \"{}\"...'.format(conf['aws_id'], conf.get('region', ''))) + session = boto3.Session(conf['aws_id'], conf['aws_secret'], region_name=conf.get('region', None)) + else: + logger.error('Both AWS access key ID and AWS secret access key has to specified if either is specified.') + elif conf.get('aws_profile', False): + logger.info('Creating AWS session using AWS profile \"{}\"...'.format(conf['aws_profile'])) + session = boto3.Session(profile_name=conf['aws_profile']) + else: + logger.info('Creating AWS session using default profile...') + session = boto3.Session() + + return session + def key_name_from_path(path): """Convert a relative path into a key name.""" @@ -122,7 +139,8 @@ def upload_to_bucket(session, site_dir, conf, dry): logger.info('Connecting to bucket {}...'.format(bucket_name)) - s3 = boto3.resource('s3') +# s3 = boto3.resource('s3') + s3 = session.resource('s3') bucket = s3.Bucket(bucket_name) logger.info('Site: {}'.format(site_dir)) @@ -204,7 +222,7 @@ def path_from_key_name(key_name): logger.info('Preparing to invalidate {}...'.format(path)) paths.append(path) - cloudfront = boto3.client('cloudfront') + cloudfront = session.client('cloudfront') if len(paths) > 0: dist_id = conf['cloudfront_distribution_id'] @@ -239,6 +257,9 @@ def main(): parser.add_argument( '-n', '--dry-run', action='store_true', dest='dry', help='run without uploading any files') + parser.add_argument( + '--profile', dest='profile', + help='AWS configuration profile') parser.add_argument( 'path', help='the .s3_website.yaml configuration file or directory', default='.', nargs='?') @@ -248,14 +269,21 @@ def main(): conf, base_path = config.load_config_file(args.path) if args.force: - conf['force'] = True + conf['force'] = True + if args.profile: + conf['aws_profile'] = args.profile + + session = create_session(conf) + if not session: + logger.error('Error creating AWS session!') + return processed_keys, updated_keys = upload_to_bucket( - None, + session, os.path.join(base_path, conf['site']), conf, args.dry) # Invalidate files in cloudfront distribution if 'cloudfront_distribution_id' in conf: - invalidate_cloudfront(None, processed_keys, updated_keys, conf, args.dry) + invalidate_cloudfront(session, processed_keys, updated_keys, conf, args.dry) From 58a29e933715024762adf743981f873a06c9ff89 Mon Sep 17 00:00:00 2001 From: Andrej Burger Date: Sat, 10 Dec 2016 20:11:40 +0100 Subject: [PATCH 3/5] README formatting fix --- README.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index 465a13c..33f87fe 100644 --- a/README.rst +++ b/README.rst @@ -25,10 +25,10 @@ The configuration is stored in a YAML file like this: site: _site - aws_id: AKIAI6XLRPEQEU6T6ZXA - aws_secret: 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 - region: eu-central-1 - aws_profile: deploy + aws_id: AKIAI6XLRPEQEU6T6ZXA + aws_secret: Ua06LWWJRnAvUZuE4yRY/f6Cyyp9R2pF0PsQnwqd + region: eu-central-1 + aws_profile: deploy s3_bucket: example.com cloudfront_distribution_id: XXXXXXXXXXX @@ -102,9 +102,9 @@ Configuration file The region used to connect to the AWS. **aws_profile** - The AWS profile used to connect to the AWS. If ``aws_profile`` and also - either ``aws_id`` or ``aws_secret`` is specified, the specified access - key is used to connect. + The AWS profile used to connect to the AWS. If ``aws_profile`` and also + either ``aws_id`` or ``aws_secret`` is specified, the specified access + key is used to connect. **s3_bucket** The name of the S3 bucket to upload the files to. You have to allow the From bb14a57223b34f4f554070763c36a427cb7e4b89 Mon Sep 17 00:00:00 2001 From: Andrej Burger Date: Sat, 10 Dec 2016 20:32:51 +0100 Subject: [PATCH 4/5] deploy.py formatting fixed --- s3_deploy/deploy.py | 66 ++++++++++++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/s3_deploy/deploy.py b/s3_deploy/deploy.py index ff2075e..d26dd82 100644 --- a/s3_deploy/deploy.py +++ b/s3_deploy/deploy.py @@ -7,12 +7,10 @@ import shutil import mimetypes from datetime import datetime +from six import BytesIO import boto3 -from six import BytesIO -from six.moves.urllib.parse import quote_plus - from . import config from .prefixcovertree import PrefixCoverTree @@ -46,20 +44,37 @@ def dst(self, dt): mimetypes.init() + def create_session(conf): session = None if conf.get('aws_id', False) or conf.get('aws_secret', False): if conf.get('aws_id', False) and conf.get('aws_secret', False): - logger.info('Creating AWS session using AWS key id \"{}\" and region \"{}\"...'.format(conf['aws_id'], conf.get('region', ''))) - session = boto3.Session(conf['aws_id'], conf['aws_secret'], region_name=conf.get('region', None)) + logger.info('Creating AWS session using AWS key id \"{}\" ' + 'and region \"{}\"...' + .format(conf['aws_id'], + conf.get('region', ''))) + + session = boto3.Session(conf['aws_id'], + conf['aws_secret'], + region_name=conf.get('region', None)) else: - logger.error('Both AWS access key ID and AWS secret access key has to specified if either is specified.') + logger.error('Both AWS access key ID and AWS secret access key has' + 'to specified if either is specified.') + elif conf.get('aws_profile', False): - logger.info('Creating AWS session using AWS profile \"{}\"...'.format(conf['aws_profile'])) - session = boto3.Session(profile_name=conf['aws_profile']) + logger.info('Creating AWS session using AWS profile \"{}\" ' + 'and region \"{}\"...' + .format(conf['aws_profile'], + conf.get('region', ''))) + + session = boto3.Session(profile_name=conf['aws_profile'], + region_name=conf.get('region', None)) else: - logger.info('Creating AWS session using default profile...') - session = boto3.Session() + logger.info('Creating AWS session using default profile ' + 'and region \"{}\"...' + .format(conf.get('region', ''))) + + session = boto3.Session(region_name=conf.get('region', None)) return session @@ -185,18 +200,21 @@ def upload_to_bucket(session, site_dir, conf, dry): logger.info('Creating key {}...'.format(obj.key)) - upload_key( - obj, path, cache_rules, dry, storage_class=storage_class) + upload_key(obj, + path, + cache_rules, + dry, + storage_class=storage_class) updated_keys.add(key_name) logger.info('Bucket update done.') - + return processed_keys, updated_keys def invalidate_cloudfront(session, processed_keys, updated_keys, conf, dry): - logger.info('Connecting to Cloudfront distribution {}...'.format( - conf['cloudfront_distribution_id'])) + logger.info('Connecting to Cloudfront distribution {}...' + .format(conf['cloudfront_distribution_id'])) index_pattern = None if 'index_document' in conf: @@ -231,12 +249,12 @@ def path_from_key_name(key_name): response = cloudfront.create_invalidation( DistributionId=dist_id, InvalidationBatch=dict( - Paths=dict( - Quantity=len(paths), - Items=paths - ), - CallerReference='s3-deploy-website' - ) + Paths=dict( + Quantity=len(paths), + Items=paths + ), + CallerReference='s3-deploy-website' + ) ) invalidation = response['Invalidation'] logger.info('Invalidation request {} is {}'.format( @@ -286,4 +304,8 @@ def main(): # Invalidate files in cloudfront distribution if 'cloudfront_distribution_id' in conf: - invalidate_cloudfront(session, processed_keys, updated_keys, conf, args.dry) + invalidate_cloudfront(session, + processed_keys, + updated_keys, + conf, + args.dry) From 285b469120c76247e53122dbf96309cde4ad9341 Mon Sep 17 00:00:00 2001 From: Andrej Burger Date: Sat, 10 Dec 2016 20:33:18 +0100 Subject: [PATCH 5/5] AWS key removed from README --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 33f87fe..9a925ef 100644 --- a/README.rst +++ b/README.rst @@ -25,8 +25,8 @@ The configuration is stored in a YAML file like this: site: _site - aws_id: AKIAI6XLRPEQEU6T6ZXA - aws_secret: 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 + aws_id: XXXXXXXXXXX + aws_secret: XXXXXXXXXXX region: eu-central-1 aws_profile: deploy