From 1cbfa930a08eb74e00b9555c4f21f0da8252041c Mon Sep 17 00:00:00 2001 From: Graham Gilbert Date: Thu, 9 May 2019 16:02:38 -0700 Subject: [PATCH 1/5] Support for s3 --- git-fat | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/git-fat b/git-fat index 135f4e2..68fa3b9 100755 --- a/git-fat +++ b/git-fat @@ -158,7 +158,40 @@ class GitFat(object): if remote is None: raise RuntimeError('No rsync.remote in %s' % cfgpath) return remote, ssh_port, ssh_user, options + + def get_aws_cmd(self, push, s3_bucket): + if not os.path.exists('/usr/local/bin/aws'): + sys.stderr.write('Could not find aws cli install. Please pip install awscli and tghen aws configure.\n') + sys.exit(1) + + if not s3_bucket.startswith('s3://'): + s3_bucket = "s3://{}".format(s3_bucket) + + if push: + cmd = [ + "aws", + "s3", + "sync", + self.objdir + "/", + s3_bucket + "/" + ] + else: + cmd = [ + "aws", + "s3", + "sync", + s3_bucket + "/", + self.objdir + "/" + ] + return cmd + + + def get_rsync_command(self,push): + cfgpath = os.path.join(self.gitroot,'.gitfat') + s3_bucket = gitconfig_get('s3.bucket', file=cfgpath) + if s3_bucket: + return self.get_aws_cmd(push, s3_bucket) (remote, ssh_port, ssh_user, options) = self.get_rsync() if push: self.verbose('Pushing to %s' % (remote)) From ab331196e11491d3632287573ef7195269aee2b0 Mon Sep 17 00:00:00 2001 From: Graham Gilbert Date: Thu, 9 May 2019 16:43:57 -0700 Subject: [PATCH 2/5] Better way to find aws binary --- git-fat | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/git-fat b/git-fat index 68fa3b9..296b7d4 100755 --- a/git-fat +++ b/git-fat @@ -46,6 +46,23 @@ except ImportError: BLOCK_SIZE = 4096 +def which(program): + def is_exe(fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + + fpath, fname = os.path.split(program) + if fpath: + if is_exe(program): + return program + else: + for path in os.environ["PATH"].split(os.pathsep): + exe_file = os.path.join(path, program) + if is_exe(exe_file): + return exe_file + + return None + + def verbose_stderr(*args, **kwargs): return print(*args, file=sys.stderr, **kwargs) def verbose_ignore(*args, **kwargs): @@ -160,8 +177,8 @@ class GitFat(object): return remote, ssh_port, ssh_user, options def get_aws_cmd(self, push, s3_bucket): - if not os.path.exists('/usr/local/bin/aws'): - sys.stderr.write('Could not find aws cli install. Please pip install awscli and tghen aws configure.\n') + if not which('aws'): + sys.stderr.write('Could not find aws cli install.\n') sys.exit(1) if not s3_bucket.startswith('s3://'): @@ -185,8 +202,6 @@ class GitFat(object): ] return cmd - - def get_rsync_command(self,push): cfgpath = os.path.join(self.gitroot,'.gitfat') s3_bucket = gitconfig_get('s3.bucket', file=cfgpath) From a0982dd07a1fcbe6459509ca5b57a0284ed91ca3 Mon Sep 17 00:00:00 2001 From: Graham Gilbert Date: Thu, 9 May 2019 16:51:54 -0700 Subject: [PATCH 3/5] readme --- README.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 1db7de8..9b95c83 100644 --- a/README.md +++ b/README.md @@ -39,18 +39,23 @@ Set a remote store for the fat objects by editing `.gitfat`. This file should typically be committed to the repository so that others will automatically have their remote set. This remote address can use -any protocol supported by rsync. +any protocol supported by rsync. -Most users will configure it to use remote ssh in a directory with shared -access. To do this, set the `sshuser` and `sshport` variables in `.gitfat` +Most users will configure it to use remote ssh in a directory with shared +access. To do this, set the `sshuser` and `sshport` variables in `.gitfat` configuration file. For example, to use rsync with ssh, with the default -port (22) and authenticate with the user "_fat_", your configuration would -look like this: +port (22) and authenticate with the user "_fat_", your configuration would +look like this: [rsync] remote = your.remote-host.org:/share/fat-store sshuser = fat +To use an Amazon S3 bucket as the backend, you should first install the AWS CLI and configure it with a user that has access to the bucket. Your configuration would then look like: + + [s3] + bucket = se://your-s3-bucket + # A worked example Before we start, let's turn on verbose reporting so we can see what's @@ -145,7 +150,7 @@ selected history. 1 file to consider 1f218834a137f7b185b498924e7a030008aee2ae 6449 100% 6.15MB/s 0:00:00 (xfer#1, to-check=0/1) - + sent 30 bytes received 6558 bytes 4392.00 bytes/sec total size is 6449 speedup is 0.98 Restoring 1f218834a137f7b185b498924e7a030008aee2ae -> master.tar.gz From 3b08269099bceee8c0d5b9cbc36f5eba92481f04 Mon Sep 17 00:00:00 2001 From: Graham Gilbert Date: Thu, 9 May 2019 20:28:59 -0700 Subject: [PATCH 4/5] a little more debug output --- git-fat | 2 ++ 1 file changed, 2 insertions(+) diff --git a/git-fat b/git-fat index 296b7d4..0c08dcb 100755 --- a/git-fat +++ b/git-fat @@ -185,6 +185,7 @@ class GitFat(object): s3_bucket = "s3://{}".format(s3_bucket) if push: + self.verbose('Pushing to %s' % (s3_bucket)) cmd = [ "aws", "s3", @@ -193,6 +194,7 @@ class GitFat(object): s3_bucket + "/" ] else: + self.verbose('Pulling from %s' % (s3_bucket)) cmd = [ "aws", "s3", From 977a329d75b797bf1a16f81b446fba233fed3692 Mon Sep 17 00:00:00 2001 From: Graham Gilbert Date: Fri, 10 May 2019 16:58:42 -0700 Subject: [PATCH 5/5] Fix typo in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9b95c83..914035f 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ look like this: To use an Amazon S3 bucket as the backend, you should first install the AWS CLI and configure it with a user that has access to the bucket. Your configuration would then look like: [s3] - bucket = se://your-s3-bucket + bucket = s3://your-s3-bucket # A worked example