Rewrite backup tarballs to support S3

This commit will do following things:
1. pbench-backup-tarballs.py: Add a new python script to
backup result to s3

This script will be invoked by the pbench-backup-tarballs.
After invoking it will check the TO-BACKUP state directory
for new results and figure out which are the results that
need to be backed up to s3.

2. pbench-verify-backup-tarballs.py: Add a new python script
for results verification

This script will be invoked by the pbench-verify-backup-tarballs.
The task will be verifying the results in archive, backup and
s3 bucket.

3. Add new configuration section for s3

4. Add new module for handling s3 configuration information

5. Fix Unit tests and dependencies

Author: tenstormavi

.travis.yml

@@ -20,7 +20,7 @@ install:
   - sudo apt-get install python3.6 --force-yes -y
   - sudo wget -O /tmp/get-pip.py https://bootstrap.pypa.io/get-pip.py
   - sudo python3.6 /tmp/get-pip.py
-  - sudo pip install 'configtools<0.4.0' elasticsearch
+  - sudo pip install 'configtools<0.4.0' elasticsearch sh boto3
   - sudo ln -sf python3.6 /usr/bin/python3
   - sudo apt-key adv --fetch-keys http://dl.yarnpkg.com/debian/pubkey.gpg
   - sudo echo "deb http://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list

server/bin/gold/test-1.txt

@@ -28,9 +28,8 @@ pbench-clean-up-dangling-results-links: Bad ARCHIVE=/var/tmp/pbench-test-server/
 pbench-edit-prefixes: Bad ARCHIVE=/var/tmp/pbench-test-server/test-1/pbench/archive/fs-version-001
 --- Finished pbench-edit-prefixes (status=1)
 +++ Running pbench-backup-tarballs
---- Finished pbench-backup-tarballs (status=1)
+--- Finished pbench-backup-tarballs (status=3)
 +++ Running pbench-verify-backup-tarballs
-pbench-verify-backup-tarballs: Bad archive: /var/tmp/pbench-test-server/test-1/pbench/archive/fs-version-001
 --- Finished pbench-verify-backup-tarballs (status=1)
 +++ Running pbench-satellite-cleanup
 pbench-satellite-cleanup: Bad ARCHIVE=/var/tmp/pbench-test-server/test-1/pbench-satellite/archive/fs-version-001
@@ -105,10 +104,11 @@ drwxrwxr-x          - public_html/users
 +++ pbench-local tree state (/var/tmp/pbench-test-server/test-1/pbench-local)
 drwxrwxr-x          - logs
 drwxrwxr-x          - logs/pbench-backup-tarballs
--rw-rw-r--        148 logs/pbench-backup-tarballs/pbench-backup-tarballs.error
--rw-rw-r--         30 logs/pbench-backup-tarballs/pbench-backup-tarballs.log
+-rw-rw-r--        280 logs/pbench-backup-tarballs/pbench-backup-tarballs.log
 drwxrwxr-x          - logs/pbench-index
 -rw-rw-r--        158 logs/pbench-index/pbench-index.log
+drwxrwxr-x          - logs/pbench-verify-backup-tarballs
+-rw-rw-r--        241 logs/pbench-verify-backup-tarballs/pbench-verify-backup-tarballs.log
 drwxrwxr-x          - pbench-move-results-receive
 drwxrwxr-x          - pbench-move-results-receive/fs-version-001
 drwxrwxr-x          - pbench-move-results-receive/fs-version-002
@@ -163,15 +163,15 @@ drwxrwxr-x          - tmp
 --- pbench-satellite-local tree state
 +++ pbench log file contents
 ++++ pbench-local/logs
-+++++ pbench-backup-tarballs/pbench-backup-tarballs.error
-pbench-backup-tarballs: The ARCHIVE directory does not resolve to a real location, /var/tmp/pbench-test-server/test-1/pbench/archive/fs-version-001
------ pbench-backup-tarballs/pbench-backup-tarballs.error
 +++++ pbench-backup-tarballs/pbench-backup-tarballs.log
-start-1900-01-01T00:00:00-UTC
+1970-01-01T00:00:00.000000 ERROR pbench-backup-tarballs.pbench-backup-tarballs sanity_check -- The ARCHIVE directory /var/tmp/pbench-test-server/test-1/pbench/archive/fs-version-001, does not resolve /var/tmp/pbench-test-server/test-1/pbench/archive/fs-version-001 to a directory
 ----- pbench-backup-tarballs/pbench-backup-tarballs.log
 +++++ pbench-index/pbench-index.log
 1970-01-01T00:00:00.000000 ERROR pbench-index.pbench-index main -- pbench-index: Bad ARCHIVE=/var/tmp/pbench-test-server/test-1/pbench/archive/fs-version-001
 ----- pbench-index/pbench-index.log
++++++ pbench-verify-backup-tarballs/pbench-verify-backup-tarballs.log
+1970-01-01T00:00:00.000000 ERROR pbench-verify-backup-tarballs.pbench-verify-backup-tarballs main -- The setting for ARCHIVE in the config file is /var/tmp/pbench-test-server/test-1/pbench/archive/fs-version-001, but that is not a directory
+----- pbench-verify-backup-tarballs/pbench-verify-backup-tarballs.log
 ---- pbench-local/logs
 ++++ pbench-satellite-local/logs
 ---- pbench-satellite-local/logs

server/bin/gold/test-11.txt

@@ -61,6 +61,7 @@ lrwxrwxrwx         70 users -> /var/tmp/pbench-test-server/test-11/pbench-satell
 drwxrwxr-x          - archive
 drwxrwxr-x          - archive/fs-version-001
 drwxrwxr-x          - archive/fs-version-001/ONE::controller
+drwxrwxr-x          - archive/fs-version-001/ONE::controller/BACKED-UP
 drwxrwxr-x          - archive/fs-version-001/ONE::controller/BAD-MD5
 drwxrwxr-x          - archive/fs-version-001/ONE::controller/COPIED-SOS
 drwxrwxr-x          - archive/fs-version-001/ONE::controller/INDEXED
@@ -72,6 +73,7 @@ drwxrwxr-x          - archive/fs-version-001/ONE::controller/SATELLITE-MD5-PASSE
 lrwxrwxrwx        113 archive/fs-version-001/ONE::controller/SATELLITE-MD5-PASSED/fio__2016-08-16_22:03:11.tar.xz -> /var/tmp/pbench-test-server/test-11/pbench/archive/fs-version-001/ONE::controller/fio__2016-08-16_22:03:11.tar.xz
 drwxrwxr-x          - archive/fs-version-001/ONE::controller/SYNCED
 drwxrwxr-x          - archive/fs-version-001/ONE::controller/TO-BACKUP
+drwxrwxr-x          - archive/fs-version-001/ONE::controller/TO-BACKUP-LARGE
 drwxrwxr-x          - archive/fs-version-001/ONE::controller/TO-COPY-SOS
 drwxrwxr-x          - archive/fs-version-001/ONE::controller/TO-DELETE
 drwxrwxr-x          - archive/fs-version-001/ONE::controller/TO-INDEX

server/bin/gold/test-16.txt

@@ -78,6 +78,7 @@ lrwxrwxrwx         70 users -> /var/tmp/pbench-test-server/test-16/pbench-satell
 drwxrwxr-x          - archive
 drwxrwxr-x          - archive/fs-version-001
 drwxrwxr-x          - archive/fs-version-001/controller00
+drwxrwxr-x          - archive/fs-version-001/controller00/BACKED-UP
 drwxrwxr-x          - archive/fs-version-001/controller00/BAD-MD5
 drwxrwxr-x          - archive/fs-version-001/controller00/COPIED-SOS
 drwxrwxr-x          - archive/fs-version-001/controller00/INDEXED
@@ -88,6 +89,7 @@ drwxrwxr-x          - archive/fs-version-001/controller00/SATELLITE-MD5-FAILED
 drwxrwxr-x          - archive/fs-version-001/controller00/SATELLITE-MD5-PASSED
 drwxrwxr-x          - archive/fs-version-001/controller00/SYNCED
 drwxrwxr-x          - archive/fs-version-001/controller00/TO-BACKUP
+drwxrwxr-x          - archive/fs-version-001/controller00/TO-BACKUP-LARGE
 drwxrwxr-x          - archive/fs-version-001/controller00/TO-COPY-SOS
 drwxrwxr-x          - archive/fs-version-001/controller00/TO-DELETE
 drwxrwxr-x          - archive/fs-version-001/controller00/TO-INDEX
@@ -101,6 +103,7 @@ drwxrwxr-x          - archive/fs-version-001/controller00/WONT-INDEX
 -rw-rw-r--       5920 archive/fs-version-001/controller00/benchmark-result-large_1900-01-01T00:00:00.tar.xz
 -rw-rw-r--         84 archive/fs-version-001/controller00/benchmark-result-large_1900-01-01T00:00:00.tar.xz.md5
 drwxrwxr-x          - archive/fs-version-001/controller01
+drwxrwxr-x          - archive/fs-version-001/controller01/BACKED-UP
 drwxrwxr-x          - archive/fs-version-001/controller01/BAD-MD5
 drwxrwxr-x          - archive/fs-version-001/controller01/COPIED-SOS
 -rw-rw-r--       3120 archive/fs-version-001/controller01/DUPLICATE__NAME.1.benchmark-result-medium_1900-01-01T00:00:00.tar.xz
@@ -113,6 +116,7 @@ drwxrwxr-x          - archive/fs-version-001/controller01/SATELLITE-MD5-FAILED
 drwxrwxr-x          - archive/fs-version-001/controller01/SATELLITE-MD5-PASSED
 drwxrwxr-x          - archive/fs-version-001/controller01/SYNCED
 drwxrwxr-x          - archive/fs-version-001/controller01/TO-BACKUP
+drwxrwxr-x          - archive/fs-version-001/controller01/TO-BACKUP-LARGE
 drwxrwxr-x          - archive/fs-version-001/controller01/TO-COPY-SOS
 drwxrwxr-x          - archive/fs-version-001/controller01/TO-DELETE
 drwxrwxr-x          - archive/fs-version-001/controller01/TO-INDEX
@@ -128,6 +132,7 @@ drwxrwxr-x          - archive/fs-version-001/controller01/WONT-INDEX
 drwxrwxr-x          - archive/fs-version-001/controller02
 drwxrwxr-x          - archive/fs-version-001/controller02/.prefix
 -rw-rw-r--          9 archive/fs-version-001/controller02/.prefix/benchmark-result-small_1900-01-01T00:00:00.prefix
+drwxrwxr-x          - archive/fs-version-001/controller02/BACKED-UP
 drwxrwxr-x          - archive/fs-version-001/controller02/BAD-MD5
 drwxrwxr-x          - archive/fs-version-001/controller02/COPIED-SOS
 drwxrwxr-x          - archive/fs-version-001/controller02/INDEXED
@@ -138,6 +143,7 @@ drwxrwxr-x          - archive/fs-version-001/controller02/SATELLITE-MD5-FAILED
 drwxrwxr-x          - archive/fs-version-001/controller02/SATELLITE-MD5-PASSED
 drwxrwxr-x          - archive/fs-version-001/controller02/SYNCED
 drwxrwxr-x          - archive/fs-version-001/controller02/TO-BACKUP
+drwxrwxr-x          - archive/fs-version-001/controller02/TO-BACKUP-LARGE
 drwxrwxr-x          - archive/fs-version-001/controller02/TO-COPY-SOS
 drwxrwxr-x          - archive/fs-version-001/controller02/TO-DELETE
 drwxrwxr-x          - archive/fs-version-001/controller02/TO-INDEX

server/bin/gold/test-17.txt

@@ -78,6 +78,7 @@ lrwxrwxrwx         70 users -> /var/tmp/pbench-test-server/test-17/pbench-satell
 drwxrwxr-x          - archive
 drwxrwxr-x          - archive/fs-version-001
 drwxrwxr-x          - archive/fs-version-001/controller00
+drwxrwxr-x          - archive/fs-version-001/controller00/BACKED-UP
 drwxrwxr-x          - archive/fs-version-001/controller00/BAD-MD5
 drwxrwxr-x          - archive/fs-version-001/controller00/COPIED-SOS
 drwxrwxr-x          - archive/fs-version-001/controller00/INDEXED
@@ -87,6 +88,7 @@ drwxrwxr-x          - archive/fs-version-001/controller00/SATELLITE-MD5-FAILED
 drwxrwxr-x          - archive/fs-version-001/controller00/SATELLITE-MD5-PASSED
 drwxrwxr-x          - archive/fs-version-001/controller00/SYNCED
 drwxrwxr-x          - archive/fs-version-001/controller00/TO-BACKUP
+drwxrwxr-x          - archive/fs-version-001/controller00/TO-BACKUP-LARGE
 drwxrwxr-x          - archive/fs-version-001/controller00/TO-COPY-SOS
 drwxrwxr-x          - archive/fs-version-001/controller00/TO-DELETE
 drwxrwxr-x          - archive/fs-version-001/controller00/TO-INDEX
@@ -101,6 +103,7 @@ drwxrwxr-x          - archive/fs-version-001/controller00/WONT-INDEX
 -rw-rw-r--       5920 archive/fs-version-001/controller00/benchmark-result-large_1900-01-01T00:00:00.tar.xz
 -rw-rw-r--         84 archive/fs-version-001/controller00/benchmark-result-large_1900-01-01T00:00:00.tar.xz.md5
 drwxrwxr-x          - archive/fs-version-001/controller01
+drwxrwxr-x          - archive/fs-version-001/controller01/BACKED-UP
 drwxrwxr-x          - archive/fs-version-001/controller01/BAD-MD5
 drwxrwxr-x          - archive/fs-version-001/controller01/COPIED-SOS
 -rw-rw-r--       3120 archive/fs-version-001/controller01/DUPLICATE__NAME.1.benchmark-result-medium_1900-01-01T00:00:00.tar.xz
@@ -112,6 +115,7 @@ drwxrwxr-x          - archive/fs-version-001/controller01/SATELLITE-MD5-FAILED
 drwxrwxr-x          - archive/fs-version-001/controller01/SATELLITE-MD5-PASSED
 drwxrwxr-x          - archive/fs-version-001/controller01/SYNCED
 drwxrwxr-x          - archive/fs-version-001/controller01/TO-BACKUP
+drwxrwxr-x          - archive/fs-version-001/controller01/TO-BACKUP-LARGE
 drwxrwxr-x          - archive/fs-version-001/controller01/TO-COPY-SOS
 drwxrwxr-x          - archive/fs-version-001/controller01/TO-DELETE
 drwxrwxr-x          - archive/fs-version-001/controller01/TO-INDEX
@@ -128,6 +132,7 @@ drwxrwxr-x          - archive/fs-version-001/controller01/WONT-INDEX
 drwxrwxr-x          - archive/fs-version-001/controller02
 drwxrwxr-x          - archive/fs-version-001/controller02/.prefix
 -rw-rw-r--          9 archive/fs-version-001/controller02/.prefix/benchmark-result-small_1900-01-01T00:00:00.prefix
+drwxrwxr-x          - archive/fs-version-001/controller02/BACKED-UP
 drwxrwxr-x          - archive/fs-version-001/controller02/BAD-MD5
 drwxrwxr-x          - archive/fs-version-001/controller02/COPIED-SOS
 drwxrwxr-x          - archive/fs-version-001/controller02/INDEXED
@@ -137,6 +142,7 @@ drwxrwxr-x          - archive/fs-version-001/controller02/SATELLITE-MD5-FAILED
 drwxrwxr-x          - archive/fs-version-001/controller02/SATELLITE-MD5-PASSED
 drwxrwxr-x          - archive/fs-version-001/controller02/SYNCED
 drwxrwxr-x          - archive/fs-version-001/controller02/TO-BACKUP
+drwxrwxr-x          - archive/fs-version-001/controller02/TO-BACKUP-LARGE
 drwxrwxr-x          - archive/fs-version-001/controller02/TO-COPY-SOS
 drwxrwxr-x          - archive/fs-version-001/controller02/TO-DELETE
 drwxrwxr-x          - archive/fs-version-001/controller02/TO-INDEX

server/bin/gold/test-19.txt

@@ -78,6 +78,7 @@ lrwxrwxrwx         70 users -> /var/tmp/pbench-test-server/test-19/pbench-satell
 drwxrwxr-x          - archive
 drwxrwxr-x          - archive/fs-version-001
 drwxrwxr-x          - archive/fs-version-001/controller00
+drwxrwxr-x          - archive/fs-version-001/controller00/BACKED-UP
 drwxrwxr-x          - archive/fs-version-001/controller00/BAD-MD5
 drwxrwxr-x          - archive/fs-version-001/controller00/COPIED-SOS
 drwxrwxr-x          - archive/fs-version-001/controller00/INDEXED
@@ -88,6 +89,7 @@ drwxrwxr-x          - archive/fs-version-001/controller00/SATELLITE-MD5-FAILED
 drwxrwxr-x          - archive/fs-version-001/controller00/SATELLITE-MD5-PASSED
 drwxrwxr-x          - archive/fs-version-001/controller00/SYNCED
 drwxrwxr-x          - archive/fs-version-001/controller00/TO-BACKUP
+drwxrwxr-x          - archive/fs-version-001/controller00/TO-BACKUP-LARGE
 drwxrwxr-x          - archive/fs-version-001/controller00/TO-COPY-SOS
 drwxrwxr-x          - archive/fs-version-001/controller00/TO-DELETE
 drwxrwxr-x          - archive/fs-version-001/controller00/TO-INDEX
@@ -100,6 +102,7 @@ drwxrwxr-x          - archive/fs-version-001/controller00/WONT-INDEX
 -rw-rw-r--       5920 archive/fs-version-001/controller00/benchmark-result-large_1900-01-01T00:00:00.tar.xz
 -rw-rw-r--         84 archive/fs-version-001/controller00/benchmark-result-large_1900-01-01T00:00:00.tar.xz.md5
 drwxrwxr-x          - archive/fs-version-001/controller01
+drwxrwxr-x          - archive/fs-version-001/controller01/BACKED-UP
 drwxrwxr-x          - archive/fs-version-001/controller01/BAD-MD5
 drwxrwxr-x          - archive/fs-version-001/controller01/COPIED-SOS
 -rw-rw-r--       3120 archive/fs-version-001/controller01/DUPLICATE__NAME.1.benchmark-result-medium_1900-01-01T00:00:00.tar.xz
@@ -112,6 +115,7 @@ drwxrwxr-x          - archive/fs-version-001/controller01/SATELLITE-MD5-FAILED
 drwxrwxr-x          - archive/fs-version-001/controller01/SATELLITE-MD5-PASSED
 drwxrwxr-x          - archive/fs-version-001/controller01/SYNCED
 drwxrwxr-x          - archive/fs-version-001/controller01/TO-BACKUP
+drwxrwxr-x          - archive/fs-version-001/controller01/TO-BACKUP-LARGE
 drwxrwxr-x          - archive/fs-version-001/controller01/TO-COPY-SOS
 drwxrwxr-x          - archive/fs-version-001/controller01/TO-DELETE
 drwxrwxr-x          - archive/fs-version-001/controller01/TO-INDEX
@@ -127,6 +131,7 @@ drwxrwxr-x          - archive/fs-version-001/controller01/WONT-INDEX
 drwxrwxr-x          - archive/fs-version-001/controller02
 drwxrwxr-x          - archive/fs-version-001/controller02/.prefix
 -rw-rw-r--          9 archive/fs-version-001/controller02/.prefix/benchmark-result-small_1900-01-01T00:00:00.prefix
+drwxrwxr-x          - archive/fs-version-001/controller02/BACKED-UP
 drwxrwxr-x          - archive/fs-version-001/controller02/BAD-MD5
 drwxrwxr-x          - archive/fs-version-001/controller02/COPIED-SOS
 drwxrwxr-x          - archive/fs-version-001/controller02/INDEXED
@@ -136,6 +141,7 @@ drwxrwxr-x          - archive/fs-version-001/controller02/SATELLITE-MD5-FAILED
 drwxrwxr-x          - archive/fs-version-001/controller02/SATELLITE-MD5-PASSED
 drwxrwxr-x          - archive/fs-version-001/controller02/SYNCED
 drwxrwxr-x          - archive/fs-version-001/controller02/TO-BACKUP
+drwxrwxr-x          - archive/fs-version-001/controller02/TO-BACKUP-LARGE
 drwxrwxr-x          - archive/fs-version-001/controller02/TO-COPY-SOS
 drwxrwxr-x          - archive/fs-version-001/controller02/TO-DELETE
 drwxrwxr-x          - archive/fs-version-001/controller02/TO-INDEX