feat: enable download to not run in parallel

Some provider fails to comply with s3 parallel downloads.
Add an option to disable it

http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/transfer/TransferManager.html#download(com.amazonaws.services.s3.model.GetObjectRequest,%20java.io.File,%20com.amazonaws.services.s3.transfer.internal.S3ProgressListener)

Author: jfroche

src/main/java/de/taimos/pipeline/aws/S3DownloadStep.java

@@ -52,12 +52,14 @@ public class S3DownloadStep extends AbstractS3Step {
 	private final String bucket;
 	private String path = "";
 	private boolean force = false;
+	private boolean disableParallelDownloads = false;
 
 	@DataBoundConstructor
-	public S3DownloadStep(String file, String bucket, boolean pathStyleAccessEnabled, boolean payloadSigningEnabled) {
+	public S3DownloadStep(String file, String bucket, boolean pathStyleAccessEnabled, boolean payloadSigningEnabled, boolean disableParallelDownloads) {
 		super(pathStyleAccessEnabled, payloadSigningEnabled);
 		this.file = file;
 		this.bucket = bucket;
+		this.disableParallelDownloads = disableParallelDownloads;
 	}
 
 	public String getFile() {
@@ -76,6 +78,10 @@ public boolean isForce() {
 		return this.force;
 	}
 
+	public boolean isDisableParallelDownloads() {
+		return this.disableParallelDownloads;
+	}
+
 	@DataBoundSetter
 	public void setForce(boolean force) {
 		this.force = force;
@@ -86,6 +92,11 @@ public void setPath(String path) {
 		this.path = path;
 	}
 
+	@DataBoundSetter
+	public void setDisableParallelDownload(boolean disableParallelDownloads) {
+		this.disableParallelDownloads = disableParallelDownloads;
+	}
+
 	@Override
 	public StepExecution start(StepContext context) throws Exception {
 		return new S3DownloadStep.Execution(this, context);
@@ -130,6 +141,7 @@ public Void run() throws Exception {
 			final String bucket = this.step.getBucket();
 			final String path = this.step.getPath();
 			final boolean force = this.step.isForce();
+			final boolean disableParallelDownloads = this.step.isDisableParallelDownloads();
 
 			Preconditions.checkArgument(bucket != null && !bucket.isEmpty(), "Bucket must not be null or empty");
 
@@ -146,7 +158,7 @@ public Void run() throws Exception {
 					throw new RuntimeException("Target exists: " + target.toURI().toString());
 				}
 			}
-			target.act(new RemoteDownloader(Execution.this.step.createS3ClientOptions(), envVars, listener, bucket, path));
+			target.act(new RemoteDownloader(Execution.this.step.createS3ClientOptions(), envVars, listener, bucket, path, disableParallelDownloads));
 			listener.getLogger().println("Download complete");
 			return null;
 		}
@@ -162,19 +174,22 @@ private static class RemoteDownloader extends MasterToSlaveFileCallable<Void> {
 		private final TaskListener taskListener;
 		private final String bucket;
 		private final String path;
+		private final Boolean disableParallelDownloads;
 
-		RemoteDownloader(S3ClientOptions amazonS3ClientOptions, EnvVars envVars, TaskListener taskListener, String bucket, String path) {
+		RemoteDownloader(S3ClientOptions amazonS3ClientOptions, EnvVars envVars, TaskListener taskListener, String bucket, String path, Boolean disableParallelDownloads) {
 			this.amazonS3ClientOptions = amazonS3ClientOptions;
 			this.envVars = envVars;
 			this.taskListener = taskListener;
 			this.bucket = bucket;
 			this.path = path;
+			this.disableParallelDownloads = disableParallelDownloads;
 		}
 
 		@Override
 		public Void invoke(File localFile, VirtualChannel channel) throws IOException, InterruptedException {
 			TransferManager mgr = TransferManagerBuilder.standard()
 					.withS3Client(AWSClientFactory.create(this.amazonS3ClientOptions.createAmazonS3ClientBuilder(), this.envVars))
+					.withDisableParallelDownloads(this.disableParallelDownloads)
 					.build();
 
 			if (this.path == null || this.path.isEmpty() || this.path.endsWith("/")) {

src/main/resources/de/taimos/pipeline/aws/S3DownloadStep/config.jelly

@@ -18,4 +18,7 @@
 	<f:entry title="${%Enable Payload Signing}" field="payloadSigningEnabled">
 		<f:checkbox />
 	</f:entry>
+	<f:entry title="${%Disable parallel downloads}" field="disableParallelDownloads">
+		<f:checkbox />
+	</f:entry>
 </j:jelly>

src/main/resources/de/taimos/pipeline/aws/S3DownloadStep/help-disableParallelDownloads.html

@@ -0,0 +1,22 @@
+<!--
+  #%L
+  Pipeline: AWS Steps
+  %%
+  Copyright (C) 2016 - 2017 Taimos GmbH
+  %%
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+  
+       http://www.apache.org/licenses/LICENSE-2.0
+  
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  #L%
+  -->
+<div>
+	Disable parallel download (for system that don't support parallel download).
+</div>

src/test/java/de/taimos/pipeline/aws/S3DownloadStepTest.java

@@ -27,20 +27,20 @@
 public class S3DownloadStepTest {
 	@Test
 	public void gettersWorkAsExpected() throws Exception {
-		S3DownloadStep step = new S3DownloadStep("my-file", "my-bucket", false, false);
+		S3DownloadStep step = new S3DownloadStep("my-file", "my-bucket", false, false, false);
 		Assert.assertEquals("my-file", step.getFile());
 		Assert.assertEquals("my-bucket", step.getBucket());
 	}
 
 	@Test
 	public void defaultPathIsEmpty() throws Exception {
-		S3DownloadStep step = new S3DownloadStep("my-file", "my-bucket", false, false);
+		S3DownloadStep step = new S3DownloadStep("my-file", "my-bucket", false, false, false);
 		Assert.assertEquals("", step.getPath());
 	}
 
 	@Test
 	public void defaultForceIsFalse() throws Exception {
-		S3DownloadStep step = new S3DownloadStep("my-file", "my-bucket", false, false);
+		S3DownloadStep step = new S3DownloadStep("my-file", "my-bucket", false, false, false);
 		Assert.assertFalse(step.isForce());
 	}
 }