Skip to content

Commit 632db70

Browse files
joshuaspenceph
authored andcommitted
Fix AWS credentials
Fixes #110. This plugin is not currently passing the correct credentials to the AWS SDK. Instead of trying to pass in `:credentials` explicitly, we should just trust that `LogStash::PluginMixins::AwsConfig::V2#aws_options_hash` does what it is expected. Currently, the plugin is failing in one of two different ways, when using instance profile credentials. If `validate_credentials_on_root_bucket => true`, the following error is thrown during plugin registration: ``` {"level":"ERROR","loggerName":"logstash.agent","timeMillis":1483509033892,"thread":"[main]-pipeline-manager","logEvent":{"message":"Pipeline aborted due to error","exception":{"metaClass":{"metaClass":{"metaClass":{"exception":"Logstash must have the privileges to write to root bucket `REDACTED`, check you credentials or your permissions.","backtrace":["/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-s3-4.0.1/lib/logstash/outputs/s3.rb:187:in `register'","/usr/share/logstash/logstash-core/lib/logstash/output_delegator_strategies/shared.rb:8:in `register'","/usr/share/logstash/logstash-core/lib/logstash/output_delegator.rb:37:in `register'","/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:229:in `start_workers'","org/jruby/RubyArray.java:1613:in `each'","/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:229:in `start_workers'","/usr/share/logstash/logstash-core/lib/logstash/pipeline.rb:183:in `run'","/usr/share/logstash/logstash-core/lib/logstash/agent.rb:292:in `start_pipeline'"]}}}}}} ``` If `validate_credentials_on_root_bucket => false`, the following error is thrown from the uploader threads: ``` {"level":"ERROR","loggerName":"logstash.outputs.s3","timeMillis":1483506824991,"thread":"S3 output uploader, file: /tmp/logstash/b1fe230c-3df6-40ce-8293-6cc8133ba31c/ap-southeast-2/dt=2017-01-04/ls.s3.f4fb1158-e180-4fec-b355-bafc6f388563.2017-01-03T23.22.part3.txt","logEvent":{"message":"Uploading failed, retrying","exception":{"metaClass":{"metaClass":{"metaClass":{"exception":"unable to sign request without credentials set","path":"/tmp/logstash/b1fe230c-3df6-40ce-8293-6cc8133ba31c/ap-southeast-2/dt=2017-01-04/ls.s3.f4fb1158-e180-4fec-b355-bafc6f388563.2017-01-03T23.22.part3.txt","backtrace":["/usr/share/logstash/vendor/bundle/jruby/1.9/gems/aws-sdk-core-2.3.22/lib/aws-sdk-core/plugins/request_signer.rb:100:in `require_credentials'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/aws-sdk-core-2.3.22/lib/aws-sdk-core/plugins/s3_request_signer.rb:14:in `call'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/aws-sdk-core-2.3.22/lib/aws-sdk-core/xml/error_handler.rb:8:in `call'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/aws-sdk-core-2.3.22/lib/aws-sdk-core/plugins/s3_request_signer.rb:65:in `call'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/aws-sdk-core-2.3.22/lib/aws-sdk-core/plugins/s3_redirects.rb:15:in `call'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/aws-sdk-core-2.3.22/lib/aws-sdk-core/plugins/retry_errors.rb:87:in `call'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/aws-sdk-core-2.3.22/lib/aws-sdk-core/plugins/s3_accelerate.rb:42:in `call'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/aws-sdk-core-2.3.22/lib/aws-sdk-core/plugins/s3_md5s.rb:31:in `call'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/aws-sdk-core-2.3.22/lib/aws-sdk-core/plugins/s3_expect_100_continue.rb:21:in `call'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/aws-sdk-core-2.3.22/lib/aws-sdk-core/plugins/s3_bucket_name_restrictions.rb:12:in `call'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/aws-sdk-core-2.3.22/lib/aws-sdk-core/plugins/s3_bucket_dns.rb:31:in `call'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/aws-sdk-core-2.3.22/lib/aws-sdk-core/rest/handler.rb:7:in `call'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/aws-sdk-core-2.3.22/lib/aws-sdk-core/plugins/user_agent.rb:12:in `call'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/aws-sdk-core-2.3.22/lib/seahorse/client/plugins/endpoint.rb:41:in `call'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/aws-sdk-core-2.3.22/lib/aws-sdk-core/plugins/param_validator.rb:21:in `call'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/aws-sdk-core-2.3.22/lib/seahorse/client/plugins/raise_response_errors.rb:14:in `call'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/aws-sdk-core-2.3.22/lib/aws-sdk-core/plugins/s3_sse_cpk.rb:19:in `call'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/aws-sdk-core-2.3.22/lib/aws-sdk-core/plugins/s3_accelerate.rb:33:in `call'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/aws-sdk-core-2.3.22/lib/aws-sdk-core/plugins/param_converter.rb:20:in `call'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/aws-sdk-core-2.3.22/lib/seahorse/client/plugins/response_target.rb:21:in `call'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/aws-sdk-core-2.3.22/lib/seahorse/client/request.rb:70:in `send_request'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/aws-sdk-core-2.3.22/lib/seahorse/client/base.rb:207:in `create_multipart_upload'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/aws-sdk-resources-2.3.22/lib/aws-sdk-resources/services/s3/multipart_file_uploader.rb:52:in `initiate_upload'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/aws-sdk-resources-2.3.22/lib/aws-sdk-resources/services/s3/multipart_file_uploader.rb:43:in `upload'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/aws-sdk-resources-2.3.22/lib/aws-sdk-resources/services/s3/file_uploader.rb:32:in `upload'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/aws-sdk-resources-2.3.22/lib/aws-sdk-resources/services/s3/object.rb:251:in `upload_file'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-s3-4.0.1/lib/logstash/outputs/s3/uploader.rb:38:in `upload'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/logstash-output-s3-4.0.1/lib/logstash/outputs/s3/uploader.rb:29:in `upload_async'","org/jruby/RubyProc.java:281:in `call'","/usr/share/logstash/vendor/bundle/jruby/1.9/gems/concurrent-ruby-1.0.0-java/lib/concurrent/executor/java_executor_service.rb:94:in `run'","Concurrent$$JavaExecutorService$$Job_1168525423.gen:13:in `run'"]}}}}}} ``` Fixes #112
1 parent bd6eed5 commit 632db70

File tree

6 files changed

+8
-7
lines changed

6 files changed

+8
-7
lines changed

CHANGELOG.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
## 4.0.2
2+
- Fixed AWS authentication when using instance profile credentials.
3+
14
## 4.0.1
25
- Improved Error logging for S3 validation. Now specific S3 perms errors are logged
36

CONTRIBUTORS

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ Contributors:
55
* Colin Surprenant (colinsurprenant)
66
* Helmut Duregger (hduregger)
77
* Jordan Sissel (jordansissel)
8+
* Joshua Spence (joshuaspence)
89
* Kurt Hurtado (kurtado)
910
* Mattia Peterle (MattiaBeast)
1011
* Nick Ethier (nickethier)

lib/logstash/outputs/s3.rb

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -253,7 +253,7 @@ def close
253253
end
254254

255255
def full_options
256-
options = { :credentials => credentials }
256+
options = Hash.new
257257
options[:s3_signature_version] = @signature_version if @signature_version
258258
options.merge(aws_options_hash)
259259
end
@@ -281,7 +281,7 @@ def stop_periodic_check
281281
end
282282

283283
def bucket_resource
284-
Aws::S3::Bucket.new(@bucket, { :credentials => credentials }.merge(aws_options_hash))
284+
Aws::S3::Bucket.new(@bucket, full_options)
285285
end
286286

287287
def aws_service_endpoint(region)

logstash-output-s3.gemspec

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
Gem::Specification.new do |s|
22
s.name = 'logstash-output-s3'
3-
s.version = '4.0.1'
3+
s.version = '4.0.2'
44
s.licenses = ['Apache-2.0']
55
s.summary = "This plugin was created for store the logstash's events into Amazon Simple Storage Service (Amazon S3)"
66
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

spec/outputs/s3/file_repository_spec.rb

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -105,9 +105,6 @@
105105

106106
@file_repository.get_file("another-prefix") { |file| file.write("hello") }
107107
expect(@file_repository.size).to eq(2)
108-
@file_repository.keys.each do |k|
109-
puts k
110-
end
111108
try(10) { expect(@file_repository.size).to eq(1) }
112109
expect(File.directory?(path)).to be_falsey
113110
end

spec/supports/helpers.rb

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
let(:time_file) { 100 }
1010
let(:tags) { [] }
1111
let(:prefix) { "home" }
12-
let(:region) { "us-east-1" }
12+
let(:region) { ENV['AWS_REGION'] }
1313

1414
let(:main_options) do
1515
{

0 commit comments

Comments
 (0)