From b427d494554e98585efbdf9590adbd07abf31bf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Di=CC=81az=20Bautista?= Date: Wed, 17 Feb 2016 23:21:16 +0100 Subject: [PATCH 1/3] Adding feature to support dynamic prefix --- CHANGELOG.md | 4 +++- CONTRIBUTORS | 3 ++- lib/logstash/outputs/s3.rb | 15 +++++++++++++-- logstash-output-s3.gemspec | 4 ++-- spec/outputs/s3_spec.rb | 19 ++++++++++++++++++- 5 files changed, 38 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7e4a654..0926bd8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +## 2.0.6 + - Feature to support dynamic prefix ## 2.0.5 - Support signature_version option for v4 S3 keys ## 2.0.4 @@ -14,4 +16,4 @@ # 1.0.1 - Fix a synchronization issue when doing file rotation and checking the size of the current file -- Fix an issue with synchronization when shutting down the plugin and closing the current temp file +- Fix an issue with synchronization when shutting down the plugin and closing the current temp file \ No newline at end of file diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 5f2fef2f..594b0f0e 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -10,8 +10,9 @@ Contributors: * Nick Ethier (nickethier) * Pier-Hugues Pellerin (ph) * Richard Pijnenburg (electrical) +* Sergio Díaz (sdiazb) Note: If you've sent us patches, bug reports, or otherwise contributed to Logstash, and you aren't on the list above and want to be, please let us know and we'll make sure you're here. Contributions from folks like you are what make -open source awesome. +open source awesome. \ No newline at end of file diff --git a/lib/logstash/outputs/s3.rb b/lib/logstash/outputs/s3.rb index 20d4b395..2603a05d 100644 --- a/lib/logstash/outputs/s3.rb +++ b/lib/logstash/outputs/s3.rb @@ -104,6 +104,12 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base config :temporary_directory, :validate => :string, :default => File.join(Dir.tmpdir, "logstash") # Specify a prefix to the uploaded filename, this can simulate directories on S3 + # Event fields can be used here, like `path/to/bucket/folder/%{host}/%{application}` + # One may also utilize the path option for date-based log + # rotation via the joda time format. This will use the event + # timestamp. + # E.g.: `prefix => "test/%{+YYYYMMdd}/"` to create + # `test/20160216/` config :prefix, :validate => :string, :default => '' # Specify how many workers to use to upload the files to S3 @@ -303,7 +309,7 @@ def get_temporary_filename(page_counter = 0) public def receive(event) - + build_prefix(event) @codec.encode(event) end @@ -343,6 +349,11 @@ def close end end + public + def build_prefix(event) + @prefix = event.sprintf(@prefix) + end + private def shutdown_upload_workers @logger.debug("S3: Gracefully shutdown the upload workers") @@ -445,4 +456,4 @@ def move_file_to_bucket_async(file) @logger.debug("S3: Sending the file to the upload queue.", :filename => File.basename(file)) @upload_queue.enq(file) end -end +end \ No newline at end of file diff --git a/logstash-output-s3.gemspec b/logstash-output-s3.gemspec index 7b2a9342..aeda31cf 100644 --- a/logstash-output-s3.gemspec +++ b/logstash-output-s3.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-output-s3' - s.version = '2.0.5' + s.version = '2.0.6' s.licenses = ['Apache License (2.0)'] s.summary = "This plugin was created for store the logstash's events into Amazon Simple Storage Service (Amazon S3)" s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program" @@ -26,4 +26,4 @@ Gem::Specification.new do |s| s.add_development_dependency 'logstash-devutils' s.add_development_dependency 'logstash-input-generator' s.add_development_dependency 'logstash-codec-line' -end +end \ No newline at end of file diff --git a/spec/outputs/s3_spec.rb b/spec/outputs/s3_spec.rb index a34ae821..2bd495a0 100644 --- a/spec/outputs/s3_spec.rb +++ b/spec/outputs/s3_spec.rb @@ -351,4 +351,21 @@ end end end -end + + describe "#build_prefix" do + it "should render event fields and date in prefix" do + config = { + "prefix" => "foo/%{foo}/%{+yyyyMMdd}/" + } + + s3 = LogStash::Outputs::S3.new(config) + + data = {"foo" => "bar", "@timestamp" => "2016-02-17T02:52:17.929Z"} + event = LogStash::Event.new(data) + + s3.build_prefix(event) + + expect(s3.prefix).to eql('foo/bar/20160217/') + end + end +end \ No newline at end of file From d247798a2b73be09cf176e5a0aab6a106090ccdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Di=CC=81az=20Bautista?= Date: Fri, 11 Mar 2016 21:45:59 +0100 Subject: [PATCH 2/3] Fix to support logtime rotation on dynamic prefix --- lib/logstash/outputs/s3.rb | 12 +++++++++--- spec/outputs/s3_spec.rb | 6 ++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/lib/logstash/outputs/s3.rb b/lib/logstash/outputs/s3.rb index 2603a05d..543dc9a9 100644 --- a/lib/logstash/outputs/s3.rb +++ b/lib/logstash/outputs/s3.rb @@ -134,6 +134,8 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base attr_reader :page_counter attr_reader :s3 + attr_accessor :original_prefix + def aws_s3_config @logger.info("Registering s3 output", :bucket => @bucket, :endpoint_region => @region) @s3 = AWS::S3.new(full_options) @@ -223,6 +225,10 @@ def register raise LogStash::ConfigurationError, "S3: prefix contains invalid characters" end + if @original_prefix.nil? + @original_prefix = @prefix + end + if !Dir.exist?(@temporary_directory) FileUtils.mkdir_p(@temporary_directory) end @@ -349,9 +355,9 @@ def close end end - public + public def build_prefix(event) - @prefix = event.sprintf(@prefix) + @prefix = event.sprintf(@original_prefix) end private @@ -456,4 +462,4 @@ def move_file_to_bucket_async(file) @logger.debug("S3: Sending the file to the upload queue.", :filename => File.basename(file)) @upload_queue.enq(file) end -end \ No newline at end of file +end diff --git a/spec/outputs/s3_spec.rb b/spec/outputs/s3_spec.rb index 2bd495a0..649cfd80 100644 --- a/spec/outputs/s3_spec.rb +++ b/spec/outputs/s3_spec.rb @@ -206,7 +206,7 @@ after(:each) do s3.close - tmp.close + tmp.close tmp.unlink end @@ -363,9 +363,11 @@ data = {"foo" => "bar", "@timestamp" => "2016-02-17T02:52:17.929Z"} event = LogStash::Event.new(data) + s3.original_prefix = s3.prefix + s3.build_prefix(event) expect(s3.prefix).to eql('foo/bar/20160217/') end end -end \ No newline at end of file +end From c50995f2ced8f62a5217be87ffda194b351dffed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sergio=20Di=CC=81az=20Bautista?= Date: Thu, 7 Apr 2016 01:05:22 +0200 Subject: [PATCH 3/3] Removing dynamic prefix by event and adding dynamic prefix by date when flushing in a thread safe way --- lib/logstash/outputs/s3.rb | 13 ++++++------- spec/outputs/s3_spec.rb | 7 ++----- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/lib/logstash/outputs/s3.rb b/lib/logstash/outputs/s3.rb index 543dc9a9..ecf967b1 100644 --- a/lib/logstash/outputs/s3.rb +++ b/lib/logstash/outputs/s3.rb @@ -104,11 +104,9 @@ class LogStash::Outputs::S3 < LogStash::Outputs::Base config :temporary_directory, :validate => :string, :default => File.join(Dir.tmpdir, "logstash") # Specify a prefix to the uploaded filename, this can simulate directories on S3 - # Event fields can be used here, like `path/to/bucket/folder/%{host}/%{application}` # One may also utilize the path option for date-based log - # rotation via the joda time format. This will use the event - # timestamp. - # E.g.: `prefix => "test/%{+YYYYMMdd}/"` to create + # rotation. This will use the event timestamp. + # E.g.: `prefix => "test/%Y%m%d/"` to create # `test/20160216/` config :prefix, :validate => :string, :default => '' @@ -173,6 +171,8 @@ def write_on_bucket(file) # find and use the bucket bucket = @s3.buckets[@bucket] + build_prefix(Time.now) + remote_filename = "#{@prefix}#{File.basename(file)}" @logger.debug("S3: ready to write file in bucket", :remote_filename => remote_filename, :bucket => @bucket) @@ -315,7 +315,6 @@ def get_temporary_filename(page_counter = 0) public def receive(event) - build_prefix(event) @codec.encode(event) end @@ -356,8 +355,8 @@ def close end public - def build_prefix(event) - @prefix = event.sprintf(@original_prefix) + def build_prefix(datetime) + @prefix = datetime.strftime(@original_prefix) end private diff --git a/spec/outputs/s3_spec.rb b/spec/outputs/s3_spec.rb index 649cfd80..0088731f 100644 --- a/spec/outputs/s3_spec.rb +++ b/spec/outputs/s3_spec.rb @@ -355,17 +355,14 @@ describe "#build_prefix" do it "should render event fields and date in prefix" do config = { - "prefix" => "foo/%{foo}/%{+yyyyMMdd}/" + "prefix" => "foo/bar/%Y%m%d/" } s3 = LogStash::Outputs::S3.new(config) - data = {"foo" => "bar", "@timestamp" => "2016-02-17T02:52:17.929Z"} - event = LogStash::Event.new(data) - s3.original_prefix = s3.prefix - s3.build_prefix(event) + s3.build_prefix(Time.new(2016, 02, 17)) expect(s3.prefix).to eql('foo/bar/20160217/') end