{"action":"created","comment":{"url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/pulls/comments/23017172","id":23017172,"diff_hunk":"@@ -73,285 +61,354 @@\n #      format => \"plain\"                        (optional)\n #      canned_acl => \"private\"                  (optional. Options are \"private\", \"public_read\", \"public_read_write\", \"authenticated_read\". Defaults to \"private\" )\n #    }\n-# }\n-#\n-# We analize this:\n-#\n-# access_key_id => \"crazy_key\"\n-# Amazon will give you the key for use their service if you buy it or try it. (not very much open source anyway)\n-#\n-# secret_access_key => \"monkey_access_key\"\n-# Amazon will give you the secret_access_key for use their service if you buy it or try it . (not very much open source anyway).\n-#\n-# endpoint_region => \"eu-west-1\"\n-# When you make a contract with Amazon, you should know where the services you use.\n-#\n-# bucket => \"boss_please_open_your_bucket\"\n-# Be careful you have the permission to write on bucket and know the name.\n-#\n-# size_file => 2048\n-# Means the size, in KB, of files who can store on temporary directory before you will be pushed on bucket.\n-# Is useful if you have a little server with poor space on disk and you don't want blow up the server with unnecessary temporary log files.\n-#\n-# time_file => 5\n-# Means, in minutes, the time  before the files will be pushed on bucket. Is useful if you want to push the files every specific time.\n-#\n-# format => \"plain\"\n-# Means the format of events you want to store in the files\n-#\n-# canned_acl => \"private\"\n-# The S3 canned ACL to use when putting the file. Defaults to \"private\".\n-#\n-# LET'S ROCK AND ROLL ON THE CODE!\n #\n class LogStash::Outputs::S3 < LogStash::Outputs::Base\n- #TODO integrate aws_config in the future\n- #  include LogStash::PluginMixins::AwsConfig\n-\n- config_name \"s3\"\n- milestone 1\n+  include LogStash::PluginMixins::AwsConfig\n+\n+  TEMPFILE_EXTENSION = \"txt\"\n+  S3_INVALID_CHARACTERS = /[\\^`><]/\n+\n+  config_name \"s3\"\n+  milestone 1\n+  default :codec, 'line'\n+\n+  # S3 bucket\n+  config :bucket, :validate => :string\n+\n+  # AWS endpoint_region\n+  config :endpoint_region, :validate => [\"us-east-1\", \"us-west-1\", \"us-west-2\",\n+                                         \"eu-west-1\", \"ap-southeast-1\", \"ap-southeast-2\",\n+                                        \"ap-northeast-1\", \"sa-east-1\", \"us-gov-west-1\"], :default => \"us-east-1\", :deprecated => 'Deprecated, use region instead.'\n+\n+  # Set the size of file in bytes, this means that files on bucket when have dimension > file_size, they are stored in two or more file.\n+  # If you have tags then it will generate a specific size file for every tags\n+  ##NOTE: define size of file is the better thing, because generate a local temporary file on disk and then put it in bucket.\n+  config :size_file, :validate => :number, :default => 0\n+\n+  # Set the time, in minutes, to close the current sub_time_section of bucket.\n+  # If you define file_size you have a number of files in consideration of the section and the current tag.\n+  # 0 stay all time on listerner, beware if you specific 0 and size_file 0, because you will not put the file on bucket,\n+  # for now the only thing this plugin can do is to put the file when logstash restart.\n+  config :time_file, :validate => :number, :default => 0\n+\n+  ## IMPORTANT: if you use multiple instance of s3, you should specify on one of them the \"restore=> true\" and on the others \"restore => false\".\n+  ## This is hack for not destroy the new files after restoring the initial files.\n+  ## If you do not specify \"restore => true\" when logstash crashes or is restarted, the files are not sent into the bucket,\n+  ## for example if you have single Instance.\n+  config :restore, :validate => :boolean, :default => false\n+\n+  # The S3 canned ACL to use when putting the file. Defaults to \"private\".\n+  config :canned_acl, :validate => [\"private\", \"public_read\", \"public_read_write\", \"authenticated_read\"],\n+         :default => \"private\"\n+\n+  # Set the directory where logstash will store the tmp files before sending it to S3\n+  # default to the current OS temporary directory in linux /tmp/logstash\n+  config :temporary_directory, :validate => :string, :default => File.join(Dir.tmpdir, \"logstash\")\n+\n+  # Specify a prefix to the uploaded filename, this can simulate directories on S3\n+  config :prefix, :validate => :string, :default => ''\n+\n+  # Specify how many workers to use to upload the files to S3\n+  config :upload_workers_count, :validate => :number, :default => 1\n+\n+  # Exposed attributes for testing purpose.\n+  attr_accessor :tempfile\n+  attr_reader :page_counter\n+  attr_reader :s3\n+\n+  def aws_s3_config\n+    @logger.info(\"Registering s3 output\", :bucket => @bucket, :endpoint_region => @region)\n+    @s3 = AWS::S3.new(aws_options_hash)\n+  end\n \n- # Aws access_key.\n- config :access_key_id, :validate => :string\n+  def aws_service_endpoint(region)\n+    # Make the deprecated endpoint_region work\n+    # TODO: (ph) Remove this after deprecation.\n+    if @endpoint_region\n+      region_to_use = @endpoint_region\n+    else\n+      region_to_use = region\n+    end\n \n- # Aws secret_access_key\n- config :secret_access_key, :validate => :string\n+    return {\n+      :s3_endpoint => region_to_use == 'us-east-1' ? 's3.amazonaws.com' : \"s3-#{region_to_use}.amazonaws.com\"\n+    }\n+  end\n \n- # S3 bucket\n- config :bucket, :validate => :string\n+  public\n+  def write_on_bucket(file)\n+    # find and use the bucket\n+    bucket = @s3.buckets[@bucket]\n \n- # Aws endpoint_region\n- config :endpoint_region, :validate => [\"us-east-1\", \"us-west-1\", \"us-west-2\",\n-                                        \"eu-west-1\", \"ap-southeast-1\", \"ap-southeast-2\",\n-                                        \"ap-northeast-1\", \"sa-east-1\", \"us-gov-west-1\"], :default => \"us-east-1\"\n+    remote_filename = \"#{@prefix}#{File.basename(file)}\"\n \n- # Set the size of file in KB, this means that files on bucket when have dimension > file_size, they are stored in two or more file.\n- # If you have tags then it will generate a specific size file for every tags\n- ##NOTE: define size of file is the better thing, because generate a local temporary file on disk and then put it in bucket.\n- config :size_file, :validate => :number, :default => 0\n+    @logger.debug(\"S3: ready to write file in bucket\", :remote_filename => remote_filename, :bucket => @bucket)\n \n- # Set the time, in minutes, to close the current sub_time_section of bucket.\n- # If you define file_size you have a number of files in consideration of the section and the current tag.\n- # 0 stay all time on listerner, beware if you specific 0 and size_file 0, because you will not put the file on bucket,\n- # for now the only thing this plugin can do is to put the file when logstash restart.\n- config :time_file, :validate => :number, :default => 0\n+    begin\n+      # prepare for write the file\n+      object = bucket.objects[remote_filename]\n+      object.write(:file => file, :acl => @canned_acl)\n+    rescue AWS::Errors::Base => e\n+      @logger.error(\"S3: AWS error\", :error => e)\n+      raise LogStash::ConfigurationError, \"AWS Configuration Error\"\n+    end\n \n- # The event format you want to store in files. Defaults to plain text.\n- config :format, :validate => [ \"json\", \"plain\", \"nil\" ], :default => \"plain\"\n+    @logger.debug(\"S3: has written remote file in bucket with canned ACL\", :remote_filename => remote_filename, :bucket  => @bucket, :canned_acl => @canned_acl)\n+  end\n \n- ## IMPORTANT: if you use multiple instance of s3, you should specify on one of them the \"restore=> true\" and on the others \"restore => false\".\n- ## This is hack for not destroy the new files after restoring the initial files.\n- ## If you do not specify \"restore => true\" when logstash crashes or is restarted, the files are not sent into the bucket,\n- ## for example if you have single Instance.\n- config :restore, :validate => :boolean, :default => false\n+  # This method is used for create new empty temporary files for use. Flag is needed for indicate new subsection time_file.\n+  public\n+  def create_temporary_file\n+    filename = File.join(@temporary_directory, get_temporary_filename(@page_counter))\n \n- # Aws canned ACL\n- config :canned_acl, :validate => [\"private\", \"public_read\", \"public_read_write\", \"authenticated_read\"],\n-        :default => \"private\"\n+    @logger.debug(\"S3: Creating a new temporary file\", :filename => filename)\n \n- # Method to set up the aws configuration and establish connection\n- def aws_s3_config\n+    @file_rotation_lock.synchronize do\n+      unless @tempfile.nil?\n+        @tempfile.close\n+      end\n \n-  @endpoint_region == 'us-east-1' ? @endpoint_region = 's3.amazonaws.com' : @endpoint_region = 's3-'+@endpoint_region+'.amazonaws.com'\n+      @tempfile = File.open(filename, \"a\")\n+    end\n+  end\n \n-  @logger.info(\"Registering s3 output\", :bucket => @bucket, :endpoint_region => @endpoint_region)\n+  public\n+  def register\n+    require \"aws-sdk\"\n+    # required if using ruby version < 2.0\n+    # http://ruby.awsblog.com/post/Tx16QY1CI5GVBFT/Threading-with-the-AWS-SDK-for-Ruby\n+    AWS.eager_autoload!(AWS::S3)\n \n-  AWS.config(\n-    :access_key_id => @access_key_id,\n-    :secret_access_key => @secret_access_key,\n-    :s3_endpoint => @endpoint_region\n-  )\n-  @s3 = AWS::S3.new\n+    workers_not_supported\n \n- end\n+    @s3 = aws_s3_config\n+    @upload_queue = Queue.new\n+    @file_rotation_lock = Mutex.new\n \n- # This method is used to manage sleep and awaken thread.\n- def time_alert(interval)\n+    if @prefix && @prefix =~ S3_INVALID_CHARACTERS\n+      @logger.error(\"S3: prefix contains invalid characters\", :prefix => @prefix, :contains => S3_INVALID_CHARACTERS)\n+      raise LogStash::ConfigurationError, \"S3: prefix contains invalid characters\"\n+    end\n \n-   Thread.new do\n-    loop do\n-      start_time = Time.now\n-      yield\n-      elapsed = Time.now - start_time\n-      sleep([interval - elapsed, 0].max)\n+    if !Dir.exist?(@temporary_directory)\n+      FileUtils.mkdir_p(@temporary_directory)\n     end\n-   end\n \n- end\n+    test_s3_write\n \n- # this method is used for write files on bucket. It accept the file and the name of file.\n- def write_on_bucket (file_data, file_basename)\n+    restore_from_crashes if @restore == true\n+    reset_page_counter\n+    create_temporary_file\n+    configure_periodic_rotation if time_file != 0\n+    configure_upload_workers\n \n-  # if you lose connection with s3, bad control implementation.\n-  if ( @s3 == nil)\n-    aws_s3_config\n+    @codec.on_event do |event, encoded_event|\n+      handle_event(encoded_event)\n+    end\n   end\n \n-  # find and use the bucket\n-  bucket = @s3.buckets[@bucket]\n-\n-  @logger.debug \"S3: ready to write \"+file_basename+\" in bucket \"+@bucket+\", Fire in the hole!\"\n-\n-  # prepare for write the file\n-  object = bucket.objects[file_basename]\n-  object.write(:file => file_data, :acl => @canned_acl)\n-\n-  @logger.debug \"S3: has written \"+file_basename+\" in bucket \"+@bucket + \" with canned ACL \\\"\" + @canned_acl + \"\\\"\"\n-\n- end\n+  public\n+  def configure_upload_workers\n+    @logger.debug(\"S3: Configure upload workers\")\n \n- # this method is used for create new path for name the file\n- def getFinalPath\n+    @upload_workers = @upload_workers_count.times.map do |worker_id|\n+      Stud::Task.new do\n+        LogStash::Util::set_thread_name(\"<S3 upload worker #{worker_id}\")\n \n-   @pass_time = Time.now\n-   return @temp_directory+\"ls.s3.\"+Socket.gethostname+\".\"+(@pass_time).strftime(\"%Y-%m-%dT%H.%M\")\n+        while true do\n+          @logger.debug(\"S3: upload worker is waiting for a new file to upload.\", :worker_id => worker_id)\n \n- end\n-\n- # This method is used for restore the previous crash of logstash or to prepare the files to send in bucket.\n- # Take two parameter: flag and name. Flag indicate if you want to restore or not, name is the name of file\n- def upFile(flag, name)\n+          upload_worker\n+        end\n+      end\n+    end\n+  end\n \n-   Dir[@temp_directory+name].each do |file|\n-     name_file = File.basename(file)\n+  private\n+  def upload_worker\n+    file = @upload_queue.deq\n+\n+    case file\n+      when LogStash::ShutdownEvent\n+        @logger.debug(\"S3: upload worker is shutting down gracefuly\")\n+        @upload_queue.enq(LogStash::ShutdownEvent)\n+      else\n+        @logger.debug(\"S3: upload working is uploading a new file\", :filename => File.basename(file))\n+        move_file_to_bucket(file)\n+    end\n+  end\n \n-     if (flag == true)\n-      @logger.warn \"S3: have found temporary file: \"+name_file+\", something has crashed before... Prepare for upload in bucket!\"\n-     end\n+  public\n+  def next_page\n+    @page_counter += 1\n+  end\n \n-     if (!File.zero?(file))\n-       write_on_bucket(file, name_file)\n+  def reset_page_counter\n+    @page_counter = 0\n+  end\n \n-       if (flag == true)\n-          @logger.debug \"S3: file: \"+name_file+\" restored on bucket \"+@bucket\n-       else\n-          @logger.debug \"S3: file: \"+name_file+\" was put on bucket \"+@bucket\n-       end\n-     end\n+  # Use the same method that Amazon use to check\n+  # permission on the user bucket by creating a small file\n+  public\n+  def test_s3_write\n+    @logger.debug(\"S3: Creating a test file on S3\")\n \n-     File.delete (file)\n+    test_filename = File.join(@temporary_directory,\n+                              \"logstash-programmatic-access-test-object-#{Time.now.to_i}\")\n \n-   end\n- end\n+    File.open(test_filename, 'a') do |file|\n+      file.write('test')\n+    end\n \n- # This method is used for create new empty temporary files for use. Flag is needed for indicate new subsection time_file.\n- def newFile (flag)\n+    begin\n+      write_on_bucket(test_filename)\n+      delete_on_bucket(test_filename)\n+    ensure\n+      File.delete(test_filename)\n+    end\n+  end\n+  \n+  def delete_on_bucket(filename)\n+    bucket = @s3.buckets[@bucket]\n \n-   if (flag == true)\n-     @current_final_path = getFinalPath\n-     @sizeCounter = 0\n-   end\n+    remote_filename = \"#{@prefix}#{File.basename(filename)}\"\n \n-   if (@tags.size != 0)\n-     @tempFile = File.new(@current_final_path+\".tag_\"+@tag_path+\"part\"+@sizeCounter.to_s+\".txt\", \"w\")\n-   else\n-     @tempFile = File.new(@current_final_path+\".part\"+@sizeCounter.to_s+\".txt\", \"w\")\n-   end\n+    @logger.debug(\"S3: delete file from bucket\", :remote_filename => remote_filename, :bucket => @bucket)\n \n- end\n+    begin\n+      # prepare for write the file\n+      object = bucket.objects[remote_filename]\n+      object.delete\n+    rescue AWS::Errors::Base => e\n+      @logger.error(\"S3: AWS error\", :error => e)\n+      raise LogStash::ConfigurationError, \"AWS Configuration Error\"\n+    end\n+  end\n \n- public\n- def register\n-   require \"aws-sdk\"\n-   @temp_directory = \"/opt/logstash/S3_temp/\"\n+  public\n+  def restore_from_crashes\n+    @logger.debug(\"S3: is attempting to verify previous crashes...\")\n \n-   if (@tags.size != 0)\n-       @tag_path = \"\"\n-       for i in (0..@tags.size-1)\n-          @tag_path += @tags[i].to_s+\".\"\n-       end\n-   end\n+    Dir[File.join(@temporary_directory, \"*.#{TEMPFILE_EXTENSION}\")].each do |file|\n+      name_file = File.basename(file)\n+      @logger.warn(\"S3: have found temporary file the upload process crashed, uploading file to S3.\", :filename => name_file)\n+      move_file_to_bucket_async(file)\n+    end\n+  end\n \n-   if !(File.directory? @temp_directory)\n-    @logger.debug \"S3: Directory \"+@temp_directory+\" doesn't exist, let's make it!\"\n-    Dir.mkdir(@temp_directory)\n-   else\n-    @logger.debug \"S3: Directory \"+@temp_directory+\" exist, nothing to do\"\n-   end\n+  public\n+  def move_file_to_bucket(file)\n+    if !File.zero?(file)\n+      write_on_bucket(file)\n+      @logger.debug(\"S3: file was put on the upload thread\", :filename => File.basename(file), :bucket => @bucket)\n+    end\n \n-   if (@restore == true )\n-     @logger.debug \"S3: is attempting to verify previous crashes...\"\n+    begin\n+      File.delete(file)\n+    rescue Errno::ENOENT\n+      # Something else deleted the file, logging but not raising the issue\n+      @logger.warn(\"S3: Cannot delete the temporary file since it doesn't exist on disk\", :filename => File.basename(file))\n+    rescue Errno::EACCES\n+      @logger.error(\"S3: Logstash doesnt have the permission to delete the file in the temporary directory.\", :filename => File.basename, :temporary_directory => @temporary_directory)\n+    end\n+  end\n \n-     upFile(true, \"*.txt\")\n-   end\n+  def move_file_to_bucket_async(file)\n+    @logger.debug(\"S3: Sending the file to the upload queue.\", :filename => File.basename(file))\n+    @upload_queue.enq(file)\n+  end\n \n-   newFile(true)\n+  public\n+  def configure_periodic_rotation\n+    @periodic_rotation_thread = Stud::Task.new do\n+      LogStash::Util::set_thread_name(\"<S3 periodic uploader\")\n \n-   if (time_file != 0)\n-      first_time = true\n-      @thread = time_alert(@time_file*60) do\n-       if (first_time == false)\n-         @logger.debug \"S3: time_file triggered,  let's bucket the file if dosen't empty  and create new file \"\n-         upFile(false, File.basename(@tempFile))\n-         newFile(true)\n-       else\n-         first_time = false\n-       end\n-     end\n-   end\n+      Stud.interval(periodic_interval, :sleep_then_run => true) do\n+        @logger.debug(\"S3: time_file triggered, bucketing the file\", :filename => @tempfile.path)\n \n- end\n-\n- public\n- def receive(event)\n-  return unless output?(event)\n-\n-  # Prepare format of Events\n-  if (@format == \"plain\")\n-     message = self.class.format_message(event)\n-  elsif (@format == \"json\")\n-     message = event.to_json\n-  else\n-     message = event.to_s\n+        move_file_to_bucket_async(@tempfile.path)\n+        next_page\n+        create_temporary_file\n+      end\n+    end\n   end\n \n-  if(time_file !=0)\n-     @logger.debug \"S3: trigger files after \"+((@pass_time+60*time_file)-Time.now).to_s\n+  public\n+  def periodic_interval\n+    @time_file * 60\n   end\n \n-  # if specific the size\n-  if(size_file !=0)\n-\n-    if (@tempFile.size < @size_file )\n+  public\n+  def get_temporary_filename(page_counter = 0)\n+    current_time = Time.now\n+    filename = \"ls.s3.#{Socket.gethostname}.#{current_time.strftime(\"%Y-%m-%dT%H.%M\")}\"\n \n-       @logger.debug \"S3: File have size: \"+@tempFile.size.to_s+\" and size_file is: \"+ @size_file.to_s\n-       @logger.debug \"S3: put event into: \"+File.basename(@tempFile)\n-\n-       # Put the event in the file, now!\n-       File.open(@tempFile, 'a') do |file|\n-         file.puts message\n-         file.write \"\\n\"\n-       end\n+    if @tags.size > 0\n+      return \"#{filename}.tag_#{@tags.join('.')}.part#{page_counter}.#{TEMPFILE_EXTENSION}\"\n+    else\n+      return \"#{filename}.part#{page_counter}.#{TEMPFILE_EXTENSION}\"\n+    end\n+  end\n \n-     else\n+  public\n+  def receive(event)\n+    return unless output?(event)\n+    @codec.encode(event)\n+  end\n \n-       @logger.debug \"S3: file: \"+File.basename(@tempFile)+\" is too large, let's bucket it and create new file\"\n-       upFile(false, File.basename(@tempFile))\n-       @sizeCounter += 1\n-       newFile(false)\n+  def handle_event(encoded_event)\n+    if write_events_to_multiple_files?\n+      if rotate_events_log?\n+        @logger.debug(\"S3: tempfile is too large, let's bucket it and create new file\", :tempfile => File.basename(@tempfile))\n+\n+        move_file_to_bucket_async(@tempfile.path)\n+        next_page\n+        create_temporary_file\n+      else\n+        @logger.debug(\"S3: tempfile file size report.\", :tempfile_size => @tempfile.size, :size_file => @size_file)\n+      end\n+\n+      write_to_tempfile(encoded_event)","path":"lib/logstash/outputs/s3.rb","position":565,"original_position":565,"commit_id":"e5b73fde4b2d9535cab7c2a324faca830c792edc","original_commit_id":"e5b73fde4b2d9535cab7c2a324faca830c792edc","user":{"login":"jsvd","id":31809,"avatar_url":"https://avatars.githubusercontent.com/u/31809?v=3","gravatar_id":"","url":"https://api.github.com/users/jsvd","html_url":"https://github.com/jsvd","followers_url":"https://api.github.com/users/jsvd/followers","following_url":"https://api.github.com/users/jsvd/following{/other_user}","gists_url":"https://api.github.com/users/jsvd/gists{/gist_id}","starred_url":"https://api.github.com/users/jsvd/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/jsvd/subscriptions","organizations_url":"https://api.github.com/users/jsvd/orgs","repos_url":"https://api.github.com/users/jsvd/repos","events_url":"https://api.github.com/users/jsvd/events{/privacy}","received_events_url":"https://api.github.com/users/jsvd/received_events","type":"User","site_admin":false},"body":"maybe move this to the outside of the if, since it will be called either way","created_at":"2015-01-15T15:33:16Z","updated_at":"2015-01-15T15:33:16Z","html_url":"https://github.com/logstash-plugins/logstash-output-s3/pull/1#discussion_r23017172","pull_request_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/pulls/1","_links":{"self":{"href":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/pulls/comments/23017172"},"html":{"href":"https://github.com/logstash-plugins/logstash-output-s3/pull/1#discussion_r23017172"},"pull_request":{"href":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/pulls/1"}}},"pull_request":{"url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/pulls/1","id":24154256,"html_url":"https://github.com/logstash-plugins/logstash-output-s3/pull/1","diff_url":"https://github.com/logstash-plugins/logstash-output-s3/pull/1.diff","patch_url":"https://github.com/logstash-plugins/logstash-output-s3/pull/1.patch","issue_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/issues/1","number":1,"state":"open","locked":false,"title":"Refactoring and cleaning of the S3 outputs","user":{"login":"ph","id":640,"avatar_url":"https://avatars.githubusercontent.com/u/640?v=3","gravatar_id":"","url":"https://api.github.com/users/ph","html_url":"https://github.com/ph","followers_url":"https://api.github.com/users/ph/followers","following_url":"https://api.github.com/users/ph/following{/other_user}","gists_url":"https://api.github.com/users/ph/gists{/gist_id}","starred_url":"https://api.github.com/users/ph/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ph/subscriptions","organizations_url":"https://api.github.com/users/ph/orgs","repos_url":"https://api.github.com/users/ph/repos","events_url":"https://api.github.com/users/ph/events{/privacy}","received_events_url":"https://api.github.com/users/ph/received_events","type":"User","site_admin":false},"body":"This PRs includes a lot of changes.\r\n\r\n- Merged @jsvd documentation improvement and added more documentation on new attributes Fix: https://github.com/elasticsearch/logstash/issues/1340\r\n- Ruby code cleanup, following the STYLE.md\r\n- It now use the aws_config mixing support IAM roles fix https://github.com/elasticsearch/logstash/issues/1778 fix https://github.com/elasticsearch/logstash/issues/1575\r\n- the plugin also use standard codec, added a new codec [s3_plain](https://github.com/logstash-plugins/logstash-codec-s3plain) to support previous version\r\n- refactoring of the code to improve clarity and allow easier testing\r\n- Created initial test suite for it.\r\n- Fixed a bug causing events to be dropped when using the size_file option and the file rotation.\r\n- Fixed a bug when the tags were nil in the plain text format. fix https://github.com/elasticsearch/logstash/issues/1626\r\n- Fixed a bug when specifying the file_size, the doc says KB but the condition was on bytes.\r\n- Added the option to specify a temporary_directory for the working files. fix https://github.com/elasticsearch/logstash/issues/1860\r\n- S3 output issue when working with multiple workers added a threaded uploader fix https://github.com/elasticsearch/logstash/issues/1915\r\n- S3 output: when the codec is started it will try to create a simple file on your bucket to test the permissions fix https://github.com/elasticsearch/logstash/issues/1914","created_at":"2014-11-10T15:35:40Z","updated_at":"2015-01-15T15:33:16Z","closed_at":null,"merged_at":null,"merge_commit_sha":"5c18101d36b6b45d577c073306ab8b64e58be8dc","assignee":null,"milestone":null,"commits_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/pulls/1/commits","review_comments_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/pulls/1/comments","review_comment_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/pulls/comments/{number}","comments_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/issues/1/comments","statuses_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/statuses/e5b73fde4b2d9535cab7c2a324faca830c792edc","head":{"label":"ph:fix/refactor-s3","ref":"fix/refactor-s3","sha":"e5b73fde4b2d9535cab7c2a324faca830c792edc","user":{"login":"ph","id":640,"avatar_url":"https://avatars.githubusercontent.com/u/640?v=3","gravatar_id":"","url":"https://api.github.com/users/ph","html_url":"https://github.com/ph","followers_url":"https://api.github.com/users/ph/followers","following_url":"https://api.github.com/users/ph/following{/other_user}","gists_url":"https://api.github.com/users/ph/gists{/gist_id}","starred_url":"https://api.github.com/users/ph/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ph/subscriptions","organizations_url":"https://api.github.com/users/ph/orgs","repos_url":"https://api.github.com/users/ph/repos","events_url":"https://api.github.com/users/ph/events{/privacy}","received_events_url":"https://api.github.com/users/ph/received_events","type":"User","site_admin":false},"repo":{"id":26442369,"name":"logstash-output-s3","full_name":"ph/logstash-output-s3","owner":{"login":"ph","id":640,"avatar_url":"https://avatars.githubusercontent.com/u/640?v=3","gravatar_id":"","url":"https://api.github.com/users/ph","html_url":"https://github.com/ph","followers_url":"https://api.github.com/users/ph/followers","following_url":"https://api.github.com/users/ph/following{/other_user}","gists_url":"https://api.github.com/users/ph/gists{/gist_id}","starred_url":"https://api.github.com/users/ph/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/ph/subscriptions","organizations_url":"https://api.github.com/users/ph/orgs","repos_url":"https://api.github.com/users/ph/repos","events_url":"https://api.github.com/users/ph/events{/privacy}","received_events_url":"https://api.github.com/users/ph/received_events","type":"User","site_admin":false},"private":false,"html_url":"https://github.com/ph/logstash-output-s3","description":null,"fork":true,"url":"https://api.github.com/repos/ph/logstash-output-s3","forks_url":"https://api.github.com/repos/ph/logstash-output-s3/forks","keys_url":"https://api.github.com/repos/ph/logstash-output-s3/keys{/key_id}","collaborators_url":"https://api.github.com/repos/ph/logstash-output-s3/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/ph/logstash-output-s3/teams","hooks_url":"https://api.github.com/repos/ph/logstash-output-s3/hooks","issue_events_url":"https://api.github.com/repos/ph/logstash-output-s3/issues/events{/number}","events_url":"https://api.github.com/repos/ph/logstash-output-s3/events","assignees_url":"https://api.github.com/repos/ph/logstash-output-s3/assignees{/user}","branches_url":"https://api.github.com/repos/ph/logstash-output-s3/branches{/branch}","tags_url":"https://api.github.com/repos/ph/logstash-output-s3/tags","blobs_url":"https://api.github.com/repos/ph/logstash-output-s3/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/ph/logstash-output-s3/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/ph/logstash-output-s3/git/refs{/sha}","trees_url":"https://api.github.com/repos/ph/logstash-output-s3/git/trees{/sha}","statuses_url":"https://api.github.com/repos/ph/logstash-output-s3/statuses/{sha}","languages_url":"https://api.github.com/repos/ph/logstash-output-s3/languages","stargazers_url":"https://api.github.com/repos/ph/logstash-output-s3/stargazers","contributors_url":"https://api.github.com/repos/ph/logstash-output-s3/contributors","subscribers_url":"https://api.github.com/repos/ph/logstash-output-s3/subscribers","subscription_url":"https://api.github.com/repos/ph/logstash-output-s3/subscription","commits_url":"https://api.github.com/repos/ph/logstash-output-s3/commits{/sha}","git_commits_url":"https://api.github.com/repos/ph/logstash-output-s3/git/commits{/sha}","comments_url":"https://api.github.com/repos/ph/logstash-output-s3/comments{/number}","issue_comment_url":"https://api.github.com/repos/ph/logstash-output-s3/issues/comments/{number}","contents_url":"https://api.github.com/repos/ph/logstash-output-s3/contents/{+path}","compare_url":"https://api.github.com/repos/ph/logstash-output-s3/compare/{base}...{head}","merges_url":"https://api.github.com/repos/ph/logstash-output-s3/merges","archive_url":"https://api.github.com/repos/ph/logstash-output-s3/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/ph/logstash-output-s3/downloads","issues_url":"https://api.github.com/repos/ph/logstash-output-s3/issues{/number}","pulls_url":"https://api.github.com/repos/ph/logstash-output-s3/pulls{/number}","milestones_url":"https://api.github.com/repos/ph/logstash-output-s3/milestones{/number}","notifications_url":"https://api.github.com/repos/ph/logstash-output-s3/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/ph/logstash-output-s3/labels{/name}","releases_url":"https://api.github.com/repos/ph/logstash-output-s3/releases{/id}","created_at":"2014-11-10T15:30:53Z","updated_at":"2014-11-01T06:34:35Z","pushed_at":"2015-01-12T18:24:29Z","git_url":"git://github.com/ph/logstash-output-s3.git","ssh_url":"git@github.com:ph/logstash-output-s3.git","clone_url":"https://github.com/ph/logstash-output-s3.git","svn_url":"https://github.com/ph/logstash-output-s3","homepage":null,"size":172,"stargazers_count":0,"watchers_count":0,"language":null,"has_issues":false,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":0,"mirror_url":null,"open_issues_count":0,"forks":0,"open_issues":0,"watchers":0,"default_branch":"master"}},"base":{"label":"logstash-plugins:master","ref":"master","sha":"0a0e9465df2d4d9fb5a6718283e552ba2ceef3f4","user":{"login":"logstash-plugins","id":8961934,"avatar_url":"https://avatars.githubusercontent.com/u/8961934?v=3","gravatar_id":"","url":"https://api.github.com/users/logstash-plugins","html_url":"https://github.com/logstash-plugins","followers_url":"https://api.github.com/users/logstash-plugins/followers","following_url":"https://api.github.com/users/logstash-plugins/following{/other_user}","gists_url":"https://api.github.com/users/logstash-plugins/gists{/gist_id}","starred_url":"https://api.github.com/users/logstash-plugins/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/logstash-plugins/subscriptions","organizations_url":"https://api.github.com/users/logstash-plugins/orgs","repos_url":"https://api.github.com/users/logstash-plugins/repos","events_url":"https://api.github.com/users/logstash-plugins/events{/privacy}","received_events_url":"https://api.github.com/users/logstash-plugins/received_events","type":"Organization","site_admin":false},"repo":{"id":25471132,"name":"logstash-output-s3","full_name":"logstash-plugins/logstash-output-s3","owner":{"login":"logstash-plugins","id":8961934,"avatar_url":"https://avatars.githubusercontent.com/u/8961934?v=3","gravatar_id":"","url":"https://api.github.com/users/logstash-plugins","html_url":"https://github.com/logstash-plugins","followers_url":"https://api.github.com/users/logstash-plugins/followers","following_url":"https://api.github.com/users/logstash-plugins/following{/other_user}","gists_url":"https://api.github.com/users/logstash-plugins/gists{/gist_id}","starred_url":"https://api.github.com/users/logstash-plugins/starred{/owner}{/repo}","subscriptions_url":"https://api.github.com/users/logstash-plugins/subscriptions","organizations_url":"https://api.github.com/users/logstash-plugins/orgs","repos_url":"https://api.github.com/users/logstash-plugins/repos","events_url":"https://api.github.com/users/logstash-plugins/events{/privacy}","received_events_url":"https://api.github.com/users/logstash-plugins/received_events","type":"Organization","site_admin":false},"private":false,"html_url":"https://github.com/logstash-plugins/logstash-output-s3","description":null,"fork":false,"url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3","forks_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/forks","keys_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/keys{/key_id}","collaborators_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/collaborators{/collaborator}","teams_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/teams","hooks_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/hooks","issue_events_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/issues/events{/number}","events_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/events","assignees_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/assignees{/user}","branches_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/branches{/branch}","tags_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/tags","blobs_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/git/blobs{/sha}","git_tags_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/git/tags{/sha}","git_refs_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/git/refs{/sha}","trees_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/git/trees{/sha}","statuses_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/statuses/{sha}","languages_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/languages","stargazers_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/stargazers","contributors_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/contributors","subscribers_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/subscribers","subscription_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/subscription","commits_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/commits{/sha}","git_commits_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/git/commits{/sha}","comments_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/comments{/number}","issue_comment_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/issues/comments/{number}","contents_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/contents/{+path}","compare_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/compare/{base}...{head}","merges_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/merges","archive_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/{archive_format}{/ref}","downloads_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/downloads","issues_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/issues{/number}","pulls_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/pulls{/number}","milestones_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/milestones{/number}","notifications_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/notifications{?since,all,participating}","labels_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/labels{/name}","releases_url":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/releases{/id}","created_at":"2014-10-20T14:49:40Z","updated_at":"2014-11-19T14:45:11Z","pushed_at":"2014-11-19T20:11:57Z","git_url":"git://github.com/logstash-plugins/logstash-output-s3.git","ssh_url":"git@github.com:logstash-plugins/logstash-output-s3.git","clone_url":"https://github.com/logstash-plugins/logstash-output-s3.git","svn_url":"https://github.com/logstash-plugins/logstash-output-s3","homepage":null,"size":207,"stargazers_count":1,"watchers_count":1,"language":"Ruby","has_issues":true,"has_downloads":true,"has_wiki":true,"has_pages":false,"forks_count":5,"mirror_url":null,"open_issues_count":5,"forks":5,"open_issues":5,"watchers":1,"default_branch":"master"}},"_links":{"self":{"href":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/pulls/1"},"html":{"href":"https://github.com/logstash-plugins/logstash-output-s3/pull/1"},"issue":{"href":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/issues/1"},"comments":{"href":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/issues/1/comments"},"review_comments":{"href":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/pulls/1/comments"},"review_comment":{"href":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/pulls/comments/{number}"},"commits":{"href":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/pulls/1/commits"},"statuses":{"href":"https://api.github.com/repos/logstash-plugins/logstash-output-s3/statuses/e5b73fde4b2d9535cab7c2a324faca830c792edc"}}}}