Fix timestamp precision mismatch causing S3 input to skip backup and delete operations

CHANGELOG.md

@@ -1,3 +1,6 @@
+## 7.3.4
+  - Use milliseconds timestamp precision in S3 input to fix the skip backup and delete object issue in S3-compatible storage services [#60](https://github.com/logstash-plugins/logstash-integration-aws/pull/60)
+
 ## 7.3.3
   - Replace deprecated `Aws::S3::Object#upload_file` in favor of `Aws::S3::TransferManager#upload_file` [#67](https://github.com/logstash-plugins/logstash-integration-aws/pull/67)
 

VERSION

@@ -1 +1 @@
-7.3.3
+7.3.4

lib/logstash/inputs/s3.rb

@@ -148,7 +148,7 @@ def list_new_files
           @logger.debug('Ignoring', :key => log.key)
         elsif log.content_length <= 0
           @logger.debug('Object Zero Length', :key => log.key)
-        elsif log.last_modified <= sincedb_time
+        elsif log.last_modified.to_i <= sincedb_time.to_i
           @logger.debug('Object Not Modified', :key => log.key)
         elsif log.last_modified > (current_time - @cutoff_second).utc # recently modified files will be processed in next cycle
           @logger.debug('Object Modified After Cutoff Time', :key => log.key)
@@ -380,7 +380,7 @@ def process_log(queue, log)
     filename = File.join(temporary_directory, File.basename(log.key))
     if download_remote_file(object, filename)
       if process_local_log(queue, filename, object)
-        if object.last_modified == log.last_modified
+        if object.last_modified.to_i == log.last_modified.to_i
           backup_to_bucket(object)
           backup_to_dir(filename)
           delete_file_from_bucket(object)

spec/inputs/s3_spec.rb

@@ -412,28 +412,14 @@
 
     context "when event doesn't have a `message` field" do
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'json.log') }
-      let(:config) {
-        {
-          "access_key_id" => "1234",
-          "secret_access_key" => "secret",
-          "bucket" => "logstash-test",
-          "codec" => "json",
-        }
-      }
+      let(:config) { super().merge({ "codec" => "json" }) }
 
       include_examples "generated events"
     end
 
     context "when event does have a `message` field" do
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'json_with_message.log') }
-      let(:config) {
-        {
-          "access_key_id" => "1234",
-          "secret_access_key" => "secret",
-          "bucket" => "logstash-test",
-          "codec" => "json",
-        }
-      }
+      let(:config) { super().merge({ "codec" => "json" }) }
 
       include_examples "generated events"
     end
@@ -476,14 +462,11 @@
 
     context 'multi-line' do
       let(:log_file) { File.join(File.dirname(__FILE__), '..', 'fixtures', 'multiline.log') }
-       let(:config) {
-           {
-              "access_key_id" => "1234",
-              "secret_access_key" => "secret",
-              "bucket" => "logstash-test",
-              "codec" => LogStash::Codecs::Multiline.new( {"pattern" => "__SEPARATOR__", "negate" => "true",  "what" => "previous"})
-           }
-        }
+      let(:config) {
+        super().merge({
+          "codec" => LogStash::Codecs::Multiline.new( {"pattern" => "__SEPARATOR__", "negate" => "true",  "what" => "previous"})
+        })
+      }
 
       include_examples "generated events"
     end