From 80325c6e0f6c906d063a680e4f3f780494a4c243 Mon Sep 17 00:00:00 2001 From: oldbelvey Date: Tue, 26 Mar 2019 23:25:09 +0800 Subject: [PATCH 1/3] FLUME-3325 fixed TaildirSource oom issue when tailing a lot of files where some file is writing frequently but the others are idle can cause the idleInodes List increasing infinitely, change it to set(list backend) can fix this problem. --- .../java/org/apache/flume/source/taildir/TaildirSource.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/flume-ng-sources/flume-taildir-source/src/main/java/org/apache/flume/source/taildir/TaildirSource.java b/flume-ng-sources/flume-taildir-source/src/main/java/org/apache/flume/source/taildir/TaildirSource.java index 9ecccd7487..57d880dcb8 100644 --- a/flume-ng-sources/flume-taildir-source/src/main/java/org/apache/flume/source/taildir/TaildirSource.java +++ b/flume-ng-sources/flume-taildir-source/src/main/java/org/apache/flume/source/taildir/TaildirSource.java @@ -28,12 +28,15 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; + import org.apache.flume.ChannelException; import org.apache.flume.Context; import org.apache.flume.Event; @@ -82,7 +85,7 @@ public class TaildirSource extends AbstractSource implements private boolean cachePatternMatching; private List existingInodes = new CopyOnWriteArrayList(); - private List idleInodes = new CopyOnWriteArrayList(); + private Set idleInodes = new CopyOnWriteArraySet(); private Long backoffSleepIncrement; private Long maxBackOffSleepInterval; private boolean fileHeader; From 6032128d2ee941890f48b0e63ccc89d5a286bd46 Mon Sep 17 00:00:00 2001 From: belvey Date: Wed, 27 Mar 2019 07:33:22 +0800 Subject: [PATCH 2/3] Revert "FLUME-3325 fixed TaildirSource oom issue" This reverts commit 80325c6e0f6c906d063a680e4f3f780494a4c243. --- .../java/org/apache/flume/source/taildir/TaildirSource.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/flume-ng-sources/flume-taildir-source/src/main/java/org/apache/flume/source/taildir/TaildirSource.java b/flume-ng-sources/flume-taildir-source/src/main/java/org/apache/flume/source/taildir/TaildirSource.java index 57d880dcb8..9ecccd7487 100644 --- a/flume-ng-sources/flume-taildir-source/src/main/java/org/apache/flume/source/taildir/TaildirSource.java +++ b/flume-ng-sources/flume-taildir-source/src/main/java/org/apache/flume/source/taildir/TaildirSource.java @@ -28,15 +28,12 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Set; -import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; - import org.apache.flume.ChannelException; import org.apache.flume.Context; import org.apache.flume.Event; @@ -85,7 +82,7 @@ public class TaildirSource extends AbstractSource implements private boolean cachePatternMatching; private List existingInodes = new CopyOnWriteArrayList(); - private Set idleInodes = new CopyOnWriteArraySet(); + private List idleInodes = new CopyOnWriteArrayList(); private Long backoffSleepIncrement; private Long maxBackOffSleepInterval; private boolean fileHeader; From 3e186a5d4ed0fa6ae09a75a4653899c59708e65c Mon Sep 17 00:00:00 2001 From: oldbelvey Date: Wed, 27 Mar 2019 07:38:16 +0800 Subject: [PATCH 3/3] FLUME-3325 fixed TaildirSource oom issue when tailing a lot of files where some file is writing frequently but the others are idle can cause the idleInodes List increasing infinitely, change it to set(list backend) can fix this problem. --- .../java/org/apache/flume/source/taildir/TaildirSource.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flume-ng-sources/flume-taildir-source/src/main/java/org/apache/flume/source/taildir/TaildirSource.java b/flume-ng-sources/flume-taildir-source/src/main/java/org/apache/flume/source/taildir/TaildirSource.java index 9ecccd7487..83602c36e5 100644 --- a/flume-ng-sources/flume-taildir-source/src/main/java/org/apache/flume/source/taildir/TaildirSource.java +++ b/flume-ng-sources/flume-taildir-source/src/main/java/org/apache/flume/source/taildir/TaildirSource.java @@ -28,6 +28,8 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -82,7 +84,7 @@ public class TaildirSource extends AbstractSource implements private boolean cachePatternMatching; private List existingInodes = new CopyOnWriteArrayList(); - private List idleInodes = new CopyOnWriteArrayList(); + private Set idleInodes = new CopyOnWriteArraySet(); private Long backoffSleepIncrement; private Long maxBackOffSleepInterval; private boolean fileHeader;