From 95d21fa264e2f1e760db38232331ac14ffe2cb08 Mon Sep 17 00:00:00 2001 From: turcsanyi Date: Thu, 22 Nov 2018 12:34:14 +0100 Subject: [PATCH] FLUME-3271 Update User Guide to follow log4j2 upgrade changes Also add support for configuration via system properties (similar to the Flume system properties used with Log4j 1.x): - flume.log.dir - flume.log.file - flume.root.logger.level (formerly: flume.root.logger) - flume.root.logger.appender (formerly: flume.root.logger) --- conf/log4j2.xml | 13 ++++++------ flume-ng-doc/sphinx/FlumeUserGuide.rst | 21 ++++++++++--------- .../apache/flume/test/util/StagedInstall.java | 12 +++++++---- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/conf/log4j2.xml b/conf/log4j2.xml index d203a24d97..64675b505b 100644 --- a/conf/log4j2.xml +++ b/conf/log4j2.xml @@ -18,13 +18,14 @@ --> - ./logs + ${sys:flume.log.dir:-./logs} + ${sys:flume.log.file:-flume.log} - + @@ -34,7 +35,7 @@ - + @@ -44,13 +45,13 @@ - + - - + + \ No newline at end of file diff --git a/flume-ng-doc/sphinx/FlumeUserGuide.rst b/flume-ng-doc/sphinx/FlumeUserGuide.rst index 163ec49256..b7812a6758 100644 --- a/flume-ng-doc/sphinx/FlumeUserGuide.rst +++ b/flume-ng-doc/sphinx/FlumeUserGuide.rst @@ -195,11 +195,11 @@ several named agents; when a given Flume process is launched a flag is passed te Given this configuration file, we can start Flume as follows:: - $ bin/flume-ng agent --conf conf --conf-file example.conf --name a1 -Dflume.root.logger=INFO,console + $ bin/flume-ng agent --conf conf --conf-file example.conf --name a1 -Dflume.root.logger.level=INFO -Dflume.root.logger.appender=Console -Note that in a full deployment we would typically include one more option: ``--conf=``. -The ```` directory would include a shell script *flume-env.sh* and potentially a log4j properties file. -In this example, we pass a Java option to force Flume to log to the console and we go without a custom environment script. +Note that in a full deployment we typically include the option: ``--conf=``. +The ```` directory would include a shell script *flume-env.sh* and a *log4j2.xml* file. +In this example, we pass a Java option to force Flume to log to the console. From a separate terminal, we can then telnet port 44444 and send Flume an event: @@ -237,7 +237,8 @@ NB: it currently works for values only, not for keys. (Ie. only on the "right si This can be enabled via Java system properties on agent invocation by setting `propertiesImplementation = org.apache.flume.node.EnvVarResolverProperties`. For example:: - $ NC_PORT=44444 bin/flume-ng agent --conf conf --conf-file example.conf --name a1 -Dflume.root.logger=INFO,console -DpropertiesImplementation=org.apache.flume.node.EnvVarResolverProperties + + $ NC_PORT=44444 bin/flume-ng agent --conf conf --conf-file example.conf --name a1 -Dflume.root.logger.level=INFO -Dflume.root.logger.appender=Console -DpropertiesImplementation=org.apache.flume.node.EnvVarResolverProperties Note the above is just an example, environment variables can be configured in other ways, including being set in `conf/flume-env.sh`. @@ -256,20 +257,20 @@ connected to a `Logger Sink`_, which will output all event data to the Flume log In some situations, however, this approach is insufficient. In order to enable logging of event- and configuration-related data, some Java system properties -must be set in addition to log4j properties. +must be set in addition to log4j2 configuration. To enable configuration-related logging, set the Java system property ``-Dorg.apache.flume.log.printconfig=true``. This can either be passed on the command line or by setting this in the ``JAVA_OPTS`` variable in *flume-env.sh*. To enable data logging, set the Java system property ``-Dorg.apache.flume.log.rawdata=true`` -in the same way described above. For most components, the log4j logging level must also be set to +in the same way described above. For most components, the log4j2 logging level must also be set to DEBUG or TRACE to make event-specific logging appear in the Flume logs. Here is an example of enabling both configuration logging and raw data logging while also -setting the Log4j loglevel to DEBUG for console output:: +setting the log4j2 logging level to DEBUG for console output:: - $ bin/flume-ng agent --conf conf --conf-file example.conf --name a1 -Dflume.root.logger=DEBUG,console -Dorg.apache.flume.log.printconfig=true -Dorg.apache.flume.log.rawdata=true + $ bin/flume-ng agent --conf conf --conf-file example.conf --name a1 -Dflume.root.logger.level=DEBUG -Dflume.root.logger.appender=Console -Dorg.apache.flume.log.printconfig=true -Dorg.apache.flume.log.rawdata=true Zookeeper based Configuration @@ -287,7 +288,7 @@ Following is how the Zookeeper Node tree would look like for agents a1 and a2 Once the configuration file is uploaded, start the agent with following options - $ bin/flume-ng agent --conf conf -z zkhost:2181,zkhost1:2181 -p /flume --name a1 -Dflume.root.logger=INFO,console + $ bin/flume-ng agent --conf conf -z zkhost:2181,zkhost1:2181 -p /flume --name a1 -Dflume.root.logger.level=INFO -Dflume.root.logger.appender=Console ================== ================ ========================================================================= Argument Name Default Description diff --git a/flume-ng-tests/src/test/java/org/apache/flume/test/util/StagedInstall.java b/flume-ng-tests/src/test/java/org/apache/flume/test/util/StagedInstall.java index e5aa0cb203..44c0a9cd9f 100644 --- a/flume-ng-tests/src/test/java/org/apache/flume/test/util/StagedInstall.java +++ b/flume-ng-tests/src/test/java/org/apache/flume/test/util/StagedInstall.java @@ -55,9 +55,11 @@ public class StagedInstall { "flume.dist.tarball"; public static final String ENV_FLUME_LOG_DIR = "flume.log.dir"; - public static final String ENV_FLUME_ROOT_LOGGER = "flume.root.logger"; - public static final String ENV_FLUME_ROOT_LOGGER_VALUE = "DEBUG,LOGFILE"; public static final String ENV_FLUME_LOG_FILE = "flume.log.file"; + public static final String ENV_FLUME_ROOT_LOGGER_LEVEL = "flume.root.logger.level"; + public static final String ENV_FLUME_ROOT_LOGGER_LEVEL_VALUE = "DEBUG"; + public static final String ENV_FLUME_ROOT_LOGGER_APPENDER = "flume.root.logger.appender"; + public static final String ENV_FLUME_ROOT_LOGGER_APPENDER_VALUE = "LogFile"; private final File stageDir; private final File baseDir; @@ -156,9 +158,11 @@ public synchronized void startAgent( builder.add("--conf-file", configFilePath); builder.add("--name", agentName); builder.add("-D" + ENV_FLUME_LOG_DIR + "=" + logDirPath); - builder.add("-D" + ENV_FLUME_ROOT_LOGGER + "=" - + ENV_FLUME_ROOT_LOGGER_VALUE); builder.add("-D" + ENV_FLUME_LOG_FILE + "=" + logFileName); + builder.add("-D" + ENV_FLUME_ROOT_LOGGER_LEVEL + "=" + + ENV_FLUME_ROOT_LOGGER_LEVEL_VALUE); + builder.add("-D" + ENV_FLUME_ROOT_LOGGER_APPENDER + "=" + + ENV_FLUME_ROOT_LOGGER_APPENDER_VALUE); commandOptions.forEach((key, value) -> builder.add(key, value));