MINIFICPP-2342 Do not overwrite config files during an upgrade

README.md

@@ -457,8 +457,22 @@ The performance tests can similarly be enabled. To execute them and see their ou
 $ ctest --verbose -L performance
 ```
 
+
 ### Configuring
-The 'conf' directory in the installation root contains a template config.yml document, minifi.properties, and minifi-log.properties. Please see our [Configuration document](CONFIGURE.md) for details on how to configure agents.
+The 'conf' directory in the installation root contains all configuration files.
+
+The files conf/minifi.properties, conf/minifi-log.properties and conf/minifi-uid.properties contain key-value pair configuration settings;
+these are the default settings supplied by the latest MiNiFi version. If you would like to modify these, you should create a corresponding
+.d directory (e.g. conf/minifi.properties.d) and put your settings in a new file inside this directory. These files are read and applied
+in lexicographic order, after the default settings file.
+The Windows installer creates a conf/minifi.properties.d/10_installer.properties file, which contains C2 connection settings.
+If C2 is enabled and settings are added/modified from the C2 server, these will be saved in conf/minifi.properties.d/90_c2.properties.
+
+The conf/config.yml file contains the flow definition (i.e. the layout of processors, controller services etc). When you start MiNiFi for
+the first time, the flow will be fetched from the C2 server (if available), or a file containing an empty flow will be created by MiNiFi.
+
+Please see our [Configuration document](CONFIGURE.md) for details on how to configure agents.
+
 
 ### Installing as a service
 

conf/CMakeLists.txt

@@ -41,12 +41,6 @@ else()
     message(FATAL_ERROR "Invalid MINIFI_PACKAGING_TYPE")
 endif()
 
-configure_file(
-        config.yml.in
-        ${CMAKE_BINARY_DIR}/conf/config.yml
-        @ONLY
-)
-
 configure_file(
         minifi.properties.in
         ${CMAKE_BINARY_DIR}/conf/minifi.properties
@@ -75,15 +69,13 @@ if (MINIFI_PACKAGING_TYPE STREQUAL "RPM")
             ${CMAKE_BINARY_DIR}/conf/minifi.properties
             ${CMAKE_BINARY_DIR}/conf/minifi-log.properties
             ${CMAKE_BINARY_DIR}/conf/minifi-uid.properties
-            ${CMAKE_BINARY_DIR}/conf/config.yml
             DESTINATION /${CMAKE_INSTALL_SYSCONFDIR}/${PROJECT_NAME}
             COMPONENT bin)
 elseif (MINIFI_PACKAGING_TYPE STREQUAL "TGZ")
     install(FILES
             ${CMAKE_BINARY_DIR}/conf/minifi.properties
             ${CMAKE_BINARY_DIR}/conf/minifi-log.properties
             ${CMAKE_BINARY_DIR}/conf/minifi-uid.properties
-            ${CMAKE_BINARY_DIR}/conf/config.yml
             DESTINATION conf
             COMPONENT bin)
 else()

conf/minifi-log.properties.in

@@ -13,6 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+###  NOTE: Changes to this file are overwritten on upgrade. Add your custom settings to new files     ###
+###  in the conf/minifi-log.properties.d directory instead. These custom settings files should have   ###
+###  a name ending with ".properties", and they are applied in lexicographic order, after this file.  ###
+###  Example filename: conf/minifi-log.properties.d/90-log-to-stderr.properties                       ###
+
 #More verbose pattern by default
 #Format details at https://github.com/gabime/spdlog/wiki/3.-Custom-formatting
 spdlog.pattern=[%Y-%m-%d %H:%M:%S.%e] [%n] [%l] %v

conf/minifi-uid.properties.in

@@ -13,6 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+###  NOTE: Changes to this file are overwritten on upgrade. Add your custom settings to new files     ###
+###  in the conf/minifi-uid.properties.d directory instead. These custom settings files should have   ###
+###  a name ending with ".properties", and they are applied in lexicographic order, after this file.  ###
+###  Example filename: conf/minifi-uid.properties.d/10-use-random.properties                          ###
+
 # Implementation for uid generation.
 # Valid values:
 # time - use uuid_generate_time

conf/minifi.properties.in

@@ -13,6 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+###  NOTE: Changes to this file are overwritten on upgrade. Add your custom settings to new files     ###
+###  in the conf/minifi.properties.d directory instead. These custom settings files should have       ###
+###  a name ending with ".properties", and they are applied in lexicographic order, after this file.  ###
+###  Example filenames: conf/minifi.properties.d/10-c2.properties, conf/minifi.properties.d/90-repository-tweaks.properties  ###
+
 # Core Properties #
 nifi.flow.configuration.file=@MINIFI_PATH_FLOW_CONFIG@
 nifi.administrative.yield.duration=30 sec

core-framework/include/utils/file/FileUtils.h

@@ -242,6 +242,12 @@ inline int copy_file(const std::filesystem::path& path_from, const std::filesyst
   return 0;
 }
 
+inline bool move_file(const std::filesystem::path& source_path, const std::filesystem::path& dest_path) {
+  std::error_code ec;
+  std::filesystem::rename(source_path, dest_path, ec);
+  return !ec;
+}
+
 inline void addFilesMatchingExtension(const std::shared_ptr<core::logging::Logger> &logger,
                                       const std::filesystem::path& originalPath,
                                       const std::filesystem::path& extension,

docker/test/integration/cluster/containers/MinifiContainer.py

@@ -238,9 +238,6 @@ def deploy(self):
         else:
             image = 'apacheminificpp:' + MinifiContainer.MINIFI_TAG_PREFIX + MinifiContainer.MINIFI_VERSION
 
-        if self.options.use_flow_config_from_url:
-            self.command = ["/bin/sh", "-c", "rm " + MinifiContainer.MINIFI_LOCATIONS.config_path + " && " + MinifiContainer.MINIFI_LOCATIONS.run_minifi_cmd]
-
         ports = {}
         if self.options.enable_prometheus or self.options.enable_prometheus_with_ssl:
             ports = {'9936/tcp': 9936}