MINIFICPP-2775 Allow manifest generation for comparison between releases

.github/workflows/ci.yml

@@ -322,6 +322,26 @@ jobs:
           ulimit -c unlimited
           ctest --timeout 300 -j$(nproc) --output-on-failure -LE performance
         working-directory: build
+      - name: Check manifest
+        id: check_manifest
+        working-directory: build
+        run: |
+          sed -i 's|nifi\.extension\.path=\.\./extensions/\*|nifi.extension.path=./*|' conf/minifi.properties
+          ./bin/minifi --manifest | diff -u ../.github/references/ubuntu_22_04_clang_arm_manifest.json - > manifest.patch || {
+            echo "::error::Manifest mismatch! Download the artifact to apply the patch."
+            cat manifest.patch
+            exit 1
+          }
+
+      - name: Upload patch file
+        if: failure() && steps.check_manifest.outcome == 'failure'
+        uses: actions/upload-artifact@v4
+        with:
+          name: update-manifest.patch
+          path: build/manifest.patch
+          if-no-files-found: ignore
+          retention-days: 5
+
       - name: performance-test
         id: performance-test
         run: ctest -j1 --verbose -L performance

.github/workflows/clear-actions-cache.yml

@@ -16,5 +16,5 @@ jobs:
         run: |
           python3 -m venv github_env
           source github_env/bin/activate
-          pip install -r github_scripts/requirements.txt
-          python3 github_scripts/github_actions_cache_cleanup.py -t ${{github.token}} -r ${{github.repository}}
+          pip install -r .github/github_scripts/requirements.txt
+          python3 .github/github_scripts/github_actions_cache_cleanup.py -t ${{github.token}} -r ${{github.repository}}

cmake/ArgParse.cmake

@@ -18,8 +18,8 @@
 include(FetchContent)
 FetchContent_Declare(
     argparse
-    URL https://github.com/p-ranav/argparse/archive/refs/tags/v3.0.tar.gz
-    URL_HASH SHA256=ba7b465759bb01069d57302855eaf4d1f7d677f21ad7b0b00b92939645c30f47
+    URL https://github.com/p-ranav/argparse/archive/refs/tags/v3.2.tar.gz
+    URL_HASH SHA256=9dcb3d8ce0a41b2a48ac8baa54b51a9f1b6a2c52dd374e28cc713bab0568ec98
     SYSTEM
 )
 FetchContent_MakeAvailable(argparse)

minifi_main/AgentDocs.cpp

@@ -28,6 +28,7 @@
 #include "agent/agent_docs.h"
 #include "core/Core.h"
 #include "core/Relationship.h"
+#include "core/state/nodes/AgentInformation.h"
 #include "minifi-cpp/agent/agent_version.h"
 #include "minifi-cpp/core/PropertyValidator.h"
 #include "range/v3/algorithm/lexicographical_compare.hpp"
@@ -41,6 +42,36 @@ namespace {
 
 namespace minifi = org::apache::nifi::minifi;
 
+
+void sortClassDescription(minifi::ClassDescription& class_description) {
+  std::ranges::sort(class_description.class_properties_, {}, &minifi::core::Property::getName);
+  std::ranges::sort(class_description.dynamic_properties_, {}, &minifi::core::DynamicProperty::name);
+  std::ranges::sort(class_description.class_relationships_, {}, &minifi::core::Relationship::getName);
+  std::ranges::sort(class_description.output_attributes_, {}, &minifi::core::OutputAttribute::name);
+  std::ranges::sort(class_description.api_implementations, {}, &minifi::core::ControllerServiceType::type);
+}
+
+void sortComponents(minifi::Components& components) {
+  auto lower_case_short_name = [](const auto& b) { return minifi::utils::string::toLower(b.short_name_); };
+  std::ranges::sort(components.processors, {}, lower_case_short_name);
+  std::ranges::sort(components.controller_services, {}, lower_case_short_name);
+  std::ranges::sort(components.parameter_providers, {}, lower_case_short_name);
+  std::ranges::sort(components.other_components, {}, lower_case_short_name);
+
+  for (auto& processors : components.processors) {
+    sortClassDescription(processors);
+  }
+  for (auto& cs : components.controller_services) {
+    sortClassDescription(cs);
+  }
+  for (auto& pp : components.parameter_providers) {
+    sortClassDescription(pp);
+  }
+  for (auto& oc : components.other_components) {
+    sortClassDescription(oc);
+  }
+}
+
 std::string formatName(std::string_view name_view, bool is_required) {
   std::string name{name_view};
   if (is_required) { return "**" + name + "**"; }
@@ -258,13 +289,6 @@ class ModularDocumentation {
   }
 
  private:
-  static void sortComponents(minifi::Components& components) {
-    auto lower_case_short_name = [](const auto& b) { return minifi::utils::string::toLower(b.short_name_); };
-    std::ranges::sort(components.processors, {}, lower_case_short_name);
-    std::ranges::sort(components.controller_services, {}, lower_case_short_name);
-    std::ranges::sort(components.parameter_providers, {}, lower_case_short_name);
-  }
-
   static void writeComponentParts(std::ostream& os, const std::vector<minifi::ClassDescription>& class_descriptions, const std::string_view h3) {
     if (!class_descriptions.empty()) {
       os << fmt::format("### {}\n\n", h3);
@@ -315,4 +339,19 @@ void AgentDocs::generate(const std::filesystem::path& docs_dir) {
   ModularDocumentation::write(docs_dir);
 }
 
+void AgentDocs::generateManifest() {
+  auto all_components = Components{};
+  for (const auto& components : minifi::ClassDescriptionRegistry::getClassDescriptions() | std::views::values) {
+    std::ranges::copy(components.processors, std::back_inserter(all_components.processors));
+    std::ranges::copy(components.controller_services, std::back_inserter(all_components.controller_services));
+    std::ranges::copy(components.parameter_providers, std::back_inserter(all_components.parameter_providers));
+    std::ranges::copy(components.other_components, std::back_inserter(all_components.other_components));
+  }
+
+  sortComponents(all_components);
+  auto serialized = minifi::state::response::serializeComponentManifest(all_components);
+  for (const auto& ser : serialized) {
+    std::cout << ser.to_pretty_string() << std::endl;
+  }}
+
 }  // namespace org::apache::nifi::minifi::docs

minifi_main/AgentDocs.h

@@ -24,6 +24,7 @@ namespace org::apache::nifi::minifi::docs {
 class AgentDocs {
  public:
   static void generate(const std::filesystem::path& docs_dir);
+  static void generateManifest();
 };
 
 }  // namespace org::apache::nifi::minifi::docs

minifi_main/MiNiFiMain.cpp

@@ -114,12 +114,6 @@ void sigHandler(int signal) {
   }
 }
 
-void dumpDocs(const std::string& dir) {
-  minifi::docs::AgentDocs docsCreator;
-
-  docsCreator.generate(dir);
-}
-
 void writeJsonSchema(std::ostream& out) {
   out << minifi::docs::generateJsonSchema() << '\n';
 }
@@ -148,7 +142,12 @@ void overridePropertiesFromCommandLine(const argparse::ArgumentParser& parser, c
   }
 
   std::cout << "Dumping docs to " << docs_params[0] << std::endl;
-  dumpDocs(docs_params[0]);
+  minifi::docs::AgentDocs::generate(docs_params[0]);
+  return 0;
+}
+
+[[nodiscard]] int generateManifest() {
+  minifi::docs::AgentDocs::generateManifest();
   return 0;
 }
 
@@ -185,6 +184,10 @@ int main(int argc, char **argv) {
     .nargs(1)
     .metavar("DIRECTORY")
     .help("Generate documentation in the specified directory");
+  bool generate_manifest = false;
+  argument_parser.add_argument("--manifest")
+    .store_into(generate_manifest)
+    .help("Generate manifest to stdout");
   argument_parser.add_argument("-s", "--schema")
     .metavar("PATH")
     .default_value("-")
@@ -214,6 +217,9 @@ int main(int argc, char **argv) {
     minifi::setSyslogLogger();
   }
   auto& logger_configuration = core::logging::LoggerConfiguration::getConfiguration();
+  if (generate_manifest) {
+    logger_configuration.disableLogging();
+  }
   const auto logger = logger_configuration.getLogger("main");
   auto startup_timepoint = std::chrono::system_clock::now();
   auto log_runtime = gsl::finally([&]() {
@@ -327,6 +333,9 @@ int main(int argc, char **argv) {
 
     if (const auto maybe_exit_code = dumpDocsIfRequested(argument_parser)) { return *maybe_exit_code; }
     if (const auto maybe_exit_code = writeSchemaIfRequested(argument_parser)) { return *maybe_exit_code; }
+    if (generate_manifest) {
+      return generateManifest();
+    }
 
     std::chrono::milliseconds stop_wait_time = configure->get(minifi::Configure::nifi_graceful_shutdown_seconds)
         | utils::andThen(utils::timeutils::StringToDuration<std::chrono::milliseconds>)