Introduce multi-slice NuGraph2 inference and filtering

CMakeLists.txt

@@ -69,6 +69,7 @@ find_package(lardata REQUIRED )
 find_package(larsim REQUIRED )
 find_package(larevt REQUIRED )
 find_package(larreco REQUIRED )
+find_package(larrecodnn REQUIRED )
 find_package(larana REQUIRED )
 find_package(larpandora REQUIRED )
 find_package(larpandoracontent REQUIRED )

fcl/caf/cafmaker_defs.fcl

@@ -319,6 +319,9 @@ cafmaker.SystWeightLabels: ["genieweight", "fluxweight"]
 cafmaker.SaveGENIEEventRecord: true # save GENIE event record by default. Turn this off for data cafmaker fcl
 cafmaker.TPCPMTBarycenterMatchLabel: "tpcpmtbarycentermatch"
 cafmaker.TrackHitFillRREndCut: 30 # include entire PID region
+cafmaker.NuGraphSlicesLabel: "NCCSlices"
+cafmaker.NuGraphFilterLabel: "NGMultiSlice:filter"
+cafmaker.NuGraphSemanticLabel: "NGMultiSlice:semantic"
 
 # Add CAFMaker to the list of producers
 caf_preprocess_producers.cafmaker: @local::cafmaker

fcl/caf/cafmakerjob_icarus_data_NuGraphReco.fcl

@@ -0,0 +1,43 @@
+#include "services_common_icarus.fcl"
+#include "channelmapping_icarus.fcl"
+
+#include "correctionservices_icarus.fcl"
+
+#include "cafmaker_defs.fcl"
+
+process_name: CAFmaker
+
+services:
+{
+  @table::icarus_wirecalibration_minimum_services
+  @table::icarus_random_services  # from services_common_icarus.fcl
+
+  SpaceChargeService: @local::icarus_spacecharge
+}
+
+physics:
+{
+
+  producers: {
+    @table::caf_preprocess_data_producers
+  }
+
+  runprod: [ @sequence::caf_preprocess_data_sequence, cafmaker]
+  trigger_paths: [ runprod ] 
+}
+
+physics.producers.cafmaker.G4Label: ""
+physics.producers.cafmaker.GenLabel: ""
+physics.producers.cafmaker.SimChannelLabel: ""
+physics.producers.cafmaker.SystWeightLabels: []
+physics.producers.cafmaker.SaveGENIEEventRecord: false
+
+physics.producers.cafmaker.TriggerLabel: "daqTrigger" # the general configuration, for MC, has a different one (see also https://github.com/SBNSoftware/icaruscode/issues/556)
+
+## Grab the second-pass Pandora reconstruction, after NuGraph2's filter
+physics.producers.cafmaker.PFParticleLabel: "pandoraGausNuGraphReco"
+
+## Use NuGraph2's PID after NuGraph2's filter
+physics.producers.cafmaker.UsePandoraAfterNuGraph: true
+physics.producers.cafmaker.NuGraphFilterLabel: "ngfilteredhits:filter"
+physics.producers.cafmaker.NuGraphSemanticLabel: "ngfilteredhits:semantic"

fcl/caf/cafmakerjob_icarus_detsim2d_NuGraphReco.fcl

@@ -0,0 +1,13 @@
+#include "cafmakerjob_icarus.fcl"
+#include "cafmaker_add_detsim2d_icarus.fcl"
+
+## Always refer to the original hit collection
+physics.producers.cafmaker.HitLabel: "cluster3D"
+
+## Grab the second-pass Pandora reconstruction, after NuGraph2's filter
+physics.producers.cafmaker.PFParticleLabel: "pandoraGausNuGraphReco"
+
+## Use NuGraph2's PID after NuGraph2's filter
+physics.producers.cafmaker.UsePandoraAfterNuGraph: true
+physics.producers.cafmaker.NuGraphFilterLabel: "ngfilteredhits:filter"
+physics.producers.cafmaker.NuGraphSemanticLabel: "ngfilteredhits:semantic"

fcl/reco/Definitions/stage1_icarus_defs.fcl

@@ -22,6 +22,8 @@
 #include "supera_modules.fcl"
 #include "crtpmtmatching_parameters.fcl"
 #include "tpcpmtbarycentermatch_config.fcl"
+#include "pandoramodules_icarus.fcl"
+#include "nugraph_icarus.fcl"
 
 BEGIN_PROLOG
 
@@ -58,10 +60,10 @@ icarus_stage1_producers:
   SBNShowerGausCryoE:             @local::icarus_pandorashower_3dTraj
 
   # pandora CALO and PID
-#  pandoraGausCaloCryoW:		      @local::icarus_calomc
-#  pandoraGausPidCryoW:		        @local::icarus_chi2pid
-#  pandoraGausCaloCryoE:		      @local::icarus_calomc
-#  pandoraGausPidCryoE:		        @local::icarus_chi2pid
+  #  pandoraGausCaloCryoW:		    @local::icarus_calomc
+  #  pandoraGausPidCryoW:		      @local::icarus_chi2pid
+  #  pandoraGausCaloCryoE:		    @local::icarus_calomc
+  #  pandoraGausPidCryoE:		      @local::icarus_chi2pid
 
   # Placeholder uncalibrated calorimetry
   caloskimCalorimetryCryoE:       @local::caloskim_calorimetry
@@ -74,12 +76,42 @@ icarus_stage1_producers:
   fmatchopCryoE:                  @local::icarus_simple_flashmatch_E_op
   fmatchopCryoW:                  @local::icarus_simple_flashmatch_W_op
 
- ## crt producer
-  crttrack:          @local::standard_crttrackproducer
-  CRTT0Tagging:         @local::icarus_crtt0tagging_data
+  ## crt producer
+  crttrack:     @local::standard_crttrackproducer
+  CRTT0Tagging: @local::icarus_crtt0tagging_data
 
   tpcpmtbarycentermatchCryoE: @local::data_tpcpmtbarycentermatchproducer_east
   tpcpmtbarycentermatchCryoW: @local::data_tpcpmtbarycentermatchproducer_west
+
+  ## NuGraph
+  nuslhitsCryoE: @local::nuslhitsCryoE
+  nuslhitsCryoW: @local::nuslhitsCryoW
+
+  NuGraphCryoE: @local::NuGraphCryoE
+  NuGraphCryoW: @local::NuGraphCryoW
+
+  NCCSlicesCryoE: @local::NCCSlicesCryoE
+  NCCSlicesCryoW: @local::NCCSlicesCryoW
+
+  NGMultiSliceCryoE: @local::NGMultiSliceCryoE
+  NGMultiSliceCryoW: @local::NGMultiSliceCryoW
+
+  ngfilteredhitsCryoE: @local::ngfilteredhitsCryoE
+  ngfilteredhitsCryoW: @local::ngfilteredhitsCryoW
+
+  ## Pandora, when running after NuGraph
+  pandoraGausNuGraphRecoCryoE: {
+    @table::icarus_pandora
+    HitFinderModuleLabel:   "ngfilteredhitsCryoE"
+    # CollectHitPredictions:  true
+    # UseHitPredictions:      true
+  }
+  pandoraGausNuGraphRecoCryoW: {
+    @table::icarus_pandora
+    HitFinderModuleLabel:   "ngfilteredhitsCryoW"
+    # CollectHitPredictions:  true
+    # UseHitPredictions:      true
+  }
 }
 
 icarus_stage1_filters:
@@ -98,7 +130,7 @@ icarus_stage1_analyzers:
   superaNu:        @local::icarus_supera_generator_PMT_CRT
   superaMPVMPR:	   @local::icarus_supera_MC_MPVMPR_cryoE_PMT_CRT
   superaCosmic:    @local::icarus_supera_cosmgen_all_cryo_PMT_CRT
-  CRTDataAnalysis: 
+  CRTDataAnalysis:
    {
      module_type:   "CRTDataAnalysis"
      CRTHitLabel:   "crthit"
@@ -128,11 +160,11 @@ icarus_stage1_analyzers:
      #  Note1: in the plots a 500 ns offset was added to have start of the gate right at 0.
      #  Note2: the inBeam parameters were determined experimentally by selecting the beam
      #  excess time window. At the current state of the CRTPMT filter, the requirement to
-     #  have the optical flashes within the BeamExcess time window is not applied. 
+     #  have the optical flashes within the BeamExcess time window is not applied.
 
       # note 19/04/2023: parameters are set to run2,
-      # they can be set to run1, but currently values are the same 
-   
+      # they can be set to run1, but currently values are the same
+
       @table::CRTMatchBNBBeamGate_run2
       @table::CRTMatchNuMIBeamGate_run2
       @table::CRTPMTmatchingparams_standard
@@ -154,15 +186,15 @@ icarus_stage1_analyzers_crthittagged.caloskimW.TopCRTDistanceCut_throughgoing: 1
 ### Below are a list of convenient sequences that can be used for production/typical users. ###
 
 # Set up the standard analysis chain
-icarus_analysis_modules:           [  caloskimE 
-                                     ,caloskimW 
+icarus_analysis_modules:           [  caloskimE
+                                     ,caloskimW
                                      ,simpleLightAna
                                      ,CRTDataAnalysis
 				                             ,CRTAnalysis
                                    ]
 
-icarus_analysis_modules_nolight:      [  caloskimE 
-                                     ,caloskimW 
+icarus_analysis_modules_nolight:   [  caloskimE
+                                     ,caloskimW
                                      ,CRTDataAnalysis
 				                             ,CRTAnalysis
                                    ]
@@ -178,10 +210,12 @@ icarus_analysis_superaNu:          [ superaNu
 
 icarus_analysis_superaMPVMPR:      [ superaMPVMPR
                                    ]
+
 icarus_analysis_superaCosmic:      [ superaCosmic
-				   ]
+				                           ]
+
 icarus_analysis_larcv_modules:     [  @sequence::icarus_analysis_modules
-                                     ,@sequence::icarus_analysis_supera 
+                                     ,@sequence::icarus_analysis_supera
                                    ]
 
 icarus_EastHits_TPC:               [ gaushit1dTPCEW,
@@ -191,6 +225,7 @@ icarus_EastHits_TPC:               [ gaushit1dTPCEW,
 icarus_WestHits_TPC:               [ gaushit1dTPCWW,
                                      gaushit1dTPCWE
                                    ]
+
 icarus_EastHits2d_TPC:             [ gaushit2dTPCEW,
                                      gaushit2dTPCEE
                                    ]
@@ -201,7 +236,7 @@ icarus_WestHits2d_TPC:             [ gaushit2dTPCWW,
 
 # Set up filtering of cluster3D hits by cryostat
 # Changed slightly to faciliate larcv processing
-icarus_filter1D_cluster3D:         [ 
+icarus_filter1D_cluster3D:         [
                                      @sequence::icarus_EastHits_TPC,
                                      @sequence::icarus_WestHits_TPC,
                                      cluster3DCryoE,
@@ -210,7 +245,7 @@ icarus_filter1D_cluster3D:         [
                                      TPCHitFilterCryoW
                                    ]
 
-icarus_filter2D_cluster3D:         [ 
+icarus_filter2D_cluster3D:         [
                                      @sequence::icarus_EastHits2d_TPC,
                                      @sequence::icarus_WestHits2d_TPC,
                                      cluster3DCryoE,
@@ -224,37 +259,61 @@ icarus_reco_cluster3DCryoW:        [ cluster3DCryoW ]
 
 icarus_reco_cluster3DCryoE:        [ cluster3DCryoE ]
 
-icarus_reco_pandoraGausCryoW:      [ pandoraGausCryoW,
+icarus_reco_pandoraGausCryoW:      [ pandoraGausCryoW ]
+
+icarus_reco_pandoraGausCryoE:      [ pandoraGausCryoE ]
+
+icarus_reco_postPandoraGausCryoW:  [ NCCSlicesCryoW,
+                                     NGMultiSliceCryoW,
                                      pandoraTrackGausCryoW,
                                      pandoraKalmanTrackGausCryoW,
                                      SBNShowerGausCryoW
                                    ]
 
-icarus_reco_pandoraGausCryoE:      [ pandoraGausCryoE,
+icarus_reco_postPandoraGausCryoW_NuGraphReco:   [ NCCSlicesCryoW,
+                                                  NGMultiSliceCryoW,
+                                                  ngfilteredhitsCryoW,
+                                                  pandoraGausNuGraphRecoCryoW,
+                                                  pandoraTrackGausCryoW,
+                                                  pandoraKalmanTrackGausCryoW,
+                                                  SBNShowerGausCryoW
+                                                ]
+
+icarus_reco_postPandoraGausCryoE:  [ NCCSlicesCryoE,
+                                     NGMultiSliceCryoE,
                                      pandoraTrackGausCryoE,
                                      pandoraKalmanTrackGausCryoE,
                                      SBNShowerGausCryoE
                                    ]
 
+icarus_reco_postPandoraGausCryoE_NuGraphReco:   [ NCCSlicesCryoE,
+                                                  NGMultiSliceCryoE,
+                                                  ngfilteredhitsCryoE,
+                                                  pandoraGausNuGraphRecoCryoE,
+                                                  pandoraTrackGausCryoE,
+                                                  pandoraKalmanTrackGausCryoE,
+                                                  SBNShowerGausCryoE
+                                                ]
+
 icarus_reco_Gauss1D_CryoW:         [ 
                                      @sequence::icarus_WestHits_TPC,
                                      @sequence::icarus_reco_cluster3DCryoW,
                                      @sequence::icarus_reco_pandoraGausCryoW
                                    ]
 
-icarus_reco_Gauss1D_CryoE:         [ 
+icarus_reco_Gauss1D_CryoE:         [
                                      @sequence::icarus_EastHits_TPC,
                                      @sequence::icarus_reco_cluster3DCryoE,
                                      @sequence::icarus_reco_pandoraGausCryoE
                                    ]
 
-icarus_reco_Gauss2D_CryoW:         [ 
+icarus_reco_Gauss2D_CryoW:         [
                                      @sequence::icarus_WestHits2d_TPC,
                                      @sequence::icarus_reco_cluster3DCryoW,
                                      @sequence::icarus_reco_pandoraGausCryoW
                                    ]
 
-icarus_reco_Gauss2D_CryoE:           [ 
+icarus_reco_Gauss2D_CryoE:           [
                                      @sequence::icarus_EastHits2d_TPC,
                                      @sequence::icarus_reco_cluster3DCryoE,
                                      @sequence::icarus_reco_pandoraGausCryoE
@@ -265,14 +324,24 @@ icarus_pandora_Gauss:              [
                                      @sequence::icarus_reco_pandoraGausCryoW
                                    ]
 
+icarus_postPandora_Gauss:          [
+                                     @sequence::icarus_reco_postPandoraGausCryoE,
+                                     @sequence::icarus_reco_postPandoraGausCryoW
+                                   ]
+
+icarus_postPandora_Gauss_NuGraphReco: [
+                                        @sequence::icarus_reco_postPandoraGausCryoE_NuGraphReco,
+                                        @sequence::icarus_reco_postPandoraGausCryoW_NuGraphReco
+                                      ]
+
 #Add flash matching
 icarus_reco_fm:                    [ fmatchCryoE,
                                      fmatchCryoW,
                                      fmatchopCryoE,
                                      fmatchopCryoW ]
 
 icarus_tpcpmtbarycentermatch:      [
-                                     tpcpmtbarycentermatchCryoE, 
+                                     tpcpmtbarycentermatchCryoE,
                                      tpcpmtbarycentermatchCryoW
                                    ]
 
@@ -316,10 +385,10 @@ icarus_stage1_producers.gaushit2dTPCWW.HitFilterAlg.MinPulseHeight:
 
 ## Overrides for filtering of cluster3D hits
 icarus_stage1_filters.TPCHitFilterCryoW.HitDataLabelVec: ["cluster3DCryoW"]
-icarus_stage1_filters.TPCHitFilterCryoW.MaximumHits:     60000 
+icarus_stage1_filters.TPCHitFilterCryoW.MaximumHits:     60000
 
 icarus_stage1_filters.TPCHitFilterCryoE.HitDataLabelVec: ["cluster3DCryoE"]
-icarus_stage1_filters.TPCHitFilterCryoE.MaximumHits:     60000 
+icarus_stage1_filters.TPCHitFilterCryoE.MaximumHits:     60000
 
 ## Definitions for running the 3D clustering by Cryostat
 icarus_stage1_producers.cluster3DCryoW.MakeSpacePointsOnly:                                      true
@@ -338,30 +407,32 @@ icarus_stage1_producers.cluster3DCryoE.Hit3DBuilderAlg.MaxHitChiSquare:
 icarus_stage1_producers.cluster3DCryoE.Hit3DBuilderAlg.MaxMythicalChiSquare:                     30.
 icarus_stage1_producers.cluster3DCryoE.Hit3DBuilderAlg.OutputHistograms:                         false
 
-### Definitions for a pandora by cryostat
+### Definitions for Pandora by cryostat
+icarus_stage1_producers.pandoraGausCryoE.HitFinderModuleLabel:                                   "cluster3DCryoE"
 icarus_stage1_producers.pandoraGausCryoW.HitFinderModuleLabel:                                   "cluster3DCryoW"
-icarus_stage1_producers.pandoraTrackGausCryoW.PFParticleLabel:                                   "pandoraGausCryoW"
-icarus_stage1_producers.pandoraTrackGausCryoW.UseAllParticles:                                   true
-icarus_stage1_producers.pandoraKalmanTrackGausCryoW.inputCollection:                             "pandoraGausCryoW"
-icarus_stage1_producers.pandoraKalmanTrackGausCryoW.trackInputTag:                               "pandoraTrackGausCryoW"
 
-icarus_stage1_producers.pandoraGausCryoE.HitFinderModuleLabel:                                   "cluster3DCryoE"
+### Definitions for Pandora tracks by cryostat
 icarus_stage1_producers.pandoraTrackGausCryoE.PFParticleLabel:                                   "pandoraGausCryoE"
 icarus_stage1_producers.pandoraTrackGausCryoE.UseAllParticles:                                   true
+icarus_stage1_producers.pandoraTrackGausCryoW.PFParticleLabel:                                   "pandoraGausCryoW"
+icarus_stage1_producers.pandoraTrackGausCryoW.UseAllParticles:                                   true
+
 icarus_stage1_producers.pandoraKalmanTrackGausCryoE.inputCollection:                             "pandoraGausCryoE"
 icarus_stage1_producers.pandoraKalmanTrackGausCryoE.trackInputTag:                               "pandoraTrackGausCryoE"
+icarus_stage1_producers.pandoraKalmanTrackGausCryoW.inputCollection:                             "pandoraGausCryoW"
+icarus_stage1_producers.pandoraKalmanTrackGausCryoW.trackInputTag:                               "pandoraTrackGausCryoW"
+
+### Definitions for Pandora showers by cryostat
+icarus_stage1_producers.SBNShowerGausCryoE.PFParticleLabel:                                      "pandoraGausCryoE"
+icarus_stage1_producers.SBNShowerGausCryoE.UseAllParticles:                                      true
+icarus_stage1_producers.SBNShowerGausCryoW.PFParticleLabel:                                      "pandoraGausCryoW"
+icarus_stage1_producers.SBNShowerGausCryoW.UseAllParticles:                                      true
 
 icarus_stage1_producers.caloskimCalorimetryCryoE.TrackModuleLabel:                               "pandoraTrackGausCryoE"
 icarus_stage1_producers.caloskimCalorimetryCryoW.TrackModuleLabel:                               "pandoraTrackGausCryoW"
 
 ## Switch pandora back to just gaushits?
-#icarus_stage1_producers.pandoraGausCryoW.ConfigFile:                                             "PandoraSettings_Master_ICARUS_RawICARUS.xml"
-#icarus_stage1_producers.pandoraGausCryoE.ConfigFile:                                             "PandoraSettings_Master_ICARUS_RawICARUS.xml"
-
-## Definitions for shower finding (both single and by cryostat)
-icarus_stage1_producers.SBNShowerGausCryoW.PFParticleLabel:                                      "pandoraGausCryoW"
-icarus_stage1_producers.SBNShowerGausCryoW.UseAllParticles:                                      true
-icarus_stage1_producers.SBNShowerGausCryoE.PFParticleLabel:                                      "pandoraGausCryoE"
-icarus_stage1_producers.SBNShowerGausCryoE.UseAllParticles:                                      true
+#icarus_stage1_producers.pandoraGausCryoW.ConfigFile:                                            "PandoraSettings_Master_ICARUS_RawICARUS.xml"
+#icarus_stage1_producers.pandoraGausCryoE.ConfigFile:                                            "PandoraSettings_Master_ICARUS_RawICARUS.xml"
 
 END_PROLOG

fcl/reco/Stage1/data/stage1_run2_1d_icarus.fcl

@@ -5,6 +5,7 @@ physics.reco: [ @sequence::icarus_filter1D_cluster3D,
                 @sequence::icarus_pandora_Gauss,
                 @sequence::icarus_reco_fm,
                 @sequence::icarus_tpcpmtbarycentermatch,
+                @sequence::icarus_postPandora_Gauss,
                 @sequence::icarus_crttrack,
                 @sequence::icarus_crtt0tagging,
                 caloskimCalorimetryCryoE, caloskimCalorimetryCryoW]