New script for converting JSON dumpstat to Twiki format

ConditionsConsumed/AlCaCondJSONtoTwiki.py

@@ -0,0 +1,98 @@
+############################################
+# get AlCa conditions consumed in the CMSSW
+# https://twiki.cern.ch/twiki/bin/view/CMSPublic/SWGuideCalAliTrigger2023
+############################################
+
+import json
+import argparse
+from contextlib import ExitStack
+
+def checkTagInJson(tag, data):
+    # processing string with "/" separator from tagFile
+    stripped_tag = tag.split('/')[0].strip()
+    requested_label = tag.split('/')[1].strip()
+    for record in data.get(stripped_tag, []):
+        if requested_label in record.get('label', '') and record.get('timeLookupPayloadIds', []):
+            return "%GREEN% Yes %ENDCOLOR%"
+    return ""
+
+
+def getOneRow(tag, jsonFiles):
+    rowName = "|" + tag.strip()
+    for file_ in jsonFiles:
+        with open(file_, 'r') as file:
+            data = json.load(file)
+            checkTagInFile_ = checkTagInJson(tag, data)
+            rowName = rowName + " | " + checkTagInFile_
+    return rowName + " | \n"
+
+
+def printAllRow(tagFile, jsonFiles, output_table, tableTitle):
+    with open(f"outputForTwiki_{output_table}.txt", 'w') as outForTwiki, open(tagFile, 'r') as tag_file:
+        outForTwiki.write(tableTitle)
+        for tag in tag_file:
+            getOneRow_ = getOneRow(tag, jsonFiles)
+            outForTwiki.write(getOneRow_)
+
+def updateTagFile(tagFile, json_files):
+    unique_rows = set()
+
+# Open files in advance using context managers
+    with ExitStack() as stack:
+        files = [stack.enter_context(open(file_, 'r')) for file_ in json_files]
+        data_list = [json.load(file_content) for file_content in files]
+
+# Process data
+    for data in data_list:
+        for record, label_info_list in data.items():
+            for label_info in label_info_list:
+                label = label_info.get('label', '')
+                row = f"{record} / {label}"
+                unique_rows.add(row)
+
+# Sort and write the sorted lines to a file
+    with open(tagFile, 'w') as new_tagFile:
+        new_tagFile.write('\n'.join(sorted(unique_rows, reverse=False)))
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Usage: python3 getAlCaCondInGT.py [options] \n')
+    parser.add_argument('-d', '--data', dest='isData', action='store_true',
+                        default=False, help='Enter --data -d for data, False for MC. Default: False')
+    args = parser.parse_args()
+
+    tagFile = "allAlCaTags.txt"
+    if args.isData:
+        jsonFiles = [
+            "output_step1_L1.json",  # step1_L1
+            "output_step2_HLT.json",  # step2_HLT
+            "output_step3_AOD.json",  # step3_AOD
+            "output_step4_MINIAOD.json",  # step4_MINIAOD
+            "output_step5_NANOAOD.json",  # step5_NANOAOD
+        ]
+    else:
+        jsonFiles = [
+            "output_step1_GEN.json",  # step1_GEN
+            "output_step2_SIM.json",  # step2_SIM
+            "output_step3_DIGI.json",  # step3_DIGI
+            "output_step4_L1.json",  # step4_L1
+            "output_step5_DIGI2RAW.json",  # step5_DIGI2RAW
+            "output_step6_HLT.json",  # step6_HLT
+            "output_step7_AODSIM.json",  # step7_AODSIM
+            "output_step8_MINIAODSIM.json",  # step8_MINIAODSIM
+            "output_step9_NANOAODSIM.json",  # step9_NANOAODSIM
+
+            # Add remaining JSON filenames if available
+        ]
+
+    output_table = "DATA" if args.isData else "MC"
+    tableTitle = "|Tags|L1|HLT|AOD|MINIAOD|NANOAOD|\n" if args.isData else "|Tags|GEN|SIM|DIGI|L1|DIGI2RAW|HLT|AOD|MINIAOD|NANOAOD|\n"
+
+    updateTagFile(tagFile, jsonFiles)
+
+    printAllRow(tagFile, jsonFiles, output_table, tableTitle)
+
+
+if __name__ == '__main__':
+    main()

ConditionsConsumed/runCMSDrivers_data2023DwithJSON.sh

@@ -0,0 +1,30 @@
+#1: L1
+echo "------------------------------"
+echo "Running step-1/5: L1"
+echo "------------------------------"
+cmsDriver.py step1  --conditions auto:run3_hlt_relval -n 5 --era Run3_2023 -s L1REPACK:Full --data --scenario pp --datatier FEVTDEBUGHLT --eventcontent FEVTDEBUGHLT --filein /store/data/Run2023D/JetMET0/RAW/v1/000/369/978/00000/00b9eba7-c847-465b-a6de-98bceae93613.root --fileout output_step1_L1.root --customise_commands='process.load("Configuration.StandardSequences.Digi_cff") \n process.GlobalTag.DumpStat =cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName = cms.untracked.string("output_step1_L1.json")' --outputCommands "keep *" |& tee output_step1_L1.log
+
+#2: HLTlt
+echo "------------------------------"
+echo "Running step-2/5: HLT"
+echo "------------------------------"
+cmsDriver.py step2  --conditions auto:run3_hlt_relval -n 5 --era Run3_2023 -s HLT:@relval2023 --processName HLT2 --data --scenario pp --datatier FEVTDEBUGHLT --eventcontent FEVTDEBUGHLT --filein file:output_step1_L1.root --fileout output_step2_HLT.root --customise_commands='process.GlobalTag.DumpStat=cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName =cms.untracked.string("output_step2_HLT.json")' |& tee output_step2_HLT.log
+#cmsDriver.py step2  --conditions auto:run3_hlt_relval -n 5 --era Run3_2023 -s HLT:@relval2023 --processName HLT2 --data --scenario pp --datatier FEVTDEBUGHLT --eventcontent FEVTDEBUGHLT --filein file:output_step1_L1.root --fileout output_step2_HLT.root --customise_commands='process.GlobalTag.DumpStat=cms.untracked.bool(True)' --outputCommands "keep *, drop  *_hlt*Legacy_*_*" |& tee output_step2_HLT.log
+
+#3 AOD (RAW2DIGI,L1Reco,RECO)
+echo "------------------------------"
+echo "Running step-3/5: AOD"
+echo "------------------------------"
+cmsDriver.py step3 --conditions auto:run3_data_prompt_relval -n 5 --era Run3_2023 -s RAW2DIGI,L1Reco,RECO --data --datatier AOD --eventcontent AOD --filein file:output_step2_HLT.root --fileout output_step3_AOD.root --customise_commands='process.GlobalTag.DumpStat =cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName =cms.untracked.string("output_step3_AOD.json")' --customise Configuration/DataProcessing/RecoTLR.customisePostEra_Run3 |& tee output_step3_AOD.log
+
+#4 MINIAOD
+echo "------------------------------"
+echo "Running step-4/5: MINIAOD"
+echo "------------------------------"
+cmsDriver.py step4 --runUnscheduled --conditions auto:run3_data_prompt_relval -n 5 --era Run3_2023 -s PAT --data --datatier MINIAOD --eventcontent MINIAOD --filein file:output_step3_AOD.root --fileout output_step4_MINIAOD.root --customise_commands='process.GlobalTag.DumpStat =cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName =cms.untracked.string("output_step4_MINIAOD.json")' --customise Configuration/DataProcessing/RecoTLR.customisePostEra_Run3 |& tee output_step4_MINIAOD.log
+
+#5 NANOAOD
+echo "------------------------------"
+echo "Running step-5/5: NANOAOD"
+echo "------------------------------"
+cmsDriver.py step5 --runUnscheduled --conditions auto:run3_data_prompt_relval -n 5 --era Run3_2023 -s NANO --data --datatier NANOAOD --eventcontent NANOAOD --filein file:output_step4_MINIAOD.root --fileout output_step5_NANOAOD.root --customise_commands='process.GlobalTag.DumpStat=cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName =cms.untracked.string("output_step5_NANOAOD.json")' |& tee output_step5_NANOAOD.log

ConditionsConsumed/runCMSDrivers_mc2023withJSON.sh

@@ -0,0 +1,65 @@
+#1: GEN
+echo "------------------------------"
+echo "Running step-1/9: GEN"
+echo "------------------------------"
+cmsDriver.py TTbar_13TeV_TuneCUETP8M1_cfi  --conditions auto:phase1_2023_realistic_postBPix -n 5 --era Run3_2023 --geometry DB:Extended -s GEN --fileout output_step1_GEN.root --beamspot Realistic25ns13p6TeVEarly2023Collision  --customise_commands='process.GlobalTag.DumpStat =cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName =cms.untracked.string("output_step1_GEN.json")'|& tee output_step1_GEN.log
+
+#2: SIM
+echo "------------------------------"
+echo "Running step-2/9: SIM"
+echo "------------------------------"
+cmsDriver.py step2  --conditions auto:phase1_2023_realistic_postBPix -n 5 --era Run3_2023 --geometry DB:Extended -s SIM --datatier GEN-SIM --eventcontent FEVTDEBUG --filein file:output_step1_GEN.root --fileout output_step2_SIM.root --beamspot Realistic25ns13p6TeVEarly2023Collision  --customise_commands='process.GlobalTag.DumpStat =cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName =cms.untracked.string("output_step2_SIM.json")' --outputCommands "keep *" |& tee output_step2_SIM.log
+
+#3: DIGI
+echo "------------------------------"
+echo "Running step-3/9: DIGI"
+echo "------------------------------"
+cmsDriver.py step3  --conditions auto:phase1_2023_realistic_postBPix -n 5 --era Run3_2023 --geometry DB:Extended -s DIGI:pdigi_valid --datatier GEN-SIM-DIGI-RAW --eventcontent FEVTDEBUGHLT --filein file:output_step2_SIM.root --fileout output_step3_DIGI.root  --customise_commands='process.GlobalTag.DumpStat =cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName =cms.untracked.string("output_step3_DIGI.json")' --outputCommands "keep *, drop *_mix_*_*" |& tee output_step3_DIGI.log
+
+#4: L1
+echo "------------------------------"
+echo "Running step-4/9: L1"
+echo "------------------------------"
+cmsDriver.py step4  --conditions auto:phase1_2023_realistic_postBPix -n 5 --era Run3_2023 --geometry DB:Extended -s L1 --datatier GEN-SIM-DIGI-RAW --eventcontent FEVTDEBUGHLT --filein file:output_step3_DIGI.root --fileout output_step4_L1.root --customise_commands='process.load("Configuration.StandardSequences.Digi_cff") \n process.GlobalTag.DumpStat =cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName =cms.untracked.string("output_step4_L1.json")' --outputCommands "keep *" |& tee output_step4_L1.log
+
+#5: DIGI2RAW
+echo "------------------------------"
+echo "Running step-5/9: DIGI2RAW"
+echo "------------------------------"
+cmsDriver.py step5  --conditions auto:phase1_2023_realistic_postBPix -n 5 --era Run3_2023 --geometry DB:Extended -s DIGI2RAW --datatier GEN-SIM-DIGI-RAW --eventcontent FEVTDEBUGHLT --filein file:output_step4_L1.root  --fileout output_step5_DIGI2RAW.root --customise_commands='process.GlobalTag.DumpStat=cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName =cms.untracked.string("output_step5_DIGI2RAW.json")' --outputCommands "keep *" |& tee output_step5_DIGI2RAW.log
+
+#6: HLT
+echo "------------------------------"
+echo "Running step-6/9: HLT"
+echo "------------------------------"
+cmsDriver.py step6  --conditions auto:phase1_2023_realistic_postBPix -n 5 --era Run3_2023 -s HLT:@relval2016  --datatier GEN-SIM-DIGI-RAW-HLTDEBUG --eventcontent FEVTDEBUGHLT --filein file:output_step5_DIGI2RAW.root --fileout output_step6_HLT.root --customise_commands='process.GlobalTag.DumpStat=cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName =cms.untracked.string("output_step6_HLT.json")' --outputCommands "keep *" |& tee output_step6_HLT.log
+
+#7 AODSIM (RAW2DIGI,L1Reco,RECO,RECOSIM)
+echo "------------------------------"
+echo "Running step-7/9: AODSIM"
+echo "------------------------------"
+cmsDriver.py step7 --conditions auto:phase1_2023_realistic_postBPix -n 5 --era Run3_2023 --geometry DB:Extended -s RAW2DIGI,L1Reco,RECO,RECOSIM --datatier AODSIM --eventcontent AODSIM --filein file:output_step6_HLT.root --fileout output_step7_AODSIM.root --customise_commands='process.GlobalTag.DumpStat =cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName =cms.untracked.string("output_step7_AODSIM.json")' |& tee output_step7_AODSIM.log
+
+#8 MINIAODSIM
+echo "------------------------------"
+echo "Running step-8/9: MINIADOSIM"
+echo "------------------------------"
+cmsDriver.py step8 --runUnscheduled --conditions auto:phase1_2023_realistic_postBPix -n 5 --era Run3_2023 -s PAT --datatier MINIAODSIM --eventcontent MINIAODSIM --filein file:output_step7_AODSIM.root --fileout output_step8_MINIAODSIM.root --customise_commands='process.GlobalTag.DumpStat =cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName =cms.untracked.string("output_step8_MINIAODSIM.json")' |& tee output_step8_MINIAODSIM.log
+
+#9 NANOAODSIM
+echo "------------------------------"
+echo "Running step-9/9: NANOAODSIM"
+echo "------------------------------"
+cmsDriver.py step9 --runUnscheduled --conditions auto:phase1_2023_realistic_postBPix -n 5 --era Run3_2023 --geometry DB:Extended -s NANO --datatier NANOAODSIM --eventcontent NANOAODSIM --filein file:output_step8_MINIAODSIM.root --fileout output_step9_NANOAODSIM.root --customise_commands='process.GlobalTag.DumpStat=cms.untracked.bool(True) \n process.GlobalTag.JsonDumpFileName =cms.untracked.string("output_step9_NANOAODSIM.json")' |& tee output_step9_NANOAODSIM.log
+
+################# step-7 can be splitted in sub steps ###################
+#7a RAW2DIGI
+#cmsDriver.py step7a  --conditions auto:phase1_2023_realistic_postBPix -n 5 --era Run3_2023 -s RAW2DIGI --datatier GEN-SIM-RECO --eventcontent RECOSIM --filein file:output_step6_HLT.root --fileout output_step7a_RAW2DIGI.root --customise_commands='process.GlobalTag.DumpStat =cms.untracked.bool(True)' --outputCommands "keep *" |& tee output_step7a_RAW2DIGI.log
+
+#7b L1Reco
+#cmsDriver.py step7b  --conditions auto:phase1_2023_realistic_postBPix -n 5 --era Run3_2023 -s L1Reco --datatier GEN-SIM-RECO --eventcontent RECOSIM --filein file:output_step7a_RAW2DIGI.root --fileout output_step7b_L1Reco.root --customise_commands='process.GlobalTag.DumpStat =cms.untracked.bool(True)' --outputCommands "keep *" |& tee output_step7b_L1Reco.log
+
+#7c RECO
+#cmsDriver.py step7c  --conditions auto:phase1_2023_realistic_postBPix -n 5 --era Run3_2023 -s RECO --datatier GEN-SIM-RECO --eventcontent RECO --filein file:output_step7b_L1Reco.root --fileout output_step7c_RECO.root --customise_commands='process.GlobalTag.DumpStat =cms.untracked.bool(True)' --outputCommands "keep *" |& tee output_step7c_RECO.log
+
+#7d, etc