diff --git a/.github/workflows/mucoll-ci.yml b/.github/workflows/mucoll-ci.yml index 7c06936..291ee2a 100644 --- a/.github/workflows/mucoll-ci.yml +++ b/.github/workflows/mucoll-ci.yml @@ -12,7 +12,7 @@ concurrency: cancel-in-progress: true jobs: - build-mucoll-image: + build-and-test-mucoll-sim-main: runs-on: ubuntu-latest steps: - name: Free Disk Space (Ubuntu) diff --git a/MAIAConfig/Common/steering.py b/MAIAConfig/Common/steering.py index 050bd8e..f0b2e21 100644 --- a/MAIAConfig/Common/steering.py +++ b/MAIAConfig/Common/steering.py @@ -24,6 +24,68 @@ def merge_alg_lists(*alg_lists): return merged +# EDM4hep collection types covering every flavour of tracker hit written by the +# MAIA chain: the simulated hits, both the plane-measurement (VXD/IT/OT) and the +# 3D digitised/merged hits, and the hit <-> simulated-hit links, which would +# otherwise be left pointing at collections that are no longer in the file. +TRACKER_HIT_TYPES = [ + "edm4hep::SimTrackerHitCollection", + "edm4hep::TrackerHitPlaneCollection", + "edm4hep::TrackerHit3DCollection", + "podio::LinkCollection", +] + +# The calorimeter counterpart: the simulated hits together with their per-hit +# contributions, the digitised/reconstructed hits (ECal, HCal and the muon +# system, which uses the same type), and the hit <-> simulated-hit links. +CALORIMETER_HIT_TYPES = [ + "edm4hep::SimCalorimeterHitCollection", + "edm4hep::CaloHitContributionCollection", + "edm4hep::CalorimeterHitCollection", + "podio::LinkCollection", +] + + +def _drop_collection_types(collection_types): + """ + Append type-based drop commands to the IOSvc keep/drop switch. + + Reading the current value back before appending keeps successive calls + (tracker + calorimeter) from overwriting each other. + """ + from k4FWCore import IOSvc + + io_svc = IOSvc("IOSvc") + commands = list(getattr(io_svc, "outputCommands", [])) or ["keep *"] + commands += ["drop type " + coll_type for coll_type in collection_types] + io_svc.outputCommands = commands + + +def drop_tracker_hits(): + """ + Exclude all TrackerHit and SimTrackerHit collections (and their relation + links) from the output file. + + Must be called after build_application, which creates the IOSvc that the + keep/drop switch belongs to. The selection is type based, so it also + catches the collections created downstream (merged hits) whose names are + not known here. + """ + _drop_collection_types(TRACKER_HIT_TYPES) + + +def drop_calorimeter_hits(): + """ + Exclude all CalorimeterHit and SimCalorimeterHit collections (plus the + calorimeter contributions and relation links) from the output file. + + Same call-after-build_application requirement as drop_tracker_hits. Note + that the Pandora clusters and PFOs are kept, but their references into the + calorimeter hits no longer resolve once the hits are gone. + """ + _drop_collection_types(CALORIMETER_HIT_TYPES) + + def build_application(args, alg_list, input_files, output_file, histo_file, evt_max=10): """ Configure the services, IO and ApplicationMgr common to every steering diff --git a/MAIAConfig/Overlay/overlay_IP.py b/MAIAConfig/Overlay/overlay_IP.py index 23f3b5c..9648283 100644 --- a/MAIAConfig/Overlay/overlay_IP.py +++ b/MAIAConfig/Overlay/overlay_IP.py @@ -71,6 +71,7 @@ def overlay_ip_cfg(args): OutputSimTrackerHits = out_tracker_hits, OutputSimCalorimeterHits = out_calo_hits, OutputCaloHitContributions = out_calo_contribs, - MergeMCParticles = False, + # Uncomment when https://github.com/key4hep/k4FWCore/pull/413 is merged. + #MergeMCParticles = False, OutputLevel = INFO ) diff --git a/MAIAConfig/Tracking/CKF_tracking.py b/MAIAConfig/Tracking/CKF_tracking.py index 91acd6f..d316722 100644 --- a/MAIAConfig/Tracking/CKF_tracking.py +++ b/MAIAConfig/Tracking/CKF_tracking.py @@ -1,5 +1,7 @@ from GaudiKernel.Constants import INFO, WARNING, DEBUG -from Configurables import ActsGeoSvc, CKFTrackingAlg, ACTSDuplicateRemoval, FilterTracksAlg, TrackTruthAlg, RefitFinal +from Configurables import ActsGeoSvc, CKFTrackingAlg, ACTSDuplicateRemoval, FilterTracksAlg, TrackTruthAlg + +import os def ActsGeoSvc_cfg(args): """Configure the ACTS GeoSvc. @@ -8,7 +10,8 @@ def ActsGeoSvc_cfg(args): """ return ActsGeoSvc( "ActsGeoSvc", - #UseDD4hepBField=args.use_dd4hep_field + UseDD4hepBField=args.use_dd4hep_field, + MaterialMapFile = args.materialMapFile, ) def CKFTracker_cfg(args): @@ -33,7 +36,9 @@ def CKFTracker_cfg(args): SeedFinding_RMax = 150, SeedFinding_MinPt = 500, SeedFinding_ImpactMax = 3, - CKF_NumMeasurementsCutOff = 1, + # CKF_NumMeasurementsCutOff: controls the CKF branching during track extension. + # Set to 1 to keep only the best candidate. + CKF_NumMeasurementsCutOff = 2, SeedFinding_SigmaScattering = 50, SeedFinding_CollisionRegion = 6, SeedFinding_RadLengthPerSeed = 0.1, @@ -57,7 +62,6 @@ def deduper_cfg(): OutputLevel = INFO ) - def track_filter_cfg(): """ Create a new FilterTracksAlg instance for filtering tracks. @@ -89,26 +93,3 @@ def track_truth_cfg(args): OutputParticle2TrackRelationName = ["SiTrackRelations"], OutputLevel = INFO ) - -def track_refitter_cfg(): - """ - Create a new TrackRefitter instance for refitting tracks. - """ - return RefitFinal( - "Refitter", -# DoCutsOnRedChi2Nhits = True, - EnergyLossOn = True, - InputRelationCollectionName = ["SiTrackRelations"], - InputTrackCollectionName = ["SiTracks"], - Max_Chi2_Incr = 1.79769e+30, - MinClustersOnTrackAfterFit = 3, - MultipleScatteringOn = True, -# NHitsCuts = ["1,2", "1", "3,4", "1", "5,6", "0"], - OutputRelationCollectionName = ["SiTracks_Refitted_Relation"], - OutputTrackCollectionName = ["SiTracks_Refitted"], -# ReducedChi2Cut = 10., - ReferencePoint = -1, - SmoothOn = False, - extrapolateForward = True, - OutputLevel = INFO - ) diff --git a/MAIAConfig/digi_args.py b/MAIAConfig/digi_args.py index 6fad7d8..571acc0 100644 --- a/MAIAConfig/digi_args.py +++ b/MAIAConfig/digi_args.py @@ -62,8 +62,7 @@ def get_digi_args(): default=False, ) - # Shared with reco_args (the merger reads the coned hits when enabled), so - # added once to allow combining the two parsers in a single job. + # Shared with reco_args (the merger reads the coned hits when enabled). add_argument_once( parser, "--doTrackerConing", diff --git a/MAIAConfig/digi_reco_steer.py b/MAIAConfig/digi_reco_steer.py index 40ce515..1c42b76 100644 --- a/MAIAConfig/digi_reco_steer.py +++ b/MAIAConfig/digi_reco_steer.py @@ -14,7 +14,12 @@ from reco_args import get_reco_args from digiAlgList import makeDigiAlgList from recoAlgList import makeRecoAlgList -from Common.steering import build_application, merge_alg_lists +from Common.steering import ( + build_application, + drop_calorimeter_hits, + drop_tracker_hits, + merge_alg_lists, +) # Register the digi arguments first, then the reco arguments; the returned reco # namespace is a superset that carries every option needed by both lists. The @@ -33,3 +38,10 @@ output_file = "digireco_output.edm4hep.root", histo_file = "digireco_histograms.root", ) + +# Same output slimming as reco_steer.py: with background overlaid the hit +# collections dominate the output file size, so drop every (Sim)TrackerHit and +# (Sim)CalorimeterHit collection unless --keepEverything asks for the full event. +if (args.doOverlayIP or args.doOverlayFull) and not args.keepEverything: + drop_tracker_hits() + drop_calorimeter_hits() diff --git a/MAIAConfig/recoAlgList.py b/MAIAConfig/recoAlgList.py index 4a5a1bc..23729cf 100644 --- a/MAIAConfig/recoAlgList.py +++ b/MAIAConfig/recoAlgList.py @@ -15,18 +15,16 @@ def makeRecoAlgList(the_args): algList.append(mergehitsrelations_cfg(the_args)) # CKF Tracking - from Tracking.CKF_tracking import CKFTracker_cfg, deduper_cfg, track_filter_cfg, track_truth_cfg, ActsGeoSvc_cfg, track_refitter_cfg + from Tracking.CKF_tracking import CKFTracker_cfg, deduper_cfg, track_filter_cfg, ActsGeoSvc_cfg ActsGeoSvc_cfg(the_args) # service: configure it, do NOT append to the algorithm list algList.append(CKFTracker_cfg(the_args)) algList.append(deduper_cfg()) algList.append(track_filter_cfg()) - algList.append(track_truth_cfg(the_args)) - #algList.append(track_refitter_cfg()) # Track Performance Monitoring if the_args.doTrackPerf: - from Diagnostics.track_performance import trackTruth_cfg - algList.append(trackTruth_cfg()) + from Diagnostics.track_performance import track_truth_cfg + algList.append(track_truth_cfg(the_args)) # Pandora PFOs from ParticleFlow.pandora import pandoraPFA_cfg, fastJet_cfg diff --git a/MAIAConfig/reco_args.py b/MAIAConfig/reco_args.py index 815166d..3f85f80 100644 --- a/MAIAConfig/reco_args.py +++ b/MAIAConfig/reco_args.py @@ -16,6 +16,14 @@ def get_reco_args(): default=os.environ.get("k4geo_DIR", "")+"/MuColl/MAIA/compact/MAIA_v0/MAIA_v0.xml", ) + add_argument_once( + parser, + "--materialMapFile", + help="Material map file", + type=str, + default=os.environ.get("ACTSTRACKING_DATA") + "/k4ActsTracking/data/MAIA_v0_gen3_material_map.json", + ) + parser.add_argument( "--doTrackPerf", help="Run Performance Analysis on Tracking", @@ -38,9 +46,7 @@ def get_reco_args(): default=1, ) - # Shared with digi_args: the digi step produces the "...Coned" collections - # and the merger here must read them. add_argument_once allows the two - # parsers to coexist in the combined digi_reco job. + # Shared with digi_args. add_argument_once( parser, "--doTrackerConing", @@ -49,6 +55,32 @@ def get_reco_args(): default=False, ) + # Shared with digi_args. + add_argument_once( + parser, + "--doOverlayFull", + help="Do BIB overlay", + action="store_true", + default=False, + ) + + add_argument_once( + parser, + "--doOverlayIP", + help="Do incoherent pairs overlay", + action="store_true", + default=False, + ) + + parser.add_argument( + "--keepEverything", + help="Write every collection to the reconstruction output, including " + "the tracker and calorimeter hits that are otherwise dropped when " + "an overlay is enabled", + action="store_true", + default=False, + ) + parser.add_argument( "--pandoraSettings", help="Pandora settings XML. A bare file name is looked up in the " diff --git a/MAIAConfig/reco_steer.py b/MAIAConfig/reco_steer.py index 25ba5d9..129055b 100644 --- a/MAIAConfig/reco_steer.py +++ b/MAIAConfig/reco_steer.py @@ -8,7 +8,7 @@ from reco_args import get_reco_args from recoAlgList import makeRecoAlgList -from Common.steering import build_application +from Common.steering import build_application, drop_calorimeter_hits, drop_tracker_hits # Collect arguments and build the reconstruction algorithm list args = get_reco_args() @@ -22,14 +22,15 @@ histo_file = "reco_histograms.root", ) -# Per-algorithm CPU monitoring via the Gaudi Auditor Service. The ChronoAuditor -# records CPU usage of each algorithm's execute() and ChronoStatSvc prints the -# per-algorithm total + per-event-average table ("Final CPU consumption") at end -# of job. Most meaningful single-threaded. -# -# The global message level is raised to INFO so the ChronoStatSvc table (emitted -# under the "*****Chrono*****" / ":Execute" message sources) is not -# suppressed by the WARNING default that build_application sets. +# With BIB, drop every (Sim)TrackerHit and (Sim)CalorimeterHit collection +# from the reco output. --keepEverything writes the full event. +if (args.doOverlayIP or args.doOverlayFull) and not args.keepEverything: + drop_tracker_hits() + drop_calorimeter_hits() + +# Per-algorithm CPU monitoring via the Gaudi Auditor Service. Most meaningful +# single-threaded. Global message level raised to INFO not to be suppressed +# by the WARNING default. from GaudiKernel.Constants import INFO from Configurables import AuditorSvc, ChronoStatSvc app.OutputLevel = INFO diff --git a/README.md b/README.md index 2b36a93..e01f98e 100644 --- a/README.md +++ b/README.md @@ -97,19 +97,45 @@ The full set is: | `--inputFiles` | both | per macro (see above) | Input EDM4hep file(s) to read; accepts several files. | | `--outputFile` | both | per macro (see above) | Output EDM4hep file to write. | | `--histoFile` | both | per macro | Output ROOT file for the histograms. | -| `--doOverlayFull` | digi | `False` | Overlay beam-induced background (BIB). | +| `--doOverlayFull` | digi + reco | `False` | Overlay beam-induced background (BIB). In the reco step it only acts as a flag: when set, all tracker and calorimeter hit collections are dropped from the reconstruction output (see below). | | `--OverlayFullPathToMuPlus` | digi | `/path/to/muplus/` | Directory of the μ⁺ BIB overlay files (used with `--doOverlayFull`). | | `--OverlayFullPathToMuMinus` | digi | `/path/to/muminus/` | Directory of the μ⁻ BIB overlay files (used with `--doOverlayFull`). | | `--OverlayFullNumberBackground` | digi | `812` | Number of BIB background files overlaid (used with `--doOverlayFull`). | -| `--doOverlayIP` | digi | `False` | Overlay incoherent pairs. When both overlays are enabled they are chained (BIB then IP) before digitisation. | +| `--doOverlayIP` | digi + reco | `False` | Overlay incoherent pairs. When both overlays are enabled they are chained (BIB then IP) before digitisation. In the reco step it only acts as a flag: when set, all tracker and calorimeter hit collections are dropped from the reconstruction output (see below). | | `--OverlayIPBackgroundFileNames` | digi | `[/path/to/pairs.slcio]` | Incoherent-pair overlay input file(s) (used with `--doOverlayIP`). | | `--doFilterDL` | digi | `False` | Double-layer hit filtering in the vertex detector. | | `--doTrackerConing` | digi + reco | `False` | Cone-filter the tracker hits around the signal MC particles (BIB cleaning). When enabled, the digi step writes the `…Coned` hit collections and the merger reads them before tracking. | | `--RandSeed` | digi | `42` | Random seed for the digitisation smearing. | | `--doTrackPerf` | reco | `False` | Run the tracking performance monitoring. | +| `--keepEverything` | reco | `False` | Write every collection to the reconstruction output, overriding the hit dropping that `--doOverlayFull`/`--doOverlayIP` would otherwise trigger (see below). | | `--TrackingThreads` | reco | `1` | Internal thread count of the CKF tracking and truth-matching algorithms (independent of `--numThreads`). | | `--numThreads` | both | `1` | Number of threads for the Gaudi event loop. `1` runs serially; any value `> 1` enables the multi-threaded Gaudi Hive event loop with that many threads (scheduler + event slots); `0` auto-detects a sensible count from the CPU count. | +### Hit collections in the overlay output + +Running with background (`--doOverlayFull` and/or `--doOverlayIP`) makes the hit +collections dominate the output file, so `reco_steer.py` drops all of them from +the reconstruction output when either flag is set: the tracker hits +(`drop_tracker_hits`, i.e. `SimTrackerHit`, `TrackerHitPlane`, `TrackerHit3D`) +and the calorimeter hits (`drop_calorimeter_hits`, i.e. `SimCalorimeterHit`, +`CaloHitContribution`, `CalorimeterHit`, which also covers the muon system), +together with the corresponding hit ↔ simulated-hit link collections. The +selection is done by collection *type* through the `IOSvc` keep/drop switch, so +it also covers the collections produced during reconstruction (e.g. +`MergedTrackerHits`). + +The reconstructed objects — tracks, Pandora clusters, PFOs, jets, vertices, +`MCParticles` and the track ↔ MC-particle links — are kept, but any reference +they hold into a dropped hit collection (`Track::trackerHits`, +`Cluster::hits`, …) no longer resolves in the output file. + +Pass `--keepEverything` to switch the dropping off and write the full event +even with an overlay enabled: + +```bash +k4run reco_steer.py --doOverlayFull --keepEverything +``` + ### BIB hit cleaning Mirroring the Marlin `steer_reco.py` workflow, once the calorimeter hits are