Tracking-related preparation for release 3.1

.github/workflows/mucoll-ci.yml

@@ -12,7 +12,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build-mucoll-image:
+  build-and-test-mucoll-sim-main:
     runs-on: ubuntu-latest
     steps:
       - name: Free Disk Space (Ubuntu)

MAIAConfig/Common/steering.py

@@ -24,6 +24,68 @@ def merge_alg_lists(*alg_lists):
     return merged
 
 
+# EDM4hep collection types covering every flavour of tracker hit written by the
+# MAIA chain: the simulated hits, both the plane-measurement (VXD/IT/OT) and the
+# 3D digitised/merged hits, and the hit <-> simulated-hit links, which would
+# otherwise be left pointing at collections that are no longer in the file.
+TRACKER_HIT_TYPES = [
+    "edm4hep::SimTrackerHitCollection",
+    "edm4hep::TrackerHitPlaneCollection",
+    "edm4hep::TrackerHit3DCollection",
+    "podio::LinkCollection<edm4hep::TrackerHit,edm4hep::SimTrackerHit>",
+]
+
+# The calorimeter counterpart: the simulated hits together with their per-hit
+# contributions, the digitised/reconstructed hits (ECal, HCal and the muon
+# system, which uses the same type), and the hit <-> simulated-hit links.
+CALORIMETER_HIT_TYPES = [
+    "edm4hep::SimCalorimeterHitCollection",
+    "edm4hep::CaloHitContributionCollection",
+    "edm4hep::CalorimeterHitCollection",
+    "podio::LinkCollection<edm4hep::CalorimeterHit,edm4hep::SimCalorimeterHit>",
+]
+
+
+def _drop_collection_types(collection_types):
+    """
+    Append type-based drop commands to the IOSvc keep/drop switch.
+
+    Reading the current value back before appending keeps successive calls
+    (tracker + calorimeter) from overwriting each other.
+    """
+    from k4FWCore import IOSvc
+
+    io_svc = IOSvc("IOSvc")
+    commands = list(getattr(io_svc, "outputCommands", [])) or ["keep *"]
+    commands += ["drop type " + coll_type for coll_type in collection_types]
+    io_svc.outputCommands = commands
+
+
+def drop_tracker_hits():
+    """
+    Exclude all TrackerHit and SimTrackerHit collections (and their relation
+    links) from the output file.
+
+    Must be called after build_application, which creates the IOSvc that the
+    keep/drop switch belongs to. The selection is type based, so it also
+    catches the collections created downstream (merged hits) whose names are
+    not known here.
+    """
+    _drop_collection_types(TRACKER_HIT_TYPES)
+
+
+def drop_calorimeter_hits():
+    """
+    Exclude all CalorimeterHit and SimCalorimeterHit collections (plus the
+    calorimeter contributions and relation links) from the output file.
+
+    Same call-after-build_application requirement as drop_tracker_hits. Note
+    that the Pandora clusters and PFOs are kept, but their references into the
+    calorimeter hits no longer resolve once the hits are gone.
+    """
+    _drop_collection_types(CALORIMETER_HIT_TYPES)
+
+
 def build_application(args, alg_list, input_files, output_file, histo_file, evt_max=10):
     """
     Configure the services, IO and ApplicationMgr common to every steering

MAIAConfig/Overlay/overlay_IP.py

@@ -71,6 +71,7 @@ def overlay_ip_cfg(args):
         OutputSimTrackerHits = out_tracker_hits,
         OutputSimCalorimeterHits = out_calo_hits,
         OutputCaloHitContributions = out_calo_contribs,
-        MergeMCParticles = False,
+        # Uncomment when https://github.com/key4hep/k4FWCore/pull/413 is merged.
+        #MergeMCParticles = False,
         OutputLevel = INFO
     )

MAIAConfig/Tracking/CKF_tracking.py

@@ -1,5 +1,7 @@
 from GaudiKernel.Constants import INFO, WARNING, DEBUG
-from Configurables import ActsGeoSvc, CKFTrackingAlg, ACTSDuplicateRemoval, FilterTracksAlg, TrackTruthAlg, RefitFinal
+from Configurables import ActsGeoSvc, CKFTrackingAlg, ACTSDuplicateRemoval, FilterTracksAlg, TrackTruthAlg
+
+import os
 
 def ActsGeoSvc_cfg(args):
     """Configure the ACTS GeoSvc.
@@ -8,7 +10,8 @@ def ActsGeoSvc_cfg(args):
     """
     return ActsGeoSvc(
         "ActsGeoSvc",
-        #UseDD4hepBField=args.use_dd4hep_field
+        UseDD4hepBField=args.use_dd4hep_field,
+        MaterialMapFile = args.materialMapFile,
     )
 
 def CKFTracker_cfg(args):
@@ -33,7 +36,9 @@ def CKFTracker_cfg(args):
         SeedFinding_RMax = 150,
         SeedFinding_MinPt = 500,
         SeedFinding_ImpactMax = 3,
-        CKF_NumMeasurementsCutOff = 1,
+        # CKF_NumMeasurementsCutOff: controls the CKF branching during track extension.
+        # Set to 1 to keep only the best candidate.
+        CKF_NumMeasurementsCutOff = 2,
         SeedFinding_SigmaScattering = 50,
         SeedFinding_CollisionRegion = 6,
         SeedFinding_RadLengthPerSeed = 0.1,
@@ -57,7 +62,6 @@ def deduper_cfg():
         OutputLevel = INFO
     )
 
-
 def track_filter_cfg():
     """
     Create a new FilterTracksAlg instance for filtering tracks.
@@ -89,26 +93,3 @@ def track_truth_cfg(args):
         OutputParticle2TrackRelationName = ["SiTrackRelations"],
         OutputLevel = INFO
     )
-
-def track_refitter_cfg():
-    """
-    Create a new TrackRefitter instance for refitting tracks.
-    """
-    return RefitFinal(
-        "Refitter",
-#        DoCutsOnRedChi2Nhits = True,
-        EnergyLossOn = True,
-        InputRelationCollectionName = ["SiTrackRelations"],
-        InputTrackCollectionName = ["SiTracks"],
-        Max_Chi2_Incr = 1.79769e+30,
-        MinClustersOnTrackAfterFit = 3,
-        MultipleScatteringOn = True,
-#        NHitsCuts = ["1,2", "1", "3,4", "1", "5,6", "0"],
-        OutputRelationCollectionName = ["SiTracks_Refitted_Relation"],
-        OutputTrackCollectionName = ["SiTracks_Refitted"],
-#        ReducedChi2Cut = 10.,
-        ReferencePoint = -1,
-        SmoothOn = False,
-        extrapolateForward = True,
-        OutputLevel = INFO
-    )

MAIAConfig/digi_args.py

@@ -62,8 +62,7 @@ def get_digi_args():
         default=False,
     )
 
-    # Shared with reco_args (the merger reads the coned hits when enabled), so
-    # added once to allow combining the two parsers in a single job.
+    # Shared with reco_args (the merger reads the coned hits when enabled).
     add_argument_once(
         parser,
         "--doTrackerConing",

MAIAConfig/digi_reco_steer.py

@@ -14,7 +14,12 @@
 from reco_args import get_reco_args
 from digiAlgList import makeDigiAlgList
 from recoAlgList import makeRecoAlgList
-from Common.steering import build_application, merge_alg_lists
+from Common.steering import (
+    build_application,
+    drop_calorimeter_hits,
+    drop_tracker_hits,
+    merge_alg_lists,
+)
 
 # Register the digi arguments first, then the reco arguments; the returned reco
 # namespace is a superset that carries every option needed by both lists. The
@@ -33,3 +38,10 @@
     output_file = "digireco_output.edm4hep.root",
     histo_file = "digireco_histograms.root",
 )
+
+# Same output slimming as reco_steer.py: with background overlaid the hit
+# collections dominate the output file size, so drop every (Sim)TrackerHit and
+# (Sim)CalorimeterHit collection unless --keepEverything asks for the full event.
+if (args.doOverlayIP or args.doOverlayFull) and not args.keepEverything:
+    drop_tracker_hits()
+    drop_calorimeter_hits()

MAIAConfig/recoAlgList.py

@@ -15,18 +15,16 @@ def makeRecoAlgList(the_args):
     algList.append(mergehitsrelations_cfg(the_args))
 
     # CKF Tracking
-    from Tracking.CKF_tracking import CKFTracker_cfg, deduper_cfg, track_filter_cfg, track_truth_cfg, ActsGeoSvc_cfg, track_refitter_cfg
+    from Tracking.CKF_tracking import CKFTracker_cfg, deduper_cfg, track_filter_cfg, ActsGeoSvc_cfg
     ActsGeoSvc_cfg(the_args)  # service: configure it, do NOT append to the algorithm list
     algList.append(CKFTracker_cfg(the_args))
     algList.append(deduper_cfg())
     algList.append(track_filter_cfg())
-    algList.append(track_truth_cfg(the_args))
-    #algList.append(track_refitter_cfg())
 
     # Track Performance Monitoring
     if the_args.doTrackPerf:
-        from Diagnostics.track_performance import trackTruth_cfg
-        algList.append(trackTruth_cfg())
+        from Diagnostics.track_performance import track_truth_cfg
+        algList.append(track_truth_cfg(the_args))
 
     # Pandora PFOs
     from ParticleFlow.pandora import pandoraPFA_cfg, fastJet_cfg

MAIAConfig/reco_args.py

@@ -16,6 +16,14 @@ def get_reco_args():
         default=os.environ.get("k4geo_DIR", "")+"/MuColl/MAIA/compact/MAIA_v0/MAIA_v0.xml",
     )
 
+    add_argument_once(
+        parser,
+        "--materialMapFile",
+        help="Material map file",
+        type=str,
+        default=os.environ.get("ACTSTRACKING_DATA") + "/k4ActsTracking/data/MAIA_v0_gen3_material_map.json",
+    )
+
     parser.add_argument(
         "--doTrackPerf",
         help="Run Performance Analysis on Tracking",
@@ -38,9 +46,7 @@ def get_reco_args():
         default=1,
     )
 
-    # Shared with digi_args: the digi step produces the "...Coned" collections
-    # and the merger here must read them. add_argument_once allows the two
-    # parsers to coexist in the combined digi_reco job.
+    # Shared with digi_args.
     add_argument_once(
         parser,
         "--doTrackerConing",
@@ -49,6 +55,32 @@ def get_reco_args():
         default=False,
     )
 
+    # Shared with digi_args.
+    add_argument_once(
+        parser,
+        "--doOverlayFull",
+        help="Do BIB overlay",
+        action="store_true",
+        default=False,
+    )
+
+    add_argument_once(
+        parser,
+        "--doOverlayIP",
+        help="Do incoherent pairs overlay",
+        action="store_true",
+        default=False,
+    )
+
+    parser.add_argument(
+        "--keepEverything",
+        help="Write every collection to the reconstruction output, including "
+             "the tracker and calorimeter hits that are otherwise dropped when "
+             "an overlay is enabled",
+        action="store_true",
+        default=False,
+    )
+
     parser.add_argument(
         "--pandoraSettings",
         help="Pandora settings XML. A bare file name is looked up in the "

MAIAConfig/reco_steer.py

@@ -8,7 +8,7 @@
 
 from reco_args import get_reco_args
 from recoAlgList import makeRecoAlgList
-from Common.steering import build_application
+from Common.steering import build_application, drop_calorimeter_hits, drop_tracker_hits
 
 # Collect arguments and build the reconstruction algorithm list
 args = get_reco_args()
@@ -22,14 +22,15 @@
     histo_file = "reco_histograms.root",
 )
 
-# Per-algorithm CPU monitoring via the Gaudi Auditor Service. The ChronoAuditor
-# records CPU usage of each algorithm's execute() and ChronoStatSvc prints the
-# per-algorithm total + per-event-average table ("Final CPU consumption") at end
-# of job. Most meaningful single-threaded.
-#
-# The global message level is raised to INFO so the ChronoStatSvc table (emitted
-# under the "*****Chrono*****" / "<Alg>:Execute" message sources) is not
-# suppressed by the WARNING default that build_application sets.
+# With BIB, drop every (Sim)TrackerHit and (Sim)CalorimeterHit collection
+# from the reco output. --keepEverything writes the full event.
+if (args.doOverlayIP or args.doOverlayFull) and not args.keepEverything:
+    drop_tracker_hits()
+    drop_calorimeter_hits()
+
+# Per-algorithm CPU monitoring via the Gaudi Auditor Service. Most meaningful
+# single-threaded. Global message level raised to INFO not to be suppressed
+# by the WARNING default.
 from GaudiKernel.Constants import INFO
 from Configurables import AuditorSvc, ChronoStatSvc
 app.OutputLevel = INFO

README.md

@@ -97,19 +97,45 @@ The full set is:
 | `--inputFiles` | both | per macro (see above) | Input EDM4hep file(s) to read; accepts several files. |
 | `--outputFile` | both | per macro (see above) | Output EDM4hep file to write. |
 | `--histoFile` | both | per macro | Output ROOT file for the histograms. |
-| `--doOverlayFull` | digi | `False` | Overlay beam-induced background (BIB). |
+| `--doOverlayFull` | digi + reco | `False` | Overlay beam-induced background (BIB). In the reco step it only acts as a flag: when set, all tracker and calorimeter hit collections are dropped from the reconstruction output (see below). |
 | `--OverlayFullPathToMuPlus` | digi | `/path/to/muplus/` | Directory of the μ⁺ BIB overlay files (used with `--doOverlayFull`). |
 | `--OverlayFullPathToMuMinus` | digi | `/path/to/muminus/` | Directory of the μ⁻ BIB overlay files (used with `--doOverlayFull`). |
 | `--OverlayFullNumberBackground` | digi | `812` | Number of BIB background files overlaid (used with `--doOverlayFull`). |
-| `--doOverlayIP` | digi | `False` | Overlay incoherent pairs. When both overlays are enabled they are chained (BIB then IP) before digitisation. |
+| `--doOverlayIP` | digi + reco | `False` | Overlay incoherent pairs. When both overlays are enabled they are chained (BIB then IP) before digitisation. In the reco step it only acts as a flag: when set, all tracker and calorimeter hit collections are dropped from the reconstruction output (see below). |
 | `--OverlayIPBackgroundFileNames` | digi | `[/path/to/pairs.slcio]` | Incoherent-pair overlay input file(s) (used with `--doOverlayIP`). |
 | `--doFilterDL` | digi | `False` | Double-layer hit filtering in the vertex detector. |
 | `--doTrackerConing` | digi + reco | `False` | Cone-filter the tracker hits around the signal MC particles (BIB cleaning). When enabled, the digi step writes the `…Coned` hit collections and the merger reads them before tracking. |
 | `--RandSeed` | digi | `42` | Random seed for the digitisation smearing. |
 | `--doTrackPerf` | reco | `False` | Run the tracking performance monitoring. |
+| `--keepEverything` | reco | `False` | Write every collection to the reconstruction output, overriding the hit dropping that `--doOverlayFull`/`--doOverlayIP` would otherwise trigger (see below). |
 | `--TrackingThreads` | reco | `1` | Internal thread count of the CKF tracking and truth-matching algorithms (independent of `--numThreads`). |
 | `--numThreads` | both | `1` | Number of threads for the Gaudi event loop. `1` runs serially; any value `> 1` enables the multi-threaded Gaudi Hive event loop with that many threads (scheduler + event slots); `0` auto-detects a sensible count from the CPU count. |
 
+### Hit collections in the overlay output
+
+Running with background (`--doOverlayFull` and/or `--doOverlayIP`) makes the hit
+collections dominate the output file, so `reco_steer.py` drops all of them from
+the reconstruction output when either flag is set: the tracker hits
+(`drop_tracker_hits`, i.e. `SimTrackerHit`, `TrackerHitPlane`, `TrackerHit3D`)
+and the calorimeter hits (`drop_calorimeter_hits`, i.e. `SimCalorimeterHit`,
+`CaloHitContribution`, `CalorimeterHit`, which also covers the muon system),
+together with the corresponding hit ↔ simulated-hit link collections. The
+selection is done by collection *type* through the `IOSvc` keep/drop switch, so
+it also covers the collections produced during reconstruction (e.g.
+`MergedTrackerHits`).
+
+The reconstructed objects — tracks, Pandora clusters, PFOs, jets, vertices,
+`MCParticles` and the track ↔ MC-particle links — are kept, but any reference
+they hold into a dropped hit collection (`Track::trackerHits`,
+`Cluster::hits`, …) no longer resolves in the output file.
+
+Pass `--keepEverything` to switch the dropping off and write the full event
+even with an overlay enabled:
+
+```bash
+k4run reco_steer.py --doOverlayFull --keepEverything
+```
+
 ### BIB hit cleaning
 
 Mirroring the Marlin `steer_reco.py` workflow, once the calorimeter hits are