Tracking-related preparation for release 3.1

MAIAConfig/Common/steering.py

@@ -24,6 +24,68 @@ def merge_alg_lists(*alg_lists):
     return merged
 
 
+# EDM4hep collection types covering every flavour of tracker hit written by the
+# MAIA chain: the simulated hits, both the plane-measurement (VXD/IT/OT) and the
+# 3D digitised/merged hits, and the hit <-> simulated-hit links, which would
+# otherwise be left pointing at collections that are no longer in the file.
+TRACKER_HIT_TYPES = [
+    "edm4hep::SimTrackerHitCollection",
+    "edm4hep::TrackerHitPlaneCollection",
+    "edm4hep::TrackerHit3DCollection",
+    "podio::LinkCollection<edm4hep::TrackerHit,edm4hep::SimTrackerHit>",
+]
+
+# The calorimeter counterpart: the simulated hits together with their per-hit
+# contributions, the digitised/reconstructed hits (ECal, HCal and the muon
+# system, which uses the same type), and the hit <-> simulated-hit links.
+CALORIMETER_HIT_TYPES = [
+    "edm4hep::SimCalorimeterHitCollection",
+    "edm4hep::CaloHitContributionCollection",
+    "edm4hep::CalorimeterHitCollection",
+    "podio::LinkCollection<edm4hep::CalorimeterHit,edm4hep::SimCalorimeterHit>",
+]
+
+
+def _drop_collection_types(collection_types):
+    """
+    Append type-based drop commands to the IOSvc keep/drop switch.
+
+    Reading the current value back before appending keeps successive calls
+    (tracker + calorimeter) from overwriting each other.
+    """
+    from k4FWCore import IOSvc
+
+    io_svc = IOSvc("IOSvc")
+    commands = list(getattr(io_svc, "outputCommands", [])) or ["keep *"]
+    commands += ["drop type " + coll_type for coll_type in collection_types]
+    io_svc.outputCommands = commands
+
+
+def drop_tracker_hits():
+    """
+    Exclude all TrackerHit and SimTrackerHit collections (and their relation
+    links) from the output file.
+
+    Must be called after build_application, which creates the IOSvc that the
+    keep/drop switch belongs to. The selection is type based, so it also
+    catches the collections created downstream (merged hits) whose names are
+    not known here.
+    """
+    _drop_collection_types(TRACKER_HIT_TYPES)
+
+
+def drop_calorimeter_hits():
+    """
+    Exclude all CalorimeterHit and SimCalorimeterHit collections (plus the
+    calorimeter contributions and relation links) from the output file.
+
+    Same call-after-build_application requirement as drop_tracker_hits. Note
+    that the Pandora clusters and PFOs are kept, but their references into the
+    calorimeter hits no longer resolve once the hits are gone.
+    """
+    _drop_collection_types(CALORIMETER_HIT_TYPES)
+
+
 def build_application(args, alg_list, input_files, output_file, histo_file, evt_max=10):
     """
     Configure the services, IO and ApplicationMgr common to every steering

MAIAConfig/Overlay/overlay_IP.py

@@ -71,6 +71,6 @@ def overlay_ip_cfg(args):
         OutputSimTrackerHits = out_tracker_hits,
         OutputSimCalorimeterHits = out_calo_hits,
         OutputCaloHitContributions = out_calo_contribs,
-        MergeMCParticles = False,
+        #MergeMCParticles = False,
         OutputLevel = INFO
     )

MAIAConfig/Tracking/CKF_tracking.py

@@ -1,14 +1,17 @@
 from GaudiKernel.Constants import INFO, WARNING, DEBUG
 from Configurables import ActsGeoSvc, CKFTrackingAlg, ACTSDuplicateRemoval, FilterTracksAlg, TrackTruthAlg, RefitFinal
 
+import os
+
 def ActsGeoSvc_cfg(args):
     """Configure the ACTS GeoSvc.
     Set use_dd4hep_field=True to make ACTS use the real, position-dependent
     DD4hep field.
     """
     return ActsGeoSvc(
         "ActsGeoSvc",
-        #UseDD4hepBField=args.use_dd4hep_field
+        UseDD4hepBField=args.use_dd4hep_field,
+        MaterialMapFile = os.environ.get("ACTSTRACKING_DATA") + "/k4ActsTracking/data/MAIA_v0_gen3_material_map.json",
     )
 
 def CKFTracker_cfg(args):
@@ -33,7 +36,7 @@ def CKFTracker_cfg(args):
         SeedFinding_RMax = 150,
         SeedFinding_MinPt = 500,
         SeedFinding_ImpactMax = 3,
-        CKF_NumMeasurementsCutOff = 1,
+        CKF_NumMeasurementsCutOff = 2,
         SeedFinding_SigmaScattering = 50,
         SeedFinding_CollisionRegion = 6,
         SeedFinding_RadLengthPerSeed = 0.1,

MAIAConfig/digi_reco_steer.py

@@ -14,7 +14,12 @@
 from reco_args import get_reco_args
 from digiAlgList import makeDigiAlgList
 from recoAlgList import makeRecoAlgList
-from Common.steering import build_application, merge_alg_lists
+from Common.steering import (
+    build_application,
+    drop_calorimeter_hits,
+    drop_tracker_hits,
+    merge_alg_lists,
+)
 
 # Register the digi arguments first, then the reco arguments; the returned reco
 # namespace is a superset that carries every option needed by both lists. The
@@ -33,3 +38,10 @@
     output_file = "digireco_output.edm4hep.root",
     histo_file = "digireco_histograms.root",
 )
+
+# Same output slimming as reco_steer.py: with background overlaid the hit
+# collections dominate the output file size, so drop every (Sim)TrackerHit and
+# (Sim)CalorimeterHit collection unless --keepEverything asks for the full event.
+if (args.doOverlayIP or args.doOverlayFull) and not args.keepEverything:
+    drop_tracker_hits()
+    drop_calorimeter_hits()

MAIAConfig/reco_args.py

@@ -22,7 +22,7 @@ def get_reco_args():
         action="store_true",
         default=False
     )
-    
+
     parser.add_argument(
         "--use_dd4hep_field",
         help="Use DD4hep field",
@@ -49,4 +49,34 @@ def get_reco_args():
         default=False,
     )
 
+    # Shared with digi_args: the overlay flags are declared at digitisation, but
+    # the reco steering needs to know about them too so that the (very large)
+    # tracker and calorimeter hit collections can be dropped from the output
+    # when background is overlaid. add_argument_once keeps the combined
+    # digi_reco job working.
+    add_argument_once(
+        parser,
+        "--doOverlayFull",
+        help="Do BIB overlay",
+        action="store_true",
+        default=False,
+    )
+
+    add_argument_once(
+        parser,
+        "--doOverlayIP",
+        help="Do incoherent pairs overlay",
+        action="store_true",
+        default=False,
+    )
+
+    parser.add_argument(
+        "--keepEverything",
+        help="Write every collection to the reconstruction output, including "
+             "the tracker and calorimeter hits that are otherwise dropped when "
+             "an overlay is enabled",
+        action="store_true",
+        default=False,
+    )
+
     return parser.parse_known_args()[0]

MAIAConfig/reco_steer.py

@@ -8,7 +8,7 @@
 
 from reco_args import get_reco_args
 from recoAlgList import makeRecoAlgList
-from Common.steering import build_application
+from Common.steering import build_application, drop_calorimeter_hits, drop_tracker_hits
 
 # Collect arguments and build the reconstruction algorithm list
 args = get_reco_args()
@@ -22,6 +22,14 @@
     histo_file = "reco_histograms.root",
 )
 
+# With background overlaid the hit collections dominate the output file size,
+# so drop every (Sim)TrackerHit and (Sim)CalorimeterHit collection from the reco
+# output. The reconstructed objects (tracks, clusters, PFOs, jets) are kept.
+# --keepEverything switches the dropping off and writes the full event.
+if (args.doOverlayIP or args.doOverlayFull) and not args.keepEverything:
+    drop_tracker_hits()
+    drop_calorimeter_hits()
+
 # Per-algorithm CPU monitoring via the Gaudi Auditor Service. The ChronoAuditor
 # records CPU usage of each algorithm's execute() and ChronoStatSvc prints the
 # per-algorithm total + per-event-average table ("Final CPU consumption") at end

README.md

@@ -96,19 +96,45 @@ The full set is:
 | `--inputFiles` | both | per macro (see above) | Input EDM4hep file(s) to read; accepts several files. |
 | `--outputFile` | both | per macro (see above) | Output EDM4hep file to write. |
 | `--histoFile` | both | per macro | Output ROOT file for the histograms. |
-| `--doOverlayFull` | digi | `False` | Overlay beam-induced background (BIB). |
+| `--doOverlayFull` | digi + reco | `False` | Overlay beam-induced background (BIB). In the reco step it only acts as a flag: when set, all tracker and calorimeter hit collections are dropped from the reconstruction output (see below). |
 | `--OverlayFullPathToMuPlus` | digi | `/path/to/muplus/` | Directory of the μ⁺ BIB overlay files (used with `--doOverlayFull`). |
 | `--OverlayFullPathToMuMinus` | digi | `/path/to/muminus/` | Directory of the μ⁻ BIB overlay files (used with `--doOverlayFull`). |
 | `--OverlayFullNumberBackground` | digi | `812` | Number of BIB background files overlaid (used with `--doOverlayFull`). |
-| `--doOverlayIP` | digi | `False` | Overlay incoherent pairs. When both overlays are enabled they are chained (BIB then IP) before digitisation. |
+| `--doOverlayIP` | digi + reco | `False` | Overlay incoherent pairs. When both overlays are enabled they are chained (BIB then IP) before digitisation. In the reco step it only acts as a flag: when set, all tracker and calorimeter hit collections are dropped from the reconstruction output (see below). |
 | `--OverlayIPBackgroundFileNames` | digi | `[/path/to/pairs.slcio]` | Incoherent-pair overlay input file(s) (used with `--doOverlayIP`). |
 | `--doFilterDL` | digi | `False` | Double-layer hit filtering in the vertex detector. |
 | `--doTrackerConing` | digi + reco | `False` | Cone-filter the tracker hits around the signal MC particles (BIB cleaning). When enabled, the digi step writes the `…Coned` hit collections and the merger reads them before tracking. |
 | `--RandSeed` | digi | `42` | Random seed for the digitisation smearing. |
 | `--doTrackPerf` | reco | `False` | Run the tracking performance monitoring. |
+| `--keepEverything` | reco | `False` | Write every collection to the reconstruction output, overriding the hit dropping that `--doOverlayFull`/`--doOverlayIP` would otherwise trigger (see below). |
 | `--TrackingThreads` | reco | `1` | Internal thread count of the CKF tracking and truth-matching algorithms (independent of `--numThreads`). |
 | `--numThreads` | both | `1` | Number of threads for the Gaudi event loop. `1` runs serially; any value `> 1` enables the multi-threaded Gaudi Hive event loop with that many threads (scheduler + event slots); `0` auto-detects a sensible count from the CPU count. |
 
+### Hit collections in the overlay output
+
+Running with background (`--doOverlayFull` and/or `--doOverlayIP`) makes the hit
+collections dominate the output file, so `reco_steer.py` drops all of them from
+the reconstruction output when either flag is set: the tracker hits
+(`drop_tracker_hits`, i.e. `SimTrackerHit`, `TrackerHitPlane`, `TrackerHit3D`)
+and the calorimeter hits (`drop_calorimeter_hits`, i.e. `SimCalorimeterHit`,
+`CaloHitContribution`, `CalorimeterHit`, which also covers the muon system),
+together with the corresponding hit ↔ simulated-hit link collections. The
+selection is done by collection *type* through the `IOSvc` keep/drop switch, so
+it also covers the collections produced during reconstruction (e.g.
+`MergedTrackerHits`).
+
+The reconstructed objects — tracks, Pandora clusters, PFOs, jets, vertices,
+`MCParticles` and the track ↔ MC-particle links — are kept, but any reference
+they hold into a dropped hit collection (`Track::trackerHits`,
+`Cluster::hits`, …) no longer resolves in the output file.
+
+Pass `--keepEverything` to switch the dropping off and write the full event
+even with an overlay enabled:
+
+```bash
+k4run reco_steer.py --doOverlayFull --keepEverything
+```
+
 ### BIB hit cleaning
 
 Mirroring the Marlin `steer_reco.py` workflow, once the calorimeter hits are