From 3c7fcbe000e8733c2fde14e19a83b0d299eae415 Mon Sep 17 00:00:00 2001 From: alexstuk <33109346+alexstuk@users.noreply.github.com> Date: Tue, 17 Mar 2026 15:13:11 -0700 Subject: [PATCH 1/5] Create wfi_microlensing_detection_efficiency_catalog_level_4.yaml --- ...nsing_detection_efficiency_catalog_level_4.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 latest/SSC/MSOS/wfi_microlensing_detection_efficiency_catalog_level_4.yaml diff --git a/latest/SSC/MSOS/wfi_microlensing_detection_efficiency_catalog_level_4.yaml b/latest/SSC/MSOS/wfi_microlensing_detection_efficiency_catalog_level_4.yaml new file mode 100644 index 00000000..cb1bc36b --- /dev/null +++ b/latest/SSC/MSOS/wfi_microlensing_detection_efficiency_catalog_level_4.yaml @@ -0,0 +1,14 @@ +%YAML 1.1 +--- +$schema: asdf://stsci.edu/datamodels/roman/schemas/rad_schema-1.1.0 +id: asdf://stsci.edu/datamodels/roman/schemas/SSC/MSOS/wfi_microlensing_detection_efficiency_catalog_level_4-1.0.0 + +title: GBTDS Level 4 Microlensing Detection Efficiency Catalog +archive_meta: Science WFI Level 4 MSOS Detection Efficiency Catalog +type: object +allOf: + - $ref: asdf://stsci.edu/datamodels/roman/schemas/SSC/ssc_basic-1.0.0 + - $ref: asdf://stsci.edu/datamodels/roman/schemas/SSC/MSOS/keywords/msos_basic-1.0.0 + - $ref: asdf://stsci.edu/datamodels/roman/schemas/SSC/MSOS/keywords/msos_program-1.0.0 + - $ref: asdf://stsci.edu/datamodels/roman/schemas/SSC/MSOS/keywords/msos_region-1.0.0 + - $ref: asdf://stsci.edu/datamodels/roman/schemas/SSC/MSOS/keywords/msos_settings-1.0.0 From 781e60d191e8f1ae378db5b0dc4e0aae8dbc9dae Mon Sep 17 00:00:00 2001 From: alexstuk <33109346+alexstuk@users.noreply.github.com> Date: Tue, 17 Mar 2026 15:36:32 -0700 Subject: [PATCH 2/5] symlink and pyproject update --- pyproject.toml | 1 + ..._microlensing_detection_efficiency_catalog_level_4-1.0.0.yaml | 1 + 2 files changed, 2 insertions(+) create mode 120000 src/rad/resources/schemas/SSC/MSOS/wfi_microlensing_detection_efficiency_catalog_level_4-1.0.0.yaml diff --git a/pyproject.toml b/pyproject.toml index ec358ed2..9413b96f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,6 +94,7 @@ asdf_schema_xfail_tests = [ "src/rad/resources/schemas/SSC/GDPS/wfi_spec_dqa_2d_level_4-1.0.0.yaml", "src/rad/resources/schemas/SSC/GDPS/wfi_spec_catalog_dqa_level_4-1.0.0.yaml", "src/rad/resources/schemas/SSC/MSOS/wfi_microlensing_event_catalog_level_4-1.0.0.yaml", + "src/rad/resources/schemas/SSC/MSOS/wfi_microlensing_detection_efficiency_catalog_level_4-1.0.0.yaml", "src/rad/resources/schemas/SSC/MSOS/wfi_microlensing_light_curve_catalog_level_4-1.0.0.yaml", "src/rad/resources/schemas/SSC/MSOS/wfi_microlensing_object_periodic_catalog_level_4-1.0.0.yaml", "src/rad/resources/schemas/SSC/MSOS/wfi_microlensing_object_fiducial_catalog_level_4-1.0.0.yaml", diff --git a/src/rad/resources/schemas/SSC/MSOS/wfi_microlensing_detection_efficiency_catalog_level_4-1.0.0.yaml b/src/rad/resources/schemas/SSC/MSOS/wfi_microlensing_detection_efficiency_catalog_level_4-1.0.0.yaml new file mode 120000 index 00000000..b3542bb5 --- /dev/null +++ b/src/rad/resources/schemas/SSC/MSOS/wfi_microlensing_detection_efficiency_catalog_level_4-1.0.0.yaml @@ -0,0 +1 @@ +../../../../../../latest/SSC/MSOS/wfi_microlensing_detection_efficiency_catalog_level_4.yaml \ No newline at end of file From 56cdec4a6255b84710a69e21560c3240a9bcb61c Mon Sep 17 00:00:00 2001 From: alexstuk <33109346+alexstuk@users.noreply.github.com> Date: Tue, 9 Jun 2026 16:12:29 -0700 Subject: [PATCH 3/5] ROMAN-4866 Add detection efficiency archive metadata keys Add Detection Efficiency-specific RAD metadata for the Level 4 MSOS Detection Efficiency Catalog. The original PR only wired the product to the shared MSOS metadata blocks. This update adds a dedicated MSOS Detection Efficiency keyword block with archive-queryable fields for: - inject_medium - tp_count - fp_count - tn_count - fn_count The TP/FP/TN/FN values are represented as file-level row counts, matching the expected final product behavior where BinaryClass remains a row-level catalog column. The Detection Efficiency catalog schema now references the shared MSOS basic/program/settings metadata plus the new DE-specific keyword block. The spatial region block was removed because the DE catalog does not have a clear file-level sky footprint requirement, and similar MSOS L4 catalog products do not all require s_region. --- .../keywords/msos_detection_efficiency.yaml | 55 +++++++++++++++++++ ..._detection_efficiency_catalog_level_4.yaml | 2 +- .../msos_detection_efficiency-1.0.0.yaml | 1 + 3 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 latest/SSC/MSOS/keywords/msos_detection_efficiency.yaml create mode 120000 src/rad/resources/schemas/SSC/MSOS/keywords/msos_detection_efficiency-1.0.0.yaml diff --git a/latest/SSC/MSOS/keywords/msos_detection_efficiency.yaml b/latest/SSC/MSOS/keywords/msos_detection_efficiency.yaml new file mode 100644 index 00000000..d8f4e5af --- /dev/null +++ b/latest/SSC/MSOS/keywords/msos_detection_efficiency.yaml @@ -0,0 +1,55 @@ +%YAML 1.1 +--- +$schema: asdf://stsci.edu/datamodels/roman/schemas/rad_schema-1.1.0 +id: asdf://stsci.edu/datamodels/roman/schemas/SSC/MSOS/keywords/msos_detection_efficiency-1.0.0 + +title: MSOS detection efficiency keywords +extName: SSC +type: object +properties: + inject_medium: + title: Injection Medium + description: Detection efficiency injection medium represented by the rows in this file. + type: string + maxLength: 16 + enum: + - images + - lightcurve + - rhie + archive_catalog: + datatype: nvarchar(16) + destination: [WFIMSOSCatalog.inject_medium] + tp_count: + title: True Positive Count + description: Number of rows in this file with BinaryClass equal to TP. + type: integer + minimum: 0 + archive_catalog: + datatype: int + destination: [WFIMSOSCatalog.tp_count] + fp_count: + title: False Positive Count + description: Number of rows in this file with BinaryClass equal to FP. + type: integer + minimum: 0 + archive_catalog: + datatype: int + destination: [WFIMSOSCatalog.fp_count] + tn_count: + title: True Negative Count + description: Number of rows in this file with BinaryClass equal to TN. + type: integer + minimum: 0 + archive_catalog: + datatype: int + destination: [WFIMSOSCatalog.tn_count] + fn_count: + title: False Negative Count + description: Number of rows in this file with BinaryClass equal to FN. + type: integer + minimum: 0 + archive_catalog: + datatype: int + destination: [WFIMSOSCatalog.fn_count] + +required: [inject_medium, tp_count, fp_count, tn_count, fn_count] diff --git a/latest/SSC/MSOS/wfi_microlensing_detection_efficiency_catalog_level_4.yaml b/latest/SSC/MSOS/wfi_microlensing_detection_efficiency_catalog_level_4.yaml index cb1bc36b..93255cfc 100644 --- a/latest/SSC/MSOS/wfi_microlensing_detection_efficiency_catalog_level_4.yaml +++ b/latest/SSC/MSOS/wfi_microlensing_detection_efficiency_catalog_level_4.yaml @@ -10,5 +10,5 @@ allOf: - $ref: asdf://stsci.edu/datamodels/roman/schemas/SSC/ssc_basic-1.0.0 - $ref: asdf://stsci.edu/datamodels/roman/schemas/SSC/MSOS/keywords/msos_basic-1.0.0 - $ref: asdf://stsci.edu/datamodels/roman/schemas/SSC/MSOS/keywords/msos_program-1.0.0 - - $ref: asdf://stsci.edu/datamodels/roman/schemas/SSC/MSOS/keywords/msos_region-1.0.0 - $ref: asdf://stsci.edu/datamodels/roman/schemas/SSC/MSOS/keywords/msos_settings-1.0.0 + - $ref: asdf://stsci.edu/datamodels/roman/schemas/SSC/MSOS/keywords/msos_detection_efficiency-1.0.0 diff --git a/src/rad/resources/schemas/SSC/MSOS/keywords/msos_detection_efficiency-1.0.0.yaml b/src/rad/resources/schemas/SSC/MSOS/keywords/msos_detection_efficiency-1.0.0.yaml new file mode 120000 index 00000000..baaa18df --- /dev/null +++ b/src/rad/resources/schemas/SSC/MSOS/keywords/msos_detection_efficiency-1.0.0.yaml @@ -0,0 +1 @@ +../../../../../../../latest/SSC/MSOS/keywords/msos_detection_efficiency.yaml \ No newline at end of file From 74ea6c0925d67e600e71a7eea0aef85c34e3249a Mon Sep 17 00:00:00 2001 From: alexstuk <33109346+alexstuk@users.noreply.github.com> Date: Tue, 9 Jun 2026 16:19:42 -0700 Subject: [PATCH 4/5] Resolve detection efficiency pyproject conflict --- pyproject.toml | 64 +++++++++++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 29 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9413b96f..ce4c456a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,14 @@ classifiers = [ "Topic :: Scientific/Engineering :: Astronomy", "Programming Language :: Python :: 3", ] -dependencies = ["asdf >=4.1.0", "asdf-astropy >=0.8.0"] +dependencies = [ + "asdf>=4.1.0", + "asdf-astropy>=0.8.0", + "asdf-standard>=1.1.0", + "deepdiff>=8.1", + "pyyaml>=6.0", + "semantic-version>=2.10.0", +] license-files = ["LICENSE"] dynamic = ["version"] @@ -18,15 +25,14 @@ content-type = "text/markdown" [project.optional-dependencies] test = [ - "pytest>=7.0.0", - "pytest-doctestplus>=1.2.1", - "pytest-asdf-plugin>=0.1.0", "crds>=12.0.4", "GitPython>=3.1.44", - "semantic-version>=2.10.0", - "textual>=3.1", - "deepdiff>=8.1", + "pytest-asdf-plugin>=0.1.0", + "pytest-doctestplus>=1.2.1", + "pytest>=7.0.0", + ] +script = ["GitPython>=3.1.44", "astropy>=6.0.0", "textual>=3.1"] docs = [ "sphinx", "sphinx-asdf>=0.1.3", @@ -75,33 +81,33 @@ addopts = [ "--strict-markers", # fail on unknown markers "-ra", # Show summary of all failures/errors ] -testpaths = ["tests", "src/rad/resources/schemas"] +testpaths = ["tests", "latest"] filterwarnings = ["error"] asdf_schema_tests_enabled = "true" # All the SSC schemas with a $ref have to be xfailed due to asdf-format/pytest-asdf-plugin#9 asdf_schema_xfail_tests = [ - "src/rad/resources/schemas/SSC/CGI/cgi_ancillary-1.0.0.yaml", - "src/rad/resources/schemas/SSC/CGI/cgi_level_1-1.0.0.yaml", - "src/rad/resources/schemas/SSC/CGI/cgi_level_2a-1.0.0.yaml", - "src/rad/resources/schemas/SSC/CGI/cgi_level_2b-1.0.0.yaml", - "src/rad/resources/schemas/SSC/CGI/cgi_level_3-1.0.0.yaml", - "src/rad/resources/schemas/SSC/CGI/cgi_level_4-1.0.0.yaml", - "src/rad/resources/schemas/SSC/GDPS/wfi_spec_catalog_level_4-1.0.0.yaml", - "src/rad/resources/schemas/SSC/GDPS/wfi_spec_combined_1d_level_4-1.0.0.yaml", - "src/rad/resources/schemas/SSC/GDPS/wfi_spec_decontam_2d_level_4-1.0.0.yaml", - "src/rad/resources/schemas/SSC/GDPS/wfi_spec_individual_1d_level_4-1.0.0.yaml", - "src/rad/resources/schemas/SSC/GDPS/wfi_spec_location_table_level_4-1.0.0.yaml", - "src/rad/resources/schemas/SSC/GDPS/wfi_spec_dqa_2d_level_4-1.0.0.yaml", - "src/rad/resources/schemas/SSC/GDPS/wfi_spec_catalog_dqa_level_4-1.0.0.yaml", - "src/rad/resources/schemas/SSC/MSOS/wfi_microlensing_event_catalog_level_4-1.0.0.yaml", - "src/rad/resources/schemas/SSC/MSOS/wfi_microlensing_detection_efficiency_catalog_level_4-1.0.0.yaml", - "src/rad/resources/schemas/SSC/MSOS/wfi_microlensing_light_curve_catalog_level_4-1.0.0.yaml", - "src/rad/resources/schemas/SSC/MSOS/wfi_microlensing_object_periodic_catalog_level_4-1.0.0.yaml", - "src/rad/resources/schemas/SSC/MSOS/wfi_microlensing_object_fiducial_catalog_level_4-1.0.0.yaml", - "src/rad/resources/schemas/SSC/MSOS/wfi_microlensing_reference_frame_level_3-1.0.0.yaml", - "src/rad/resources/schemas/SSC/MSOS/wfi_microlensing_variability_catalog_level_4-1.0.0.yaml", + "latest/SSC/CGI/cgi_ancillary.yaml", + "latest/SSC/CGI/cgi_level_1.yaml", + "latest/SSC/CGI/cgi_level_2a.yaml", + "latest/SSC/CGI/cgi_level_2b.yaml", + "latest/SSC/CGI/cgi_level_3.yaml", + "latest/SSC/CGI/cgi_level_4.yaml", + "latest/SSC/GDPS/wfi_spec_catalog_level_4.yaml", + "latest/SSC/GDPS/wfi_spec_combined_1d_level_4.yaml", + "latest/SSC/GDPS/wfi_spec_decontam_2d_level_4.yaml", + "latest/SSC/GDPS/wfi_spec_individual_1d_level_4.yaml", + "latest/SSC/GDPS/wfi_spec_location_table_level_4.yaml", + "latest/SSC/GDPS/wfi_spec_dqa_2d_level_4.yaml", + "latest/SSC/GDPS/wfi_spec_catalog_dqa_level_4.yaml", + "latest/SSC/MSOS/wfi_microlensing_event_catalog_level_4.yaml", + "latest/SSC/MSOS/wfi_microlensing_detection_efficiency_catalog_level_4.yaml", + "latest/SSC/MSOS/wfi_microlensing_light_curve_catalog_level_4.yaml", + "latest/SSC/MSOS/wfi_microlensing_object_periodic_catalog_level_4.yaml", + "latest/SSC/MSOS/wfi_microlensing_object_fiducial_catalog_level_4.yaml", + "latest/SSC/MSOS/wfi_microlensing_reference_frame_level_3.yaml", + "latest/SSC/MSOS/wfi_microlensing_variability_catalog_level_4.yaml", ] -asdf_schema_root = "src/rad/resources/schemas" +asdf_schema_root = "latest" [tool.ruff] line-length = 130 From 9e8e903513a9375f789c42deec9b54e39fbf9a06 Mon Sep 17 00:00:00 2001 From: alexstuk <33109346+alexstuk@users.noreply.github.com> Date: Tue, 23 Jun 2026 11:46:43 -0700 Subject: [PATCH 5/5] Add detection efficiency changelog fragment --- changes/838.feature.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changes/838.feature.rst diff --git a/changes/838.feature.rst b/changes/838.feature.rst new file mode 100644 index 00000000..850da7ab --- /dev/null +++ b/changes/838.feature.rst @@ -0,0 +1 @@ +Add the SSC MSOS Level 4 detection efficiency catalog schema.