diff --git a/cime_config/namelist_definition_cam.xml b/cime_config/namelist_definition_cam.xml index c07fe112d..090e9f0cc 100644 --- a/cime_config/namelist_definition_cam.xml +++ b/cime_config/namelist_definition_cam.xml @@ -206,6 +206,39 @@ .false. + + char*256(200) + initial_conditions + physics_nl + + Ordered list of glob patterns ('*' matches any run of characters) + excluding variables from the `{{ hilight }}ncdata_check{{ closehilight }}` + comparison, matched against the name each row is reported under in the + physics check output (for constituent-dimensioned variables, the + per-constituent file variable name). + + gitignore-style keep is supported: + A pattern with a leading '!' keeps (still compares) the name, any other pattern excludes it; + names matching no pattern are compared. + The first pattern that matches a name decides. + + Example: '!aerochem_vmr_post*', 'aerochem_vmr_*' excludes the aerochem_vmr rows except the aerochem_vmr_post* ones. + + Excluded rows are listed in the check output and are not silently dropped. + + This is intended to be used in conjunction with ncdata_check_err=.true. in testmods to exclude any expected + differences (i.e., where some variables are not to be compared against this snapshot) by design, + this would allow for a hard pass/fail test despite such expected differences. + + Such exclusions should be annotated with a reason next to the ncdata_check_exclude + clause in the testmod user_nl_cam. + + Default: '' (no exclusions) + + + '' + + real diagnostics diff --git a/src/core_utils/string_core_utils.F90 b/src/core_utils/string_core_utils.F90 index 1970ac9f3..ca52b3513 100644 --- a/src/core_utils/string_core_utils.F90 +++ b/src/core_utils/string_core_utils.F90 @@ -12,6 +12,8 @@ module string_core_utils public :: increment_string ! Increment a string whose ending characters are digits. public :: last_non_digit ! Get position of last non-digit in the input string. public :: get_last_significant_char ! Get position of last significant (non-blank, non-null) character in string. + public :: core_glob_match ! Match a string against a '*'-wildcard glob pattern. + public :: core_glob_list_excluded ! First-match-wins exclusion decision over an ordered glob pattern list ('!' keeps). interface tokenize module procedure tokenize_into_first_last @@ -405,4 +407,93 @@ integer pure function get_last_significant_char(cs) end function get_last_significant_char + !> Match `string` against a glob `pattern` in which `*` matches any run of + !> characters, including an empty one; every other character, including `?`, + !> matches only itself. Trailing blanks in both arguments are not significant + !> (leading and embedded blanks are). An empty pattern matches only an empty + !> string. (2026-07-02) + pure logical function core_glob_match(string, pattern) result(is_match) + character(len=*), intent(in) :: string + character(len=*), intent(in) :: pattern + + integer :: ls, lp ! significant lengths of string/pattern + integer :: s, p ! current positions in string/pattern + integer :: star_p ! position of the most recent '*' in pattern (0 = none seen) + integer :: star_s ! string position currently tried as that star's first unmatched character + + ls = len_trim(string) + lp = len_trim(pattern) + + s = 1 + p = 1 + star_p = 0 + star_s = 0 + + do while (s <= ls) + if (p <= lp) then + if (pattern(p:p) == '*') then + ! Record the star and first try matching it to nothing + star_p = p + star_s = s + p = p + 1 + cycle + else if (pattern(p:p) == string(s:s)) then + p = p + 1 + s = s + 1 + cycle + end if + end if + ! Mismatch: backtrack to the most recent star and extend its match + ! by one character; with no star to extend, the match fails. + if (star_p > 0) then + star_s = star_s + 1 + s = star_s + p = star_p + 1 + else + is_match = .false. + return + end if + end do + + ! String fully consumed; the pattern matches if only stars remain + do while (p <= lp) + if (pattern(p:p) /= '*') exit + p = p + 1 + end do + is_match = (p > lp) + + end function core_glob_match + + !> Decide whether `name` is excluded by an ordered list of glob `patterns` + !> (see `core_glob_match`). Patterns are evaluated in order and the FIRST + !> pattern whose glob matches decides: a pattern with a leading `!` keeps + !> the name (not excluded), any other pattern excludes it. A name matching + !> no pattern is not excluded; blank patterns are skipped, so fixed-size + !> namelist arrays can be passed directly. This enables gitignore-style + !> lists such as ['!aero_post*', 'aero_*'], which excludes the `aero_` + !> names except those beginning with `aero_post`. (2026-07-02) + pure logical function core_glob_list_excluded(name, patterns) result(excluded) + character(len=*), intent(in) :: name + character(len=*), intent(in) :: patterns(:) + + integer :: i + + excluded = .false. + do i = 1, size(patterns) + if (len_trim(patterns(i)) == 0) cycle + if (patterns(i)(1:1) == '!') then + if (core_glob_match(name, patterns(i)(2:))) then + ! Keep-verb: a match means the name stays compared + return + end if + else + if (core_glob_match(name, patterns(i))) then + excluded = .true. + return + end if + end if + end do + + end function core_glob_list_excluded + end module string_core_utils diff --git a/src/physics/utils/phys_comp.F90 b/src/physics/utils/phys_comp.F90 index 4db7c9c71..741bdad8a 100644 --- a/src/physics/utils/phys_comp.F90 +++ b/src/physics/utils/phys_comp.F90 @@ -31,6 +31,11 @@ module phys_comp character(len=SHR_KIND_CS) :: suite_parts_expect(2) = (/"physics_before_coupler", "physics_after_coupler "/) character(len=SHR_KIND_CS), allocatable :: suite_parts(:) logical :: ncdata_check_err = .false. + ! ncdata_check_exclude: ordered glob patterns excluding rows from the + ! ncdata_check comparison; first match wins, a leading '!' keeps a + ! matching row (see set_check_field_exclusions in physics_data) + integer, parameter :: max_check_exclude = 200 + character(len=SHR_KIND_CL) :: ncdata_check_exclude(max_check_exclude) character(len=SHR_KIND_CL) :: cam_physics_mesh = unset_str character(len=SHR_KIND_CS) :: cam_take_snapshot_before = unset_str character(len=SHR_KIND_CS) :: cam_take_snapshot_after = unset_str @@ -50,6 +55,7 @@ subroutine phys_readnl(nlfilename) use cam_abortutils, only: endrun use cam_initfiles, only: unset_path_str use cam_ccpp_cap, only: ccpp_physics_suite_list + use physics_data, only: set_check_field_exclusions ! filepath for file containing namelist input character(len=*), intent(in) :: nlfilename @@ -62,7 +68,7 @@ subroutine phys_readnl(nlfilename) namelist /physics_nl/ ncdata_check, min_difference, min_relative_value,& cam_take_snapshot_before, cam_take_snapshot_after, cam_physics_mesh,& - physics_suite, ncdata_check_err + physics_suite, ncdata_check_err, ncdata_check_exclude ! Initialize namelist variables to invalid values min_difference = HUGE(1.0_kind_phys) @@ -73,6 +79,7 @@ subroutine phys_readnl(nlfilename) ncdata_check = unset_path_str physics_suite = unset_str ncdata_check_err = .false. + ncdata_check_exclude(:) = '' ! Read namelist if (masterproc) then @@ -104,6 +111,12 @@ subroutine phys_readnl(nlfilename) mpi_character, masterprocid, mpicom, ierr) call mpi_bcast(ncdata_check_err, 1, mpi_logical, masterprocid, & mpicom, ierr) + call mpi_bcast(ncdata_check_exclude, & + len(ncdata_check_exclude(1))*max_check_exclude, mpi_character, & + masterprocid, mpicom, ierr) + + ! Store the check-exclusion patterns for use by check_field + call set_check_field_exclusions(ncdata_check_exclude) ! Check that the listed physics suite is actually present ! in the CCPP physics suite list: @@ -134,6 +147,15 @@ subroutine phys_readnl(nlfilename) min_difference write(iulog,*) 'Value Under Which Absolute Difference Calculated: ', & min_relative_value + if (any(len_trim(ncdata_check_exclude) > 0)) then + write(iulog,*) ' Rows excluded from the check by pattern ', & + '(first match wins, ''!'' keeps):' + do i = 1, max_check_exclude + if (len_trim(ncdata_check_exclude(i)) > 0) then + write(iulog,*) ' ', trim(ncdata_check_exclude(i)) + end if + end do + end if else write(iulog,*) ' Physics data check will not be performed' end if diff --git a/src/physics/utils/physics_data.F90 b/src/physics/utils/physics_data.F90 index 67eb78919..9758183d0 100644 --- a/src/physics/utils/physics_data.F90 +++ b/src/physics/utils/physics_data.F90 @@ -8,6 +8,7 @@ module physics_data public :: read_constituent_dimensioned_field public :: check_field public :: flush_check_field_verbose + public :: set_check_field_exclusions ! Non-standard variable indices: integer, public, parameter :: no_exist_idx = -1 @@ -43,6 +44,17 @@ module physics_data real(8), save :: verbose_avg_model(max_verbose_entries) real(8), save :: verbose_avg_snapshot(max_verbose_entries) + ! Check-exclusion patterns (the ncdata_check_exclude namelist option, + ! stored via set_check_field_exclusions): ordered '*'-glob patterns + ! matched against the label each check row is reported under; the first + ! matching pattern decides and a leading '!' keeps (still compares) a + ! matching row. Excluded rows skip the comparison entirely but are + ! buffered here and listed by flush_check_field_verbose, so exclusion + ! is never silent. + character(len=verbose_name_len), allocatable, save :: check_exclude_patterns(:) + integer, save :: num_excluded_entries = 0 + character(len=verbose_name_len), save :: excluded_stdnames(max_verbose_entries) + !============================================================================== CONTAINS !============================================================================== @@ -720,6 +732,11 @@ subroutine check_field_2d(file, var_names, timestep, current_value, & real(kind_phys) :: global_avg_model ! Global average of model state real(kind_phys) :: global_avg_snapshot! Global average of snapshot + !Skip rows excluded by the ncdata_check_exclude namelist option; + !excluded rows are buffered and listed by flush_check_field_verbose: + diff_found = .false. + if (check_field_excluded(stdname)) return + !Initialize output variables ierr = 0 allocate(buffer(size(current_value)), stat=ierr) @@ -926,6 +943,11 @@ subroutine check_field_3d(file, var_names, vcoord_name, timestep, & real(kind_phys) :: global_avg_model ! Global average of model state real(kind_phys) :: global_avg_snapshot! Global average of snapshot + !Skip rows excluded by the ncdata_check_exclude namelist option; + !excluded rows are buffered and listed by flush_check_field_verbose: + diff_found = .false. + if (check_field_excluded(stdname)) return + !Initialize output variables ierr = 0 allocate(buffer(size(current_value, 1), size(current_value, 2)), & @@ -1156,6 +1178,11 @@ subroutine check_field_4d(file, var_names, vcoord_name, timestep, & real(kind_phys) :: global_avg_model ! Global average of model state real(kind_phys) :: global_avg_snapshot! Global average of snapshot + !Skip rows excluded by the ncdata_check_exclude namelist option; + !excluded rows are buffered and listed by flush_check_field_verbose: + diff_found = .false. + if (check_field_excluded(stdname)) return + !Initialize output variables ierr = 0 allocate(buffer(size(current_value, 1), size(current_value, 2), & @@ -1337,6 +1364,77 @@ subroutine check_field_4d(file, var_names, vcoord_name, timestep, & end subroutine check_field_4d + subroutine set_check_field_exclusions(patterns) + ! + ! Store the ordered check-exclusion glob patterns (the + ! ncdata_check_exclude namelist option). Blank entries are dropped; + ! the relative order of the rest is preserved (first match wins in + ! check_field_excluded). + ! + use cam_abortutils, only: check_allocate + + !Dummy variables: + character(len=*), intent(in) :: patterns(:) + + !Local variables: + integer :: i, num_patterns, ierr + character(len=*), parameter :: subname = 'set_check_field_exclusions' + + num_patterns = 0 + do i = 1, size(patterns) + if (len_trim(patterns(i)) > 0) then + num_patterns = num_patterns + 1 + end if + end do + + if (allocated(check_exclude_patterns)) then + deallocate(check_exclude_patterns) + end if + allocate(check_exclude_patterns(num_patterns), stat=ierr) + call check_allocate(ierr, subname, 'check_exclude_patterns') + + num_patterns = 0 + do i = 1, size(patterns) + if (len_trim(patterns(i)) > 0) then + num_patterns = num_patterns + 1 + check_exclude_patterns(num_patterns) = patterns(i) + end if + end do + + end subroutine set_check_field_exclusions + + logical function check_field_excluded(stdname) + ! + ! Decide whether a check row is excluded from comparison by the + ! ncdata_check_exclude patterns; excluded rows are buffered and later + ! listed by flush_check_field_verbose. Runs identically on all ranks + ! (the patterns arrive via the namelist broadcast), so the collective + ! MPI calls inside check_field stay aligned. + ! + use string_core_utils, only: core_glob_list_excluded + + !Dummy variables: + character(len=*), intent(in) :: stdname + + check_field_excluded = .false. + if (.not. allocated(check_exclude_patterns)) then + return + end if + if (size(check_exclude_patterns) == 0) then + return + end if + + check_field_excluded = core_glob_list_excluded(trim(stdname), & + check_exclude_patterns) + + if (check_field_excluded .and. & + (num_excluded_entries < max_verbose_entries)) then + num_excluded_entries = num_excluded_entries + 1 + excluded_stdnames(num_excluded_entries) = stdname + end if + + end function check_field_excluded + subroutine write_check_field_entry(stdname, & diff_count, nan_count, & max_diff, max_diff_rank, & @@ -1438,6 +1536,19 @@ subroutine flush_check_field_verbose() integer :: i, slen integer, parameter :: indent_level = 50 + !List and reset the rows excluded by ncdata_check_exclude (all ranks + !buffer identically; only masterproc prints): + if (masterproc .and. (num_excluded_entries > 0)) then + write(iulog, *) '' + write(iulog, '(1x,a,i0,a)') & + 'Excluded from comparison by ncdata_check_exclude (', & + num_excluded_entries, ' rows, no diffs computed):' + do i = 1, num_excluded_entries + write(iulog, '(4x,a)') trim(excluded_stdnames(i)) + end do + end if + num_excluded_entries = 0 + if (num_verbose_entries == 0) return if (.not. masterproc) return diff --git a/test/unit/fortran/src/core_utils/test_string_core_utils.pf b/test/unit/fortran/src/core_utils/test_string_core_utils.pf index 1dc68e0d4..c14dfa84b 100644 --- a/test/unit/fortran/src/core_utils/test_string_core_utils.pf +++ b/test/unit/fortran/src/core_utils/test_string_core_utils.pf @@ -851,3 +851,188 @@ subroutine test_increment_string_positive_overflow() @assertEqual(-2, result) end subroutine test_increment_string_positive_overflow + +@test +subroutine test_glob_match_exact() + use funit + use string_core_utils, only: core_glob_match + + @assertTrue(core_glob_match('aerochem_vmr_bc_a1', 'aerochem_vmr_bc_a1')) + @assertFalse(core_glob_match('aerochem_vmr_bc_a1', 'aerochem_vmr_bc_a2')) +end subroutine test_glob_match_exact + +@test +subroutine test_glob_match_trailing_star() + use funit + use string_core_utils, only: core_glob_match + + @assertTrue(core_glob_match('aerochem_vmr_bc_a1', 'aerochem_vmr_*')) + @assertFalse(core_glob_match('other_var', 'aerochem_vmr_*')) +end subroutine test_glob_match_trailing_star + +@test +subroutine test_glob_match_star_matches_empty() + use funit + use string_core_utils, only: core_glob_match + + @assertTrue(core_glob_match('aerochem_vmr_', 'aerochem_vmr_*')) + @assertTrue(core_glob_match('ab', 'a*b')) +end subroutine test_glob_match_star_matches_empty + +@test +subroutine test_glob_match_leading_star() + use funit + use string_core_utils, only: core_glob_match + + @assertTrue(core_glob_match('aerochem_vmr_so4_c1', '*_c1')) + @assertFalse(core_glob_match('aerochem_vmr_so4_a1', '*_c1')) +end subroutine test_glob_match_leading_star + +@test +subroutine test_glob_match_middle_star_backtracks() + use funit + use string_core_utils, only: core_glob_match + + ! The star must be able to extend past an early partial match of the tail + @assertTrue(core_glob_match('a_x_y_b', 'a_*_b')) + @assertTrue(core_glob_match('mam_del_h2so4_gasprod', 'mam_*_gasprod')) + @assertFalse(core_glob_match('mam_del_h2so4_aeruptk', 'mam_*_gasprod')) +end subroutine test_glob_match_middle_star_backtracks + +@test +subroutine test_glob_match_multiple_stars() + use funit + use string_core_utils, only: core_glob_match + + @assertTrue(core_glob_match('xAAyBBz', 'x*y*z')) + @assertTrue(core_glob_match('xyz', 'x*y*z')) + @assertFalse(core_glob_match('xAAzBBy', 'x*y*z')) +end subroutine test_glob_match_multiple_stars + +@test +subroutine test_glob_match_star_only_and_empty() + use funit + use string_core_utils, only: core_glob_match + + @assertTrue(core_glob_match('anything_at_all', '*')) + @assertTrue(core_glob_match('', '*')) + @assertTrue(core_glob_match('', '')) + @assertFalse(core_glob_match('a', '')) +end subroutine test_glob_match_star_only_and_empty + +@test +subroutine test_glob_match_question_mark_is_literal() + use funit + use string_core_utils, only: core_glob_match + + ! Only '*' is a wildcard; '?' matches itself + @assertTrue(core_glob_match('what?', 'what?')) + @assertFalse(core_glob_match('whatx', 'what?')) +end subroutine test_glob_match_question_mark_is_literal + +@test +subroutine test_glob_match_trailing_blanks_insignificant() + use funit + use string_core_utils, only: core_glob_match + + character(len=32) :: padded_string, padded_pattern + + padded_string = 'aerochem_vmr_bc_a1' + padded_pattern = 'aerochem_vmr_*' + + @assertTrue(core_glob_match(padded_string, padded_pattern)) +end subroutine test_glob_match_trailing_blanks_insignificant + +@test +subroutine test_glob_list_excluded_simple() + use funit + use string_core_utils, only: core_glob_list_excluded + + character(len=32) :: patterns(1) + + patterns(1) = 'aerochem_vmr_*' + + @assertTrue(core_glob_list_excluded('aerochem_vmr_bc_a1', patterns)) + @assertFalse(core_glob_list_excluded('other_var', patterns)) +end subroutine test_glob_list_excluded_simple + +@test +subroutine test_glob_list_excluded_keep_verb_first_match_wins() + use funit + use string_core_utils, only: core_glob_list_excluded + + character(len=32) :: patterns(2) + + ! gitignore-style: keep the verdict rows, exclude the rest of the family + patterns(1) = '!aerochem_vmr_post*' + patterns(2) = 'aerochem_vmr_*' + + @assertFalse(core_glob_list_excluded('aerochem_vmr_postcoag_bc_a1', patterns)) + @assertTrue(core_glob_list_excluded('aerochem_vmr_bc_a1', patterns)) + @assertFalse(core_glob_list_excluded('unrelated_var', patterns)) +end subroutine test_glob_list_excluded_keep_verb_first_match_wins + +@test +subroutine test_glob_list_excluded_order_matters() + use funit + use string_core_utils, only: core_glob_list_excluded + + character(len=32) :: patterns(2) + + ! With the broad exclude first, the keep-verb is never reached + patterns(1) = 'aerochem_vmr_*' + patterns(2) = '!aerochem_vmr_post*' + + @assertTrue(core_glob_list_excluded('aerochem_vmr_postcoag_bc_a1', patterns)) +end subroutine test_glob_list_excluded_order_matters + +@test +subroutine test_glob_list_excluded_blank_entries_skipped() + use funit + use string_core_utils, only: core_glob_list_excluded + + character(len=32) :: patterns(4) + + ! Fixed-size namelist arrays arrive mostly blank + patterns(1) = '' + patterns(2) = '!aerochem_vmr_post*' + patterns(3) = '' + patterns(4) = 'aerochem_vmr_*' + + @assertFalse(core_glob_list_excluded('aerochem_vmr_postnewnuc_so4_a1', patterns)) + @assertTrue(core_glob_list_excluded('aerochem_vmr_so4_a1', patterns)) +end subroutine test_glob_list_excluded_blank_entries_skipped + +@test +subroutine test_glob_list_excluded_empty_list() + use funit + use string_core_utils, only: core_glob_list_excluded + + character(len=32) :: patterns(0) + + @assertFalse(core_glob_list_excluded('any_var', patterns)) +end subroutine test_glob_list_excluded_empty_list + +@test +subroutine test_glob_list_excluded_multiple_keep_verbs() + use funit + use string_core_utils, only: core_glob_list_excluded + + character(len=32) :: patterns(3) + + ! Keep the verdict rows AND the cloud-borne (_c) rows, exclude the rest + ! of the family; names outside the family are compared as usual + patterns(1) = '!aerochem_vmr_post*' + patterns(2) = '!aerochem_vmr_*_c*' + patterns(3) = 'aerochem_vmr_*' + + @assertFalse(core_glob_list_excluded('aerochem_vmr_postnewnuc_so4_a1', patterns)) + @assertFalse(core_glob_list_excluded('aerochem_vmr_postcoag_bc_a1', patterns)) + @assertFalse(core_glob_list_excluded('aerochem_vmr_num_c1', patterns)) + @assertFalse(core_glob_list_excluded('aerochem_vmr_so4_c2', patterns)) + @assertTrue(core_glob_list_excluded('aerochem_vmr_so4_a1', patterns)) + @assertTrue(core_glob_list_excluded('aerochem_vmr_ncl_a3', patterns)) + @assertTrue(core_glob_list_excluded('aerochem_vmr_H2SO4', patterns)) + @assertTrue(core_glob_list_excluded('aerochem_vmr_SOAG', patterns)) + @assertFalse(core_glob_list_excluded('mam_del_h2so4_gasprod', patterns)) +end subroutine test_glob_list_excluded_multiple_keep_verbs