Reader-defined derived variables (local BP5 path)

bindings/C/adios2/c/adios2_c_io.cpp

@@ -181,6 +181,24 @@ adios2_derived_variable *adios2_define_derived_variable(adios2_io *io, const cha
     }
     return variable;
 }
+
+adios2_error adios2_define_reader_derived_variable(adios2_io *io, const char *name,
+                                                   const char *expression)
+{
+    try
+    {
+        adios2::helper::CheckForNullptr(
+            io, "for adios2_io, in call to adios2_define_reader_derived_variable");
+        adios2::core::IO &ioCpp = *reinterpret_cast<adios2::core::IO *>(io);
+        ioCpp.DefineReaderDerivedVariable(name, expression);
+        return adios2_error_none;
+    }
+    catch (...)
+    {
+        return static_cast<adios2_error>(
+            adios2::helper::ExceptionToError("adios2_define_reader_derived_variable"));
+    }
+}
 #endif
 
 adios2_variable *adios2_define_variable(adios2_io *io, const char *name, const adios2_type type,

bindings/C/adios2/c/adios2_c_io.h

@@ -136,6 +136,21 @@ adios2_variable *adios2_define_variable(adios2_io *io, const char *name, const a
 adios2_derived_variable *adios2_define_derived_variable(adios2_io *io, const char *name,
                                                         const char *expression,
                                                         const adios2_derived_var_type type);
+
+/**
+ * @brief Define a reader-side derived variable within io: an expression the
+ * reader computes over variables in an opened file, without those variables
+ * having been marked derived at write time. It has no derived-variable type
+ * (the reader always computes, never stores) and no returned handle; it is
+ * resolved lazily against the file's variables, then found via adios2_inquire_
+ * variable. Use on a reading io.
+ * @param io handler that owns the variable
+ * @param name unique variable identifier
+ * @param expression derived expression over file variable names
+ * @return adios2_error 0: success, see enum adios2_error for errors
+ */
+adios2_error adios2_define_reader_derived_variable(adios2_io *io, const char *name,
+                                                   const char *expression);
 #endif
 
 /**

bindings/CXX/adios2/cxx/IO.cpp

@@ -191,6 +191,13 @@ VariableDerived IO::DefineDerivedVariable(const std::string &name, const std::st
 
     return VariableDerived(&m_IO->DefineDerivedVariable(name, expression, varType));
 }
+
+void IO::DefineReaderDerivedVariable(const std::string &name, const std::string &expression)
+{
+    helper::CheckForNullptr(m_IO, "for variable name " + name +
+                                      ", in call to IO::DefineReaderDerivedVariable");
+    m_IO->DefineReaderDerivedVariable(name, expression);
+}
 #endif
 StructDefinition IO::DefineStruct(const std::string &name, const size_t size)
 {

bindings/CXX/adios2/cxx/IO.h

@@ -159,6 +159,18 @@ class IO
 #ifdef ADIOS2_HAVE_DERIVED_VARIABLE
     VariableDerived DefineDerivedVariable(const std::string &name, const std::string &expression,
                                           const DerivedVarType varType = DerivedVarType::StatsOnly);
+
+    /**
+     * Define a reader-side derived variable: an expression the reader computes
+     * over variables in a file it opens, without those variables having been
+     * marked derived at write time. Has no DerivedVarType (the reader always
+     * computes). May be called before Open; it is resolved lazily against the
+     * file's variables, and InquireVariable finds it once an engine has resolved
+     * it. Use on a reading IO.
+     * @param name unique identifier, shares the IO namespace with all variables
+     * @param expression derived expression over file variable names
+     */
+    void DefineReaderDerivedVariable(const std::string &name, const std::string &expression);
 #endif
     VariableNT DefineVariable(const DataType type, const std::string &name,
                               const Dims &shape = Dims(), const Dims &start = Dims(),

bindings/Fortran/f2c/adios2_f2c_io.cpp

@@ -218,6 +218,21 @@ void FC_GLOBAL(adios2_define_derived_variable_f2c,
 #endif
 }
 
+void FC_GLOBAL(adios2_define_reader_derived_variable_f2c,
+               ADIOS2_DEFINE_READER_DERIVED_VARIABLE_F2C)(adios2_io **io, const char *name,
+                                                          const char *expression, int *ierr)
+{
+#ifdef ADIOS2_HAVE_DERIVED_VARIABLE
+    *ierr = static_cast<int>(adios2_define_reader_derived_variable(*io, name, expression));
+#else
+    std::cout << "ADIOS2 Warning:  adios2_define_reader_derived_variable() is not supported in the "
+                 "current ADIOS2 build. The expression "
+              << expression << " will be ignored and the variable " << name
+              << " will not be produced." << std::endl;
+    *ierr = static_cast<int>(adios2_error_exception);
+#endif
+}
+
 struct cnamelist
 {
     char **names;

bindings/Fortran/modules/adios2_io_define_derived_variable_mod.f90

@@ -8,6 +8,7 @@ module adios2_io_define_derived_variable_mod
    implicit none
 
    external adios2_define_derived_variable_f2c
+   external adios2_define_reader_derived_variable_f2c
 
 contains
 
@@ -31,4 +32,18 @@ subroutine adios2_define_derived_variable(variable, io, name, expression, adios2
 
    end subroutine
 
+   ! Reader-side derived variable: the reader computes the expression over
+   ! variables in an opened file. No derived-var type and no returned handle
+   ! (resolved lazily; find it afterwards with adios2_inquire_variable).
+   subroutine adios2_define_reader_derived_variable(io, name, expression, ierr)
+      type(adios2_io), intent(in) :: io
+      character*(*), intent(in) :: name
+      character*(*), intent(in) :: expression
+      integer, intent(out) :: ierr
+
+      call adios2_define_reader_derived_variable_f2c(io%f2c, &
+         TRIM(ADJUSTL(name))//char(0), TRIM(ADJUSTL(expression))//char(0), ierr)
+
+   end subroutine
+
 end module

bindings/Python/nbGlue.cpp

@@ -237,6 +237,9 @@ NB_MODULE(ADIOS2_PYTHON_MODULE_NAME, m)
              nb::arg("name"), nb::arg("expression"),
              nb::arg("vartype") = adios2::DerivedVarType::StatsOnly)
 
+        .def("DefineReaderDerivedVariable", &adios2::py11::IO::DefineReaderDerivedVariable,
+             nb::arg("name"), nb::arg("expression"))
+
         .def("InquireVariable", &adios2::py11::IO::InquireVariable)
 
         .def("InquireAttribute",

bindings/Python/nbIO.cpp

@@ -161,6 +161,19 @@ VariableDerived IO::DefineDerivedVariable(const std::string &name, const std::st
     return vd;
 }
 
+void IO::DefineReaderDerivedVariable(const std::string &name, const std::string &expression)
+{
+    helper::CheckForNullptr(m_IO, "for variable " + name +
+                                      ", in call to IO::DefineReaderDerivedVariable");
+
+#ifdef ADIOS2_HAVE_DERIVED_VARIABLE
+    m_IO->DefineReaderDerivedVariable(name, expression);
+#else
+    throw std::invalid_argument("ERROR: Derived Variables are not supported in this adios2 build "
+                                ", in call to DefineReaderDerivedVariable\n");
+#endif
+}
+
 Variable IO::InquireVariable(const std::string &name)
 {
     helper::CheckForNullptr(m_IO, "for variable " + name + ", in call to IO::InquireVariable");

bindings/Python/nbIO.h

@@ -64,6 +64,8 @@ class IO
     VariableDerived DefineDerivedVariable(const std::string &name, const std::string &expression,
                                           const DerivedVarType varType = DerivedVarType::StatsOnly);
 
+    void DefineReaderDerivedVariable(const std::string &name, const std::string &expression);
+
     Variable InquireVariable(const std::string &name);
 
     Attribute DefineAttribute(const std::string &name, const nb::ndarray<nb::numpy> &array,

source/adios2/core/IO.cpp

@@ -1054,6 +1054,146 @@ VariableDerived &IO::DefineDerivedVariable(const std::string &name, const std::s
     }
     return variable;
 }
+
+void IO::DefineReaderDerivedVariable(const std::string &name, const std::string &expression)
+{
+    PERFSTUBS_SCOPED_TIMER("IO::DefineReaderDerivedVariable");
+
+    // Name shares the IO namespace with every variable. Reject the collisions we
+    // can see now; a clash with a file variable not yet installed is caught at
+    // resolve time, when creating the placeholder throws on the duplicate name.
+    if (m_ReaderDerivedVariables.find(name) != m_ReaderDerivedVariables.end() ||
+        m_VariablesDerived.find(name) != m_VariablesDerived.end() ||
+        m_Variables.find(name) != m_Variables.end())
+    {
+        helper::Throw<std::invalid_argument>("Core", "IO", "DefineReaderDerivedVariable",
+                                             "variable " + name + " already defined in IO " +
+                                                 m_Name);
+    }
+
+    // Parse now so syntax errors surface at define time. Input variables are not
+    // resolved here: on a read IO the file's variables need not exist until Open.
+    derived::ExprNode exprTree = detail::ParseToExprNode(expression);
+
+    ReaderDerivedVariable rec;
+    rec.Expression = expression;
+    rec.ExprTree = std::move(exprTree);
+    m_ReaderDerivedVariables.emplace(name, std::move(rec));
+}
+
+std::vector<std::string> IO::GetUnresolvedReaderDerivedVariables() const
+{
+    std::vector<std::string> names;
+    for (const auto &pair : m_ReaderDerivedVariables)
+    {
+        if (!pair.second.Variable)
+        {
+            names.push_back(pair.first);
+        }
+    }
+    return names;
+}
+
+VariableDerived *IO::ResolveReaderDerivedVariable(const std::string &name)
+{
+    PERFSTUBS_SCOPED_TIMER("IO::ResolveReaderDerivedVariable");
+
+    auto it = m_ReaderDerivedVariables.find(name);
+    if (it == m_ReaderDerivedVariables.end())
+    {
+        return nullptr;
+    }
+    ReaderDerivedVariable &rec = it->second;
+    if (rec.Variable)
+    {
+        return rec.Variable.get(); // already resolved
+    }
+
+    // The name shares the IO namespace. A clash with a file variable could not be
+    // seen at define time (before Open); reject it clearly here rather than let
+    // the placeholder creation fail deep in the engine. rec.Variable is null, so
+    // the placeholder does not yet exist and any match is a foreign variable.
+    if (m_Variables.find(name) != m_Variables.end())
+    {
+        helper::Throw<std::invalid_argument>(
+            "Core", "IO", "ResolveReaderDerivedVariable",
+            "reader derived variable " + name +
+                " collides with a variable of the same name in the opened file");
+    }
+
+    // Bind inputs against the file's variables. If any input is not yet
+    // installed (e.g. a step that has not written it), stay pending and let a
+    // later step resolve. Check before ResolveTreeTypes, which annotates the
+    // tree in place and needs every input type.
+    std::vector<std::string> var_list = derived::VariableNameList(rec.ExprTree);
+    if (var_list.empty())
+    {
+        // A reader-side derived variable must read something from the file; an
+        // expression over no variables (e.g. all constants) has nothing to
+        // resolve against and no block structure to inherit.
+        helper::Throw<std::invalid_argument>("Core", "IO", "ResolveReaderDerivedVariable",
+                                             "reader derived variable " + name +
+                                                 " must reference at least one file variable");
+    }
+    std::map<std::string, DataType> name_to_type;
+    std::map<std::string, std::tuple<Dims, Dims, Dims>> name_to_dims;
+    for (const auto &var_name : var_list)
+    {
+        auto itVariable = m_Variables.find(var_name);
+        if (itVariable == m_Variables.end())
+        {
+            return nullptr; // input not present yet
+        }
+        name_to_type.insert({var_name, InquireVariableType(var_name)});
+        name_to_dims.insert({var_name,
+                             {itVariable->second->m_Start, itVariable->second->m_Count,
+                              itVariable->second->m_Shape}});
+    }
+
+    // Require input congruence. The reader-side read path takes the derived
+    // variable's block structure from its first input, so every input must share
+    // the same global shape (differing per-rank start/count is fine). This is the
+    // reader-side contract; shape-changing or broadcasting expressions are not
+    // supported on read.
+    const Dims &refShape = std::get<2>(name_to_dims.at(var_list.front()));
+    for (const auto &var_name : var_list)
+    {
+        if (std::get<2>(name_to_dims.at(var_name)) != refShape)
+        {
+            helper::Throw<std::invalid_argument>(
+                "Core", "IO", "ResolveReaderDerivedVariable",
+                "reader derived variable " + name + " requires congruent inputs, but " + var_name +
+                    " and " + var_list.front() + " have different shapes");
+        }
+    }
+
+    // Work on a copy so the stored tree stays pristine: ResolveTreeTypes mutates
+    // it in place, and a re-Open on the same IO must be able to resolve again.
+    derived::ExprNode exprTree = rec.ExprTree;
+    derived::ResolveTreeTypes(exprTree, name_to_type);
+    derived::ExprCodeStream codeStream = derived::GenerateCode(exprTree);
+    codeStream.ExprString = rec.Expression;
+    derived::SemanticsPass(codeStream, name_to_type);
+    derived::PlanBuffers(codeStream);
+    DataType expressionType = codeStream.OutputType;
+    auto outDims = derived::GetDims(codeStream, name_to_dims);
+
+    // varType is inert on the read side (the reader always computes, never
+    // stores); StatsOnly is the neutral value.
+    rec.Variable = std::unique_ptr<VariableDerived>(
+        new VariableDerived(name, std::move(exprTree), std::move(codeStream), rec.Expression,
+                            expressionType, std::get<2>(outDims), std::get<0>(outDims),
+                            std::get<1>(outDims), false, DerivedVarType::StatsOnly, name_to_type));
+    return rec.Variable.get();
+}
+
+void IO::ResetReaderDerivedResolutions() noexcept
+{
+    for (auto &pair : m_ReaderDerivedVariables)
+    {
+        pair.second.Variable.reset();
+    }
+}
 #endif
 
 StructDefinition &IO::DefineStruct(const std::string &name, const size_t size)

source/adios2/core/IO.h

@@ -183,6 +183,49 @@ class IO
     VariableDerived &
     DefineDerivedVariable(const std::string &name, const std::string &expression,
                           const DerivedVarType varType = DerivedVarType::StatsOnly);
+
+    /**
+     * @brief Define a reader-side derived variable: an expression the reader
+     * supplies over variables present in a file it opens, without those
+     * variables having been marked derived at write time. The reader's own
+     * process computes the result on Get.
+     *
+     * Unlike DefineDerivedVariable (a writer-side concept), this has no
+     * DerivedVarType: the reader always computes, never stores. The expression
+     * is parsed here (syntax errors throw now) but resolved lazily against the
+     * file's variables the first time an opened engine can satisfy it, so it may
+     * be defined before Open. Kept separate from the writer-side derived
+     * registry so a write engine sharing this IO never sees it.
+     * @param name unique identifier, shares the IO namespace with all variables
+     * @param expression derived expression over file variable names
+     * @exception std::invalid_argument if the name is already in use or the
+     * expression fails to parse
+     */
+    void DefineReaderDerivedVariable(const std::string &name, const std::string &expression);
+
+    /** @brief Whether any reader-side derived variable is defined on this IO.
+     * Cheap guard for the engine's per-step resolution pass. */
+    bool HasReaderDerivedVariables() const noexcept { return !m_ReaderDerivedVariables.empty(); }
+
+    /**
+     * @brief Names of reader-side derived variables not yet resolved. Used by the
+     * engine to drive lazy resolution once the file's variables are installed.
+     */
+    std::vector<std::string> GetUnresolvedReaderDerivedVariables() const;
+
+    /**
+     * @brief Resolve a reader-side derived variable against the currently
+     * installed file variables: bind input types and dims, run the derived
+     * pipeline, and build its VariableDerived. Idempotent. Returns the resolved
+     * object, or nullptr if it is unknown or an input is not yet installed (in
+     * which case it stays pending and can be resolved on a later step).
+     */
+    VariableDerived *ResolveReaderDerivedVariable(const std::string &name);
+
+    /** @brief Drop the resolved state of every reader-side derived variable,
+     * keeping the definitions. Called on engine close so a later Open on this IO
+     * re-resolves against the new file. */
+    void ResetReaderDerivedResolutions() noexcept;
 #endif
     VariableStruct &DefineStructVariable(const std::string &name, StructDefinition &def,
                                          const Dims &shape = Dims(), const Dims &start = Dims(),
@@ -552,6 +595,18 @@ class IO
     VarMap m_Variables;
 #ifdef ADIOS2_HAVE_DERIVED_VARIABLE
     VarMap m_VariablesDerived;
+
+    /** A reader-side derived variable, before and after lazy resolution.
+     *  Parsed at define time; Variable stays null until an opened engine
+     *  resolves the expression against the file's variables. Segregated from
+     *  m_VariablesDerived so a write engine sharing this IO never picks it up. */
+    struct ReaderDerivedVariable
+    {
+        std::string Expression;
+        adios2::derived::ExprNode ExprTree;
+        std::unique_ptr<VariableDerived> Variable; // null until resolved
+    };
+    std::unordered_map<std::string, ReaderDerivedVariable> m_ReaderDerivedVariables;
 #endif
 
     AttrMap m_Attributes;