From 805f0fa26ead832b1b665066fd053dd580221bb6 Mon Sep 17 00:00:00 2001 From: akust <211008703+akustn@users.noreply.github.com> Date: Thu, 13 Aug 2026 09:55:31 +0200 Subject: [PATCH] Make _observations.py use polars --- src/ert/config/_observations.py | 54 +++++++++---------- .../config/test_observation_declaration.py | 6 +-- 2 files changed, 27 insertions(+), 33 deletions(-) diff --git a/src/ert/config/_observations.py b/src/ert/config/_observations.py index 26d7ee76e7e..cfc6b6330a7 100644 --- a/src/ert/config/_observations.py +++ b/src/ert/config/_observations.py @@ -18,7 +18,6 @@ ) import numpy as np -import pandas as pd import polars as pl import scipy as sp from pydantic import BaseModel, ConfigDict, Field, model_serializer @@ -686,11 +685,7 @@ def from_csv( f"The CSV file ({filename}) does not exist or is not accessible.", filename, ) - csv_file = pd.read_csv( - filename, - encoding="utf-8", - on_bad_lines="error", - ) + csv_file = pl.read_csv(filename, encoding="utf-8") required_columns = { "WELL_NAME", @@ -701,7 +696,9 @@ def from_csv( "EAST", "TVD", } - missing_required_columns = required_columns - set(csv_file.keys()) + + columns = set(csv_file.columns) + missing_required_columns = required_columns - columns if missing_required_columns: raise ObservationConfigError.with_context( f"The rft observations file {filename} is missing required column(s) " @@ -711,9 +708,9 @@ def from_csv( rft_observations = [] invalid_observations = [] - for row in csv_file.itertuples(index=True): - east_val = validate_float(str(row.EAST), "EAST") - north_val = validate_float(str(row.NORTH), "NORTH") + for index, row in enumerate(csv_file.iter_rows(named=True)): + east_val = validate_float(str(row["EAST"]), "EAST") + north_val = validate_float(str(row["NORTH"]), "NORTH") radius = radius if radius is not None else DEFAULT_LOCALIZATION_RADIUS shape_id = shape_registry.register( @@ -725,22 +722,20 @@ def from_csv( ) rft_observation = cls( - name=f"{observation_dict['name']}[{row.Index}]", - well=str(row.WELL_NAME), - date=str(row.DATE), + name=f"{observation_dict['name']}[{index}]", + well=str(row["WELL_NAME"]), + date=str(row["DATE"]), property=observed_property, - value=validate_float( - str(getattr(row, observed_property)), observed_property - ), - error=validate_float(str(row.ERROR), "ERROR"), + value=validate_float(str(row[observed_property]), observed_property), + error=validate_float(str(row["ERROR"]), "ERROR"), east=east_val, north=north_val, shape_id=shape_id, - tvd=validate_float(str(row.TVD), "TVD"), - md=validate_float(str(row.MD), "MD") if "MD" in csv_file else None, + tvd=validate_float(str(row["TVD"]), "TVD"), + md=validate_float(str(row["MD"]), "MD") if "MD" in columns else None, zone=( - str(row.ZONE) - if "ZONE" in csv_file and row.ZONE is not None + str(row["ZONE"]) + if "ZONE" in columns and row["ZONE"] is not None else None ), ) @@ -1007,11 +1002,10 @@ class SeismicObservation(BaseObservation): TOLERANCE: ClassVar[float] = 0.1 @staticmethod - def _load_observations(filepath: Path) -> pd.DataFrame: - df = pd.read_csv( + def _load_observations(filepath: Path) -> pl.DataFrame: + df = pl.read_csv( filepath, encoding="utf-8", - on_bad_lines="error", ) required_columns = { @@ -1020,7 +1014,7 @@ def _load_observations(filepath: Path) -> pd.DataFrame: "OBS", "OBS_ERROR", } - missing_required_columns = required_columns - set(df.keys()) + missing_required_columns = required_columns - set(df.columns) if missing_required_columns: raise ObservationConfigError.with_context( f"The seismic observations file {filepath} " @@ -1119,11 +1113,11 @@ def from_obs_dict( boundary_id = shape_registry.register(boundary) seismic_observations = [] - for row in df.itertuples(): - east = validate_float(str(row.X_UTME), "X_UTME") - north = validate_float(str(row.Y_UTMN), "Y_UTMN") - value = validate_float(str(row.OBS), "OBS") - error = validate_float(str(row.OBS_ERROR), "OBS_ERROR") + for row in df.iter_rows(named=True): + east = validate_float(str(row["X_UTME"]), "X_UTME") + north = validate_float(str(row["Y_UTMN"]), "Y_UTMN") + value = validate_float(str(row["OBS"]), "OBS") + error = validate_float(str(row["OBS_ERROR"]), "OBS_ERROR") # Currently supports only default localization radius as behavior of # LOCALIZATION keyword is undefined. All shapes are being registered diff --git a/tests/ert/unit_tests/config/test_observation_declaration.py b/tests/ert/unit_tests/config/test_observation_declaration.py index 3ff3be8f267..74a9a4e049d 100644 --- a/tests/ert/unit_tests/config/test_observation_declaration.py +++ b/tests/ert/unit_tests/config/test_observation_declaration.py @@ -841,9 +841,9 @@ def test_that_seismic_observation_instantiates(file_context_token): create_seismic_observation( name="NAME", filepath=Path("obs.csv"), - east=461231.5537527473, + east=461231.55375274725, north=5933187.729869121, - value=-0.0003566695393886, + value=-0.00035666953938864876, error=0.005, shape_id=0, boundary_id=None, @@ -853,7 +853,7 @@ def test_that_seismic_observation_instantiates(file_context_token): filepath=Path("obs.csv"), east=461156.9532936567, north=5933317.28138355, - value=-0.0005293887515127, + value=-0.0005293887515127136, error=0.005, shape_id=1, boundary_id=None,