Skip to content

Make _observations.py use polars #14169

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
akustn merged 1 commit into from
Aug 14, 2026
+27 −33
Merged

Make _observations.py use polars #14169

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 24 additions & 30 deletions src/ert/config/_observations.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
)

import numpy as np
import pandas as pd
import polars as pl
import scipy as sp
from pydantic import BaseModel, ConfigDict, Field, model_serializer
Expand Down Expand Up @@ -686,11 +685,7 @@ def from_csv(
f"The CSV file ({filename}) does not exist or is not accessible.",
filename,
)
csv_file = pd.read_csv(
filename,
encoding="utf-8",
on_bad_lines="error",

@SAKavli SAKavli Aug 13, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this setting default in polars?

@akustn akustn Aug 13, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Polars throws an error by default when there's malformed rows or parsing failures

)
csv_file = pl.read_csv(filename, encoding="utf-8")

required_columns = {
"WELL_NAME",
Expand All @@ -701,7 +696,9 @@ def from_csv(
"EAST",
"TVD",
}
missing_required_columns = required_columns - set(csv_file.keys())

columns = set(csv_file.columns)
missing_required_columns = required_columns - columns
if missing_required_columns:
raise ObservationConfigError.with_context(
f"The rft observations file {filename} is missing required column(s) "
Expand All @@ -711,9 +708,9 @@ def from_csv(

rft_observations = []
invalid_observations = []
for row in csv_file.itertuples(index=True):
east_val = validate_float(str(row.EAST), "EAST")
north_val = validate_float(str(row.NORTH), "NORTH")
for index, row in enumerate(csv_file.iter_rows(named=True)):
east_val = validate_float(str(row["EAST"]), "EAST")
north_val = validate_float(str(row["NORTH"]), "NORTH")
radius = radius if radius is not None else DEFAULT_LOCALIZATION_RADIUS

shape_id = shape_registry.register(
Expand All @@ -725,22 +722,20 @@ def from_csv(
)

rft_observation = cls(
name=f"{observation_dict['name']}[{row.Index}]",
well=str(row.WELL_NAME),
date=str(row.DATE),
name=f"{observation_dict['name']}[{index}]",
well=str(row["WELL_NAME"]),
date=str(row["DATE"]),

@SAKavli SAKavli Aug 13, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you ensure it is tested that a file while doesn't contain the required rows raised an error nicely without traceback?
It is probably tested somewhere already, but nice to make sure.

@akustn akustn Aug 13, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

there is tests in test_observation_declaration.py covering this. :)
(test_that_missing_user_specified_property_raises_error and test_that_missing_columns_in_rft_observations_file_raises_error)

property=observed_property,
value=validate_float(
str(getattr(row, observed_property)), observed_property
),
error=validate_float(str(row.ERROR), "ERROR"),
value=validate_float(str(row[observed_property]), observed_property),
error=validate_float(str(row["ERROR"]), "ERROR"),
east=east_val,
north=north_val,
shape_id=shape_id,
tvd=validate_float(str(row.TVD), "TVD"),
md=validate_float(str(row.MD), "MD") if "MD" in csv_file else None,
tvd=validate_float(str(row["TVD"]), "TVD"),
md=validate_float(str(row["MD"]), "MD") if "MD" in columns else None,
zone=(
str(row.ZONE)
if "ZONE" in csv_file and row.ZONE is not None
str(row["ZONE"])
if "ZONE" in columns and row["ZONE"] is not None
else None
),
)
Expand Down Expand Up @@ -1007,11 +1002,10 @@ class SeismicObservation(BaseObservation):
TOLERANCE: ClassVar[float] = 0.1

@staticmethod
def _load_observations(filepath: Path) -> pd.DataFrame:
df = pd.read_csv(
def _load_observations(filepath: Path) -> pl.DataFrame:
df = pl.read_csv(
filepath,
encoding="utf-8",
on_bad_lines="error",
)

required_columns = {
Expand All @@ -1020,7 +1014,7 @@ def _load_observations(filepath: Path) -> pd.DataFrame:
"OBS",
"OBS_ERROR",
}
missing_required_columns = required_columns - set(df.keys())
missing_required_columns = required_columns - set(df.columns)
if missing_required_columns:
raise ObservationConfigError.with_context(
f"The seismic observations file {filepath} "
Expand Down Expand Up @@ -1119,11 +1113,11 @@ def from_obs_dict(
boundary_id = shape_registry.register(boundary)

seismic_observations = []
for row in df.itertuples():
east = validate_float(str(row.X_UTME), "X_UTME")
north = validate_float(str(row.Y_UTMN), "Y_UTMN")
value = validate_float(str(row.OBS), "OBS")
error = validate_float(str(row.OBS_ERROR), "OBS_ERROR")
for row in df.iter_rows(named=True):
east = validate_float(str(row["X_UTME"]), "X_UTME")
north = validate_float(str(row["Y_UTMN"]), "Y_UTMN")
value = validate_float(str(row["OBS"]), "OBS")
error = validate_float(str(row["OBS_ERROR"]), "OBS_ERROR")

# Currently supports only default localization radius as behavior of
# LOCALIZATION keyword is undefined. All shapes are being registered
Expand Down
6 changes: 3 additions & 3 deletions tests/ert/unit_tests/config/test_observation_declaration.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -841,9 +841,9 @@ def test_that_seismic_observation_instantiates(file_context_token):
create_seismic_observation(
name="NAME",
filepath=Path("obs.csv"),
east=461231.5537527473,
east=461231.55375274725,
north=5933187.729869121,
value=-0.0003566695393886,
value=-0.00035666953938864876,
error=0.005,
shape_id=0,
boundary_id=None,
Expand All @@ -853,7 +853,7 @@ def test_that_seismic_observation_instantiates(file_context_token):
filepath=Path("obs.csv"),
east=461156.9532936567,
north=5933317.28138355,
value=-0.0005293887515127,
value=-0.0005293887515127136,
error=0.005,
shape_id=1,
boundary_id=None,
Expand Down
Loading