Skip to content

refactor: build paths with pathlib instead of os.path #153

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
cmulk merged 1 commit into from
Oct 10, 2026
+161 −161
Merged

refactor: build paths with pathlib instead of os.path #153

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion src/python/phenix_apps/apps/otsim/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
from pathlib import Path

import lxml.etree as ET


Expand Down Expand Up @@ -289,5 +291,5 @@ def append_to_cpu(self, child):
self.cpu.append(child)

def to_file(self, path):
with open(path, "w") as f:
with Path(path).open("w") as f:
f.write(ET.tostring(self.root, pretty_print=True).decode())
2 changes: 1 addition & 1 deletion src/python/phenix_apps/apps/scale/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ Plugins can override the default Minimega template (`minimega.mako`) to customiz
class MyPlugin(ScalePlugin):
def __init__(self):
# Set templates directory relative to this file
self.templates_dir = os.path.join(os.path.dirname(__file__), "templates")
self.templates_dir = Path(__file__).parent / "templates"

def get_template_name(self) -> str:
return "my_custom_vm.mako"
Expand Down
29 changes: 13 additions & 16 deletions src/python/phenix_apps/apps/scorch/app.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,19 +83,19 @@ def __init__(self, typ: str) -> None:
) from ex

self.exp_name: str = self.experiment.spec.experimentName
self.exp_dir: str = self.experiment.spec.baseDir
self.exp_dir: Path = Path(self.experiment.spec.baseDir)
self.metadata: Box | None = self.extract_metadata()

self.root_dir: str = os.path.join(PHENIX_DIR, "images")
self.files_dir: str = os.getenv(
"PHENIX_FILES_DIR", os.path.join(self.root_dir, self.exp_name, "files")

@GhostofGoes GhostofGoes Oct 2, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

make all of these path objects yeah?

@adamsrnmsu adamsrnmsu Oct 4, 2026

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

exp_dir is a Path now as well. I also dropped the leftover Path(self.base_dir) wraps in cc and ssh.

self.root_dir: Path = Path(PHENIX_DIR) / "images"
self.files_dir: Path = Path(
os.getenv("PHENIX_FILES_DIR", self.root_dir / self.exp_name / "files")
)
self.base_dir: str = os.path.join(
self.files_dir,
f"scorch/run-{self.run}/{self.name}/loop-{self.loop}-count-{self.count}",
self.base_dir: Path = (
self.files_dir
/ f"scorch/run-{self.run}/{self.name}/loop-{self.loop}-count-{self.count}"
)

os.makedirs(self.base_dir, exist_ok=True)
self.base_dir.mkdir(parents=True, exist_ok=True)

self._mm: minimega.minimega | None = None # minimega instance
self._es: Elasticsearch | None = None # Elasticsearch instance
Expand Down Expand Up @@ -175,10 +175,7 @@ def execute_stage(self) -> None:
# filenames can't have colons, so replace with dashes
start_ts_filename = start_dt.strftime("%Y-%m-%dT%H-%M-%SZ")

info_file = os.path.join(
self.base_dir,
f"{self.stage}-{start_ts_filename}.json",
)
info_file = self.base_dir / f"{self.stage}-{start_ts_filename}.json"

content = {
"experiment_name": self.exp_name,
Expand All @@ -194,7 +191,7 @@ def execute_stage(self) -> None:
"stderr": self._format_stream(stderr_mirror.getvalue()),
"logs": self._format_stream(log_buffer.getvalue()),
}
with open(info_file, "w") as f:
with info_file.open("w") as f:
json.dump(content, f, indent=2)

if error_occurred:
Expand Down Expand Up @@ -226,7 +223,7 @@ def mm_init(self, namespaced: bool = True) -> minimega.minimega:

saved_stdout = sys.stdout

sys.stdout = open("/dev/null", "w")
sys.stdout = Path("/dev/null").open("w")

mm = None

Expand Down Expand Up @@ -374,9 +371,9 @@ def get_host_and_iface(self, config: Box) -> tuple[str, int]:

def recv_file(self, vm: str, src: list[str] | str, dst: str = "") -> None:
if not dst and isinstance(src, str):
dst = os.path.join(self.base_dir, Path(src).name)
dst = str(self.base_dir / Path(src).name)
elif not dst and isinstance(src, list):
dst = self.base_dir
dst = str(self.base_dir)

logger.info(f"copying file from {vm} (src={src}, dst={dst})")

Expand Down
66 changes: 33 additions & 33 deletions src/python/phenix_apps/apps/scorch/caldera/caldera.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import json
import os
import time
import uuid
from pathlib import PurePath

from phenix_apps.apps.scorch import ComponentBase
from phenix_apps.common import utils
Expand Down Expand Up @@ -64,17 +64,17 @@ def __run(self, stage):
logger.info(f"looking up ID for '{adversary}' adversary...")

cmd_file = f"run-{self.extract_run_name()}_{uuid.uuid4()!s}.sh"
cmd_src = os.path.join(self.root_dir, self.exp_name, cmd_file)
cmd_dst = os.path.join("/tmp/miniccc/files", self.exp_name, cmd_file)
cmd_src = self.root_dir / self.exp_name / cmd_file
cmd_dst = PurePath("/tmp/miniccc/files") / self.exp_name / cmd_file

with open(cmd_src, "w") as f:
with cmd_src.open("w") as f:
utils.mako_serve_template(
"api_call.mako", templates, f, model="adversaries"
)

utils.mm_cc_send_wait(mm, server, cmd_src, self.exp_name)
utils.mm_cc_send_wait(mm, server, str(cmd_src), self.exp_name)

os.remove(cmd_src)
cmd_src.unlink()

result = utils.mm_exec_wait(mm, server, f"bash {cmd_dst}", once=True)

Expand All @@ -101,17 +101,17 @@ def __run(self, stage):
logger.info(f"looking up ID for '{facts}' fact source...")

cmd_file = f"run-{self.extract_run_name()}_{uuid.uuid4()!s}.sh"
cmd_src = os.path.join(self.root_dir, self.exp_name, cmd_file)
cmd_dst = os.path.join("/tmp/miniccc/files", self.exp_name, cmd_file)
cmd_src = self.root_dir / self.exp_name / cmd_file
cmd_dst = PurePath("/tmp/miniccc/files") / self.exp_name / cmd_file

with open(cmd_src, "w") as f:
with cmd_src.open("w") as f:
utils.mako_serve_template(
"api_call.mako", templates, f, model="sources"
)

utils.mm_cc_send_wait(mm, server, cmd_src, self.exp_name)
utils.mm_cc_send_wait(mm, server, str(cmd_src), self.exp_name)

os.remove(cmd_src)
cmd_src.unlink()

result = utils.mm_exec_wait(mm, server, f"bash {cmd_dst}", once=True)

Expand All @@ -138,17 +138,17 @@ def __run(self, stage):
logger.info(f"looking up ID for '{planner}' planner...")

cmd_file = f"run-{self.extract_run_name()}_{uuid.uuid4()!s}.sh"
cmd_src = os.path.join(self.root_dir, self.exp_name, cmd_file)
cmd_dst = os.path.join("/tmp/miniccc/files", self.exp_name, cmd_file)
cmd_src = self.root_dir / self.exp_name / cmd_file
cmd_dst = PurePath("/tmp/miniccc/files") / self.exp_name / cmd_file

with open(cmd_src, "w") as f:
with cmd_src.open("w") as f:
utils.mako_serve_template(
"api_call.mako", templates, f, model="planners"
)

utils.mm_cc_send_wait(mm, server, cmd_src, self.exp_name)
utils.mm_cc_send_wait(mm, server, str(cmd_src), self.exp_name)

os.remove(cmd_src)
cmd_src.unlink()

result = utils.mm_exec_wait(mm, server, f"bash {cmd_dst}", once=True)

Expand All @@ -171,15 +171,15 @@ def __run(self, stage):
)

cmd_file = f"run-{self.extract_run_name()}_{uuid.uuid4()!s}.sh"
cmd_src = os.path.join(self.root_dir, self.exp_name, cmd_file)
cmd_dst = os.path.join("/tmp/miniccc/files", self.exp_name, cmd_file)
cmd_src = self.root_dir / self.exp_name / cmd_file
cmd_dst = PurePath("/tmp/miniccc/files") / self.exp_name / cmd_file

with open(cmd_src, "w") as f:
with cmd_src.open("w") as f:
utils.mako_serve_template("new_operation.mako", templates, f, op=op)

utils.mm_cc_send_wait(mm, server, cmd_src, self.exp_name)
utils.mm_cc_send_wait(mm, server, str(cmd_src), self.exp_name)

os.remove(cmd_src)
cmd_src.unlink()

result = utils.mm_exec_wait(mm, server, f"bash {cmd_dst}", once=True)

Expand All @@ -192,17 +192,17 @@ def __run(self, stage):
time.sleep(10)

cmd_file = f"run-{self.extract_run_name()}_{uuid.uuid4()!s}.sh"
cmd_src = os.path.join(self.root_dir, self.exp_name, cmd_file)
cmd_dst = os.path.join("/tmp/miniccc/files", self.exp_name, cmd_file)
cmd_src = self.root_dir / self.exp_name / cmd_file
cmd_dst = PurePath("/tmp/miniccc/files") / self.exp_name / cmd_file

with open(cmd_src, "w") as f:
with cmd_src.open("w") as f:
utils.mako_serve_template(
"get_operation.mako", templates, f, op=op["id"]
)

utils.mm_cc_send_wait(mm, server, cmd_src, self.exp_name)
utils.mm_cc_send_wait(mm, server, str(cmd_src), self.exp_name)

os.remove(cmd_src)
cmd_src.unlink()

result = utils.mm_exec_wait(mm, server, f"bash {cmd_dst}", once=True)

Expand All @@ -218,26 +218,26 @@ def __run(self, stage):
logger.info(f"exporting Caldera report for '{self.name}' operation...")

cmd_file = f"run-{self.extract_run_name()}_{uuid.uuid4()!s}.sh"
cmd_src = os.path.join(self.root_dir, self.exp_name, cmd_file)
cmd_dst = os.path.join("/tmp/miniccc/files", self.exp_name, cmd_file)
cmd_src = self.root_dir / self.exp_name / cmd_file
cmd_dst = PurePath("/tmp/miniccc/files") / self.exp_name / cmd_file

with open(cmd_src, "w") as f:
with cmd_src.open("w") as f:
utils.mako_serve_template(
"get_operation_report.mako", templates, f, op=op["id"]
)

utils.mm_cc_send_wait(mm, server, cmd_src, self.exp_name)
utils.mm_cc_send_wait(mm, server, str(cmd_src), self.exp_name)

os.remove(cmd_src)
cmd_src.unlink()

result = utils.mm_exec_wait(mm, server, f"bash {cmd_dst}", once=True)

if result["exitcode"]:
raise RuntimeError("failed to make API call for operation report")
report = json.loads(result["stdout"])

output_file = os.path.join(self.base_dir, "caldera-report.json")
with open(output_file, "w") as f:
output_file = self.base_dir / "caldera-report.json"
with output_file.open("w") as f:
json.dump(report, f, indent=2)


Expand Down
3 changes: 2 additions & 1 deletion src/python/phenix_apps/apps/scorch/cc/cc.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import os
import subprocess
import uuid
from pathlib import PurePath

from phenix_apps.apps.scorch import ComponentBase
from phenix_apps.common import utils
Expand Down Expand Up @@ -207,7 +208,7 @@ def __run(self, stage: str) -> None:

if len(args) == 1:
src = args[0]
dst = self.base_dir + "/" + os.path.basename(src)
dst = str(self.base_dir / PurePath(src).name)
elif len(args) == 2:
src = args[0]
dst = args[1]
Expand Down
5 changes: 2 additions & 3 deletions src/python/phenix_apps/apps/scorch/hoststats/hoststats.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import json
import os
import signal
import threading
import time
Expand Down Expand Up @@ -63,10 +62,10 @@ def __flush_buffer(self):
if not self.resdata:
return

output_file = os.path.join(self.base_dir, "host_stats.jsonl")
output_file = self.base_dir / "host_stats.jsonl"

# write jsonl
with open(output_file, "a+") as f:
with output_file.open("a+") as f:
for datum in self.resdata:
json_record = json.dumps(datum)
f.write(json_record + "\n")
Expand Down
39 changes: 18 additions & 21 deletions src/python/phenix_apps/apps/scorch/iperf/iperf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import os.path
from collections import Counter
from pathlib import Path
from pathlib import Path, PurePath
from time import sleep

from box import Box
Expand Down Expand Up @@ -162,7 +161,7 @@ def configure(self):

# Generate iperf mapping and save it to a JSON file
mapping = self._build_iperf_mapping()
mapping_path = Path(self.base_dir, "iperf_mapping.json")
mapping_path = self.base_dir / "iperf_mapping.json"
logger.info(f"Saving iperf mapping to {mapping_path}")
utils.write_json(mapping_path, mapping)

Expand Down Expand Up @@ -417,8 +416,8 @@ def _get_netstat_info(self, node_info: dict) -> None:
ss_outputs += f"\n{ss_res['stdout']}\n\n"

logger.info("saving netstat info to files")
Path(self.base_dir, "netstat_outputs.txt").write_text(ns_outputs)
Path(self.base_dir, "ss_outputs.txt").write_text(ss_outputs)
(self.base_dir / "netstat_outputs.txt").write_text(ns_outputs)
(self.base_dir / "ss_outputs.txt").write_text(ss_outputs)

def stop(self):
logger.info(f"Stopping user component: {self.name}")
Expand Down Expand Up @@ -463,8 +462,8 @@ def stop(self):
mm=self.mm,
vm=client.hostname,
src=client["client_log_path"],
dst=os.path.join(
self.base_dir, os.path.basename(client["client_log_path"])
dst=str(
self.base_dir / PurePath(client["client_log_path"]).name
),
)

Expand All @@ -476,8 +475,8 @@ def stop(self):
mm=self.mm,
vm=server.hostname,
src=client["server_log_path"],
dst=os.path.join(
self.base_dir, os.path.basename(client["server_log_path"])
dst=str(
self.base_dir / PurePath(client["server_log_path"]).name
),
)

Expand All @@ -486,24 +485,24 @@ def stop(self):
mm=self.mm,
vm=client.hostname,
src=f"/iperf_client-log_client-{client.hostname}_server-{server.hostname}.log",
dst=os.path.join(
self.base_dir,
f"iperf_client-log_client-{client.hostname}_server-{server.hostname}.log",
dst=str(
self.base_dir
/ f"iperf_client-log_client-{client.hostname}_server-{server.hostname}.log"
),
)
if node_info["os_types"][server.hostname] == "linux":
utils.mm_recv(
mm=self.mm,
vm=server.hostname,
src=f"/iperf_server-log_client-{client.hostname}_server-{server.hostname}.log",
dst=os.path.join(
self.base_dir,
f"iperf_server-log_client-{client.hostname}_server-{server.hostname}.log",
dst=str(
self.base_dir
/ f"iperf_server-log_client-{client.hostname}_server-{server.hostname}.log"
),
)

# verify error field isn't set in iperf results
for file in Path(self.base_dir).glob("*.json"):
for file in self.base_dir.glob("*.json"):
try:
data = utils.read_json(file)
except Exception as ex:
Expand All @@ -525,8 +524,8 @@ def stop(self):
logger.info(
f"generating RTT histogram for client '{client.hostname}' and server '{server.hostname}'"
)
c_path = Path(
self.base_dir, os.path.basename(client["client_log_path"])
c_path = (
self.base_dir / PurePath(client["client_log_path"]).name
)
results = utils.read_json(c_path) # type: dict

Expand All @@ -553,9 +552,7 @@ def stop(self):
.replace("_", "-")
)

rtt_hist_file = Path(
self.base_dir, f"rtt_histogram_{pair_name}.txt"
)
rtt_hist_file = self.base_dir / f"rtt_histogram_{pair_name}.txt"
logger.info(
f"Writing RTT histogram to {rtt_hist_file} (client={client.hostname}, server={server.hostname})"
)
Expand Down
Loading
Loading