From fddd14f2da50fc5b5e0d344840690bb422b5f0dc Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Wed, 17 Jun 2026 13:50:23 +0200 Subject: [PATCH 01/59] Reimplemented pbook through typer --- packages/full/pyproject.toml | 1 + packages/light/pyproject.toml | 1 + scripts/pbook | 2 +- setup.py | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/full/pyproject.toml b/packages/full/pyproject.toml index 5cf4945c..5ddc8354 100644 --- a/packages/full/pyproject.toml +++ b/packages/full/pyproject.toml @@ -27,6 +27,7 @@ classifiers = [ dependencies = [ "rich", + "typer" ] [project.optional-dependencies] diff --git a/packages/light/pyproject.toml b/packages/light/pyproject.toml index 31eeeadd..b0da9f4d 100644 --- a/packages/light/pyproject.toml +++ b/packages/light/pyproject.toml @@ -27,6 +27,7 @@ classifiers = [ dependencies = [ "rich", + "typer" ] [project.urls] diff --git a/scripts/pbook b/scripts/pbook index b44250ee..8ed2fe5f 100755 --- a/scripts/pbook +++ b/scripts/pbook @@ -2,4 +2,4 @@ source ${PANDA_SYS}/etc/panda/share/functions.sh -exec_p_command "import pandaclient.PBookScript as pbook; pbook.main()" "$@" +exec_p_command "import pandaclient.PBookTyper as pbook; pbook.main()" "$@" diff --git a/setup.py b/setup.py index 2855326a..60a6840d 100644 --- a/setup.py +++ b/setup.py @@ -154,7 +154,7 @@ def run(self): "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", ], - install_requires=["rich"], + install_requires=["rich", "typer"], # optional pip dependencies extras_require={ "jupyter": ["pandas", "jupyter-dash"], From dd82b3ced4254b8dc3224ee4128e662ff0057347 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Wed, 17 Jun 2026 13:57:00 +0200 Subject: [PATCH 02/59] Reimplemented pbook through typer --- pandaclient/PBookTyper.py | 653 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 653 insertions(+) create mode 100644 pandaclient/PBookTyper.py diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py new file mode 100644 index 00000000..8657548f --- /dev/null +++ b/pandaclient/PBookTyper.py @@ -0,0 +1,653 @@ +""" +pbook CLI — PanDA task bookkeeper with typer-based shell autocompletion. +""" + +from __future__ import annotations + +import atexit +import code +import os +import signal +import sys +import tempfile +from concurrent.futures import ThreadPoolExecutor +from typing import Optional + +import typer + +from pandaclient import Client, PandaToolsPkgInfo +from pandaclient.MiscUtils import commands_get_output + +# ─── Runtime state ──────────────────────────────────────────────────────────── +_tmp_dir: Optional[str] = None +_history_file: Optional[str] = None +_fork_child_pid: Optional[int] = None +_setup_done: bool = False +_ctx_state: dict = {} + +app = typer.Typer( + name="pbook", + help="PanDA task bookkeeper. Run without arguments for interactive mode.", + invoke_without_command=True, + no_args_is_help=False, +) + +# ─── Utilities ──────────────────────────────────────────────────────────────── + +def _parallel(func, items): + with ThreadPoolExecutor(8) as pool: + return list(pool.map(func, items)) + + +def _parse_ids(raw: str): + """'all' → str, '42' → int, '1,2,3' → [int,...].""" + if raw == "all": + return "all" + parts = raw.split(",") + try: + ids = [int(p) for p in parts] + return ids[0] if len(ids) == 1 else ids + except ValueError: + typer.echo(f"Error: invalid task ID(s): {raw}", err=True) + raise typer.Exit(1) + + +def _setup() -> None: + global _tmp_dir, _history_file, _setup_done + if _setup_done: + return + _setup_done = True + + import readline + readline.parse_and_bind("tab: complete") + readline.parse_and_bind("set show-all-if-ambiguous On") + + if "CMTSITE" not in os.environ: + os.environ["CMTSITE"] = "" + + pconf_dir = os.path.expanduser(os.environ.get("PANDA_CONFIG_ROOT", "~/.panda")) + os.makedirs(pconf_dir, exist_ok=True) + + _history_file = os.path.join(pconf_dir, ".history") + if os.path.exists(_history_file): + try: + readline.read_history_file(_history_file) + except Exception: + pass + readline.set_history_length(1024) + + _tmp_dir = tempfile.mkdtemp() + Client.setGlobalTmpDir(_tmp_dir) + + for path in sys.path: + real = path or "." + if ( + os.path.exists(real) + and os.path.isdir(real) + and "pandaclient" in os.listdir(real) + and os.path.exists(os.path.join(real, "pandaclient", "__init__.py")) + ): + link = os.path.join(_tmp_dir, "taskbuffer") + if not os.path.exists(link): + os.symlink(os.path.join(real, "pandaclient"), link) + break + if _tmp_dir not in sys.path: + sys.path.insert(0, _tmp_dir) + + atexit.register(_cleanup) + + +def _cleanup() -> None: + if _fork_child_pid == 0 and _history_file: + import readline + readline.write_history_file(_history_file) + if _tmp_dir: + commands_get_output(f"rm -rf {_tmp_dir}") + + +def _make_core(verbose: bool = False): + from pandaclient import PBookCore + return PBookCore.PBookCore(verbose=verbose) + + +def _get_core(): + _setup() + return _make_core(_ctx_state.get("verbose", False)) + + +def _catch_sig(sig, frame): + _cleanup() + commands_get_output(f"kill -9 -- -{os.getpgrp()}") + + +# ─── Interactive REPL namespace ─────────────────────────────────────────────── + +_RETRY_ALLOWED_OPTS = [ + "site", "excludedSite", "includedSite", "nFilesPerJob", "nMaxFilesPerJob", + "nGBPerJob", "nFiles", "nEvents", "loopingCheck", "maxNFilesPerJob", + "memory", "ramCount", "avoidVP", "ignoreMissingInDS", "forceStaged", "maxCore", +] + + +def _build_namespace(core) -> dict: + import pydoc + + def help(*arg): + """Show the help doc.""" + if arg: + try: + func = ns[arg[0]] if isinstance(arg[0], str) else arg[0] + print(pydoc.plain(pydoc.render_doc(func))) + return + except Exception: + print(f"Unknown command: {arg[0]}") + return + print(""" +Available commands: + help show showl kill finish retry debug + get_user_job_metadata recover_lost_files reload_input + show_workflow kill_workflow retry_workflow finish_workflow + pause_workflow resume_workflow + set_secret list_secrets delete_secret delete_all_secrets + generate_credential + +Usage: help(show) or pbook show --help +""") + + def show(*args, **kwargs): + """Print task records. Args: [taskID|'run'|'fin']. Kwargs: username, limit, taskname, days, jeditaskid, reqid, status, superstatus, format.""" + return core.show(*args, **kwargs) + + def showl(*args, **kwargs): + """Print task records in long format (shortcut for show(..., format='long')).""" + kwargs["format"] = "long" + return core.show(*args, **kwargs) + + def kill(taskIDs): + """Kill tasks. taskIDs: int, [int,...], or 'all'.""" + if taskIDs == "all": + return _parallel(lambda t: core.kill(t.jeditaskid), core.get_active_tasks()) + elif isinstance(taskIDs, (list, tuple)): + return _parallel(core.kill, taskIDs) + elif isinstance(taskIDs, int): + return [core.kill(taskIDs)] + print("Error: Invalid argument") + + def finish(taskIDs, soft=False): + """Finish tasks. taskIDs: int, [int,...], or 'all'. soft=True waits for running jobs.""" + if taskIDs == "all": + return _parallel( + lambda t: core.finish.original_func(core, t.jeditaskid, soft=soft), + core.get_active_tasks(), + ) + elif isinstance(taskIDs, (list, tuple)): + return _parallel(lambda tid: core.finish(tid, soft=soft), taskIDs) + elif isinstance(taskIDs, int): + return [core.finish(taskIDs, soft=soft)] + print("Error: Invalid argument") + + def retry(taskIDs, newOpts=None, days=14, limit=1000, **kwargs): + """Retry failed/cancelled tasks. taskIDs: int, [int,...], or 'all'.""" + if newOpts is None: + newOpts = dict(kwargs) + for key in list(newOpts): + if key == "memory": + newOpts["ramCount"] = newOpts.pop(key) + elif key == "maxCore": + newOpts["maxCoreCount"] = newOpts.pop(key) + elif key not in _RETRY_ALLOWED_OPTS: + print(f'Error: Unknown option "{key}"') + return None + opts = newOpts or None + if isinstance(taskIDs, (list, tuple)): + return _parallel(lambda tid: core.retry(tid, newOpts=opts), taskIDs) + elif isinstance(taskIDs, int): + return [core.retry(taskIDs, newOpts=opts)] + elif taskIDs == "all": + data = core.show(status="finished", days=days, limit=limit, format="json") + return _parallel(lambda d: core.retry.original_func(core, d["jediTaskID"], newOpts=opts), data) + print("Error: Invalid argument") + + def debug(PandaID, modeOn): + """Toggle debug mode for a subjob. modeOn: True/False.""" + core.debug(PandaID, modeOn) + + def get_user_job_metadata(taskID, outputFileName): + """Write user metadata of successful jobs to a JSON file.""" + core.getUserJobMetadata(taskID, outputFileName) + + def reload_input(task_id): + """Reload input dataset and retry with new contents.""" + core.reload_input(task_id) + + def recover_lost_files(taskID, test_mode=False): + """Request recovery of lost files from a task.""" + core.recover_lost_files(taskID, test_mode) + + def show_workflow(request_id): + """Show workflow status.""" + _, output = core.execute_workflow_command("get_status", request_id) + if output: + print(output) + + def kill_workflow(request_id): + """Kill a workflow.""" + _, output = core.execute_workflow_command("abort", request_id) + if output: + print(output[0][-1]) + + def retry_workflow(request_id): + """Retry a workflow.""" + _, output = core.execute_workflow_command("retry", request_id) + if output: + print(output[0][-1]) + + def finish_workflow(request_id): + """Finish a workflow.""" + _, output = core.execute_workflow_command("finish", request_id) + if output: + print(output[0][-1]) + + def pause_workflow(request_id): + """Pause a workflow.""" + _, output = core.execute_workflow_command("suspend", request_id) + if output: + print(output[0][-1]) + + def resume_workflow(request_id): + """Resume a workflow.""" + _, output = core.execute_workflow_command("resume", request_id) + if output: + print(output[0][-1]) + + def set_secret(key, value, is_file=False): + """Set a secret key-value pair. is_file=True to upload a file.""" + core.set_secret(key, value, is_file) + + def delete_secret(key): + """Delete a secret.""" + core.set_secret(key, None) + + def delete_all_secrets(): + """Delete all secrets.""" + core.set_secret(None, None) + + def list_secrets(full=False): + """List secrets. full=True to show full values.""" + core.list_secrets(full) + + def generate_credential(): + """Generate a new proxy or token.""" + core.generate_credential() + + ns = {k: v for k, v in locals().items() if callable(v)} + return ns + + +# ─── Top-level callback ─────────────────────────────────────────────────────── + +@app.callback(invoke_without_command=True) +def _main( + ctx: typer.Context, + verbose: bool = typer.Option(False, "-v", help="Verbose"), + command_string: Optional[str] = typer.Option(None, "-c", help="Execute a Python code snippet"), + version: bool = typer.Option(False, "--version", is_eager=True, help="Display version"), + dev_srv: bool = typer.Option(False, "--devSrv", hidden=True), + intr_srv: bool = typer.Option(False, "--intrSrv", hidden=True), + prompt_with_newline: bool = typer.Option(False, "--prompt_with_newline", hidden=True), +) -> None: + """PanDA task bookkeeper. Run without arguments for interactive mode.""" + if version: + typer.echo(f"Version: {PandaToolsPkgInfo.release_version}") + raise typer.Exit() + + if dev_srv: + Client.useDevServer() + if intr_srv: + Client.useIntrServer() + + _ctx_state.update({"verbose": verbose}) + + if ctx.invoked_subcommand is not None: + return + + # Interactive or snippet mode + _setup() + global _fork_child_pid + _fork_child_pid = os.fork() + if _fork_child_pid == -1: + typer.echo("ERROR: Failed to fork", err=True) + raise typer.Exit(1) + + if _fork_child_pid == 0: + if verbose: + typer.echo(str(ctx.params)) + if prompt_with_newline: + sys.ps1 = ">>> \n" + core = _make_core(verbose) + ns = _build_namespace(core) + if command_string: + core.init() + exec(command_string, {}, ns) # noqa: S102 + from pandaclient import PBookCore as _PBC + raise typer.Exit(0 if _PBC.func_return_value else 1) + core.init() + code.interact(banner=f"\nStart pBook {PandaToolsPkgInfo.release_version}", local=ns) + else: + signal.signal(signal.SIGINT, _catch_sig) + signal.signal(signal.SIGHUP, _catch_sig) + signal.signal(signal.SIGTERM, _catch_sig) + pid, status = os.wait() + if os.WIFSIGNALED(status): + raise typer.Exit(-os.WTERMSIG(status)) + elif os.WIFEXITED(status): + raise typer.Exit(os.WEXITSTATUS(status)) + raise typer.Exit(0) + + +# ─── Subcommands ────────────────────────────────────────────────────────────── + +@app.command() +def show( + task_id: Optional[str] = typer.Argument(None, help="jediTaskID, reqID, 'run' (active only), or 'fin' (terminated only)"), + username: Optional[str] = typer.Option(None, "--username", help="Filter by username"), + limit: int = typer.Option(1000, "--limit", help="Maximum number of records"), + taskname: Optional[str] = typer.Option(None, "--taskname", help="Filter by task name"), + days: int = typer.Option(14, "--days", help="Look back N days (capped at 90 without a task ID)"), + jeditaskid: Optional[int] = typer.Option(None, "--jeditaskid", help="Filter by jediTaskID"), + reqid: Optional[int] = typer.Option(None, "--reqid", help="Filter by reqID"), + status: Optional[str] = typer.Option(None, "--status", help="Filter by task status"), + superstatus: Optional[str] = typer.Option(None, "--superstatus", help="Filter by super-status"), + output_format: str = typer.Option("standard", "--format", help="Output format: standard|long|json|plain"), +) -> None: + """Print task records.""" + core = _get_core() + core.init(sanity_check=False) + kwargs: dict = {"limit": limit, "days": days, "format": output_format} + for k, v in [("username", username), ("taskname", taskname), ("jeditaskid", jeditaskid), + ("reqid", reqid), ("status", status), ("superstatus", superstatus)]: + if v is not None: + kwargs[k] = v + if task_id is not None: + try: + first_arg = int(task_id) + except ValueError: + first_arg = task_id + core.show(first_arg, **kwargs) + else: + core.show(**kwargs) + + +@app.command() +def showl( + task_id: Optional[str] = typer.Argument(None, help="jediTaskID, reqID, 'run', or 'fin'"), + username: Optional[str] = typer.Option(None, "--username"), + limit: int = typer.Option(1000, "--limit"), + taskname: Optional[str] = typer.Option(None, "--taskname"), + days: int = typer.Option(14, "--days"), + jeditaskid: Optional[int] = typer.Option(None, "--jeditaskid"), + reqid: Optional[int] = typer.Option(None, "--reqid"), + status: Optional[str] = typer.Option(None, "--status"), + superstatus: Optional[str] = typer.Option(None, "--superstatus"), +) -> None: + """Print task records in long format (shortcut for show --format long).""" + core = _get_core() + core.init(sanity_check=False) + kwargs: dict = {"limit": limit, "days": days, "format": "long"} + for k, v in [("username", username), ("taskname", taskname), ("jeditaskid", jeditaskid), + ("reqid", reqid), ("status", status), ("superstatus", superstatus)]: + if v is not None: + kwargs[k] = v + if task_id is not None: + try: + first_arg = int(task_id) + except ValueError: + first_arg = task_id + core.show(first_arg, **kwargs) + else: + core.show(**kwargs) + + +@app.command() +def kill( + task_ids: str = typer.Argument(..., help="Task ID, comma-separated IDs, or 'all'"), +) -> None: + """Kill tasks.""" + core = _get_core() + core.init(sanity_check=False) + ids = _parse_ids(task_ids) + if ids == "all": + _parallel(lambda t: core.kill(t.jeditaskid), core.get_active_tasks()) + elif isinstance(ids, list): + _parallel(core.kill, ids) + else: + core.kill(ids) + + +@app.command() +def finish( + task_ids: str = typer.Argument(..., help="Task ID, comma-separated IDs, or 'all'"), + soft: bool = typer.Option(False, "--soft", help="Wait for running jobs to finish instead of killing them"), +) -> None: + """Finish tasks.""" + core = _get_core() + core.init(sanity_check=False) + ids = _parse_ids(task_ids) + if ids == "all": + _parallel(lambda t: core.finish.original_func(core, t.jeditaskid, soft=soft), core.get_active_tasks()) + elif isinstance(ids, list): + _parallel(lambda tid: core.finish(tid, soft=soft), ids) + else: + core.finish(ids, soft=soft) + + +@app.command() +def retry( + task_ids: str = typer.Argument(..., help="Task ID, comma-separated IDs, or 'all'"), + days: int = typer.Option(14, "--days", help="Look-back window when task_ids='all'"), + limit: int = typer.Option(1000, "--limit", help="Max tasks to retry when task_ids='all'"), + site: Optional[str] = typer.Option(None, "--site"), + excluded_site: Optional[str] = typer.Option(None, "--excludedSite"), + included_site: Optional[str] = typer.Option(None, "--includedSite"), + n_files_per_job: Optional[int] = typer.Option(None, "--nFilesPerJob"), + n_max_files_per_job: Optional[int] = typer.Option(None, "--nMaxFilesPerJob"), + n_gb_per_job: Optional[float] = typer.Option(None, "--nGBPerJob"), + n_files: Optional[int] = typer.Option(None, "--nFiles"), + n_events: Optional[int] = typer.Option(None, "--nEvents"), + looping_check: Optional[bool] = typer.Option(None, "--loopingCheck"), + memory: Optional[int] = typer.Option(None, "--memory"), + avoid_vp: Optional[bool] = typer.Option(None, "--avoidVP"), + ignore_missing_in_ds: Optional[bool] = typer.Option(None, "--ignoreMissingInDS"), + force_staged: Optional[bool] = typer.Option(None, "--forceStaged"), + max_core: Optional[int] = typer.Option(None, "--maxCore"), +) -> None: + """Retry failed/cancelled tasks.""" + core = _get_core() + core.init(sanity_check=False) + new_opts = { + k: v for k, v in { + "site": site, "excludedSite": excluded_site, "includedSite": included_site, + "nFilesPerJob": n_files_per_job, "nMaxFilesPerJob": n_max_files_per_job, + "nGBPerJob": n_gb_per_job, "nFiles": n_files, "nEvents": n_events, + "loopingCheck": looping_check, "ramCount": memory, "avoidVP": avoid_vp, + "ignoreMissingInDS": ignore_missing_in_ds, "forceStaged": force_staged, + "maxCoreCount": max_core, + }.items() if v is not None + } + opts = new_opts or None + ids = _parse_ids(task_ids) + if isinstance(ids, list): + _parallel(lambda tid: core.retry(tid, newOpts=opts), ids) + elif isinstance(ids, int): + core.retry(ids, newOpts=opts) + else: + data = core.show(status="finished", days=days, limit=limit, format="json") + _parallel(lambda d: core.retry.original_func(core, d["jediTaskID"], newOpts=opts), data) + + +@app.command() +def debug( + panda_id: int = typer.Argument(..., help="PanDA subjob ID"), + mode_on: bool = typer.Argument(..., help="True to enable, False to disable"), +) -> None: + """Toggle debug mode for a subjob.""" + core = _get_core() + core.init(sanity_check=False) + core.debug(panda_id, mode_on) + + +@app.command(name="get-user-job-metadata") +def get_user_job_metadata( + task_id: int = typer.Argument(..., help="Task ID"), + output_file: str = typer.Argument(..., help="Output JSON file path"), +) -> None: + """Write user metadata of successful jobs to a JSON file.""" + core = _get_core() + core.init(sanity_check=False) + core.getUserJobMetadata(task_id, output_file) + + +@app.command(name="reload-input") +def reload_input( + task_id: int = typer.Argument(..., help="Task ID"), +) -> None: + """Reload input dataset and retry the task with new contents.""" + core = _get_core() + core.init(sanity_check=False) + core.reload_input(task_id) + + +@app.command(name="recover-lost-files") +def recover_lost_files( + task_id: int = typer.Argument(..., help="Task ID"), + test_mode: bool = typer.Option(False, "--test-mode", help="Dry-run mode"), +) -> None: + """Request recovery of lost files from a task.""" + core = _get_core() + core.init(sanity_check=False) + core.recover_lost_files(task_id, test_mode) + + +@app.command(name="show-workflow") +def show_workflow( + request_id: int = typer.Argument(..., help="Workflow request ID"), +) -> None: + """Show workflow status.""" + core = _get_core() + core.init(sanity_check=False) + _, output = core.execute_workflow_command("get_status", request_id) + if output: + print(output) + + +@app.command(name="kill-workflow") +def kill_workflow( + request_id: int = typer.Argument(..., help="Workflow request ID"), +) -> None: + """Kill a workflow.""" + core = _get_core() + core.init(sanity_check=False) + _, output = core.execute_workflow_command("abort", request_id) + if output: + print(output[0][-1]) + + +@app.command(name="retry-workflow") +def retry_workflow( + request_id: int = typer.Argument(..., help="Workflow request ID"), +) -> None: + """Retry a workflow.""" + core = _get_core() + core.init(sanity_check=False) + _, output = core.execute_workflow_command("retry", request_id) + if output: + print(output[0][-1]) + + +@app.command(name="finish-workflow") +def finish_workflow( + request_id: int = typer.Argument(..., help="Workflow request ID"), +) -> None: + """Finish a workflow.""" + core = _get_core() + core.init(sanity_check=False) + _, output = core.execute_workflow_command("finish", request_id) + if output: + print(output[0][-1]) + + +@app.command(name="pause-workflow") +def pause_workflow( + request_id: int = typer.Argument(..., help="Workflow request ID"), +) -> None: + """Pause a workflow.""" + core = _get_core() + core.init(sanity_check=False) + _, output = core.execute_workflow_command("suspend", request_id) + if output: + print(output[0][-1]) + + +@app.command(name="resume-workflow") +def resume_workflow( + request_id: int = typer.Argument(..., help="Workflow request ID"), +) -> None: + """Resume a workflow.""" + core = _get_core() + core.init(sanity_check=False) + _, output = core.execute_workflow_command("resume", request_id) + if output: + print(output[0][-1]) + + +@app.command(name="set-secret") +def set_secret( + key: str = typer.Argument(..., help="Secret key"), + value: str = typer.Argument(..., help="Secret value or file path"), + is_file: bool = typer.Option(False, "--is-file", help="Treat value as a file path to upload"), +) -> None: + """Set a secret key-value pair.""" + core = _get_core() + core.init(sanity_check=False) + core.set_secret(key, value, is_file) + + +@app.command(name="list-secrets") +def list_secrets( + full: bool = typer.Option(False, "--full", help="Show full secret values"), +) -> None: + """List secrets.""" + core = _get_core() + core.init(sanity_check=False) + core.list_secrets(full) + + +@app.command(name="delete-secret") +def delete_secret( + key: str = typer.Argument(..., help="Secret key to delete"), +) -> None: + """Delete a secret.""" + core = _get_core() + core.init(sanity_check=False) + core.set_secret(key, None) + + +@app.command(name="delete-all-secrets") +def delete_all_secrets() -> None: + """Delete all secrets.""" + core = _get_core() + core.init(sanity_check=False) + core.set_secret(None, None) + + +@app.command(name="generate-credential") +def generate_credential() -> None: + """Generate a new proxy or token.""" + core = _get_core() + core.generate_credential() + + +# ─── Entry point ────────────────────────────────────────────────────────────── + +def main() -> None: + app() \ No newline at end of file From cb95b91771ef3cb8d7a8f0cd43d87d2910adec94 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Wed, 17 Jun 2026 15:59:10 +0200 Subject: [PATCH 03/59] Fix in wrapper script --- pandaclient/PBookTyper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index 8657548f..40dfd864 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -650,4 +650,5 @@ def generate_credential() -> None: # ─── Entry point ────────────────────────────────────────────────────────────── def main() -> None: + sys.argv[0] = "pbook" app() \ No newline at end of file From a14c60470a83162f0d46ad38c33336b702c815a0 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Wed, 17 Jun 2026 16:14:44 +0200 Subject: [PATCH 04/59] Autocomplete inside REPL --- pandaclient/PBookTyper.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index 40dfd864..4aa04b5e 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -331,6 +331,9 @@ def _main( exec(command_string, {}, ns) # noqa: S102 from pandaclient import PBookCore as _PBC raise typer.Exit(0 if _PBC.func_return_value else 1) + import rlcompleter + import readline as _rl + _rl.set_completer(rlcompleter.Completer(ns).complete) core.init() code.interact(banner=f"\nStart pBook {PandaToolsPkgInfo.release_version}", local=ns) else: From eca3b6d7ef9ba873f675d192a22c7b7215c44109 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Wed, 17 Jun 2026 16:20:29 +0200 Subject: [PATCH 05/59] Autocomplete arguments inside REPL --- pandaclient/PBookTyper.py | 54 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 2 deletions(-) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index 4aa04b5e..ea5bdbf5 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -7,6 +7,7 @@ import atexit import code import os +import re import signal import sys import tempfile @@ -122,6 +123,56 @@ def _catch_sig(sig, frame): # ─── Interactive REPL namespace ─────────────────────────────────────────────── +# ─── REPL kwarg completer ───────────────────────────────────────────────────── + +_FUNC_KWARGS: dict[str, list[str]] = { + "show": ["username=", "limit=", "taskname=", "days=", "jeditaskid=", "reqid=", "status=", "superstatus=", "format="], + "showl": ["username=", "limit=", "taskname=", "days=", "jeditaskid=", "reqid=", "status=", "superstatus="], + "kill": [], + "finish": ["soft="], + "retry": [ + "newOpts=", "days=", "limit=", "site=", "excludedSite=", "includedSite=", + "nFilesPerJob=", "nMaxFilesPerJob=", "nGBPerJob=", "nFiles=", "nEvents=", + "loopingCheck=", "memory=", "avoidVP=", "ignoreMissingInDS=", "forceStaged=", "maxCore=", + ], + "debug": ["modeOn="], + "get_user_job_metadata": [], + "recover_lost_files": ["test_mode="], + "set_secret": ["is_file="], + "list_secrets": ["full="], +} + + +class _PBookCompleter: + """Readline completer that adds kwarg hints when cursor is inside a call.""" + + def __init__(self, ns: dict) -> None: + import rlcompleter + self._base = rlcompleter.Completer(ns) + self._matches: list[str] = [] + + def complete(self, text: str, state: int) -> Optional[str]: + if state == 0: + self._matches = self._compute(text) + return self._matches[state] if state < len(self._matches) else None + + def _compute(self, text: str) -> list[str]: + import readline + line = readline.get_line_buffer() + m = re.search(r"(\w+)\s*\([^)]*$", line) + if m: + kwargs = _FUNC_KWARGS.get(m.group(1), []) + hits = [k for k in kwargs if k.startswith(text)] + if hits: + return hits + # Fall back to standard name completion + results, i = [], 0 + while (c := self._base.complete(text, i)) is not None: + results.append(c) + i += 1 + return results + + _RETRY_ALLOWED_OPTS = [ "site", "excludedSite", "includedSite", "nFilesPerJob", "nMaxFilesPerJob", "nGBPerJob", "nFiles", "nEvents", "loopingCheck", "maxNFilesPerJob", @@ -331,9 +382,8 @@ def _main( exec(command_string, {}, ns) # noqa: S102 from pandaclient import PBookCore as _PBC raise typer.Exit(0 if _PBC.func_return_value else 1) - import rlcompleter import readline as _rl - _rl.set_completer(rlcompleter.Completer(ns).complete) + _rl.set_completer(_PBookCompleter(ns).complete) core.init() code.interact(banner=f"\nStart pBook {PandaToolsPkgInfo.release_version}", local=ns) else: From a0f87fd22c6b114bd995ba599cf73f348c55f391 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Wed, 17 Jun 2026 16:25:00 +0200 Subject: [PATCH 06/59] Autocomplete arguments inside REPL --- pandaclient/PBookTyper.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index ea5bdbf5..5f4f5fab 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -126,20 +126,20 @@ def _catch_sig(sig, frame): # ─── REPL kwarg completer ───────────────────────────────────────────────────── _FUNC_KWARGS: dict[str, list[str]] = { - "show": ["username=", "limit=", "taskname=", "days=", "jeditaskid=", "reqid=", "status=", "superstatus=", "format="], - "showl": ["username=", "limit=", "taskname=", "days=", "jeditaskid=", "reqid=", "status=", "superstatus="], + "show": ["username", "limit", "taskname", "days", "jeditaskid", "reqid", "status", "superstatus", "format"], + "showl": ["username", "limit", "taskname", "days", "jeditaskid", "reqid", "status", "superstatus"], "kill": [], - "finish": ["soft="], + "finish": ["soft"], "retry": [ - "newOpts=", "days=", "limit=", "site=", "excludedSite=", "includedSite=", - "nFilesPerJob=", "nMaxFilesPerJob=", "nGBPerJob=", "nFiles=", "nEvents=", - "loopingCheck=", "memory=", "avoidVP=", "ignoreMissingInDS=", "forceStaged=", "maxCore=", + "newOpts", "days", "limit", "site", "excludedSite", "includedSite", + "nFilesPerJob", "nMaxFilesPerJob", "nGBPerJob", "nFiles", "nEvents", + "loopingCheck", "memory", "avoidVP", "ignoreMissingInDS", "forceStaged", "maxCore", ], - "debug": ["modeOn="], + "debug": ["modeOn"], "get_user_job_metadata": [], - "recover_lost_files": ["test_mode="], - "set_secret": ["is_file="], - "list_secrets": ["full="], + "recover_lost_files": ["test_mode"], + "set_secret": ["is_file"], + "list_secrets": ["full"], } @@ -165,10 +165,10 @@ def _compute(self, text: str) -> list[str]: hits = [k for k in kwargs if k.startswith(text)] if hits: return hits - # Fall back to standard name completion + # Fall back to standard name completion, stripping the trailing '(' rlcompleter adds to callables results, i = [], 0 while (c := self._base.complete(text, i)) is not None: - results.append(c) + results.append(c.rstrip("(")) i += 1 return results From fd26c015ddac08c2ec2c8b9e8097721f96f9754f Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Wed, 17 Jun 2026 16:30:05 +0200 Subject: [PATCH 07/59] Autocomplete options inside REPL --- pandaclient/PBookTyper.py | 53 +++++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index 5f4f5fab..732c8fd6 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -142,9 +142,39 @@ def _catch_sig(sig, frame): "list_secrets": ["full"], } +_KWARG_VALUES: dict[str, dict[str, list[str]]] = { + "show": { + "format": ["standard", "long", "json", "plain"], + }, + "showl": { + "format": ["standard", "long", "json", "plain"], + }, + "finish": { + "soft": ["True", "False"], + }, + "debug": { + "modeOn": ["True", "False"], + }, + "recover_lost_files": { + "test_mode": ["True", "False"], + }, + "list_secrets": { + "full": ["True", "False"], + }, + "set_secret": { + "is_file": ["True", "False"], + }, + "retry": { + "loopingCheck": ["True", "False"], + "avoidVP": ["True", "False"], + "ignoreMissingInDS": ["True", "False"], + "forceStaged": ["True", "False"], + }, +} + class _PBookCompleter: - """Readline completer that adds kwarg hints when cursor is inside a call.""" + """Readline completer: kwarg names and values when inside a call, names otherwise.""" def __init__(self, ns: dict) -> None: import rlcompleter @@ -159,16 +189,29 @@ def complete(self, text: str, state: int) -> Optional[str]: def _compute(self, text: str) -> list[str]: import readline line = readline.get_line_buffer() + + # Value completion: last token is kwarg= or kwarg='partial + m_val = re.search(r"\b(\w+)\s*=\s*(['\"]?)(\w*)$", line) + m_func = re.match(r"(\w+)\s*\(", line) + if m_val and m_func: + kwarg, quote, partial = m_val.group(1), m_val.group(2), m_val.group(3) + func_name = m_func.group(1) + values = _KWARG_VALUES.get(func_name, {}).get(kwarg, []) + hits = [f"{v}{quote}" for v in values if v.startswith(partial)] + if hits: + return hits + + # Kwarg name completion: cursor is inside an open call m = re.search(r"(\w+)\s*\([^)]*$", line) if m: - kwargs = _FUNC_KWARGS.get(m.group(1), []) - hits = [k for k in kwargs if k.startswith(text)] + hits = [k for k in _FUNC_KWARGS.get(m.group(1), []) if k.startswith(text)] if hits: return hits - # Fall back to standard name completion, stripping the trailing '(' rlcompleter adds to callables + + # Fallback: standard name completion, stripping trailing '(' rlcompleter adds to callables results, i = [], 0 while (c := self._base.complete(text, i)) is not None: - results.append(c.rstrip("(")) + results.append(c.rstrip("()").rstrip("(")) i += 1 return results From 3ef4c47eb05f89cd3caaca92f8a62ba8036ce225 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Wed, 17 Jun 2026 16:34:28 +0200 Subject: [PATCH 08/59] Autocomplete options inside REPL --- pandaclient/PBookTyper.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index 732c8fd6..caa58d87 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -173,6 +173,19 @@ def _catch_sig(sig, frame): } +class _PBookConsole(code.InteractiveConsole): + """InteractiveConsole that keeps our custom completer active even after code.interact() resets it.""" + + def __init__(self, local: dict, completer) -> None: + super().__init__(local) + self._completer = completer + + def raw_input(self, prompt: str = "") -> str: + import readline + readline.set_completer(self._completer) + return super().raw_input(prompt) + + class _PBookCompleter: """Readline completer: kwarg names and values when inside a call, names otherwise.""" @@ -425,10 +438,9 @@ def _main( exec(command_string, {}, ns) # noqa: S102 from pandaclient import PBookCore as _PBC raise typer.Exit(0 if _PBC.func_return_value else 1) - import readline as _rl - _rl.set_completer(_PBookCompleter(ns).complete) + completer = _PBookCompleter(ns) core.init() - code.interact(banner=f"\nStart pBook {PandaToolsPkgInfo.release_version}", local=ns) + _PBookConsole(ns, completer.complete).interact(banner=f"\nStart pBook {PandaToolsPkgInfo.release_version}") else: signal.signal(signal.SIGINT, _catch_sig) signal.signal(signal.SIGHUP, _catch_sig) From ccb38a5a5482ed837679e59621e1f876ef278f38 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Wed, 17 Jun 2026 16:38:09 +0200 Subject: [PATCH 09/59] Autocomplete options inside REPL --- pandaclient/PBookTyper.py | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index caa58d87..22ca4e02 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -173,17 +173,34 @@ def _catch_sig(sig, frame): } -class _PBookConsole(code.InteractiveConsole): - """InteractiveConsole that keeps our custom completer active even after code.interact() resets it.""" - def __init__(self, local: dict, completer) -> None: - super().__init__(local) - self._completer = completer +def _run_repl(ns: dict, banner: str) -> None: + """Manual REPL using InteractiveConsole.push() so we own readline setup entirely.""" + import readline - def raw_input(self, prompt: str = "") -> str: - import readline - readline.set_completer(self._completer) - return super().raw_input(prompt) + completer = _PBookCompleter(ns) + readline.set_completer(completer.complete) + readline.parse_and_bind("tab: complete") + readline.parse_and_bind("set show-all-if-ambiguous On") + + console = code.InteractiveConsole(ns) + print(banner) + + more = False + while True: + prompt = "... " if more else ">>> " + try: + readline.set_completer(completer.complete) + line = input(prompt) + except EOFError: + print() + break + except KeyboardInterrupt: + print("\nKeyboardInterrupt") + console.resetbuffer() + more = False + continue + more = console.push(line) class _PBookCompleter: @@ -438,9 +455,8 @@ def _main( exec(command_string, {}, ns) # noqa: S102 from pandaclient import PBookCore as _PBC raise typer.Exit(0 if _PBC.func_return_value else 1) - completer = _PBookCompleter(ns) core.init() - _PBookConsole(ns, completer.complete).interact(banner=f"\nStart pBook {PandaToolsPkgInfo.release_version}") + _run_repl(ns, banner=f"\nStart pBook {PandaToolsPkgInfo.release_version}") else: signal.signal(signal.SIGINT, _catch_sig) signal.signal(signal.SIGHUP, _catch_sig) From 65a7bbec5e73b511723f64751f0271edf2fc64c2 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Wed, 17 Jun 2026 16:50:30 +0200 Subject: [PATCH 10/59] Autocomplete options inside REPL --- pandaclient/PBookTyper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index 22ca4e02..54ab6e67 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -227,7 +227,7 @@ def _compute(self, text: str) -> list[str]: kwarg, quote, partial = m_val.group(1), m_val.group(2), m_val.group(3) func_name = m_func.group(1) values = _KWARG_VALUES.get(func_name, {}).get(kwarg, []) - hits = [f"{v}{quote}" for v in values if v.startswith(partial)] + hits = [v for v in values if v.startswith(partial)] if hits: return hits From da8ab37221de1e99cb27a93b450418d681907c75 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Wed, 17 Jun 2026 17:18:14 +0200 Subject: [PATCH 11/59] Better help --- pandaclient/PBookTyper.py | 107 +++++++++++++++++++++++++++----------- 1 file changed, 76 insertions(+), 31 deletions(-) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index 54ab6e67..f19e9320 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -254,41 +254,79 @@ def _compute(self, text: str) -> list[str]: def _build_namespace(core) -> dict: - import pydoc - - def help(*arg): - """Show the help doc.""" - if arg: - try: - func = ns[arg[0]] if isinstance(arg[0], str) else arg[0] - print(pydoc.plain(pydoc.render_doc(func))) + from inspect import signature + from rich.console import Console + from rich.table import Table + from rich import box + + _console = Console() + + def help(command=None): + """Show available commands, or detailed help for a specific command.""" + if command is not None: + name = command if isinstance(command, str) else command.__name__ + func = ns.get(name, command if callable(command) else None) + if func is None: + _console.print(f"[red]Unknown command:[/red] {name}") return - except Exception: - print(f"Unknown command: {arg[0]}") + sig = str(signature(func)).replace("(", f"[bold cyan]{name}[/bold cyan](", 1) + _console.print(f"\n[bold]{sig}[/bold]") + doc = (func.__doc__ or "No description.").strip() + _console.print(f"\n{doc}\n") return - print(""" -Available commands: - help show showl kill finish retry debug - get_user_job_metadata recover_lost_files reload_input - show_workflow kill_workflow retry_workflow finish_workflow - pause_workflow resume_workflow - set_secret list_secrets delete_secret delete_all_secrets - generate_credential - -Usage: help(show) or pbook show --help -""") - - def show(*args, **kwargs): - """Print task records. Args: [taskID|'run'|'fin']. Kwargs: username, limit, taskname, days, jeditaskid, reqid, status, superstatus, format.""" - return core.show(*args, **kwargs) - - def showl(*args, **kwargs): + + table = Table(box=box.SIMPLE, show_header=True, header_style="bold magenta") + table.add_column("Command", style="bold cyan", no_wrap=True) + table.add_column("Signature", style="dim", no_wrap=True) + table.add_column("Description") + + _GROUPS = [ + ("Tasks", ["show", "showl", "kill", "finish", "retry", "debug"]), + ("Files & input", ["get_user_job_metadata", "recover_lost_files", "reload_input"]), + ("Workflows", ["show_workflow", "kill_workflow", "retry_workflow", + "finish_workflow", "pause_workflow", "resume_workflow"]), + ("Secrets", ["set_secret", "list_secrets", "delete_secret", + "delete_all_secrets"]), + ("Auth", ["generate_credential"]), + ] + for group, names in _GROUPS: + table.add_section() + table.add_row(f"[bold white]{group}[/bold white]", "", "") + for name in names: + func = ns.get(name) + if func is None: + continue + sig = str(signature(func)) + doc = (func.__doc__ or "").strip().splitlines()[0] + table.add_row(f" {name}", sig, doc) + + _console.print(table) + _console.print("Usage: [bold]help(show)[/bold] or [bold]pbook show --help[/bold]\n") + + def show(taskID=None, *, username=None, limit=1000, taskname=None, days=14, + jeditaskid=None, reqid=None, status=None, superstatus=None, format="standard"): + """Print task records. + + taskID: jediTaskID / reqID / 'run' (active) / 'fin' (terminated) / omit for all. + format: standard | long | json | plain + """ + kwargs = {k: v for k, v in dict(username=username, limit=limit, taskname=taskname, + days=days, jeditaskid=jeditaskid, reqid=reqid, status=status, + superstatus=superstatus, format=format).items() if v is not None} + kwargs.setdefault("limit", limit) + kwargs.setdefault("days", days) + kwargs["format"] = format + return core.show(taskID, **kwargs) if taskID is not None else core.show(**kwargs) + + def showl(taskID=None, *, username=None, limit=1000, taskname=None, days=14, + jeditaskid=None, reqid=None, status=None, superstatus=None): """Print task records in long format (shortcut for show(..., format='long')).""" - kwargs["format"] = "long" - return core.show(*args, **kwargs) + return show(taskID, username=username, limit=limit, taskname=taskname, days=days, + jeditaskid=jeditaskid, reqid=reqid, status=status, + superstatus=superstatus, format="long") def kill(taskIDs): - """Kill tasks. taskIDs: int, [int,...], or 'all'.""" + """Kill tasks. taskIDs: int, list of ints, or 'all'.""" if taskIDs == "all": return _parallel(lambda t: core.kill(t.jeditaskid), core.get_active_tasks()) elif isinstance(taskIDs, (list, tuple)): @@ -311,7 +349,14 @@ def finish(taskIDs, soft=False): print("Error: Invalid argument") def retry(taskIDs, newOpts=None, days=14, limit=1000, **kwargs): - """Retry failed/cancelled tasks. taskIDs: int, [int,...], or 'all'.""" + """Retry failed/cancelled tasks. + + taskIDs (required): int, list of ints, or 'all'. + Allowed kwargs: site, excludedSite, includedSite, nFilesPerJob, nMaxFilesPerJob, + nGBPerJob, nFiles, nEvents, loopingCheck, memory, avoidVP, + ignoreMissingInDS, forceStaged, maxCore. + Example: retry('all', loopingCheck=True) + """ if newOpts is None: newOpts = dict(kwargs) for key in list(newOpts): From 76a953c9acc30badecb97185b2ee229276e7edf1 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Tue, 21 Jul 2026 12:48:21 +0200 Subject: [PATCH 12/59] Typo: missing comma in light pyproject.toml --- .pre-commit-config.yaml | 4 ++-- packages/light/pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7c4550ce..9534f2ba 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,14 +1,14 @@ repos: - repo: https://github.com/psf/black - rev: 23.9.1 + rev: 26.5.1 hooks: - id: black types: [python] args: ["--config", "pyproject.toml"] - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 9.0.0b1 hooks: - id: isort name: isort (python) diff --git a/packages/light/pyproject.toml b/packages/light/pyproject.toml index 934148dc..a722f0f3 100644 --- a/packages/light/pyproject.toml +++ b/packages/light/pyproject.toml @@ -27,7 +27,7 @@ classifiers = [ dependencies = [ "rich", - "typer" + "typer", "certifi", ] From 20a83fb8380a8c5346df1ae39b2033731c50bf69 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Tue, 28 Jul 2026 13:38:22 +0200 Subject: [PATCH 13/59] Removed python2 compatibility leftover --- pandaclient/PBookScript.py | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/pandaclient/PBookScript.py b/pandaclient/PBookScript.py index c1d5ad8e..f28b1e68 100644 --- a/pandaclient/PBookScript.py +++ b/pandaclient/PBookScript.py @@ -3,36 +3,27 @@ Import PBookCore instead. """ +import argparse import atexit import code import os +import pydoc +import readline import signal import sys import tempfile +from concurrent.futures import ThreadPoolExecutor +from pandaclient import Client, PandaToolsPkgInfo from pandaclient.MiscUtils import commands_get_output -try: - from concurrent.futures import ThreadPoolExecutor -except ImportError: - - def list_parallel_exec(func, array): - return [func(x) for x in array] - -else: - def list_parallel_exec(func, array): - with ThreadPoolExecutor(8) as thread_pool: - dataIterator = thread_pool.map(func, array) - return list(dataIterator) +def list_parallel_exec(func, array): + with ThreadPoolExecutor(8) as thread_pool: + dataIterator = thread_pool.map(func, array) + return list(dataIterator) -import argparse -import pydoc -import readline - -from pandaclient import Client, PandaToolsPkgInfo - # readline support readline.parse_and_bind("tab: complete") readline.parse_and_bind("set show-all-if-ambiguous On") From e675a33d7dddd644242f599a746b4dd13a6ecc50 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Tue, 28 Jul 2026 13:51:32 +0200 Subject: [PATCH 14/59] Moved import to top --- pandaclient/PBookScript.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pandaclient/PBookScript.py b/pandaclient/PBookScript.py index f28b1e68..c8e44334 100644 --- a/pandaclient/PBookScript.py +++ b/pandaclient/PBookScript.py @@ -12,6 +12,7 @@ import signal import sys import tempfile +import textwrap from concurrent.futures import ThreadPoolExecutor from pandaclient import Client, PandaToolsPkgInfo @@ -288,8 +289,6 @@ def retry(taskIDs, newOpts=None, days=14, limit=1000, **kwargs): ret = None return ret - import textwrap - _opts_list = ", ".join(_retry_allowed_opts) # 8-space docstring indent _first_indent = " " * 8 From cb4582bb7fe9a287dcc5f68624c596fed433b91e Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Tue, 28 Jul 2026 14:05:30 +0200 Subject: [PATCH 15/59] Removed obsoleted setGlobalTmpDir --- pandaclient/PBookTyper.py | 145 +++++++++++++++++++++++++++++--------- 1 file changed, 110 insertions(+), 35 deletions(-) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index f19e9320..a5a12d1c 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -35,6 +35,7 @@ # ─── Utilities ──────────────────────────────────────────────────────────────── + def _parallel(func, items): with ThreadPoolExecutor(8) as pool: return list(pool.map(func, items)) @@ -60,6 +61,7 @@ def _setup() -> None: _setup_done = True import readline + readline.parse_and_bind("tab: complete") readline.parse_and_bind("set show-all-if-ambiguous On") @@ -78,7 +80,6 @@ def _setup() -> None: readline.set_history_length(1024) _tmp_dir = tempfile.mkdtemp() - Client.setGlobalTmpDir(_tmp_dir) for path in sys.path: real = path or "." @@ -101,6 +102,7 @@ def _setup() -> None: def _cleanup() -> None: if _fork_child_pid == 0 and _history_file: import readline + readline.write_history_file(_history_file) if _tmp_dir: commands_get_output(f"rm -rf {_tmp_dir}") @@ -108,6 +110,7 @@ def _cleanup() -> None: def _make_core(verbose: bool = False): from pandaclient import PBookCore + return PBookCore.PBookCore(verbose=verbose) @@ -131,9 +134,23 @@ def _catch_sig(sig, frame): "kill": [], "finish": ["soft"], "retry": [ - "newOpts", "days", "limit", "site", "excludedSite", "includedSite", - "nFilesPerJob", "nMaxFilesPerJob", "nGBPerJob", "nFiles", "nEvents", - "loopingCheck", "memory", "avoidVP", "ignoreMissingInDS", "forceStaged", "maxCore", + "newOpts", + "days", + "limit", + "site", + "excludedSite", + "includedSite", + "nFilesPerJob", + "nMaxFilesPerJob", + "nGBPerJob", + "nFiles", + "nEvents", + "loopingCheck", + "memory", + "avoidVP", + "ignoreMissingInDS", + "forceStaged", + "maxCore", ], "debug": ["modeOn"], "get_user_job_metadata": [], @@ -173,7 +190,6 @@ def _catch_sig(sig, frame): } - def _run_repl(ns: dict, banner: str) -> None: """Manual REPL using InteractiveConsole.push() so we own readline setup entirely.""" import readline @@ -208,6 +224,7 @@ class _PBookCompleter: def __init__(self, ns: dict) -> None: import rlcompleter + self._base = rlcompleter.Completer(ns) self._matches: list[str] = [] @@ -218,6 +235,7 @@ def complete(self, text: str, state: int) -> Optional[str]: def _compute(self, text: str) -> list[str]: import readline + line = readline.get_line_buffer() # Value completion: last token is kwarg= or kwarg='partial @@ -247,17 +265,31 @@ def _compute(self, text: str) -> list[str]: _RETRY_ALLOWED_OPTS = [ - "site", "excludedSite", "includedSite", "nFilesPerJob", "nMaxFilesPerJob", - "nGBPerJob", "nFiles", "nEvents", "loopingCheck", "maxNFilesPerJob", - "memory", "ramCount", "avoidVP", "ignoreMissingInDS", "forceStaged", "maxCore", + "site", + "excludedSite", + "includedSite", + "nFilesPerJob", + "nMaxFilesPerJob", + "nGBPerJob", + "nFiles", + "nEvents", + "loopingCheck", + "maxNFilesPerJob", + "memory", + "ramCount", + "avoidVP", + "ignoreMissingInDS", + "forceStaged", + "maxCore", ] def _build_namespace(core) -> dict: from inspect import signature + + from rich import box from rich.console import Console from rich.table import Table - from rich import box _console = Console() @@ -283,10 +315,8 @@ def help(command=None): _GROUPS = [ ("Tasks", ["show", "showl", "kill", "finish", "retry", "debug"]), ("Files & input", ["get_user_job_metadata", "recover_lost_files", "reload_input"]), - ("Workflows", ["show_workflow", "kill_workflow", "retry_workflow", - "finish_workflow", "pause_workflow", "resume_workflow"]), - ("Secrets", ["set_secret", "list_secrets", "delete_secret", - "delete_all_secrets"]), + ("Workflows", ["show_workflow", "kill_workflow", "retry_workflow", "finish_workflow", "pause_workflow", "resume_workflow"]), + ("Secrets", ["set_secret", "list_secrets", "delete_secret", "delete_all_secrets"]), ("Auth", ["generate_credential"]), ] for group, names in _GROUPS: @@ -303,27 +333,46 @@ def help(command=None): _console.print(table) _console.print("Usage: [bold]help(show)[/bold] or [bold]pbook show --help[/bold]\n") - def show(taskID=None, *, username=None, limit=1000, taskname=None, days=14, - jeditaskid=None, reqid=None, status=None, superstatus=None, format="standard"): + def show(taskID=None, *, username=None, limit=1000, taskname=None, days=14, jeditaskid=None, reqid=None, status=None, superstatus=None, format="standard"): """Print task records. taskID: jediTaskID / reqID / 'run' (active) / 'fin' (terminated) / omit for all. format: standard | long | json | plain """ - kwargs = {k: v for k, v in dict(username=username, limit=limit, taskname=taskname, - days=days, jeditaskid=jeditaskid, reqid=reqid, status=status, - superstatus=superstatus, format=format).items() if v is not None} + kwargs = { + k: v + for k, v in dict( + username=username, + limit=limit, + taskname=taskname, + days=days, + jeditaskid=jeditaskid, + reqid=reqid, + status=status, + superstatus=superstatus, + format=format, + ).items() + if v is not None + } kwargs.setdefault("limit", limit) kwargs.setdefault("days", days) kwargs["format"] = format return core.show(taskID, **kwargs) if taskID is not None else core.show(**kwargs) - def showl(taskID=None, *, username=None, limit=1000, taskname=None, days=14, - jeditaskid=None, reqid=None, status=None, superstatus=None): + def showl(taskID=None, *, username=None, limit=1000, taskname=None, days=14, jeditaskid=None, reqid=None, status=None, superstatus=None): """Print task records in long format (shortcut for show(..., format='long')).""" - return show(taskID, username=username, limit=limit, taskname=taskname, days=days, - jeditaskid=jeditaskid, reqid=reqid, status=status, - superstatus=superstatus, format="long") + return show( + taskID, + username=username, + limit=limit, + taskname=taskname, + days=days, + jeditaskid=jeditaskid, + reqid=reqid, + status=status, + superstatus=superstatus, + format="long", + ) def kill(taskIDs): """Kill tasks. taskIDs: int, list of ints, or 'all'.""" @@ -455,6 +504,7 @@ def generate_credential(): # ─── Top-level callback ─────────────────────────────────────────────────────── + @app.callback(invoke_without_command=True) def _main( ctx: typer.Context, @@ -499,6 +549,7 @@ def _main( core.init() exec(command_string, {}, ns) # noqa: S102 from pandaclient import PBookCore as _PBC + raise typer.Exit(0 if _PBC.func_return_value else 1) core.init() _run_repl(ns, banner=f"\nStart pBook {PandaToolsPkgInfo.release_version}") @@ -516,6 +567,7 @@ def _main( # ─── Subcommands ────────────────────────────────────────────────────────────── + @app.command() def show( task_id: Optional[str] = typer.Argument(None, help="jediTaskID, reqID, 'run' (active only), or 'fin' (terminated only)"), @@ -533,8 +585,14 @@ def show( core = _get_core() core.init(sanity_check=False) kwargs: dict = {"limit": limit, "days": days, "format": output_format} - for k, v in [("username", username), ("taskname", taskname), ("jeditaskid", jeditaskid), - ("reqid", reqid), ("status", status), ("superstatus", superstatus)]: + for k, v in [ + ("username", username), + ("taskname", taskname), + ("jeditaskid", jeditaskid), + ("reqid", reqid), + ("status", status), + ("superstatus", superstatus), + ]: if v is not None: kwargs[k] = v if task_id is not None: @@ -563,8 +621,14 @@ def showl( core = _get_core() core.init(sanity_check=False) kwargs: dict = {"limit": limit, "days": days, "format": "long"} - for k, v in [("username", username), ("taskname", taskname), ("jeditaskid", jeditaskid), - ("reqid", reqid), ("status", status), ("superstatus", superstatus)]: + for k, v in [ + ("username", username), + ("taskname", taskname), + ("jeditaskid", jeditaskid), + ("reqid", reqid), + ("status", status), + ("superstatus", superstatus), + ]: if v is not None: kwargs[k] = v if task_id is not None: @@ -634,14 +698,24 @@ def retry( core = _get_core() core.init(sanity_check=False) new_opts = { - k: v for k, v in { - "site": site, "excludedSite": excluded_site, "includedSite": included_site, - "nFilesPerJob": n_files_per_job, "nMaxFilesPerJob": n_max_files_per_job, - "nGBPerJob": n_gb_per_job, "nFiles": n_files, "nEvents": n_events, - "loopingCheck": looping_check, "ramCount": memory, "avoidVP": avoid_vp, - "ignoreMissingInDS": ignore_missing_in_ds, "forceStaged": force_staged, + k: v + for k, v in { + "site": site, + "excludedSite": excluded_site, + "includedSite": included_site, + "nFilesPerJob": n_files_per_job, + "nMaxFilesPerJob": n_max_files_per_job, + "nGBPerJob": n_gb_per_job, + "nFiles": n_files, + "nEvents": n_events, + "loopingCheck": looping_check, + "ramCount": memory, + "avoidVP": avoid_vp, + "ignoreMissingInDS": ignore_missing_in_ds, + "forceStaged": force_staged, "maxCoreCount": max_core, - }.items() if v is not None + }.items() + if v is not None } opts = new_opts or None ids = _parse_ids(task_ids) @@ -818,6 +892,7 @@ def generate_credential() -> None: # ─── Entry point ────────────────────────────────────────────────────────────── + def main() -> None: sys.argv[0] = "pbook" - app() \ No newline at end of file + app() From c83f35cd81a4b05faef2d14f80e54f57b9fe4d3b Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Wed, 29 Jul 2026 11:47:42 +0200 Subject: [PATCH 16/59] Fix when nothing typed yet and pressing tab --- pandaclient/PBookTyper.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index a5a12d1c..7aa87bf8 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -257,6 +257,11 @@ def _compute(self, text: str) -> list[str]: return hits # Fallback: standard name completion, stripping trailing '(' rlcompleter adds to callables + if not text: + # rlcompleter.complete() special-cases blank text by calling readline.insert_text() + # itself, which re-enters readline from inside this callback and confuses the active + # Tab press; list the namespace directly instead of delegating to it here + return sorted(k for k in self._base.namespace if not k.startswith("_")) results, i = [], 0 while (c := self._base.complete(text, i)) is not None: results.append(c.rstrip("()").rstrip("(")) From 48be21032281c059c52373a1ddcd2d01694f0f7d Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Wed, 29 Jul 2026 11:51:40 +0200 Subject: [PATCH 17/59] Skip Console and Table in tab completion --- pandaclient/PBookTyper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index 7aa87bf8..540c95a3 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -503,7 +503,7 @@ def generate_credential(): """Generate a new proxy or token.""" core.generate_credential() - ns = {k: v for k, v in locals().items() if callable(v)} + ns = {k: v for k, v in locals().items() if callable(v) and getattr(v, "__module__", None) == __name__} return ns From ef12fa61c62565f78785297ad8f3ea10b606bc94 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Fri, 31 Jul 2026 10:54:36 +0200 Subject: [PATCH 18/59] Reimplemented Pbook through typer a second time, but without duplication of functions --- pandaclient/PBookTyper.py | 68 ++-- pandaclient/PBookTyper2.py | 753 +++++++++++++++++++++++++++++++++++++ scripts/pbook | 2 +- 3 files changed, 799 insertions(+), 24 deletions(-) create mode 100644 pandaclient/PBookTyper2.py diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index 540c95a3..28ff41d5 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -8,13 +8,19 @@ import code import os import re +import readline +import rlcompleter import signal import sys import tempfile from concurrent.futures import ThreadPoolExecutor +from inspect import signature from typing import Optional import typer +from rich import box +from rich.console import Console +from rich.table import Table from pandaclient import Client, PandaToolsPkgInfo from pandaclient.MiscUtils import commands_get_output @@ -37,6 +43,7 @@ def _parallel(func, items): + # Parallel execution in a thread pool of 8 threads, for example when the user wants to act on a list of task IDs with ThreadPoolExecutor(8) as pool: return list(pool.map(func, items)) @@ -60,8 +67,6 @@ def _setup() -> None: return _setup_done = True - import readline - readline.parse_and_bind("tab: complete") readline.parse_and_bind("set show-all-if-ambiguous On") @@ -101,9 +106,8 @@ def _setup() -> None: def _cleanup() -> None: if _fork_child_pid == 0 and _history_file: - import readline - readline.write_history_file(_history_file) + if _tmp_dir: commands_get_output(f"rm -rf {_tmp_dir}") @@ -121,6 +125,7 @@ def _get_core(): def _catch_sig(sig, frame): _cleanup() + # Hard kill all processes in the group commands_get_output(f"kill -9 -- -{os.getpgrp()}") @@ -163,9 +168,6 @@ def _catch_sig(sig, frame): "show": { "format": ["standard", "long", "json", "plain"], }, - "showl": { - "format": ["standard", "long", "json", "plain"], - }, "finish": { "soft": ["True", "False"], }, @@ -192,8 +194,6 @@ def _catch_sig(sig, frame): def _run_repl(ns: dict, banner: str) -> None: """Manual REPL using InteractiveConsole.push() so we own readline setup entirely.""" - import readline - completer = _PBookCompleter(ns) readline.set_completer(completer.complete) readline.parse_and_bind("tab: complete") @@ -223,8 +223,6 @@ class _PBookCompleter: """Readline completer: kwarg names and values when inside a call, names otherwise.""" def __init__(self, ns: dict) -> None: - import rlcompleter - self._base = rlcompleter.Completer(ns) self._matches: list[str] = [] @@ -234,11 +232,13 @@ def complete(self, text: str, state: int) -> Optional[str]: return self._matches[state] if state < len(self._matches) else None def _compute(self, text: str) -> list[str]: - import readline - line = readline.get_line_buffer() - # Value completion: last token is kwarg= or kwarg='partial + # Kwarg value completion(tier 1, highest priority): last token is kwarg= or kwarg='partial + # >>> show(format='j│ + # Now the line matches both m_func (show() and m_val (line 234's regex: kwarg="format", quote="'", partial="j"). + # It looks up _KWARG_VALUES["show"]["format"] (line 165) = ["standard", "long", "json", "plain"], filters to those starting with j → json. + # This branch is checked before tier 2, so once you're past the =, you get value suggestions instead of more kwarg names. m_val = re.search(r"\b(\w+)\s*=\s*(['\"]?)(\w*)$", line) m_func = re.match(r"(\w+)\s*\(", line) if m_val and m_func: @@ -250,13 +250,25 @@ def _compute(self, text: str) -> list[str]: return hits # Kwarg name completion: cursor is inside an open call + # >>> show(│ + # readline.get_line_buffer() returns "show(". The regex on line 245, (\w+)\s*\([^)]*$, matches with func_name = "show" and nothing after the (. + # So it looks up _FUNC_KWARGS["show"] (line 133) and lists username, limit, taskname, days, jeditaskid, reqid, status, superstatus, format — + # rlcompleter is never consulted here. + + # >>> show(form│ + # Same regex still matches (func_name = "show", and [^)]* swallows form), but we filter _FUNC_KWARGS["show"] down to names starting with + # form → just format. m = re.search(r"(\w+)\s*\([^)]*$", line) if m: hits = [k for k in _FUNC_KWARGS.get(m.group(1), []) if k.startswith(text)] if hits: return hits - # Fallback: standard name completion, stripping trailing '(' rlcompleter adds to callables + # Plain name completion (tier 3, the rlcompleter fallback): standard name completion, stripping trailing '(' rlcompleter adds to callables + # >>> sho│ + # Not inside any (...), so tiers 1 and 2 don't match. Falls through to rlcompleter (line 257-260), which scans ns for names starting with + # sho → matches show, showl. Since these are callables, rlcompleter.complete() would normally return "show("/"showl("; the .rstrip("()").rstrip("(") + # on line 259 strips that back to plain show, showl. if not text: # rlcompleter.complete() special-cases blank text by calling readline.insert_text() # itself, which re-enters readline from inside this callback and confuses the active @@ -290,12 +302,6 @@ def _compute(self, text: str) -> list[str]: def _build_namespace(core) -> dict: - from inspect import signature - - from rich import box - from rich.console import Console - from rich.table import Table - _console = Console() def help(command=None): @@ -312,6 +318,7 @@ def help(command=None): _console.print(f"\n{doc}\n") return + # No argument - summary table table = Table(box=box.SIMPLE, show_header=True, header_style="bold magenta") table.add_column("Command", style="bold cyan", no_wrap=True) table.add_column("Signature", style="dim", no_wrap=True) @@ -380,17 +387,19 @@ def showl(taskID=None, *, username=None, limit=1000, taskname=None, days=14, jed ) def kill(taskIDs): - """Kill tasks. taskIDs: int, list of ints, or 'all'.""" + """Kill tasks. taskIDs: int, list of ints, or 'all' for all active tasks.""" if taskIDs == "all": return _parallel(lambda t: core.kill(t.jeditaskid), core.get_active_tasks()) elif isinstance(taskIDs, (list, tuple)): return _parallel(core.kill, taskIDs) elif isinstance(taskIDs, int): return [core.kill(taskIDs)] + print("Error: Invalid argument") + return None def finish(taskIDs, soft=False): - """Finish tasks. taskIDs: int, [int,...], or 'all'. soft=True waits for running jobs.""" + """Finish tasks. taskIDs: int, [int,...], or 'all' for all active tasks. soft=True waits for running jobs.""" if taskIDs == "all": return _parallel( lambda t: core.finish.original_func(core, t.jeditaskid, soft=soft), @@ -400,7 +409,9 @@ def finish(taskIDs, soft=False): return _parallel(lambda tid: core.finish(tid, soft=soft), taskIDs) elif isinstance(taskIDs, int): return [core.finish(taskIDs, soft=soft)] + print("Error: Invalid argument") + return None def retry(taskIDs, newOpts=None, days=14, limit=1000, **kwargs): """Retry failed/cancelled tasks. @@ -429,7 +440,9 @@ def retry(taskIDs, newOpts=None, days=14, limit=1000, **kwargs): elif taskIDs == "all": data = core.show(status="finished", days=days, limit=limit, format="json") return _parallel(lambda d: core.retry.original_func(core, d["jediTaskID"], newOpts=opts), data) + print("Error: Invalid argument") + return None def debug(PandaID, modeOn): """Toggle debug mode for a subjob. modeOn: True/False.""" @@ -503,6 +516,7 @@ def generate_credential(): """Generate a new proxy or token.""" core.generate_credential() + # Generate the namespace with the local functions and exclude any imported functions or variables ns = {k: v for k, v in locals().items() if callable(v) and getattr(v, "__module__", None) == __name__} return ns @@ -525,6 +539,7 @@ def _main( typer.echo(f"Version: {PandaToolsPkgInfo.release_version}") raise typer.Exit() + # Set up the development or integration server if dev_srv: Client.useDevServer() if intr_srv: @@ -539,10 +554,13 @@ def _main( _setup() global _fork_child_pid _fork_child_pid = os.fork() + + # Fork failed if _fork_child_pid == -1: typer.echo("ERROR: Failed to fork", err=True) raise typer.Exit(1) + # Child process if _fork_child_pid == 0: if verbose: typer.echo(str(ctx.params)) @@ -550,6 +568,8 @@ def _main( sys.ps1 = ">>> \n" core = _make_core(verbose) ns = _build_namespace(core) + + # The user wants to execute a Python code snippet instead of entering the REPL if command_string: core.init() exec(command_string, {}, ns) # noqa: S102 @@ -558,6 +578,8 @@ def _main( raise typer.Exit(0 if _PBC.func_return_value else 1) core.init() _run_repl(ns, banner=f"\nStart pBook {PandaToolsPkgInfo.release_version}") + + # Parent process else: signal.signal(signal.SIGINT, _catch_sig) signal.signal(signal.SIGHUP, _catch_sig) diff --git a/pandaclient/PBookTyper2.py b/pandaclient/PBookTyper2.py new file mode 100644 index 00000000..1c7dc65d --- /dev/null +++ b/pandaclient/PBookTyper2.py @@ -0,0 +1,753 @@ +""" +pbook CLI — PanDA task bookkeeper. + +Each command below is defined exactly once, as a Typer command using the +Annotated[...] parameter style. Because the Typer/Click metadata lives in the +annotation rather than in the default value, these functions remain ordinary +callables with ordinary defaults - the same function is used to build the +`pbook --flag ...` CLI (with real shell completion) *and* is placed +directly into the interactive REPL namespace (`>>> command(...)`), with no +separate REPL-only copy of the command's logic, docstring, or option list. +""" + +from __future__ import annotations + +import atexit +import code +import os +import re +import readline +import rlcompleter +import signal +import sys +import tempfile +from concurrent.futures import ThreadPoolExecutor +from inspect import Parameter, signature +from typing import ( + Annotated, + Literal, + Optional, + Union, + get_args, + get_origin, + get_type_hints, +) + +import typer +from rich import box +from rich.console import Console +from rich.markup import escape as _esc +from rich.table import Table + +from pandaclient import Client, PandaToolsPkgInfo +from pandaclient.MiscUtils import commands_get_output + +# ─── Runtime state ──────────────────────────────────────────────────────────── +_tmp_dir: Optional[str] = None +_history_file: Optional[str] = None +_fork_child_pid: Optional[int] = None +_setup_done: bool = False +_ctx_state: dict = {} +_core = None +_core_inited: bool = False + +app = typer.Typer( + name="pbook", + help="PanDA task bookkeeper. Run without arguments for interactive mode.", + invoke_without_command=True, + no_args_is_help=False, +) + +# ─── Utilities ──────────────────────────────────────────────────────────────── + + +def _parallel(func, items): + with ThreadPoolExecutor(8) as pool: + return list(pool.map(func, items)) + + +def _parse_ids(raw): + """'all' -> 'all'; '42' -> 42; '1,2,3' -> [1,2,3]; anything not a string passes through as-is.""" + if not isinstance(raw, str): + return raw + if raw == "all": + return "all" + parts = raw.split(",") + try: + ids = [int(p) for p in parts] + return ids[0] if len(ids) == 1 else ids + except ValueError: + typer.echo(f"Error: invalid task ID(s): {raw}", err=True) + raise typer.Exit(1) + + +def _setup() -> None: + global _tmp_dir, _history_file, _setup_done + if _setup_done: + return + _setup_done = True + + readline.parse_and_bind("tab: complete") + readline.parse_and_bind("set show-all-if-ambiguous On") + + if "CMTSITE" not in os.environ: + os.environ["CMTSITE"] = "" + + pconf_dir = os.path.expanduser(os.environ.get("PANDA_CONFIG_ROOT", "~/.panda")) + os.makedirs(pconf_dir, exist_ok=True) + + _history_file = os.path.join(pconf_dir, ".history") + if os.path.exists(_history_file): + try: + readline.read_history_file(_history_file) + except Exception: + pass + readline.set_history_length(1024) + + _tmp_dir = tempfile.mkdtemp() + + for path in sys.path: + real = path or "." + if ( + os.path.exists(real) + and os.path.isdir(real) + and "pandaclient" in os.listdir(real) + and os.path.exists(os.path.join(real, "pandaclient", "__init__.py")) + ): + link = os.path.join(_tmp_dir, "taskbuffer") + if not os.path.exists(link): + os.symlink(os.path.join(real, "pandaclient"), link) + break + if _tmp_dir not in sys.path: + sys.path.insert(0, _tmp_dir) + + atexit.register(_cleanup) + + +def _cleanup() -> None: + if _fork_child_pid == 0 and _history_file: + readline.write_history_file(_history_file) + + if _tmp_dir: + commands_get_output(f"rm -rf {_tmp_dir}") + + +def _make_core(verbose: bool = False): + from pandaclient import PBookCore + + return PBookCore.PBookCore(verbose=verbose) + + +def _get_core(): + """Return the (memoized) core for this process, uninitialized.""" + global _core + _setup() + if _core is None: + _core = _make_core(_ctx_state.get("verbose", False)) + return _core + + +def _ensure_init(sanity_check: bool = False): + """Return the core, running PBookCore.init() exactly once per process. + + The REPL calls this once upfront with sanity_check=True; every command + function then calls it again with the default before touching the core, + which is a no-op once already initialized - so commands stay correct + whether they run once (batch mode) or repeatedly (REPL session). + """ + global _core_inited + core = _get_core() + if not _core_inited: + core.init(sanity_check=sanity_check) + _core_inited = True + return core + + +def _catch_sig(sig, frame): + _cleanup() + # Hard kill all processes in the group + commands_get_output(f"kill -9 -- -{os.getpgrp()}") + + +# ─── REPL namespace & completion ────────────────────────────────────────────── + + +def _build_namespace() -> dict: + """The REPL namespace: every registered Typer command, keyed by its real Python name.""" + return {info.callback.__name__: info.callback for info in app.registered_commands} + + +def _kwarg_names(func) -> list: + """All parameter names a function accepts - candidates for `name=` completion.""" + return list(signature(func).parameters) + + +def _kwarg_choices(func, name: str) -> list: + """Value choices for a parameter, derived from its type hint: Literal[...] members or True/False for bool.""" + try: + hints = get_type_hints(func, include_extras=True) + except Exception: + return [] + ann = hints.get(name) + if ann is None: + return [] + while hasattr(ann, "__metadata__"): + ann = ann.__origin__ + if get_origin(ann) is Union: + non_none = [a for a in get_args(ann) if a is not type(None)] + if len(non_none) == 1: + ann = non_none[0] + if get_origin(ann) is Literal: + return [str(v) for v in get_args(ann)] + if ann is bool: + return ["True", "False"] + return [] + + +class _PBookCompleter: + """Readline completer: kwarg names and values when inside a call, names otherwise.""" + + def __init__(self, ns: dict) -> None: + self._ns = ns + self._base = rlcompleter.Completer(ns) + self._matches: list = [] + + def complete(self, text: str, state: int) -> Optional[str]: + if state == 0: + self._matches = self._compute(text) + return self._matches[state] if state < len(self._matches) else None + + def _compute(self, text: str) -> list: + line = readline.get_line_buffer() + + # Kwarg value completion (tier 1): last token is kwarg= or kwarg='partial + m_val = re.search(r"\b(\w+)\s*=\s*(['\"]?)(\w*)$", line) + m_func = re.match(r"(\w+)\s*\(", line) + if m_val and m_func: + kwarg, partial = m_val.group(1), m_val.group(3) + func = self._ns.get(m_func.group(1)) + if func is not None: + hits = [v for v in _kwarg_choices(func, kwarg) if v.startswith(partial)] + if hits: + return hits + + # Kwarg name completion (tier 2): cursor is inside an open call + m = re.search(r"(\w+)\s*\([^)]*$", line) + if m: + func = self._ns.get(m.group(1)) + if func is not None: + hits = [k for k in _kwarg_names(func) if k.startswith(text)] + if hits: + return hits + + # Plain name completion (tier 3, rlcompleter fallback) + if not text: + # rlcompleter.complete() special-cases blank text by calling readline.insert_text() + # itself, which re-enters readline from inside this callback and confuses the active + # Tab press; list the namespace directly instead of delegating to it here + return sorted(k for k in self._base.namespace if not k.startswith("_")) + results, i = [], 0 + while (c := self._base.complete(text, i)) is not None: + results.append(c.rstrip("()").rstrip("(")) + i += 1 + return results + + +def _run_repl(ns: dict, banner: str) -> None: + """Manual REPL using InteractiveConsole.push() so we own readline setup entirely.""" + completer = _PBookCompleter(ns) + readline.set_completer(completer.complete) + readline.parse_and_bind("tab: complete") + readline.parse_and_bind("set show-all-if-ambiguous On") + + console = code.InteractiveConsole(ns) + print(banner) + + more = False + while True: + prompt = "... " if more else ">>> " + try: + readline.set_completer(completer.complete) + line = input(prompt) + except EOFError: + print() + break + except KeyboardInterrupt: + print("\nKeyboardInterrupt") + console.resetbuffer() + more = False + continue + more = console.push(line) + + +# ─── Top-level callback ─────────────────────────────────────────────────────── + + +@app.callback(invoke_without_command=True) +def _main( + ctx: typer.Context, + verbose: bool = typer.Option(False, "-v", help="Verbose"), + command_string: Optional[str] = typer.Option(None, "-c", help="Execute a Python code snippet"), + version: bool = typer.Option(False, "--version", is_eager=True, help="Display version"), + dev_srv: bool = typer.Option(False, "--devSrv", hidden=True), + intr_srv: bool = typer.Option(False, "--intrSrv", hidden=True), + prompt_with_newline: bool = typer.Option(False, "--prompt_with_newline", hidden=True), +) -> None: + """PanDA task bookkeeper. Run without arguments for interactive mode.""" + if version: + typer.echo(f"Version: {PandaToolsPkgInfo.release_version}") + raise typer.Exit() + + if dev_srv: + Client.useDevServer() + if intr_srv: + Client.useIntrServer() + + _ctx_state.update({"verbose": verbose}) + + if ctx.invoked_subcommand is not None: + return + + # Interactive or snippet mode + _setup() + global _fork_child_pid + _fork_child_pid = os.fork() + + if _fork_child_pid == -1: + typer.echo("ERROR: Failed to fork", err=True) + raise typer.Exit(1) + + if _fork_child_pid == 0: + if verbose: + typer.echo(str(ctx.params)) + if prompt_with_newline: + sys.ps1 = ">>> \n" + _ensure_init(sanity_check=True) + ns = _build_namespace() + + if command_string: + exec(command_string, {}, ns) # noqa: S102 + from pandaclient import PBookCore as _PBC + + raise typer.Exit(0 if _PBC.func_return_value else 1) + _run_repl(ns, banner=f"\nStart pBook {PandaToolsPkgInfo.release_version}") + + else: + signal.signal(signal.SIGINT, _catch_sig) + signal.signal(signal.SIGHUP, _catch_sig) + signal.signal(signal.SIGTERM, _catch_sig) + pid, status = os.wait() + if os.WIFSIGNALED(status): + raise typer.Exit(-os.WTERMSIG(status)) + elif os.WIFEXITED(status): + raise typer.Exit(os.WEXITSTATUS(status)) + raise typer.Exit(0) + + +# ─── Commands ────────────────────────────────────────────────────────────────── + +_HELP_GROUPS = [ + ("Tasks", ["show", "showl", "kill", "finish", "retry", "debug"]), + ("Files & input", ["get_user_job_metadata", "recover_lost_files", "reload_input"]), + ("Workflows", ["show_workflow", "kill_workflow", "retry_workflow", "finish_workflow", "pause_workflow", "resume_workflow"]), + ("Secrets", ["set_secret", "list_secrets", "delete_secret", "delete_all_secrets"]), + ("Auth", ["generate_credential"]), +] + + +def _type_name(ann) -> str: + """Render a resolved type annotation as a short, human-readable name (Optional[str], Literal[...], etc.).""" + if ann is None or ann is type(None): + return "" + origin = get_origin(ann) + if origin is Union: + args = get_args(ann) + non_none = [a for a in args if a is not type(None)] + if len(non_none) == 1 and len(args) == 2: + return f"Optional[{_type_name(non_none[0])}]" + return " | ".join(_type_name(a) for a in args) + if origin is Literal: + return "Literal[" + ", ".join(repr(v) for v in get_args(ann)) + "]" + if origin is not None: + args = get_args(ann) + origin_name = getattr(origin, "__name__", str(origin)) + return f"{origin_name}[{', '.join(_type_name(a) for a in args)}]" if args else origin_name + return getattr(ann, "__name__", str(ann)) + + +def _format_signature(func) -> str: + """A clean '(param: Type = default, ...)' string, stripping the Typer/Annotated plumbing.""" + try: + hints = get_type_hints(func, include_extras=True) + except Exception: + hints = {} + parts = [] + for pname, p in signature(func).parameters.items(): + ann = hints.get(pname) + while hasattr(ann, "__metadata__"): + ann = ann.__origin__ + piece = pname + type_str = _type_name(ann) + if type_str: + piece += f": {type_str}" + if p.default is not Parameter.empty: + piece += f" = {p.default!r}" + parts.append(piece) + return f"({', '.join(parts)})" + + +@app.command() +def help( + command: Annotated[Optional[str], typer.Argument(help="Command name for detailed help")] = None, +) -> None: + """Show available commands, or detailed help for a specific command.""" + ns = _build_namespace() + console = Console() + + if command is not None: + name = command if isinstance(command, str) else command.__name__ + func = ns.get(name, command if callable(command) else None) + if func is None: + console.print(f"[red]Unknown command:[/red] {_esc(name)}") + return + console.print(f"\n[bold cyan]{name}[/bold cyan][bold]{_esc(_format_signature(func))}[/bold]") + doc = (func.__doc__ or "No description.").strip() + console.print(f"\n{_esc(doc)}\n") + return + + table = Table(box=box.SIMPLE, show_header=True, header_style="bold magenta") + table.add_column("Command", style="bold cyan", no_wrap=True) + table.add_column("Signature", style="dim") + table.add_column("Description") + + for group, names in _HELP_GROUPS: + table.add_section() + table.add_row(f"[bold white]{group}[/bold white]", "", "") + for name in names: + func = ns.get(name) + if func is None: + continue + sig = _format_signature(func) + doc = (func.__doc__ or "").strip().splitlines()[0] if func.__doc__ else "" + table.add_row(f" {name}", _esc(sig), _esc(doc)) + + console.print(table) + console.print("Usage: [bold]help(show)[/bold] or [bold]pbook show --help[/bold]\n") + + +@app.command() +def show( + task_id: Annotated[Optional[str], typer.Argument(help="jediTaskID, reqID, 'run' (active only), or 'fin' (terminated only)")] = None, + username: Annotated[Optional[str], typer.Option(help="Filter by username")] = None, + limit: Annotated[int, typer.Option(help="Maximum number of records")] = 1000, + taskname: Annotated[Optional[str], typer.Option(help="Filter by task name")] = None, + days: Annotated[int, typer.Option(help="Look back N days (capped at 90 without a task ID)")] = 14, + jeditaskid: Annotated[Optional[int], typer.Option(help="Filter by jediTaskID")] = None, + reqid: Annotated[Optional[int], typer.Option(help="Filter by reqID")] = None, + status: Annotated[Optional[str], typer.Option(help="Filter by task status")] = None, + superstatus: Annotated[Optional[str], typer.Option(help="Filter by super-status")] = None, + format: Annotated[Literal["standard", "long", "json", "plain"], typer.Option("--format", help="Output format")] = "standard", +) -> None: + """Print task records. + + taskID: jediTaskID / reqID / 'run' (active) / 'fin' (terminated) / omit for all. + """ + core = _ensure_init() + kwargs = { + k: v + for k, v in dict( + username=username, + limit=limit, + taskname=taskname, + days=days, + jeditaskid=jeditaskid, + reqid=reqid, + status=status, + superstatus=superstatus, + ).items() + if v is not None + } + kwargs["format"] = format + if task_id is not None: + try: + first_arg = int(task_id) + except (TypeError, ValueError): + first_arg = task_id + return core.show(first_arg, **kwargs) + return core.show(**kwargs) + + +@app.command() +def showl( + task_id: Annotated[Optional[str], typer.Argument(help="jediTaskID, reqID, 'run', or 'fin'")] = None, + username: Annotated[Optional[str], typer.Option(help="Filter by username")] = None, + limit: Annotated[int, typer.Option(help="Maximum number of records")] = 1000, + taskname: Annotated[Optional[str], typer.Option(help="Filter by task name")] = None, + days: Annotated[int, typer.Option(help="Look back N days (capped at 90 without a task ID)")] = 14, + jeditaskid: Annotated[Optional[int], typer.Option(help="Filter by jediTaskID")] = None, + reqid: Annotated[Optional[int], typer.Option(help="Filter by reqID")] = None, + status: Annotated[Optional[str], typer.Option(help="Filter by task status")] = None, + superstatus: Annotated[Optional[str], typer.Option(help="Filter by super-status")] = None, +) -> None: + """Print task records in long format (shortcut for show --format long).""" + return show( + task_id, + username=username, + limit=limit, + taskname=taskname, + days=days, + jeditaskid=jeditaskid, + reqid=reqid, + status=status, + superstatus=superstatus, + format="long", + ) + + +@app.command() +def kill( + task_ids: Annotated[str, typer.Argument(help="Task ID, comma-separated IDs, or 'all'")], +) -> None: + """Kill tasks.""" + core = _ensure_init() + ids = _parse_ids(task_ids) + if ids == "all": + return _parallel(lambda t: core.kill(t.jeditaskid), core.get_active_tasks()) + elif isinstance(ids, list): + return _parallel(core.kill, ids) + return core.kill(ids) + + +@app.command() +def finish( + task_ids: Annotated[str, typer.Argument(help="Task ID, comma-separated IDs, or 'all'")], + soft: Annotated[bool, typer.Option("--soft", help="Wait for running jobs to finish instead of killing them")] = False, +) -> None: + """Finish tasks.""" + core = _ensure_init() + ids = _parse_ids(task_ids) + if ids == "all": + return _parallel(lambda t: core.finish.original_func(core, t.jeditaskid, soft=soft), core.get_active_tasks()) + elif isinstance(ids, list): + return _parallel(lambda tid: core.finish(tid, soft=soft), ids) + return core.finish(ids, soft=soft) + + +@app.command() +def retry( + task_ids: Annotated[str, typer.Argument(help="Task ID, comma-separated IDs, or 'all'")], + days: Annotated[int, typer.Option("--days", help="Look-back window when task_ids='all'")] = 14, + limit: Annotated[int, typer.Option("--limit", help="Max tasks to retry when task_ids='all'")] = 1000, + site: Annotated[Optional[str], typer.Option("--site")] = None, + excludedSite: Annotated[Optional[str], typer.Option("--excludedSite")] = None, + includedSite: Annotated[Optional[str], typer.Option("--includedSite")] = None, + nFilesPerJob: Annotated[Optional[int], typer.Option("--nFilesPerJob")] = None, + nMaxFilesPerJob: Annotated[Optional[int], typer.Option("--nMaxFilesPerJob")] = None, + nGBPerJob: Annotated[Optional[float], typer.Option("--nGBPerJob")] = None, + nFiles: Annotated[Optional[int], typer.Option("--nFiles")] = None, + nEvents: Annotated[Optional[int], typer.Option("--nEvents")] = None, + loopingCheck: Annotated[Optional[bool], typer.Option("--loopingCheck")] = None, + memory: Annotated[Optional[int], typer.Option("--memory")] = None, + avoidVP: Annotated[Optional[bool], typer.Option("--avoidVP")] = None, + ignoreMissingInDS: Annotated[Optional[bool], typer.Option("--ignoreMissingInDS")] = None, + forceStaged: Annotated[Optional[bool], typer.Option("--forceStaged")] = None, + maxCore: Annotated[Optional[int], typer.Option("--maxCore")] = None, +) -> None: + """Retry failed/cancelled tasks. + + Allowed options: site, excludedSite, includedSite, nFilesPerJob, nMaxFilesPerJob, + nGBPerJob, nFiles, nEvents, loopingCheck, memory, avoidVP, ignoreMissingInDS, + forceStaged, maxCore. + + example: + >>> retry(123) + >>> retry([123, 345, 567]) + >>> retry(789, excludedSite='siteA,siteB') + >>> retry('all') + >>> retry('all', days=30, limit=2000) + """ + core = _ensure_init() + new_opts = { + k: v + for k, v in { + "site": site, + "excludedSite": excludedSite, + "includedSite": includedSite, + "nFilesPerJob": nFilesPerJob, + "nMaxFilesPerJob": nMaxFilesPerJob, + "nGBPerJob": nGBPerJob, + "nFiles": nFiles, + "nEvents": nEvents, + "loopingCheck": loopingCheck, + "ramCount": memory, + "avoidVP": avoidVP, + "ignoreMissingInDS": ignoreMissingInDS, + "forceStaged": forceStaged, + "maxCoreCount": maxCore, + }.items() + if v is not None + } + opts = new_opts or None + ids = _parse_ids(task_ids) + if isinstance(ids, list): + return _parallel(lambda tid: core.retry(tid, newOpts=opts), ids) + elif ids == "all": + data = core.show(status="finished", days=days, limit=limit, format="json") + return _parallel(lambda d: core.retry.original_func(core, d["jediTaskID"], newOpts=opts), data) + return core.retry(ids, newOpts=opts) + + +@app.command() +def debug( + panda_id: Annotated[int, typer.Argument(help="PanDA subjob ID")], + mode_on: Annotated[bool, typer.Argument(help="True to enable, False to disable")], +) -> None: + """Toggle debug mode for a subjob.""" + core = _ensure_init() + core.debug(panda_id, mode_on) + + +@app.command(name="get-user-job-metadata") +def get_user_job_metadata( + task_id: Annotated[int, typer.Argument(help="Task ID")], + output_file: Annotated[str, typer.Argument(help="Output JSON file path")], +) -> None: + """Write user metadata of successful jobs to a JSON file.""" + core = _ensure_init() + core.getUserJobMetadata(task_id, output_file) + + +@app.command(name="reload-input") +def reload_input( + task_id: Annotated[int, typer.Argument(help="Task ID")], +) -> None: + """Reload input dataset and retry the task with new contents.""" + core = _ensure_init() + core.reload_input(task_id) + + +@app.command(name="recover-lost-files") +def recover_lost_files( + task_id: Annotated[int, typer.Argument(help="Task ID")], + test_mode: Annotated[bool, typer.Option("--test-mode", help="Dry-run mode")] = False, +) -> None: + """Request recovery of lost files from a task.""" + core = _ensure_init() + core.recover_lost_files(task_id, test_mode) + + +@app.command(name="show-workflow") +def show_workflow( + request_id: Annotated[int, typer.Argument(help="Workflow request ID")], +) -> None: + """Show workflow status.""" + core = _ensure_init() + _, output = core.execute_workflow_command("get_status", request_id) + if output: + print(output) + + +@app.command(name="kill-workflow") +def kill_workflow( + request_id: Annotated[int, typer.Argument(help="Workflow request ID")], +) -> None: + """Kill a workflow.""" + core = _ensure_init() + _, output = core.execute_workflow_command("abort", request_id) + if output: + print(output[0][-1]) + + +@app.command(name="retry-workflow") +def retry_workflow( + request_id: Annotated[int, typer.Argument(help="Workflow request ID")], +) -> None: + """Retry a workflow.""" + core = _ensure_init() + _, output = core.execute_workflow_command("retry", request_id) + if output: + print(output[0][-1]) + + +@app.command(name="finish-workflow") +def finish_workflow( + request_id: Annotated[int, typer.Argument(help="Workflow request ID")], +) -> None: + """Finish a workflow.""" + core = _ensure_init() + _, output = core.execute_workflow_command("finish", request_id) + if output: + print(output[0][-1]) + + +@app.command(name="pause-workflow") +def pause_workflow( + request_id: Annotated[int, typer.Argument(help="Workflow request ID")], +) -> None: + """Pause a workflow.""" + core = _ensure_init() + _, output = core.execute_workflow_command("suspend", request_id) + if output: + print(output[0][-1]) + + +@app.command(name="resume-workflow") +def resume_workflow( + request_id: Annotated[int, typer.Argument(help="Workflow request ID")], +) -> None: + """Resume a workflow.""" + core = _ensure_init() + _, output = core.execute_workflow_command("resume", request_id) + if output: + print(output[0][-1]) + + +@app.command(name="set-secret") +def set_secret( + key: Annotated[str, typer.Argument(help="Secret key")], + value: Annotated[str, typer.Argument(help="Secret value or file path")], + is_file: Annotated[bool, typer.Option("--is-file", help="Treat value as a file path to upload")] = False, +) -> None: + """Set a secret key-value pair.""" + core = _ensure_init() + core.set_secret(key, value, is_file) + + +@app.command(name="delete-secret") +def delete_secret( + key: Annotated[str, typer.Argument(help="Secret key to delete")], +) -> None: + """Delete a secret.""" + core = _ensure_init() + core.set_secret(key, None) + + +@app.command(name="delete-all-secrets") +def delete_all_secrets() -> None: + """Delete all secrets.""" + core = _ensure_init() + core.set_secret(None, None) + + +@app.command(name="list-secrets") +def list_secrets( + full: Annotated[bool, typer.Option("--full", help="Show full secret values")] = False, +) -> None: + """List secrets.""" + core = _ensure_init() + core.list_secrets(full) + + +@app.command(name="generate-credential") +def generate_credential() -> None: + """Generate a new proxy or token.""" + core = _get_core() + core.generate_credential() + + +# ─── Entry point ────────────────────────────────────────────────────────────── + + +def main() -> None: + sys.argv[0] = "pbook" + app() diff --git a/scripts/pbook b/scripts/pbook index 8ed2fe5f..81949cf6 100755 --- a/scripts/pbook +++ b/scripts/pbook @@ -2,4 +2,4 @@ source ${PANDA_SYS}/etc/panda/share/functions.sh -exec_p_command "import pandaclient.PBookTyper as pbook; pbook.main()" "$@" +exec_p_command "import pandaclient.PBookTyper2 as pbook; pbook.main()" "$@" From 0d3ea8a9cdce85babf3413dfa972bc28612b6c92 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Fri, 31 Jul 2026 11:23:57 +0200 Subject: [PATCH 19/59] Colour coding in function signature --- pandaclient/PBookTyper2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandaclient/PBookTyper2.py b/pandaclient/PBookTyper2.py index 1c7dc65d..c3dac91c 100644 --- a/pandaclient/PBookTyper2.py +++ b/pandaclient/PBookTyper2.py @@ -427,9 +427,9 @@ def help( func = ns.get(name) if func is None: continue - sig = _format_signature(func) + sig = console.highlighter(_format_signature(func)) doc = (func.__doc__ or "").strip().splitlines()[0] if func.__doc__ else "" - table.add_row(f" {name}", _esc(sig), _esc(doc)) + table.add_row(f" {name}", sig, _esc(doc)) console.print(table) console.print("Usage: [bold]help(show)[/bold] or [bold]pbook show --help[/bold]\n") From adc2cd61ca6e78ff8c0004c0f3735ced4916d440 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Fri, 31 Jul 2026 11:25:34 +0200 Subject: [PATCH 20/59] Remove dimming in function signature --- pandaclient/PBookTyper2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandaclient/PBookTyper2.py b/pandaclient/PBookTyper2.py index c3dac91c..ad2d0b05 100644 --- a/pandaclient/PBookTyper2.py +++ b/pandaclient/PBookTyper2.py @@ -417,7 +417,7 @@ def help( table = Table(box=box.SIMPLE, show_header=True, header_style="bold magenta") table.add_column("Command", style="bold cyan", no_wrap=True) - table.add_column("Signature", style="dim") + table.add_column("Signature") table.add_column("Description") for group, names in _HELP_GROUPS: From b8af0baaab88a1aafb530efdcf79f7b66ccd1cea Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Fri, 31 Jul 2026 11:49:15 +0200 Subject: [PATCH 21/59] Extend documentation --- pandaclient/PBookTyper2.py | 173 ++++++++++++++++++++++++++++++++----- 1 file changed, 153 insertions(+), 20 deletions(-) diff --git a/pandaclient/PBookTyper2.py b/pandaclient/PBookTyper2.py index ad2d0b05..850b47e2 100644 --- a/pandaclient/PBookTyper2.py +++ b/pandaclient/PBookTyper2.py @@ -450,7 +450,21 @@ def show( ) -> None: """Print task records. - taskID: jediTaskID / reqID / 'run' (active) / 'fin' (terminated) / omit for all. + The first argument (task_id) can be a jediTaskID or reqID, or 'run' (show active tasks + only), or 'fin' (show terminated tasks only), or can be omitted. Records are fetched + directly from the PanDA server, so they are always up to date. Note that days is capped + at 90 days unless a jediTaskID or reqID is specified, in which case tasks of any age are + returned. The default filter conditions are: username=(name from user voms proxy), + limit=1000, days=14, format='standard'. + + example: + >>> show() + >>> show(123) + >>> show(12345678, format='long') + >>> show(taskname='my_task_name') + >>> show('run') + >>> show('fin', days=7, limit=100) + >>> show(format='json') """ core = _ensure_init() kwargs = { @@ -489,7 +503,16 @@ def showl( status: Annotated[Optional[str], typer.Option(help="Filter by task status")] = None, superstatus: Annotated[Optional[str], typer.Option(help="Filter by super-status")] = None, ) -> None: - """Print task records in long format (shortcut for show --format long).""" + """Print task records in long format (shortcut for show --format long). + + See help(show) for the available filter keywords. + + example: + >>> showl() + >>> showl(123) + >>> showl(12345678) + >>> showl(taskname='my_task_name') + """ return show( task_id, username=username, @@ -508,7 +531,16 @@ def showl( def kill( task_ids: Annotated[str, typer.Argument(help="Task ID, comma-separated IDs, or 'all'")], ) -> None: - """Kill tasks.""" + """Kill tasks. + + Kill all subJobs in task_ids (ID or a list of IDs, can be either jediTaskID or reqID). + If 'all', kill all active tasks of the user. + + example: + >>> kill(123) + >>> kill([123, 345, 567]) + >>> kill('all') + """ core = _ensure_init() ids = _parse_ids(task_ids) if ids == "all": @@ -523,7 +555,19 @@ def finish( task_ids: Annotated[str, typer.Argument(help="Task ID, comma-separated IDs, or 'all'")], soft: Annotated[bool, typer.Option("--soft", help="Wait for running jobs to finish instead of killing them")] = False, ) -> None: - """Finish tasks.""" + """Finish tasks. + + Finish all subJobs in task_ids (ID or a list of IDs, can be either jediTaskID or reqID). + If task_ids is 'all', finish all active tasks of the user. If soft is False (default), + all running jobs are killed and the task finishes immediately. If soft is True, new jobs + are not generated and the task finishes once all running jobs finish. + + example: + >>> finish(123) + >>> finish(234, soft=True) + >>> finish([123, 345, 567]) + >>> finish('all') + """ core = _ensure_init() ids = _parse_ids(task_ids) if ids == "all": @@ -555,9 +599,16 @@ def retry( ) -> None: """Retry failed/cancelled tasks. - Allowed options: site, excludedSite, includedSite, nFilesPerJob, nMaxFilesPerJob, - nGBPerJob, nFiles, nEvents, loopingCheck, memory, avoidVP, ignoreMissingInDS, - forceStaged, maxCore. + Retry failed/cancelled subJobs in task_ids (ID or a list of IDs, can be either jediTaskID + or reqID). Allowed options to overwrite task parameters for new attempts: site, + excludedSite, includedSite, nFilesPerJob, nMaxFilesPerJob, nGBPerJob, nFiles, nEvents, + loopingCheck, memory, avoidVP, ignoreMissingInDS, forceStaged, maxCore. If input files + were used or are being used by other jobs for the same output dataset container, those + files are skipped to avoid job duplication when retrying failed subjobs. + + If task_ids is 'all', it retries 1000 tasks at most that have finished for the last 14 + days. It is possible to retry more tasks by setting the days and limit options. If + named arguments are specified, they are applied to all retried tasks. example: >>> retry(123) @@ -602,7 +653,15 @@ def debug( panda_id: Annotated[int, typer.Argument(help="PanDA subjob ID")], mode_on: Annotated[bool, typer.Argument(help="True to enable, False to disable")], ) -> None: - """Toggle debug mode for a subjob.""" + """Toggle debug mode for a subjob. + + mode_on is True/False to enable/disable the debug mode. Note that the maximum number of + debug subjobs is limited. If you already hit the limit you need to disable the debug mode + for a subjob before debugging another subjob. + + example: + >>> debug(1234, True) + """ core = _ensure_init() core.debug(panda_id, mode_on) @@ -612,7 +671,13 @@ def get_user_job_metadata( task_id: Annotated[int, typer.Argument(help="Task ID")], output_file: Annotated[str, typer.Argument(help="Output JSON file path")], ) -> None: - """Write user metadata of successful jobs to a JSON file.""" + """Write user metadata of successful jobs to a JSON file. + + Get user metadata of successful jobs in a task and write them in a json file. + + example: + >>> get_user_job_metadata(123, 'output.json') + """ core = _ensure_init() core.getUserJobMetadata(task_id, output_file) @@ -621,7 +686,13 @@ def get_user_job_metadata( def reload_input( task_id: Annotated[int, typer.Argument(help="Task ID")], ) -> None: - """Reload input dataset and retry the task with new contents.""" + """Reload input dataset and retry the task with new contents. + + This is useful when input dataset contents are changed after the task is submitted. + + example: + >>> reload_input(123) + """ core = _ensure_init() core.reload_input(task_id) @@ -631,7 +702,14 @@ def recover_lost_files( task_id: Annotated[int, typer.Argument(help="Task ID")], test_mode: Annotated[bool, typer.Option("--test-mode", help="Dry-run mode")] = False, ) -> None: - """Request recovery of lost files from a task.""" + """Request recovery of lost files from a task. + + Send a request to recover lost files produced by a task. Set test_mode=True for testing. + + example: + >>> recover_lost_files(123) + >>> recover_lost_files(123, test_mode=True) + """ core = _ensure_init() core.recover_lost_files(task_id, test_mode) @@ -640,7 +718,13 @@ def recover_lost_files( def show_workflow( request_id: Annotated[int, typer.Argument(help="Workflow request ID")], ) -> None: - """Show workflow status.""" + """Show workflow status. + + Send a request to show the status of a workflow. + + example: + >>> show_workflow(456) + """ core = _ensure_init() _, output = core.execute_workflow_command("get_status", request_id) if output: @@ -651,7 +735,13 @@ def show_workflow( def kill_workflow( request_id: Annotated[int, typer.Argument(help="Workflow request ID")], ) -> None: - """Kill a workflow.""" + """Kill a workflow. + + Send a request to kill a workflow. + + example: + >>> kill_workflow(456) + """ core = _ensure_init() _, output = core.execute_workflow_command("abort", request_id) if output: @@ -662,7 +752,13 @@ def kill_workflow( def retry_workflow( request_id: Annotated[int, typer.Argument(help="Workflow request ID")], ) -> None: - """Retry a workflow.""" + """Retry a workflow. + + Send a request to retry a workflow. + + example: + >>> retry_workflow(456) + """ core = _ensure_init() _, output = core.execute_workflow_command("retry", request_id) if output: @@ -673,7 +769,13 @@ def retry_workflow( def finish_workflow( request_id: Annotated[int, typer.Argument(help="Workflow request ID")], ) -> None: - """Finish a workflow.""" + """Finish a workflow. + + Send a request to finish a workflow. + + example: + >>> finish_workflow(456) + """ core = _ensure_init() _, output = core.execute_workflow_command("finish", request_id) if output: @@ -684,7 +786,13 @@ def finish_workflow( def pause_workflow( request_id: Annotated[int, typer.Argument(help="Workflow request ID")], ) -> None: - """Pause a workflow.""" + """Pause a workflow. + + Send a request to pause a workflow. + + example: + >>> pause_workflow(456) + """ core = _ensure_init() _, output = core.execute_workflow_command("suspend", request_id) if output: @@ -695,7 +803,13 @@ def pause_workflow( def resume_workflow( request_id: Annotated[int, typer.Argument(help="Workflow request ID")], ) -> None: - """Resume a workflow.""" + """Resume a workflow. + + Send a request to resume a workflow. + + example: + >>> resume_workflow(456) + """ core = _ensure_init() _, output = core.execute_workflow_command("resume", request_id) if output: @@ -708,7 +822,15 @@ def set_secret( value: Annotated[str, typer.Argument(help="Secret value or file path")], is_file: Annotated[bool, typer.Option("--is-file", help="Treat value as a file path to upload")] = False, ) -> None: - """Set a secret key-value pair.""" + """Set a secret key-value pair. + + Define a pair of secret key-value strings. The value can be a file path to upload a + secret file when is_file=True. + + example: + >>> set_secret('mykey', 'myvalue') + >>> set_secret('mykey', '/path/to/file', is_file=True) + """ core = _ensure_init() core.set_secret(key, value, is_file) @@ -717,7 +839,11 @@ def set_secret( def delete_secret( key: Annotated[str, typer.Argument(help="Secret key to delete")], ) -> None: - """Delete a secret.""" + """Delete a secret. + + example: + >>> delete_secret('mykey') + """ core = _ensure_init() core.set_secret(key, None) @@ -733,7 +859,14 @@ def delete_all_secrets() -> None: def list_secrets( full: Annotated[bool, typer.Option("--full", help="Show full secret values")] = False, ) -> None: - """List secrets.""" + """List secrets. + + Value strings are truncated by default. full=True to see entire strings. + + example: + >>> list_secrets() + >>> list_secrets(full=True) + """ core = _ensure_init() core.list_secrets(full) From 013c6f2a408e55b2dfb3b5f02bd3e597088313c3 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Fri, 31 Jul 2026 14:05:04 +0200 Subject: [PATCH 22/59] Swap column order in pbook help --- pandaclient/PBookTyper2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandaclient/PBookTyper2.py b/pandaclient/PBookTyper2.py index 850b47e2..0eb79fea 100644 --- a/pandaclient/PBookTyper2.py +++ b/pandaclient/PBookTyper2.py @@ -417,8 +417,8 @@ def help( table = Table(box=box.SIMPLE, show_header=True, header_style="bold magenta") table.add_column("Command", style="bold cyan", no_wrap=True) - table.add_column("Signature") table.add_column("Description") + table.add_column("Signature") for group, names in _HELP_GROUPS: table.add_section() @@ -427,8 +427,8 @@ def help( func = ns.get(name) if func is None: continue - sig = console.highlighter(_format_signature(func)) doc = (func.__doc__ or "").strip().splitlines()[0] if func.__doc__ else "" + sig = console.highlighter(_format_signature(func)) table.add_row(f" {name}", sig, _esc(doc)) console.print(table) From 4f3050b8dc7d58c743e0f8f6d0e6565f69f43d83 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Fri, 31 Jul 2026 14:08:54 +0200 Subject: [PATCH 23/59] Swap column order in pbook help --- pandaclient/PBookTyper2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandaclient/PBookTyper2.py b/pandaclient/PBookTyper2.py index 0eb79fea..a6393648 100644 --- a/pandaclient/PBookTyper2.py +++ b/pandaclient/PBookTyper2.py @@ -429,7 +429,7 @@ def help( continue doc = (func.__doc__ or "").strip().splitlines()[0] if func.__doc__ else "" sig = console.highlighter(_format_signature(func)) - table.add_row(f" {name}", sig, _esc(doc)) + table.add_row(f" {name}", _esc(doc), sig) console.print(table) console.print("Usage: [bold]help(show)[/bold] or [bold]pbook show --help[/bold]\n") From 03e9c781e592fdc8ab62e9168235de92e2f5ce02 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Fri, 31 Jul 2026 15:32:07 +0200 Subject: [PATCH 24/59] Extended basic help text --- pandaclient/PBookTyper2.py | 46 +++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/pandaclient/PBookTyper2.py b/pandaclient/PBookTyper2.py index a6393648..b7d6f527 100644 --- a/pandaclient/PBookTyper2.py +++ b/pandaclient/PBookTyper2.py @@ -51,9 +51,53 @@ _core = None _core_inited: bool = False +help_text = """ +$ pbook [options] # interactive mode +$ pbook [options] command [args] [kwargs] # batch mode + +The same command can be executed in interactive mode: + +$ pbook +>>> command(*args, **kwargs) + +or in batch mode: + +$ pbook command arg1 arg2 ... argN kwarg1=value1 kwarg2=value2 ... kwargN=valueN + +E.g. + +$ pbook +>>> show(123, format='long') + +is equivalent to + +$ pbook show 123 format='long' + +If arg or value is a list in interactive mode, it is represented as a comma-separate list in batch mode. E.g. +to kill three tasks in interactive mode: + +$ pbook +>>> kill([123, 456, 789]) + +or in batch mode: + +$ pbook kill 123,456,789 + +To see the list of commands and help of each command, + +$ pbook +>>> help() +>>> help(command_name) + +or + +$ pbook help +$ pbook help command_name +""" + app = typer.Typer( name="pbook", - help="PanDA task bookkeeper. Run without arguments for interactive mode.", + help=help_text, invoke_without_command=True, no_args_is_help=False, ) From 00c7beacbf5af53db5e77a7d6161c16e8422f574 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Fri, 31 Jul 2026 15:40:59 +0200 Subject: [PATCH 25/59] -h was not working, changing in typer context settings --- pandaclient/PBookTyper2.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pandaclient/PBookTyper2.py b/pandaclient/PBookTyper2.py index b7d6f527..6fdce08b 100644 --- a/pandaclient/PBookTyper2.py +++ b/pandaclient/PBookTyper2.py @@ -52,6 +52,8 @@ _core_inited: bool = False help_text = """ +PanDA task bookkeeper. Run without arguments for interactive mode. + $ pbook [options] # interactive mode $ pbook [options] command [args] [kwargs] # batch mode @@ -100,6 +102,7 @@ help=help_text, invoke_without_command=True, no_args_is_help=False, + context_settings={"help_option_names": ["-h", "--help"]}, ) # ─── Utilities ──────────────────────────────────────────────────────────────── From 7e68a4a82bdd3a0db20a424a55cb8edae5c9ab2d Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Fri, 31 Jul 2026 15:48:44 +0200 Subject: [PATCH 26/59] added back -3 option --- pandaclient/PBookTyper2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandaclient/PBookTyper2.py b/pandaclient/PBookTyper2.py index 6fdce08b..017fd0c8 100644 --- a/pandaclient/PBookTyper2.py +++ b/pandaclient/PBookTyper2.py @@ -339,6 +339,7 @@ def _main( dev_srv: bool = typer.Option(False, "--devSrv", hidden=True), intr_srv: bool = typer.Option(False, "--intrSrv", hidden=True), prompt_with_newline: bool = typer.Option(False, "--prompt_with_newline", hidden=True), + python3: bool = typer.Option(False, "-3", hidden=True), ) -> None: """PanDA task bookkeeper. Run without arguments for interactive mode.""" if version: From d7c5b3c8b6686c98bba3a5851e06bd4dc4a40233 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Fri, 31 Jul 2026 16:08:22 +0200 Subject: [PATCH 27/59] Corrected how to call the CLI help --- pandaclient/PBookTyper2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandaclient/PBookTyper2.py b/pandaclient/PBookTyper2.py index 017fd0c8..27cfd956 100644 --- a/pandaclient/PBookTyper2.py +++ b/pandaclient/PBookTyper2.py @@ -94,7 +94,7 @@ or $ pbook help -$ pbook help command_name +$ pbook command_name --help """ app = typer.Typer( From add507329fcec6767ed6c90b34813866253987c0 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Fri, 31 Jul 2026 17:04:21 +0200 Subject: [PATCH 28/59] Trying to improve the tab-completion --- pandaclient/PBookTyper2.py | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/pandaclient/PBookTyper2.py b/pandaclient/PBookTyper2.py index 27cfd956..07597dda 100644 --- a/pandaclient/PBookTyper2.py +++ b/pandaclient/PBookTyper2.py @@ -229,15 +229,20 @@ def _kwarg_names(func) -> list: return list(signature(func).parameters) -def _kwarg_choices(func, name: str) -> list: - """Value choices for a parameter, derived from its type hint: Literal[...] members or True/False for bool.""" +def _kwarg_choices(func, name: str) -> tuple: + """Value choices for a parameter, derived from its type hint. + + Returns (values, is_str): values are Literal[...] members (as str) or True/False for + bool; is_str tells the completer whether these need to be quoted as Python string + literals (True/False must stay bare, unquoted keywords). + """ try: hints = get_type_hints(func, include_extras=True) except Exception: - return [] + return [], False ann = hints.get(name) if ann is None: - return [] + return [], False while hasattr(ann, "__metadata__"): ann = ann.__origin__ if get_origin(ann) is Union: @@ -245,10 +250,11 @@ def _kwarg_choices(func, name: str) -> list: if len(non_none) == 1: ann = non_none[0] if get_origin(ann) is Literal: - return [str(v) for v in get_args(ann)] + args = get_args(ann) + return [str(v) for v in args], all(isinstance(v, str) for v in args) if ann is bool: - return ["True", "False"] - return [] + return ["True", "False"], False + return [], False class _PBookCompleter: @@ -268,15 +274,23 @@ def _compute(self, text: str) -> list: line = readline.get_line_buffer() # Kwarg value completion (tier 1): last token is kwarg= or kwarg='partial + # readline's default delimiters include =, ', " - so the "word" it will replace + # already excludes any quote the user typed; we just need to add whichever quote(s) + # are still missing so the result reads as a valid quoted string either way. m_val = re.search(r"\b(\w+)\s*=\s*(['\"]?)(\w*)$", line) m_func = re.match(r"(\w+)\s*\(", line) if m_val and m_func: - kwarg, partial = m_val.group(1), m_val.group(3) + kwarg, quote, partial = m_val.group(1), m_val.group(2), m_val.group(3) func = self._ns.get(m_func.group(1)) if func is not None: - hits = [v for v in _kwarg_choices(func, kwarg) if v.startswith(partial)] - if hits: - return hits + values, is_str = _kwarg_choices(func, kwarg) + matches = [v for v in values if v.startswith(partial)] + if matches: + if not is_str: + return matches + if quote: + return [f"{v}{quote}" for v in matches] + return [f"'{v}'" for v in matches] # Kwarg name completion (tier 2): cursor is inside an open call m = re.search(r"(\w+)\s*\([^)]*$", line) From c171ce971a86e87b8828b1dfe882f8c6c3b1eb05 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Fri, 31 Jul 2026 17:09:23 +0200 Subject: [PATCH 29/59] Trying to improve the tab-completion --- pandaclient/PBookTyper2.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pandaclient/PBookTyper2.py b/pandaclient/PBookTyper2.py index 07597dda..b8e14ebc 100644 --- a/pandaclient/PBookTyper2.py +++ b/pandaclient/PBookTyper2.py @@ -277,7 +277,11 @@ def _compute(self, text: str) -> list: # readline's default delimiters include =, ', " - so the "word" it will replace # already excludes any quote the user typed; we just need to add whichever quote(s) # are still missing so the result reads as a valid quoted string either way. - m_val = re.search(r"\b(\w+)\s*=\s*(['\"]?)(\w*)$", line) + # The quote group is '*' (not '?') because readline's own ambiguous-completion + # common-prefix insertion can leave a stray quote in the buffer before the user + # types their own - tolerate any number of leading quote characters rather than + # silently failing to match and falling through to the wrong tier. + m_val = re.search(r"\b(\w+)\s*=\s*(['\"]*)(\w*)$", line) m_func = re.match(r"(\w+)\s*\(", line) if m_val and m_func: kwarg, quote, partial = m_val.group(1), m_val.group(2), m_val.group(3) @@ -289,7 +293,7 @@ def _compute(self, text: str) -> list: if not is_str: return matches if quote: - return [f"{v}{quote}" for v in matches] + return [f"{v}{quote[-1]}" for v in matches] return [f"'{v}'" for v in matches] # Kwarg name completion (tier 2): cursor is inside an open call From 5dc56da920552328c413677db89fc9071abf5d9a Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Fri, 31 Jul 2026 17:20:56 +0200 Subject: [PATCH 30/59] Trying to improve the tab-completion --- pandaclient/PBookTyper2.py | 44 ++++++++++++++------------------------ 1 file changed, 16 insertions(+), 28 deletions(-) diff --git a/pandaclient/PBookTyper2.py b/pandaclient/PBookTyper2.py index b8e14ebc..d7bb33f7 100644 --- a/pandaclient/PBookTyper2.py +++ b/pandaclient/PBookTyper2.py @@ -229,20 +229,19 @@ def _kwarg_names(func) -> list: return list(signature(func).parameters) -def _kwarg_choices(func, name: str) -> tuple: - """Value choices for a parameter, derived from its type hint. +def _kwarg_choices(func, name: str) -> list: + """Value choices for a parameter, derived from its type hint: Literal[...] members or True/False for bool. - Returns (values, is_str): values are Literal[...] members (as str) or True/False for - bool; is_str tells the completer whether these need to be quoted as Python string - literals (True/False must stay bare, unquoted keywords). + Returned bare (unquoted) - readline's own quote-matching auto-closes an opening quote + the user already typed, so we don't need to (and shouldn't try to) add one ourselves. """ try: hints = get_type_hints(func, include_extras=True) except Exception: - return [], False + return [] ann = hints.get(name) if ann is None: - return [], False + return [] while hasattr(ann, "__metadata__"): ann = ann.__origin__ if get_origin(ann) is Union: @@ -250,11 +249,10 @@ def _kwarg_choices(func, name: str) -> tuple: if len(non_none) == 1: ann = non_none[0] if get_origin(ann) is Literal: - args = get_args(ann) - return [str(v) for v in args], all(isinstance(v, str) for v in args) + return [str(v) for v in get_args(ann)] if ann is bool: - return ["True", "False"], False - return [], False + return ["True", "False"] + return [] class _PBookCompleter: @@ -274,27 +272,17 @@ def _compute(self, text: str) -> list: line = readline.get_line_buffer() # Kwarg value completion (tier 1): last token is kwarg= or kwarg='partial - # readline's default delimiters include =, ', " - so the "word" it will replace - # already excludes any quote the user typed; we just need to add whichever quote(s) - # are still missing so the result reads as a valid quoted string either way. - # The quote group is '*' (not '?') because readline's own ambiguous-completion - # common-prefix insertion can leave a stray quote in the buffer before the user - # types their own - tolerate any number of leading quote characters rather than - # silently failing to match and falling through to the wrong tier. - m_val = re.search(r"\b(\w+)\s*=\s*(['\"]*)(\w*)$", line) + # Return bare values - readline's own quote-matching auto-closes an opening quote + # the user already typed, so we deliberately don't add quotes ourselves here. + m_val = re.search(r"\b(\w+)\s*=\s*(['\"]?)(\w*)$", line) m_func = re.match(r"(\w+)\s*\(", line) if m_val and m_func: - kwarg, quote, partial = m_val.group(1), m_val.group(2), m_val.group(3) + kwarg, partial = m_val.group(1), m_val.group(3) func = self._ns.get(m_func.group(1)) if func is not None: - values, is_str = _kwarg_choices(func, kwarg) - matches = [v for v in values if v.startswith(partial)] - if matches: - if not is_str: - return matches - if quote: - return [f"{v}{quote[-1]}" for v in matches] - return [f"'{v}'" for v in matches] + hits = [v for v in _kwarg_choices(func, kwarg) if v.startswith(partial)] + if hits: + return hits # Kwarg name completion (tier 2): cursor is inside an open call m = re.search(r"(\w+)\s*\([^)]*$", line) From db4d569d6f307f5748f574af9a4651a648e17138 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Fri, 31 Jul 2026 17:24:26 +0200 Subject: [PATCH 31/59] Trying to improve the tab-completion --- pandaclient/PBookTyper2.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pandaclient/PBookTyper2.py b/pandaclient/PBookTyper2.py index d7bb33f7..47e164e6 100644 --- a/pandaclient/PBookTyper2.py +++ b/pandaclient/PBookTyper2.py @@ -280,9 +280,11 @@ def _compute(self, text: str) -> list: kwarg, partial = m_val.group(1), m_val.group(3) func = self._ns.get(m_func.group(1)) if func is not None: - hits = [v for v in _kwarg_choices(func, kwarg) if v.startswith(partial)] - if hits: - return hits + # We're unambiguously past a `kwarg=` - this is a value position, not a + # name position, even if this particular kwarg has no enumerable choices + # (e.g. limit: int). Return here regardless, so an empty result doesn't + # fall through to tier 2's kwarg-name completion. + return [v for v in _kwarg_choices(func, kwarg) if v.startswith(partial)] # Kwarg name completion (tier 2): cursor is inside an open call m = re.search(r"(\w+)\s*\([^)]*$", line) From 64effe40db3f7752d658cfeeacefb2ae2bcba760 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Fri, 31 Jul 2026 17:59:34 +0200 Subject: [PATCH 32/59] Backwards compatibility for options and not requiring -- --- pandaclient/PBookTyper2.py | 58 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/pandaclient/PBookTyper2.py b/pandaclient/PBookTyper2.py index 47e164e6..fbdecf63 100644 --- a/pandaclient/PBookTyper2.py +++ b/pandaclient/PBookTyper2.py @@ -936,7 +936,65 @@ def generate_credential() -> None: # ─── Entry point ────────────────────────────────────────────────────────────── +# Global options that consume the following argv token as their own value, so the +# subcommand-token scan below can skip over both. +_GLOBAL_VALUE_OPTS = {"-c"} + + +def _rewrite_legacy_kwargs(argv: list) -> list: + """Rewrite legacy bare `key=value` batch args into `--key=value`. + + The pre-Typer pbook batch mode accepted `pbook show format=long`; Click only + recognizes `--format=long`. Find the subcommand, look up its real option names via + introspection (never a separately maintained list), and rewrite any later bare + `key=value` token whose key matches one of them. Anything else - already-dashed + flags, positional args that happen to contain "=" - passes through untouched. + """ + i = 0 + while i < len(argv): + tok = argv[i] + if tok in _GLOBAL_VALUE_OPTS: + i += 2 + continue + if tok.startswith("-"): + i += 1 + continue + break + if i >= len(argv): + return argv + + sub_cmd = typer.main.get_command(app).commands.get(argv[i]) + if sub_cmd is None: + return argv + + option_flags = {} + flag_only = set() + for param in sub_cmd.params: + flags = [o for o in getattr(param, "opts", []) if o.startswith("--")] + if flags: + option_flags[param.name] = flags[0] + if getattr(param, "is_flag", False): + flag_only.add(param.name) + + rewritten = argv[: i + 1] + for tok in argv[i + 1 :]: + key, sep, value = tok.partition("=") + if sep and not tok.startswith("-") and key in option_flags: + flag = option_flags[key] + if key in flag_only: + # Click flag-style options (e.g. --soft) take no value at all; the legacy + # syntax passed an explicit True/False, so translate that into presence + # (truthy) or absence (falsy - same as the option's own default) instead. + if value.strip().lower() in ("true", "1", "yes"): + rewritten.append(flag) + continue + rewritten.append(f"{flag}={value}") + else: + rewritten.append(tok) + return rewritten + def main() -> None: sys.argv[0] = "pbook" + sys.argv[1:] = _rewrite_legacy_kwargs(sys.argv[1:]) app() From c4f91d0bac30792811eba4eb0089f1d72071271c Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Mon, 3 Aug 2026 13:51:21 +0200 Subject: [PATCH 33/59] Help text --- pandaclient/PBookTyper2.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pandaclient/PBookTyper2.py b/pandaclient/PBookTyper2.py index fbdecf63..c63f06d9 100644 --- a/pandaclient/PBookTyper2.py +++ b/pandaclient/PBookTyper2.py @@ -64,7 +64,9 @@ or in batch mode: +$ pbook command arg1 arg2 ... argN --kwarg1=value1 --kwarg2=value2 ... --kwargN=valueN $ pbook command arg1 arg2 ... argN kwarg1=value1 kwarg2=value2 ... kwargN=valueN +Please note that the latter option is kept for backward compatibility, but we plan to drop it in the future. E.g. @@ -73,6 +75,7 @@ is equivalent to +$ pbook show 123 --format='long' $ pbook show 123 format='long' If arg or value is a list in interactive mode, it is represented as a comma-separate list in batch mode. E.g. From d36ff15bff2ea065e05e46cd4dab31b919687f6e Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Mon, 3 Aug 2026 13:52:48 +0200 Subject: [PATCH 34/59] Reshuffled PBookTyper.py versions --- pandaclient/PBookTyper.py | 1172 +++++++++++++++++++----------------- pandaclient/PBookTyper2.py | 1003 ------------------------------ scripts/pbook | 2 +- 3 files changed, 626 insertions(+), 1551 deletions(-) delete mode 100644 pandaclient/PBookTyper2.py diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index 28ff41d5..c63f06d9 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -1,5 +1,13 @@ """ -pbook CLI — PanDA task bookkeeper with typer-based shell autocompletion. +pbook CLI — PanDA task bookkeeper. + +Each command below is defined exactly once, as a Typer command using the +Annotated[...] parameter style. Because the Typer/Click metadata lives in the +annotation rather than in the default value, these functions remain ordinary +callables with ordinary defaults - the same function is used to build the +`pbook --flag ...` CLI (with real shell completion) *and* is placed +directly into the interactive REPL namespace (`>>> command(...)`), with no +separate REPL-only copy of the command's logic, docstring, or option list. """ from __future__ import annotations @@ -14,12 +22,21 @@ import sys import tempfile from concurrent.futures import ThreadPoolExecutor -from inspect import signature -from typing import Optional +from inspect import Parameter, signature +from typing import ( + Annotated, + Literal, + Optional, + Union, + get_args, + get_origin, + get_type_hints, +) import typer from rich import box from rich.console import Console +from rich.markup import escape as _esc from rich.table import Table from pandaclient import Client, PandaToolsPkgInfo @@ -31,25 +48,78 @@ _fork_child_pid: Optional[int] = None _setup_done: bool = False _ctx_state: dict = {} +_core = None +_core_inited: bool = False + +help_text = """ +PanDA task bookkeeper. Run without arguments for interactive mode. + +$ pbook [options] # interactive mode +$ pbook [options] command [args] [kwargs] # batch mode + +The same command can be executed in interactive mode: + +$ pbook +>>> command(*args, **kwargs) + +or in batch mode: + +$ pbook command arg1 arg2 ... argN --kwarg1=value1 --kwarg2=value2 ... --kwargN=valueN +$ pbook command arg1 arg2 ... argN kwarg1=value1 kwarg2=value2 ... kwargN=valueN +Please note that the latter option is kept for backward compatibility, but we plan to drop it in the future. + +E.g. + +$ pbook +>>> show(123, format='long') + +is equivalent to + +$ pbook show 123 --format='long' +$ pbook show 123 format='long' + +If arg or value is a list in interactive mode, it is represented as a comma-separate list in batch mode. E.g. +to kill three tasks in interactive mode: + +$ pbook +>>> kill([123, 456, 789]) + +or in batch mode: + +$ pbook kill 123,456,789 + +To see the list of commands and help of each command, + +$ pbook +>>> help() +>>> help(command_name) + +or + +$ pbook help +$ pbook command_name --help +""" app = typer.Typer( name="pbook", - help="PanDA task bookkeeper. Run without arguments for interactive mode.", + help=help_text, invoke_without_command=True, no_args_is_help=False, + context_settings={"help_option_names": ["-h", "--help"]}, ) # ─── Utilities ──────────────────────────────────────────────────────────────── def _parallel(func, items): - # Parallel execution in a thread pool of 8 threads, for example when the user wants to act on a list of task IDs with ThreadPoolExecutor(8) as pool: return list(pool.map(func, items)) -def _parse_ids(raw: str): - """'all' → str, '42' → int, '1,2,3' → [int,...].""" +def _parse_ids(raw): + """'all' -> 'all'; '42' -> 42; '1,2,3' -> [1,2,3]; anything not a string passes through as-is.""" + if not isinstance(raw, str): + return raw if raw == "all": return "all" parts = raw.split(",") @@ -119,8 +189,28 @@ def _make_core(verbose: bool = False): def _get_core(): + """Return the (memoized) core for this process, uninitialized.""" + global _core _setup() - return _make_core(_ctx_state.get("verbose", False)) + if _core is None: + _core = _make_core(_ctx_state.get("verbose", False)) + return _core + + +def _ensure_init(sanity_check: bool = False): + """Return the core, running PBookCore.init() exactly once per process. + + The REPL calls this once upfront with sanity_check=True; every command + function then calls it again with the default before touching the core, + which is a no-op once already initialized - so commands stay correct + whether they run once (batch mode) or repeatedly (REPL session). + """ + global _core_inited + core = _get_core() + if not _core_inited: + core.init(sanity_check=sanity_check) + _core_inited = True + return core def _catch_sig(sig, frame): @@ -129,146 +219,86 @@ def _catch_sig(sig, frame): commands_get_output(f"kill -9 -- -{os.getpgrp()}") -# ─── Interactive REPL namespace ─────────────────────────────────────────────── - -# ─── REPL kwarg completer ───────────────────────────────────────────────────── - -_FUNC_KWARGS: dict[str, list[str]] = { - "show": ["username", "limit", "taskname", "days", "jeditaskid", "reqid", "status", "superstatus", "format"], - "showl": ["username", "limit", "taskname", "days", "jeditaskid", "reqid", "status", "superstatus"], - "kill": [], - "finish": ["soft"], - "retry": [ - "newOpts", - "days", - "limit", - "site", - "excludedSite", - "includedSite", - "nFilesPerJob", - "nMaxFilesPerJob", - "nGBPerJob", - "nFiles", - "nEvents", - "loopingCheck", - "memory", - "avoidVP", - "ignoreMissingInDS", - "forceStaged", - "maxCore", - ], - "debug": ["modeOn"], - "get_user_job_metadata": [], - "recover_lost_files": ["test_mode"], - "set_secret": ["is_file"], - "list_secrets": ["full"], -} - -_KWARG_VALUES: dict[str, dict[str, list[str]]] = { - "show": { - "format": ["standard", "long", "json", "plain"], - }, - "finish": { - "soft": ["True", "False"], - }, - "debug": { - "modeOn": ["True", "False"], - }, - "recover_lost_files": { - "test_mode": ["True", "False"], - }, - "list_secrets": { - "full": ["True", "False"], - }, - "set_secret": { - "is_file": ["True", "False"], - }, - "retry": { - "loopingCheck": ["True", "False"], - "avoidVP": ["True", "False"], - "ignoreMissingInDS": ["True", "False"], - "forceStaged": ["True", "False"], - }, -} +# ─── REPL namespace & completion ────────────────────────────────────────────── -def _run_repl(ns: dict, banner: str) -> None: - """Manual REPL using InteractiveConsole.push() so we own readline setup entirely.""" - completer = _PBookCompleter(ns) - readline.set_completer(completer.complete) - readline.parse_and_bind("tab: complete") - readline.parse_and_bind("set show-all-if-ambiguous On") +def _build_namespace() -> dict: + """The REPL namespace: every registered Typer command, keyed by its real Python name.""" + return {info.callback.__name__: info.callback for info in app.registered_commands} - console = code.InteractiveConsole(ns) - print(banner) - more = False - while True: - prompt = "... " if more else ">>> " - try: - readline.set_completer(completer.complete) - line = input(prompt) - except EOFError: - print() - break - except KeyboardInterrupt: - print("\nKeyboardInterrupt") - console.resetbuffer() - more = False - continue - more = console.push(line) +def _kwarg_names(func) -> list: + """All parameter names a function accepts - candidates for `name=` completion.""" + return list(signature(func).parameters) + + +def _kwarg_choices(func, name: str) -> list: + """Value choices for a parameter, derived from its type hint: Literal[...] members or True/False for bool. + + Returned bare (unquoted) - readline's own quote-matching auto-closes an opening quote + the user already typed, so we don't need to (and shouldn't try to) add one ourselves. + """ + try: + hints = get_type_hints(func, include_extras=True) + except Exception: + return [] + ann = hints.get(name) + if ann is None: + return [] + while hasattr(ann, "__metadata__"): + ann = ann.__origin__ + if get_origin(ann) is Union: + non_none = [a for a in get_args(ann) if a is not type(None)] + if len(non_none) == 1: + ann = non_none[0] + if get_origin(ann) is Literal: + return [str(v) for v in get_args(ann)] + if ann is bool: + return ["True", "False"] + return [] class _PBookCompleter: """Readline completer: kwarg names and values when inside a call, names otherwise.""" def __init__(self, ns: dict) -> None: + self._ns = ns self._base = rlcompleter.Completer(ns) - self._matches: list[str] = [] + self._matches: list = [] def complete(self, text: str, state: int) -> Optional[str]: if state == 0: self._matches = self._compute(text) return self._matches[state] if state < len(self._matches) else None - def _compute(self, text: str) -> list[str]: + def _compute(self, text: str) -> list: line = readline.get_line_buffer() - # Kwarg value completion(tier 1, highest priority): last token is kwarg= or kwarg='partial - # >>> show(format='j│ - # Now the line matches both m_func (show() and m_val (line 234's regex: kwarg="format", quote="'", partial="j"). - # It looks up _KWARG_VALUES["show"]["format"] (line 165) = ["standard", "long", "json", "plain"], filters to those starting with j → json. - # This branch is checked before tier 2, so once you're past the =, you get value suggestions instead of more kwarg names. + # Kwarg value completion (tier 1): last token is kwarg= or kwarg='partial + # Return bare values - readline's own quote-matching auto-closes an opening quote + # the user already typed, so we deliberately don't add quotes ourselves here. m_val = re.search(r"\b(\w+)\s*=\s*(['\"]?)(\w*)$", line) m_func = re.match(r"(\w+)\s*\(", line) if m_val and m_func: - kwarg, quote, partial = m_val.group(1), m_val.group(2), m_val.group(3) - func_name = m_func.group(1) - values = _KWARG_VALUES.get(func_name, {}).get(kwarg, []) - hits = [v for v in values if v.startswith(partial)] - if hits: - return hits - - # Kwarg name completion: cursor is inside an open call - # >>> show(│ - # readline.get_line_buffer() returns "show(". The regex on line 245, (\w+)\s*\([^)]*$, matches with func_name = "show" and nothing after the (. - # So it looks up _FUNC_KWARGS["show"] (line 133) and lists username, limit, taskname, days, jeditaskid, reqid, status, superstatus, format — - # rlcompleter is never consulted here. - - # >>> show(form│ - # Same regex still matches (func_name = "show", and [^)]* swallows form), but we filter _FUNC_KWARGS["show"] down to names starting with - # form → just format. + kwarg, partial = m_val.group(1), m_val.group(3) + func = self._ns.get(m_func.group(1)) + if func is not None: + # We're unambiguously past a `kwarg=` - this is a value position, not a + # name position, even if this particular kwarg has no enumerable choices + # (e.g. limit: int). Return here regardless, so an empty result doesn't + # fall through to tier 2's kwarg-name completion. + return [v for v in _kwarg_choices(func, kwarg) if v.startswith(partial)] + + # Kwarg name completion (tier 2): cursor is inside an open call m = re.search(r"(\w+)\s*\([^)]*$", line) if m: - hits = [k for k in _FUNC_KWARGS.get(m.group(1), []) if k.startswith(text)] - if hits: - return hits - - # Plain name completion (tier 3, the rlcompleter fallback): standard name completion, stripping trailing '(' rlcompleter adds to callables - # >>> sho│ - # Not inside any (...), so tiers 1 and 2 don't match. Falls through to rlcompleter (line 257-260), which scans ns for names starting with - # sho → matches show, showl. Since these are callables, rlcompleter.complete() would normally return "show("/"showl("; the .rstrip("()").rstrip("(") - # on line 259 strips that back to plain show, showl. + func = self._ns.get(m.group(1)) + if func is not None: + hits = [k for k in _kwarg_names(func) if k.startswith(text)] + if hits: + return hits + + # Plain name completion (tier 3, rlcompleter fallback) if not text: # rlcompleter.complete() special-cases blank text by calling readline.insert_text() # itself, which re-enters readline from inside this callback and confuses the active @@ -281,244 +311,31 @@ def _compute(self, text: str) -> list[str]: return results -_RETRY_ALLOWED_OPTS = [ - "site", - "excludedSite", - "includedSite", - "nFilesPerJob", - "nMaxFilesPerJob", - "nGBPerJob", - "nFiles", - "nEvents", - "loopingCheck", - "maxNFilesPerJob", - "memory", - "ramCount", - "avoidVP", - "ignoreMissingInDS", - "forceStaged", - "maxCore", -] - - -def _build_namespace(core) -> dict: - _console = Console() +def _run_repl(ns: dict, banner: str) -> None: + """Manual REPL using InteractiveConsole.push() so we own readline setup entirely.""" + completer = _PBookCompleter(ns) + readline.set_completer(completer.complete) + readline.parse_and_bind("tab: complete") + readline.parse_and_bind("set show-all-if-ambiguous On") - def help(command=None): - """Show available commands, or detailed help for a specific command.""" - if command is not None: - name = command if isinstance(command, str) else command.__name__ - func = ns.get(name, command if callable(command) else None) - if func is None: - _console.print(f"[red]Unknown command:[/red] {name}") - return - sig = str(signature(func)).replace("(", f"[bold cyan]{name}[/bold cyan](", 1) - _console.print(f"\n[bold]{sig}[/bold]") - doc = (func.__doc__ or "No description.").strip() - _console.print(f"\n{doc}\n") - return + console = code.InteractiveConsole(ns) + print(banner) - # No argument - summary table - table = Table(box=box.SIMPLE, show_header=True, header_style="bold magenta") - table.add_column("Command", style="bold cyan", no_wrap=True) - table.add_column("Signature", style="dim", no_wrap=True) - table.add_column("Description") - - _GROUPS = [ - ("Tasks", ["show", "showl", "kill", "finish", "retry", "debug"]), - ("Files & input", ["get_user_job_metadata", "recover_lost_files", "reload_input"]), - ("Workflows", ["show_workflow", "kill_workflow", "retry_workflow", "finish_workflow", "pause_workflow", "resume_workflow"]), - ("Secrets", ["set_secret", "list_secrets", "delete_secret", "delete_all_secrets"]), - ("Auth", ["generate_credential"]), - ] - for group, names in _GROUPS: - table.add_section() - table.add_row(f"[bold white]{group}[/bold white]", "", "") - for name in names: - func = ns.get(name) - if func is None: - continue - sig = str(signature(func)) - doc = (func.__doc__ or "").strip().splitlines()[0] - table.add_row(f" {name}", sig, doc) - - _console.print(table) - _console.print("Usage: [bold]help(show)[/bold] or [bold]pbook show --help[/bold]\n") - - def show(taskID=None, *, username=None, limit=1000, taskname=None, days=14, jeditaskid=None, reqid=None, status=None, superstatus=None, format="standard"): - """Print task records. - - taskID: jediTaskID / reqID / 'run' (active) / 'fin' (terminated) / omit for all. - format: standard | long | json | plain - """ - kwargs = { - k: v - for k, v in dict( - username=username, - limit=limit, - taskname=taskname, - days=days, - jeditaskid=jeditaskid, - reqid=reqid, - status=status, - superstatus=superstatus, - format=format, - ).items() - if v is not None - } - kwargs.setdefault("limit", limit) - kwargs.setdefault("days", days) - kwargs["format"] = format - return core.show(taskID, **kwargs) if taskID is not None else core.show(**kwargs) - - def showl(taskID=None, *, username=None, limit=1000, taskname=None, days=14, jeditaskid=None, reqid=None, status=None, superstatus=None): - """Print task records in long format (shortcut for show(..., format='long')).""" - return show( - taskID, - username=username, - limit=limit, - taskname=taskname, - days=days, - jeditaskid=jeditaskid, - reqid=reqid, - status=status, - superstatus=superstatus, - format="long", - ) - - def kill(taskIDs): - """Kill tasks. taskIDs: int, list of ints, or 'all' for all active tasks.""" - if taskIDs == "all": - return _parallel(lambda t: core.kill(t.jeditaskid), core.get_active_tasks()) - elif isinstance(taskIDs, (list, tuple)): - return _parallel(core.kill, taskIDs) - elif isinstance(taskIDs, int): - return [core.kill(taskIDs)] - - print("Error: Invalid argument") - return None - - def finish(taskIDs, soft=False): - """Finish tasks. taskIDs: int, [int,...], or 'all' for all active tasks. soft=True waits for running jobs.""" - if taskIDs == "all": - return _parallel( - lambda t: core.finish.original_func(core, t.jeditaskid, soft=soft), - core.get_active_tasks(), - ) - elif isinstance(taskIDs, (list, tuple)): - return _parallel(lambda tid: core.finish(tid, soft=soft), taskIDs) - elif isinstance(taskIDs, int): - return [core.finish(taskIDs, soft=soft)] - - print("Error: Invalid argument") - return None - - def retry(taskIDs, newOpts=None, days=14, limit=1000, **kwargs): - """Retry failed/cancelled tasks. - - taskIDs (required): int, list of ints, or 'all'. - Allowed kwargs: site, excludedSite, includedSite, nFilesPerJob, nMaxFilesPerJob, - nGBPerJob, nFiles, nEvents, loopingCheck, memory, avoidVP, - ignoreMissingInDS, forceStaged, maxCore. - Example: retry('all', loopingCheck=True) - """ - if newOpts is None: - newOpts = dict(kwargs) - for key in list(newOpts): - if key == "memory": - newOpts["ramCount"] = newOpts.pop(key) - elif key == "maxCore": - newOpts["maxCoreCount"] = newOpts.pop(key) - elif key not in _RETRY_ALLOWED_OPTS: - print(f'Error: Unknown option "{key}"') - return None - opts = newOpts or None - if isinstance(taskIDs, (list, tuple)): - return _parallel(lambda tid: core.retry(tid, newOpts=opts), taskIDs) - elif isinstance(taskIDs, int): - return [core.retry(taskIDs, newOpts=opts)] - elif taskIDs == "all": - data = core.show(status="finished", days=days, limit=limit, format="json") - return _parallel(lambda d: core.retry.original_func(core, d["jediTaskID"], newOpts=opts), data) - - print("Error: Invalid argument") - return None - - def debug(PandaID, modeOn): - """Toggle debug mode for a subjob. modeOn: True/False.""" - core.debug(PandaID, modeOn) - - def get_user_job_metadata(taskID, outputFileName): - """Write user metadata of successful jobs to a JSON file.""" - core.getUserJobMetadata(taskID, outputFileName) - - def reload_input(task_id): - """Reload input dataset and retry with new contents.""" - core.reload_input(task_id) - - def recover_lost_files(taskID, test_mode=False): - """Request recovery of lost files from a task.""" - core.recover_lost_files(taskID, test_mode) - - def show_workflow(request_id): - """Show workflow status.""" - _, output = core.execute_workflow_command("get_status", request_id) - if output: - print(output) - - def kill_workflow(request_id): - """Kill a workflow.""" - _, output = core.execute_workflow_command("abort", request_id) - if output: - print(output[0][-1]) - - def retry_workflow(request_id): - """Retry a workflow.""" - _, output = core.execute_workflow_command("retry", request_id) - if output: - print(output[0][-1]) - - def finish_workflow(request_id): - """Finish a workflow.""" - _, output = core.execute_workflow_command("finish", request_id) - if output: - print(output[0][-1]) - - def pause_workflow(request_id): - """Pause a workflow.""" - _, output = core.execute_workflow_command("suspend", request_id) - if output: - print(output[0][-1]) - - def resume_workflow(request_id): - """Resume a workflow.""" - _, output = core.execute_workflow_command("resume", request_id) - if output: - print(output[0][-1]) - - def set_secret(key, value, is_file=False): - """Set a secret key-value pair. is_file=True to upload a file.""" - core.set_secret(key, value, is_file) - - def delete_secret(key): - """Delete a secret.""" - core.set_secret(key, None) - - def delete_all_secrets(): - """Delete all secrets.""" - core.set_secret(None, None) - - def list_secrets(full=False): - """List secrets. full=True to show full values.""" - core.list_secrets(full) - - def generate_credential(): - """Generate a new proxy or token.""" - core.generate_credential() - - # Generate the namespace with the local functions and exclude any imported functions or variables - ns = {k: v for k, v in locals().items() if callable(v) and getattr(v, "__module__", None) == __name__} - return ns + more = False + while True: + prompt = "... " if more else ">>> " + try: + readline.set_completer(completer.complete) + line = input(prompt) + except EOFError: + print() + break + except KeyboardInterrupt: + print("\nKeyboardInterrupt") + console.resetbuffer() + more = False + continue + more = console.push(line) # ─── Top-level callback ─────────────────────────────────────────────────────── @@ -533,13 +350,13 @@ def _main( dev_srv: bool = typer.Option(False, "--devSrv", hidden=True), intr_srv: bool = typer.Option(False, "--intrSrv", hidden=True), prompt_with_newline: bool = typer.Option(False, "--prompt_with_newline", hidden=True), + python3: bool = typer.Option(False, "-3", hidden=True), ) -> None: """PanDA task bookkeeper. Run without arguments for interactive mode.""" if version: typer.echo(f"Version: {PandaToolsPkgInfo.release_version}") raise typer.Exit() - # Set up the development or integration server if dev_srv: Client.useDevServer() if intr_srv: @@ -555,31 +372,25 @@ def _main( global _fork_child_pid _fork_child_pid = os.fork() - # Fork failed if _fork_child_pid == -1: typer.echo("ERROR: Failed to fork", err=True) raise typer.Exit(1) - # Child process if _fork_child_pid == 0: if verbose: typer.echo(str(ctx.params)) if prompt_with_newline: sys.ps1 = ">>> \n" - core = _make_core(verbose) - ns = _build_namespace(core) + _ensure_init(sanity_check=True) + ns = _build_namespace() - # The user wants to execute a Python code snippet instead of entering the REPL if command_string: - core.init() exec(command_string, {}, ns) # noqa: S102 from pandaclient import PBookCore as _PBC raise typer.Exit(0 if _PBC.func_return_value else 1) - core.init() _run_repl(ns, banner=f"\nStart pBook {PandaToolsPkgInfo.release_version}") - # Parent process else: signal.signal(signal.SIGINT, _catch_sig) signal.signal(signal.SIGHUP, _catch_sig) @@ -592,219 +403,388 @@ def _main( raise typer.Exit(0) -# ─── Subcommands ────────────────────────────────────────────────────────────── +# ─── Commands ────────────────────────────────────────────────────────────────── + +_HELP_GROUPS = [ + ("Tasks", ["show", "showl", "kill", "finish", "retry", "debug"]), + ("Files & input", ["get_user_job_metadata", "recover_lost_files", "reload_input"]), + ("Workflows", ["show_workflow", "kill_workflow", "retry_workflow", "finish_workflow", "pause_workflow", "resume_workflow"]), + ("Secrets", ["set_secret", "list_secrets", "delete_secret", "delete_all_secrets"]), + ("Auth", ["generate_credential"]), +] + + +def _type_name(ann) -> str: + """Render a resolved type annotation as a short, human-readable name (Optional[str], Literal[...], etc.).""" + if ann is None or ann is type(None): + return "" + origin = get_origin(ann) + if origin is Union: + args = get_args(ann) + non_none = [a for a in args if a is not type(None)] + if len(non_none) == 1 and len(args) == 2: + return f"Optional[{_type_name(non_none[0])}]" + return " | ".join(_type_name(a) for a in args) + if origin is Literal: + return "Literal[" + ", ".join(repr(v) for v in get_args(ann)) + "]" + if origin is not None: + args = get_args(ann) + origin_name = getattr(origin, "__name__", str(origin)) + return f"{origin_name}[{', '.join(_type_name(a) for a in args)}]" if args else origin_name + return getattr(ann, "__name__", str(ann)) + + +def _format_signature(func) -> str: + """A clean '(param: Type = default, ...)' string, stripping the Typer/Annotated plumbing.""" + try: + hints = get_type_hints(func, include_extras=True) + except Exception: + hints = {} + parts = [] + for pname, p in signature(func).parameters.items(): + ann = hints.get(pname) + while hasattr(ann, "__metadata__"): + ann = ann.__origin__ + piece = pname + type_str = _type_name(ann) + if type_str: + piece += f": {type_str}" + if p.default is not Parameter.empty: + piece += f" = {p.default!r}" + parts.append(piece) + return f"({', '.join(parts)})" + + +@app.command() +def help( + command: Annotated[Optional[str], typer.Argument(help="Command name for detailed help")] = None, +) -> None: + """Show available commands, or detailed help for a specific command.""" + ns = _build_namespace() + console = Console() + + if command is not None: + name = command if isinstance(command, str) else command.__name__ + func = ns.get(name, command if callable(command) else None) + if func is None: + console.print(f"[red]Unknown command:[/red] {_esc(name)}") + return + console.print(f"\n[bold cyan]{name}[/bold cyan][bold]{_esc(_format_signature(func))}[/bold]") + doc = (func.__doc__ or "No description.").strip() + console.print(f"\n{_esc(doc)}\n") + return + + table = Table(box=box.SIMPLE, show_header=True, header_style="bold magenta") + table.add_column("Command", style="bold cyan", no_wrap=True) + table.add_column("Description") + table.add_column("Signature") + + for group, names in _HELP_GROUPS: + table.add_section() + table.add_row(f"[bold white]{group}[/bold white]", "", "") + for name in names: + func = ns.get(name) + if func is None: + continue + doc = (func.__doc__ or "").strip().splitlines()[0] if func.__doc__ else "" + sig = console.highlighter(_format_signature(func)) + table.add_row(f" {name}", _esc(doc), sig) + + console.print(table) + console.print("Usage: [bold]help(show)[/bold] or [bold]pbook show --help[/bold]\n") @app.command() def show( - task_id: Optional[str] = typer.Argument(None, help="jediTaskID, reqID, 'run' (active only), or 'fin' (terminated only)"), - username: Optional[str] = typer.Option(None, "--username", help="Filter by username"), - limit: int = typer.Option(1000, "--limit", help="Maximum number of records"), - taskname: Optional[str] = typer.Option(None, "--taskname", help="Filter by task name"), - days: int = typer.Option(14, "--days", help="Look back N days (capped at 90 without a task ID)"), - jeditaskid: Optional[int] = typer.Option(None, "--jeditaskid", help="Filter by jediTaskID"), - reqid: Optional[int] = typer.Option(None, "--reqid", help="Filter by reqID"), - status: Optional[str] = typer.Option(None, "--status", help="Filter by task status"), - superstatus: Optional[str] = typer.Option(None, "--superstatus", help="Filter by super-status"), - output_format: str = typer.Option("standard", "--format", help="Output format: standard|long|json|plain"), + task_id: Annotated[Optional[str], typer.Argument(help="jediTaskID, reqID, 'run' (active only), or 'fin' (terminated only)")] = None, + username: Annotated[Optional[str], typer.Option(help="Filter by username")] = None, + limit: Annotated[int, typer.Option(help="Maximum number of records")] = 1000, + taskname: Annotated[Optional[str], typer.Option(help="Filter by task name")] = None, + days: Annotated[int, typer.Option(help="Look back N days (capped at 90 without a task ID)")] = 14, + jeditaskid: Annotated[Optional[int], typer.Option(help="Filter by jediTaskID")] = None, + reqid: Annotated[Optional[int], typer.Option(help="Filter by reqID")] = None, + status: Annotated[Optional[str], typer.Option(help="Filter by task status")] = None, + superstatus: Annotated[Optional[str], typer.Option(help="Filter by super-status")] = None, + format: Annotated[Literal["standard", "long", "json", "plain"], typer.Option("--format", help="Output format")] = "standard", ) -> None: - """Print task records.""" - core = _get_core() - core.init(sanity_check=False) - kwargs: dict = {"limit": limit, "days": days, "format": output_format} - for k, v in [ - ("username", username), - ("taskname", taskname), - ("jeditaskid", jeditaskid), - ("reqid", reqid), - ("status", status), - ("superstatus", superstatus), - ]: - if v is not None: - kwargs[k] = v + """Print task records. + + The first argument (task_id) can be a jediTaskID or reqID, or 'run' (show active tasks + only), or 'fin' (show terminated tasks only), or can be omitted. Records are fetched + directly from the PanDA server, so they are always up to date. Note that days is capped + at 90 days unless a jediTaskID or reqID is specified, in which case tasks of any age are + returned. The default filter conditions are: username=(name from user voms proxy), + limit=1000, days=14, format='standard'. + + example: + >>> show() + >>> show(123) + >>> show(12345678, format='long') + >>> show(taskname='my_task_name') + >>> show('run') + >>> show('fin', days=7, limit=100) + >>> show(format='json') + """ + core = _ensure_init() + kwargs = { + k: v + for k, v in dict( + username=username, + limit=limit, + taskname=taskname, + days=days, + jeditaskid=jeditaskid, + reqid=reqid, + status=status, + superstatus=superstatus, + ).items() + if v is not None + } + kwargs["format"] = format if task_id is not None: try: first_arg = int(task_id) - except ValueError: + except (TypeError, ValueError): first_arg = task_id - core.show(first_arg, **kwargs) - else: - core.show(**kwargs) + return core.show(first_arg, **kwargs) + return core.show(**kwargs) @app.command() def showl( - task_id: Optional[str] = typer.Argument(None, help="jediTaskID, reqID, 'run', or 'fin'"), - username: Optional[str] = typer.Option(None, "--username"), - limit: int = typer.Option(1000, "--limit"), - taskname: Optional[str] = typer.Option(None, "--taskname"), - days: int = typer.Option(14, "--days"), - jeditaskid: Optional[int] = typer.Option(None, "--jeditaskid"), - reqid: Optional[int] = typer.Option(None, "--reqid"), - status: Optional[str] = typer.Option(None, "--status"), - superstatus: Optional[str] = typer.Option(None, "--superstatus"), + task_id: Annotated[Optional[str], typer.Argument(help="jediTaskID, reqID, 'run', or 'fin'")] = None, + username: Annotated[Optional[str], typer.Option(help="Filter by username")] = None, + limit: Annotated[int, typer.Option(help="Maximum number of records")] = 1000, + taskname: Annotated[Optional[str], typer.Option(help="Filter by task name")] = None, + days: Annotated[int, typer.Option(help="Look back N days (capped at 90 without a task ID)")] = 14, + jeditaskid: Annotated[Optional[int], typer.Option(help="Filter by jediTaskID")] = None, + reqid: Annotated[Optional[int], typer.Option(help="Filter by reqID")] = None, + status: Annotated[Optional[str], typer.Option(help="Filter by task status")] = None, + superstatus: Annotated[Optional[str], typer.Option(help="Filter by super-status")] = None, ) -> None: - """Print task records in long format (shortcut for show --format long).""" - core = _get_core() - core.init(sanity_check=False) - kwargs: dict = {"limit": limit, "days": days, "format": "long"} - for k, v in [ - ("username", username), - ("taskname", taskname), - ("jeditaskid", jeditaskid), - ("reqid", reqid), - ("status", status), - ("superstatus", superstatus), - ]: - if v is not None: - kwargs[k] = v - if task_id is not None: - try: - first_arg = int(task_id) - except ValueError: - first_arg = task_id - core.show(first_arg, **kwargs) - else: - core.show(**kwargs) + """Print task records in long format (shortcut for show --format long). + + See help(show) for the available filter keywords. + + example: + >>> showl() + >>> showl(123) + >>> showl(12345678) + >>> showl(taskname='my_task_name') + """ + return show( + task_id, + username=username, + limit=limit, + taskname=taskname, + days=days, + jeditaskid=jeditaskid, + reqid=reqid, + status=status, + superstatus=superstatus, + format="long", + ) @app.command() def kill( - task_ids: str = typer.Argument(..., help="Task ID, comma-separated IDs, or 'all'"), + task_ids: Annotated[str, typer.Argument(help="Task ID, comma-separated IDs, or 'all'")], ) -> None: - """Kill tasks.""" - core = _get_core() - core.init(sanity_check=False) + """Kill tasks. + + Kill all subJobs in task_ids (ID or a list of IDs, can be either jediTaskID or reqID). + If 'all', kill all active tasks of the user. + + example: + >>> kill(123) + >>> kill([123, 345, 567]) + >>> kill('all') + """ + core = _ensure_init() ids = _parse_ids(task_ids) if ids == "all": - _parallel(lambda t: core.kill(t.jeditaskid), core.get_active_tasks()) + return _parallel(lambda t: core.kill(t.jeditaskid), core.get_active_tasks()) elif isinstance(ids, list): - _parallel(core.kill, ids) - else: - core.kill(ids) + return _parallel(core.kill, ids) + return core.kill(ids) @app.command() def finish( - task_ids: str = typer.Argument(..., help="Task ID, comma-separated IDs, or 'all'"), - soft: bool = typer.Option(False, "--soft", help="Wait for running jobs to finish instead of killing them"), + task_ids: Annotated[str, typer.Argument(help="Task ID, comma-separated IDs, or 'all'")], + soft: Annotated[bool, typer.Option("--soft", help="Wait for running jobs to finish instead of killing them")] = False, ) -> None: - """Finish tasks.""" - core = _get_core() - core.init(sanity_check=False) + """Finish tasks. + + Finish all subJobs in task_ids (ID or a list of IDs, can be either jediTaskID or reqID). + If task_ids is 'all', finish all active tasks of the user. If soft is False (default), + all running jobs are killed and the task finishes immediately. If soft is True, new jobs + are not generated and the task finishes once all running jobs finish. + + example: + >>> finish(123) + >>> finish(234, soft=True) + >>> finish([123, 345, 567]) + >>> finish('all') + """ + core = _ensure_init() ids = _parse_ids(task_ids) if ids == "all": - _parallel(lambda t: core.finish.original_func(core, t.jeditaskid, soft=soft), core.get_active_tasks()) + return _parallel(lambda t: core.finish.original_func(core, t.jeditaskid, soft=soft), core.get_active_tasks()) elif isinstance(ids, list): - _parallel(lambda tid: core.finish(tid, soft=soft), ids) - else: - core.finish(ids, soft=soft) + return _parallel(lambda tid: core.finish(tid, soft=soft), ids) + return core.finish(ids, soft=soft) @app.command() def retry( - task_ids: str = typer.Argument(..., help="Task ID, comma-separated IDs, or 'all'"), - days: int = typer.Option(14, "--days", help="Look-back window when task_ids='all'"), - limit: int = typer.Option(1000, "--limit", help="Max tasks to retry when task_ids='all'"), - site: Optional[str] = typer.Option(None, "--site"), - excluded_site: Optional[str] = typer.Option(None, "--excludedSite"), - included_site: Optional[str] = typer.Option(None, "--includedSite"), - n_files_per_job: Optional[int] = typer.Option(None, "--nFilesPerJob"), - n_max_files_per_job: Optional[int] = typer.Option(None, "--nMaxFilesPerJob"), - n_gb_per_job: Optional[float] = typer.Option(None, "--nGBPerJob"), - n_files: Optional[int] = typer.Option(None, "--nFiles"), - n_events: Optional[int] = typer.Option(None, "--nEvents"), - looping_check: Optional[bool] = typer.Option(None, "--loopingCheck"), - memory: Optional[int] = typer.Option(None, "--memory"), - avoid_vp: Optional[bool] = typer.Option(None, "--avoidVP"), - ignore_missing_in_ds: Optional[bool] = typer.Option(None, "--ignoreMissingInDS"), - force_staged: Optional[bool] = typer.Option(None, "--forceStaged"), - max_core: Optional[int] = typer.Option(None, "--maxCore"), + task_ids: Annotated[str, typer.Argument(help="Task ID, comma-separated IDs, or 'all'")], + days: Annotated[int, typer.Option("--days", help="Look-back window when task_ids='all'")] = 14, + limit: Annotated[int, typer.Option("--limit", help="Max tasks to retry when task_ids='all'")] = 1000, + site: Annotated[Optional[str], typer.Option("--site")] = None, + excludedSite: Annotated[Optional[str], typer.Option("--excludedSite")] = None, + includedSite: Annotated[Optional[str], typer.Option("--includedSite")] = None, + nFilesPerJob: Annotated[Optional[int], typer.Option("--nFilesPerJob")] = None, + nMaxFilesPerJob: Annotated[Optional[int], typer.Option("--nMaxFilesPerJob")] = None, + nGBPerJob: Annotated[Optional[float], typer.Option("--nGBPerJob")] = None, + nFiles: Annotated[Optional[int], typer.Option("--nFiles")] = None, + nEvents: Annotated[Optional[int], typer.Option("--nEvents")] = None, + loopingCheck: Annotated[Optional[bool], typer.Option("--loopingCheck")] = None, + memory: Annotated[Optional[int], typer.Option("--memory")] = None, + avoidVP: Annotated[Optional[bool], typer.Option("--avoidVP")] = None, + ignoreMissingInDS: Annotated[Optional[bool], typer.Option("--ignoreMissingInDS")] = None, + forceStaged: Annotated[Optional[bool], typer.Option("--forceStaged")] = None, + maxCore: Annotated[Optional[int], typer.Option("--maxCore")] = None, ) -> None: - """Retry failed/cancelled tasks.""" - core = _get_core() - core.init(sanity_check=False) + """Retry failed/cancelled tasks. + + Retry failed/cancelled subJobs in task_ids (ID or a list of IDs, can be either jediTaskID + or reqID). Allowed options to overwrite task parameters for new attempts: site, + excludedSite, includedSite, nFilesPerJob, nMaxFilesPerJob, nGBPerJob, nFiles, nEvents, + loopingCheck, memory, avoidVP, ignoreMissingInDS, forceStaged, maxCore. If input files + were used or are being used by other jobs for the same output dataset container, those + files are skipped to avoid job duplication when retrying failed subjobs. + + If task_ids is 'all', it retries 1000 tasks at most that have finished for the last 14 + days. It is possible to retry more tasks by setting the days and limit options. If + named arguments are specified, they are applied to all retried tasks. + + example: + >>> retry(123) + >>> retry([123, 345, 567]) + >>> retry(789, excludedSite='siteA,siteB') + >>> retry('all') + >>> retry('all', days=30, limit=2000) + """ + core = _ensure_init() new_opts = { k: v for k, v in { "site": site, - "excludedSite": excluded_site, - "includedSite": included_site, - "nFilesPerJob": n_files_per_job, - "nMaxFilesPerJob": n_max_files_per_job, - "nGBPerJob": n_gb_per_job, - "nFiles": n_files, - "nEvents": n_events, - "loopingCheck": looping_check, + "excludedSite": excludedSite, + "includedSite": includedSite, + "nFilesPerJob": nFilesPerJob, + "nMaxFilesPerJob": nMaxFilesPerJob, + "nGBPerJob": nGBPerJob, + "nFiles": nFiles, + "nEvents": nEvents, + "loopingCheck": loopingCheck, "ramCount": memory, - "avoidVP": avoid_vp, - "ignoreMissingInDS": ignore_missing_in_ds, - "forceStaged": force_staged, - "maxCoreCount": max_core, + "avoidVP": avoidVP, + "ignoreMissingInDS": ignoreMissingInDS, + "forceStaged": forceStaged, + "maxCoreCount": maxCore, }.items() if v is not None } opts = new_opts or None ids = _parse_ids(task_ids) if isinstance(ids, list): - _parallel(lambda tid: core.retry(tid, newOpts=opts), ids) - elif isinstance(ids, int): - core.retry(ids, newOpts=opts) - else: + return _parallel(lambda tid: core.retry(tid, newOpts=opts), ids) + elif ids == "all": data = core.show(status="finished", days=days, limit=limit, format="json") - _parallel(lambda d: core.retry.original_func(core, d["jediTaskID"], newOpts=opts), data) + return _parallel(lambda d: core.retry.original_func(core, d["jediTaskID"], newOpts=opts), data) + return core.retry(ids, newOpts=opts) @app.command() def debug( - panda_id: int = typer.Argument(..., help="PanDA subjob ID"), - mode_on: bool = typer.Argument(..., help="True to enable, False to disable"), + panda_id: Annotated[int, typer.Argument(help="PanDA subjob ID")], + mode_on: Annotated[bool, typer.Argument(help="True to enable, False to disable")], ) -> None: - """Toggle debug mode for a subjob.""" - core = _get_core() - core.init(sanity_check=False) + """Toggle debug mode for a subjob. + + mode_on is True/False to enable/disable the debug mode. Note that the maximum number of + debug subjobs is limited. If you already hit the limit you need to disable the debug mode + for a subjob before debugging another subjob. + + example: + >>> debug(1234, True) + """ + core = _ensure_init() core.debug(panda_id, mode_on) @app.command(name="get-user-job-metadata") def get_user_job_metadata( - task_id: int = typer.Argument(..., help="Task ID"), - output_file: str = typer.Argument(..., help="Output JSON file path"), + task_id: Annotated[int, typer.Argument(help="Task ID")], + output_file: Annotated[str, typer.Argument(help="Output JSON file path")], ) -> None: - """Write user metadata of successful jobs to a JSON file.""" - core = _get_core() - core.init(sanity_check=False) + """Write user metadata of successful jobs to a JSON file. + + Get user metadata of successful jobs in a task and write them in a json file. + + example: + >>> get_user_job_metadata(123, 'output.json') + """ + core = _ensure_init() core.getUserJobMetadata(task_id, output_file) @app.command(name="reload-input") def reload_input( - task_id: int = typer.Argument(..., help="Task ID"), + task_id: Annotated[int, typer.Argument(help="Task ID")], ) -> None: - """Reload input dataset and retry the task with new contents.""" - core = _get_core() - core.init(sanity_check=False) + """Reload input dataset and retry the task with new contents. + + This is useful when input dataset contents are changed after the task is submitted. + + example: + >>> reload_input(123) + """ + core = _ensure_init() core.reload_input(task_id) @app.command(name="recover-lost-files") def recover_lost_files( - task_id: int = typer.Argument(..., help="Task ID"), - test_mode: bool = typer.Option(False, "--test-mode", help="Dry-run mode"), + task_id: Annotated[int, typer.Argument(help="Task ID")], + test_mode: Annotated[bool, typer.Option("--test-mode", help="Dry-run mode")] = False, ) -> None: - """Request recovery of lost files from a task.""" - core = _get_core() - core.init(sanity_check=False) + """Request recovery of lost files from a task. + + Send a request to recover lost files produced by a task. Set test_mode=True for testing. + + example: + >>> recover_lost_files(123) + >>> recover_lost_files(123, test_mode=True) + """ + core = _ensure_init() core.recover_lost_files(task_id, test_mode) @app.command(name="show-workflow") def show_workflow( - request_id: int = typer.Argument(..., help="Workflow request ID"), + request_id: Annotated[int, typer.Argument(help="Workflow request ID")], ) -> None: - """Show workflow status.""" - core = _get_core() - core.init(sanity_check=False) + """Show workflow status. + + Send a request to show the status of a workflow. + + example: + >>> show_workflow(456) + """ + core = _ensure_init() _, output = core.execute_workflow_command("get_status", request_id) if output: print(output) @@ -812,11 +792,16 @@ def show_workflow( @app.command(name="kill-workflow") def kill_workflow( - request_id: int = typer.Argument(..., help="Workflow request ID"), + request_id: Annotated[int, typer.Argument(help="Workflow request ID")], ) -> None: - """Kill a workflow.""" - core = _get_core() - core.init(sanity_check=False) + """Kill a workflow. + + Send a request to kill a workflow. + + example: + >>> kill_workflow(456) + """ + core = _ensure_init() _, output = core.execute_workflow_command("abort", request_id) if output: print(output[0][-1]) @@ -824,11 +809,16 @@ def kill_workflow( @app.command(name="retry-workflow") def retry_workflow( - request_id: int = typer.Argument(..., help="Workflow request ID"), + request_id: Annotated[int, typer.Argument(help="Workflow request ID")], ) -> None: - """Retry a workflow.""" - core = _get_core() - core.init(sanity_check=False) + """Retry a workflow. + + Send a request to retry a workflow. + + example: + >>> retry_workflow(456) + """ + core = _ensure_init() _, output = core.execute_workflow_command("retry", request_id) if output: print(output[0][-1]) @@ -836,11 +826,16 @@ def retry_workflow( @app.command(name="finish-workflow") def finish_workflow( - request_id: int = typer.Argument(..., help="Workflow request ID"), + request_id: Annotated[int, typer.Argument(help="Workflow request ID")], ) -> None: - """Finish a workflow.""" - core = _get_core() - core.init(sanity_check=False) + """Finish a workflow. + + Send a request to finish a workflow. + + example: + >>> finish_workflow(456) + """ + core = _ensure_init() _, output = core.execute_workflow_command("finish", request_id) if output: print(output[0][-1]) @@ -848,11 +843,16 @@ def finish_workflow( @app.command(name="pause-workflow") def pause_workflow( - request_id: int = typer.Argument(..., help="Workflow request ID"), + request_id: Annotated[int, typer.Argument(help="Workflow request ID")], ) -> None: - """Pause a workflow.""" - core = _get_core() - core.init(sanity_check=False) + """Pause a workflow. + + Send a request to pause a workflow. + + example: + >>> pause_workflow(456) + """ + core = _ensure_init() _, output = core.execute_workflow_command("suspend", request_id) if output: print(output[0][-1]) @@ -860,11 +860,16 @@ def pause_workflow( @app.command(name="resume-workflow") def resume_workflow( - request_id: int = typer.Argument(..., help="Workflow request ID"), + request_id: Annotated[int, typer.Argument(help="Workflow request ID")], ) -> None: - """Resume a workflow.""" - core = _get_core() - core.init(sanity_check=False) + """Resume a workflow. + + Send a request to resume a workflow. + + example: + >>> resume_workflow(456) + """ + core = _ensure_init() _, output = core.execute_workflow_command("resume", request_id) if output: print(output[0][-1]) @@ -872,44 +877,59 @@ def resume_workflow( @app.command(name="set-secret") def set_secret( - key: str = typer.Argument(..., help="Secret key"), - value: str = typer.Argument(..., help="Secret value or file path"), - is_file: bool = typer.Option(False, "--is-file", help="Treat value as a file path to upload"), + key: Annotated[str, typer.Argument(help="Secret key")], + value: Annotated[str, typer.Argument(help="Secret value or file path")], + is_file: Annotated[bool, typer.Option("--is-file", help="Treat value as a file path to upload")] = False, ) -> None: - """Set a secret key-value pair.""" - core = _get_core() - core.init(sanity_check=False) - core.set_secret(key, value, is_file) + """Set a secret key-value pair. + Define a pair of secret key-value strings. The value can be a file path to upload a + secret file when is_file=True. -@app.command(name="list-secrets") -def list_secrets( - full: bool = typer.Option(False, "--full", help="Show full secret values"), -) -> None: - """List secrets.""" - core = _get_core() - core.init(sanity_check=False) - core.list_secrets(full) + example: + >>> set_secret('mykey', 'myvalue') + >>> set_secret('mykey', '/path/to/file', is_file=True) + """ + core = _ensure_init() + core.set_secret(key, value, is_file) @app.command(name="delete-secret") def delete_secret( - key: str = typer.Argument(..., help="Secret key to delete"), + key: Annotated[str, typer.Argument(help="Secret key to delete")], ) -> None: - """Delete a secret.""" - core = _get_core() - core.init(sanity_check=False) + """Delete a secret. + + example: + >>> delete_secret('mykey') + """ + core = _ensure_init() core.set_secret(key, None) @app.command(name="delete-all-secrets") def delete_all_secrets() -> None: """Delete all secrets.""" - core = _get_core() - core.init(sanity_check=False) + core = _ensure_init() core.set_secret(None, None) +@app.command(name="list-secrets") +def list_secrets( + full: Annotated[bool, typer.Option("--full", help="Show full secret values")] = False, +) -> None: + """List secrets. + + Value strings are truncated by default. full=True to see entire strings. + + example: + >>> list_secrets() + >>> list_secrets(full=True) + """ + core = _ensure_init() + core.list_secrets(full) + + @app.command(name="generate-credential") def generate_credential() -> None: """Generate a new proxy or token.""" @@ -919,7 +939,65 @@ def generate_credential() -> None: # ─── Entry point ────────────────────────────────────────────────────────────── +# Global options that consume the following argv token as their own value, so the +# subcommand-token scan below can skip over both. +_GLOBAL_VALUE_OPTS = {"-c"} + + +def _rewrite_legacy_kwargs(argv: list) -> list: + """Rewrite legacy bare `key=value` batch args into `--key=value`. + + The pre-Typer pbook batch mode accepted `pbook show format=long`; Click only + recognizes `--format=long`. Find the subcommand, look up its real option names via + introspection (never a separately maintained list), and rewrite any later bare + `key=value` token whose key matches one of them. Anything else - already-dashed + flags, positional args that happen to contain "=" - passes through untouched. + """ + i = 0 + while i < len(argv): + tok = argv[i] + if tok in _GLOBAL_VALUE_OPTS: + i += 2 + continue + if tok.startswith("-"): + i += 1 + continue + break + if i >= len(argv): + return argv + + sub_cmd = typer.main.get_command(app).commands.get(argv[i]) + if sub_cmd is None: + return argv + + option_flags = {} + flag_only = set() + for param in sub_cmd.params: + flags = [o for o in getattr(param, "opts", []) if o.startswith("--")] + if flags: + option_flags[param.name] = flags[0] + if getattr(param, "is_flag", False): + flag_only.add(param.name) + + rewritten = argv[: i + 1] + for tok in argv[i + 1 :]: + key, sep, value = tok.partition("=") + if sep and not tok.startswith("-") and key in option_flags: + flag = option_flags[key] + if key in flag_only: + # Click flag-style options (e.g. --soft) take no value at all; the legacy + # syntax passed an explicit True/False, so translate that into presence + # (truthy) or absence (falsy - same as the option's own default) instead. + if value.strip().lower() in ("true", "1", "yes"): + rewritten.append(flag) + continue + rewritten.append(f"{flag}={value}") + else: + rewritten.append(tok) + return rewritten + def main() -> None: sys.argv[0] = "pbook" + sys.argv[1:] = _rewrite_legacy_kwargs(sys.argv[1:]) app() diff --git a/pandaclient/PBookTyper2.py b/pandaclient/PBookTyper2.py deleted file mode 100644 index c63f06d9..00000000 --- a/pandaclient/PBookTyper2.py +++ /dev/null @@ -1,1003 +0,0 @@ -""" -pbook CLI — PanDA task bookkeeper. - -Each command below is defined exactly once, as a Typer command using the -Annotated[...] parameter style. Because the Typer/Click metadata lives in the -annotation rather than in the default value, these functions remain ordinary -callables with ordinary defaults - the same function is used to build the -`pbook --flag ...` CLI (with real shell completion) *and* is placed -directly into the interactive REPL namespace (`>>> command(...)`), with no -separate REPL-only copy of the command's logic, docstring, or option list. -""" - -from __future__ import annotations - -import atexit -import code -import os -import re -import readline -import rlcompleter -import signal -import sys -import tempfile -from concurrent.futures import ThreadPoolExecutor -from inspect import Parameter, signature -from typing import ( - Annotated, - Literal, - Optional, - Union, - get_args, - get_origin, - get_type_hints, -) - -import typer -from rich import box -from rich.console import Console -from rich.markup import escape as _esc -from rich.table import Table - -from pandaclient import Client, PandaToolsPkgInfo -from pandaclient.MiscUtils import commands_get_output - -# ─── Runtime state ──────────────────────────────────────────────────────────── -_tmp_dir: Optional[str] = None -_history_file: Optional[str] = None -_fork_child_pid: Optional[int] = None -_setup_done: bool = False -_ctx_state: dict = {} -_core = None -_core_inited: bool = False - -help_text = """ -PanDA task bookkeeper. Run without arguments for interactive mode. - -$ pbook [options] # interactive mode -$ pbook [options] command [args] [kwargs] # batch mode - -The same command can be executed in interactive mode: - -$ pbook ->>> command(*args, **kwargs) - -or in batch mode: - -$ pbook command arg1 arg2 ... argN --kwarg1=value1 --kwarg2=value2 ... --kwargN=valueN -$ pbook command arg1 arg2 ... argN kwarg1=value1 kwarg2=value2 ... kwargN=valueN -Please note that the latter option is kept for backward compatibility, but we plan to drop it in the future. - -E.g. - -$ pbook ->>> show(123, format='long') - -is equivalent to - -$ pbook show 123 --format='long' -$ pbook show 123 format='long' - -If arg or value is a list in interactive mode, it is represented as a comma-separate list in batch mode. E.g. -to kill three tasks in interactive mode: - -$ pbook ->>> kill([123, 456, 789]) - -or in batch mode: - -$ pbook kill 123,456,789 - -To see the list of commands and help of each command, - -$ pbook ->>> help() ->>> help(command_name) - -or - -$ pbook help -$ pbook command_name --help -""" - -app = typer.Typer( - name="pbook", - help=help_text, - invoke_without_command=True, - no_args_is_help=False, - context_settings={"help_option_names": ["-h", "--help"]}, -) - -# ─── Utilities ──────────────────────────────────────────────────────────────── - - -def _parallel(func, items): - with ThreadPoolExecutor(8) as pool: - return list(pool.map(func, items)) - - -def _parse_ids(raw): - """'all' -> 'all'; '42' -> 42; '1,2,3' -> [1,2,3]; anything not a string passes through as-is.""" - if not isinstance(raw, str): - return raw - if raw == "all": - return "all" - parts = raw.split(",") - try: - ids = [int(p) for p in parts] - return ids[0] if len(ids) == 1 else ids - except ValueError: - typer.echo(f"Error: invalid task ID(s): {raw}", err=True) - raise typer.Exit(1) - - -def _setup() -> None: - global _tmp_dir, _history_file, _setup_done - if _setup_done: - return - _setup_done = True - - readline.parse_and_bind("tab: complete") - readline.parse_and_bind("set show-all-if-ambiguous On") - - if "CMTSITE" not in os.environ: - os.environ["CMTSITE"] = "" - - pconf_dir = os.path.expanduser(os.environ.get("PANDA_CONFIG_ROOT", "~/.panda")) - os.makedirs(pconf_dir, exist_ok=True) - - _history_file = os.path.join(pconf_dir, ".history") - if os.path.exists(_history_file): - try: - readline.read_history_file(_history_file) - except Exception: - pass - readline.set_history_length(1024) - - _tmp_dir = tempfile.mkdtemp() - - for path in sys.path: - real = path or "." - if ( - os.path.exists(real) - and os.path.isdir(real) - and "pandaclient" in os.listdir(real) - and os.path.exists(os.path.join(real, "pandaclient", "__init__.py")) - ): - link = os.path.join(_tmp_dir, "taskbuffer") - if not os.path.exists(link): - os.symlink(os.path.join(real, "pandaclient"), link) - break - if _tmp_dir not in sys.path: - sys.path.insert(0, _tmp_dir) - - atexit.register(_cleanup) - - -def _cleanup() -> None: - if _fork_child_pid == 0 and _history_file: - readline.write_history_file(_history_file) - - if _tmp_dir: - commands_get_output(f"rm -rf {_tmp_dir}") - - -def _make_core(verbose: bool = False): - from pandaclient import PBookCore - - return PBookCore.PBookCore(verbose=verbose) - - -def _get_core(): - """Return the (memoized) core for this process, uninitialized.""" - global _core - _setup() - if _core is None: - _core = _make_core(_ctx_state.get("verbose", False)) - return _core - - -def _ensure_init(sanity_check: bool = False): - """Return the core, running PBookCore.init() exactly once per process. - - The REPL calls this once upfront with sanity_check=True; every command - function then calls it again with the default before touching the core, - which is a no-op once already initialized - so commands stay correct - whether they run once (batch mode) or repeatedly (REPL session). - """ - global _core_inited - core = _get_core() - if not _core_inited: - core.init(sanity_check=sanity_check) - _core_inited = True - return core - - -def _catch_sig(sig, frame): - _cleanup() - # Hard kill all processes in the group - commands_get_output(f"kill -9 -- -{os.getpgrp()}") - - -# ─── REPL namespace & completion ────────────────────────────────────────────── - - -def _build_namespace() -> dict: - """The REPL namespace: every registered Typer command, keyed by its real Python name.""" - return {info.callback.__name__: info.callback for info in app.registered_commands} - - -def _kwarg_names(func) -> list: - """All parameter names a function accepts - candidates for `name=` completion.""" - return list(signature(func).parameters) - - -def _kwarg_choices(func, name: str) -> list: - """Value choices for a parameter, derived from its type hint: Literal[...] members or True/False for bool. - - Returned bare (unquoted) - readline's own quote-matching auto-closes an opening quote - the user already typed, so we don't need to (and shouldn't try to) add one ourselves. - """ - try: - hints = get_type_hints(func, include_extras=True) - except Exception: - return [] - ann = hints.get(name) - if ann is None: - return [] - while hasattr(ann, "__metadata__"): - ann = ann.__origin__ - if get_origin(ann) is Union: - non_none = [a for a in get_args(ann) if a is not type(None)] - if len(non_none) == 1: - ann = non_none[0] - if get_origin(ann) is Literal: - return [str(v) for v in get_args(ann)] - if ann is bool: - return ["True", "False"] - return [] - - -class _PBookCompleter: - """Readline completer: kwarg names and values when inside a call, names otherwise.""" - - def __init__(self, ns: dict) -> None: - self._ns = ns - self._base = rlcompleter.Completer(ns) - self._matches: list = [] - - def complete(self, text: str, state: int) -> Optional[str]: - if state == 0: - self._matches = self._compute(text) - return self._matches[state] if state < len(self._matches) else None - - def _compute(self, text: str) -> list: - line = readline.get_line_buffer() - - # Kwarg value completion (tier 1): last token is kwarg= or kwarg='partial - # Return bare values - readline's own quote-matching auto-closes an opening quote - # the user already typed, so we deliberately don't add quotes ourselves here. - m_val = re.search(r"\b(\w+)\s*=\s*(['\"]?)(\w*)$", line) - m_func = re.match(r"(\w+)\s*\(", line) - if m_val and m_func: - kwarg, partial = m_val.group(1), m_val.group(3) - func = self._ns.get(m_func.group(1)) - if func is not None: - # We're unambiguously past a `kwarg=` - this is a value position, not a - # name position, even if this particular kwarg has no enumerable choices - # (e.g. limit: int). Return here regardless, so an empty result doesn't - # fall through to tier 2's kwarg-name completion. - return [v for v in _kwarg_choices(func, kwarg) if v.startswith(partial)] - - # Kwarg name completion (tier 2): cursor is inside an open call - m = re.search(r"(\w+)\s*\([^)]*$", line) - if m: - func = self._ns.get(m.group(1)) - if func is not None: - hits = [k for k in _kwarg_names(func) if k.startswith(text)] - if hits: - return hits - - # Plain name completion (tier 3, rlcompleter fallback) - if not text: - # rlcompleter.complete() special-cases blank text by calling readline.insert_text() - # itself, which re-enters readline from inside this callback and confuses the active - # Tab press; list the namespace directly instead of delegating to it here - return sorted(k for k in self._base.namespace if not k.startswith("_")) - results, i = [], 0 - while (c := self._base.complete(text, i)) is not None: - results.append(c.rstrip("()").rstrip("(")) - i += 1 - return results - - -def _run_repl(ns: dict, banner: str) -> None: - """Manual REPL using InteractiveConsole.push() so we own readline setup entirely.""" - completer = _PBookCompleter(ns) - readline.set_completer(completer.complete) - readline.parse_and_bind("tab: complete") - readline.parse_and_bind("set show-all-if-ambiguous On") - - console = code.InteractiveConsole(ns) - print(banner) - - more = False - while True: - prompt = "... " if more else ">>> " - try: - readline.set_completer(completer.complete) - line = input(prompt) - except EOFError: - print() - break - except KeyboardInterrupt: - print("\nKeyboardInterrupt") - console.resetbuffer() - more = False - continue - more = console.push(line) - - -# ─── Top-level callback ─────────────────────────────────────────────────────── - - -@app.callback(invoke_without_command=True) -def _main( - ctx: typer.Context, - verbose: bool = typer.Option(False, "-v", help="Verbose"), - command_string: Optional[str] = typer.Option(None, "-c", help="Execute a Python code snippet"), - version: bool = typer.Option(False, "--version", is_eager=True, help="Display version"), - dev_srv: bool = typer.Option(False, "--devSrv", hidden=True), - intr_srv: bool = typer.Option(False, "--intrSrv", hidden=True), - prompt_with_newline: bool = typer.Option(False, "--prompt_with_newline", hidden=True), - python3: bool = typer.Option(False, "-3", hidden=True), -) -> None: - """PanDA task bookkeeper. Run without arguments for interactive mode.""" - if version: - typer.echo(f"Version: {PandaToolsPkgInfo.release_version}") - raise typer.Exit() - - if dev_srv: - Client.useDevServer() - if intr_srv: - Client.useIntrServer() - - _ctx_state.update({"verbose": verbose}) - - if ctx.invoked_subcommand is not None: - return - - # Interactive or snippet mode - _setup() - global _fork_child_pid - _fork_child_pid = os.fork() - - if _fork_child_pid == -1: - typer.echo("ERROR: Failed to fork", err=True) - raise typer.Exit(1) - - if _fork_child_pid == 0: - if verbose: - typer.echo(str(ctx.params)) - if prompt_with_newline: - sys.ps1 = ">>> \n" - _ensure_init(sanity_check=True) - ns = _build_namespace() - - if command_string: - exec(command_string, {}, ns) # noqa: S102 - from pandaclient import PBookCore as _PBC - - raise typer.Exit(0 if _PBC.func_return_value else 1) - _run_repl(ns, banner=f"\nStart pBook {PandaToolsPkgInfo.release_version}") - - else: - signal.signal(signal.SIGINT, _catch_sig) - signal.signal(signal.SIGHUP, _catch_sig) - signal.signal(signal.SIGTERM, _catch_sig) - pid, status = os.wait() - if os.WIFSIGNALED(status): - raise typer.Exit(-os.WTERMSIG(status)) - elif os.WIFEXITED(status): - raise typer.Exit(os.WEXITSTATUS(status)) - raise typer.Exit(0) - - -# ─── Commands ────────────────────────────────────────────────────────────────── - -_HELP_GROUPS = [ - ("Tasks", ["show", "showl", "kill", "finish", "retry", "debug"]), - ("Files & input", ["get_user_job_metadata", "recover_lost_files", "reload_input"]), - ("Workflows", ["show_workflow", "kill_workflow", "retry_workflow", "finish_workflow", "pause_workflow", "resume_workflow"]), - ("Secrets", ["set_secret", "list_secrets", "delete_secret", "delete_all_secrets"]), - ("Auth", ["generate_credential"]), -] - - -def _type_name(ann) -> str: - """Render a resolved type annotation as a short, human-readable name (Optional[str], Literal[...], etc.).""" - if ann is None or ann is type(None): - return "" - origin = get_origin(ann) - if origin is Union: - args = get_args(ann) - non_none = [a for a in args if a is not type(None)] - if len(non_none) == 1 and len(args) == 2: - return f"Optional[{_type_name(non_none[0])}]" - return " | ".join(_type_name(a) for a in args) - if origin is Literal: - return "Literal[" + ", ".join(repr(v) for v in get_args(ann)) + "]" - if origin is not None: - args = get_args(ann) - origin_name = getattr(origin, "__name__", str(origin)) - return f"{origin_name}[{', '.join(_type_name(a) for a in args)}]" if args else origin_name - return getattr(ann, "__name__", str(ann)) - - -def _format_signature(func) -> str: - """A clean '(param: Type = default, ...)' string, stripping the Typer/Annotated plumbing.""" - try: - hints = get_type_hints(func, include_extras=True) - except Exception: - hints = {} - parts = [] - for pname, p in signature(func).parameters.items(): - ann = hints.get(pname) - while hasattr(ann, "__metadata__"): - ann = ann.__origin__ - piece = pname - type_str = _type_name(ann) - if type_str: - piece += f": {type_str}" - if p.default is not Parameter.empty: - piece += f" = {p.default!r}" - parts.append(piece) - return f"({', '.join(parts)})" - - -@app.command() -def help( - command: Annotated[Optional[str], typer.Argument(help="Command name for detailed help")] = None, -) -> None: - """Show available commands, or detailed help for a specific command.""" - ns = _build_namespace() - console = Console() - - if command is not None: - name = command if isinstance(command, str) else command.__name__ - func = ns.get(name, command if callable(command) else None) - if func is None: - console.print(f"[red]Unknown command:[/red] {_esc(name)}") - return - console.print(f"\n[bold cyan]{name}[/bold cyan][bold]{_esc(_format_signature(func))}[/bold]") - doc = (func.__doc__ or "No description.").strip() - console.print(f"\n{_esc(doc)}\n") - return - - table = Table(box=box.SIMPLE, show_header=True, header_style="bold magenta") - table.add_column("Command", style="bold cyan", no_wrap=True) - table.add_column("Description") - table.add_column("Signature") - - for group, names in _HELP_GROUPS: - table.add_section() - table.add_row(f"[bold white]{group}[/bold white]", "", "") - for name in names: - func = ns.get(name) - if func is None: - continue - doc = (func.__doc__ or "").strip().splitlines()[0] if func.__doc__ else "" - sig = console.highlighter(_format_signature(func)) - table.add_row(f" {name}", _esc(doc), sig) - - console.print(table) - console.print("Usage: [bold]help(show)[/bold] or [bold]pbook show --help[/bold]\n") - - -@app.command() -def show( - task_id: Annotated[Optional[str], typer.Argument(help="jediTaskID, reqID, 'run' (active only), or 'fin' (terminated only)")] = None, - username: Annotated[Optional[str], typer.Option(help="Filter by username")] = None, - limit: Annotated[int, typer.Option(help="Maximum number of records")] = 1000, - taskname: Annotated[Optional[str], typer.Option(help="Filter by task name")] = None, - days: Annotated[int, typer.Option(help="Look back N days (capped at 90 without a task ID)")] = 14, - jeditaskid: Annotated[Optional[int], typer.Option(help="Filter by jediTaskID")] = None, - reqid: Annotated[Optional[int], typer.Option(help="Filter by reqID")] = None, - status: Annotated[Optional[str], typer.Option(help="Filter by task status")] = None, - superstatus: Annotated[Optional[str], typer.Option(help="Filter by super-status")] = None, - format: Annotated[Literal["standard", "long", "json", "plain"], typer.Option("--format", help="Output format")] = "standard", -) -> None: - """Print task records. - - The first argument (task_id) can be a jediTaskID or reqID, or 'run' (show active tasks - only), or 'fin' (show terminated tasks only), or can be omitted. Records are fetched - directly from the PanDA server, so they are always up to date. Note that days is capped - at 90 days unless a jediTaskID or reqID is specified, in which case tasks of any age are - returned. The default filter conditions are: username=(name from user voms proxy), - limit=1000, days=14, format='standard'. - - example: - >>> show() - >>> show(123) - >>> show(12345678, format='long') - >>> show(taskname='my_task_name') - >>> show('run') - >>> show('fin', days=7, limit=100) - >>> show(format='json') - """ - core = _ensure_init() - kwargs = { - k: v - for k, v in dict( - username=username, - limit=limit, - taskname=taskname, - days=days, - jeditaskid=jeditaskid, - reqid=reqid, - status=status, - superstatus=superstatus, - ).items() - if v is not None - } - kwargs["format"] = format - if task_id is not None: - try: - first_arg = int(task_id) - except (TypeError, ValueError): - first_arg = task_id - return core.show(first_arg, **kwargs) - return core.show(**kwargs) - - -@app.command() -def showl( - task_id: Annotated[Optional[str], typer.Argument(help="jediTaskID, reqID, 'run', or 'fin'")] = None, - username: Annotated[Optional[str], typer.Option(help="Filter by username")] = None, - limit: Annotated[int, typer.Option(help="Maximum number of records")] = 1000, - taskname: Annotated[Optional[str], typer.Option(help="Filter by task name")] = None, - days: Annotated[int, typer.Option(help="Look back N days (capped at 90 without a task ID)")] = 14, - jeditaskid: Annotated[Optional[int], typer.Option(help="Filter by jediTaskID")] = None, - reqid: Annotated[Optional[int], typer.Option(help="Filter by reqID")] = None, - status: Annotated[Optional[str], typer.Option(help="Filter by task status")] = None, - superstatus: Annotated[Optional[str], typer.Option(help="Filter by super-status")] = None, -) -> None: - """Print task records in long format (shortcut for show --format long). - - See help(show) for the available filter keywords. - - example: - >>> showl() - >>> showl(123) - >>> showl(12345678) - >>> showl(taskname='my_task_name') - """ - return show( - task_id, - username=username, - limit=limit, - taskname=taskname, - days=days, - jeditaskid=jeditaskid, - reqid=reqid, - status=status, - superstatus=superstatus, - format="long", - ) - - -@app.command() -def kill( - task_ids: Annotated[str, typer.Argument(help="Task ID, comma-separated IDs, or 'all'")], -) -> None: - """Kill tasks. - - Kill all subJobs in task_ids (ID or a list of IDs, can be either jediTaskID or reqID). - If 'all', kill all active tasks of the user. - - example: - >>> kill(123) - >>> kill([123, 345, 567]) - >>> kill('all') - """ - core = _ensure_init() - ids = _parse_ids(task_ids) - if ids == "all": - return _parallel(lambda t: core.kill(t.jeditaskid), core.get_active_tasks()) - elif isinstance(ids, list): - return _parallel(core.kill, ids) - return core.kill(ids) - - -@app.command() -def finish( - task_ids: Annotated[str, typer.Argument(help="Task ID, comma-separated IDs, or 'all'")], - soft: Annotated[bool, typer.Option("--soft", help="Wait for running jobs to finish instead of killing them")] = False, -) -> None: - """Finish tasks. - - Finish all subJobs in task_ids (ID or a list of IDs, can be either jediTaskID or reqID). - If task_ids is 'all', finish all active tasks of the user. If soft is False (default), - all running jobs are killed and the task finishes immediately. If soft is True, new jobs - are not generated and the task finishes once all running jobs finish. - - example: - >>> finish(123) - >>> finish(234, soft=True) - >>> finish([123, 345, 567]) - >>> finish('all') - """ - core = _ensure_init() - ids = _parse_ids(task_ids) - if ids == "all": - return _parallel(lambda t: core.finish.original_func(core, t.jeditaskid, soft=soft), core.get_active_tasks()) - elif isinstance(ids, list): - return _parallel(lambda tid: core.finish(tid, soft=soft), ids) - return core.finish(ids, soft=soft) - - -@app.command() -def retry( - task_ids: Annotated[str, typer.Argument(help="Task ID, comma-separated IDs, or 'all'")], - days: Annotated[int, typer.Option("--days", help="Look-back window when task_ids='all'")] = 14, - limit: Annotated[int, typer.Option("--limit", help="Max tasks to retry when task_ids='all'")] = 1000, - site: Annotated[Optional[str], typer.Option("--site")] = None, - excludedSite: Annotated[Optional[str], typer.Option("--excludedSite")] = None, - includedSite: Annotated[Optional[str], typer.Option("--includedSite")] = None, - nFilesPerJob: Annotated[Optional[int], typer.Option("--nFilesPerJob")] = None, - nMaxFilesPerJob: Annotated[Optional[int], typer.Option("--nMaxFilesPerJob")] = None, - nGBPerJob: Annotated[Optional[float], typer.Option("--nGBPerJob")] = None, - nFiles: Annotated[Optional[int], typer.Option("--nFiles")] = None, - nEvents: Annotated[Optional[int], typer.Option("--nEvents")] = None, - loopingCheck: Annotated[Optional[bool], typer.Option("--loopingCheck")] = None, - memory: Annotated[Optional[int], typer.Option("--memory")] = None, - avoidVP: Annotated[Optional[bool], typer.Option("--avoidVP")] = None, - ignoreMissingInDS: Annotated[Optional[bool], typer.Option("--ignoreMissingInDS")] = None, - forceStaged: Annotated[Optional[bool], typer.Option("--forceStaged")] = None, - maxCore: Annotated[Optional[int], typer.Option("--maxCore")] = None, -) -> None: - """Retry failed/cancelled tasks. - - Retry failed/cancelled subJobs in task_ids (ID or a list of IDs, can be either jediTaskID - or reqID). Allowed options to overwrite task parameters for new attempts: site, - excludedSite, includedSite, nFilesPerJob, nMaxFilesPerJob, nGBPerJob, nFiles, nEvents, - loopingCheck, memory, avoidVP, ignoreMissingInDS, forceStaged, maxCore. If input files - were used or are being used by other jobs for the same output dataset container, those - files are skipped to avoid job duplication when retrying failed subjobs. - - If task_ids is 'all', it retries 1000 tasks at most that have finished for the last 14 - days. It is possible to retry more tasks by setting the days and limit options. If - named arguments are specified, they are applied to all retried tasks. - - example: - >>> retry(123) - >>> retry([123, 345, 567]) - >>> retry(789, excludedSite='siteA,siteB') - >>> retry('all') - >>> retry('all', days=30, limit=2000) - """ - core = _ensure_init() - new_opts = { - k: v - for k, v in { - "site": site, - "excludedSite": excludedSite, - "includedSite": includedSite, - "nFilesPerJob": nFilesPerJob, - "nMaxFilesPerJob": nMaxFilesPerJob, - "nGBPerJob": nGBPerJob, - "nFiles": nFiles, - "nEvents": nEvents, - "loopingCheck": loopingCheck, - "ramCount": memory, - "avoidVP": avoidVP, - "ignoreMissingInDS": ignoreMissingInDS, - "forceStaged": forceStaged, - "maxCoreCount": maxCore, - }.items() - if v is not None - } - opts = new_opts or None - ids = _parse_ids(task_ids) - if isinstance(ids, list): - return _parallel(lambda tid: core.retry(tid, newOpts=opts), ids) - elif ids == "all": - data = core.show(status="finished", days=days, limit=limit, format="json") - return _parallel(lambda d: core.retry.original_func(core, d["jediTaskID"], newOpts=opts), data) - return core.retry(ids, newOpts=opts) - - -@app.command() -def debug( - panda_id: Annotated[int, typer.Argument(help="PanDA subjob ID")], - mode_on: Annotated[bool, typer.Argument(help="True to enable, False to disable")], -) -> None: - """Toggle debug mode for a subjob. - - mode_on is True/False to enable/disable the debug mode. Note that the maximum number of - debug subjobs is limited. If you already hit the limit you need to disable the debug mode - for a subjob before debugging another subjob. - - example: - >>> debug(1234, True) - """ - core = _ensure_init() - core.debug(panda_id, mode_on) - - -@app.command(name="get-user-job-metadata") -def get_user_job_metadata( - task_id: Annotated[int, typer.Argument(help="Task ID")], - output_file: Annotated[str, typer.Argument(help="Output JSON file path")], -) -> None: - """Write user metadata of successful jobs to a JSON file. - - Get user metadata of successful jobs in a task and write them in a json file. - - example: - >>> get_user_job_metadata(123, 'output.json') - """ - core = _ensure_init() - core.getUserJobMetadata(task_id, output_file) - - -@app.command(name="reload-input") -def reload_input( - task_id: Annotated[int, typer.Argument(help="Task ID")], -) -> None: - """Reload input dataset and retry the task with new contents. - - This is useful when input dataset contents are changed after the task is submitted. - - example: - >>> reload_input(123) - """ - core = _ensure_init() - core.reload_input(task_id) - - -@app.command(name="recover-lost-files") -def recover_lost_files( - task_id: Annotated[int, typer.Argument(help="Task ID")], - test_mode: Annotated[bool, typer.Option("--test-mode", help="Dry-run mode")] = False, -) -> None: - """Request recovery of lost files from a task. - - Send a request to recover lost files produced by a task. Set test_mode=True for testing. - - example: - >>> recover_lost_files(123) - >>> recover_lost_files(123, test_mode=True) - """ - core = _ensure_init() - core.recover_lost_files(task_id, test_mode) - - -@app.command(name="show-workflow") -def show_workflow( - request_id: Annotated[int, typer.Argument(help="Workflow request ID")], -) -> None: - """Show workflow status. - - Send a request to show the status of a workflow. - - example: - >>> show_workflow(456) - """ - core = _ensure_init() - _, output = core.execute_workflow_command("get_status", request_id) - if output: - print(output) - - -@app.command(name="kill-workflow") -def kill_workflow( - request_id: Annotated[int, typer.Argument(help="Workflow request ID")], -) -> None: - """Kill a workflow. - - Send a request to kill a workflow. - - example: - >>> kill_workflow(456) - """ - core = _ensure_init() - _, output = core.execute_workflow_command("abort", request_id) - if output: - print(output[0][-1]) - - -@app.command(name="retry-workflow") -def retry_workflow( - request_id: Annotated[int, typer.Argument(help="Workflow request ID")], -) -> None: - """Retry a workflow. - - Send a request to retry a workflow. - - example: - >>> retry_workflow(456) - """ - core = _ensure_init() - _, output = core.execute_workflow_command("retry", request_id) - if output: - print(output[0][-1]) - - -@app.command(name="finish-workflow") -def finish_workflow( - request_id: Annotated[int, typer.Argument(help="Workflow request ID")], -) -> None: - """Finish a workflow. - - Send a request to finish a workflow. - - example: - >>> finish_workflow(456) - """ - core = _ensure_init() - _, output = core.execute_workflow_command("finish", request_id) - if output: - print(output[0][-1]) - - -@app.command(name="pause-workflow") -def pause_workflow( - request_id: Annotated[int, typer.Argument(help="Workflow request ID")], -) -> None: - """Pause a workflow. - - Send a request to pause a workflow. - - example: - >>> pause_workflow(456) - """ - core = _ensure_init() - _, output = core.execute_workflow_command("suspend", request_id) - if output: - print(output[0][-1]) - - -@app.command(name="resume-workflow") -def resume_workflow( - request_id: Annotated[int, typer.Argument(help="Workflow request ID")], -) -> None: - """Resume a workflow. - - Send a request to resume a workflow. - - example: - >>> resume_workflow(456) - """ - core = _ensure_init() - _, output = core.execute_workflow_command("resume", request_id) - if output: - print(output[0][-1]) - - -@app.command(name="set-secret") -def set_secret( - key: Annotated[str, typer.Argument(help="Secret key")], - value: Annotated[str, typer.Argument(help="Secret value or file path")], - is_file: Annotated[bool, typer.Option("--is-file", help="Treat value as a file path to upload")] = False, -) -> None: - """Set a secret key-value pair. - - Define a pair of secret key-value strings. The value can be a file path to upload a - secret file when is_file=True. - - example: - >>> set_secret('mykey', 'myvalue') - >>> set_secret('mykey', '/path/to/file', is_file=True) - """ - core = _ensure_init() - core.set_secret(key, value, is_file) - - -@app.command(name="delete-secret") -def delete_secret( - key: Annotated[str, typer.Argument(help="Secret key to delete")], -) -> None: - """Delete a secret. - - example: - >>> delete_secret('mykey') - """ - core = _ensure_init() - core.set_secret(key, None) - - -@app.command(name="delete-all-secrets") -def delete_all_secrets() -> None: - """Delete all secrets.""" - core = _ensure_init() - core.set_secret(None, None) - - -@app.command(name="list-secrets") -def list_secrets( - full: Annotated[bool, typer.Option("--full", help="Show full secret values")] = False, -) -> None: - """List secrets. - - Value strings are truncated by default. full=True to see entire strings. - - example: - >>> list_secrets() - >>> list_secrets(full=True) - """ - core = _ensure_init() - core.list_secrets(full) - - -@app.command(name="generate-credential") -def generate_credential() -> None: - """Generate a new proxy or token.""" - core = _get_core() - core.generate_credential() - - -# ─── Entry point ────────────────────────────────────────────────────────────── - -# Global options that consume the following argv token as their own value, so the -# subcommand-token scan below can skip over both. -_GLOBAL_VALUE_OPTS = {"-c"} - - -def _rewrite_legacy_kwargs(argv: list) -> list: - """Rewrite legacy bare `key=value` batch args into `--key=value`. - - The pre-Typer pbook batch mode accepted `pbook show format=long`; Click only - recognizes `--format=long`. Find the subcommand, look up its real option names via - introspection (never a separately maintained list), and rewrite any later bare - `key=value` token whose key matches one of them. Anything else - already-dashed - flags, positional args that happen to contain "=" - passes through untouched. - """ - i = 0 - while i < len(argv): - tok = argv[i] - if tok in _GLOBAL_VALUE_OPTS: - i += 2 - continue - if tok.startswith("-"): - i += 1 - continue - break - if i >= len(argv): - return argv - - sub_cmd = typer.main.get_command(app).commands.get(argv[i]) - if sub_cmd is None: - return argv - - option_flags = {} - flag_only = set() - for param in sub_cmd.params: - flags = [o for o in getattr(param, "opts", []) if o.startswith("--")] - if flags: - option_flags[param.name] = flags[0] - if getattr(param, "is_flag", False): - flag_only.add(param.name) - - rewritten = argv[: i + 1] - for tok in argv[i + 1 :]: - key, sep, value = tok.partition("=") - if sep and not tok.startswith("-") and key in option_flags: - flag = option_flags[key] - if key in flag_only: - # Click flag-style options (e.g. --soft) take no value at all; the legacy - # syntax passed an explicit True/False, so translate that into presence - # (truthy) or absence (falsy - same as the option's own default) instead. - if value.strip().lower() in ("true", "1", "yes"): - rewritten.append(flag) - continue - rewritten.append(f"{flag}={value}") - else: - rewritten.append(tok) - return rewritten - - -def main() -> None: - sys.argv[0] = "pbook" - sys.argv[1:] = _rewrite_legacy_kwargs(sys.argv[1:]) - app() diff --git a/scripts/pbook b/scripts/pbook index 81949cf6..8ed2fe5f 100755 --- a/scripts/pbook +++ b/scripts/pbook @@ -2,4 +2,4 @@ source ${PANDA_SYS}/etc/panda/share/functions.sh -exec_p_command "import pandaclient.PBookTyper2 as pbook; pbook.main()" "$@" +exec_p_command "import pandaclient.PBookTyper as pbook; pbook.main()" "$@" From 6f89a8bb041dc9dc1f9d8adb36d3ae44ff764aae Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Mon, 3 Aug 2026 14:43:43 +0200 Subject: [PATCH 35/59] Validating True/False options --- pandaclient/PBookTyper.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index c63f06d9..68fa19bb 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -497,7 +497,7 @@ def help( @app.command() def show( task_id: Annotated[Optional[str], typer.Argument(help="jediTaskID, reqID, 'run' (active only), or 'fin' (terminated only)")] = None, - username: Annotated[Optional[str], typer.Option(help="Filter by username")] = None, + username: Annotated[Optional[str], typer.Option(help="Filter by username. By default, the name from the voms/token is used.")] = None, limit: Annotated[int, typer.Option(help="Maximum number of records")] = 1000, taskname: Annotated[Optional[str], typer.Option(help="Filter by task name")] = None, days: Annotated[int, typer.Option(help="Look back N days (capped at 90 without a task ID)")] = 14, @@ -509,12 +509,11 @@ def show( ) -> None: """Print task records. - The first argument (task_id) can be a jediTaskID or reqID, or 'run' (show active tasks + The first non-keyword argument (task_id) can be a jediTaskID or reqID, or 'run' (show active tasks only), or 'fin' (show terminated tasks only), or can be omitted. Records are fetched directly from the PanDA server, so they are always up to date. Note that days is capped at 90 days unless a jediTaskID or reqID is specified, in which case tasks of any age are - returned. The default filter conditions are: username=(name from user voms proxy), - limit=1000, days=14, format='standard'. + returned. See the default filter conditions in the annotations. example: >>> show() @@ -562,9 +561,7 @@ def showl( status: Annotated[Optional[str], typer.Option(help="Filter by task status")] = None, superstatus: Annotated[Optional[str], typer.Option(help="Filter by super-status")] = None, ) -> None: - """Print task records in long format (shortcut for show --format long). - - See help(show) for the available filter keywords. + """Print task records in long format (shortcut for show --format='long'). example: >>> showl() @@ -599,6 +596,8 @@ def kill( >>> kill(123) >>> kill([123, 345, 567]) >>> kill('all') + + $ pbook kill 123,345,567 """ core = _ensure_init() ids = _parse_ids(task_ids) @@ -626,6 +625,8 @@ def finish( >>> finish(234, soft=True) >>> finish([123, 345, 567]) >>> finish('all') + + $ pbook finish 123,345,567 --soft """ core = _ensure_init() ids = _parse_ids(task_ids) @@ -988,8 +989,15 @@ def _rewrite_legacy_kwargs(argv: list) -> list: # Click flag-style options (e.g. --soft) take no value at all; the legacy # syntax passed an explicit True/False, so translate that into presence # (truthy) or absence (falsy - same as the option's own default) instead. - if value.strip().lower() in ("true", "1", "yes"): + normalized = value.strip().lower() + if normalized in ("true", "1", "yes"): rewritten.append(flag) + elif normalized not in ("false", "0", "no"): + typer.echo( + f"Error: '{key}' is a flag and expects true/false (got '{value}' in '{tok}')", + err=True, + ) + raise typer.Exit(1) continue rewritten.append(f"{flag}={value}") else: From f32fed73c1acc9a1628e00837c1c5dd603b274ef Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Mon, 3 Aug 2026 14:45:21 +0200 Subject: [PATCH 36/59] Validating True/False options --- pandaclient/PBookTyper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index 68fa19bb..4bdfa419 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -994,7 +994,7 @@ def _rewrite_legacy_kwargs(argv: list) -> list: rewritten.append(flag) elif normalized not in ("false", "0", "no"): typer.echo( - f"Error: '{key}' is a flag and expects true/false (got '{value}' in '{tok}')", + f"Error: '{key}' is a flag and expects True/False (got '{value}' in '{tok}')", err=True, ) raise typer.Exit(1) From 123563639a837c15f85e5dcc02d444b3aad55637 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Mon, 3 Aug 2026 14:46:05 +0200 Subject: [PATCH 37/59] Validating True/False options --- pandaclient/PBookTyper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index 4bdfa419..f033ca72 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -997,7 +997,7 @@ def _rewrite_legacy_kwargs(argv: list) -> list: f"Error: '{key}' is a flag and expects True/False (got '{value}' in '{tok}')", err=True, ) - raise typer.Exit(1) + sys.exit(1) continue rewritten.append(f"{flag}={value}") else: From a2571d294fcebce02365a06f46d35c30b9f3ad96 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Mon, 3 Aug 2026 14:57:35 +0200 Subject: [PATCH 38/59] Validating True/False options --- pandaclient/PBookTyper.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index f033ca72..fc6089a0 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -131,6 +131,20 @@ def _parse_ids(raw): raise typer.Exit(1) +def _require_bool(name: str, value): + """Reject non-bool values for a bool parameter. + + Click enforces this on the CLI path (a flag is present or absent, never a stray + string), but commands are also called directly from the REPL, where a plain Python + call - e.g. finish(123, soft='gfd') - bypasses that check entirely and would + otherwise silently treat any truthy string as on. + """ + if not isinstance(value, bool): + typer.echo(f"Error: '{name}' must be True or False, got {value!r}", err=True) + raise typer.Exit(1) + return value + + def _setup() -> None: global _tmp_dir, _history_file, _setup_done if _setup_done: @@ -628,6 +642,7 @@ def finish( $ pbook finish 123,345,567 --soft """ + soft = _require_bool("soft", soft) core = _ensure_init() ids = _parse_ids(task_ids) if ids == "all": @@ -722,6 +737,7 @@ def debug( example: >>> debug(1234, True) """ + mode_on = _require_bool("mode_on", mode_on) core = _ensure_init() core.debug(panda_id, mode_on) @@ -770,6 +786,7 @@ def recover_lost_files( >>> recover_lost_files(123) >>> recover_lost_files(123, test_mode=True) """ + test_mode = _require_bool("test_mode", test_mode) core = _ensure_init() core.recover_lost_files(task_id, test_mode) @@ -891,6 +908,7 @@ def set_secret( >>> set_secret('mykey', 'myvalue') >>> set_secret('mykey', '/path/to/file', is_file=True) """ + is_file = _require_bool("is_file", is_file) core = _ensure_init() core.set_secret(key, value, is_file) @@ -927,6 +945,7 @@ def list_secrets( >>> list_secrets() >>> list_secrets(full=True) """ + full = _require_bool("full", full) core = _ensure_init() core.list_secrets(full) From 68540e65062c1fab37088a30b6a52823667cfa8b Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Mon, 3 Aug 2026 15:18:44 +0200 Subject: [PATCH 39/59] Validating True/False options --- pandaclient/PBookTyper.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index fc6089a0..01eeb48b 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -135,13 +135,14 @@ def _require_bool(name: str, value): """Reject non-bool values for a bool parameter. Click enforces this on the CLI path (a flag is present or absent, never a stray - string), but commands are also called directly from the REPL, where a plain Python - call - e.g. finish(123, soft='gfd') - bypasses that check entirely and would - otherwise silently treat any truthy string as on. + string) - unreachable from real CLI dispatch. Commands are also called directly + from the REPL though, where a plain Python call - e.g. finish(123, soft='gfd') - + bypasses that check entirely and would otherwise silently treat any truthy string + as on. A plain ValueError is the right signal there: the REPL's console reports it + with one clear line, same as any other misused Python call. """ if not isinstance(value, bool): - typer.echo(f"Error: '{name}' must be True or False, got {value!r}", err=True) - raise typer.Exit(1) + raise ValueError(f"'{name}' must be True or False, got {value!r}") return value From 92d48a8d0602f504eeaeb03e382d351cadd98276 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Mon, 3 Aug 2026 15:21:20 +0200 Subject: [PATCH 40/59] Validating True/False options --- pandaclient/PBookTyper.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index 01eeb48b..ecc4ee9d 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -131,18 +131,23 @@ def _parse_ids(raw): raise typer.Exit(1) +_INVALID = object() + + def _require_bool(name: str, value): - """Reject non-bool values for a bool parameter. + """Report non-bool values for a bool parameter, without raising. Click enforces this on the CLI path (a flag is present or absent, never a stray string) - unreachable from real CLI dispatch. Commands are also called directly from the REPL though, where a plain Python call - e.g. finish(123, soft='gfd') - bypasses that check entirely and would otherwise silently treat any truthy string - as on. A plain ValueError is the right signal there: the REPL's console reports it - with one clear line, same as any other misused Python call. + as on. Raising here would just dump a traceback into the interactive session, so + report the problem and let the caller return early instead: `if soft is _INVALID: + return`. """ if not isinstance(value, bool): - raise ValueError(f"'{name}' must be True or False, got {value!r}") + typer.echo(f"Error: '{name}' must be True or False, got {value!r}", err=True) + return _INVALID return value @@ -644,6 +649,8 @@ def finish( $ pbook finish 123,345,567 --soft """ soft = _require_bool("soft", soft) + if soft is _INVALID: + return core = _ensure_init() ids = _parse_ids(task_ids) if ids == "all": @@ -739,6 +746,8 @@ def debug( >>> debug(1234, True) """ mode_on = _require_bool("mode_on", mode_on) + if mode_on is _INVALID: + return core = _ensure_init() core.debug(panda_id, mode_on) @@ -788,6 +797,8 @@ def recover_lost_files( >>> recover_lost_files(123, test_mode=True) """ test_mode = _require_bool("test_mode", test_mode) + if test_mode is _INVALID: + return core = _ensure_init() core.recover_lost_files(task_id, test_mode) @@ -910,6 +921,8 @@ def set_secret( >>> set_secret('mykey', '/path/to/file', is_file=True) """ is_file = _require_bool("is_file", is_file) + if is_file is _INVALID: + return core = _ensure_init() core.set_secret(key, value, is_file) @@ -947,6 +960,8 @@ def list_secrets( >>> list_secrets(full=True) """ full = _require_bool("full", full) + if full is _INVALID: + return core = _ensure_init() core.list_secrets(full) From 6fc2761edc8d1c99598bf27734a194090dc1a0bd Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Tue, 4 Aug 2026 13:58:06 +0200 Subject: [PATCH 41/59] Re-enable retry.newOpts --- pandaclient/PBookTyper.py | 62 ++++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 24 deletions(-) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index ecc4ee9d..417b926d 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -669,16 +669,17 @@ def retry( excludedSite: Annotated[Optional[str], typer.Option("--excludedSite")] = None, includedSite: Annotated[Optional[str], typer.Option("--includedSite")] = None, nFilesPerJob: Annotated[Optional[int], typer.Option("--nFilesPerJob")] = None, - nMaxFilesPerJob: Annotated[Optional[int], typer.Option("--nMaxFilesPerJob")] = None, + nMaxFilesPerJob: Annotated[Optional[int], typer.Option("--nMaxFilesPerJob", "--maxNFilesPerJob")] = None, nGBPerJob: Annotated[Optional[float], typer.Option("--nGBPerJob")] = None, nFiles: Annotated[Optional[int], typer.Option("--nFiles")] = None, nEvents: Annotated[Optional[int], typer.Option("--nEvents")] = None, loopingCheck: Annotated[Optional[bool], typer.Option("--loopingCheck")] = None, - memory: Annotated[Optional[int], typer.Option("--memory")] = None, + memory: Annotated[Optional[int], typer.Option("--memory", "--ramCount")] = None, avoidVP: Annotated[Optional[bool], typer.Option("--avoidVP")] = None, ignoreMissingInDS: Annotated[Optional[bool], typer.Option("--ignoreMissingInDS")] = None, forceStaged: Annotated[Optional[bool], typer.Option("--forceStaged")] = None, maxCore: Annotated[Optional[int], typer.Option("--maxCore")] = None, + newOpts: Annotated[Optional[str], typer.Option("--new-opts", hidden=True)] = None, ) -> None: """Retry failed/cancelled tasks. @@ -693,35 +694,44 @@ def retry( days. It is possible to retry more tasks by setting the days and limit options. If named arguments are specified, they are applied to all retried tasks. + newOpts, a raw dict of task-retry options, overrides all of the individual options + above - kept for backward compatibility with pre-Typer pbook scripts/REPL usage. It is + a REPL-only convenience; there is no supported way to pass it from the shell CLI. + example: >>> retry(123) >>> retry([123, 345, 567]) + >>> retry(789, newOpts={'excludedSite': 'siteA,siteB'}) >>> retry(789, excludedSite='siteA,siteB') >>> retry('all') >>> retry('all', days=30, limit=2000) + >>> retry('all', newOpts={'excludedSite': 'siteA,siteB'}) """ core = _ensure_init() - new_opts = { - k: v - for k, v in { - "site": site, - "excludedSite": excludedSite, - "includedSite": includedSite, - "nFilesPerJob": nFilesPerJob, - "nMaxFilesPerJob": nMaxFilesPerJob, - "nGBPerJob": nGBPerJob, - "nFiles": nFiles, - "nEvents": nEvents, - "loopingCheck": loopingCheck, - "ramCount": memory, - "avoidVP": avoidVP, - "ignoreMissingInDS": ignoreMissingInDS, - "forceStaged": forceStaged, - "maxCoreCount": maxCore, - }.items() - if v is not None - } - opts = new_opts or None + if newOpts is not None: + opts = newOpts + else: + new_opts = { + k: v + for k, v in { + "site": site, + "excludedSite": excludedSite, + "includedSite": includedSite, + "nFilesPerJob": nFilesPerJob, + "nMaxFilesPerJob": nMaxFilesPerJob, + "nGBPerJob": nGBPerJob, + "nFiles": nFiles, + "nEvents": nEvents, + "loopingCheck": loopingCheck, + "ramCount": memory, + "avoidVP": avoidVP, + "ignoreMissingInDS": ignoreMissingInDS, + "forceStaged": forceStaged, + "maxCoreCount": maxCore, + }.items() + if v is not None + } + opts = new_opts or None ids = _parse_ids(task_ids) if isinstance(ids, list): return _parallel(lambda tid: core.retry(tid, newOpts=opts), ids) @@ -1011,7 +1021,11 @@ def _rewrite_legacy_kwargs(argv: list) -> list: for param in sub_cmd.params: flags = [o for o in getattr(param, "opts", []) if o.startswith("--")] if flags: - option_flags[param.name] = flags[0] + primary = flags[0] + # Register every alias (e.g. --ramCount for --memory), not just the primary + # name, so the legacy bare `key=value` syntax recognizes them too. + for flag in flags: + option_flags[flag.lstrip("-")] = primary if getattr(param, "is_flag", False): flag_only.add(param.name) From 0ffc5da777a05702c494737684e1749c9ff09f60 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Tue, 4 Aug 2026 15:10:58 +0200 Subject: [PATCH 42/59] Error in documentation - probably some flynt side-effect --- pandaclient/PBookScript.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandaclient/PBookScript.py b/pandaclient/PBookScript.py index c8e44334..e11fe0c5 100644 --- a/pandaclient/PBookScript.py +++ b/pandaclient/PBookScript.py @@ -254,11 +254,11 @@ def retry(taskIDs, newOpts=None, days=14, limit=1000, **kwargs): example: >>> retry(123) >>> retry([123, 345, 567]) - >>> retry(789, newOpts={{'excludedSite':'siteA,siteB'}}) + >>> retry(789, newOpts={'excludedSite':'siteA,siteB'}) >>> retry(789, excludedSite='siteA,siteB') >>> retry('all') >>> retry('all', days=30, limit=2000) - >>> retry('all', newOpts={{'excludedSite':'siteA,siteB'}}) + >>> retry('all', newOpts={'excludedSite':'siteA,siteB'}) """ if newOpts is None: newOpts = kwargs From 40b3d7df20845e844138fa03964a8d47122dd32f Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Tue, 4 Aug 2026 15:14:48 +0200 Subject: [PATCH 43/59] Error in documentation --- pandaclient/PBookScript.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandaclient/PBookScript.py b/pandaclient/PBookScript.py index e11fe0c5..c8e44334 100644 --- a/pandaclient/PBookScript.py +++ b/pandaclient/PBookScript.py @@ -254,11 +254,11 @@ def retry(taskIDs, newOpts=None, days=14, limit=1000, **kwargs): example: >>> retry(123) >>> retry([123, 345, 567]) - >>> retry(789, newOpts={'excludedSite':'siteA,siteB'}) + >>> retry(789, newOpts={{'excludedSite':'siteA,siteB'}}) >>> retry(789, excludedSite='siteA,siteB') >>> retry('all') >>> retry('all', days=30, limit=2000) - >>> retry('all', newOpts={'excludedSite':'siteA,siteB'}) + >>> retry('all', newOpts={{'excludedSite':'siteA,siteB'}}) """ if newOpts is None: newOpts = kwargs From cb9b18b359d49d773a6390e70210ba47ee18b9a7 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Tue, 4 Aug 2026 15:15:29 +0200 Subject: [PATCH 44/59] Extended documentation in retry --- pandaclient/PBookTyper.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index 417b926d..e0ece729 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -666,8 +666,8 @@ def retry( days: Annotated[int, typer.Option("--days", help="Look-back window when task_ids='all'")] = 14, limit: Annotated[int, typer.Option("--limit", help="Max tasks to retry when task_ids='all'")] = 1000, site: Annotated[Optional[str], typer.Option("--site")] = None, - excludedSite: Annotated[Optional[str], typer.Option("--excludedSite")] = None, - includedSite: Annotated[Optional[str], typer.Option("--includedSite")] = None, + excludedSite: Annotated[Optional[str], typer.Option("--excludedSite", help="Comma separated list of sites to exclude, e.g. 'siteA,siteB'")] = None, + includedSite: Annotated[Optional[str], typer.Option("--includedSite", help="Comma separated list of sites to include, e.g. 'siteA,siteB'")] = None, nFilesPerJob: Annotated[Optional[int], typer.Option("--nFilesPerJob")] = None, nMaxFilesPerJob: Annotated[Optional[int], typer.Option("--nMaxFilesPerJob", "--maxNFilesPerJob")] = None, nGBPerJob: Annotated[Optional[float], typer.Option("--nGBPerJob")] = None, @@ -681,9 +681,9 @@ def retry( maxCore: Annotated[Optional[int], typer.Option("--maxCore")] = None, newOpts: Annotated[Optional[str], typer.Option("--new-opts", hidden=True)] = None, ) -> None: - """Retry failed/cancelled tasks. + """Retry failed/canceled tasks. - Retry failed/cancelled subJobs in task_ids (ID or a list of IDs, can be either jediTaskID + Retry failed/canceled subJobs in task_ids (ID or a list of IDs, can be either jediTaskID or reqID). Allowed options to overwrite task parameters for new attempts: site, excludedSite, includedSite, nFilesPerJob, nMaxFilesPerJob, nGBPerJob, nFiles, nEvents, loopingCheck, memory, avoidVP, ignoreMissingInDS, forceStaged, maxCore. If input files From 250ae70918e49b8a64a91d224efffa292a0e92d2 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Wed, 5 Aug 2026 10:55:56 +0200 Subject: [PATCH 45/59] Cosmetic --- pandaclient/PBookTyper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index e0ece729..183e0a61 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -769,7 +769,7 @@ def get_user_job_metadata( ) -> None: """Write user metadata of successful jobs to a JSON file. - Get user metadata of successful jobs in a task and write them in a json file. + Get user metadata of successful jobs in a task and write them locally to a JSON file. example: >>> get_user_job_metadata(123, 'output.json') From 29ed2934f7b32b68464281410bfbad1fdfdcc6f7 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Thu, 6 Aug 2026 14:59:51 +0200 Subject: [PATCH 46/59] CLI names to _ --- pandaclient/PBookTyper.py | 46 +++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index 183e0a61..c4666bc3 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -535,7 +535,7 @@ def show( at 90 days unless a jediTaskID or reqID is specified, in which case tasks of any age are returned. See the default filter conditions in the annotations. - example: + examples: >>> show() >>> show(123) >>> show(12345678, format='long') @@ -543,6 +543,8 @@ def show( >>> show('run') >>> show('fin', days=7, limit=100) >>> show(format='json') + + $ pbook show --format=long --status=done --limit=100 """ core = _ensure_init() kwargs = { @@ -583,11 +585,13 @@ def showl( ) -> None: """Print task records in long format (shortcut for show --format='long'). - example: + examples: >>> showl() >>> showl(123) >>> showl(12345678) >>> showl(taskname='my_task_name') + + $ pbook showl --status=done --limit=100 """ return show( task_id, @@ -617,7 +621,9 @@ def kill( >>> kill([123, 345, 567]) >>> kill('all') + $ pbook kill 123 $ pbook kill 123,345,567 + $ pbook kill all """ core = _ensure_init() ids = _parse_ids(task_ids) @@ -706,6 +712,10 @@ def retry( >>> retry('all') >>> retry('all', days=30, limit=2000) >>> retry('all', newOpts={'excludedSite': 'siteA,siteB'}) + + $ pbook retry 123 + $ pbook retry 123 --excludedSite=siteA,siteB + """ core = _ensure_init() if newOpts is not None: @@ -754,6 +764,8 @@ def debug( example: >>> debug(1234, True) + + $ pbook debug 1234 True """ mode_on = _require_bool("mode_on", mode_on) if mode_on is _INVALID: @@ -762,7 +774,7 @@ def debug( core.debug(panda_id, mode_on) -@app.command(name="get-user-job-metadata") +@app.command(name="get_user_job_metadata") def get_user_job_metadata( task_id: Annotated[int, typer.Argument(help="Task ID")], output_file: Annotated[str, typer.Argument(help="Output JSON file path")], @@ -773,12 +785,14 @@ def get_user_job_metadata( example: >>> get_user_job_metadata(123, 'output.json') + + $ pbook get_user_job_metadata 123 output.json """ core = _ensure_init() core.getUserJobMetadata(task_id, output_file) -@app.command(name="reload-input") +@app.command(name="reload_input") def reload_input( task_id: Annotated[int, typer.Argument(help="Task ID")], ) -> None: @@ -793,7 +807,7 @@ def reload_input( core.reload_input(task_id) -@app.command(name="recover-lost-files") +@app.command(name="recover_lost_files") def recover_lost_files( task_id: Annotated[int, typer.Argument(help="Task ID")], test_mode: Annotated[bool, typer.Option("--test-mode", help="Dry-run mode")] = False, @@ -813,7 +827,7 @@ def recover_lost_files( core.recover_lost_files(task_id, test_mode) -@app.command(name="show-workflow") +@app.command(name="show_workflow") def show_workflow( request_id: Annotated[int, typer.Argument(help="Workflow request ID")], ) -> None: @@ -830,7 +844,7 @@ def show_workflow( print(output) -@app.command(name="kill-workflow") +@app.command(name="kill_workflow") def kill_workflow( request_id: Annotated[int, typer.Argument(help="Workflow request ID")], ) -> None: @@ -847,7 +861,7 @@ def kill_workflow( print(output[0][-1]) -@app.command(name="retry-workflow") +@app.command(name="retry_workflow") def retry_workflow( request_id: Annotated[int, typer.Argument(help="Workflow request ID")], ) -> None: @@ -864,7 +878,7 @@ def retry_workflow( print(output[0][-1]) -@app.command(name="finish-workflow") +@app.command(name="finish_workflow") def finish_workflow( request_id: Annotated[int, typer.Argument(help="Workflow request ID")], ) -> None: @@ -881,7 +895,7 @@ def finish_workflow( print(output[0][-1]) -@app.command(name="pause-workflow") +@app.command(name="pause_workflow") def pause_workflow( request_id: Annotated[int, typer.Argument(help="Workflow request ID")], ) -> None: @@ -898,7 +912,7 @@ def pause_workflow( print(output[0][-1]) -@app.command(name="resume-workflow") +@app.command(name="resume_workflow") def resume_workflow( request_id: Annotated[int, typer.Argument(help="Workflow request ID")], ) -> None: @@ -915,7 +929,7 @@ def resume_workflow( print(output[0][-1]) -@app.command(name="set-secret") +@app.command(name="set_secret") def set_secret( key: Annotated[str, typer.Argument(help="Secret key")], value: Annotated[str, typer.Argument(help="Secret value or file path")], @@ -937,7 +951,7 @@ def set_secret( core.set_secret(key, value, is_file) -@app.command(name="delete-secret") +@app.command(name="delete_secret") def delete_secret( key: Annotated[str, typer.Argument(help="Secret key to delete")], ) -> None: @@ -950,14 +964,14 @@ def delete_secret( core.set_secret(key, None) -@app.command(name="delete-all-secrets") +@app.command(name="delete_all_secrets") def delete_all_secrets() -> None: """Delete all secrets.""" core = _ensure_init() core.set_secret(None, None) -@app.command(name="list-secrets") +@app.command(name="list_secrets") def list_secrets( full: Annotated[bool, typer.Option("--full", help="Show full secret values")] = False, ) -> None: @@ -976,7 +990,7 @@ def list_secrets( core.list_secrets(full) -@app.command(name="generate-credential") +@app.command(name="generate_credential") def generate_credential() -> None: """Generate a new proxy or token.""" core = _get_core() From a9b70a55035de684fc7c22b005156e8642ce7d8c Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Thu, 6 Aug 2026 16:11:51 +0200 Subject: [PATCH 47/59] Bug in reload_input --- pandaclient/PBookCore.py | 4 ++-- pandaclient/PBookTyper.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pandaclient/PBookCore.py b/pandaclient/PBookCore.py index 30c2fc4c..cf6de082 100644 --- a/pandaclient/PBookCore.py +++ b/pandaclient/PBookCore.py @@ -497,10 +497,10 @@ def reload_input(self, task_id): tmp_log.error(output) tmp_log.error(f"Failed to reload input {task_id}") return False - elif output[0] != 0: + elif output[0] is False: tmp_log.error(output[-1]) tmp_log.error(f"Failed to reload input {task_id}") return False # done - tmp_log.info("command is registered. will be executed in a few minutes") + tmp_log.info(f"command is registered for task {task_id} and will be executed in a few minutes") return True diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index c4666bc3..887f11f5 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -802,6 +802,8 @@ def reload_input( example: >>> reload_input(123) + + $ pbook reload_input 123 """ core = _ensure_init() core.reload_input(task_id) From 0e3eeadd8b588058c5effc2bcbf815ac3d53cec6 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Thu, 6 Aug 2026 16:13:42 +0200 Subject: [PATCH 48/59] Bug in reload_input --- pandaclient/PBookCore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandaclient/PBookCore.py b/pandaclient/PBookCore.py index cf6de082..e3184b22 100644 --- a/pandaclient/PBookCore.py +++ b/pandaclient/PBookCore.py @@ -497,7 +497,7 @@ def reload_input(self, task_id): tmp_log.error(output) tmp_log.error(f"Failed to reload input {task_id}") return False - elif output[0] is False: + elif not output[0]: tmp_log.error(output[-1]) tmp_log.error(f"Failed to reload input {task_id}") return False From 841a5b0f2241dd3da0378ab088fd3e42a58b944c Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Thu, 6 Aug 2026 16:20:40 +0200 Subject: [PATCH 49/59] _ vs - bug --- pandaclient/PBookTyper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index 887f11f5..5ac1f451 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -812,7 +812,7 @@ def reload_input( @app.command(name="recover_lost_files") def recover_lost_files( task_id: Annotated[int, typer.Argument(help="Task ID")], - test_mode: Annotated[bool, typer.Option("--test-mode", help="Dry-run mode")] = False, + test_mode: Annotated[bool, typer.Option("--test_mode", help="Dry-run mode")] = False, ) -> None: """Request recovery of lost files from a task. From 8e127abdbb819dea31d42be786f261b9556ce5bb Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Thu, 6 Aug 2026 16:29:30 +0200 Subject: [PATCH 50/59] Some more examples --- pandaclient/PBookTyper.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index 5ac1f451..1b287b32 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -816,11 +816,13 @@ def recover_lost_files( ) -> None: """Request recovery of lost files from a task. - Send a request to recover lost files produced by a task. Set test_mode=True for testing. + Send a request to recover lost files produced by a task. Use test_mode for testing. example: >>> recover_lost_files(123) >>> recover_lost_files(123, test_mode=True) + + $ pbook recover_lost_files 123 --test_mode """ test_mode = _require_bool("test_mode", test_mode) if test_mode is _INVALID: @@ -935,7 +937,7 @@ def resume_workflow( def set_secret( key: Annotated[str, typer.Argument(help="Secret key")], value: Annotated[str, typer.Argument(help="Secret value or file path")], - is_file: Annotated[bool, typer.Option("--is-file", help="Treat value as a file path to upload")] = False, + is_file: Annotated[bool, typer.Option("--is_file", help="Treat value as a file path to upload")] = False, ) -> None: """Set a secret key-value pair. @@ -945,6 +947,9 @@ def set_secret( example: >>> set_secret('mykey', 'myvalue') >>> set_secret('mykey', '/path/to/file', is_file=True) + + $ pbook set_secret mykey myvalue + $ pbook set_secret mykey /path/to/file --is_file """ is_file = _require_bool("is_file", is_file) if is_file is _INVALID: @@ -961,6 +966,8 @@ def delete_secret( example: >>> delete_secret('mykey') + + $ pbook delete_secret mykey """ core = _ensure_init() core.set_secret(key, None) @@ -968,7 +975,13 @@ def delete_secret( @app.command(name="delete_all_secrets") def delete_all_secrets() -> None: - """Delete all secrets.""" + """Delete all secrets. + + example: + >>> delete_all_secrets + + $ pbook delete_all_secrets + """ core = _ensure_init() core.set_secret(None, None) @@ -984,6 +997,9 @@ def list_secrets( example: >>> list_secrets() >>> list_secrets(full=True) + + $ pbook list_secrets + $ pbook list_secrets --full """ full = _require_bool("full", full) if full is _INVALID: From c8747d62f3b42a24f93795dad9fd29fe6d1c9bef Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Thu, 6 Aug 2026 16:32:02 +0200 Subject: [PATCH 51/59] Some more examples --- pandaclient/PBookTyper.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index 1b287b32..15e81b55 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -822,6 +822,7 @@ def recover_lost_files( >>> recover_lost_files(123) >>> recover_lost_files(123, test_mode=True) + $ pbook recover_lost_files 123 $ pbook recover_lost_files 123 --test_mode """ test_mode = _require_bool("test_mode", test_mode) @@ -841,6 +842,8 @@ def show_workflow( example: >>> show_workflow(456) + + $ pbook show_workflow 456 """ core = _ensure_init() _, output = core.execute_workflow_command("get_status", request_id) @@ -858,6 +861,8 @@ def kill_workflow( example: >>> kill_workflow(456) + + $ pbook kill_workflow 456 """ core = _ensure_init() _, output = core.execute_workflow_command("abort", request_id) @@ -875,6 +880,8 @@ def retry_workflow( example: >>> retry_workflow(456) + + $ pbook retry_workflow 456 """ core = _ensure_init() _, output = core.execute_workflow_command("retry", request_id) @@ -892,6 +899,8 @@ def finish_workflow( example: >>> finish_workflow(456) + + $ pbook finish_workflow 456 """ core = _ensure_init() _, output = core.execute_workflow_command("finish", request_id) @@ -909,6 +918,8 @@ def pause_workflow( example: >>> pause_workflow(456) + + $ pbook pause_workflow 456 """ core = _ensure_init() _, output = core.execute_workflow_command("suspend", request_id) @@ -926,6 +937,8 @@ def resume_workflow( example: >>> resume_workflow(456) + + $ pbook resume_workflow 456 """ core = _ensure_init() _, output = core.execute_workflow_command("resume", request_id) From 569e98c98142581a17c6edce538e5162c66af52d Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Fri, 7 Aug 2026 11:37:10 +0200 Subject: [PATCH 52/59] PBookTyper documentation --- pandaclient/PBookTyper.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index 15e81b55..755fd7bb 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -1,4 +1,7 @@ """ +Do NOT import this module in your code. +Import PBookCore instead. + pbook CLI — PanDA task bookkeeper. Each command below is defined exactly once, as a Typer command using the @@ -6,8 +9,7 @@ annotation rather than in the default value, these functions remain ordinary callables with ordinary defaults - the same function is used to build the `pbook --flag ...` CLI (with real shell completion) *and* is placed -directly into the interactive REPL namespace (`>>> command(...)`), with no -separate REPL-only copy of the command's logic, docstring, or option list. +directly into the interactive REPL namespace (`>>> command(...)`). """ from __future__ import annotations From 200f138e2669300e281c89e19f306ed1d947de70 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Fri, 7 Aug 2026 11:56:12 +0200 Subject: [PATCH 53/59] PBookTyper documentation --- pandaclient/PBookTyper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index 755fd7bb..11ee847e 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -98,7 +98,7 @@ or -$ pbook help +$ pbook --help $ pbook command_name --help """ From bdaaf5f9105afed63ecd910ff7081069412eed32 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Fri, 7 Aug 2026 14:26:17 +0200 Subject: [PATCH 54/59] Detailed documentation per show option --- pandaclient/PBookTyper.py | 63 +++++++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 22 deletions(-) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index 11ee847e..919fb0e0 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -673,38 +673,56 @@ def retry( task_ids: Annotated[str, typer.Argument(help="Task ID, comma-separated IDs, or 'all'")], days: Annotated[int, typer.Option("--days", help="Look-back window when task_ids='all'")] = 14, limit: Annotated[int, typer.Option("--limit", help="Max tasks to retry when task_ids='all'")] = 1000, - site: Annotated[Optional[str], typer.Option("--site")] = None, - excludedSite: Annotated[Optional[str], typer.Option("--excludedSite", help="Comma separated list of sites to exclude, e.g. 'siteA,siteB'")] = None, - includedSite: Annotated[Optional[str], typer.Option("--includedSite", help="Comma separated list of sites to include, e.g. 'siteA,siteB'")] = None, - nFilesPerJob: Annotated[Optional[int], typer.Option("--nFilesPerJob")] = None, - nMaxFilesPerJob: Annotated[Optional[int], typer.Option("--nMaxFilesPerJob", "--maxNFilesPerJob")] = None, - nGBPerJob: Annotated[Optional[float], typer.Option("--nGBPerJob")] = None, - nFiles: Annotated[Optional[int], typer.Option("--nFiles")] = None, - nEvents: Annotated[Optional[int], typer.Option("--nEvents")] = None, - loopingCheck: Annotated[Optional[bool], typer.Option("--loopingCheck")] = None, - memory: Annotated[Optional[int], typer.Option("--memory", "--ramCount")] = None, - avoidVP: Annotated[Optional[bool], typer.Option("--avoidVP")] = None, - ignoreMissingInDS: Annotated[Optional[bool], typer.Option("--ignoreMissingInDS")] = None, - forceStaged: Annotated[Optional[bool], typer.Option("--forceStaged")] = None, - maxCore: Annotated[Optional[int], typer.Option("--maxCore")] = None, + site: Annotated[Optional[str], typer.Option("--site", help="Run the task on a particular PanDA queue")] = None, + excludedSite: Annotated[Optional[str], typer.Option("--excludedSite", help="Comma separated list of PanDA queues to exclude, e.g. 'siteA,siteB'")] = None, + includedSite: Annotated[Optional[str], typer.Option("--includedSite", help="Comma separated list of PanDA queues to include, e.g. 'siteA,siteB'")] = None, + nFilesPerJob: Annotated[Optional[int], typer.Option("--nFilesPerJob", help="Number of files on which each sub-job runs (default 50)")] = None, + nMaxFilesPerJob: Annotated[ + Optional[int], + typer.Option( + "--nMaxFilesPerJob", + "--maxNFilesPerJob", + help="Maximum number of input files to be processed by a single job in the task.", + ), + ] = None, + nGBPerJob: Annotated[ + Optional[float], + typer.Option("--nGBPerJob", help="Maximum input size in GB to be processed by a single job in the task.."), + ] = None, + nFiles: Annotated[Optional[int], typer.Option("--nFiles", help="Total number of input files to be processed by the task.")] = None, + nEvents: Annotated[ + Optional[int], + typer.Option("--nEvents", help="Total number of events to be processed by the task."), + ] = None, + loopingCheck: Annotated[ + Optional[bool], + typer.Option("--loopingCheck", help="Enable (True) or disable (False) the automatic check that kills jobs suspected of being stuck in a loop"), + ] = None, + memory: Annotated[Optional[int], typer.Option("--memory", "--ramCount", help="Required memory size in MB per core")] = None, + avoidVP: Annotated[Optional[bool], typer.Option("--avoidVP", help="Avoid PanDA queues which use Virtual Placement")] = None, + ignoreMissingInDS: Annotated[ + Optional[bool], typer.Option("--ignoreMissingInDS", help="Ignore missing input datasets which were deleted after the task is submitted.") + ] = None, + forceStaged: Annotated[ + Optional[bool], + typer.Option("--forceStaged", help="Force files from the primary dataset to be staged to local disk instead of using direct access"), + ] = None, + maxCore: Annotated[Optional[int], typer.Option("--maxCore", help="Maximum number of CPU cores that a single job is allowed to utilize")] = None, newOpts: Annotated[Optional[str], typer.Option("--new-opts", hidden=True)] = None, ) -> None: """Retry failed/canceled tasks. Retry failed/canceled subJobs in task_ids (ID or a list of IDs, can be either jediTaskID - or reqID). Allowed options to overwrite task parameters for new attempts: site, - excludedSite, includedSite, nFilesPerJob, nMaxFilesPerJob, nGBPerJob, nFiles, nEvents, - loopingCheck, memory, avoidVP, ignoreMissingInDS, forceStaged, maxCore. If input files - were used or are being used by other jobs for the same output dataset container, those + or reqID). You can specify options (site, excludedSite,...) to overwrite task parameters for new attempts. + If input files were used or are being used by other jobs for the same output dataset container, those files are skipped to avoid job duplication when retrying failed subjobs. If task_ids is 'all', it retries 1000 tasks at most that have finished for the last 14 days. It is possible to retry more tasks by setting the days and limit options. If named arguments are specified, they are applied to all retried tasks. - newOpts, a raw dict of task-retry options, overrides all of the individual options - above - kept for backward compatibility with pre-Typer pbook scripts/REPL usage. It is - a REPL-only convenience; there is no supported way to pass it from the shell CLI. + In interactive mode, newOpts can be passed with a raw dict of task-retry options and it overrides + all of the individual options above. newOpts is not supported from the shell CLI. example: >>> retry(123) @@ -716,7 +734,8 @@ def retry( >>> retry('all', newOpts={'excludedSite': 'siteA,siteB'}) $ pbook retry 123 - $ pbook retry 123 --excludedSite=siteA,siteB + $ pbook retry 123,345,567 --excludedSite=siteA,siteB + $ pbook retry all --days=30 --limit=2000 """ core = _ensure_init() From c3fff3e8c1cb6267188cb0089f970c1fc40d6de9 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Fri, 7 Aug 2026 14:34:37 +0200 Subject: [PATCH 55/59] Docstrings --- pandaclient/PBookTyper.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py index 919fb0e0..01d30922 100644 --- a/pandaclient/PBookTyper.py +++ b/pandaclient/PBookTyper.py @@ -114,6 +114,9 @@ def _parallel(func, items): + """ + Parallel execution of func with items in a pool of 8 threads. + """ with ThreadPoolExecutor(8) as pool: return list(pool.map(func, items)) @@ -1022,7 +1025,7 @@ def delete_all_secrets() -> None: @app.command(name="list_secrets") def list_secrets( - full: Annotated[bool, typer.Option("--full", help="Show full secret values")] = False, + full: Annotated[bool, typer.Option("--full", help="Show complete secret values instead of truncating them")] = False, ) -> None: """List secrets. From 91b21638823f80b5e3870346497c9f001fd6d78f Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Fri, 7 Aug 2026 14:36:09 +0200 Subject: [PATCH 56/59] Rename PBookTyper.py to PBookScript.py --- pandaclient/PBookScript.py | 1694 +++++++++++++++++++++++------------- 1 file changed, 1083 insertions(+), 611 deletions(-) diff --git a/pandaclient/PBookScript.py b/pandaclient/PBookScript.py index c8e44334..01d30922 100644 --- a/pandaclient/PBookScript.py +++ b/pandaclient/PBookScript.py @@ -1,657 +1,1129 @@ """ Do NOT import this module in your code. Import PBookCore instead. + +pbook CLI — PanDA task bookkeeper. + +Each command below is defined exactly once, as a Typer command using the +Annotated[...] parameter style. Because the Typer/Click metadata lives in the +annotation rather than in the default value, these functions remain ordinary +callables with ordinary defaults - the same function is used to build the +`pbook --flag ...` CLI (with real shell completion) *and* is placed +directly into the interactive REPL namespace (`>>> command(...)`). """ -import argparse +from __future__ import annotations + import atexit import code import os -import pydoc +import re import readline +import rlcompleter import signal import sys import tempfile -import textwrap from concurrent.futures import ThreadPoolExecutor +from inspect import Parameter, signature +from typing import ( + Annotated, + Literal, + Optional, + Union, + get_args, + get_origin, + get_type_hints, +) + +import typer +from rich import box +from rich.console import Console +from rich.markup import escape as _esc +from rich.table import Table from pandaclient import Client, PandaToolsPkgInfo from pandaclient.MiscUtils import commands_get_output +# ─── Runtime state ──────────────────────────────────────────────────────────── +_tmp_dir: Optional[str] = None +_history_file: Optional[str] = None +_fork_child_pid: Optional[int] = None +_setup_done: bool = False +_ctx_state: dict = {} +_core = None +_core_inited: bool = False -def list_parallel_exec(func, array): - with ThreadPoolExecutor(8) as thread_pool: - dataIterator = thread_pool.map(func, array) - return list(dataIterator) +help_text = """ +PanDA task bookkeeper. Run without arguments for interactive mode. +$ pbook [options] # interactive mode +$ pbook [options] command [args] [kwargs] # batch mode -# readline support -readline.parse_and_bind("tab: complete") -readline.parse_and_bind("set show-all-if-ambiguous On") +The same command can be executed in interactive mode: -# history support -pconfDir = os.path.expanduser(os.environ["PANDA_CONFIG_ROOT"]) -if not os.path.exists(pconfDir): - os.makedirs(pconfDir) -historyFile = "%s/.history" % pconfDir -# history file -if os.path.exists(historyFile): - try: - # except for macOS X - readline.read_history_file(historyFile) - except Exception: - pass -readline.set_history_length(1024) +$ pbook +>>> command(*args, **kwargs) -# set dummy CMTSITE -if "CMTSITE" not in os.environ: - os.environ["CMTSITE"] = "" +or in batch mode: -# make tmp dir -tmpDir = tempfile.mkdtemp() +$ pbook command arg1 arg2 ... argN --kwarg1=value1 --kwarg2=value2 ... --kwargN=valueN +$ pbook command arg1 arg2 ... argN kwarg1=value1 kwarg2=value2 ... kwargN=valueN +Please note that the latter option is kept for backward compatibility, but we plan to drop it in the future. -# fork PID -fork_child_pid = None +E.g. -# options of client tools -client_options = None +$ pbook +>>> show(123, format='long') +is equivalent to -# exit action -def _onExit(dirName, hFile): - # save history only for master process - if fork_child_pid == 0: - readline.write_history_file(hFile) - # remove tmp dir - commands_get_output("rm -rf %s" % dirName) +$ pbook show 123 --format='long' +$ pbook show 123 format='long' +If arg or value is a list in interactive mode, it is represented as a comma-separate list in batch mode. E.g. +to kill three tasks in interactive mode: -atexit.register(_onExit, tmpDir, historyFile) +$ pbook +>>> kill([123, 456, 789]) +or in batch mode: -# look for PandaTools package -for path in sys.path: - if path == "": - path = "." - if os.path.exists(path) and os.path.isdir(path) and "pandaclient" in os.listdir(path) and os.path.exists("%s/pandaclient/__init__.py" % path): - # make symlink for module name - os.symlink("%s/pandaclient" % path, "%s/taskbuffer" % tmpDir) - break -sys.path = [tmpDir] + sys.path - -from pandaclient import PBookCore # noqa: E402 - - -# main for interactive session -def intmain(pbookCore, comString, args_list): - # help - def help(*arg): - """ - Show the help doc - """ - if len(arg) > 0: - try: - if isinstance(arg[0], str): - func = main_locals[arg[0]] - else: - func = arg[0] - print(pydoc.plain(pydoc.render_doc(func))) - return - except Exception: - print(f"Unknown command : {str(arg[0])}") - # print available methods - tmp_str = """ -The following commands are available: - - help - show - showl - kill - retry - finish - debug - get_user_job_metadata - recover_lost_files - reload_input - show_workflow - kill_workflow - retry_workflow - finish_workflow - pause_workflow - resume_workflow - set_secret - list_secrets - delete_secret - delete_all_secrets - generate_credential - -For more info of each command, e.g. do "help(show)" in interactive mode or "help show" in batch mode. +$ pbook kill 123,456,789 + +To see the list of commands and help of each command, + +$ pbook +>>> help() +>>> help(command_name) + +or + +$ pbook --help +$ pbook command_name --help """ - print(tmp_str) - - # show status - def show(*args, **kwargs): - """ - Print task records. The first argument (non-keyword) can be an jediTaskID or reqID, or 'run' (show active tasks only), or 'fin' (show terminated tasks only), or can be omitted. The following keyword arguments are available to filter the tasks: [username, limit, taskname, days, jeditaskid, reqid, status, superstatus]. - Records are fetched directly from the PanDA server, so they are always up to date (the sync keyword is kept for backward compatibility but has no effect). Note that days is capped at 90 days unless a jediTaskID or reqID is specified, in which case tasks of any age are returned. - Specify display format with format='xxx', available formats are ['standard', 'long', 'json', 'plain']. - The default filter conditions are: username=(name from user voms proxy), limit=1000, days=14, format='standard'. - - example: - >>> show() - >>> show(123) - >>> show(12345678, format='long') - >>> show(taskname='my_task_name') - >>> show('run') - >>> show('fin', days=7, limit=100) - >>> show(format='json') - """ - return pbookCore.show(*args, **kwargs) - - # shortcut to show long status - def showl(*args, **kwargs): - """ - Print task records in long format; shortcut function of show(..., format='long'). See help message of show() for other keywords arguments - - example: - >>> showl() - >>> showl(123) - >>> showl(12345678) - >>> showl(taskname='my_task_name') - """ - kwargs["format"] = "long" - return pbookCore.show(*args, **kwargs) - - # kill - def kill(taskIDs): - """ - Kill all subJobs in taskIDs (ID or a list of ID, can be either jediTaskID or reqID). If 'all', kill all active tasks of the user. - - example: - >>> kill(123) - >>> kill([123, 345, 567]) - >>> kill('all') - """ - if taskIDs == "all": - # active tasks - task_list = pbookCore.get_active_tasks() - ret = list_parallel_exec(lambda task: pbookCore.kill(task.jeditaskid), task_list) - elif isinstance(taskIDs, (list, tuple)): - ret = list_parallel_exec(lambda taskID: pbookCore.kill(taskID), taskIDs) - elif isinstance(taskIDs, int): - ret = [pbookCore.kill(taskIDs)] - else: - print("Error: Invalid argument") - ret = None - return ret - - # finish - def finish(taskIDs, soft=False): - """ - Finish all subJobs in taskIDs (ID or a list of ID, can be either jediTaskID or reqID). If taskIDs is 'all', - finish all active tasks of the user. If soft is False (default), all running jobs are killed and the task - finishes immediately. If soft is True, new jobs are not generated and the task finishes once all running - jobs finish. - - example: - >>> finish(123) - >>> finish(234, soft=True) - >>> finish([123, 345, 567]) - >>> finish('all') - """ - if taskIDs == "all": - # active tasks - task_list = pbookCore.get_active_tasks() - ret = list_parallel_exec( - lambda task: pbookCore.finish.original_func(pbookCore, task.jeditaskid, soft=soft), - task_list, - ) - elif isinstance(taskIDs, (list, tuple)): - ret = list_parallel_exec(lambda taskID: pbookCore.finish(taskID, soft=soft), taskIDs) - elif isinstance(taskIDs, int): - ret = [pbookCore.finish(taskIDs, soft=soft)] - else: - print("Error: Invalid argument") - ret = None - return ret - - # retry - _retry_allowed_opts = [ - "site", - "excludedSite", - "includedSite", - "nFilesPerJob", - "nMaxFilesPerJob", - "nGBPerJob", - "nFiles", - "nEvents", - "loopingCheck", - "maxNFilesPerJob", - "memory", - "ramCount", - "avoidVP", - "ignoreMissingInDS", - "forceStaged", - "maxCore", - ] - - def retry(taskIDs, newOpts=None, days=14, limit=1000, **kwargs): - """ - Retry failed/cancelled subJobs in taskIDs (ID or a list of ID, can be either jediTaskID or reqID). - It is possible to specify newOpts, which is a map of options and new arguments like - {{'nFilesPerJob':10,'excludedSite':'ABC,XYZ'}}, to overwrite task parameters for new attempts. - The list of changeable parameters is - {allowed_opts} - It is also possible to specify those parameters as named arguments of the retry function, - e.g. nFilesPerJob=10, excludedSite='ABC,XYZ'. If input files were used or are being used by other - jobs for the same output dataset container, those files are skipped to avoid job duplication when - retrying failed subjobs. - - If taskIDs is 'all', it retries 1000 tasks at most that have finished for the last 14 days. It is possible - to retry more tasks by setting the days and limit options. If newOpts and/or named arguments are specified, - they are applied to all retried tasks. - - example: - >>> retry(123) - >>> retry([123, 345, 567]) - >>> retry(789, newOpts={{'excludedSite':'siteA,siteB'}}) - >>> retry(789, excludedSite='siteA,siteB') - >>> retry('all') - >>> retry('all', days=30, limit=2000) - >>> retry('all', newOpts={{'excludedSite':'siteA,siteB'}}) - """ - if newOpts is None: - newOpts = kwargs - if newOpts: - # check options against the allowed list - for key in list(newOpts): - if key == "memory": - newOpts["ramCount"] = newOpts[key] - del newOpts[key] - elif key == "maxCore": - newOpts["maxCoreCount"] = newOpts[key] - del newOpts[key] - elif key not in _retry_allowed_opts: - print('Error: Unknown option name "%s"' % key) - return None - if isinstance(taskIDs, (list, tuple)): - ret = list_parallel_exec(lambda taskID: pbookCore.retry(taskID, newOpts=newOpts), taskIDs) - elif isinstance(taskIDs, int): - ret = [pbookCore.retry(taskIDs, newOpts=newOpts)] - elif taskIDs == "all": - dataList = pbookCore.show(status="finished", days=days, limit=limit, format="json") - ret = list_parallel_exec( - lambda data: pbookCore.retry.original_func(pbookCore, data["jediTaskID"], newOpts=newOpts), - dataList, - ) - else: - print("Error: Invalid argument") - ret = None - return ret - - _opts_list = ", ".join(_retry_allowed_opts) - # 8-space docstring indent - _first_indent = " " * 8 - _subsequent_indent = " " * 16 - _wrapped_list = textwrap.fill(_first_indent + _opts_list, width=80, subsequent_indent=_subsequent_indent) - _wrapped = _wrapped_list - retry.__doc__ = ( - retry.__doc__.replace( - "{allowed_opts}", - _wrapped, - ) - .replace("{{", "{") - .replace("}}", "}") - ) - # debug mode - def debug(PandaID, modeOn): - """ - Turn the debug mode on/off for a subjob with PandaID. modeOn is True/False to enable/disable the debug mode. Note that the maxinum number of debug subjobs is limited. If you already hit the limit you need to disable the debug mode for a subjob before debugging another subjob - - example: - >>> debug(1234, True) - """ - pbookCore.debug(PandaID, modeOn) - - # get user job metadata - def getUserJobMetadata(taskID, outputFileName): - pbookCore.getUserJobMetadata(taskID, outputFileName) - - # get user job metadata - def get_user_job_metadata(taskID, outputFileName): - """ - Get user metadata of successful jobs in a task and write them in a json file - - example: - >>> get_user_job_metadata(123, 'output.json') - """ - getUserJobMetadata(taskID, outputFileName) - - # reload input dataset and retry - def reload_input(task_id): - """ - Reload input dataset and retry the task with new contents. This is useful when input dataset contents are - changed after the task is submitted - - example: - >>> reload_input(123) - """ - pbookCore.reload_input(task_id) - - # recover lost files - def recover_lost_files(taskID, test_mode=False): - """ - Send a request to recover lost files produced by a task. Set test_mode=True for testing - - example: - >>> recover_lost_files(123) - >>> recover_lost_files(123, test_mode=True) - """ - pbookCore.recover_lost_files(taskID, test_mode) - - # finish a workflow - def finish_workflow(request_id): - """ - Send a request to finish a workflow - - """ - status, output = pbookCore.execute_workflow_command("finish", request_id) - if output: - print(output[0][-1]) - - # kill a workflow - def kill_workflow(request_id): - """ - Send a request to kill a workflow - - """ - status, output = pbookCore.execute_workflow_command("abort", request_id) - if output: - print(output[0][-1]) - - # pause a workflow - def pause_workflow(request_id): - """ - Send a request to pause a workflow - - """ - status, output = pbookCore.execute_workflow_command("suspend", request_id) - if output: - print(output[0][-1]) - - # resume a workflow - def resume_workflow(request_id): - """ - Send a request to resume a workflow - - """ - status, output = pbookCore.execute_workflow_command("resume", request_id) - if output: - print(output[0][-1]) - - # retry a workflow - def retry_workflow(request_id): - """ - Send a request to retry a workflow - - """ - status, output = pbookCore.execute_workflow_command("retry", request_id) - if output: - print(output[0][-1]) - - # show a workflow - def show_workflow(request_id): - """ - Show a workflow - - """ - status, output = pbookCore.execute_workflow_command("get_status", request_id) - if output: - print(output) - - # set a secret - def set_secret(key, value, is_file=False): - """ - Define a pair of secret key-value strings. The value can be a file path to upload a secret file when is_file=True - - """ - pbookCore.set_secret(key, value, is_file) - - # delete a secret - def delete_secret(key): - """ - Delete a secret - - """ - pbookCore.set_secret(key, None) - - # delete all secrets - def delete_all_secrets(): - """ - Delete all secrets - - """ - pbookCore.set_secret(None, None) - - # list secrets - def list_secrets(full=False): - """ - List secrets. Value strings are truncated by default. full=True to see entire strings - - """ - pbookCore.list_secrets(full) - - # generate credential - def generate_credential(): - """ - Generate a new proxy or token - - """ - pbookCore.generate_credential() - - main_locals = locals() - - # execute command in the batch mode - if comString != "": - pbookCore.init() - exec(comString) in globals(), locals() - # exit - if PBookCore.func_return_value: - sys.exit(0) - else: - sys.exit(1) - - # execute with args in the batch mode - if args_list: - func_name = args_list.pop(0) - if func_name not in locals(): - print(f"ERROR : function {func_name} is undefined") - sys.exit(1) - - # convert arg string - def _conv_str(some_string): - if "," in some_string: - try: - return [int(s) for s in some_string.split(",")] - except Exception: - return some_string.split(",") - else: - if some_string == "None": - return None - if some_string == "True": - return True - if some_string == "False": - return False - try: - return int(some_string) - except Exception: - return some_string - - # separate args and kwargs - args = [] - kwargs = {} - for arg in args_list: - if "=" in arg: - k, v = arg.split("=") - kwargs[k] = _conv_str(v) - else: - args.append(_conv_str(arg)) - # execute - if func_name not in ["help", "generate_credential"]: - pbookCore.init(sanity_check=False) - locals()[func_name](*args, **kwargs) - - # exit - if PBookCore.func_return_value: - sys.exit(0) - else: - sys.exit(1) - - # go to interactive prompt - pbookCore.init() - code.interact(banner="\nStart pBook %s" % PandaToolsPkgInfo.release_version, local=locals()) - - -# kill whole process -def catch_sig(sig, frame): - # cleanup - _onExit(tmpDir, historyFile) - # kill - commands_get_output("kill -9 -- -%s" % os.getpgrp()) - - -# overall main -def main(): - # parse option - usage = """ - $ pbook [options] # interactive mode - $ pbook [options] command [args] [kwargs] # batch mode - - The same command can be executed in interactive mode: - - $ pbook - >>> command(*args, **kwargs) - - or in batch mode: - - $ pbook command arg1 arg2 ... argN kwarg1=value1 kwarg2=value2 ... kwargN=valueN - - E.g. - - $ pbook - >>> show(123, format='long') - - is equivalent to - - $ pbook show 123 format='long' - - If arg or value is a list in interactive mode, it is represented as a comma-separate list in batch mode. E.g. - to kill three tasks in interactive mode: - - $ pbook - >>> kill([123, 456, 789]) - - or in batch mode: - - $ pbook kill 123,456,789 - - To see the list of commands and help of each command, - - $ pbook - >>> help() - >>> help(command_name) - - or - - $ pbook help - $ pbook help command_name +app = typer.Typer( + name="pbook", + help=help_text, + invoke_without_command=True, + no_args_is_help=False, + context_settings={"help_option_names": ["-h", "--help"]}, +) + +# ─── Utilities ──────────────────────────────────────────────────────────────── + + +def _parallel(func, items): """ - parser = argparse.ArgumentParser(conflict_handler="resolve", usage=usage) - parser.add_argument("-v", action="store_true", dest="verbose", default=False, help="Verbose") - parser.add_argument( - "-c", - action="store", - dest="comString", - default="", - type=str, - help="Execute a python code snippet", - ) - parser.add_argument("-3", action="store_true", dest="python3", default=False, help="Use python3") - parser.add_argument( - "--version", - action="store_const", - const=True, - dest="version", - default=False, - help="Displays version", - ) - parser.add_argument( - "--devSrv", - action="store_const", - const=True, - dest="devSrv", - default=False, - help=argparse.SUPPRESS, - ) - parser.add_argument( - "--intrSrv", - action="store_const", - const=True, - dest="intrSrv", - default=False, - help=argparse.SUPPRESS, - ) - # option for jupyter notebook - parser.add_argument( - "--prompt_with_newline", - action="store_const", - const=True, - dest="prompt_with_newline", - default=False, - help=argparse.SUPPRESS, - ) + Parallel execution of func with items in a pool of 8 threads. + """ + with ThreadPoolExecutor(8) as pool: + return list(pool.map(func, items)) + - options, args = parser.parse_known_args() +def _parse_ids(raw): + """'all' -> 'all'; '42' -> 42; '1,2,3' -> [1,2,3]; anything not a string passes through as-is.""" + if not isinstance(raw, str): + return raw + if raw == "all": + return "all" + parts = raw.split(",") + try: + ids = [int(p) for p in parts] + return ids[0] if len(ids) == 1 else ids + except ValueError: + typer.echo(f"Error: invalid task ID(s): {raw}", err=True) + raise typer.Exit(1) - # display version - if options.version: - print("Version: %s" % PandaToolsPkgInfo.release_version) - sys.exit(0) - # use dev server - if options.devSrv: - Client.useDevServer() +_INVALID = object() + + +def _require_bool(name: str, value): + """Report non-bool values for a bool parameter, without raising. + + Click enforces this on the CLI path (a flag is present or absent, never a stray + string) - unreachable from real CLI dispatch. Commands are also called directly + from the REPL though, where a plain Python call - e.g. finish(123, soft='gfd') - + bypasses that check entirely and would otherwise silently treat any truthy string + as on. Raising here would just dump a traceback into the interactive session, so + report the problem and let the caller return early instead: `if soft is _INVALID: + return`. + """ + if not isinstance(value, bool): + typer.echo(f"Error: '{name}' must be True or False, got {value!r}", err=True) + return _INVALID + return value + + +def _setup() -> None: + global _tmp_dir, _history_file, _setup_done + if _setup_done: + return + _setup_done = True + + readline.parse_and_bind("tab: complete") + readline.parse_and_bind("set show-all-if-ambiguous On") + + if "CMTSITE" not in os.environ: + os.environ["CMTSITE"] = "" + + pconf_dir = os.path.expanduser(os.environ.get("PANDA_CONFIG_ROOT", "~/.panda")) + os.makedirs(pconf_dir, exist_ok=True) + + _history_file = os.path.join(pconf_dir, ".history") + if os.path.exists(_history_file): + try: + readline.read_history_file(_history_file) + except Exception: + pass + readline.set_history_length(1024) + + _tmp_dir = tempfile.mkdtemp() + + for path in sys.path: + real = path or "." + if ( + os.path.exists(real) + and os.path.isdir(real) + and "pandaclient" in os.listdir(real) + and os.path.exists(os.path.join(real, "pandaclient", "__init__.py")) + ): + link = os.path.join(_tmp_dir, "taskbuffer") + if not os.path.exists(link): + os.symlink(os.path.join(real, "pandaclient"), link) + break + if _tmp_dir not in sys.path: + sys.path.insert(0, _tmp_dir) + + atexit.register(_cleanup) + + +def _cleanup() -> None: + if _fork_child_pid == 0 and _history_file: + readline.write_history_file(_history_file) + + if _tmp_dir: + commands_get_output(f"rm -rf {_tmp_dir}") + - # use INTR server - if options.intrSrv: +def _make_core(verbose: bool = False): + from pandaclient import PBookCore + + return PBookCore.PBookCore(verbose=verbose) + + +def _get_core(): + """Return the (memoized) core for this process, uninitialized.""" + global _core + _setup() + if _core is None: + _core = _make_core(_ctx_state.get("verbose", False)) + return _core + + +def _ensure_init(sanity_check: bool = False): + """Return the core, running PBookCore.init() exactly once per process. + + The REPL calls this once upfront with sanity_check=True; every command + function then calls it again with the default before touching the core, + which is a no-op once already initialized - so commands stay correct + whether they run once (batch mode) or repeatedly (REPL session). + """ + global _core_inited + core = _get_core() + if not _core_inited: + core.init(sanity_check=sanity_check) + _core_inited = True + return core + + +def _catch_sig(sig, frame): + _cleanup() + # Hard kill all processes in the group + commands_get_output(f"kill -9 -- -{os.getpgrp()}") + + +# ─── REPL namespace & completion ────────────────────────────────────────────── + + +def _build_namespace() -> dict: + """The REPL namespace: every registered Typer command, keyed by its real Python name.""" + return {info.callback.__name__: info.callback for info in app.registered_commands} + + +def _kwarg_names(func) -> list: + """All parameter names a function accepts - candidates for `name=` completion.""" + return list(signature(func).parameters) + + +def _kwarg_choices(func, name: str) -> list: + """Value choices for a parameter, derived from its type hint: Literal[...] members or True/False for bool. + + Returned bare (unquoted) - readline's own quote-matching auto-closes an opening quote + the user already typed, so we don't need to (and shouldn't try to) add one ourselves. + """ + try: + hints = get_type_hints(func, include_extras=True) + except Exception: + return [] + ann = hints.get(name) + if ann is None: + return [] + while hasattr(ann, "__metadata__"): + ann = ann.__origin__ + if get_origin(ann) is Union: + non_none = [a for a in get_args(ann) if a is not type(None)] + if len(non_none) == 1: + ann = non_none[0] + if get_origin(ann) is Literal: + return [str(v) for v in get_args(ann)] + if ann is bool: + return ["True", "False"] + return [] + + +class _PBookCompleter: + """Readline completer: kwarg names and values when inside a call, names otherwise.""" + + def __init__(self, ns: dict) -> None: + self._ns = ns + self._base = rlcompleter.Completer(ns) + self._matches: list = [] + + def complete(self, text: str, state: int) -> Optional[str]: + if state == 0: + self._matches = self._compute(text) + return self._matches[state] if state < len(self._matches) else None + + def _compute(self, text: str) -> list: + line = readline.get_line_buffer() + + # Kwarg value completion (tier 1): last token is kwarg= or kwarg='partial + # Return bare values - readline's own quote-matching auto-closes an opening quote + # the user already typed, so we deliberately don't add quotes ourselves here. + m_val = re.search(r"\b(\w+)\s*=\s*(['\"]?)(\w*)$", line) + m_func = re.match(r"(\w+)\s*\(", line) + if m_val and m_func: + kwarg, partial = m_val.group(1), m_val.group(3) + func = self._ns.get(m_func.group(1)) + if func is not None: + # We're unambiguously past a `kwarg=` - this is a value position, not a + # name position, even if this particular kwarg has no enumerable choices + # (e.g. limit: int). Return here regardless, so an empty result doesn't + # fall through to tier 2's kwarg-name completion. + return [v for v in _kwarg_choices(func, kwarg) if v.startswith(partial)] + + # Kwarg name completion (tier 2): cursor is inside an open call + m = re.search(r"(\w+)\s*\([^)]*$", line) + if m: + func = self._ns.get(m.group(1)) + if func is not None: + hits = [k for k in _kwarg_names(func) if k.startswith(text)] + if hits: + return hits + + # Plain name completion (tier 3, rlcompleter fallback) + if not text: + # rlcompleter.complete() special-cases blank text by calling readline.insert_text() + # itself, which re-enters readline from inside this callback and confuses the active + # Tab press; list the namespace directly instead of delegating to it here + return sorted(k for k in self._base.namespace if not k.startswith("_")) + results, i = [], 0 + while (c := self._base.complete(text, i)) is not None: + results.append(c.rstrip("()").rstrip("(")) + i += 1 + return results + + +def _run_repl(ns: dict, banner: str) -> None: + """Manual REPL using InteractiveConsole.push() so we own readline setup entirely.""" + completer = _PBookCompleter(ns) + readline.set_completer(completer.complete) + readline.parse_and_bind("tab: complete") + readline.parse_and_bind("set show-all-if-ambiguous On") + + console = code.InteractiveConsole(ns) + print(banner) + + more = False + while True: + prompt = "... " if more else ">>> " + try: + readline.set_completer(completer.complete) + line = input(prompt) + except EOFError: + print() + break + except KeyboardInterrupt: + print("\nKeyboardInterrupt") + console.resetbuffer() + more = False + continue + more = console.push(line) + + +# ─── Top-level callback ─────────────────────────────────────────────────────── + + +@app.callback(invoke_without_command=True) +def _main( + ctx: typer.Context, + verbose: bool = typer.Option(False, "-v", help="Verbose"), + command_string: Optional[str] = typer.Option(None, "-c", help="Execute a Python code snippet"), + version: bool = typer.Option(False, "--version", is_eager=True, help="Display version"), + dev_srv: bool = typer.Option(False, "--devSrv", hidden=True), + intr_srv: bool = typer.Option(False, "--intrSrv", hidden=True), + prompt_with_newline: bool = typer.Option(False, "--prompt_with_newline", hidden=True), + python3: bool = typer.Option(False, "-3", hidden=True), +) -> None: + """PanDA task bookkeeper. Run without arguments for interactive mode.""" + if version: + typer.echo(f"Version: {PandaToolsPkgInfo.release_version}") + raise typer.Exit() + + if dev_srv: + Client.useDevServer() + if intr_srv: Client.useIntrServer() - # fork for Ctl-c - global fork_child_pid - fork_child_pid = os.fork() - if fork_child_pid == -1: - print("ERROR : Failed to fork") - sys.exit(1) - if fork_child_pid == 0: - # main - if options.verbose: - print(options) - if options.prompt_with_newline: + _ctx_state.update({"verbose": verbose}) + + if ctx.invoked_subcommand is not None: + return + + # Interactive or snippet mode + _setup() + global _fork_child_pid + _fork_child_pid = os.fork() + + if _fork_child_pid == -1: + typer.echo("ERROR: Failed to fork", err=True) + raise typer.Exit(1) + + if _fork_child_pid == 0: + if verbose: + typer.echo(str(ctx.params)) + if prompt_with_newline: sys.ps1 = ">>> \n" - # instantiate core - pbookCore = PBookCore.PBookCore(verbose=options.verbose) - # execute - intmain(pbookCore, options.comString, args) + _ensure_init(sanity_check=True) + ns = _build_namespace() + + if command_string: + exec(command_string, {}, ns) # noqa: S102 + from pandaclient import PBookCore as _PBC + + raise typer.Exit(0 if _PBC.func_return_value else 1) + _run_repl(ns, banner=f"\nStart pBook {PandaToolsPkgInfo.release_version}") + else: - # set handler - signal.signal(signal.SIGINT, catch_sig) - signal.signal(signal.SIGHUP, catch_sig) - signal.signal(signal.SIGTERM, catch_sig) + signal.signal(signal.SIGINT, _catch_sig) + signal.signal(signal.SIGHUP, _catch_sig) + signal.signal(signal.SIGTERM, _catch_sig) pid, status = os.wait() if os.WIFSIGNALED(status): - sys.exit(-os.WTERMSIG(status)) + raise typer.Exit(-os.WTERMSIG(status)) elif os.WIFEXITED(status): - sys.exit(os.WEXITSTATUS(status)) + raise typer.Exit(os.WEXITSTATUS(status)) + raise typer.Exit(0) + + +# ─── Commands ────────────────────────────────────────────────────────────────── + +_HELP_GROUPS = [ + ("Tasks", ["show", "showl", "kill", "finish", "retry", "debug"]), + ("Files & input", ["get_user_job_metadata", "recover_lost_files", "reload_input"]), + ("Workflows", ["show_workflow", "kill_workflow", "retry_workflow", "finish_workflow", "pause_workflow", "resume_workflow"]), + ("Secrets", ["set_secret", "list_secrets", "delete_secret", "delete_all_secrets"]), + ("Auth", ["generate_credential"]), +] + + +def _type_name(ann) -> str: + """Render a resolved type annotation as a short, human-readable name (Optional[str], Literal[...], etc.).""" + if ann is None or ann is type(None): + return "" + origin = get_origin(ann) + if origin is Union: + args = get_args(ann) + non_none = [a for a in args if a is not type(None)] + if len(non_none) == 1 and len(args) == 2: + return f"Optional[{_type_name(non_none[0])}]" + return " | ".join(_type_name(a) for a in args) + if origin is Literal: + return "Literal[" + ", ".join(repr(v) for v in get_args(ann)) + "]" + if origin is not None: + args = get_args(ann) + origin_name = getattr(origin, "__name__", str(origin)) + return f"{origin_name}[{', '.join(_type_name(a) for a in args)}]" if args else origin_name + return getattr(ann, "__name__", str(ann)) + + +def _format_signature(func) -> str: + """A clean '(param: Type = default, ...)' string, stripping the Typer/Annotated plumbing.""" + try: + hints = get_type_hints(func, include_extras=True) + except Exception: + hints = {} + parts = [] + for pname, p in signature(func).parameters.items(): + ann = hints.get(pname) + while hasattr(ann, "__metadata__"): + ann = ann.__origin__ + piece = pname + type_str = _type_name(ann) + if type_str: + piece += f": {type_str}" + if p.default is not Parameter.empty: + piece += f" = {p.default!r}" + parts.append(piece) + return f"({', '.join(parts)})" + + +@app.command() +def help( + command: Annotated[Optional[str], typer.Argument(help="Command name for detailed help")] = None, +) -> None: + """Show available commands, or detailed help for a specific command.""" + ns = _build_namespace() + console = Console() + + if command is not None: + name = command if isinstance(command, str) else command.__name__ + func = ns.get(name, command if callable(command) else None) + if func is None: + console.print(f"[red]Unknown command:[/red] {_esc(name)}") + return + console.print(f"\n[bold cyan]{name}[/bold cyan][bold]{_esc(_format_signature(func))}[/bold]") + doc = (func.__doc__ or "No description.").strip() + console.print(f"\n{_esc(doc)}\n") + return + + table = Table(box=box.SIMPLE, show_header=True, header_style="bold magenta") + table.add_column("Command", style="bold cyan", no_wrap=True) + table.add_column("Description") + table.add_column("Signature") + + for group, names in _HELP_GROUPS: + table.add_section() + table.add_row(f"[bold white]{group}[/bold white]", "", "") + for name in names: + func = ns.get(name) + if func is None: + continue + doc = (func.__doc__ or "").strip().splitlines()[0] if func.__doc__ else "" + sig = console.highlighter(_format_signature(func)) + table.add_row(f" {name}", _esc(doc), sig) + + console.print(table) + console.print("Usage: [bold]help(show)[/bold] or [bold]pbook show --help[/bold]\n") + + +@app.command() +def show( + task_id: Annotated[Optional[str], typer.Argument(help="jediTaskID, reqID, 'run' (active only), or 'fin' (terminated only)")] = None, + username: Annotated[Optional[str], typer.Option(help="Filter by username. By default, the name from the voms/token is used.")] = None, + limit: Annotated[int, typer.Option(help="Maximum number of records")] = 1000, + taskname: Annotated[Optional[str], typer.Option(help="Filter by task name")] = None, + days: Annotated[int, typer.Option(help="Look back N days (capped at 90 without a task ID)")] = 14, + jeditaskid: Annotated[Optional[int], typer.Option(help="Filter by jediTaskID")] = None, + reqid: Annotated[Optional[int], typer.Option(help="Filter by reqID")] = None, + status: Annotated[Optional[str], typer.Option(help="Filter by task status")] = None, + superstatus: Annotated[Optional[str], typer.Option(help="Filter by super-status")] = None, + format: Annotated[Literal["standard", "long", "json", "plain"], typer.Option("--format", help="Output format")] = "standard", +) -> None: + """Print task records. + + The first non-keyword argument (task_id) can be a jediTaskID or reqID, or 'run' (show active tasks + only), or 'fin' (show terminated tasks only), or can be omitted. Records are fetched + directly from the PanDA server, so they are always up to date. Note that days is capped + at 90 days unless a jediTaskID or reqID is specified, in which case tasks of any age are + returned. See the default filter conditions in the annotations. + + examples: + >>> show() + >>> show(123) + >>> show(12345678, format='long') + >>> show(taskname='my_task_name') + >>> show('run') + >>> show('fin', days=7, limit=100) + >>> show(format='json') + + $ pbook show --format=long --status=done --limit=100 + """ + core = _ensure_init() + kwargs = { + k: v + for k, v in dict( + username=username, + limit=limit, + taskname=taskname, + days=days, + jeditaskid=jeditaskid, + reqid=reqid, + status=status, + superstatus=superstatus, + ).items() + if v is not None + } + kwargs["format"] = format + if task_id is not None: + try: + first_arg = int(task_id) + except (TypeError, ValueError): + first_arg = task_id + return core.show(first_arg, **kwargs) + return core.show(**kwargs) + + +@app.command() +def showl( + task_id: Annotated[Optional[str], typer.Argument(help="jediTaskID, reqID, 'run', or 'fin'")] = None, + username: Annotated[Optional[str], typer.Option(help="Filter by username")] = None, + limit: Annotated[int, typer.Option(help="Maximum number of records")] = 1000, + taskname: Annotated[Optional[str], typer.Option(help="Filter by task name")] = None, + days: Annotated[int, typer.Option(help="Look back N days (capped at 90 without a task ID)")] = 14, + jeditaskid: Annotated[Optional[int], typer.Option(help="Filter by jediTaskID")] = None, + reqid: Annotated[Optional[int], typer.Option(help="Filter by reqID")] = None, + status: Annotated[Optional[str], typer.Option(help="Filter by task status")] = None, + superstatus: Annotated[Optional[str], typer.Option(help="Filter by super-status")] = None, +) -> None: + """Print task records in long format (shortcut for show --format='long'). + + examples: + >>> showl() + >>> showl(123) + >>> showl(12345678) + >>> showl(taskname='my_task_name') + + $ pbook showl --status=done --limit=100 + """ + return show( + task_id, + username=username, + limit=limit, + taskname=taskname, + days=days, + jeditaskid=jeditaskid, + reqid=reqid, + status=status, + superstatus=superstatus, + format="long", + ) + + +@app.command() +def kill( + task_ids: Annotated[str, typer.Argument(help="Task ID, comma-separated IDs, or 'all'")], +) -> None: + """Kill tasks. + + Kill all subJobs in task_ids (ID or a list of IDs, can be either jediTaskID or reqID). + If 'all', kill all active tasks of the user. + + example: + >>> kill(123) + >>> kill([123, 345, 567]) + >>> kill('all') + + $ pbook kill 123 + $ pbook kill 123,345,567 + $ pbook kill all + """ + core = _ensure_init() + ids = _parse_ids(task_ids) + if ids == "all": + return _parallel(lambda t: core.kill(t.jeditaskid), core.get_active_tasks()) + elif isinstance(ids, list): + return _parallel(core.kill, ids) + return core.kill(ids) + + +@app.command() +def finish( + task_ids: Annotated[str, typer.Argument(help="Task ID, comma-separated IDs, or 'all'")], + soft: Annotated[bool, typer.Option("--soft", help="Wait for running jobs to finish instead of killing them")] = False, +) -> None: + """Finish tasks. + + Finish all subJobs in task_ids (ID or a list of IDs, can be either jediTaskID or reqID). + If task_ids is 'all', finish all active tasks of the user. If soft is False (default), + all running jobs are killed and the task finishes immediately. If soft is True, new jobs + are not generated and the task finishes once all running jobs finish. + + example: + >>> finish(123) + >>> finish(234, soft=True) + >>> finish([123, 345, 567]) + >>> finish('all') + + $ pbook finish 123,345,567 --soft + """ + soft = _require_bool("soft", soft) + if soft is _INVALID: + return + core = _ensure_init() + ids = _parse_ids(task_ids) + if ids == "all": + return _parallel(lambda t: core.finish.original_func(core, t.jeditaskid, soft=soft), core.get_active_tasks()) + elif isinstance(ids, list): + return _parallel(lambda tid: core.finish(tid, soft=soft), ids) + return core.finish(ids, soft=soft) + + +@app.command() +def retry( + task_ids: Annotated[str, typer.Argument(help="Task ID, comma-separated IDs, or 'all'")], + days: Annotated[int, typer.Option("--days", help="Look-back window when task_ids='all'")] = 14, + limit: Annotated[int, typer.Option("--limit", help="Max tasks to retry when task_ids='all'")] = 1000, + site: Annotated[Optional[str], typer.Option("--site", help="Run the task on a particular PanDA queue")] = None, + excludedSite: Annotated[Optional[str], typer.Option("--excludedSite", help="Comma separated list of PanDA queues to exclude, e.g. 'siteA,siteB'")] = None, + includedSite: Annotated[Optional[str], typer.Option("--includedSite", help="Comma separated list of PanDA queues to include, e.g. 'siteA,siteB'")] = None, + nFilesPerJob: Annotated[Optional[int], typer.Option("--nFilesPerJob", help="Number of files on which each sub-job runs (default 50)")] = None, + nMaxFilesPerJob: Annotated[ + Optional[int], + typer.Option( + "--nMaxFilesPerJob", + "--maxNFilesPerJob", + help="Maximum number of input files to be processed by a single job in the task.", + ), + ] = None, + nGBPerJob: Annotated[ + Optional[float], + typer.Option("--nGBPerJob", help="Maximum input size in GB to be processed by a single job in the task.."), + ] = None, + nFiles: Annotated[Optional[int], typer.Option("--nFiles", help="Total number of input files to be processed by the task.")] = None, + nEvents: Annotated[ + Optional[int], + typer.Option("--nEvents", help="Total number of events to be processed by the task."), + ] = None, + loopingCheck: Annotated[ + Optional[bool], + typer.Option("--loopingCheck", help="Enable (True) or disable (False) the automatic check that kills jobs suspected of being stuck in a loop"), + ] = None, + memory: Annotated[Optional[int], typer.Option("--memory", "--ramCount", help="Required memory size in MB per core")] = None, + avoidVP: Annotated[Optional[bool], typer.Option("--avoidVP", help="Avoid PanDA queues which use Virtual Placement")] = None, + ignoreMissingInDS: Annotated[ + Optional[bool], typer.Option("--ignoreMissingInDS", help="Ignore missing input datasets which were deleted after the task is submitted.") + ] = None, + forceStaged: Annotated[ + Optional[bool], + typer.Option("--forceStaged", help="Force files from the primary dataset to be staged to local disk instead of using direct access"), + ] = None, + maxCore: Annotated[Optional[int], typer.Option("--maxCore", help="Maximum number of CPU cores that a single job is allowed to utilize")] = None, + newOpts: Annotated[Optional[str], typer.Option("--new-opts", hidden=True)] = None, +) -> None: + """Retry failed/canceled tasks. + + Retry failed/canceled subJobs in task_ids (ID or a list of IDs, can be either jediTaskID + or reqID). You can specify options (site, excludedSite,...) to overwrite task parameters for new attempts. + If input files were used or are being used by other jobs for the same output dataset container, those + files are skipped to avoid job duplication when retrying failed subjobs. + + If task_ids is 'all', it retries 1000 tasks at most that have finished for the last 14 + days. It is possible to retry more tasks by setting the days and limit options. If + named arguments are specified, they are applied to all retried tasks. + + In interactive mode, newOpts can be passed with a raw dict of task-retry options and it overrides + all of the individual options above. newOpts is not supported from the shell CLI. + + example: + >>> retry(123) + >>> retry([123, 345, 567]) + >>> retry(789, newOpts={'excludedSite': 'siteA,siteB'}) + >>> retry(789, excludedSite='siteA,siteB') + >>> retry('all') + >>> retry('all', days=30, limit=2000) + >>> retry('all', newOpts={'excludedSite': 'siteA,siteB'}) + + $ pbook retry 123 + $ pbook retry 123,345,567 --excludedSite=siteA,siteB + $ pbook retry all --days=30 --limit=2000 + + """ + core = _ensure_init() + if newOpts is not None: + opts = newOpts + else: + new_opts = { + k: v + for k, v in { + "site": site, + "excludedSite": excludedSite, + "includedSite": includedSite, + "nFilesPerJob": nFilesPerJob, + "nMaxFilesPerJob": nMaxFilesPerJob, + "nGBPerJob": nGBPerJob, + "nFiles": nFiles, + "nEvents": nEvents, + "loopingCheck": loopingCheck, + "ramCount": memory, + "avoidVP": avoidVP, + "ignoreMissingInDS": ignoreMissingInDS, + "forceStaged": forceStaged, + "maxCoreCount": maxCore, + }.items() + if v is not None + } + opts = new_opts or None + ids = _parse_ids(task_ids) + if isinstance(ids, list): + return _parallel(lambda tid: core.retry(tid, newOpts=opts), ids) + elif ids == "all": + data = core.show(status="finished", days=days, limit=limit, format="json") + return _parallel(lambda d: core.retry.original_func(core, d["jediTaskID"], newOpts=opts), data) + return core.retry(ids, newOpts=opts) + + +@app.command() +def debug( + panda_id: Annotated[int, typer.Argument(help="PanDA subjob ID")], + mode_on: Annotated[bool, typer.Argument(help="True to enable, False to disable")], +) -> None: + """Toggle debug mode for a subjob. + + mode_on is True/False to enable/disable the debug mode. Note that the maximum number of + debug subjobs is limited. If you already hit the limit you need to disable the debug mode + for a subjob before debugging another subjob. + + example: + >>> debug(1234, True) + + $ pbook debug 1234 True + """ + mode_on = _require_bool("mode_on", mode_on) + if mode_on is _INVALID: + return + core = _ensure_init() + core.debug(panda_id, mode_on) + + +@app.command(name="get_user_job_metadata") +def get_user_job_metadata( + task_id: Annotated[int, typer.Argument(help="Task ID")], + output_file: Annotated[str, typer.Argument(help="Output JSON file path")], +) -> None: + """Write user metadata of successful jobs to a JSON file. + + Get user metadata of successful jobs in a task and write them locally to a JSON file. + + example: + >>> get_user_job_metadata(123, 'output.json') + + $ pbook get_user_job_metadata 123 output.json + """ + core = _ensure_init() + core.getUserJobMetadata(task_id, output_file) + + +@app.command(name="reload_input") +def reload_input( + task_id: Annotated[int, typer.Argument(help="Task ID")], +) -> None: + """Reload input dataset and retry the task with new contents. + + This is useful when input dataset contents are changed after the task is submitted. + + example: + >>> reload_input(123) + + $ pbook reload_input 123 + """ + core = _ensure_init() + core.reload_input(task_id) + + +@app.command(name="recover_lost_files") +def recover_lost_files( + task_id: Annotated[int, typer.Argument(help="Task ID")], + test_mode: Annotated[bool, typer.Option("--test_mode", help="Dry-run mode")] = False, +) -> None: + """Request recovery of lost files from a task. + + Send a request to recover lost files produced by a task. Use test_mode for testing. + + example: + >>> recover_lost_files(123) + >>> recover_lost_files(123, test_mode=True) + + $ pbook recover_lost_files 123 + $ pbook recover_lost_files 123 --test_mode + """ + test_mode = _require_bool("test_mode", test_mode) + if test_mode is _INVALID: + return + core = _ensure_init() + core.recover_lost_files(task_id, test_mode) + + +@app.command(name="show_workflow") +def show_workflow( + request_id: Annotated[int, typer.Argument(help="Workflow request ID")], +) -> None: + """Show workflow status. + + Send a request to show the status of a workflow. + + example: + >>> show_workflow(456) + + $ pbook show_workflow 456 + """ + core = _ensure_init() + _, output = core.execute_workflow_command("get_status", request_id) + if output: + print(output) + + +@app.command(name="kill_workflow") +def kill_workflow( + request_id: Annotated[int, typer.Argument(help="Workflow request ID")], +) -> None: + """Kill a workflow. + + Send a request to kill a workflow. + + example: + >>> kill_workflow(456) + + $ pbook kill_workflow 456 + """ + core = _ensure_init() + _, output = core.execute_workflow_command("abort", request_id) + if output: + print(output[0][-1]) + + +@app.command(name="retry_workflow") +def retry_workflow( + request_id: Annotated[int, typer.Argument(help="Workflow request ID")], +) -> None: + """Retry a workflow. + + Send a request to retry a workflow. + + example: + >>> retry_workflow(456) + + $ pbook retry_workflow 456 + """ + core = _ensure_init() + _, output = core.execute_workflow_command("retry", request_id) + if output: + print(output[0][-1]) + + +@app.command(name="finish_workflow") +def finish_workflow( + request_id: Annotated[int, typer.Argument(help="Workflow request ID")], +) -> None: + """Finish a workflow. + + Send a request to finish a workflow. + + example: + >>> finish_workflow(456) + + $ pbook finish_workflow 456 + """ + core = _ensure_init() + _, output = core.execute_workflow_command("finish", request_id) + if output: + print(output[0][-1]) + + +@app.command(name="pause_workflow") +def pause_workflow( + request_id: Annotated[int, typer.Argument(help="Workflow request ID")], +) -> None: + """Pause a workflow. + + Send a request to pause a workflow. + + example: + >>> pause_workflow(456) + + $ pbook pause_workflow 456 + """ + core = _ensure_init() + _, output = core.execute_workflow_command("suspend", request_id) + if output: + print(output[0][-1]) + + +@app.command(name="resume_workflow") +def resume_workflow( + request_id: Annotated[int, typer.Argument(help="Workflow request ID")], +) -> None: + """Resume a workflow. + + Send a request to resume a workflow. + + example: + >>> resume_workflow(456) + + $ pbook resume_workflow 456 + """ + core = _ensure_init() + _, output = core.execute_workflow_command("resume", request_id) + if output: + print(output[0][-1]) + + +@app.command(name="set_secret") +def set_secret( + key: Annotated[str, typer.Argument(help="Secret key")], + value: Annotated[str, typer.Argument(help="Secret value or file path")], + is_file: Annotated[bool, typer.Option("--is_file", help="Treat value as a file path to upload")] = False, +) -> None: + """Set a secret key-value pair. + + Define a pair of secret key-value strings. The value can be a file path to upload a + secret file when is_file=True. + + example: + >>> set_secret('mykey', 'myvalue') + >>> set_secret('mykey', '/path/to/file', is_file=True) + + $ pbook set_secret mykey myvalue + $ pbook set_secret mykey /path/to/file --is_file + """ + is_file = _require_bool("is_file", is_file) + if is_file is _INVALID: + return + core = _ensure_init() + core.set_secret(key, value, is_file) + + +@app.command(name="delete_secret") +def delete_secret( + key: Annotated[str, typer.Argument(help="Secret key to delete")], +) -> None: + """Delete a secret. + + example: + >>> delete_secret('mykey') + + $ pbook delete_secret mykey + """ + core = _ensure_init() + core.set_secret(key, None) + + +@app.command(name="delete_all_secrets") +def delete_all_secrets() -> None: + """Delete all secrets. + + example: + >>> delete_all_secrets + + $ pbook delete_all_secrets + """ + core = _ensure_init() + core.set_secret(None, None) + + +@app.command(name="list_secrets") +def list_secrets( + full: Annotated[bool, typer.Option("--full", help="Show complete secret values instead of truncating them")] = False, +) -> None: + """List secrets. + + Value strings are truncated by default. full=True to see entire strings. + + example: + >>> list_secrets() + >>> list_secrets(full=True) + + $ pbook list_secrets + $ pbook list_secrets --full + """ + full = _require_bool("full", full) + if full is _INVALID: + return + core = _ensure_init() + core.list_secrets(full) + + +@app.command(name="generate_credential") +def generate_credential() -> None: + """Generate a new proxy or token.""" + core = _get_core() + core.generate_credential() + + +# ─── Entry point ────────────────────────────────────────────────────────────── + +# Global options that consume the following argv token as their own value, so the +# subcommand-token scan below can skip over both. +_GLOBAL_VALUE_OPTS = {"-c"} + + +def _rewrite_legacy_kwargs(argv: list) -> list: + """Rewrite legacy bare `key=value` batch args into `--key=value`. + + The pre-Typer pbook batch mode accepted `pbook show format=long`; Click only + recognizes `--format=long`. Find the subcommand, look up its real option names via + introspection (never a separately maintained list), and rewrite any later bare + `key=value` token whose key matches one of them. Anything else - already-dashed + flags, positional args that happen to contain "=" - passes through untouched. + """ + i = 0 + while i < len(argv): + tok = argv[i] + if tok in _GLOBAL_VALUE_OPTS: + i += 2 + continue + if tok.startswith("-"): + i += 1 + continue + break + if i >= len(argv): + return argv + + sub_cmd = typer.main.get_command(app).commands.get(argv[i]) + if sub_cmd is None: + return argv + + option_flags = {} + flag_only = set() + for param in sub_cmd.params: + flags = [o for o in getattr(param, "opts", []) if o.startswith("--")] + if flags: + primary = flags[0] + # Register every alias (e.g. --ramCount for --memory), not just the primary + # name, so the legacy bare `key=value` syntax recognizes them too. + for flag in flags: + option_flags[flag.lstrip("-")] = primary + if getattr(param, "is_flag", False): + flag_only.add(param.name) + + rewritten = argv[: i + 1] + for tok in argv[i + 1 :]: + key, sep, value = tok.partition("=") + if sep and not tok.startswith("-") and key in option_flags: + flag = option_flags[key] + if key in flag_only: + # Click flag-style options (e.g. --soft) take no value at all; the legacy + # syntax passed an explicit True/False, so translate that into presence + # (truthy) or absence (falsy - same as the option's own default) instead. + normalized = value.strip().lower() + if normalized in ("true", "1", "yes"): + rewritten.append(flag) + elif normalized not in ("false", "0", "no"): + typer.echo( + f"Error: '{key}' is a flag and expects True/False (got '{value}' in '{tok}')", + err=True, + ) + sys.exit(1) + continue + rewritten.append(f"{flag}={value}") else: - sys.exit(0) + rewritten.append(tok) + return rewritten + + +def main() -> None: + sys.argv[0] = "pbook" + sys.argv[1:] = _rewrite_legacy_kwargs(sys.argv[1:]) + app() From 2c06b7ef4745b50a01f5b8c7602f89fc486d4af4 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Fri, 7 Aug 2026 14:58:53 +0200 Subject: [PATCH 57/59] Allow negative values for avoidVP, loopingCheck etc --- pandaclient/PBookScript.py | 22 +- pandaclient/PBookTyper.py | 1129 ------------------------------------ scripts/pbook | 2 +- 3 files changed, 17 insertions(+), 1136 deletions(-) delete mode 100644 pandaclient/PBookTyper.py diff --git a/pandaclient/PBookScript.py b/pandaclient/PBookScript.py index 01d30922..668fbf4c 100644 --- a/pandaclient/PBookScript.py +++ b/pandaclient/PBookScript.py @@ -697,20 +697,30 @@ def retry( Optional[int], typer.Option("--nEvents", help="Total number of events to be processed by the task."), ] = None, + memory: Annotated[Optional[int], typer.Option("--memory", "--ramCount", help="Required memory size in MB per core")] = None, + maxCore: Annotated[Optional[int], typer.Option("--maxCore", help="Maximum number of CPU cores that a single job is allowed to utilize")] = None, loopingCheck: Annotated[ Optional[bool], - typer.Option("--loopingCheck", help="Enable (True) or disable (False) the automatic check that kills jobs suspected of being stuck in a loop"), + typer.Option( + "--loopingCheck/--no-loopingCheck", + help="Enable/disable the automatic check that kills jobs suspected of being stuck in a loop", + ), ] = None, - memory: Annotated[Optional[int], typer.Option("--memory", "--ramCount", help="Required memory size in MB per core")] = None, - avoidVP: Annotated[Optional[bool], typer.Option("--avoidVP", help="Avoid PanDA queues which use Virtual Placement")] = None, + avoidVP: Annotated[Optional[bool], typer.Option("--avoidVP/--no-avoidVP", help="Avoid PanDA queues which use Virtual Placement")] = None, ignoreMissingInDS: Annotated[ - Optional[bool], typer.Option("--ignoreMissingInDS", help="Ignore missing input datasets which were deleted after the task is submitted.") + Optional[bool], + typer.Option( + "--ignoreMissingInDS/--no-ignoreMissingInDS", + help="Ignore missing input datasets which were deleted after the task is submitted.", + ), ] = None, forceStaged: Annotated[ Optional[bool], - typer.Option("--forceStaged", help="Force files from the primary dataset to be staged to local disk instead of using direct access"), + typer.Option( + "--forceStaged/--no-forceStaged", + help="Force files from the primary dataset to be staged to local disk instead of using direct access", + ), ] = None, - maxCore: Annotated[Optional[int], typer.Option("--maxCore", help="Maximum number of CPU cores that a single job is allowed to utilize")] = None, newOpts: Annotated[Optional[str], typer.Option("--new-opts", hidden=True)] = None, ) -> None: """Retry failed/canceled tasks. diff --git a/pandaclient/PBookTyper.py b/pandaclient/PBookTyper.py deleted file mode 100644 index 01d30922..00000000 --- a/pandaclient/PBookTyper.py +++ /dev/null @@ -1,1129 +0,0 @@ -""" -Do NOT import this module in your code. -Import PBookCore instead. - -pbook CLI — PanDA task bookkeeper. - -Each command below is defined exactly once, as a Typer command using the -Annotated[...] parameter style. Because the Typer/Click metadata lives in the -annotation rather than in the default value, these functions remain ordinary -callables with ordinary defaults - the same function is used to build the -`pbook --flag ...` CLI (with real shell completion) *and* is placed -directly into the interactive REPL namespace (`>>> command(...)`). -""" - -from __future__ import annotations - -import atexit -import code -import os -import re -import readline -import rlcompleter -import signal -import sys -import tempfile -from concurrent.futures import ThreadPoolExecutor -from inspect import Parameter, signature -from typing import ( - Annotated, - Literal, - Optional, - Union, - get_args, - get_origin, - get_type_hints, -) - -import typer -from rich import box -from rich.console import Console -from rich.markup import escape as _esc -from rich.table import Table - -from pandaclient import Client, PandaToolsPkgInfo -from pandaclient.MiscUtils import commands_get_output - -# ─── Runtime state ──────────────────────────────────────────────────────────── -_tmp_dir: Optional[str] = None -_history_file: Optional[str] = None -_fork_child_pid: Optional[int] = None -_setup_done: bool = False -_ctx_state: dict = {} -_core = None -_core_inited: bool = False - -help_text = """ -PanDA task bookkeeper. Run without arguments for interactive mode. - -$ pbook [options] # interactive mode -$ pbook [options] command [args] [kwargs] # batch mode - -The same command can be executed in interactive mode: - -$ pbook ->>> command(*args, **kwargs) - -or in batch mode: - -$ pbook command arg1 arg2 ... argN --kwarg1=value1 --kwarg2=value2 ... --kwargN=valueN -$ pbook command arg1 arg2 ... argN kwarg1=value1 kwarg2=value2 ... kwargN=valueN -Please note that the latter option is kept for backward compatibility, but we plan to drop it in the future. - -E.g. - -$ pbook ->>> show(123, format='long') - -is equivalent to - -$ pbook show 123 --format='long' -$ pbook show 123 format='long' - -If arg or value is a list in interactive mode, it is represented as a comma-separate list in batch mode. E.g. -to kill three tasks in interactive mode: - -$ pbook ->>> kill([123, 456, 789]) - -or in batch mode: - -$ pbook kill 123,456,789 - -To see the list of commands and help of each command, - -$ pbook ->>> help() ->>> help(command_name) - -or - -$ pbook --help -$ pbook command_name --help -""" - -app = typer.Typer( - name="pbook", - help=help_text, - invoke_without_command=True, - no_args_is_help=False, - context_settings={"help_option_names": ["-h", "--help"]}, -) - -# ─── Utilities ──────────────────────────────────────────────────────────────── - - -def _parallel(func, items): - """ - Parallel execution of func with items in a pool of 8 threads. - """ - with ThreadPoolExecutor(8) as pool: - return list(pool.map(func, items)) - - -def _parse_ids(raw): - """'all' -> 'all'; '42' -> 42; '1,2,3' -> [1,2,3]; anything not a string passes through as-is.""" - if not isinstance(raw, str): - return raw - if raw == "all": - return "all" - parts = raw.split(",") - try: - ids = [int(p) for p in parts] - return ids[0] if len(ids) == 1 else ids - except ValueError: - typer.echo(f"Error: invalid task ID(s): {raw}", err=True) - raise typer.Exit(1) - - -_INVALID = object() - - -def _require_bool(name: str, value): - """Report non-bool values for a bool parameter, without raising. - - Click enforces this on the CLI path (a flag is present or absent, never a stray - string) - unreachable from real CLI dispatch. Commands are also called directly - from the REPL though, where a plain Python call - e.g. finish(123, soft='gfd') - - bypasses that check entirely and would otherwise silently treat any truthy string - as on. Raising here would just dump a traceback into the interactive session, so - report the problem and let the caller return early instead: `if soft is _INVALID: - return`. - """ - if not isinstance(value, bool): - typer.echo(f"Error: '{name}' must be True or False, got {value!r}", err=True) - return _INVALID - return value - - -def _setup() -> None: - global _tmp_dir, _history_file, _setup_done - if _setup_done: - return - _setup_done = True - - readline.parse_and_bind("tab: complete") - readline.parse_and_bind("set show-all-if-ambiguous On") - - if "CMTSITE" not in os.environ: - os.environ["CMTSITE"] = "" - - pconf_dir = os.path.expanduser(os.environ.get("PANDA_CONFIG_ROOT", "~/.panda")) - os.makedirs(pconf_dir, exist_ok=True) - - _history_file = os.path.join(pconf_dir, ".history") - if os.path.exists(_history_file): - try: - readline.read_history_file(_history_file) - except Exception: - pass - readline.set_history_length(1024) - - _tmp_dir = tempfile.mkdtemp() - - for path in sys.path: - real = path or "." - if ( - os.path.exists(real) - and os.path.isdir(real) - and "pandaclient" in os.listdir(real) - and os.path.exists(os.path.join(real, "pandaclient", "__init__.py")) - ): - link = os.path.join(_tmp_dir, "taskbuffer") - if not os.path.exists(link): - os.symlink(os.path.join(real, "pandaclient"), link) - break - if _tmp_dir not in sys.path: - sys.path.insert(0, _tmp_dir) - - atexit.register(_cleanup) - - -def _cleanup() -> None: - if _fork_child_pid == 0 and _history_file: - readline.write_history_file(_history_file) - - if _tmp_dir: - commands_get_output(f"rm -rf {_tmp_dir}") - - -def _make_core(verbose: bool = False): - from pandaclient import PBookCore - - return PBookCore.PBookCore(verbose=verbose) - - -def _get_core(): - """Return the (memoized) core for this process, uninitialized.""" - global _core - _setup() - if _core is None: - _core = _make_core(_ctx_state.get("verbose", False)) - return _core - - -def _ensure_init(sanity_check: bool = False): - """Return the core, running PBookCore.init() exactly once per process. - - The REPL calls this once upfront with sanity_check=True; every command - function then calls it again with the default before touching the core, - which is a no-op once already initialized - so commands stay correct - whether they run once (batch mode) or repeatedly (REPL session). - """ - global _core_inited - core = _get_core() - if not _core_inited: - core.init(sanity_check=sanity_check) - _core_inited = True - return core - - -def _catch_sig(sig, frame): - _cleanup() - # Hard kill all processes in the group - commands_get_output(f"kill -9 -- -{os.getpgrp()}") - - -# ─── REPL namespace & completion ────────────────────────────────────────────── - - -def _build_namespace() -> dict: - """The REPL namespace: every registered Typer command, keyed by its real Python name.""" - return {info.callback.__name__: info.callback for info in app.registered_commands} - - -def _kwarg_names(func) -> list: - """All parameter names a function accepts - candidates for `name=` completion.""" - return list(signature(func).parameters) - - -def _kwarg_choices(func, name: str) -> list: - """Value choices for a parameter, derived from its type hint: Literal[...] members or True/False for bool. - - Returned bare (unquoted) - readline's own quote-matching auto-closes an opening quote - the user already typed, so we don't need to (and shouldn't try to) add one ourselves. - """ - try: - hints = get_type_hints(func, include_extras=True) - except Exception: - return [] - ann = hints.get(name) - if ann is None: - return [] - while hasattr(ann, "__metadata__"): - ann = ann.__origin__ - if get_origin(ann) is Union: - non_none = [a for a in get_args(ann) if a is not type(None)] - if len(non_none) == 1: - ann = non_none[0] - if get_origin(ann) is Literal: - return [str(v) for v in get_args(ann)] - if ann is bool: - return ["True", "False"] - return [] - - -class _PBookCompleter: - """Readline completer: kwarg names and values when inside a call, names otherwise.""" - - def __init__(self, ns: dict) -> None: - self._ns = ns - self._base = rlcompleter.Completer(ns) - self._matches: list = [] - - def complete(self, text: str, state: int) -> Optional[str]: - if state == 0: - self._matches = self._compute(text) - return self._matches[state] if state < len(self._matches) else None - - def _compute(self, text: str) -> list: - line = readline.get_line_buffer() - - # Kwarg value completion (tier 1): last token is kwarg= or kwarg='partial - # Return bare values - readline's own quote-matching auto-closes an opening quote - # the user already typed, so we deliberately don't add quotes ourselves here. - m_val = re.search(r"\b(\w+)\s*=\s*(['\"]?)(\w*)$", line) - m_func = re.match(r"(\w+)\s*\(", line) - if m_val and m_func: - kwarg, partial = m_val.group(1), m_val.group(3) - func = self._ns.get(m_func.group(1)) - if func is not None: - # We're unambiguously past a `kwarg=` - this is a value position, not a - # name position, even if this particular kwarg has no enumerable choices - # (e.g. limit: int). Return here regardless, so an empty result doesn't - # fall through to tier 2's kwarg-name completion. - return [v for v in _kwarg_choices(func, kwarg) if v.startswith(partial)] - - # Kwarg name completion (tier 2): cursor is inside an open call - m = re.search(r"(\w+)\s*\([^)]*$", line) - if m: - func = self._ns.get(m.group(1)) - if func is not None: - hits = [k for k in _kwarg_names(func) if k.startswith(text)] - if hits: - return hits - - # Plain name completion (tier 3, rlcompleter fallback) - if not text: - # rlcompleter.complete() special-cases blank text by calling readline.insert_text() - # itself, which re-enters readline from inside this callback and confuses the active - # Tab press; list the namespace directly instead of delegating to it here - return sorted(k for k in self._base.namespace if not k.startswith("_")) - results, i = [], 0 - while (c := self._base.complete(text, i)) is not None: - results.append(c.rstrip("()").rstrip("(")) - i += 1 - return results - - -def _run_repl(ns: dict, banner: str) -> None: - """Manual REPL using InteractiveConsole.push() so we own readline setup entirely.""" - completer = _PBookCompleter(ns) - readline.set_completer(completer.complete) - readline.parse_and_bind("tab: complete") - readline.parse_and_bind("set show-all-if-ambiguous On") - - console = code.InteractiveConsole(ns) - print(banner) - - more = False - while True: - prompt = "... " if more else ">>> " - try: - readline.set_completer(completer.complete) - line = input(prompt) - except EOFError: - print() - break - except KeyboardInterrupt: - print("\nKeyboardInterrupt") - console.resetbuffer() - more = False - continue - more = console.push(line) - - -# ─── Top-level callback ─────────────────────────────────────────────────────── - - -@app.callback(invoke_without_command=True) -def _main( - ctx: typer.Context, - verbose: bool = typer.Option(False, "-v", help="Verbose"), - command_string: Optional[str] = typer.Option(None, "-c", help="Execute a Python code snippet"), - version: bool = typer.Option(False, "--version", is_eager=True, help="Display version"), - dev_srv: bool = typer.Option(False, "--devSrv", hidden=True), - intr_srv: bool = typer.Option(False, "--intrSrv", hidden=True), - prompt_with_newline: bool = typer.Option(False, "--prompt_with_newline", hidden=True), - python3: bool = typer.Option(False, "-3", hidden=True), -) -> None: - """PanDA task bookkeeper. Run without arguments for interactive mode.""" - if version: - typer.echo(f"Version: {PandaToolsPkgInfo.release_version}") - raise typer.Exit() - - if dev_srv: - Client.useDevServer() - if intr_srv: - Client.useIntrServer() - - _ctx_state.update({"verbose": verbose}) - - if ctx.invoked_subcommand is not None: - return - - # Interactive or snippet mode - _setup() - global _fork_child_pid - _fork_child_pid = os.fork() - - if _fork_child_pid == -1: - typer.echo("ERROR: Failed to fork", err=True) - raise typer.Exit(1) - - if _fork_child_pid == 0: - if verbose: - typer.echo(str(ctx.params)) - if prompt_with_newline: - sys.ps1 = ">>> \n" - _ensure_init(sanity_check=True) - ns = _build_namespace() - - if command_string: - exec(command_string, {}, ns) # noqa: S102 - from pandaclient import PBookCore as _PBC - - raise typer.Exit(0 if _PBC.func_return_value else 1) - _run_repl(ns, banner=f"\nStart pBook {PandaToolsPkgInfo.release_version}") - - else: - signal.signal(signal.SIGINT, _catch_sig) - signal.signal(signal.SIGHUP, _catch_sig) - signal.signal(signal.SIGTERM, _catch_sig) - pid, status = os.wait() - if os.WIFSIGNALED(status): - raise typer.Exit(-os.WTERMSIG(status)) - elif os.WIFEXITED(status): - raise typer.Exit(os.WEXITSTATUS(status)) - raise typer.Exit(0) - - -# ─── Commands ────────────────────────────────────────────────────────────────── - -_HELP_GROUPS = [ - ("Tasks", ["show", "showl", "kill", "finish", "retry", "debug"]), - ("Files & input", ["get_user_job_metadata", "recover_lost_files", "reload_input"]), - ("Workflows", ["show_workflow", "kill_workflow", "retry_workflow", "finish_workflow", "pause_workflow", "resume_workflow"]), - ("Secrets", ["set_secret", "list_secrets", "delete_secret", "delete_all_secrets"]), - ("Auth", ["generate_credential"]), -] - - -def _type_name(ann) -> str: - """Render a resolved type annotation as a short, human-readable name (Optional[str], Literal[...], etc.).""" - if ann is None or ann is type(None): - return "" - origin = get_origin(ann) - if origin is Union: - args = get_args(ann) - non_none = [a for a in args if a is not type(None)] - if len(non_none) == 1 and len(args) == 2: - return f"Optional[{_type_name(non_none[0])}]" - return " | ".join(_type_name(a) for a in args) - if origin is Literal: - return "Literal[" + ", ".join(repr(v) for v in get_args(ann)) + "]" - if origin is not None: - args = get_args(ann) - origin_name = getattr(origin, "__name__", str(origin)) - return f"{origin_name}[{', '.join(_type_name(a) for a in args)}]" if args else origin_name - return getattr(ann, "__name__", str(ann)) - - -def _format_signature(func) -> str: - """A clean '(param: Type = default, ...)' string, stripping the Typer/Annotated plumbing.""" - try: - hints = get_type_hints(func, include_extras=True) - except Exception: - hints = {} - parts = [] - for pname, p in signature(func).parameters.items(): - ann = hints.get(pname) - while hasattr(ann, "__metadata__"): - ann = ann.__origin__ - piece = pname - type_str = _type_name(ann) - if type_str: - piece += f": {type_str}" - if p.default is not Parameter.empty: - piece += f" = {p.default!r}" - parts.append(piece) - return f"({', '.join(parts)})" - - -@app.command() -def help( - command: Annotated[Optional[str], typer.Argument(help="Command name for detailed help")] = None, -) -> None: - """Show available commands, or detailed help for a specific command.""" - ns = _build_namespace() - console = Console() - - if command is not None: - name = command if isinstance(command, str) else command.__name__ - func = ns.get(name, command if callable(command) else None) - if func is None: - console.print(f"[red]Unknown command:[/red] {_esc(name)}") - return - console.print(f"\n[bold cyan]{name}[/bold cyan][bold]{_esc(_format_signature(func))}[/bold]") - doc = (func.__doc__ or "No description.").strip() - console.print(f"\n{_esc(doc)}\n") - return - - table = Table(box=box.SIMPLE, show_header=True, header_style="bold magenta") - table.add_column("Command", style="bold cyan", no_wrap=True) - table.add_column("Description") - table.add_column("Signature") - - for group, names in _HELP_GROUPS: - table.add_section() - table.add_row(f"[bold white]{group}[/bold white]", "", "") - for name in names: - func = ns.get(name) - if func is None: - continue - doc = (func.__doc__ or "").strip().splitlines()[0] if func.__doc__ else "" - sig = console.highlighter(_format_signature(func)) - table.add_row(f" {name}", _esc(doc), sig) - - console.print(table) - console.print("Usage: [bold]help(show)[/bold] or [bold]pbook show --help[/bold]\n") - - -@app.command() -def show( - task_id: Annotated[Optional[str], typer.Argument(help="jediTaskID, reqID, 'run' (active only), or 'fin' (terminated only)")] = None, - username: Annotated[Optional[str], typer.Option(help="Filter by username. By default, the name from the voms/token is used.")] = None, - limit: Annotated[int, typer.Option(help="Maximum number of records")] = 1000, - taskname: Annotated[Optional[str], typer.Option(help="Filter by task name")] = None, - days: Annotated[int, typer.Option(help="Look back N days (capped at 90 without a task ID)")] = 14, - jeditaskid: Annotated[Optional[int], typer.Option(help="Filter by jediTaskID")] = None, - reqid: Annotated[Optional[int], typer.Option(help="Filter by reqID")] = None, - status: Annotated[Optional[str], typer.Option(help="Filter by task status")] = None, - superstatus: Annotated[Optional[str], typer.Option(help="Filter by super-status")] = None, - format: Annotated[Literal["standard", "long", "json", "plain"], typer.Option("--format", help="Output format")] = "standard", -) -> None: - """Print task records. - - The first non-keyword argument (task_id) can be a jediTaskID or reqID, or 'run' (show active tasks - only), or 'fin' (show terminated tasks only), or can be omitted. Records are fetched - directly from the PanDA server, so they are always up to date. Note that days is capped - at 90 days unless a jediTaskID or reqID is specified, in which case tasks of any age are - returned. See the default filter conditions in the annotations. - - examples: - >>> show() - >>> show(123) - >>> show(12345678, format='long') - >>> show(taskname='my_task_name') - >>> show('run') - >>> show('fin', days=7, limit=100) - >>> show(format='json') - - $ pbook show --format=long --status=done --limit=100 - """ - core = _ensure_init() - kwargs = { - k: v - for k, v in dict( - username=username, - limit=limit, - taskname=taskname, - days=days, - jeditaskid=jeditaskid, - reqid=reqid, - status=status, - superstatus=superstatus, - ).items() - if v is not None - } - kwargs["format"] = format - if task_id is not None: - try: - first_arg = int(task_id) - except (TypeError, ValueError): - first_arg = task_id - return core.show(first_arg, **kwargs) - return core.show(**kwargs) - - -@app.command() -def showl( - task_id: Annotated[Optional[str], typer.Argument(help="jediTaskID, reqID, 'run', or 'fin'")] = None, - username: Annotated[Optional[str], typer.Option(help="Filter by username")] = None, - limit: Annotated[int, typer.Option(help="Maximum number of records")] = 1000, - taskname: Annotated[Optional[str], typer.Option(help="Filter by task name")] = None, - days: Annotated[int, typer.Option(help="Look back N days (capped at 90 without a task ID)")] = 14, - jeditaskid: Annotated[Optional[int], typer.Option(help="Filter by jediTaskID")] = None, - reqid: Annotated[Optional[int], typer.Option(help="Filter by reqID")] = None, - status: Annotated[Optional[str], typer.Option(help="Filter by task status")] = None, - superstatus: Annotated[Optional[str], typer.Option(help="Filter by super-status")] = None, -) -> None: - """Print task records in long format (shortcut for show --format='long'). - - examples: - >>> showl() - >>> showl(123) - >>> showl(12345678) - >>> showl(taskname='my_task_name') - - $ pbook showl --status=done --limit=100 - """ - return show( - task_id, - username=username, - limit=limit, - taskname=taskname, - days=days, - jeditaskid=jeditaskid, - reqid=reqid, - status=status, - superstatus=superstatus, - format="long", - ) - - -@app.command() -def kill( - task_ids: Annotated[str, typer.Argument(help="Task ID, comma-separated IDs, or 'all'")], -) -> None: - """Kill tasks. - - Kill all subJobs in task_ids (ID or a list of IDs, can be either jediTaskID or reqID). - If 'all', kill all active tasks of the user. - - example: - >>> kill(123) - >>> kill([123, 345, 567]) - >>> kill('all') - - $ pbook kill 123 - $ pbook kill 123,345,567 - $ pbook kill all - """ - core = _ensure_init() - ids = _parse_ids(task_ids) - if ids == "all": - return _parallel(lambda t: core.kill(t.jeditaskid), core.get_active_tasks()) - elif isinstance(ids, list): - return _parallel(core.kill, ids) - return core.kill(ids) - - -@app.command() -def finish( - task_ids: Annotated[str, typer.Argument(help="Task ID, comma-separated IDs, or 'all'")], - soft: Annotated[bool, typer.Option("--soft", help="Wait for running jobs to finish instead of killing them")] = False, -) -> None: - """Finish tasks. - - Finish all subJobs in task_ids (ID or a list of IDs, can be either jediTaskID or reqID). - If task_ids is 'all', finish all active tasks of the user. If soft is False (default), - all running jobs are killed and the task finishes immediately. If soft is True, new jobs - are not generated and the task finishes once all running jobs finish. - - example: - >>> finish(123) - >>> finish(234, soft=True) - >>> finish([123, 345, 567]) - >>> finish('all') - - $ pbook finish 123,345,567 --soft - """ - soft = _require_bool("soft", soft) - if soft is _INVALID: - return - core = _ensure_init() - ids = _parse_ids(task_ids) - if ids == "all": - return _parallel(lambda t: core.finish.original_func(core, t.jeditaskid, soft=soft), core.get_active_tasks()) - elif isinstance(ids, list): - return _parallel(lambda tid: core.finish(tid, soft=soft), ids) - return core.finish(ids, soft=soft) - - -@app.command() -def retry( - task_ids: Annotated[str, typer.Argument(help="Task ID, comma-separated IDs, or 'all'")], - days: Annotated[int, typer.Option("--days", help="Look-back window when task_ids='all'")] = 14, - limit: Annotated[int, typer.Option("--limit", help="Max tasks to retry when task_ids='all'")] = 1000, - site: Annotated[Optional[str], typer.Option("--site", help="Run the task on a particular PanDA queue")] = None, - excludedSite: Annotated[Optional[str], typer.Option("--excludedSite", help="Comma separated list of PanDA queues to exclude, e.g. 'siteA,siteB'")] = None, - includedSite: Annotated[Optional[str], typer.Option("--includedSite", help="Comma separated list of PanDA queues to include, e.g. 'siteA,siteB'")] = None, - nFilesPerJob: Annotated[Optional[int], typer.Option("--nFilesPerJob", help="Number of files on which each sub-job runs (default 50)")] = None, - nMaxFilesPerJob: Annotated[ - Optional[int], - typer.Option( - "--nMaxFilesPerJob", - "--maxNFilesPerJob", - help="Maximum number of input files to be processed by a single job in the task.", - ), - ] = None, - nGBPerJob: Annotated[ - Optional[float], - typer.Option("--nGBPerJob", help="Maximum input size in GB to be processed by a single job in the task.."), - ] = None, - nFiles: Annotated[Optional[int], typer.Option("--nFiles", help="Total number of input files to be processed by the task.")] = None, - nEvents: Annotated[ - Optional[int], - typer.Option("--nEvents", help="Total number of events to be processed by the task."), - ] = None, - loopingCheck: Annotated[ - Optional[bool], - typer.Option("--loopingCheck", help="Enable (True) or disable (False) the automatic check that kills jobs suspected of being stuck in a loop"), - ] = None, - memory: Annotated[Optional[int], typer.Option("--memory", "--ramCount", help="Required memory size in MB per core")] = None, - avoidVP: Annotated[Optional[bool], typer.Option("--avoidVP", help="Avoid PanDA queues which use Virtual Placement")] = None, - ignoreMissingInDS: Annotated[ - Optional[bool], typer.Option("--ignoreMissingInDS", help="Ignore missing input datasets which were deleted after the task is submitted.") - ] = None, - forceStaged: Annotated[ - Optional[bool], - typer.Option("--forceStaged", help="Force files from the primary dataset to be staged to local disk instead of using direct access"), - ] = None, - maxCore: Annotated[Optional[int], typer.Option("--maxCore", help="Maximum number of CPU cores that a single job is allowed to utilize")] = None, - newOpts: Annotated[Optional[str], typer.Option("--new-opts", hidden=True)] = None, -) -> None: - """Retry failed/canceled tasks. - - Retry failed/canceled subJobs in task_ids (ID or a list of IDs, can be either jediTaskID - or reqID). You can specify options (site, excludedSite,...) to overwrite task parameters for new attempts. - If input files were used or are being used by other jobs for the same output dataset container, those - files are skipped to avoid job duplication when retrying failed subjobs. - - If task_ids is 'all', it retries 1000 tasks at most that have finished for the last 14 - days. It is possible to retry more tasks by setting the days and limit options. If - named arguments are specified, they are applied to all retried tasks. - - In interactive mode, newOpts can be passed with a raw dict of task-retry options and it overrides - all of the individual options above. newOpts is not supported from the shell CLI. - - example: - >>> retry(123) - >>> retry([123, 345, 567]) - >>> retry(789, newOpts={'excludedSite': 'siteA,siteB'}) - >>> retry(789, excludedSite='siteA,siteB') - >>> retry('all') - >>> retry('all', days=30, limit=2000) - >>> retry('all', newOpts={'excludedSite': 'siteA,siteB'}) - - $ pbook retry 123 - $ pbook retry 123,345,567 --excludedSite=siteA,siteB - $ pbook retry all --days=30 --limit=2000 - - """ - core = _ensure_init() - if newOpts is not None: - opts = newOpts - else: - new_opts = { - k: v - for k, v in { - "site": site, - "excludedSite": excludedSite, - "includedSite": includedSite, - "nFilesPerJob": nFilesPerJob, - "nMaxFilesPerJob": nMaxFilesPerJob, - "nGBPerJob": nGBPerJob, - "nFiles": nFiles, - "nEvents": nEvents, - "loopingCheck": loopingCheck, - "ramCount": memory, - "avoidVP": avoidVP, - "ignoreMissingInDS": ignoreMissingInDS, - "forceStaged": forceStaged, - "maxCoreCount": maxCore, - }.items() - if v is not None - } - opts = new_opts or None - ids = _parse_ids(task_ids) - if isinstance(ids, list): - return _parallel(lambda tid: core.retry(tid, newOpts=opts), ids) - elif ids == "all": - data = core.show(status="finished", days=days, limit=limit, format="json") - return _parallel(lambda d: core.retry.original_func(core, d["jediTaskID"], newOpts=opts), data) - return core.retry(ids, newOpts=opts) - - -@app.command() -def debug( - panda_id: Annotated[int, typer.Argument(help="PanDA subjob ID")], - mode_on: Annotated[bool, typer.Argument(help="True to enable, False to disable")], -) -> None: - """Toggle debug mode for a subjob. - - mode_on is True/False to enable/disable the debug mode. Note that the maximum number of - debug subjobs is limited. If you already hit the limit you need to disable the debug mode - for a subjob before debugging another subjob. - - example: - >>> debug(1234, True) - - $ pbook debug 1234 True - """ - mode_on = _require_bool("mode_on", mode_on) - if mode_on is _INVALID: - return - core = _ensure_init() - core.debug(panda_id, mode_on) - - -@app.command(name="get_user_job_metadata") -def get_user_job_metadata( - task_id: Annotated[int, typer.Argument(help="Task ID")], - output_file: Annotated[str, typer.Argument(help="Output JSON file path")], -) -> None: - """Write user metadata of successful jobs to a JSON file. - - Get user metadata of successful jobs in a task and write them locally to a JSON file. - - example: - >>> get_user_job_metadata(123, 'output.json') - - $ pbook get_user_job_metadata 123 output.json - """ - core = _ensure_init() - core.getUserJobMetadata(task_id, output_file) - - -@app.command(name="reload_input") -def reload_input( - task_id: Annotated[int, typer.Argument(help="Task ID")], -) -> None: - """Reload input dataset and retry the task with new contents. - - This is useful when input dataset contents are changed after the task is submitted. - - example: - >>> reload_input(123) - - $ pbook reload_input 123 - """ - core = _ensure_init() - core.reload_input(task_id) - - -@app.command(name="recover_lost_files") -def recover_lost_files( - task_id: Annotated[int, typer.Argument(help="Task ID")], - test_mode: Annotated[bool, typer.Option("--test_mode", help="Dry-run mode")] = False, -) -> None: - """Request recovery of lost files from a task. - - Send a request to recover lost files produced by a task. Use test_mode for testing. - - example: - >>> recover_lost_files(123) - >>> recover_lost_files(123, test_mode=True) - - $ pbook recover_lost_files 123 - $ pbook recover_lost_files 123 --test_mode - """ - test_mode = _require_bool("test_mode", test_mode) - if test_mode is _INVALID: - return - core = _ensure_init() - core.recover_lost_files(task_id, test_mode) - - -@app.command(name="show_workflow") -def show_workflow( - request_id: Annotated[int, typer.Argument(help="Workflow request ID")], -) -> None: - """Show workflow status. - - Send a request to show the status of a workflow. - - example: - >>> show_workflow(456) - - $ pbook show_workflow 456 - """ - core = _ensure_init() - _, output = core.execute_workflow_command("get_status", request_id) - if output: - print(output) - - -@app.command(name="kill_workflow") -def kill_workflow( - request_id: Annotated[int, typer.Argument(help="Workflow request ID")], -) -> None: - """Kill a workflow. - - Send a request to kill a workflow. - - example: - >>> kill_workflow(456) - - $ pbook kill_workflow 456 - """ - core = _ensure_init() - _, output = core.execute_workflow_command("abort", request_id) - if output: - print(output[0][-1]) - - -@app.command(name="retry_workflow") -def retry_workflow( - request_id: Annotated[int, typer.Argument(help="Workflow request ID")], -) -> None: - """Retry a workflow. - - Send a request to retry a workflow. - - example: - >>> retry_workflow(456) - - $ pbook retry_workflow 456 - """ - core = _ensure_init() - _, output = core.execute_workflow_command("retry", request_id) - if output: - print(output[0][-1]) - - -@app.command(name="finish_workflow") -def finish_workflow( - request_id: Annotated[int, typer.Argument(help="Workflow request ID")], -) -> None: - """Finish a workflow. - - Send a request to finish a workflow. - - example: - >>> finish_workflow(456) - - $ pbook finish_workflow 456 - """ - core = _ensure_init() - _, output = core.execute_workflow_command("finish", request_id) - if output: - print(output[0][-1]) - - -@app.command(name="pause_workflow") -def pause_workflow( - request_id: Annotated[int, typer.Argument(help="Workflow request ID")], -) -> None: - """Pause a workflow. - - Send a request to pause a workflow. - - example: - >>> pause_workflow(456) - - $ pbook pause_workflow 456 - """ - core = _ensure_init() - _, output = core.execute_workflow_command("suspend", request_id) - if output: - print(output[0][-1]) - - -@app.command(name="resume_workflow") -def resume_workflow( - request_id: Annotated[int, typer.Argument(help="Workflow request ID")], -) -> None: - """Resume a workflow. - - Send a request to resume a workflow. - - example: - >>> resume_workflow(456) - - $ pbook resume_workflow 456 - """ - core = _ensure_init() - _, output = core.execute_workflow_command("resume", request_id) - if output: - print(output[0][-1]) - - -@app.command(name="set_secret") -def set_secret( - key: Annotated[str, typer.Argument(help="Secret key")], - value: Annotated[str, typer.Argument(help="Secret value or file path")], - is_file: Annotated[bool, typer.Option("--is_file", help="Treat value as a file path to upload")] = False, -) -> None: - """Set a secret key-value pair. - - Define a pair of secret key-value strings. The value can be a file path to upload a - secret file when is_file=True. - - example: - >>> set_secret('mykey', 'myvalue') - >>> set_secret('mykey', '/path/to/file', is_file=True) - - $ pbook set_secret mykey myvalue - $ pbook set_secret mykey /path/to/file --is_file - """ - is_file = _require_bool("is_file", is_file) - if is_file is _INVALID: - return - core = _ensure_init() - core.set_secret(key, value, is_file) - - -@app.command(name="delete_secret") -def delete_secret( - key: Annotated[str, typer.Argument(help="Secret key to delete")], -) -> None: - """Delete a secret. - - example: - >>> delete_secret('mykey') - - $ pbook delete_secret mykey - """ - core = _ensure_init() - core.set_secret(key, None) - - -@app.command(name="delete_all_secrets") -def delete_all_secrets() -> None: - """Delete all secrets. - - example: - >>> delete_all_secrets - - $ pbook delete_all_secrets - """ - core = _ensure_init() - core.set_secret(None, None) - - -@app.command(name="list_secrets") -def list_secrets( - full: Annotated[bool, typer.Option("--full", help="Show complete secret values instead of truncating them")] = False, -) -> None: - """List secrets. - - Value strings are truncated by default. full=True to see entire strings. - - example: - >>> list_secrets() - >>> list_secrets(full=True) - - $ pbook list_secrets - $ pbook list_secrets --full - """ - full = _require_bool("full", full) - if full is _INVALID: - return - core = _ensure_init() - core.list_secrets(full) - - -@app.command(name="generate_credential") -def generate_credential() -> None: - """Generate a new proxy or token.""" - core = _get_core() - core.generate_credential() - - -# ─── Entry point ────────────────────────────────────────────────────────────── - -# Global options that consume the following argv token as their own value, so the -# subcommand-token scan below can skip over both. -_GLOBAL_VALUE_OPTS = {"-c"} - - -def _rewrite_legacy_kwargs(argv: list) -> list: - """Rewrite legacy bare `key=value` batch args into `--key=value`. - - The pre-Typer pbook batch mode accepted `pbook show format=long`; Click only - recognizes `--format=long`. Find the subcommand, look up its real option names via - introspection (never a separately maintained list), and rewrite any later bare - `key=value` token whose key matches one of them. Anything else - already-dashed - flags, positional args that happen to contain "=" - passes through untouched. - """ - i = 0 - while i < len(argv): - tok = argv[i] - if tok in _GLOBAL_VALUE_OPTS: - i += 2 - continue - if tok.startswith("-"): - i += 1 - continue - break - if i >= len(argv): - return argv - - sub_cmd = typer.main.get_command(app).commands.get(argv[i]) - if sub_cmd is None: - return argv - - option_flags = {} - flag_only = set() - for param in sub_cmd.params: - flags = [o for o in getattr(param, "opts", []) if o.startswith("--")] - if flags: - primary = flags[0] - # Register every alias (e.g. --ramCount for --memory), not just the primary - # name, so the legacy bare `key=value` syntax recognizes them too. - for flag in flags: - option_flags[flag.lstrip("-")] = primary - if getattr(param, "is_flag", False): - flag_only.add(param.name) - - rewritten = argv[: i + 1] - for tok in argv[i + 1 :]: - key, sep, value = tok.partition("=") - if sep and not tok.startswith("-") and key in option_flags: - flag = option_flags[key] - if key in flag_only: - # Click flag-style options (e.g. --soft) take no value at all; the legacy - # syntax passed an explicit True/False, so translate that into presence - # (truthy) or absence (falsy - same as the option's own default) instead. - normalized = value.strip().lower() - if normalized in ("true", "1", "yes"): - rewritten.append(flag) - elif normalized not in ("false", "0", "no"): - typer.echo( - f"Error: '{key}' is a flag and expects True/False (got '{value}' in '{tok}')", - err=True, - ) - sys.exit(1) - continue - rewritten.append(f"{flag}={value}") - else: - rewritten.append(tok) - return rewritten - - -def main() -> None: - sys.argv[0] = "pbook" - sys.argv[1:] = _rewrite_legacy_kwargs(sys.argv[1:]) - app() diff --git a/scripts/pbook b/scripts/pbook index 8ed2fe5f..b44250ee 100755 --- a/scripts/pbook +++ b/scripts/pbook @@ -2,4 +2,4 @@ source ${PANDA_SYS}/etc/panda/share/functions.sh -exec_p_command "import pandaclient.PBookTyper as pbook; pbook.main()" "$@" +exec_p_command "import pandaclient.PBookScript as pbook; pbook.main()" "$@" From f874b19f057a263cc78bea30a77b2d68de284776 Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Fri, 7 Aug 2026 17:10:54 +0200 Subject: [PATCH 58/59] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- pandaclient/PBookScript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandaclient/PBookScript.py b/pandaclient/PBookScript.py index 668fbf4c..34b0e2b8 100644 --- a/pandaclient/PBookScript.py +++ b/pandaclient/PBookScript.py @@ -80,7 +80,7 @@ $ pbook show 123 --format='long' $ pbook show 123 format='long' -If arg or value is a list in interactive mode, it is represented as a comma-separate list in batch mode. E.g. +If arg or value is a list in interactive mode, it is represented as a comma-separated list in batch mode. E.g. to kill three tasks in interactive mode: $ pbook From c072578b2721d08411bdedd14a9eb0342048e49d Mon Sep 17 00:00:00 2001 From: Fernando Barreiro Date: Fri, 7 Aug 2026 17:11:12 +0200 Subject: [PATCH 59/59] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- pandaclient/PBookScript.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandaclient/PBookScript.py b/pandaclient/PBookScript.py index 34b0e2b8..e2fd842a 100644 --- a/pandaclient/PBookScript.py +++ b/pandaclient/PBookScript.py @@ -1025,7 +1025,7 @@ def delete_all_secrets() -> None: """Delete all secrets. example: - >>> delete_all_secrets + >>> delete_all_secrets() $ pbook delete_all_secrets """