From 8637a31a042a2294a9c2ff943b8ce60bd4f6e97d Mon Sep 17 00:00:00 2001 From: Ian McEwen Date: Tue, 28 Oct 2025 11:36:30 -0700 Subject: [PATCH 1/7] initial todo commit --- container.go | 2 ++ jobs.go | 2 ++ 2 files changed, 4 insertions(+) diff --git a/container.go b/container.go index be54c6e..bddb7b5 100644 --- a/container.go +++ b/container.go @@ -60,6 +60,8 @@ type Container struct { MinMemoryLimit int64 `json:"min_memory_limit"` // The minimum the container needs. MaxCPUCores float32 `json:"max_cpu_cores"` // The maximum number of cores the container needs. MinCPUCores float32 `json:"min_cpu_cores"` // The minimum number of cores the container needs. + MaxGPUs int64 `json:"max_gpus"` // The maximum number of GPUs the container needs. + MinGPUs int64 `json:"min_gpus"` // The minimum number of GPUs the container needs. MinDiskSpace int64 `json:"min_disk_space"` // The minimum amount of disk space that the container needs. PIDsLimit int64 `json:"pids_limit"` Image ContainerImage `json:"image"` diff --git a/jobs.go b/jobs.go index ddaea81..09a0e88 100644 --- a/jobs.go +++ b/jobs.go @@ -446,6 +446,8 @@ func (job *Job) CPURequest() float32 { return cpu } +// TODO: GPURequest? + // MemoryRequest calculates the highest minimum memory among the steps of a job // (i.e. the largest slot size the job will need), or 0 if no steps have min // memory set. From 339b95e0b255333975e305fe83b3e2140be2fad3 Mon Sep 17 00:00:00 2001 From: Ian McEwen Date: Tue, 28 Oct 2025 11:45:26 -0700 Subject: [PATCH 2/7] add GPURequest functions and stuff --- jobs.go | 15 ++++++++++++++- jobs_test.go | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/jobs.go b/jobs.go index 09a0e88..3434486 100644 --- a/jobs.go +++ b/jobs.go @@ -446,7 +446,20 @@ func (job *Job) CPURequest() float32 { return cpu } -// TODO: GPURequest? +// GPURequest calculates the highest minimum GPU count among the steps of a job +// (i.e. the largest slot size the job will need in terms of GPUs), or 0 if no +// steps have min GPUs set. +func (job *Job) GPURequest() int64 { + var gpus int64 + + for _, step := range job.Steps { + if step.Component.Container.MinGPUs > gpus { + gpus = step.Component.Container.MinGPUs + } + } + + return gpus +} // MemoryRequest calculates the highest minimum memory among the steps of a job // (i.e. the largest slot size the job will need), or 0 if no steps have min diff --git a/jobs_test.go b/jobs_test.go index 272b2b8..6478a1d 100644 --- a/jobs_test.go +++ b/jobs_test.go @@ -718,6 +718,40 @@ func TestDiskRequest(t *testing.T) { } } +func TestGPURequest(t *testing.T) { + // Start with a fresh initialized submission + s := _inittests(t, false) + + // Default test data does not specify GPUs; expect 0 + gpus := s.GPURequest() + var expected int64 + expected = 0 + if gpus != expected { + t.Errorf("GPU request was %d, not %d", gpus, expected) + } + + // Set a minimum GPU requirement on the existing step and recheck + s.Steps[0].Component.Container.MinGPUs = 1 + gpus = s.GPURequest() + expected = 1 + if gpus != expected { + t.Errorf("GPU request was %d, not %d after setting MinGPUs=1", gpus, expected) + } + + // Add a second step with a higher MinGPUs to ensure we take the maximum of mins + second := s.Steps[0] // shallow copy is fine for test purposes + second.Component.Container.MinGPUs = 4 + s.Steps = append(s.Steps, second) + gpus = s.GPURequest() + expected = 4 + if gpus != expected { + t.Errorf("GPU request was %d, not %d after adding second step with MinGPUs=4", gpus, expected) + } + + // Reset memoized submission for other tests + _inittests(t, false) +} + func TestExtractJobID(t *testing.T) { testData := []byte(`1000 job(s) submitted to cluster 100000000.0000.`) actual := ExtractJobID(testData) From 9f2b84b59bbe8ca201b85053f194202765e219b1 Mon Sep 17 00:00:00 2001 From: Ian McEwen Date: Tue, 28 Oct 2025 11:52:26 -0700 Subject: [PATCH 3/7] Add copilot instructions file --- .github/copilot-instructions.md | 70 +++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 .github/copilot-instructions.md diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..4ce8cf2 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,70 @@ +## Project overview + +This repository contains Go types and helpers for CyVerse job submissions (analyses). +Primary responsibilities: +- Define data models for jobs, steps, containers, volumes, and I/O (see `jobs.go`, `step.go`, `container.go`, `io.go`, `interapps.go`). +- Provide helper methods used by higher-level services to build Condor submissions and prepare container run arguments. + +The code is a library consumed by other CyVerse services; it does not contain a main binary. Expect callers to provide configuration via `viper` and JSON payloads that map onto the types here (for example `Job`, `Step`, `Container`). + +## Key files to reference +- `go.mod` — project dependencies and Go version (module path `github.com/cyverse-de/model/v8`). +- `jobs.go` — central Job type and many helpers (DirectoryName, OutputDirectory, FinalOutputArguments, resource request helpers). +- `step.go` — Step, StepConfig, params/arguments ordering, and env/exec helpers used when assembling runtime commands. +- `container.go` — Container, Volume, ContainerImage and container-specific helpers (WorkingDirectory, UsesVolumes). +- `io.go` — Input/Output types and porklock argument builders used for iRODS transfers. +- `interapps.go` — Interactive app reverse-proxy and websocket configuration fields. + +## Patterns & conventions +- Data-first: types are plain structs with JSON tags. External services submit JSON that is unmarshaled into these structs. Example: `NewFromData(cfg, data []byte)` in `jobs.go`. +- Computed accessors: many derived values are exposed via methods (e.g. `OutputDirectory()`, `WorkingDirectory()`, `Arguments()`); prefer these helpers rather than directly reading fields. +- Parameter ordering: step parameters are ordered by `Order` and returned via `StepConfig.Parameters()`; use that function to construct CLI args. +- Backwards compatibility: some logic checks image names for legacy compatibility (see `Step.IsBackwardsCompatible()`). + +## Build / test / dev workflows +- This is a Go module. Use the module-aware Go toolchain. Common commands: + + - Run unit tests quickly: `go test ./...` + - Tidy dependencies: `go mod tidy` + - Lint with golangci-lint (same as CI): `golangci-lint run ./...` + - Optionally also run `go vet` if available. + +- The repo contains unit tests (files ending with `_test.go`); tests exercise models and helpers. Tests are the recommended way to validate that model changes don't break callers. + +### Linting +- CI uses golangci-lint (v1.64+). Run locally before committing: + - `golangci-lint run ./...` +- Fixes should be minimal and avoid changing public APIs unless intended. Re-run `go test ./...` after fixes. + +## Integration points & expectations +- iRODS: the io helpers generate porklock-style arguments and expect a mounted `/configs/irods-config` inside runtime containers. +- HTCondor: job metadata, log paths, and submission content assume Condor-style executions (see `CondorLogDirectory`, `FinalOutputArguments`). +- Containers: this package describes image names, tags and resource requests; other services read `Container` values to build runtime tasks. + +## Examples (copyable snippets) +- Build porklock get args for an input: + +```go +args := input.Arguments(username, job.FileMetadata) +// returns []string like: ["get","--user",username,"--source", input.IRODSPath(), "--config", "/configs/irods-config", ...] +``` + +- Get ordered parameters for a step: + +```go +params := step.Config.Parameters() +for _, p := range params { // stable order by p.Order + // use p.Name and p.Value +} +``` + +## What to watch for when editing +- Preserve JSON tags and method contracts. Changes to struct fields or JSON tags can break callers. +- Keep computed helper behavior stable (return formats of directory paths, trailing slashes for collections, etc.). Tests in the repo exercise many of these behaviors — update/add tests when changing behavior. +- This module is versioned as v8 in the module path. If releasing a new major version, update module path and coordinate with downstream consumers. + +## Where to look next / follow-ups +- If you need runtime examples of how these models are consumed, search in the `cyverse-de` organization for imports of `github.com/cyverse-de/model/v8`. +- Add small focused tests for behavioral changes: a happy path + one edge case (e.g., collection trailing slash handling). + +If any section is unclear or you'd like the file to include more examples (test examples, typical JSON payloads, or a small quick-start snippet for local testing), tell me which part and I'll iterate. From 7f688f1a1a5d6124f7cbe6cd7d93efaa474eaf05 Mon Sep 17 00:00:00 2001 From: Ian McEwen Date: Tue, 28 Oct 2025 11:54:28 -0700 Subject: [PATCH 4/7] update version to be v9 --- .github/copilot-instructions.md | 2 +- go.mod | 2 +- jobs.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 4ce8cf2..c73a1cc 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -8,7 +8,7 @@ Primary responsibilities: The code is a library consumed by other CyVerse services; it does not contain a main binary. Expect callers to provide configuration via `viper` and JSON payloads that map onto the types here (for example `Job`, `Step`, `Container`). ## Key files to reference -- `go.mod` — project dependencies and Go version (module path `github.com/cyverse-de/model/v8`). +- `go.mod` — project dependencies and Go version (module path `github.com/cyverse-de/model/**`). - `jobs.go` — central Job type and many helpers (DirectoryName, OutputDirectory, FinalOutputArguments, resource request helpers). - `step.go` — Step, StepConfig, params/arguments ordering, and env/exec helpers used when assembling runtime commands. - `container.go` — Container, Volume, ContainerImage and container-specific helpers (WorkingDirectory, UsesVolumes). diff --git a/go.mod b/go.mod index 00d4943..a0aeb1b 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/cyverse-de/model/v8 +module github.com/cyverse-de/model/v9 go 1.24 diff --git a/jobs.go b/jobs.go index 3434486..d898213 100644 --- a/jobs.go +++ b/jobs.go @@ -13,7 +13,7 @@ import ( "strings" "time" - "github.com/cyverse-de/model/v8/submitfile" + "github.com/cyverse-de/model/v9/submitfile" "github.com/spf13/viper" ) From 1ef101cf8c82e329180f9ed7a6cbcc89c78f4228 Mon Sep 17 00:00:00 2001 From: Ian McEwen Date: Tue, 28 Oct 2025 11:59:35 -0700 Subject: [PATCH 5/7] Update .github/copilot-instructions.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .github/copilot-instructions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index c73a1cc..680f857 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -61,7 +61,7 @@ for _, p := range params { // stable order by p.Order ## What to watch for when editing - Preserve JSON tags and method contracts. Changes to struct fields or JSON tags can break callers. - Keep computed helper behavior stable (return formats of directory paths, trailing slashes for collections, etc.). Tests in the repo exercise many of these behaviors — update/add tests when changing behavior. -- This module is versioned as v8 in the module path. If releasing a new major version, update module path and coordinate with downstream consumers. +- This module is versioned as v9 in the module path. If releasing a new major version, update module path and coordinate with downstream consumers. ## Where to look next / follow-ups - If you need runtime examples of how these models are consumed, search in the `cyverse-de` organization for imports of `github.com/cyverse-de/model/v8`. From fc7ea6d9c5ac16059862fdd31e553233d01509e3 Mon Sep 17 00:00:00 2001 From: Ian McEwen Date: Tue, 28 Oct 2025 11:59:54 -0700 Subject: [PATCH 6/7] Update .github/copilot-instructions.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .github/copilot-instructions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 680f857..a74036e 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -64,7 +64,7 @@ for _, p := range params { // stable order by p.Order - This module is versioned as v9 in the module path. If releasing a new major version, update module path and coordinate with downstream consumers. ## Where to look next / follow-ups -- If you need runtime examples of how these models are consumed, search in the `cyverse-de` organization for imports of `github.com/cyverse-de/model/v8`. +- If you need runtime examples of how these models are consumed, search in the `cyverse-de` organization for imports of `github.com/cyverse-de/model/v9`. - Add small focused tests for behavioral changes: a happy path + one edge case (e.g., collection trailing slash handling). If any section is unclear or you'd like the file to include more examples (test examples, typical JSON payloads, or a small quick-start snippet for local testing), tell me which part and I'll iterate. From 70d79558174171cd6416e58ce149845d6d188c19 Mon Sep 17 00:00:00 2001 From: Ian McEwen Date: Tue, 28 Oct 2025 12:00:13 -0700 Subject: [PATCH 7/7] Update jobs_test.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- jobs_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jobs_test.go b/jobs_test.go index 6478a1d..3514598 100644 --- a/jobs_test.go +++ b/jobs_test.go @@ -724,8 +724,7 @@ func TestGPURequest(t *testing.T) { // Default test data does not specify GPUs; expect 0 gpus := s.GPURequest() - var expected int64 - expected = 0 + expected := int64(0) if gpus != expected { t.Errorf("GPU request was %d, not %d", gpus, expected) }