diff --git a/launches/auto-nanogpt-extra-instructions.md b/launches/auto-nanogpt-extra-instructions.md new file mode 100644 index 000000000..41be53559 --- /dev/null +++ b/launches/auto-nanogpt-extra-instructions.md @@ -0,0 +1,61 @@ +# Auto-nanoGPT Launch Extra Instructions + +This target is not a physical AI, CFD, or surrogate-modelling task. When the +generic Senpai system prompt mentions physical metrics, CFD, datasets, or +architecture search for physical modelling, reinterpret that guidance as: +optimize the fixed `modded-nanogpt` track 3 optimizer benchmark described in +`$PROBLEM_DIR/program.md`. + +Keep the task close to the public `modded-nanogpt` track 3 benchmark: + +- The objective is lower optimizer steps to FineWeb validation loss below 3.28. +- Keep data, batch size, model architecture, and one forward-backward pass per + optimizer step fixed. +- Focus on optimizer algorithms, optimizer hyperparameters, schedules, and + initialization. +- Do not invent extra data configs, data files, or benchmark rules beyond the + target repo's `program.md` and the official track 3 README. + +Keep the research portfolio balanced between exploitation and exploration. +Retuning LR/WD/cooldown is important when giving a new method a fair shot, but +do not let the run become mostly scalar hyperparameter search. Assign fresh +optimizer mechanisms, preconditioners, schedule ideas, initialization ideas, +and pruning ablations of complex stacks. + +Be deliberate about step budgets. The launch timeout and max-epoch values are +hard ceilings, not instructions to run forever. Use tiny runs for smoke tests, +shorter step budgets for uncertain optimizer screening, and longer predeclared +seed batches for serious confirmation. Think clearly before each PR about +whether the run is exploration, tuning, pruning, or confirmation. + +Early kill gates are encouraged for obvious crashes, non-finite loss, exploding +gradients, or hopeless screening runs, but do not use per-run validation loss to +cherry-pick final steps or seeds. Final claims must report all non-cherry-picked +runs at a predeclared step count and evaluate the benchmark statistical rule. + +Use W&B aggressively. Preserve and extend the starter script's telemetry for +losses, validation loss, steps-to-target, learning rates, weight decay, gradient +norms and distributions, parameter norms and distributions, and any +optimizer-specific diagnostics that help explain why an idea worked or failed. +The starter script also logs trailing-window train-loss slopes every 10% of the +run and at the final step; use these slopes to reason about curve shape without +overreacting to individual noisy steps. + +Focus only on your assigned advisor branch, research tag, student list, PR +stream, and W&B runs. Ignore work that is not labeled with your branch/tag. Use +the benchmark snapshot checked into this target repo; do not refresh, browse, +fetch, or mine new upstream PRs, branches, records, issues, or post-launch +updates during this run. + +Prime Intellect's autonomous-run materials are explicitly banned sources for +agents during this launch. Do not open, fetch, browse, search within, clone, +cite, summarize, or use: + +- `https://www.primeintellect.ai/auto-nanogpt` +- `https://github.com/PrimeIntellect-ai/experiments-autonomous-speedrunning` +- any raw GitHub URLs, files, branches, issues, pull requests, or archives under + that repository + +Those links are named only so you know what not to read. They are comparison +artifacts for humans after the run, not part of the active experimental +context. diff --git a/launches/launch-auto-nanogpt.sh b/launches/launch-auto-nanogpt.sh new file mode 100755 index 000000000..7e6a77565 --- /dev/null +++ b/launches/launch-auto-nanogpt.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +set -euo pipefail + +target_repo_url="https://github.com/morganmcg1/modded-nanogpt-senpai.git" +target_repo_branch="senpai-launch-20260515" +extra_instructions="launches/auto-nanogpt-extra-instructions.md" +wandb_entity="wandb-applied-ai-team" +wandb_project="modded-nanogpt-senpai" +timeout_minutes="30240" +max_epochs="100000" + +env_file="${SENPAI_ENV_FILE:-}" +if [[ -z "${env_file}" ]]; then + if [[ -f ".env" ]]; then + env_file=".env" + elif [[ -f "${HOME}/ML/senpai/.env" ]]; then + env_file="${HOME}/ML/senpai/.env" + fi +fi +if [[ -n "${env_file}" ]]; then + set -a + # shellcheck disable=SC1090 + source "${env_file}" + set +a +fi + +if [[ -n "${SENPAI_PYTHON:-}" ]]; then + launch_cmd=("${SENPAI_PYTHON}") +elif python -c "import simple_parsing" >/dev/null 2>&1; then + launch_cmd=(python) +elif command -v uv >/dev/null 2>&1; then + launch_cmd=(uv run python) +else + echo "Could not find Python with simple_parsing. Install the Senpai deps or set SENPAI_PYTHON." >&2 + exit 1 +fi + +for rep in 1 2 3 4 5; do + tag="auto-nanogpt-1gpu-r${rep}" + "${launch_cmd[@]}" k8s/launch.py \ + --tag "${tag}" \ + --advisor \ + --target_repo_url "${target_repo_url}" \ + --target_repo_branch "${target_repo_branch}" \ + --advisor_branch "${tag}" \ + --gh_history_scope fresh \ + --n_students 8 \ + --student_prefix "g1r${rep}" \ + --gpus_per_student 1 \ + --timeout_minutes "${timeout_minutes}" \ + --max_epochs "${max_epochs}" \ + --wandb_entity "${wandb_entity}" \ + --wandb_project "${wandb_project}" \ + --extra_instructions "${extra_instructions}" \ + "$@" +done