diff --git a/.github/labeler.yaml b/.github/labeler.yaml index 2481a112ce0ff6..45eac213d7f796 100644 --- a/.github/labeler.yaml +++ b/.github/labeler.yaml @@ -4,7 +4,7 @@ CI / testing: car: - changed-files: - - any-glob-to-all-files: '{openpilot/selfdrive/car/**,opendbc_repo}' + - any-glob-to-all-files: '{openpilot/selfdrive/car/**,first_party/opendbc_repo}' simulation: - changed-files: diff --git a/.github/workflows/model_review.yaml b/.github/workflows/model_review.yaml index 66f0398d09d393..d67e67f550bfee 100644 --- a/.github/workflows/model_review.yaml +++ b/.github/workflows/model_review.yaml @@ -32,7 +32,7 @@ jobs: run: | echo "content<> $GITHUB_OUTPUT echo "## Model Review" >> $GITHUB_OUTPUT - PYTHONPATH=${{ github.workspace }} MASTER_PATH=${{ github.workspace }}/base python scripts/reporter.py >> $GITHUB_OUTPUT + PYTHONPATH=${{ github.workspace }}:${{ github.workspace }}/first_party MASTER_PATH=${{ github.workspace }}/base python scripts/reporter.py >> $GITHUB_OUTPUT echo "EOF" >> $GITHUB_OUTPUT - name: Post model report comment diff --git a/.github/workflows/repo-maintenance.yaml b/.github/workflows/repo-maintenance.yaml index 14c613fbe433d3..45c4509abac989 100644 --- a/.github/workflows/repo-maintenance.yaml +++ b/.github/workflows/repo-maintenance.yaml @@ -6,7 +6,7 @@ on: workflow_dispatch: env: - PYTHONPATH: ${{ github.workspace }} + PYTHONPATH: "${{ github.workspace }}:${{ github.workspace }}/first_party" jobs: package_updates: diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 3ed0829a2647ed..cd0bf46de30652 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -19,7 +19,7 @@ concurrency: env: CI: 1 - PYTHONPATH: ${{ github.workspace }} + PYTHONPATH: "${{ github.workspace }}:${{ github.workspace }}/first_party" PYTEST: pytest --continue-on-collection-errors --durations=0 -n logical jobs: @@ -33,7 +33,7 @@ jobs: || fromJSON('["ubuntu-24.04"]') }} env: STRIPPED_DIR: /tmp/releasepilot - PYTHONPATH: /tmp/releasepilot + PYTHONPATH: "/tmp/releasepilot:/tmp/releasepilot/first_party" steps: - uses: actions/checkout@v6 with: diff --git a/.github/workflows/ui_preview.yaml b/.github/workflows/ui_preview.yaml index 7bfaea63aa7c0d..015b4b1905a0f5 100644 --- a/.github/workflows/ui_preview.yaml +++ b/.github/workflows/ui_preview.yaml @@ -103,7 +103,7 @@ jobs: if: github.event_name == 'pull_request_target' id: find_diff run: | - export PYTHONPATH=${{ github.workspace }} + export PYTHONPATH=${{ github.workspace }}:${{ github.workspace }}/first_party baseurl="https://github.com/commaai/ci-artifacts/raw/refs/heads/${{ env.BRANCH_NAME }}" COMMENT="" diff --git a/.gitmodules b/.gitmodules index ad6530de9ac910..4eed302bef423e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,18 +1,18 @@ [submodule "panda"] - path = panda + path = first_party/panda url = ../../commaai/panda.git [submodule "opendbc"] - path = opendbc_repo + path = first_party/opendbc_repo url = ../../commaai/opendbc.git [submodule "msgq"] - path = msgq_repo + path = first_party/msgq_repo url = ../../commaai/msgq.git [submodule "rednose_repo"] - path = rednose_repo + path = first_party/rednose_repo url = ../../commaai/rednose.git [submodule "teleoprtc_repo"] - path = teleoprtc_repo + path = first_party/teleoprtc_repo url = ../../commaai/teleoprtc [submodule "tinygrad"] - path = tinygrad_repo + path = first_party/tinygrad_repo url = https://github.com/tinygrad/tinygrad.git diff --git a/.vscode/launch.json b/.vscode/launch.json index 5a052a112aabd6..695c1fc0002e4d 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -53,7 +53,7 @@ "request": "attach", "pid": "${command:pickMyProcess}", "sourceMap": { - ".": "${workspaceFolder}/opendbc/safety" + ".": "${workspaceFolder}/first_party/opendbc/safety" }, "initCommands": [ "script import time; time.sleep(3)" @@ -63,14 +63,14 @@ "name": "Replay drive", "type": "debugpy", "request": "launch", - "program": "${workspaceFolder}/opendbc/safety/tests/safety_replay/replay_drive.py", + "program": "${workspaceFolder}/first_party/opendbc/safety/tests/safety_replay/replay_drive.py", "args": [ "${input:replayArg}" ], "console": "integratedTerminal", "justMyCode": false, "env": { - "PYTHONPATH": "${workspaceFolder}" + "PYTHONPATH": "${workspaceFolder}:${workspaceFolder}/first_party" }, "subProcess": true, "stopOnEntry": false diff --git a/Jenkinsfile b/Jenkinsfile index c06674a5c547b8..7a2dde331bc917 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -65,6 +65,10 @@ fi ln -snf ${env.TEST_DIR} /data/pythonpath +# submodules now live under first_party/; add it to the path (set after sourcing +# profiles since those reset PYTHONPATH to /data/pythonpath) +export PYTHONPATH="/data/pythonpath:/data/pythonpath/first_party" + cd ${env.TEST_DIR} || true time ${cmd} END""" @@ -237,8 +241,8 @@ node { }, 'replay': { deviceStage("model-replay", "tizi-replay", ["UNSAFE=1"], [ - step("build", "cd openpilot/system/manager && ./build.py", [diffPaths: ["openpilot/selfdrive/modeld/", "tinygrad_repo", "openpilot/selfdrive/test/process_replay/model_replay.py"]]), - step("model replay", "openpilot/selfdrive/test/process_replay/model_replay.py", [diffPaths: ["openpilot/selfdrive/modeld/", "tinygrad_repo", "openpilot/selfdrive/test/process_replay/model_replay.py"]]), + step("build", "cd openpilot/system/manager && ./build.py", [diffPaths: ["openpilot/selfdrive/modeld/", "first_party/tinygrad_repo", "openpilot/selfdrive/test/process_replay/model_replay.py"]]), + step("model replay", "openpilot/selfdrive/test/process_replay/model_replay.py", [diffPaths: ["openpilot/selfdrive/modeld/", "first_party/tinygrad_repo", "openpilot/selfdrive/test/process_replay/model_replay.py"]]), ]) }, 'tizi': { diff --git a/SConstruct b/SConstruct index 2e4002f1f2e318..6a9cb72d17377f 100644 --- a/SConstruct +++ b/SConstruct @@ -91,7 +91,7 @@ def _libflags(target, source, env, for_signature): env = Environment( ENV={ "PATH": os.environ['PATH'], - "PYTHONPATH": Dir("#").abspath, + "PYTHONPATH": Dir("#").abspath + os.pathsep + Dir("#first_party").abspath, "ACADOS_SOURCE_DIR": acados.DIR, "ACADOS_PYTHON_INTERFACE_PATH": acados.TEMPLATE_DIR, "TERA_PATH": acados.TERA_PATH @@ -114,24 +114,25 @@ env = Environment( CPPPATH=[ "#", "#openpilot", - "#msgq", + "#first_party", + "#first_party/msgq", "#openpilot/cereal/gen/cpp", acados_include_dirs, [x.INCLUDE_DIR for x in pkgs], ], LIBPATH=[ "#openpilot/common", - "#msgq_repo", + "#first_party/msgq_repo", "#openpilot/selfdrive/pandad", - "#rednose/helpers", + "#first_party/rednose/helpers", [x.LIB_DIR for x in pkgs], ], RPATH=[], CYTHONCFILESUFFIX=".cpp", COMPILATIONDB_USE_ABSPATH=True, - REDNOSE_ROOT="#", + REDNOSE_ROOT="#first_party", tools=["default", "cython", "compilation_db", "rednose_filter"], - toolpath=["#site_scons/site_tools", "#rednose_repo/site_scons/site_tools"], + toolpath=["#site_scons/site_tools", "#first_party/rednose_repo/site_scons/site_tools"], ) if arch != "larch64": env['_LIBFLAGS'] = _libflags @@ -221,7 +222,7 @@ Export('common') # Enable swaglog include in submodules env_swaglog = env.Clone() env_swaglog['CXXFLAGS'].append('-DSWAGLOG="\\"common/swaglog.h\\""') -SConscript(['msgq_repo/SConscript'], exports={'env': env_swaglog}) +SConscript(['first_party/msgq_repo/SConscript'], exports={'env': env_swaglog}) SConscript(['openpilot/cereal/SConscript']) @@ -231,10 +232,10 @@ Export('messaging') # Build other submodules -SConscript(['panda/SConscript']) +SConscript(['first_party/panda/SConscript']) # Build rednose library -SConscript(['rednose/SConscript']) +SConscript(['first_party/rednose/SConscript']) # Build system services SConscript([ diff --git a/msgq b/first_party/msgq similarity index 100% rename from msgq rename to first_party/msgq diff --git a/msgq_repo b/first_party/msgq_repo similarity index 100% rename from msgq_repo rename to first_party/msgq_repo diff --git a/opendbc b/first_party/opendbc similarity index 100% rename from opendbc rename to first_party/opendbc diff --git a/opendbc_repo b/first_party/opendbc_repo similarity index 100% rename from opendbc_repo rename to first_party/opendbc_repo diff --git a/panda b/first_party/panda similarity index 100% rename from panda rename to first_party/panda diff --git a/rednose b/first_party/rednose similarity index 100% rename from rednose rename to first_party/rednose diff --git a/rednose_repo b/first_party/rednose_repo similarity index 100% rename from rednose_repo rename to first_party/rednose_repo diff --git a/teleoprtc b/first_party/teleoprtc similarity index 100% rename from teleoprtc rename to first_party/teleoprtc diff --git a/teleoprtc_repo b/first_party/teleoprtc_repo similarity index 100% rename from teleoprtc_repo rename to first_party/teleoprtc_repo diff --git a/tinygrad b/first_party/tinygrad similarity index 100% rename from tinygrad rename to first_party/tinygrad diff --git a/tinygrad_repo b/first_party/tinygrad_repo similarity index 100% rename from tinygrad_repo rename to first_party/tinygrad_repo diff --git a/launch_chffrplus.sh b/launch_chffrplus.sh index 17a765e076a3b3..0aebd4a5a75b62 100755 --- a/launch_chffrplus.sh +++ b/launch_chffrplus.sh @@ -68,7 +68,7 @@ function launch { # handle pythonpath ln -sfn $(pwd) /data/pythonpath - export PYTHONPATH="$PWD" + export PYTHONPATH="$PWD:$PWD/first_party" # hardware specific init if [ -f /AGNOS ]; then diff --git a/openpilot/cereal/SConscript b/openpilot/cereal/SConscript index bee7620b1ee589..47baaca0ec12cd 100644 --- a/openpilot/cereal/SConscript +++ b/openpilot/cereal/SConscript @@ -5,11 +5,11 @@ gen_dir = Dir('gen') # Build cereal schema_files = ['log.capnp', 'deprecated.capnp', 'custom.capnp'] -car_capnp = '#opendbc_repo/opendbc/car/car.capnp' +car_capnp = '#first_party/opendbc_repo/opendbc/car/car.capnp' all_output = schema_files + ['car.capnp'] env.Command([f'gen/cpp/{s}.c++' for s in all_output] + [f'gen/cpp/{s}.h' for s in all_output], schema_files + [car_capnp], - f"capnpc --src-prefix={cereal_dir.path} --src-prefix=opendbc_repo/opendbc/car --import-path=opendbc_repo/opendbc/car $SOURCES -o c++:{gen_dir.path}/cpp/") + f"capnpc --src-prefix={cereal_dir.path} --src-prefix=first_party/opendbc_repo/opendbc/car --import-path=first_party/opendbc_repo/opendbc/car $SOURCES -o c++:{gen_dir.path}/cpp/") cereal = env.Library('cereal', [f'gen/cpp/{s}.c++' for s in all_output]) diff --git a/openpilot/selfdrive/modeld/SConscript b/openpilot/selfdrive/modeld/SConscript index 5f6281556f2539..bba5ab36b8ed26 100644 --- a/openpilot/selfdrive/modeld/SConscript +++ b/openpilot/selfdrive/modeld/SConscript @@ -20,7 +20,7 @@ chunker_file = File("#openpilot/common/file_chunker.py") lenv = env.Clone() tinygrad_root = env.Dir("#").abspath -tinygrad_files = ["#"+x for x in glob.glob(env.Dir("#tinygrad_repo").relpath + "/**", recursive=True, root_dir=tinygrad_root) +tinygrad_files = ["#"+x for x in glob.glob(env.Dir("#first_party/tinygrad_repo").relpath + "/**", recursive=True, root_dir=tinygrad_root) if 'pycache' not in x and os.path.isfile(os.path.join(tinygrad_root, x))] def estimate_pickle_max_size(onnx_size): @@ -123,7 +123,7 @@ for cam_w, cam_h in CAMERA_CONFIGS: lenv.Command(dm_pkl_path, tinygrad_files + compile_dm_warp_script + compile_modeld_script + [tg_devices_node], cmd) def tg_compile(flags, model_name): - pythonpath_string = 'PYTHONPATH="${PYTHONPATH}:' + env.Dir("#tinygrad_repo").abspath + '"' + pythonpath_string = 'PYTHONPATH="${PYTHONPATH}:' + env.Dir("#first_party/tinygrad_repo").abspath + '"' fn = File(f"models/{model_name}").abspath pkl = fn + "_tinygrad.pkl" onnx_path = fn + ".onnx" @@ -133,7 +133,7 @@ def tg_compile(flags, model_name): return lenv.Command( chunk_targets, [onnx_path] + tinygrad_files + [Value(chunk_targets), chunker_file, tg_devices_node], - [f'{pythonpath_string} {flags} python3 {Dir("#tinygrad_repo").abspath}/examples/openpilot/compile3.py {fn}.onnx {pkl}', + [f'{pythonpath_string} {flags} python3 {Dir("#first_party/tinygrad_repo").abspath}/examples/openpilot/compile3.py {fn}.onnx {pkl}', Action(do_chunk, " [CHUNK] $TARGET")], ) diff --git a/openpilot/selfdrive/pandad/panda.cc b/openpilot/selfdrive/pandad/panda.cc index d067a73107cd4a..e3e919dbb9708a 100644 --- a/openpilot/selfdrive/pandad/panda.cc +++ b/openpilot/selfdrive/pandad/panda.cc @@ -116,7 +116,7 @@ std::optional Panda::get_serial() { bool Panda::up_to_date() { if (auto fw_sig = get_firmware_version()) { for (auto fn : { "panda.bin.signed", "panda_h7.bin.signed" }) { - auto content = util::read_file(std::string("../../../panda/board/obj/") + fn); + auto content = util::read_file(std::string("../../../first_party/panda/board/obj/") + fn); if (content.size() >= fw_sig->size() && memcmp(content.data() + content.size() - fw_sig->size(), fw_sig->data(), fw_sig->size()) == 0) { return true; diff --git a/openpilot/selfdrive/test/scons_build_test.sh b/openpilot/selfdrive/test/scons_build_test.sh index 6f6eafcad15f73..8e1476fa1ea30e 100755 --- a/openpilot/selfdrive/test/scons_build_test.sh +++ b/openpilot/selfdrive/test/scons_build_test.sh @@ -11,7 +11,7 @@ cd $BASEDIR # helpful commands: # scons -Q --tree=derived -cd $BASEDIR/opendbc_repo/ +cd $BASEDIR/first_party/opendbc_repo/ scons --clean scons --no-cache --random if ! scons -q; then diff --git a/openpilot/tools/cabana/SConscript b/openpilot/tools/cabana/SConscript index 92e95d3cc235e7..fd66c90bbdf1e8 100644 --- a/openpilot/tools/cabana/SConscript +++ b/openpilot/tools/cabana/SConscript @@ -116,4 +116,4 @@ output_json_file = 'openpilot/tools/cabana/dbc/car_fingerprint_to_dbc.json' generate_dbc = cabana_env.Command('#' + output_json_file, ['dbc/generate_dbc_json.py'], "python3 openpilot/tools/cabana/dbc/generate_dbc_json.py --out " + output_json_file) -cabana_env.Depends(generate_dbc, ["#openpilot/common", '#opendbc_repo', "#openpilot/cereal", "#msgq_repo"]) +cabana_env.Depends(generate_dbc, ["#openpilot/common", '#first_party/opendbc_repo', "#openpilot/cereal", "#first_party/msgq_repo"]) diff --git a/openpilot/tools/jotpluggler/SConscript b/openpilot/tools/jotpluggler/SConscript index 3e235bb07a6ec3..9871aac95e99aa 100644 --- a/openpilot/tools/jotpluggler/SConscript +++ b/openpilot/tools/jotpluggler/SConscript @@ -95,8 +95,8 @@ event_extractors = jot_env.Command("generated_event_extractors.h", [ "generate_event_extractors.py", jot_env.Glob("#openpilot/cereal/*.capnp"), jot_env.Glob("#openpilot/cereal/include/*.capnp"), - "#opendbc_repo/opendbc/car/car.capnp", - "#opendbc_repo/opendbc/car/include/c++.capnp", + "#first_party/opendbc_repo/opendbc/car/car.capnp", + "#first_party/opendbc_repo/opendbc/car/include/c++.capnp", ], generate_event_extractors, ) diff --git a/openpilot/tools/jotpluggler/generate_event_extractors.py b/openpilot/tools/jotpluggler/generate_event_extractors.py index be9bd49003afe0..6da58c61f64888 100644 --- a/openpilot/tools/jotpluggler/generate_event_extractors.py +++ b/openpilot/tools/jotpluggler/generate_event_extractors.py @@ -321,7 +321,7 @@ def generate(self): repo_root = Path(sys.argv[1]).resolve() output = Path(sys.argv[2]) capnp.remove_import_hook() - log = capnp.load(str(repo_root / "openpilot" / "cereal" / "log.capnp"), imports=[str(repo_root / "opendbc_repo" / "opendbc" / "car")]) + log = capnp.load(str(repo_root / "openpilot" / "cereal" / "log.capnp"), imports=[str(repo_root / "first_party" / "opendbc_repo" / "opendbc" / "car")]) generated = Generator(log.Event.schema).generate() output.parent.mkdir(parents=True, exist_ok=True) output.write_text(generated) diff --git a/openpilot/tools/op.sh b/openpilot/tools/op.sh index 3d0af526bac7da..846add674255b2 100755 --- a/openpilot/tools/op.sh +++ b/openpilot/tools/op.sh @@ -281,7 +281,7 @@ function op_script() { op_before_cmd case $1 in - som-debug ) op_run_command panda/scripts/som_debug.sh "${@:2}" ;; + som-debug ) op_run_command first_party/panda/scripts/som_debug.sh "${@:2}" ;; * ) echo -e "Unknown script '$1'. Available scripts:" echo -e " ${BOLD}som-debug${NC} SOM serial debug console via panda" diff --git a/openpilot/tools/plotjuggler/juggle.py b/openpilot/tools/plotjuggler/juggle.py index 593192ed3031fa..9fc1dc915b191e 100755 --- a/openpilot/tools/plotjuggler/juggle.py +++ b/openpilot/tools/plotjuggler/juggle.py @@ -95,7 +95,7 @@ def start_juggler(fn=None, dbc=None, layout=None, route_or_segment_name=None, pl with open(os.path.join(tmp_cereal, schema), "w") as dst: dst.write(contents) os.symlink(os.path.join(BASEDIR, "openpilot", "cereal", "include"), os.path.join(tmp_cereal, "include"), target_is_directory=True) - os.symlink(os.path.join(BASEDIR, "opendbc_repo", "opendbc", "car", "car.capnp"), os.path.join(tmp_cereal, "car.capnp")) + os.symlink(os.path.join(BASEDIR, "first_party", "opendbc_repo", "opendbc", "car", "car.capnp"), os.path.join(tmp_cereal, "car.capnp")) env["BASEDIR"] = schema_root subprocess.call(cmd, shell=True, env=env, cwd=juggle_dir) diff --git a/pyproject.toml b/pyproject.toml index 15efc934fe3270..2a0610040d5f6c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -123,13 +123,15 @@ build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] packages = [ "openpilot", - "msgq", - "opendbc", - "panda", - "rednose", - "teleoprtc", - "tinygrad", + "first_party/msgq", + "first_party/opendbc", + "first_party/panda", + "first_party/rednose", + "first_party/teleoprtc", + "first_party/tinygrad", ] +# submodules live under first_party/; add it to the editable install's import path +dev-mode-dirs = [ ".", "first_party" ] [tool.hatch.metadata] allow-direct-references = true diff --git a/release/build_release.sh b/release/build_release.sh index 4f609aea3fc25a..17b15879c3d7cd 100755 --- a/release/build_release.sh +++ b/release/build_release.sh @@ -38,8 +38,8 @@ cp -pR --parents $(./release/release_files.py) $BUILD_DIR/ # in the directory cd $BUILD_DIR -rm -f panda/board/obj/panda.bin.signed -rm -f panda/board/obj/panda_h7.bin.signed +rm -f first_party/panda/board/obj/panda.bin.signed +rm -f first_party/panda/board/obj/panda_h7.bin.signed VERSION=$(cat openpilot/common/version.h | awk -F[\"-] '{print $2}') echo "[-] committing version $VERSION T=$SECONDS" @@ -47,15 +47,15 @@ git add -f . git commit -a -m "openpilot v$VERSION release" # Build -export PYTHONPATH="$BUILD_DIR" +export PYTHONPATH="$BUILD_DIR:$BUILD_DIR/first_party" scons if [ -z "$PANDA_DEBUG_BUILD" ]; then # release panda fw - CERT=/data/pandaextra/certs/release RELEASE=1 scons panda/ + CERT=/data/pandaextra/certs/release RELEASE=1 scons first_party/panda/ else # build with ALLOW_DEBUG=1 to enable features like experimental longitudinal - scons panda/ + scons first_party/panda/ fi # Ensure no submodules in release diff --git a/release/build_stripped.sh b/release/build_stripped.sh index ea74aac859796c..5bf7b3cab7e478 100755 --- a/release/build_stripped.sh +++ b/release/build_stripped.sh @@ -43,7 +43,7 @@ cp -pR --parents $(./release/release_files.py) $TARGET_DIR/ # in the directory cd $TARGET_DIR rm -rf .git/modules/ -rm -f panda/board/obj/panda.bin.signed +rm -f first_party/panda/board/obj/panda.bin.signed find openpilot/selfdrive/modeld/models -name '*.onnx' -size +95M -exec ./openpilot/common/file_chunker.py {} \; diff --git a/release/check-submodules.sh b/release/check-submodules.sh index 93869a740347d9..083e319fc6b1ee 100755 --- a/release/check-submodules.sh +++ b/release/check-submodules.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash while read hash submodule ref; do - if [ "$submodule" = "tinygrad_repo" ]; then + if [ "$submodule" = "first_party/tinygrad_repo" ]; then echo "Skipping $submodule" continue fi diff --git a/scripts/docs.py b/scripts/docs.py index d60bfb791f65f3..d4508ee2fc87d5 100644 --- a/scripts/docs.py +++ b/scripts/docs.py @@ -14,6 +14,7 @@ DOCS_DIR = REPO_ROOT / "docs" SITE_DIR = REPO_ROOT / "docs_site" sys.path.insert(0, str(REPO_ROOT)) +sys.path.insert(0, str(REPO_ROOT / "first_party")) # Local docs build helpers live under docs/ so they stay near the content # source. The wrapper prunes them from docs_site/ after build. sys.path.insert(0, str(DOCS_DIR))