cherry-pick: add MongoDB external table scan

.github/workflows/mongodb-connector.yml

@@ -0,0 +1,87 @@
+name: MongoDB Connector E2E
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+    paths:
+      - '.github/workflows/entrypoint.yaml'
+      - '.github/workflows/mongodb-connector.yml'
+      - 'Makefile'
+      - 'go.mod'
+      - 'go.sum'
+      - 'optools/mongodb_ci.bash'
+      - 'etc/launch-mongodb-local/**'
+      - 'etc/launch/**'
+      - 'test/mongodb/**'
+      - 'pkg/sql/mongodb/**'
+      - 'pkg/sql/colexec/mongoscan/**'
+      - 'pkg/sql/colexec/timewin/**'
+      - 'pkg/sql/colexec/aggexec/**'
+      - 'pkg/sql/plan/**'
+      - 'pkg/sql/compile/**'
+      - 'pkg/frontend/**'
+      - 'pkg/sql/parsers/**'
+      - 'pkg/config/**'
+      - 'pkg/defines/**'
+      - 'pkg/bootstrap/**'
+      - 'pkg/cnservice/**'
+      - 'pkg/clusterservice/**'
+      - 'pkg/queryservice/**'
+      - 'pkg/vm/**'
+      - 'pkg/pb/plan/**'
+      - 'pkg/pb/pipeline/**'
+      - 'pkg/pb/query/**'
+      - 'pkg/util/metric/v2/**'
+      - 'proto/plan.proto'
+      - 'proto/pipeline.proto'
+      - 'proto/query.proto'
+      - 'vendor/go.mongodb.org/**'
+      - 'vendor/github.com/xdg-go/**'
+      - 'vendor/github.com/youmark/pkcs8/**'
+      - 'vendor/github.com/montanaflynn/stats/**'
+      - 'vendor/github.com/klauspost/compress/**'
+      - 'vendor/github.com/golang/snappy/**'
+      - 'vendor/modules.txt'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  mongodb-e2e-local:
+    name: MongoDB E2E Local
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    env:
+      MO_MONGODB_REPORT_DIR: test/mongodb/reports/e2e-${{ github.run_id }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache: true
+
+      - name: Run MongoDB connector E2E
+        run: make test-mongodb-e2e-local
+
+      - name: Summarize MongoDB E2E
+        if: always()
+        run: |
+          {
+            echo "## MongoDB connector E2E"
+            echo ""
+            echo "Artifact: \`mongodb-e2e-local-${{ github.sha }}-${{ github.run_id }}\`"
+            echo ""
+            if [ -f "${MO_MONGODB_REPORT_DIR}/summary.md" ]; then
+              cat "${MO_MONGODB_REPORT_DIR}/summary.md"
+            else
+              echo "No summary was created."
+            fi
+          } >> "$GITHUB_STEP_SUMMARY"
+
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: mongodb-e2e-local-${{ github.sha }}-${{ github.run_id }}
+          path: test/mongodb/reports/**

.gitignore

@@ -18,6 +18,12 @@
 *.py
 !pkg/iceberg/metadata/testdata/generate_golden_vectors.py
 !optools/iceberg_external_runner.py
+!suites/scenarios/14_issue_regression/issue_25599_proxy_disconnect_cancel.py
+!suites/scenarios/14_issue_regression/issue_25599_proxy_disconnect_cancel_test.py
+!optools/mongodb_changed_files.py
+!test/mongodb/compare_results.py
+!test/mongodb/test_changes.py
+!test/mongodb/test_compare_results.py
 *.sh
 !etc/launch-minio-local/tier-b/seed-nyc-tlc-iceberg.sh
 bin/

Makefile

@@ -421,6 +421,14 @@ test-iceberg-readiness:
 test-iceberg-e2e-local:
 	@optools/iceberg_ci.bash e2e-local
 
+.PHONY: test-mongodb-e2e-local
+test-mongodb-e2e-local:
+	@optools/mongodb_ci.bash e2e-local
+
+.PHONY: test-mongodb-unit
+test-mongodb-unit:
+	@optools/mongodb_ci.bash unit
+
 .PHONY: test-iceberg-local
 test-iceberg-local:
 	@optools/iceberg_ci.bash local

etc/launch-mongodb-local/README.md

@@ -0,0 +1,7 @@
+# Local MongoDB connector fixture
+
+`optools/mongodb_ci.bash` is the only supported entry point. It creates a unique Compose project, port, keyfile, credentials, MongoDB volume, MO data directory and report directory; waits for a writable ReplicaSet primary; seeds a read-only MongoDB user; and cleans everything on exit.
+
+The connector uses a direct connection to the published localhost port for this single-node fixture. Production ReplicaSets should use normal discovery and a dedicated read-only role with only `find`, `listCollections`, `listIndexes` and `collStats` on the mapped databases/collections. Do not grant write or cluster-administration roles to a MatrixOne source identity.
+
+The built-in environment resolver is intended for local/controlled deployments. System-account references must start with `secret://env/MO_MONGODB_`; tenant account `N` is restricted to `secret://env/MO_MONGODB_ACCOUNT_N_`, preventing one tenant from naming another tenant's or an unrelated process secret. Production secret-manager implementations receive the account ID and must enforce an equivalent namespace and authorization check.

etc/launch-mongodb-local/compose.yaml

@@ -0,0 +1,31 @@
+# Copyright 2026 Matrix Origin
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+
+services:
+  mongo:
+    image: mongo@sha256:95a98776f273721a295b03098578b06bc10281bb56aa828c77e9f60ecc70b150
+    entrypoint: ["bash", "-c", "cp /run/key-source /tmp/mongodb-keyfile && chown mongodb:mongodb /tmp/mongodb-keyfile && chmod 400 /tmp/mongodb-keyfile && exec /usr/local/bin/docker-entrypoint.sh \"$$@\"", "--"]
+    command: ["mongod", "--replSet", "rs0", "--bind_ip_all", "--auth", "--keyFile", "/tmp/mongodb-keyfile"]
+    ports:
+      - "127.0.0.1:${MONGODB_PORT}:27017"
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: ${MONGODB_ROOT_USER}
+      MONGO_INITDB_ROOT_PASSWORD: ${MONGODB_ROOT_PASSWORD}
+    volumes:
+      - mongodb-data:/data/db
+      - ${MONGODB_KEYFILE}:/run/key-source:ro
+    healthcheck:
+      test: ["CMD-SHELL", "mongosh --quiet --username $$MONGO_INITDB_ROOT_USERNAME --password $$MONGO_INITDB_ROOT_PASSWORD --authenticationDatabase admin --eval 'quit(db.adminCommand({ping:1}).ok ? 0 : 2)' >/dev/null"]
+      interval: 2s
+      timeout: 5s
+      retries: 60
+    networks: [mongodb-e2e]
+
+volumes:
+  mongodb-data:
+
+networks:
+  mongodb-e2e:
+    driver: bridge

etc/launch-mongodb-local/init_and_seed.js

@@ -0,0 +1,27 @@
+// Copyright 2026 Matrix Origin
+// Licensed under the Apache License, Version 2.0.
+
+const source = db.getSiblingDB("mongodb_source");
+if (!source.getUser("mo_reader")) {
+  source.createUser({
+    user: "mo_reader",
+    pwd: process.env.MONGODB_READER_PASSWORD,
+    roles: [{role: "read", db: "mongodb_source"}]
+  });
+}
+if (!source.getUser("mo_reader_next")) {
+  source.createUser({
+    user: "mo_reader_next",
+    pwd: process.env.MONGODB_READER_NEXT_PASSWORD,
+    roles: [{role: "read", db: "mongodb_source"}]
+  });
+}
+source.events.drop();
+source.events.insertMany([
+  {_id: ObjectId("64b000000000000000000001"), device_id: "device-001", site_id: "site-east", ts: ISODate("2026-07-27T10:00:05Z"), measurement: 10.0, source_batch: "batch-001"},
+  {_id: ObjectId("64b000000000000000000002"), device_id: "device-001", site_id: "site-east", ts: ISODate("2026-07-27T10:00:35Z"), measurement: 14.0, source_batch: null},
+  {_id: ObjectId("64b000000000000000000003"), device_id: "device-001", site_id: "site-east", ts: ISODate("2026-07-27T10:02:05Z"), measurement: 20.0},
+  {_id: ObjectId("64b000000000000000000004"), device_id: "device-001", site_id: "site-west", ts: ISODate("2026-07-27T10:00:15Z"), measurement: 30.0, source_batch: "batch-002"},
+  {_id: ObjectId("64b000000000000000000005"), device_id: "device-002", site_id: "site-east", ts: ISODate("2026-07-27T10:01:00Z"), measurement: "malformed"}
+]);
+source.events.createIndex({ts: 1, _id: 1});

go.mod

@@ -97,6 +97,7 @@ require (
 	github.com/unum-cloud/usearch/golang v0.0.0-20260524141737-9fd6b0115dcd
 	github.com/xeipuuv/gojsonschema v1.2.0
 	github.com/yanyiwu/gojieba v1.4.7
+	go.mongodb.org/mongo-driver/v2 v2.8.0
 	go.starlark.net v0.0.0-20250701195324-d457b4515e0e
 	go.uber.org/automaxprocs v1.5.3
 	go.uber.org/ratelimit v0.2.0
@@ -241,9 +242,13 @@ require (
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/valyala/fastrand v1.1.0 // indirect
 	github.com/valyala/histogram v1.2.0 // indirect
+	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
+	github.com/xdg-go/scram v1.2.0 // indirect
+	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
+	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect
 	go.opentelemetry.io/proto/otlp v1.3.1 // indirect
 	go.uber.org/atomic v1.11.0 // indirect

go.sum

@@ -902,6 +902,12 @@ github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tz
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
 github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8=
 github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
+github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.2.0 h1:bYKF2AEwG5rqd1BumT4gAnvwU/M9nBp2pTSxeZw7Wvs=
+github.com/xdg-go/scram v1.2.0/go.mod h1:3dlrS0iBaWKYVt2ZfA4cj48umJZ+cAEbR6/SjLA88I8=
+github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
+github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
 github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
@@ -916,6 +922,8 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:
 github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI=
 github.com/yanyiwu/gojieba v1.4.7 h1:2YkXELcYLTE0SJetq6xv4MjpEikWga6VpFn4jIFFQ/k=
 github.com/yanyiwu/gojieba v1.4.7/go.mod h1:JUq4DddFVGdHXJHxxepxRmhrKlDpaBxR8O28v6fKYLY=
+github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 h1:ilQV1hzziu+LLM3zUTJ0trRztfwgjqKnBWNtSRkbmwM=
+github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfSfmXjznFBSZNN13rSJjlIOI1fUNAtF7rmI=
 github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
 github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
 github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
@@ -926,6 +934,8 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t
 github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
 github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
+go.mongodb.org/mongo-driver/v2 v2.8.0 h1:CxWDGQYY8QQwNjAl/aq2sfWakdnWZynnqJ9F4DhHbP8=
+go.mongodb.org/mongo-driver/v2 v2.8.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0 h1:A3SayB3rNyt+1S6qpI9mHPkeHTZbD7XILEqWnYZb2l0=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.51.0/go.mod h1:27iA5uvhuRNmalO+iEUdVn5ZMj2qy10Mm+XRIpRmyuU=
 go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace v0.45.0 h1:2ea0IkZBsWH+HA2GkD+7+hRw2u97jzdFyRtXuO14a1s=
@@ -1137,6 +1147,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=

optools/iceberg_ci.bash

@@ -365,13 +365,13 @@ preflight() {
   fi
 
   if contains_profile tier-d; then
-    require_env MO_ICEBERG_NESR
-    [[ "$MO_ICEBERG_NESR" == "1" ]] || die "MO_ICEBERG_NESR must be 1 for tier-d profile"
-    require_file MO_ICEBERG_NESR_SCENARIOS
-    require_sql_template MO_ICEBERG_NESR_MO_SQL_CMD
-    require_sql_template MO_ICEBERG_NESR_EXTERNAL_SQL_CMD
-    require_file MO_ICEBERG_NESR_EXPECTED_KPI
-    require_env MO_ICEBERG_NESR_RESIDENCY_ERROR
+    require_env MO_ICEBERG_REFERENCE
+    [[ "$MO_ICEBERG_REFERENCE" == "1" ]] || die "MO_ICEBERG_REFERENCE must be 1 for tier-d profile"
+    require_file MO_ICEBERG_REFERENCE_SCENARIOS
+    require_sql_template MO_ICEBERG_REFERENCE_MO_SQL_CMD
+    require_sql_template MO_ICEBERG_REFERENCE_EXTERNAL_SQL_CMD
+    require_file MO_ICEBERG_REFERENCE_EXPECTED_KPI
+    require_env MO_ICEBERG_REFERENCE_RESIDENCY_ERROR
   fi
 
   log "preflight passed for MO_ICEBERG_CI_PROFILE=${profile}"
@@ -432,9 +432,9 @@ run_remaining_external_if_enabled() {
   fi
 
   if contains_profile tier-d; then
-    run_external_profile tier-d "$MO_ICEBERG_NESR_SCENARIOS"
+    run_external_profile tier-d "$MO_ICEBERG_REFERENCE_SCENARIOS"
   else
-    log "tier-d profile not enabled; skipping NESR scenarios"
+    log "tier-d profile not enabled; skipping reference scenarios"
   fi
 }
 
@@ -444,7 +444,7 @@ validate_external_templates() {
     "${ROOT_DIR}/test/iceberg/credential_vending_scenarios.example.json" \
     "${ROOT_DIR}/test/iceberg/tier_b_public_dataset_scenarios.example.json" \
     "${ROOT_DIR}/test/iceberg/tier_c_sandbox_scenarios.example.json" \
-    "${ROOT_DIR}/test/iceberg/tier_d_nesr_scenarios.example.json"; do
+    "${ROOT_DIR}/test/iceberg/tier_d_reference_scenarios.example.json"; do
     run python3 "${ROOT_DIR}/optools/iceberg_external_runner.py" \
       --profile template \
       --scenario-file "$template" \
@@ -876,14 +876,14 @@ profiles = [
         "name": "tier-d",
         "tests": ["ICE-TEST-135"],
         "required_env": [
-            "MO_ICEBERG_NESR",
-            "MO_ICEBERG_NESR_SCENARIOS",
-            "MO_ICEBERG_NESR_MO_SQL_CMD",
-            "MO_ICEBERG_NESR_EXTERNAL_SQL_CMD",
-            "MO_ICEBERG_NESR_EXPECTED_KPI",
-            "MO_ICEBERG_NESR_RESIDENCY_ERROR",
+            "MO_ICEBERG_REFERENCE",
+            "MO_ICEBERG_REFERENCE_SCENARIOS",
+            "MO_ICEBERG_REFERENCE_MO_SQL_CMD",
+            "MO_ICEBERG_REFERENCE_EXTERNAL_SQL_CMD",
+            "MO_ICEBERG_REFERENCE_EXPECTED_KPI",
+            "MO_ICEBERG_REFERENCE_RESIDENCY_ERROR",
         ],
-        "help": "NESR de-identified demo data and expected KPI file are required.",
+        "help": "Reference-profile de-identified demo data and expected KPI file are required.",
     },
     {
         "name": "golden-real",