K8s replaced legacy systems + CI improvement

.github/workflows/tests.yml

@@ -34,12 +34,50 @@ jobs:
           sudo apt-get update
           sudo apt-get install k6
 
+      - name: Clear previous k6 summaries
+        run: rm -f test/k6/outputs/*_summary.json
+
       - name: Run all direct tests
         shell: pwsh
         working-directory: test
         run: |
           ./run-tests.ps1 -Test all-direct -Workers ${{ env.WORKERS }} -Gateways ${{ env.GATEWAYS }} -EntrypointUrl ${{ env.ENTRYPOINT_URL }}
 
+      - name: Validate k6 thresholds (direct)
+        run: |
+          python - <<'PY'
+          import glob
+          import json
+          import os
+          import sys
+
+          files = sorted(glob.glob("test/k6/outputs/*_summary.json"))
+          if not files:
+              print("No k6 summary files found in test/k6/outputs.")
+              sys.exit(1)
+
+          failures = []
+          checked = 0
+          for path in files:
+              with open(path, "r", encoding="utf-8") as f:
+                  data = json.load(f)
+              metrics = data.get("metrics", {})
+              for metric_name, metric in metrics.items():
+                  thresholds = metric.get("thresholds") or {}
+                  for threshold_name, result in thresholds.items():
+                      checked += 1
+                      if isinstance(result, dict) and result.get("ok") is False:
+                          failures.append((os.path.basename(path), metric_name, threshold_name))
+
+          if failures:
+              print("k6 threshold failures detected:")
+              for file_name, metric_name, threshold_name in failures:
+                  print(f"- {file_name}: {metric_name} -> {threshold_name}")
+              sys.exit(1)
+
+          print(f"All k6 thresholds passed ({checked} checks across {len(files)} summaries).")
+          PY
+
       - name: Upload test results
         uses: actions/upload-artifact@v4
         if: always()
@@ -81,12 +119,65 @@ jobs:
           sudo apt-get update
           sudo apt-get install k6
 
+      - name: Clear previous k6 summaries
+        run: rm -f test/k6/outputs/*_summary.json
+
       - name: Run all orchestrated tests
         shell: pwsh
         working-directory: test
         run: |
           ./run-tests.ps1 -Test all-orchestrated -Workers ${{ env.WORKERS }} -Gateways ${{ env.GATEWAYS }} -EntrypointUrl ${{ env.ENTRYPOINT_URL }}
 
+      - name: Validate k6 thresholds (orchestrated)
+        run: |
+          python - <<'PY'
+          import glob
+          import json
+          import os
+          import sys
+
+          expected_failure_files = {
+              "registry_disruption_summary.json",
+              "split_brain_summary.json",
+          }
+
+          files = sorted(glob.glob("test/k6/outputs/*_summary.json"))
+          if not files:
+              print("No k6 summary files found in test/k6/outputs.")
+              sys.exit(1)
+
+          failures = []
+          ignored_failures = []
+          checked = 0
+          for path in files:
+              file_name = os.path.basename(path)
+              with open(path, "r", encoding="utf-8") as f:
+                  data = json.load(f)
+              metrics = data.get("metrics", {})
+              for metric_name, metric in metrics.items():
+                  thresholds = metric.get("thresholds") or {}
+                  for threshold_name, result in thresholds.items():
+                      checked += 1
+                      if isinstance(result, dict) and result.get("ok") is False:
+                          if file_name in expected_failure_files:
+                              ignored_failures.append((file_name, metric_name, threshold_name))
+                          else:
+                              failures.append((file_name, metric_name, threshold_name))
+
+          if ignored_failures:
+              print("Ignoring expected k6 threshold failures:")
+              for file_name, metric_name, threshold_name in ignored_failures:
+                  print(f"- {file_name}: {metric_name} -> {threshold_name}")
+
+          if failures:
+              print("k6 threshold failures detected:")
+              for file_name, metric_name, threshold_name in failures:
+                  print(f"- {file_name}: {metric_name} -> {threshold_name}")
+              sys.exit(1)
+
+          print(f"All k6 thresholds passed ({checked} checks across {len(files)} summaries).")
+          PY
+
       - name: Upload orchestrated test results
         uses: actions/upload-artifact@v4
         if: always()

docker-compose.yaml

@@ -61,7 +61,10 @@ services:
       - grafana-data:/var/lib/grafana
       - ./grafana/provisioning:/etc/grafana/provisioning
       - ./grafana/provisioning/datasources-dc:/etc/grafana/provisioning/datasources:ro
-      - ./grafana/dashboards:/var/lib/grafana/dashboards
+      - ./grafana/dashboards/gateway-overview.json:/var/lib/grafana/dashboards/gateway-overview.json:ro
+      - ./grafana/dashboards/registry-overview.json:/var/lib/grafana/dashboards/registry-overview.json:ro
+      - ./grafana/dashboards/system-overview.json:/var/lib/grafana/dashboards/system-overview.json:ro
+      - ./grafana/dashboards/worker-overview.json:/var/lib/grafana/dashboards/worker-overview.json:ro
 
   prometheus:
     image: prom/prometheus:v3.9.1

grafana/dashboards/system-overview.cAdvisor.json

@@ -0,0 +1,144 @@
+{
+    "title": "System Overview",
+    "uid": "system-overview",
+    "version": 1,
+    "time": {
+        "from": "now-1h",
+        "to": "now"
+    },
+    "timepicker": {
+        "refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"]
+    },
+    "panels": [
+      {
+        "title": "System CPU Usage",
+        "type": "graph",
+        "gridPos": {"h": 8, "w": 8, "x": 0, "y": 8},
+        "targets": [
+          {
+            "expr": "(1 - avg by(instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100",
+            "legendFormat": "CPU Usage"
+          }
+        ],
+        "lines": true,
+        "linewidth": 2,
+        "fill": 1,
+        "fillGradient": 0,
+        "points": false,
+        "pointradius": 2,
+        "bars": false,
+        "stack": false,
+        "fieldConfig": {
+          "defaults": {
+            "min": 0,
+            "unit": "percent",
+            "decimals": 2
+          }
+        }
+      },
+      {
+        "title": "System Memory Usage",
+        "type": "graph",
+        "gridPos": {"h": 8, "w": 8, "x": 8, "y": 8},
+        "targets": [
+          {
+            "expr": "(node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100",
+            "legendFormat": "Memory Usage"
+          }
+        ],
+        "lines": true,
+        "linewidth": 2,
+        "fill": 1,
+        "fillGradient": 0,
+        "points": false,
+        "pointradius": 2,
+        "bars": false,
+        "stack": false,
+        "fieldConfig": {
+          "defaults": {
+            "min": 0,
+            "unit": "percent",
+            "decimals": 2
+          }
+        }
+      },
+      {
+        "title": "System Disk Usage",
+        "type": "graph",
+        "gridPos": {"h": 8, "w": 8, "x": 16, "y": 8},
+        "targets": [
+          {
+            "expr": "topk(1, (node_filesystem_size_bytes{fstype=~\"ext4|xfs|btrfs\",mountpoint!=\"/boot\"} - node_filesystem_free_bytes{fstype=~\"ext4|xfs|btrfs\",mountpoint!=\"/boot\"}) / node_filesystem_size_bytes{fstype=~\"ext4|xfs|btrfs\",mountpoint!=\"/boot\"} * 100)",
+            "legendFormat": "Disk Usage"
+          }
+        ],
+        "lines": true,
+        "linewidth": 2,
+        "fill": 1,
+        "fillGradient": 0,
+        "points": false,
+        "pointradius": 2,
+        "bars": false,
+        "stack": false,
+        "fieldConfig": {
+          "defaults": {
+            "min": 0,
+            "unit": "percent",
+            "decimals": 2
+          }
+        }
+      },
+      {
+        "title": "Container CPU Usage",
+        "type": "graph",
+        "gridPos": {"h": 8, "w": 12, "x": 0, "y": 16},
+        "targets": [
+          {
+            "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"geostreamdb\",pod!=\"\"}[1m])) * 100",
+            "legendFormat": "{{pod}}"
+          }
+        ],
+        "lines": true,
+        "linewidth": 2,
+        "fill": 1,
+        "fillGradient": 0,
+        "points": false,
+        "pointradius": 2,
+        "bars": false,
+        "stack": false,
+        "fieldConfig": {
+          "defaults": {
+            "min": 0,
+            "unit": "percent",
+            "decimals": 2
+          }
+        }
+      },
+      {
+        "title": "Container Memory Usage",
+        "type": "graph",
+        "gridPos": {"h": 8, "w": 12, "x": 12, "y": 16},
+        "targets": [
+          {
+            "expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"geostreamdb\",pod!=\"\"})",
+            "legendFormat": "{{pod}}"
+          }
+        ],
+        "lines": true,
+        "linewidth": 2,
+        "fill": 1,
+        "fillGradient": 0,
+        "points": false,
+        "pointradius": 2,
+        "bars": false,
+        "stack": false,
+        "fieldConfig": {
+          "defaults": {
+            "min": 0,
+            "unit": "bytes",
+            "decimals": 0
+          }
+        }
+      }
+    ]
+  }

k8s/deployment-node-exporter.yaml → k8s/daemonset-node-exporter.yaml

@@ -1,12 +1,11 @@
 apiVersion: apps/v1
-kind: Deployment
+kind: DaemonSet
 metadata:
   name: node-exporter
   namespace: geostreamdb
   labels:
     app: node-exporter
 spec:
-  replicas: 1
   selector:
     matchLabels:
       app: node-exporter

k8s/ingress-gateway.yaml

@@ -0,0 +1,17 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: geostreamdb-ingress
+  namespace: geostreamdb
+spec:
+  ingressClassName: nginx
+  rules:
+    - http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: gateway-service
+                port:
+                  number: 8080