From 354320bcf129721e0162d80b4e890893963e768f Mon Sep 17 00:00:00 2001 From: MrPoll0 Date: Tue, 17 Feb 2026 22:21:12 +0100 Subject: [PATCH 1/4] feat: replaced docker-exporter for kubelet cAdvisor in k8s --- docker-compose.yaml | 5 +- .../dashboards/system-overview.cAdvisor.json | 144 +++++ k8s/deployment-docker-exporter.yaml | 34 -- k8s/kustomization.yaml | 2 +- k8s/service-docker-exporter.yaml | 12 - overlays/minikube/kustomization.yaml | 14 +- overlays/prod/kustomization.yaml | 9 + prometheus/prometheus.minikube.yml | 507 ------------------ prometheus/prometheus.yml | 18 + 9 files changed, 183 insertions(+), 562 deletions(-) create mode 100644 grafana/dashboards/system-overview.cAdvisor.json delete mode 100644 k8s/deployment-docker-exporter.yaml delete mode 100644 k8s/service-docker-exporter.yaml delete mode 100644 prometheus/prometheus.minikube.yml diff --git a/docker-compose.yaml b/docker-compose.yaml index cd3a44e..a42e0d0 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -61,7 +61,10 @@ services: - grafana-data:/var/lib/grafana - ./grafana/provisioning:/etc/grafana/provisioning - ./grafana/provisioning/datasources-dc:/etc/grafana/provisioning/datasources:ro - - ./grafana/dashboards:/var/lib/grafana/dashboards + - ./grafana/dashboards/gateway-overview.json:/var/lib/grafana/dashboards/gateway-overview.json:ro + - ./grafana/dashboards/registry-overview.json:/var/lib/grafana/dashboards/registry-overview.json:ro + - ./grafana/dashboards/system-overview.json:/var/lib/grafana/dashboards/system-overview.json:ro + - ./grafana/dashboards/worker-overview.json:/var/lib/grafana/dashboards/worker-overview.json:ro prometheus: image: prom/prometheus:v3.9.1 diff --git a/grafana/dashboards/system-overview.cAdvisor.json b/grafana/dashboards/system-overview.cAdvisor.json new file mode 100644 index 0000000..ef1d0d3 --- /dev/null +++ b/grafana/dashboards/system-overview.cAdvisor.json @@ -0,0 +1,144 @@ +{ + "title": "System Overview", + "uid": "system-overview", + "version": 1, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"] + }, + "panels": [ + { + "title": "System CPU Usage", + "type": "graph", + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 8}, + "targets": [ + { + "expr": "(1 - avg by(instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100", + "legendFormat": "CPU Usage" + } + ], + "lines": true, + "linewidth": 2, + "fill": 1, + "fillGradient": 0, + "points": false, + "pointradius": 2, + "bars": false, + "stack": false, + "fieldConfig": { + "defaults": { + "min": 0, + "unit": "percent", + "decimals": 2 + } + } + }, + { + "title": "System Memory Usage", + "type": "graph", + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 8}, + "targets": [ + { + "expr": "(node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100", + "legendFormat": "Memory Usage" + } + ], + "lines": true, + "linewidth": 2, + "fill": 1, + "fillGradient": 0, + "points": false, + "pointradius": 2, + "bars": false, + "stack": false, + "fieldConfig": { + "defaults": { + "min": 0, + "unit": "percent", + "decimals": 2 + } + } + }, + { + "title": "System Disk Usage", + "type": "graph", + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 8}, + "targets": [ + { + "expr": "topk(1, (node_filesystem_size_bytes{fstype=~\"ext4|xfs|btrfs\",mountpoint!=\"/boot\"} - node_filesystem_free_bytes{fstype=~\"ext4|xfs|btrfs\",mountpoint!=\"/boot\"}) / node_filesystem_size_bytes{fstype=~\"ext4|xfs|btrfs\",mountpoint!=\"/boot\"} * 100)", + "legendFormat": "Disk Usage" + } + ], + "lines": true, + "linewidth": 2, + "fill": 1, + "fillGradient": 0, + "points": false, + "pointradius": 2, + "bars": false, + "stack": false, + "fieldConfig": { + "defaults": { + "min": 0, + "unit": "percent", + "decimals": 2 + } + } + }, + { + "title": "Container CPU Usage", + "type": "graph", + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 16}, + "targets": [ + { + "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{namespace=\"geostreamdb\",pod!=\"\"}[1m])) * 100", + "legendFormat": "{{pod}}" + } + ], + "lines": true, + "linewidth": 2, + "fill": 1, + "fillGradient": 0, + "points": false, + "pointradius": 2, + "bars": false, + "stack": false, + "fieldConfig": { + "defaults": { + "min": 0, + "unit": "percent", + "decimals": 2 + } + } + }, + { + "title": "Container Memory Usage", + "type": "graph", + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 16}, + "targets": [ + { + "expr": "sum by (pod) (container_memory_working_set_bytes{namespace=\"geostreamdb\",pod!=\"\"})", + "legendFormat": "{{pod}}" + } + ], + "lines": true, + "linewidth": 2, + "fill": 1, + "fillGradient": 0, + "points": false, + "pointradius": 2, + "bars": false, + "stack": false, + "fieldConfig": { + "defaults": { + "min": 0, + "unit": "bytes", + "decimals": 0 + } + } + } + ] + } \ No newline at end of file diff --git a/k8s/deployment-docker-exporter.yaml b/k8s/deployment-docker-exporter.yaml deleted file mode 100644 index def20f2..0000000 --- a/k8s/deployment-docker-exporter.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: docker-exporter - namespace: geostreamdb - labels: - app: docker-exporter -spec: - replicas: 1 - selector: - matchLabels: - app: docker-exporter - template: - metadata: - labels: - app: docker-exporter - spec: - containers: - - name: docker-exporter - image: ghcr.io/davidborzek/docker-exporter:v0.3.0 - ports: - - containerPort: 8080 - name: http - securityContext: - runAsUser: 0 - volumeMounts: - - name: docker-sock - mountPath: /var/run/docker.sock - readOnly: true - volumes: - - name: docker-sock - hostPath: - path: /var/run/docker.sock - type: Socket diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml index 3acbca9..8a1435b 100644 --- a/k8s/kustomization.yaml +++ b/k8s/kustomization.yaml @@ -39,5 +39,5 @@ configMapGenerator: files: - gateway-overview.json=../grafana/dashboards/gateway-overview.json - registry-overview.json=../grafana/dashboards/registry-overview.json - - system-overview.json=../grafana/dashboards/system-overview.json + - system-overview.json=../grafana/dashboards/system-overview.cAdvisor.json - worker-overview.json=../grafana/dashboards/worker-overview.json \ No newline at end of file diff --git a/k8s/service-docker-exporter.yaml b/k8s/service-docker-exporter.yaml deleted file mode 100644 index ab6d773..0000000 --- a/k8s/service-docker-exporter.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: docker-exporter-service - namespace: geostreamdb -spec: - selector: - app: docker-exporter - ports: - - name: http - port: 8080 - targetPort: 8080 diff --git a/overlays/minikube/kustomization.yaml b/overlays/minikube/kustomization.yaml index 8a17db9..6c9e979 100644 --- a/overlays/minikube/kustomization.yaml +++ b/overlays/minikube/kustomization.yaml @@ -5,12 +5,12 @@ namespace: geostreamdb resources: - ../../k8s - - ../../k8s/deployment-docker-exporter.yaml - - ../../k8s/service-docker-exporter.yaml + #- ../../k8s/deployment-docker-exporter.yaml + #- ../../k8s/service-docker-exporter.yaml # replace prometheus-config with version that includes docker-exporter scrape job (minikube/Docker runtime only) -configMapGenerator: - - name: prometheus-config - behavior: replace - files: - - prometheus.yml=../../prometheus/prometheus.minikube.yml \ No newline at end of file +#configMapGenerator: +# - name: prometheus-config +# behavior: replace +# files: +# - prometheus.yml=../../prometheus/prometheus.minikube.yml \ No newline at end of file diff --git a/overlays/prod/kustomization.yaml b/overlays/prod/kustomization.yaml index 4058b65..193c96c 100644 --- a/overlays/prod/kustomization.yaml +++ b/overlays/prod/kustomization.yaml @@ -5,3 +5,12 @@ namespace: geostreamdb resources: - ../../k8s + +#configMapGenerator: +# - name: grafana-dashboards +# behavior: replace +# files: +# - gateway-overview.json=../../grafana/dashboards/gateway-overview.json +# - registry-overview.json=../../grafana/dashboards/registry-overview.json +# - system-overview.json=../../grafana/dashboards/system-overview.cAdvisor.json +# - worker-overview.json=../../grafana/dashboards/worker-overview.json diff --git a/prometheus/prometheus.minikube.yml b/prometheus/prometheus.minikube.yml deleted file mode 100644 index 1f62007..0000000 --- a/prometheus/prometheus.minikube.yml +++ /dev/null @@ -1,507 +0,0 @@ -# global config -global: - scrape_interval: 15s - #evaluation_interval: 30s - #body_size_limit: 15MB - #sample_limit: 1500 - #target_limit: 30 - #label_limit: 30 - #label_name_length_limit: 200 - #label_value_length_limit: 200 - #query_log_file: query.log - #scrape_failure_log_file: fail.log - # scrape_timeout is set to the global default (10s). - - #external_labels: - # monitor: codelab - # foo: bar - -#runtime: -# gogc: 42 - -#rule_files: -# - "first.rules" -# - "my/*.rules" - -#remote_write: -# - url: http://remote1/push -# name: drop_expensive -# write_relabel_configs: -# - source_labels: [__name__] -# regex: expensive.* -# action: drop -# oauth2: -# client_id: "123" -# client_secret: "456" -# token_url: "http://remote1/auth" -# tls_config: -# cert_file: valid_cert_file -# key_file: valid_key_file - -# - url: http://remote2/push -# protobuf_message: io.prometheus.write.v2.Request -# name: rw_tls -# tls_config: -# cert_file: valid_cert_file -# key_file: valid_key_file -# headers: -# name: value - -#otlp: -# promote_resource_attributes: ["k8s.cluster.name", "k8s.job.name", "k8s.namespace.name"] - -#remote_read: - #- url: http://remote1/read - # read_recent: true - # name: default - # enable_http2: false - #- url: http://remote3/read - # read_recent: false - # name: read_special - # required_matchers: - # job: special - # tls_config: - # cert_file: valid_cert_file - # key_file: valid_key_file - -scrape_configs: - - job_name: prometheus - scrape_interval: 30s - static_configs: - - targets: - - prometheus-service:9090 - - - job_name: node-exporter - scrape_interval: 10s - static_configs: - - targets: - - node-exporter-service:9100 - - - job_name: docker-exporter - scrape_interval: 10s - static_configs: - - targets: - - docker-exporter-service:8080 - - - job_name: gateway - scrape_interval: 10s - kubernetes_sd_configs: - - role: pod - namespaces: - names: - - geostreamdb - relabel_configs: - - source_labels: [__meta_kubernetes_pod_label_app] - regex: gateway - action: keep - - source_labels: [__meta_kubernetes_pod_ip] - target_label: __address__ - replacement: $1:8080 - - - job_name: worker-node - scrape_interval: 10s - kubernetes_sd_configs: - - role: pod - namespaces: - names: - - geostreamdb - relabel_configs: - - source_labels: [__meta_kubernetes_pod_label_app] - regex: worker-node - action: keep - - source_labels: [__meta_kubernetes_pod_ip] - target_label: __address__ - replacement: $1:2112 - - - job_name: registry - scrape_interval: 10s - static_configs: - - targets: - - registry-service:2112 - - #- job_name: prometheus - - # honor_labels: true - # scrape_interval is defined by the configured global (15s). - # scrape_timeout is defined by the global default (10s). - - # metrics_path defaults to '/metrics' - # scheme defaults to 'http'. - - # fallback_scrape_protocol: PrometheusText0.0.4 - - # scrape_failure_log_file: fail_prom.log - # file_sd_configs: - # - files: - # - foo/*.slow.json - # - foo/*.slow.yml - # - single/file.yml - # refresh_interval: 10m - # - files: - # - bar/*.yaml - - # static_configs: - # - targets: ["localhost:9090", "localhost:9191"] - # labels: - # my: label - # your: label - - # http_headers: - # foo: - # values: ["foobar"] - # secrets: ["bar", "foo"] - # files: ["valid_password_file"] - - # relabel_configs: - # - source_labels: [job, __meta_dns_name] - # regex: (.*)some-[regex] - # target_label: job - # replacement: foo-${1} - # action defaults to 'replace' - # - source_labels: [abc] - # target_label: cde - # - replacement: static - # target_label: abc - # - regex: - # replacement: static - # target_label: abc - # - source_labels: [foo] - # target_label: abc - # action: keepequal - # - source_labels: [foo] - # target_label: abc - # action: dropequal - - # authorization: - # credentials_file: valid_token_file - - # tls_config: - # min_version: TLS10 - - #- job_name: service-x - - # basic_auth: - # username: admin_name - # password: "multiline\nmysecret\ntest" - - # scrape_interval: 50s - # scrape_timeout: 5s - # scrape_protocols: ["PrometheusText0.0.4"] - - # body_size_limit: 10MB - # sample_limit: 1000 - # target_limit: 35 - # label_limit: 35 - # label_name_length_limit: 210 - # label_value_length_limit: 210 - - # dns_sd_configs: - # - refresh_interval: 15s - # names: - # - first.dns.address.domain.com - # - second.dns.address.domain.com - # - names: - # - first.dns.address.domain.com - - # relabel_configs: - # - source_labels: [job] - # regex: (.*)some-[regex] - # action: drop - # - source_labels: [__address__] - # modulus: 8 - # target_label: __tmp_hash - # action: hashmod - # - source_labels: [__tmp_hash] - # regex: 1 - # action: keep - # - action: labelmap - # regex: 1 - # - action: labeldrop - # regex: d - # - action: labelkeep - # regex: k - - # metric_relabel_configs: - # - source_labels: [__name__] - # regex: expensive_metric.* - # action: drop - - #- job_name: service-y - - # consul_sd_configs: - # - server: "localhost:1234" - # token: mysecret - # path_prefix: /consul - # services: ["nginx", "cache", "mysql"] - # tags: ["canary", "v1"] - # node_meta: - # rack: "123" - # allow_stale: true - # scheme: https - # tls_config: - # ca_file: valid_ca_file - # cert_file: valid_cert_file - # key_file: valid_key_file - # insecure_skip_verify: false - - # relabel_configs: - # - source_labels: [__meta_sd_consul_tags] - # separator: "," - # regex: label:([^=]+)=([^,]+) - # target_label: ${1} - # replacement: ${2} - - #- job_name: service-z - - # tls_config: - # cert_file: valid_cert_file - # key_file: valid_key_file - - # authorization: - # credentials: mysecret - - #- job_name: service-kubernetes - - # kubernetes_sd_configs: - # - role: endpoints - # api_server: "https://localhost:1234" - # tls_config: - # cert_file: valid_cert_file - # key_file: valid_key_file - - # basic_auth: - # username: "myusername" - # password: "mysecret" - - #- job_name: service-kubernetes-namespaces - - # kubernetes_sd_configs: - # - role: endpoints - # api_server: "https://localhost:1234" - # namespaces: - # names: - # - default - - # basic_auth: - # username: "myusername" - # password_file: valid_password_file - - #- job_name: service-kuma - - # kuma_sd_configs: - # - server: http://kuma-control-plane.kuma-system.svc:5676 - # client_id: main-prometheus - - #- job_name: service-marathon - # marathon_sd_configs: - # - servers: - # - "https://marathon.example.com:443" - - # auth_token: "mysecret" - # tls_config: - # cert_file: valid_cert_file - # key_file: valid_key_file - - #- job_name: service-nomad - # nomad_sd_configs: - # - server: 'http://localhost:4646' - - #- job_name: service-ec2 - # ec2_sd_configs: - # - region: us-east-1 - # access_key: access - # secret_key: mysecret - # profile: profile - # filters: - # - name: tag:environment - # values: - # - prod - - #- job_name: service-lightsail - # lightsail_sd_configs: - # - region: us-east-1 - # access_key: access - # secret_key: mysecret - # profile: profile - - #- job_name: service-azure - # azure_sd_configs: - # - environment: AzurePublicCloud - # authentication_method: OAuth - # subscription_id: 11AAAA11-A11A-111A-A111-1111A1111A11 - # resource_group: my-resource-group - # tenant_id: BBBB222B-B2B2-2B22-B222-2BB2222BB2B2 - # client_id: 333333CC-3C33-3333-CCC3-33C3CCCCC33C - # client_secret: mysecret - # port: 9100 - - #- job_name: service-nerve - # nerve_sd_configs: - # - servers: - # - localhost - # paths: - # - /monitoring - - #- job_name: 0123service-xxx - # metrics_path: /metrics - # static_configs: - # - targets: - # - localhost:9090 - - #- job_name: badfederation - # honor_timestamps: false - # metrics_path: /federate - # static_configs: - # - targets: - # - localhost:9090 - - #- job_name: 測試 - # metrics_path: /metrics - # static_configs: - # - targets: - # - localhost:9090 - - #- job_name: httpsd - # http_sd_configs: - # - url: "http://example.com/prometheus" - - #- job_name: service-triton - # triton_sd_configs: - # - account: "testAccount" - # dns_suffix: "triton.example.com" - # endpoint: "triton.example.com" - # port: 9163 - # refresh_interval: 1m - # version: 1 - # tls_config: - # cert_file: valid_cert_file - # key_file: valid_key_file - - #- job_name: digitalocean-droplets - # digitalocean_sd_configs: - # - authorization: - # credentials: abcdef - - #- job_name: service-openstack - # openstack_sd_configs: - # - role: instance - # region: RegionOne - # port: 80 - # refresh_interval: 1m - # tls_config: - # ca_file: valid_ca_file - # cert_file: valid_cert_file - # key_file: valid_key_file - - #- job_name: service-puppetdb - # puppetdb_sd_configs: - # - url: https://puppetserver/ - # query: 'resources { type = "Package" and title = "httpd" }' - # include_parameters: true - # port: 80 - # refresh_interval: 1m - # tls_config: - # ca_file: valid_ca_file - # cert_file: valid_cert_file - # key_file: valid_key_file - - #- job_name: hetzner - # scrape_native_histograms: true - # relabel_configs: - # - action: uppercase - # source_labels: [instance] - # target_label: instance - # hetzner_sd_configs: - # - role: hcloud - # authorization: - # credentials: abcdef - # - role: robot - # basic_auth: - # username: abcdef - # password: abcdef - - #- job_name: service-eureka - # eureka_sd_configs: - # - server: "http://eureka.example.com:8761/eureka" - - #- job_name: ovhcloud - # ovhcloud_sd_configs: - # - service: vps - # endpoint: ovh-eu - # application_key: testAppKey - # application_secret: testAppSecret - # consumer_key: testConsumerKey - # refresh_interval: 1m - # - service: dedicated_server - # endpoint: ovh-eu - # application_key: testAppKey - # application_secret: testAppSecret - # consumer_key: testConsumerKey - # refresh_interval: 1m - - #- job_name: scaleway - # scaleway_sd_configs: - # - role: instance - # project_id: 11111111-1111-1111-1111-111111111112 - # access_key: SCWXXXXXXXXXXXXXXXXX - # secret_key: 11111111-1111-1111-1111-111111111111 - # - role: baremetal - # project_id: 11111111-1111-1111-1111-111111111112 - # access_key: SCWXXXXXXXXXXXXXXXXX - # secret_key: 11111111-1111-1111-1111-111111111111 - - #- job_name: linode-instances - # linode_sd_configs: - # - authorization: - # credentials: abcdef - - #- job_name: stackit-servers - # stackit_sd_configs: - # - project: 11111111-1111-1111-1111-111111111111 - # authorization: - # credentials: abcdef - - #- job_name: uyuni - # uyuni_sd_configs: - # - server: https://localhost:1234 - # username: gopher - # password: hole - - #- job_name: ionos - # ionos_sd_configs: - # - datacenter_id: 8feda53f-15f0-447f-badf-ebe32dad2fc0 - # authorization: - # credentials: abcdef - - #- job_name: vultr - # vultr_sd_configs: - # - authorization: - # credentials: abcdef - -#alerting: -# alertmanagers: -# - scheme: https -# static_configs: -# - targets: -# - "1.2.3.4:9093" -# - "1.2.3.5:9093" -# - "1.2.3.6:9093" - -#storage: -# tsdb: -# out_of_order_time_window: 30m -# retention: -# time: 1d -# size: 1GB - -#tracing: -# endpoint: "localhost:4317" -# client_type: "grpc" -# headers: -# foo: "bar" -# timeout: 5s -# compression: "gzip" -# tls_config: -# cert_file: valid_cert_file -# key_file: valid_key_file -# insecure_skip_verify: true \ No newline at end of file diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index de50004..6ccccb5 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -76,6 +76,24 @@ scrape_configs: static_configs: - targets: - node-exporter-service:9100 + + # scrape kubelet-embedded cAdvisor via API server proxy for containerd clusters + - job_name: kubelet-cadvisor + scrape_interval: 15s + scheme: https + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: [__meta_kubernetes_node_name] + target_label: __metrics_path__ + replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor - job_name: gateway scrape_interval: 10s From aacbcfa89ad8f6fce114602e24fae0c9802ae187 Mon Sep 17 00:00:00 2001 From: MrPoll0 Date: Wed, 18 Feb 2026 02:39:20 +0100 Subject: [PATCH 2/4] feat: node-exporter as DaemonSet --- ...exporter.yaml => daemonset-node-exporter.yaml} | 3 +-- prometheus/prometheus.yml | 15 ++++++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) rename k8s/{deployment-node-exporter.yaml => daemonset-node-exporter.yaml} (97%) diff --git a/k8s/deployment-node-exporter.yaml b/k8s/daemonset-node-exporter.yaml similarity index 97% rename from k8s/deployment-node-exporter.yaml rename to k8s/daemonset-node-exporter.yaml index 80cf255..b7f1220 100644 --- a/k8s/deployment-node-exporter.yaml +++ b/k8s/daemonset-node-exporter.yaml @@ -1,12 +1,11 @@ apiVersion: apps/v1 -kind: Deployment +kind: DaemonSet metadata: name: node-exporter namespace: geostreamdb labels: app: node-exporter spec: - replicas: 1 selector: matchLabels: app: node-exporter diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index 6ccccb5..b27ec33 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -73,9 +73,18 @@ scrape_configs: - job_name: node-exporter scrape_interval: 10s - static_configs: - - targets: - - node-exporter-service:9100 + kubernetes_sd_configs: + - role: pod + namespaces: + names: + - geostreamdb + relabel_configs: + - source_labels: [__meta_kubernetes_pod_label_app] + regex: node-exporter + action: keep + - source_labels: [__meta_kubernetes_pod_ip] + target_label: __address__ + replacement: $1:9100 # scrape kubelet-embedded cAdvisor via API server proxy for containerd clusters - job_name: kubelet-cadvisor From 2b34cd53f6f7aad154bac02e1aad917215bc8c06 Mon Sep 17 00:00:00 2001 From: MrPoll0 Date: Wed, 18 Feb 2026 04:14:00 +0100 Subject: [PATCH 3/4] feat: switch nginx LB for Ingress pointing to gateway service with nginx ingress controller in k8s --- k8s/deployment-loadbalancer.yaml | 32 -------------------------------- k8s/ingress-gateway.yaml | 17 +++++++++++++++++ k8s/kustomization.yaml | 8 ++------ k8s/service-loadbalancer.yaml | 14 -------------- loadbalancer/nginx.conf | 20 -------------------- test/run-tests.ps1 | 16 ++++++++++------ 6 files changed, 29 insertions(+), 78 deletions(-) delete mode 100644 k8s/deployment-loadbalancer.yaml create mode 100644 k8s/ingress-gateway.yaml delete mode 100644 k8s/service-loadbalancer.yaml delete mode 100644 loadbalancer/nginx.conf diff --git a/k8s/deployment-loadbalancer.yaml b/k8s/deployment-loadbalancer.yaml deleted file mode 100644 index f9581f6..0000000 --- a/k8s/deployment-loadbalancer.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: loadbalancer - namespace: geostreamdb - labels: - app: loadbalancer -spec: - replicas: 1 - selector: - matchLabels: - app: loadbalancer - template: - metadata: - labels: - app: loadbalancer - spec: - containers: - - name: nginx - image: nginx:alpine - ports: - - containerPort: 80 - name: http - volumeMounts: - - name: nginx-config - mountPath: /etc/nginx/nginx.conf - subPath: nginx.conf - readOnly: true - volumes: - - name: nginx-config - configMap: - name: nginx-config diff --git a/k8s/ingress-gateway.yaml b/k8s/ingress-gateway.yaml new file mode 100644 index 0000000..e02efcc --- /dev/null +++ b/k8s/ingress-gateway.yaml @@ -0,0 +1,17 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: geostreamdb-ingress + namespace: geostreamdb +spec: + ingressClassName: nginx + rules: + - http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: gateway-service + port: + number: 8080 \ No newline at end of file diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml index 8a1435b..8df4675 100644 --- a/k8s/kustomization.yaml +++ b/k8s/kustomization.yaml @@ -10,24 +10,20 @@ resources: - deployment-gateway.yaml - pdb-gateway.yaml - deployment-worker.yaml - - deployment-loadbalancer.yaml - deployment-prometheus.yaml - deployment-grafana.yaml - - deployment-node-exporter.yaml + - daemonset-node-exporter.yaml - service-registry.yaml - service-gateway.yaml - service-worker.yaml - - service-loadbalancer.yaml - service-prometheus.yaml - service-grafana.yaml - service-node-exporter.yaml + - ingress-gateway.yaml - pvc-grafana.yaml - pvc-prometheus.yaml configMapGenerator: - - name: nginx-config - files: - - nginx.conf=../loadbalancer/nginx.conf - name: prometheus-config files: - prometheus.yml=../prometheus/prometheus.yml diff --git a/k8s/service-loadbalancer.yaml b/k8s/service-loadbalancer.yaml deleted file mode 100644 index 373f179..0000000 --- a/k8s/service-loadbalancer.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: loadbalancer-service - namespace: geostreamdb -spec: - type: NodePort # TODO: NodePort for local development, LoadBalancer for cloud providers - selector: - app: loadbalancer - ports: - - name: http - port: 8080 - targetPort: 80 - nodePort: 30080 # only used if type is NodePort diff --git a/loadbalancer/nginx.conf b/loadbalancer/nginx.conf deleted file mode 100644 index 62ad3de..0000000 --- a/loadbalancer/nginx.conf +++ /dev/null @@ -1,20 +0,0 @@ -events {} - -http { - # kubernetes DNS resolver (CoreDNS) - resolver kube-dns.kube-system.svc.cluster.local valid=10s ipv6=off; - - server { - listen 80; - - location / { - # use variable to force DNS re-resolution on each request - # k8s Service will return multiple IPs for gateway, nginx will round-robin - # use FQDN for proper DNS resolution - set $backend "http://gateway-service.geostreamdb.svc.cluster.local:8080"; - proxy_pass $backend; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - } - } -} \ No newline at end of file diff --git a/test/run-tests.ps1 b/test/run-tests.ps1 index 112392d..334979d 100644 --- a/test/run-tests.ps1 +++ b/test/run-tests.ps1 @@ -85,6 +85,12 @@ if (-not $SkipInfra) { Write-Host "minikube start failed. If using Hyper-V, run it with administrator privileges." -ForegroundColor Red exit 1 } + Write-Host "Ensuring ingress addon is enabled..." -ForegroundColor Yellow + minikube addons enable ingress | Out-Null + if ($LASTEXITCODE -ne 0) { + Write-Host "Failed to enable minikube ingress addon." -ForegroundColor Red + exit 1 + } # install Chaos Mesh only when running orchestrated tests if ($NeedsChaosMesh) { @@ -128,7 +134,6 @@ if (-not $SkipInfra) { # load public images minikube image load prom/prometheus:v3.9.1 2>&1 | Out-Null minikube image load grafana/grafana:12.3.3 2>&1 | Out-Null - minikube image load nginx:alpine 2>&1 | Out-Null minikube image load prom/node-exporter:v1.10.2 2>&1 | Out-Null minikube image load ghcr.io/davidborzek/docker-exporter:v0.3.0 2>&1 | Out-Null } catch { @@ -146,9 +151,9 @@ if (-not $SkipInfra) { kubectl wait --for=condition=ready pod -l app=gateway -n $Namespace --timeout=120s kubectl wait --for=condition=ready pod -l app=worker-node -n $Namespace --timeout=120s kubectl wait --for=condition=ready pod -l app=registry -n $Namespace --timeout=120s - kubectl wait --for=condition=ready pod -l app=loadbalancer -n $Namespace --timeout=120s kubectl wait --for=condition=ready pod -l app=prometheus -n $Namespace --timeout=120s kubectl wait --for=condition=ready pod -l app=grafana -n $Namespace --timeout=120s + kubectl wait --for=condition=available deployment/ingress-nginx-controller -n ingress-nginx --timeout=180s # port-forward services for local access during tests (localhost:8080/9090/3000) Write-Host "Starting Kubernetes port-forwards..." -ForegroundColor Yellow @@ -166,10 +171,9 @@ if (-not $SkipInfra) { exit 1 } - $PortForwardJobs += Start-Job -Name "pf-loadbalancer-8080" -ScriptBlock { - param($ns) - kubectl port-forward -n $ns svc/loadbalancer-service 8080:8080 2>&1 - } -ArgumentList $Namespace + $PortForwardJobs += Start-Job -Name "pf-ingress-controller-8080" -ScriptBlock { + kubectl port-forward -n ingress-nginx svc/ingress-nginx-controller 8080:80 2>&1 + } $PortForwardJobs += Start-Job -Name "pf-prometheus-9090" -ScriptBlock { param($ns) kubectl port-forward -n $ns svc/prometheus-service 9090:9090 2>&1 From c5666519c82e20b9de6bb1904bf9107e3ba49a77 Mon Sep 17 00:00:00 2001 From: MrPoll0 Date: Wed, 18 Feb 2026 21:58:52 +0100 Subject: [PATCH 4/4] feat[CI]: validate k6 tests thresholds --- .github/workflows/tests.yml | 91 +++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index fd67896..520e59d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -34,12 +34,50 @@ jobs: sudo apt-get update sudo apt-get install k6 + - name: Clear previous k6 summaries + run: rm -f test/k6/outputs/*_summary.json + - name: Run all direct tests shell: pwsh working-directory: test run: | ./run-tests.ps1 -Test all-direct -Workers ${{ env.WORKERS }} -Gateways ${{ env.GATEWAYS }} -EntrypointUrl ${{ env.ENTRYPOINT_URL }} + - name: Validate k6 thresholds (direct) + run: | + python - <<'PY' + import glob + import json + import os + import sys + + files = sorted(glob.glob("test/k6/outputs/*_summary.json")) + if not files: + print("No k6 summary files found in test/k6/outputs.") + sys.exit(1) + + failures = [] + checked = 0 + for path in files: + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + metrics = data.get("metrics", {}) + for metric_name, metric in metrics.items(): + thresholds = metric.get("thresholds") or {} + for threshold_name, result in thresholds.items(): + checked += 1 + if isinstance(result, dict) and result.get("ok") is False: + failures.append((os.path.basename(path), metric_name, threshold_name)) + + if failures: + print("k6 threshold failures detected:") + for file_name, metric_name, threshold_name in failures: + print(f"- {file_name}: {metric_name} -> {threshold_name}") + sys.exit(1) + + print(f"All k6 thresholds passed ({checked} checks across {len(files)} summaries).") + PY + - name: Upload test results uses: actions/upload-artifact@v4 if: always() @@ -81,12 +119,65 @@ jobs: sudo apt-get update sudo apt-get install k6 + - name: Clear previous k6 summaries + run: rm -f test/k6/outputs/*_summary.json + - name: Run all orchestrated tests shell: pwsh working-directory: test run: | ./run-tests.ps1 -Test all-orchestrated -Workers ${{ env.WORKERS }} -Gateways ${{ env.GATEWAYS }} -EntrypointUrl ${{ env.ENTRYPOINT_URL }} + - name: Validate k6 thresholds (orchestrated) + run: | + python - <<'PY' + import glob + import json + import os + import sys + + expected_failure_files = { + "registry_disruption_summary.json", + "split_brain_summary.json", + } + + files = sorted(glob.glob("test/k6/outputs/*_summary.json")) + if not files: + print("No k6 summary files found in test/k6/outputs.") + sys.exit(1) + + failures = [] + ignored_failures = [] + checked = 0 + for path in files: + file_name = os.path.basename(path) + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + metrics = data.get("metrics", {}) + for metric_name, metric in metrics.items(): + thresholds = metric.get("thresholds") or {} + for threshold_name, result in thresholds.items(): + checked += 1 + if isinstance(result, dict) and result.get("ok") is False: + if file_name in expected_failure_files: + ignored_failures.append((file_name, metric_name, threshold_name)) + else: + failures.append((file_name, metric_name, threshold_name)) + + if ignored_failures: + print("Ignoring expected k6 threshold failures:") + for file_name, metric_name, threshold_name in ignored_failures: + print(f"- {file_name}: {metric_name} -> {threshold_name}") + + if failures: + print("k6 threshold failures detected:") + for file_name, metric_name, threshold_name in failures: + print(f"- {file_name}: {metric_name} -> {threshold_name}") + sys.exit(1) + + print(f"All k6 thresholds passed ({checked} checks across {len(files)} summaries).") + PY + - name: Upload orchestrated test results uses: actions/upload-artifact@v4 if: always()