From dcd8a65353b29b54776998312cb5dacae42e6b7c Mon Sep 17 00:00:00 2001 From: Andrzej Stencel Date: Wed, 22 Jul 2026 15:36:41 +0200 Subject: [PATCH 1/4] chore: import autodiscover repository (#51804) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [elastic-agent-autodiscover] Add extra resources in k8s informer Signed-off-by: ChrsMark * [elastic-agent-autodiscover] Add data plane team as code owners. * [elastic-agent-autodiscover] ci: enable orka (#12) * [elastic-agent-autodiscover] automation: bump golang version (#11) * [elastic-agent-autodiscover] [Automation] Update go release version to 1.17.10 * [elastic-agent-autodiscover] [Automation] Update go release version to 1.17.11 * [elastic-agent-autodiscover] ci: use windows-2022 and default go-version (#13) * [elastic-agent-autodiscover] Upgrade containerd to fix depandabot alerts. * [elastic-agent-autodiscover] Read Go version for CI from .go-version file * [elastic-agent-autodiscover] Reuse mage targets from elastic-agent-libs * [elastic-agent-autodiscover] Update lint rules. See discussion in https://github.com/elastic/beats/pull/31683 * [elastic-agent-autodiscover] Fix new lint warnings. * [elastic-agent-autodiscover] Upgrade github.com/containerd/containerd to 1.5.13 * [elastic-agent-autodiscover] Add a notice target, generate a notice file. * [elastic-agent-autodiscover] Fix lint warnings. Ported in the rule to ignore use of fmt.Println in magefiles. * [elastic-agent-autodiscover] Add hints' utils to autodiscover library (#19) Adding hints' helpers from https://github.com/elastic/beats/blob/ba3bce42590dbf722061d7d92bfdb0bd903e9014/libbeat/autodiscover/builder/helper.go to the external library so as to be re-used by elastic-agent. * [elastic-agent-autodiscover] Move hints helpers under utils and add tests Signed-off-by: ChrsMark * [elastic-agent-autodiscover] Fix linter Signed-off-by: ChrsMark * [elastic-agent-autodiscover] Update README.md * [elastic-agent-autodiscover] Update README.md * [elastic-agent-autodiscover] Update README.md * [elastic-agent-autodiscover] [Automation] Update go release version to 1.17.12 * [elastic-agent-autodiscover] [Automation] Update go release version to 1.18.4 * [elastic-agent-autodiscover] [Automation] Update go release version to 1.18.5 * [elastic-agent-autodiscover] Update elastic-agent-libs to v0.2.11. Upgrades the local version of the linter. * [elastic-agent-autodiscover] Update linter github action. * [elastic-agent-autodiscover] Remove obsolete comment in .golangci-lint * [elastic-agent-autodiscover] add StorageClass resource in k8s informer Signed-off-by: Tetiana Kravchenko * [elastic-agent-autodiscover] jjbb: use fleet-ci (#27) * [elastic-agent-autodiscover] [Automation] Update go release version to 1.18.6 * [elastic-agent-autodiscover] Add util function to return pod labels * [elastic-agent-autodiscover] Revert "Add util function to return pod labels" This reverts commit c4a7baeca736128f2036eb2ee139864373c97740. * [elastic-agent-autodiscover] Add util function to return pod labels * [elastic-agent-autodiscover] [Automation] Update go release version to 1.18.7 * [elastic-agent-autodiscover] Create add-issues-to-ingest-board.yml * [elastic-agent-autodiscover] [Automation] Update go release version to 1.18.8 * [elastic-agent-autodiscover] add namespace MetaGen to the generic Resource Signed-off-by: Tetiana Kravchenko * [elastic-agent-autodiscover] clean up some local tests adjustments Signed-off-by: Tetiana Kravchenko * [elastic-agent-autodiscover] remove comment Signed-off-by: Tetiana Kravchenko * [elastic-agent-autodiscover] [Automation] Update go release version to 1.18.9 * [elastic-agent-autodiscover] [Automation] Update go release version to 1.18.10 * [elastic-agent-autodiscover] Update copyright year. * [elastic-agent-autodiscover] golang: bump automation with updatecli and GH actions (#36) * [elastic-agent-autodiscover] chore: Updated the content of the file "/var/folders/hn/j6gds1qx1f5by... ... sfn0qp0t03c0000gn/T/updatecli/github/elastic/elastic-agent-autodiscover/.golangci.yml" Made with ❤️️ by updatecli * [elastic-agent-autodiscover] chore: Updated the content of the file "/var/folders/hn/j6gds1qx1f5by... ... sfn0qp0t03c0000gn/T/updatecli/github/elastic/elastic-agent-autodiscover/.go-version" Made with ❤️️ by updatecli * [elastic-agent-autodiscover] chore: Updated the content of the file "/var/folders/hn/j6gds1qx1f5by... ... sfn0qp0t03c0000gn/T/updatecli/github/elastic/elastic-agent-autodiscover/go.mod" Made with ❤️️ by updatecli * [elastic-agent-autodiscover] Bump up containderd * [elastic-agent-autodiscover] lint * [elastic-agent-autodiscover] Update linter and libs * [elastic-agent-autodiscover] chore: Updated the content of the file "/tmp/updatecli/github/elastic... ... /elastic-agent-autodiscover/.golangci.yml" Made with ❤️️ by updatecli * [elastic-agent-autodiscover] chore: Updated the content of the file "/tmp/updatecli/github/elastic... ... /elastic-agent-autodiscover/.go-version" Made with ❤️️ by updatecli * [elastic-agent-autodiscover] chore: Updated the content of the file "/tmp/updatecli/github/elastic... ... /elastic-agent-autodiscover/.go-version" Made with ❤️️ by updatecli * [elastic-agent-autodiscover] chore: Updated the content of the file "/tmp/updatecli/github/elastic... ... /elastic-agent-autodiscover/.golangci.yml" Made with ❤️️ by updatecli * [elastic-agent-autodiscover] chore: Updated the content of the file "/tmp/updatecli/github/elastic... ... /elastic-agent-autodiscover/.golangci.yml" Made with ❤️️ by updatecli * [elastic-agent-autodiscover] chore: Updated the content of the file "/tmp/updatecli/github/elastic... ... /elastic-agent-autodiscover/.go-version" Made with ❤️️ by updatecli * [elastic-agent-autodiscover] chore: Updated the content of the file "/tmp/updatecli/github/elastic... ... /elastic-agent-autodiscover/.golangci.yml" Made with ❤️️ by updatecli * [elastic-agent-autodiscover] chore: Updated the content of the file "/tmp/updatecli/github/elastic... ... /elastic-agent-autodiscover/.go-version" Made with ❤️️ by updatecli * [elastic-agent-autodiscover] Add replicaset metaGenerator based on watcher Signed-off-by: ChrsMark * [elastic-agent-autodiscover] Add job metaGenerator based on watcher Signed-off-by: ChrsMark * [elastic-agent-autodiscover] fix types Signed-off-by: ChrsMark * [elastic-agent-autodiscover] Fix references of objects Signed-off-by: ChrsMark * [elastic-agent-autodiscover] Fix resource name format Signed-off-by: ChrsMark * [elastic-agent-autodiscover] Fix tests Signed-off-by: ChrsMark * [elastic-agent-autodiscover] Adjust tests Signed-off-by: ChrsMark * [elastic-agent-autodiscover] fix docstrings Signed-off-by: ChrsMark * [elastic-agent-autodiscover] Add Jobs tests Signed-off-by: ChrsMark * [elastic-agent-autodiscover] Add rs tests Signed-off-by: ChrsMark * [elastic-agent-autodiscover] Fix test Signed-off-by: ChrsMark * [elastic-agent-autodiscover] Bump golang.org/x/net from 0.0.0-20220225172249-27dd8689420f to 0.7.0 Bumps [golang.org/x/net](https://github.com/golang/net) from 0.0.0-20220225172249-27dd8689420f to 0.7.0. - [Commits](https://github.com/golang/net/commits/v0.7.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: direct:production ... Signed-off-by: dependabot[bot] * [elastic-agent-autodiscover] Update notice. * [elastic-agent-autodiscover] Bump golang.org/x/crypto from 0.0.0-20210817164053-32db794688a5 to 0.1.0 Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.0.0-20210817164053-32db794688a5 to 0.1.0. - [Commits](https://github.com/golang/crypto/commits/v0.1.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: indirect ... Signed-off-by: dependabot[bot] * [elastic-agent-autodiscover] Bump github.com/docker/docker Bumps [github.com/docker/docker](https://github.com/docker/docker) from 20.10.12+incompatible to 20.10.24+incompatible. - [Release notes](https://github.com/docker/docker/releases) - [Commits](https://github.com/docker/docker/compare/v20.10.12...v20.10.24) --- updated-dependencies: - dependency-name: github.com/docker/docker dependency-type: direct:production ... Signed-off-by: dependabot[bot] * [elastic-agent-autodiscover] Update notice. * [elastic-agent-autodiscover] Update notice. * [elastic-agent-autodiscover] Bump github.com/docker/distribution Bumps [github.com/docker/distribution](https://github.com/docker/distribution) from 2.8.0+incompatible to 2.8.2+incompatible. - [Release notes](https://github.com/docker/distribution/releases) - [Commits](https://github.com/docker/distribution/compare/v2.8.0...v2.8.2) --- updated-dependencies: - dependency-name: github.com/docker/distribution dependency-type: indirect ... Signed-off-by: dependabot[bot] * [elastic-agent-autodiscover] Update notice. * [elastic-agent-autodiscover] chore: Updated the content of the file "/tmp/updatecli/github/elastic... ... /elastic-agent-autodiscover/.golangci.yml" Made with ❤️️ by updatecli * [elastic-agent-autodiscover] chore: Updated the content of the file "/tmp/updatecli/github/elastic... ... /elastic-agent-autodiscover/.go-version" Made with ❤️️ by updatecli * [elastic-agent-autodiscover] Initial backstage anc buildkite boilerplate This commit adds the initial boilerplate for initializing the backstage and buildkite resources. Signed-off-by: Alexandros, Sapranidis * [elastic-agent-autodiscover] Re order kubernetes node discovery * [elastic-agent-autodiscover] Add a test and a note * [elastic-agent-autodiscover] Update kubernetes/util_test.go Co-authored-by: Chris Mark * [elastic-agent-autodiscover] Add changelog * [elastic-agent-autodiscover] Updated protobuf to 1.29.1 * [elastic-agent-autodiscover] notice * [elastic-agent-autodiscover] Update catalog-info.yaml * [elastic-agent-autodiscover] migrate the jenkins pipeline to the Buildkite * [elastic-agent-autodiscover] change the config of the pull-requiests.json * [elastic-agent-autodiscover] empty commit for the trigger tests * [elastic-agent-autodiscover] decrese the perfirmance of the k8s agents * [elastic-agent-autodiscover] status messages to comments * [elastic-agent-autodiscover] upgrade Go to 1.20.5 * [elastic-agent-autodiscover] Update .go-version * [elastic-agent-autodiscover] chore: Updated the content of the file "/tmp/updatecli/github/elastic... ... /elastic-agent-autodiscover/go.mod" Made with ❤️️ by updatecli * [elastic-agent-autodiscover] chore: Updated the content of the file "/tmp/updatecli/github/elastic... ... /elastic-agent-autodiscover/.golangci.yml" Made with ❤️️ by updatecli * [elastic-agent-autodiscover] chore: Updated the content of the file "/tmp/updatecli/github/elastic... ... /elastic-agent-autodiscover/.go-version" Made with ❤️️ by updatecli * [elastic-agent-autodiscover] increase Buildkite linux agent's memory (#57) * [elastic-agent-autodiscover] [updatecli][githubrelease] Bump version to 1.20.7 (#56) # [updatecli][githubrelease] Bump version to 1.20.7 ## Report Source: ✔ [gomod] Get version in go.mod format(shell) ✔ [latestGoVersion] Get Latest Go Release(githubrelease) ✔ [minor] Get minor version in .go-version(shell) Condition: ✔ [dockerTag] Is docker image golang:1.20.7 published(dockerimage) ✔ [goDefaultVersion-check] Check if defined golang version differs(shell) Target: ⚠ [update-go-version] Update .go-version(file) ⚠ [update-golang.ci] Update .golangci.yml(file) ✔ [update-gomod] Update go.mod(file) ## Changelog
Click to expand ```` no GitHub Release found for go1.20.7 on "https://github.com/golang/go" ````
## Remark This pull request was automatically created using [Updatecli](https://www.updatecli.io). Please report any issues with this tool [here](https://github.com/updatecli/updatecli/issues/) --------- Co-authored-by: apmmachine * [elastic-agent-autodiscover] Delete all Jenkins resources and rather use buildkite (#58) * [elastic-agent-autodiscover] Add development note (#59) This PR adds a note on how to use the local dependency during the development process. * [elastic-agent-autodiscover] Bump github.com/cyphar/filepath-securejoin from 0.2.3 to 0.2.4 (#61) Bumps [github.com/cyphar/filepath-securejoin](https://github.com/cyphar/filepath-securejoin) from 0.2.3 to 0.2.4.
Release notes

Sourced from github.com/cyphar/filepath-securejoin's releases.

v0.2.4

This release fixes a potential security issue in filepath-securejoin when used on Windows (GHSA-6xv5-86q9-7xr8, which could be used to generate paths outside of the provided rootfs in certain cases), as well as improving the overall behaviour of filepath-securejoin when dealing with Windows paths that contain volume names. Thanks to Paulo Gomes for discovering and fixing these issues.

In addition, we've switched (at long last) to GitHub Actions and have continuous integration testing on Linux, MacOS, and Windows.

Thanks to the following contributors for making this release possible:

Signed-off-by: Aleksa Sarai cyphar@cyphar.com

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/cyphar/filepath-securejoin&package-manager=go_modules&previous-version=0.2.3&new-version=0.2.4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/elastic/elastic-agent-autodiscover/network/alerts).
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Craig MacKenzie * [elastic-agent-autodiscover] Regex support for incuding labels and annotations in elastic-agent-autodiscovery library (#60) This PR introduces the main functionality in order to support the regex expression for kubernetes provider as described here: https://github.com/elastic/elastic-agent/issues/3346 * [elastic-agent-autodiscover] Disable Deployment and Replicaset enrichment by default (#62) Disabling AddResourceMetadataConfig.Deployment: false and AddResourceMetadataConfig.Cronjob: false as per discussion here https://github.com/elastic/elastic-agent-autodiscover/issues/31#issuecomment-1745057020 * [elastic-agent-autodiscover] Bump golang.org/x/net from 0.7.0 to 0.17.0 (#63) Bumps [golang.org/x/net](https://github.com/golang/net) from 0.7.0 to 0.17.0.
Commits
  • b225e7c http2: limit maximum handler goroutines to MaxConcurrentStreams
  • 88194ad go.mod: update golang.org/x dependencies
  • 2b60a61 quic: fix several bugs in flow control accounting
  • 73d82ef quic: handle DATA_BLOCKED frames
  • 5d5a036 quic: handle streams moving from the data queue to the meta queue
  • 350aad2 quic: correctly extend peer's flow control window after MAX_DATA
  • 21814e7 quic: validate connection id transport parameters
  • a600b35 quic: avoid redundant MAX_DATA updates
  • ea63359 http2: check stream body is present on read timeout
  • ddd8598 quic: version negotiation
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/net&package-manager=go_modules&previous-version=0.7.0&new-version=0.17.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/elastic/elastic-agent-autodiscover/network/alerts).
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Craig MacKenzie * [elastic-agent-autodiscover] Bump github.com/docker/docker from 20.10.24+incompatible to 24.0.7+incompatible (#64) Bumps [github.com/docker/docker](https://github.com/docker/docker) from 20.10.24+incompatible to 24.0.7+incompatible.
Release notes

Sourced from github.com/docker/docker's releases.

v24.0.7

24.0.7

For a full list of pull requests and changes in this release, refer to the relevant GitHub milestones:

Bug fixes and enhancements

  • Write overlay2 layer metadata atomically. moby/moby#46703
  • Fix "Rootful-in-Rootless" Docker-in-Docker on systemd version 250 and later. moby/moby#46626
  • Fix dockerd-rootless-setuptools.sh when username contains a backslash. moby/moby#46407
  • Fix a bug that would prevent network sandboxes to be fully deleted when stopping containers with no network attachments and when dockerd --bridge=none is used. moby/moby#46702
  • Fix a bug where cancelling an API request could interrupt container restart. moby/moby#46697
  • Fix an issue where containers would fail to start when providing --ip-range with a range larger than the subnet. docker/for-mac#6870
  • Fix data corruption with zstd output. moby/moby#46709
  • Fix the conditions under which the container's MAC address is applied. moby/moby#46478
  • Improve the performance of the stats collector. moby/moby#46448
  • Fix an issue with source policy rules ending up in the wrong order. moby/moby#46441

Packaging updates

Security

v24.0.6

24.0.6

For a full list of pull requests and changes in this release, refer to the relevant GitHub milestones:

Bug fixes and enhancements

  • containerd storage backend: Fix docker ps failing when a container image is no longer present in the content store. moby/moby#46095
  • containerd storage backend: Fix docker ps -s -a and docker container prune failing when a container image config is no longer present in the content store. moby/moby#46097
  • containerd storage backend: Fix docker inspect failing when a container image config is no longer (or was never) present in the content store. moby/moby#46244
  • containerd storage backend: Fix diff and export with the overlayfs snapshotter by using reference-counted rootfs mounts. moby/moby#46266
  • containerd storage backend: Fix a misleading error message when the image platforms available locally do not match the desired platform. moby/moby#46300
  • containerd storage backend: Fix the FROM scratch Dockerfile instruction with the classic builder. moby/moby#46302
  • containerd storage backend: Fix mismatched image rootfs and manifest layers errors with the classic builder. moby/moby#46310

... (truncated)

Commits
  • 311b9ff Merge pull request #46697 from thaJeztah/24.0_backport_restart_nocancel
  • af60804 Merge pull request from GHSA-jq35-85cj-fj4p
  • 3cf363e Merge pull request #46709 from thaJeztah/24.0_backport_bump_compress
  • 05d7386 daemon: daemon.containerRestart: don't cancel restart on context cancel
  • 649c944 Merge pull request #46703 from thaJeztah/24.0_backport_atomic-layer-data-write
  • 9b20b1a Merge pull request #46702 from thaJeztah/24.0_backport_releaseNetwork_Network...
  • dd37b0b vendor: github.com/klauspost/compress v1.17.2
  • 7058c0d vendor: github.com/klauspost/compress v1.16.5
  • 57bd388 daemon: overlay2: Write layer metadata atomically
  • 05d95fd daemon: release sandbox even when NetworkDisabled
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/docker/docker&package-manager=go_modules&previous-version=20.10.24+incompatible&new-version=24.0.7+incompatible)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/elastic/elastic-agent-autodiscover/network/alerts).
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Craig MacKenzie * [elastic-agent-autodiscover] enable auto-bump golang (#65) * [elastic-agent-autodiscover] [updatecli] Bump Golang version to 1.20.10 (#66) ### What Bump go release version with the latest release. ### Further details See [changelog](https://github.com/golang/go/issues?q=milestone%3AGo1.20.10+label%3ACherryPickApproved) for 1.20.10 Generated automatically with https://github.com/elastic/elastic-agent-autodiscover/actions/runs/6768260729 ---

Bump golang-version to latest version

Update .go-version

1 file(s) updated with "1.20.10": * .go-version

1.20.10 
no GitHub Release found for go1.20.10 on
"https://github.com/golang/go"
Update .golangci.yml

1 file(s) updated with "1.20.10": * .golangci.yml

1.20.10 
no GitHub Release found for go1.20.10 on
"https://github.com/golang/go"
 ---
Updatecli logo

Created automatically by Updatecli

Options:

Most of Updatecli configuration is done via its manifest(s).

  • If you close this pull request, Updatecli will automatically reopen it, the next time it runs.
  • If you close this pull request and delete the base branch, Updatecli will automatically recreate it, erasing all previous commits made.

Feel free to report any issues at github.com/updatecli/updatecli.
If you find this tool useful, do not hesitate to star our GitHub repository as a sign of appreciation, and/or to tell us directly on our chat!

--------- Co-authored-by: apmmachine * [elastic-agent-autodiscover] [updatecli] Bump Golang version to 1.20.11 (#67) ### What Bump go release version with the latest release. ### Further details See [changelog](https://github.com/golang/go/issues?q=milestone%3AGo1.20.11+label%3ACherryPickApproved) for 1.20.11 Generated automatically with https://github.com/elastic/elastic-agent-autodiscover/actions/runs/6836279610 ---

Bump golang-version to latest version

Update .go-version

1 file(s) updated with "1.20.11": * .go-version

1.20.11 
no GitHub Release found for go1.20.11 on
"https://github.com/golang/go"
Update .golangci.yml

1 file(s) updated with "1.20.11": * .golangci.yml

1.20.11 
no GitHub Release found for go1.20.11 on
"https://github.com/golang/go"
 ---
Updatecli logo

Created automatically by Updatecli

Options:

Most of Updatecli configuration is done via its manifest(s).

  • If you close this pull request, Updatecli will automatically reopen it, the next time it runs.
  • If you close this pull request and delete the base branch, Updatecli will automatically recreate it, erasing all previous commits made.

Feel free to report any issues at github.com/updatecli/updatecli.
If you find this tool useful, do not hesitate to star our GitHub repository as a sign of appreciation, and/or to tell us directly on our chat!

--------- Co-authored-by: apmmachine * [elastic-agent-autodiscover] Buidlkite: add macos (#68) * [elastic-agent-autodiscover] [updatecli] Bump Golang version to 1.20.12 (#70) ### What Bump go release version with the latest release. ### Further details See [changelog](https://github.com/golang/go/issues?q=milestone%3AGo1.20.12+label%3ACherryPickApproved) for 1.20.12 Generated automatically with https://github.com/elastic/elastic-agent-autodiscover/actions/runs/7153247505 ---

Bump golang-version to latest version

Update .golangci.yml

1 file(s) updated with "1.20.12": * .golangci.yml

1.20.12 
no GitHub Release found for go1.20.12 on
"https://github.com/golang/go"
Update .go-version

1 file(s) updated with "1.20.12": * .go-version

1.20.12 
no GitHub Release found for go1.20.12 on
"https://github.com/golang/go"
 ---
Updatecli logo

Created automatically by Updatecli

Options:

Most of Updatecli configuration is done via its manifest(s).

  • If you close this pull request, Updatecli will automatically reopen it, the next time it runs.
  • If you close this pull request and delete the base branch, Updatecli will automatically recreate it, erasing all previous commits made.

Feel free to report any issues at github.com/updatecli/updatecli.
If you find this tool useful, do not hesitate to star our GitHub repository as a sign of appreciation, and/or to tell us directly on our chat!

--------- Co-authored-by: apmmachine * [elastic-agent-autodiscover] Fix default resource metadata config (#72) fix default metadata config: ensure that AddResourceMetadataConfig.Node and AddResourceMetadataConfig.Namespace are different objects Relates https://github.com/elastic/elastic-agent/issues/3636 Signed-off-by: Tetiana Kravchenko * [elastic-agent-autodiscover] Bump golang.org/x/crypto from 0.14.0 to 0.17.0 (#73) Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.14.0 to 0.17.0.
Commits
  • 9d2ee97 ssh: implement strict KEX protocol changes
  • 4e5a261 ssh: close net.Conn on all NewServerConn errors
  • 152cdb1 x509roots/fallback: update bundle
  • fdfe1f8 ssh: defer channel window adjustment
  • b8ffc16 blake2b: drop Go 1.6, Go 1.8 compatibility
  • 7e6fbd8 ssh: wrap errors from client handshake
  • bda2f3f argon2: avoid clobbering BP
  • 325b735 ssh/test: skip TestSSHCLIAuth on Windows
  • 1eadac5 go.mod: update golang.org/x dependencies
  • b2d7c26 ssh: add (*Client).DialContext method
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/crypto&package-manager=go_modules&previous-version=0.14.0&new-version=0.17.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/elastic/elastic-agent-autodiscover/network/alerts).
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Craig MacKenzie * [elastic-agent-autodiscover] Update notice to 2024. (#74) What the title says. * [elastic-agent-autodiscover] [updatecli] Bump Golang version to 1.21.5 (#75) # [updatecli] Bump Golang version to 1.21.5 ### What Bump go release version with the latest release. ### Further details See [changelog](https://github.com/golang/go/issues?q=milestone%3AGo1.21.5+label%3ACherryPickApproved) for 1.21.5 Generated automatically with manual CLI ## Report Source: ✔ [gomod] Get version in go.mod format(shell) ✔ [latestGoVersion] Get Latest Go Release(githubrelease) ✔ [minor] Get minor version in .go-version(shell) Condition: ✔ [dockerTag] Is docker image golang:1.21.5 published(dockerimage) ✔ [goDefaultVersion-check] Check if defined golang version differs(shell) Target: ⚠ [update-go-version] Update .go-version(file) ⚠ [update-golang.ci] Update .golangci.yml(file) ⚠ [update-gomod] Update go.mod(file) ## Changelog
Click to expand ```` no GitHub Release found for go1.21.5 on "https://github.com/golang/go" ````
## Remark This pull request was automatically created using [Updatecli](https://www.updatecli.io). Please report any issues with this tool [here](https://github.com/updatecli/updatecli/issues/) * [elastic-agent-autodiscover] Conditions on podupdater functions of kubernetes autodiscovery (#71) - Enhancement ## Proposed commit message - WHAT: Update the PodUpdater fucntion with additonal checks before actaully triggering Pod watcher restarts - WHY: There where node and namespace events that dont always inlcude Metadata changes. We want only update events that change labels and annotations of node and namespace to trigger pod updates. This leads to bettwr management of watchers ## How to test this PR locally See info https://github.com/elastic/beats/issues/37338 ## Related issues - Relates https://github.com/elastic/beats/issues/37338 - Relates https://github.com/elastic/beats/pull/37431 --------- Co-authored-by: Tetiana Kravchenko Co-authored-by: Giuseppe Santoro * [elastic-agent-autodiscover] [updatecli] Bump Golang version to 1.21.6 (#76) ### What Bump go release version with the latest release. ### Further details See [changelog](https://github.com/golang/go/issues?q=milestone%3AGo1.21.6+label%3ACherryPickApproved) for 1.21.6 Generated automatically with https://github.com/elastic/elastic-agent-autodiscover/actions/runs/7514558421 ---

Bump golang-version to latest version

Update .golangci.yml

1 file(s) updated with "1.21.6": * .golangci.yml

1.21.6 
no GitHub Release found for go1.21.6 on
"https://github.com/golang/go"
Update .go-version

1 file(s) updated with "1.21.6": * .go-version

1.21.6 
no GitHub Release found for go1.21.6 on
"https://github.com/golang/go"
 ---
Updatecli logo

Created automatically by Updatecli

Options:

Most of Updatecli configuration is done via its manifest(s).

  • If you close this pull request, Updatecli will automatically reopen it, the next time it runs.
  • If you close this pull request and delete the base branch, Updatecli will automatically recreate it, erasing all previous commits made.

Feel free to report any issues at github.com/updatecli/updatecli.
If you find this tool useful, do not hesitate to star our GitHub repository as a sign of appreciation, and/or to tell us directly on our chat!

--------- Co-authored-by: apmmachine * [elastic-agent-autodiscover] [updatecli] Bump Golang version to 1.21.7 (#78) ### What Bump go release version with the latest release. ### Further details See [changelog](https://github.com/golang/go/issues?q=milestone%3AGo1.21.7+label%3ACherryPickApproved) for 1.21.7 Generated automatically with https://github.com/elastic/elastic-agent-autodiscover/actions/runs/7857106506 ---

Bump golang-version to latest version

Update .go-version

1 file(s) updated with "1.21.7": * .go-version

1.21.7 
no GitHub Release found for go1.21.7 on
"https://github.com/golang/go"
Update .golangci.yml

1 file(s) updated with "1.21.7": * .golangci.yml

1.21.7 
no GitHub Release found for go1.21.7 on
"https://github.com/golang/go"
 ---
Updatecli logo

Created automatically by Updatecli

Options:

Most of Updatecli configuration is done via its manifest(s).

  • If you close this pull request, Updatecli will automatically reopen it, the next time it runs.
  • If you close this pull request and delete the base branch, Updatecli will automatically recreate it, erasing all previous commits made.

Feel free to report any issues at github.com/updatecli/updatecli.
If you find this tool useful, do not hesitate to star our GitHub repository as a sign of appreciation, and/or to tell us directly on our chat!

--------- Co-authored-by: apmmachine * [elastic-agent-autodiscover] Add method to return a watcher's event handler (#79) This PR adds a method to kubernetes Watcher interface in order to return the event handler of a watcher. This is needed as part of refactoring of metadata enrichment (https://github.com/elastic/beats/pull/37332). * [elastic-agent-autodiscover] [updatecli] Bump Golang version to 1.21.8 (#82) ### What Bump go release version with the latest release. ### Further details See [changelog](https://github.com/golang/go/issues?q=milestone%3AGo1.21.8+label%3ACherryPickApproved) for 1.21.8 Generated automatically with https://github.com/elastic/elastic-agent-autodiscover/actions/runs/8216870501 ---

Bump golang-version to latest version

Update .go-version

1 file(s) updated with "1.21.8": * .go-version

1.21.8 
no GitHub Release found for go1.21.8 on
"https://github.com/golang/go"
Update .golangci.yml

1 file(s) updated with "1.21.8": * .golangci.yml

1.21.8 
no GitHub Release found for go1.21.8 on
"https://github.com/golang/go"
 ---
Updatecli logo

Created automatically by Updatecli

Options:

Most of Updatecli configuration is done via its manifest(s).

  • If you close this pull request, Updatecli will automatically reopen it, the next time it runs.
  • If you close this pull request and delete the base branch, Updatecli will automatically recreate it, erasing all previous commits made.

Feel free to report any issues at github.com/updatecli/updatecli.
If you find this tool useful, do not hesitate to star our GitHub repository as a sign of appreciation, and/or to tell us directly on our chat!

--------- Co-authored-by: apmmachine * [elastic-agent-autodiscover] security: add permissions block to workflows (#80) ## Details ⚠️ This PR was created by an automated tool. Please review the changes carefully. ⚠️ We want to set the default permissions for workflows to read-only for contents. This is a security measure to prevent accidental changes to the repository. This change adds a top-level permissions block to all workflows in the .github/workflows directory. ```yaml permissions: contents: read ``` In some cases workflows might need more permissions than just `contents: read`. Please checkout this branch and add the necessary permissions to the workflows. If your workflow uses a Personal Access Token (PAT), we can still add the permissions block, but it will not have any effect. Merging this PR as is might cause workflows that need more permissions to fail. If there are any questions, please reach out to the @elastic/observablt-ci * [elastic-agent-autodiscover] Introduce checks for not supported annotations for Hints based autodiscover of Elastic Agent (#81) This PR relates to https://github.com/elastic/elastic-agent/issues/3064 and provides the checks for the supported hints. * [elastic-agent-autodiscover] Bump google.golang.org/protobuf from 1.29.1 to 1.33.0 (#84) Bumps google.golang.org/protobuf from 1.29.1 to 1.33.0. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=google.golang.org/protobuf&package-manager=go_modules&previous-version=1.29.1&new-version=1.33.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/elastic/elastic-agent-autodiscover/network/alerts).
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Craig MacKenzie * [elastic-agent-autodiscover] Bump github.com/docker/docker from 24.0.7+incompatible to 24.0.9+incompatible (#85) Bumps [github.com/docker/docker](https://github.com/docker/docker) from 24.0.7+incompatible to 24.0.9+incompatible.
Release notes

Sourced from github.com/docker/docker's releases.

v24.0.9

24.0.9

For a full list of pull requests and changes in this release, refer to the relevant GitHub milestones:

Security

This release contains security fixes for the following CVEs affecting Docker Engine and its components.

CVE Component Fix version Severity
CVE-2024-21626 runc 1.1.12 High, CVSS 8.6
CVE-2024-24557 Docker Engine 24.0.9 Medium, CVSS 6.9

Important ⚠️

Note that this release of Docker Engine doesn't include fixes for the following known vulnerabilities in BuildKit:

To address these vulnerabilities, upgrade to Docker Engine v25.0.2.

For more information about the security issues addressed in this release, and the unaddressed vulnerabilities in BuildKit, refer to the blog post. For details about each vulnerability, see the relevant security advisory:

Packaging updates

v24.0.8

24.0.8

For a full list of pull requests and changes in this release, refer to the relevant GitHub milestones:

Bug fixes and enhancements

  • Live restore: Containers with auto remove (docker run --rm) are no longer forcibly removed on engine restart. moby/moby#46857

... (truncated)

Commits
  • fca702d Merge pull request from GHSA-xw73-rw38-6vjc
  • f78a772 Merge pull request #47281 from thaJeztah/24.0_backport_bump_containerd_binary...
  • 61afffe Merge pull request #47270 from thaJeztah/24.0_backport_bump_runc_binary_1.1.12
  • b38e74c Merge pull request #47276 from thaJeztah/24.0_backport_bump_runc_1.1.12
  • dac5663 update containerd binary to v1.7.13
  • 20e1af3 vendor: github.com/opencontainers/runc v1.1.12
  • 858919d update runc binary to v1.1.12
  • 141ad39 Merge pull request #47266 from vvoland/ci-fix-makeps1-templatefail-24
  • db968c6 hack/make.ps1: Fix go list pattern
  • 61c51fb Merge pull request #47221 from vvoland/pkg-pools-close-noop-24
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/docker/docker&package-manager=go_modules&previous-version=24.0.7+incompatible&new-version=24.0.9+incompatible)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/elastic/elastic-agent-autodiscover/network/alerts).
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Craig MacKenzie * [elastic-agent-autodiscover] [updatecli] Bump Golang version to 1.21.9 (#86) ### What Bump go release version with the latest release. ### Further details See [changelog](https://github.com/golang/go/issues?q=milestone%3AGo1.21.9+label%3ACherryPickApproved) for 1.21.9 Generated automatically with https://github.com/elastic/elastic-agent-autodiscover/actions/runs/8583485714 ---

Bump golang-version to latest version

Update .go-version

1 file(s) updated with "1.21.9": * .go-version

1.21.9 
no GitHub Release found for go1.21.9 on
"https://github.com/golang/go"
Update .golangci.yml

1 file(s) updated with "1.21.9": * .golangci.yml

1.21.9 
no GitHub Release found for go1.21.9 on
"https://github.com/golang/go"
 ---
Updatecli logo

Created automatically by Updatecli

Options:

Most of Updatecli configuration is done via its manifest(s).

  • If you close this pull request, Updatecli will automatically reopen it, the next time it runs.
  • If you close this pull request and delete the base branch, Updatecli will automatically recreate it, erasing all previous commits made.

Feel free to report any issues at github.com/updatecli/updatecli.
If you find this tool useful, do not hesitate to star our GitHub repository as a sign of appreciation, and/or to tell us directly on our chat!

--------- Co-authored-by: apmmachine * [elastic-agent-autodiscover] Logshints (#83) Continuation of https://github.com/elastic/elastic-agent-autodiscover/pull/81 We enhance the autodiscovery checks for supported hints to check multiple data_streams (in case of agent) and multiple metrcisets (in case of beats) * [elastic-agent-autodiscover] Correcting changelog for v0.6.12 (#88) Correcting cahngelog for latest release * [elastic-agent-autodiscover] Adding validate flag for GenerateHints (#89) Based on the https://github.com/elastic/beats/pull/38213/files#r1523532163, we consider adding a flag to enable the validation of GenerateHints. The specific function is called in other places apart form kubernetes and we dont want to add additional function calls to other beats * [elastic-agent-autodiscover] Bump golang.org/x/net from 0.17.0 to 0.23.0 (#90) Bumps [golang.org/x/net](https://github.com/golang/net) from 0.17.0 to 0.23.0.
Commits
  • c48da13 http2: fix TestServerContinuationFlood flakes
  • 762b58d http2: fix tipos in comment
  • ba87210 http2: close connections when receiving too many headers
  • ebc8168 all: fix some typos
  • 3678185 http2: make TestCanonicalHeaderCacheGrowth faster
  • 448c44f http2: remove clientTester
  • c7877ac http2: convert the remaining clientTester tests to testClientConn
  • d8870b0 http2: use synthetic time in TestIdleConnTimeout
  • d73acff http2: only set up deadline when Server.IdleTimeout is positive
  • 89f602b http2: validate client/outgoing trailers
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/net&package-manager=go_modules&previous-version=0.17.0&new-version=0.23.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/elastic/elastic-agent-autodiscover/network/alerts).
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Craig MacKenzie * [elastic-agent-autodiscover] [updatecli] Bump Golang version to 1.21.10 (#92) ### What Bump go release version with the latest release. ### Further details See [changelog](https://github.com/golang/go/issues?q=milestone%3AGo1.21.10+label%3ACherryPickApproved) for 1.21.10 Generated automatically with https://github.com/elastic/elastic-agent-autodiscover/actions/runs/9046098086 ---

Bump golang-version to latest version

Update .golangci.yml

1 file(s) updated with "1.21.10": * .golangci.yml

1.21.10 
no GitHub Release found for go1.21.10 on
"https://github.com/golang/go"
Update .go-version

1 file(s) updated with "1.21.10": * .go-version

1.21.10 
no GitHub Release found for go1.21.10 on
"https://github.com/golang/go"
 ---
Updatecli logo

Created automatically by Updatecli

Options:

Most of Updatecli configuration is done via its manifest(s).

  • If you close this pull request, Updatecli will automatically reopen it, the next time it runs.
  • If you close this pull request and delete the base branch, Updatecli will automatically recreate it, erasing all previous commits made.

Feel free to report any issues at github.com/updatecli/updatecli.
If you find this tool useful, do not hesitate to star our GitHub repository as a sign of appreciation, and/or to tell us directly on our chat!

--------- Co-authored-by: obltmachine <150269514+obltmachine@users.noreply.github.com> * [elastic-agent-autodiscover] Update k8s.io/client-go to v0.29.5 (#93) This updates the `client-go` dependency to a more recent version. This is a breaking change, as it removes support for `PodSecurityPolicy`, which has been [removed](https://kubernetes.io/docs/concepts/security/pod-security-policy/) since Kubernetes v1.25. I have upgraded to `v0.29` as this is the latest version that supports Go 1.21 - `client-go@v0.30` requires Go 1.22. We can also upgrade to a version lower than v0.29 if that is needed by any of the downstream libraries depending on this package. * [elastic-agent-autodiscover] Remove CHANGELOG (#95) This PR removes the `CHANGELOG.md` file. It has not been in use since the initial release. Also this repository has been maintaining it's release notes as part the [release documentation](https://github.com/elastic/elastic-agent-autodiscover/releases) itself. * [elastic-agent-autodiscover] [updatecli] Bump Golang version to 1.21.11 (#96) ### What Bump go release version with the latest release. ### Further details See [changelog](https://github.com/golang/go/issues?q=milestone%3AGo1.21.11+label%3ACherryPickApproved) for 1.21.11 Generated automatically with https://github.com/elastic/elastic-agent-autodiscover/actions/runs/9431204465 ---

Bump golang-version to latest version

Update .go-version

1 file(s) updated with "1.21.11": * .go-version

1.21.11 
no GitHub Release found for go1.21.11 on
"https://github.com/golang/go"
Update .golangci.yml

1 file(s) updated with "1.21.11": * .golangci.yml

1.21.11 
no GitHub Release found for go1.21.11 on
"https://github.com/golang/go"
 ---
Updatecli logo

Created automatically by Updatecli

Options:

Most of Updatecli configuration is done via its manifest(s).

  • If you close this pull request, Updatecli will automatically reopen it, the next time it runs.
  • If you close this pull request and delete the base branch, Updatecli will automatically recreate it, erasing all previous commits made.

Feel free to report any issues at github.com/updatecli/updatecli.
If you find this tool useful, do not hesitate to star our GitHub repository as a sign of appreciation, and/or to tell us directly on our chat!

--------- Co-authored-by: obltmachine <150269514+obltmachine@users.noreply.github.com> * [elastic-agent-autodiscover] Use core-windows-2022 image instead of unmaintained ci-windows-2022 (#97) The image family `ci-windows-2022` has been unmaintained for a while and has been replaced by `core-windows-2022`. This PR updates them to this instead. Related to [incident 505.](https://elastic.slack.com/archives/C078RND5G5C) * [elastic-agent-autodiscover] [updatecli] Bump Golang version to 1.21.12 (#99) ### What Bump go release version with the latest release. ### Further details See [changelog](https://github.com/golang/go/issues?q=milestone%3AGo1.21.12+label%3ACherryPickApproved) for 1.21.12 Generated automatically with https://github.com/elastic/elastic-agent-autodiscover/actions/runs/9821763055 ---

Bump golang-version to latest version

Update .go-version

1 file(s) updated with "1.21.12": * .go-version

1.21.12 
no GitHub Release found for go1.21.12 on
"https://github.com/golang/go"
Update .golangci.yml

1 file(s) updated with "1.21.12": * .golangci.yml

1.21.12 
no GitHub Release found for go1.21.12 on
"https://github.com/golang/go"
 ---
Updatecli logo

Created automatically by Updatecli

Options:

Most of Updatecli configuration is done via its manifest(s).

  • If you close this pull request, Updatecli will automatically reopen it, the next time it runs.
  • If you close this pull request and delete the base branch, Updatecli will automatically recreate it, erasing all previous commits made.

Feel free to report any issues at github.com/updatecli/updatecli.
If you find this tool useful, do not hesitate to star our GitHub repository as a sign of appreciation, and/or to tell us directly on our chat!

--------- Co-authored-by: obltmachine <150269514+obltmachine@users.noreply.github.com> * [elastic-agent-autodiscover] Align CI go version (#101) trying to align the go version used in CI with the one we used for this project - https://github.com/elastic/elastic-agent-autodiscover/blob/main/.go-version * [elastic-agent-autodiscover] Update docker dependency (#100) Updated dependency is needed for agent. Replaced deprecated type `type.ContainerListOptions` with `container.ListOptions` * [elastic-agent-autodiscover] Update docker to v26.1.5 (#102) Co-authored-by: Julien Lind * [elastic-agent-autodiscover] refactor: drop x/net dependency (#104) replace x/net/context with stdlib context fix compile errors after docker bump * [elastic-agent-autodiscover] [autodiscover] - Providing config option to disable Kubeadm config api requests (#98) Related changes needed for https://github.com/elastic/beats/pull/40086 * [elastic-agent-autodiscover] [updatecli] Bump Golang version to 1.21.13 (#106) ### What Bump go release version with the latest release. ### Further details See [changelog](https://github.com/golang/go/issues?q=milestone%3AGo1.21.13+label%3ACherryPickApproved) for 1.21.13 Generated automatically with https://github.com/elastic/elastic-agent-autodiscover/actions/runs/10334132092 ---

Bump golang-version to latest version

Update .go-version

1 file(s) updated with "1.21.13": * .go-version

1.21.13 
no GitHub Release found for go1.21.13 on
"https://github.com/golang/go"
Update .golangci.yml

1 file(s) updated with "1.21.13": * .golangci.yml

1.21.13 
no GitHub Release found for go1.21.13 on
"https://github.com/golang/go"
 ---
Updatecli logo

Created automatically by Updatecli

Options:

Most of Updatecli configuration is done via its manifest(s).

  • If you close this pull request, Updatecli will automatically reopen it, the next time it runs.
  • If you close this pull request and delete the base branch, Updatecli will automatically recreate it, erasing all previous commits made.

Feel free to report any issues at github.com/updatecli/updatecli.
If you find this tool useful, do not hesitate to star our GitHub repository as a sign of appreciation, and/or to tell us directly on our chat!

--------- Co-authored-by: obltmachine <150269514+obltmachine@users.noreply.github.com> Co-authored-by: Craig MacKenzie * [elastic-agent-autodiscover] github-action: use oblt-actions/git/setup (#107) * [elastic-agent-autodiscover] Update tools and fix CI (#110) Update github.com/elastic/elastic-agent-libs so we get a more recent version of golangci-lint locally. Now running `mage lint` gives the same result as the Github Action. Fix linter errors in existing code, drop deprecated linters, and revert the `nakedret` linter to its default config. I've also updated the default Go version to 1.22.7, as the updated tooling requires it. * [elastic-agent-autodiscover] Allow ReplicaSet and Job metadata generators to use partial meta objects (#109) We only use metadata from Jobs and ReplicaSets, but require that full resources are supplied. This change relaxes this requirement, allowing PartialObjectMetadata resources to be used. This allows callers to use metadata informers and avoid having to receive and deserialize non-metadata updates from the API Server. See https://github.com/elastic/elastic-agent/pull/5580 for an example of how this could be used. I'm planning to add the metadata informer from that PR to this library as well. Together, these will allow us to greatly reduce memory used for processing and storing ReplicaSets and Jobs in beats and elastic-agent. This is will help https://github.com/elastic/elastic-agent/pull/5580 and https://github.com/elastic/elastic-agent/issues/4729 specifically, and https://github.com/elastic/elastic-agent/issues/3801 in general. * [elastic-agent-autodiscover] Add metadata watcher and informer (#111) Introduce metadata-only watchers to the kubernetes package. These are useful if we only need to track metadata for a resource - a good example are ReplicaSets, for which we usually only care about the OwnerReferences. As a result, we only store the metadata, reducing steady-state memory consumption, but also only get updates involving metadata, reducing churn greatly in larger clusters. The implementation introduces new constructors for the Watcher, allowing an informer to be passed in. Existing constructors are implemented using the new constructor, though none of the code actually changes. As a result, it is now possible to unit test the watcher, and I've added some basic unit tests for it. We also add two helper functions: - `GetKubernetesMetadataClient` creates a metadata-only kubernetes client, and is very similar to the existing `GetKubernetesClient` - `RemoveUnnecessaryReplicaSetData` is a transform function that can be passed into an informer so it only stores the metadata we actually use I tested these new functions in both beats and agent, in a kind cluster as well as one of our staging clusters. This is part of the solution to https://github.com/elastic/elastic-agent/pull/5580. --------- Co-authored-by: Mauri de Souza Meneguzzo * [elastic-agent-autodiscover] Update catalog-info file with correct system property (#112) Relates: https://github.com/elastic/ingest-dev/issues/3398 * [elastic-agent-autodiscover] Bump golang.org/x/crypto from 0.24.0 to 0.31.0 (#114) Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.24.0 to 0.31.0.
Commits
  • b4f1988 ssh: make the public key cache a 1-entry FIFO cache
  • 7042ebc openpgp/clearsign: just use rand.Reader in tests
  • 3e90321 go.mod: update golang.org/x dependencies
  • 8c4e668 x509roots/fallback: update bundle
  • 6018723 go.mod: update golang.org/x dependencies
  • 71ed71b README: don't recommend go get
  • 750a45f sha3: add MarshalBinary, AppendBinary, and UnmarshalBinary
  • 36b1725 sha3: avoid trailing permutation
  • 80ea76e sha3: fix padding for long cSHAKE parameters
  • c17aa50 sha3: avoid buffer copy
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/crypto&package-manager=go_modules&previous-version=0.24.0&new-version=0.31.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/elastic/elastic-agent-autodiscover/network/alerts).
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Craig MacKenzie * [elastic-agent-autodiscover] Update github.com/docker/docker dependency to v28.1.1 (#116) Updates the github.com/docker/docker module dependency to v28.1.1. Also does a bit of related cleanup: * Replaces the use of `types.EventsOptions` with `events.ListOptions` as the former has been removed from the Docker API in the new version. * Replaces the use of several deprecated types in the Docker API as they are slated for removal in a future version: * `types.Container` -> `container.Summary` * `types.ContainerJSON` -> `container.InspectResponse` * `types.SummaryNetworkSettings` -> `container.NetworkSettingsSummary` * Removes redundant type usages * [elastic-agent-autodiscover] github-actions: use them instead of buildkite (#118) * [elastic-agent-autodiscover] buildkite: remove buildkite (#120) See https://github.com/elastic/elastic-agent-autodiscover/pull/118 * [elastic-agent-autodiscover] github-action: add catalog-validate for GitHub actions (#117) * [elastic-agent-autodiscover] Bump the github-actions group across 1 directory with 4 updates (#119) Bumps the github-actions group with 4 updates in the / directory: [actions/checkout](https://github.com/actions/checkout), [updatecli/updatecli-action](https://github.com/updatecli/updatecli-action), [actions/setup-go](https://github.com/actions/setup-go) and [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action). Updates `actions/checkout` from 3 to 4
Release notes

Sourced from actions/checkout's releases.

v4.0.0

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v3...v4.0.0

v3.6.0

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v3.5.3...v3.6.0

v3.5.3

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v3...v3.5.3

v3.5.2

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v3.5.1...v3.5.2

v3.5.1

What's Changed

New Contributors

... (truncated)

Changelog

Sourced from actions/checkout's changelog.

Changelog

v4.2.2

v4.2.1

v4.2.0

v4.1.7

v4.1.6

v4.1.5

v4.1.4

v4.1.3

v4.1.2

v4.1.1

v4.1.0

... (truncated)

Commits

Updates `updatecli/updatecli-action` from 2.48.0 to 2.84.0
Release notes

Sourced from updatecli/updatecli-action's releases.

v2.84.0 🌈

Changes

🧰 Maintenance

Contributors

@​dependabot[bot], @​updateclibot[bot], dependabot[bot] and updateclibot[bot]

v2.83.0 🌈

Changes

🧰 Maintenance

Contributors

@​olblak, @​updateclibot[bot] and updateclibot[bot]

v2.82.0 🌈

Changes

🧰 Maintenance

Contributors

@​updateclibot[bot] and updateclibot[bot]

v2.81.0 🌈

Changes

🧰 Maintenance

Contributors

... (truncated)

Commits
  • 307ce72 chore(deps): Bump actions/setup-node from 4.3.0 to 4.4.0 (#892)
  • 11f06f8 Bump "eslint-plugin-prettier" package version (#886)
  • be3db6a Bump "@​types/node" package version (#879)
  • e06f6cf deps: bump Updatecli GitHub action to v0.100.0 (#893)
  • cf94222 deps: update ubuntu in release-drafter.yml workflow
  • 8fcd21f deps: bump updatecli version to v0.99.0 (#890)
  • ae3030c deps: bump updatecli version (#889)
  • 8b8b71b deps: bump updatecli version (#888)
  • 6257796 deps: bump updatecli version (#887)
  • 436410c Bump "eslint-plugin-prettier" package version (#882)
  • Additional commits viewable in compare view

Updates `actions/setup-go` from 3 to 5
Release notes

Sourced from actions/setup-go's releases.

v5.0.0

What's Changed

In scope of this release, we change Nodejs runtime from node16 to node20 (actions/setup-go#421). Moreover, we update some dependencies to the latest versions (actions/setup-go#445).

Besides, this release contains such changes as:

New Contributors

Full Changelog: https://github.com/actions/setup-go/compare/v4...v5.0.0

v4.2.1

What's Changed

Full Changelog: https://github.com/actions/setup-go/compare/v4...v4.2.1

v4.2.0

What's Changed

  • Upgrade @actions/cache by @​priyagupta108 in actions/setup-go#578. In scope of this release we updated actions/cache package to 4.0.3. This update ensures continued support and compatibility, as older versions of the package are now deprecated. For more information please refer to the toolkit/cache.

Full Changelog: https://github.com/actions/setup-go/compare/v4...v4.2.0

v4.1.0

What's Changed

In scope of this release, slow installation on Windows was fixed by @​dsame in actions/setup-go#393 and OS version was added to primaryKey for Ubuntu runners to avoid conflicts (actions/setup-go#383)

This release also includes the following changes:

New Contributors

Full Changelog: https://github.com/actions/setup-go/compare/v4...v4.1.0

... (truncated)

Commits
  • d35c59a chore: update discussions url (#527)
  • 29694d7 Add manifest validation and improve error handling (#586)
  • 78535dd Bump eslint-plugin-jest from 27.9.0 to 28.11.0 (#537)
  • bb65d88 Bump ts-jest from 29.1.2 to 29.3.2 (#582)
  • 7f17e83 Bump @​actions/glob from 0.4.0 to 0.5.0 (#573)
  • dca8468 Update self-hosted environment validation and bump undici version (#556)
  • 691cc35 upgrade actions/cache to 4.0.3 (#574)
  • 0aaccfd Bump undici from 5.28.4 to 5.28.5 (#541)
  • c4c1141 upgrade actions/cache to 4.0.2 (#568)
  • 5a083d0 Bump eslint-config-prettier from 8.10.0 to 10.0.1 (#536)
  • Additional commits viewable in compare view

Updates `golangci/golangci-lint-action` from 3 to 8
Release notes

Sourced from golangci/golangci-lint-action's releases.

v8.0.0

Requires golangci-lint version >= v2.1.0

What's Changed

Changes

Full Changelog: https://github.com/golangci/golangci-lint-action/compare/v7...v8.0.0

v7.0.1

What's Changed

Documentation

Dependencies

New Contributors

Full Changelog: https://github.com/golangci/golangci-lint-action/compare/v7.0.0...v7.0.1

v7.0.0

⚠️ The GitHub Action v7 supports golangci-lint v2 only. ⚠️

What's Changed

Changes

Documentation

Dependencies

New Contributors

... (truncated)

Commits
  • 4afd733 8.0.0
  • 7774f98 feat: use absolute paths by default when using working-directory option (#1231)
  • 9fae48a 7.0.1
  • 16ece5e docs: clarify that ’args: --path-mode=abs’ is needed for working-directory (...
  • a3942e2 build(deps-dev): bump the dev-dependencies group with 2 updates (#1227)
  • 7ecb048 build(deps): bump @​types/node from 22.14.0 to 22.14.1 in the dependencies gro...
  • 63a0d0e build(deps-dev): bump the dev-dependencies group with 3 updates (#1224)
  • c2427fe docs: update problem matchers section
  • 642f8ee build(deps): bump @​types/node from 22.13.14 to 22.14.0 in the dependencies gr...
  • d84be92 build(deps-dev): bump the dev-dependencies group with 4 updates (#1220)
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Olga Naydyonock * [elastic-agent-autodiscover] [updatecli] Bump Golang version to 1.24.4 (#121) ### What Bump go release version with the latest release. ### Further details See [changelog](https://github.com/golang/go/issues?q=milestone%3AGo1.24.4+label%3ACherryPickApproved) for 1.24.4 Generated automatically with https://github.com/elastic/elastic-agent-autodiscover/actions/runs/15511220764 ---

Bump golang-version to latest version

Update .go-version

1 file(s) updated with "1.24.4": * .go-version

GitHub Action workflow link ---
Updatecli logo

Created automatically by Updatecli

Options:

Most of Updatecli configuration is done via its manifest(s).

  • If you close this pull request, Updatecli will automatically reopen it, the next time it runs.
  • If you close this pull request and delete the base branch, Updatecli will automatically recreate it, erasing all previous commits made.

Feel free to report any issues at github.com/updatecli/updatecli.
If you find this tool useful, do not hesitate to star our GitHub repository as a sign of appreciation, and/or to tell us directly on our chat!

Co-authored-by: obltmachine * [elastic-agent-autodiscover] Bump golang.org/x/net from 0.34.0 to 0.38.0 (#122) Bumps [golang.org/x/net](https://github.com/golang/net) from 0.34.0 to 0.38.0.
Commits
  • e1fcd82 html: properly handle trailing solidus in unquoted attribute value in foreign...
  • ebed060 internal/http3: fix build of tests with GOEXPERIMENT=nosynctest
  • 1f1fa29 publicsuffix: regenerate table
  • 1215081 http2: improve error when server sends HTTP/1
  • 312450e html: ensure <search> tag closes <p> and update tests
  • 09731f9 http2: improve handling of lost PING in Server
  • 55989e2 http2/h2c: use ResponseController for hijacking connections
  • 2914f46 websocket: re-recommend gorilla/websocket
  • 99b3ae0 go.mod: update golang.org/x dependencies
  • 85d1d54 go.mod: update golang.org/x dependencies
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/net&package-manager=go_modules&previous-version=0.34.0&new-version=0.38.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/elastic/elastic-agent-autodiscover/network/alerts).
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Craig MacKenzie * [elastic-agent-autodiscover] Bump updatecli/updatecli-action from 2.84.0 to 2.86.0 in the github-actions group across 1 directory (#123) Bumps the github-actions group with 1 update in the / directory: [updatecli/updatecli-action](https://github.com/updatecli/updatecli-action). Updates `updatecli/updatecli-action` from 2.84.0 to 2.86.0 * [elastic-agent-autodiscover] Bump updatecli/updatecli-action from 2.86.0 to 2.87.0 in the github-actions group across 1 directory (#124) Bumps the github-actions group with 1 update in the / directory: [updatecli/updatecli-action](https://github.com/updatecli/updatecli-action). Updates `updatecli/updatecli-action` from 2.86.0 to 2.87.0 * [elastic-agent-autodiscover] Replace global logger with local logger (#127) Part of effort to get rid of global logger and replace them with local versions See Issue https://github.com/elastic/ingest-dev/issues/5251 * [elastic-agent-autodiscover] fix: notice (#128) * [elastic-agent-autodiscover] feat: retract v1.0.0 and future v1.0.1 version (#129) we don't want to bump a major version for now * [elastic-agent-autodiscover] Bump updatecli/updatecli-action from 2.87.0 to 2.88.0 in the github-actions group across 1 directory (#126) Bumps the github-actions group with 1 update in the / directory: [updatecli/updatecli-action](https://github.com/updatecli/updatecli-action). Updates `updatecli/updatecli-action` from 2.87.0 to 2.88.0
Release notes

Sourced from updatecli/updatecli-action's releases.

v2.88.0 🌈

Changes

🧰 Maintenance

Contributors

@​updateclibot[bot] and updateclibot[bot]

v2.87.1 🌈

Changes

Contributors

@​updateclibot[bot] and updateclibot[bot]

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=updatecli/updatecli-action&package-manager=github_actions&previous-version=2.87.0&new-version=2.88.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Victor Martinez * [elastic-agent-autodiscover] Bump updatecli/updatecli-action from 2.88.0 to 2.89.0 in the github-actions group across 1 directory (#131) Bumps the github-actions group with 1 update in the / directory: [updatecli/updatecli-action](https://github.com/updatecli/updatecli-action). Updates `updatecli/updatecli-action` from 2.88.0 to 2.89.0
Release notes

Sourced from updatecli/updatecli-action's releases.

v2.89.0 🌈

Changes

Dependencies

Contributors

@​updateclibot[bot] and updateclibot[bot]

Commits
  • 419b75c deps: update updatecli version to v0.105.0 (#922)
  • 6aef077 Bump "@​types/node" package version (#921)
  • d4b5fd4 Bump "eslint-plugin-prettier" package version (#920)
  • e39aca5 Bump "eslint-config-prettier" package version (#919)
  • 3b1aa0a Bump "eslint-config-prettier" package version (#918)
  • caba4ef Bump "eslint-plugin-prettier" package version (#917)
  • dd19655 Bump "eslint-config-prettier" package version (#916)
  • ccc4369 Bump "@​types/node" package version (#914)
  • 779eb0e deps(updatecli/policies): bump all policies (#915)
  • 51c0b80 deps: bump Updatecli GitHub action to v2.88.0 (#913)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=updatecli/updatecli-action&package-manager=github_actions&previous-version=2.88.0&new-version=2.89.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * [elastic-agent-autodiscover] Bump the github-actions group across 1 directory with 2 updates (#133) Bumps the github-actions group with 2 updates in the / directory: [actions/checkout](https://github.com/actions/checkout) and [updatecli/updatecli-action](https://github.com/updatecli/updatecli-action). Updates `actions/checkout` from 4 to 5
Release notes

Sourced from actions/checkout's releases.

v5.0.0

What's Changed

⚠️ Minimum Compatible Runner Version

v2.327.1
Release Notes

Make sure your runner is updated to this version or newer to use this release.

Full Changelog: https://github.com/actions/checkout/compare/v4...v5.0.0

v4.3.0

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v4...v4.3.0

v4.2.2

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v4.2.1...v4.2.2

v4.2.1

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v4.2.0...v4.2.1

... (truncated)

Changelog

Sourced from actions/checkout's changelog.

Changelog

V5.0.0

V4.3.0

v4.2.2

v4.2.1

v4.2.0

v4.1.7

v4.1.6

v4.1.5

v4.1.4

v4.1.3

... (truncated)

Commits

Updates `updatecli/updatecli-action` from 2.89.0 to 2.90.0
Release notes

Sourced from updatecli/updatecli-action's releases.

v2.90.0 🌈

Changes

Contributors

@​updateclibot[bot] and updateclibot[bot]

Commits

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * [elastic-agent-autodiscover] Bump updatecli/updatecli-action from 2.90.0 to 2.91.0 in the github-actions group across 1 directory (#135) Bumps the github-actions group with 1 update in the / directory: [updatecli/updatecli-action](https://github.com/updatecli/updatecli-action). Updates `updatecli/updatecli-action` from 2.90.0 to 2.91.0
Release notes

Sourced from updatecli/updatecli-action's releases.

v2.91.0 🌈

Changes

Contributors

@​updateclibot[bot] and updateclibot[bot]

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=updatecli/updatecli-action&package-manager=github_actions&previous-version=2.90.0&new-version=2.91.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * [elastic-agent-autodiscover] [updatecli] Bump Golang version to 1.24.7 (#136) ### What Bump go release version with the latest release. ### Further details See [changelog](https://github.com/golang/go/issues?q=milestone%3AGo1.24.7+label%3ACherryPickApproved) for 1.24.7 Generated automatically with https://github.com/elastic/elastic-agent-autodiscover/actions/runs/17518933838 ---

Bump golang-version to latest version

Update .go-version

1 file(s) updated with "1.24.7": * .go-version

GitHub Action workflow link ---
Updatecli logo

Created automatically by Updatecli

Options:

Most of Updatecli configuration is done via its manifest(s).

  • If you close this pull request, Updatecli will automatically reopen it, the next time it runs.
  • If you close this pull request and delete the base branch, Updatecli will automatically recreate it, erasing all previous commits made.

Feel free to report any issues at github.com/updatecli/updatecli.
If you find this tool useful, do not hesitate to star our GitHub repository as a sign of appreciation, and/or to tell us directly on our chat!

Co-authored-by: obltmachine * [elastic-agent-autodiscover] [updatecli] Bump Golang version to 1.24.8 (#139) ### What Bump go release version with the latest release. ### Further details See [changelog](https://github.com/golang/go/issues?q=milestone%3AGo1.24.8+label%3ACherryPickApproved) for 1.24.8 Generated automatically with https://github.com/elastic/elastic-agent-autodiscover/actions/runs/18434091812 ---

Bump golang-version to latest version

Update .go-version

1 file(s) updated with "1.24.8": * .go-version

GitHub Action workflow link ---
Updatecli logo

Created automatically by Updatecli

Options:

Most of Updatecli configuration is done via its manifest(s).

  • If you close this pull request, Updatecli will automatically reopen it, the next time it runs.
  • If you close this pull request and delete the base branch, Updatecli will automatically recreate it, erasing all previous commits made.

Feel free to report any issues at github.com/updatecli/updatecli.
If you find this tool useful, do not hesitate to star our GitHub repository as a sign of appreciation, and/or to tell us directly on our chat!

Co-authored-by: obltmachine * [elastic-agent-autodiscover] Bump the github-actions group across 2 directories with 2 updates (#138) Bumps the github-actions group with 2 updates in the / directory: [updatecli/updatecli-action](https://github.com/updatecli/updatecli-action) and [actions/setup-go](https://github.com/actions/setup-go). Bumps the github-actions group with 1 update in the /.github/actions/setup directory: [actions/setup-go](https://github.com/actions/setup-go). * [elastic-agent-autodiscover] [updatecli] Bump Golang version to 1.24.9 (#140) ### What Bump go release version with the latest release. ### Further details See [changelog](https://github.com/golang/go/issues?q=milestone%3AGo1.24.9+label%3ACherryPickApproved) for 1.24.9 Generated automatically with https://github.com/elastic/elastic-agent-autodiscover/actions/runs/18620315629 ---

Bump golang-version to latest version

Update .go-version

1 file(s) updated with "1.24.9": * .go-version

GitHub Action workflow link ---
Updatecli logo

Created automatically by Updatecli

Options:

Most of Updatecli configuration is done via its manifest(s).

  • If you close this pull request, Updatecli will automatically reopen it, the next time it runs.
  • If you close this pull request and delete the base branch, Updatecli will automatically recreate it, erasing all previous commits made.

Feel free to report any issues at github.com/updatecli/updatecli.
If you find this tool useful, do not hesitate to star our GitHub repository as a sign of appreciation, and/or to tell us directly on our chat!

Co-authored-by: obltmachine * [elastic-agent-autodiscover] [updatecli] Bump Golang version to 1.24.10 (#142) ### What Bump go release version with the latest release. ### Further details See [changelog](https://github.com/golang/go/issues?q=milestone%3AGo1.24.10+label%3ACherryPickApproved) for 1.24.10 Generated automatically with https://github.com/elastic/elastic-agent-autodiscover/actions/runs/19197904244 ---

Bump golang-version to latest version

Update .go-version

1 file(s) updated with "1.24.10": * .go-version

GitHub Action workflow link ---
Updatecli logo

Created automatically by Updatecli

Options:

Most of Updatecli configuration is done via its manifest(s).

  • If you close this pull request, Updatecli will automatically reopen it, the next time it runs.
  • If you close this pull request and delete the base branch, Updatecli will automatically recreate it, erasing all previous commits made.

Feel free to report any issues at github.com/updatecli/updatecli.
If you find this tool useful, do not hesitate to star our GitHub repository as a sign of appreciation, and/or to tell us directly on our chat!

Co-authored-by: obltmachine * [elastic-agent-autodiscover] Bump the github-actions group across 1 directory with 2 updates (#143) Bumps the github-actions group with 2 updates in the / directory: [updatecli/updatecli-action](https://github.com/updatecli/updatecli-action) and [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action). Updates `updatecli/updatecli-action` from 2.93.0 to 2.94.0
Release notes

Sourced from updatecli/updatecli-action's releases.

v2.94.0 🌈

Changes

🧰 Maintenance

Contributors

@​olblak, @​updateclibot[bot] and updateclibot[bot]

Commits

Updates `golangci/golangci-lint-action` from 8 to 9
Release notes

Sourced from golangci/golangci-lint-action's releases.

v9.0.0

In the scope of this release, we change Nodejs runtime from node20 to node24 (https://github.blog/changelog/2025-09-19-deprecation-of-node-20-on-github-actions-runners/).

What's Changed

Changes

Full Changelog: https://github.com/golangci/golangci-lint-action/compare/v8.0.0...v9.0.0

Commits
  • 0a35821 docs: update readme
  • 043b1b8 feat: support Module Plugin System (#1306)
  • a66d26a feat: add install-only option (#1305)
  • 7fe1b22 build(deps): bump the dependencies group with 2 updates (#1303)
  • 14973f1 build(deps-dev): bump the dev-dependencies group with 2 updates (#1299)
  • 8c2d575 build(deps): bump @​types/node from 24.8.1 to 24.9.1 in the dependencies group...
  • b002b6e build(deps): bump actions/setup-node from 5 to 6 (#1296)
  • c13f4ed build(deps): bump @​types/node from 24.7.2 to 24.8.1 in the dependencies group...
  • b68d21b docs: improve readme
  • 06188a2 build(deps): bump github/codeql-action from 3 to 4 (#1293)
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * [elastic-agent-autodiscover] [docker] don't return error on watcher creation (#145) Log the error if `listContainers` fails. Propagating the error to the caller causes the entire elastic-agent to shut down. Relates https://github.com/elastic/elastic-agent/issues/11171 ## Screenshots - The monitoring components are back to healthy state after this change: Screenshot 2025-11-21 at 4 22 46 PM * [elastic-agent-autodiscover] Bump golang.org/x/oauth2 from 0.10.0 to 0.27.0 (#148) Bumps [golang.org/x/oauth2](https://github.com/golang/oauth2) from 0.10.0 to 0.27.0.
Commits
  • 681b4d8 jws: split token into fixed number of parts
  • 3f78298 all: upgrade go directive to at least 1.23.0 [generated]
  • 109dabf endpoints: add links/provider for Discord
  • ac571fa oauth2: fix docs for Config.DeviceAuth
  • 314ee5b endpoints: add patreon endpoint
  • b9c813b google: add warning about externally-provided credentials
  • 49a531d all: make method and struct comments match the names
  • 22134a4 README: don't recommend go get
  • 3e64809 x/oauth2: add Token.ExpiresIn
  • 16a9973 jwt: rename example to avoid vet error
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/oauth2&package-manager=go_modules&previous-version=0.10.0&new-version=0.27.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/elastic/elastic-agent-autodiscover/network/alerts).
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Andrzej Stencel * [elastic-agent-autodiscover] [updatecli] Bump Golang version to 1.24.11 (#149) ### What Bump go release version with the latest release. ### Further details See [changelog](https://github.com/golang/go/issues?q=milestone%3AGo1.24.11+label%3ACherryPickApproved) for 1.24.11 Generated automatically with https://github.com/elastic/elastic-agent-autodiscover/actions/runs/19993560951 ---

Bump golang-version to latest version

Update .go-version

1 file(s) updated with "1.24.11": * .go-version

GitHub Action workflow link ---
Updatecli logo

Created automatically by Updatecli

Options:

Most of Updatecli configuration is done via its manifest(s).

  • If you close this pull request, Updatecli will automatically reopen it, the next time it runs.
  • If you close this pull request and delete the base branch, Updatecli will automatically recreate it, erasing all previous commits made.

Feel free to report any issues at github.com/updatecli/updatecli.
If you find this tool useful, do not hesitate to star our GitHub repository as a sign of appreciation, and/or to tell us directly on our chat!

Co-authored-by: obltmachine * [elastic-agent-autodiscover] Bump golang.org/x/crypto from 0.36.0 to 0.45.0 (#147) Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.36.0 to 0.45.0.
Commits
  • 4e0068c go.mod: update golang.org/x dependencies
  • e79546e ssh: curb GSSAPI DoS risk by limiting number of specified OIDs
  • f91f7a7 ssh/agent: prevent panic on malformed constraint
  • 2df4153 acme/autocert: let automatic renewal work with short lifetime certs
  • bcf6a84 acme: pass context to request
  • b4f2b62 ssh: fix error message on unsupported cipher
  • 79ec3a5 ssh: allow to bind to a hostname in remote forwarding
  • 122a78f go.mod: update golang.org/x dependencies
  • c0531f9 all: eliminate vet diagnostics
  • 0997000 all: fix some comments
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/crypto&package-manager=go_modules&previous-version=0.36.0&new-version=0.45.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/elastic/elastic-agent-autodiscover/network/alerts).
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * [elastic-agent-autodiscover] [updatecli] Bump Golang version to 1.24.12 (#150) ### What Bump go release version with the latest release. ### Further details See [changelog](https://github.com/golang/go/issues?q=milestone%3AGo1.24.12+label%3ACherryPickApproved) for 1.24.12 Generated automatically with https://github.com/elastic/elastic-agent-autodiscover/actions/runs/21100075616 ---

Bump golang-version to latest version

Update .go-version

1 file(s) updated with "1.24.12": * .go-version

GitHub Action workflow link ---
Updatecli logo

Created automatically by Updatecli

Options:

Most of Updatecli configuration is done via its manifest(s).

  • If you close this pull request, Updatecli will automatically reopen it, the next time it runs.
  • If you close this pull request and delete the base branch, Updatecli will automatically recreate it, erasing all previous commits made.

Feel free to report any issues at github.com/updatecli/updatecli.
If you find this tool useful, do not hesitate to star our GitHub repository as a sign of appreciation, and/or to tell us directly on our chat!

Co-authored-by: obltmachine * [elastic-agent-autodiscover] [updatecli] Bump Golang version to 1.24.13 (#151) ### What Bump go release version with the latest release. ### Further details See [changelog](https://github.com/golang/go/issues?q=milestone%3AGo1.24.13+label%3ACherryPickApproved) for 1.24.13 Generated automatically with https://github.com/elastic/elastic-agent-autodiscover/actions/runs/21786051745 ---

Bump golang-version to latest version

Update .go-version

1 file(s) updated with "1.24.13": * .go-version

GitHub Action workflow link ---
Updatecli logo

Created automatically by Updatecli

Options:

Most of Updatecli configuration is done via its manifest(s).

  • If you close this pull request, Updatecli will automatically reopen it, the next time it runs.
  • If you close this pull request and delete the base branch, Updatecli will automatically recreate it, erasing all previous commits made.

Feel free to report any issues at github.com/updatecli/updatecli.
If you find this tool useful, do not hesitate to star our GitHub repository as a sign of appreciation, and/or to tell us directly on our chat!

Co-authored-by: obltmachine * [elastic-agent-autodiscover] Bump the github-actions group across 1 directory with 2 updates (#152) Bumps the github-actions group with 2 updates in the / directory: [actions/checkout](https://github.com/actions/checkout) and [updatecli/updatecli-action](https://github.com/updatecli/updatecli-action). Updates `actions/checkout` from 5 to 6
Release notes

Sourced from actions/checkout's releases.

v6.0.0

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v5.0.0...v6.0.0

v6-beta

What's Changed

Updated persist-credentials to store the credentials under $RUNNER_TEMP instead of directly in the local git config.

This requires a minimum Actions Runner version of v2.329.0 to access the persisted credentials for Docker container action scenarios.

v5.0.1

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v5...v5.0.1

Changelog

Sourced from actions/checkout's changelog.

Changelog

v6.0.2

v6.0.1

v6.0.0

v5.0.1

v5.0.0

v4.3.1

v4.3.0

v4.2.2

v4.2.1

v4.2.0

v4.1.7

v4.1.6

... (truncated)

Commits

Updates `updatecli/updatecli-action` from 2.94.0 to 2.99.0
Release notes

Sourced from updatecli/updatecli-action's releases.

v2.99.0 🌈

Changes

🐛 Bug Fixes

🧰 Maintenance

Contributors

@​olblak, @​updateclibot[bot] and updateclibot[bot]

v2.98.0 🌈

Changes

Contributors

@​updateclibot[bot] and updateclibot[bot]

v2.97.0 🌈

Changes

🧰 Maintenance

... (truncated)

Commits

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * [elastic-agent-autodiscover] Bump go.opentelemetry.io/otel/sdk from 1.28.0 to 1.40.0 (#154) Bumps [go.opentelemetry.io/otel/sdk](https://github.com/open-telemetry/opentelemetry-go) from 1.28.0 to 1.40.0.
Changelog

Sourced from go.opentelemetry.io/otel/sdk's changelog.

[1.40.0/0.62.0/0.16.0] 2026-02-02

Added

  • Add AlwaysRecord sampler in go.opentelemetry.io/otel/sdk/trace. (#7724)
  • Add Enabled method to all synchronous instrument interfaces (Float64Counter, Float64UpDownCounter, Float64Histogram, Float64Gauge, Int64Counter, Int64UpDownCounter, Int64Histogram, Int64Gauge,) in go.opentelemetry.io/otel/metric. This stabilizes the synchronous instrument enabled feature, allowing users to check if an instrument will process measurements before performing computationally expensive operations. (#7763)
  • Add go.opentelemetry.io/otel/semconv/v1.39.0 package. The package contains semantic conventions from the v1.39.0 version of the OpenTelemetry Semantic Conventions. See the migration documentation for information on how to upgrade from go.opentelemetry.io/otel/semconv/v1.38.0. (#7783, #7789)

Changed

  • Improve the concurrent performance of HistogramReservoir in go.opentelemetry.io/otel/sdk/metric/exemplar by 4x. (#7443)
  • Improve the concurrent performance of FixedSizeReservoir in go.opentelemetry.io/otel/sdk/metric/exemplar. (#7447)
  • Improve performance of concurrent histogram measurements in go.opentelemetry.io/otel/sdk/metric. (#7474)
  • Improve performance of concurrent synchronous gauge measurements in go.opentelemetry.io/otel/sdk/metric. (#7478)
  • Add experimental observability metrics in go.opentelemetry.io/otel/exporters/stdout/stdoutmetric. (#7492)
  • Exporter in go.opentelemetry.io/otel/exporters/prometheus ignores metrics with the scope go.opentelemetry.io/contrib/bridges/prometheus. This prevents scrape failures when the Prometheus exporter is misconfigured to get data from the Prometheus bridge. (#7688)
  • Improve performance of concurrent exponential histogram measurements in go.opentelemetry.io/otel/sdk/metric. (#7702)
  • The rpc.grpc.status_code attribute in the experimental metrics emitted from go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc is replaced with the rpc.response.status_code attribute to align with the semantic conventions. (#7854)
  • The rpc.grpc.status_code attribute in the experimental metrics emitted from go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc is replaced with the rpc.response.status_code attribute to align with the semantic conventions. (#7854)

Fixed

  • Fix bad log message when key-value pairs are dropped because of key duplication in go.opentelemetry.io/otel/sdk/log. (#7662)
  • Fix DroppedAttributes on Record in go.opentelemetry.io/otel/sdk/log to not count the non-attribute key-value pairs dropped because of key duplication. (#7662)
  • Fix SetAttributes on Record in go.opentelemetry.io/otel/sdk/log to not log that attributes are dropped when they are actually not dropped. (#7662)
  • Fix missing request.GetBody in go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp to correctly handle HTTP/2 GOAWAY frame. (#7794)
  • WithHostID detector in go.opentelemetry.io/otel/sdk/resource to use full path for ioreg command on Darwin (macOS). (#7818)

Deprecated

[1.39.0/0.61.0/0.15.0/0.0.14] 2025-12-05

Added

  • Greatly reduce the cost of recording metrics in go.opentelemetry.io/otel/sdk/metric using hashing for map keys. (#7175)
  • Add WithInstrumentationAttributeSet option to go.opentelemetry.io/otel/log, go.opentelemetry.io/otel/metric, and go.opentelemetry.io/otel/trace packages. This provides a concurrent-safe and performant alternative to WithInstrumentationAttributes by accepting a pre-constructed attribute.Set. (#7287)
  • Add experimental observability for the Prometheus exporter in go.opentelemetry.io/otel/exporters/prometheus. Check the go.opentelemetry.io/otel/exporters/prometheus/internal/x package documentation for more information. (#7345)
  • Add experimental observability metrics in go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc. (#7353)
  • Add temporality selector functions DeltaTemporalitySelector, CumulativeTemporalitySelector, LowMemoryTemporalitySelector to go.opentelemetry.io/otel/sdk/metric. (#7434)
  • Add experimental observability metrics for simple log processor in go.opentelemetry.io/otel/sdk/log. (#7548)
  • Add experimental observability metrics in go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc. (#7459)

... (truncated)

Commits
  • a3a5317 Release v1.40.0 (#7859)
  • 77785da chore(deps): update github/codeql-action action to v4.32.1 (#7858)
  • 56fa1c2 chore(deps): update module github.com/clipperhouse/uax29/v2 to v2.5.0 (#7857)
  • 298cbed Upgrade semconv use to v1.39.0 (#7854)
  • 3264bf1 refactor: modernize code (#7850)
  • fd5d030 chore(deps): update module github.com/grpc-ecosystem/grpc-gateway/v2 to v2.27...
  • 8d3b4cb chore(deps): update actions/cache action to v5.0.3 (#7847)
  • 91f7cad chore(deps): update github.com/timakin/bodyclose digest to 73d1f95 (#7845)
  • fdad1eb chore(deps): update module github.com/grpc-ecosystem/grpc-gateway/v2 to v2.27...
  • c46d3ba chore(deps): update golang.org/x/telemetry digest to fcf36f6 (#7843)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=go.opentelemetry.io/otel/sdk&package-manager=go_modules&previous-version=1.28.0&new-version=1.40.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself) You can disable automated security fix PRs for this repo from the [Security Alerts page](https://github.com/elastic/elastic-agent-autodiscover/network/alerts).
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Mauri de Souza Meneguzzo * [elastic-agent-autodiscover] Bump updatecli/updatecli-action from 2.99.0 to 2.100.0 in the github-actions group across 1 directory (#153) Bumps the github-actions group with 1 update in the / directory: [updatecli/updatecli-action](https://github.com/updatecli/updatecli-action). Updates `updatecli/updatecli-action` from 2.99.0 to 2.100.0 * [elastic-agent-autodiscover] Bump updatecli/updatecli-action from 2.100.0 to 3.0.0 in the github-actions group across 1 directory (#155) Bumps the github-actions group with 1 update in the / directory: [updatecli/updatecli-action](https://github.com/updatecli/updatecli-action). * [elastic-agent-autodiscover] Bump updatecli/updatecli-action from 3.0.0 to 3.1.1 in the github-actions group across 1 directory (#156) Bumps the github-actions group with 1 update in the / directory: [updatecli/updatecli-action](https://github.com/updatecli/updatecli-action). Updates `updatecli/updatecli-action` from 3.0.0 to 3.1.1
Release notes

Sourced from updatecli/updatecli-action's releases.

v3.1.1 🌈

Changes

🧰 Maintenance

Contributors

@​updateclibot[bot] and updateclibot[bot]

v3.1.0 🌈

Changes

🧰 Maintenance

Contributors

@​olblak, @​updateclibot[bot] and updateclibot[bot]

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=updatecli/updatecli-action&package-manager=github_actions&previous-version=3.0.0&new-version=3.1.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * [elastic-agent-autodiscover] Bump updatecli/updatecli-action from 3.1.1 to 3.1.2 in the github-actions group across 1 directory (#157) Bumps the github-actions group with 1 update in the / directory: [updatecli/updatecli-action](https://github.com/updatecli/updatecli-action). Updates `updatecli/updatecli-action` from 3.1.1 to 3.1.2
Release notes

Sourced from updatecli/updatecli-action's releases.

v3.1.2 🌈

Changes

🧰 Maintenance

Contributors

@​dependabot[bot], @​updateclibot[bot], dependabot[bot] and updateclibot[bot]

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=updatecli/updatecli-action&package-manager=github_actions&previous-version=3.1.1&new-version=3.1.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * [elastic-agent-autodiscover] Migrate from deprecated docker/docker to moby/moby (#158) ## What does this PR do? Migrates the `docker/` package from the deprecated `github.com/docker/docker` Go module to the new split modules: - `github.com/moby/moby/client` v0.4.1 - `github.com/moby/moby/api` v1.54.2 ## Why is it important? The `github.com/docker/docker` Go module is deprecated as of v29.0.0. Security fixes are only available via the new `github.com/moby/moby` module path. ## Checklist - [x] My code follows the style guidelines of this project - [x] I have commented my code, particularly in hard-to-understand areas - [x] I have made corresponding changes to the documentation - [x] I have made corresponding change to the default configuration files - [x] I have added tests that prove my fix is effective or that my feature works - [x] I have added an entry in `CHANGELOG.md` or `CHANGELOG.next.asciidoc` ## :warning: Breaking changes The `docker.Client` interface and `docker.Container.Ports` field type changed to match the new SDK: - `Client.ContainerList` now takes `client.ContainerListOptions` and returns `client.ContainerListResult` - `Client.ContainerInspect` now takes an additional `client.ContainerInspectOptions` param and returns `client.ContainerInspectResult` - `Client.Events` now takes `client.EventsListOptions` and returns `client.EventsResult` (struct, not two channels) - `Container.Ports` changed from `[]container.Port` to `[]container.PortSummary` Downstream consumers that call `NewWatcherWithClient` or reference the `Container` struct will need corresponding updates. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.6 * [elastic-agent-autodiscover] Bump updatecli/updatecli-action from 3.1.2 to 3.1.3 in the github-actions group across 1 directory (#159) Bumps the github-actions group with 1 update in the / directory: [updatecli/updatecli-action](https://github.com/updatecli/updatecli-action). Updates `updatecli/updatecli-action` from 3.1.2 to 3.1.3
Release notes

Sourced from updatecli/updatecli-action's releases.

v3.1.3 🌈

Changes

Contributors

@​updateclibot[bot] and updateclibot[bot]

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=updatecli/updatecli-action&package-manager=github_actions&previous-version=3.1.2&new-version=3.1.3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * [elastic-agent-autodiscover] Bump updatecli/updatecli-action from 3.1.3 to 3.2.0 in the github-actions group across 1 directory (#160) Bumps the github-actions group with 1 update in the / directory: [updatecli/updatecli-action](https://github.com/updatecli/updatecli-action). Updates `updatecli/updatecli-action` from 3.1.3 to 3.2.0
Release notes

Sourced from updatecli/updatecli-action's releases.

v3.2.0 🌈

Changes

🧰 Maintenance

Contributors

@​updateclibot[bot] and updateclibot[bot]

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=updatecli/updatecli-action&package-manager=github_actions&previous-version=3.1.3&new-version=3.2.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * [elastic-agent-autodiscover] Bump updatecli/updatecli-action from 3.2.0 to 3.3.0 in the github-actions group across 1 directory (#161) Bumps the github-actions group with 1 update in the / directory: [updatecli/updatecli-action](https://github.com/updatecli/updatecli-action). Updates `updatecli/updatecli-action` from 3.2.0 to 3.3.0
Release notes

Sourced from updatecli/updatecli-action's releases.

v3.3.0 🌈

Changes

🧰 Maintenance

Contributors

@​updateclibot[bot] and updateclibot[bot]

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=updatecli/updatecli-action&package-manager=github_actions&previous-version=3.2.0&new-version=3.3.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * [elastic-agent-autodiscover] Add exponential backoff to Docker watcher. (#162) When Docker is not available, the add_docker_metadata processor that uses this code can spam the logs with `Error watching for docker events: Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?` messages once per second. Add simple exponential backoff to make the logs less verbose. * [elastic-agent-autodiscover] Bump actions/checkout from 6 to 7 in the github-actions group across 1 directory (#163) Bumps the github-actions group with 1 update in the / directory: [actions/checkout](https://github.com/actions/checkout). Updates `actions/checkout` from 6 to 7
Release notes

Sourced from actions/checkout's releases.

v7.0.0

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v6.0.3...v7.0.0

v6.0.3

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v6...v6.0.3

v6.0.2

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v6.0.1...v6.0.2

v6.0.1

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v6...v6.0.1

Changelog

Sourced from actions/checkout's changelog.

Changelog

v7.0.0

v6.0.3

v6.0.2

v6.0.1

v6.0.0

v5.0.1

v5.0.0

v4.3.1

v4.3.0

v4.2.2

v4.2.1

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/checkout&package-manager=github_actions&previous-version=6&new-version=7)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * [elastic-agent-autodiscover] chore: align Go and linter with Beats repository (#164) Relates https://github.com/elastic/ingest-dev/issues/7710 Brings Go version and linter version and config in line with Beats repository, to make the merging of this repository into Beats easier. Also ran "go fix". See individual commits for easier review. * refactor: remove unneeded files Moved missing deps into root license `overrides.json`, even though they seem to not be used actually used. Still doing this for visibility, removal can be done as a separate step. * ci: Add `pkg/autodiscover` to `CODEOWNERS` * ci: remove elastic-agent-autodiscover repo from allow list in dependabot config * update import paths * make notice * make check * add nolint directives on unmodified code to satisfy linter * add more nolint directives on unmodified code to satisfy linter * chore: fix fixable linter issues in modified files `golangci-lint run --new-from-rev=origin/main --whole-files --fix` * fix linter issues by adding `nolint` directives * fix linter issues * Apply automatic lint fixes where possible with `golangci-lint --fix` * restore missing `nolint` directives * remove unused notice overrides from autodiscover repository --------- Signed-off-by: ChrsMark Signed-off-by: Tetiana Kravchenko Signed-off-by: dependabot[bot] Signed-off-by: Alexandros, Sapranidis Co-authored-by: ChrsMark Co-authored-by: Craig MacKenzie Co-authored-by: Victor Martinez Co-authored-by: apmmachine Co-authored-by: Tetiana Kravchenko Co-authored-by: MichaelKatsoulis Co-authored-by: Denis Rechkunov Co-authored-by: Kyle Pollich Co-authored-by: Michal Pristas Co-authored-by: Denis Rechkunov Co-authored-by: apmmachine Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Alexandros, Sapranidis Co-authored-by: Julien Lind Co-authored-by: Siarhei Harbuz Co-authored-by: Anderson Queiroz Co-authored-by: Anderson Queiroz Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Andrew Gizas Co-authored-by: Svetlana Maltseva <58.svetlana.maltseva@gmail.com> Co-authored-by: Giuseppe Santoro Co-authored-by: Jan Calanog Co-authored-by: obltmachine <150269514+obltmachine@users.noreply.github.com> Co-authored-by: Shaunak Kashyap Co-authored-by: Linh H. Nguyen Co-authored-by: kruskall <99559985+kruskall@users.noreply.github.com> Co-authored-by: Mikołaj Świątek Co-authored-by: Mauri de Souza Meneguzzo Co-authored-by: Yehor Shvedov <146825775+ev1yehor@users.noreply.github.com> Co-authored-by: Olga Naydyonock Co-authored-by: obltmachine Co-authored-by: Khushi Jain Co-authored-by: Vihas Makwana <121151420+VihasMakwana@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 (cherry picked from commit cf032f423c8423de7b5c01cbe6a192697bb1d888) # Conflicts: # filebeat/tests/integration/autodiscover_test.go # heartbeat/autodiscover/builder/hints/monitors_test.go # libbeat/autodiscover/providers/kubernetes/node.go # libbeat/processors/add_kubernetes_metadata/kubernetes.go # x-pack/osquerybeat/ext/osquery-extension/pkg/encoding/encoding_test.go --- .github/CODEOWNERS | 1 + .github/dependabot.yml | 1 - NOTICE.txt | 211 ----- filebeat/autodiscover/builder/hints/logs.go | 4 +- .../autodiscover/builder/hints/logs_test.go | 14 +- .../tests/integration/autodiscover_test.go | 754 ++++++++++++++++ .../translate_ldap_attribute_test.go | 2 +- go.mod | 1 - go.sum | 2 - .../autodiscover/builder/hints/monitors.go | 4 +- .../builder/hints/monitors_test.go | 19 +- libbeat/autodiscover/appender.go | 2 +- libbeat/autodiscover/appender_test.go | 6 +- .../autodiscover/appenders/config/config.go | 8 +- .../appenders/config/config_test.go | 4 +- libbeat/autodiscover/autodiscover.go | 2 +- libbeat/autodiscover/autodiscover_test.go | 2 +- libbeat/autodiscover/builder.go | 2 +- libbeat/autodiscover/builder_test.go | 2 +- libbeat/autodiscover/eventselect.go | 4 +- libbeat/autodiscover/provider.go | 2 +- .../autodiscover/providers/docker/config.go | 2 +- .../autodiscover/providers/docker/docker.go | 6 +- .../docker/docker_integration_test.go | 2 +- .../providers/docker/docker_test.go | 4 +- .../providers/jolokia/discovery.go | 4 +- .../autodiscover/providers/jolokia/jolokia.go | 2 +- .../providers/kubernetes/config.go | 8 +- .../providers/kubernetes/config_test.go | 2 +- .../providers/kubernetes/kubernetes.go | 6 +- .../autodiscover/providers/kubernetes/node.go | 18 +- .../providers/kubernetes/node_test.go | 6 +- .../autodiscover/providers/kubernetes/pod.go | 16 +- .../providers/kubernetes/pod_test.go | 10 +- .../providers/kubernetes/service.go | 16 +- .../providers/kubernetes/service_test.go | 8 +- libbeat/autodiscover/template/config.go | 2 +- libbeat/autodiscover/template/config_test.go | 8 +- .../add_docker_metadata.go | 2 +- .../add_docker_metadata_integration_test.go | 7 +- .../add_docker_metadata_test.go | 4 +- .../processors/add_docker_metadata/config.go | 2 +- .../add_kubernetes_metadata/config.go | 6 +- .../add_kubernetes_metadata/indexers.go | 4 +- .../add_kubernetes_metadata/indexers_test.go | 4 +- .../add_kubernetes_metadata/kubernetes.go | 7 + libbeat/tests/compose/wrapper.go | 2 +- libbeat/tests/docker/docker.go | 2 +- .../appender/kubernetes/token/token.go | 8 +- .../appender/kubernetes/token/token_test.go | 6 +- .../autodiscover/builder/hints/metrics.go | 4 +- .../builder/hints/metrics_test.go | 2 +- metricbeat/module/docker/container/data.go | 2 +- metricbeat/module/docker/docker.go | 2 +- .../docker/event/event_integration_test.go | 2 +- metricbeat/module/docker/helper.go | 2 +- metricbeat/module/docker/helper_test.go | 2 +- metricbeat/module/docker/image/data.go | 2 +- metricbeat/module/kubernetes/event/config.go | 2 +- metricbeat/module/kubernetes/event/event.go | 36 +- .../module/kubernetes/util/kubernetes.go | 44 +- .../module/kubernetes/util/kubernetes_test.go | 36 +- pkg/autodiscover/bus/bus.go | 162 ++++ pkg/autodiscover/bus/bus_test.go | 122 +++ pkg/autodiscover/docker/client.go | 52 ++ pkg/autodiscover/docker/client_test.go | 63 ++ pkg/autodiscover/docker/helpers.go | 60 ++ pkg/autodiscover/docker/watcher.go | 494 ++++++++++ pkg/autodiscover/docker/watcher_test.go | 559 ++++++++++++ pkg/autodiscover/kubernetes/eventhandler.go | 257 ++++++ pkg/autodiscover/kubernetes/informer.go | 334 +++++++ .../k8skeystore/kubernetes_keystore.go | 144 +++ .../k8skeystore/kubernetes_keystore_test.go | 146 +++ .../kubernetes/metadata/config.go | 71 ++ pkg/autodiscover/kubernetes/metadata/job.go | 94 ++ .../kubernetes/metadata/job_test.go | 203 +++++ .../kubernetes/metadata/metadata.go | 202 +++++ .../kubernetes/metadata/namespace.go | 132 +++ .../kubernetes/metadata/namespace_test.go | 173 ++++ pkg/autodiscover/kubernetes/metadata/node.go | 109 +++ .../kubernetes/metadata/node_test.go | 155 ++++ pkg/autodiscover/kubernetes/metadata/pod.go | 159 ++++ .../kubernetes/metadata/pod_test.go | 847 ++++++++++++++++++ .../kubernetes/metadata/replicaset.go | 128 +++ .../kubernetes/metadata/replicaset_test.go | 359 ++++++++ .../kubernetes/metadata/resource.go | 231 +++++ .../kubernetes/metadata/resource_test.go | 366 ++++++++ .../kubernetes/metadata/service.go | 104 +++ .../kubernetes/metadata/service_test.go | 325 +++++++ pkg/autodiscover/kubernetes/options.go | 23 + pkg/autodiscover/kubernetes/types.go | 154 ++++ pkg/autodiscover/kubernetes/util.go | 374 ++++++++ pkg/autodiscover/kubernetes/util_test.go | 334 +++++++ pkg/autodiscover/kubernetes/watcher.go | 375 ++++++++ pkg/autodiscover/kubernetes/watcher_test.go | 167 ++++ pkg/autodiscover/utils/hints.go | 423 +++++++++ pkg/autodiscover/utils/hints_test.go | 515 +++++++++++ pkg/autodiscover/utils/key.go | 26 + .../pipelinemanager/clientLogReader.go | 2 +- x-pack/filebeat/input/streaming/config.go | 2 +- .../providers/aws/ec2/provider.go | 2 +- .../providers/aws/ec2/provider_test.go | 4 +- .../providers/aws/elb/provider.go | 2 +- .../providers/aws/elb/provider_test.go | 4 +- .../providers/aws/test/provider.go | 4 +- .../autodiscover/providers/nomad/nomad.go | 4 +- .../providers/nomad/nomad_test.go | 2 +- .../module/awsfargate/task_stats/container.go | 4 +- .../pkg/encoding/encoding_test.go | 847 ++++++++++++++++++ x-pack/winlogbeat/module/testing.go | 2 +- .../winlogbeat/module/wintest/docker_test.go | 3 +- .../module/wintest/simulate_test.go | 4 +- 112 files changed, 10263 insertions(+), 422 deletions(-) create mode 100644 filebeat/tests/integration/autodiscover_test.go create mode 100644 pkg/autodiscover/bus/bus.go create mode 100644 pkg/autodiscover/bus/bus_test.go create mode 100644 pkg/autodiscover/docker/client.go create mode 100644 pkg/autodiscover/docker/client_test.go create mode 100644 pkg/autodiscover/docker/helpers.go create mode 100644 pkg/autodiscover/docker/watcher.go create mode 100644 pkg/autodiscover/docker/watcher_test.go create mode 100644 pkg/autodiscover/kubernetes/eventhandler.go create mode 100644 pkg/autodiscover/kubernetes/informer.go create mode 100644 pkg/autodiscover/kubernetes/k8skeystore/kubernetes_keystore.go create mode 100644 pkg/autodiscover/kubernetes/k8skeystore/kubernetes_keystore_test.go create mode 100644 pkg/autodiscover/kubernetes/metadata/config.go create mode 100644 pkg/autodiscover/kubernetes/metadata/job.go create mode 100644 pkg/autodiscover/kubernetes/metadata/job_test.go create mode 100644 pkg/autodiscover/kubernetes/metadata/metadata.go create mode 100644 pkg/autodiscover/kubernetes/metadata/namespace.go create mode 100644 pkg/autodiscover/kubernetes/metadata/namespace_test.go create mode 100644 pkg/autodiscover/kubernetes/metadata/node.go create mode 100644 pkg/autodiscover/kubernetes/metadata/node_test.go create mode 100644 pkg/autodiscover/kubernetes/metadata/pod.go create mode 100644 pkg/autodiscover/kubernetes/metadata/pod_test.go create mode 100644 pkg/autodiscover/kubernetes/metadata/replicaset.go create mode 100644 pkg/autodiscover/kubernetes/metadata/replicaset_test.go create mode 100644 pkg/autodiscover/kubernetes/metadata/resource.go create mode 100644 pkg/autodiscover/kubernetes/metadata/resource_test.go create mode 100644 pkg/autodiscover/kubernetes/metadata/service.go create mode 100644 pkg/autodiscover/kubernetes/metadata/service_test.go create mode 100644 pkg/autodiscover/kubernetes/options.go create mode 100644 pkg/autodiscover/kubernetes/types.go create mode 100644 pkg/autodiscover/kubernetes/util.go create mode 100644 pkg/autodiscover/kubernetes/util_test.go create mode 100644 pkg/autodiscover/kubernetes/watcher.go create mode 100644 pkg/autodiscover/kubernetes/watcher_test.go create mode 100644 pkg/autodiscover/utils/hints.go create mode 100644 pkg/autodiscover/utils/hints_test.go create mode 100644 pkg/autodiscover/utils/key.go create mode 100644 x-pack/osquerybeat/ext/osquery-extension/pkg/encoding/encoding_test.go diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index fb9cd2a5a927..5f3495af5a12 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -110,6 +110,7 @@ CHANGELOG* /metricbeat/module/windows/wmi @elastic/obs-infraobs-integrations /metricbeat/module/zookeeper @elastic/obs-infraobs-integrations /packetbeat/ @elastic/sec-linux-platform +/pkg/autodiscover/ @elastic/elastic-agent-data-plane /script/ @elastic/elastic-agent-data-plane /testing/ @elastic/elastic-agent-data-plane /tools/ @elastic/elastic-agent-data-plane diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2b2bc132b1c3..045f84bac207 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -10,7 +10,6 @@ updates: - dependabot allow: # Team:Elastic-Agent-Data-Plane - - dependency-name: github.com/elastic/elastic-agent-autodiscover - dependency-name: github.com/elastic/elastic-agent-client/* - dependency-name: github.com/elastic/elastic-agent-libs - dependency-name: github.com/elastic/elastic-agent-shipper-client diff --git a/NOTICE.txt b/NOTICE.txt index 8ea596ca701a..69d2bad52b98 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -10464,217 +10464,6 @@ various licenses: version 2 (LICENSES/Apache-2.0.txt). --------------------------------------------------------------------------------- -Dependency : github.com/elastic/elastic-agent-autodiscover -Version: v0.10.3-0.20260423154939-e990715f9426 -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.10.3-0.20260423154939-e990715f9426/LICENSE: - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-client/v7 Version: v7.18.1 diff --git a/filebeat/autodiscover/builder/hints/logs.go b/filebeat/autodiscover/builder/hints/logs.go index bdf13535d6a5..47de15194c8e 100644 --- a/filebeat/autodiscover/builder/hints/logs.go +++ b/filebeat/autodiscover/builder/hints/logs.go @@ -23,8 +23,8 @@ import ( "github.com/elastic/go-ucfg" - "github.com/elastic/elastic-agent-autodiscover/bus" - "github.com/elastic/elastic-agent-autodiscover/utils" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/utils" "github.com/elastic/beats/v7/filebeat/fileset" "github.com/elastic/beats/v7/filebeat/harvester" diff --git a/filebeat/autodiscover/builder/hints/logs_test.go b/filebeat/autodiscover/builder/hints/logs_test.go index 760336847021..862a6e10d7b0 100644 --- a/filebeat/autodiscover/builder/hints/logs_test.go +++ b/filebeat/autodiscover/builder/hints/logs_test.go @@ -25,8 +25,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" "github.com/elastic/beats/v7/testing/testutils" - "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp/logptest" @@ -616,7 +616,7 @@ func TestGenerateHints(t *testing.T) { "logs": mapstr.M{ "processors": mapstr.M{ "1": mapstr.M{ - "dissect": mapstr.M{ + "dissect": mapstr.M{ //nolint:gosec // test tokenizer fixture "tokenizer": "%{key1} %{key2}", }, }, @@ -635,7 +635,7 @@ func TestGenerateHints(t *testing.T) { "close_timeout": "true", "processors": []any{ map[string]any{ - "dissect": map[string]any{ + "dissect": map[string]any{ //nolint:gosec // test tokenizer fixture "tokenizer": "%{key1} %{key2}", }, }, @@ -665,7 +665,7 @@ func TestGenerateHints(t *testing.T) { "logs": mapstr.M{ "processors": mapstr.M{ "1": mapstr.M{ - "dissect": mapstr.M{ + "dissect": mapstr.M{ //nolint:gosec // test tokenizer fixture "tokenizer": "%{key1} %{key2}", }, }, @@ -690,7 +690,7 @@ func TestGenerateHints(t *testing.T) { }, }, map[string]any{ - "dissect": map[string]any{ + "dissect": map[string]any{ //nolint:gosec // test tokenizer fixture "tokenizer": "%{key1} %{key2}", }, }, @@ -1194,7 +1194,7 @@ func TestGenerateHintsWithPaths(t *testing.T) { "logs": mapstr.M{ "processors": mapstr.M{ "1": mapstr.M{ - "dissect": mapstr.M{ + "dissect": mapstr.M{ //nolint:gosec // test tokenizer fixture "tokenizer": "%{key1} %{key2}", }, }, @@ -1213,7 +1213,7 @@ func TestGenerateHintsWithPaths(t *testing.T) { "close_timeout": "true", "processors": []any{ map[string]any{ - "dissect": map[string]any{ + "dissect": map[string]any{ //nolint:gosec // test tokenizer fixture "tokenizer": "%{key1} %{key2}", }, }, diff --git a/filebeat/tests/integration/autodiscover_test.go b/filebeat/tests/integration/autodiscover_test.go new file mode 100644 index 000000000000..9259a74f8eca --- /dev/null +++ b/filebeat/tests/integration/autodiscover_test.go @@ -0,0 +1,754 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build integration && !requirefips + +package integration + +import ( + "bufio" + "context" + "fmt" + "io" + "os" + "os/user" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/gofrs/uuid/v5" + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/client" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/kind/pkg/apis/config/v1alpha4" + "sigs.k8s.io/kind/pkg/cluster" + "sigs.k8s.io/kind/pkg/cluster/nodeutils" + + "github.com/elastic/beats/v7/libbeat/tests/integration" + "github.com/elastic/beats/v7/libbeat/version" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/testing/fs" +) + +func TestHintsDocker(t *testing.T) { + containerID := startFlogDocker(t) + filebeat := integration.NewBeat( + t, + "filebeat", + "../../filebeat.test", + ) + + cfgYAML := getConfig(t, nil, "autodiscover", "docker.yml") + filebeat.WriteConfigFile(cfgYAML) + filebeat.Start() + + // By ensuring the Filestream input started with the correct ID, we're + // testing that the whole autodiscover + hints is working as expected. + filebeat.WaitLogsContains( + fmt.Sprintf( + `"message":"Input 'filestream' starting","service.name":"filebeat","id":"container-logs-%s"`, + containerID, + ), + 30*time.Second, + "Filestream did not start for the test container") +} + +func TestHintsKubernetes(t *testing.T) { + filebeat := integration.NewBeat( + t, + "filebeat", + "../../filebeat.test", + ) + + kubeConfigPath, _ := createKindCluster(t, filebeat.TempDir()) + noneName, _, containerID := startFlogKubernetes(t, kubeConfigPath) + + cfgYAML := getConfig( + t, + map[string]any{ + "kubeConfig": kubeConfigPath, + "nodeName": noneName, + }, + "autodiscover", + "k8s.yml") + filebeat.WriteConfigFile(cfgYAML) + filebeat.Start() + + // By ensuring the Filestream input started with the correct ID, we're + // testing that the whole autodiscover + hints is working as expected. + filebeat.WaitLogsContains( + fmt.Sprintf( + `"message":"Input 'filestream' starting","service.name":"filebeat","id":"container-logs-%s"`, + containerID, + ), + 30*time.Second, + "Filestream did not start for the test container") +} + +func TestAutodiscoverFilestreamTakeOverDoesNotReingest(t *testing.T) { + integration.EnsureESIsRunning(t) + filebeatImage := "docker.elastic.co/beats/filebeat-oss-wolfi" + ":" + version.GetDefaultVersion() + "-SNAPSHOT" + + workDir := fs.TempDir(t, "..", "..", "build", "integration-tests") + + kubeConfigPath, clusterName := createKindCluster(t, workDir, + cluster.CreateWithV1Alpha4Config(&v1alpha4.Cluster{ + Nodes: []v1alpha4.Node{ + { + Role: v1alpha4.ControlPlaneRole, + ExtraMounts: []v1alpha4.Mount{ + { + HostPath: workDir, + ContainerPath: workDir, + }, + }, + }, + }, + })) + nodeName, podName, _ := startFlogKubernetes(t, kubeConfigPath) + + loadDockerImageIntoKind(t, clusterName, filebeatImage) + grantClusterAdminToDefaultServiceAccount(t, kubeConfigPath) + + filebeatPodName := "filebeat-pod-" + uuid.Must(uuid.NewV4()).String() + logInputConfigPath := filepath.Join(workDir, "filebeat-log.yml") + filestreamInputConfigPath := filepath.Join(workDir, "filebeat-filestream.yml") + + esURL := integration.GetESAdminURL(t, "http") + esHost := fmt.Sprintf("%s://%s", esURL.Scheme, esURL.Host) + if esURL.Hostname() == "localhost" || esURL.Hostname() == "127.0.0.1" { + esHost = fmt.Sprintf("%s://%s:%s", esURL.Scheme, kindNodeGatewayIP(t, nodeName), esURL.Port()) + } + esUser := esURL.User.Username() + esPass, _ := esURL.User.Password() + + index := fmt.Sprintf("test-autodiscover-take-over-%s", uuid.Must(uuid.NewV4()).String()) + t.Cleanup(func() { + if t.Failed() { + t.Logf("Elasticsearch index used: %q", index) + } + }) + + tmplVars := map[string]any{ + "nodeName": nodeName, + "podName": podName, + "esHost": esHost, + "esUser": esUser, + "esPass": esPass, + "index": index, + } + + writeFile( + t, + logInputConfigPath, + getConfig(t, tmplVars, "autodiscover", "take-over-log-input-k8s.yml"), + ) + + startFilebeatPodForTakeOver( + t, + kubeConfigPath, + nodeName, + filebeatPodName, + filebeatImage, + workDir, + logInputConfigPath, + ) + + // Wait until at least 5 events are ingested + require.Eventually( + t, + func() bool { return countEventsInES(t, index, 1000) >= 5 }, + 30*time.Second, + 200*time.Millisecond, + "did not ingest the initial events") + + deletePodK8s(t, kubeConfigPath, filebeatPodName) + logInputIngested := countEventsInES(t, index, 1000) + + // Re-Start Filebeat with Filestream and take_over enabled + writeFile( + t, + filestreamInputConfigPath, + getConfig(t, tmplVars, "autodiscover", "take-over-filestream-input-k8s.yml"), + ) + + startFilebeatPodForTakeOver( + t, + kubeConfigPath, + nodeName, + filebeatPodName, + filebeatImage, + workDir, + filestreamInputConfigPath, + ) + + // Wait for at least two extra events to be ingested + require.EventuallyWithT( + t, + func(collect *assert.CollectT) { + totalEvents := countEventsInES(t, index, 1000) + if totalEvents <= logInputIngested+2 { + collect.Errorf( + "expecting more events in Elasticsearch than the %d from the first run", + logInputIngested, + ) + } + }, + 10*time.Second, + time.Second, + "No new events ingested") + + // Stop the pod and get the total number of events generated + generatedEvents := stopPodK8sAndCountLogs(t, kubeConfigPath, podName) + + // Wait for Filebeat to fully ingest the file, we do it by waiting the file + // to be closed due to inactivity. + waitFilebeatLogContains( + t, + workDir, + "File is inactive. Closing.", + 20*time.Second, + ) + + // Wait until at least all published events can be found in Elasticsearch + require.Eventuallyf( + t, + func() bool { return countEventsInES(t, index, generatedEvents+100) >= generatedEvents }, + 30*time.Second, + 200*time.Millisecond, + "did not ingest all generated events from pod %s", podName, + ) + + deletePodK8s(t, kubeConfigPath, filebeatPodName) + + totalEventsIngested := countEventsInES(t, index, generatedEvents+100) + + require.Equalf( + t, + generatedEvents, + totalEventsIngested, + "file re-ingestion has occurred\n"+ + "Generated Events: %d\n"+ + "Events ingested by the Log input: %d\n"+ + "Total number of events ingested: %d", + generatedEvents, + logInputIngested, + totalEventsIngested, + ) +} + +func createKindCluster( + t *testing.T, + workDir string, + options ...cluster.CreateOption, +) (kubeConfigPath, clusterName string) { + + uid := uuid.Must(uuid.NewV4()).String() + + defer func() { + if t.Failed() { + t.Log("To see the Kind logs search for 'cluster.ProviderWithLogger' and uncomment it.") + } + }() + + provider := cluster.NewProvider( + // Uncomment the next line to have Kind logs written to stderr. + // You will also have to import "sigs.k8s.io/kind/pkg/cmd" + // cluster.ProviderWithLogger(cmd.NewLogger()), + ) + + clusterName = fmt.Sprintf("test-cluster-%s", uid) + err := provider.Create( + clusterName, + append( + []cluster.CreateOption{cluster.CreateWithWaitForReady(30 * time.Second)}, + options...)..., + ) + if err != nil { + t.Fatalf("could not create cluster: %s", err) + } + t.Cleanup(func() { + if err := provider.Delete(clusterName, ""); err != nil { + t.Logf("could not delete K8s cluster: %s", err) + } + }) + + var kubeConfig string + require.Eventually(t, func() bool { + kubeConfig, err = provider.KubeConfig(clusterName, false) + return err == nil + }, 30*time.Second, 100*time.Millisecond, "could not get kube config") + + kubeConfigPath = filepath.Join(workDir, "kube-config") + if err = os.WriteFile(kubeConfigPath, []byte(kubeConfig), 0666); err != nil { + t.Fatalf("cannot write kube config file: %s", err) + } + + return kubeConfigPath, clusterName +} + +func startFlogKubernetes(t *testing.T, kubeConfigPath string) (nodeName, podName, containerID string) { + clientset := newK8sClientsetFromKubeConfigPath(t, kubeConfigPath) + + podName = "flog-pod-" + uuid.Must(uuid.NewV4()).String() + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: "default", + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "flog", + Image: "mingrammer/flog", + Args: []string{"-s", "0.2", "-d", "0.2", "-l"}, + }, + }, + }, + } + + pod, err := clientset.CoreV1().Pods("default").Create(context.Background(), pod, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("could not create pod: %s", err) + } + + t.Cleanup(func() { + _, err := clientset.CoreV1().Pods("default").Get(context.Background(), pod.Name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + // The pod has already been removed, return + return + } + + // by the time Cleanup runs, t.Context has been cancelled, so we need a new context here + err = clientset.CoreV1().Pods("default").Delete(context.Background(), pod.Name, metav1.DeleteOptions{}) + if err != nil { + t.Logf("could not remove pod: %s", err) + } + }) + + require.Eventually( + t, + func() bool { + pod, err = clientset.CoreV1().Pods("default").Get(t.Context(), pod.Name, metav1.GetOptions{}) + if err != nil { + return false + } + + if pod.Status.Phase == corev1.PodRunning && len(pod.Status.ContainerStatuses) > 0 { + containerID = pod.Status.ContainerStatuses[0].ContainerID + if containerID != "" { + nodeName = pod.Spec.NodeName + // Remove the runtime prefix (e.g., "containerd://") + if idx := strings.Index(containerID, "://"); idx != -1 { + containerID = containerID[idx+3:] + return true + } + return true + } + } + + return false + }, + 60*time.Second, + 100*time.Millisecond, + "pod did not start within timeout", + ) + + return nodeName, podName, containerID +} + +// startFlogDocker starts a `mingrammer/flog` that logs one line every +// second. The container ID is returned and the container is stopped at the +// end of the test. On error the test fails by calling t.Fatalf +func startFlogDocker(t *testing.T) string { + ctx := t.Context() + img := "mingrammer/flog:0.4.3" + cli, err := docker.NewClient(client.DefaultDockerHost, nil, nil, logp.NewNopLogger()) + if err != nil { + t.Fatalf("cannot create Docker client: %s", err) + } + + // Pull the image first + reader, err := cli.ImagePull(ctx, img, client.ImagePullOptions{}) + if err != nil { + t.Fatalf("cannot pull image %q: %s", img, err) + } + defer reader.Close() + + // Wait for the pull to complete by reading the response + _, err = io.Copy(io.Discard, reader) + if err != nil { + t.Fatalf("error while pulling image %q: %s", img, err) + } + + resp, err := cli.ContainerCreate( + ctx, + client.ContainerCreateOptions{ + Config: &container.Config{ + Image: img, + Cmd: []string{"-l", "-d", "1", "-s", "1"}, + }, + }) + if err != nil { + t.Fatalf("cannot create container for %q: %s", img, err) + } + + _, err = cli.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}) + if err != nil { + t.Fatalf("cannot start container: %s", err) + } + + t.Cleanup(func() { + ctx := context.Background() + if _, err := cli.ContainerStop(ctx, resp.ID, client.ContainerStopOptions{}); err != nil { + t.Errorf("cannot stop container: %s", err) + } + if _, err := cli.ContainerRemove(ctx, resp.ID, client.ContainerRemoveOptions{}); err != nil { + t.Errorf("cannot remove container: %s", err) + } + }) + return resp.ID +} + +func loadDockerImageIntoKind(t *testing.T, clusterName, imageName string) { + provider := cluster.NewProvider() + nodes, err := provider.ListInternalNodes(clusterName) + if err != nil { + t.Fatalf( + "cannot list nodes for kind cluster %q: %s", + clusterName, + err, + ) + } + if len(nodes) == 0 { + t.Fatalf("no nodes found for kind cluster %q", clusterName) + } + + cli, err := docker.NewClient(client.DefaultDockerHost, nil, nil, logp.NewNopLogger()) + if err != nil { + t.Fatalf("cannot create Docker client: %s", err) + } + + for _, node := range nodes { + imageReader, err := cli.ImageSave(t.Context(), []string{imageName}) + if err != nil { + t.Fatalf("cannot save image %q from local docker daemon: %s", imageName, err) + } + + if err := nodeutils.LoadImageArchive(node, imageReader); err != nil { + _ = imageReader.Close() + t.Fatalf( + "cannot load image %q into kind node %q: %s", + imageName, + node.String(), + err, + ) + } + + if err := imageReader.Close(); err != nil { + t.Fatalf( + "cannot close image stream for image %q on node %q: %s", + imageName, + node.String(), + err, + ) + } + } +} + +func kindNodeGatewayIP(t *testing.T, nodeName string) string { + cli, err := docker.NewClient(client.DefaultDockerHost, nil, nil, logp.NewNopLogger()) + if err != nil { + t.Fatalf("cannot create Docker client: %s", err) + } + + inspectResult, err := cli.ContainerInspect(context.Background(), nodeName, client.ContainerInspectOptions{}) + if err != nil { + t.Fatalf("cannot inspect Kind node %q: %s", nodeName, err) + } + + if inspectResult.Container.NetworkSettings != nil { + for _, networkSettings := range inspectResult.Container.NetworkSettings.Networks { + if networkSettings != nil && networkSettings.Gateway.IsValid() { + return networkSettings.Gateway.String() + } + } + } + + t.Fatalf("cannot determine gateway IP for Kind node %q", nodeName) + return "" +} + +func grantClusterAdminToDefaultServiceAccount(t *testing.T, kubeConfigPath string) { + cs := newK8sClientsetFromKubeConfigPath(t, kubeConfigPath) + + bindingName := "filebeat-autodiscover-admin-" + uuid.Must(uuid.NewV4()).String() + _, err := cs.RbacV1().ClusterRoleBindings().Create(t.Context(), &rbacv1.ClusterRoleBinding{ + ObjectMeta: metav1.ObjectMeta{ + Name: bindingName, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: "default", + Namespace: "default", + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: "cluster-admin", + }, + }, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("cannot create cluster role binding for default service account: %s", err) + } +} + +func writeFile(t *testing.T, path, content string) { + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("cannot write file %q: %s", path, err) + } +} + +func startFilebeatPodForTakeOver( + t *testing.T, + kubeConfigPath, + nodeName, + podName, + imageName, + workDir, + configPath string, +) { + + cs := newK8sClientsetFromKubeConfigPath(t, kubeConfigPath) + hostPathDir := corev1.HostPathDirectory + + user, err := user.Current() + if err != nil { + t.Fatalf("cannot get current user: %s", err) + } + udi, err := strconv.Atoi(user.Uid) + if err != nil { + t.Fatalf("cannot convert UID from string to integer: %s", err) + } + + // Small hack to make sure the test can read the files generated by the + // container (Filebeat): run Filebeat (the container) with the same UID + // as the current user. That is required for CI + uid := int64(udi) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: "default", + }, + Spec: corev1.PodSpec{ + NodeName: nodeName, + RestartPolicy: corev1.RestartPolicyNever, + Containers: []corev1.Container{ + { + Name: "filebeat", + Image: imageName, + ImagePullPolicy: corev1.PullNever, + Args: []string{ + "--strict.perms=false", + "-c", configPath, + "-E", fmt.Sprintf("path.home=%s", workDir), + }, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &uid, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "home-folder", + MountPath: workDir, + }, + { + Name: "varlogcontainers", + MountPath: "/var/log/containers", + ReadOnly: true, + }, + { + Name: "varlogpods", + MountPath: "/var/log/pods", + ReadOnly: true, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "home-folder", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: workDir, + Type: &hostPathDir, + }, + }, + }, + { + Name: "varlogcontainers", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/var/log/containers", + Type: &hostPathDir, + }, + }, + }, + { + Name: "varlogpods", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/var/log/pods", + Type: &hostPathDir, + }, + }, + }, + }, + }, + } + + if _, err := cs.CoreV1().Pods("default").Create(t.Context(), pod, metav1.CreateOptions{}); err != nil { + t.Fatalf("could not create filebeat pod: %s", err) + } + + require.Eventuallyf( + t, + func() bool { + pod, err := cs.CoreV1().Pods("default").Get(t.Context(), podName, metav1.GetOptions{}) + if err != nil { + return false + } + if pod.Status.Phase == corev1.PodFailed { + t.Logf("filebeat pod failed: %v", pod.Status) + return false + } + return pod.Status.Phase == corev1.PodRunning + }, + 60*time.Second, + 200*time.Millisecond, + "filebeat pod %q did not start", podName, + ) +} + +func waitFilebeatLogContains(t *testing.T, workDir, msg string, timeout time.Duration) { + t.Helper() + // Glob to match the date, it will stop working in about 1000 years + paths, err := filepath.Glob(filepath.Join(workDir, "logs", "filebeat-2*.ndjson")) + if err != nil { + t.Fatalf("cannot resolve glob for log files: %s", err) + } + + if len(paths) != 1 { + t.Fatalf("There must be a single log file for Filebeat, found %d", len(paths)) + } + + f, err := os.Open(paths[0]) + if err != nil { + t.Fatalf("cannot open Filebeat log file: %s", err) + } + defer f.Close() + + logFile := fs.LogFile{File: f} + logFile.WaitLogsContains(t, msg, timeout, "Filebeat logs did not contain '%s'", msg) +} + +func deletePodK8s(t *testing.T, kubeConfigPath, podName string) { + cs := newK8sClientsetFromKubeConfigPath(t, kubeConfigPath) + + err := cs.CoreV1().Pods("default").Delete(context.Background(), podName, metav1.DeleteOptions{}) + if err != nil && !apierrors.IsNotFound(err) { + t.Fatalf("cannot delete pod %q: %s", podName, err) + } + + require.Eventuallyf( + t, + func() bool { + _, err := cs.CoreV1().Pods("default").Get(context.Background(), podName, metav1.GetOptions{}) + return apierrors.IsNotFound(err) + }, + 30*time.Second, + 100*time.Millisecond, + "pod %q was not deleted", podName, + ) +} + +// stopPodK8sAndCountLogs returns the number of log lines generated by the pod +// and then deletes it. +func stopPodK8sAndCountLogs(t *testing.T, kubeConfigPath, podName string) int { + cs := newK8sClientsetFromKubeConfigPath(t, kubeConfigPath) + + logsReader, err := cs.CoreV1().Pods("default").GetLogs(podName, &corev1.PodLogOptions{ + Container: "flog", + Follow: true, + }).Stream(t.Context()) + if err != nil { + t.Fatalf("cannot get logs for pod %q: %s", podName, err) + } + defer logsReader.Close() + + if err := cs.CoreV1().Pods("default").Delete(context.Background(), podName, metav1.DeleteOptions{}); err != nil { + t.Fatalf("cannot delete pod %q: %s", podName, err) + } + + logLines := countReaderLines(t, logsReader) + + return logLines +} + +func newK8sClientsetFromKubeConfigPath(t *testing.T, kubeConfigPath string) *kubernetes.Clientset { + data, err := os.ReadFile(kubeConfigPath) + if err != nil { + t.Fatalf("cannot read kube config: %s", err) + } + config, err := clientcmd.RESTConfigFromKubeConfig(data) + if err != nil { + t.Fatalf("cannot build REST config: %s", err) + } + cs, err := kubernetes.NewForConfig(config) + if err != nil { + t.Fatalf("cannot create clientset: %s", err) + } + return cs +} + +func countEventsInES(t *testing.T, index string, size int) int { + return len(integration.GetEventsMsgFromES(t, index, size)) +} + +func countReaderLines(t *testing.T, r io.Reader) int { + s := bufio.NewScanner(r) + s.Buffer(make([]byte, 0, 64*1024), 1024*1024) + n := 0 + for s.Scan() { + n++ + } + if err := s.Err(); err != nil { + t.Fatalf("cannot scan logs: %s", err) + } + return n +} diff --git a/filebeat/tests/integration/translate_ldap_attribute_test.go b/filebeat/tests/integration/translate_ldap_attribute_test.go index bd2b9de3ce7f..961e8485da19 100644 --- a/filebeat/tests/integration/translate_ldap_attribute_test.go +++ b/filebeat/tests/integration/translate_ldap_attribute_test.go @@ -38,7 +38,7 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/libbeat/tests/integration" - "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/elastic-agent-libs/logp/logptest" ) diff --git a/go.mod b/go.mod index 31f1def46e52..aaaa3f4dbe9d 100644 --- a/go.mod +++ b/go.mod @@ -170,7 +170,6 @@ require ( github.com/dgraph-io/badger/v4 v4.6.0 github.com/elastic/bayeux v1.0.5 github.com/elastic/ebpfevents v0.9.0 - github.com/elastic/elastic-agent-autodiscover v0.10.3-0.20260423154939-e990715f9426 github.com/elastic/elastic-agent-libs v0.46.1 github.com/elastic/elastic-agent-system-metrics v0.14.4 github.com/elastic/go-elasticsearch/v8 v8.19.0 diff --git a/go.sum b/go.sum index 00e57696d464..0042057f5a7b 100644 --- a/go.sum +++ b/go.sum @@ -358,8 +358,6 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/ebpfevents v0.9.0 h1:La6+7+hbnzlM6IOo9edRJfa4qQ6Uo4ALsKRhS2gwBKI= github.com/elastic/ebpfevents v0.9.0/go.mod h1:RDOopYpSomVu55gCO8CPqDMzcU1poDvBeQUEb7pkI2k= -github.com/elastic/elastic-agent-autodiscover v0.10.3-0.20260423154939-e990715f9426 h1:zqt0GOf6/e287nhLlAODfW6Pncydjr4ArEpM589MIXI= -github.com/elastic/elastic-agent-autodiscover v0.10.3-0.20260423154939-e990715f9426/go.mod h1:HyjdiEVP/A7iMN0jeac2EoUohpHKAZIFRDQsvPDg2Dw= github.com/elastic/elastic-agent-client/v7 v7.18.1 h1:WnM53JjaukeysrAuiTyrhDPmFxJG07ZAByc2TrkcKcs= github.com/elastic/elastic-agent-client/v7 v7.18.1/go.mod h1:uDpSGZ+YCKgqgtkwCA0qjwX0gU/wmixDsVbPjY3GkPs= github.com/elastic/elastic-agent-libs v0.46.1 h1:1dVRKOaGWBxqDQPb/NCtoJHI8Nx3QbJOmr4v6y+UdKQ= diff --git a/heartbeat/autodiscover/builder/hints/monitors.go b/heartbeat/autodiscover/builder/hints/monitors.go index 3f31cbf4d123..2b087b1323c0 100644 --- a/heartbeat/autodiscover/builder/hints/monitors.go +++ b/heartbeat/autodiscover/builder/hints/monitors.go @@ -23,8 +23,8 @@ import ( "github.com/elastic/go-ucfg" - "github.com/elastic/elastic-agent-autodiscover/bus" - "github.com/elastic/elastic-agent-autodiscover/utils" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/utils" "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/autodiscover/template" diff --git a/heartbeat/autodiscover/builder/hints/monitors_test.go b/heartbeat/autodiscover/builder/hints/monitors_test.go index 935c11cad762..96124c29434f 100644 --- a/heartbeat/autodiscover/builder/hints/monitors_test.go +++ b/heartbeat/autodiscover/builder/hints/monitors_test.go @@ -23,8 +23,13 @@ import ( "github.com/stretchr/testify/assert" +<<<<<<< HEAD "github.com/elastic/elastic-agent-autodiscover/bus" "github.com/elastic/elastic-agent-libs/logp" +======= + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/elastic-agent-libs/logp/logptest" +>>>>>>> cf032f423 (chore: import autodiscover repository (#51804)) "github.com/elastic/elastic-agent-libs/mapstr" ) @@ -140,9 +145,9 @@ func TestGenerateHints(t *testing.T) { "type": "http", "hosts": []string{"1.2.3.4:9090"}, "schedule": "@every 5s", - "processors": []interface{}{ - map[string]interface{}{ - "add_locale": map[string]interface{}{ + "processors": []any{ + map[string]any{ + "add_locale": map[string]any{ "abbrevation": "MST", }, }, @@ -203,19 +208,19 @@ func TestGenerateHints(t *testing.T) { logger: logp.L(), } cfgs := m.CreateConfig(test.event) - assert.Equal(t, test.len, len(cfgs), test.message) + assert.Len(t, cfgs, test.len, test.message) if len(cfgs) != 0 { config := mapstr.M{} err := cfgs[0].Unpack(&config) - assert.Nil(t, err, test.message) + assert.NoError(t, err, test.message) // Autodiscover can return configs with different sort orders here, which is irrelevant // To make tests pass consistently we sort the host list hostStrs := []string{} - if hostsSlice, ok := config["hosts"].([]interface{}); ok && len(hostsSlice) > 0 { + if hostsSlice, ok := config["hosts"].([]any); ok && len(hostsSlice) > 0 { for _, hi := range hostsSlice { - hostStrs = append(hostStrs, hi.(string)) + hostStrs = append(hostStrs, hi.(string)) //nolint:errcheck // test fixture type } sort.Strings(hostStrs) config["hosts"] = hostStrs diff --git a/libbeat/autodiscover/appender.go b/libbeat/autodiscover/appender.go index a643cdb8b740..91e1d59a2cfe 100644 --- a/libbeat/autodiscover/appender.go +++ b/libbeat/autodiscover/appender.go @@ -21,7 +21,7 @@ import ( "fmt" "strings" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" ) diff --git a/libbeat/autodiscover/appender_test.go b/libbeat/autodiscover/appender_test.go index 6473bb3752d1..ef59cd18c2da 100644 --- a/libbeat/autodiscover/appender_test.go +++ b/libbeat/autodiscover/appender_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" ) @@ -40,7 +40,7 @@ func newFakeAppender(_ *conf.C, _ *logp.Logger) (Appender, error) { func TestAppenderRegistry(t *testing.T) { // Add a new builder reg := NewRegistry() - reg.AddAppender("fake", newFakeAppender) + reg.AddAppender("fake", newFakeAppender) //nolint:errcheck // test registration // Check if that appender is available in registry b := reg.GetAppender("fake") @@ -60,7 +60,7 @@ func TestAppenderRegistry(t *testing.T) { assert.NotNil(t, appender) // Attempt to build using an array of configs - Registry.AddAppender("fake", newFakeAppender) + Registry.AddAppender("fake", newFakeAppender) //nolint:errcheck // test registration cfgs := []*conf.C{cfg} appenders, err := NewAppenders(cfgs) assert.NoError(t, err) diff --git a/libbeat/autodiscover/appenders/config/config.go b/libbeat/autodiscover/appenders/config/config.go index 09e3b89b0bf0..27a8cf2b36b0 100644 --- a/libbeat/autodiscover/appenders/config/config.go +++ b/libbeat/autodiscover/appenders/config/config.go @@ -24,14 +24,14 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover/template" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/libbeat/conditions" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) func init() { - autodiscover.Registry.AddAppender("config", NewConfigAppender) + autodiscover.Registry.AddAppender("config", NewConfigAppender) //nolint:errcheck // provider registration } type config struct { @@ -52,7 +52,7 @@ func NewConfigAppender(cfg *conf.C, logger *logp.Logger) (autodiscover.Appender, config := config{} err := cfg.Unpack(&config) if err != nil { - return nil, fmt.Errorf("unable to unpack config appender due to error: %+v", err) + return nil, fmt.Errorf("unable to unpack config appender due to error: %+w", err) } var cond conditions.Condition @@ -88,7 +88,7 @@ func (c *configAppender) Append(event bus.Event) { if !ok { return } - if c.condition == nil || c.condition.Check(mapstr.M(event)) == true { + if c.condition == nil || c.condition.Check(mapstr.M(event)) { // Merge the template with all the configs for _, cfg := range cfgs { cf := mapstr.M{} diff --git a/libbeat/autodiscover/appenders/config/config_test.go b/libbeat/autodiscover/appenders/config/config_test.go index 375e71c19da3..a0f328a0bd90 100644 --- a/libbeat/autodiscover/appenders/config/config_test.go +++ b/libbeat/autodiscover/appenders/config/config_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp/logptest" "github.com/elastic/elastic-agent-libs/mapstr" @@ -107,7 +107,7 @@ condition.equals: assert.Len(t, cfgs, 1) out := mapstr.M{} - cfgs[0].Unpack(&out) + cfgs[0].Unpack(&out) //nolint:errcheck // test assertion follows assert.Equal(t, out, test.result) }) diff --git a/libbeat/autodiscover/autodiscover.go b/libbeat/autodiscover/autodiscover.go index 6f4e2433f42d..4a36cdb68306 100644 --- a/libbeat/autodiscover/autodiscover.go +++ b/libbeat/autodiscover/autodiscover.go @@ -25,7 +25,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/cfgfile" "github.com/elastic/beats/v7/libbeat/common/reload" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" diff --git a/libbeat/autodiscover/autodiscover_test.go b/libbeat/autodiscover/autodiscover_test.go index b0b7c274c46b..588cb63b8dd9 100644 --- a/libbeat/autodiscover/autodiscover_test.go +++ b/libbeat/autodiscover/autodiscover_test.go @@ -32,7 +32,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/cfgfile" "github.com/elastic/beats/v7/libbeat/tests/resources" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" diff --git a/libbeat/autodiscover/builder.go b/libbeat/autodiscover/builder.go index dfb1e06dc5ec..39293b303822 100644 --- a/libbeat/autodiscover/builder.go +++ b/libbeat/autodiscover/builder.go @@ -22,7 +22,7 @@ import ( "fmt" "strings" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" diff --git a/libbeat/autodiscover/builder_test.go b/libbeat/autodiscover/builder_test.go index 7aa6bf2a0456..d89c5a88b384 100644 --- a/libbeat/autodiscover/builder_test.go +++ b/libbeat/autodiscover/builder_test.go @@ -23,7 +23,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/paths" diff --git a/libbeat/autodiscover/eventselect.go b/libbeat/autodiscover/eventselect.go index e7a05123be00..c7a69e2611cc 100644 --- a/libbeat/autodiscover/eventselect.go +++ b/libbeat/autodiscover/eventselect.go @@ -20,7 +20,7 @@ package autodiscover import ( "fmt" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" "github.com/elastic/elastic-agent-libs/config" ) @@ -47,7 +47,7 @@ func (q queryConfigFrom) CreateConfig(e bus.Event) ([]*config.C, error) { fieldName := string(q) config, ok := e[fieldName].([]*config.C) if !ok { - return nil, fmt.Errorf("Event field '%v' does not contain a valid configuration object", fieldName) + return nil, fmt.Errorf("Event field '%v' does not contain a valid configuration object", fieldName) //nolint:staticcheck // error message format } return config, nil } diff --git a/libbeat/autodiscover/provider.go b/libbeat/autodiscover/provider.go index 60d37cc1ca11..295f1e433dd2 100644 --- a/libbeat/autodiscover/provider.go +++ b/libbeat/autodiscover/provider.go @@ -24,7 +24,7 @@ import ( "github.com/gofrs/uuid/v5" "github.com/elastic/beats/v7/libbeat/cfgfile" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" diff --git a/libbeat/autodiscover/providers/docker/config.go b/libbeat/autodiscover/providers/docker/config.go index 86ab487a15a6..fc1498b4345f 100644 --- a/libbeat/autodiscover/providers/docker/config.go +++ b/libbeat/autodiscover/providers/docker/config.go @@ -23,7 +23,7 @@ import ( "time" "github.com/elastic/beats/v7/libbeat/autodiscover/template" - "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/elastic-agent-libs/config" ) diff --git a/libbeat/autodiscover/providers/docker/docker.go b/libbeat/autodiscover/providers/docker/docker.go index 2adca04992f6..5c96cb087dae 100644 --- a/libbeat/autodiscover/providers/docker/docker.go +++ b/libbeat/autodiscover/providers/docker/docker.go @@ -32,9 +32,9 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover/template" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/elastic-agent-autodiscover/bus" - "github.com/elastic/elastic-agent-autodiscover/docker" - "github.com/elastic/elastic-agent-autodiscover/utils" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/utils" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" diff --git a/libbeat/autodiscover/providers/docker/docker_integration_test.go b/libbeat/autodiscover/providers/docker/docker_integration_test.go index 63f815b911ec..fe2a6df33146 100644 --- a/libbeat/autodiscover/providers/docker/docker_integration_test.go +++ b/libbeat/autodiscover/providers/docker/docker_integration_test.go @@ -28,7 +28,7 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover/template" dk "github.com/elastic/beats/v7/libbeat/tests/docker" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp/logptest" diff --git a/libbeat/autodiscover/providers/docker/docker_test.go b/libbeat/autodiscover/providers/docker/docker_test.go index 309a06f0b676..6211fe22f6f3 100644 --- a/libbeat/autodiscover/providers/docker/docker_test.go +++ b/libbeat/autodiscover/providers/docker/docker_test.go @@ -22,8 +22,8 @@ import ( "github.com/stretchr/testify/assert" - "github.com/elastic/elastic-agent-autodiscover/bus" - "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/libbeat/autodiscover/providers/jolokia/discovery.go b/libbeat/autodiscover/providers/jolokia/discovery.go index 03d2a96e326c..cbac73bff145 100644 --- a/libbeat/autodiscover/providers/jolokia/discovery.go +++ b/libbeat/autodiscover/providers/jolokia/discovery.go @@ -29,7 +29,7 @@ import ( s "github.com/elastic/beats/v7/libbeat/common/schema" c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) @@ -222,7 +222,7 @@ func (d *Discovery) sendProbe(config InterfaceConfig) { wg.Go(func() { - conn, err := net.ListenPacket("udp4", net.JoinHostPort(ip.String(), "0")) + conn, err := net.ListenPacket("udp4", net.JoinHostPort(ip.String(), "0")) //nolint:noctx // legacy UDP discovery if err != nil { log.Error(err.Error()) return diff --git a/libbeat/autodiscover/providers/jolokia/jolokia.go b/libbeat/autodiscover/providers/jolokia/jolokia.go index e35ee42b276b..020b845ccfa5 100644 --- a/libbeat/autodiscover/providers/jolokia/jolokia.go +++ b/libbeat/autodiscover/providers/jolokia/jolokia.go @@ -24,7 +24,7 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/autodiscover/template" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" diff --git a/libbeat/autodiscover/providers/kubernetes/config.go b/libbeat/autodiscover/providers/kubernetes/config.go index 884ad6e810fa..d0cfd3e89369 100644 --- a/libbeat/autodiscover/providers/kubernetes/config.go +++ b/libbeat/autodiscover/providers/kubernetes/config.go @@ -24,8 +24,8 @@ import ( "time" "github.com/elastic/beats/v7/libbeat/autodiscover/template" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/metadata" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" ) @@ -105,7 +105,7 @@ func (c *Config) Validate() error { default: if c.Scope == "node" { - logp.L().Warnf("can not set scope to `node` when using resource %s. resetting scope to `cluster`", c.Resource) + logp.L().Warnf("can not set scope to `node` when using resource %s. resetting scope to `cluster`", c.Resource) //nolint:forbidigo // validation fallback logging } c.Scope = "cluster" } @@ -117,7 +117,7 @@ func (c *Config) Validate() error { return nil } -// checkUnsupportedParams checks if unsupported/deprecated/discouraged paramaters are set and logs a warning +// checkUnsupportedParams checks if unsupported/deprecated/discouraged parameters are set and logs a warning func (c Config) checkUnsupportedParams(logger *logp.Logger) { if c.Unique && c.Scope != "cluster" { logger.Warn("can only set `unique` when scope is `cluster`") diff --git a/libbeat/autodiscover/providers/kubernetes/config_test.go b/libbeat/autodiscover/providers/kubernetes/config_test.go index 2f255996b9f9..818a84df5be4 100644 --- a/libbeat/autodiscover/providers/kubernetes/config_test.go +++ b/libbeat/autodiscover/providers/kubernetes/config_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/libbeat/autodiscover/providers/kubernetes/kubernetes.go b/libbeat/autodiscover/providers/kubernetes/kubernetes.go index c73138693b15..d08c57e1deb0 100644 --- a/libbeat/autodiscover/providers/kubernetes/kubernetes.go +++ b/libbeat/autodiscover/providers/kubernetes/kubernetes.go @@ -33,9 +33,9 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/autodiscover/template" - "github.com/elastic/elastic-agent-autodiscover/bus" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/k8skeystore" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/k8skeystore" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" diff --git a/libbeat/autodiscover/providers/kubernetes/node.go b/libbeat/autodiscover/providers/kubernetes/node.go index 1ad69ea218f4..776bedaa26f0 100644 --- a/libbeat/autodiscover/providers/kubernetes/node.go +++ b/libbeat/autodiscover/providers/kubernetes/node.go @@ -23,16 +23,16 @@ import ( "fmt" "time" - "github.com/elastic/elastic-agent-autodiscover/utils" + "github.com/elastic/beats/v7/pkg/autodiscover/utils" "github.com/gofrs/uuid/v5" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" k8s "k8s.io/client-go/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/bus" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/metadata" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" @@ -147,11 +147,21 @@ func (n *node) GenerateHints(event bus.Event) bus.Event { var kubeMeta mapstr.M rawMeta, ok := event["kubernetes"] if ok { +<<<<<<< HEAD kubeMeta = rawMeta.(mapstr.M) // The builder base config can configure any of the field values of kubernetes if need be. e["kubernetes"] = kubeMeta if rawAnn, ok := kubeMeta["annotations"]; ok { annotations = rawAnn.(mapstr.M) +======= + kubeMeta, ok = rawMeta.(mapstr.M) + if ok { + // The builder base config can configure any of the field values of kubernetes if need be. + e["kubernetes"] = kubeMeta + if rawAnn, ok := kubeMeta["annotations"]; ok { + annotations = rawAnn.(mapstr.M) //nolint:errcheck // type validated by map lookup + } +>>>>>>> cf032f423 (chore: import autodiscover repository (#51804)) } } if host, ok := event["host"]; ok { diff --git a/libbeat/autodiscover/providers/kubernetes/node_test.go b/libbeat/autodiscover/providers/kubernetes/node_test.go index 881cffadba96..cc473f20aa1a 100644 --- a/libbeat/autodiscover/providers/kubernetes/node_test.go +++ b/libbeat/autodiscover/providers/kubernetes/node_test.go @@ -29,9 +29,9 @@ import ( k8sfake "k8s.io/client-go/kubernetes/fake" "github.com/elastic/beats/v7/libbeat/autodiscover/template" - "github.com/elastic/elastic-agent-autodiscover/bus" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/metadata" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp/logptest" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/libbeat/autodiscover/providers/kubernetes/pod.go b/libbeat/autodiscover/providers/kubernetes/pod.go index c356d3996050..95a8ed2b7d97 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod.go +++ b/libbeat/autodiscover/providers/kubernetes/pod.go @@ -29,10 +29,10 @@ import ( "github.com/gofrs/uuid/v5" k8s "k8s.io/client-go/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/bus" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" - "github.com/elastic/elastic-agent-autodiscover/utils" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/metadata" + "github.com/elastic/beats/v7/pkg/autodiscover/utils" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" @@ -215,7 +215,7 @@ func (p *pod) OnAdd(obj any) { defer p.crossUpdate.RUnlock() p.logger.Debugf("Watcher Pod add: %+v", obj) - p.emit(obj.(*kubernetes.Pod), "start") + p.emit(obj.(*kubernetes.Pod), "start") //nolint:errcheck // informer object type is validated } // OnUpdate handles events for pods that have been updated. @@ -228,8 +228,8 @@ func (p *pod) OnUpdate(obj any) { func (p *pod) unlockedUpdate(obj any) { p.logger.Debugf("Watcher Pod update: %+v", obj) - p.emit(obj.(*kubernetes.Pod), "stop") - p.emit(obj.(*kubernetes.Pod), "start") + p.emit(obj.(*kubernetes.Pod), "stop") //nolint:errcheck // informer object type is validated + p.emit(obj.(*kubernetes.Pod), "start") //nolint:errcheck // informer object type is validated } // OnDelete stops pod objects that are deleted. @@ -238,7 +238,7 @@ func (p *pod) OnDelete(obj any) { defer p.crossUpdate.RUnlock() p.logger.Debugf("Watcher Pod delete: %+v", obj) - p.emit(obj.(*kubernetes.Pod), "stop") + p.emit(obj.(*kubernetes.Pod), "stop") //nolint:errcheck // informer object type is validated } // GenerateHints creates hints needed for hints builder. diff --git a/libbeat/autodiscover/providers/kubernetes/pod_test.go b/libbeat/autodiscover/providers/kubernetes/pod_test.go index 560a081d997f..c9a2d2f286d0 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod_test.go +++ b/libbeat/autodiscover/providers/kubernetes/pod_test.go @@ -34,9 +34,9 @@ import ( caches "k8s.io/client-go/tools/cache" "github.com/elastic/beats/v7/libbeat/autodiscover/template" - "github.com/elastic/elastic-agent-autodiscover/bus" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/metadata" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp/logptest" "github.com/elastic/elastic-agent-libs/mapstr" @@ -2219,8 +2219,8 @@ func TestPodEventer_Namespace_Node_Watcher(t *testing.T) { t.Fatal(err) } - namespaceWatcher := eventer.(*pod).namespaceWatcher - nodeWatcher := eventer.(*pod).nodeWatcher + namespaceWatcher := eventer.(*pod).namespaceWatcher //nolint:errcheck // test type assertion + nodeWatcher := eventer.(*pod).nodeWatcher //nolint:errcheck // test type assertion if test.expectedNil { assert.Nilf(t, namespaceWatcher, "Namespace "+test.msg) diff --git a/libbeat/autodiscover/providers/kubernetes/service.go b/libbeat/autodiscover/providers/kubernetes/service.go index 0e84f5255222..5faba57fda01 100644 --- a/libbeat/autodiscover/providers/kubernetes/service.go +++ b/libbeat/autodiscover/providers/kubernetes/service.go @@ -23,14 +23,14 @@ import ( "fmt" "time" - "github.com/elastic/elastic-agent-autodiscover/utils" + "github.com/elastic/beats/v7/pkg/autodiscover/utils" "github.com/gofrs/uuid/v5" k8s "k8s.io/client-go/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/bus" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/metadata" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" @@ -115,12 +115,12 @@ func NewServiceEventer( // OnAdd ensures processing of service objects that are newly created func (s *service) OnAdd(obj any) { s.logger.Debugf("Watcher service add: %+v", obj) - s.emit(obj.(*kubernetes.Service), "start") + s.emit(obj.(*kubernetes.Service), "start") //nolint:errcheck // informer object type is validated } // OnUpdate ensures processing of service objects that are updated func (s *service) OnUpdate(obj any) { - svc := obj.(*kubernetes.Service) + svc := obj.(*kubernetes.Service) //nolint:errcheck // informer object type is validated // Once service is in terminated state, mark it for deletion if svc.GetObjectMeta().GetDeletionTimestamp() != nil { time.AfterFunc(s.config.CleanupTimeout, func() { s.emit(svc, "stop") }) @@ -134,7 +134,7 @@ func (s *service) OnUpdate(obj any) { // OnDelete ensures processing of service objects that are deleted func (s *service) OnDelete(obj any) { s.logger.Debugf("Watcher service delete: %+v", obj) - time.AfterFunc(s.config.CleanupTimeout, func() { s.emit(obj.(*kubernetes.Service), "stop") }) + time.AfterFunc(s.config.CleanupTimeout, func() { s.emit(obj.(*kubernetes.Service), "stop") }) //nolint:errcheck // informer object type is validated } // GenerateHints creates hints needed for hints builder @@ -147,7 +147,7 @@ func (s *service) GenerateHints(event bus.Event) bus.Event { annotations := make(mapstr.M, 0) rawMeta, ok := event["kubernetes"] if ok { - kubeMeta = rawMeta.(mapstr.M) + kubeMeta = rawMeta.(mapstr.M) //nolint:errcheck // type validated by map lookup // The builder base config can configure any of the field values of kubernetes if need be. e["kubernetes"] = kubeMeta if rawAnn, ok := kubeMeta["annotations"]; ok { diff --git a/libbeat/autodiscover/providers/kubernetes/service_test.go b/libbeat/autodiscover/providers/kubernetes/service_test.go index 5f852a834671..1840840ab44b 100644 --- a/libbeat/autodiscover/providers/kubernetes/service_test.go +++ b/libbeat/autodiscover/providers/kubernetes/service_test.go @@ -29,9 +29,9 @@ import ( k8sfake "k8s.io/client-go/kubernetes/fake" "github.com/elastic/beats/v7/libbeat/autodiscover/template" - "github.com/elastic/elastic-agent-autodiscover/bus" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/metadata" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp/logptest" "github.com/elastic/elastic-agent-libs/mapstr" @@ -520,7 +520,7 @@ func TestServiceEventer_NamespaceWatcher(t *testing.T) { t.Fatal(err) } - namespaceWatcher := eventer.(*service).namespaceWatcher + namespaceWatcher := eventer.(*service).namespaceWatcher //nolint:errcheck // test type assertion if test.expectedNil { assert.Nilf(t, namespaceWatcher, test.msg) diff --git a/libbeat/autodiscover/template/config.go b/libbeat/autodiscover/template/config.go index d28fd6d6553d..82fdb1b04d5c 100644 --- a/libbeat/autodiscover/template/config.go +++ b/libbeat/autodiscover/template/config.go @@ -26,7 +26,7 @@ import ( "github.com/elastic/go-ucfg/parse" "github.com/elastic/beats/v7/libbeat/conditions" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" ) diff --git a/libbeat/autodiscover/template/config_test.go b/libbeat/autodiscover/template/config_test.go index e90cdfcd0ddc..ab09fea70d79 100644 --- a/libbeat/autodiscover/template/config_test.go +++ b/libbeat/autodiscover/template/config_test.go @@ -24,7 +24,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" @@ -33,7 +33,7 @@ import ( ) func TestConfigsMapping(t *testing.T) { - logp.TestingSetup() + logp.TestingSetup() //nolint:staticcheck // legacy test logger setup config, _ := conf.NewConfigFrom(map[string]any{ "correct": "config", @@ -331,8 +331,8 @@ func createAnExistingKeystore(path string, secret string) keystore.Keystore { panic(err) } - writableKeystore.Store("PASSWORD", []byte(secret)) - writableKeystore.Save() + writableKeystore.Store("PASSWORD", []byte(secret)) //nolint:errcheck // test helper + writableKeystore.Save() //nolint:errcheck // test helper return keyStore } diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata.go b/libbeat/processors/add_docker_metadata/add_docker_metadata.go index 0131a951ca4c..24131c1763af 100644 --- a/libbeat/processors/add_docker_metadata/add_docker_metadata.go +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata.go @@ -34,7 +34,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/processors/actions" - "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata_integration_test.go b/libbeat/processors/add_docker_metadata/add_docker_metadata_integration_test.go index d88c3f9d21db..e2b28543447a 100644 --- a/libbeat/processors/add_docker_metadata/add_docker_metadata_integration_test.go +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata_integration_test.go @@ -30,7 +30,7 @@ import ( "github.com/elastic/beats/v7/libbeat/processors" dockertest "github.com/elastic/beats/v7/libbeat/tests/docker" "github.com/elastic/beats/v7/libbeat/tests/resources" - "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/logp/logptest" @@ -61,7 +61,7 @@ func TestAddDockerMetadata(t *testing.T) { labels := map[string]string{"label": "foo"} id, err := testClient.ContainerStart(image, cmd, labels) require.NoError(t, err) - defer testClient.ContainerRemove(id) + defer testClient.ContainerRemove(id) //nolint:errcheck // test cleanup info, err := testClient.ContainerInspect(id) require.NoError(t, err) @@ -70,8 +70,9 @@ func TestAddDockerMetadata(t *testing.T) { config, err := config.NewConfigFrom(map[string]any{ "match_fields": []string{"cid"}, }) + require.NoError(t, err) watcherConstructor := newWatcherWith(client) - processor, err := buildDockerMetadataProcessor(logp.L(), config, watcherConstructor) + processor, err := buildDockerMetadataProcessor(logp.L(), config, watcherConstructor) //nolint:forbidigo // integration test setup require.NoError(t, err) t.Run("match container by container id", func(t *testing.T) { diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go b/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go index 1d854d4579f3..1445ef0726db 100644 --- a/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go @@ -34,8 +34,8 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/tests/resources" - "github.com/elastic/elastic-agent-autodiscover/bus" - "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/logp/logptest" diff --git a/libbeat/processors/add_docker_metadata/config.go b/libbeat/processors/add_docker_metadata/config.go index ea79bdbc9faf..0e928ce9aa1f 100644 --- a/libbeat/processors/add_docker_metadata/config.go +++ b/libbeat/processors/add_docker_metadata/config.go @@ -22,7 +22,7 @@ package add_docker_metadata import ( "time" - "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" ) // Config for docker processor. diff --git a/libbeat/processors/add_kubernetes_metadata/config.go b/libbeat/processors/add_kubernetes_metadata/config.go index aaf219f8642b..902c9891b094 100644 --- a/libbeat/processors/add_kubernetes_metadata/config.go +++ b/libbeat/processors/add_kubernetes_metadata/config.go @@ -21,8 +21,8 @@ import ( "fmt" "time" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/metadata" "github.com/elastic/elastic-agent-libs/config" ) @@ -90,7 +90,7 @@ func (k *kubeAnnotatorConfig) Validate() error { if logsPathMatcher.ResourceType != "pod" && logsPathMatcher.ResourceType != "container" { return fmt.Errorf("invalid resource_type %s, valid values include `pod`, `container`", logsPathMatcher.ResourceType) } - if logsPathMatcher.ResourceType == "pod" && !(logsPathMatcher.LogsPath == "/var/lib/kubelet/pods/" || logsPathMatcher.LogsPath == "/var/log/pods/") { + if logsPathMatcher.ResourceType == "pod" && (logsPathMatcher.LogsPath != "/var/lib/kubelet/pods/" && logsPathMatcher.LogsPath != "/var/log/pods/") { return fmt.Errorf("invalid logs_path defined for resource_type: %s, valid values include `/var/lib/kubelet/pods/`, `/var/log/pods/`", logsPathMatcher.ResourceType) } if logsPathMatcher.ResourceType == "container" && logsPathMatcher.LogsPath != "/var/log/containers/" { diff --git a/libbeat/processors/add_kubernetes_metadata/indexers.go b/libbeat/processors/add_kubernetes_metadata/indexers.go index a60ee2e21ea7..a44ef12474d2 100644 --- a/libbeat/processors/add_kubernetes_metadata/indexers.go +++ b/libbeat/processors/add_kubernetes_metadata/indexers.go @@ -22,8 +22,8 @@ import ( "net" "strconv" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/metadata" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/libbeat/processors/add_kubernetes_metadata/indexers_test.go b/libbeat/processors/add_kubernetes_metadata/indexers_test.go index 43d31a9d3611..d6dd420c1e1b 100644 --- a/libbeat/processors/add_kubernetes_metadata/indexers_test.go +++ b/libbeat/processors/add_kubernetes_metadata/indexers_test.go @@ -23,8 +23,8 @@ import ( "strconv" "testing" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/metadata" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/libbeat/processors/add_kubernetes_metadata/kubernetes.go b/libbeat/processors/add_kubernetes_metadata/kubernetes.go index e66e5c72aaaa..d1d661e8a22e 100644 --- a/libbeat/processors/add_kubernetes_metadata/kubernetes.go +++ b/libbeat/processors/add_kubernetes_metadata/kubernetes.go @@ -30,8 +30,13 @@ import ( k8sclient "k8s.io/client-go/kubernetes" +<<<<<<< HEAD "github.com/elastic/elastic-agent-autodiscover/kubernetes" "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" +======= + "go.opentelemetry.io/collector/pdata/pcommon" + +>>>>>>> cf032f423 (chore: import autodiscover repository (#51804)) "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -39,6 +44,8 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/processors/shared" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/metadata" ) const ( diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go index f9b013f5f27b..166d0e57d800 100644 --- a/libbeat/tests/compose/wrapper.go +++ b/libbeat/tests/compose/wrapper.go @@ -37,7 +37,7 @@ import ( "github.com/moby/moby/api/types/container" dockerclient "github.com/moby/moby/client" - "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/elastic-agent-libs/logp" ) diff --git a/libbeat/tests/docker/docker.go b/libbeat/tests/docker/docker.go index d2c88af10a82..c4d6f70a190f 100644 --- a/libbeat/tests/docker/docker.go +++ b/libbeat/tests/docker/docker.go @@ -23,7 +23,7 @@ import ( "io" "io/ioutil" - "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/elastic-agent-libs/logp" "github.com/moby/moby/api/types/container" diff --git a/metricbeat/autodiscover/appender/kubernetes/token/token.go b/metricbeat/autodiscover/appender/kubernetes/token/token.go index 09f06203fd6f..19db90e8fbca 100644 --- a/metricbeat/autodiscover/appender/kubernetes/token/token.go +++ b/metricbeat/autodiscover/appender/kubernetes/token/token.go @@ -24,7 +24,7 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/libbeat/conditions" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -52,7 +52,7 @@ func NewTokenAppender(cfg *conf.C, logger *logp.Logger) (autodiscover.Appender, err := cfg.Unpack(&conf) if err != nil { - return nil, fmt.Errorf("unable to unpack config due to error: %v", err) + return nil, fmt.Errorf("unable to unpack config due to error: %w", err) } var cond conditions.Condition @@ -60,7 +60,7 @@ func NewTokenAppender(cfg *conf.C, logger *logp.Logger) (autodiscover.Appender, // Attempt to create a condition. If fails then report error cond, err = conditions.NewCondition(conf.ConditionConfig, logger) if err != nil { - return nil, fmt.Errorf("unable to create condition due to error: %v", err) + return nil, fmt.Errorf("unable to create condition due to error: %w", err) } } appender := tokenAppender{ @@ -87,7 +87,7 @@ func (t *tokenAppender) Append(event bus.Event) { } // Check if the condition is met. Attempt to append only if that is the case. - if t.Condition == nil || t.Condition.Check(mapstr.M(event)) == true { + if t.Condition == nil || t.Condition.Check(mapstr.M(event)) { tok := t.getAuthHeaderFromToken() // If token is empty then just return if tok == "" { diff --git a/metricbeat/autodiscover/appender/kubernetes/token/token_test.go b/metricbeat/autodiscover/appender/kubernetes/token/token_test.go index f4d27a554bdd..7b14f3208efe 100644 --- a/metricbeat/autodiscover/appender/kubernetes/token/token_test.go +++ b/metricbeat/autodiscover/appender/kubernetes/token/token_test.go @@ -23,7 +23,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp/logptest" "github.com/elastic/elastic-agent-libs/mapstr" @@ -99,7 +99,7 @@ token_path: "test" assert.Len(t, cfgs, 1) out := mapstr.M{} - cfgs[0].Unpack(&out) + cfgs[0].Unpack(&out) //nolint:errcheck // test assertion follows assert.Equal(t, out, test.result) deleteFile("test") @@ -107,7 +107,7 @@ token_path: "test" } func writeFile(name, message string) { - os.WriteFile(name, []byte(message), os.ModePerm) + os.WriteFile(name, []byte(message), os.ModePerm) //nolint:errcheck // test helper } func deleteFile(name string) { diff --git a/metricbeat/autodiscover/builder/hints/metrics.go b/metricbeat/autodiscover/builder/hints/metrics.go index 9d56b334b836..60ae6f7a8cbf 100644 --- a/metricbeat/autodiscover/builder/hints/metrics.go +++ b/metricbeat/autodiscover/builder/hints/metrics.go @@ -29,8 +29,8 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/elastic-agent-libs/paths" - "github.com/elastic/elastic-agent-autodiscover/bus" - "github.com/elastic/elastic-agent-autodiscover/utils" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/utils" "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/autodiscover/template" diff --git a/metricbeat/autodiscover/builder/hints/metrics_test.go b/metricbeat/autodiscover/builder/hints/metrics_test.go index f9d771ccf893..7011afb357be 100644 --- a/metricbeat/autodiscover/builder/hints/metrics_test.go +++ b/metricbeat/autodiscover/builder/hints/metrics_test.go @@ -26,8 +26,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" "github.com/elastic/beats/v7/testing/testutils" - "github.com/elastic/elastic-agent-autodiscover/bus" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp/logptest" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/metricbeat/module/docker/container/data.go b/metricbeat/module/docker/container/data.go index effaa75c24cc..42f69be5b09d 100644 --- a/metricbeat/module/docker/container/data.go +++ b/metricbeat/module/docker/container/data.go @@ -25,7 +25,7 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover/providers/kubernetes" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" - "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/metricbeat/module/docker/docker.go b/metricbeat/module/docker/docker.go index 45e470c8f798..88d07a6bb457 100644 --- a/metricbeat/module/docker/docker.go +++ b/metricbeat/module/docker/docker.go @@ -32,7 +32,7 @@ import ( "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" - "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/elastic-agent-libs/logp" ) diff --git a/metricbeat/module/docker/event/event_integration_test.go b/metricbeat/module/docker/event/event_integration_test.go index d418a7403620..80a3349822f7 100644 --- a/metricbeat/module/docker/event/event_integration_test.go +++ b/metricbeat/module/docker/event/event_integration_test.go @@ -32,7 +32,7 @@ import ( "github.com/elastic/beats/v7/auditbeat/core" "github.com/elastic/beats/v7/metricbeat/mb" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" - "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/elastic-agent-libs/logp/logptest" ) diff --git a/metricbeat/module/docker/helper.go b/metricbeat/module/docker/helper.go index 5003384e1212..04c4085df7c5 100644 --- a/metricbeat/module/docker/helper.go +++ b/metricbeat/module/docker/helper.go @@ -20,7 +20,7 @@ package docker import ( "github.com/moby/moby/api/types/container" - helpers "github.com/elastic/elastic-agent-autodiscover/docker" + helpers "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/metricbeat/module/docker/helper_test.go b/metricbeat/module/docker/helper_test.go index 0c9238f7506f..e58708fdbeca 100644 --- a/metricbeat/module/docker/helper_test.go +++ b/metricbeat/module/docker/helper_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/assert" - helpers "github.com/elastic/elastic-agent-autodiscover/docker" + helpers "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/metricbeat/module/docker/image/data.go b/metricbeat/module/docker/image/data.go index cc2f9d9bdb91..c030efdfca61 100644 --- a/metricbeat/module/docker/image/data.go +++ b/metricbeat/module/docker/image/data.go @@ -23,7 +23,7 @@ import ( "github.com/moby/moby/api/types/image" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/metricbeat/module/kubernetes/event/config.go b/metricbeat/module/kubernetes/event/config.go index cc62c3395c7e..84728ff17b22 100644 --- a/metricbeat/module/kubernetes/event/config.go +++ b/metricbeat/module/kubernetes/event/config.go @@ -20,7 +20,7 @@ package event import ( "time" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" ) type kubeEventsConfig struct { diff --git a/metricbeat/module/kubernetes/event/event.go b/metricbeat/module/kubernetes/event/event.go index 97f2deb23f56..f3405066486c 100644 --- a/metricbeat/module/kubernetes/event/event.go +++ b/metricbeat/module/kubernetes/event/event.go @@ -25,7 +25,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/kubernetes/util" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -114,17 +114,17 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { func (m *MetricSet) Run(reporter mb.PushReporterV2) { now := time.Now() handler := kubernetes.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { + AddFunc: func(obj any) { m.reportEvent(obj, reporter) }, - UpdateFunc: func(obj interface{}) { + UpdateFunc: func(obj any) { m.reportEvent(obj, reporter) }, // ignore events that are deleted DeleteFunc: nil, } m.watcher.AddEventHandler(kubernetes.FilteringResourceEventHandler{ - FilterFunc: func(obj interface{}) bool { + FilterFunc: func(obj any) bool { eve, ok := obj.(*kubernetes.Event) if !ok { m.Logger().Debugf("Error while casting event. Got type: %T", obj) @@ -154,8 +154,8 @@ func (m *MetricSet) Run(reporter mb.PushReporterV2) { m.watcher.Stop() } -func (m *MetricSet) reportEvent(obj interface{}, reporter mb.PushReporterV2) { - mapStrEvent := generateMapStrFromEvent(obj.(*kubernetes.Event), m.dedotConfig, m.Logger()) +func (m *MetricSet) reportEvent(obj any, reporter mb.PushReporterV2) { + mapStrEvent := generateMapStrFromEvent(obj.(*kubernetes.Event), m.dedotConfig, m.Logger()) //nolint:errcheck // informer object type is validated event := mb.TransformMapStrToEvent("kubernetes", mapStrEvent, nil) if m.clusterMeta != nil { event.RootFields.DeepUpdate(m.clusterMeta) @@ -168,17 +168,17 @@ func generateMapStrFromEvent(eve *kubernetes.Event, dedotConfig dedotConfig, log "timestamp": mapstr.M{ "created": kubernetes.Time(&eve.ObjectMeta.CreationTimestamp).UTC(), }, - "name": eve.ObjectMeta.GetName(), - "namespace": eve.ObjectMeta.GetNamespace(), - "self_link": eve.ObjectMeta.GetSelfLink(), - "generate_name": eve.ObjectMeta.GetGenerateName(), - "uid": eve.ObjectMeta.GetUID(), - "resource_version": eve.ObjectMeta.GetResourceVersion(), + "name": eve.GetName(), + "namespace": eve.GetNamespace(), + "self_link": eve.GetSelfLink(), + "generate_name": eve.GetGenerateName(), + "uid": eve.GetUID(), + "resource_version": eve.GetResourceVersion(), } - if len(eve.ObjectMeta.Labels) != 0 { - labels := make(mapstr.M, len(eve.ObjectMeta.Labels)) - for k, v := range eve.ObjectMeta.Labels { + if len(eve.Labels) != 0 { + labels := make(mapstr.M, len(eve.Labels)) + for k, v := range eve.Labels { if dedotConfig.LabelsDedot { label := common.DeDot(k) kubernetes2.ShouldPut(labels, label, v, logger) @@ -194,9 +194,9 @@ func generateMapStrFromEvent(eve *kubernetes.Event, dedotConfig dedotConfig, log eventMeta["labels"] = labels } - if len(eve.ObjectMeta.Annotations) != 0 { - annotations := make(mapstr.M, len(eve.ObjectMeta.Annotations)) - for k, v := range eve.ObjectMeta.Annotations { + if len(eve.Annotations) != 0 { + annotations := make(mapstr.M, len(eve.Annotations)) + for k, v := range eve.Annotations { if dedotConfig.AnnotationsDedot { annotation := common.DeDot(k) kubernetes2.ShouldPut(annotations, annotation, v, logger) diff --git a/metricbeat/module/kubernetes/util/kubernetes.go b/metricbeat/module/kubernetes/util/kubernetes.go index 720fabb8e527..1eb344bed36c 100644 --- a/metricbeat/module/kubernetes/util/kubernetes.go +++ b/metricbeat/module/kubernetes/util/kubernetes.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "maps" + "slices" "strings" "sync" "time" @@ -35,8 +36,8 @@ import ( "k8s.io/apimachinery/pkg/api/meta" k8sresource "k8s.io/apimachinery/pkg/api/resource" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/metadata" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -326,10 +327,8 @@ func createWatcher( ) (bool, error) { // We need to check the node scope to decide on whether a watcher should be updated or not. - nodeScope := false - if options.Node != "" { - nodeScope = true - } + nodeScope := options.Node != "" + // The nodescope for extra watchers node, namespace, replicaset and job should be always false. if extraWatcher { nodeScope = false @@ -466,7 +465,7 @@ func addEventHandlersToWatcher( nodeStore.SetNodeMetrics(metrics) } - clearMetadataCacheFunc := func(obj interface{}) { + clearMetadataCacheFunc := func(obj any) { enrichers := make(map[string]*enricher, len(metaWatcher.enrichers)) resourceWatchers.lock.Lock() @@ -475,7 +474,7 @@ func addEventHandlersToWatcher( for _, enricher := range enrichers { enricher.Lock() - ids := enricher.deleteFunc(obj.(kubernetes.Resource)) + ids := enricher.deleteFunc(obj.(kubernetes.Resource)) //nolint:errcheck // handler type is validated by informer // update this watcher events by removing all the metadata[id] for _, id := range ids { delete(enricher.metadataCache, id) @@ -485,7 +484,7 @@ func addEventHandlersToWatcher( } metaWatcher.watcher.AddEventHandler(kubernetes.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { + AddFunc: func(obj any) { switch res := obj.(type) { case *kubernetes.Pod: containerMetricsUpdateFunc(res) @@ -493,7 +492,7 @@ func addEventHandlersToWatcher( nodeMetricsUpdateFunc(res) } }, - UpdateFunc: func(obj interface{}) { + UpdateFunc: func(obj any) { clearMetadataCacheFunc(obj) switch res := obj.(type) { case *kubernetes.Pod: @@ -502,7 +501,7 @@ func addEventHandlersToWatcher( nodeMetricsUpdateFunc(res) } }, - DeleteFunc: func(obj interface{}) { + DeleteFunc: func(obj any) { clearMetadataCacheFunc(obj) switch res := obj.(type) { case *kubernetes.Pod: @@ -523,13 +522,7 @@ func addToMetricsetsUsing(resourceName string, metricsetUsing string, resourceWa data, ok := resourceWatchers.metaWatchersMap[resourceName] if ok { - contains := false - for _, which := range data.metricsetsUsing { - if which == metricsetUsing { - contains = true - break - } - } + contains := slices.Contains(data.metricsetsUsing, metricsetUsing) // add this resource to the list of resources using it if !contains { data.metricsetsUsing = append(data.metricsetsUsing, metricsetUsing) @@ -662,7 +655,8 @@ func createMetadataGenSpecific(client k8sclient.Interface, commonConfig *conf.C, } var metaGen metadata.MetaGen - if resourceName == PodResource { + switch resourceName { + case PodResource: var nodeWatcher kubernetes.Watcher if nodeMetaWatcher := resourceWatchers.metaWatchersMap[NodeResource]; nodeMetaWatcher != nil { nodeWatcher = (*nodeMetaWatcher).watcher @@ -684,7 +678,7 @@ func createMetadataGenSpecific(client k8sclient.Interface, commonConfig *conf.C, metaGen = metadata.GetPodMetaGen(commonConfig, mainWatcher, nodeWatcher, namespaceWatcher, replicaSetWatcher, jobWatcher, addResourceMetadata) return metaGen, nil - } else if resourceName == ServiceResource { + case ServiceResource: namespaceMetaWatcher := resourceWatchers.metaWatchersMap[NamespaceResource] if namespaceMetaWatcher == nil { return nil, fmt.Errorf("could not create the metadata generator, as the watcher for namespace does not exist") @@ -914,7 +908,7 @@ func NewContainerMetadataEnricher( } for _, container := range append(pod.Spec.Containers, pod.Spec.InitContainers...) { - id := join(pod.ObjectMeta.GetNamespace(), pod.GetObjectMeta().GetName(), container.Name) + id := join(pod.GetNamespace(), pod.GetObjectMeta().GetName(), container.Name) ids = append(ids, id) } @@ -1164,10 +1158,8 @@ func (e *enricher) getMetadata(event mapstr.M) mapstr.M { func (e *enricher) updateMetadataCacheFromWatcher(key string) { storeKey := getWatcherStoreKeyFromMetadataKey(key) if res, exists, _ := e.watcher.watcher.Store().GetByKey(storeKey); exists { - eventMetaMap := e.updateFunc(res.(kubernetes.Resource)) - for k, v := range eventMetaMap { - e.metadataCache[k] = v - } + eventMetaMap := e.updateFunc(res.(kubernetes.Resource)) //nolint:errcheck // store object type is validated by watcher + maps.Copy(e.metadataCache, eventMetaMap) } } @@ -1257,7 +1249,7 @@ func AddClusterECSMeta(base mb.BaseMetricSet) mapstr.M { // transformReplicaSetMetadata ensures that the PartialObjectMetadata resources we get from a metadata watcher // can be correctly interpreted by the update function returned by getEventMetadataFunc. // This really just involves adding missing type information. -func transformReplicaSetMetadata(obj interface{}) (interface{}, error) { +func transformReplicaSetMetadata(obj any) (any, error) { old, ok := obj.(*metav1.PartialObjectMetadata) if !ok { return nil, fmt.Errorf("obj of type %T neither a ReplicaSet nor a PartialObjectMetadata", obj) diff --git a/metricbeat/module/kubernetes/util/kubernetes_test.go b/metricbeat/module/kubernetes/util/kubernetes_test.go index bcf556e05b87..f5251e1c8e72 100644 --- a/metricbeat/module/kubernetes/util/kubernetes_test.go +++ b/metricbeat/module/kubernetes/util/kubernetes_test.go @@ -38,7 +38,7 @@ import ( "k8s.io/client-go/tools/cache" kubernetes2 "github.com/elastic/beats/v7/libbeat/autodiscover/providers/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/metadata" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/logp/logptest" @@ -48,7 +48,7 @@ import ( k8sfake "k8s.io/client-go/kubernetes/fake" k8smetafake "k8s.io/client-go/metadata/fake" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" ) func TestWatchOptions(t *testing.T) { @@ -102,7 +102,7 @@ func TestCreateWatcher(t *testing.T) { require.NoError(t, err) resourceWatchers.lock.Lock() - require.Equal(t, 1, len(resourceWatchers.metaWatchersMap)) + require.Len(t, resourceWatchers.metaWatchersMap, 1) require.NotNil(t, resourceWatchers.metaWatchersMap[NamespaceResource]) require.NotNil(t, resourceWatchers.metaWatchersMap[NamespaceResource].watcher) resourceWatchers.lock.Unlock() @@ -120,7 +120,7 @@ func TestCreateWatcher(t *testing.T) { require.NoError(t, err) resourceWatchers.lock.Lock() - require.Equal(t, 1, len(resourceWatchers.metaWatchersMap)) + require.Len(t, resourceWatchers.metaWatchersMap, 1) require.NotNil(t, resourceWatchers.metaWatchersMap[NamespaceResource]) require.NotNil(t, resourceWatchers.metaWatchersMap[NamespaceResource].watcher) resourceWatchers.lock.Unlock() @@ -138,7 +138,7 @@ func TestCreateWatcher(t *testing.T) { require.NoError(t, err) resourceWatchers.lock.Lock() - require.Equal(t, 2, len(resourceWatchers.metaWatchersMap)) + require.Len(t, resourceWatchers.metaWatchersMap, 2) require.NotNil(t, resourceWatchers.metaWatchersMap[DeploymentResource]) require.NotNil(t, resourceWatchers.metaWatchersMap[NamespaceResource]) resourceWatchers.lock.Unlock() @@ -294,15 +294,15 @@ func TestWatcherContainerMetrics(t *testing.T) { containerStore := metricsRepo.GetNodeStore(pod.Spec.NodeName).GetPodStore(podId).GetContainerStore(containerName) metrics := containerStore.GetContainerMetrics() require.NotNil(t, metrics) - assert.Equal(t, 0.1, metrics.CoresLimit.Value) - assert.Equal(t, 100*1024*1024.0, metrics.MemoryLimit.Value) + assert.Equal(t, 0.1, metrics.CoresLimit.Value) //nolint:testifylint // exact float comparison + assert.Equal(t, 100*1024*1024.0, metrics.MemoryLimit.Value) //nolint:testifylint // exact float comparison // modify the limit and verify the new value is present pod.Spec.Containers[0].Resources.Limits[v1.ResourceCPU] = resource.MustParse("200m") watcher.handler.OnUpdate(pod) metrics = containerStore.GetContainerMetrics() require.NotNil(t, metrics) - assert.Equal(t, 0.2, metrics.CoresLimit.Value) + assert.Equal(t, 0.2, metrics.CoresLimit.Value) //nolint:testifylint // exact float comparison // delete the pod and verify no metrics are present watcher.handler.OnDelete(pod) @@ -355,15 +355,15 @@ func TestWatcherNodeMetrics(t *testing.T) { nodeStore := metricsRepo.GetNodeStore(node.Name) metrics := nodeStore.GetNodeMetrics() require.NotNil(t, metrics) - assert.Equal(t, 0.1, metrics.CoresAllocatable.Value) - assert.Equal(t, 100*1024*1024.0, metrics.MemoryAllocatable.Value) + assert.Equal(t, 0.1, metrics.CoresAllocatable.Value) //nolint:testifylint // exact float comparison + assert.Equal(t, 100*1024*1024.0, metrics.MemoryAllocatable.Value) //nolint:testifylint // exact float comparison // modify the limit and verify the new value is present node.Status.Allocatable[v1.ResourceCPU] = resource.MustParse("200m") watcher.handler.OnUpdate(node) metrics = nodeStore.GetNodeMetrics() require.NotNil(t, metrics) - assert.Equal(t, 0.2, metrics.CoresAllocatable.Value) + assert.Equal(t, 0.2, metrics.CoresAllocatable.Value) //nolint:testifylint // exact float comparison // delete the node and verify no metrics are present watcher.handler.OnDelete(node) @@ -404,7 +404,7 @@ func TestCreateAllWatchers(t *testing.T) { metricsRepo) require.Error(t, err) resourceWatchers.lock.Lock() - require.Equal(t, 0, len(resourceWatchers.metaWatchersMap)) + require.Empty(t, resourceWatchers.metaWatchersMap) resourceWatchers.lock.Unlock() // Start watcher for a resource that requires other resources, should start all the watchers @@ -425,7 +425,7 @@ func TestCreateAllWatchers(t *testing.T) { // Check that all the required watchers are in the map resourceWatchers.lock.Lock() // we add 1 to the expected result to represent the resource itself - require.Equal(t, len(extras)+1, len(resourceWatchers.metaWatchersMap)) + require.Len(t, resourceWatchers.metaWatchersMap, len(extras)+1) for _, extra := range extras { require.NotNil(t, resourceWatchers.metaWatchersMap[extra]) } @@ -487,7 +487,7 @@ func TestCreateMetaGenSpecific(t *testing.T) { log := logptest.NewTestingLogger(t, "test") - namespaceConfig, err := conf.NewConfigFrom(map[string]interface{}{ + namespaceConfig, err := conf.NewConfigFrom(map[string]any{ "enabled": true, }) require.NoError(t, err) @@ -573,7 +573,7 @@ func TestBuildMetadataEnricher_Start_Stop(t *testing.T) { resourceWatchers.lock.Unlock() funcs := mockFuncs{} - namespaceConfig, err := conf.NewConfigFrom(map[string]interface{}{ + namespaceConfig, err := conf.NewConfigFrom(map[string]any{ "enabled": true, }) require.NoError(t, err) @@ -1001,7 +1001,7 @@ func (f *mockFuncs) update(obj kubernetes.Resource) map[string]mapstr.M { }, }, } - logger := logp.NewLogger("kubernetes") + logger := logp.NewLogger("kubernetes") //nolint:forbidigo // test helper for k, v := range accessor.GetLabels() { kubernetes2.ShouldPut(meta, fmt.Sprintf("kubernetes.%v", k), v, logger) } @@ -1018,7 +1018,7 @@ func (f *mockFuncs) delete(obj kubernetes.Resource) []string { func (f *mockFuncs) index(m mapstr.M) string { f.indexed = m - return m["name"].(string) + return m["name"].(string) //nolint:errcheck // test mock } type mockWatcher struct { @@ -1028,7 +1028,7 @@ type mockWatcher struct { func newMockWatcher() *mockWatcher { return &mockWatcher{ - store: cache.NewStore(func(obj interface{}) (string, error) { + store: cache.NewStore(func(obj any) (string, error) { objName, err := cache.ObjectToName(obj) if err != nil { return "", err diff --git a/pkg/autodiscover/bus/bus.go b/pkg/autodiscover/bus/bus.go new file mode 100644 index 000000000000..621e23ba64e3 --- /dev/null +++ b/pkg/autodiscover/bus/bus.go @@ -0,0 +1,162 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package bus + +import ( + "sync" + + "github.com/elastic/elastic-agent-libs/keystore" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +// Event sent to the bus +type Event mapstr.M + +// Bus provides a common channel to emit and listen for Events +type Bus interface { + // Publish an event to the bus + Publish(Event) + + // Subscribe to all events, filter them to the ones containing *all* the keys in filter + Subscribe(filter ...string) Listener +} + +// Provider for keystore +type KeystoreProvider interface { + GetKeystore(event Event) keystore.Keystore +} + +// Listener retrieves Events from a Bus subscription until Stop is called +type Listener interface { + // Events channel + Events() <-chan Event + + // Stop listening and removes itself from the bus + Stop() +} + +type bus struct { + sync.RWMutex + log *logp.Logger + listeners []*listener + store chan Event +} + +type listener struct { + filter []string + channel chan Event + bus *bus +} + +// New initializes a new bus with the given name and returns it +func New(log *logp.Logger, name string) Bus { + return &bus{ + log: createLogger(log, name), + listeners: make([]*listener, 0), + } +} + +// NewBusWithStore allows to create a buffered bus when producers send data without +// listeners being subscribed to them. size determines the size of the buffer. +func NewBusWithStore(log *logp.Logger, name string, size int) Bus { + return &bus{ + log: createLogger(log, name), + listeners: make([]*listener, 0), + store: make(chan Event, size), + } +} + +func createLogger(log *logp.Logger, name string) *logp.Logger { + selector := "bus-" + name + return log.Named(selector).With("elastic-agent-autodiscover.bus", name) +} + +func (b *bus) Publish(e Event) { + b.RLock() + defer b.RUnlock() + + b.log.Debugf("%+v", e) + if len(b.listeners) == 0 && b.store != nil { + b.store <- e + return + } + + if len(b.store) != 0 { + doBreak := false + for !doBreak { + select { + case eve := <-b.store: + for _, listener := range b.listeners { + if listener.interested(eve) { + listener.channel <- eve + } + } + default: + doBreak = true + } + } + } + + for _, listener := range b.listeners { + if listener.interested(e) { + listener.channel <- e + } + } +} + +func (b *bus) Subscribe(filter ...string) Listener { + listener := &listener{ + filter: filter, + bus: b, + channel: make(chan Event, 100), + } + + b.Lock() + defer b.Unlock() + b.listeners = append(b.listeners, listener) + + return listener +} + +func (l *listener) Events() <-chan Event { + return l.channel +} + +func (l *listener) Stop() { + l.bus.Lock() + defer l.bus.Unlock() + + for i, listener := range l.bus.listeners { + if l == listener { + l.bus.listeners = append(l.bus.listeners[:i], l.bus.listeners[i+1:]...) + } + } + + close(l.channel) +} + +// Return true if listener is interested on the given event +func (l *listener) interested(e Event) bool { + for _, key := range l.filter { + if _, ok := e[key]; !ok { + return false + } + } + return true +} diff --git a/pkg/autodiscover/bus/bus_test.go b/pkg/autodiscover/bus/bus_test.go new file mode 100644 index 000000000000..4ef5e642fd1f --- /dev/null +++ b/pkg/autodiscover/bus/bus_test.go @@ -0,0 +1,122 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//nolint:forbidigo // tests verify bus behavior with the global logger +package bus + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/elastic-agent-libs/logp" +) + +func TestEmit(t *testing.T) { + bus := New(logp.L(), "name") + listener := bus.Subscribe() + + bus.Publish(Event{ + "foo": "bar", + }) + + event := <-listener.Events() + assert.Equal(t, "bar", event["foo"]) +} + +func TestEmitOrder(t *testing.T) { + bus := New(logp.L(), "name") + listener := bus.Subscribe() + bus.Publish(Event{"first": "event"}) + bus.Publish(Event{"second": "event"}) + + event1 := <-listener.Events() + event2 := <-listener.Events() + assert.Equal(t, Event{"first": "event"}, event1) + assert.Equal(t, Event{"second": "event"}, event2) +} + +func TestSubscribeFilter(t *testing.T) { + bus := New(logp.L(), "name") + listener := bus.Subscribe("second") + + bus.Publish(Event{"first": "event"}) + bus.Publish(Event{"second": "event"}) + + event := <-listener.Events() + assert.Equal(t, Event{"second": "event"}, event) +} + +func TestMultipleListeners(t *testing.T) { + bus := New(logp.L(), "name") + listener1 := bus.Subscribe("a") + listener2 := bus.Subscribe("a", "b") + + bus.Publish(Event{"a": "event"}) + bus.Publish(Event{"a": 1, "b": 2}) + + event1 := <-listener1.Events() + event2 := <-listener1.Events() + assert.Equal(t, Event{"a": "event"}, event1) + assert.Equal(t, Event{"a": 1, "b": 2}, event2) + + event1 = <-listener2.Events() + assert.Equal(t, Event{"a": 1, "b": 2}, event1) + + select { + case event2 = <-listener2.Events(): + t.Error("Got unexpected event:", event2) + default: + } +} + +func TestListenerClose(t *testing.T) { + bus := New(logp.L(), "name") + listener := bus.Subscribe() + + bus.Publish(Event{"first": "event"}) + bus.Publish(Event{"second": "event"}) + + listener.Stop() + + bus.Publish(Event{"third": "event"}) + + event := <-listener.Events() + assert.Equal(t, Event{"first": "event"}, event) + event = <-listener.Events() + assert.Equal(t, Event{"second": "event"}, event) + + // Channel was closed, we get an empty event + event = <-listener.Events() + assert.Equal(t, event, Event(nil)) +} + +func TestUnsubscribedBus(t *testing.T) { + bus := NewBusWithStore(logp.L(), "name", 2) + bus.Publish(Event{"first": "event"}) + + listener := bus.Subscribe() + bus.Publish(Event{"second": "event"}) + event := <-listener.Events() + event1 := <-listener.Events() + assert.Equal(t, Event{"first": "event"}, event) + assert.Equal(t, Event{"second": "event"}, event1) + + bus.Publish(Event{"a": 1, "b": 2}) + event2 := <-listener.Events() + assert.Equal(t, Event{"a": 1, "b": 2}, event2) +} diff --git a/pkg/autodiscover/docker/client.go b/pkg/autodiscover/docker/client.go new file mode 100644 index 000000000000..85251120590d --- /dev/null +++ b/pkg/autodiscover/docker/client.go @@ -0,0 +1,52 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux || darwin || windows + +package docker + +import ( + "net/http" + "os" + + "github.com/moby/moby/client" + + "github.com/elastic/elastic-agent-libs/logp" +) + +// NewClient builds and returns a new Docker client. On the first request the +// client will negotiate the API version with the server unless +// DOCKER_API_VERSION is set in the environment. +func NewClient(host string, httpClient *http.Client, httpHeaders map[string]string, logger *logp.Logger) (*client.Client, error) { + log := logger.Named("docker") + + opts := []client.Opt{ + client.WithHost(host), + client.WithHTTPClient(httpClient), + client.WithHTTPHeaders(httpHeaders), + } + + version := os.Getenv("DOCKER_API_VERSION") + if version != "" { + log.Debugf("Docker client will use API version %v as set by the DOCKER_API_VERSION environment variable.", version) + opts = append(opts, client.WithAPIVersion(version)) + } else { + log.Debug("Docker client will negotiate the API version on the first request.") + } + + return client.New(opts...) +} diff --git a/pkg/autodiscover/docker/client_test.go b/pkg/autodiscover/docker/client_test.go new file mode 100644 index 000000000000..7cea0f956e29 --- /dev/null +++ b/pkg/autodiscover/docker/client_test.go @@ -0,0 +1,63 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build integration && (linux || darwin || windows) + +package docker + +import ( + "context" + "os" + "testing" + + "github.com/moby/moby/client" + "github.com/stretchr/testify/assert" + + "github.com/elastic/elastic-agent-libs/logp/logptest" +) + +func TestNewClient(t *testing.T) { + logger := logptest.NewTestingLogger(t, "") + + c, err := NewClient(client.DefaultDockerHost, nil, nil, logger) + assert.NoError(t, err) + assert.NotNil(t, c) + + _, err = c.ContainerList(context.Background(), client.ContainerListOptions{}) + assert.NoError(t, err) + + // This test only works on newer Docker versions (any supported one really) + switch c.ClientVersion() { + case "1.22": + t.Skip("Docker version is too old for this test") + case client.MaxAPIVersion: + t.Logf("Using default API version: %s", client.MaxAPIVersion) + default: + t.Logf("Negotiated version: %s", c.ClientVersion()) + } + + // Test we can hardcode version + os.Setenv("DOCKER_API_VERSION", "1.22") + + c, err = NewClient(client.DefaultDockerHost, nil, nil, logger) + assert.NoError(t, err) + assert.NotNil(t, c) + assert.Equal(t, "1.22", c.ClientVersion()) + + _, err = c.ContainerList(context.Background(), client.ContainerListOptions{}) + assert.NoError(t, err) +} diff --git a/pkg/autodiscover/docker/helpers.go b/pkg/autodiscover/docker/helpers.go new file mode 100644 index 000000000000..ab2ff164c884 --- /dev/null +++ b/pkg/autodiscover/docker/helpers.go @@ -0,0 +1,60 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package docker + +import ( + "strings" + + "github.com/elastic/beats/v7/pkg/autodiscover/utils" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/elastic-agent-libs/safemapstr" +) + +// ExtractContainerName strips the `/` characters that frequently appear in container names +func ExtractContainerName(names []string) string { + output := names[0] + + if len(names) > 1 { + for _, name := range names { + if strings.Count(output, "/") > strings.Count(name, "/") { + output = name + } + } + } + return strings.Trim(output, "/") +} + +// DeDotLabels returns a new mapstr.M containing a copy of the labels +// where the dots have been converted into nested structure, avoiding +// possible mapping errors +func DeDotLabels(labels map[string]string, dedot bool) mapstr.M { + outputLabels := mapstr.M{} + for k, v := range labels { + if dedot { + // This is necessary so that ES does not interpret '.' fields as new + // nested JSON objects, and also makes this compatible with ES 2.x. + label := utils.DeDot(k) + _, _ = outputLabels.Put(label, v) + } else { + // If we don't dedot we ensure there are no mapping errors with safemapstr + _ = safemapstr.Put(outputLabels, k, v) + } + } + + return outputLabels +} diff --git a/pkg/autodiscover/docker/watcher.go b/pkg/autodiscover/docker/watcher.go new file mode 100644 index 000000000000..bf4fa22ec50f --- /dev/null +++ b/pkg/autodiscover/docker/watcher.go @@ -0,0 +1,494 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux || darwin || windows + +package docker + +import ( + "context" + "errors" + "io" + "net/http" + "sync" + "time" + + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/api/types/events" + dockerclient "github.com/moby/moby/client" + + "github.com/docker/go-connections/tlsconfig" + + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/elastic-agent-libs/logp" +) + +// Select Docker API version +const ( + shortIDLen = 12 + dockerRequestTimeout = 10 * time.Second + dockerEventsWatchPityTimerInterval = 10 * time.Second + dockerEventsWatchPityTimerTimeout = 10 * time.Minute + dockerEventsRetryBackoffInitial = 1 * time.Second + dockerEventsRetryBackoffMax = 60 * time.Second +) + +// Watcher reads docker events and keeps a list of known containers +type Watcher interface { + // Start watching docker API for new containers + Start() error + + // Stop watching docker API for new containers + Stop() + + // Container returns the running container with the given ID or nil if unknown + Container(ID string) *Container + + // Containers returns the list of known containers + Containers() map[string]*Container + + // ListenStart returns a bus listener to receive container started events, with a `container` key holding it + ListenStart() bus.Listener + + // ListenStop returns a bus listener to receive container stopped events, with a `container` key holding it + ListenStop() bus.Listener +} + +// TLSConfig for docker socket connection +type TLSConfig struct { + CA string `config:"certificate_authority"` + Certificate string `config:"certificate"` + Key string `config:"key"` +} + +type watcher struct { + sync.RWMutex + log *logp.Logger + client Client + ctx context.Context + stop context.CancelFunc + containers map[string]*Container + deleted map[string]time.Time // deleted annotations key -> last access time + cleanupTimeout time.Duration + clock clock + stopped sync.WaitGroup + bus bus.Bus + shortID bool // whether to store short ID in "containers" too +} + +// clock is an interface used to provide mocked time on testing +type clock interface { + Now() time.Time +} + +// systemClock implements the clock interface using the system clock via the time package +type systemClock struct{} + +// Now returns the current time +func (*systemClock) Now() time.Time { return time.Now() } + +// Container info retrieved by the watcher +type Container struct { + ID string + Name string + Image string + Labels map[string]string + IPAddresses []string + Ports []container.PortSummary +} + +// Client for docker interface +type Client interface { + ContainerList(ctx context.Context, options dockerclient.ContainerListOptions) (dockerclient.ContainerListResult, error) + ContainerInspect(ctx context.Context, container string, options dockerclient.ContainerInspectOptions) (dockerclient.ContainerInspectResult, error) + Events(ctx context.Context, options dockerclient.EventsListOptions) dockerclient.EventsResult +} + +// WatcherConstructor represent a function that creates a new Watcher from giving parameters +type WatcherConstructor func(logp *logp.Logger, host string, tls *TLSConfig, storeShortID bool) (Watcher, error) + +// NewWatcher returns a watcher running for the given settings +func NewWatcher(log *logp.Logger, host string, tls *TLSConfig, storeShortID bool) (Watcher, error) { + var httpClient *http.Client + if tls != nil { + options := tlsconfig.Options{ + CAFile: tls.CA, + CertFile: tls.Certificate, + KeyFile: tls.Key, + } + + tlsc, err := tlsconfig.Client(options) + if err != nil { + return nil, err + } + + httpClient = &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: tlsc, + }, + } + } + + client, err := NewClient(host, httpClient, nil, log) + if err != nil { + return nil, err + } + + // Extra check to confirm that Docker is available + _, err = client.Info(context.Background(), dockerclient.InfoOptions{}) + if err != nil { + client.Close() + return nil, err + } + + return NewWatcherWithClient(log, client, 60*time.Second, storeShortID) +} + +// NewWatcherWithClient creates a new Watcher from a given Docker client +func NewWatcherWithClient(log *logp.Logger, client Client, cleanupTimeout time.Duration, storeShortID bool) (Watcher, error) { + ctx, cancel := context.WithCancel(context.Background()) + return &watcher{ + log: log, + client: client, + ctx: ctx, + stop: cancel, + containers: make(map[string]*Container), + deleted: make(map[string]time.Time), + cleanupTimeout: cleanupTimeout, + bus: bus.New(log, "docker"), + shortID: storeShortID, + clock: &systemClock{}, + }, nil +} + +// Container returns the running container with the given ID or nil if unknown +func (w *watcher) Container(ID string) *Container { + w.RLock() + container := w.containers[ID] + if container == nil { + w.RUnlock() + return nil + } + _, ok := w.deleted[container.ID] + w.RUnlock() + + // Update last access time if it's deleted + if ok { + w.Lock() + w.deleted[container.ID] = w.clock.Now() + w.Unlock() + } + + return container +} + +// Containers returns the list of known containers +func (w *watcher) Containers() map[string]*Container { + w.RLock() + defer w.RUnlock() + res := make(map[string]*Container) + for k, v := range w.containers { + if !w.shortID || len(k) != shortIDLen { + res[k] = v + } + } + return res +} + +// Start watching docker API for new containers +func (w *watcher) Start() error { + // Do initial scan of existing containers + w.log.Debug("Start docker containers scanner") + + w.Lock() + defer w.Unlock() + containers, err := w.listContainers(dockerclient.ContainerListOptions{}) + if err != nil { + w.log.Errorf("Failed to call listContainers: %v", err) + } + + for _, c := range containers { + w.containers[c.ID] = c + if w.shortID { + w.containers[c.ID[:shortIDLen]] = c + } + } + + // Emit all start events (avoid blocking if the bus get's blocked) + go func() { + for _, c := range containers { + w.bus.Publish(bus.Event{ + "start": true, + "container": c, + }) + } + }() + + w.stopped.Add(2) + go w.watch() + go w.cleanupWorker() + + return nil +} + +func (w *watcher) Stop() { + w.stop() + w.stopped.Wait() +} + +func (w *watcher) watch() { + defer w.stopped.Done() + + filter := make(dockerclient.Filters).Add("type", "container") + + // Ticker to restart the watcher when no events are received after some time. + tickChan := time.NewTicker(dockerEventsWatchPityTimerInterval) + defer tickChan.Stop() + + lastValidTimestamp := w.clock.Now() + retryDelay := dockerEventsRetryBackoffInitial + + watch := func() bool { + lastReceivedEventTime := w.clock.Now() + + w.log.Debugf("Fetching events since %s", lastValidTimestamp) + + options := dockerclient.EventsListOptions{ + Since: lastValidTimestamp.Format(time.RFC3339Nano), + Filters: filter, + } + + ctx, cancel := context.WithCancel(w.ctx) + defer cancel() + + result := w.client.Events(ctx, options) + for { + select { + case event := <-result.Messages: + retryDelay = dockerEventsRetryBackoffInitial + w.log.Debugf("Got a new docker event: %v", event) + if event.TimeNano > 0 { + lastValidTimestamp = time.Unix(0, event.TimeNano) + } else { + lastValidTimestamp = time.Unix(event.Time, 0) + } + lastReceivedEventTime = w.clock.Now() + + switch event.Action { + case "start", "update": + w.containerUpdate(event) + case "die": + w.containerDelete(event) + } + case err := <-result.Err: + if errors.Is(err, io.EOF) { + // Client disconnected, watch is not done, reconnect + w.log.Debug("EOF received in events stream, restarting watch call") + } else if errors.Is(err, context.DeadlineExceeded) { + w.log.Debug("Context deadline exceeded for docker request, restarting watch call") + } else if errors.Is(err, context.Canceled) { + // Parent context has been canceled, watch is done. + return true + } else { + w.log.Errorf("Error watching for docker events: %+v", err) + } + return false + case <-tickChan.C: + if time.Since(lastReceivedEventTime) > dockerEventsWatchPityTimerTimeout { + w.log.Infof("No events received within %s, restarting watch call", dockerEventsWatchPityTimerTimeout) + return false + } + case <-w.ctx.Done(): + w.log.Debug("Watcher stopped") + return true + } + } + } + + for { + done := watch() + if done { + return + } + // Wait before trying to reconnect, using exponential backoff to avoid + // log spam when the Docker daemon is unavailable (e.g. Docker Desktop on macOS). + select { + case <-w.ctx.Done(): + return + case <-time.After(retryDelay): + } + if retryDelay < dockerEventsRetryBackoffMax { + retryDelay *= 2 + if retryDelay > dockerEventsRetryBackoffMax { + retryDelay = dockerEventsRetryBackoffMax + } + } + } +} + +func (w *watcher) containerUpdate(event events.Message) { + filter := make(dockerclient.Filters).Add("id", event.Actor.ID) + + containers, err := w.listContainers(dockerclient.ContainerListOptions{ + Filters: filter, + }) + if err != nil || len(containers) != 1 { + w.log.Errorf("Error getting container info: %v", err) + return + } + container := containers[0] + + w.Lock() + w.containers[event.Actor.ID] = container + if w.shortID { + w.containers[event.Actor.ID[:shortIDLen]] = container + } + // un-delete if it's flagged (in case of update or recreation) + delete(w.deleted, event.Actor.ID) + w.Unlock() + + w.bus.Publish(bus.Event{ + "start": true, + "container": container, + }) +} + +func (w *watcher) containerDelete(event events.Message) { + container := w.Container(event.Actor.ID) + + w.Lock() + w.deleted[event.Actor.ID] = w.clock.Now() + w.Unlock() + + if container != nil { + w.bus.Publish(bus.Event{ + "stop": true, + "container": container, + }) + } +} + +func (w *watcher) listContainers(options dockerclient.ContainerListOptions) ([]*Container, error) { + log := w.log + + log.Debug("List containers") + ctx, cancel := context.WithTimeout(w.ctx, dockerRequestTimeout) + defer cancel() + + listResult, err := w.client.ContainerList(ctx, options) + if err != nil { + return nil, err + } + + containers := listResult.Items + result := make([]*Container, len(containers)) + for idx, c := range containers { + var ipaddresses []string + if c.NetworkSettings != nil { + // Handle alternate platforms like VMWare's VIC that might not have this data. + for _, net := range c.NetworkSettings.Networks { + if net.IPAddress.IsValid() { + ipaddresses = append(ipaddresses, net.IPAddress.String()) + } + } + } + + // If there are no network interfaces, assume that the container is on host network + // Inspect the container directly and use the hostname as the IP address in order + if len(ipaddresses) == 0 { + log.Debugf("Inspect container %s", c.ID) + ctx, cancel := context.WithTimeout(w.ctx, dockerRequestTimeout) + defer cancel() + inspectResult, err := w.client.ContainerInspect(ctx, c.ID, dockerclient.ContainerInspectOptions{}) + if err == nil { + ipaddresses = append(ipaddresses, inspectResult.Container.Config.Hostname) + } else { + log.Warnf("unable to inspect container %s due to error %+v", c.ID, err) + } + } + result[idx] = &Container{ + ID: c.ID, + Name: c.Names[0][1:], // Strip '/' from container names + Image: c.Image, + Labels: c.Labels, + Ports: c.Ports, + IPAddresses: ipaddresses, + } + } + + return result, nil +} + +// Clean up deleted containers after they are not used anymore +func (w *watcher) cleanupWorker() { + defer w.stopped.Done() + + for { + select { + case <-w.ctx.Done(): + return + // Wait a full period + case <-time.After(w.cleanupTimeout): + w.runCleanup() + } + } +} + +func (w *watcher) runCleanup() { + // Check entries for timeout + var toDelete []string + timeout := w.clock.Now().Add(-w.cleanupTimeout) + w.RLock() + for key, lastSeen := range w.deleted { + if lastSeen.Before(timeout) { + w.log.Debugf("Removing container %s after cool down timeout", key) + toDelete = append(toDelete, key) + } + } + w.RUnlock() + + // Delete timed out entries: + for _, key := range toDelete { + container := w.Container(key) + if container != nil { + w.bus.Publish(bus.Event{ + "delete": true, + "container": container, + }) + } + } + + w.Lock() + for _, key := range toDelete { + delete(w.deleted, key) + delete(w.containers, key) + if w.shortID { + delete(w.containers, key[:shortIDLen]) + } + } + w.Unlock() +} + +// ListenStart returns a bus listener to receive container started events, with a `container` key holding it +func (w *watcher) ListenStart() bus.Listener { + return w.bus.Subscribe("start") +} + +// ListenStop returns a bus listener to receive container stopped events, with a `container` key holding it +func (w *watcher) ListenStop() bus.Listener { + return w.bus.Subscribe("stop") +} diff --git a/pkg/autodiscover/docker/watcher_test.go b/pkg/autodiscover/docker/watcher_test.go new file mode 100644 index 000000000000..e9a93acff9b4 --- /dev/null +++ b/pkg/autodiscover/docker/watcher_test.go @@ -0,0 +1,559 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux || darwin || windows + +package docker + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/api/types/events" + dockerclient "github.com/moby/moby/client" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" + + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/logp/logptest" +) + +type MockClient struct { + // containers to return on ContainerList call + containers [][]container.Summary + containersErr error + // event list to send on Events call + events []any + // done channel is closed when the client has sent all events + done chan any +} + +func (m *MockClient) ContainerList(ctx context.Context, options dockerclient.ContainerListOptions) (dockerclient.ContainerListResult, error) { + if m.containersErr != nil { + return dockerclient.ContainerListResult{}, m.containersErr + } + res := m.containers[0] + m.containers = m.containers[1:] + return dockerclient.ContainerListResult{Items: res}, nil +} + +func (m *MockClient) Events(ctx context.Context, options dockerclient.EventsListOptions) dockerclient.EventsResult { + eventsC := make(chan events.Message) + errorsC := make(chan error) + + go func() { + for _, event := range m.events { + switch e := event.(type) { + case events.Message: + eventsC <- e + case error: + errorsC <- e + } + } + close(m.done) + }() + + return dockerclient.EventsResult{Messages: eventsC, Err: errorsC} +} + +func (m *MockClient) ContainerInspect(ctx context.Context, containerID string, options dockerclient.ContainerInspectOptions) (dockerclient.ContainerInspectResult, error) { + return dockerclient.ContainerInspectResult{}, errors.New("unimplemented") +} + +func TestWatcherInitialization(t *testing.T) { + watcher := runAndWait(testWatcher(t, + [][]container.Summary{ + { + container.Summary{ + ID: "0332dbd79e20", + Names: []string{"/containername", "othername"}, + Image: "busybox", + Labels: map[string]string{"foo": "bar"}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + container.Summary{ + ID: "6ac6ee8df5d4", + Names: []string{"/other"}, + Image: "nginx", + Labels: map[string]string{}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + }, + }, + nil, + )) + + assert.Equal(t, map[string]*Container{ + "0332dbd79e20": { + ID: "0332dbd79e20", + Name: "containername", + Image: "busybox", + Labels: map[string]string{"foo": "bar"}, + }, + "6ac6ee8df5d4": { + ID: "6ac6ee8df5d4", + Name: "other", + Image: "nginx", + Labels: map[string]string{}, + }, + }, watcher.Containers()) +} + +func TestWatcherInitializationShortID(t *testing.T) { + watcher := runAndWait(testWatcherShortID(t, + [][]container.Summary{ + { + container.Summary{ + ID: "1234567890123", + Names: []string{"/containername", "othername"}, + Image: "busybox", + Labels: map[string]string{"foo": "bar"}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + container.Summary{ + ID: "2345678901234", + Names: []string{"/other"}, + Image: "nginx", + Labels: map[string]string{}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + }, + }, + nil, + true, + )) + + assert.Equal(t, map[string]*Container{ + "1234567890123": { + ID: "1234567890123", + Name: "containername", + Image: "busybox", + Labels: map[string]string{"foo": "bar"}, + }, + "2345678901234": { + ID: "2345678901234", + Name: "other", + Image: "nginx", + Labels: map[string]string{}, + }, + }, watcher.Containers()) + + assert.Equal(t, &Container{ + ID: "1234567890123", + Name: "containername", + Image: "busybox", + Labels: map[string]string{"foo": "bar"}, + }, watcher.Container("123456789012")) +} + +func TestWatcherAddEvents(t *testing.T) { + watcher := runAndWait(testWatcher(t, + [][]container.Summary{ + { + container.Summary{ + ID: "0332dbd79e20", + Names: []string{"/containername", "othername"}, + Image: "busybox", + Labels: map[string]string{"foo": "bar"}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + }, + { + container.Summary{ + ID: "6ac6ee8df5d4", + Names: []string{"/other"}, + Image: "nginx", + Labels: map[string]string{"label": "value"}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + }, + }, + []any{ + events.Message{ + Action: "start", + Actor: events.Actor{ + ID: "6ac6ee8df5d4", + Attributes: map[string]string{ + "name": "other", + "image": "nginx", + "label": "value", + }, + }, + }, + }, + )) + + assert.Equal(t, map[string]*Container{ + "0332dbd79e20": { + ID: "0332dbd79e20", + Name: "containername", + Image: "busybox", + Labels: map[string]string{"foo": "bar"}, + }, + "6ac6ee8df5d4": { + ID: "6ac6ee8df5d4", + Name: "other", + Image: "nginx", + Labels: map[string]string{"label": "value"}, + }, + }, watcher.Containers()) +} + +func TestWatcherAddEventsShortID(t *testing.T) { + watcher := runAndWait(testWatcherShortID(t, + [][]container.Summary{ + { + container.Summary{ + ID: "1234567890123", + Names: []string{"/containername", "othername"}, + Image: "busybox", + Labels: map[string]string{"foo": "bar"}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + }, + { + container.Summary{ + ID: "2345678901234", + Names: []string{"/other"}, + Image: "nginx", + Labels: map[string]string{"label": "value"}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + }, + }, + []any{ + events.Message{ + Action: "start", + Actor: events.Actor{ + ID: "2345678901234", + Attributes: map[string]string{ + "name": "other", + "image": "nginx", + "label": "value", + }, + }, + }, + }, + true, + )) + + assert.Equal(t, map[string]*Container{ + "1234567890123": { + ID: "1234567890123", + Name: "containername", + Image: "busybox", + Labels: map[string]string{"foo": "bar"}, + }, + "2345678901234": { + ID: "2345678901234", + Name: "other", + Image: "nginx", + Labels: map[string]string{"label": "value"}, + }, + }, watcher.Containers()) +} + +func TestWatcherUpdateEvent(t *testing.T) { + watcher := runAndWait(testWatcher(t, + [][]container.Summary{ + { + { + ID: "0332dbd79e20", + Names: []string{"/containername", "othername"}, + Image: "busybox", + Labels: map[string]string{"label": "foo"}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + }, + { + container.Summary{ + ID: "0332dbd79e20", + Names: []string{"/containername", "othername"}, + Image: "busybox", + Labels: map[string]string{"label": "bar"}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + }, + }, + []any{ + events.Message{ + Action: "update", + Actor: events.Actor{ + ID: "0332dbd79e20", + Attributes: map[string]string{ + "name": "containername", + "image": "busybox", + "label": "bar", + }, + }, + }, + }, + )) + + assert.Equal(t, map[string]*Container{ + "0332dbd79e20": { + ID: "0332dbd79e20", + Name: "containername", + Image: "busybox", + Labels: map[string]string{"label": "bar"}, + }, + }, watcher.Containers()) + assert.Empty(t, watcher.deleted) +} + +func TestWatcherUpdateEventShortID(t *testing.T) { + watcher := runAndWait(testWatcherShortID(t, + [][]container.Summary{ + { + container.Summary{ + ID: "1234567890123", + Names: []string{"/containername", "othername"}, + Image: "busybox", + Labels: map[string]string{"label": "foo"}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + }, + { + container.Summary{ + ID: "1234567890123", + Names: []string{"/containername", "othername"}, + Image: "busybox", + Labels: map[string]string{"label": "bar"}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + }, + }, + []any{ + events.Message{ + Action: "update", + Actor: events.Actor{ + ID: "1234567890123", + Attributes: map[string]string{ + "name": "containername", + "image": "busybox", + "label": "bar", + }, + }, + }, + }, + true, + )) + + assert.Equal(t, map[string]*Container{ + "1234567890123": { + ID: "1234567890123", + Name: "containername", + Image: "busybox", + Labels: map[string]string{"label": "bar"}, + }, + }, watcher.Containers()) + assert.Empty(t, watcher.deleted) +} + +func TestWatcherDie(t *testing.T) { + watcher, clientDone := testWatcher(t, + [][]container.Summary{ + { + container.Summary{ + ID: "0332dbd79e20", + Names: []string{"/containername", "othername"}, + Image: "busybox", + Labels: map[string]string{"label": "foo"}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + }, + }, + []any{ + events.Message{ + Action: "die", + Actor: events.Actor{ + ID: "0332dbd79e20", + }, + }, + }, + ) + + clock := newTestClock() + watcher.clock = clock + + stopListener := watcher.ListenStop() + + err := watcher.Start() + require.NoError(t, err) + defer watcher.Stop() + + // Check it doesn't get removed while we request meta for the container + for range 18 { + watcher.Container("0332dbd79e20") + clock.Sleep(watcher.cleanupTimeout / 2) + watcher.runCleanup() + if !assert.Len(t, watcher.Containers(), 1) { + break + } + } + + // Wait to be sure that the delete event has been processed + <-clientDone + <-stopListener.Events() + + // Check that after the cleanup period the container is removed + clock.Sleep(watcher.cleanupTimeout + 1*time.Second) + watcher.runCleanup() + assert.Empty(t, watcher.Containers()) +} + +func TestWatcherDieShortID(t *testing.T) { + watcher, clientDone := testWatcherShortID(t, + [][]container.Summary{ + { + container.Summary{ + ID: "0332dbd79e20aaa", + Names: []string{"/containername", "othername"}, + Image: "busybox", + Labels: map[string]string{"label": "foo"}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + }, + }, + []any{ + events.Message{ + Action: "die", + Actor: events.Actor{ + ID: "0332dbd79e20aaa", + }, + }, + }, + true, + ) + + clock := newTestClock() + watcher.clock = clock + + stopListener := watcher.ListenStop() + + err := watcher.Start() + require.NoError(t, err) + defer watcher.Stop() + + // Check it doesn't get removed while we request meta for the container + for range 18 { + watcher.Container("0332dbd79e20") + clock.Sleep(watcher.cleanupTimeout / 2) + watcher.runCleanup() + if !assert.Len(t, watcher.Containers(), 1) { + break + } + } + + // Wait to be sure that the delete event has been processed + <-clientDone + <-stopListener.Events() + + // Check that after the cleanup period the container is removed + clock.Sleep(watcher.cleanupTimeout + 1*time.Second) + watcher.runCleanup() + assert.Empty(t, watcher.Containers()) +} + +func TestWatcherNoError(t *testing.T) { + core, obs := observer.New(zapcore.DebugLevel) + l, err := logp.ConfigureWithCoreLocal(logp.DefaultConfig(logp.DefaultEnvironment), core) + require.NoError(t, err) + client := &MockClient{ + containersErr: errors.New("test error"), + events: nil, + done: make(chan any), + } + w, err := NewWatcherWithClient(l, client, 200*time.Millisecond, true) + if err != nil { + t.Fatal(err) + } + watcher, ok := w.(*watcher) + if !ok { + t.Fatal("'watcher' was supposed to be pointer to the watcher structure") + } + clock := newTestClock() + watcher.clock = clock + + err = watcher.Start() + require.NoError(t, err) + watcher.Stop() + require.Len(t, obs.FilterMessageSnippet("Failed to call listContainers").All(), 1) +} + +func testWatcher(t *testing.T, containers [][]container.Summary, events []any) (*watcher, chan any) { + return testWatcherShortID(t, containers, events, false) +} + +func testWatcherShortID(t *testing.T, containers [][]container.Summary, events []any, enable bool) (*watcher, chan any) { + logger := logptest.NewTestingLogger(t, "") + + client := &MockClient{ + containers: containers, + events: events, + done: make(chan any), + } + + w, err := NewWatcherWithClient(logger, client, 200*time.Millisecond, enable) + if err != nil { + t.Fatal(err) + } + watcher, ok := w.(*watcher) + if !ok { + t.Fatal("'watcher' was supposed to be pointer to the watcher structure") + } + + return watcher, client.done +} + +func runAndWait(w *watcher, done chan any) *watcher { + _ = w.Start() + <-done + w.Stop() + return w +} + +type testClock struct { + sync.Mutex + + now time.Time +} + +func newTestClock() *testClock { + return &testClock{now: time.Time{}} +} + +func (c *testClock) Now() time.Time { + c.Lock() + defer c.Unlock() + + c.now = c.now.Add(1) + return c.now +} + +func (c *testClock) Sleep(d time.Duration) { + c.Lock() + defer c.Unlock() + + c.now = c.now.Add(d) +} diff --git a/pkg/autodiscover/kubernetes/eventhandler.go b/pkg/autodiscover/kubernetes/eventhandler.go new file mode 100644 index 000000000000..0a71cf2b38e8 --- /dev/null +++ b/pkg/autodiscover/kubernetes/eventhandler.go @@ -0,0 +1,257 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package kubernetes + +import ( + "reflect" + "sync" +) + +// ResourceEventHandler can handle notifications for events that happen to a +// resource. The events are informational only, so you can't return an +// error. +// - OnAdd is called when an object is added. +// - OnUpdate is called when an object is modified. Note that oldObj is the +// last known state of the object-- it is possible that several changes +// were combined together, so you can't use this to see every single +// change. OnUpdate is also called when a re-list happens, and it will +// get called even if nothing changed. This is useful for periodically +// evaluating or syncing something. +// - OnDelete will get the final state of the item if it is known, otherwise +// it will get an object of type DeletedFinalStateUnknown. This can +// happen if the watch is closed and misses the delete event and we don't +// notice the deletion until the subsequent re-list. +// +// idea: allow the On* methods to return an error so that the RateLimited WorkQueue +// idea: can requeue the failed event processing. +type ResourceEventHandler interface { + OnAdd(obj any) + OnUpdate(obj any) + OnDelete(obj any) +} + +// ResourceEventHandlerFuncs is an adaptor to let you easily specify as many or +// as few of the notification functions as you want while still implementing +// ResourceEventHandler. +type ResourceEventHandlerFuncs struct { + AddFunc func(obj any) + UpdateFunc func(obj any) + DeleteFunc func(obj any) +} + +// OnAdd calls AddFunc if it's not nil. +func (r ResourceEventHandlerFuncs) OnAdd(obj any) { + if r.AddFunc != nil { + r.AddFunc(obj) + } +} + +// OnUpdate calls UpdateFunc if it's not nil. +func (r ResourceEventHandlerFuncs) OnUpdate(obj any) { + if r.UpdateFunc != nil { + r.UpdateFunc(obj) + } +} + +// OnDelete calls DeleteFunc if it's not nil. +func (r ResourceEventHandlerFuncs) OnDelete(obj any) { + if r.DeleteFunc != nil { + r.DeleteFunc(obj) + } +} + +// NoOpEventHandlerFuncs ensures that watcher reconciliation can happen even without the required funcs +type NoOpEventHandlerFuncs struct { +} + +// OnAdd does a no-op on an add event +func (n NoOpEventHandlerFuncs) OnAdd(obj any) { + +} + +// OnUpdate does a no-op on an update event +func (n NoOpEventHandlerFuncs) OnUpdate(obj any) { + +} + +// OnDelete does a no-op on a delete event +func (n NoOpEventHandlerFuncs) OnDelete(obj any) { + +} + +// FilteringResourceEventHandler applies the provided filter to all events coming +// in, ensuring the appropriate nested handler method is invoked. An object +// that starts passing the filter after an update is considered an add, and an +// object that stops passing the filter after an update is considered a delete. +type FilteringResourceEventHandler struct { + FilterFunc func(obj any) bool + Handler ResourceEventHandler +} + +// OnAdd calls the nested handler only if the filter succeeds +func (r FilteringResourceEventHandler) OnAdd(obj any) { + if !r.FilterFunc(obj) { + return + } + r.Handler.OnAdd(obj) +} + +// OnUpdate ensures the proper handler is called depending on whether the filter matches +func (r FilteringResourceEventHandler) OnUpdate(obj any) { + if !r.FilterFunc(obj) { + return + } + r.Handler.OnUpdate(obj) +} + +// OnDelete calls the nested handler only if the filter succeeds +func (r FilteringResourceEventHandler) OnDelete(obj any) { + if !r.FilterFunc(obj) { + return + } + r.Handler.OnDelete(obj) +} + +// podUpdaterHandlerFunc is a function that handles pod updater notifications. +type podUpdaterHandlerFunc func(any) + +// podUpdaterStore is the interface that an object needs to implement to be +// used as a pod updater store. +type podUpdaterStore interface { + List() []any +} + +// namespacePodUpdater notifies updates on pods when their namespaces are updated. +type namespacePodUpdater struct { + handler podUpdaterHandlerFunc + store podUpdaterStore + namespaceWatcher Watcher + locker sync.Locker +} + +// NewNamespacePodUpdater creates a namespacePodUpdater +func NewNamespacePodUpdater(handler podUpdaterHandlerFunc, store podUpdaterStore, namespaceWatcher Watcher, locker sync.Locker) *namespacePodUpdater { + return &namespacePodUpdater{ + handler: handler, + store: store, + namespaceWatcher: namespaceWatcher, + locker: locker, + } +} + +// OnUpdate handles update events on namespaces. +func (n *namespacePodUpdater) OnUpdate(obj any) { + ns, ok := obj.(*Namespace) + if !ok { + return + } + // n.store.List() returns a snapshot at this point. If a delete is received + // from the main watcher, this loop may generate an update event after the + // delete is processed, leaving configurations that would never be deleted. + // Also this loop can miss updates, what could leave outdated configurations. + // Avoid these issues by locking the processing of events from the main watcher. + if n.locker != nil { + n.locker.Lock() + defer n.locker.Unlock() + } + + cachedObject := n.namespaceWatcher.CachedObject() + cachedNamespace, ok := cachedObject.(*Namespace) + + if ok && ns.Name == cachedNamespace.Name { + labelscheck := reflect.DeepEqual(ns.Labels, cachedNamespace.Labels) + annotationscheck := reflect.DeepEqual(ns.Annotations, cachedNamespace.Annotations) + // Only if there is a difference in Metadata labels or annotations proceed to Pod update + if !labelscheck || !annotationscheck { + for _, pod := range n.store.List() { + pod, ok := pod.(*Pod) + if ok && pod.Namespace == ns.Name { + n.handler(pod) + } + } + } + } + +} + +// OnAdd handles add events on namespaces. Nothing to do, if pods are added to this +// namespace they will generate their own add events. +func (*namespacePodUpdater) OnAdd(any) {} + +// OnDelete handles delete events on namespaces. Nothing to do, if pods are deleted from this +// namespace they will generate their own delete events. +func (*namespacePodUpdater) OnDelete(any) {} + +// nodePodUpdater notifies updates on pods when their nodes are updated. +type nodePodUpdater struct { + handler podUpdaterHandlerFunc + store podUpdaterStore + nodeWatcher Watcher + locker sync.Locker +} + +// NewNodePodUpdater creates a nodePodUpdater +func NewNodePodUpdater(handler podUpdaterHandlerFunc, store podUpdaterStore, nodeWatcher Watcher, locker sync.Locker) *nodePodUpdater { + return &nodePodUpdater{ + handler: handler, + store: store, + nodeWatcher: nodeWatcher, + locker: locker, + } +} + +// OnUpdate handles update events on nodes. +func (n *nodePodUpdater) OnUpdate(obj any) { + node, ok := obj.(*Node) + if !ok { + return + } + // n.store.List() returns a snapshot at this point. If a delete is received + // from the main watcher, this loop may generate an update event after the + // delete is processed, leaving configurations that would never be deleted. + // Also this loop can miss updates, what could leave outdated configurations. + // Avoid these issues by locking the processing of events from the main watcher. + if n.locker != nil { + n.locker.Lock() + defer n.locker.Unlock() + } + cachedObject := n.nodeWatcher.CachedObject() + cachedNode, ok := cachedObject.(*Node) + + if ok && node.Name == cachedNode.Name { + labelscheck := reflect.DeepEqual(node.Labels, cachedNode.Labels) + annotationscheck := reflect.DeepEqual(node.Annotations, cachedNode.Annotations) + // Only if there is a difference in Metadata labels or annotations proceed to Pod update + if !labelscheck || !annotationscheck { + for _, pod := range n.store.List() { + pod, ok := pod.(*Pod) + if ok && pod.Spec.NodeName == node.Name { + n.handler(pod) + } + } + } + } +} + +// OnAdd handles add events on namespaces. Nothing to do, if pods are added to this +// namespace they will generate their own add events. +func (*nodePodUpdater) OnAdd(any) {} + +// OnDelete handles delete events on namespaces. Nothing to do, if pods are deleted from this +// namespace they will generate their own delete events. +func (*nodePodUpdater) OnDelete(any) {} diff --git a/pkg/autodiscover/kubernetes/informer.go b/pkg/autodiscover/kubernetes/informer.go new file mode 100644 index 000000000000..ed7e7d4bd753 --- /dev/null +++ b/pkg/autodiscover/kubernetes/informer.go @@ -0,0 +1,334 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package kubernetes + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/metadata" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" +) + +func nodeSelector(options *metav1.ListOptions, opt WatchOptions) { + if opt.Node != "" { + options.FieldSelector = "spec.nodeName=" + opt.Node + } +} + +func nameSelector(options *metav1.ListOptions, name string) { + if name != "" { + options.FieldSelector = "metadata.name=" + name + } +} + +// NewInformer creates an informer for a given resource +func NewInformer(client kubernetes.Interface, resource Resource, opts WatchOptions, indexers cache.Indexers) (cache.SharedInformer, string, error) { + var objType string + + var listwatch *cache.ListWatch + ctx := context.TODO() + switch resource.(type) { + case *Pod: + p := client.CoreV1().Pods(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + nodeSelector(&options, opts) + return p.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + nodeSelector(&options, opts) + return p.Watch(ctx, options) + }, + } + + objType = "pod" + case *Event: + e := client.CoreV1().Events(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return e.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return e.Watch(ctx, options) + }, + } + + objType = "event" + case *Node: + n := client.CoreV1().Nodes() + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + nameSelector(&options, opts.Node) + return n.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + nameSelector(&options, opts.Node) + return n.Watch(ctx, options) + }, + } + + objType = "node" + case *Namespace: + ns := client.CoreV1().Namespaces() + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + nameSelector(&options, opts.Namespace) + return ns.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + nameSelector(&options, opts.Namespace) + return ns.Watch(ctx, options) + }, + } + + objType = "namespace" + case *Deployment: + d := client.AppsV1().Deployments(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return d.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return d.Watch(ctx, options) + }, + } + + objType = "deployment" + case *ReplicaSet: + rs := client.AppsV1().ReplicaSets(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return rs.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return rs.Watch(ctx, options) + }, + } + + objType = "replicaset" + case *StatefulSet: + ss := client.AppsV1().StatefulSets(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return ss.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return ss.Watch(ctx, options) + }, + } + + objType = "statefulset" + case *DaemonSet: + ss := client.AppsV1().DaemonSets(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return ss.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return ss.Watch(ctx, options) + }, + } + + objType = "daemonset" + case *Service: + svc := client.CoreV1().Services(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return svc.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return svc.Watch(ctx, options) + }, + } + + objType = "service" + case *ServiceAccount: + sa := client.CoreV1().ServiceAccounts(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return sa.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return sa.Watch(ctx, options) + }, + } + + objType = "serviceAccount" + case *CronJob: + cronjob := client.BatchV1().CronJobs(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return cronjob.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return cronjob.Watch(ctx, options) + }, + } + + objType = "cronjob" + case *Job: + job := client.BatchV1().Jobs(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return job.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return job.Watch(ctx, options) + }, + } + + objType = "job" + case *PersistentVolume: + ss := client.CoreV1().PersistentVolumes() + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return ss.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return ss.Watch(ctx, options) + }, + } + + objType = "persistentvolume" + case *PersistentVolumeClaim: + ss := client.CoreV1().PersistentVolumeClaims(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return ss.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return ss.Watch(ctx, options) + }, + } + + objType = "persistentvolumeclaim" + case *StorageClass: + sc := client.StorageV1().StorageClasses() + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return sc.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return sc.Watch(ctx, options) + }, + } + + objType = "storageclass" + case *Role: + r := client.RbacV1().Roles(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return r.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return r.Watch(ctx, options) + }, + } + + objType = "role" + + case *RoleBinding: + rb := client.RbacV1().RoleBindings(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return rb.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return rb.Watch(ctx, options) + }, + } + + objType = "rolebinding" + + case *ClusterRole: + cr := client.RbacV1().ClusterRoles() + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return cr.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return cr.Watch(ctx, options) + }, + } + + objType = "clusterrole" + + case *ClusterRoleBinding: + crb := client.RbacV1().ClusterRoleBindings() + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return crb.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return crb.Watch(ctx, options) + }, + } + + objType = "clusterrolebinding" + + case *NetworkPolicy: + np := client.ExtensionsV1beta1().NetworkPolicies(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return np.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return np.Watch(ctx, options) + }, + } + + objType = "networkpolicy" + + default: + return nil, "", fmt.Errorf("unsupported resource type for watching %T", resource) + } + + if indexers == nil { + indexers = cache.Indexers{} + } + return cache.NewSharedIndexInformer(listwatch, resource, opts.SyncTimeout, indexers), objType, nil +} + +// NewMetadataInformer creates an informer for a given resource that only tracks the resource metadata. +func NewMetadataInformer(client metadata.Interface, gvr schema.GroupVersionResource, opts WatchOptions, indexers cache.Indexers) cache.SharedInformer { + ctx := context.Background() + if indexers == nil { + indexers = cache.Indexers{} + } + informer := cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return client.Resource(gvr).List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return client.Resource(gvr).Watch(ctx, options) + }, + }, + &metav1.PartialObjectMetadata{}, + opts.SyncTimeout, + indexers, + ) + return informer +} diff --git a/pkg/autodiscover/kubernetes/k8skeystore/kubernetes_keystore.go b/pkg/autodiscover/kubernetes/k8skeystore/kubernetes_keystore.go new file mode 100644 index 000000000000..437db7ce3976 --- /dev/null +++ b/pkg/autodiscover/kubernetes/k8skeystore/kubernetes_keystore.go @@ -0,0 +1,144 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package k8skeystore + +import ( + "context" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8s "k8s.io/client-go/kubernetes" + + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/keystore" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +// KubernetesKeystoresRegistry implements a Provider for Keystore. +type KubernetesKeystoresRegistry struct { + logger *logp.Logger + client k8s.Interface +} + +// KubernetesSecretsKeystore allows to retrieve passwords from Kubernetes secrets for a given namespace +type KubernetesSecretsKeystore struct { + namespace string + client k8s.Interface + logger *logp.Logger +} + +// Factoryk8s Create the right keystore with the configured options +func Factoryk8s(keystoreNamespace string, ks8client k8s.Interface, logger *logp.Logger) (keystore.Keystore, error) { + keystore, err := NewKubernetesSecretsKeystore(keystoreNamespace, ks8client, logger) + return keystore, err +} + +// NewKubernetesKeystoresRegistry initializes a KubernetesKeystoresRegistry +func NewKubernetesKeystoresRegistry(logger *logp.Logger, client k8s.Interface) bus.KeystoreProvider { + return &KubernetesKeystoresRegistry{ + logger: logger, + client: client, + } +} + +// GetKeystore return a KubernetesSecretsKeystore if it already exists for a given namespace or creates a new one. +func (kr *KubernetesKeystoresRegistry) GetKeystore(event bus.Event) keystore.Keystore { + namespace := "" + if val, ok := event["kubernetes"]; ok { + kubernetesMeta, ok := val.(mapstr.M) + if !ok { + kr.logger.Debugf("Unexpected type for kubernetes: %v", kubernetesMeta) + return nil + } + ns, err := kubernetesMeta.GetValue("namespace") + if err != nil { + kr.logger.Debugf("Cannot retrieve kubernetes namespace from event: %s", event) + return nil + } + namespace, ok = ns.(string) + if !ok { + return nil + } + } + if namespace != "" { + k8sKeystore, _ := Factoryk8s(namespace, kr.client, kr.logger) + return k8sKeystore + } + kr.logger.Debugf("Cannot retrieve kubernetes namespace from event: %s", event) + return nil +} + +// NewKubernetesSecretsKeystore returns an new k8s Keystore +func NewKubernetesSecretsKeystore(keystoreNamespace string, ks8client k8s.Interface, logger *logp.Logger) (keystore.Keystore, error) { + keystore := KubernetesSecretsKeystore{ + namespace: keystoreNamespace, + client: ks8client, + logger: logger, + } + return &keystore, nil +} + +// Retrieve return a SecureString instance that will contains both the key and the secret. +func (k *KubernetesSecretsKeystore) Retrieve(key string) (*keystore.SecureString, error) { + // key = "kubernetes.somenamespace.somesecret.value" + tokens := strings.Split(key, ".") + if len(tokens) > 0 && tokens[0] != "kubernetes" { + return nil, keystore.ErrKeyDoesntExists + } + if len(tokens) != 4 { + k.logger.Debugf( + "not valid secret key: %v. Secrets should be of the following format %v", + key, + "kubernetes.somenamespace.somesecret.value", + ) + return nil, keystore.ErrKeyDoesntExists + } + ns := tokens[1] + secretName := tokens[2] + secretVar := tokens[3] + if ns != k.namespace { + k.logger.Debugf("cannot access Kubernetes secrets from a different namespace (%v) than: %v", ns, k.namespace) + return nil, keystore.ErrKeyDoesntExists + } + secretIntefrace := k.client.CoreV1().Secrets(ns) + ctx := context.TODO() + secret, err := secretIntefrace.Get(ctx, secretName, metav1.GetOptions{}) + if err != nil { + k.logger.Errorf("Could not retrieve secret from k8s API: %v", err) + return nil, keystore.ErrKeyDoesntExists + } + if _, ok := secret.Data[secretVar]; !ok { + k.logger.Errorf("Could not retrieve value %v for secret %v", secretVar, secretName) + return nil, keystore.ErrKeyDoesntExists + } + secretString := secret.Data[secretVar] + return keystore.NewSecureString(secretString), nil +} + +// GetConfig returns config.C representation of the key / secret pair to be merged with other +// loaded configuration. +func (k *KubernetesSecretsKeystore) GetConfig() (*config.C, error) { + return nil, nil +} + +// IsPersisted return if the keystore is physically persisted on disk. +func (k *KubernetesSecretsKeystore) IsPersisted() bool { + return true +} diff --git a/pkg/autodiscover/kubernetes/k8skeystore/kubernetes_keystore_test.go b/pkg/autodiscover/kubernetes/k8skeystore/kubernetes_keystore_test.go new file mode 100644 index 000000000000..6097fb3dcdb2 --- /dev/null +++ b/pkg/autodiscover/kubernetes/k8skeystore/kubernetes_keystore_test.go @@ -0,0 +1,146 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package k8skeystore + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/elastic-agent-libs/logp/logptest" + "github.com/elastic/elastic-agent-libs/mapstr" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sfake "k8s.io/client-go/kubernetes/fake" +) + +const ( + ns = "test_namespace" + correctKey = "kubernetes.test_namespace.testing_secret.secret_value" + pass = "testing_passpass" +) + +func TestGetKeystore(t *testing.T) { + kRegistry := NewKubernetesKeystoresRegistry(nil, nil) + k1 := kRegistry.GetKeystore(bus.Event{"kubernetes": mapstr.M{"namespace": "my_namespace"}}) + k2 := kRegistry.GetKeystore(bus.Event{"kubernetes": mapstr.M{"namespace": "my_namespace"}}) + assert.Equal(t, k1, k2) + k3 := kRegistry.GetKeystore(bus.Event{"kubernetes": mapstr.M{"namespace": "my_namespace_2"}}) + assert.NotEqual(t, k2, k3) +} + +func TestGetKeystoreAndRetrieve(t *testing.T) { + client := k8sfake.NewSimpleClientset() + secret := &v1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "apps/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "testing_secret", + Namespace: ns, + }, + Data: map[string][]byte{ + "secret_value": []byte(pass), + }, + } + _, err := client.CoreV1().Secrets(ns).Create(context.Background(), secret, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("failed to create k8s secret: %v", err) + } + + kRegistry := NewKubernetesKeystoresRegistry(nil, client) + k1 := kRegistry.GetKeystore(bus.Event{"kubernetes": mapstr.M{"namespace": ns}}) + secure, err := k1.Retrieve(correctKey) + if err != nil { + t.Fatalf("could not retrieve k8s secret: %v", err) + } + secretVal, err := secure.Get() + assert.NoError(t, err) + bytePassword := []byte(pass) + assert.Equal(t, bytePassword, secretVal) +} + +func TestGetKeystoreAndRetrieveWithNonAllowedNamespace(t *testing.T) { + kRegistry := getKeystoreForWrongValue(t) + k1 := kRegistry.GetKeystore(bus.Event{"kubernetes": mapstr.M{"namespace": ns}}) + key := "kubernetes.test_namespace_HACK.testing_secret.secret_value" + _, err := k1.Retrieve(key) + assert.Error(t, err) +} + +func TestGetKeystoreAndRetrieveWithWrongKeyFormat(t *testing.T) { + kRegistry := getKeystoreForWrongValue(t) + k1 := kRegistry.GetKeystore(bus.Event{"kubernetes": mapstr.M{"namespace": ns}}) + key := "HACK_test_namespace_HACK.testing_secret.secret_value" + _, err := k1.Retrieve(key) + assert.Error(t, err) +} + +func TestGetKeystoreAndRetrieveWithNoSecretsExistent(t *testing.T) { + logger := logptest.NewTestingLogger(t, "test_k8s_secrets") + client := k8sfake.NewSimpleClientset() + + kRegistry := NewKubernetesKeystoresRegistry(logger, client) + k1 := kRegistry.GetKeystore(bus.Event{"kubernetes": mapstr.M{"namespace": ns}}) + _, err := k1.Retrieve(correctKey) + assert.Error(t, err) +} + +func TestGetKeystoreAndRetrieveWithWrongSecretName(t *testing.T) { + kRegistry := getKeystoreForWrongValue(t) + k1 := kRegistry.GetKeystore(bus.Event{"kubernetes": mapstr.M{"namespace": ns}}) + key := "kubernetes.test_namespace.testing_secret_WRONG.secret_value" + _, err := k1.Retrieve(key) + assert.Error(t, err) +} + +func TestGetKeystoreAndRetrieveWithWrongSecretValue(t *testing.T) { + kRegistry := getKeystoreForWrongValue(t) + k1 := kRegistry.GetKeystore(bus.Event{"kubernetes": mapstr.M{"namespace": ns}}) + key := "kubernetes.test_namespace.testing_secret.secret_value_WRONG" + _, err := k1.Retrieve(key) + assert.Error(t, err) +} + +func getKeystoreForWrongValue(t *testing.T) bus.KeystoreProvider { + logger := logptest.NewTestingLogger(t, "test_k8s_secrets") + client := k8sfake.NewSimpleClientset() + secret := &v1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "apps/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "testing_secret", + Namespace: ns, + }, + Data: map[string][]byte{ + "secret_value": []byte(pass), + }, + } + _, err := client.CoreV1().Secrets(ns).Create(context.Background(), secret, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("failed to create k8s secret: %v", err) + } + + return NewKubernetesKeystoresRegistry(logger, client) +} diff --git a/pkg/autodiscover/kubernetes/metadata/config.go b/pkg/autodiscover/kubernetes/metadata/config.go new file mode 100644 index 000000000000..eafbb9dbd3ae --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/config.go @@ -0,0 +1,71 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + "github.com/elastic/elastic-agent-libs/config" +) + +// Config declares supported configuration for metadata generation +type Config struct { + KubeConfig string `config:"kube_config"` + KubeAdm bool `config:"use_kubeadm"` + UseRegexInclude bool `config:"use_regex_include"` + UseRegexExclude bool `config:"use_regex_exclude"` + IncludeLabels []string `config:"include_labels"` + ExcludeLabels []string `config:"exclude_labels"` + IncludeAnnotations []string `config:"include_annotations"` + + LabelsDedot bool `config:"labels.dedot"` + AnnotationsDedot bool `config:"annotations.dedot"` +} + +// AddResourceMetadataConfig allows adding config for enriching additional resources +type AddResourceMetadataConfig struct { + Node *config.C `config:"node"` + Namespace *config.C `config:"namespace"` + Deployment bool `config:"deployment"` + CronJob bool `config:"cronjob"` +} + +// InitDefaults initializes the defaults for the config. +func (c *Config) InitDefaults() { + c.KubeAdm = true + c.LabelsDedot = true + c.AnnotationsDedot = true + c.UseRegexInclude = false + c.UseRegexExclude = false +} + +// Unmarshal unpacks a Config into the metagen Config +func (c *Config) Unmarshal(cfg *config.C) error { + return cfg.Unpack(c) +} + +func GetDefaultResourceMetadataConfig() *AddResourceMetadataConfig { + metaConfig := Config{} + metaConfig.InitDefaults() + nodeCfg, _ := config.NewConfigFrom(metaConfig) + nsCfg, _ := config.NewConfigFrom(metaConfig) + return &AddResourceMetadataConfig{ + Node: nodeCfg, + Namespace: nsCfg, + Deployment: false, + CronJob: false, + } +} diff --git a/pkg/autodiscover/kubernetes/metadata/job.go b/pkg/autodiscover/kubernetes/metadata/job.go new file mode 100644 index 000000000000..7dbdbcfc1073 --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/job.go @@ -0,0 +1,94 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8s "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +type job struct { + store cache.Store + resource *Resource +} + +// NewJobMetadataGenerator creates a metagen for job resources +func NewJobMetadataGenerator(cfg *config.C, jobs cache.Store, client k8s.Interface) MetaGen { + return &job{ + resource: NewResourceMetadataGenerator(cfg, client), + store: jobs, + } +} + +// Generate generates job metadata from a resource object +// Metadata map is in the following form: +// +// { +// "kubernetes": {}, +// "some.ecs.field": "asdf" +// } +// +// All Kubernetes fields that need to be stored under kuberentes. prefix are populetad by +// GenerateK8s method while fields that are part of ECS are generated by GenerateECS method +func (jb *job) Generate(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + ecsFields := jb.GenerateECS(obj) + meta := mapstr.M{ + "kubernetes": jb.GenerateK8s(obj, opts...), + } + meta.DeepUpdate(ecsFields) + return meta +} + +// GenerateECS generates job ECS metadata from a resource object +func (jb *job) GenerateECS(obj kubernetes.Resource) mapstr.M { + return jb.resource.GenerateECS(obj) +} + +// GenerateK8s generates job metadata from a resource object +func (jb *job) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + _, ok := obj.(metav1.Object) + if !ok { + return nil + } + + meta := jb.resource.GenerateK8s("job", obj, opts...) + return meta +} + +// GenerateFromName generates job metadata from a namespace name +func (jb *job) GenerateFromName(name string, opts ...FieldOptions) mapstr.M { + if jb.store == nil { + return nil + } + + if obj, ok, _ := jb.store.GetByKey(name); ok { + res, ok := obj.(kubernetes.Resource) + if !ok { + return nil + } + + return jb.GenerateK8s(res, opts...) + } + + return nil +} diff --git a/pkg/autodiscover/kubernetes/metadata/job_test.go b/pkg/autodiscover/kubernetes/metadata/job_test.go new file mode 100644 index 000000000000..37384a8ae487 --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/job_test.go @@ -0,0 +1,203 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + batchv1 "k8s.io/api/batch/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + k8sfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +func TestJob_Generate(t *testing.T) { + client := k8sfake.NewSimpleClientset() + boolean := true + tests := []struct { + input *batchv1.Job + output mapstr.M + name string + }{ + { + name: "test simple object with owner", + input: &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "CronJob", + Name: "nginx-job", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Job", + APIVersion: "v1", + }, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "job": mapstr.M{ + "name": name, + "uid": uid, + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "cronjob": mapstr.M{ + "name": "nginx-job", + }, + "namespace": defaultNs, + }, + }, + }, + } + + cfg := config.NewConfig() + metagen := NewJobMetadataGenerator(cfg, nil, client) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.Generate(test.input)) + objMeta := &metav1.PartialObjectMetadata{ + ObjectMeta: test.input.ObjectMeta, + } + assert.Equal(t, test.output, metagen.Generate(objMeta)) + }) + } +} + +func TestJob_GenerateFromName(t *testing.T) { + client := k8sfake.NewSimpleClientset() + boolean := true + tests := []struct { + input kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object with owner", + input: &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "CronJob", + Name: "nginx-job", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Job", + APIVersion: "v1", + }, + }, + output: mapstr.M{ + "job": mapstr.M{ + "name": name, + "uid": uid, + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "cronjob": mapstr.M{ + "name": "nginx-job", + }, + "namespace": defaultNs, + }, + }, + { + name: "test simple object with owner", + input: &metav1.PartialObjectMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "CronJob", + Name: "nginx-job", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + }, + output: mapstr.M{ + "job": mapstr.M{ + "name": name, + "uid": uid, + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "cronjob": mapstr.M{ + "name": "nginx-job", + }, + "namespace": defaultNs, + }, + }, + } + + for _, test := range tests { + cfg := config.NewConfig() + jobs := cache.NewStore(cache.MetaNamespaceKeyFunc) + err := jobs.Add(test.input) + require.NoError(t, err) + metagen := NewJobMetadataGenerator(cfg, jobs, client) + + accessor, err := meta.Accessor(test.input) + require.NoError(t, err) + + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.GenerateFromName(fmt.Sprint(accessor.GetNamespace(), "/", accessor.GetName()))) + }) + } +} diff --git a/pkg/autodiscover/kubernetes/metadata/metadata.go b/pkg/autodiscover/kubernetes/metadata/metadata.go new file mode 100644 index 000000000000..17be2460975c --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/metadata.go @@ -0,0 +1,202 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + "context" + "fmt" + "strings" + + "gopkg.in/yaml.v2" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/elastic-agent-libs/safemapstr" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sclient "k8s.io/client-go/kubernetes" + + "k8s.io/client-go/tools/clientcmd" +) + +// MetaGen allows creation of metadata from either Kubernetes resources or their Resource names. +type MetaGen interface { + // Generate generates metadata for a given resource. + // Metadata map is formed in the following format: + // { + // "kubernetes": GenerateK8s(), + // "some.ecs.field": "asdf, // populated by GenerateECS() + // } + // This method is called in top level and returns the complete map of metadata. + Generate(kubernetes.Resource, ...FieldOptions) mapstr.M + // GenerateFromName generates metadata for a given resource based on it's name + GenerateFromName(string, ...FieldOptions) mapstr.M + // GenerateK8s generates kubernetes metadata for a given resource + GenerateK8s(kubernetes.Resource, ...FieldOptions) mapstr.M + // GenerateECS generates ECS metadata for a given resource + GenerateECS(kubernetes.Resource) mapstr.M +} + +// FieldOptions allows additional enrichment to be done on top of existing metadata +type FieldOptions func(mapstr.M) + +type ClusterInfo struct { + URL string + Name string +} + +type ClusterConfiguration struct { + ControlPlaneEndpoint string `yaml:"controlPlaneEndpoint"` + ClusterName string `yaml:"clusterName"` +} + +// WithFields FieldOption allows adding specific fields into the generated metadata +func WithFields(key string, value any) FieldOptions { + return func(meta mapstr.M) { + _ = safemapstr.Put(meta, key, value) + } +} + +// WithMetadata FieldOption allows adding labels and annotations under sub-resource(kind) +// example if kind=namespace namespace.labels key will be added +func WithMetadata(kind string) FieldOptions { + return func(meta mapstr.M) { + if meta["labels"] != nil { + _ = safemapstr.Put(meta, strings.ToLower(kind)+".labels", meta["labels"]) + } + if meta["annotations"] != nil { + _ = safemapstr.Put(meta, strings.ToLower(kind)+".annotations", meta["annotations"]) + } + } +} + +// GetPodMetaGen is a wrapper function that creates a metaGen for pod resource and has embeeded +// nodeMetaGen and namespaceMetaGen +func GetPodMetaGen( + cfg *config.C, + podWatcher kubernetes.Watcher, + nodeWatcher kubernetes.Watcher, + namespaceWatcher kubernetes.Watcher, + replicasetWatcher kubernetes.Watcher, + jobWatcher kubernetes.Watcher, + metaConf *AddResourceMetadataConfig) MetaGen { + var nodeMetaGen, namespaceMetaGen, rsMetaGen, jobMetaGen MetaGen + if nodeWatcher != nil && metaConf.Node.Enabled() { + nodeMetaGen = NewNodeMetadataGenerator(metaConf.Node, nodeWatcher.Store(), nodeWatcher.Client()) + } + if namespaceWatcher != nil && metaConf.Namespace.Enabled() { + namespaceMetaGen = NewNamespaceMetadataGenerator(metaConf.Namespace, namespaceWatcher.Store(), namespaceWatcher.Client()) + } + if replicasetWatcher != nil && metaConf.Deployment { + rsMetaGen = NewReplicasetMetadataGenerator(cfg, replicasetWatcher.Store(), replicasetWatcher.Client()) + } + if jobWatcher != nil && metaConf.CronJob { + jobMetaGen = NewJobMetadataGenerator(cfg, jobWatcher.Store(), jobWatcher.Client()) + } + metaGen := NewPodMetadataGenerator( + cfg, + podWatcher.Store(), + podWatcher.Client(), + nodeMetaGen, + namespaceMetaGen, + rsMetaGen, + jobMetaGen, + metaConf) + return metaGen +} + +// GetKubernetesClusterIdentifier returns ClusterInfo for k8s if available +func GetKubernetesClusterIdentifier(cfg *config.C, client k8sclient.Interface) (ClusterInfo, error) { + // try with kube config file + var c Config + err := c.Unmarshal(cfg) + if err != nil { + return ClusterInfo{}, err + } + clusterInfo, err := getClusterInfoFromKubeConfigFile(c.KubeConfig) + if err == nil { + return clusterInfo, nil + } + + // try with kubeadm-config configmap only if config_kubeAdm == true + clusterInfo, err = getClusterInfoFromKubeadmConfigMap(client, c.KubeAdm) + if err == nil { + return clusterInfo, nil + } + return ClusterInfo{}, fmt.Errorf("unable to retrieve cluster identifiers") +} + +func getClusterInfoFromKubeadmConfigMap(client k8sclient.Interface, kubeadm bool) (ClusterInfo, error) { + clusterInfo := ClusterInfo{} + if client == nil { + return clusterInfo, fmt.Errorf("unable to get cluster identifiers from kubeadm-config") + } + if !kubeadm { + return clusterInfo, nil + } + cm, err := client.CoreV1().ConfigMaps("kube-system").Get(context.TODO(), "kubeadm-config", metav1.GetOptions{}) + if err != nil { + return clusterInfo, fmt.Errorf("unable to get cluster identifiers from kubeadm-config: %w", err) + } + p, ok := cm.Data["ClusterConfiguration"] + if !ok { + return clusterInfo, fmt.Errorf("unable to get cluster identifiers from ClusterConfiguration") + } + + cc := &ClusterConfiguration{} + err = yaml.Unmarshal([]byte(p), cc) + if err != nil { + return ClusterInfo{}, err + } + if cc.ClusterName != "" { + clusterInfo.Name = cc.ClusterName + } + if cc.ControlPlaneEndpoint != "" { + clusterInfo.URL = cc.ControlPlaneEndpoint + } + + return clusterInfo, nil +} + +func getClusterInfoFromKubeConfigFile(kubeconfig string) (ClusterInfo, error) { + if kubeconfig == "" { + kubeconfig = kubernetes.GetKubeConfigEnvironmentVariable() + } + + if kubeconfig == "" { + return ClusterInfo{}, fmt.Errorf("unable to get cluster identifiers from kube_config from env") + } + + cfg, err := kubernetes.BuildConfig(kubeconfig) + if err != nil { + return ClusterInfo{}, fmt.Errorf("unable to build kube config due to error: %w", err) + } + + kubeCfg, err := clientcmd.LoadFromFile(kubeconfig) + if err != nil { + return ClusterInfo{}, fmt.Errorf("unable to load kube_config due to error: %w", err) + } + + for key, element := range kubeCfg.Clusters { + if element.Server == cfg.Host { + return ClusterInfo{element.Server, key}, nil + } + } + return ClusterInfo{}, fmt.Errorf("unable to get cluster identifiers from kube_config") +} diff --git a/pkg/autodiscover/kubernetes/metadata/namespace.go b/pkg/autodiscover/kubernetes/metadata/namespace.go new file mode 100644 index 000000000000..0bde7ac7a24a --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/namespace.go @@ -0,0 +1,132 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + k8s "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +const resource = "namespace" + +type namespace struct { + store cache.Store + resource *Resource +} + +// NewNamespaceMetadataGenerator creates a metagen for namespace resources +func NewNamespaceMetadataGenerator(cfg *config.C, namespaces cache.Store, client k8s.Interface) MetaGen { + return &namespace{ + resource: NewResourceMetadataGenerator(cfg, client), + store: namespaces, + } +} + +// Generate generates pod metadata from a resource object +// Metadata map is in the following form: +// +// { +// "kubernetes": {}, +// "some.ecs.field": "asdf" +// } +// +// All Kubernetes fields that need to be stored under kuberentes. prefix are populetad by +// GenerateK8s method while fields that are part of ECS are generated by GenerateECS method +func (n *namespace) Generate(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + ecsFields := n.GenerateECS(obj) + meta := mapstr.M{ + "kubernetes": n.GenerateK8s(obj, opts...), + } + meta.DeepUpdate(ecsFields) + return meta +} + +// GenerateECS generates namespace ECS metadata from a resource object +func (n *namespace) GenerateECS(obj kubernetes.Resource) mapstr.M { + return n.resource.GenerateECS(obj) +} + +// GenerateK8s generates namespace metadata from a resource object +func (n *namespace) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + _, ok := obj.(*kubernetes.Namespace) + if !ok { + return nil + } + + meta := n.resource.GenerateK8s(resource, obj, opts...) + meta = flattenMetadata(meta) + + // Add extra fields in here if need be + return meta +} + +// GenerateFromName generates pod metadata from a namespace name +func (n *namespace) GenerateFromName(name string, opts ...FieldOptions) mapstr.M { + if n.store == nil { + return nil + } + + if obj, ok, _ := n.store.GetByKey(name); ok { + no, ok := obj.(*kubernetes.Namespace) + if !ok { + return nil + } + + return n.GenerateK8s(no, opts...) + } + + return nil +} + +func flattenMetadata(in mapstr.M) mapstr.M { + out := mapstr.M{} + rawFields, err := in.GetValue(resource) + if err != nil { + return nil + } + + fields, ok := rawFields.(mapstr.M) + if !ok { + return nil + } + for k, v := range fields { + if k == "name" { + out[resource] = v + } else { + out[resource+"_"+k] = v + } + } + + populateFromKeys := []string{"labels", "annotations"} + for _, key := range populateFromKeys { + rawValues, err := in.GetValue(key) + if err != nil { + continue + } + values, ok := rawValues.(mapstr.M) + if ok { + out[resource+"_"+key] = values + } + } + + return out +} diff --git a/pkg/autodiscover/kubernetes/metadata/namespace_test.go b/pkg/autodiscover/kubernetes/metadata/namespace_test.go new file mode 100644 index 000000000000..0b8a0eb56afd --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/namespace_test.go @@ -0,0 +1,173 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + k8sfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +func TestNamespace_Generate(t *testing.T) { + client := k8sfake.NewSimpleClientset() + tests := []struct { + input kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object", + input: &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "spam": "baz", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: "v1", + }, + }, + // Use this for 8.0 + /* + output: mapstr.M{ + "kubernetes": mapstr.M{ + "namespace": mapstr.M{ + "name": name, + "uid": uid, + "labels": mapstr.M{ + "foo": "bar", + }, + }, + }, + },*/ + output: mapstr.M{"kubernetes": mapstr.M{ + "namespace": name, + "namespace_uid": uid, + "namespace_labels": mapstr.M{ + "foo": "bar", + }, + "namespace_annotations": mapstr.M{ + "spam": "baz", + }, + }}, + }, + } + + cfg, err := config.NewConfigFrom(Config{ + IncludeAnnotations: []string{"spam"}, + }) + if err != nil { + t.Fatalf("Could not merge configs") + } + + metagen := NewNamespaceMetadataGenerator(cfg, nil, client) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.Generate(test.input)) + }) + } +} + +func TestNamespace_GenerateFromName(t *testing.T) { + client := k8sfake.NewSimpleClientset() + tests := []struct { + input kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object", + input: &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "spam": "baz", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: "v1", + }, + }, + // Use this for 8.0 + /* + output: mapstr.M{ + "namespace": mapstr.M{ + "name": name, + "uid": uid, + "labels": mapstr.M{ + "foo": "bar", + }, + }, + },*/ + output: mapstr.M{ + "namespace": name, + "namespace_uid": uid, + "namespace_labels": mapstr.M{ + "foo": "bar", + }, + "namespace_annotations": mapstr.M{ + "spam": "baz", + }, + }, + }, + } + + for _, test := range tests { + cfg, err := config.NewConfigFrom(Config{ + IncludeAnnotations: []string{"spam"}, + }) + if err != nil { + t.Fatalf("Could not merge configs") + } + + namespaces := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = namespaces.Add(test.input) + require.NoError(t, err) + metagen := NewNamespaceMetadataGenerator(cfg, namespaces, client) + + accessor, err := meta.Accessor(test.input) + require.NoError(t, err) + + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.GenerateFromName(fmt.Sprint(accessor.GetName()))) + }) + } +} diff --git a/pkg/autodiscover/kubernetes/metadata/node.go b/pkg/autodiscover/kubernetes/metadata/node.go new file mode 100644 index 000000000000..3e678e54b3c6 --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/node.go @@ -0,0 +1,109 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + v1 "k8s.io/api/core/v1" + k8s "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +type node struct { + store cache.Store + resource *Resource +} + +// NewNodeMetadataGenerator creates a metagen for service resources +func NewNodeMetadataGenerator(cfg *config.C, nodes cache.Store, client k8s.Interface) MetaGen { + return &node{ + resource: NewResourceMetadataGenerator(cfg, client), + store: nodes, + } +} + +// Generate generates node metadata from a resource object +// Metadata map is in the following form: +// +// { +// "kubernetes": {}, +// "some.ecs.field": "asdf" +// } +// +// All Kubernetes fields that need to be stored under kuberentes. prefix are populetad by +// GenerateK8s method while fields that are part of ECS are generated by GenerateECS method +func (n *node) Generate(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + ecsFields := n.GenerateECS(obj) + meta := mapstr.M{ + "kubernetes": n.GenerateK8s(obj, opts...), + } + meta.DeepUpdate(ecsFields) + return meta +} + +// GenerateECS generates node ECS metadata from a resource object +func (n *node) GenerateECS(obj kubernetes.Resource) mapstr.M { + return n.resource.GenerateECS(obj) +} + +// GenerateK8s generates node metadata from a resource object +func (n *node) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + node, ok := obj.(*kubernetes.Node) + if !ok { + return nil + } + + meta := n.resource.GenerateK8s("node", obj, opts...) + // Add extra fields in here if need be + hostname := getHostName(node) + if hostname != "" { + _, _ = meta.Put("node.hostname", hostname) + } + return meta +} + +// GenerateFromName generates pod metadata from a service name +func (n *node) GenerateFromName(name string, opts ...FieldOptions) mapstr.M { + if n.store == nil { + return nil + } + + if obj, ok, _ := n.store.GetByKey(name); ok { + no, ok := obj.(*kubernetes.Node) + if !ok { + return nil + } + + return n.GenerateK8s(no, opts...) + } + + return nil +} + +// getHostName returns the HostName address of the node +func getHostName(node *v1.Node) string { + for _, adr := range node.Status.Addresses { + if adr.Type == v1.NodeHostName { + return adr.Address + } + } + return "" +} diff --git a/pkg/autodiscover/kubernetes/metadata/node_test.go b/pkg/autodiscover/kubernetes/metadata/node_test.go new file mode 100644 index 000000000000..c59e5bd90a8d --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/node_test.go @@ -0,0 +1,155 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + k8sfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +func TestNode_Generate(t *testing.T) { + client := k8sfake.NewSimpleClientset() + tests := []struct { + input kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object", + input: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Node", + APIVersion: "v1", + }, + Status: v1.NodeStatus{ + Addresses: []v1.NodeAddress{{Type: v1.NodeHostName, Address: "node1"}}, + }, + }, + output: mapstr.M{"kubernetes": mapstr.M{ + "node": mapstr.M{ + "name": name, + "uid": uid, + "hostname": "node1", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "annotations": mapstr.M{ + "key2": "value2", + }, + }}, + }, + } + + cfg, _ := config.NewConfigFrom(Config{ + IncludeAnnotations: []string{"key2"}, + }) + metagen := NewNodeMetadataGenerator(cfg, nil, client) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.Generate(test.input)) + }) + } +} + +func TestNode_GenerateFromName(t *testing.T) { + client := k8sfake.NewSimpleClientset() + tests := []struct { + input kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object", + input: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "key": "value", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Node", + APIVersion: "v1", + }, + Status: v1.NodeStatus{ + Addresses: []v1.NodeAddress{{Type: v1.NodeHostName, Address: "node1"}}, + }, + }, + output: mapstr.M{ + "node": mapstr.M{ + "name": name, + "uid": uid, + "hostname": "node1", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "annotations": mapstr.M{ + "key": "value", + }, + }, + }, + } + + for _, test := range tests { + cfg, err := config.NewConfigFrom(Config{ + IncludeAnnotations: []string{"key"}, + }) + require.NoError(t, err) + nodes := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = nodes.Add(test.input) + require.NoError(t, err) + metagen := NewNodeMetadataGenerator(cfg, nodes, client) + + accessor, err := meta.Accessor(test.input) + require.NoError(t, err) + + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.GenerateFromName(fmt.Sprint(accessor.GetName()))) + }) + } +} diff --git a/pkg/autodiscover/kubernetes/metadata/pod.go b/pkg/autodiscover/kubernetes/metadata/pod.go new file mode 100644 index 000000000000..51edbc9bba01 --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/pod.go @@ -0,0 +1,159 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + k8s "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +type pod struct { + store cache.Store + client k8s.Interface + node MetaGen + replicaset MetaGen + job MetaGen + resource *Resource + addResourceMetadata *AddResourceMetadataConfig +} + +// NewPodMetadataGenerator creates a metagen for pod resources +func NewPodMetadataGenerator( + cfg *config.C, + pods cache.Store, + client k8s.Interface, + node MetaGen, + namespace MetaGen, + replicaset MetaGen, + job MetaGen, + addResourceMetadata *AddResourceMetadataConfig) MetaGen { + + return &pod{ + resource: NewNamespaceAwareResourceMetadataGenerator(cfg, client, namespace), + store: pods, + node: node, + replicaset: replicaset, + job: job, + client: client, + addResourceMetadata: addResourceMetadata, + } +} + +// Generate generates pod metadata from a resource object +// Metadata map is in the following form: +// +// { +// "kubernetes": {}, +// "some.ecs.field": "asdf" +// } +// +// All Kubernetes fields that need to be stored under kubernetes. prefix are populated by +// GenerateK8s method while fields that are part of ECS are generated by GenerateECS method +func (p *pod) Generate(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + ecsFields := p.GenerateECS(obj) + meta := mapstr.M{ + "kubernetes": p.GenerateK8s(obj, opts...), + } + meta.DeepUpdate(ecsFields) + + return meta +} + +// GenerateECS generates pod ECS metadata from a resource object +func (p *pod) GenerateECS(obj kubernetes.Resource) mapstr.M { + return p.resource.GenerateECS(obj) +} + +// GenerateK8s generates pod metadata from a resource object +func (p *pod) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + po, ok := obj.(*kubernetes.Pod) + if !ok { + return nil + } + + out := p.resource.GenerateK8s("pod", obj, opts...) + + // check if Pod is handled by a ReplicaSet which is controlled by a Deployment. + // The hierarchy there is Deployment->ReplicaSet->Pod. + if p.addResourceMetadata.Deployment { + if p.replicaset != nil { + rsName, _ := out.GetValue("replicaset.name") + if rsName, ok := rsName.(string); ok { + meta := p.replicaset.GenerateFromName(po.Namespace + "/" + rsName) + deploymentName, _ := meta.GetValue("deployment.name") + if deploymentName != "" { + _, _ = out.Put("deployment.name", deploymentName) + } + } + } + } + + // check if Pod is handled by a Job which is controlled by a CronJob. + // The hierarchy there is CronJob->Job->Pod + if p.addResourceMetadata.CronJob { + if p.job != nil { + jobName, _ := out.GetValue("job.name") + if jobName, ok := jobName.(string); ok { + meta := p.job.GenerateFromName(po.Namespace + "/" + jobName) + cronjobName, _ := meta.GetValue("cronjob.name") + if cronjobName != "" { + _, _ = out.Put("cronjob.name", cronjobName) + } + } + } + } + + if p.node != nil { + meta := p.node.GenerateFromName(po.Spec.NodeName, WithMetadata("node")) + if meta != nil { + _, _ = out.Put("node", meta["node"]) + } else { + _, _ = out.Put("node.name", po.Spec.NodeName) + } + } else { + _, _ = out.Put("node.name", po.Spec.NodeName) + } + + if po.Status.PodIP != "" { + _, _ = out.Put("pod.ip", po.Status.PodIP) + } + + return out +} + +// GenerateFromName generates pod metadata from a pod name +func (p *pod) GenerateFromName(name string, opts ...FieldOptions) mapstr.M { + if p.store == nil { + return nil + } + + if obj, ok, _ := p.store.GetByKey(name); ok { + po, ok := obj.(*kubernetes.Pod) + if !ok { + return nil + } + + return p.GenerateK8s(po, opts...) + } + + return nil +} diff --git a/pkg/autodiscover/kubernetes/metadata/pod_test.go b/pkg/autodiscover/kubernetes/metadata/pod_test.go new file mode 100644 index 000000000000..228601db79a5 --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/pod_test.go @@ -0,0 +1,847 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + k8sfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +var addResourceMetadata = GetDefaultResourceMetadataConfig() + +func TestPod_Generate(t *testing.T) { + client := k8sfake.NewSimpleClientset() + boolean := true + rs := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-rs", + Namespace: defaultNs, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "nginx-deployment", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + Spec: appsv1.ReplicaSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "demo", + }, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "demo", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "nginx", + Image: "nginx:1.12", + Ports: []v1.ContainerPort{ + { + Name: "http", + Protocol: v1.ProtocolTCP, + ContainerPort: 80, + }, + }, + }, + }, + }, + }, + }, + } + + _, err := client.AppsV1().ReplicaSets(defaultNs).Create(context.Background(), rs, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("failed to create k8s deployment: %v", err) + } + + tests := []struct { + input kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object", + input: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "app": "production", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + Spec: v1.PodSpec{ + NodeName: "testnode", + }, + Status: v1.PodStatus{PodIP: "127.0.0.5"}, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "pod": mapstr.M{ + "name": "obj", + "uid": uid, + "ip": "127.0.0.5", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "annotations": mapstr.M{ + "app": "production", + }, + "namespace": defaultNs, + "node": mapstr.M{ + "name": "testnode", + }, + }, + }, + }, + { + name: "test object with owner reference to Deployment", + input: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "app": "production", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "owner", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + Spec: v1.PodSpec{ + NodeName: "testnode", + }, + Status: v1.PodStatus{PodIP: "127.0.0.5"}, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "pod": mapstr.M{ + "name": "obj", + "uid": uid, + "ip": "127.0.0.5", + }, + "namespace": defaultNs, + "deployment": mapstr.M{ + "name": "owner", + }, + "node": mapstr.M{ + "name": "testnode", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "annotations": mapstr.M{ + "app": "production", + }, + }, + }, + }, + { + name: "test object with owner reference to DaemonSet", + input: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "app": "production", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "DaemonSet", + Name: "owner", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + Spec: v1.PodSpec{ + NodeName: "testnode", + }, + Status: v1.PodStatus{PodIP: "127.0.0.5"}, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "pod": mapstr.M{ + "name": "obj", + "uid": uid, + "ip": "127.0.0.5", + }, + "namespace": defaultNs, + "daemonset": mapstr.M{ + "name": "owner", + }, + "node": mapstr.M{ + "name": "testnode", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "annotations": mapstr.M{ + "app": "production", + }, + }, + }, + }, + { + name: "test object with owner reference to Job", + input: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "app": "production", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "batch/v1", + Kind: "Job", + Name: "owner", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + Spec: v1.PodSpec{ + NodeName: "testnode", + }, + Status: v1.PodStatus{PodIP: "127.0.0.5"}, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "pod": mapstr.M{ + "name": "obj", + "uid": uid, + "ip": "127.0.0.5", + }, + "namespace": defaultNs, + "job": mapstr.M{ + "name": "owner", + }, + "node": mapstr.M{ + "name": "testnode", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "annotations": mapstr.M{ + "app": "production", + }, + }, + }, + }, + { + name: "test object with owner reference to replicaset", + input: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "app": "production", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "ReplicaSet", + Name: "nginx-rs", + UID: "005f3b90-4b9d-12f8-acf0-31020a8409087", + Controller: &boolean, + }, + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + Spec: v1.PodSpec{ + NodeName: "testnode", + }, + Status: v1.PodStatus{PodIP: "127.0.0.5"}, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "pod": mapstr.M{ + "name": "obj", + "uid": uid, + "ip": "127.0.0.5", + }, + "namespace": defaultNs, + // We comment below block because add_resource_metadata.deployment: false by default + // "deployment": mapstr.M{ + // "name": "nginx-deployment", + // }, + "replicaset": mapstr.M{ + "name": "nginx-rs", + }, + "node": mapstr.M{ + "name": "testnode", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "annotations": mapstr.M{ + "app": "production", + }, + }, + }, + }, + { + name: "test object with owner reference to replicaset honors annotations.dedot: false", + input: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "k8s.app": "production", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "ReplicaSet", + Name: "nginx-rs", + UID: "005f3b90-4b9d-12f8-acf0-31020a8409087", + Controller: &boolean, + }, + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + Spec: v1.PodSpec{ + NodeName: "testnode", + }, + Status: v1.PodStatus{PodIP: "127.0.0.5"}, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "pod": mapstr.M{ + "name": "obj", + "uid": uid, + "ip": "127.0.0.5", + }, + "namespace": defaultNs, + // We comment below block because add_resource_metadata.deployment: false by default + // "deployment": mapstr.M{ + // "name": "nginx-deployment", + // }, + "replicaset": mapstr.M{ + "name": "nginx-rs", + }, + "node": mapstr.M{ + "name": "testnode", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "annotations": mapstr.M{ + "k8s": mapstr.M{"app": "production"}, + }, + }, + }, + }, + } + + config, err := config.NewConfigFrom(map[string]any{ + "include_annotations": []string{"app", "k8s.app"}, + "annotations.dedot": false, + }) + assert.NoError(t, err) + assert.NotEqual(t, *addResourceMetadata.Namespace, *addResourceMetadata.Node) + + replicaSets := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = replicaSets.Add(rs) + require.NoError(t, err) + rsMeta := NewReplicasetMetadataGenerator(config, replicaSets, client) + metagen := NewPodMetadataGenerator(config, nil, client, nil, nil, rsMeta, nil, addResourceMetadata) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.Generate(test.input)) + }) + } +} + +func TestPod_GenerateFromName(t *testing.T) { + client := k8sfake.NewSimpleClientset() + boolean := true + tests := []struct { + input kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object", + input: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "k8s.app": "production", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + Spec: v1.PodSpec{ + NodeName: "testnode", + }, + Status: v1.PodStatus{PodIP: "127.0.0.5"}, + }, + output: mapstr.M{ + "pod": mapstr.M{ + "name": "obj", + "uid": uid, + "ip": "127.0.0.5", + }, + "namespace": defaultNs, + "node": mapstr.M{ + "name": "testnode", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "annotations": mapstr.M{ + "k8s_app": "production", + }, + }, + }, + { + name: "test object with owner reference", + input: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "app": "production", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "owner", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + Spec: v1.PodSpec{ + NodeName: "testnode", + }, + Status: v1.PodStatus{PodIP: "127.0.0.5"}, + }, + output: mapstr.M{ + "pod": mapstr.M{ + "name": "obj", + "uid": uid, + "ip": "127.0.0.5", + }, + "namespace": defaultNs, + "deployment": mapstr.M{ + "name": "owner", + }, + "node": mapstr.M{ + "name": "testnode", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "annotations": mapstr.M{ + "app": "production", + }, + }, + }, + } + + for _, test := range tests { + config, err := config.NewConfigFrom(map[string]any{ + "include_annotations": []string{"app", "k8s.app"}, + }) + assert.NoError(t, err) + pods := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = pods.Add(test.input) + require.NoError(t, err) + metagen := NewPodMetadataGenerator(config, pods, client, nil, nil, nil, nil, addResourceMetadata) + + accessor, err := meta.Accessor(test.input) + require.NoError(t, err) + + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.GenerateFromName(fmt.Sprint(accessor.GetNamespace(), "/", accessor.GetName()))) + }) + } +} + +func TestPod_GenerateWithNodeNamespace(t *testing.T) { + client := k8sfake.NewSimpleClientset() + tests := []struct { + input kubernetes.Resource + node kubernetes.Resource + namespace kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object", + input: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "app": "production", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + Spec: v1.PodSpec{ + NodeName: "testnode", + }, + Status: v1.PodStatus{PodIP: "127.0.0.5"}, + }, + node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testnode", + UID: types.UID(uid), + Labels: map[string]string{ + "nodekey": "nodevalue", + }, + Annotations: map[string]string{}, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Node", + APIVersion: "v1", + }, + Status: v1.NodeStatus{ + Addresses: []v1.NodeAddress{{Type: v1.NodeHostName, Address: "node1"}}, + }, + }, + namespace: &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultNs, + UID: types.UID(uid), + Labels: map[string]string{ + "nskey": "nsvalue", + }, + Annotations: map[string]string{}, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: "v1", + }, + }, + output: mapstr.M{"kubernetes": mapstr.M{ + "pod": mapstr.M{ + "name": "obj", + "uid": uid, + "ip": "127.0.0.5", + }, + "namespace": defaultNs, + "namespace_uid": uid, + "namespace_labels": mapstr.M{ + "nskey": "nsvalue", + }, + "node": mapstr.M{ + "name": "testnode", + "uid": uid, + "labels": mapstr.M{ + "nodekey": "nodevalue", + }, + "hostname": "node1", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "annotations": mapstr.M{ + "app": "production", + }, + }}, + }, + } + + for _, test := range tests { + config, err := config.NewConfigFrom(map[string]any{ + "include_annotations": []string{"app"}, + }) + assert.NoError(t, err) + pods := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = pods.Add(test.input) + require.NoError(t, err) + + nodes := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = nodes.Add(test.node) + require.NoError(t, err) + nodeMeta := NewNodeMetadataGenerator(config, nodes, client) + + namespaces := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = namespaces.Add(test.namespace) + require.NoError(t, err) + nsMeta := NewNamespaceMetadataGenerator(config, namespaces, client) + + metagen := NewPodMetadataGenerator(config, pods, client, nodeMeta, nsMeta, nil, nil, addResourceMetadata) + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.Generate(test.input)) + }) + } +} + +func TestPod_GenerateWithNodeNamespaceWithAddResourceConfig(t *testing.T) { + client := k8sfake.NewSimpleClientset() + boolean := true + + tests := []struct { + input kubernetes.Resource + node kubernetes.Resource + namespace kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object", + input: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "app.kubernetes.io/component": "exporter", + }, + Annotations: map[string]string{ + "app": "production", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "ReplicaSet", + Name: "nginx-rs", + UID: "005f3b90-4b9d-12f8-acf0-31020a8409087", + Controller: &boolean, + }, + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + + Spec: v1.PodSpec{ + NodeName: "testnode", + }, + Status: v1.PodStatus{PodIP: "127.0.0.5"}, + }, + node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testnode", + UID: types.UID(uid), + Labels: map[string]string{ + "nodekey": "nodevalue", + "nodekey2": "nodevalue2", + }, + Annotations: map[string]string{ + "node.annotation": "node.value", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Node", + APIVersion: "v1", + }, + Status: v1.NodeStatus{ + Addresses: []v1.NodeAddress{{Type: v1.NodeHostName, Address: "node1"}}, + }, + }, + namespace: &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultNs, + UID: types.UID(uid), + Labels: map[string]string{ + "app.kubernetes.io/name": "kube-state-metrics", + "nskey2": "nsvalue2", + }, + Annotations: map[string]string{ + "ns.annotation": "ns.value", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: "v1", + }, + }, + output: mapstr.M{"kubernetes": mapstr.M{ + "pod": mapstr.M{ + "name": "obj", + "uid": uid, + "ip": "127.0.0.5", + }, + "namespace": defaultNs, + "namespace_uid": uid, + "namespace_labels": mapstr.M{ + "app_kubernetes_io/name": "kube-state-metrics", + }, + "namespace_annotations": mapstr.M{ + "ns_annotation": "ns.value", + }, + "node": mapstr.M{ + "name": "testnode", + "uid": uid, + "labels": mapstr.M{ + "nodekey2": "nodevalue2", + }, + "hostname": "node1", + "annotations": mapstr.M{ + "node_annotation": "node.value", + }, + }, + "labels": mapstr.M{ + "app_kubernetes_io/component": "exporter", + }, + "annotations": mapstr.M{ + "app": "production", + }, + "replicaset": mapstr.M{ + "name": "nginx-rs", + }, + }}, + }, + } + + for _, test := range tests { + c, err := config.NewConfigFrom(map[string]any{ + "include_annotations": []string{"app"}, + }) + + assert.NoError(t, err) + + namespaceConfig, _ := config.NewConfigFrom(map[string]any{ + "include_labels": []string{"app.kubernetes.io/name"}, + "include_annotations": []string{"ns.annotation"}, + }) + nodeConfig, _ := config.NewConfigFrom(map[string]any{ + "include_labels": []string{"nodekey2"}, + "include_annotations": []string{"node.annotation"}, + }) + metaConfig := AddResourceMetadataConfig{ + Namespace: namespaceConfig, + Node: nodeConfig, + Deployment: false, + } + + pods := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = pods.Add(test.input) + require.NoError(t, err) + + nodes := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = nodes.Add(test.node) + require.NoError(t, err) + nodeMeta := NewNodeMetadataGenerator(nodeConfig, nodes, client) + + namespaces := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = namespaces.Add(test.namespace) + require.NoError(t, err) + nsMeta := NewNamespaceMetadataGenerator(namespaceConfig, namespaces, client) + + metagen := NewPodMetadataGenerator(c, pods, client, nodeMeta, nsMeta, nil, nil, &metaConfig) + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.Generate(test.input)) + }) + } +} diff --git a/pkg/autodiscover/kubernetes/metadata/replicaset.go b/pkg/autodiscover/kubernetes/metadata/replicaset.go new file mode 100644 index 000000000000..40fc86c80db7 --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/replicaset.go @@ -0,0 +1,128 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + "fmt" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8s "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +const resourceType = "replicaset" + +type replicaset struct { + store cache.Store + resource *Resource +} + +// NewReplicasetMetadataGenerator creates a metagen for replicaset resources +func NewReplicasetMetadataGenerator(cfg *config.C, replicasets cache.Store, client k8s.Interface) MetaGen { + return &replicaset{ + resource: NewResourceMetadataGenerator(cfg, client), + store: replicasets, + } +} + +// Generate generates replicaset metadata from a resource object +// Metadata map is in the following form: +// +// { +// "kubernetes": {}, +// "some.ecs.field": "asdf" +// } +// +// All Kubernetes fields that need to be stored under kuberentes. prefix are populetad by +// GenerateK8s method while fields that are part of ECS are generated by GenerateECS method +func (rs *replicaset) Generate(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + ecsFields := rs.GenerateECS(obj) + meta := mapstr.M{ + "kubernetes": rs.GenerateK8s(obj, opts...), + } + meta.DeepUpdate(ecsFields) + return meta +} + +// GenerateECS generates replicaset ECS metadata from a resource object +func (rs *replicaset) GenerateECS(obj kubernetes.Resource) mapstr.M { + return rs.resource.GenerateECS(obj) +} + +// GenerateK8s generates replicaset metadata from a resource object +func (rs *replicaset) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + _, ok := obj.(metav1.Object) + if !ok { + return nil + } + + meta := rs.resource.GenerateK8s(resourceType, obj, opts...) + return meta +} + +// GenerateFromName generates replicaset metadata from a replicaset name +func (rs *replicaset) GenerateFromName(name string, opts ...FieldOptions) mapstr.M { + if rs.store == nil { + return nil + } + + if obj, ok, _ := rs.store.GetByKey(name); ok { + res, ok := obj.(kubernetes.Resource) + if !ok { + return nil + } + + return rs.GenerateK8s(res, opts...) + } + + return nil +} + +// RemoveUnnecessaryReplicaSetData removes all data from a ReplicaSet resource, except what we need to compute +// Pod metadata. This function works for both ReplicaSet and PartialObjectMetadata. +func RemoveUnnecessaryReplicaSetData(obj any) (any, error) { + switch old := obj.(type) { + case *appsv1.ReplicaSet: + transformed := &appsv1.ReplicaSet{ + ObjectMeta: kubernetes.ObjectMeta{ + Name: old.GetName(), + Namespace: old.GetNamespace(), + OwnerReferences: old.GetOwnerReferences(), + ResourceVersion: old.GetResourceVersion(), + }, + } + return transformed, nil + case *metav1.PartialObjectMetadata: + transformed := &metav1.PartialObjectMetadata{ + ObjectMeta: kubernetes.ObjectMeta{ + Name: old.GetName(), + Namespace: old.GetNamespace(), + OwnerReferences: old.GetOwnerReferences(), + ResourceVersion: old.GetResourceVersion(), + }, + } + return transformed, nil + default: + return nil, fmt.Errorf("obj of type %T neither a ReplicaSet nor a PartialObjectMetadata", obj) + } +} diff --git a/pkg/autodiscover/kubernetes/metadata/replicaset_test.go b/pkg/autodiscover/kubernetes/metadata/replicaset_test.go new file mode 100644 index 000000000000..3b4b2b9c9e60 --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/replicaset_test.go @@ -0,0 +1,359 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +func TestReplicaset_Generate(t *testing.T) { + client := k8sfake.NewSimpleClientset() + boolean := true + tests := []struct { + input *appsv1.ReplicaSet + output mapstr.M + name string + }{ + { + name: "test simple object with owner", + input: &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-rs", + Namespace: defaultNs, + UID: uid, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "nginx-deployment", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + Spec: appsv1.ReplicaSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "demo", + }, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "demo", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "nginx", + Image: "nginx:1.12", + Ports: []v1.ContainerPort{ + { + Name: "http", + Protocol: v1.ProtocolTCP, + ContainerPort: 80, + }, + }, + }, + }, + }, + }, + }, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "replicaset": mapstr.M{ + "name": "nginx-rs", + "uid": uid, + }, + "deployment": mapstr.M{ + "name": "nginx-deployment", + }, + "namespace": defaultNs, + }, + }, + }, + } + + cfg := config.NewConfig() + metagen := NewReplicasetMetadataGenerator(cfg, nil, client) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.Generate(test.input)) + // verify this works with just the metadata + objMeta := &metav1.PartialObjectMetadata{ + ObjectMeta: test.input.ObjectMeta, + } + assert.Equal(t, test.output, metagen.Generate(objMeta)) + }) + } +} + +func TestReplicaset_GenerateFromName(t *testing.T) { + client := k8sfake.NewSimpleClientset() + boolean := true + tests := []struct { + input kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object with owner", + input: &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-rs", + Namespace: defaultNs, + UID: uid, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "nginx-deployment", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + Spec: appsv1.ReplicaSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "demo", + }, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "demo", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "nginx", + Image: "nginx:1.12", + Ports: []v1.ContainerPort{ + { + Name: "http", + Protocol: v1.ProtocolTCP, + ContainerPort: 80, + }, + }, + }, + }, + }, + }, + }, + }, + output: mapstr.M{ + "replicaset": mapstr.M{ + "name": "nginx-rs", + "uid": uid, + }, + "deployment": mapstr.M{ + "name": "nginx-deployment", + }, + "namespace": defaultNs, + }, + }, + { + name: "test simple object with owner", + input: &metav1.PartialObjectMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-rs", + Namespace: defaultNs, + UID: uid, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "nginx-deployment", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + }, + output: mapstr.M{ + "replicaset": mapstr.M{ + "name": "nginx-rs", + "uid": uid, + }, + "deployment": mapstr.M{ + "name": "nginx-deployment", + }, + "namespace": defaultNs, + }, + }, + } + + for _, test := range tests { + cfg := config.NewConfig() + replicasets := cache.NewStore(cache.MetaNamespaceKeyFunc) + err := replicasets.Add(test.input) + require.NoError(t, err) + metagen := NewReplicasetMetadataGenerator(cfg, replicasets, client) + + accessor, err := meta.Accessor(test.input) + require.NoError(t, err) + + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.GenerateFromName(fmt.Sprint(accessor.GetNamespace(), "/", accessor.GetName()))) + }) + } +} + +func TestReplicaset_RemoveUnnecessaryData(t *testing.T) { + boolean := true + tests := []struct { + input kubernetes.Resource + output kubernetes.Resource + name string + err error + }{ + { + name: "test simple object with owner", + input: &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-rs", + Namespace: defaultNs, + UID: uid, + ResourceVersion: "688594", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "nginx-deployment", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + Spec: appsv1.ReplicaSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "demo", + }, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "demo", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "nginx", + Image: "nginx:1.12", + Ports: []v1.ContainerPort{ + { + Name: "http", + Protocol: v1.ProtocolTCP, + ContainerPort: 80, + }, + }, + }, + }, + }, + }, + }, + }, + output: &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-rs", + Namespace: defaultNs, + ResourceVersion: "688594", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "nginx-deployment", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + }, + }, + { + name: "test simple object with owner", + input: &metav1.PartialObjectMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-rs", + Namespace: defaultNs, + UID: uid, + ResourceVersion: "688594", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "nginx-deployment", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + }, + output: &metav1.PartialObjectMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-rs", + Namespace: defaultNs, + ResourceVersion: "688594", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "nginx-deployment", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + }, + }, + { + name: "wrong resource type", + input: &v1.Pod{}, + err: fmt.Errorf("obj of type *v1.Pod neither a ReplicaSet nor a PartialObjectMetadata"), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + transformed, err := RemoveUnnecessaryReplicaSetData(test.input) + assert.Equal(t, test.err, err) + assert.Equal(t, test.output, transformed) + }) + } +} diff --git a/pkg/autodiscover/kubernetes/metadata/resource.go b/pkg/autodiscover/kubernetes/metadata/resource.go new file mode 100644 index 000000000000..bded74e7c301 --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/resource.go @@ -0,0 +1,231 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + "regexp" + "strings" + + "k8s.io/apimachinery/pkg/api/meta" + + k8s "k8s.io/client-go/kubernetes" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/utils" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/elastic-agent-libs/safemapstr" +) + +const deploymentType = "Deployment" + +// Resource generates metadata for any kubernetes resource +type Resource struct { + config *Config + clusterInfo ClusterInfo + namespace MetaGen +} + +// NewResourceMetadataGenerator creates a metadata generator for a generic resource +func NewResourceMetadataGenerator(cfg *config.C, client k8s.Interface) *Resource { + var c Config + err := c.Unmarshal(cfg) + if err != nil { + return nil + } + + r := &Resource{ + config: &c, + } + clusterInfo, err := GetKubernetesClusterIdentifier(cfg, client) + if err == nil { + r.clusterInfo = clusterInfo + } + return r +} + +// NewNamespaceAwareResourceMetadataGenerator creates a metadata generator with informatuon about namespace +func NewNamespaceAwareResourceMetadataGenerator(cfg *config.C, client k8s.Interface, namespace MetaGen) *Resource { + r := NewResourceMetadataGenerator(cfg, client) + r.namespace = namespace + return r +} + +// Generate generates metadata from a resource object +// Generate method returns metadata in the following form: +// +// { +// "kubernetes": {}, +// "ecs.a.field": 42, +// } +// +// This method should be called in top level and not as part of other metadata generators. +// For retrieving metadata without 'kubernetes.' prefix one should call GenerateK8s instead. +func (r *Resource) Generate(kind string, obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + ecsFields := r.GenerateECS(obj) + meta := mapstr.M{ + "kubernetes": r.GenerateK8s(kind, obj, opts...), + } + meta.DeepUpdate(ecsFields) + return meta +} + +// GenerateECS generates ECS metadata from a resource object +func (r *Resource) GenerateECS(_ kubernetes.Resource) mapstr.M { + ecsMeta := mapstr.M{} + if r.clusterInfo.URL != "" { + _, _ = ecsMeta.Put("orchestrator.cluster.url", r.clusterInfo.URL) + } + if r.clusterInfo.Name != "" { + _, _ = ecsMeta.Put("orchestrator.cluster.name", r.clusterInfo.Name) + } + return ecsMeta +} + +// GenerateK8s takes a kind and an object and creates metadata for the same +func (r *Resource) GenerateK8s(kind string, obj kubernetes.Resource, options ...FieldOptions) mapstr.M { + accessor, err := meta.Accessor(obj) + if err != nil { + return nil + } + + var labelMap mapstr.M + if len(r.config.IncludeLabels) == 0 { + labelMap = GenerateMap(accessor.GetLabels(), r.config.LabelsDedot) + } else { + labelMap = generateMapSubset(accessor.GetLabels(), r.config.IncludeLabels, r.config.LabelsDedot, r.config.UseRegexInclude) + } + + var labelMaptoExclude mapstr.M + if len(r.config.ExcludeLabels) != 0 { + labelMaptoExclude = generateMapSubset(accessor.GetLabels(), r.config.ExcludeLabels, r.config.LabelsDedot, r.config.UseRegexExclude) + } + + // Exclude any labels that are present in the exclude_labels config + for label := range labelMaptoExclude { + _ = labelMap.Delete(label) + } + + annotationsMap := generateMapSubset(accessor.GetAnnotations(), r.config.IncludeAnnotations, r.config.AnnotationsDedot, r.config.UseRegexInclude) + + meta := mapstr.M{ + strings.ToLower(kind): mapstr.M{ + "name": accessor.GetName(), + "uid": string(accessor.GetUID()), + }, + } + + namespaceName := accessor.GetNamespace() + if namespaceName != "" { + _ = safemapstr.Put(meta, "namespace", namespaceName) + + if r.namespace != nil { + nsMeta := r.namespace.GenerateFromName(namespaceName) + if nsMeta != nil { + meta.DeepUpdate(nsMeta) + } + } + } + + // Add controller metadata if present + for _, ref := range accessor.GetOwnerReferences() { + if ref.Controller != nil && *ref.Controller { + switch ref.Kind { + // grow this list as we keep adding more `state_*` metricsets + case deploymentType, + "ReplicaSet", + "StatefulSet", + "DaemonSet", + "Job", + "CronJob": + _ = safemapstr.Put(meta, strings.ToLower(ref.Kind)+".name", ref.Name) + } + } + } + + if len(labelMap) != 0 { + _ = safemapstr.Put(meta, "labels", labelMap) + } + + if len(annotationsMap) != 0 { + _ = safemapstr.Put(meta, "annotations", annotationsMap) + } + + for _, option := range options { + option(meta) + } + + return meta +} + +func generateMapSubset(input map[string]string, keys []string, dedot bool, useregex bool) mapstr.M { + output := mapstr.M{} + if input == nil { + return output + } + + for _, key := range keys { + //This is the part where use_regex is enabled + if useregex { + pattern, err := regexp.Compile(key) + if err == nil { + for label, value := range input { + matched := pattern.MatchString(label) + if matched { + if dedot { + dedotKey := utils.DeDot(label) + _, _ = output.Put(dedotKey, value) + } else { + _ = safemapstr.Put(output, label, value) + } + } + + } + } + } else { + value, ok := input[key] + if ok { + if dedot { + dedotKey := utils.DeDot(key) + _, _ = output.Put(dedotKey, value) + } else { + _ = safemapstr.Put(output, key, value) + } + } + } + } + return output +} + +func GenerateMap(input map[string]string, dedot bool) mapstr.M { + output := mapstr.M{} + if input == nil { + return output + } + + for k, v := range input { + if dedot { + label := utils.DeDot(k) + _, _ = output.Put(label, v) + } else { + _ = safemapstr.Put(output, k, v) + } + } + + return output +} diff --git a/pkg/autodiscover/kubernetes/metadata/resource_test.go b/pkg/autodiscover/kubernetes/metadata/resource_test.go new file mode 100644 index 000000000000..fe95a8e66358 --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/resource_test.go @@ -0,0 +1,366 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + k8sfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/go-ucfg" +) + +const ( + uid = "005f3b90-4b9d-12f8-acf0-31020a840133" + defaultNs = "default" + name = "obj" +) + +func TestResource_Generate(t *testing.T) { + boolean := true + tests := []struct { + input kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object", + input: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "pod": mapstr.M{ + "name": name, + "uid": uid, + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "namespace": defaultNs, + }, + }, + }, + { + name: "test object with owner reference", + input: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "owner", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "pod": mapstr.M{ + "name": name, + "uid": uid, + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "namespace": defaultNs, + "deployment": mapstr.M{ + "name": "owner", + }, + }, + }, + }, + } + + var cfg Config + err := ucfg.New().Unpack(&cfg) + require.NoError(t, err) + metagen := &Resource{ + config: &cfg, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.Generate("pod", test.input)) + }) + } +} + +func TestNamespaceAwareResource_GenerateWithNamespace(t *testing.T) { + client := k8sfake.NewSimpleClientset() + tests := []struct { + resourceName string + input kubernetes.Resource + namespace kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test not namespaced kubernetes resource - PersistentVolume", + resourceName: "persistentvolume", + input: &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc-18705cfb-9fb8-441f-9b32-0d67a21af839", + UID: "020fd954-3674-496a-9e77-c25f0f2257ea", + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "PersistentVolume", + APIVersion: "v1", + }, + }, + namespace: &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultNs, + UID: types.UID(uid), + Labels: map[string]string{ + "nskey": "nsvalue", + }, + Annotations: map[string]string{ + "ns.annotation": "value", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: "v1", + }, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "persistentvolume": mapstr.M{ + "name": "pvc-18705cfb-9fb8-441f-9b32-0d67a21af839", + "uid": "020fd954-3674-496a-9e77-c25f0f2257ea", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + }, + }, + }, + { + name: "test namespaced kubernetes resource", + resourceName: "deployment", + input: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + UID: "f33ca314-8cc5-48ea-90b7-3102c7430f75", + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + }, + namespace: &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultNs, + UID: types.UID(uid), + Labels: map[string]string{ + "nskey": "nsvalue", + }, + Annotations: map[string]string{ + "ns.annotation": "value", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: "v1", + }, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "deployment": mapstr.M{ + "name": name, + "uid": "f33ca314-8cc5-48ea-90b7-3102c7430f75", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "namespace": "default", + "namespace_uid": uid, + "namespace_labels": mapstr.M{ + "nskey": "nsvalue", + }, + "namespace_annotations": mapstr.M{ + "ns_annotation": "value", + }, + }, + }, + }, + } + + for _, test := range tests { + nsConfig, err := config.NewConfigFrom(map[string]any{ + "include_annotations": []string{"ns.annotation"}, + }) + require.NoError(t, err) + + namespaces := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = namespaces.Add(test.namespace) + require.NoError(t, err) + nsMeta := NewNamespaceMetadataGenerator(nsConfig, namespaces, client) + + metagen := NewNamespaceAwareResourceMetadataGenerator(nsConfig, client, nsMeta) + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.Generate(test.resourceName, test.input)) + }) + } +} + +func Test_generateMapSubset(t *testing.T) { + + Labelsdedot := map[string]string{ + "app.kubernetes.io/name": "no", + "foo": "bar", + } + + Labels := map[string]string{ + "nottomatch": "no", + "foo": "bar", + "foo1": "bar1", + "foo2": "bar2", + "foo-example": "bar-example", + "test": "test1", + "footest": "footest1", + "simplefo-example": "simplefo-example1", + } + + tests := []struct { + name string + key []string + expectedresult mapstr.M + }{ + {name: "Validating wildcards", + key: []string{ + "foo*", + "test", + }, + expectedresult: mapstr.M{ + "foo": "bar", + "foo1": "bar1", + "foo2": "bar2", + "foo-example": "bar-example", + "test": "test1", + "footest": "footest1", + "simplefo-example": "simplefo-example1", + }, + }, { + name: "Validating ?", + key: []string{ + "0?1", + }, + expectedresult: mapstr.M{ + "foo1": "bar1", + }, + }, { + name: "Validating start of a string", + key: []string{ + "^test", + }, + expectedresult: mapstr.M{ + "test": "test1", + }, + }, { + name: "Validating end of a string", + key: []string{ + "test$", + }, + expectedresult: mapstr.M{ + "test": "test1", + "footest": "footest1", + }, + }, { + name: "Exact matches", + key: []string{ + "t{2}", + }, + expectedresult: mapstr.M{ + "nottomatch": "no", + }, + }, { + name: " //Dedot Validation", + key: []string{ + "app.kubernetes.io", + }, + expectedresult: mapstr.M{ + "app_kubernetes_io/name": "no", + }, + }, { + name: "Default code, UsegexInclude=false Validation", + key: []string{ + "app.kubernetes.io/name", + }, + expectedresult: mapstr.M{ + "app_kubernetes_io/name": "no", + }, + }, + } + + for i := 0; i <= 4; i++ { + output := generateMapSubset(Labels, tests[i].key, false, true) + assert.Equal(t, tests[i].expectedresult, output) + } + + output := generateMapSubset(Labelsdedot, tests[5].key, true, true) + assert.Equal(t, tests[5].expectedresult, output) + + output = generateMapSubset(Labelsdedot, tests[6].key, true, false) + assert.Equal(t, tests[6].expectedresult, output) + +} diff --git a/pkg/autodiscover/kubernetes/metadata/service.go b/pkg/autodiscover/kubernetes/metadata/service.go new file mode 100644 index 000000000000..be93ae7554d6 --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/service.go @@ -0,0 +1,104 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + k8s "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/elastic-agent-libs/safemapstr" +) + +type service struct { + store cache.Store + resource *Resource +} + +// NewServiceMetadataGenerator creates a metagen for service resources +func NewServiceMetadataGenerator(cfg *config.C, services cache.Store, namespace MetaGen, client k8s.Interface) MetaGen { + return &service{ + resource: NewNamespaceAwareResourceMetadataGenerator(cfg, client, namespace), + store: services, + } +} + +// Generate generates service metadata from a resource object +// Metadata map is in the following form: +// +// { +// "kubernetes": {}, +// "some.ecs.field": "asdf" +// } +// +// All Kubernetes fields that need to be stored under kuberentes. prefix are populetad by +// GenerateK8s method while fields that are part of ECS are generated by GenerateECS method +func (s *service) Generate(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + ecsFields := s.GenerateECS(obj) + meta := mapstr.M{ + "kubernetes": s.GenerateK8s(obj, opts...), + } + meta.DeepUpdate(ecsFields) + return meta +} + +// GenerateECS generates service ECS metadata from a resource object +func (s *service) GenerateECS(obj kubernetes.Resource) mapstr.M { + return s.resource.GenerateECS(obj) +} + +// GenerateK8s generates service metadata from a resource object +func (s *service) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + svc, ok := obj.(*kubernetes.Service) + if !ok { + return nil + } + + out := s.resource.GenerateK8s("service", obj, opts...) + + selectors := svc.Spec.Selector + if len(selectors) == 0 { + return out + } + svcMap := GenerateMap(selectors, s.resource.config.LabelsDedot) + if len(svcMap) != 0 { + _ = safemapstr.Put(out, "selectors", svcMap) + } + + return out +} + +// GenerateFromName generates pod metadata from a service name +func (s *service) GenerateFromName(name string, opts ...FieldOptions) mapstr.M { + if s.store == nil { + return nil + } + + if obj, ok, _ := s.store.GetByKey(name); ok { + svc, ok := obj.(*kubernetes.Service) + if !ok { + return nil + } + + return s.GenerateK8s(svc, opts...) + } + + return nil +} diff --git a/pkg/autodiscover/kubernetes/metadata/service_test.go b/pkg/autodiscover/kubernetes/metadata/service_test.go new file mode 100644 index 000000000000..63b3556b9d2d --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/service_test.go @@ -0,0 +1,325 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + k8sfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +func TestService_Generate(t *testing.T) { + client := k8sfake.NewSimpleClientset() + boolean := true + tests := []struct { + input kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object", + input: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + Spec: v1.ServiceSpec{ + Selector: map[string]string{ + "app": "istiod", + "istio": "pilot", + }, + }, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "service": mapstr.M{ + "name": name, + "uid": uid, + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "selectors": mapstr.M{ + "app": "istiod", + "istio": "pilot", + }, + "namespace": defaultNs, + }, + }, + }, + { + name: "test object with owner reference", + input: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "owner", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + Spec: v1.ServiceSpec{ + Selector: map[string]string{ + "app": "istiod", + "istio": "pilot", + }, + }, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "service": mapstr.M{ + "name": name, + "uid": uid, + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "selectors": mapstr.M{ + "app": "istiod", + "istio": "pilot", + }, + "namespace": defaultNs, + "deployment": mapstr.M{ + "name": "owner", + }, + }, + }, + }, + } + + cfg := config.NewConfig() + metagen := NewServiceMetadataGenerator(cfg, nil, nil, client) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.Generate(test.input)) + }) + } +} + +func TestService_GenerateFromName(t *testing.T) { + client := k8sfake.NewSimpleClientset() + boolean := true + tests := []struct { + input kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object", + input: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + }, + output: mapstr.M{ + "service": mapstr.M{ + "name": name, + "uid": uid, + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "namespace": defaultNs, + }, + }, + { + name: "test object with owner reference", + input: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "owner", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + }, + output: mapstr.M{ + "service": mapstr.M{ + "name": name, + "uid": uid, + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "namespace": defaultNs, + "deployment": mapstr.M{ + "name": "owner", + }, + }, + }, + } + + for _, test := range tests { + cfg := config.NewConfig() + services := cache.NewStore(cache.MetaNamespaceKeyFunc) + err := services.Add(test.input) + require.NoError(t, err) + metagen := NewServiceMetadataGenerator(cfg, services, nil, client) + + accessor, err := meta.Accessor(test.input) + require.NoError(t, err) + + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.GenerateFromName(fmt.Sprint(accessor.GetNamespace(), "/", accessor.GetName()))) + }) + } +} + +func TestService_GenerateWithNamespace(t *testing.T) { + client := k8sfake.NewSimpleClientset() + tests := []struct { + input kubernetes.Resource + namespace kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object", + input: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + }, + namespace: &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultNs, + UID: types.UID(uid), + Labels: map[string]string{ + "nskey": "nsvalue", + }, + Annotations: map[string]string{ + "ns.annotation": "value", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: "v1", + }, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "service": mapstr.M{ + "name": name, + "uid": uid, + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "namespace": defaultNs, + "namespace_uid": uid, + "namespace_labels": mapstr.M{ + "nskey": "nsvalue", + }, + "namespace_annotations": mapstr.M{ + "ns_annotation": "value", + }, + }, + }, + }, + } + + for _, test := range tests { + nsConfig, err := config.NewConfigFrom(map[string]any{ + "include_annotations": []string{"ns.annotation"}, + }) + require.NoError(t, err) + services := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = services.Add(test.input) + require.NoError(t, err) + + namespaces := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = namespaces.Add(test.namespace) + require.NoError(t, err) + nsMeta := NewNamespaceMetadataGenerator(nsConfig, namespaces, client) + + metagen := NewServiceMetadataGenerator(nsConfig, services, nsMeta, client) + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.Generate(test.input)) + }) + } +} diff --git a/pkg/autodiscover/kubernetes/options.go b/pkg/autodiscover/kubernetes/options.go new file mode 100644 index 000000000000..63850f3f1cca --- /dev/null +++ b/pkg/autodiscover/kubernetes/options.go @@ -0,0 +1,23 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package kubernetes + +type KubeClientOptions struct { + QPS float32 `config:"qps"` + Burst int `config:"burst"` +} diff --git a/pkg/autodiscover/kubernetes/types.go b/pkg/autodiscover/kubernetes/types.go new file mode 100644 index 000000000000..200a2f7b1505 --- /dev/null +++ b/pkg/autodiscover/kubernetes/types.go @@ -0,0 +1,154 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package kubernetes + +import ( + "strings" + "time" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + v1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + rbacv1 "k8s.io/api/rbac/v1" + storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// Resource data +type Resource = runtime.Object + +// ObjectMeta data +type ObjectMeta = metav1.ObjectMeta + +// Pod data +type Pod = v1.Pod + +// PodSpec data +type PodSpec = v1.PodSpec + +// PodStatus data +type PodStatus = v1.PodStatus + +// Node data +type Node = v1.Node + +// Namespace data +type Namespace = v1.Namespace + +// Container data +type Container = v1.Container + +// ContainerPort data +type ContainerPort = v1.ContainerPort + +// Event data +type Event = v1.Event + +// PodContainerStatus data +type PodContainerStatus = v1.ContainerStatus + +// Deployment data +type Deployment = appsv1.Deployment + +// ReplicaSet data +type ReplicaSet = appsv1.ReplicaSet + +// StatefulSet data +type StatefulSet = appsv1.StatefulSet + +// DaemonSet data +type DaemonSet = appsv1.DaemonSet + +// Service data +type Service = v1.Service + +// ServiceAccount data +type ServiceAccount = v1.ServiceAccount + +// Job data +type Job = batchv1.Job + +// CronJob data +type CronJob = batchv1.CronJob + +// PersistentVolume data +type PersistentVolume = v1.PersistentVolume + +// PersistentVolumeClaim data +type PersistentVolumeClaim = v1.PersistentVolumeClaim + +// StorageClass data +type StorageClass = storagev1.StorageClass + +// Role data +type Role = rbacv1.Role + +// RoleBinding data +type RoleBinding = rbacv1.RoleBinding + +// ClusterRole data +type ClusterRole = rbacv1.ClusterRole + +// ClusterRoleBinding data +type ClusterRoleBinding = rbacv1.ClusterRoleBinding + +// NetworkPolicy data +type NetworkPolicy = networkingv1.NetworkPolicy + +const ( + // PodPending phase + PodPending = v1.PodPending + // PodRunning phase + PodRunning = v1.PodRunning + // PodSucceeded phase + PodSucceeded = v1.PodSucceeded + // PodFailed phase + PodFailed = v1.PodFailed + // PodUnknown phase + PodUnknown = v1.PodUnknown +) + +// Time extracts time from k8s.Time type +func Time(t *metav1.Time) time.Time { + return t.Time +} + +// MicroTime extracts time from k8s.MicroTime type +func MicroTime(t *metav1.MicroTime) time.Time { + return t.Time +} + +// ContainerID parses the container ID to get the actual ID string +func ContainerID(s PodContainerStatus) string { + cID, _ := ContainerIDWithRuntime(s) + return cID +} + +// ContainerIDWithRuntime parses the container ID to get the actual ID string +func ContainerIDWithRuntime(s PodContainerStatus) (string, string) { + cID := s.ContainerID + if cID != "" { + parts := strings.Split(cID, "://") + if len(parts) == 2 { + return parts[1], parts[0] + } + } + return "", "" +} diff --git a/pkg/autodiscover/kubernetes/util.go b/pkg/autodiscover/kubernetes/util.go new file mode 100644 index 000000000000..73e9b33c9ac2 --- /dev/null +++ b/pkg/autodiscover/kubernetes/util.go @@ -0,0 +1,374 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package kubernetes + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + + "k8s.io/client-go/metadata" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/elastic-agent-libs/safemapstr" +) + +type HostDiscoveryUtils interface { + GetNamespace() (string, error) + GetPodName() (string, error) + GetMachineID() string +} + +// DiscoverKubernetesNodeParams includes parameters for discovering kubernetes node +type DiscoverKubernetesNodeParams struct { + ConfigHost string + Client kubernetes.Interface + IsInCluster bool + HostUtils HostDiscoveryUtils +} + +// DefaultDiscoveryUtils implements functions of HostDiscoveryUtils interface +type DefaultDiscoveryUtils struct{} + +func GetKubeConfigEnvironmentVariable() string { + envKubeConfig := os.Getenv("KUBECONFIG") + if envKubeConfig == "" { + return "" + } + if _, err := os.Stat(envKubeConfig); !os.IsNotExist(err) { //nolint:gosec // KUBECONFIG is a standard environment variable for kubeconfig path + return envKubeConfig + } + return "" +} + +// GetKubernetesClient returns a kubernetes client. If inCluster is true, it returns an +// in cluster configuration based on the secrets mounted in the Pod. If kubeConfig is passed, +// it parses the config file to get the config required to build a client. +func GetKubernetesClient(kubeconfig string, opt KubeClientOptions) (kubernetes.Interface, error) { + if kubeconfig == "" { + kubeconfig = GetKubeConfigEnvironmentVariable() + } + + cfg, err := BuildConfig(kubeconfig) + if err != nil { + return nil, fmt.Errorf("unable to build kube config due to error: %w", err) + } + cfg.QPS = opt.QPS + cfg.Burst = opt.Burst + client, err := kubernetes.NewForConfig(cfg) + if err != nil { + return nil, fmt.Errorf("unable to build kubernetes clientset: %w", err) + } + + return client, nil +} + +// GetKubernetesMetadataClient returns a kubernetes metadata-only client. If inCluster is true, it returns an +// in cluster configuration based on the secrets mounted in the Pod. If kubeConfig is passed, +// it parses the config file to get the config required to build a client. +func GetKubernetesMetadataClient(kubeconfig string, opt KubeClientOptions) (metadata.Interface, error) { + if kubeconfig == "" { + kubeconfig = GetKubeConfigEnvironmentVariable() + } + + cfg, err := BuildConfig(kubeconfig) + if err != nil { + return nil, fmt.Errorf("unable to build kube config due to error: %w", err) + } + cfg.QPS = opt.QPS + cfg.Burst = opt.Burst + client, err := metadata.NewForConfig(cfg) + if err != nil { + return nil, fmt.Errorf("unable to build kubernetes clientset: %w", err) + } + + return client, nil +} + +// BuildConfig is a helper function that builds configs from a kubeconfig filepath. +// If kubeconfigPath is not passed in we fallback to inClusterConfig. +// If inClusterConfig fails, we fallback to the default config. +// This is a copy of `clientcmd.BuildConfigFromFlags` of `client-go` but without the annoying +// klog messages that are not possible to be disabled. +func BuildConfig(kubeconfigPath string) (*restclient.Config, error) { + if kubeconfigPath == "" { + kubeconfig, err := restclient.InClusterConfig() + if err == nil { + return kubeconfig, nil + } + } + return clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath}, + &clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: ""}}).ClientConfig() +} + +// IsInCluster takes a kubeconfig file path as input and deduces if Beats is running in cluster or not, +// taking into consideration the existence of KUBECONFIG variable +func IsInCluster(kubeconfig string) bool { + if kubeconfig != "" || GetKubeConfigEnvironmentVariable() != "" { + return false + } + return true +} + +// DiscoverKubernetesNode figures out the Kubernetes node to use. +// If host is provided in the config use it directly. +// If it is empty then try +// 1. If beat is deployed in k8s cluster, use hostname of pod as the pod name to query pod metadata for node name. +// 2. If step 1 fails or beat is deployed outside k8s cluster, use NODE_NAME env var. +// 3. If node cannot be discovered with step 1,2, use machine-id to match against k8s nodes for node name. In case it is not set return error. +// Note: There have been cases where machine-id reported by compute instances of some cloud providers where k8s nodes run on, has the wrong value. +func DiscoverKubernetesNode(log *logp.Logger, nd *DiscoverKubernetesNodeParams) (string, error) { + ctx := context.TODO() + // Discover node by configuration file (NODE) if set + if nd.ConfigHost != "" { + log.Infof("kubernetes: Using node %s provided in the config", nd.ConfigHost) + return nd.ConfigHost, nil + } + // Discover node by serviceaccount namespace and pod's hostname in case Beats is running in cluster + if nd.IsInCluster { + node, err := discoverInCluster(nd, ctx) + if err == nil { + log.Infof("kubernetes: Node %s discovered by in cluster pod node query", node) + return node, nil + } + log.Debug(err) + } + + // use environment variable NODE_NAME + node := os.Getenv("NODE_NAME") + if node != "" { + log.Infof("kubernetes: Node %s discovered by NODE_NAME environment variable", node) + return node, nil + } + log.Debug(errors.New("NODE_NAME environment variable is not set")) + + // try discover node by machine id + node, err := discoverByMachineID(nd, ctx) + if err == nil { + log.Infof("kubernetes: Node %s discovered by machine-id matching", node) + return node, nil + } + log.Debug(err) + + return "", errors.New("kubernetes: Node could not be discovered with any known method. Consider setting env var NODE_NAME") +} + +func discoverInCluster(nd *DiscoverKubernetesNodeParams, ctx context.Context) (node string, errorMsg error) { + ns, err := nd.HostUtils.GetNamespace() + if err != nil { + errorMsg = fmt.Errorf("kubernetes: Couldn't get namespace when beat is in cluster with error: %w", err) + return + } + podName, err := nd.HostUtils.GetPodName() + if err != nil { + errorMsg = fmt.Errorf("kubernetes: Couldn't get hostname as beat pod name in cluster with error: %w", err) + return + } + pod, err := nd.Client.CoreV1().Pods(ns).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + errorMsg = fmt.Errorf("kubernetes: Querying for pod failed with error: %w", err) + return + } + return pod.Spec.NodeName, nil +} + +func discoverByMachineID(nd *DiscoverKubernetesNodeParams, ctx context.Context) (nodeName string, errorMsg error) { + mid := nd.HostUtils.GetMachineID() + if mid == "" { + errorMsg = errors.New("kubernetes: Couldn't collect info from any of the files in /etc/machine-id /var/lib/dbus/machine-id") + return + } + + nodes, err := nd.Client.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + errorMsg = fmt.Errorf("kubernetes: Querying for nodes failed with error: %w", err) + return + } + for _, n := range nodes.Items { + if n.Status.NodeInfo.MachineID == mid { + nodeName = n.GetObjectMeta().GetName() + return nodeName, nil + } + } + errorMsg = fmt.Errorf("kubernetes: Couldn't discover node %s", mid) + return nodeName, errorMsg +} + +// GetMachineID returns the machine-idadd_kubernetes_metadata/indexers_test.go +// borrowed from machineID of cadvisor. +func (hd *DefaultDiscoveryUtils) GetMachineID() string { + for _, file := range []string{ + "/etc/machine-id", + "/var/lib/dbus/machine-id", + } { + id, err := os.ReadFile(file) + if err == nil { + return strings.TrimSpace(string(id)) + } + } + return "" +} + +// GetNamespace gets namespace from serviceaccount when beat is in cluster. +func (hd *DefaultDiscoveryUtils) GetNamespace() (string, error) { + return InClusterNamespace() +} + +// GetPodName returns the hostname of the pod +func (hd *DefaultDiscoveryUtils) GetPodName() (string, error) { + return os.Hostname() +} + +// InClusterNamespace gets namespace from serviceaccount when beat is in cluster. // code borrowed from client-go with some changes. +func InClusterNamespace() (string, error) { + // get namespace associated with the service account token, if available + data, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace") + if err != nil { + return "", err + } + return strings.TrimSpace(string(data)), nil +} + +type ContainerInPod struct { + ID string + Runtime string + Spec Container + Status PodContainerStatus +} + +// GetContainersInPod returns all the containers defined in a pod and their statuses. +// It includes init and ephemeral containers. +func GetContainersInPod(pod *Pod) []*ContainerInPod { + containers := make([]*ContainerInPod, len(pod.Spec.Containers)+len(pod.Spec.InitContainers)+len(pod.Spec.EphemeralContainers)) + idx := 0 + for _, c := range pod.Spec.Containers { + containers[idx] = &ContainerInPod{Spec: c} + idx++ + } + for _, c := range pod.Spec.InitContainers { + containers[idx] = &ContainerInPod{Spec: c} + idx++ + } + for _, c := range pod.Spec.EphemeralContainers { + c := Container(c.EphemeralContainerCommon) + containers[idx] = &ContainerInPod{Spec: c} + idx++ + } + + statuses := make(map[string]*PodContainerStatus) + mapStatuses := func(s []PodContainerStatus) { + for i := range s { + statuses[s[i].Name] = &s[i] + } + } + mapStatuses(pod.Status.ContainerStatuses) + mapStatuses(pod.Status.InitContainerStatuses) + mapStatuses(pod.Status.EphemeralContainerStatuses) + for _, c := range containers { + if s, ok := statuses[c.Spec.Name]; ok { + c.ID, c.Runtime = ContainerIDWithRuntime(*s) + c.Status = *s + } + } + + return containers +} + +// PodLabels returns the labels in a pod +func PodLabels(pod *Pod) mapstr.M { + labels := mapstr.M{} + for k, v := range pod.GetObjectMeta().GetLabels() { + _ = safemapstr.Put(labels, k, v) + } + return labels +} + +// PodAnnotations returns the annotations in a pod +func PodAnnotations(pod *Pod) mapstr.M { + annotations := mapstr.M{} + for k, v := range pod.GetObjectMeta().GetAnnotations() { + _ = safemapstr.Put(annotations, k, v) + } + return annotations +} + +// PodNamespaceAnnotations returns the annotations of the namespace of the pod +func PodNamespaceAnnotations(pod *Pod, watcher Watcher) mapstr.M { + if watcher == nil { + return nil + } + + rawNs, ok, err := watcher.Store().GetByKey(pod.Namespace) + if !ok || err != nil { + return nil + } + + namespace, ok := rawNs.(*Namespace) + if !ok { + return nil + } + + annotations := mapstr.M{} + for k, v := range namespace.GetAnnotations() { + _ = safemapstr.Put(annotations, k, v) + } + return annotations +} + +// PodTerminating returns true if a pod is marked for deletion or is in a phase beyond running. +func PodTerminating(pod *Pod) bool { + if pod.GetObjectMeta().GetDeletionTimestamp() != nil { + return true + } + + switch pod.Status.Phase { + case PodRunning, PodPending: + default: + return true + } + + return false +} + +// PodTerminated returns true if a pod is terminated, this method considers a +// pod as terminated if none of its containers are running (or going to be running). +func PodTerminated(pod *Pod, containers []*ContainerInPod) bool { + // Pod is not marked for termination, so it is not terminated. + if !PodTerminating(pod) { + return false + } + + // If any container is running, the pod is not terminated yet. + for _, container := range containers { + if container.Status.State.Running != nil { + return false + } + } + + return true +} diff --git a/pkg/autodiscover/kubernetes/util_test.go b/pkg/autodiscover/kubernetes/util_test.go new file mode 100644 index 000000000000..e6f1d905e0b9 --- /dev/null +++ b/pkg/autodiscover/kubernetes/util_test.go @@ -0,0 +1,334 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package kubernetes + +import ( + "context" + "errors" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" + k8sfake "k8s.io/client-go/kubernetes/fake" + + "github.com/elastic/elastic-agent-libs/logp/logptest" + + core "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +func TestDiscoverKubernetesNode(t *testing.T) { + client := k8sfake.NewSimpleClientset() + logger := logptest.NewTestingLogger(t, "autodiscover.node") + ge := errors.New("kubernetes: Node could not be discovered with any known method. Consider setting env var NODE_NAME") + tests := []struct { + host string + node string + err error + name string + setEnv bool + isInCluster bool + machineid string + podname string + namespace string + init func(*testing.T, kubernetes.Interface) + }{ + { + name: "test value from config", + host: "worker-1", + node: "worker-1", + err: nil, + setEnv: false, + isInCluster: true, + machineid: "", + podname: "", + namespace: "", + }, + { + name: "test value with env var", + host: "", + node: "worker-2", + err: nil, + setEnv: true, + isInCluster: false, + machineid: "", + podname: "", + namespace: "", + }, + { + name: "test value with not incluster, machine ID not retrieved, env var not set", + host: "", + node: "", + err: ge, + setEnv: false, + isInCluster: false, + machineid: "", + podname: "", + namespace: "", + }, + { + name: "test value with inCluster , serviceaccount namespace not found and env var not set", + host: "", + node: "", + err: ge, + setEnv: false, + isInCluster: true, + machineid: "", + podname: "", + namespace: "none", + }, + { + name: "test value with inCluster, pod not found and env var not set", + host: "", + isInCluster: true, + node: "", + err: ge, + setEnv: false, + machineid: "", + podname: "test-pod", + namespace: "default", + }, + { + name: "test value with inCluster, pod found and env var not set", + host: "", + isInCluster: true, + node: "test-node", + err: nil, + setEnv: false, + machineid: "", + podname: "test-pod", + namespace: "default", + init: createResources, + }, + { + name: "test value with inCluster, pod found and env var set", + host: "", + isInCluster: true, + node: "test-node", + err: nil, + setEnv: true, + machineid: "", + podname: "test-pod", + namespace: "default", + init: createResources, + }, + { + name: "test value without inCluster, machine-ID empty and env var not set", + host: "", + isInCluster: false, + node: "", + err: ge, + setEnv: false, + machineid: "", + podname: "", + namespace: "", + }, + { + name: "test value without inCluster, machine-ID set, node not found and env var not set", + host: "", + isInCluster: false, + node: "", + err: ge, + setEnv: false, + machineid: "worker-2", + podname: "", + namespace: "", + }, + { + name: "test value without inCluster, machine-ID set, node found and env var not set", + host: "", + isInCluster: false, + node: "worker-2", + err: nil, + setEnv: false, + machineid: "worker-2", + podname: "", + namespace: "", + init: createResources, + }, + { + name: "test value without inCluster, machine-ID set, node not found and env var set", + host: "", + isInCluster: false, + node: "worker-2", + err: nil, + setEnv: true, + machineid: "worker-2", + podname: "", + namespace: "", + }, + { + name: "test value without inCluster, machine-ID set, node not found and env var set: priority given to the env var as more accurate", + host: "", + isInCluster: false, + node: "worker-2", + err: nil, + setEnv: true, + machineid: "worker-1", + podname: "", + namespace: "", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + + if test.setEnv { + + if err := os.Setenv("NODE_NAME", "worker-2"); err != nil { + t.Fatal(err) + } + defer func() { + if err := os.Unsetenv("NODE_NAME"); err != nil { + t.Fatal(err) + } + }() + } + mdu := createMockdu(test.namespace, test.podname, test.machineid) + if test.init != nil { + test.init(t, client) + } + + var nodeName string + var error error + nd := &DiscoverKubernetesNodeParams{ConfigHost: test.host, Client: client, IsInCluster: test.isInCluster, HostUtils: mdu} + nodeName, error = DiscoverKubernetesNode(logger, nd) + + assert.Equal(t, test.node, nodeName) + if error != nil { + assert.Equal(t, test.err.Error(), error.Error()) + } else { + assert.Equal(t, test.err, error) + } + + }) + } +} + +func createPod(client kubernetes.Interface) error { + pod := getPodObject() + + _, err := client.CoreV1().Pods(pod.Namespace).Create(context.Background(), pod, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create k8s pod: %w", err) + } + return nil +} + +func createResources(t *testing.T, client kubernetes.Interface) { + err := createPod(client) + if err != nil { + t.Fatal(err) + } + + err = createNode(client, "worker-2") + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { + pod := "test-pod" + err := client.CoreV1().Pods("default").Delete(context.Background(), pod, metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("failed to delete k8s pod: %+v", err) + } + + err = client.CoreV1().Nodes().Delete(context.Background(), "worker-2", metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("failed to delete k8s node: %+v", err) + } + + }) +} + +func createNode(client kubernetes.Interface, name string) error { + node := getNodeObject(name) + + _, err := client.CoreV1().Nodes().Create(context.Background(), node, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create k8s node: %w", err) + } + return nil +} + +func getPodObject() *core.Pod { + return &core.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Labels: map[string]string{ + "app": "demo", + }, + }, + Spec: core.PodSpec{ + NodeName: "test-node", + Containers: []core.Container{ + { + Name: "busybox", + Image: "busybox", + ImagePullPolicy: core.PullIfNotPresent, + Command: []string{ + "sleep", + "3600", + }, + }, + }, + }, + } +} + +func getNodeObject(name string) *core.Node { + return &core.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "name": name, + }, + }, + Spec: core.NodeSpec{}, + Status: core.NodeStatus{NodeInfo: core.NodeSystemInfo{MachineID: name}}, + } +} + +func createMockdu(namespace, podname, machineid string) *mockDiscoveryUtils { + return &mockDiscoveryUtils{namespace: namespace, podname: podname, machineid: machineid} +} + +type mockDiscoveryUtils struct { + namespace string + podname string + machineid string +} + +func (hd *mockDiscoveryUtils) GetMachineID() string { + return hd.machineid +} + +func (hd *mockDiscoveryUtils) GetNamespace() (string, error) { + var error error + if hd.namespace == "none" { + error = errors.New("open /var/run/secrets/kubernetes.io/serviceaccount/namespace: no such file or directory") + } + return hd.namespace, error +} + +func (hd *mockDiscoveryUtils) GetPodName() (string, error) { + return hd.podname, nil +} diff --git a/pkg/autodiscover/kubernetes/watcher.go b/pkg/autodiscover/kubernetes/watcher.go new file mode 100644 index 000000000000..75f2bc489a07 --- /dev/null +++ b/pkg/autodiscover/kubernetes/watcher.go @@ -0,0 +1,375 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package kubernetes + +import ( + "context" + "fmt" + "time" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/metadata" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + + "github.com/elastic/elastic-agent-libs/logp" +) + +const ( + add = "add" + update = "update" + delete = "delete" +) + +var ( + accessor = meta.NewAccessor() +) + +// Watcher watches Kubernetes resources events +type Watcher interface { + // Start watching Kubernetes API for new events after resources were listed + Start() error + + // Stop watching Kubernetes API for new events + Stop() + + // AddEventHandler add event handlers for corresponding event type watched + AddEventHandler(ResourceEventHandler) + + // GetEventHandler returns the event handlers for corresponding event type watched + GetEventHandler() ResourceEventHandler + + // Store returns the store object for the watcher + Store() cache.Store + + // Client returns the kubernetes client object used by the watcher + Client() kubernetes.Interface + + // CachedObject returns the old object before change during the last updated event + CachedObject() runtime.Object +} + +// WatchOptions controls watch behaviors +type WatchOptions struct { + // SyncTimeout is a timeout for listing historical resources + SyncTimeout time.Duration + // Node is used for filtering watched resource to given node, use "" for all nodes + Node string + // Namespace is used for filtering watched resource to given namespace, use "" for all namespaces + Namespace string + // IsUpdated allows registering a func that allows the invoker of the Watch to decide what amounts to an update + // vs what does not. + IsUpdated func(old, new any) bool + // HonorReSyncs allows resync events to be requeued on the worker + HonorReSyncs bool +} + +type item struct { + object any + objectRaw any + state string +} + +type watcher struct { + client kubernetes.Interface + informer cache.SharedInformer + store cache.Store + queue workqueue.Interface //nolint:staticcheck // client-go workqueue API + ctx context.Context + stop context.CancelFunc + handler ResourceEventHandler + logger *logp.Logger + cachedObject runtime.Object +} + +// NewWatcher initializes the watcher client to provide a events handler for +// resource from the cluster (filtered to the given node) +// Note: This watcher won't emit workqueue metrics. Use NewNamedWatcher to provide an explicit queue name. +func NewWatcher(client kubernetes.Interface, resource Resource, opts WatchOptions, indexers cache.Indexers, logger *logp.Logger) (Watcher, error) { + return NewNamedWatcher("", client, resource, opts, indexers, logger) +} + +// NewNamedWatcher initializes the watcher client to provide an events handler for +// resource from the cluster (filtered to the given node) and also allows to name the k8s +// client's workqueue that is used by the watcher. Workqueue name is important for exposing workqueue +// metrics, if it is empty, its metrics will not be logged by the k8s client. +func NewNamedWatcher(name string, client kubernetes.Interface, resource Resource, opts WatchOptions, indexers cache.Indexers, logger *logp.Logger) (Watcher, error) { + informer, _, err := NewInformer(client, resource, opts, indexers) + if err != nil { + return nil, err + } + return NewNamedWatcherWithInformer(name, client, resource, informer, logger, opts) +} + +// NewNamedWatcherWithInformer initializes the watcher client to provide an events handler for +// resource from the cluster (filtered to the given node) and also allows to name the k8s +// client's workqueue that is used by the watcher. Workqueue name is important for exposing workqueue +// metrics, if it is empty, its metrics will not be logged by the k8s client. +// This function requires the underlying informer to be passed by the caller. +func NewNamedWatcherWithInformer( + name string, + client kubernetes.Interface, + resource Resource, + informer cache.SharedInformer, + logger *logp.Logger, + opts WatchOptions, +) (Watcher, error) { + var store cache.Store + var queue workqueue.Interface //nolint:staticcheck // client-go workqueue API + var cachedObject runtime.Object + + store = informer.GetStore() + queue = workqueue.NewNamed(name) + + if opts.IsUpdated == nil { + opts.IsUpdated = func(o, n any) bool { + oldObj, ok := o.(runtime.Object) + if !ok { + return true + } + newObj, ok := n.(runtime.Object) + if !ok { + return true + } + old, _ := accessor.ResourceVersion(oldObj) + new, _ := accessor.ResourceVersion(newObj) + + // Only enqueue changes that have a different resource versions to avoid processing resyncs. + return old != new + } + } + + ctx, cancel := context.WithCancel(context.TODO()) + w := &watcher{ + client: client, + informer: informer, + store: store, + queue: queue, + ctx: ctx, + cachedObject: cachedObject, + stop: cancel, + logger: logger.Named("kubernetes"), + handler: NoOpEventHandlerFuncs{}, + } + + _, err := w.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(o any) { + w.enqueue(o, add) + }, + DeleteFunc: func(o any) { + w.enqueue(o, delete) + }, + UpdateFunc: func(o, n any) { + if opts.IsUpdated(o, n) { + w.enqueue(n, update) + } else if opts.HonorReSyncs { + // HonorReSyncs ensure that at the time when the kubernetes client does a "resync", i.e, a full list of all + // objects we make sure that autodiscover processes them. Why is this necessary? An effective control loop works + // based on two state changes, a list and a watch. A watch is triggered each time the state of the system changes. + // However, there is no guarantee that all events from a watch are processed by the receiver. To ensure that missed events + // are properly handled, a period re-list is done to ensure that every state within the system is effectively handled. + // In this case, we are making sure that we are enqueueing an "add" event because, an runner that is already in Running + // state should just be deduped by autodiscover and not stop/started periodically as would be the case with an update. + w.enqueue(n, add) + } + + //We check the type of resource and only if it is namespace or node return the cacheObject + switch resource.(type) { + case *Namespace: + w.cacheObject(o) + case *Node: + w.cacheObject(o) + } + }, + }) + if err != nil { + return nil, err + } + + return w, nil +} + +// NewMetadataWatcher initializes a metadata-only watcher client to provide an events handler for +// resource from the cluster (filtered to the given node). +// Event handlers defined on this watcher receive PartialObjectMetadata resources. +// Note: This watcher won't emit workqueue metrics. Use NewNamedWatcher to provide an explicit queue name. +func NewMetadataWatcher( + client kubernetes.Interface, + metadataClient metadata.Interface, + gvr schema.GroupVersionResource, + opts WatchOptions, + indexers cache.Indexers, + transformFunc cache.TransformFunc, + logger *logp.Logger, +) (Watcher, error) { + return NewNamedMetadataWatcher("", client, metadataClient, gvr, opts, indexers, transformFunc, logger) +} + +// NewNamedMetadataWatcher initializes a metadata-only watcher client to provide an events handler for +// resource from the cluster (filtered to the given node) and also allows to name the k8s +// client's workqueue that is used by the watcher. Workqueue name is important for exposing workqueue +// metrics, if it is empty, its metrics will not be logged by the k8s client. +// Event handlers defined on this watcher receive PartialObjectMetadata resources. +func NewNamedMetadataWatcher( + name string, + client kubernetes.Interface, + metadataClient metadata.Interface, + gvr schema.GroupVersionResource, + opts WatchOptions, + indexers cache.Indexers, + transformFunc cache.TransformFunc, + logger *logp.Logger, +) (Watcher, error) { + informer := NewMetadataInformer(metadataClient, gvr, opts, indexers) + + if transformFunc != nil { + err := informer.SetTransform(transformFunc) + if err != nil { + return nil, err + } + } + + return NewNamedWatcherWithInformer(name, client, &v1.PartialObjectMetadata{}, informer, logger, opts) +} + +// AddEventHandler adds a resource handler to process each request that is coming into the watcher +func (w *watcher) AddEventHandler(h ResourceEventHandler) { + w.handler = h +} + +// GetEventHandler returns the watcher's event handler +func (w *watcher) GetEventHandler() ResourceEventHandler { + return w.handler +} + +// Store returns the store object for the resource that is being watched +func (w *watcher) Store() cache.Store { + return w.store +} + +// Client returns the kubernetes client object used by the watcher +func (w *watcher) Client() kubernetes.Interface { + return w.client +} + +// CachedObject returns the old object in cache during the last updated event +func (w *watcher) CachedObject() runtime.Object { + return w.cachedObject +} + +// Start watching pods +func (w *watcher) Start() error { + go w.informer.Run(w.ctx.Done()) + + if !cache.WaitForCacheSync(w.ctx.Done(), w.informer.HasSynced) { + return fmt.Errorf("kubernetes informer unable to sync cache") + } + + w.logger.Debugf("cache sync done") + + // Wrap the process function with wait.Until so that if the controller crashes, it starts up again after a second. + go wait.Until(func() { + for w.process(w.ctx) { + } + }, time.Second*1, w.ctx.Done()) + + return nil +} + +func (w *watcher) Stop() { + w.queue.ShutDown() + w.stop() +} + +// enqueue takes the most recent object that was received, figures out the namespace/name of the object +// and adds it to the work queue for processing. +func (w *watcher) enqueue(obj any, state string) { + // DeletionHandlingMetaNamespaceKeyFunc that we get a key only if the resource's state is not Unknown. + key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) + if err != nil { + return + } + if deleted, ok := obj.(cache.DeletedFinalStateUnknown); ok { + w.logger.Debugf("Enqueued DeletedFinalStateUnknown contained object: %+v", deleted.Obj) + obj = deleted.Obj + } + w.queue.Add(&item{key, obj, state}) +} + +// cacheObject updates watcher with the old version of cache objects before change during update events +func (w *watcher) cacheObject(o any) { + if old, ok := o.(runtime.Object); !ok { + utilruntime.HandleError(fmt.Errorf("expected object in cache got %#v", o)) + } else { + w.cachedObject = old + } +} + +// process gets the top of the work queue and processes the object that is received. +func (w *watcher) process(_ context.Context) bool { + obj, quit := w.queue.Get() + if quit { + return false + } + defer w.queue.Done(obj) + + var entry *item + var ok bool + if entry, ok = obj.(*item); !ok { + utilruntime.HandleError(fmt.Errorf("expected *item in workqueue but got %#v", obj)) + return true + } + + key, ok := entry.object.(string) + if !ok { + return false + } + + o, exists, err := w.store.GetByKey(key) + if err != nil { + utilruntime.HandleError(fmt.Errorf("getting object %#v from cache: %w", obj, err)) + return true + } + if !exists { + if entry.state == delete { + w.logger.Debugf("Object %+v was not found in the store, deleting anyway!", key) + // delete anyway in order to clean states + w.handler.OnDelete(entry.objectRaw) + } + return true + } + + switch entry.state { + case add: + w.handler.OnAdd(o) + case update: + w.handler.OnUpdate(o) + case delete: + w.handler.OnDelete(o) + } + + return true +} diff --git a/pkg/autodiscover/kubernetes/watcher_test.go b/pkg/autodiscover/kubernetes/watcher_test.go new file mode 100644 index 000000000000..1edd3caa833d --- /dev/null +++ b/pkg/autodiscover/kubernetes/watcher_test.go @@ -0,0 +1,167 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package kubernetes + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" + cachetest "k8s.io/client-go/tools/cache/testing" + + "github.com/elastic/elastic-agent-libs/logp/logptest" +) + +func TestWatcherStartAndStop(t *testing.T) { + client := fake.NewSimpleClientset() + listWatch := cachetest.NewFakeControllerSource() + resource := &Pod{} + informer := cache.NewSharedInformer(listWatch, resource, 0) + watcher, err := NewNamedWatcherWithInformer("test", client, resource, informer, logptest.NewTestingLogger(t, ""), WatchOptions{}) + require.NoError(t, err) + require.NoError(t, watcher.Start()) + watcher.Stop() +} + +func TestWatcherHandlers(t *testing.T) { + client := fake.NewSimpleClientset() + listWatch := cachetest.NewFakeControllerSource() + resource := &Pod{} + informer := cache.NewSharedInformer(listWatch, resource, 0) + watcher, err := NewNamedWatcherWithInformer("test", client, resource, informer, logptest.NewTestingLogger(t, ""), WatchOptions{}) + require.NoError(t, err) + + var added, updated, deleted bool + + watcher.AddEventHandler(ResourceEventHandlerFuncs{ + AddFunc: func(obj any) { + added = true + }, + UpdateFunc: func(obj any) { + updated = true + }, + DeleteFunc: func(obj any) { + deleted = true + }, + }) + + require.NoError(t, watcher.Start()) + defer watcher.Stop() + + pod := &Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + UID: types.UID("poduid"), + Namespace: "test", + ResourceVersion: "1", + }, + } + // add a resource + listWatch.Add(pod) + assert.Eventually(t, func() bool { + return added + }, time.Second*5, time.Millisecond) + + // update the resource + modifiedPod := pod.DeepCopy() + modifiedPod.SetResourceVersion("2") + listWatch.Modify(modifiedPod) + assert.Eventually(t, func() bool { + return updated + }, time.Second*5, time.Millisecond) + + // delete the resource + listWatch.Delete(modifiedPod) + assert.Eventually(t, func() bool { + return deleted + }, time.Second*5, time.Millisecond) +} + +func TestWatcherIsUpdated(t *testing.T) { + client := fake.NewSimpleClientset() + listWatch := cachetest.NewFakeControllerSource() + resource := &Pod{} + informer := cache.NewSharedInformer(listWatch, resource, 0) + // set a custom IsUpdated that always returns true + watcher, err := NewNamedWatcherWithInformer("test", client, resource, informer, + logptest.NewTestingLogger(t, ""), + WatchOptions{IsUpdated: func(old, new any) bool { + return true + }}) + require.NoError(t, err) + + var updated bool + + watcher.AddEventHandler(ResourceEventHandlerFuncs{ + UpdateFunc: func(obj any) { + updated = true + }, + }) + + require.NoError(t, watcher.Start()) + defer watcher.Stop() + + pod := &Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + UID: types.UID("poduid"), + Namespace: "test", + }, + } + listWatch.Add(pod) + + // update the resource, but don't actually change it + // with the default IsUpdated, our handler wouldn't be called, but with our custom one, it will + modifiedPod := pod.DeepCopy() + listWatch.Modify(modifiedPod) + assert.Eventually(t, func() bool { + return updated + }, time.Second*5, time.Millisecond) + +} + +func TestCachedObject(t *testing.T) { + t.Skip("Currently bugged, and not used anywhere") + client := fake.NewSimpleClientset() + listWatch := cachetest.NewFakeControllerSource() + resource := &Namespace{} + informer := cache.NewSharedInformer(listWatch, resource, 0) + watcher, err := NewNamedWatcherWithInformer("test", client, resource, informer, logptest.NewTestingLogger(t, ""), WatchOptions{}) + require.NoError(t, err) + + require.NoError(t, watcher.Start()) + defer watcher.Stop() + + namespace := &Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + UID: types.UID("poduid"), + Namespace: "test", + ResourceVersion: "1", + }, + } + listWatch.Add(namespace) + assert.EventuallyWithT(t, func(collectT *assert.CollectT) { + assert.Equal(collectT, namespace, watcher.CachedObject()) + }, time.Second*5, time.Millisecond) +} diff --git a/pkg/autodiscover/utils/hints.go b/pkg/autodiscover/utils/hints.go new file mode 100644 index 000000000000..5dbba4696ee5 --- /dev/null +++ b/pkg/autodiscover/utils/hints.go @@ -0,0 +1,423 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import ( + "encoding/json" + "fmt" + "regexp" + "slices" + "sort" + "strconv" + "strings" + + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +const logName = "hints" + +// GetContainerID returns the id of a container +func GetContainerID(container mapstr.M) string { + id, _ := container["id"].(string) + return id +} + +// GetContainerName returns the name of a container +func GetContainerName(container mapstr.M) string { + name, _ := container["name"].(string) + return name +} + +// GetHintString takes a hint and returns its value as a string +func GetHintString(hints mapstr.M, key, config string) string { + base := config + if base == "" { + base = key + } else if key != "" { + base = fmt.Sprint(key, ".", config) + } + if iface, err := hints.GetValue(base); err == nil { + if str, ok := iface.(string); ok { + return str + } + } + + return "" +} + +// GetHintMapStr takes a hint and returns a MapStr +func GetHintMapStr(hints mapstr.M, key, config string) mapstr.M { + base := config + if base == "" { + base = key + } else if key != "" { + base = fmt.Sprint(key, ".", config) + } + if iface, err := hints.GetValue(base); err == nil { + if mapstr, ok := iface.(mapstr.M); ok { + return mapstr + } + } + + return nil +} + +// GetHintAsList takes a hint and returns the value as lists. +func GetHintAsList(hints mapstr.M, key, config string) []string { + if str := GetHintString(hints, key, config); str != "" { + return getStringAsList(str) + } + + return nil +} + +// GetProcessors gets processor definitions from the hints and returns a list of configs as a MapStr +func GetProcessors(hints mapstr.M, key string, logger *logp.Logger) []mapstr.M { + processors := GetConfigs(hints, key, "processors") + for _, proc := range processors { + for key, value := range proc { + if str, ok := value.(string); ok { + cfg := mapstr.M{} + if err := json.Unmarshal([]byte(str), &cfg); err != nil { + logger.Named(logName).Debugw("Unable to unmarshal json due to error", "error", err) + continue + } + proc[key] = cfg + } + } + } + return processors +} + +// GetConfigs takes in a key and returns a list of configs as a slice of MapStr +func GetConfigs(hints mapstr.M, key, name string) []mapstr.M { + raw := GetHintMapStr(hints, key, name) + if raw == nil { + return nil + } + + var words, nums []string + + for key := range raw { + if _, err := strconv.Atoi(key); err != nil { + words = append(words, key) + continue + } else { + nums = append(nums, key) + } + } + + sort.Strings(nums) + + var configs []mapstr.M + for _, key := range nums { + rawCfg := raw[key] + if config, ok := rawCfg.(mapstr.M); ok { + configs = append(configs, config) + } + } + + for _, word := range words { + configs = append(configs, mapstr.M{ + word: raw[word], + }) + } + + return configs +} + +func getStringAsList(input string) []string { + if input == "" { + return []string{} + } + list := strings.Split(input, ",") + + for i := range list { + list[i] = strings.TrimSpace(list[i]) + } + + return list +} + +// GetHintAsConfigs can read a hint in the form of a stringified JSON and return a mapstr.M +func GetHintAsConfigs(hints mapstr.M, key string, logger *logp.Logger) []mapstr.M { + if str := GetHintString(hints, key, "raw"); str != "" { + // check if it is a single config + if str[0] != '[' { + cfg := mapstr.M{} + if err := json.Unmarshal([]byte(str), &cfg); err != nil { + logger.Named(logName).Debugw("Unable to unmarshal json due to error", "error", err) + return nil + } + return []mapstr.M{cfg} + } + + var cfg []mapstr.M + if err := json.Unmarshal([]byte(str), &cfg); err != nil { + logger.Named(logName).Debugw("Unable to unmarshal json due to error", "error", err) + return nil + } + return cfg + } + return nil +} + +// IsEnabled will return true when 'enabled' is **explicitly** set to true. +func IsEnabled(hints mapstr.M, key string) bool { + if value, err := hints.GetValue(fmt.Sprintf("%s.enabled", key)); err == nil { + str, ok := value.(string) + if !ok { + return false + } + enabled, _ := strconv.ParseBool(str) + return enabled + } + + return false +} + +// IsDisabled will return true when 'enabled' is **explicitly** set to false. +func IsDisabled(hints mapstr.M, key string, logger *logp.Logger) bool { + if value, err := hints.GetValue(fmt.Sprintf("%s.enabled", key)); err == nil { + str, ok := value.(string) + if !ok { + return false + } + enabled, err := strconv.ParseBool(str) + if err != nil { + logger.Named(logName).Debugw("Error parsing 'enabled' hint.", + "error", err, "autodiscover.hints", hints) + return false + } + return !enabled + } + + return false +} + +// GenerateHints parses annotations based on a prefix and sets up hints that can be picked up by individual Beats. +// Arguments: annotations: include the annotatons defined in the container, container: is the container name, +// prefix: the prefix of the annotation to check, validate: boolean variable that enables the validation of hints format and allSupportedHints: list of supported annotations to validate against +func GenerateHints(annotations mapstr.M, container, prefix string, validate bool, allSupportedHints []string) (mapstr.M, []string) { + hints := mapstr.M{} + var incorrecthints []string + var incorrecthint string + var digitCheck = regexp.MustCompile(`^[0-9]+$`) + + if rawEntries, err := annotations.GetValue(prefix); err == nil { + if entries, ok := rawEntries.(mapstr.M); ok { + + if validate { + //Start of Annotation Check: whether the annotation follows the supported format and vocabulary. The check happens for annotations that have prefix co.elastic + datastreamlist := GetHintAsList(entries, logName+"/"+"data_streams", "") + // We check if multiple data_streams are defined and we retrieve the hints per data_stream. Only applicable in elastic-agent + // See Metrics_apache_package_and_specific_config_per_datastream test case in hints_test.go + for _, stream := range datastreamlist { + allSupportedHints = append(allSupportedHints, stream) + incorrecthints = checkSupportedHintsSets(annotations, prefix, stream, logName, allSupportedHints, incorrecthints) + } + metricsetlist := GetHintAsList(entries, "metrics"+"/"+"metricsets", "") + // We check if multiple metrcisets are defined and we retrieve the hints per metricset. Only applicable in beats + //See Metrics_istio_module_and_specific_config_per_metricset test case in hints_test.go + for _, metric := range metricsetlist { + allSupportedHints = append(allSupportedHints, metric) + incorrecthints = checkSupportedHintsSets(annotations, prefix, metric, "metrics", allSupportedHints, incorrecthints) + } + } //end of annotation check + + for key, rawValue := range entries { + enumeratedmodules := []string{} + // If there are top level hints like co.elastic.logs/ then just add the values after the / + // Only consider namespaced annotations + parts := strings.Split(key, "/") + if len(parts) == 2 { + hintKey := fmt.Sprintf("%s.%s", parts[0], parts[1]) + + if validate { + checkdigit := digitCheck.MatchString(parts[1]) // With this regex we check if enumeration for modules is provided + if checkdigit { + allSupportedHints = append(allSupportedHints, parts[1]) + + specificlist, _ := entries.GetValue(key) + if specificentries, ok := specificlist.(mapstr.M); ok { + for keyspec := range specificentries { + // enumeratedmodules will be populated only in cases we have module enumeration, like: + // "co.elastic.metrics/1.module": "prometheus", + // "co.elastic.metrics/2.module": "istiod", + enumeratedmodules = append(enumeratedmodules, keyspec) + } + } + } + // We check if multiple metrcisets are defined and we retrieve the hints per metricset. Only applicable in beats + // See Metrics_multiple_modules_and_specific_config_per_module test case in hints_test.go + for _, metric := range enumeratedmodules { + _, incorrecthint = checkSupportedHints(metric, fmt.Sprintf("%s.%s", key, metric), allSupportedHints) + if incorrecthint != "" { + incorrecthints = append(incorrecthints, incorrecthint) + } + + } + //We check whether the provided annotation follows the supported format and vocabulary. The check happens for annotations that have prefix co.elastic + _, incorrecthint = checkSupportedHints(parts[1], key, allSupportedHints) + } //end of annotation check + + // Insert only if there is no entry already. container level annotations take + // higher priority. + if _, err := hints.GetValue(hintKey); err != nil { + _, err = hints.Put(hintKey, rawValue) + if err != nil { + continue + } + } + } else if container != "" { + // Only consider annotations that are of type mapstr.M as we are looking for + // container level nesting + builderHints, ok := rawValue.(mapstr.M) + if !ok { + continue + } + + // Check for / prefix + for hintKey, rawVal := range builderHints { + if strings.HasPrefix(hintKey, container) { + // Split the key to get part[1] to be the hint + parts := strings.Split(hintKey, "/") + + if validate { + checkdigit := digitCheck.MatchString(parts[1]) // With this regex we check if enumeration for modules is provided + if checkdigit { + allSupportedHints = append(allSupportedHints, parts[1]) + + specificlist, _ := entries.GetValue(key) + if specificentries, ok := specificlist.(mapstr.M); ok { + for keyspec := range specificentries { + // enumeratedmodules will be populated only in cases we have module enumeration, like: + // "co.elastic.metrics/1.module": "prometheus", + // "co.elastic.metrics/2.module": "istiod", + enumeratedmodules = append(enumeratedmodules, keyspec) + } + } + } + + // We check if multiple metrcisets are defined and we retrieve the hints per metricset. Only applicable in beats + // See Metrics_multiple_modules_and_specific_config_per_module test case in hints_test.go + for _, metric := range enumeratedmodules { + _, incorrecthint = checkSupportedHints(metric, fmt.Sprintf("%s.%s", key, metric), allSupportedHints) + if incorrecthint != "" { + incorrecthints = append(incorrecthints, incorrecthint) + } + + } + //We check whether the provided annotation follows the supported format and vocabulary. The check happens for annotations that have prefix co.elastic + _, incorrecthint = checkSupportedHints(parts[1], key, allSupportedHints) + } //end of annotation check + + if len(parts) == 2 { + // key will be the hint type + hintKey := fmt.Sprintf("%s.%s", key, parts[1]) + _, err := hints.Put(hintKey, rawVal) + if err != nil { + continue + } + } + } + } + } + if validate && incorrecthint != "" { + incorrecthints = append(incorrecthints, incorrecthint) + } + } + } + } + + return hints, incorrecthints +} + +// GetHintsAsList gets a set of hints and tries to convert them into a list of hints +func GetHintsAsList(hints mapstr.M, key string) []mapstr.M { + raw := GetHintMapStr(hints, key, "") + if raw == nil { + return nil + } + + var words, nums []string + + for key := range raw { + if _, err := strconv.Atoi(key); err != nil { + words = append(words, key) + continue + } else { + nums = append(nums, key) + } + } + + sort.Strings(nums) + + var configs []mapstr.M + for _, key := range nums { + rawCfg := raw[key] + if config, ok := rawCfg.(mapstr.M); ok { + configs = append(configs, config) + } + } + + defaultMap := mapstr.M{} + for _, word := range words { + defaultMap[word] = raw[word] + } + + if len(defaultMap) != 0 { + configs = append(configs, defaultMap) + } + return configs +} + +// checkSupportedHints gets a specific hint annotation and compares it with the supported list of hints +func checkSupportedHints(actualannotation, key string, allSupportedHints []string) (bool, string) { + found := false + var incorrecthint string + + if slices.Contains(allSupportedHints, actualannotation) { + found = true + } + if !found { + incorrecthint = key + } + return found, incorrecthint +} + +// checkSupportedHintsSets gest the data_streams or metricset lists that are defined. Searches inside specific list and returns the unsupported list of hints found +// This function will merge the incorrect hints found in metricsets of data_streams with rest incorrect hints +func checkSupportedHintsSets(annotations mapstr.M, prefix, stream, kind string, allSupportedHints, incorrecthints []string) []string { + var incorrecthint string + + if hintsindatastream, err := annotations.GetValue(prefix + "." + kind + "/" + stream); err == nil { + if hintsentries, ok := hintsindatastream.(mapstr.M); ok { + for hintkey := range hintsentries { + _, incorrecthint = checkSupportedHints(hintkey, kind+"/"+stream+"."+hintkey, allSupportedHints) + if incorrecthint != "" { + incorrecthints = append(incorrecthints, incorrecthint) + } + } + + } + } + + return incorrecthints +} diff --git a/pkg/autodiscover/utils/hints_test.go b/pkg/autodiscover/utils/hints_test.go new file mode 100644 index 000000000000..def50ee57c4b --- /dev/null +++ b/pkg/autodiscover/utils/hints_test.go @@ -0,0 +1,515 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/elastic-agent-libs/logp/logptest" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +func TestGetProcessors(t *testing.T) { + hints := mapstr.M{ + "co": mapstr.M{ + "elastic": mapstr.M{ + "logs": mapstr.M{ + "processors": mapstr.M{ + "add_fields": `{"fields": {"foo": "bar"}}`, + }, + }, + }, + }, + } + procs := GetProcessors(hints, "co.elastic.logs", logptest.NewTestingLogger(t, "")) + assert.Equal(t, []mapstr.M{ + mapstr.M{ + "add_fields": mapstr.M{ + "fields": map[string]any{ + "foo": "bar", + }, + }, + }, + }, procs) +} + +func TestGenerateHints(t *testing.T) { + + var allSupportedHints = []string{"enabled", "package", "module", "integration", "data_streams", "metricsets", "host", "period", "timeout", "metrics_path", "username", "password", "stream", "processors", "multiline", "json", "disable"} + + tests := []struct { + name string + annotations map[string]string + result mapstr.M + expectedIncorrectHints int // We set the number of hints that will be marked as incorrect and wont be included in the acceptable supported list + }{ + //Empty annotations should return empty hints + { + name: "Empty_Annotations", + annotations: map[string]string{}, + result: mapstr.M{}, + expectedIncorrectHints: 0, + }, + + // Scenarios being tested: + // logs/multiline.pattern must be a nested mapstr.M under hints.logs + // logs/processors.add_fields must be nested mapstr.M under hints.logs + // logs/json.keys_under_root must be a nested mapstr.M under hints.logs + // metrics/module must be found in hints.metrics + // not.to.include must not be part of hints + // period is annotated at both container and pod level. Container level value must be in hints + { + name: "Logs_multiline_and_metrics", + annotations: map[string]string{ + "co.elastic.logs/multiline.pattern": "^test", + "co.elastic.logs/json.keys_under_root": "true", + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + "co.elastic.metrics.foobar/period": "15s", + "co.elastic.metrics.foobar1/period": "15s", + "not.to.include": "true", + }, + result: mapstr.M{ + "logs": mapstr.M{ + "multiline": mapstr.M{ + "pattern": "^test", + }, + "json": mapstr.M{ + "keys_under_root": "true", + }, + }, + "metrics": mapstr.M{ + "module": "prometheus", + "period": "15s", + }, + }, + expectedIncorrectHints: 0, + }, + // Scenarios being tested: + // logs/multiline.pattern must be a nested mapstr.M under hints.logs + // metrics/module must be found in hints.metrics + // not.to.include must not be part of hints + // metrics/metrics_path must be found in hints.metrics + { + name: "Logs_multiline_and_metrics_with_metrics_path", + annotations: map[string]string{ + "co.elastic.logs/multiline.pattern": "^test", + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + "co.elastic.metrics/metrics_path": "/metrics/prometheus", + "co.elastic.metrics/username": "user", + "co.elastic.metrics/password": "pass", + "co.elastic.metrics.foobar/period": "15s", + "co.elastic.metrics.foobar1/period": "15s", + "co.elastic.hints/streamssssssssss": "stdout", // On purpose this added with typo + "not.to.include": "true", + }, + result: mapstr.M{ + "logs": mapstr.M{ + "multiline": mapstr.M{ + "pattern": "^test", + }, + }, + "hints": mapstr.M{"streamssssssssss": "stdout"}, + "metrics": mapstr.M{ + "module": "prometheus", + "period": "15s", + "metrics_path": "/metrics/prometheus", + "username": "user", + "password": "pass", + }, + }, + expectedIncorrectHints: 1, // Due to co.elastic.hints/streamsteamssssssssss + }, + // Scenarios being tested: + // logs/multiline.pattern must be a nested mapstr.M under hints.logs + // metrics/module must be found in hints.metrics + // not.to.include must not be part of hints + // period is annotated at both container and pod level. Container level value must be in hints + { + name: "Logs_multiline_and_metrics", + annotations: map[string]string{ + "co.elastic.logs/multiline.pattern": "^test", + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + "co.elastic.metrics.foobar/period": "15s", + "co.elastic.metrics.foobar1/period": "15s", + "not.to.include": "true", + }, + result: mapstr.M{ + "logs": mapstr.M{ + "multiline": mapstr.M{ + "pattern": "^test", + }, + }, + "metrics": mapstr.M{ + "module": "prometheus", + "period": "15s", + }, + }, + expectedIncorrectHints: 0, + }, + // Scenarios being tested: + // have co.elastic.logs/disable set to false. + // logs/multiline.pattern must be a nested mapstr.M under hints.logs + // metrics/module must be found in hints.metrics + // not.to.include must not be part of hints + // period is annotated at both container and pod level. Container level value must be in hints + { + name: "Logs_disabled_false_and_metrics", + annotations: map[string]string{ + "co.elastic.logs/disable": "false", + "co.elastic.logs/multiline.pattern": "^test", + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + "co.elastic.metrics.foobar/period": "15s", + "co.elastic.metrics.foobar1/period": "15s", + "not.to.include": "true", + }, + result: mapstr.M{ + "logs": mapstr.M{ + "multiline": mapstr.M{ + "pattern": "^test", + }, + "disable": "false", + }, + "metrics": mapstr.M{ + "module": "prometheus", + "period": "15s", + }, + }, + expectedIncorrectHints: 0, + }, + // Scenarios being tested: + // have co.elastic.logs/disable set to true. + // logs/multiline.pattern must be a nested mapstr.M under hints.logs + // metrics/module must be found in hints.metrics + // not.to.include must not be part of hints + // period is annotated at both container and pod level. Container level value must be in hints + { + name: "Logs_disabled_true_and_metrics", + annotations: map[string]string{ + "co.elastic.logs/disable": "true", + "co.elastic.logs/multiline.pattern": "^test", + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + "co.elastic.metrics.foobar/period": "15s", + "co.elastic.metrics.foobar1/period": "15s", + "not.to.include": "true", + }, + result: mapstr.M{ + "logs": mapstr.M{ + "multiline": mapstr.M{ + "pattern": "^test", + }, + "disable": "true", + }, + "metrics": mapstr.M{ + "module": "prometheus", + "period": "15s", + }, + }, + expectedIncorrectHints: 0, + }, + // Scenarios being tested: + // have co.elastic.hints/package set. + // Define multiple co.elastic.hints/data_streams and also specific configuration for each one + // Typo errors introduced for "co.elastic.hints/access.streams" and "co.elastic.hints/error.streams" + { + name: "Metrics_apache_package_and_specific_config_per_datastream", + annotations: map[string]string{ + "co.elastic.hints/package": "apache", + "co.elastic.hints/data_streams": "access,error", + "co.elastic.hints/access.period": "5m", + "co.elastic.hints/access.streamssssssssss": "stdout", // On purpose this added with typo + "co.elastic.hints/error.period": "5m", + "co.elastic.hints/error.streamssssssssss": "stderr", // On purpose this added with typo + }, + result: mapstr.M{ + "hints": mapstr.M{ + "data_streams": "access,error", + "access": mapstr.M{"period": "5m", "streamssssssssss": "stdout"}, + "error": mapstr.M{"period": "5m", "streamssssssssss": "stderr"}, + "package": "apache", + }}, + expectedIncorrectHints: 2, // Due to co.elastic.hints/access.streamssssssssss and co.elastic.hints/error.streamssssssssss typo errors + }, + // Scenarios being tested: + // have co.elastic.metrics/module set. + // Define multiple co.elastic.hints/data_streams and also specific configuration for each one + // A typo error introduced for "co.elastic.metrics/istiod.streams" + { + name: "Metrics_istio_module_and_specific_config_per_metricset", + annotations: map[string]string{ + "co.elastic.metrics/module": "istio", + "co.elastic.metrics/metricsets": "istiod,proxy", + "co.elastic.metrics/istiod.period": "5m", + "co.elastic.metrics/istiod.streamssssssssss": "stdout", // On purpose this added with typo + "co.elastic.metrics/proxy.period": "5m", + "co.elastic.metrics/proxy.stream": "stderr", + }, + result: mapstr.M{ + "metrics": mapstr.M{ + "metricsets": "istiod,proxy", + "istiod": mapstr.M{"period": "5m", "streamssssssssss": "stdout"}, + "proxy": mapstr.M{"period": "5m", "stream": "stderr"}, + "module": "istio", + }}, + expectedIncorrectHints: 1, // Due to co.elastic.metrics/istiod.streamssssssssss + }, + // Scenarios being tested: + // have co.elastic.metrics/module set for multiple enumerations. + // Define different hints for each one enumeration + // A typo error introduced for "co.elastic.metrics/1.periods" and "co.elastic.metrics/2.streams" + { + name: "Metrics_multiple_modules_and_specific_config_per_module", + annotations: map[string]string{ + "co.elastic.metrics/1.module": "prometheus", + "co.elastic.metrics/1.periodssssssssss": "15s", // On purpose this added with typo + "co.elastic.metrics/2.module": "istiod", + "co.elastic.metrics/2.period": "15s", + "co.elastic.metrics/2.streamssssssssss": "stderr", // On purpose this added with typo + }, + result: mapstr.M{ + "metrics": mapstr.M{ + "1": mapstr.M{ + "module": "prometheus", + "periodssssssssss": "15s", + }, + "2": mapstr.M{ + "module": "istiod", + "period": "15s", + "streamssssssssss": "stderr", + }, + }}, + expectedIncorrectHints: 2, // Due to co.elastic.metrics/1.periodssssssssss and co.elastic.metrics/2.streamssssssssss typo errors + }, + } + + for _, test := range tests { + annMap := mapstr.M{} + for k, v := range test.annotations { + _, err := annMap.Put(k, v) + if err != nil { + continue + } + } + + generateHints, incorrectHints := GenerateHints(annMap, "foobar", "co.elastic", true, allSupportedHints) + assert.Len(t, incorrectHints, test.expectedIncorrectHints) // We validate how many incorrect hints are provided per test case. + assert.Equal(t, test.result, generateHints) + } +} + +func TestGenerateHintsWithValidatedisabled(t *testing.T) { + + var allSupportedHints = []string{"enabled", "package", "module", "integration", "data_streams", "metricsets", "host", "period", "timeout", "metrics_path", "username", "password", "stream", "processors", "multiline", "json", "disable"} + + tests := []struct { + name string + annotations map[string]string + result mapstr.M + expectedIncorrectHints int // We set the number of hints that will be marked as incorrect and wont be included in the acceptable supported list + }{ + + // Scenarios being tested: + // have co.elastic.hints/package set. + // Define multiple co.elastic.hints/data_streams and also specific configuration for each one + // Typo errors introduced for "co.elastic.hints/access.streams" and "co.elastic.hints/error.streams" + { + name: "Metrics_apache_package_and_specific_config_per_datastream", + annotations: map[string]string{ + "co.elastic.hints/package": "apache", + "co.elastic.hints/data_streams": "access,error", + "co.elastic.hints/access.period": "5m", + "co.elastic.hints/access.streamssssssssss": "stdout", // On purpose this added with typo + "co.elastic.hints/error.period": "5m", + "co.elastic.hints/error.streamssssssssss": "stderr", // On purpose this added with typo + }, + result: mapstr.M{ + "hints": mapstr.M{ + "data_streams": "access,error", + "access": mapstr.M{"period": "5m", "streamssssssssss": "stdout"}, + "error": mapstr.M{"period": "5m", "streamssssssssss": "stderr"}, + "package": "apache", + }}, + expectedIncorrectHints: 0, // Validate flag= false in GenerateHints + }, + // Scenarios being tested: + // have co.elastic.metrics/module set. + // Define multiple co.elastic.hints/data_streams and also specific configuration for each one + // A typo error introduced for "co.elastic.metrics/istiod.streams" + { + name: "Metrics_istio_module_and_specific_config_per_metricset", + annotations: map[string]string{ + "co.elastic.metrics/module": "istio", + "co.elastic.metrics/metricsets": "istiod,proxy", + "co.elastic.metrics/istiod.period": "5m", + "co.elastic.metrics/istiod.streamssssssssss": "stdout", // On purpose this added with typo + "co.elastic.metrics/proxy.period": "5m", + "co.elastic.metrics/proxy.stream": "stderr", + }, + result: mapstr.M{ + "metrics": mapstr.M{ + "metricsets": "istiod,proxy", + "istiod": mapstr.M{"period": "5m", "streamssssssssss": "stdout"}, + "proxy": mapstr.M{"period": "5m", "stream": "stderr"}, + "module": "istio", + }}, + expectedIncorrectHints: 0, // Validate flag= false in GenerateHints + }, + // Scenarios being tested: + // have co.elastic.metrics/module set for multiple enumerations. + // Define different hints for each one enumeration + // A typo error introduced for "co.elastic.metrics/1.periods" and "co.elastic.metrics/2.streams" + { + name: "Metrics_multiple_modules_and_specific_config_per_module", + annotations: map[string]string{ + "co.elastic.metrics/1.module": "prometheus", + "co.elastic.metrics/1.periodssssssssss": "15s", // On purpose this added with typo + "co.elastic.metrics/2.module": "istiod", + "co.elastic.metrics/2.period": "15s", + "co.elastic.metrics/2.streamssssssssss": "stderr", // On purpose this added with typo + }, + result: mapstr.M{ + "metrics": mapstr.M{ + "1": mapstr.M{ + "module": "prometheus", + "periodssssssssss": "15s", + }, + "2": mapstr.M{ + "module": "istiod", + "period": "15s", + "streamssssssssss": "stderr", + }, + }}, + expectedIncorrectHints: 0, // Validate flag= false in GenerateHints + }, + } + + for _, test := range tests { + annMap := mapstr.M{} + for k, v := range test.annotations { + _, err := annMap.Put(k, v) + if err != nil { + continue + } + } + + generateHints, incorrectHints := GenerateHints(annMap, "foobar", "co.elastic", false, allSupportedHints) + assert.Len(t, incorrectHints, test.expectedIncorrectHints) // We validate how many incorrect hints are provided per test case. + assert.Equal(t, test.result, generateHints) + } +} + +func TestGetHintsAsList(t *testing.T) { + tests := []struct { + input mapstr.M + output []mapstr.M + message string + }{ + { + input: mapstr.M{ + "metrics": mapstr.M{ + "module": "prometheus", + "period": "15s", + }, + }, + output: []mapstr.M{ + { + "module": "prometheus", + "period": "15s", + }, + }, + message: "Single hint should return a single set of configs", + }, + { + input: mapstr.M{ + "metrics": mapstr.M{ + "1": mapstr.M{ + "module": "prometheus", + "period": "15s", + }, + }, + }, + output: []mapstr.M{ + { + "module": "prometheus", + "period": "15s", + }, + }, + message: "Single hint with numeric prefix should return a single set of configs", + }, + { + input: mapstr.M{ + "metrics": mapstr.M{ + "1": mapstr.M{ + "module": "prometheus", + "period": "15s", + }, + "2": mapstr.M{ + "module": "dropwizard", + "period": "20s", + }, + }, + }, + output: []mapstr.M{ + { + "module": "prometheus", + "period": "15s", + }, + { + "module": "dropwizard", + "period": "20s", + }, + }, + message: "Multiple hints with numeric prefix should return configs in numeric ordering", + }, + { + input: mapstr.M{ + "metrics": mapstr.M{ + "1": mapstr.M{ + "module": "prometheus", + "period": "15s", + }, + "module": "dropwizard", + "period": "20s", + }, + }, + output: []mapstr.M{ + { + "module": "prometheus", + "period": "15s", + }, + { + "module": "dropwizard", + "period": "20s", + }, + }, + message: "Multiple hints with numeric prefix and default should return configs with defaults at the last", + }, + } + + for _, test := range tests { + t.Run(test.message, func(t *testing.T) { + assert.Equal(t, test.output, GetHintsAsList(test.input, "metrics")) + }) + } +} diff --git a/pkg/autodiscover/utils/key.go b/pkg/autodiscover/utils/key.go new file mode 100644 index 000000000000..a54fdcf38bb2 --- /dev/null +++ b/pkg/autodiscover/utils/key.go @@ -0,0 +1,26 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import "strings" + +// DeDot a string by replacing all . with _ +// This helps when sending data to Elasticsearch to prevent object and key collisions. +func DeDot(s string) string { + return strings.ReplaceAll(s, ".", "_") +} diff --git a/x-pack/dockerlogbeat/pipelinemanager/clientLogReader.go b/x-pack/dockerlogbeat/pipelinemanager/clientLogReader.go index 1826ed0eee12..901bf93ddc73 100644 --- a/x-pack/dockerlogbeat/pipelinemanager/clientLogReader.go +++ b/x-pack/dockerlogbeat/pipelinemanager/clientLogReader.go @@ -18,8 +18,8 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common/acker" + helper "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/beats/v7/x-pack/dockerlogbeat/pipereader" - helper "github.com/elastic/elastic-agent-autodiscover/docker" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/x-pack/filebeat/input/streaming/config.go b/x-pack/filebeat/input/streaming/config.go index eb604d438db8..45779e76ca7b 100644 --- a/x-pack/filebeat/input/streaming/config.go +++ b/x-pack/filebeat/input/streaming/config.go @@ -39,7 +39,7 @@ type config struct { // program. If it has a cursor field, that field will // be overwritten by any stored cursor, but will be // available if no stored cursor exists. - State map[string]interface{} `config:"state"` + State map[string]any `config:"state"` // Auth is the authentication config for connection. Auth authConfig `config:"auth"` // URL is the websocket url to connect to. diff --git a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go index 0489c62da3ec..6cce0849f3aa 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go +++ b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go @@ -13,9 +13,9 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/autodiscover/template" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" awsauto "github.com/elastic/beats/v7/x-pack/libbeat/autodiscover/providers/aws" awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" - "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" diff --git a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider_test.go b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider_test.go index f72f600bc542..244069655cb7 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider_test.go +++ b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider_test.go @@ -13,9 +13,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" awsauto "github.com/elastic/beats/v7/x-pack/libbeat/autodiscover/providers/aws" "github.com/elastic/beats/v7/x-pack/libbeat/autodiscover/providers/aws/test" - "github.com/elastic/elastic-agent-autodiscover/bus" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/logp/logptest" @@ -26,7 +26,7 @@ func Test_internalBuilder(t *testing.T) { instance := fakeEC2Instance() instances := []*ec2Instance{instance} fetcher := newMockFetcher(instances, nil) - log := logp.NewLogger("ec2") + log := logp.NewLogger("ec2") //nolint:forbidigo // test setup pBus := bus.New(log, "test") cfg := &awsauto.Config{ diff --git a/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go b/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go index 4012460b5dbd..e3afa7ba1f39 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go +++ b/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go @@ -13,9 +13,9 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/autodiscover/template" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" awsauto "github.com/elastic/beats/v7/x-pack/libbeat/autodiscover/providers/aws" awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" - "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" diff --git a/x-pack/libbeat/autodiscover/providers/aws/elb/provider_test.go b/x-pack/libbeat/autodiscover/providers/aws/elb/provider_test.go index a2f151a5ece9..4f720c7baffc 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/elb/provider_test.go +++ b/x-pack/libbeat/autodiscover/providers/aws/elb/provider_test.go @@ -14,8 +14,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" awsauto "github.com/elastic/beats/v7/x-pack/libbeat/autodiscover/providers/aws" - "github.com/elastic/elastic-agent-autodiscover/bus" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/logp/logptest" @@ -64,7 +64,7 @@ func (tea *testEventAccumulator) waitForNumEvents(t *testing.T, targetLen int, t } func Test_internalBuilder(t *testing.T) { - log := logp.NewLogger("elb") + log := logp.NewLogger("elb") //nolint:forbidigo // test setup lbl := fakeLbl() lbls := []*lbListener{lbl} fetcher := newMockFetcher(lbls, nil) diff --git a/x-pack/libbeat/autodiscover/providers/aws/test/provider.go b/x-pack/libbeat/autodiscover/providers/aws/test/provider.go index 8779c99b7e66..d07183f07f51 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/test/provider.go +++ b/x-pack/libbeat/autodiscover/providers/aws/test/provider.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" ) // TestEventAccumulator defined a list of events for testing @@ -48,7 +48,7 @@ func (tea *TestEventAccumulator) Get() []bus.Event { func (tea *TestEventAccumulator) WaitForNumEvents(t *testing.T, targetLen int, timeout time.Duration) { start := time.Now() - for time.Now().Sub(start) < timeout { + for time.Since(start) < timeout { if tea.Len() >= targetLen { return } diff --git a/x-pack/libbeat/autodiscover/providers/nomad/nomad.go b/x-pack/libbeat/autodiscover/providers/nomad/nomad.go index 417e01fe7507..803b5452a353 100644 --- a/x-pack/libbeat/autodiscover/providers/nomad/nomad.go +++ b/x-pack/libbeat/autodiscover/providers/nomad/nomad.go @@ -11,13 +11,13 @@ import ( "github.com/gofrs/uuid/v5" - "github.com/elastic/elastic-agent-autodiscover/utils" + "github.com/elastic/beats/v7/pkg/autodiscover/utils" "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/autodiscover/template" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" "github.com/elastic/beats/v7/x-pack/libbeat/common/nomad" - "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" diff --git a/x-pack/libbeat/autodiscover/providers/nomad/nomad_test.go b/x-pack/libbeat/autodiscover/providers/nomad/nomad_test.go index a465aef0364f..59c18bb2c048 100644 --- a/x-pack/libbeat/autodiscover/providers/nomad/nomad_test.go +++ b/x-pack/libbeat/autodiscover/providers/nomad/nomad_test.go @@ -18,8 +18,8 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover/template" "github.com/elastic/beats/v7/libbeat/tests/resources" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" "github.com/elastic/beats/v7/x-pack/libbeat/common/nomad" - "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp/logptest" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/container.go b/x-pack/metricbeat/module/awsfargate/task_stats/container.go index 9c4d7d0d0e61..6f827c0a03b8 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/container.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/container.go @@ -5,8 +5,8 @@ package task_stats import ( - helpers "github.com/elastic/elastic-agent-autodiscover/docker" - "github.com/elastic/elastic-agent-autodiscover/utils" + helpers "github.com/elastic/beats/v7/pkg/autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/utils" ) // container is a struct representation of a container diff --git a/x-pack/osquerybeat/ext/osquery-extension/pkg/encoding/encoding_test.go b/x-pack/osquerybeat/ext/osquery-extension/pkg/encoding/encoding_test.go new file mode 100644 index 000000000000..2edf84021b6a --- /dev/null +++ b/x-pack/osquerybeat/ext/osquery-extension/pkg/encoding/encoding_test.go @@ -0,0 +1,847 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package encoding + +import ( + "reflect" + "testing" + "time" + + "github.com/osquery/osquery-go/plugin/table" +) + +func TestEncodingFlagHas(t *testing.T) { + tests := []struct { + flag EncodingFlag + option EncodingFlag + expected bool + }{ + {EncodingFlagUseNumbersZeroValues, EncodingFlagUseNumbersZeroValues, true}, + } + + for _, test := range tests { + result := test.flag.has(test.option) + if result != test.expected { + t.Errorf("has(%v) = %v; expected %v", test.option, result, test.expected) + } + } +} + +func TestMarshalToMapWithFlags(t *testing.T) { + tests := []struct { + name string + input any + flags EncodingFlag + expected map[string]string + err bool + }{ + { + name: "nil input", + input: nil, + flags: 0, + expected: nil, + err: true, + }, + { + name: "struct with osquery tag", + input: &struct { + Name string `osquery:"name"` + }{Name: "test"}, + flags: 0, + expected: map[string]string{"name": "test"}, + err: false, + }, + { + name: "map input", + input: map[string]any{"key1": "value1", "key2": "value2", "key3": 1}, + flags: 0, + expected: map[string]string{"key1": "value1", "key2": "value2", "key3": "1"}, + err: false, + }, + { + name: "struct with hidden field", + input: &struct { + HiddenField int `osquery:"-"` + }{HiddenField: 42}, + flags: 0, + expected: map[string]string{}, + err: false, + }, + { + name: "invalid type", + input: &struct { + InvalidType map[int]string + }{InvalidType: map[int]string{1: "value"}}, + flags: 0, + expected: map[string]string{"InvalidType": "map[1:value]"}, + err: false, + }, + { + name: "zero value int", + input: &struct { + ZeroVal int + }{ZeroVal: 0}, + flags: 0, + expected: map[string]string{"ZeroVal": ""}, + err: false, + }, + { + name: "zero value int with flag", + input: &struct { + ZeroVal int + }{ZeroVal: 0}, + flags: EncodingFlagUseNumbersZeroValues, + expected: map[string]string{"ZeroVal": "0"}, + err: false, + }, + // Test bool type + { + name: "bool type", + input: &struct { + IsActive bool + }{IsActive: true}, + flags: 0, + expected: map[string]string{"IsActive": "1"}, + err: false, + }, + { + name: "bool type false", + input: &struct { + IsActive bool + }{IsActive: false}, + flags: 0, + expected: map[string]string{"IsActive": "0"}, + err: false, + }, + // Test uint type + { + name: "uint type", + input: &struct { + Count uint + }{Count: 42}, + flags: 0, + expected: map[string]string{"Count": "42"}, + err: false, + }, + { + name: "zero value uint", + input: &struct { + Count uint + }{Count: 0}, + flags: 0, + expected: map[string]string{"Count": ""}, + err: false, + }, + { + name: "zero value uint with flag", + input: &struct { + Count uint + }{Count: 0}, + flags: EncodingFlagUseNumbersZeroValues, + expected: map[string]string{"Count": "0"}, + err: false, + }, + // Test float type + { + name: "float64 type", + input: &struct { + Price float64 + }{Price: 99.99}, + flags: 0, + expected: map[string]string{"Price": "99.99"}, + err: false, + }, + { + name: "float32 type", + input: &struct { + Price float32 + }{Price: 12.5}, + flags: 0, + expected: map[string]string{"Price": "12.5"}, + err: false, + }, + // Test non-pointer struct + { + name: "non-pointer struct", + input: struct { + Name string + }{Name: "test"}, + flags: 0, + expected: map[string]string{"Name": "test"}, + err: false, + }, + // Test pointer maps + { + name: "pointer map", + input: &map[string]string{ + "key1": "value1", + "key2": "value2", + }, + flags: 0, + expected: map[string]string{"key1": "value1", "key2": "value2"}, + err: false, + }, + // Test pointer fields + { + name: "pointer fields", + input: &struct { + StrPtr *string + IntPtr *int + }{ + StrPtr: new("hello"), + IntPtr: new(123), + }, + flags: 0, + expected: map[string]string{"StrPtr": "hello", "IntPtr": "123"}, + err: false, + }, + // Test nil pointer fields + { + name: "nil pointer fields", + input: &struct { + StrPtr *string + IntPtr *int + }{ + StrPtr: nil, + IntPtr: nil, + }, + flags: 0, + expected: map[string]string{"StrPtr": "", "IntPtr": ""}, + err: false, + }, + // Test time.Time type + { + name: "time.Time type", + input: &struct { + Time time.Time `osquery:"time"` + }{Time: time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)}, + flags: 0, + expected: map[string]string{"time": "2023-06-15T14:30:00Z"}, + err: false, + }, + { + name: "time.Time", + input: &struct { + Time time.Time `osquery:"time" format:"unix"` + }{Time: time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)}, + flags: 0, + expected: map[string]string{"time": "1686839400"}, + err: false, + }, + } + + for _, test := range tests { + result, err := MarshalToMapWithFlags(test.input, test.flags) + if (err != nil) != test.err { + t.Errorf("%s: MarshalToMapWithFlags(%v, %v) error = %v; expected error = %v", test.name, test.input, test.flags, err, test.err) + continue + } + if !reflect.DeepEqual(result, test.expected) { + t.Errorf("%s: MarshalToMapWithFlags(%v, %v) = %v; expected %v", test.name, test.input, test.flags, result, test.expected) + continue + } + } +} + +func tagPtr(tag string) *reflect.StructTag { + st := reflect.StructTag(tag) + return &st +} + +func Test_formatTimeWithTagFormat(t *testing.T) { + tests := []struct { + name string // description of this test case + // Named input parameters for target function. + fieldValue reflect.Value + flag EncodingFlag + tag *reflect.StructTag + want string + wantErr bool + }{ + { + name: "RFC3339 format", + fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), + flag: 0, + tag: tagPtr(`format:"rfc3339"`), + want: "2023-06-15T14:30:00Z", + wantErr: false, + }, + { + name: "RFC3339Nano format", + fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 123456789, time.UTC)), + flag: 0, + tag: tagPtr(`format:"rfc3339nano"`), + want: "2023-06-15T14:30:00.123456789Z", + wantErr: false, + }, + { + name: "Unix timestamp format", + fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), + flag: 0, + tag: tagPtr(`format:"unix"`), + want: "1686839400", + wantErr: false, + }, + { + name: "RFC822 format", + fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), + flag: 0, + tag: tagPtr(`format:"rfc822"`), + want: "15 Jun 23 14:30 UTC", + wantErr: false, + }, + { + name: "RFC822Z format", + fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), + flag: 0, + tag: tagPtr(`format:"rfc822z"`), + want: "15 Jun 23 14:30 +0000", + wantErr: false, + }, + { + name: "RFC850 format", + fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), + flag: 0, + tag: tagPtr(`format:"rfc850"`), + want: "Thursday, 15-Jun-23 14:30:00 UTC", + wantErr: false, + }, + { + name: "RFC1123 format", + fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), + flag: 0, + tag: tagPtr(`format:"rfc1123"`), + want: "Thu, 15 Jun 2023 14:30:00 UTC", + wantErr: false, + }, + { + name: "RFC1123Z format", + fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), + flag: 0, + tag: tagPtr(`format:"rfc1123z"`), + want: "Thu, 15 Jun 2023 14:30:00 +0000", + wantErr: false, + }, + { + name: "Kitchen format", + fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), + flag: 0, + tag: tagPtr(`format:"kitchen"`), + want: "2:30PM", + wantErr: false, + }, + { + name: "StampMilli format", + fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 123000000, time.UTC)), + flag: 0, + tag: tagPtr(`format:"stampmilli"`), + want: "Jun 15 14:30:00.123", + wantErr: false, + }, + { + name: "StampMicro format", + fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 123456000, time.UTC)), + flag: 0, + tag: tagPtr(`format:"stampmicro"`), + want: "Jun 15 14:30:00.123456", + wantErr: false, + }, + { + name: "StampNano format", + fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 123456789, time.UTC)), + flag: 0, + tag: tagPtr(`format:"stampnano"`), + want: "Jun 15 14:30:00.123456789", + wantErr: false, + }, + { + name: "Invalid format", + fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), + flag: 0, + tag: tagPtr(`format:"invalid"`), + want: "", + wantErr: true, + }, + { + name: "With timezone", + fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), + flag: 0, + tag: tagPtr(`format:"rfc3339" tz:"America/New_York"`), + want: "2023-06-15T10:30:00-04:00", + wantErr: false, + }, + { + name: "Invalid timezone", + fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), + flag: 0, + tag: tagPtr(`format:"rfc3339" tz:"Invalid/Timezone"`), + want: "", + wantErr: true, + }, + { + name: "Zero time with default flags", + fieldValue: reflect.ValueOf(time.Time{}), + flag: 0, + tag: tagPtr(`format:"rfc3339"`), + want: "", + wantErr: false, + }, + { + name: "Zero time with UseNumbersZeroValues flag", + fieldValue: reflect.ValueOf(time.Time{}), + flag: EncodingFlagUseNumbersZeroValues, + tag: tagPtr(`format:"rfc3339"`), + want: "0001-01-01T00:00:00Z", + wantErr: false, + }, + { + name: "Zero unix timestamp with UseNumbersZeroValues flag", + fieldValue: reflect.ValueOf(time.Time{}), + flag: EncodingFlagUseNumbersZeroValues, + tag: tagPtr(`format:"unix"`), + want: "0", + wantErr: false, + }, + { + name: "With timezone Asia/Tokyo", + fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), + flag: 0, + tag: tagPtr(`format:"rfc3339" tz:"Asia/Tokyo"`), + want: "2023-06-15T23:30:00+09:00", + wantErr: false, + }, + { + name: "With timezone Europe/London", + fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), + flag: 0, + tag: tagPtr(`format:"rfc3339" tz:"Europe/London"`), + want: "2023-06-15T15:30:00+01:00", + wantErr: false, + }, + { + name: "With timezone Australia/Sydney", + fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), + flag: 0, + tag: tagPtr(`format:"rfc3339" tz:"Australia/Sydney"`), + want: "2023-06-16T00:30:00+10:00", + wantErr: false, + }, + { + name: "Unix milliseconds format", + fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), + flag: 0, + tag: tagPtr(`format:"unixmilli"`), + want: "1686839400000", + wantErr: false, + }, + { + name: "Unix nanoseconds format", + fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), + flag: 0, + tag: tagPtr(`format:"unixnano"`), + want: "1686839400000000000", + wantErr: false, + }, + { + name: "Unix microseconds format", + fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), + flag: 0, + tag: tagPtr(`format:"unixmicro"`), + want: "1686839400000000", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, gotErr := formatTimeWithTagFormat(tt.fieldValue, tt.flag, tt.tag) + if gotErr != nil { + if !tt.wantErr { + t.Errorf("Name: %s, formatTimeWithTagFormat() failed: %v", tt.name, gotErr) + } + return + } + if tt.wantErr { + t.Fatalf("Name: %s, formatTimeWithTagFormat() succeeded unexpectedly", tt.name) + } + // TODO: update the condition below to compare got with tt.want. + if got != tt.want { + t.Errorf("Name: %s, formatTimeWithTagFormat() = %v, want %v", tt.name, got, tt.want) + } + }) + } +} + +func TestMarshalToMapWithFlags_embeddedStruct(t *testing.T) { + type EmbeddedStruct struct { + EmbeddedField string `osquery:"embedded_field"` + } + + type TestStruct struct { + EmbeddedStruct + TestField string `osquery:"test_field"` + } + + type TestStructWithPointer struct { + *EmbeddedStruct + TestField string `osquery:"test_field"` + } + + cases := []struct { + name string + input any + expected map[string]string + }{ + { + name: "embedded struct", + input: &TestStruct{ + EmbeddedStruct: EmbeddedStruct{EmbeddedField: "embedded_value"}, + TestField: "test_value", + }, + expected: map[string]string{ + "embedded_field": "embedded_value", + "test_field": "test_value", + }, + }, + { + name: "embedded struct with pointer", + input: &TestStructWithPointer{ + EmbeddedStruct: &EmbeddedStruct{EmbeddedField: "embedded_value"}, + TestField: "test_value", + }, + expected: map[string]string{ + "embedded_field": "embedded_value", + "test_field": "test_value", + }, + }, + } + + for _, tt := range cases { + t.Run(tt.name, func(t *testing.T) { + result, err := MarshalToMapWithFlags(tt.input, 0) + if err != nil { + t.Errorf("MarshalToMapWithFlags() error = %v", err) + } + if !reflect.DeepEqual(result, tt.expected) { + t.Errorf("%s: MarshalToMapWithFlags() = %v, want %v", tt.name, result, tt.expected) + } + }) + } +} + +func TestGenerateColumnDefinitions(t *testing.T) { + tests := []struct { + name string + input any + expectedCols []table.ColumnDefinition + expectedError bool + }{ + { + name: "basic struct with all supported types", + input: struct { + Name string `osquery:"name"` + Count int64 `osquery:"count"` + Active bool `osquery:"active"` + Score float64 `osquery:"score"` + SmallNum int `osquery:"small_num"` + BigNum uint64 `osquery:"big_num"` + Precision float32 `osquery:"precision"` + }{}, + expectedCols: []table.ColumnDefinition{ + table.TextColumn("name"), + table.BigIntColumn("count"), + table.IntegerColumn("active"), + table.DoubleColumn("score"), + table.IntegerColumn("small_num"), + table.BigIntColumn("big_num"), + table.DoubleColumn("precision"), + }, + expectedError: false, + }, + { + name: "struct with skipped fields", + input: struct { + Included string `osquery:"included"` + Skipped string `osquery:"-"` + NoTag string + }{}, + expectedCols: []table.ColumnDefinition{ + table.TextColumn("included"), + table.TextColumn("NoTag"), // Fields without osquery tag use field name + }, + expectedError: false, + }, + { + name: "struct with pointer fields", + input: struct { + Name *string `osquery:"name"` + Count *int64 `osquery:"count"` + }{}, + expectedCols: []table.ColumnDefinition{ + table.TextColumn("name"), + table.BigIntColumn("count"), + }, + expectedError: false, + }, + { + name: "nil input", + input: nil, + expectedCols: nil, + expectedError: true, + }, + { + name: "non-struct input", + input: "string", + expectedCols: nil, + expectedError: true, + }, + { + name: "pointer to struct", + input: &struct { + Field string `osquery:"field"` + }{}, + expectedCols: []table.ColumnDefinition{ + table.TextColumn("field"), + }, + expectedError: false, + }, + { + name: "all integer types", + input: struct { + I int `osquery:"i"` + I8 int8 `osquery:"i8"` + I16 int16 `osquery:"i16"` + I32 int32 `osquery:"i32"` + I64 int64 `osquery:"i64"` + U uint `osquery:"u"` + U8 uint8 `osquery:"u8"` + U16 uint16 `osquery:"u16"` + U32 uint32 `osquery:"u32"` + U64 uint64 `osquery:"u64"` + }{}, + expectedCols: []table.ColumnDefinition{ + table.IntegerColumn("i"), + table.IntegerColumn("i8"), + table.IntegerColumn("i16"), + table.IntegerColumn("i32"), + table.BigIntColumn("i64"), + table.IntegerColumn("u"), + table.IntegerColumn("u8"), + table.IntegerColumn("u16"), + table.IntegerColumn("u32"), + table.BigIntColumn("u64"), + }, + expectedError: false, + }, + { + name: "fields without osquery tag use field name", + input: struct { + CustomName string `osquery:"custom_name"` + DefaultName string + }{}, + expectedCols: []table.ColumnDefinition{ + table.TextColumn("custom_name"), + table.TextColumn("DefaultName"), // No tag, uses field name + }, + expectedError: false, + }, + { + name: "mixed exported and unexported fields", + input: struct { + Public string `osquery:"public"` + private string `osquery:"private"` // will be skipped + Another string `osquery:"another"` + }{}, + expectedCols: []table.ColumnDefinition{ + table.TextColumn("public"), + table.TextColumn("another"), + }, + expectedError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cols, err := GenerateColumnDefinitions(tt.input) + + if (err != nil) != tt.expectedError { + t.Errorf("GenerateColumnDefinitions() error = %v, expectedError %v", err, tt.expectedError) + return + } + + if !tt.expectedError { + if len(cols) != len(tt.expectedCols) { + t.Errorf("GenerateColumnDefinitions() returned %d columns, expected %d", len(cols), len(tt.expectedCols)) + return + } + + for i := range cols { + if cols[i].Name != tt.expectedCols[i].Name { + t.Errorf("Column %d: name = %s, expected %s", i, cols[i].Name, tt.expectedCols[i].Name) + } + if cols[i].Type != tt.expectedCols[i].Type { + t.Errorf("Column %d (%s): type = %s, expected %s", i, cols[i].Name, cols[i].Type, tt.expectedCols[i].Type) + } + } + } + }) + } +} +func TestGenerateColumnDefinitions_basic(t *testing.T) { + type testStruct struct { + Name string `osquery:"name"` + Active bool `osquery:"active"` + Count int `osquery:"count"` + Score float64 `osquery:"score"` + } + expected := []table.ColumnDefinition{ + table.TextColumn("name"), + table.IntegerColumn("active"), + table.IntegerColumn("count"), + table.DoubleColumn("score"), + } + cols, err := GenerateColumnDefinitions(testStruct{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(cols) != len(expected) { + t.Fatalf("expected %d columns, got %d", len(expected), len(cols)) + } + for i := range cols { + if cols[i].Name != expected[i].Name || cols[i].Type != expected[i].Type { + t.Errorf("column %d: got (%s, %s), want (%s, %s)", i, cols[i].Name, cols[i].Type, expected[i].Name, expected[i].Type) + } + } +} + +func TestGenerateColumnDefinitions_pointerStruct(t *testing.T) { + type testStruct struct { + Name *string `osquery:"name"` + } + expected := []table.ColumnDefinition{ + table.TextColumn("name"), + } + val := &testStruct{} + cols, err := GenerateColumnDefinitions(val) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(cols) != len(expected) { + t.Fatalf("expected %d columns, got %d", len(expected), len(cols)) + } + for i := range cols { + if cols[i].Name != expected[i].Name || cols[i].Type != expected[i].Type { //nolint:gosec // lengths checked above + t.Errorf("column %d: got (%s, %s), want (%s, %s)", i, cols[i].Name, cols[i].Type, expected[i].Name, expected[i].Type) //nolint:gosec // lengths checked above + } + } +} + +func TestGenerateColumnDefinitions_timeFormats(t *testing.T) { + type testStruct struct { + CreatedAt time.Time `osquery:"created_at" format:"unix"` + UpdatedAt time.Time `osquery:"updated_at"` + } + expected := []table.ColumnDefinition{ + table.BigIntColumn("created_at"), + table.TextColumn("updated_at"), + } + cols, err := GenerateColumnDefinitions(testStruct{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(cols) != len(expected) { + t.Fatalf("expected %d columns, got %d", len(expected), len(cols)) + } + for i := range cols { + if cols[i].Name != expected[i].Name || cols[i].Type != expected[i].Type { + t.Errorf("column %d: got (%s, %s), want (%s, %s)", i, cols[i].Name, cols[i].Type, expected[i].Name, expected[i].Type) + } + } +} + +func TestGenerateColumnDefinitions_skippedFields(t *testing.T) { + type testStruct struct { + Visible string `osquery:"visible"` + Hidden string `osquery:"-"` + } + expected := []table.ColumnDefinition{ + table.TextColumn("visible"), + } + cols, err := GenerateColumnDefinitions(testStruct{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(cols) != len(expected) { + t.Fatalf("expected %d columns, got %d", len(expected), len(cols)) + } + for i := range cols { + if cols[i].Name != expected[i].Name || cols[i].Type != expected[i].Type { //nolint:gosec // lengths checked above + t.Errorf("column %d: got (%s, %s), want (%s, %s)", i, cols[i].Name, cols[i].Type, expected[i].Name, expected[i].Type) //nolint:gosec // lengths checked above + } + } +} + +func TestGenerateColumnDefinitions_nonStructInput(t *testing.T) { + _, err := GenerateColumnDefinitions("not a struct") + if err == nil { + t.Error("expected error for non-struct input, got nil") + } +} + +func TestGenerateColumnDefinitions_nilInput(t *testing.T) { + _, err := GenerateColumnDefinitions(nil) + if err == nil { + t.Error("expected error for nil input, got nil") + } +} + +func TestGenerateColumnDefinitions_unexportedFields(t *testing.T) { + type testStruct struct { + Public string `osquery:"public"` + private string `osquery:"private"` //nolint:unused // meaningful for test coverage + Exported2 int `osquery:"exported2"` + } + expected := []table.ColumnDefinition{ + table.TextColumn("public"), + table.IntegerColumn("exported2"), + } + cols, err := GenerateColumnDefinitions(testStruct{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(cols) != len(expected) { + t.Fatalf("expected %d columns, got %d", len(expected), len(cols)) + } + for i := range cols { + if cols[i].Name != expected[i].Name || cols[i].Type != expected[i].Type { + t.Errorf("column %d: got (%s, %s), want (%s, %s)", i, cols[i].Name, cols[i].Type, expected[i].Name, expected[i].Type) + } + } +} + +func TestGenerateColumnDefinitions_embeddedStruct(t *testing.T) { + type EmbeddedStruct struct { + EmbeddedField string `osquery:"embedded_field"` + } + type testStruct struct { + EmbeddedStruct + TestField string `osquery:"test_field"` + } + expected := []table.ColumnDefinition{ + table.TextColumn("embedded_field"), + table.TextColumn("test_field"), + } + cols, err := GenerateColumnDefinitions(testStruct{}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(cols) != len(expected) { + t.Fatalf("expected %d columns, got %d", len(expected), len(cols)) + } + for i := range cols { + if cols[i].Name != expected[i].Name || cols[i].Type != expected[i].Type { + t.Errorf("column %d: got (%s, %s), want (%s, %s)", i, cols[i].Name, cols[i].Type, expected[i].Name, expected[i].Type) + } + } +} diff --git a/x-pack/winlogbeat/module/testing.go b/x-pack/winlogbeat/module/testing.go index 0add079c4343..c4a0f4ab23f6 100644 --- a/x-pack/winlogbeat/module/testing.go +++ b/x-pack/winlogbeat/module/testing.go @@ -207,7 +207,7 @@ func testIngestPipeline(t *testing.T, pipeline, pattern string, p *params) { // assertEqual asserts that the two objects are deeply equal. If not it will // error the test and output a diff of the two objects' JSON representation. -func assertEqual(t testing.TB, expected, actual interface{}) bool { +func assertEqual(t testing.TB, expected, actual any) bool { t.Helper() if reflect.DeepEqual(expected, actual) { diff --git a/x-pack/winlogbeat/module/wintest/docker_test.go b/x-pack/winlogbeat/module/wintest/docker_test.go index 46687a6d1d36..96c57926051d 100644 --- a/x-pack/winlogbeat/module/wintest/docker_test.go +++ b/x-pack/winlogbeat/module/wintest/docker_test.go @@ -83,8 +83,7 @@ func TestDocker(t *testing.T) { } defer conn.Close() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() err = conn.Connect(ctx) if err != nil { t.Fatalf("unexpected error making connection: %v", err) diff --git a/x-pack/winlogbeat/module/wintest/simulate_test.go b/x-pack/winlogbeat/module/wintest/simulate_test.go index a5e79f7cc00d..a22c0c30c0ce 100644 --- a/x-pack/winlogbeat/module/wintest/simulate_test.go +++ b/x-pack/winlogbeat/module/wintest/simulate_test.go @@ -11,7 +11,6 @@ package wintest_test import ( - "context" "encoding/json" "fmt" "os" @@ -74,8 +73,7 @@ func TestSimulate(t *testing.T) { } defer conn.Close() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() err = conn.Connect(ctx) if err != nil { t.Fatalf("unexpected error making connection: %v", err) From f60aabdee92fdc30c9556ed97a31aafc8b20bc16 Mon Sep 17 00:00:00 2001 From: Andrzej Stencel Date: Wed, 22 Jul 2026 16:17:04 +0200 Subject: [PATCH 2/4] fix conflicts --- heartbeat/autodiscover/builder/hints/monitors_test.go | 7 +------ libbeat/autodiscover/providers/kubernetes/node.go | 8 -------- libbeat/processors/add_kubernetes_metadata/kubernetes.go | 7 ------- 3 files changed, 1 insertion(+), 21 deletions(-) diff --git a/heartbeat/autodiscover/builder/hints/monitors_test.go b/heartbeat/autodiscover/builder/hints/monitors_test.go index 96124c29434f..2372fd843962 100644 --- a/heartbeat/autodiscover/builder/hints/monitors_test.go +++ b/heartbeat/autodiscover/builder/hints/monitors_test.go @@ -23,13 +23,8 @@ import ( "github.com/stretchr/testify/assert" -<<<<<<< HEAD - "github.com/elastic/elastic-agent-autodiscover/bus" - "github.com/elastic/elastic-agent-libs/logp" -======= "github.com/elastic/beats/v7/pkg/autodiscover/bus" - "github.com/elastic/elastic-agent-libs/logp/logptest" ->>>>>>> cf032f423 (chore: import autodiscover repository (#51804)) + "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/libbeat/autodiscover/providers/kubernetes/node.go b/libbeat/autodiscover/providers/kubernetes/node.go index 776bedaa26f0..176ec8396fcd 100644 --- a/libbeat/autodiscover/providers/kubernetes/node.go +++ b/libbeat/autodiscover/providers/kubernetes/node.go @@ -147,13 +147,6 @@ func (n *node) GenerateHints(event bus.Event) bus.Event { var kubeMeta mapstr.M rawMeta, ok := event["kubernetes"] if ok { -<<<<<<< HEAD - kubeMeta = rawMeta.(mapstr.M) - // The builder base config can configure any of the field values of kubernetes if need be. - e["kubernetes"] = kubeMeta - if rawAnn, ok := kubeMeta["annotations"]; ok { - annotations = rawAnn.(mapstr.M) -======= kubeMeta, ok = rawMeta.(mapstr.M) if ok { // The builder base config can configure any of the field values of kubernetes if need be. @@ -161,7 +154,6 @@ func (n *node) GenerateHints(event bus.Event) bus.Event { if rawAnn, ok := kubeMeta["annotations"]; ok { annotations = rawAnn.(mapstr.M) //nolint:errcheck // type validated by map lookup } ->>>>>>> cf032f423 (chore: import autodiscover repository (#51804)) } } if host, ok := event["host"]; ok { diff --git a/libbeat/processors/add_kubernetes_metadata/kubernetes.go b/libbeat/processors/add_kubernetes_metadata/kubernetes.go index d1d661e8a22e..dbe79a72bce4 100644 --- a/libbeat/processors/add_kubernetes_metadata/kubernetes.go +++ b/libbeat/processors/add_kubernetes_metadata/kubernetes.go @@ -30,13 +30,6 @@ import ( k8sclient "k8s.io/client-go/kubernetes" -<<<<<<< HEAD - "github.com/elastic/elastic-agent-autodiscover/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" -======= - "go.opentelemetry.io/collector/pdata/pcommon" - ->>>>>>> cf032f423 (chore: import autodiscover repository (#51804)) "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" From ac2a1c2d339cf756765818d4ba469cdafbf14dfa Mon Sep 17 00:00:00 2001 From: Andrzej Stencel Date: Wed, 22 Jul 2026 16:24:20 +0200 Subject: [PATCH 3/4] go mod tidy && make notice --- NOTICE.txt | 565 +++++++++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 5 + go.sum | 10 + 3 files changed, 580 insertions(+) diff --git a/NOTICE.txt b/NOTICE.txt index 69d2bad52b98..b789e6f3961a 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -32080,12 +32080,254 @@ library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. +-------------------------------------------------------------------------------- +Dependency : sigs.k8s.io/kind +Version: v0.32.0 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/sigs.k8s.io/kind@v0.32.0/LICENSE: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + ================================================================================ Indirect dependencies +-------------------------------------------------------------------------------- +Dependency : al.essio.dev/pkg/shellescape +Version: v1.5.1 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/al.essio.dev/pkg/shellescape@v1.5.1/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2016 Alessio Treglia + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : aqwari.net/xml Version: v0.0.0-20210331023308-d9421b293817 @@ -37386,6 +37628,37 @@ Contents of probable licence file $GOMODCACHE/github.com/!azure!a!d/microsoft-au SOFTWARE +-------------------------------------------------------------------------------- +Dependency : github.com/BurntSushi/toml +Version: v1.4.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/!burnt!sushi/toml@v1.4.0/COPYING: + +The MIT License (MIT) + +Copyright (c) 2013 TOML authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp Version: v1.31.0 @@ -48249,6 +48522,41 @@ Contents of probable licence file $GOMODCACHE/github.com/envoyproxy/protoc-gen-v limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/evanphx/json-patch/v5 +Version: v5.6.0 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/evanphx/json-patch/v5@v5.6.0/LICENSE: + +Copyright (c) 2014, Evan Phoenix +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +* Neither the name of the Evan Phoenix nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + -------------------------------------------------------------------------------- Dependency : github.com/fearful-symmetry/gomsr Version: v0.0.1 @@ -67075,6 +67383,263 @@ Contents of probable licence file $GOMODCACHE/github.com/oxtoacart/bpool@v0.0.0- limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/pelletier/go-toml +Version: v1.9.5 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/pelletier/go-toml@v1.9.5/LICENSE: + +The bulk of github.com/pelletier/go-toml is distributed under the MIT license +(see below), with the exception of localtime.go and localtime.test.go. +Those two files have been copied over from Google's civil library at revision +ed46f5086358513cf8c25f8e3f022cb838a49d66, and are distributed under the Apache +2.0 license (see below). + + +github.com/pelletier/go-toml: + + +The MIT License (MIT) + +Copyright (c) 2013 - 2021 Thomas Pelletier, Eric Anderton + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +localtime.go, localtime_test.go: + +Originals: + https://raw.githubusercontent.com/googleapis/google-cloud-go/ed46f5086358513cf8c25f8e3f022cb838a49d66/civil/civil.go + https://raw.githubusercontent.com/googleapis/google-cloud-go/ed46f5086358513cf8c25f8e3f022cb838a49d66/civil/civil_test.go +Changes: + * Renamed files from civil* to localtime*. + * Package changed from civil to toml. + * 'Local' prefix added to all structs. +License: + https://raw.githubusercontent.com/googleapis/google-cloud-go/ed46f5086358513cf8c25f8e3f022cb838a49d66/LICENSE + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Dependency : github.com/pkg/browser Version: v0.0.0-20240102092130-5ac0b6a4141c diff --git a/go.mod b/go.mod index aaaa3f4dbe9d..02e8f75a97aa 100644 --- a/go.mod +++ b/go.mod @@ -254,10 +254,15 @@ require ( go.opentelemetry.io/collector/service v0.155.0 go.opentelemetry.io/otel/sdk/metric v1.44.0 go.uber.org/goleak v1.3.0 + sigs.k8s.io/kind v0.32.0 ) require ( + al.essio.dev/pkg/shellescape v1.5.1 // indirect + github.com/BurntSushi/toml v1.4.0 // indirect github.com/cenkalti/backoff/v6 v6.0.0 // indirect + github.com/evanphx/json-patch/v5 v5.6.0 // indirect + github.com/pelletier/go-toml v1.9.5 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/twmb/franz-go v1.21.3 // indirect go.opentelemetry.io/collector/exporter v1.61.0 // indirect diff --git a/go.sum b/go.sum index 0042057f5a7b..f39958ea3aa5 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +al.essio.dev/pkg/shellescape v1.5.1 h1:86HrALUujYS/h+GtqoB26SBEdkWfmMI6FubjXlsXyho= +al.essio.dev/pkg/shellescape v1.5.1/go.mod h1:6sIqp7X2P6mThCQ7twERpZTuigpr6KbZWtls1U8I890= aqwari.net/xml v0.0.0-20210331023308-d9421b293817 h1:+3Rh5EaTzNLnzWx3/uy/mAaH/dGI7svJ6e0oOIDcPuE= aqwari.net/xml v0.0.0-20210331023308-d9421b293817/go.mod h1:c7kkWzc7HS/t8Q2DcVY8P2d1dyWNEhEVT5pL0ZHO11c= cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= @@ -128,6 +130,8 @@ github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mo github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= +github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 h1:DHa2U07rk8syqvCge0QIGMCE1WxGj9njT44GH7zNJLQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= @@ -440,6 +444,8 @@ github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJP github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= +github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= +github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= @@ -900,6 +906,8 @@ github.com/otiai10/mint v1.6.3 h1:87qsV/aw1F5as1eH1zS/yqHY85ANKVMgkDrf9rcxbQs= github.com/otiai10/mint v1.6.3/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM= github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2 h1:CXwSGu/LYmbjEab5aMCs5usQRVBGThelUKBNnoSOuso= github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2/go.mod h1:L3UMQOThbttwfYRNFOWLLVXMhk5Lkio4GGOtw5UrxS0= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.27 h1:+PhzhWDrjRj89TH2sw43nE3+4+W8lSxIuQadEHZyjUk= @@ -1571,6 +1579,8 @@ pgregory.net/rapid v1.3.0 h1:vBvO0VSqti75J1jjYqpgPNBLKMd1+gxa9fYo7vk/Exc= pgregory.net/rapid v1.3.0/go.mod h1:dPlE4OBBxgXPqkP79flB6sJL1dx5azpI7HQ9MY9Z7uk= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/kind v0.32.0 h1:p9hscbj98u/qyrjVpjId86LI70nQmbSsipV7wCG10Xk= +sigs.k8s.io/kind v0.32.0/go.mod h1:FSqriGaoTPruiXWfRnUXNykF8r2t+fHtK0P0m1AbGF8= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= From 0de2756e50809526d2ffe1b16a9fbbeca9eb7b22 Mon Sep 17 00:00:00 2001 From: Andrzej Stencel Date: Thu, 23 Jul 2026 13:34:02 +0200 Subject: [PATCH 4/4] remove files that do not belong in 8.19 --- .../tests/integration/autodiscover_test.go | 754 ---------------- .../pkg/encoding/encoding_test.go | 847 ------------------ 2 files changed, 1601 deletions(-) delete mode 100644 filebeat/tests/integration/autodiscover_test.go delete mode 100644 x-pack/osquerybeat/ext/osquery-extension/pkg/encoding/encoding_test.go diff --git a/filebeat/tests/integration/autodiscover_test.go b/filebeat/tests/integration/autodiscover_test.go deleted file mode 100644 index 9259a74f8eca..000000000000 --- a/filebeat/tests/integration/autodiscover_test.go +++ /dev/null @@ -1,754 +0,0 @@ -// Licensed to Elasticsearch B.V. under one or more contributor -// license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright -// ownership. Elasticsearch B.V. licenses this file to you under -// the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -//go:build integration && !requirefips - -package integration - -import ( - "bufio" - "context" - "fmt" - "io" - "os" - "os/user" - "path/filepath" - "strconv" - "strings" - "testing" - "time" - - "github.com/gofrs/uuid/v5" - "github.com/moby/moby/api/types/container" - "github.com/moby/moby/client" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/kubernetes" - "k8s.io/client-go/tools/clientcmd" - "sigs.k8s.io/kind/pkg/apis/config/v1alpha4" - "sigs.k8s.io/kind/pkg/cluster" - "sigs.k8s.io/kind/pkg/cluster/nodeutils" - - "github.com/elastic/beats/v7/libbeat/tests/integration" - "github.com/elastic/beats/v7/libbeat/version" - "github.com/elastic/beats/v7/pkg/autodiscover/docker" - "github.com/elastic/elastic-agent-libs/logp" - "github.com/elastic/elastic-agent-libs/testing/fs" -) - -func TestHintsDocker(t *testing.T) { - containerID := startFlogDocker(t) - filebeat := integration.NewBeat( - t, - "filebeat", - "../../filebeat.test", - ) - - cfgYAML := getConfig(t, nil, "autodiscover", "docker.yml") - filebeat.WriteConfigFile(cfgYAML) - filebeat.Start() - - // By ensuring the Filestream input started with the correct ID, we're - // testing that the whole autodiscover + hints is working as expected. - filebeat.WaitLogsContains( - fmt.Sprintf( - `"message":"Input 'filestream' starting","service.name":"filebeat","id":"container-logs-%s"`, - containerID, - ), - 30*time.Second, - "Filestream did not start for the test container") -} - -func TestHintsKubernetes(t *testing.T) { - filebeat := integration.NewBeat( - t, - "filebeat", - "../../filebeat.test", - ) - - kubeConfigPath, _ := createKindCluster(t, filebeat.TempDir()) - noneName, _, containerID := startFlogKubernetes(t, kubeConfigPath) - - cfgYAML := getConfig( - t, - map[string]any{ - "kubeConfig": kubeConfigPath, - "nodeName": noneName, - }, - "autodiscover", - "k8s.yml") - filebeat.WriteConfigFile(cfgYAML) - filebeat.Start() - - // By ensuring the Filestream input started with the correct ID, we're - // testing that the whole autodiscover + hints is working as expected. - filebeat.WaitLogsContains( - fmt.Sprintf( - `"message":"Input 'filestream' starting","service.name":"filebeat","id":"container-logs-%s"`, - containerID, - ), - 30*time.Second, - "Filestream did not start for the test container") -} - -func TestAutodiscoverFilestreamTakeOverDoesNotReingest(t *testing.T) { - integration.EnsureESIsRunning(t) - filebeatImage := "docker.elastic.co/beats/filebeat-oss-wolfi" + ":" + version.GetDefaultVersion() + "-SNAPSHOT" - - workDir := fs.TempDir(t, "..", "..", "build", "integration-tests") - - kubeConfigPath, clusterName := createKindCluster(t, workDir, - cluster.CreateWithV1Alpha4Config(&v1alpha4.Cluster{ - Nodes: []v1alpha4.Node{ - { - Role: v1alpha4.ControlPlaneRole, - ExtraMounts: []v1alpha4.Mount{ - { - HostPath: workDir, - ContainerPath: workDir, - }, - }, - }, - }, - })) - nodeName, podName, _ := startFlogKubernetes(t, kubeConfigPath) - - loadDockerImageIntoKind(t, clusterName, filebeatImage) - grantClusterAdminToDefaultServiceAccount(t, kubeConfigPath) - - filebeatPodName := "filebeat-pod-" + uuid.Must(uuid.NewV4()).String() - logInputConfigPath := filepath.Join(workDir, "filebeat-log.yml") - filestreamInputConfigPath := filepath.Join(workDir, "filebeat-filestream.yml") - - esURL := integration.GetESAdminURL(t, "http") - esHost := fmt.Sprintf("%s://%s", esURL.Scheme, esURL.Host) - if esURL.Hostname() == "localhost" || esURL.Hostname() == "127.0.0.1" { - esHost = fmt.Sprintf("%s://%s:%s", esURL.Scheme, kindNodeGatewayIP(t, nodeName), esURL.Port()) - } - esUser := esURL.User.Username() - esPass, _ := esURL.User.Password() - - index := fmt.Sprintf("test-autodiscover-take-over-%s", uuid.Must(uuid.NewV4()).String()) - t.Cleanup(func() { - if t.Failed() { - t.Logf("Elasticsearch index used: %q", index) - } - }) - - tmplVars := map[string]any{ - "nodeName": nodeName, - "podName": podName, - "esHost": esHost, - "esUser": esUser, - "esPass": esPass, - "index": index, - } - - writeFile( - t, - logInputConfigPath, - getConfig(t, tmplVars, "autodiscover", "take-over-log-input-k8s.yml"), - ) - - startFilebeatPodForTakeOver( - t, - kubeConfigPath, - nodeName, - filebeatPodName, - filebeatImage, - workDir, - logInputConfigPath, - ) - - // Wait until at least 5 events are ingested - require.Eventually( - t, - func() bool { return countEventsInES(t, index, 1000) >= 5 }, - 30*time.Second, - 200*time.Millisecond, - "did not ingest the initial events") - - deletePodK8s(t, kubeConfigPath, filebeatPodName) - logInputIngested := countEventsInES(t, index, 1000) - - // Re-Start Filebeat with Filestream and take_over enabled - writeFile( - t, - filestreamInputConfigPath, - getConfig(t, tmplVars, "autodiscover", "take-over-filestream-input-k8s.yml"), - ) - - startFilebeatPodForTakeOver( - t, - kubeConfigPath, - nodeName, - filebeatPodName, - filebeatImage, - workDir, - filestreamInputConfigPath, - ) - - // Wait for at least two extra events to be ingested - require.EventuallyWithT( - t, - func(collect *assert.CollectT) { - totalEvents := countEventsInES(t, index, 1000) - if totalEvents <= logInputIngested+2 { - collect.Errorf( - "expecting more events in Elasticsearch than the %d from the first run", - logInputIngested, - ) - } - }, - 10*time.Second, - time.Second, - "No new events ingested") - - // Stop the pod and get the total number of events generated - generatedEvents := stopPodK8sAndCountLogs(t, kubeConfigPath, podName) - - // Wait for Filebeat to fully ingest the file, we do it by waiting the file - // to be closed due to inactivity. - waitFilebeatLogContains( - t, - workDir, - "File is inactive. Closing.", - 20*time.Second, - ) - - // Wait until at least all published events can be found in Elasticsearch - require.Eventuallyf( - t, - func() bool { return countEventsInES(t, index, generatedEvents+100) >= generatedEvents }, - 30*time.Second, - 200*time.Millisecond, - "did not ingest all generated events from pod %s", podName, - ) - - deletePodK8s(t, kubeConfigPath, filebeatPodName) - - totalEventsIngested := countEventsInES(t, index, generatedEvents+100) - - require.Equalf( - t, - generatedEvents, - totalEventsIngested, - "file re-ingestion has occurred\n"+ - "Generated Events: %d\n"+ - "Events ingested by the Log input: %d\n"+ - "Total number of events ingested: %d", - generatedEvents, - logInputIngested, - totalEventsIngested, - ) -} - -func createKindCluster( - t *testing.T, - workDir string, - options ...cluster.CreateOption, -) (kubeConfigPath, clusterName string) { - - uid := uuid.Must(uuid.NewV4()).String() - - defer func() { - if t.Failed() { - t.Log("To see the Kind logs search for 'cluster.ProviderWithLogger' and uncomment it.") - } - }() - - provider := cluster.NewProvider( - // Uncomment the next line to have Kind logs written to stderr. - // You will also have to import "sigs.k8s.io/kind/pkg/cmd" - // cluster.ProviderWithLogger(cmd.NewLogger()), - ) - - clusterName = fmt.Sprintf("test-cluster-%s", uid) - err := provider.Create( - clusterName, - append( - []cluster.CreateOption{cluster.CreateWithWaitForReady(30 * time.Second)}, - options...)..., - ) - if err != nil { - t.Fatalf("could not create cluster: %s", err) - } - t.Cleanup(func() { - if err := provider.Delete(clusterName, ""); err != nil { - t.Logf("could not delete K8s cluster: %s", err) - } - }) - - var kubeConfig string - require.Eventually(t, func() bool { - kubeConfig, err = provider.KubeConfig(clusterName, false) - return err == nil - }, 30*time.Second, 100*time.Millisecond, "could not get kube config") - - kubeConfigPath = filepath.Join(workDir, "kube-config") - if err = os.WriteFile(kubeConfigPath, []byte(kubeConfig), 0666); err != nil { - t.Fatalf("cannot write kube config file: %s", err) - } - - return kubeConfigPath, clusterName -} - -func startFlogKubernetes(t *testing.T, kubeConfigPath string) (nodeName, podName, containerID string) { - clientset := newK8sClientsetFromKubeConfigPath(t, kubeConfigPath) - - podName = "flog-pod-" + uuid.Must(uuid.NewV4()).String() - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: podName, - Namespace: "default", - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "flog", - Image: "mingrammer/flog", - Args: []string{"-s", "0.2", "-d", "0.2", "-l"}, - }, - }, - }, - } - - pod, err := clientset.CoreV1().Pods("default").Create(context.Background(), pod, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("could not create pod: %s", err) - } - - t.Cleanup(func() { - _, err := clientset.CoreV1().Pods("default").Get(context.Background(), pod.Name, metav1.GetOptions{}) - if apierrors.IsNotFound(err) { - // The pod has already been removed, return - return - } - - // by the time Cleanup runs, t.Context has been cancelled, so we need a new context here - err = clientset.CoreV1().Pods("default").Delete(context.Background(), pod.Name, metav1.DeleteOptions{}) - if err != nil { - t.Logf("could not remove pod: %s", err) - } - }) - - require.Eventually( - t, - func() bool { - pod, err = clientset.CoreV1().Pods("default").Get(t.Context(), pod.Name, metav1.GetOptions{}) - if err != nil { - return false - } - - if pod.Status.Phase == corev1.PodRunning && len(pod.Status.ContainerStatuses) > 0 { - containerID = pod.Status.ContainerStatuses[0].ContainerID - if containerID != "" { - nodeName = pod.Spec.NodeName - // Remove the runtime prefix (e.g., "containerd://") - if idx := strings.Index(containerID, "://"); idx != -1 { - containerID = containerID[idx+3:] - return true - } - return true - } - } - - return false - }, - 60*time.Second, - 100*time.Millisecond, - "pod did not start within timeout", - ) - - return nodeName, podName, containerID -} - -// startFlogDocker starts a `mingrammer/flog` that logs one line every -// second. The container ID is returned and the container is stopped at the -// end of the test. On error the test fails by calling t.Fatalf -func startFlogDocker(t *testing.T) string { - ctx := t.Context() - img := "mingrammer/flog:0.4.3" - cli, err := docker.NewClient(client.DefaultDockerHost, nil, nil, logp.NewNopLogger()) - if err != nil { - t.Fatalf("cannot create Docker client: %s", err) - } - - // Pull the image first - reader, err := cli.ImagePull(ctx, img, client.ImagePullOptions{}) - if err != nil { - t.Fatalf("cannot pull image %q: %s", img, err) - } - defer reader.Close() - - // Wait for the pull to complete by reading the response - _, err = io.Copy(io.Discard, reader) - if err != nil { - t.Fatalf("error while pulling image %q: %s", img, err) - } - - resp, err := cli.ContainerCreate( - ctx, - client.ContainerCreateOptions{ - Config: &container.Config{ - Image: img, - Cmd: []string{"-l", "-d", "1", "-s", "1"}, - }, - }) - if err != nil { - t.Fatalf("cannot create container for %q: %s", img, err) - } - - _, err = cli.ContainerStart(ctx, resp.ID, client.ContainerStartOptions{}) - if err != nil { - t.Fatalf("cannot start container: %s", err) - } - - t.Cleanup(func() { - ctx := context.Background() - if _, err := cli.ContainerStop(ctx, resp.ID, client.ContainerStopOptions{}); err != nil { - t.Errorf("cannot stop container: %s", err) - } - if _, err := cli.ContainerRemove(ctx, resp.ID, client.ContainerRemoveOptions{}); err != nil { - t.Errorf("cannot remove container: %s", err) - } - }) - return resp.ID -} - -func loadDockerImageIntoKind(t *testing.T, clusterName, imageName string) { - provider := cluster.NewProvider() - nodes, err := provider.ListInternalNodes(clusterName) - if err != nil { - t.Fatalf( - "cannot list nodes for kind cluster %q: %s", - clusterName, - err, - ) - } - if len(nodes) == 0 { - t.Fatalf("no nodes found for kind cluster %q", clusterName) - } - - cli, err := docker.NewClient(client.DefaultDockerHost, nil, nil, logp.NewNopLogger()) - if err != nil { - t.Fatalf("cannot create Docker client: %s", err) - } - - for _, node := range nodes { - imageReader, err := cli.ImageSave(t.Context(), []string{imageName}) - if err != nil { - t.Fatalf("cannot save image %q from local docker daemon: %s", imageName, err) - } - - if err := nodeutils.LoadImageArchive(node, imageReader); err != nil { - _ = imageReader.Close() - t.Fatalf( - "cannot load image %q into kind node %q: %s", - imageName, - node.String(), - err, - ) - } - - if err := imageReader.Close(); err != nil { - t.Fatalf( - "cannot close image stream for image %q on node %q: %s", - imageName, - node.String(), - err, - ) - } - } -} - -func kindNodeGatewayIP(t *testing.T, nodeName string) string { - cli, err := docker.NewClient(client.DefaultDockerHost, nil, nil, logp.NewNopLogger()) - if err != nil { - t.Fatalf("cannot create Docker client: %s", err) - } - - inspectResult, err := cli.ContainerInspect(context.Background(), nodeName, client.ContainerInspectOptions{}) - if err != nil { - t.Fatalf("cannot inspect Kind node %q: %s", nodeName, err) - } - - if inspectResult.Container.NetworkSettings != nil { - for _, networkSettings := range inspectResult.Container.NetworkSettings.Networks { - if networkSettings != nil && networkSettings.Gateway.IsValid() { - return networkSettings.Gateway.String() - } - } - } - - t.Fatalf("cannot determine gateway IP for Kind node %q", nodeName) - return "" -} - -func grantClusterAdminToDefaultServiceAccount(t *testing.T, kubeConfigPath string) { - cs := newK8sClientsetFromKubeConfigPath(t, kubeConfigPath) - - bindingName := "filebeat-autodiscover-admin-" + uuid.Must(uuid.NewV4()).String() - _, err := cs.RbacV1().ClusterRoleBindings().Create(t.Context(), &rbacv1.ClusterRoleBinding{ - ObjectMeta: metav1.ObjectMeta{ - Name: bindingName, - }, - Subjects: []rbacv1.Subject{ - { - Kind: "ServiceAccount", - Name: "default", - Namespace: "default", - }, - }, - RoleRef: rbacv1.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "ClusterRole", - Name: "cluster-admin", - }, - }, metav1.CreateOptions{}) - if err != nil { - t.Fatalf("cannot create cluster role binding for default service account: %s", err) - } -} - -func writeFile(t *testing.T, path, content string) { - if err := os.WriteFile(path, []byte(content), 0o644); err != nil { - t.Fatalf("cannot write file %q: %s", path, err) - } -} - -func startFilebeatPodForTakeOver( - t *testing.T, - kubeConfigPath, - nodeName, - podName, - imageName, - workDir, - configPath string, -) { - - cs := newK8sClientsetFromKubeConfigPath(t, kubeConfigPath) - hostPathDir := corev1.HostPathDirectory - - user, err := user.Current() - if err != nil { - t.Fatalf("cannot get current user: %s", err) - } - udi, err := strconv.Atoi(user.Uid) - if err != nil { - t.Fatalf("cannot convert UID from string to integer: %s", err) - } - - // Small hack to make sure the test can read the files generated by the - // container (Filebeat): run Filebeat (the container) with the same UID - // as the current user. That is required for CI - uid := int64(udi) - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: podName, - Namespace: "default", - }, - Spec: corev1.PodSpec{ - NodeName: nodeName, - RestartPolicy: corev1.RestartPolicyNever, - Containers: []corev1.Container{ - { - Name: "filebeat", - Image: imageName, - ImagePullPolicy: corev1.PullNever, - Args: []string{ - "--strict.perms=false", - "-c", configPath, - "-E", fmt.Sprintf("path.home=%s", workDir), - }, - SecurityContext: &corev1.SecurityContext{ - RunAsUser: &uid, - }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "home-folder", - MountPath: workDir, - }, - { - Name: "varlogcontainers", - MountPath: "/var/log/containers", - ReadOnly: true, - }, - { - Name: "varlogpods", - MountPath: "/var/log/pods", - ReadOnly: true, - }, - }, - }, - }, - Volumes: []corev1.Volume{ - { - Name: "home-folder", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: workDir, - Type: &hostPathDir, - }, - }, - }, - { - Name: "varlogcontainers", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/var/log/containers", - Type: &hostPathDir, - }, - }, - }, - { - Name: "varlogpods", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/var/log/pods", - Type: &hostPathDir, - }, - }, - }, - }, - }, - } - - if _, err := cs.CoreV1().Pods("default").Create(t.Context(), pod, metav1.CreateOptions{}); err != nil { - t.Fatalf("could not create filebeat pod: %s", err) - } - - require.Eventuallyf( - t, - func() bool { - pod, err := cs.CoreV1().Pods("default").Get(t.Context(), podName, metav1.GetOptions{}) - if err != nil { - return false - } - if pod.Status.Phase == corev1.PodFailed { - t.Logf("filebeat pod failed: %v", pod.Status) - return false - } - return pod.Status.Phase == corev1.PodRunning - }, - 60*time.Second, - 200*time.Millisecond, - "filebeat pod %q did not start", podName, - ) -} - -func waitFilebeatLogContains(t *testing.T, workDir, msg string, timeout time.Duration) { - t.Helper() - // Glob to match the date, it will stop working in about 1000 years - paths, err := filepath.Glob(filepath.Join(workDir, "logs", "filebeat-2*.ndjson")) - if err != nil { - t.Fatalf("cannot resolve glob for log files: %s", err) - } - - if len(paths) != 1 { - t.Fatalf("There must be a single log file for Filebeat, found %d", len(paths)) - } - - f, err := os.Open(paths[0]) - if err != nil { - t.Fatalf("cannot open Filebeat log file: %s", err) - } - defer f.Close() - - logFile := fs.LogFile{File: f} - logFile.WaitLogsContains(t, msg, timeout, "Filebeat logs did not contain '%s'", msg) -} - -func deletePodK8s(t *testing.T, kubeConfigPath, podName string) { - cs := newK8sClientsetFromKubeConfigPath(t, kubeConfigPath) - - err := cs.CoreV1().Pods("default").Delete(context.Background(), podName, metav1.DeleteOptions{}) - if err != nil && !apierrors.IsNotFound(err) { - t.Fatalf("cannot delete pod %q: %s", podName, err) - } - - require.Eventuallyf( - t, - func() bool { - _, err := cs.CoreV1().Pods("default").Get(context.Background(), podName, metav1.GetOptions{}) - return apierrors.IsNotFound(err) - }, - 30*time.Second, - 100*time.Millisecond, - "pod %q was not deleted", podName, - ) -} - -// stopPodK8sAndCountLogs returns the number of log lines generated by the pod -// and then deletes it. -func stopPodK8sAndCountLogs(t *testing.T, kubeConfigPath, podName string) int { - cs := newK8sClientsetFromKubeConfigPath(t, kubeConfigPath) - - logsReader, err := cs.CoreV1().Pods("default").GetLogs(podName, &corev1.PodLogOptions{ - Container: "flog", - Follow: true, - }).Stream(t.Context()) - if err != nil { - t.Fatalf("cannot get logs for pod %q: %s", podName, err) - } - defer logsReader.Close() - - if err := cs.CoreV1().Pods("default").Delete(context.Background(), podName, metav1.DeleteOptions{}); err != nil { - t.Fatalf("cannot delete pod %q: %s", podName, err) - } - - logLines := countReaderLines(t, logsReader) - - return logLines -} - -func newK8sClientsetFromKubeConfigPath(t *testing.T, kubeConfigPath string) *kubernetes.Clientset { - data, err := os.ReadFile(kubeConfigPath) - if err != nil { - t.Fatalf("cannot read kube config: %s", err) - } - config, err := clientcmd.RESTConfigFromKubeConfig(data) - if err != nil { - t.Fatalf("cannot build REST config: %s", err) - } - cs, err := kubernetes.NewForConfig(config) - if err != nil { - t.Fatalf("cannot create clientset: %s", err) - } - return cs -} - -func countEventsInES(t *testing.T, index string, size int) int { - return len(integration.GetEventsMsgFromES(t, index, size)) -} - -func countReaderLines(t *testing.T, r io.Reader) int { - s := bufio.NewScanner(r) - s.Buffer(make([]byte, 0, 64*1024), 1024*1024) - n := 0 - for s.Scan() { - n++ - } - if err := s.Err(); err != nil { - t.Fatalf("cannot scan logs: %s", err) - } - return n -} diff --git a/x-pack/osquerybeat/ext/osquery-extension/pkg/encoding/encoding_test.go b/x-pack/osquerybeat/ext/osquery-extension/pkg/encoding/encoding_test.go deleted file mode 100644 index 2edf84021b6a..000000000000 --- a/x-pack/osquerybeat/ext/osquery-extension/pkg/encoding/encoding_test.go +++ /dev/null @@ -1,847 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License; -// you may not use this file except in compliance with the Elastic License. - -package encoding - -import ( - "reflect" - "testing" - "time" - - "github.com/osquery/osquery-go/plugin/table" -) - -func TestEncodingFlagHas(t *testing.T) { - tests := []struct { - flag EncodingFlag - option EncodingFlag - expected bool - }{ - {EncodingFlagUseNumbersZeroValues, EncodingFlagUseNumbersZeroValues, true}, - } - - for _, test := range tests { - result := test.flag.has(test.option) - if result != test.expected { - t.Errorf("has(%v) = %v; expected %v", test.option, result, test.expected) - } - } -} - -func TestMarshalToMapWithFlags(t *testing.T) { - tests := []struct { - name string - input any - flags EncodingFlag - expected map[string]string - err bool - }{ - { - name: "nil input", - input: nil, - flags: 0, - expected: nil, - err: true, - }, - { - name: "struct with osquery tag", - input: &struct { - Name string `osquery:"name"` - }{Name: "test"}, - flags: 0, - expected: map[string]string{"name": "test"}, - err: false, - }, - { - name: "map input", - input: map[string]any{"key1": "value1", "key2": "value2", "key3": 1}, - flags: 0, - expected: map[string]string{"key1": "value1", "key2": "value2", "key3": "1"}, - err: false, - }, - { - name: "struct with hidden field", - input: &struct { - HiddenField int `osquery:"-"` - }{HiddenField: 42}, - flags: 0, - expected: map[string]string{}, - err: false, - }, - { - name: "invalid type", - input: &struct { - InvalidType map[int]string - }{InvalidType: map[int]string{1: "value"}}, - flags: 0, - expected: map[string]string{"InvalidType": "map[1:value]"}, - err: false, - }, - { - name: "zero value int", - input: &struct { - ZeroVal int - }{ZeroVal: 0}, - flags: 0, - expected: map[string]string{"ZeroVal": ""}, - err: false, - }, - { - name: "zero value int with flag", - input: &struct { - ZeroVal int - }{ZeroVal: 0}, - flags: EncodingFlagUseNumbersZeroValues, - expected: map[string]string{"ZeroVal": "0"}, - err: false, - }, - // Test bool type - { - name: "bool type", - input: &struct { - IsActive bool - }{IsActive: true}, - flags: 0, - expected: map[string]string{"IsActive": "1"}, - err: false, - }, - { - name: "bool type false", - input: &struct { - IsActive bool - }{IsActive: false}, - flags: 0, - expected: map[string]string{"IsActive": "0"}, - err: false, - }, - // Test uint type - { - name: "uint type", - input: &struct { - Count uint - }{Count: 42}, - flags: 0, - expected: map[string]string{"Count": "42"}, - err: false, - }, - { - name: "zero value uint", - input: &struct { - Count uint - }{Count: 0}, - flags: 0, - expected: map[string]string{"Count": ""}, - err: false, - }, - { - name: "zero value uint with flag", - input: &struct { - Count uint - }{Count: 0}, - flags: EncodingFlagUseNumbersZeroValues, - expected: map[string]string{"Count": "0"}, - err: false, - }, - // Test float type - { - name: "float64 type", - input: &struct { - Price float64 - }{Price: 99.99}, - flags: 0, - expected: map[string]string{"Price": "99.99"}, - err: false, - }, - { - name: "float32 type", - input: &struct { - Price float32 - }{Price: 12.5}, - flags: 0, - expected: map[string]string{"Price": "12.5"}, - err: false, - }, - // Test non-pointer struct - { - name: "non-pointer struct", - input: struct { - Name string - }{Name: "test"}, - flags: 0, - expected: map[string]string{"Name": "test"}, - err: false, - }, - // Test pointer maps - { - name: "pointer map", - input: &map[string]string{ - "key1": "value1", - "key2": "value2", - }, - flags: 0, - expected: map[string]string{"key1": "value1", "key2": "value2"}, - err: false, - }, - // Test pointer fields - { - name: "pointer fields", - input: &struct { - StrPtr *string - IntPtr *int - }{ - StrPtr: new("hello"), - IntPtr: new(123), - }, - flags: 0, - expected: map[string]string{"StrPtr": "hello", "IntPtr": "123"}, - err: false, - }, - // Test nil pointer fields - { - name: "nil pointer fields", - input: &struct { - StrPtr *string - IntPtr *int - }{ - StrPtr: nil, - IntPtr: nil, - }, - flags: 0, - expected: map[string]string{"StrPtr": "", "IntPtr": ""}, - err: false, - }, - // Test time.Time type - { - name: "time.Time type", - input: &struct { - Time time.Time `osquery:"time"` - }{Time: time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)}, - flags: 0, - expected: map[string]string{"time": "2023-06-15T14:30:00Z"}, - err: false, - }, - { - name: "time.Time", - input: &struct { - Time time.Time `osquery:"time" format:"unix"` - }{Time: time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)}, - flags: 0, - expected: map[string]string{"time": "1686839400"}, - err: false, - }, - } - - for _, test := range tests { - result, err := MarshalToMapWithFlags(test.input, test.flags) - if (err != nil) != test.err { - t.Errorf("%s: MarshalToMapWithFlags(%v, %v) error = %v; expected error = %v", test.name, test.input, test.flags, err, test.err) - continue - } - if !reflect.DeepEqual(result, test.expected) { - t.Errorf("%s: MarshalToMapWithFlags(%v, %v) = %v; expected %v", test.name, test.input, test.flags, result, test.expected) - continue - } - } -} - -func tagPtr(tag string) *reflect.StructTag { - st := reflect.StructTag(tag) - return &st -} - -func Test_formatTimeWithTagFormat(t *testing.T) { - tests := []struct { - name string // description of this test case - // Named input parameters for target function. - fieldValue reflect.Value - flag EncodingFlag - tag *reflect.StructTag - want string - wantErr bool - }{ - { - name: "RFC3339 format", - fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), - flag: 0, - tag: tagPtr(`format:"rfc3339"`), - want: "2023-06-15T14:30:00Z", - wantErr: false, - }, - { - name: "RFC3339Nano format", - fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 123456789, time.UTC)), - flag: 0, - tag: tagPtr(`format:"rfc3339nano"`), - want: "2023-06-15T14:30:00.123456789Z", - wantErr: false, - }, - { - name: "Unix timestamp format", - fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), - flag: 0, - tag: tagPtr(`format:"unix"`), - want: "1686839400", - wantErr: false, - }, - { - name: "RFC822 format", - fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), - flag: 0, - tag: tagPtr(`format:"rfc822"`), - want: "15 Jun 23 14:30 UTC", - wantErr: false, - }, - { - name: "RFC822Z format", - fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), - flag: 0, - tag: tagPtr(`format:"rfc822z"`), - want: "15 Jun 23 14:30 +0000", - wantErr: false, - }, - { - name: "RFC850 format", - fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), - flag: 0, - tag: tagPtr(`format:"rfc850"`), - want: "Thursday, 15-Jun-23 14:30:00 UTC", - wantErr: false, - }, - { - name: "RFC1123 format", - fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), - flag: 0, - tag: tagPtr(`format:"rfc1123"`), - want: "Thu, 15 Jun 2023 14:30:00 UTC", - wantErr: false, - }, - { - name: "RFC1123Z format", - fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), - flag: 0, - tag: tagPtr(`format:"rfc1123z"`), - want: "Thu, 15 Jun 2023 14:30:00 +0000", - wantErr: false, - }, - { - name: "Kitchen format", - fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), - flag: 0, - tag: tagPtr(`format:"kitchen"`), - want: "2:30PM", - wantErr: false, - }, - { - name: "StampMilli format", - fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 123000000, time.UTC)), - flag: 0, - tag: tagPtr(`format:"stampmilli"`), - want: "Jun 15 14:30:00.123", - wantErr: false, - }, - { - name: "StampMicro format", - fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 123456000, time.UTC)), - flag: 0, - tag: tagPtr(`format:"stampmicro"`), - want: "Jun 15 14:30:00.123456", - wantErr: false, - }, - { - name: "StampNano format", - fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 123456789, time.UTC)), - flag: 0, - tag: tagPtr(`format:"stampnano"`), - want: "Jun 15 14:30:00.123456789", - wantErr: false, - }, - { - name: "Invalid format", - fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), - flag: 0, - tag: tagPtr(`format:"invalid"`), - want: "", - wantErr: true, - }, - { - name: "With timezone", - fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), - flag: 0, - tag: tagPtr(`format:"rfc3339" tz:"America/New_York"`), - want: "2023-06-15T10:30:00-04:00", - wantErr: false, - }, - { - name: "Invalid timezone", - fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), - flag: 0, - tag: tagPtr(`format:"rfc3339" tz:"Invalid/Timezone"`), - want: "", - wantErr: true, - }, - { - name: "Zero time with default flags", - fieldValue: reflect.ValueOf(time.Time{}), - flag: 0, - tag: tagPtr(`format:"rfc3339"`), - want: "", - wantErr: false, - }, - { - name: "Zero time with UseNumbersZeroValues flag", - fieldValue: reflect.ValueOf(time.Time{}), - flag: EncodingFlagUseNumbersZeroValues, - tag: tagPtr(`format:"rfc3339"`), - want: "0001-01-01T00:00:00Z", - wantErr: false, - }, - { - name: "Zero unix timestamp with UseNumbersZeroValues flag", - fieldValue: reflect.ValueOf(time.Time{}), - flag: EncodingFlagUseNumbersZeroValues, - tag: tagPtr(`format:"unix"`), - want: "0", - wantErr: false, - }, - { - name: "With timezone Asia/Tokyo", - fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), - flag: 0, - tag: tagPtr(`format:"rfc3339" tz:"Asia/Tokyo"`), - want: "2023-06-15T23:30:00+09:00", - wantErr: false, - }, - { - name: "With timezone Europe/London", - fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), - flag: 0, - tag: tagPtr(`format:"rfc3339" tz:"Europe/London"`), - want: "2023-06-15T15:30:00+01:00", - wantErr: false, - }, - { - name: "With timezone Australia/Sydney", - fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), - flag: 0, - tag: tagPtr(`format:"rfc3339" tz:"Australia/Sydney"`), - want: "2023-06-16T00:30:00+10:00", - wantErr: false, - }, - { - name: "Unix milliseconds format", - fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), - flag: 0, - tag: tagPtr(`format:"unixmilli"`), - want: "1686839400000", - wantErr: false, - }, - { - name: "Unix nanoseconds format", - fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), - flag: 0, - tag: tagPtr(`format:"unixnano"`), - want: "1686839400000000000", - wantErr: false, - }, - { - name: "Unix microseconds format", - fieldValue: reflect.ValueOf(time.Date(2023, 6, 15, 14, 30, 0, 0, time.UTC)), - flag: 0, - tag: tagPtr(`format:"unixmicro"`), - want: "1686839400000000", - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, gotErr := formatTimeWithTagFormat(tt.fieldValue, tt.flag, tt.tag) - if gotErr != nil { - if !tt.wantErr { - t.Errorf("Name: %s, formatTimeWithTagFormat() failed: %v", tt.name, gotErr) - } - return - } - if tt.wantErr { - t.Fatalf("Name: %s, formatTimeWithTagFormat() succeeded unexpectedly", tt.name) - } - // TODO: update the condition below to compare got with tt.want. - if got != tt.want { - t.Errorf("Name: %s, formatTimeWithTagFormat() = %v, want %v", tt.name, got, tt.want) - } - }) - } -} - -func TestMarshalToMapWithFlags_embeddedStruct(t *testing.T) { - type EmbeddedStruct struct { - EmbeddedField string `osquery:"embedded_field"` - } - - type TestStruct struct { - EmbeddedStruct - TestField string `osquery:"test_field"` - } - - type TestStructWithPointer struct { - *EmbeddedStruct - TestField string `osquery:"test_field"` - } - - cases := []struct { - name string - input any - expected map[string]string - }{ - { - name: "embedded struct", - input: &TestStruct{ - EmbeddedStruct: EmbeddedStruct{EmbeddedField: "embedded_value"}, - TestField: "test_value", - }, - expected: map[string]string{ - "embedded_field": "embedded_value", - "test_field": "test_value", - }, - }, - { - name: "embedded struct with pointer", - input: &TestStructWithPointer{ - EmbeddedStruct: &EmbeddedStruct{EmbeddedField: "embedded_value"}, - TestField: "test_value", - }, - expected: map[string]string{ - "embedded_field": "embedded_value", - "test_field": "test_value", - }, - }, - } - - for _, tt := range cases { - t.Run(tt.name, func(t *testing.T) { - result, err := MarshalToMapWithFlags(tt.input, 0) - if err != nil { - t.Errorf("MarshalToMapWithFlags() error = %v", err) - } - if !reflect.DeepEqual(result, tt.expected) { - t.Errorf("%s: MarshalToMapWithFlags() = %v, want %v", tt.name, result, tt.expected) - } - }) - } -} - -func TestGenerateColumnDefinitions(t *testing.T) { - tests := []struct { - name string - input any - expectedCols []table.ColumnDefinition - expectedError bool - }{ - { - name: "basic struct with all supported types", - input: struct { - Name string `osquery:"name"` - Count int64 `osquery:"count"` - Active bool `osquery:"active"` - Score float64 `osquery:"score"` - SmallNum int `osquery:"small_num"` - BigNum uint64 `osquery:"big_num"` - Precision float32 `osquery:"precision"` - }{}, - expectedCols: []table.ColumnDefinition{ - table.TextColumn("name"), - table.BigIntColumn("count"), - table.IntegerColumn("active"), - table.DoubleColumn("score"), - table.IntegerColumn("small_num"), - table.BigIntColumn("big_num"), - table.DoubleColumn("precision"), - }, - expectedError: false, - }, - { - name: "struct with skipped fields", - input: struct { - Included string `osquery:"included"` - Skipped string `osquery:"-"` - NoTag string - }{}, - expectedCols: []table.ColumnDefinition{ - table.TextColumn("included"), - table.TextColumn("NoTag"), // Fields without osquery tag use field name - }, - expectedError: false, - }, - { - name: "struct with pointer fields", - input: struct { - Name *string `osquery:"name"` - Count *int64 `osquery:"count"` - }{}, - expectedCols: []table.ColumnDefinition{ - table.TextColumn("name"), - table.BigIntColumn("count"), - }, - expectedError: false, - }, - { - name: "nil input", - input: nil, - expectedCols: nil, - expectedError: true, - }, - { - name: "non-struct input", - input: "string", - expectedCols: nil, - expectedError: true, - }, - { - name: "pointer to struct", - input: &struct { - Field string `osquery:"field"` - }{}, - expectedCols: []table.ColumnDefinition{ - table.TextColumn("field"), - }, - expectedError: false, - }, - { - name: "all integer types", - input: struct { - I int `osquery:"i"` - I8 int8 `osquery:"i8"` - I16 int16 `osquery:"i16"` - I32 int32 `osquery:"i32"` - I64 int64 `osquery:"i64"` - U uint `osquery:"u"` - U8 uint8 `osquery:"u8"` - U16 uint16 `osquery:"u16"` - U32 uint32 `osquery:"u32"` - U64 uint64 `osquery:"u64"` - }{}, - expectedCols: []table.ColumnDefinition{ - table.IntegerColumn("i"), - table.IntegerColumn("i8"), - table.IntegerColumn("i16"), - table.IntegerColumn("i32"), - table.BigIntColumn("i64"), - table.IntegerColumn("u"), - table.IntegerColumn("u8"), - table.IntegerColumn("u16"), - table.IntegerColumn("u32"), - table.BigIntColumn("u64"), - }, - expectedError: false, - }, - { - name: "fields without osquery tag use field name", - input: struct { - CustomName string `osquery:"custom_name"` - DefaultName string - }{}, - expectedCols: []table.ColumnDefinition{ - table.TextColumn("custom_name"), - table.TextColumn("DefaultName"), // No tag, uses field name - }, - expectedError: false, - }, - { - name: "mixed exported and unexported fields", - input: struct { - Public string `osquery:"public"` - private string `osquery:"private"` // will be skipped - Another string `osquery:"another"` - }{}, - expectedCols: []table.ColumnDefinition{ - table.TextColumn("public"), - table.TextColumn("another"), - }, - expectedError: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cols, err := GenerateColumnDefinitions(tt.input) - - if (err != nil) != tt.expectedError { - t.Errorf("GenerateColumnDefinitions() error = %v, expectedError %v", err, tt.expectedError) - return - } - - if !tt.expectedError { - if len(cols) != len(tt.expectedCols) { - t.Errorf("GenerateColumnDefinitions() returned %d columns, expected %d", len(cols), len(tt.expectedCols)) - return - } - - for i := range cols { - if cols[i].Name != tt.expectedCols[i].Name { - t.Errorf("Column %d: name = %s, expected %s", i, cols[i].Name, tt.expectedCols[i].Name) - } - if cols[i].Type != tt.expectedCols[i].Type { - t.Errorf("Column %d (%s): type = %s, expected %s", i, cols[i].Name, cols[i].Type, tt.expectedCols[i].Type) - } - } - } - }) - } -} -func TestGenerateColumnDefinitions_basic(t *testing.T) { - type testStruct struct { - Name string `osquery:"name"` - Active bool `osquery:"active"` - Count int `osquery:"count"` - Score float64 `osquery:"score"` - } - expected := []table.ColumnDefinition{ - table.TextColumn("name"), - table.IntegerColumn("active"), - table.IntegerColumn("count"), - table.DoubleColumn("score"), - } - cols, err := GenerateColumnDefinitions(testStruct{}) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(cols) != len(expected) { - t.Fatalf("expected %d columns, got %d", len(expected), len(cols)) - } - for i := range cols { - if cols[i].Name != expected[i].Name || cols[i].Type != expected[i].Type { - t.Errorf("column %d: got (%s, %s), want (%s, %s)", i, cols[i].Name, cols[i].Type, expected[i].Name, expected[i].Type) - } - } -} - -func TestGenerateColumnDefinitions_pointerStruct(t *testing.T) { - type testStruct struct { - Name *string `osquery:"name"` - } - expected := []table.ColumnDefinition{ - table.TextColumn("name"), - } - val := &testStruct{} - cols, err := GenerateColumnDefinitions(val) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(cols) != len(expected) { - t.Fatalf("expected %d columns, got %d", len(expected), len(cols)) - } - for i := range cols { - if cols[i].Name != expected[i].Name || cols[i].Type != expected[i].Type { //nolint:gosec // lengths checked above - t.Errorf("column %d: got (%s, %s), want (%s, %s)", i, cols[i].Name, cols[i].Type, expected[i].Name, expected[i].Type) //nolint:gosec // lengths checked above - } - } -} - -func TestGenerateColumnDefinitions_timeFormats(t *testing.T) { - type testStruct struct { - CreatedAt time.Time `osquery:"created_at" format:"unix"` - UpdatedAt time.Time `osquery:"updated_at"` - } - expected := []table.ColumnDefinition{ - table.BigIntColumn("created_at"), - table.TextColumn("updated_at"), - } - cols, err := GenerateColumnDefinitions(testStruct{}) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(cols) != len(expected) { - t.Fatalf("expected %d columns, got %d", len(expected), len(cols)) - } - for i := range cols { - if cols[i].Name != expected[i].Name || cols[i].Type != expected[i].Type { - t.Errorf("column %d: got (%s, %s), want (%s, %s)", i, cols[i].Name, cols[i].Type, expected[i].Name, expected[i].Type) - } - } -} - -func TestGenerateColumnDefinitions_skippedFields(t *testing.T) { - type testStruct struct { - Visible string `osquery:"visible"` - Hidden string `osquery:"-"` - } - expected := []table.ColumnDefinition{ - table.TextColumn("visible"), - } - cols, err := GenerateColumnDefinitions(testStruct{}) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(cols) != len(expected) { - t.Fatalf("expected %d columns, got %d", len(expected), len(cols)) - } - for i := range cols { - if cols[i].Name != expected[i].Name || cols[i].Type != expected[i].Type { //nolint:gosec // lengths checked above - t.Errorf("column %d: got (%s, %s), want (%s, %s)", i, cols[i].Name, cols[i].Type, expected[i].Name, expected[i].Type) //nolint:gosec // lengths checked above - } - } -} - -func TestGenerateColumnDefinitions_nonStructInput(t *testing.T) { - _, err := GenerateColumnDefinitions("not a struct") - if err == nil { - t.Error("expected error for non-struct input, got nil") - } -} - -func TestGenerateColumnDefinitions_nilInput(t *testing.T) { - _, err := GenerateColumnDefinitions(nil) - if err == nil { - t.Error("expected error for nil input, got nil") - } -} - -func TestGenerateColumnDefinitions_unexportedFields(t *testing.T) { - type testStruct struct { - Public string `osquery:"public"` - private string `osquery:"private"` //nolint:unused // meaningful for test coverage - Exported2 int `osquery:"exported2"` - } - expected := []table.ColumnDefinition{ - table.TextColumn("public"), - table.IntegerColumn("exported2"), - } - cols, err := GenerateColumnDefinitions(testStruct{}) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(cols) != len(expected) { - t.Fatalf("expected %d columns, got %d", len(expected), len(cols)) - } - for i := range cols { - if cols[i].Name != expected[i].Name || cols[i].Type != expected[i].Type { - t.Errorf("column %d: got (%s, %s), want (%s, %s)", i, cols[i].Name, cols[i].Type, expected[i].Name, expected[i].Type) - } - } -} - -func TestGenerateColumnDefinitions_embeddedStruct(t *testing.T) { - type EmbeddedStruct struct { - EmbeddedField string `osquery:"embedded_field"` - } - type testStruct struct { - EmbeddedStruct - TestField string `osquery:"test_field"` - } - expected := []table.ColumnDefinition{ - table.TextColumn("embedded_field"), - table.TextColumn("test_field"), - } - cols, err := GenerateColumnDefinitions(testStruct{}) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if len(cols) != len(expected) { - t.Fatalf("expected %d columns, got %d", len(expected), len(cols)) - } - for i := range cols { - if cols[i].Name != expected[i].Name || cols[i].Type != expected[i].Type { - t.Errorf("column %d: got (%s, %s), want (%s, %s)", i, cols[i].Name, cols[i].Type, expected[i].Name, expected[i].Type) - } - } -}