diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index fb9cd2a5a927..5f3495af5a12 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -110,6 +110,7 @@ CHANGELOG* /metricbeat/module/windows/wmi @elastic/obs-infraobs-integrations /metricbeat/module/zookeeper @elastic/obs-infraobs-integrations /packetbeat/ @elastic/sec-linux-platform +/pkg/autodiscover/ @elastic/elastic-agent-data-plane /script/ @elastic/elastic-agent-data-plane /testing/ @elastic/elastic-agent-data-plane /tools/ @elastic/elastic-agent-data-plane diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 2b2bc132b1c3..045f84bac207 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -10,7 +10,6 @@ updates: - dependabot allow: # Team:Elastic-Agent-Data-Plane - - dependency-name: github.com/elastic/elastic-agent-autodiscover - dependency-name: github.com/elastic/elastic-agent-client/* - dependency-name: github.com/elastic/elastic-agent-libs - dependency-name: github.com/elastic/elastic-agent-shipper-client diff --git a/NOTICE.txt b/NOTICE.txt index 8ea596ca701a..b789e6f3961a 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -10464,217 +10464,6 @@ various licenses: version 2 (LICENSES/Apache-2.0.txt). --------------------------------------------------------------------------------- -Dependency : github.com/elastic/elastic-agent-autodiscover -Version: v0.10.3-0.20260423154939-e990715f9426 -Licence type (autodetected): Apache-2.0 --------------------------------------------------------------------------------- - -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-autodiscover@v0.10.3-0.20260423154939-e990715f9426/LICENSE: - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-client/v7 Version: v7.18.1 @@ -32291,12 +32080,254 @@ library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. +-------------------------------------------------------------------------------- +Dependency : sigs.k8s.io/kind +Version: v0.32.0 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/sigs.k8s.io/kind@v0.32.0/LICENSE: + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + ================================================================================ Indirect dependencies +-------------------------------------------------------------------------------- +Dependency : al.essio.dev/pkg/shellescape +Version: v1.5.1 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/al.essio.dev/pkg/shellescape@v1.5.1/LICENSE: + +The MIT License (MIT) + +Copyright (c) 2016 Alessio Treglia + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : aqwari.net/xml Version: v0.0.0-20210331023308-d9421b293817 @@ -37597,6 +37628,37 @@ Contents of probable licence file $GOMODCACHE/github.com/!azure!a!d/microsoft-au SOFTWARE +-------------------------------------------------------------------------------- +Dependency : github.com/BurntSushi/toml +Version: v1.4.0 +Licence type (autodetected): MIT +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/!burnt!sushi/toml@v1.4.0/COPYING: + +The MIT License (MIT) + +Copyright (c) 2013 TOML authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + -------------------------------------------------------------------------------- Dependency : github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp Version: v1.31.0 @@ -48460,6 +48522,41 @@ Contents of probable licence file $GOMODCACHE/github.com/envoyproxy/protoc-gen-v limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/evanphx/json-patch/v5 +Version: v5.6.0 +Licence type (autodetected): BSD-3-Clause +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/evanphx/json-patch/v5@v5.6.0/LICENSE: + +Copyright (c) 2014, Evan Phoenix +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +* Neither the name of the Evan Phoenix nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + -------------------------------------------------------------------------------- Dependency : github.com/fearful-symmetry/gomsr Version: v0.0.1 @@ -67286,6 +67383,263 @@ Contents of probable licence file $GOMODCACHE/github.com/oxtoacart/bpool@v0.0.0- limitations under the License. +-------------------------------------------------------------------------------- +Dependency : github.com/pelletier/go-toml +Version: v1.9.5 +Licence type (autodetected): Apache-2.0 +-------------------------------------------------------------------------------- + +Contents of probable licence file $GOMODCACHE/github.com/pelletier/go-toml@v1.9.5/LICENSE: + +The bulk of github.com/pelletier/go-toml is distributed under the MIT license +(see below), with the exception of localtime.go and localtime.test.go. +Those two files have been copied over from Google's civil library at revision +ed46f5086358513cf8c25f8e3f022cb838a49d66, and are distributed under the Apache +2.0 license (see below). + + +github.com/pelletier/go-toml: + + +The MIT License (MIT) + +Copyright (c) 2013 - 2021 Thomas Pelletier, Eric Anderton + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +localtime.go, localtime_test.go: + +Originals: + https://raw.githubusercontent.com/googleapis/google-cloud-go/ed46f5086358513cf8c25f8e3f022cb838a49d66/civil/civil.go + https://raw.githubusercontent.com/googleapis/google-cloud-go/ed46f5086358513cf8c25f8e3f022cb838a49d66/civil/civil_test.go +Changes: + * Renamed files from civil* to localtime*. + * Package changed from civil to toml. + * 'Local' prefix added to all structs. +License: + https://raw.githubusercontent.com/googleapis/google-cloud-go/ed46f5086358513cf8c25f8e3f022cb838a49d66/LICENSE + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + -------------------------------------------------------------------------------- Dependency : github.com/pkg/browser Version: v0.0.0-20240102092130-5ac0b6a4141c diff --git a/filebeat/autodiscover/builder/hints/logs.go b/filebeat/autodiscover/builder/hints/logs.go index bdf13535d6a5..47de15194c8e 100644 --- a/filebeat/autodiscover/builder/hints/logs.go +++ b/filebeat/autodiscover/builder/hints/logs.go @@ -23,8 +23,8 @@ import ( "github.com/elastic/go-ucfg" - "github.com/elastic/elastic-agent-autodiscover/bus" - "github.com/elastic/elastic-agent-autodiscover/utils" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/utils" "github.com/elastic/beats/v7/filebeat/fileset" "github.com/elastic/beats/v7/filebeat/harvester" diff --git a/filebeat/autodiscover/builder/hints/logs_test.go b/filebeat/autodiscover/builder/hints/logs_test.go index 760336847021..862a6e10d7b0 100644 --- a/filebeat/autodiscover/builder/hints/logs_test.go +++ b/filebeat/autodiscover/builder/hints/logs_test.go @@ -25,8 +25,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" "github.com/elastic/beats/v7/testing/testutils" - "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp/logptest" @@ -616,7 +616,7 @@ func TestGenerateHints(t *testing.T) { "logs": mapstr.M{ "processors": mapstr.M{ "1": mapstr.M{ - "dissect": mapstr.M{ + "dissect": mapstr.M{ //nolint:gosec // test tokenizer fixture "tokenizer": "%{key1} %{key2}", }, }, @@ -635,7 +635,7 @@ func TestGenerateHints(t *testing.T) { "close_timeout": "true", "processors": []any{ map[string]any{ - "dissect": map[string]any{ + "dissect": map[string]any{ //nolint:gosec // test tokenizer fixture "tokenizer": "%{key1} %{key2}", }, }, @@ -665,7 +665,7 @@ func TestGenerateHints(t *testing.T) { "logs": mapstr.M{ "processors": mapstr.M{ "1": mapstr.M{ - "dissect": mapstr.M{ + "dissect": mapstr.M{ //nolint:gosec // test tokenizer fixture "tokenizer": "%{key1} %{key2}", }, }, @@ -690,7 +690,7 @@ func TestGenerateHints(t *testing.T) { }, }, map[string]any{ - "dissect": map[string]any{ + "dissect": map[string]any{ //nolint:gosec // test tokenizer fixture "tokenizer": "%{key1} %{key2}", }, }, @@ -1194,7 +1194,7 @@ func TestGenerateHintsWithPaths(t *testing.T) { "logs": mapstr.M{ "processors": mapstr.M{ "1": mapstr.M{ - "dissect": mapstr.M{ + "dissect": mapstr.M{ //nolint:gosec // test tokenizer fixture "tokenizer": "%{key1} %{key2}", }, }, @@ -1213,7 +1213,7 @@ func TestGenerateHintsWithPaths(t *testing.T) { "close_timeout": "true", "processors": []any{ map[string]any{ - "dissect": map[string]any{ + "dissect": map[string]any{ //nolint:gosec // test tokenizer fixture "tokenizer": "%{key1} %{key2}", }, }, diff --git a/filebeat/tests/integration/translate_ldap_attribute_test.go b/filebeat/tests/integration/translate_ldap_attribute_test.go index bd2b9de3ce7f..961e8485da19 100644 --- a/filebeat/tests/integration/translate_ldap_attribute_test.go +++ b/filebeat/tests/integration/translate_ldap_attribute_test.go @@ -38,7 +38,7 @@ import ( "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/libbeat/tests/integration" - "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/elastic-agent-libs/logp/logptest" ) diff --git a/go.mod b/go.mod index 31f1def46e52..02e8f75a97aa 100644 --- a/go.mod +++ b/go.mod @@ -170,7 +170,6 @@ require ( github.com/dgraph-io/badger/v4 v4.6.0 github.com/elastic/bayeux v1.0.5 github.com/elastic/ebpfevents v0.9.0 - github.com/elastic/elastic-agent-autodiscover v0.10.3-0.20260423154939-e990715f9426 github.com/elastic/elastic-agent-libs v0.46.1 github.com/elastic/elastic-agent-system-metrics v0.14.4 github.com/elastic/go-elasticsearch/v8 v8.19.0 @@ -255,10 +254,15 @@ require ( go.opentelemetry.io/collector/service v0.155.0 go.opentelemetry.io/otel/sdk/metric v1.44.0 go.uber.org/goleak v1.3.0 + sigs.k8s.io/kind v0.32.0 ) require ( + al.essio.dev/pkg/shellescape v1.5.1 // indirect + github.com/BurntSushi/toml v1.4.0 // indirect github.com/cenkalti/backoff/v6 v6.0.0 // indirect + github.com/evanphx/json-patch/v5 v5.6.0 // indirect + github.com/pelletier/go-toml v1.9.5 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/twmb/franz-go v1.21.3 // indirect go.opentelemetry.io/collector/exporter v1.61.0 // indirect diff --git a/go.sum b/go.sum index 00e57696d464..f39958ea3aa5 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +al.essio.dev/pkg/shellescape v1.5.1 h1:86HrALUujYS/h+GtqoB26SBEdkWfmMI6FubjXlsXyho= +al.essio.dev/pkg/shellescape v1.5.1/go.mod h1:6sIqp7X2P6mThCQ7twERpZTuigpr6KbZWtls1U8I890= aqwari.net/xml v0.0.0-20210331023308-d9421b293817 h1:+3Rh5EaTzNLnzWx3/uy/mAaH/dGI7svJ6e0oOIDcPuE= aqwari.net/xml v0.0.0-20210331023308-d9421b293817/go.mod h1:c7kkWzc7HS/t8Q2DcVY8P2d1dyWNEhEVT5pL0ZHO11c= cel.dev/expr v0.25.1 h1:1KrZg61W6TWSxuNZ37Xy49ps13NUovb66QLprthtwi4= @@ -128,6 +130,8 @@ github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mo github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0 h1:XRzhVemXdgvJqCH0sFfrBUTnUJSBrBf7++ypk+twtRs= github.com/AzureAD/microsoft-authentication-library-for-go v1.6.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= +github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 h1:DHa2U07rk8syqvCge0QIGMCE1WxGj9njT44GH7zNJLQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0/go.mod h1:P4WPRUkOhJC13W//jWpyfJNDAIpvRbAUIYLX/4jtlE0= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.55.0 h1:UnDZ/zFfG1JhH/DqxIZYU/1CUAlTUScoXD/LcM2Ykk8= @@ -358,8 +362,6 @@ github.com/elastic/bayeux v1.0.5 h1:UceFq01ipmT3S8DzFK+uVAkbCdiPR0Bqei8qIGmUeY0= github.com/elastic/bayeux v1.0.5/go.mod h1:CSI4iP7qeo5MMlkznGvYKftp8M7qqP/3nzmVZoXHY68= github.com/elastic/ebpfevents v0.9.0 h1:La6+7+hbnzlM6IOo9edRJfa4qQ6Uo4ALsKRhS2gwBKI= github.com/elastic/ebpfevents v0.9.0/go.mod h1:RDOopYpSomVu55gCO8CPqDMzcU1poDvBeQUEb7pkI2k= -github.com/elastic/elastic-agent-autodiscover v0.10.3-0.20260423154939-e990715f9426 h1:zqt0GOf6/e287nhLlAODfW6Pncydjr4ArEpM589MIXI= -github.com/elastic/elastic-agent-autodiscover v0.10.3-0.20260423154939-e990715f9426/go.mod h1:HyjdiEVP/A7iMN0jeac2EoUohpHKAZIFRDQsvPDg2Dw= github.com/elastic/elastic-agent-client/v7 v7.18.1 h1:WnM53JjaukeysrAuiTyrhDPmFxJG07ZAByc2TrkcKcs= github.com/elastic/elastic-agent-client/v7 v7.18.1/go.mod h1:uDpSGZ+YCKgqgtkwCA0qjwX0gU/wmixDsVbPjY3GkPs= github.com/elastic/elastic-agent-libs v0.46.1 h1:1dVRKOaGWBxqDQPb/NCtoJHI8Nx3QbJOmr4v6y+UdKQ= @@ -442,6 +444,8 @@ github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJP github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= +github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= +github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= @@ -902,6 +906,8 @@ github.com/otiai10/mint v1.6.3 h1:87qsV/aw1F5as1eH1zS/yqHY85ANKVMgkDrf9rcxbQs= github.com/otiai10/mint v1.6.3/go.mod h1:MJm72SBthJjz8qhefc4z1PYEieWmy8Bku7CjcAqyUSM= github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2 h1:CXwSGu/LYmbjEab5aMCs5usQRVBGThelUKBNnoSOuso= github.com/oxtoacart/bpool v0.0.0-20150712133111-4e1c5567d7c2/go.mod h1:L3UMQOThbttwfYRNFOWLLVXMhk5Lkio4GGOtw5UrxS0= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.27 h1:+PhzhWDrjRj89TH2sw43nE3+4+W8lSxIuQadEHZyjUk= @@ -1573,6 +1579,8 @@ pgregory.net/rapid v1.3.0 h1:vBvO0VSqti75J1jjYqpgPNBLKMd1+gxa9fYo7vk/Exc= pgregory.net/rapid v1.3.0/go.mod h1:dPlE4OBBxgXPqkP79flB6sJL1dx5azpI7HQ9MY9Z7uk= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/kind v0.32.0 h1:p9hscbj98u/qyrjVpjId86LI70nQmbSsipV7wCG10Xk= +sigs.k8s.io/kind v0.32.0/go.mod h1:FSqriGaoTPruiXWfRnUXNykF8r2t+fHtK0P0m1AbGF8= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= diff --git a/heartbeat/autodiscover/builder/hints/monitors.go b/heartbeat/autodiscover/builder/hints/monitors.go index 3f31cbf4d123..2b087b1323c0 100644 --- a/heartbeat/autodiscover/builder/hints/monitors.go +++ b/heartbeat/autodiscover/builder/hints/monitors.go @@ -23,8 +23,8 @@ import ( "github.com/elastic/go-ucfg" - "github.com/elastic/elastic-agent-autodiscover/bus" - "github.com/elastic/elastic-agent-autodiscover/utils" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/utils" "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/autodiscover/template" diff --git a/heartbeat/autodiscover/builder/hints/monitors_test.go b/heartbeat/autodiscover/builder/hints/monitors_test.go index 935c11cad762..2372fd843962 100644 --- a/heartbeat/autodiscover/builder/hints/monitors_test.go +++ b/heartbeat/autodiscover/builder/hints/monitors_test.go @@ -23,7 +23,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) @@ -140,9 +140,9 @@ func TestGenerateHints(t *testing.T) { "type": "http", "hosts": []string{"1.2.3.4:9090"}, "schedule": "@every 5s", - "processors": []interface{}{ - map[string]interface{}{ - "add_locale": map[string]interface{}{ + "processors": []any{ + map[string]any{ + "add_locale": map[string]any{ "abbrevation": "MST", }, }, @@ -203,19 +203,19 @@ func TestGenerateHints(t *testing.T) { logger: logp.L(), } cfgs := m.CreateConfig(test.event) - assert.Equal(t, test.len, len(cfgs), test.message) + assert.Len(t, cfgs, test.len, test.message) if len(cfgs) != 0 { config := mapstr.M{} err := cfgs[0].Unpack(&config) - assert.Nil(t, err, test.message) + assert.NoError(t, err, test.message) // Autodiscover can return configs with different sort orders here, which is irrelevant // To make tests pass consistently we sort the host list hostStrs := []string{} - if hostsSlice, ok := config["hosts"].([]interface{}); ok && len(hostsSlice) > 0 { + if hostsSlice, ok := config["hosts"].([]any); ok && len(hostsSlice) > 0 { for _, hi := range hostsSlice { - hostStrs = append(hostStrs, hi.(string)) + hostStrs = append(hostStrs, hi.(string)) //nolint:errcheck // test fixture type } sort.Strings(hostStrs) config["hosts"] = hostStrs diff --git a/libbeat/autodiscover/appender.go b/libbeat/autodiscover/appender.go index a643cdb8b740..91e1d59a2cfe 100644 --- a/libbeat/autodiscover/appender.go +++ b/libbeat/autodiscover/appender.go @@ -21,7 +21,7 @@ import ( "fmt" "strings" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" ) diff --git a/libbeat/autodiscover/appender_test.go b/libbeat/autodiscover/appender_test.go index 6473bb3752d1..ef59cd18c2da 100644 --- a/libbeat/autodiscover/appender_test.go +++ b/libbeat/autodiscover/appender_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" ) @@ -40,7 +40,7 @@ func newFakeAppender(_ *conf.C, _ *logp.Logger) (Appender, error) { func TestAppenderRegistry(t *testing.T) { // Add a new builder reg := NewRegistry() - reg.AddAppender("fake", newFakeAppender) + reg.AddAppender("fake", newFakeAppender) //nolint:errcheck // test registration // Check if that appender is available in registry b := reg.GetAppender("fake") @@ -60,7 +60,7 @@ func TestAppenderRegistry(t *testing.T) { assert.NotNil(t, appender) // Attempt to build using an array of configs - Registry.AddAppender("fake", newFakeAppender) + Registry.AddAppender("fake", newFakeAppender) //nolint:errcheck // test registration cfgs := []*conf.C{cfg} appenders, err := NewAppenders(cfgs) assert.NoError(t, err) diff --git a/libbeat/autodiscover/appenders/config/config.go b/libbeat/autodiscover/appenders/config/config.go index 09e3b89b0bf0..27a8cf2b36b0 100644 --- a/libbeat/autodiscover/appenders/config/config.go +++ b/libbeat/autodiscover/appenders/config/config.go @@ -24,14 +24,14 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover/template" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/libbeat/conditions" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) func init() { - autodiscover.Registry.AddAppender("config", NewConfigAppender) + autodiscover.Registry.AddAppender("config", NewConfigAppender) //nolint:errcheck // provider registration } type config struct { @@ -52,7 +52,7 @@ func NewConfigAppender(cfg *conf.C, logger *logp.Logger) (autodiscover.Appender, config := config{} err := cfg.Unpack(&config) if err != nil { - return nil, fmt.Errorf("unable to unpack config appender due to error: %+v", err) + return nil, fmt.Errorf("unable to unpack config appender due to error: %+w", err) } var cond conditions.Condition @@ -88,7 +88,7 @@ func (c *configAppender) Append(event bus.Event) { if !ok { return } - if c.condition == nil || c.condition.Check(mapstr.M(event)) == true { + if c.condition == nil || c.condition.Check(mapstr.M(event)) { // Merge the template with all the configs for _, cfg := range cfgs { cf := mapstr.M{} diff --git a/libbeat/autodiscover/appenders/config/config_test.go b/libbeat/autodiscover/appenders/config/config_test.go index 375e71c19da3..a0f328a0bd90 100644 --- a/libbeat/autodiscover/appenders/config/config_test.go +++ b/libbeat/autodiscover/appenders/config/config_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp/logptest" "github.com/elastic/elastic-agent-libs/mapstr" @@ -107,7 +107,7 @@ condition.equals: assert.Len(t, cfgs, 1) out := mapstr.M{} - cfgs[0].Unpack(&out) + cfgs[0].Unpack(&out) //nolint:errcheck // test assertion follows assert.Equal(t, out, test.result) }) diff --git a/libbeat/autodiscover/autodiscover.go b/libbeat/autodiscover/autodiscover.go index 6f4e2433f42d..4a36cdb68306 100644 --- a/libbeat/autodiscover/autodiscover.go +++ b/libbeat/autodiscover/autodiscover.go @@ -25,7 +25,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/cfgfile" "github.com/elastic/beats/v7/libbeat/common/reload" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" diff --git a/libbeat/autodiscover/autodiscover_test.go b/libbeat/autodiscover/autodiscover_test.go index b0b7c274c46b..588cb63b8dd9 100644 --- a/libbeat/autodiscover/autodiscover_test.go +++ b/libbeat/autodiscover/autodiscover_test.go @@ -32,7 +32,7 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/cfgfile" "github.com/elastic/beats/v7/libbeat/tests/resources" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" diff --git a/libbeat/autodiscover/builder.go b/libbeat/autodiscover/builder.go index dfb1e06dc5ec..39293b303822 100644 --- a/libbeat/autodiscover/builder.go +++ b/libbeat/autodiscover/builder.go @@ -22,7 +22,7 @@ import ( "fmt" "strings" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" diff --git a/libbeat/autodiscover/builder_test.go b/libbeat/autodiscover/builder_test.go index 7aa6bf2a0456..d89c5a88b384 100644 --- a/libbeat/autodiscover/builder_test.go +++ b/libbeat/autodiscover/builder_test.go @@ -23,7 +23,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/paths" diff --git a/libbeat/autodiscover/eventselect.go b/libbeat/autodiscover/eventselect.go index e7a05123be00..c7a69e2611cc 100644 --- a/libbeat/autodiscover/eventselect.go +++ b/libbeat/autodiscover/eventselect.go @@ -20,7 +20,7 @@ package autodiscover import ( "fmt" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" "github.com/elastic/elastic-agent-libs/config" ) @@ -47,7 +47,7 @@ func (q queryConfigFrom) CreateConfig(e bus.Event) ([]*config.C, error) { fieldName := string(q) config, ok := e[fieldName].([]*config.C) if !ok { - return nil, fmt.Errorf("Event field '%v' does not contain a valid configuration object", fieldName) + return nil, fmt.Errorf("Event field '%v' does not contain a valid configuration object", fieldName) //nolint:staticcheck // error message format } return config, nil } diff --git a/libbeat/autodiscover/provider.go b/libbeat/autodiscover/provider.go index 60d37cc1ca11..295f1e433dd2 100644 --- a/libbeat/autodiscover/provider.go +++ b/libbeat/autodiscover/provider.go @@ -24,7 +24,7 @@ import ( "github.com/gofrs/uuid/v5" "github.com/elastic/beats/v7/libbeat/cfgfile" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" diff --git a/libbeat/autodiscover/providers/docker/config.go b/libbeat/autodiscover/providers/docker/config.go index 86ab487a15a6..fc1498b4345f 100644 --- a/libbeat/autodiscover/providers/docker/config.go +++ b/libbeat/autodiscover/providers/docker/config.go @@ -23,7 +23,7 @@ import ( "time" "github.com/elastic/beats/v7/libbeat/autodiscover/template" - "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/elastic-agent-libs/config" ) diff --git a/libbeat/autodiscover/providers/docker/docker.go b/libbeat/autodiscover/providers/docker/docker.go index 2adca04992f6..5c96cb087dae 100644 --- a/libbeat/autodiscover/providers/docker/docker.go +++ b/libbeat/autodiscover/providers/docker/docker.go @@ -32,9 +32,9 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover/template" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/elastic-agent-autodiscover/bus" - "github.com/elastic/elastic-agent-autodiscover/docker" - "github.com/elastic/elastic-agent-autodiscover/utils" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/utils" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" diff --git a/libbeat/autodiscover/providers/docker/docker_integration_test.go b/libbeat/autodiscover/providers/docker/docker_integration_test.go index 63f815b911ec..fe2a6df33146 100644 --- a/libbeat/autodiscover/providers/docker/docker_integration_test.go +++ b/libbeat/autodiscover/providers/docker/docker_integration_test.go @@ -28,7 +28,7 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover/template" dk "github.com/elastic/beats/v7/libbeat/tests/docker" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp/logptest" diff --git a/libbeat/autodiscover/providers/docker/docker_test.go b/libbeat/autodiscover/providers/docker/docker_test.go index 309a06f0b676..6211fe22f6f3 100644 --- a/libbeat/autodiscover/providers/docker/docker_test.go +++ b/libbeat/autodiscover/providers/docker/docker_test.go @@ -22,8 +22,8 @@ import ( "github.com/stretchr/testify/assert" - "github.com/elastic/elastic-agent-autodiscover/bus" - "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/libbeat/autodiscover/providers/jolokia/discovery.go b/libbeat/autodiscover/providers/jolokia/discovery.go index 03d2a96e326c..cbac73bff145 100644 --- a/libbeat/autodiscover/providers/jolokia/discovery.go +++ b/libbeat/autodiscover/providers/jolokia/discovery.go @@ -29,7 +29,7 @@ import ( s "github.com/elastic/beats/v7/libbeat/common/schema" c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) @@ -222,7 +222,7 @@ func (d *Discovery) sendProbe(config InterfaceConfig) { wg.Go(func() { - conn, err := net.ListenPacket("udp4", net.JoinHostPort(ip.String(), "0")) + conn, err := net.ListenPacket("udp4", net.JoinHostPort(ip.String(), "0")) //nolint:noctx // legacy UDP discovery if err != nil { log.Error(err.Error()) return diff --git a/libbeat/autodiscover/providers/jolokia/jolokia.go b/libbeat/autodiscover/providers/jolokia/jolokia.go index e35ee42b276b..020b845ccfa5 100644 --- a/libbeat/autodiscover/providers/jolokia/jolokia.go +++ b/libbeat/autodiscover/providers/jolokia/jolokia.go @@ -24,7 +24,7 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/autodiscover/template" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" diff --git a/libbeat/autodiscover/providers/kubernetes/config.go b/libbeat/autodiscover/providers/kubernetes/config.go index 884ad6e810fa..d0cfd3e89369 100644 --- a/libbeat/autodiscover/providers/kubernetes/config.go +++ b/libbeat/autodiscover/providers/kubernetes/config.go @@ -24,8 +24,8 @@ import ( "time" "github.com/elastic/beats/v7/libbeat/autodiscover/template" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/metadata" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" ) @@ -105,7 +105,7 @@ func (c *Config) Validate() error { default: if c.Scope == "node" { - logp.L().Warnf("can not set scope to `node` when using resource %s. resetting scope to `cluster`", c.Resource) + logp.L().Warnf("can not set scope to `node` when using resource %s. resetting scope to `cluster`", c.Resource) //nolint:forbidigo // validation fallback logging } c.Scope = "cluster" } @@ -117,7 +117,7 @@ func (c *Config) Validate() error { return nil } -// checkUnsupportedParams checks if unsupported/deprecated/discouraged paramaters are set and logs a warning +// checkUnsupportedParams checks if unsupported/deprecated/discouraged parameters are set and logs a warning func (c Config) checkUnsupportedParams(logger *logp.Logger) { if c.Unique && c.Scope != "cluster" { logger.Warn("can only set `unique` when scope is `cluster`") diff --git a/libbeat/autodiscover/providers/kubernetes/config_test.go b/libbeat/autodiscover/providers/kubernetes/config_test.go index 2f255996b9f9..818a84df5be4 100644 --- a/libbeat/autodiscover/providers/kubernetes/config_test.go +++ b/libbeat/autodiscover/providers/kubernetes/config_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/libbeat/autodiscover/providers/kubernetes/kubernetes.go b/libbeat/autodiscover/providers/kubernetes/kubernetes.go index c73138693b15..d08c57e1deb0 100644 --- a/libbeat/autodiscover/providers/kubernetes/kubernetes.go +++ b/libbeat/autodiscover/providers/kubernetes/kubernetes.go @@ -33,9 +33,9 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/autodiscover/template" - "github.com/elastic/elastic-agent-autodiscover/bus" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/k8skeystore" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/k8skeystore" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" diff --git a/libbeat/autodiscover/providers/kubernetes/node.go b/libbeat/autodiscover/providers/kubernetes/node.go index 1ad69ea218f4..176ec8396fcd 100644 --- a/libbeat/autodiscover/providers/kubernetes/node.go +++ b/libbeat/autodiscover/providers/kubernetes/node.go @@ -23,16 +23,16 @@ import ( "fmt" "time" - "github.com/elastic/elastic-agent-autodiscover/utils" + "github.com/elastic/beats/v7/pkg/autodiscover/utils" "github.com/gofrs/uuid/v5" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" k8s "k8s.io/client-go/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/bus" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/metadata" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" @@ -147,11 +147,13 @@ func (n *node) GenerateHints(event bus.Event) bus.Event { var kubeMeta mapstr.M rawMeta, ok := event["kubernetes"] if ok { - kubeMeta = rawMeta.(mapstr.M) - // The builder base config can configure any of the field values of kubernetes if need be. - e["kubernetes"] = kubeMeta - if rawAnn, ok := kubeMeta["annotations"]; ok { - annotations = rawAnn.(mapstr.M) + kubeMeta, ok = rawMeta.(mapstr.M) + if ok { + // The builder base config can configure any of the field values of kubernetes if need be. + e["kubernetes"] = kubeMeta + if rawAnn, ok := kubeMeta["annotations"]; ok { + annotations = rawAnn.(mapstr.M) //nolint:errcheck // type validated by map lookup + } } } if host, ok := event["host"]; ok { diff --git a/libbeat/autodiscover/providers/kubernetes/node_test.go b/libbeat/autodiscover/providers/kubernetes/node_test.go index 881cffadba96..cc473f20aa1a 100644 --- a/libbeat/autodiscover/providers/kubernetes/node_test.go +++ b/libbeat/autodiscover/providers/kubernetes/node_test.go @@ -29,9 +29,9 @@ import ( k8sfake "k8s.io/client-go/kubernetes/fake" "github.com/elastic/beats/v7/libbeat/autodiscover/template" - "github.com/elastic/elastic-agent-autodiscover/bus" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/metadata" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp/logptest" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/libbeat/autodiscover/providers/kubernetes/pod.go b/libbeat/autodiscover/providers/kubernetes/pod.go index c356d3996050..95a8ed2b7d97 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod.go +++ b/libbeat/autodiscover/providers/kubernetes/pod.go @@ -29,10 +29,10 @@ import ( "github.com/gofrs/uuid/v5" k8s "k8s.io/client-go/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/bus" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" - "github.com/elastic/elastic-agent-autodiscover/utils" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/metadata" + "github.com/elastic/beats/v7/pkg/autodiscover/utils" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" @@ -215,7 +215,7 @@ func (p *pod) OnAdd(obj any) { defer p.crossUpdate.RUnlock() p.logger.Debugf("Watcher Pod add: %+v", obj) - p.emit(obj.(*kubernetes.Pod), "start") + p.emit(obj.(*kubernetes.Pod), "start") //nolint:errcheck // informer object type is validated } // OnUpdate handles events for pods that have been updated. @@ -228,8 +228,8 @@ func (p *pod) OnUpdate(obj any) { func (p *pod) unlockedUpdate(obj any) { p.logger.Debugf("Watcher Pod update: %+v", obj) - p.emit(obj.(*kubernetes.Pod), "stop") - p.emit(obj.(*kubernetes.Pod), "start") + p.emit(obj.(*kubernetes.Pod), "stop") //nolint:errcheck // informer object type is validated + p.emit(obj.(*kubernetes.Pod), "start") //nolint:errcheck // informer object type is validated } // OnDelete stops pod objects that are deleted. @@ -238,7 +238,7 @@ func (p *pod) OnDelete(obj any) { defer p.crossUpdate.RUnlock() p.logger.Debugf("Watcher Pod delete: %+v", obj) - p.emit(obj.(*kubernetes.Pod), "stop") + p.emit(obj.(*kubernetes.Pod), "stop") //nolint:errcheck // informer object type is validated } // GenerateHints creates hints needed for hints builder. diff --git a/libbeat/autodiscover/providers/kubernetes/pod_test.go b/libbeat/autodiscover/providers/kubernetes/pod_test.go index 560a081d997f..c9a2d2f286d0 100644 --- a/libbeat/autodiscover/providers/kubernetes/pod_test.go +++ b/libbeat/autodiscover/providers/kubernetes/pod_test.go @@ -34,9 +34,9 @@ import ( caches "k8s.io/client-go/tools/cache" "github.com/elastic/beats/v7/libbeat/autodiscover/template" - "github.com/elastic/elastic-agent-autodiscover/bus" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/metadata" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp/logptest" "github.com/elastic/elastic-agent-libs/mapstr" @@ -2219,8 +2219,8 @@ func TestPodEventer_Namespace_Node_Watcher(t *testing.T) { t.Fatal(err) } - namespaceWatcher := eventer.(*pod).namespaceWatcher - nodeWatcher := eventer.(*pod).nodeWatcher + namespaceWatcher := eventer.(*pod).namespaceWatcher //nolint:errcheck // test type assertion + nodeWatcher := eventer.(*pod).nodeWatcher //nolint:errcheck // test type assertion if test.expectedNil { assert.Nilf(t, namespaceWatcher, "Namespace "+test.msg) diff --git a/libbeat/autodiscover/providers/kubernetes/service.go b/libbeat/autodiscover/providers/kubernetes/service.go index 0e84f5255222..5faba57fda01 100644 --- a/libbeat/autodiscover/providers/kubernetes/service.go +++ b/libbeat/autodiscover/providers/kubernetes/service.go @@ -23,14 +23,14 @@ import ( "fmt" "time" - "github.com/elastic/elastic-agent-autodiscover/utils" + "github.com/elastic/beats/v7/pkg/autodiscover/utils" "github.com/gofrs/uuid/v5" k8s "k8s.io/client-go/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/bus" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/metadata" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" @@ -115,12 +115,12 @@ func NewServiceEventer( // OnAdd ensures processing of service objects that are newly created func (s *service) OnAdd(obj any) { s.logger.Debugf("Watcher service add: %+v", obj) - s.emit(obj.(*kubernetes.Service), "start") + s.emit(obj.(*kubernetes.Service), "start") //nolint:errcheck // informer object type is validated } // OnUpdate ensures processing of service objects that are updated func (s *service) OnUpdate(obj any) { - svc := obj.(*kubernetes.Service) + svc := obj.(*kubernetes.Service) //nolint:errcheck // informer object type is validated // Once service is in terminated state, mark it for deletion if svc.GetObjectMeta().GetDeletionTimestamp() != nil { time.AfterFunc(s.config.CleanupTimeout, func() { s.emit(svc, "stop") }) @@ -134,7 +134,7 @@ func (s *service) OnUpdate(obj any) { // OnDelete ensures processing of service objects that are deleted func (s *service) OnDelete(obj any) { s.logger.Debugf("Watcher service delete: %+v", obj) - time.AfterFunc(s.config.CleanupTimeout, func() { s.emit(obj.(*kubernetes.Service), "stop") }) + time.AfterFunc(s.config.CleanupTimeout, func() { s.emit(obj.(*kubernetes.Service), "stop") }) //nolint:errcheck // informer object type is validated } // GenerateHints creates hints needed for hints builder @@ -147,7 +147,7 @@ func (s *service) GenerateHints(event bus.Event) bus.Event { annotations := make(mapstr.M, 0) rawMeta, ok := event["kubernetes"] if ok { - kubeMeta = rawMeta.(mapstr.M) + kubeMeta = rawMeta.(mapstr.M) //nolint:errcheck // type validated by map lookup // The builder base config can configure any of the field values of kubernetes if need be. e["kubernetes"] = kubeMeta if rawAnn, ok := kubeMeta["annotations"]; ok { diff --git a/libbeat/autodiscover/providers/kubernetes/service_test.go b/libbeat/autodiscover/providers/kubernetes/service_test.go index 5f852a834671..1840840ab44b 100644 --- a/libbeat/autodiscover/providers/kubernetes/service_test.go +++ b/libbeat/autodiscover/providers/kubernetes/service_test.go @@ -29,9 +29,9 @@ import ( k8sfake "k8s.io/client-go/kubernetes/fake" "github.com/elastic/beats/v7/libbeat/autodiscover/template" - "github.com/elastic/elastic-agent-autodiscover/bus" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/metadata" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp/logptest" "github.com/elastic/elastic-agent-libs/mapstr" @@ -520,7 +520,7 @@ func TestServiceEventer_NamespaceWatcher(t *testing.T) { t.Fatal(err) } - namespaceWatcher := eventer.(*service).namespaceWatcher + namespaceWatcher := eventer.(*service).namespaceWatcher //nolint:errcheck // test type assertion if test.expectedNil { assert.Nilf(t, namespaceWatcher, test.msg) diff --git a/libbeat/autodiscover/template/config.go b/libbeat/autodiscover/template/config.go index d28fd6d6553d..82fdb1b04d5c 100644 --- a/libbeat/autodiscover/template/config.go +++ b/libbeat/autodiscover/template/config.go @@ -26,7 +26,7 @@ import ( "github.com/elastic/go-ucfg/parse" "github.com/elastic/beats/v7/libbeat/conditions" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" ) diff --git a/libbeat/autodiscover/template/config_test.go b/libbeat/autodiscover/template/config_test.go index e90cdfcd0ddc..ab09fea70d79 100644 --- a/libbeat/autodiscover/template/config_test.go +++ b/libbeat/autodiscover/template/config_test.go @@ -24,7 +24,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" @@ -33,7 +33,7 @@ import ( ) func TestConfigsMapping(t *testing.T) { - logp.TestingSetup() + logp.TestingSetup() //nolint:staticcheck // legacy test logger setup config, _ := conf.NewConfigFrom(map[string]any{ "correct": "config", @@ -331,8 +331,8 @@ func createAnExistingKeystore(path string, secret string) keystore.Keystore { panic(err) } - writableKeystore.Store("PASSWORD", []byte(secret)) - writableKeystore.Save() + writableKeystore.Store("PASSWORD", []byte(secret)) //nolint:errcheck // test helper + writableKeystore.Save() //nolint:errcheck // test helper return keyStore } diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata.go b/libbeat/processors/add_docker_metadata/add_docker_metadata.go index 0131a951ca4c..24131c1763af 100644 --- a/libbeat/processors/add_docker_metadata/add_docker_metadata.go +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata.go @@ -34,7 +34,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/processors/actions" - "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata_integration_test.go b/libbeat/processors/add_docker_metadata/add_docker_metadata_integration_test.go index d88c3f9d21db..e2b28543447a 100644 --- a/libbeat/processors/add_docker_metadata/add_docker_metadata_integration_test.go +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata_integration_test.go @@ -30,7 +30,7 @@ import ( "github.com/elastic/beats/v7/libbeat/processors" dockertest "github.com/elastic/beats/v7/libbeat/tests/docker" "github.com/elastic/beats/v7/libbeat/tests/resources" - "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/logp/logptest" @@ -61,7 +61,7 @@ func TestAddDockerMetadata(t *testing.T) { labels := map[string]string{"label": "foo"} id, err := testClient.ContainerStart(image, cmd, labels) require.NoError(t, err) - defer testClient.ContainerRemove(id) + defer testClient.ContainerRemove(id) //nolint:errcheck // test cleanup info, err := testClient.ContainerInspect(id) require.NoError(t, err) @@ -70,8 +70,9 @@ func TestAddDockerMetadata(t *testing.T) { config, err := config.NewConfigFrom(map[string]any{ "match_fields": []string{"cid"}, }) + require.NoError(t, err) watcherConstructor := newWatcherWith(client) - processor, err := buildDockerMetadataProcessor(logp.L(), config, watcherConstructor) + processor, err := buildDockerMetadataProcessor(logp.L(), config, watcherConstructor) //nolint:forbidigo // integration test setup require.NoError(t, err) t.Run("match container by container id", func(t *testing.T) { diff --git a/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go b/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go index 1d854d4579f3..1445ef0726db 100644 --- a/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go +++ b/libbeat/processors/add_docker_metadata/add_docker_metadata_test.go @@ -34,8 +34,8 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/tests/resources" - "github.com/elastic/elastic-agent-autodiscover/bus" - "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/logp/logptest" diff --git a/libbeat/processors/add_docker_metadata/config.go b/libbeat/processors/add_docker_metadata/config.go index ea79bdbc9faf..0e928ce9aa1f 100644 --- a/libbeat/processors/add_docker_metadata/config.go +++ b/libbeat/processors/add_docker_metadata/config.go @@ -22,7 +22,7 @@ package add_docker_metadata import ( "time" - "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" ) // Config for docker processor. diff --git a/libbeat/processors/add_kubernetes_metadata/config.go b/libbeat/processors/add_kubernetes_metadata/config.go index aaf219f8642b..902c9891b094 100644 --- a/libbeat/processors/add_kubernetes_metadata/config.go +++ b/libbeat/processors/add_kubernetes_metadata/config.go @@ -21,8 +21,8 @@ import ( "fmt" "time" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/metadata" "github.com/elastic/elastic-agent-libs/config" ) @@ -90,7 +90,7 @@ func (k *kubeAnnotatorConfig) Validate() error { if logsPathMatcher.ResourceType != "pod" && logsPathMatcher.ResourceType != "container" { return fmt.Errorf("invalid resource_type %s, valid values include `pod`, `container`", logsPathMatcher.ResourceType) } - if logsPathMatcher.ResourceType == "pod" && !(logsPathMatcher.LogsPath == "/var/lib/kubelet/pods/" || logsPathMatcher.LogsPath == "/var/log/pods/") { + if logsPathMatcher.ResourceType == "pod" && (logsPathMatcher.LogsPath != "/var/lib/kubelet/pods/" && logsPathMatcher.LogsPath != "/var/log/pods/") { return fmt.Errorf("invalid logs_path defined for resource_type: %s, valid values include `/var/lib/kubelet/pods/`, `/var/log/pods/`", logsPathMatcher.ResourceType) } if logsPathMatcher.ResourceType == "container" && logsPathMatcher.LogsPath != "/var/log/containers/" { diff --git a/libbeat/processors/add_kubernetes_metadata/indexers.go b/libbeat/processors/add_kubernetes_metadata/indexers.go index a60ee2e21ea7..a44ef12474d2 100644 --- a/libbeat/processors/add_kubernetes_metadata/indexers.go +++ b/libbeat/processors/add_kubernetes_metadata/indexers.go @@ -22,8 +22,8 @@ import ( "net" "strconv" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/metadata" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/libbeat/processors/add_kubernetes_metadata/indexers_test.go b/libbeat/processors/add_kubernetes_metadata/indexers_test.go index 43d31a9d3611..d6dd420c1e1b 100644 --- a/libbeat/processors/add_kubernetes_metadata/indexers_test.go +++ b/libbeat/processors/add_kubernetes_metadata/indexers_test.go @@ -23,8 +23,8 @@ import ( "strconv" "testing" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/metadata" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/libbeat/processors/add_kubernetes_metadata/kubernetes.go b/libbeat/processors/add_kubernetes_metadata/kubernetes.go index e66e5c72aaaa..dbe79a72bce4 100644 --- a/libbeat/processors/add_kubernetes_metadata/kubernetes.go +++ b/libbeat/processors/add_kubernetes_metadata/kubernetes.go @@ -30,8 +30,6 @@ import ( k8sclient "k8s.io/client-go/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -39,6 +37,8 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/processors" "github.com/elastic/beats/v7/libbeat/processors/shared" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/metadata" ) const ( diff --git a/libbeat/tests/compose/wrapper.go b/libbeat/tests/compose/wrapper.go index f9b013f5f27b..166d0e57d800 100644 --- a/libbeat/tests/compose/wrapper.go +++ b/libbeat/tests/compose/wrapper.go @@ -37,7 +37,7 @@ import ( "github.com/moby/moby/api/types/container" dockerclient "github.com/moby/moby/client" - "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/elastic-agent-libs/logp" ) diff --git a/libbeat/tests/docker/docker.go b/libbeat/tests/docker/docker.go index d2c88af10a82..c4d6f70a190f 100644 --- a/libbeat/tests/docker/docker.go +++ b/libbeat/tests/docker/docker.go @@ -23,7 +23,7 @@ import ( "io" "io/ioutil" - "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/elastic-agent-libs/logp" "github.com/moby/moby/api/types/container" diff --git a/metricbeat/autodiscover/appender/kubernetes/token/token.go b/metricbeat/autodiscover/appender/kubernetes/token/token.go index 09f06203fd6f..19db90e8fbca 100644 --- a/metricbeat/autodiscover/appender/kubernetes/token/token.go +++ b/metricbeat/autodiscover/appender/kubernetes/token/token.go @@ -24,7 +24,7 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" "github.com/elastic/beats/v7/libbeat/conditions" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -52,7 +52,7 @@ func NewTokenAppender(cfg *conf.C, logger *logp.Logger) (autodiscover.Appender, err := cfg.Unpack(&conf) if err != nil { - return nil, fmt.Errorf("unable to unpack config due to error: %v", err) + return nil, fmt.Errorf("unable to unpack config due to error: %w", err) } var cond conditions.Condition @@ -60,7 +60,7 @@ func NewTokenAppender(cfg *conf.C, logger *logp.Logger) (autodiscover.Appender, // Attempt to create a condition. If fails then report error cond, err = conditions.NewCondition(conf.ConditionConfig, logger) if err != nil { - return nil, fmt.Errorf("unable to create condition due to error: %v", err) + return nil, fmt.Errorf("unable to create condition due to error: %w", err) } } appender := tokenAppender{ @@ -87,7 +87,7 @@ func (t *tokenAppender) Append(event bus.Event) { } // Check if the condition is met. Attempt to append only if that is the case. - if t.Condition == nil || t.Condition.Check(mapstr.M(event)) == true { + if t.Condition == nil || t.Condition.Check(mapstr.M(event)) { tok := t.getAuthHeaderFromToken() // If token is empty then just return if tok == "" { diff --git a/metricbeat/autodiscover/appender/kubernetes/token/token_test.go b/metricbeat/autodiscover/appender/kubernetes/token/token_test.go index f4d27a554bdd..7b14f3208efe 100644 --- a/metricbeat/autodiscover/appender/kubernetes/token/token_test.go +++ b/metricbeat/autodiscover/appender/kubernetes/token/token_test.go @@ -23,7 +23,7 @@ import ( "github.com/stretchr/testify/assert" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp/logptest" "github.com/elastic/elastic-agent-libs/mapstr" @@ -99,7 +99,7 @@ token_path: "test" assert.Len(t, cfgs, 1) out := mapstr.M{} - cfgs[0].Unpack(&out) + cfgs[0].Unpack(&out) //nolint:errcheck // test assertion follows assert.Equal(t, out, test.result) deleteFile("test") @@ -107,7 +107,7 @@ token_path: "test" } func writeFile(name, message string) { - os.WriteFile(name, []byte(message), os.ModePerm) + os.WriteFile(name, []byte(message), os.ModePerm) //nolint:errcheck // test helper } func deleteFile(name string) { diff --git a/metricbeat/autodiscover/builder/hints/metrics.go b/metricbeat/autodiscover/builder/hints/metrics.go index 9d56b334b836..60ae6f7a8cbf 100644 --- a/metricbeat/autodiscover/builder/hints/metrics.go +++ b/metricbeat/autodiscover/builder/hints/metrics.go @@ -29,8 +29,8 @@ import ( "github.com/elastic/elastic-agent-libs/mapstr" "github.com/elastic/elastic-agent-libs/paths" - "github.com/elastic/elastic-agent-autodiscover/bus" - "github.com/elastic/elastic-agent-autodiscover/utils" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/utils" "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/autodiscover/template" diff --git a/metricbeat/autodiscover/builder/hints/metrics_test.go b/metricbeat/autodiscover/builder/hints/metrics_test.go index f9d771ccf893..7011afb357be 100644 --- a/metricbeat/autodiscover/builder/hints/metrics_test.go +++ b/metricbeat/autodiscover/builder/hints/metrics_test.go @@ -26,8 +26,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" "github.com/elastic/beats/v7/testing/testutils" - "github.com/elastic/elastic-agent-autodiscover/bus" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp/logptest" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/metricbeat/module/docker/container/data.go b/metricbeat/module/docker/container/data.go index effaa75c24cc..42f69be5b09d 100644 --- a/metricbeat/module/docker/container/data.go +++ b/metricbeat/module/docker/container/data.go @@ -25,7 +25,7 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover/providers/kubernetes" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" - "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/metricbeat/module/docker/docker.go b/metricbeat/module/docker/docker.go index 45e470c8f798..88d07a6bb457 100644 --- a/metricbeat/module/docker/docker.go +++ b/metricbeat/module/docker/docker.go @@ -32,7 +32,7 @@ import ( "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" - "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/elastic-agent-libs/logp" ) diff --git a/metricbeat/module/docker/event/event_integration_test.go b/metricbeat/module/docker/event/event_integration_test.go index d418a7403620..80a3349822f7 100644 --- a/metricbeat/module/docker/event/event_integration_test.go +++ b/metricbeat/module/docker/event/event_integration_test.go @@ -32,7 +32,7 @@ import ( "github.com/elastic/beats/v7/auditbeat/core" "github.com/elastic/beats/v7/metricbeat/mb" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" - "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/elastic-agent-libs/logp/logptest" ) diff --git a/metricbeat/module/docker/helper.go b/metricbeat/module/docker/helper.go index 5003384e1212..04c4085df7c5 100644 --- a/metricbeat/module/docker/helper.go +++ b/metricbeat/module/docker/helper.go @@ -20,7 +20,7 @@ package docker import ( "github.com/moby/moby/api/types/container" - helpers "github.com/elastic/elastic-agent-autodiscover/docker" + helpers "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/metricbeat/module/docker/helper_test.go b/metricbeat/module/docker/helper_test.go index 0c9238f7506f..e58708fdbeca 100644 --- a/metricbeat/module/docker/helper_test.go +++ b/metricbeat/module/docker/helper_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/assert" - helpers "github.com/elastic/elastic-agent-autodiscover/docker" + helpers "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/metricbeat/module/docker/image/data.go b/metricbeat/module/docker/image/data.go index cc2f9d9bdb91..c030efdfca61 100644 --- a/metricbeat/module/docker/image/data.go +++ b/metricbeat/module/docker/image/data.go @@ -23,7 +23,7 @@ import ( "github.com/moby/moby/api/types/image" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/elastic-agent-autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/metricbeat/module/kubernetes/event/config.go b/metricbeat/module/kubernetes/event/config.go index cc62c3395c7e..84728ff17b22 100644 --- a/metricbeat/module/kubernetes/event/config.go +++ b/metricbeat/module/kubernetes/event/config.go @@ -20,7 +20,7 @@ package event import ( "time" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" ) type kubeEventsConfig struct { diff --git a/metricbeat/module/kubernetes/event/event.go b/metricbeat/module/kubernetes/event/event.go index 97f2deb23f56..f3405066486c 100644 --- a/metricbeat/module/kubernetes/event/event.go +++ b/metricbeat/module/kubernetes/event/event.go @@ -25,7 +25,7 @@ import ( "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/module/kubernetes/util" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -114,17 +114,17 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { func (m *MetricSet) Run(reporter mb.PushReporterV2) { now := time.Now() handler := kubernetes.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { + AddFunc: func(obj any) { m.reportEvent(obj, reporter) }, - UpdateFunc: func(obj interface{}) { + UpdateFunc: func(obj any) { m.reportEvent(obj, reporter) }, // ignore events that are deleted DeleteFunc: nil, } m.watcher.AddEventHandler(kubernetes.FilteringResourceEventHandler{ - FilterFunc: func(obj interface{}) bool { + FilterFunc: func(obj any) bool { eve, ok := obj.(*kubernetes.Event) if !ok { m.Logger().Debugf("Error while casting event. Got type: %T", obj) @@ -154,8 +154,8 @@ func (m *MetricSet) Run(reporter mb.PushReporterV2) { m.watcher.Stop() } -func (m *MetricSet) reportEvent(obj interface{}, reporter mb.PushReporterV2) { - mapStrEvent := generateMapStrFromEvent(obj.(*kubernetes.Event), m.dedotConfig, m.Logger()) +func (m *MetricSet) reportEvent(obj any, reporter mb.PushReporterV2) { + mapStrEvent := generateMapStrFromEvent(obj.(*kubernetes.Event), m.dedotConfig, m.Logger()) //nolint:errcheck // informer object type is validated event := mb.TransformMapStrToEvent("kubernetes", mapStrEvent, nil) if m.clusterMeta != nil { event.RootFields.DeepUpdate(m.clusterMeta) @@ -168,17 +168,17 @@ func generateMapStrFromEvent(eve *kubernetes.Event, dedotConfig dedotConfig, log "timestamp": mapstr.M{ "created": kubernetes.Time(&eve.ObjectMeta.CreationTimestamp).UTC(), }, - "name": eve.ObjectMeta.GetName(), - "namespace": eve.ObjectMeta.GetNamespace(), - "self_link": eve.ObjectMeta.GetSelfLink(), - "generate_name": eve.ObjectMeta.GetGenerateName(), - "uid": eve.ObjectMeta.GetUID(), - "resource_version": eve.ObjectMeta.GetResourceVersion(), + "name": eve.GetName(), + "namespace": eve.GetNamespace(), + "self_link": eve.GetSelfLink(), + "generate_name": eve.GetGenerateName(), + "uid": eve.GetUID(), + "resource_version": eve.GetResourceVersion(), } - if len(eve.ObjectMeta.Labels) != 0 { - labels := make(mapstr.M, len(eve.ObjectMeta.Labels)) - for k, v := range eve.ObjectMeta.Labels { + if len(eve.Labels) != 0 { + labels := make(mapstr.M, len(eve.Labels)) + for k, v := range eve.Labels { if dedotConfig.LabelsDedot { label := common.DeDot(k) kubernetes2.ShouldPut(labels, label, v, logger) @@ -194,9 +194,9 @@ func generateMapStrFromEvent(eve *kubernetes.Event, dedotConfig dedotConfig, log eventMeta["labels"] = labels } - if len(eve.ObjectMeta.Annotations) != 0 { - annotations := make(mapstr.M, len(eve.ObjectMeta.Annotations)) - for k, v := range eve.ObjectMeta.Annotations { + if len(eve.Annotations) != 0 { + annotations := make(mapstr.M, len(eve.Annotations)) + for k, v := range eve.Annotations { if dedotConfig.AnnotationsDedot { annotation := common.DeDot(k) kubernetes2.ShouldPut(annotations, annotation, v, logger) diff --git a/metricbeat/module/kubernetes/util/kubernetes.go b/metricbeat/module/kubernetes/util/kubernetes.go index 720fabb8e527..1eb344bed36c 100644 --- a/metricbeat/module/kubernetes/util/kubernetes.go +++ b/metricbeat/module/kubernetes/util/kubernetes.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "maps" + "slices" "strings" "sync" "time" @@ -35,8 +36,8 @@ import ( "k8s.io/apimachinery/pkg/api/meta" k8sresource "k8s.io/apimachinery/pkg/api/resource" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/metadata" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" @@ -326,10 +327,8 @@ func createWatcher( ) (bool, error) { // We need to check the node scope to decide on whether a watcher should be updated or not. - nodeScope := false - if options.Node != "" { - nodeScope = true - } + nodeScope := options.Node != "" + // The nodescope for extra watchers node, namespace, replicaset and job should be always false. if extraWatcher { nodeScope = false @@ -466,7 +465,7 @@ func addEventHandlersToWatcher( nodeStore.SetNodeMetrics(metrics) } - clearMetadataCacheFunc := func(obj interface{}) { + clearMetadataCacheFunc := func(obj any) { enrichers := make(map[string]*enricher, len(metaWatcher.enrichers)) resourceWatchers.lock.Lock() @@ -475,7 +474,7 @@ func addEventHandlersToWatcher( for _, enricher := range enrichers { enricher.Lock() - ids := enricher.deleteFunc(obj.(kubernetes.Resource)) + ids := enricher.deleteFunc(obj.(kubernetes.Resource)) //nolint:errcheck // handler type is validated by informer // update this watcher events by removing all the metadata[id] for _, id := range ids { delete(enricher.metadataCache, id) @@ -485,7 +484,7 @@ func addEventHandlersToWatcher( } metaWatcher.watcher.AddEventHandler(kubernetes.ResourceEventHandlerFuncs{ - AddFunc: func(obj interface{}) { + AddFunc: func(obj any) { switch res := obj.(type) { case *kubernetes.Pod: containerMetricsUpdateFunc(res) @@ -493,7 +492,7 @@ func addEventHandlersToWatcher( nodeMetricsUpdateFunc(res) } }, - UpdateFunc: func(obj interface{}) { + UpdateFunc: func(obj any) { clearMetadataCacheFunc(obj) switch res := obj.(type) { case *kubernetes.Pod: @@ -502,7 +501,7 @@ func addEventHandlersToWatcher( nodeMetricsUpdateFunc(res) } }, - DeleteFunc: func(obj interface{}) { + DeleteFunc: func(obj any) { clearMetadataCacheFunc(obj) switch res := obj.(type) { case *kubernetes.Pod: @@ -523,13 +522,7 @@ func addToMetricsetsUsing(resourceName string, metricsetUsing string, resourceWa data, ok := resourceWatchers.metaWatchersMap[resourceName] if ok { - contains := false - for _, which := range data.metricsetsUsing { - if which == metricsetUsing { - contains = true - break - } - } + contains := slices.Contains(data.metricsetsUsing, metricsetUsing) // add this resource to the list of resources using it if !contains { data.metricsetsUsing = append(data.metricsetsUsing, metricsetUsing) @@ -662,7 +655,8 @@ func createMetadataGenSpecific(client k8sclient.Interface, commonConfig *conf.C, } var metaGen metadata.MetaGen - if resourceName == PodResource { + switch resourceName { + case PodResource: var nodeWatcher kubernetes.Watcher if nodeMetaWatcher := resourceWatchers.metaWatchersMap[NodeResource]; nodeMetaWatcher != nil { nodeWatcher = (*nodeMetaWatcher).watcher @@ -684,7 +678,7 @@ func createMetadataGenSpecific(client k8sclient.Interface, commonConfig *conf.C, metaGen = metadata.GetPodMetaGen(commonConfig, mainWatcher, nodeWatcher, namespaceWatcher, replicaSetWatcher, jobWatcher, addResourceMetadata) return metaGen, nil - } else if resourceName == ServiceResource { + case ServiceResource: namespaceMetaWatcher := resourceWatchers.metaWatchersMap[NamespaceResource] if namespaceMetaWatcher == nil { return nil, fmt.Errorf("could not create the metadata generator, as the watcher for namespace does not exist") @@ -914,7 +908,7 @@ func NewContainerMetadataEnricher( } for _, container := range append(pod.Spec.Containers, pod.Spec.InitContainers...) { - id := join(pod.ObjectMeta.GetNamespace(), pod.GetObjectMeta().GetName(), container.Name) + id := join(pod.GetNamespace(), pod.GetObjectMeta().GetName(), container.Name) ids = append(ids, id) } @@ -1164,10 +1158,8 @@ func (e *enricher) getMetadata(event mapstr.M) mapstr.M { func (e *enricher) updateMetadataCacheFromWatcher(key string) { storeKey := getWatcherStoreKeyFromMetadataKey(key) if res, exists, _ := e.watcher.watcher.Store().GetByKey(storeKey); exists { - eventMetaMap := e.updateFunc(res.(kubernetes.Resource)) - for k, v := range eventMetaMap { - e.metadataCache[k] = v - } + eventMetaMap := e.updateFunc(res.(kubernetes.Resource)) //nolint:errcheck // store object type is validated by watcher + maps.Copy(e.metadataCache, eventMetaMap) } } @@ -1257,7 +1249,7 @@ func AddClusterECSMeta(base mb.BaseMetricSet) mapstr.M { // transformReplicaSetMetadata ensures that the PartialObjectMetadata resources we get from a metadata watcher // can be correctly interpreted by the update function returned by getEventMetadataFunc. // This really just involves adding missing type information. -func transformReplicaSetMetadata(obj interface{}) (interface{}, error) { +func transformReplicaSetMetadata(obj any) (any, error) { old, ok := obj.(*metav1.PartialObjectMetadata) if !ok { return nil, fmt.Errorf("obj of type %T neither a ReplicaSet nor a PartialObjectMetadata", obj) diff --git a/metricbeat/module/kubernetes/util/kubernetes_test.go b/metricbeat/module/kubernetes/util/kubernetes_test.go index bcf556e05b87..f5251e1c8e72 100644 --- a/metricbeat/module/kubernetes/util/kubernetes_test.go +++ b/metricbeat/module/kubernetes/util/kubernetes_test.go @@ -38,7 +38,7 @@ import ( "k8s.io/client-go/tools/cache" kubernetes2 "github.com/elastic/beats/v7/libbeat/autodiscover/providers/kubernetes" - "github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes/metadata" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/logp/logptest" @@ -48,7 +48,7 @@ import ( k8sfake "k8s.io/client-go/kubernetes/fake" k8smetafake "k8s.io/client-go/metadata/fake" - "github.com/elastic/elastic-agent-autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" ) func TestWatchOptions(t *testing.T) { @@ -102,7 +102,7 @@ func TestCreateWatcher(t *testing.T) { require.NoError(t, err) resourceWatchers.lock.Lock() - require.Equal(t, 1, len(resourceWatchers.metaWatchersMap)) + require.Len(t, resourceWatchers.metaWatchersMap, 1) require.NotNil(t, resourceWatchers.metaWatchersMap[NamespaceResource]) require.NotNil(t, resourceWatchers.metaWatchersMap[NamespaceResource].watcher) resourceWatchers.lock.Unlock() @@ -120,7 +120,7 @@ func TestCreateWatcher(t *testing.T) { require.NoError(t, err) resourceWatchers.lock.Lock() - require.Equal(t, 1, len(resourceWatchers.metaWatchersMap)) + require.Len(t, resourceWatchers.metaWatchersMap, 1) require.NotNil(t, resourceWatchers.metaWatchersMap[NamespaceResource]) require.NotNil(t, resourceWatchers.metaWatchersMap[NamespaceResource].watcher) resourceWatchers.lock.Unlock() @@ -138,7 +138,7 @@ func TestCreateWatcher(t *testing.T) { require.NoError(t, err) resourceWatchers.lock.Lock() - require.Equal(t, 2, len(resourceWatchers.metaWatchersMap)) + require.Len(t, resourceWatchers.metaWatchersMap, 2) require.NotNil(t, resourceWatchers.metaWatchersMap[DeploymentResource]) require.NotNil(t, resourceWatchers.metaWatchersMap[NamespaceResource]) resourceWatchers.lock.Unlock() @@ -294,15 +294,15 @@ func TestWatcherContainerMetrics(t *testing.T) { containerStore := metricsRepo.GetNodeStore(pod.Spec.NodeName).GetPodStore(podId).GetContainerStore(containerName) metrics := containerStore.GetContainerMetrics() require.NotNil(t, metrics) - assert.Equal(t, 0.1, metrics.CoresLimit.Value) - assert.Equal(t, 100*1024*1024.0, metrics.MemoryLimit.Value) + assert.Equal(t, 0.1, metrics.CoresLimit.Value) //nolint:testifylint // exact float comparison + assert.Equal(t, 100*1024*1024.0, metrics.MemoryLimit.Value) //nolint:testifylint // exact float comparison // modify the limit and verify the new value is present pod.Spec.Containers[0].Resources.Limits[v1.ResourceCPU] = resource.MustParse("200m") watcher.handler.OnUpdate(pod) metrics = containerStore.GetContainerMetrics() require.NotNil(t, metrics) - assert.Equal(t, 0.2, metrics.CoresLimit.Value) + assert.Equal(t, 0.2, metrics.CoresLimit.Value) //nolint:testifylint // exact float comparison // delete the pod and verify no metrics are present watcher.handler.OnDelete(pod) @@ -355,15 +355,15 @@ func TestWatcherNodeMetrics(t *testing.T) { nodeStore := metricsRepo.GetNodeStore(node.Name) metrics := nodeStore.GetNodeMetrics() require.NotNil(t, metrics) - assert.Equal(t, 0.1, metrics.CoresAllocatable.Value) - assert.Equal(t, 100*1024*1024.0, metrics.MemoryAllocatable.Value) + assert.Equal(t, 0.1, metrics.CoresAllocatable.Value) //nolint:testifylint // exact float comparison + assert.Equal(t, 100*1024*1024.0, metrics.MemoryAllocatable.Value) //nolint:testifylint // exact float comparison // modify the limit and verify the new value is present node.Status.Allocatable[v1.ResourceCPU] = resource.MustParse("200m") watcher.handler.OnUpdate(node) metrics = nodeStore.GetNodeMetrics() require.NotNil(t, metrics) - assert.Equal(t, 0.2, metrics.CoresAllocatable.Value) + assert.Equal(t, 0.2, metrics.CoresAllocatable.Value) //nolint:testifylint // exact float comparison // delete the node and verify no metrics are present watcher.handler.OnDelete(node) @@ -404,7 +404,7 @@ func TestCreateAllWatchers(t *testing.T) { metricsRepo) require.Error(t, err) resourceWatchers.lock.Lock() - require.Equal(t, 0, len(resourceWatchers.metaWatchersMap)) + require.Empty(t, resourceWatchers.metaWatchersMap) resourceWatchers.lock.Unlock() // Start watcher for a resource that requires other resources, should start all the watchers @@ -425,7 +425,7 @@ func TestCreateAllWatchers(t *testing.T) { // Check that all the required watchers are in the map resourceWatchers.lock.Lock() // we add 1 to the expected result to represent the resource itself - require.Equal(t, len(extras)+1, len(resourceWatchers.metaWatchersMap)) + require.Len(t, resourceWatchers.metaWatchersMap, len(extras)+1) for _, extra := range extras { require.NotNil(t, resourceWatchers.metaWatchersMap[extra]) } @@ -487,7 +487,7 @@ func TestCreateMetaGenSpecific(t *testing.T) { log := logptest.NewTestingLogger(t, "test") - namespaceConfig, err := conf.NewConfigFrom(map[string]interface{}{ + namespaceConfig, err := conf.NewConfigFrom(map[string]any{ "enabled": true, }) require.NoError(t, err) @@ -573,7 +573,7 @@ func TestBuildMetadataEnricher_Start_Stop(t *testing.T) { resourceWatchers.lock.Unlock() funcs := mockFuncs{} - namespaceConfig, err := conf.NewConfigFrom(map[string]interface{}{ + namespaceConfig, err := conf.NewConfigFrom(map[string]any{ "enabled": true, }) require.NoError(t, err) @@ -1001,7 +1001,7 @@ func (f *mockFuncs) update(obj kubernetes.Resource) map[string]mapstr.M { }, }, } - logger := logp.NewLogger("kubernetes") + logger := logp.NewLogger("kubernetes") //nolint:forbidigo // test helper for k, v := range accessor.GetLabels() { kubernetes2.ShouldPut(meta, fmt.Sprintf("kubernetes.%v", k), v, logger) } @@ -1018,7 +1018,7 @@ func (f *mockFuncs) delete(obj kubernetes.Resource) []string { func (f *mockFuncs) index(m mapstr.M) string { f.indexed = m - return m["name"].(string) + return m["name"].(string) //nolint:errcheck // test mock } type mockWatcher struct { @@ -1028,7 +1028,7 @@ type mockWatcher struct { func newMockWatcher() *mockWatcher { return &mockWatcher{ - store: cache.NewStore(func(obj interface{}) (string, error) { + store: cache.NewStore(func(obj any) (string, error) { objName, err := cache.ObjectToName(obj) if err != nil { return "", err diff --git a/pkg/autodiscover/bus/bus.go b/pkg/autodiscover/bus/bus.go new file mode 100644 index 000000000000..621e23ba64e3 --- /dev/null +++ b/pkg/autodiscover/bus/bus.go @@ -0,0 +1,162 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package bus + +import ( + "sync" + + "github.com/elastic/elastic-agent-libs/keystore" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +// Event sent to the bus +type Event mapstr.M + +// Bus provides a common channel to emit and listen for Events +type Bus interface { + // Publish an event to the bus + Publish(Event) + + // Subscribe to all events, filter them to the ones containing *all* the keys in filter + Subscribe(filter ...string) Listener +} + +// Provider for keystore +type KeystoreProvider interface { + GetKeystore(event Event) keystore.Keystore +} + +// Listener retrieves Events from a Bus subscription until Stop is called +type Listener interface { + // Events channel + Events() <-chan Event + + // Stop listening and removes itself from the bus + Stop() +} + +type bus struct { + sync.RWMutex + log *logp.Logger + listeners []*listener + store chan Event +} + +type listener struct { + filter []string + channel chan Event + bus *bus +} + +// New initializes a new bus with the given name and returns it +func New(log *logp.Logger, name string) Bus { + return &bus{ + log: createLogger(log, name), + listeners: make([]*listener, 0), + } +} + +// NewBusWithStore allows to create a buffered bus when producers send data without +// listeners being subscribed to them. size determines the size of the buffer. +func NewBusWithStore(log *logp.Logger, name string, size int) Bus { + return &bus{ + log: createLogger(log, name), + listeners: make([]*listener, 0), + store: make(chan Event, size), + } +} + +func createLogger(log *logp.Logger, name string) *logp.Logger { + selector := "bus-" + name + return log.Named(selector).With("elastic-agent-autodiscover.bus", name) +} + +func (b *bus) Publish(e Event) { + b.RLock() + defer b.RUnlock() + + b.log.Debugf("%+v", e) + if len(b.listeners) == 0 && b.store != nil { + b.store <- e + return + } + + if len(b.store) != 0 { + doBreak := false + for !doBreak { + select { + case eve := <-b.store: + for _, listener := range b.listeners { + if listener.interested(eve) { + listener.channel <- eve + } + } + default: + doBreak = true + } + } + } + + for _, listener := range b.listeners { + if listener.interested(e) { + listener.channel <- e + } + } +} + +func (b *bus) Subscribe(filter ...string) Listener { + listener := &listener{ + filter: filter, + bus: b, + channel: make(chan Event, 100), + } + + b.Lock() + defer b.Unlock() + b.listeners = append(b.listeners, listener) + + return listener +} + +func (l *listener) Events() <-chan Event { + return l.channel +} + +func (l *listener) Stop() { + l.bus.Lock() + defer l.bus.Unlock() + + for i, listener := range l.bus.listeners { + if l == listener { + l.bus.listeners = append(l.bus.listeners[:i], l.bus.listeners[i+1:]...) + } + } + + close(l.channel) +} + +// Return true if listener is interested on the given event +func (l *listener) interested(e Event) bool { + for _, key := range l.filter { + if _, ok := e[key]; !ok { + return false + } + } + return true +} diff --git a/pkg/autodiscover/bus/bus_test.go b/pkg/autodiscover/bus/bus_test.go new file mode 100644 index 000000000000..4ef5e642fd1f --- /dev/null +++ b/pkg/autodiscover/bus/bus_test.go @@ -0,0 +1,122 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//nolint:forbidigo // tests verify bus behavior with the global logger +package bus + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/elastic-agent-libs/logp" +) + +func TestEmit(t *testing.T) { + bus := New(logp.L(), "name") + listener := bus.Subscribe() + + bus.Publish(Event{ + "foo": "bar", + }) + + event := <-listener.Events() + assert.Equal(t, "bar", event["foo"]) +} + +func TestEmitOrder(t *testing.T) { + bus := New(logp.L(), "name") + listener := bus.Subscribe() + bus.Publish(Event{"first": "event"}) + bus.Publish(Event{"second": "event"}) + + event1 := <-listener.Events() + event2 := <-listener.Events() + assert.Equal(t, Event{"first": "event"}, event1) + assert.Equal(t, Event{"second": "event"}, event2) +} + +func TestSubscribeFilter(t *testing.T) { + bus := New(logp.L(), "name") + listener := bus.Subscribe("second") + + bus.Publish(Event{"first": "event"}) + bus.Publish(Event{"second": "event"}) + + event := <-listener.Events() + assert.Equal(t, Event{"second": "event"}, event) +} + +func TestMultipleListeners(t *testing.T) { + bus := New(logp.L(), "name") + listener1 := bus.Subscribe("a") + listener2 := bus.Subscribe("a", "b") + + bus.Publish(Event{"a": "event"}) + bus.Publish(Event{"a": 1, "b": 2}) + + event1 := <-listener1.Events() + event2 := <-listener1.Events() + assert.Equal(t, Event{"a": "event"}, event1) + assert.Equal(t, Event{"a": 1, "b": 2}, event2) + + event1 = <-listener2.Events() + assert.Equal(t, Event{"a": 1, "b": 2}, event1) + + select { + case event2 = <-listener2.Events(): + t.Error("Got unexpected event:", event2) + default: + } +} + +func TestListenerClose(t *testing.T) { + bus := New(logp.L(), "name") + listener := bus.Subscribe() + + bus.Publish(Event{"first": "event"}) + bus.Publish(Event{"second": "event"}) + + listener.Stop() + + bus.Publish(Event{"third": "event"}) + + event := <-listener.Events() + assert.Equal(t, Event{"first": "event"}, event) + event = <-listener.Events() + assert.Equal(t, Event{"second": "event"}, event) + + // Channel was closed, we get an empty event + event = <-listener.Events() + assert.Equal(t, event, Event(nil)) +} + +func TestUnsubscribedBus(t *testing.T) { + bus := NewBusWithStore(logp.L(), "name", 2) + bus.Publish(Event{"first": "event"}) + + listener := bus.Subscribe() + bus.Publish(Event{"second": "event"}) + event := <-listener.Events() + event1 := <-listener.Events() + assert.Equal(t, Event{"first": "event"}, event) + assert.Equal(t, Event{"second": "event"}, event1) + + bus.Publish(Event{"a": 1, "b": 2}) + event2 := <-listener.Events() + assert.Equal(t, Event{"a": 1, "b": 2}, event2) +} diff --git a/pkg/autodiscover/docker/client.go b/pkg/autodiscover/docker/client.go new file mode 100644 index 000000000000..85251120590d --- /dev/null +++ b/pkg/autodiscover/docker/client.go @@ -0,0 +1,52 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux || darwin || windows + +package docker + +import ( + "net/http" + "os" + + "github.com/moby/moby/client" + + "github.com/elastic/elastic-agent-libs/logp" +) + +// NewClient builds and returns a new Docker client. On the first request the +// client will negotiate the API version with the server unless +// DOCKER_API_VERSION is set in the environment. +func NewClient(host string, httpClient *http.Client, httpHeaders map[string]string, logger *logp.Logger) (*client.Client, error) { + log := logger.Named("docker") + + opts := []client.Opt{ + client.WithHost(host), + client.WithHTTPClient(httpClient), + client.WithHTTPHeaders(httpHeaders), + } + + version := os.Getenv("DOCKER_API_VERSION") + if version != "" { + log.Debugf("Docker client will use API version %v as set by the DOCKER_API_VERSION environment variable.", version) + opts = append(opts, client.WithAPIVersion(version)) + } else { + log.Debug("Docker client will negotiate the API version on the first request.") + } + + return client.New(opts...) +} diff --git a/pkg/autodiscover/docker/client_test.go b/pkg/autodiscover/docker/client_test.go new file mode 100644 index 000000000000..7cea0f956e29 --- /dev/null +++ b/pkg/autodiscover/docker/client_test.go @@ -0,0 +1,63 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build integration && (linux || darwin || windows) + +package docker + +import ( + "context" + "os" + "testing" + + "github.com/moby/moby/client" + "github.com/stretchr/testify/assert" + + "github.com/elastic/elastic-agent-libs/logp/logptest" +) + +func TestNewClient(t *testing.T) { + logger := logptest.NewTestingLogger(t, "") + + c, err := NewClient(client.DefaultDockerHost, nil, nil, logger) + assert.NoError(t, err) + assert.NotNil(t, c) + + _, err = c.ContainerList(context.Background(), client.ContainerListOptions{}) + assert.NoError(t, err) + + // This test only works on newer Docker versions (any supported one really) + switch c.ClientVersion() { + case "1.22": + t.Skip("Docker version is too old for this test") + case client.MaxAPIVersion: + t.Logf("Using default API version: %s", client.MaxAPIVersion) + default: + t.Logf("Negotiated version: %s", c.ClientVersion()) + } + + // Test we can hardcode version + os.Setenv("DOCKER_API_VERSION", "1.22") + + c, err = NewClient(client.DefaultDockerHost, nil, nil, logger) + assert.NoError(t, err) + assert.NotNil(t, c) + assert.Equal(t, "1.22", c.ClientVersion()) + + _, err = c.ContainerList(context.Background(), client.ContainerListOptions{}) + assert.NoError(t, err) +} diff --git a/pkg/autodiscover/docker/helpers.go b/pkg/autodiscover/docker/helpers.go new file mode 100644 index 000000000000..ab2ff164c884 --- /dev/null +++ b/pkg/autodiscover/docker/helpers.go @@ -0,0 +1,60 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package docker + +import ( + "strings" + + "github.com/elastic/beats/v7/pkg/autodiscover/utils" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/elastic-agent-libs/safemapstr" +) + +// ExtractContainerName strips the `/` characters that frequently appear in container names +func ExtractContainerName(names []string) string { + output := names[0] + + if len(names) > 1 { + for _, name := range names { + if strings.Count(output, "/") > strings.Count(name, "/") { + output = name + } + } + } + return strings.Trim(output, "/") +} + +// DeDotLabels returns a new mapstr.M containing a copy of the labels +// where the dots have been converted into nested structure, avoiding +// possible mapping errors +func DeDotLabels(labels map[string]string, dedot bool) mapstr.M { + outputLabels := mapstr.M{} + for k, v := range labels { + if dedot { + // This is necessary so that ES does not interpret '.' fields as new + // nested JSON objects, and also makes this compatible with ES 2.x. + label := utils.DeDot(k) + _, _ = outputLabels.Put(label, v) + } else { + // If we don't dedot we ensure there are no mapping errors with safemapstr + _ = safemapstr.Put(outputLabels, k, v) + } + } + + return outputLabels +} diff --git a/pkg/autodiscover/docker/watcher.go b/pkg/autodiscover/docker/watcher.go new file mode 100644 index 000000000000..bf4fa22ec50f --- /dev/null +++ b/pkg/autodiscover/docker/watcher.go @@ -0,0 +1,494 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux || darwin || windows + +package docker + +import ( + "context" + "errors" + "io" + "net/http" + "sync" + "time" + + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/api/types/events" + dockerclient "github.com/moby/moby/client" + + "github.com/docker/go-connections/tlsconfig" + + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/elastic-agent-libs/logp" +) + +// Select Docker API version +const ( + shortIDLen = 12 + dockerRequestTimeout = 10 * time.Second + dockerEventsWatchPityTimerInterval = 10 * time.Second + dockerEventsWatchPityTimerTimeout = 10 * time.Minute + dockerEventsRetryBackoffInitial = 1 * time.Second + dockerEventsRetryBackoffMax = 60 * time.Second +) + +// Watcher reads docker events and keeps a list of known containers +type Watcher interface { + // Start watching docker API for new containers + Start() error + + // Stop watching docker API for new containers + Stop() + + // Container returns the running container with the given ID or nil if unknown + Container(ID string) *Container + + // Containers returns the list of known containers + Containers() map[string]*Container + + // ListenStart returns a bus listener to receive container started events, with a `container` key holding it + ListenStart() bus.Listener + + // ListenStop returns a bus listener to receive container stopped events, with a `container` key holding it + ListenStop() bus.Listener +} + +// TLSConfig for docker socket connection +type TLSConfig struct { + CA string `config:"certificate_authority"` + Certificate string `config:"certificate"` + Key string `config:"key"` +} + +type watcher struct { + sync.RWMutex + log *logp.Logger + client Client + ctx context.Context + stop context.CancelFunc + containers map[string]*Container + deleted map[string]time.Time // deleted annotations key -> last access time + cleanupTimeout time.Duration + clock clock + stopped sync.WaitGroup + bus bus.Bus + shortID bool // whether to store short ID in "containers" too +} + +// clock is an interface used to provide mocked time on testing +type clock interface { + Now() time.Time +} + +// systemClock implements the clock interface using the system clock via the time package +type systemClock struct{} + +// Now returns the current time +func (*systemClock) Now() time.Time { return time.Now() } + +// Container info retrieved by the watcher +type Container struct { + ID string + Name string + Image string + Labels map[string]string + IPAddresses []string + Ports []container.PortSummary +} + +// Client for docker interface +type Client interface { + ContainerList(ctx context.Context, options dockerclient.ContainerListOptions) (dockerclient.ContainerListResult, error) + ContainerInspect(ctx context.Context, container string, options dockerclient.ContainerInspectOptions) (dockerclient.ContainerInspectResult, error) + Events(ctx context.Context, options dockerclient.EventsListOptions) dockerclient.EventsResult +} + +// WatcherConstructor represent a function that creates a new Watcher from giving parameters +type WatcherConstructor func(logp *logp.Logger, host string, tls *TLSConfig, storeShortID bool) (Watcher, error) + +// NewWatcher returns a watcher running for the given settings +func NewWatcher(log *logp.Logger, host string, tls *TLSConfig, storeShortID bool) (Watcher, error) { + var httpClient *http.Client + if tls != nil { + options := tlsconfig.Options{ + CAFile: tls.CA, + CertFile: tls.Certificate, + KeyFile: tls.Key, + } + + tlsc, err := tlsconfig.Client(options) + if err != nil { + return nil, err + } + + httpClient = &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: tlsc, + }, + } + } + + client, err := NewClient(host, httpClient, nil, log) + if err != nil { + return nil, err + } + + // Extra check to confirm that Docker is available + _, err = client.Info(context.Background(), dockerclient.InfoOptions{}) + if err != nil { + client.Close() + return nil, err + } + + return NewWatcherWithClient(log, client, 60*time.Second, storeShortID) +} + +// NewWatcherWithClient creates a new Watcher from a given Docker client +func NewWatcherWithClient(log *logp.Logger, client Client, cleanupTimeout time.Duration, storeShortID bool) (Watcher, error) { + ctx, cancel := context.WithCancel(context.Background()) + return &watcher{ + log: log, + client: client, + ctx: ctx, + stop: cancel, + containers: make(map[string]*Container), + deleted: make(map[string]time.Time), + cleanupTimeout: cleanupTimeout, + bus: bus.New(log, "docker"), + shortID: storeShortID, + clock: &systemClock{}, + }, nil +} + +// Container returns the running container with the given ID or nil if unknown +func (w *watcher) Container(ID string) *Container { + w.RLock() + container := w.containers[ID] + if container == nil { + w.RUnlock() + return nil + } + _, ok := w.deleted[container.ID] + w.RUnlock() + + // Update last access time if it's deleted + if ok { + w.Lock() + w.deleted[container.ID] = w.clock.Now() + w.Unlock() + } + + return container +} + +// Containers returns the list of known containers +func (w *watcher) Containers() map[string]*Container { + w.RLock() + defer w.RUnlock() + res := make(map[string]*Container) + for k, v := range w.containers { + if !w.shortID || len(k) != shortIDLen { + res[k] = v + } + } + return res +} + +// Start watching docker API for new containers +func (w *watcher) Start() error { + // Do initial scan of existing containers + w.log.Debug("Start docker containers scanner") + + w.Lock() + defer w.Unlock() + containers, err := w.listContainers(dockerclient.ContainerListOptions{}) + if err != nil { + w.log.Errorf("Failed to call listContainers: %v", err) + } + + for _, c := range containers { + w.containers[c.ID] = c + if w.shortID { + w.containers[c.ID[:shortIDLen]] = c + } + } + + // Emit all start events (avoid blocking if the bus get's blocked) + go func() { + for _, c := range containers { + w.bus.Publish(bus.Event{ + "start": true, + "container": c, + }) + } + }() + + w.stopped.Add(2) + go w.watch() + go w.cleanupWorker() + + return nil +} + +func (w *watcher) Stop() { + w.stop() + w.stopped.Wait() +} + +func (w *watcher) watch() { + defer w.stopped.Done() + + filter := make(dockerclient.Filters).Add("type", "container") + + // Ticker to restart the watcher when no events are received after some time. + tickChan := time.NewTicker(dockerEventsWatchPityTimerInterval) + defer tickChan.Stop() + + lastValidTimestamp := w.clock.Now() + retryDelay := dockerEventsRetryBackoffInitial + + watch := func() bool { + lastReceivedEventTime := w.clock.Now() + + w.log.Debugf("Fetching events since %s", lastValidTimestamp) + + options := dockerclient.EventsListOptions{ + Since: lastValidTimestamp.Format(time.RFC3339Nano), + Filters: filter, + } + + ctx, cancel := context.WithCancel(w.ctx) + defer cancel() + + result := w.client.Events(ctx, options) + for { + select { + case event := <-result.Messages: + retryDelay = dockerEventsRetryBackoffInitial + w.log.Debugf("Got a new docker event: %v", event) + if event.TimeNano > 0 { + lastValidTimestamp = time.Unix(0, event.TimeNano) + } else { + lastValidTimestamp = time.Unix(event.Time, 0) + } + lastReceivedEventTime = w.clock.Now() + + switch event.Action { + case "start", "update": + w.containerUpdate(event) + case "die": + w.containerDelete(event) + } + case err := <-result.Err: + if errors.Is(err, io.EOF) { + // Client disconnected, watch is not done, reconnect + w.log.Debug("EOF received in events stream, restarting watch call") + } else if errors.Is(err, context.DeadlineExceeded) { + w.log.Debug("Context deadline exceeded for docker request, restarting watch call") + } else if errors.Is(err, context.Canceled) { + // Parent context has been canceled, watch is done. + return true + } else { + w.log.Errorf("Error watching for docker events: %+v", err) + } + return false + case <-tickChan.C: + if time.Since(lastReceivedEventTime) > dockerEventsWatchPityTimerTimeout { + w.log.Infof("No events received within %s, restarting watch call", dockerEventsWatchPityTimerTimeout) + return false + } + case <-w.ctx.Done(): + w.log.Debug("Watcher stopped") + return true + } + } + } + + for { + done := watch() + if done { + return + } + // Wait before trying to reconnect, using exponential backoff to avoid + // log spam when the Docker daemon is unavailable (e.g. Docker Desktop on macOS). + select { + case <-w.ctx.Done(): + return + case <-time.After(retryDelay): + } + if retryDelay < dockerEventsRetryBackoffMax { + retryDelay *= 2 + if retryDelay > dockerEventsRetryBackoffMax { + retryDelay = dockerEventsRetryBackoffMax + } + } + } +} + +func (w *watcher) containerUpdate(event events.Message) { + filter := make(dockerclient.Filters).Add("id", event.Actor.ID) + + containers, err := w.listContainers(dockerclient.ContainerListOptions{ + Filters: filter, + }) + if err != nil || len(containers) != 1 { + w.log.Errorf("Error getting container info: %v", err) + return + } + container := containers[0] + + w.Lock() + w.containers[event.Actor.ID] = container + if w.shortID { + w.containers[event.Actor.ID[:shortIDLen]] = container + } + // un-delete if it's flagged (in case of update or recreation) + delete(w.deleted, event.Actor.ID) + w.Unlock() + + w.bus.Publish(bus.Event{ + "start": true, + "container": container, + }) +} + +func (w *watcher) containerDelete(event events.Message) { + container := w.Container(event.Actor.ID) + + w.Lock() + w.deleted[event.Actor.ID] = w.clock.Now() + w.Unlock() + + if container != nil { + w.bus.Publish(bus.Event{ + "stop": true, + "container": container, + }) + } +} + +func (w *watcher) listContainers(options dockerclient.ContainerListOptions) ([]*Container, error) { + log := w.log + + log.Debug("List containers") + ctx, cancel := context.WithTimeout(w.ctx, dockerRequestTimeout) + defer cancel() + + listResult, err := w.client.ContainerList(ctx, options) + if err != nil { + return nil, err + } + + containers := listResult.Items + result := make([]*Container, len(containers)) + for idx, c := range containers { + var ipaddresses []string + if c.NetworkSettings != nil { + // Handle alternate platforms like VMWare's VIC that might not have this data. + for _, net := range c.NetworkSettings.Networks { + if net.IPAddress.IsValid() { + ipaddresses = append(ipaddresses, net.IPAddress.String()) + } + } + } + + // If there are no network interfaces, assume that the container is on host network + // Inspect the container directly and use the hostname as the IP address in order + if len(ipaddresses) == 0 { + log.Debugf("Inspect container %s", c.ID) + ctx, cancel := context.WithTimeout(w.ctx, dockerRequestTimeout) + defer cancel() + inspectResult, err := w.client.ContainerInspect(ctx, c.ID, dockerclient.ContainerInspectOptions{}) + if err == nil { + ipaddresses = append(ipaddresses, inspectResult.Container.Config.Hostname) + } else { + log.Warnf("unable to inspect container %s due to error %+v", c.ID, err) + } + } + result[idx] = &Container{ + ID: c.ID, + Name: c.Names[0][1:], // Strip '/' from container names + Image: c.Image, + Labels: c.Labels, + Ports: c.Ports, + IPAddresses: ipaddresses, + } + } + + return result, nil +} + +// Clean up deleted containers after they are not used anymore +func (w *watcher) cleanupWorker() { + defer w.stopped.Done() + + for { + select { + case <-w.ctx.Done(): + return + // Wait a full period + case <-time.After(w.cleanupTimeout): + w.runCleanup() + } + } +} + +func (w *watcher) runCleanup() { + // Check entries for timeout + var toDelete []string + timeout := w.clock.Now().Add(-w.cleanupTimeout) + w.RLock() + for key, lastSeen := range w.deleted { + if lastSeen.Before(timeout) { + w.log.Debugf("Removing container %s after cool down timeout", key) + toDelete = append(toDelete, key) + } + } + w.RUnlock() + + // Delete timed out entries: + for _, key := range toDelete { + container := w.Container(key) + if container != nil { + w.bus.Publish(bus.Event{ + "delete": true, + "container": container, + }) + } + } + + w.Lock() + for _, key := range toDelete { + delete(w.deleted, key) + delete(w.containers, key) + if w.shortID { + delete(w.containers, key[:shortIDLen]) + } + } + w.Unlock() +} + +// ListenStart returns a bus listener to receive container started events, with a `container` key holding it +func (w *watcher) ListenStart() bus.Listener { + return w.bus.Subscribe("start") +} + +// ListenStop returns a bus listener to receive container stopped events, with a `container` key holding it +func (w *watcher) ListenStop() bus.Listener { + return w.bus.Subscribe("stop") +} diff --git a/pkg/autodiscover/docker/watcher_test.go b/pkg/autodiscover/docker/watcher_test.go new file mode 100644 index 000000000000..e9a93acff9b4 --- /dev/null +++ b/pkg/autodiscover/docker/watcher_test.go @@ -0,0 +1,559 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build linux || darwin || windows + +package docker + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/moby/moby/api/types/container" + "github.com/moby/moby/api/types/events" + dockerclient "github.com/moby/moby/client" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zapcore" + "go.uber.org/zap/zaptest/observer" + + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/logp/logptest" +) + +type MockClient struct { + // containers to return on ContainerList call + containers [][]container.Summary + containersErr error + // event list to send on Events call + events []any + // done channel is closed when the client has sent all events + done chan any +} + +func (m *MockClient) ContainerList(ctx context.Context, options dockerclient.ContainerListOptions) (dockerclient.ContainerListResult, error) { + if m.containersErr != nil { + return dockerclient.ContainerListResult{}, m.containersErr + } + res := m.containers[0] + m.containers = m.containers[1:] + return dockerclient.ContainerListResult{Items: res}, nil +} + +func (m *MockClient) Events(ctx context.Context, options dockerclient.EventsListOptions) dockerclient.EventsResult { + eventsC := make(chan events.Message) + errorsC := make(chan error) + + go func() { + for _, event := range m.events { + switch e := event.(type) { + case events.Message: + eventsC <- e + case error: + errorsC <- e + } + } + close(m.done) + }() + + return dockerclient.EventsResult{Messages: eventsC, Err: errorsC} +} + +func (m *MockClient) ContainerInspect(ctx context.Context, containerID string, options dockerclient.ContainerInspectOptions) (dockerclient.ContainerInspectResult, error) { + return dockerclient.ContainerInspectResult{}, errors.New("unimplemented") +} + +func TestWatcherInitialization(t *testing.T) { + watcher := runAndWait(testWatcher(t, + [][]container.Summary{ + { + container.Summary{ + ID: "0332dbd79e20", + Names: []string{"/containername", "othername"}, + Image: "busybox", + Labels: map[string]string{"foo": "bar"}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + container.Summary{ + ID: "6ac6ee8df5d4", + Names: []string{"/other"}, + Image: "nginx", + Labels: map[string]string{}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + }, + }, + nil, + )) + + assert.Equal(t, map[string]*Container{ + "0332dbd79e20": { + ID: "0332dbd79e20", + Name: "containername", + Image: "busybox", + Labels: map[string]string{"foo": "bar"}, + }, + "6ac6ee8df5d4": { + ID: "6ac6ee8df5d4", + Name: "other", + Image: "nginx", + Labels: map[string]string{}, + }, + }, watcher.Containers()) +} + +func TestWatcherInitializationShortID(t *testing.T) { + watcher := runAndWait(testWatcherShortID(t, + [][]container.Summary{ + { + container.Summary{ + ID: "1234567890123", + Names: []string{"/containername", "othername"}, + Image: "busybox", + Labels: map[string]string{"foo": "bar"}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + container.Summary{ + ID: "2345678901234", + Names: []string{"/other"}, + Image: "nginx", + Labels: map[string]string{}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + }, + }, + nil, + true, + )) + + assert.Equal(t, map[string]*Container{ + "1234567890123": { + ID: "1234567890123", + Name: "containername", + Image: "busybox", + Labels: map[string]string{"foo": "bar"}, + }, + "2345678901234": { + ID: "2345678901234", + Name: "other", + Image: "nginx", + Labels: map[string]string{}, + }, + }, watcher.Containers()) + + assert.Equal(t, &Container{ + ID: "1234567890123", + Name: "containername", + Image: "busybox", + Labels: map[string]string{"foo": "bar"}, + }, watcher.Container("123456789012")) +} + +func TestWatcherAddEvents(t *testing.T) { + watcher := runAndWait(testWatcher(t, + [][]container.Summary{ + { + container.Summary{ + ID: "0332dbd79e20", + Names: []string{"/containername", "othername"}, + Image: "busybox", + Labels: map[string]string{"foo": "bar"}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + }, + { + container.Summary{ + ID: "6ac6ee8df5d4", + Names: []string{"/other"}, + Image: "nginx", + Labels: map[string]string{"label": "value"}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + }, + }, + []any{ + events.Message{ + Action: "start", + Actor: events.Actor{ + ID: "6ac6ee8df5d4", + Attributes: map[string]string{ + "name": "other", + "image": "nginx", + "label": "value", + }, + }, + }, + }, + )) + + assert.Equal(t, map[string]*Container{ + "0332dbd79e20": { + ID: "0332dbd79e20", + Name: "containername", + Image: "busybox", + Labels: map[string]string{"foo": "bar"}, + }, + "6ac6ee8df5d4": { + ID: "6ac6ee8df5d4", + Name: "other", + Image: "nginx", + Labels: map[string]string{"label": "value"}, + }, + }, watcher.Containers()) +} + +func TestWatcherAddEventsShortID(t *testing.T) { + watcher := runAndWait(testWatcherShortID(t, + [][]container.Summary{ + { + container.Summary{ + ID: "1234567890123", + Names: []string{"/containername", "othername"}, + Image: "busybox", + Labels: map[string]string{"foo": "bar"}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + }, + { + container.Summary{ + ID: "2345678901234", + Names: []string{"/other"}, + Image: "nginx", + Labels: map[string]string{"label": "value"}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + }, + }, + []any{ + events.Message{ + Action: "start", + Actor: events.Actor{ + ID: "2345678901234", + Attributes: map[string]string{ + "name": "other", + "image": "nginx", + "label": "value", + }, + }, + }, + }, + true, + )) + + assert.Equal(t, map[string]*Container{ + "1234567890123": { + ID: "1234567890123", + Name: "containername", + Image: "busybox", + Labels: map[string]string{"foo": "bar"}, + }, + "2345678901234": { + ID: "2345678901234", + Name: "other", + Image: "nginx", + Labels: map[string]string{"label": "value"}, + }, + }, watcher.Containers()) +} + +func TestWatcherUpdateEvent(t *testing.T) { + watcher := runAndWait(testWatcher(t, + [][]container.Summary{ + { + { + ID: "0332dbd79e20", + Names: []string{"/containername", "othername"}, + Image: "busybox", + Labels: map[string]string{"label": "foo"}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + }, + { + container.Summary{ + ID: "0332dbd79e20", + Names: []string{"/containername", "othername"}, + Image: "busybox", + Labels: map[string]string{"label": "bar"}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + }, + }, + []any{ + events.Message{ + Action: "update", + Actor: events.Actor{ + ID: "0332dbd79e20", + Attributes: map[string]string{ + "name": "containername", + "image": "busybox", + "label": "bar", + }, + }, + }, + }, + )) + + assert.Equal(t, map[string]*Container{ + "0332dbd79e20": { + ID: "0332dbd79e20", + Name: "containername", + Image: "busybox", + Labels: map[string]string{"label": "bar"}, + }, + }, watcher.Containers()) + assert.Empty(t, watcher.deleted) +} + +func TestWatcherUpdateEventShortID(t *testing.T) { + watcher := runAndWait(testWatcherShortID(t, + [][]container.Summary{ + { + container.Summary{ + ID: "1234567890123", + Names: []string{"/containername", "othername"}, + Image: "busybox", + Labels: map[string]string{"label": "foo"}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + }, + { + container.Summary{ + ID: "1234567890123", + Names: []string{"/containername", "othername"}, + Image: "busybox", + Labels: map[string]string{"label": "bar"}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + }, + }, + []any{ + events.Message{ + Action: "update", + Actor: events.Actor{ + ID: "1234567890123", + Attributes: map[string]string{ + "name": "containername", + "image": "busybox", + "label": "bar", + }, + }, + }, + }, + true, + )) + + assert.Equal(t, map[string]*Container{ + "1234567890123": { + ID: "1234567890123", + Name: "containername", + Image: "busybox", + Labels: map[string]string{"label": "bar"}, + }, + }, watcher.Containers()) + assert.Empty(t, watcher.deleted) +} + +func TestWatcherDie(t *testing.T) { + watcher, clientDone := testWatcher(t, + [][]container.Summary{ + { + container.Summary{ + ID: "0332dbd79e20", + Names: []string{"/containername", "othername"}, + Image: "busybox", + Labels: map[string]string{"label": "foo"}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + }, + }, + []any{ + events.Message{ + Action: "die", + Actor: events.Actor{ + ID: "0332dbd79e20", + }, + }, + }, + ) + + clock := newTestClock() + watcher.clock = clock + + stopListener := watcher.ListenStop() + + err := watcher.Start() + require.NoError(t, err) + defer watcher.Stop() + + // Check it doesn't get removed while we request meta for the container + for range 18 { + watcher.Container("0332dbd79e20") + clock.Sleep(watcher.cleanupTimeout / 2) + watcher.runCleanup() + if !assert.Len(t, watcher.Containers(), 1) { + break + } + } + + // Wait to be sure that the delete event has been processed + <-clientDone + <-stopListener.Events() + + // Check that after the cleanup period the container is removed + clock.Sleep(watcher.cleanupTimeout + 1*time.Second) + watcher.runCleanup() + assert.Empty(t, watcher.Containers()) +} + +func TestWatcherDieShortID(t *testing.T) { + watcher, clientDone := testWatcherShortID(t, + [][]container.Summary{ + { + container.Summary{ + ID: "0332dbd79e20aaa", + Names: []string{"/containername", "othername"}, + Image: "busybox", + Labels: map[string]string{"label": "foo"}, + NetworkSettings: &container.NetworkSettingsSummary{}, + }, + }, + }, + []any{ + events.Message{ + Action: "die", + Actor: events.Actor{ + ID: "0332dbd79e20aaa", + }, + }, + }, + true, + ) + + clock := newTestClock() + watcher.clock = clock + + stopListener := watcher.ListenStop() + + err := watcher.Start() + require.NoError(t, err) + defer watcher.Stop() + + // Check it doesn't get removed while we request meta for the container + for range 18 { + watcher.Container("0332dbd79e20") + clock.Sleep(watcher.cleanupTimeout / 2) + watcher.runCleanup() + if !assert.Len(t, watcher.Containers(), 1) { + break + } + } + + // Wait to be sure that the delete event has been processed + <-clientDone + <-stopListener.Events() + + // Check that after the cleanup period the container is removed + clock.Sleep(watcher.cleanupTimeout + 1*time.Second) + watcher.runCleanup() + assert.Empty(t, watcher.Containers()) +} + +func TestWatcherNoError(t *testing.T) { + core, obs := observer.New(zapcore.DebugLevel) + l, err := logp.ConfigureWithCoreLocal(logp.DefaultConfig(logp.DefaultEnvironment), core) + require.NoError(t, err) + client := &MockClient{ + containersErr: errors.New("test error"), + events: nil, + done: make(chan any), + } + w, err := NewWatcherWithClient(l, client, 200*time.Millisecond, true) + if err != nil { + t.Fatal(err) + } + watcher, ok := w.(*watcher) + if !ok { + t.Fatal("'watcher' was supposed to be pointer to the watcher structure") + } + clock := newTestClock() + watcher.clock = clock + + err = watcher.Start() + require.NoError(t, err) + watcher.Stop() + require.Len(t, obs.FilterMessageSnippet("Failed to call listContainers").All(), 1) +} + +func testWatcher(t *testing.T, containers [][]container.Summary, events []any) (*watcher, chan any) { + return testWatcherShortID(t, containers, events, false) +} + +func testWatcherShortID(t *testing.T, containers [][]container.Summary, events []any, enable bool) (*watcher, chan any) { + logger := logptest.NewTestingLogger(t, "") + + client := &MockClient{ + containers: containers, + events: events, + done: make(chan any), + } + + w, err := NewWatcherWithClient(logger, client, 200*time.Millisecond, enable) + if err != nil { + t.Fatal(err) + } + watcher, ok := w.(*watcher) + if !ok { + t.Fatal("'watcher' was supposed to be pointer to the watcher structure") + } + + return watcher, client.done +} + +func runAndWait(w *watcher, done chan any) *watcher { + _ = w.Start() + <-done + w.Stop() + return w +} + +type testClock struct { + sync.Mutex + + now time.Time +} + +func newTestClock() *testClock { + return &testClock{now: time.Time{}} +} + +func (c *testClock) Now() time.Time { + c.Lock() + defer c.Unlock() + + c.now = c.now.Add(1) + return c.now +} + +func (c *testClock) Sleep(d time.Duration) { + c.Lock() + defer c.Unlock() + + c.now = c.now.Add(d) +} diff --git a/pkg/autodiscover/kubernetes/eventhandler.go b/pkg/autodiscover/kubernetes/eventhandler.go new file mode 100644 index 000000000000..0a71cf2b38e8 --- /dev/null +++ b/pkg/autodiscover/kubernetes/eventhandler.go @@ -0,0 +1,257 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package kubernetes + +import ( + "reflect" + "sync" +) + +// ResourceEventHandler can handle notifications for events that happen to a +// resource. The events are informational only, so you can't return an +// error. +// - OnAdd is called when an object is added. +// - OnUpdate is called when an object is modified. Note that oldObj is the +// last known state of the object-- it is possible that several changes +// were combined together, so you can't use this to see every single +// change. OnUpdate is also called when a re-list happens, and it will +// get called even if nothing changed. This is useful for periodically +// evaluating or syncing something. +// - OnDelete will get the final state of the item if it is known, otherwise +// it will get an object of type DeletedFinalStateUnknown. This can +// happen if the watch is closed and misses the delete event and we don't +// notice the deletion until the subsequent re-list. +// +// idea: allow the On* methods to return an error so that the RateLimited WorkQueue +// idea: can requeue the failed event processing. +type ResourceEventHandler interface { + OnAdd(obj any) + OnUpdate(obj any) + OnDelete(obj any) +} + +// ResourceEventHandlerFuncs is an adaptor to let you easily specify as many or +// as few of the notification functions as you want while still implementing +// ResourceEventHandler. +type ResourceEventHandlerFuncs struct { + AddFunc func(obj any) + UpdateFunc func(obj any) + DeleteFunc func(obj any) +} + +// OnAdd calls AddFunc if it's not nil. +func (r ResourceEventHandlerFuncs) OnAdd(obj any) { + if r.AddFunc != nil { + r.AddFunc(obj) + } +} + +// OnUpdate calls UpdateFunc if it's not nil. +func (r ResourceEventHandlerFuncs) OnUpdate(obj any) { + if r.UpdateFunc != nil { + r.UpdateFunc(obj) + } +} + +// OnDelete calls DeleteFunc if it's not nil. +func (r ResourceEventHandlerFuncs) OnDelete(obj any) { + if r.DeleteFunc != nil { + r.DeleteFunc(obj) + } +} + +// NoOpEventHandlerFuncs ensures that watcher reconciliation can happen even without the required funcs +type NoOpEventHandlerFuncs struct { +} + +// OnAdd does a no-op on an add event +func (n NoOpEventHandlerFuncs) OnAdd(obj any) { + +} + +// OnUpdate does a no-op on an update event +func (n NoOpEventHandlerFuncs) OnUpdate(obj any) { + +} + +// OnDelete does a no-op on a delete event +func (n NoOpEventHandlerFuncs) OnDelete(obj any) { + +} + +// FilteringResourceEventHandler applies the provided filter to all events coming +// in, ensuring the appropriate nested handler method is invoked. An object +// that starts passing the filter after an update is considered an add, and an +// object that stops passing the filter after an update is considered a delete. +type FilteringResourceEventHandler struct { + FilterFunc func(obj any) bool + Handler ResourceEventHandler +} + +// OnAdd calls the nested handler only if the filter succeeds +func (r FilteringResourceEventHandler) OnAdd(obj any) { + if !r.FilterFunc(obj) { + return + } + r.Handler.OnAdd(obj) +} + +// OnUpdate ensures the proper handler is called depending on whether the filter matches +func (r FilteringResourceEventHandler) OnUpdate(obj any) { + if !r.FilterFunc(obj) { + return + } + r.Handler.OnUpdate(obj) +} + +// OnDelete calls the nested handler only if the filter succeeds +func (r FilteringResourceEventHandler) OnDelete(obj any) { + if !r.FilterFunc(obj) { + return + } + r.Handler.OnDelete(obj) +} + +// podUpdaterHandlerFunc is a function that handles pod updater notifications. +type podUpdaterHandlerFunc func(any) + +// podUpdaterStore is the interface that an object needs to implement to be +// used as a pod updater store. +type podUpdaterStore interface { + List() []any +} + +// namespacePodUpdater notifies updates on pods when their namespaces are updated. +type namespacePodUpdater struct { + handler podUpdaterHandlerFunc + store podUpdaterStore + namespaceWatcher Watcher + locker sync.Locker +} + +// NewNamespacePodUpdater creates a namespacePodUpdater +func NewNamespacePodUpdater(handler podUpdaterHandlerFunc, store podUpdaterStore, namespaceWatcher Watcher, locker sync.Locker) *namespacePodUpdater { + return &namespacePodUpdater{ + handler: handler, + store: store, + namespaceWatcher: namespaceWatcher, + locker: locker, + } +} + +// OnUpdate handles update events on namespaces. +func (n *namespacePodUpdater) OnUpdate(obj any) { + ns, ok := obj.(*Namespace) + if !ok { + return + } + // n.store.List() returns a snapshot at this point. If a delete is received + // from the main watcher, this loop may generate an update event after the + // delete is processed, leaving configurations that would never be deleted. + // Also this loop can miss updates, what could leave outdated configurations. + // Avoid these issues by locking the processing of events from the main watcher. + if n.locker != nil { + n.locker.Lock() + defer n.locker.Unlock() + } + + cachedObject := n.namespaceWatcher.CachedObject() + cachedNamespace, ok := cachedObject.(*Namespace) + + if ok && ns.Name == cachedNamespace.Name { + labelscheck := reflect.DeepEqual(ns.Labels, cachedNamespace.Labels) + annotationscheck := reflect.DeepEqual(ns.Annotations, cachedNamespace.Annotations) + // Only if there is a difference in Metadata labels or annotations proceed to Pod update + if !labelscheck || !annotationscheck { + for _, pod := range n.store.List() { + pod, ok := pod.(*Pod) + if ok && pod.Namespace == ns.Name { + n.handler(pod) + } + } + } + } + +} + +// OnAdd handles add events on namespaces. Nothing to do, if pods are added to this +// namespace they will generate their own add events. +func (*namespacePodUpdater) OnAdd(any) {} + +// OnDelete handles delete events on namespaces. Nothing to do, if pods are deleted from this +// namespace they will generate their own delete events. +func (*namespacePodUpdater) OnDelete(any) {} + +// nodePodUpdater notifies updates on pods when their nodes are updated. +type nodePodUpdater struct { + handler podUpdaterHandlerFunc + store podUpdaterStore + nodeWatcher Watcher + locker sync.Locker +} + +// NewNodePodUpdater creates a nodePodUpdater +func NewNodePodUpdater(handler podUpdaterHandlerFunc, store podUpdaterStore, nodeWatcher Watcher, locker sync.Locker) *nodePodUpdater { + return &nodePodUpdater{ + handler: handler, + store: store, + nodeWatcher: nodeWatcher, + locker: locker, + } +} + +// OnUpdate handles update events on nodes. +func (n *nodePodUpdater) OnUpdate(obj any) { + node, ok := obj.(*Node) + if !ok { + return + } + // n.store.List() returns a snapshot at this point. If a delete is received + // from the main watcher, this loop may generate an update event after the + // delete is processed, leaving configurations that would never be deleted. + // Also this loop can miss updates, what could leave outdated configurations. + // Avoid these issues by locking the processing of events from the main watcher. + if n.locker != nil { + n.locker.Lock() + defer n.locker.Unlock() + } + cachedObject := n.nodeWatcher.CachedObject() + cachedNode, ok := cachedObject.(*Node) + + if ok && node.Name == cachedNode.Name { + labelscheck := reflect.DeepEqual(node.Labels, cachedNode.Labels) + annotationscheck := reflect.DeepEqual(node.Annotations, cachedNode.Annotations) + // Only if there is a difference in Metadata labels or annotations proceed to Pod update + if !labelscheck || !annotationscheck { + for _, pod := range n.store.List() { + pod, ok := pod.(*Pod) + if ok && pod.Spec.NodeName == node.Name { + n.handler(pod) + } + } + } + } +} + +// OnAdd handles add events on namespaces. Nothing to do, if pods are added to this +// namespace they will generate their own add events. +func (*nodePodUpdater) OnAdd(any) {} + +// OnDelete handles delete events on namespaces. Nothing to do, if pods are deleted from this +// namespace they will generate their own delete events. +func (*nodePodUpdater) OnDelete(any) {} diff --git a/pkg/autodiscover/kubernetes/informer.go b/pkg/autodiscover/kubernetes/informer.go new file mode 100644 index 000000000000..ed7e7d4bd753 --- /dev/null +++ b/pkg/autodiscover/kubernetes/informer.go @@ -0,0 +1,334 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package kubernetes + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/metadata" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" +) + +func nodeSelector(options *metav1.ListOptions, opt WatchOptions) { + if opt.Node != "" { + options.FieldSelector = "spec.nodeName=" + opt.Node + } +} + +func nameSelector(options *metav1.ListOptions, name string) { + if name != "" { + options.FieldSelector = "metadata.name=" + name + } +} + +// NewInformer creates an informer for a given resource +func NewInformer(client kubernetes.Interface, resource Resource, opts WatchOptions, indexers cache.Indexers) (cache.SharedInformer, string, error) { + var objType string + + var listwatch *cache.ListWatch + ctx := context.TODO() + switch resource.(type) { + case *Pod: + p := client.CoreV1().Pods(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + nodeSelector(&options, opts) + return p.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + nodeSelector(&options, opts) + return p.Watch(ctx, options) + }, + } + + objType = "pod" + case *Event: + e := client.CoreV1().Events(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return e.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return e.Watch(ctx, options) + }, + } + + objType = "event" + case *Node: + n := client.CoreV1().Nodes() + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + nameSelector(&options, opts.Node) + return n.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + nameSelector(&options, opts.Node) + return n.Watch(ctx, options) + }, + } + + objType = "node" + case *Namespace: + ns := client.CoreV1().Namespaces() + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + nameSelector(&options, opts.Namespace) + return ns.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + nameSelector(&options, opts.Namespace) + return ns.Watch(ctx, options) + }, + } + + objType = "namespace" + case *Deployment: + d := client.AppsV1().Deployments(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return d.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return d.Watch(ctx, options) + }, + } + + objType = "deployment" + case *ReplicaSet: + rs := client.AppsV1().ReplicaSets(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return rs.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return rs.Watch(ctx, options) + }, + } + + objType = "replicaset" + case *StatefulSet: + ss := client.AppsV1().StatefulSets(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return ss.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return ss.Watch(ctx, options) + }, + } + + objType = "statefulset" + case *DaemonSet: + ss := client.AppsV1().DaemonSets(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return ss.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return ss.Watch(ctx, options) + }, + } + + objType = "daemonset" + case *Service: + svc := client.CoreV1().Services(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return svc.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return svc.Watch(ctx, options) + }, + } + + objType = "service" + case *ServiceAccount: + sa := client.CoreV1().ServiceAccounts(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return sa.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return sa.Watch(ctx, options) + }, + } + + objType = "serviceAccount" + case *CronJob: + cronjob := client.BatchV1().CronJobs(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return cronjob.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return cronjob.Watch(ctx, options) + }, + } + + objType = "cronjob" + case *Job: + job := client.BatchV1().Jobs(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return job.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return job.Watch(ctx, options) + }, + } + + objType = "job" + case *PersistentVolume: + ss := client.CoreV1().PersistentVolumes() + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return ss.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return ss.Watch(ctx, options) + }, + } + + objType = "persistentvolume" + case *PersistentVolumeClaim: + ss := client.CoreV1().PersistentVolumeClaims(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return ss.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return ss.Watch(ctx, options) + }, + } + + objType = "persistentvolumeclaim" + case *StorageClass: + sc := client.StorageV1().StorageClasses() + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return sc.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return sc.Watch(ctx, options) + }, + } + + objType = "storageclass" + case *Role: + r := client.RbacV1().Roles(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return r.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return r.Watch(ctx, options) + }, + } + + objType = "role" + + case *RoleBinding: + rb := client.RbacV1().RoleBindings(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return rb.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return rb.Watch(ctx, options) + }, + } + + objType = "rolebinding" + + case *ClusterRole: + cr := client.RbacV1().ClusterRoles() + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return cr.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return cr.Watch(ctx, options) + }, + } + + objType = "clusterrole" + + case *ClusterRoleBinding: + crb := client.RbacV1().ClusterRoleBindings() + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return crb.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return crb.Watch(ctx, options) + }, + } + + objType = "clusterrolebinding" + + case *NetworkPolicy: + np := client.ExtensionsV1beta1().NetworkPolicies(opts.Namespace) + listwatch = &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return np.List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return np.Watch(ctx, options) + }, + } + + objType = "networkpolicy" + + default: + return nil, "", fmt.Errorf("unsupported resource type for watching %T", resource) + } + + if indexers == nil { + indexers = cache.Indexers{} + } + return cache.NewSharedIndexInformer(listwatch, resource, opts.SyncTimeout, indexers), objType, nil +} + +// NewMetadataInformer creates an informer for a given resource that only tracks the resource metadata. +func NewMetadataInformer(client metadata.Interface, gvr schema.GroupVersionResource, opts WatchOptions, indexers cache.Indexers) cache.SharedInformer { + ctx := context.Background() + if indexers == nil { + indexers = cache.Indexers{} + } + informer := cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options metav1.ListOptions) (runtime.Object, error) { + return client.Resource(gvr).List(ctx, options) + }, + WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) { + return client.Resource(gvr).Watch(ctx, options) + }, + }, + &metav1.PartialObjectMetadata{}, + opts.SyncTimeout, + indexers, + ) + return informer +} diff --git a/pkg/autodiscover/kubernetes/k8skeystore/kubernetes_keystore.go b/pkg/autodiscover/kubernetes/k8skeystore/kubernetes_keystore.go new file mode 100644 index 000000000000..437db7ce3976 --- /dev/null +++ b/pkg/autodiscover/kubernetes/k8skeystore/kubernetes_keystore.go @@ -0,0 +1,144 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package k8skeystore + +import ( + "context" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8s "k8s.io/client-go/kubernetes" + + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/keystore" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +// KubernetesKeystoresRegistry implements a Provider for Keystore. +type KubernetesKeystoresRegistry struct { + logger *logp.Logger + client k8s.Interface +} + +// KubernetesSecretsKeystore allows to retrieve passwords from Kubernetes secrets for a given namespace +type KubernetesSecretsKeystore struct { + namespace string + client k8s.Interface + logger *logp.Logger +} + +// Factoryk8s Create the right keystore with the configured options +func Factoryk8s(keystoreNamespace string, ks8client k8s.Interface, logger *logp.Logger) (keystore.Keystore, error) { + keystore, err := NewKubernetesSecretsKeystore(keystoreNamespace, ks8client, logger) + return keystore, err +} + +// NewKubernetesKeystoresRegistry initializes a KubernetesKeystoresRegistry +func NewKubernetesKeystoresRegistry(logger *logp.Logger, client k8s.Interface) bus.KeystoreProvider { + return &KubernetesKeystoresRegistry{ + logger: logger, + client: client, + } +} + +// GetKeystore return a KubernetesSecretsKeystore if it already exists for a given namespace or creates a new one. +func (kr *KubernetesKeystoresRegistry) GetKeystore(event bus.Event) keystore.Keystore { + namespace := "" + if val, ok := event["kubernetes"]; ok { + kubernetesMeta, ok := val.(mapstr.M) + if !ok { + kr.logger.Debugf("Unexpected type for kubernetes: %v", kubernetesMeta) + return nil + } + ns, err := kubernetesMeta.GetValue("namespace") + if err != nil { + kr.logger.Debugf("Cannot retrieve kubernetes namespace from event: %s", event) + return nil + } + namespace, ok = ns.(string) + if !ok { + return nil + } + } + if namespace != "" { + k8sKeystore, _ := Factoryk8s(namespace, kr.client, kr.logger) + return k8sKeystore + } + kr.logger.Debugf("Cannot retrieve kubernetes namespace from event: %s", event) + return nil +} + +// NewKubernetesSecretsKeystore returns an new k8s Keystore +func NewKubernetesSecretsKeystore(keystoreNamespace string, ks8client k8s.Interface, logger *logp.Logger) (keystore.Keystore, error) { + keystore := KubernetesSecretsKeystore{ + namespace: keystoreNamespace, + client: ks8client, + logger: logger, + } + return &keystore, nil +} + +// Retrieve return a SecureString instance that will contains both the key and the secret. +func (k *KubernetesSecretsKeystore) Retrieve(key string) (*keystore.SecureString, error) { + // key = "kubernetes.somenamespace.somesecret.value" + tokens := strings.Split(key, ".") + if len(tokens) > 0 && tokens[0] != "kubernetes" { + return nil, keystore.ErrKeyDoesntExists + } + if len(tokens) != 4 { + k.logger.Debugf( + "not valid secret key: %v. Secrets should be of the following format %v", + key, + "kubernetes.somenamespace.somesecret.value", + ) + return nil, keystore.ErrKeyDoesntExists + } + ns := tokens[1] + secretName := tokens[2] + secretVar := tokens[3] + if ns != k.namespace { + k.logger.Debugf("cannot access Kubernetes secrets from a different namespace (%v) than: %v", ns, k.namespace) + return nil, keystore.ErrKeyDoesntExists + } + secretIntefrace := k.client.CoreV1().Secrets(ns) + ctx := context.TODO() + secret, err := secretIntefrace.Get(ctx, secretName, metav1.GetOptions{}) + if err != nil { + k.logger.Errorf("Could not retrieve secret from k8s API: %v", err) + return nil, keystore.ErrKeyDoesntExists + } + if _, ok := secret.Data[secretVar]; !ok { + k.logger.Errorf("Could not retrieve value %v for secret %v", secretVar, secretName) + return nil, keystore.ErrKeyDoesntExists + } + secretString := secret.Data[secretVar] + return keystore.NewSecureString(secretString), nil +} + +// GetConfig returns config.C representation of the key / secret pair to be merged with other +// loaded configuration. +func (k *KubernetesSecretsKeystore) GetConfig() (*config.C, error) { + return nil, nil +} + +// IsPersisted return if the keystore is physically persisted on disk. +func (k *KubernetesSecretsKeystore) IsPersisted() bool { + return true +} diff --git a/pkg/autodiscover/kubernetes/k8skeystore/kubernetes_keystore_test.go b/pkg/autodiscover/kubernetes/k8skeystore/kubernetes_keystore_test.go new file mode 100644 index 000000000000..6097fb3dcdb2 --- /dev/null +++ b/pkg/autodiscover/kubernetes/k8skeystore/kubernetes_keystore_test.go @@ -0,0 +1,146 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package k8skeystore + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/v7/pkg/autodiscover/bus" + "github.com/elastic/elastic-agent-libs/logp/logptest" + "github.com/elastic/elastic-agent-libs/mapstr" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sfake "k8s.io/client-go/kubernetes/fake" +) + +const ( + ns = "test_namespace" + correctKey = "kubernetes.test_namespace.testing_secret.secret_value" + pass = "testing_passpass" +) + +func TestGetKeystore(t *testing.T) { + kRegistry := NewKubernetesKeystoresRegistry(nil, nil) + k1 := kRegistry.GetKeystore(bus.Event{"kubernetes": mapstr.M{"namespace": "my_namespace"}}) + k2 := kRegistry.GetKeystore(bus.Event{"kubernetes": mapstr.M{"namespace": "my_namespace"}}) + assert.Equal(t, k1, k2) + k3 := kRegistry.GetKeystore(bus.Event{"kubernetes": mapstr.M{"namespace": "my_namespace_2"}}) + assert.NotEqual(t, k2, k3) +} + +func TestGetKeystoreAndRetrieve(t *testing.T) { + client := k8sfake.NewSimpleClientset() + secret := &v1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "apps/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "testing_secret", + Namespace: ns, + }, + Data: map[string][]byte{ + "secret_value": []byte(pass), + }, + } + _, err := client.CoreV1().Secrets(ns).Create(context.Background(), secret, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("failed to create k8s secret: %v", err) + } + + kRegistry := NewKubernetesKeystoresRegistry(nil, client) + k1 := kRegistry.GetKeystore(bus.Event{"kubernetes": mapstr.M{"namespace": ns}}) + secure, err := k1.Retrieve(correctKey) + if err != nil { + t.Fatalf("could not retrieve k8s secret: %v", err) + } + secretVal, err := secure.Get() + assert.NoError(t, err) + bytePassword := []byte(pass) + assert.Equal(t, bytePassword, secretVal) +} + +func TestGetKeystoreAndRetrieveWithNonAllowedNamespace(t *testing.T) { + kRegistry := getKeystoreForWrongValue(t) + k1 := kRegistry.GetKeystore(bus.Event{"kubernetes": mapstr.M{"namespace": ns}}) + key := "kubernetes.test_namespace_HACK.testing_secret.secret_value" + _, err := k1.Retrieve(key) + assert.Error(t, err) +} + +func TestGetKeystoreAndRetrieveWithWrongKeyFormat(t *testing.T) { + kRegistry := getKeystoreForWrongValue(t) + k1 := kRegistry.GetKeystore(bus.Event{"kubernetes": mapstr.M{"namespace": ns}}) + key := "HACK_test_namespace_HACK.testing_secret.secret_value" + _, err := k1.Retrieve(key) + assert.Error(t, err) +} + +func TestGetKeystoreAndRetrieveWithNoSecretsExistent(t *testing.T) { + logger := logptest.NewTestingLogger(t, "test_k8s_secrets") + client := k8sfake.NewSimpleClientset() + + kRegistry := NewKubernetesKeystoresRegistry(logger, client) + k1 := kRegistry.GetKeystore(bus.Event{"kubernetes": mapstr.M{"namespace": ns}}) + _, err := k1.Retrieve(correctKey) + assert.Error(t, err) +} + +func TestGetKeystoreAndRetrieveWithWrongSecretName(t *testing.T) { + kRegistry := getKeystoreForWrongValue(t) + k1 := kRegistry.GetKeystore(bus.Event{"kubernetes": mapstr.M{"namespace": ns}}) + key := "kubernetes.test_namespace.testing_secret_WRONG.secret_value" + _, err := k1.Retrieve(key) + assert.Error(t, err) +} + +func TestGetKeystoreAndRetrieveWithWrongSecretValue(t *testing.T) { + kRegistry := getKeystoreForWrongValue(t) + k1 := kRegistry.GetKeystore(bus.Event{"kubernetes": mapstr.M{"namespace": ns}}) + key := "kubernetes.test_namespace.testing_secret.secret_value_WRONG" + _, err := k1.Retrieve(key) + assert.Error(t, err) +} + +func getKeystoreForWrongValue(t *testing.T) bus.KeystoreProvider { + logger := logptest.NewTestingLogger(t, "test_k8s_secrets") + client := k8sfake.NewSimpleClientset() + secret := &v1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "apps/v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "testing_secret", + Namespace: ns, + }, + Data: map[string][]byte{ + "secret_value": []byte(pass), + }, + } + _, err := client.CoreV1().Secrets(ns).Create(context.Background(), secret, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("failed to create k8s secret: %v", err) + } + + return NewKubernetesKeystoresRegistry(logger, client) +} diff --git a/pkg/autodiscover/kubernetes/metadata/config.go b/pkg/autodiscover/kubernetes/metadata/config.go new file mode 100644 index 000000000000..eafbb9dbd3ae --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/config.go @@ -0,0 +1,71 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + "github.com/elastic/elastic-agent-libs/config" +) + +// Config declares supported configuration for metadata generation +type Config struct { + KubeConfig string `config:"kube_config"` + KubeAdm bool `config:"use_kubeadm"` + UseRegexInclude bool `config:"use_regex_include"` + UseRegexExclude bool `config:"use_regex_exclude"` + IncludeLabels []string `config:"include_labels"` + ExcludeLabels []string `config:"exclude_labels"` + IncludeAnnotations []string `config:"include_annotations"` + + LabelsDedot bool `config:"labels.dedot"` + AnnotationsDedot bool `config:"annotations.dedot"` +} + +// AddResourceMetadataConfig allows adding config for enriching additional resources +type AddResourceMetadataConfig struct { + Node *config.C `config:"node"` + Namespace *config.C `config:"namespace"` + Deployment bool `config:"deployment"` + CronJob bool `config:"cronjob"` +} + +// InitDefaults initializes the defaults for the config. +func (c *Config) InitDefaults() { + c.KubeAdm = true + c.LabelsDedot = true + c.AnnotationsDedot = true + c.UseRegexInclude = false + c.UseRegexExclude = false +} + +// Unmarshal unpacks a Config into the metagen Config +func (c *Config) Unmarshal(cfg *config.C) error { + return cfg.Unpack(c) +} + +func GetDefaultResourceMetadataConfig() *AddResourceMetadataConfig { + metaConfig := Config{} + metaConfig.InitDefaults() + nodeCfg, _ := config.NewConfigFrom(metaConfig) + nsCfg, _ := config.NewConfigFrom(metaConfig) + return &AddResourceMetadataConfig{ + Node: nodeCfg, + Namespace: nsCfg, + Deployment: false, + CronJob: false, + } +} diff --git a/pkg/autodiscover/kubernetes/metadata/job.go b/pkg/autodiscover/kubernetes/metadata/job.go new file mode 100644 index 000000000000..7dbdbcfc1073 --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/job.go @@ -0,0 +1,94 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8s "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +type job struct { + store cache.Store + resource *Resource +} + +// NewJobMetadataGenerator creates a metagen for job resources +func NewJobMetadataGenerator(cfg *config.C, jobs cache.Store, client k8s.Interface) MetaGen { + return &job{ + resource: NewResourceMetadataGenerator(cfg, client), + store: jobs, + } +} + +// Generate generates job metadata from a resource object +// Metadata map is in the following form: +// +// { +// "kubernetes": {}, +// "some.ecs.field": "asdf" +// } +// +// All Kubernetes fields that need to be stored under kuberentes. prefix are populetad by +// GenerateK8s method while fields that are part of ECS are generated by GenerateECS method +func (jb *job) Generate(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + ecsFields := jb.GenerateECS(obj) + meta := mapstr.M{ + "kubernetes": jb.GenerateK8s(obj, opts...), + } + meta.DeepUpdate(ecsFields) + return meta +} + +// GenerateECS generates job ECS metadata from a resource object +func (jb *job) GenerateECS(obj kubernetes.Resource) mapstr.M { + return jb.resource.GenerateECS(obj) +} + +// GenerateK8s generates job metadata from a resource object +func (jb *job) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + _, ok := obj.(metav1.Object) + if !ok { + return nil + } + + meta := jb.resource.GenerateK8s("job", obj, opts...) + return meta +} + +// GenerateFromName generates job metadata from a namespace name +func (jb *job) GenerateFromName(name string, opts ...FieldOptions) mapstr.M { + if jb.store == nil { + return nil + } + + if obj, ok, _ := jb.store.GetByKey(name); ok { + res, ok := obj.(kubernetes.Resource) + if !ok { + return nil + } + + return jb.GenerateK8s(res, opts...) + } + + return nil +} diff --git a/pkg/autodiscover/kubernetes/metadata/job_test.go b/pkg/autodiscover/kubernetes/metadata/job_test.go new file mode 100644 index 000000000000..37384a8ae487 --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/job_test.go @@ -0,0 +1,203 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + batchv1 "k8s.io/api/batch/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + k8sfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +func TestJob_Generate(t *testing.T) { + client := k8sfake.NewSimpleClientset() + boolean := true + tests := []struct { + input *batchv1.Job + output mapstr.M + name string + }{ + { + name: "test simple object with owner", + input: &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "CronJob", + Name: "nginx-job", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Job", + APIVersion: "v1", + }, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "job": mapstr.M{ + "name": name, + "uid": uid, + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "cronjob": mapstr.M{ + "name": "nginx-job", + }, + "namespace": defaultNs, + }, + }, + }, + } + + cfg := config.NewConfig() + metagen := NewJobMetadataGenerator(cfg, nil, client) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.Generate(test.input)) + objMeta := &metav1.PartialObjectMetadata{ + ObjectMeta: test.input.ObjectMeta, + } + assert.Equal(t, test.output, metagen.Generate(objMeta)) + }) + } +} + +func TestJob_GenerateFromName(t *testing.T) { + client := k8sfake.NewSimpleClientset() + boolean := true + tests := []struct { + input kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object with owner", + input: &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "CronJob", + Name: "nginx-job", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Job", + APIVersion: "v1", + }, + }, + output: mapstr.M{ + "job": mapstr.M{ + "name": name, + "uid": uid, + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "cronjob": mapstr.M{ + "name": "nginx-job", + }, + "namespace": defaultNs, + }, + }, + { + name: "test simple object with owner", + input: &metav1.PartialObjectMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "CronJob", + Name: "nginx-job", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + }, + output: mapstr.M{ + "job": mapstr.M{ + "name": name, + "uid": uid, + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "cronjob": mapstr.M{ + "name": "nginx-job", + }, + "namespace": defaultNs, + }, + }, + } + + for _, test := range tests { + cfg := config.NewConfig() + jobs := cache.NewStore(cache.MetaNamespaceKeyFunc) + err := jobs.Add(test.input) + require.NoError(t, err) + metagen := NewJobMetadataGenerator(cfg, jobs, client) + + accessor, err := meta.Accessor(test.input) + require.NoError(t, err) + + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.GenerateFromName(fmt.Sprint(accessor.GetNamespace(), "/", accessor.GetName()))) + }) + } +} diff --git a/pkg/autodiscover/kubernetes/metadata/metadata.go b/pkg/autodiscover/kubernetes/metadata/metadata.go new file mode 100644 index 000000000000..17be2460975c --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/metadata.go @@ -0,0 +1,202 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + "context" + "fmt" + "strings" + + "gopkg.in/yaml.v2" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/elastic-agent-libs/safemapstr" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sclient "k8s.io/client-go/kubernetes" + + "k8s.io/client-go/tools/clientcmd" +) + +// MetaGen allows creation of metadata from either Kubernetes resources or their Resource names. +type MetaGen interface { + // Generate generates metadata for a given resource. + // Metadata map is formed in the following format: + // { + // "kubernetes": GenerateK8s(), + // "some.ecs.field": "asdf, // populated by GenerateECS() + // } + // This method is called in top level and returns the complete map of metadata. + Generate(kubernetes.Resource, ...FieldOptions) mapstr.M + // GenerateFromName generates metadata for a given resource based on it's name + GenerateFromName(string, ...FieldOptions) mapstr.M + // GenerateK8s generates kubernetes metadata for a given resource + GenerateK8s(kubernetes.Resource, ...FieldOptions) mapstr.M + // GenerateECS generates ECS metadata for a given resource + GenerateECS(kubernetes.Resource) mapstr.M +} + +// FieldOptions allows additional enrichment to be done on top of existing metadata +type FieldOptions func(mapstr.M) + +type ClusterInfo struct { + URL string + Name string +} + +type ClusterConfiguration struct { + ControlPlaneEndpoint string `yaml:"controlPlaneEndpoint"` + ClusterName string `yaml:"clusterName"` +} + +// WithFields FieldOption allows adding specific fields into the generated metadata +func WithFields(key string, value any) FieldOptions { + return func(meta mapstr.M) { + _ = safemapstr.Put(meta, key, value) + } +} + +// WithMetadata FieldOption allows adding labels and annotations under sub-resource(kind) +// example if kind=namespace namespace.labels key will be added +func WithMetadata(kind string) FieldOptions { + return func(meta mapstr.M) { + if meta["labels"] != nil { + _ = safemapstr.Put(meta, strings.ToLower(kind)+".labels", meta["labels"]) + } + if meta["annotations"] != nil { + _ = safemapstr.Put(meta, strings.ToLower(kind)+".annotations", meta["annotations"]) + } + } +} + +// GetPodMetaGen is a wrapper function that creates a metaGen for pod resource and has embeeded +// nodeMetaGen and namespaceMetaGen +func GetPodMetaGen( + cfg *config.C, + podWatcher kubernetes.Watcher, + nodeWatcher kubernetes.Watcher, + namespaceWatcher kubernetes.Watcher, + replicasetWatcher kubernetes.Watcher, + jobWatcher kubernetes.Watcher, + metaConf *AddResourceMetadataConfig) MetaGen { + var nodeMetaGen, namespaceMetaGen, rsMetaGen, jobMetaGen MetaGen + if nodeWatcher != nil && metaConf.Node.Enabled() { + nodeMetaGen = NewNodeMetadataGenerator(metaConf.Node, nodeWatcher.Store(), nodeWatcher.Client()) + } + if namespaceWatcher != nil && metaConf.Namespace.Enabled() { + namespaceMetaGen = NewNamespaceMetadataGenerator(metaConf.Namespace, namespaceWatcher.Store(), namespaceWatcher.Client()) + } + if replicasetWatcher != nil && metaConf.Deployment { + rsMetaGen = NewReplicasetMetadataGenerator(cfg, replicasetWatcher.Store(), replicasetWatcher.Client()) + } + if jobWatcher != nil && metaConf.CronJob { + jobMetaGen = NewJobMetadataGenerator(cfg, jobWatcher.Store(), jobWatcher.Client()) + } + metaGen := NewPodMetadataGenerator( + cfg, + podWatcher.Store(), + podWatcher.Client(), + nodeMetaGen, + namespaceMetaGen, + rsMetaGen, + jobMetaGen, + metaConf) + return metaGen +} + +// GetKubernetesClusterIdentifier returns ClusterInfo for k8s if available +func GetKubernetesClusterIdentifier(cfg *config.C, client k8sclient.Interface) (ClusterInfo, error) { + // try with kube config file + var c Config + err := c.Unmarshal(cfg) + if err != nil { + return ClusterInfo{}, err + } + clusterInfo, err := getClusterInfoFromKubeConfigFile(c.KubeConfig) + if err == nil { + return clusterInfo, nil + } + + // try with kubeadm-config configmap only if config_kubeAdm == true + clusterInfo, err = getClusterInfoFromKubeadmConfigMap(client, c.KubeAdm) + if err == nil { + return clusterInfo, nil + } + return ClusterInfo{}, fmt.Errorf("unable to retrieve cluster identifiers") +} + +func getClusterInfoFromKubeadmConfigMap(client k8sclient.Interface, kubeadm bool) (ClusterInfo, error) { + clusterInfo := ClusterInfo{} + if client == nil { + return clusterInfo, fmt.Errorf("unable to get cluster identifiers from kubeadm-config") + } + if !kubeadm { + return clusterInfo, nil + } + cm, err := client.CoreV1().ConfigMaps("kube-system").Get(context.TODO(), "kubeadm-config", metav1.GetOptions{}) + if err != nil { + return clusterInfo, fmt.Errorf("unable to get cluster identifiers from kubeadm-config: %w", err) + } + p, ok := cm.Data["ClusterConfiguration"] + if !ok { + return clusterInfo, fmt.Errorf("unable to get cluster identifiers from ClusterConfiguration") + } + + cc := &ClusterConfiguration{} + err = yaml.Unmarshal([]byte(p), cc) + if err != nil { + return ClusterInfo{}, err + } + if cc.ClusterName != "" { + clusterInfo.Name = cc.ClusterName + } + if cc.ControlPlaneEndpoint != "" { + clusterInfo.URL = cc.ControlPlaneEndpoint + } + + return clusterInfo, nil +} + +func getClusterInfoFromKubeConfigFile(kubeconfig string) (ClusterInfo, error) { + if kubeconfig == "" { + kubeconfig = kubernetes.GetKubeConfigEnvironmentVariable() + } + + if kubeconfig == "" { + return ClusterInfo{}, fmt.Errorf("unable to get cluster identifiers from kube_config from env") + } + + cfg, err := kubernetes.BuildConfig(kubeconfig) + if err != nil { + return ClusterInfo{}, fmt.Errorf("unable to build kube config due to error: %w", err) + } + + kubeCfg, err := clientcmd.LoadFromFile(kubeconfig) + if err != nil { + return ClusterInfo{}, fmt.Errorf("unable to load kube_config due to error: %w", err) + } + + for key, element := range kubeCfg.Clusters { + if element.Server == cfg.Host { + return ClusterInfo{element.Server, key}, nil + } + } + return ClusterInfo{}, fmt.Errorf("unable to get cluster identifiers from kube_config") +} diff --git a/pkg/autodiscover/kubernetes/metadata/namespace.go b/pkg/autodiscover/kubernetes/metadata/namespace.go new file mode 100644 index 000000000000..0bde7ac7a24a --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/namespace.go @@ -0,0 +1,132 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + k8s "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +const resource = "namespace" + +type namespace struct { + store cache.Store + resource *Resource +} + +// NewNamespaceMetadataGenerator creates a metagen for namespace resources +func NewNamespaceMetadataGenerator(cfg *config.C, namespaces cache.Store, client k8s.Interface) MetaGen { + return &namespace{ + resource: NewResourceMetadataGenerator(cfg, client), + store: namespaces, + } +} + +// Generate generates pod metadata from a resource object +// Metadata map is in the following form: +// +// { +// "kubernetes": {}, +// "some.ecs.field": "asdf" +// } +// +// All Kubernetes fields that need to be stored under kuberentes. prefix are populetad by +// GenerateK8s method while fields that are part of ECS are generated by GenerateECS method +func (n *namespace) Generate(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + ecsFields := n.GenerateECS(obj) + meta := mapstr.M{ + "kubernetes": n.GenerateK8s(obj, opts...), + } + meta.DeepUpdate(ecsFields) + return meta +} + +// GenerateECS generates namespace ECS metadata from a resource object +func (n *namespace) GenerateECS(obj kubernetes.Resource) mapstr.M { + return n.resource.GenerateECS(obj) +} + +// GenerateK8s generates namespace metadata from a resource object +func (n *namespace) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + _, ok := obj.(*kubernetes.Namespace) + if !ok { + return nil + } + + meta := n.resource.GenerateK8s(resource, obj, opts...) + meta = flattenMetadata(meta) + + // Add extra fields in here if need be + return meta +} + +// GenerateFromName generates pod metadata from a namespace name +func (n *namespace) GenerateFromName(name string, opts ...FieldOptions) mapstr.M { + if n.store == nil { + return nil + } + + if obj, ok, _ := n.store.GetByKey(name); ok { + no, ok := obj.(*kubernetes.Namespace) + if !ok { + return nil + } + + return n.GenerateK8s(no, opts...) + } + + return nil +} + +func flattenMetadata(in mapstr.M) mapstr.M { + out := mapstr.M{} + rawFields, err := in.GetValue(resource) + if err != nil { + return nil + } + + fields, ok := rawFields.(mapstr.M) + if !ok { + return nil + } + for k, v := range fields { + if k == "name" { + out[resource] = v + } else { + out[resource+"_"+k] = v + } + } + + populateFromKeys := []string{"labels", "annotations"} + for _, key := range populateFromKeys { + rawValues, err := in.GetValue(key) + if err != nil { + continue + } + values, ok := rawValues.(mapstr.M) + if ok { + out[resource+"_"+key] = values + } + } + + return out +} diff --git a/pkg/autodiscover/kubernetes/metadata/namespace_test.go b/pkg/autodiscover/kubernetes/metadata/namespace_test.go new file mode 100644 index 000000000000..0b8a0eb56afd --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/namespace_test.go @@ -0,0 +1,173 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + k8sfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +func TestNamespace_Generate(t *testing.T) { + client := k8sfake.NewSimpleClientset() + tests := []struct { + input kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object", + input: &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "spam": "baz", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: "v1", + }, + }, + // Use this for 8.0 + /* + output: mapstr.M{ + "kubernetes": mapstr.M{ + "namespace": mapstr.M{ + "name": name, + "uid": uid, + "labels": mapstr.M{ + "foo": "bar", + }, + }, + }, + },*/ + output: mapstr.M{"kubernetes": mapstr.M{ + "namespace": name, + "namespace_uid": uid, + "namespace_labels": mapstr.M{ + "foo": "bar", + }, + "namespace_annotations": mapstr.M{ + "spam": "baz", + }, + }}, + }, + } + + cfg, err := config.NewConfigFrom(Config{ + IncludeAnnotations: []string{"spam"}, + }) + if err != nil { + t.Fatalf("Could not merge configs") + } + + metagen := NewNamespaceMetadataGenerator(cfg, nil, client) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.Generate(test.input)) + }) + } +} + +func TestNamespace_GenerateFromName(t *testing.T) { + client := k8sfake.NewSimpleClientset() + tests := []struct { + input kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object", + input: &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "spam": "baz", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: "v1", + }, + }, + // Use this for 8.0 + /* + output: mapstr.M{ + "namespace": mapstr.M{ + "name": name, + "uid": uid, + "labels": mapstr.M{ + "foo": "bar", + }, + }, + },*/ + output: mapstr.M{ + "namespace": name, + "namespace_uid": uid, + "namespace_labels": mapstr.M{ + "foo": "bar", + }, + "namespace_annotations": mapstr.M{ + "spam": "baz", + }, + }, + }, + } + + for _, test := range tests { + cfg, err := config.NewConfigFrom(Config{ + IncludeAnnotations: []string{"spam"}, + }) + if err != nil { + t.Fatalf("Could not merge configs") + } + + namespaces := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = namespaces.Add(test.input) + require.NoError(t, err) + metagen := NewNamespaceMetadataGenerator(cfg, namespaces, client) + + accessor, err := meta.Accessor(test.input) + require.NoError(t, err) + + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.GenerateFromName(fmt.Sprint(accessor.GetName()))) + }) + } +} diff --git a/pkg/autodiscover/kubernetes/metadata/node.go b/pkg/autodiscover/kubernetes/metadata/node.go new file mode 100644 index 000000000000..3e678e54b3c6 --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/node.go @@ -0,0 +1,109 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + v1 "k8s.io/api/core/v1" + k8s "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +type node struct { + store cache.Store + resource *Resource +} + +// NewNodeMetadataGenerator creates a metagen for service resources +func NewNodeMetadataGenerator(cfg *config.C, nodes cache.Store, client k8s.Interface) MetaGen { + return &node{ + resource: NewResourceMetadataGenerator(cfg, client), + store: nodes, + } +} + +// Generate generates node metadata from a resource object +// Metadata map is in the following form: +// +// { +// "kubernetes": {}, +// "some.ecs.field": "asdf" +// } +// +// All Kubernetes fields that need to be stored under kuberentes. prefix are populetad by +// GenerateK8s method while fields that are part of ECS are generated by GenerateECS method +func (n *node) Generate(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + ecsFields := n.GenerateECS(obj) + meta := mapstr.M{ + "kubernetes": n.GenerateK8s(obj, opts...), + } + meta.DeepUpdate(ecsFields) + return meta +} + +// GenerateECS generates node ECS metadata from a resource object +func (n *node) GenerateECS(obj kubernetes.Resource) mapstr.M { + return n.resource.GenerateECS(obj) +} + +// GenerateK8s generates node metadata from a resource object +func (n *node) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + node, ok := obj.(*kubernetes.Node) + if !ok { + return nil + } + + meta := n.resource.GenerateK8s("node", obj, opts...) + // Add extra fields in here if need be + hostname := getHostName(node) + if hostname != "" { + _, _ = meta.Put("node.hostname", hostname) + } + return meta +} + +// GenerateFromName generates pod metadata from a service name +func (n *node) GenerateFromName(name string, opts ...FieldOptions) mapstr.M { + if n.store == nil { + return nil + } + + if obj, ok, _ := n.store.GetByKey(name); ok { + no, ok := obj.(*kubernetes.Node) + if !ok { + return nil + } + + return n.GenerateK8s(no, opts...) + } + + return nil +} + +// getHostName returns the HostName address of the node +func getHostName(node *v1.Node) string { + for _, adr := range node.Status.Addresses { + if adr.Type == v1.NodeHostName { + return adr.Address + } + } + return "" +} diff --git a/pkg/autodiscover/kubernetes/metadata/node_test.go b/pkg/autodiscover/kubernetes/metadata/node_test.go new file mode 100644 index 000000000000..c59e5bd90a8d --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/node_test.go @@ -0,0 +1,155 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + k8sfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +func TestNode_Generate(t *testing.T) { + client := k8sfake.NewSimpleClientset() + tests := []struct { + input kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object", + input: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "key1": "value1", + "key2": "value2", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Node", + APIVersion: "v1", + }, + Status: v1.NodeStatus{ + Addresses: []v1.NodeAddress{{Type: v1.NodeHostName, Address: "node1"}}, + }, + }, + output: mapstr.M{"kubernetes": mapstr.M{ + "node": mapstr.M{ + "name": name, + "uid": uid, + "hostname": "node1", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "annotations": mapstr.M{ + "key2": "value2", + }, + }}, + }, + } + + cfg, _ := config.NewConfigFrom(Config{ + IncludeAnnotations: []string{"key2"}, + }) + metagen := NewNodeMetadataGenerator(cfg, nil, client) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.Generate(test.input)) + }) + } +} + +func TestNode_GenerateFromName(t *testing.T) { + client := k8sfake.NewSimpleClientset() + tests := []struct { + input kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object", + input: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "key": "value", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Node", + APIVersion: "v1", + }, + Status: v1.NodeStatus{ + Addresses: []v1.NodeAddress{{Type: v1.NodeHostName, Address: "node1"}}, + }, + }, + output: mapstr.M{ + "node": mapstr.M{ + "name": name, + "uid": uid, + "hostname": "node1", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "annotations": mapstr.M{ + "key": "value", + }, + }, + }, + } + + for _, test := range tests { + cfg, err := config.NewConfigFrom(Config{ + IncludeAnnotations: []string{"key"}, + }) + require.NoError(t, err) + nodes := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = nodes.Add(test.input) + require.NoError(t, err) + metagen := NewNodeMetadataGenerator(cfg, nodes, client) + + accessor, err := meta.Accessor(test.input) + require.NoError(t, err) + + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.GenerateFromName(fmt.Sprint(accessor.GetName()))) + }) + } +} diff --git a/pkg/autodiscover/kubernetes/metadata/pod.go b/pkg/autodiscover/kubernetes/metadata/pod.go new file mode 100644 index 000000000000..51edbc9bba01 --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/pod.go @@ -0,0 +1,159 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + k8s "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +type pod struct { + store cache.Store + client k8s.Interface + node MetaGen + replicaset MetaGen + job MetaGen + resource *Resource + addResourceMetadata *AddResourceMetadataConfig +} + +// NewPodMetadataGenerator creates a metagen for pod resources +func NewPodMetadataGenerator( + cfg *config.C, + pods cache.Store, + client k8s.Interface, + node MetaGen, + namespace MetaGen, + replicaset MetaGen, + job MetaGen, + addResourceMetadata *AddResourceMetadataConfig) MetaGen { + + return &pod{ + resource: NewNamespaceAwareResourceMetadataGenerator(cfg, client, namespace), + store: pods, + node: node, + replicaset: replicaset, + job: job, + client: client, + addResourceMetadata: addResourceMetadata, + } +} + +// Generate generates pod metadata from a resource object +// Metadata map is in the following form: +// +// { +// "kubernetes": {}, +// "some.ecs.field": "asdf" +// } +// +// All Kubernetes fields that need to be stored under kubernetes. prefix are populated by +// GenerateK8s method while fields that are part of ECS are generated by GenerateECS method +func (p *pod) Generate(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + ecsFields := p.GenerateECS(obj) + meta := mapstr.M{ + "kubernetes": p.GenerateK8s(obj, opts...), + } + meta.DeepUpdate(ecsFields) + + return meta +} + +// GenerateECS generates pod ECS metadata from a resource object +func (p *pod) GenerateECS(obj kubernetes.Resource) mapstr.M { + return p.resource.GenerateECS(obj) +} + +// GenerateK8s generates pod metadata from a resource object +func (p *pod) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + po, ok := obj.(*kubernetes.Pod) + if !ok { + return nil + } + + out := p.resource.GenerateK8s("pod", obj, opts...) + + // check if Pod is handled by a ReplicaSet which is controlled by a Deployment. + // The hierarchy there is Deployment->ReplicaSet->Pod. + if p.addResourceMetadata.Deployment { + if p.replicaset != nil { + rsName, _ := out.GetValue("replicaset.name") + if rsName, ok := rsName.(string); ok { + meta := p.replicaset.GenerateFromName(po.Namespace + "/" + rsName) + deploymentName, _ := meta.GetValue("deployment.name") + if deploymentName != "" { + _, _ = out.Put("deployment.name", deploymentName) + } + } + } + } + + // check if Pod is handled by a Job which is controlled by a CronJob. + // The hierarchy there is CronJob->Job->Pod + if p.addResourceMetadata.CronJob { + if p.job != nil { + jobName, _ := out.GetValue("job.name") + if jobName, ok := jobName.(string); ok { + meta := p.job.GenerateFromName(po.Namespace + "/" + jobName) + cronjobName, _ := meta.GetValue("cronjob.name") + if cronjobName != "" { + _, _ = out.Put("cronjob.name", cronjobName) + } + } + } + } + + if p.node != nil { + meta := p.node.GenerateFromName(po.Spec.NodeName, WithMetadata("node")) + if meta != nil { + _, _ = out.Put("node", meta["node"]) + } else { + _, _ = out.Put("node.name", po.Spec.NodeName) + } + } else { + _, _ = out.Put("node.name", po.Spec.NodeName) + } + + if po.Status.PodIP != "" { + _, _ = out.Put("pod.ip", po.Status.PodIP) + } + + return out +} + +// GenerateFromName generates pod metadata from a pod name +func (p *pod) GenerateFromName(name string, opts ...FieldOptions) mapstr.M { + if p.store == nil { + return nil + } + + if obj, ok, _ := p.store.GetByKey(name); ok { + po, ok := obj.(*kubernetes.Pod) + if !ok { + return nil + } + + return p.GenerateK8s(po, opts...) + } + + return nil +} diff --git a/pkg/autodiscover/kubernetes/metadata/pod_test.go b/pkg/autodiscover/kubernetes/metadata/pod_test.go new file mode 100644 index 000000000000..228601db79a5 --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/pod_test.go @@ -0,0 +1,847 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + k8sfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +var addResourceMetadata = GetDefaultResourceMetadataConfig() + +func TestPod_Generate(t *testing.T) { + client := k8sfake.NewSimpleClientset() + boolean := true + rs := &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-rs", + Namespace: defaultNs, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "nginx-deployment", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + Spec: appsv1.ReplicaSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "demo", + }, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "demo", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "nginx", + Image: "nginx:1.12", + Ports: []v1.ContainerPort{ + { + Name: "http", + Protocol: v1.ProtocolTCP, + ContainerPort: 80, + }, + }, + }, + }, + }, + }, + }, + } + + _, err := client.AppsV1().ReplicaSets(defaultNs).Create(context.Background(), rs, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("failed to create k8s deployment: %v", err) + } + + tests := []struct { + input kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object", + input: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "app": "production", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + Spec: v1.PodSpec{ + NodeName: "testnode", + }, + Status: v1.PodStatus{PodIP: "127.0.0.5"}, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "pod": mapstr.M{ + "name": "obj", + "uid": uid, + "ip": "127.0.0.5", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "annotations": mapstr.M{ + "app": "production", + }, + "namespace": defaultNs, + "node": mapstr.M{ + "name": "testnode", + }, + }, + }, + }, + { + name: "test object with owner reference to Deployment", + input: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "app": "production", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "owner", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + Spec: v1.PodSpec{ + NodeName: "testnode", + }, + Status: v1.PodStatus{PodIP: "127.0.0.5"}, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "pod": mapstr.M{ + "name": "obj", + "uid": uid, + "ip": "127.0.0.5", + }, + "namespace": defaultNs, + "deployment": mapstr.M{ + "name": "owner", + }, + "node": mapstr.M{ + "name": "testnode", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "annotations": mapstr.M{ + "app": "production", + }, + }, + }, + }, + { + name: "test object with owner reference to DaemonSet", + input: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "app": "production", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "DaemonSet", + Name: "owner", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + Spec: v1.PodSpec{ + NodeName: "testnode", + }, + Status: v1.PodStatus{PodIP: "127.0.0.5"}, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "pod": mapstr.M{ + "name": "obj", + "uid": uid, + "ip": "127.0.0.5", + }, + "namespace": defaultNs, + "daemonset": mapstr.M{ + "name": "owner", + }, + "node": mapstr.M{ + "name": "testnode", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "annotations": mapstr.M{ + "app": "production", + }, + }, + }, + }, + { + name: "test object with owner reference to Job", + input: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "app": "production", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "batch/v1", + Kind: "Job", + Name: "owner", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + Spec: v1.PodSpec{ + NodeName: "testnode", + }, + Status: v1.PodStatus{PodIP: "127.0.0.5"}, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "pod": mapstr.M{ + "name": "obj", + "uid": uid, + "ip": "127.0.0.5", + }, + "namespace": defaultNs, + "job": mapstr.M{ + "name": "owner", + }, + "node": mapstr.M{ + "name": "testnode", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "annotations": mapstr.M{ + "app": "production", + }, + }, + }, + }, + { + name: "test object with owner reference to replicaset", + input: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "app": "production", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "ReplicaSet", + Name: "nginx-rs", + UID: "005f3b90-4b9d-12f8-acf0-31020a8409087", + Controller: &boolean, + }, + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + Spec: v1.PodSpec{ + NodeName: "testnode", + }, + Status: v1.PodStatus{PodIP: "127.0.0.5"}, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "pod": mapstr.M{ + "name": "obj", + "uid": uid, + "ip": "127.0.0.5", + }, + "namespace": defaultNs, + // We comment below block because add_resource_metadata.deployment: false by default + // "deployment": mapstr.M{ + // "name": "nginx-deployment", + // }, + "replicaset": mapstr.M{ + "name": "nginx-rs", + }, + "node": mapstr.M{ + "name": "testnode", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "annotations": mapstr.M{ + "app": "production", + }, + }, + }, + }, + { + name: "test object with owner reference to replicaset honors annotations.dedot: false", + input: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "k8s.app": "production", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "ReplicaSet", + Name: "nginx-rs", + UID: "005f3b90-4b9d-12f8-acf0-31020a8409087", + Controller: &boolean, + }, + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + Spec: v1.PodSpec{ + NodeName: "testnode", + }, + Status: v1.PodStatus{PodIP: "127.0.0.5"}, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "pod": mapstr.M{ + "name": "obj", + "uid": uid, + "ip": "127.0.0.5", + }, + "namespace": defaultNs, + // We comment below block because add_resource_metadata.deployment: false by default + // "deployment": mapstr.M{ + // "name": "nginx-deployment", + // }, + "replicaset": mapstr.M{ + "name": "nginx-rs", + }, + "node": mapstr.M{ + "name": "testnode", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "annotations": mapstr.M{ + "k8s": mapstr.M{"app": "production"}, + }, + }, + }, + }, + } + + config, err := config.NewConfigFrom(map[string]any{ + "include_annotations": []string{"app", "k8s.app"}, + "annotations.dedot": false, + }) + assert.NoError(t, err) + assert.NotEqual(t, *addResourceMetadata.Namespace, *addResourceMetadata.Node) + + replicaSets := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = replicaSets.Add(rs) + require.NoError(t, err) + rsMeta := NewReplicasetMetadataGenerator(config, replicaSets, client) + metagen := NewPodMetadataGenerator(config, nil, client, nil, nil, rsMeta, nil, addResourceMetadata) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.Generate(test.input)) + }) + } +} + +func TestPod_GenerateFromName(t *testing.T) { + client := k8sfake.NewSimpleClientset() + boolean := true + tests := []struct { + input kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object", + input: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "k8s.app": "production", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + Spec: v1.PodSpec{ + NodeName: "testnode", + }, + Status: v1.PodStatus{PodIP: "127.0.0.5"}, + }, + output: mapstr.M{ + "pod": mapstr.M{ + "name": "obj", + "uid": uid, + "ip": "127.0.0.5", + }, + "namespace": defaultNs, + "node": mapstr.M{ + "name": "testnode", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "annotations": mapstr.M{ + "k8s_app": "production", + }, + }, + }, + { + name: "test object with owner reference", + input: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "app": "production", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "owner", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + Spec: v1.PodSpec{ + NodeName: "testnode", + }, + Status: v1.PodStatus{PodIP: "127.0.0.5"}, + }, + output: mapstr.M{ + "pod": mapstr.M{ + "name": "obj", + "uid": uid, + "ip": "127.0.0.5", + }, + "namespace": defaultNs, + "deployment": mapstr.M{ + "name": "owner", + }, + "node": mapstr.M{ + "name": "testnode", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "annotations": mapstr.M{ + "app": "production", + }, + }, + }, + } + + for _, test := range tests { + config, err := config.NewConfigFrom(map[string]any{ + "include_annotations": []string{"app", "k8s.app"}, + }) + assert.NoError(t, err) + pods := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = pods.Add(test.input) + require.NoError(t, err) + metagen := NewPodMetadataGenerator(config, pods, client, nil, nil, nil, nil, addResourceMetadata) + + accessor, err := meta.Accessor(test.input) + require.NoError(t, err) + + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.GenerateFromName(fmt.Sprint(accessor.GetNamespace(), "/", accessor.GetName()))) + }) + } +} + +func TestPod_GenerateWithNodeNamespace(t *testing.T) { + client := k8sfake.NewSimpleClientset() + tests := []struct { + input kubernetes.Resource + node kubernetes.Resource + namespace kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object", + input: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{ + "app": "production", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + Spec: v1.PodSpec{ + NodeName: "testnode", + }, + Status: v1.PodStatus{PodIP: "127.0.0.5"}, + }, + node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testnode", + UID: types.UID(uid), + Labels: map[string]string{ + "nodekey": "nodevalue", + }, + Annotations: map[string]string{}, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Node", + APIVersion: "v1", + }, + Status: v1.NodeStatus{ + Addresses: []v1.NodeAddress{{Type: v1.NodeHostName, Address: "node1"}}, + }, + }, + namespace: &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultNs, + UID: types.UID(uid), + Labels: map[string]string{ + "nskey": "nsvalue", + }, + Annotations: map[string]string{}, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: "v1", + }, + }, + output: mapstr.M{"kubernetes": mapstr.M{ + "pod": mapstr.M{ + "name": "obj", + "uid": uid, + "ip": "127.0.0.5", + }, + "namespace": defaultNs, + "namespace_uid": uid, + "namespace_labels": mapstr.M{ + "nskey": "nsvalue", + }, + "node": mapstr.M{ + "name": "testnode", + "uid": uid, + "labels": mapstr.M{ + "nodekey": "nodevalue", + }, + "hostname": "node1", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "annotations": mapstr.M{ + "app": "production", + }, + }}, + }, + } + + for _, test := range tests { + config, err := config.NewConfigFrom(map[string]any{ + "include_annotations": []string{"app"}, + }) + assert.NoError(t, err) + pods := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = pods.Add(test.input) + require.NoError(t, err) + + nodes := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = nodes.Add(test.node) + require.NoError(t, err) + nodeMeta := NewNodeMetadataGenerator(config, nodes, client) + + namespaces := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = namespaces.Add(test.namespace) + require.NoError(t, err) + nsMeta := NewNamespaceMetadataGenerator(config, namespaces, client) + + metagen := NewPodMetadataGenerator(config, pods, client, nodeMeta, nsMeta, nil, nil, addResourceMetadata) + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.Generate(test.input)) + }) + } +} + +func TestPod_GenerateWithNodeNamespaceWithAddResourceConfig(t *testing.T) { + client := k8sfake.NewSimpleClientset() + boolean := true + + tests := []struct { + input kubernetes.Resource + node kubernetes.Resource + namespace kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object", + input: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "app.kubernetes.io/component": "exporter", + }, + Annotations: map[string]string{ + "app": "production", + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "ReplicaSet", + Name: "nginx-rs", + UID: "005f3b90-4b9d-12f8-acf0-31020a8409087", + Controller: &boolean, + }, + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + + Spec: v1.PodSpec{ + NodeName: "testnode", + }, + Status: v1.PodStatus{PodIP: "127.0.0.5"}, + }, + node: &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "testnode", + UID: types.UID(uid), + Labels: map[string]string{ + "nodekey": "nodevalue", + "nodekey2": "nodevalue2", + }, + Annotations: map[string]string{ + "node.annotation": "node.value", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Node", + APIVersion: "v1", + }, + Status: v1.NodeStatus{ + Addresses: []v1.NodeAddress{{Type: v1.NodeHostName, Address: "node1"}}, + }, + }, + namespace: &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultNs, + UID: types.UID(uid), + Labels: map[string]string{ + "app.kubernetes.io/name": "kube-state-metrics", + "nskey2": "nsvalue2", + }, + Annotations: map[string]string{ + "ns.annotation": "ns.value", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: "v1", + }, + }, + output: mapstr.M{"kubernetes": mapstr.M{ + "pod": mapstr.M{ + "name": "obj", + "uid": uid, + "ip": "127.0.0.5", + }, + "namespace": defaultNs, + "namespace_uid": uid, + "namespace_labels": mapstr.M{ + "app_kubernetes_io/name": "kube-state-metrics", + }, + "namespace_annotations": mapstr.M{ + "ns_annotation": "ns.value", + }, + "node": mapstr.M{ + "name": "testnode", + "uid": uid, + "labels": mapstr.M{ + "nodekey2": "nodevalue2", + }, + "hostname": "node1", + "annotations": mapstr.M{ + "node_annotation": "node.value", + }, + }, + "labels": mapstr.M{ + "app_kubernetes_io/component": "exporter", + }, + "annotations": mapstr.M{ + "app": "production", + }, + "replicaset": mapstr.M{ + "name": "nginx-rs", + }, + }}, + }, + } + + for _, test := range tests { + c, err := config.NewConfigFrom(map[string]any{ + "include_annotations": []string{"app"}, + }) + + assert.NoError(t, err) + + namespaceConfig, _ := config.NewConfigFrom(map[string]any{ + "include_labels": []string{"app.kubernetes.io/name"}, + "include_annotations": []string{"ns.annotation"}, + }) + nodeConfig, _ := config.NewConfigFrom(map[string]any{ + "include_labels": []string{"nodekey2"}, + "include_annotations": []string{"node.annotation"}, + }) + metaConfig := AddResourceMetadataConfig{ + Namespace: namespaceConfig, + Node: nodeConfig, + Deployment: false, + } + + pods := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = pods.Add(test.input) + require.NoError(t, err) + + nodes := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = nodes.Add(test.node) + require.NoError(t, err) + nodeMeta := NewNodeMetadataGenerator(nodeConfig, nodes, client) + + namespaces := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = namespaces.Add(test.namespace) + require.NoError(t, err) + nsMeta := NewNamespaceMetadataGenerator(namespaceConfig, namespaces, client) + + metagen := NewPodMetadataGenerator(c, pods, client, nodeMeta, nsMeta, nil, nil, &metaConfig) + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.Generate(test.input)) + }) + } +} diff --git a/pkg/autodiscover/kubernetes/metadata/replicaset.go b/pkg/autodiscover/kubernetes/metadata/replicaset.go new file mode 100644 index 000000000000..40fc86c80db7 --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/replicaset.go @@ -0,0 +1,128 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + "fmt" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8s "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +const resourceType = "replicaset" + +type replicaset struct { + store cache.Store + resource *Resource +} + +// NewReplicasetMetadataGenerator creates a metagen for replicaset resources +func NewReplicasetMetadataGenerator(cfg *config.C, replicasets cache.Store, client k8s.Interface) MetaGen { + return &replicaset{ + resource: NewResourceMetadataGenerator(cfg, client), + store: replicasets, + } +} + +// Generate generates replicaset metadata from a resource object +// Metadata map is in the following form: +// +// { +// "kubernetes": {}, +// "some.ecs.field": "asdf" +// } +// +// All Kubernetes fields that need to be stored under kuberentes. prefix are populetad by +// GenerateK8s method while fields that are part of ECS are generated by GenerateECS method +func (rs *replicaset) Generate(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + ecsFields := rs.GenerateECS(obj) + meta := mapstr.M{ + "kubernetes": rs.GenerateK8s(obj, opts...), + } + meta.DeepUpdate(ecsFields) + return meta +} + +// GenerateECS generates replicaset ECS metadata from a resource object +func (rs *replicaset) GenerateECS(obj kubernetes.Resource) mapstr.M { + return rs.resource.GenerateECS(obj) +} + +// GenerateK8s generates replicaset metadata from a resource object +func (rs *replicaset) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + _, ok := obj.(metav1.Object) + if !ok { + return nil + } + + meta := rs.resource.GenerateK8s(resourceType, obj, opts...) + return meta +} + +// GenerateFromName generates replicaset metadata from a replicaset name +func (rs *replicaset) GenerateFromName(name string, opts ...FieldOptions) mapstr.M { + if rs.store == nil { + return nil + } + + if obj, ok, _ := rs.store.GetByKey(name); ok { + res, ok := obj.(kubernetes.Resource) + if !ok { + return nil + } + + return rs.GenerateK8s(res, opts...) + } + + return nil +} + +// RemoveUnnecessaryReplicaSetData removes all data from a ReplicaSet resource, except what we need to compute +// Pod metadata. This function works for both ReplicaSet and PartialObjectMetadata. +func RemoveUnnecessaryReplicaSetData(obj any) (any, error) { + switch old := obj.(type) { + case *appsv1.ReplicaSet: + transformed := &appsv1.ReplicaSet{ + ObjectMeta: kubernetes.ObjectMeta{ + Name: old.GetName(), + Namespace: old.GetNamespace(), + OwnerReferences: old.GetOwnerReferences(), + ResourceVersion: old.GetResourceVersion(), + }, + } + return transformed, nil + case *metav1.PartialObjectMetadata: + transformed := &metav1.PartialObjectMetadata{ + ObjectMeta: kubernetes.ObjectMeta{ + Name: old.GetName(), + Namespace: old.GetNamespace(), + OwnerReferences: old.GetOwnerReferences(), + ResourceVersion: old.GetResourceVersion(), + }, + } + return transformed, nil + default: + return nil, fmt.Errorf("obj of type %T neither a ReplicaSet nor a PartialObjectMetadata", obj) + } +} diff --git a/pkg/autodiscover/kubernetes/metadata/replicaset_test.go b/pkg/autodiscover/kubernetes/metadata/replicaset_test.go new file mode 100644 index 000000000000..3b4b2b9c9e60 --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/replicaset_test.go @@ -0,0 +1,359 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +func TestReplicaset_Generate(t *testing.T) { + client := k8sfake.NewSimpleClientset() + boolean := true + tests := []struct { + input *appsv1.ReplicaSet + output mapstr.M + name string + }{ + { + name: "test simple object with owner", + input: &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-rs", + Namespace: defaultNs, + UID: uid, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "nginx-deployment", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + Spec: appsv1.ReplicaSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "demo", + }, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "demo", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "nginx", + Image: "nginx:1.12", + Ports: []v1.ContainerPort{ + { + Name: "http", + Protocol: v1.ProtocolTCP, + ContainerPort: 80, + }, + }, + }, + }, + }, + }, + }, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "replicaset": mapstr.M{ + "name": "nginx-rs", + "uid": uid, + }, + "deployment": mapstr.M{ + "name": "nginx-deployment", + }, + "namespace": defaultNs, + }, + }, + }, + } + + cfg := config.NewConfig() + metagen := NewReplicasetMetadataGenerator(cfg, nil, client) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.Generate(test.input)) + // verify this works with just the metadata + objMeta := &metav1.PartialObjectMetadata{ + ObjectMeta: test.input.ObjectMeta, + } + assert.Equal(t, test.output, metagen.Generate(objMeta)) + }) + } +} + +func TestReplicaset_GenerateFromName(t *testing.T) { + client := k8sfake.NewSimpleClientset() + boolean := true + tests := []struct { + input kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object with owner", + input: &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-rs", + Namespace: defaultNs, + UID: uid, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "nginx-deployment", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + Spec: appsv1.ReplicaSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "demo", + }, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "demo", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "nginx", + Image: "nginx:1.12", + Ports: []v1.ContainerPort{ + { + Name: "http", + Protocol: v1.ProtocolTCP, + ContainerPort: 80, + }, + }, + }, + }, + }, + }, + }, + }, + output: mapstr.M{ + "replicaset": mapstr.M{ + "name": "nginx-rs", + "uid": uid, + }, + "deployment": mapstr.M{ + "name": "nginx-deployment", + }, + "namespace": defaultNs, + }, + }, + { + name: "test simple object with owner", + input: &metav1.PartialObjectMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-rs", + Namespace: defaultNs, + UID: uid, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "nginx-deployment", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + }, + output: mapstr.M{ + "replicaset": mapstr.M{ + "name": "nginx-rs", + "uid": uid, + }, + "deployment": mapstr.M{ + "name": "nginx-deployment", + }, + "namespace": defaultNs, + }, + }, + } + + for _, test := range tests { + cfg := config.NewConfig() + replicasets := cache.NewStore(cache.MetaNamespaceKeyFunc) + err := replicasets.Add(test.input) + require.NoError(t, err) + metagen := NewReplicasetMetadataGenerator(cfg, replicasets, client) + + accessor, err := meta.Accessor(test.input) + require.NoError(t, err) + + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.GenerateFromName(fmt.Sprint(accessor.GetNamespace(), "/", accessor.GetName()))) + }) + } +} + +func TestReplicaset_RemoveUnnecessaryData(t *testing.T) { + boolean := true + tests := []struct { + input kubernetes.Resource + output kubernetes.Resource + name string + err error + }{ + { + name: "test simple object with owner", + input: &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-rs", + Namespace: defaultNs, + UID: uid, + ResourceVersion: "688594", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "nginx-deployment", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + Spec: appsv1.ReplicaSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "demo", + }, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "demo", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "nginx", + Image: "nginx:1.12", + Ports: []v1.ContainerPort{ + { + Name: "http", + Protocol: v1.ProtocolTCP, + ContainerPort: 80, + }, + }, + }, + }, + }, + }, + }, + }, + output: &appsv1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-rs", + Namespace: defaultNs, + ResourceVersion: "688594", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "nginx-deployment", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + }, + }, + { + name: "test simple object with owner", + input: &metav1.PartialObjectMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-rs", + Namespace: defaultNs, + UID: uid, + ResourceVersion: "688594", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "nginx-deployment", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + }, + output: &metav1.PartialObjectMetadata{ + ObjectMeta: metav1.ObjectMeta{ + Name: "nginx-rs", + Namespace: defaultNs, + ResourceVersion: "688594", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "nginx-deployment", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + }, + }, + { + name: "wrong resource type", + input: &v1.Pod{}, + err: fmt.Errorf("obj of type *v1.Pod neither a ReplicaSet nor a PartialObjectMetadata"), + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + transformed, err := RemoveUnnecessaryReplicaSetData(test.input) + assert.Equal(t, test.err, err) + assert.Equal(t, test.output, transformed) + }) + } +} diff --git a/pkg/autodiscover/kubernetes/metadata/resource.go b/pkg/autodiscover/kubernetes/metadata/resource.go new file mode 100644 index 000000000000..bded74e7c301 --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/resource.go @@ -0,0 +1,231 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + "regexp" + "strings" + + "k8s.io/apimachinery/pkg/api/meta" + + k8s "k8s.io/client-go/kubernetes" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/beats/v7/pkg/autodiscover/utils" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/elastic-agent-libs/safemapstr" +) + +const deploymentType = "Deployment" + +// Resource generates metadata for any kubernetes resource +type Resource struct { + config *Config + clusterInfo ClusterInfo + namespace MetaGen +} + +// NewResourceMetadataGenerator creates a metadata generator for a generic resource +func NewResourceMetadataGenerator(cfg *config.C, client k8s.Interface) *Resource { + var c Config + err := c.Unmarshal(cfg) + if err != nil { + return nil + } + + r := &Resource{ + config: &c, + } + clusterInfo, err := GetKubernetesClusterIdentifier(cfg, client) + if err == nil { + r.clusterInfo = clusterInfo + } + return r +} + +// NewNamespaceAwareResourceMetadataGenerator creates a metadata generator with informatuon about namespace +func NewNamespaceAwareResourceMetadataGenerator(cfg *config.C, client k8s.Interface, namespace MetaGen) *Resource { + r := NewResourceMetadataGenerator(cfg, client) + r.namespace = namespace + return r +} + +// Generate generates metadata from a resource object +// Generate method returns metadata in the following form: +// +// { +// "kubernetes": {}, +// "ecs.a.field": 42, +// } +// +// This method should be called in top level and not as part of other metadata generators. +// For retrieving metadata without 'kubernetes.' prefix one should call GenerateK8s instead. +func (r *Resource) Generate(kind string, obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + ecsFields := r.GenerateECS(obj) + meta := mapstr.M{ + "kubernetes": r.GenerateK8s(kind, obj, opts...), + } + meta.DeepUpdate(ecsFields) + return meta +} + +// GenerateECS generates ECS metadata from a resource object +func (r *Resource) GenerateECS(_ kubernetes.Resource) mapstr.M { + ecsMeta := mapstr.M{} + if r.clusterInfo.URL != "" { + _, _ = ecsMeta.Put("orchestrator.cluster.url", r.clusterInfo.URL) + } + if r.clusterInfo.Name != "" { + _, _ = ecsMeta.Put("orchestrator.cluster.name", r.clusterInfo.Name) + } + return ecsMeta +} + +// GenerateK8s takes a kind and an object and creates metadata for the same +func (r *Resource) GenerateK8s(kind string, obj kubernetes.Resource, options ...FieldOptions) mapstr.M { + accessor, err := meta.Accessor(obj) + if err != nil { + return nil + } + + var labelMap mapstr.M + if len(r.config.IncludeLabels) == 0 { + labelMap = GenerateMap(accessor.GetLabels(), r.config.LabelsDedot) + } else { + labelMap = generateMapSubset(accessor.GetLabels(), r.config.IncludeLabels, r.config.LabelsDedot, r.config.UseRegexInclude) + } + + var labelMaptoExclude mapstr.M + if len(r.config.ExcludeLabels) != 0 { + labelMaptoExclude = generateMapSubset(accessor.GetLabels(), r.config.ExcludeLabels, r.config.LabelsDedot, r.config.UseRegexExclude) + } + + // Exclude any labels that are present in the exclude_labels config + for label := range labelMaptoExclude { + _ = labelMap.Delete(label) + } + + annotationsMap := generateMapSubset(accessor.GetAnnotations(), r.config.IncludeAnnotations, r.config.AnnotationsDedot, r.config.UseRegexInclude) + + meta := mapstr.M{ + strings.ToLower(kind): mapstr.M{ + "name": accessor.GetName(), + "uid": string(accessor.GetUID()), + }, + } + + namespaceName := accessor.GetNamespace() + if namespaceName != "" { + _ = safemapstr.Put(meta, "namespace", namespaceName) + + if r.namespace != nil { + nsMeta := r.namespace.GenerateFromName(namespaceName) + if nsMeta != nil { + meta.DeepUpdate(nsMeta) + } + } + } + + // Add controller metadata if present + for _, ref := range accessor.GetOwnerReferences() { + if ref.Controller != nil && *ref.Controller { + switch ref.Kind { + // grow this list as we keep adding more `state_*` metricsets + case deploymentType, + "ReplicaSet", + "StatefulSet", + "DaemonSet", + "Job", + "CronJob": + _ = safemapstr.Put(meta, strings.ToLower(ref.Kind)+".name", ref.Name) + } + } + } + + if len(labelMap) != 0 { + _ = safemapstr.Put(meta, "labels", labelMap) + } + + if len(annotationsMap) != 0 { + _ = safemapstr.Put(meta, "annotations", annotationsMap) + } + + for _, option := range options { + option(meta) + } + + return meta +} + +func generateMapSubset(input map[string]string, keys []string, dedot bool, useregex bool) mapstr.M { + output := mapstr.M{} + if input == nil { + return output + } + + for _, key := range keys { + //This is the part where use_regex is enabled + if useregex { + pattern, err := regexp.Compile(key) + if err == nil { + for label, value := range input { + matched := pattern.MatchString(label) + if matched { + if dedot { + dedotKey := utils.DeDot(label) + _, _ = output.Put(dedotKey, value) + } else { + _ = safemapstr.Put(output, label, value) + } + } + + } + } + } else { + value, ok := input[key] + if ok { + if dedot { + dedotKey := utils.DeDot(key) + _, _ = output.Put(dedotKey, value) + } else { + _ = safemapstr.Put(output, key, value) + } + } + } + } + return output +} + +func GenerateMap(input map[string]string, dedot bool) mapstr.M { + output := mapstr.M{} + if input == nil { + return output + } + + for k, v := range input { + if dedot { + label := utils.DeDot(k) + _, _ = output.Put(label, v) + } else { + _ = safemapstr.Put(output, k, v) + } + } + + return output +} diff --git a/pkg/autodiscover/kubernetes/metadata/resource_test.go b/pkg/autodiscover/kubernetes/metadata/resource_test.go new file mode 100644 index 000000000000..fe95a8e66358 --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/resource_test.go @@ -0,0 +1,366 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + k8sfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/go-ucfg" +) + +const ( + uid = "005f3b90-4b9d-12f8-acf0-31020a840133" + defaultNs = "default" + name = "obj" +) + +func TestResource_Generate(t *testing.T) { + boolean := true + tests := []struct { + input kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object", + input: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "pod": mapstr.M{ + "name": name, + "uid": uid, + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "namespace": defaultNs, + }, + }, + }, + { + name: "test object with owner reference", + input: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "owner", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "pod": mapstr.M{ + "name": name, + "uid": uid, + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "namespace": defaultNs, + "deployment": mapstr.M{ + "name": "owner", + }, + }, + }, + }, + } + + var cfg Config + err := ucfg.New().Unpack(&cfg) + require.NoError(t, err) + metagen := &Resource{ + config: &cfg, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.Generate("pod", test.input)) + }) + } +} + +func TestNamespaceAwareResource_GenerateWithNamespace(t *testing.T) { + client := k8sfake.NewSimpleClientset() + tests := []struct { + resourceName string + input kubernetes.Resource + namespace kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test not namespaced kubernetes resource - PersistentVolume", + resourceName: "persistentvolume", + input: &v1.PersistentVolume{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pvc-18705cfb-9fb8-441f-9b32-0d67a21af839", + UID: "020fd954-3674-496a-9e77-c25f0f2257ea", + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "PersistentVolume", + APIVersion: "v1", + }, + }, + namespace: &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultNs, + UID: types.UID(uid), + Labels: map[string]string{ + "nskey": "nsvalue", + }, + Annotations: map[string]string{ + "ns.annotation": "value", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: "v1", + }, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "persistentvolume": mapstr.M{ + "name": "pvc-18705cfb-9fb8-441f-9b32-0d67a21af839", + "uid": "020fd954-3674-496a-9e77-c25f0f2257ea", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + }, + }, + }, + { + name: "test namespaced kubernetes resource", + resourceName: "deployment", + input: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + UID: "f33ca314-8cc5-48ea-90b7-3102c7430f75", + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + }, + namespace: &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultNs, + UID: types.UID(uid), + Labels: map[string]string{ + "nskey": "nsvalue", + }, + Annotations: map[string]string{ + "ns.annotation": "value", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: "v1", + }, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "deployment": mapstr.M{ + "name": name, + "uid": "f33ca314-8cc5-48ea-90b7-3102c7430f75", + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "namespace": "default", + "namespace_uid": uid, + "namespace_labels": mapstr.M{ + "nskey": "nsvalue", + }, + "namespace_annotations": mapstr.M{ + "ns_annotation": "value", + }, + }, + }, + }, + } + + for _, test := range tests { + nsConfig, err := config.NewConfigFrom(map[string]any{ + "include_annotations": []string{"ns.annotation"}, + }) + require.NoError(t, err) + + namespaces := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = namespaces.Add(test.namespace) + require.NoError(t, err) + nsMeta := NewNamespaceMetadataGenerator(nsConfig, namespaces, client) + + metagen := NewNamespaceAwareResourceMetadataGenerator(nsConfig, client, nsMeta) + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.Generate(test.resourceName, test.input)) + }) + } +} + +func Test_generateMapSubset(t *testing.T) { + + Labelsdedot := map[string]string{ + "app.kubernetes.io/name": "no", + "foo": "bar", + } + + Labels := map[string]string{ + "nottomatch": "no", + "foo": "bar", + "foo1": "bar1", + "foo2": "bar2", + "foo-example": "bar-example", + "test": "test1", + "footest": "footest1", + "simplefo-example": "simplefo-example1", + } + + tests := []struct { + name string + key []string + expectedresult mapstr.M + }{ + {name: "Validating wildcards", + key: []string{ + "foo*", + "test", + }, + expectedresult: mapstr.M{ + "foo": "bar", + "foo1": "bar1", + "foo2": "bar2", + "foo-example": "bar-example", + "test": "test1", + "footest": "footest1", + "simplefo-example": "simplefo-example1", + }, + }, { + name: "Validating ?", + key: []string{ + "0?1", + }, + expectedresult: mapstr.M{ + "foo1": "bar1", + }, + }, { + name: "Validating start of a string", + key: []string{ + "^test", + }, + expectedresult: mapstr.M{ + "test": "test1", + }, + }, { + name: "Validating end of a string", + key: []string{ + "test$", + }, + expectedresult: mapstr.M{ + "test": "test1", + "footest": "footest1", + }, + }, { + name: "Exact matches", + key: []string{ + "t{2}", + }, + expectedresult: mapstr.M{ + "nottomatch": "no", + }, + }, { + name: " //Dedot Validation", + key: []string{ + "app.kubernetes.io", + }, + expectedresult: mapstr.M{ + "app_kubernetes_io/name": "no", + }, + }, { + name: "Default code, UsegexInclude=false Validation", + key: []string{ + "app.kubernetes.io/name", + }, + expectedresult: mapstr.M{ + "app_kubernetes_io/name": "no", + }, + }, + } + + for i := 0; i <= 4; i++ { + output := generateMapSubset(Labels, tests[i].key, false, true) + assert.Equal(t, tests[i].expectedresult, output) + } + + output := generateMapSubset(Labelsdedot, tests[5].key, true, true) + assert.Equal(t, tests[5].expectedresult, output) + + output = generateMapSubset(Labelsdedot, tests[6].key, true, false) + assert.Equal(t, tests[6].expectedresult, output) + +} diff --git a/pkg/autodiscover/kubernetes/metadata/service.go b/pkg/autodiscover/kubernetes/metadata/service.go new file mode 100644 index 000000000000..be93ae7554d6 --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/service.go @@ -0,0 +1,104 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + k8s "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/elastic-agent-libs/safemapstr" +) + +type service struct { + store cache.Store + resource *Resource +} + +// NewServiceMetadataGenerator creates a metagen for service resources +func NewServiceMetadataGenerator(cfg *config.C, services cache.Store, namespace MetaGen, client k8s.Interface) MetaGen { + return &service{ + resource: NewNamespaceAwareResourceMetadataGenerator(cfg, client, namespace), + store: services, + } +} + +// Generate generates service metadata from a resource object +// Metadata map is in the following form: +// +// { +// "kubernetes": {}, +// "some.ecs.field": "asdf" +// } +// +// All Kubernetes fields that need to be stored under kuberentes. prefix are populetad by +// GenerateK8s method while fields that are part of ECS are generated by GenerateECS method +func (s *service) Generate(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + ecsFields := s.GenerateECS(obj) + meta := mapstr.M{ + "kubernetes": s.GenerateK8s(obj, opts...), + } + meta.DeepUpdate(ecsFields) + return meta +} + +// GenerateECS generates service ECS metadata from a resource object +func (s *service) GenerateECS(obj kubernetes.Resource) mapstr.M { + return s.resource.GenerateECS(obj) +} + +// GenerateK8s generates service metadata from a resource object +func (s *service) GenerateK8s(obj kubernetes.Resource, opts ...FieldOptions) mapstr.M { + svc, ok := obj.(*kubernetes.Service) + if !ok { + return nil + } + + out := s.resource.GenerateK8s("service", obj, opts...) + + selectors := svc.Spec.Selector + if len(selectors) == 0 { + return out + } + svcMap := GenerateMap(selectors, s.resource.config.LabelsDedot) + if len(svcMap) != 0 { + _ = safemapstr.Put(out, "selectors", svcMap) + } + + return out +} + +// GenerateFromName generates pod metadata from a service name +func (s *service) GenerateFromName(name string, opts ...FieldOptions) mapstr.M { + if s.store == nil { + return nil + } + + if obj, ok, _ := s.store.GetByKey(name); ok { + svc, ok := obj.(*kubernetes.Service) + if !ok { + return nil + } + + return s.GenerateK8s(svc, opts...) + } + + return nil +} diff --git a/pkg/autodiscover/kubernetes/metadata/service_test.go b/pkg/autodiscover/kubernetes/metadata/service_test.go new file mode 100644 index 000000000000..63b3556b9d2d --- /dev/null +++ b/pkg/autodiscover/kubernetes/metadata/service_test.go @@ -0,0 +1,325 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package metadata + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + k8sfake "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" + + "github.com/elastic/beats/v7/pkg/autodiscover/kubernetes" + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +func TestService_Generate(t *testing.T) { + client := k8sfake.NewSimpleClientset() + boolean := true + tests := []struct { + input kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object", + input: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + Spec: v1.ServiceSpec{ + Selector: map[string]string{ + "app": "istiod", + "istio": "pilot", + }, + }, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "service": mapstr.M{ + "name": name, + "uid": uid, + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "selectors": mapstr.M{ + "app": "istiod", + "istio": "pilot", + }, + "namespace": defaultNs, + }, + }, + }, + { + name: "test object with owner reference", + input: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "owner", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + Spec: v1.ServiceSpec{ + Selector: map[string]string{ + "app": "istiod", + "istio": "pilot", + }, + }, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "service": mapstr.M{ + "name": name, + "uid": uid, + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "selectors": mapstr.M{ + "app": "istiod", + "istio": "pilot", + }, + "namespace": defaultNs, + "deployment": mapstr.M{ + "name": "owner", + }, + }, + }, + }, + } + + cfg := config.NewConfig() + metagen := NewServiceMetadataGenerator(cfg, nil, nil, client) + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.Generate(test.input)) + }) + } +} + +func TestService_GenerateFromName(t *testing.T) { + client := k8sfake.NewSimpleClientset() + boolean := true + tests := []struct { + input kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object", + input: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + }, + output: mapstr.M{ + "service": mapstr.M{ + "name": name, + "uid": uid, + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "namespace": defaultNs, + }, + }, + { + name: "test object with owner reference", + input: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "apps", + Kind: "Deployment", + Name: "owner", + UID: "005f3b90-4b9d-12f8-acf0-31020a840144", + Controller: &boolean, + }, + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + }, + output: mapstr.M{ + "service": mapstr.M{ + "name": name, + "uid": uid, + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "namespace": defaultNs, + "deployment": mapstr.M{ + "name": "owner", + }, + }, + }, + } + + for _, test := range tests { + cfg := config.NewConfig() + services := cache.NewStore(cache.MetaNamespaceKeyFunc) + err := services.Add(test.input) + require.NoError(t, err) + metagen := NewServiceMetadataGenerator(cfg, services, nil, client) + + accessor, err := meta.Accessor(test.input) + require.NoError(t, err) + + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.GenerateFromName(fmt.Sprint(accessor.GetNamespace(), "/", accessor.GetName()))) + }) + } +} + +func TestService_GenerateWithNamespace(t *testing.T) { + client := k8sfake.NewSimpleClientset() + tests := []struct { + input kubernetes.Resource + namespace kubernetes.Resource + output mapstr.M + name string + }{ + { + name: "test simple object", + input: &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + UID: types.UID(uid), + Namespace: defaultNs, + Labels: map[string]string{ + "foo": "bar", + }, + Annotations: map[string]string{}, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + }, + namespace: &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultNs, + UID: types.UID(uid), + Labels: map[string]string{ + "nskey": "nsvalue", + }, + Annotations: map[string]string{ + "ns.annotation": "value", + }, + }, + TypeMeta: metav1.TypeMeta{ + Kind: "Namespace", + APIVersion: "v1", + }, + }, + output: mapstr.M{ + "kubernetes": mapstr.M{ + "service": mapstr.M{ + "name": name, + "uid": uid, + }, + "labels": mapstr.M{ + "foo": "bar", + }, + "namespace": defaultNs, + "namespace_uid": uid, + "namespace_labels": mapstr.M{ + "nskey": "nsvalue", + }, + "namespace_annotations": mapstr.M{ + "ns_annotation": "value", + }, + }, + }, + }, + } + + for _, test := range tests { + nsConfig, err := config.NewConfigFrom(map[string]any{ + "include_annotations": []string{"ns.annotation"}, + }) + require.NoError(t, err) + services := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = services.Add(test.input) + require.NoError(t, err) + + namespaces := cache.NewStore(cache.MetaNamespaceKeyFunc) + err = namespaces.Add(test.namespace) + require.NoError(t, err) + nsMeta := NewNamespaceMetadataGenerator(nsConfig, namespaces, client) + + metagen := NewServiceMetadataGenerator(nsConfig, services, nsMeta, client) + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.output, metagen.Generate(test.input)) + }) + } +} diff --git a/pkg/autodiscover/kubernetes/options.go b/pkg/autodiscover/kubernetes/options.go new file mode 100644 index 000000000000..63850f3f1cca --- /dev/null +++ b/pkg/autodiscover/kubernetes/options.go @@ -0,0 +1,23 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package kubernetes + +type KubeClientOptions struct { + QPS float32 `config:"qps"` + Burst int `config:"burst"` +} diff --git a/pkg/autodiscover/kubernetes/types.go b/pkg/autodiscover/kubernetes/types.go new file mode 100644 index 000000000000..200a2f7b1505 --- /dev/null +++ b/pkg/autodiscover/kubernetes/types.go @@ -0,0 +1,154 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package kubernetes + +import ( + "strings" + "time" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + v1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + rbacv1 "k8s.io/api/rbac/v1" + storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// Resource data +type Resource = runtime.Object + +// ObjectMeta data +type ObjectMeta = metav1.ObjectMeta + +// Pod data +type Pod = v1.Pod + +// PodSpec data +type PodSpec = v1.PodSpec + +// PodStatus data +type PodStatus = v1.PodStatus + +// Node data +type Node = v1.Node + +// Namespace data +type Namespace = v1.Namespace + +// Container data +type Container = v1.Container + +// ContainerPort data +type ContainerPort = v1.ContainerPort + +// Event data +type Event = v1.Event + +// PodContainerStatus data +type PodContainerStatus = v1.ContainerStatus + +// Deployment data +type Deployment = appsv1.Deployment + +// ReplicaSet data +type ReplicaSet = appsv1.ReplicaSet + +// StatefulSet data +type StatefulSet = appsv1.StatefulSet + +// DaemonSet data +type DaemonSet = appsv1.DaemonSet + +// Service data +type Service = v1.Service + +// ServiceAccount data +type ServiceAccount = v1.ServiceAccount + +// Job data +type Job = batchv1.Job + +// CronJob data +type CronJob = batchv1.CronJob + +// PersistentVolume data +type PersistentVolume = v1.PersistentVolume + +// PersistentVolumeClaim data +type PersistentVolumeClaim = v1.PersistentVolumeClaim + +// StorageClass data +type StorageClass = storagev1.StorageClass + +// Role data +type Role = rbacv1.Role + +// RoleBinding data +type RoleBinding = rbacv1.RoleBinding + +// ClusterRole data +type ClusterRole = rbacv1.ClusterRole + +// ClusterRoleBinding data +type ClusterRoleBinding = rbacv1.ClusterRoleBinding + +// NetworkPolicy data +type NetworkPolicy = networkingv1.NetworkPolicy + +const ( + // PodPending phase + PodPending = v1.PodPending + // PodRunning phase + PodRunning = v1.PodRunning + // PodSucceeded phase + PodSucceeded = v1.PodSucceeded + // PodFailed phase + PodFailed = v1.PodFailed + // PodUnknown phase + PodUnknown = v1.PodUnknown +) + +// Time extracts time from k8s.Time type +func Time(t *metav1.Time) time.Time { + return t.Time +} + +// MicroTime extracts time from k8s.MicroTime type +func MicroTime(t *metav1.MicroTime) time.Time { + return t.Time +} + +// ContainerID parses the container ID to get the actual ID string +func ContainerID(s PodContainerStatus) string { + cID, _ := ContainerIDWithRuntime(s) + return cID +} + +// ContainerIDWithRuntime parses the container ID to get the actual ID string +func ContainerIDWithRuntime(s PodContainerStatus) (string, string) { + cID := s.ContainerID + if cID != "" { + parts := strings.Split(cID, "://") + if len(parts) == 2 { + return parts[1], parts[0] + } + } + return "", "" +} diff --git a/pkg/autodiscover/kubernetes/util.go b/pkg/autodiscover/kubernetes/util.go new file mode 100644 index 000000000000..73e9b33c9ac2 --- /dev/null +++ b/pkg/autodiscover/kubernetes/util.go @@ -0,0 +1,374 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package kubernetes + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + + "k8s.io/client-go/metadata" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/elastic/elastic-agent-libs/safemapstr" +) + +type HostDiscoveryUtils interface { + GetNamespace() (string, error) + GetPodName() (string, error) + GetMachineID() string +} + +// DiscoverKubernetesNodeParams includes parameters for discovering kubernetes node +type DiscoverKubernetesNodeParams struct { + ConfigHost string + Client kubernetes.Interface + IsInCluster bool + HostUtils HostDiscoveryUtils +} + +// DefaultDiscoveryUtils implements functions of HostDiscoveryUtils interface +type DefaultDiscoveryUtils struct{} + +func GetKubeConfigEnvironmentVariable() string { + envKubeConfig := os.Getenv("KUBECONFIG") + if envKubeConfig == "" { + return "" + } + if _, err := os.Stat(envKubeConfig); !os.IsNotExist(err) { //nolint:gosec // KUBECONFIG is a standard environment variable for kubeconfig path + return envKubeConfig + } + return "" +} + +// GetKubernetesClient returns a kubernetes client. If inCluster is true, it returns an +// in cluster configuration based on the secrets mounted in the Pod. If kubeConfig is passed, +// it parses the config file to get the config required to build a client. +func GetKubernetesClient(kubeconfig string, opt KubeClientOptions) (kubernetes.Interface, error) { + if kubeconfig == "" { + kubeconfig = GetKubeConfigEnvironmentVariable() + } + + cfg, err := BuildConfig(kubeconfig) + if err != nil { + return nil, fmt.Errorf("unable to build kube config due to error: %w", err) + } + cfg.QPS = opt.QPS + cfg.Burst = opt.Burst + client, err := kubernetes.NewForConfig(cfg) + if err != nil { + return nil, fmt.Errorf("unable to build kubernetes clientset: %w", err) + } + + return client, nil +} + +// GetKubernetesMetadataClient returns a kubernetes metadata-only client. If inCluster is true, it returns an +// in cluster configuration based on the secrets mounted in the Pod. If kubeConfig is passed, +// it parses the config file to get the config required to build a client. +func GetKubernetesMetadataClient(kubeconfig string, opt KubeClientOptions) (metadata.Interface, error) { + if kubeconfig == "" { + kubeconfig = GetKubeConfigEnvironmentVariable() + } + + cfg, err := BuildConfig(kubeconfig) + if err != nil { + return nil, fmt.Errorf("unable to build kube config due to error: %w", err) + } + cfg.QPS = opt.QPS + cfg.Burst = opt.Burst + client, err := metadata.NewForConfig(cfg) + if err != nil { + return nil, fmt.Errorf("unable to build kubernetes clientset: %w", err) + } + + return client, nil +} + +// BuildConfig is a helper function that builds configs from a kubeconfig filepath. +// If kubeconfigPath is not passed in we fallback to inClusterConfig. +// If inClusterConfig fails, we fallback to the default config. +// This is a copy of `clientcmd.BuildConfigFromFlags` of `client-go` but without the annoying +// klog messages that are not possible to be disabled. +func BuildConfig(kubeconfigPath string) (*restclient.Config, error) { + if kubeconfigPath == "" { + kubeconfig, err := restclient.InClusterConfig() + if err == nil { + return kubeconfig, nil + } + } + return clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ExplicitPath: kubeconfigPath}, + &clientcmd.ConfigOverrides{ClusterInfo: clientcmdapi.Cluster{Server: ""}}).ClientConfig() +} + +// IsInCluster takes a kubeconfig file path as input and deduces if Beats is running in cluster or not, +// taking into consideration the existence of KUBECONFIG variable +func IsInCluster(kubeconfig string) bool { + if kubeconfig != "" || GetKubeConfigEnvironmentVariable() != "" { + return false + } + return true +} + +// DiscoverKubernetesNode figures out the Kubernetes node to use. +// If host is provided in the config use it directly. +// If it is empty then try +// 1. If beat is deployed in k8s cluster, use hostname of pod as the pod name to query pod metadata for node name. +// 2. If step 1 fails or beat is deployed outside k8s cluster, use NODE_NAME env var. +// 3. If node cannot be discovered with step 1,2, use machine-id to match against k8s nodes for node name. In case it is not set return error. +// Note: There have been cases where machine-id reported by compute instances of some cloud providers where k8s nodes run on, has the wrong value. +func DiscoverKubernetesNode(log *logp.Logger, nd *DiscoverKubernetesNodeParams) (string, error) { + ctx := context.TODO() + // Discover node by configuration file (NODE) if set + if nd.ConfigHost != "" { + log.Infof("kubernetes: Using node %s provided in the config", nd.ConfigHost) + return nd.ConfigHost, nil + } + // Discover node by serviceaccount namespace and pod's hostname in case Beats is running in cluster + if nd.IsInCluster { + node, err := discoverInCluster(nd, ctx) + if err == nil { + log.Infof("kubernetes: Node %s discovered by in cluster pod node query", node) + return node, nil + } + log.Debug(err) + } + + // use environment variable NODE_NAME + node := os.Getenv("NODE_NAME") + if node != "" { + log.Infof("kubernetes: Node %s discovered by NODE_NAME environment variable", node) + return node, nil + } + log.Debug(errors.New("NODE_NAME environment variable is not set")) + + // try discover node by machine id + node, err := discoverByMachineID(nd, ctx) + if err == nil { + log.Infof("kubernetes: Node %s discovered by machine-id matching", node) + return node, nil + } + log.Debug(err) + + return "", errors.New("kubernetes: Node could not be discovered with any known method. Consider setting env var NODE_NAME") +} + +func discoverInCluster(nd *DiscoverKubernetesNodeParams, ctx context.Context) (node string, errorMsg error) { + ns, err := nd.HostUtils.GetNamespace() + if err != nil { + errorMsg = fmt.Errorf("kubernetes: Couldn't get namespace when beat is in cluster with error: %w", err) + return + } + podName, err := nd.HostUtils.GetPodName() + if err != nil { + errorMsg = fmt.Errorf("kubernetes: Couldn't get hostname as beat pod name in cluster with error: %w", err) + return + } + pod, err := nd.Client.CoreV1().Pods(ns).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + errorMsg = fmt.Errorf("kubernetes: Querying for pod failed with error: %w", err) + return + } + return pod.Spec.NodeName, nil +} + +func discoverByMachineID(nd *DiscoverKubernetesNodeParams, ctx context.Context) (nodeName string, errorMsg error) { + mid := nd.HostUtils.GetMachineID() + if mid == "" { + errorMsg = errors.New("kubernetes: Couldn't collect info from any of the files in /etc/machine-id /var/lib/dbus/machine-id") + return + } + + nodes, err := nd.Client.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + errorMsg = fmt.Errorf("kubernetes: Querying for nodes failed with error: %w", err) + return + } + for _, n := range nodes.Items { + if n.Status.NodeInfo.MachineID == mid { + nodeName = n.GetObjectMeta().GetName() + return nodeName, nil + } + } + errorMsg = fmt.Errorf("kubernetes: Couldn't discover node %s", mid) + return nodeName, errorMsg +} + +// GetMachineID returns the machine-idadd_kubernetes_metadata/indexers_test.go +// borrowed from machineID of cadvisor. +func (hd *DefaultDiscoveryUtils) GetMachineID() string { + for _, file := range []string{ + "/etc/machine-id", + "/var/lib/dbus/machine-id", + } { + id, err := os.ReadFile(file) + if err == nil { + return strings.TrimSpace(string(id)) + } + } + return "" +} + +// GetNamespace gets namespace from serviceaccount when beat is in cluster. +func (hd *DefaultDiscoveryUtils) GetNamespace() (string, error) { + return InClusterNamespace() +} + +// GetPodName returns the hostname of the pod +func (hd *DefaultDiscoveryUtils) GetPodName() (string, error) { + return os.Hostname() +} + +// InClusterNamespace gets namespace from serviceaccount when beat is in cluster. // code borrowed from client-go with some changes. +func InClusterNamespace() (string, error) { + // get namespace associated with the service account token, if available + data, err := os.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace") + if err != nil { + return "", err + } + return strings.TrimSpace(string(data)), nil +} + +type ContainerInPod struct { + ID string + Runtime string + Spec Container + Status PodContainerStatus +} + +// GetContainersInPod returns all the containers defined in a pod and their statuses. +// It includes init and ephemeral containers. +func GetContainersInPod(pod *Pod) []*ContainerInPod { + containers := make([]*ContainerInPod, len(pod.Spec.Containers)+len(pod.Spec.InitContainers)+len(pod.Spec.EphemeralContainers)) + idx := 0 + for _, c := range pod.Spec.Containers { + containers[idx] = &ContainerInPod{Spec: c} + idx++ + } + for _, c := range pod.Spec.InitContainers { + containers[idx] = &ContainerInPod{Spec: c} + idx++ + } + for _, c := range pod.Spec.EphemeralContainers { + c := Container(c.EphemeralContainerCommon) + containers[idx] = &ContainerInPod{Spec: c} + idx++ + } + + statuses := make(map[string]*PodContainerStatus) + mapStatuses := func(s []PodContainerStatus) { + for i := range s { + statuses[s[i].Name] = &s[i] + } + } + mapStatuses(pod.Status.ContainerStatuses) + mapStatuses(pod.Status.InitContainerStatuses) + mapStatuses(pod.Status.EphemeralContainerStatuses) + for _, c := range containers { + if s, ok := statuses[c.Spec.Name]; ok { + c.ID, c.Runtime = ContainerIDWithRuntime(*s) + c.Status = *s + } + } + + return containers +} + +// PodLabels returns the labels in a pod +func PodLabels(pod *Pod) mapstr.M { + labels := mapstr.M{} + for k, v := range pod.GetObjectMeta().GetLabels() { + _ = safemapstr.Put(labels, k, v) + } + return labels +} + +// PodAnnotations returns the annotations in a pod +func PodAnnotations(pod *Pod) mapstr.M { + annotations := mapstr.M{} + for k, v := range pod.GetObjectMeta().GetAnnotations() { + _ = safemapstr.Put(annotations, k, v) + } + return annotations +} + +// PodNamespaceAnnotations returns the annotations of the namespace of the pod +func PodNamespaceAnnotations(pod *Pod, watcher Watcher) mapstr.M { + if watcher == nil { + return nil + } + + rawNs, ok, err := watcher.Store().GetByKey(pod.Namespace) + if !ok || err != nil { + return nil + } + + namespace, ok := rawNs.(*Namespace) + if !ok { + return nil + } + + annotations := mapstr.M{} + for k, v := range namespace.GetAnnotations() { + _ = safemapstr.Put(annotations, k, v) + } + return annotations +} + +// PodTerminating returns true if a pod is marked for deletion or is in a phase beyond running. +func PodTerminating(pod *Pod) bool { + if pod.GetObjectMeta().GetDeletionTimestamp() != nil { + return true + } + + switch pod.Status.Phase { + case PodRunning, PodPending: + default: + return true + } + + return false +} + +// PodTerminated returns true if a pod is terminated, this method considers a +// pod as terminated if none of its containers are running (or going to be running). +func PodTerminated(pod *Pod, containers []*ContainerInPod) bool { + // Pod is not marked for termination, so it is not terminated. + if !PodTerminating(pod) { + return false + } + + // If any container is running, the pod is not terminated yet. + for _, container := range containers { + if container.Status.State.Running != nil { + return false + } + } + + return true +} diff --git a/pkg/autodiscover/kubernetes/util_test.go b/pkg/autodiscover/kubernetes/util_test.go new file mode 100644 index 000000000000..e6f1d905e0b9 --- /dev/null +++ b/pkg/autodiscover/kubernetes/util_test.go @@ -0,0 +1,334 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package kubernetes + +import ( + "context" + "errors" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" + k8sfake "k8s.io/client-go/kubernetes/fake" + + "github.com/elastic/elastic-agent-libs/logp/logptest" + + core "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +func TestDiscoverKubernetesNode(t *testing.T) { + client := k8sfake.NewSimpleClientset() + logger := logptest.NewTestingLogger(t, "autodiscover.node") + ge := errors.New("kubernetes: Node could not be discovered with any known method. Consider setting env var NODE_NAME") + tests := []struct { + host string + node string + err error + name string + setEnv bool + isInCluster bool + machineid string + podname string + namespace string + init func(*testing.T, kubernetes.Interface) + }{ + { + name: "test value from config", + host: "worker-1", + node: "worker-1", + err: nil, + setEnv: false, + isInCluster: true, + machineid: "", + podname: "", + namespace: "", + }, + { + name: "test value with env var", + host: "", + node: "worker-2", + err: nil, + setEnv: true, + isInCluster: false, + machineid: "", + podname: "", + namespace: "", + }, + { + name: "test value with not incluster, machine ID not retrieved, env var not set", + host: "", + node: "", + err: ge, + setEnv: false, + isInCluster: false, + machineid: "", + podname: "", + namespace: "", + }, + { + name: "test value with inCluster , serviceaccount namespace not found and env var not set", + host: "", + node: "", + err: ge, + setEnv: false, + isInCluster: true, + machineid: "", + podname: "", + namespace: "none", + }, + { + name: "test value with inCluster, pod not found and env var not set", + host: "", + isInCluster: true, + node: "", + err: ge, + setEnv: false, + machineid: "", + podname: "test-pod", + namespace: "default", + }, + { + name: "test value with inCluster, pod found and env var not set", + host: "", + isInCluster: true, + node: "test-node", + err: nil, + setEnv: false, + machineid: "", + podname: "test-pod", + namespace: "default", + init: createResources, + }, + { + name: "test value with inCluster, pod found and env var set", + host: "", + isInCluster: true, + node: "test-node", + err: nil, + setEnv: true, + machineid: "", + podname: "test-pod", + namespace: "default", + init: createResources, + }, + { + name: "test value without inCluster, machine-ID empty and env var not set", + host: "", + isInCluster: false, + node: "", + err: ge, + setEnv: false, + machineid: "", + podname: "", + namespace: "", + }, + { + name: "test value without inCluster, machine-ID set, node not found and env var not set", + host: "", + isInCluster: false, + node: "", + err: ge, + setEnv: false, + machineid: "worker-2", + podname: "", + namespace: "", + }, + { + name: "test value without inCluster, machine-ID set, node found and env var not set", + host: "", + isInCluster: false, + node: "worker-2", + err: nil, + setEnv: false, + machineid: "worker-2", + podname: "", + namespace: "", + init: createResources, + }, + { + name: "test value without inCluster, machine-ID set, node not found and env var set", + host: "", + isInCluster: false, + node: "worker-2", + err: nil, + setEnv: true, + machineid: "worker-2", + podname: "", + namespace: "", + }, + { + name: "test value without inCluster, machine-ID set, node not found and env var set: priority given to the env var as more accurate", + host: "", + isInCluster: false, + node: "worker-2", + err: nil, + setEnv: true, + machineid: "worker-1", + podname: "", + namespace: "", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + + if test.setEnv { + + if err := os.Setenv("NODE_NAME", "worker-2"); err != nil { + t.Fatal(err) + } + defer func() { + if err := os.Unsetenv("NODE_NAME"); err != nil { + t.Fatal(err) + } + }() + } + mdu := createMockdu(test.namespace, test.podname, test.machineid) + if test.init != nil { + test.init(t, client) + } + + var nodeName string + var error error + nd := &DiscoverKubernetesNodeParams{ConfigHost: test.host, Client: client, IsInCluster: test.isInCluster, HostUtils: mdu} + nodeName, error = DiscoverKubernetesNode(logger, nd) + + assert.Equal(t, test.node, nodeName) + if error != nil { + assert.Equal(t, test.err.Error(), error.Error()) + } else { + assert.Equal(t, test.err, error) + } + + }) + } +} + +func createPod(client kubernetes.Interface) error { + pod := getPodObject() + + _, err := client.CoreV1().Pods(pod.Namespace).Create(context.Background(), pod, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create k8s pod: %w", err) + } + return nil +} + +func createResources(t *testing.T, client kubernetes.Interface) { + err := createPod(client) + if err != nil { + t.Fatal(err) + } + + err = createNode(client, "worker-2") + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { + pod := "test-pod" + err := client.CoreV1().Pods("default").Delete(context.Background(), pod, metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("failed to delete k8s pod: %+v", err) + } + + err = client.CoreV1().Nodes().Delete(context.Background(), "worker-2", metav1.DeleteOptions{}) + if err != nil { + t.Fatalf("failed to delete k8s node: %+v", err) + } + + }) +} + +func createNode(client kubernetes.Interface, name string) error { + node := getNodeObject(name) + + _, err := client.CoreV1().Nodes().Create(context.Background(), node, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create k8s node: %w", err) + } + return nil +} + +func getPodObject() *core.Pod { + return &core.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Labels: map[string]string{ + "app": "demo", + }, + }, + Spec: core.PodSpec{ + NodeName: "test-node", + Containers: []core.Container{ + { + Name: "busybox", + Image: "busybox", + ImagePullPolicy: core.PullIfNotPresent, + Command: []string{ + "sleep", + "3600", + }, + }, + }, + }, + } +} + +func getNodeObject(name string) *core.Node { + return &core.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{ + "name": name, + }, + }, + Spec: core.NodeSpec{}, + Status: core.NodeStatus{NodeInfo: core.NodeSystemInfo{MachineID: name}}, + } +} + +func createMockdu(namespace, podname, machineid string) *mockDiscoveryUtils { + return &mockDiscoveryUtils{namespace: namespace, podname: podname, machineid: machineid} +} + +type mockDiscoveryUtils struct { + namespace string + podname string + machineid string +} + +func (hd *mockDiscoveryUtils) GetMachineID() string { + return hd.machineid +} + +func (hd *mockDiscoveryUtils) GetNamespace() (string, error) { + var error error + if hd.namespace == "none" { + error = errors.New("open /var/run/secrets/kubernetes.io/serviceaccount/namespace: no such file or directory") + } + return hd.namespace, error +} + +func (hd *mockDiscoveryUtils) GetPodName() (string, error) { + return hd.podname, nil +} diff --git a/pkg/autodiscover/kubernetes/watcher.go b/pkg/autodiscover/kubernetes/watcher.go new file mode 100644 index 000000000000..75f2bc489a07 --- /dev/null +++ b/pkg/autodiscover/kubernetes/watcher.go @@ -0,0 +1,375 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package kubernetes + +import ( + "context" + "fmt" + "time" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/metadata" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + + "github.com/elastic/elastic-agent-libs/logp" +) + +const ( + add = "add" + update = "update" + delete = "delete" +) + +var ( + accessor = meta.NewAccessor() +) + +// Watcher watches Kubernetes resources events +type Watcher interface { + // Start watching Kubernetes API for new events after resources were listed + Start() error + + // Stop watching Kubernetes API for new events + Stop() + + // AddEventHandler add event handlers for corresponding event type watched + AddEventHandler(ResourceEventHandler) + + // GetEventHandler returns the event handlers for corresponding event type watched + GetEventHandler() ResourceEventHandler + + // Store returns the store object for the watcher + Store() cache.Store + + // Client returns the kubernetes client object used by the watcher + Client() kubernetes.Interface + + // CachedObject returns the old object before change during the last updated event + CachedObject() runtime.Object +} + +// WatchOptions controls watch behaviors +type WatchOptions struct { + // SyncTimeout is a timeout for listing historical resources + SyncTimeout time.Duration + // Node is used for filtering watched resource to given node, use "" for all nodes + Node string + // Namespace is used for filtering watched resource to given namespace, use "" for all namespaces + Namespace string + // IsUpdated allows registering a func that allows the invoker of the Watch to decide what amounts to an update + // vs what does not. + IsUpdated func(old, new any) bool + // HonorReSyncs allows resync events to be requeued on the worker + HonorReSyncs bool +} + +type item struct { + object any + objectRaw any + state string +} + +type watcher struct { + client kubernetes.Interface + informer cache.SharedInformer + store cache.Store + queue workqueue.Interface //nolint:staticcheck // client-go workqueue API + ctx context.Context + stop context.CancelFunc + handler ResourceEventHandler + logger *logp.Logger + cachedObject runtime.Object +} + +// NewWatcher initializes the watcher client to provide a events handler for +// resource from the cluster (filtered to the given node) +// Note: This watcher won't emit workqueue metrics. Use NewNamedWatcher to provide an explicit queue name. +func NewWatcher(client kubernetes.Interface, resource Resource, opts WatchOptions, indexers cache.Indexers, logger *logp.Logger) (Watcher, error) { + return NewNamedWatcher("", client, resource, opts, indexers, logger) +} + +// NewNamedWatcher initializes the watcher client to provide an events handler for +// resource from the cluster (filtered to the given node) and also allows to name the k8s +// client's workqueue that is used by the watcher. Workqueue name is important for exposing workqueue +// metrics, if it is empty, its metrics will not be logged by the k8s client. +func NewNamedWatcher(name string, client kubernetes.Interface, resource Resource, opts WatchOptions, indexers cache.Indexers, logger *logp.Logger) (Watcher, error) { + informer, _, err := NewInformer(client, resource, opts, indexers) + if err != nil { + return nil, err + } + return NewNamedWatcherWithInformer(name, client, resource, informer, logger, opts) +} + +// NewNamedWatcherWithInformer initializes the watcher client to provide an events handler for +// resource from the cluster (filtered to the given node) and also allows to name the k8s +// client's workqueue that is used by the watcher. Workqueue name is important for exposing workqueue +// metrics, if it is empty, its metrics will not be logged by the k8s client. +// This function requires the underlying informer to be passed by the caller. +func NewNamedWatcherWithInformer( + name string, + client kubernetes.Interface, + resource Resource, + informer cache.SharedInformer, + logger *logp.Logger, + opts WatchOptions, +) (Watcher, error) { + var store cache.Store + var queue workqueue.Interface //nolint:staticcheck // client-go workqueue API + var cachedObject runtime.Object + + store = informer.GetStore() + queue = workqueue.NewNamed(name) + + if opts.IsUpdated == nil { + opts.IsUpdated = func(o, n any) bool { + oldObj, ok := o.(runtime.Object) + if !ok { + return true + } + newObj, ok := n.(runtime.Object) + if !ok { + return true + } + old, _ := accessor.ResourceVersion(oldObj) + new, _ := accessor.ResourceVersion(newObj) + + // Only enqueue changes that have a different resource versions to avoid processing resyncs. + return old != new + } + } + + ctx, cancel := context.WithCancel(context.TODO()) + w := &watcher{ + client: client, + informer: informer, + store: store, + queue: queue, + ctx: ctx, + cachedObject: cachedObject, + stop: cancel, + logger: logger.Named("kubernetes"), + handler: NoOpEventHandlerFuncs{}, + } + + _, err := w.informer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: func(o any) { + w.enqueue(o, add) + }, + DeleteFunc: func(o any) { + w.enqueue(o, delete) + }, + UpdateFunc: func(o, n any) { + if opts.IsUpdated(o, n) { + w.enqueue(n, update) + } else if opts.HonorReSyncs { + // HonorReSyncs ensure that at the time when the kubernetes client does a "resync", i.e, a full list of all + // objects we make sure that autodiscover processes them. Why is this necessary? An effective control loop works + // based on two state changes, a list and a watch. A watch is triggered each time the state of the system changes. + // However, there is no guarantee that all events from a watch are processed by the receiver. To ensure that missed events + // are properly handled, a period re-list is done to ensure that every state within the system is effectively handled. + // In this case, we are making sure that we are enqueueing an "add" event because, an runner that is already in Running + // state should just be deduped by autodiscover and not stop/started periodically as would be the case with an update. + w.enqueue(n, add) + } + + //We check the type of resource and only if it is namespace or node return the cacheObject + switch resource.(type) { + case *Namespace: + w.cacheObject(o) + case *Node: + w.cacheObject(o) + } + }, + }) + if err != nil { + return nil, err + } + + return w, nil +} + +// NewMetadataWatcher initializes a metadata-only watcher client to provide an events handler for +// resource from the cluster (filtered to the given node). +// Event handlers defined on this watcher receive PartialObjectMetadata resources. +// Note: This watcher won't emit workqueue metrics. Use NewNamedWatcher to provide an explicit queue name. +func NewMetadataWatcher( + client kubernetes.Interface, + metadataClient metadata.Interface, + gvr schema.GroupVersionResource, + opts WatchOptions, + indexers cache.Indexers, + transformFunc cache.TransformFunc, + logger *logp.Logger, +) (Watcher, error) { + return NewNamedMetadataWatcher("", client, metadataClient, gvr, opts, indexers, transformFunc, logger) +} + +// NewNamedMetadataWatcher initializes a metadata-only watcher client to provide an events handler for +// resource from the cluster (filtered to the given node) and also allows to name the k8s +// client's workqueue that is used by the watcher. Workqueue name is important for exposing workqueue +// metrics, if it is empty, its metrics will not be logged by the k8s client. +// Event handlers defined on this watcher receive PartialObjectMetadata resources. +func NewNamedMetadataWatcher( + name string, + client kubernetes.Interface, + metadataClient metadata.Interface, + gvr schema.GroupVersionResource, + opts WatchOptions, + indexers cache.Indexers, + transformFunc cache.TransformFunc, + logger *logp.Logger, +) (Watcher, error) { + informer := NewMetadataInformer(metadataClient, gvr, opts, indexers) + + if transformFunc != nil { + err := informer.SetTransform(transformFunc) + if err != nil { + return nil, err + } + } + + return NewNamedWatcherWithInformer(name, client, &v1.PartialObjectMetadata{}, informer, logger, opts) +} + +// AddEventHandler adds a resource handler to process each request that is coming into the watcher +func (w *watcher) AddEventHandler(h ResourceEventHandler) { + w.handler = h +} + +// GetEventHandler returns the watcher's event handler +func (w *watcher) GetEventHandler() ResourceEventHandler { + return w.handler +} + +// Store returns the store object for the resource that is being watched +func (w *watcher) Store() cache.Store { + return w.store +} + +// Client returns the kubernetes client object used by the watcher +func (w *watcher) Client() kubernetes.Interface { + return w.client +} + +// CachedObject returns the old object in cache during the last updated event +func (w *watcher) CachedObject() runtime.Object { + return w.cachedObject +} + +// Start watching pods +func (w *watcher) Start() error { + go w.informer.Run(w.ctx.Done()) + + if !cache.WaitForCacheSync(w.ctx.Done(), w.informer.HasSynced) { + return fmt.Errorf("kubernetes informer unable to sync cache") + } + + w.logger.Debugf("cache sync done") + + // Wrap the process function with wait.Until so that if the controller crashes, it starts up again after a second. + go wait.Until(func() { + for w.process(w.ctx) { + } + }, time.Second*1, w.ctx.Done()) + + return nil +} + +func (w *watcher) Stop() { + w.queue.ShutDown() + w.stop() +} + +// enqueue takes the most recent object that was received, figures out the namespace/name of the object +// and adds it to the work queue for processing. +func (w *watcher) enqueue(obj any, state string) { + // DeletionHandlingMetaNamespaceKeyFunc that we get a key only if the resource's state is not Unknown. + key, err := cache.DeletionHandlingMetaNamespaceKeyFunc(obj) + if err != nil { + return + } + if deleted, ok := obj.(cache.DeletedFinalStateUnknown); ok { + w.logger.Debugf("Enqueued DeletedFinalStateUnknown contained object: %+v", deleted.Obj) + obj = deleted.Obj + } + w.queue.Add(&item{key, obj, state}) +} + +// cacheObject updates watcher with the old version of cache objects before change during update events +func (w *watcher) cacheObject(o any) { + if old, ok := o.(runtime.Object); !ok { + utilruntime.HandleError(fmt.Errorf("expected object in cache got %#v", o)) + } else { + w.cachedObject = old + } +} + +// process gets the top of the work queue and processes the object that is received. +func (w *watcher) process(_ context.Context) bool { + obj, quit := w.queue.Get() + if quit { + return false + } + defer w.queue.Done(obj) + + var entry *item + var ok bool + if entry, ok = obj.(*item); !ok { + utilruntime.HandleError(fmt.Errorf("expected *item in workqueue but got %#v", obj)) + return true + } + + key, ok := entry.object.(string) + if !ok { + return false + } + + o, exists, err := w.store.GetByKey(key) + if err != nil { + utilruntime.HandleError(fmt.Errorf("getting object %#v from cache: %w", obj, err)) + return true + } + if !exists { + if entry.state == delete { + w.logger.Debugf("Object %+v was not found in the store, deleting anyway!", key) + // delete anyway in order to clean states + w.handler.OnDelete(entry.objectRaw) + } + return true + } + + switch entry.state { + case add: + w.handler.OnAdd(o) + case update: + w.handler.OnUpdate(o) + case delete: + w.handler.OnDelete(o) + } + + return true +} diff --git a/pkg/autodiscover/kubernetes/watcher_test.go b/pkg/autodiscover/kubernetes/watcher_test.go new file mode 100644 index 000000000000..1edd3caa833d --- /dev/null +++ b/pkg/autodiscover/kubernetes/watcher_test.go @@ -0,0 +1,167 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package kubernetes + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" + cachetest "k8s.io/client-go/tools/cache/testing" + + "github.com/elastic/elastic-agent-libs/logp/logptest" +) + +func TestWatcherStartAndStop(t *testing.T) { + client := fake.NewSimpleClientset() + listWatch := cachetest.NewFakeControllerSource() + resource := &Pod{} + informer := cache.NewSharedInformer(listWatch, resource, 0) + watcher, err := NewNamedWatcherWithInformer("test", client, resource, informer, logptest.NewTestingLogger(t, ""), WatchOptions{}) + require.NoError(t, err) + require.NoError(t, watcher.Start()) + watcher.Stop() +} + +func TestWatcherHandlers(t *testing.T) { + client := fake.NewSimpleClientset() + listWatch := cachetest.NewFakeControllerSource() + resource := &Pod{} + informer := cache.NewSharedInformer(listWatch, resource, 0) + watcher, err := NewNamedWatcherWithInformer("test", client, resource, informer, logptest.NewTestingLogger(t, ""), WatchOptions{}) + require.NoError(t, err) + + var added, updated, deleted bool + + watcher.AddEventHandler(ResourceEventHandlerFuncs{ + AddFunc: func(obj any) { + added = true + }, + UpdateFunc: func(obj any) { + updated = true + }, + DeleteFunc: func(obj any) { + deleted = true + }, + }) + + require.NoError(t, watcher.Start()) + defer watcher.Stop() + + pod := &Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + UID: types.UID("poduid"), + Namespace: "test", + ResourceVersion: "1", + }, + } + // add a resource + listWatch.Add(pod) + assert.Eventually(t, func() bool { + return added + }, time.Second*5, time.Millisecond) + + // update the resource + modifiedPod := pod.DeepCopy() + modifiedPod.SetResourceVersion("2") + listWatch.Modify(modifiedPod) + assert.Eventually(t, func() bool { + return updated + }, time.Second*5, time.Millisecond) + + // delete the resource + listWatch.Delete(modifiedPod) + assert.Eventually(t, func() bool { + return deleted + }, time.Second*5, time.Millisecond) +} + +func TestWatcherIsUpdated(t *testing.T) { + client := fake.NewSimpleClientset() + listWatch := cachetest.NewFakeControllerSource() + resource := &Pod{} + informer := cache.NewSharedInformer(listWatch, resource, 0) + // set a custom IsUpdated that always returns true + watcher, err := NewNamedWatcherWithInformer("test", client, resource, informer, + logptest.NewTestingLogger(t, ""), + WatchOptions{IsUpdated: func(old, new any) bool { + return true + }}) + require.NoError(t, err) + + var updated bool + + watcher.AddEventHandler(ResourceEventHandlerFuncs{ + UpdateFunc: func(obj any) { + updated = true + }, + }) + + require.NoError(t, watcher.Start()) + defer watcher.Stop() + + pod := &Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + UID: types.UID("poduid"), + Namespace: "test", + }, + } + listWatch.Add(pod) + + // update the resource, but don't actually change it + // with the default IsUpdated, our handler wouldn't be called, but with our custom one, it will + modifiedPod := pod.DeepCopy() + listWatch.Modify(modifiedPod) + assert.Eventually(t, func() bool { + return updated + }, time.Second*5, time.Millisecond) + +} + +func TestCachedObject(t *testing.T) { + t.Skip("Currently bugged, and not used anywhere") + client := fake.NewSimpleClientset() + listWatch := cachetest.NewFakeControllerSource() + resource := &Namespace{} + informer := cache.NewSharedInformer(listWatch, resource, 0) + watcher, err := NewNamedWatcherWithInformer("test", client, resource, informer, logptest.NewTestingLogger(t, ""), WatchOptions{}) + require.NoError(t, err) + + require.NoError(t, watcher.Start()) + defer watcher.Stop() + + namespace := &Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + UID: types.UID("poduid"), + Namespace: "test", + ResourceVersion: "1", + }, + } + listWatch.Add(namespace) + assert.EventuallyWithT(t, func(collectT *assert.CollectT) { + assert.Equal(collectT, namespace, watcher.CachedObject()) + }, time.Second*5, time.Millisecond) +} diff --git a/pkg/autodiscover/utils/hints.go b/pkg/autodiscover/utils/hints.go new file mode 100644 index 000000000000..5dbba4696ee5 --- /dev/null +++ b/pkg/autodiscover/utils/hints.go @@ -0,0 +1,423 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import ( + "encoding/json" + "fmt" + "regexp" + "slices" + "sort" + "strconv" + "strings" + + "github.com/elastic/elastic-agent-libs/logp" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +const logName = "hints" + +// GetContainerID returns the id of a container +func GetContainerID(container mapstr.M) string { + id, _ := container["id"].(string) + return id +} + +// GetContainerName returns the name of a container +func GetContainerName(container mapstr.M) string { + name, _ := container["name"].(string) + return name +} + +// GetHintString takes a hint and returns its value as a string +func GetHintString(hints mapstr.M, key, config string) string { + base := config + if base == "" { + base = key + } else if key != "" { + base = fmt.Sprint(key, ".", config) + } + if iface, err := hints.GetValue(base); err == nil { + if str, ok := iface.(string); ok { + return str + } + } + + return "" +} + +// GetHintMapStr takes a hint and returns a MapStr +func GetHintMapStr(hints mapstr.M, key, config string) mapstr.M { + base := config + if base == "" { + base = key + } else if key != "" { + base = fmt.Sprint(key, ".", config) + } + if iface, err := hints.GetValue(base); err == nil { + if mapstr, ok := iface.(mapstr.M); ok { + return mapstr + } + } + + return nil +} + +// GetHintAsList takes a hint and returns the value as lists. +func GetHintAsList(hints mapstr.M, key, config string) []string { + if str := GetHintString(hints, key, config); str != "" { + return getStringAsList(str) + } + + return nil +} + +// GetProcessors gets processor definitions from the hints and returns a list of configs as a MapStr +func GetProcessors(hints mapstr.M, key string, logger *logp.Logger) []mapstr.M { + processors := GetConfigs(hints, key, "processors") + for _, proc := range processors { + for key, value := range proc { + if str, ok := value.(string); ok { + cfg := mapstr.M{} + if err := json.Unmarshal([]byte(str), &cfg); err != nil { + logger.Named(logName).Debugw("Unable to unmarshal json due to error", "error", err) + continue + } + proc[key] = cfg + } + } + } + return processors +} + +// GetConfigs takes in a key and returns a list of configs as a slice of MapStr +func GetConfigs(hints mapstr.M, key, name string) []mapstr.M { + raw := GetHintMapStr(hints, key, name) + if raw == nil { + return nil + } + + var words, nums []string + + for key := range raw { + if _, err := strconv.Atoi(key); err != nil { + words = append(words, key) + continue + } else { + nums = append(nums, key) + } + } + + sort.Strings(nums) + + var configs []mapstr.M + for _, key := range nums { + rawCfg := raw[key] + if config, ok := rawCfg.(mapstr.M); ok { + configs = append(configs, config) + } + } + + for _, word := range words { + configs = append(configs, mapstr.M{ + word: raw[word], + }) + } + + return configs +} + +func getStringAsList(input string) []string { + if input == "" { + return []string{} + } + list := strings.Split(input, ",") + + for i := range list { + list[i] = strings.TrimSpace(list[i]) + } + + return list +} + +// GetHintAsConfigs can read a hint in the form of a stringified JSON and return a mapstr.M +func GetHintAsConfigs(hints mapstr.M, key string, logger *logp.Logger) []mapstr.M { + if str := GetHintString(hints, key, "raw"); str != "" { + // check if it is a single config + if str[0] != '[' { + cfg := mapstr.M{} + if err := json.Unmarshal([]byte(str), &cfg); err != nil { + logger.Named(logName).Debugw("Unable to unmarshal json due to error", "error", err) + return nil + } + return []mapstr.M{cfg} + } + + var cfg []mapstr.M + if err := json.Unmarshal([]byte(str), &cfg); err != nil { + logger.Named(logName).Debugw("Unable to unmarshal json due to error", "error", err) + return nil + } + return cfg + } + return nil +} + +// IsEnabled will return true when 'enabled' is **explicitly** set to true. +func IsEnabled(hints mapstr.M, key string) bool { + if value, err := hints.GetValue(fmt.Sprintf("%s.enabled", key)); err == nil { + str, ok := value.(string) + if !ok { + return false + } + enabled, _ := strconv.ParseBool(str) + return enabled + } + + return false +} + +// IsDisabled will return true when 'enabled' is **explicitly** set to false. +func IsDisabled(hints mapstr.M, key string, logger *logp.Logger) bool { + if value, err := hints.GetValue(fmt.Sprintf("%s.enabled", key)); err == nil { + str, ok := value.(string) + if !ok { + return false + } + enabled, err := strconv.ParseBool(str) + if err != nil { + logger.Named(logName).Debugw("Error parsing 'enabled' hint.", + "error", err, "autodiscover.hints", hints) + return false + } + return !enabled + } + + return false +} + +// GenerateHints parses annotations based on a prefix and sets up hints that can be picked up by individual Beats. +// Arguments: annotations: include the annotatons defined in the container, container: is the container name, +// prefix: the prefix of the annotation to check, validate: boolean variable that enables the validation of hints format and allSupportedHints: list of supported annotations to validate against +func GenerateHints(annotations mapstr.M, container, prefix string, validate bool, allSupportedHints []string) (mapstr.M, []string) { + hints := mapstr.M{} + var incorrecthints []string + var incorrecthint string + var digitCheck = regexp.MustCompile(`^[0-9]+$`) + + if rawEntries, err := annotations.GetValue(prefix); err == nil { + if entries, ok := rawEntries.(mapstr.M); ok { + + if validate { + //Start of Annotation Check: whether the annotation follows the supported format and vocabulary. The check happens for annotations that have prefix co.elastic + datastreamlist := GetHintAsList(entries, logName+"/"+"data_streams", "") + // We check if multiple data_streams are defined and we retrieve the hints per data_stream. Only applicable in elastic-agent + // See Metrics_apache_package_and_specific_config_per_datastream test case in hints_test.go + for _, stream := range datastreamlist { + allSupportedHints = append(allSupportedHints, stream) + incorrecthints = checkSupportedHintsSets(annotations, prefix, stream, logName, allSupportedHints, incorrecthints) + } + metricsetlist := GetHintAsList(entries, "metrics"+"/"+"metricsets", "") + // We check if multiple metrcisets are defined and we retrieve the hints per metricset. Only applicable in beats + //See Metrics_istio_module_and_specific_config_per_metricset test case in hints_test.go + for _, metric := range metricsetlist { + allSupportedHints = append(allSupportedHints, metric) + incorrecthints = checkSupportedHintsSets(annotations, prefix, metric, "metrics", allSupportedHints, incorrecthints) + } + } //end of annotation check + + for key, rawValue := range entries { + enumeratedmodules := []string{} + // If there are top level hints like co.elastic.logs/ then just add the values after the / + // Only consider namespaced annotations + parts := strings.Split(key, "/") + if len(parts) == 2 { + hintKey := fmt.Sprintf("%s.%s", parts[0], parts[1]) + + if validate { + checkdigit := digitCheck.MatchString(parts[1]) // With this regex we check if enumeration for modules is provided + if checkdigit { + allSupportedHints = append(allSupportedHints, parts[1]) + + specificlist, _ := entries.GetValue(key) + if specificentries, ok := specificlist.(mapstr.M); ok { + for keyspec := range specificentries { + // enumeratedmodules will be populated only in cases we have module enumeration, like: + // "co.elastic.metrics/1.module": "prometheus", + // "co.elastic.metrics/2.module": "istiod", + enumeratedmodules = append(enumeratedmodules, keyspec) + } + } + } + // We check if multiple metrcisets are defined and we retrieve the hints per metricset. Only applicable in beats + // See Metrics_multiple_modules_and_specific_config_per_module test case in hints_test.go + for _, metric := range enumeratedmodules { + _, incorrecthint = checkSupportedHints(metric, fmt.Sprintf("%s.%s", key, metric), allSupportedHints) + if incorrecthint != "" { + incorrecthints = append(incorrecthints, incorrecthint) + } + + } + //We check whether the provided annotation follows the supported format and vocabulary. The check happens for annotations that have prefix co.elastic + _, incorrecthint = checkSupportedHints(parts[1], key, allSupportedHints) + } //end of annotation check + + // Insert only if there is no entry already. container level annotations take + // higher priority. + if _, err := hints.GetValue(hintKey); err != nil { + _, err = hints.Put(hintKey, rawValue) + if err != nil { + continue + } + } + } else if container != "" { + // Only consider annotations that are of type mapstr.M as we are looking for + // container level nesting + builderHints, ok := rawValue.(mapstr.M) + if !ok { + continue + } + + // Check for / prefix + for hintKey, rawVal := range builderHints { + if strings.HasPrefix(hintKey, container) { + // Split the key to get part[1] to be the hint + parts := strings.Split(hintKey, "/") + + if validate { + checkdigit := digitCheck.MatchString(parts[1]) // With this regex we check if enumeration for modules is provided + if checkdigit { + allSupportedHints = append(allSupportedHints, parts[1]) + + specificlist, _ := entries.GetValue(key) + if specificentries, ok := specificlist.(mapstr.M); ok { + for keyspec := range specificentries { + // enumeratedmodules will be populated only in cases we have module enumeration, like: + // "co.elastic.metrics/1.module": "prometheus", + // "co.elastic.metrics/2.module": "istiod", + enumeratedmodules = append(enumeratedmodules, keyspec) + } + } + } + + // We check if multiple metrcisets are defined and we retrieve the hints per metricset. Only applicable in beats + // See Metrics_multiple_modules_and_specific_config_per_module test case in hints_test.go + for _, metric := range enumeratedmodules { + _, incorrecthint = checkSupportedHints(metric, fmt.Sprintf("%s.%s", key, metric), allSupportedHints) + if incorrecthint != "" { + incorrecthints = append(incorrecthints, incorrecthint) + } + + } + //We check whether the provided annotation follows the supported format and vocabulary. The check happens for annotations that have prefix co.elastic + _, incorrecthint = checkSupportedHints(parts[1], key, allSupportedHints) + } //end of annotation check + + if len(parts) == 2 { + // key will be the hint type + hintKey := fmt.Sprintf("%s.%s", key, parts[1]) + _, err := hints.Put(hintKey, rawVal) + if err != nil { + continue + } + } + } + } + } + if validate && incorrecthint != "" { + incorrecthints = append(incorrecthints, incorrecthint) + } + } + } + } + + return hints, incorrecthints +} + +// GetHintsAsList gets a set of hints and tries to convert them into a list of hints +func GetHintsAsList(hints mapstr.M, key string) []mapstr.M { + raw := GetHintMapStr(hints, key, "") + if raw == nil { + return nil + } + + var words, nums []string + + for key := range raw { + if _, err := strconv.Atoi(key); err != nil { + words = append(words, key) + continue + } else { + nums = append(nums, key) + } + } + + sort.Strings(nums) + + var configs []mapstr.M + for _, key := range nums { + rawCfg := raw[key] + if config, ok := rawCfg.(mapstr.M); ok { + configs = append(configs, config) + } + } + + defaultMap := mapstr.M{} + for _, word := range words { + defaultMap[word] = raw[word] + } + + if len(defaultMap) != 0 { + configs = append(configs, defaultMap) + } + return configs +} + +// checkSupportedHints gets a specific hint annotation and compares it with the supported list of hints +func checkSupportedHints(actualannotation, key string, allSupportedHints []string) (bool, string) { + found := false + var incorrecthint string + + if slices.Contains(allSupportedHints, actualannotation) { + found = true + } + if !found { + incorrecthint = key + } + return found, incorrecthint +} + +// checkSupportedHintsSets gest the data_streams or metricset lists that are defined. Searches inside specific list and returns the unsupported list of hints found +// This function will merge the incorrect hints found in metricsets of data_streams with rest incorrect hints +func checkSupportedHintsSets(annotations mapstr.M, prefix, stream, kind string, allSupportedHints, incorrecthints []string) []string { + var incorrecthint string + + if hintsindatastream, err := annotations.GetValue(prefix + "." + kind + "/" + stream); err == nil { + if hintsentries, ok := hintsindatastream.(mapstr.M); ok { + for hintkey := range hintsentries { + _, incorrecthint = checkSupportedHints(hintkey, kind+"/"+stream+"."+hintkey, allSupportedHints) + if incorrecthint != "" { + incorrecthints = append(incorrecthints, incorrecthint) + } + } + + } + } + + return incorrecthints +} diff --git a/pkg/autodiscover/utils/hints_test.go b/pkg/autodiscover/utils/hints_test.go new file mode 100644 index 000000000000..def50ee57c4b --- /dev/null +++ b/pkg/autodiscover/utils/hints_test.go @@ -0,0 +1,515 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/elastic-agent-libs/logp/logptest" + "github.com/elastic/elastic-agent-libs/mapstr" +) + +func TestGetProcessors(t *testing.T) { + hints := mapstr.M{ + "co": mapstr.M{ + "elastic": mapstr.M{ + "logs": mapstr.M{ + "processors": mapstr.M{ + "add_fields": `{"fields": {"foo": "bar"}}`, + }, + }, + }, + }, + } + procs := GetProcessors(hints, "co.elastic.logs", logptest.NewTestingLogger(t, "")) + assert.Equal(t, []mapstr.M{ + mapstr.M{ + "add_fields": mapstr.M{ + "fields": map[string]any{ + "foo": "bar", + }, + }, + }, + }, procs) +} + +func TestGenerateHints(t *testing.T) { + + var allSupportedHints = []string{"enabled", "package", "module", "integration", "data_streams", "metricsets", "host", "period", "timeout", "metrics_path", "username", "password", "stream", "processors", "multiline", "json", "disable"} + + tests := []struct { + name string + annotations map[string]string + result mapstr.M + expectedIncorrectHints int // We set the number of hints that will be marked as incorrect and wont be included in the acceptable supported list + }{ + //Empty annotations should return empty hints + { + name: "Empty_Annotations", + annotations: map[string]string{}, + result: mapstr.M{}, + expectedIncorrectHints: 0, + }, + + // Scenarios being tested: + // logs/multiline.pattern must be a nested mapstr.M under hints.logs + // logs/processors.add_fields must be nested mapstr.M under hints.logs + // logs/json.keys_under_root must be a nested mapstr.M under hints.logs + // metrics/module must be found in hints.metrics + // not.to.include must not be part of hints + // period is annotated at both container and pod level. Container level value must be in hints + { + name: "Logs_multiline_and_metrics", + annotations: map[string]string{ + "co.elastic.logs/multiline.pattern": "^test", + "co.elastic.logs/json.keys_under_root": "true", + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + "co.elastic.metrics.foobar/period": "15s", + "co.elastic.metrics.foobar1/period": "15s", + "not.to.include": "true", + }, + result: mapstr.M{ + "logs": mapstr.M{ + "multiline": mapstr.M{ + "pattern": "^test", + }, + "json": mapstr.M{ + "keys_under_root": "true", + }, + }, + "metrics": mapstr.M{ + "module": "prometheus", + "period": "15s", + }, + }, + expectedIncorrectHints: 0, + }, + // Scenarios being tested: + // logs/multiline.pattern must be a nested mapstr.M under hints.logs + // metrics/module must be found in hints.metrics + // not.to.include must not be part of hints + // metrics/metrics_path must be found in hints.metrics + { + name: "Logs_multiline_and_metrics_with_metrics_path", + annotations: map[string]string{ + "co.elastic.logs/multiline.pattern": "^test", + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + "co.elastic.metrics/metrics_path": "/metrics/prometheus", + "co.elastic.metrics/username": "user", + "co.elastic.metrics/password": "pass", + "co.elastic.metrics.foobar/period": "15s", + "co.elastic.metrics.foobar1/period": "15s", + "co.elastic.hints/streamssssssssss": "stdout", // On purpose this added with typo + "not.to.include": "true", + }, + result: mapstr.M{ + "logs": mapstr.M{ + "multiline": mapstr.M{ + "pattern": "^test", + }, + }, + "hints": mapstr.M{"streamssssssssss": "stdout"}, + "metrics": mapstr.M{ + "module": "prometheus", + "period": "15s", + "metrics_path": "/metrics/prometheus", + "username": "user", + "password": "pass", + }, + }, + expectedIncorrectHints: 1, // Due to co.elastic.hints/streamsteamssssssssss + }, + // Scenarios being tested: + // logs/multiline.pattern must be a nested mapstr.M under hints.logs + // metrics/module must be found in hints.metrics + // not.to.include must not be part of hints + // period is annotated at both container and pod level. Container level value must be in hints + { + name: "Logs_multiline_and_metrics", + annotations: map[string]string{ + "co.elastic.logs/multiline.pattern": "^test", + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + "co.elastic.metrics.foobar/period": "15s", + "co.elastic.metrics.foobar1/period": "15s", + "not.to.include": "true", + }, + result: mapstr.M{ + "logs": mapstr.M{ + "multiline": mapstr.M{ + "pattern": "^test", + }, + }, + "metrics": mapstr.M{ + "module": "prometheus", + "period": "15s", + }, + }, + expectedIncorrectHints: 0, + }, + // Scenarios being tested: + // have co.elastic.logs/disable set to false. + // logs/multiline.pattern must be a nested mapstr.M under hints.logs + // metrics/module must be found in hints.metrics + // not.to.include must not be part of hints + // period is annotated at both container and pod level. Container level value must be in hints + { + name: "Logs_disabled_false_and_metrics", + annotations: map[string]string{ + "co.elastic.logs/disable": "false", + "co.elastic.logs/multiline.pattern": "^test", + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + "co.elastic.metrics.foobar/period": "15s", + "co.elastic.metrics.foobar1/period": "15s", + "not.to.include": "true", + }, + result: mapstr.M{ + "logs": mapstr.M{ + "multiline": mapstr.M{ + "pattern": "^test", + }, + "disable": "false", + }, + "metrics": mapstr.M{ + "module": "prometheus", + "period": "15s", + }, + }, + expectedIncorrectHints: 0, + }, + // Scenarios being tested: + // have co.elastic.logs/disable set to true. + // logs/multiline.pattern must be a nested mapstr.M under hints.logs + // metrics/module must be found in hints.metrics + // not.to.include must not be part of hints + // period is annotated at both container and pod level. Container level value must be in hints + { + name: "Logs_disabled_true_and_metrics", + annotations: map[string]string{ + "co.elastic.logs/disable": "true", + "co.elastic.logs/multiline.pattern": "^test", + "co.elastic.metrics/module": "prometheus", + "co.elastic.metrics/period": "10s", + "co.elastic.metrics.foobar/period": "15s", + "co.elastic.metrics.foobar1/period": "15s", + "not.to.include": "true", + }, + result: mapstr.M{ + "logs": mapstr.M{ + "multiline": mapstr.M{ + "pattern": "^test", + }, + "disable": "true", + }, + "metrics": mapstr.M{ + "module": "prometheus", + "period": "15s", + }, + }, + expectedIncorrectHints: 0, + }, + // Scenarios being tested: + // have co.elastic.hints/package set. + // Define multiple co.elastic.hints/data_streams and also specific configuration for each one + // Typo errors introduced for "co.elastic.hints/access.streams" and "co.elastic.hints/error.streams" + { + name: "Metrics_apache_package_and_specific_config_per_datastream", + annotations: map[string]string{ + "co.elastic.hints/package": "apache", + "co.elastic.hints/data_streams": "access,error", + "co.elastic.hints/access.period": "5m", + "co.elastic.hints/access.streamssssssssss": "stdout", // On purpose this added with typo + "co.elastic.hints/error.period": "5m", + "co.elastic.hints/error.streamssssssssss": "stderr", // On purpose this added with typo + }, + result: mapstr.M{ + "hints": mapstr.M{ + "data_streams": "access,error", + "access": mapstr.M{"period": "5m", "streamssssssssss": "stdout"}, + "error": mapstr.M{"period": "5m", "streamssssssssss": "stderr"}, + "package": "apache", + }}, + expectedIncorrectHints: 2, // Due to co.elastic.hints/access.streamssssssssss and co.elastic.hints/error.streamssssssssss typo errors + }, + // Scenarios being tested: + // have co.elastic.metrics/module set. + // Define multiple co.elastic.hints/data_streams and also specific configuration for each one + // A typo error introduced for "co.elastic.metrics/istiod.streams" + { + name: "Metrics_istio_module_and_specific_config_per_metricset", + annotations: map[string]string{ + "co.elastic.metrics/module": "istio", + "co.elastic.metrics/metricsets": "istiod,proxy", + "co.elastic.metrics/istiod.period": "5m", + "co.elastic.metrics/istiod.streamssssssssss": "stdout", // On purpose this added with typo + "co.elastic.metrics/proxy.period": "5m", + "co.elastic.metrics/proxy.stream": "stderr", + }, + result: mapstr.M{ + "metrics": mapstr.M{ + "metricsets": "istiod,proxy", + "istiod": mapstr.M{"period": "5m", "streamssssssssss": "stdout"}, + "proxy": mapstr.M{"period": "5m", "stream": "stderr"}, + "module": "istio", + }}, + expectedIncorrectHints: 1, // Due to co.elastic.metrics/istiod.streamssssssssss + }, + // Scenarios being tested: + // have co.elastic.metrics/module set for multiple enumerations. + // Define different hints for each one enumeration + // A typo error introduced for "co.elastic.metrics/1.periods" and "co.elastic.metrics/2.streams" + { + name: "Metrics_multiple_modules_and_specific_config_per_module", + annotations: map[string]string{ + "co.elastic.metrics/1.module": "prometheus", + "co.elastic.metrics/1.periodssssssssss": "15s", // On purpose this added with typo + "co.elastic.metrics/2.module": "istiod", + "co.elastic.metrics/2.period": "15s", + "co.elastic.metrics/2.streamssssssssss": "stderr", // On purpose this added with typo + }, + result: mapstr.M{ + "metrics": mapstr.M{ + "1": mapstr.M{ + "module": "prometheus", + "periodssssssssss": "15s", + }, + "2": mapstr.M{ + "module": "istiod", + "period": "15s", + "streamssssssssss": "stderr", + }, + }}, + expectedIncorrectHints: 2, // Due to co.elastic.metrics/1.periodssssssssss and co.elastic.metrics/2.streamssssssssss typo errors + }, + } + + for _, test := range tests { + annMap := mapstr.M{} + for k, v := range test.annotations { + _, err := annMap.Put(k, v) + if err != nil { + continue + } + } + + generateHints, incorrectHints := GenerateHints(annMap, "foobar", "co.elastic", true, allSupportedHints) + assert.Len(t, incorrectHints, test.expectedIncorrectHints) // We validate how many incorrect hints are provided per test case. + assert.Equal(t, test.result, generateHints) + } +} + +func TestGenerateHintsWithValidatedisabled(t *testing.T) { + + var allSupportedHints = []string{"enabled", "package", "module", "integration", "data_streams", "metricsets", "host", "period", "timeout", "metrics_path", "username", "password", "stream", "processors", "multiline", "json", "disable"} + + tests := []struct { + name string + annotations map[string]string + result mapstr.M + expectedIncorrectHints int // We set the number of hints that will be marked as incorrect and wont be included in the acceptable supported list + }{ + + // Scenarios being tested: + // have co.elastic.hints/package set. + // Define multiple co.elastic.hints/data_streams and also specific configuration for each one + // Typo errors introduced for "co.elastic.hints/access.streams" and "co.elastic.hints/error.streams" + { + name: "Metrics_apache_package_and_specific_config_per_datastream", + annotations: map[string]string{ + "co.elastic.hints/package": "apache", + "co.elastic.hints/data_streams": "access,error", + "co.elastic.hints/access.period": "5m", + "co.elastic.hints/access.streamssssssssss": "stdout", // On purpose this added with typo + "co.elastic.hints/error.period": "5m", + "co.elastic.hints/error.streamssssssssss": "stderr", // On purpose this added with typo + }, + result: mapstr.M{ + "hints": mapstr.M{ + "data_streams": "access,error", + "access": mapstr.M{"period": "5m", "streamssssssssss": "stdout"}, + "error": mapstr.M{"period": "5m", "streamssssssssss": "stderr"}, + "package": "apache", + }}, + expectedIncorrectHints: 0, // Validate flag= false in GenerateHints + }, + // Scenarios being tested: + // have co.elastic.metrics/module set. + // Define multiple co.elastic.hints/data_streams and also specific configuration for each one + // A typo error introduced for "co.elastic.metrics/istiod.streams" + { + name: "Metrics_istio_module_and_specific_config_per_metricset", + annotations: map[string]string{ + "co.elastic.metrics/module": "istio", + "co.elastic.metrics/metricsets": "istiod,proxy", + "co.elastic.metrics/istiod.period": "5m", + "co.elastic.metrics/istiod.streamssssssssss": "stdout", // On purpose this added with typo + "co.elastic.metrics/proxy.period": "5m", + "co.elastic.metrics/proxy.stream": "stderr", + }, + result: mapstr.M{ + "metrics": mapstr.M{ + "metricsets": "istiod,proxy", + "istiod": mapstr.M{"period": "5m", "streamssssssssss": "stdout"}, + "proxy": mapstr.M{"period": "5m", "stream": "stderr"}, + "module": "istio", + }}, + expectedIncorrectHints: 0, // Validate flag= false in GenerateHints + }, + // Scenarios being tested: + // have co.elastic.metrics/module set for multiple enumerations. + // Define different hints for each one enumeration + // A typo error introduced for "co.elastic.metrics/1.periods" and "co.elastic.metrics/2.streams" + { + name: "Metrics_multiple_modules_and_specific_config_per_module", + annotations: map[string]string{ + "co.elastic.metrics/1.module": "prometheus", + "co.elastic.metrics/1.periodssssssssss": "15s", // On purpose this added with typo + "co.elastic.metrics/2.module": "istiod", + "co.elastic.metrics/2.period": "15s", + "co.elastic.metrics/2.streamssssssssss": "stderr", // On purpose this added with typo + }, + result: mapstr.M{ + "metrics": mapstr.M{ + "1": mapstr.M{ + "module": "prometheus", + "periodssssssssss": "15s", + }, + "2": mapstr.M{ + "module": "istiod", + "period": "15s", + "streamssssssssss": "stderr", + }, + }}, + expectedIncorrectHints: 0, // Validate flag= false in GenerateHints + }, + } + + for _, test := range tests { + annMap := mapstr.M{} + for k, v := range test.annotations { + _, err := annMap.Put(k, v) + if err != nil { + continue + } + } + + generateHints, incorrectHints := GenerateHints(annMap, "foobar", "co.elastic", false, allSupportedHints) + assert.Len(t, incorrectHints, test.expectedIncorrectHints) // We validate how many incorrect hints are provided per test case. + assert.Equal(t, test.result, generateHints) + } +} + +func TestGetHintsAsList(t *testing.T) { + tests := []struct { + input mapstr.M + output []mapstr.M + message string + }{ + { + input: mapstr.M{ + "metrics": mapstr.M{ + "module": "prometheus", + "period": "15s", + }, + }, + output: []mapstr.M{ + { + "module": "prometheus", + "period": "15s", + }, + }, + message: "Single hint should return a single set of configs", + }, + { + input: mapstr.M{ + "metrics": mapstr.M{ + "1": mapstr.M{ + "module": "prometheus", + "period": "15s", + }, + }, + }, + output: []mapstr.M{ + { + "module": "prometheus", + "period": "15s", + }, + }, + message: "Single hint with numeric prefix should return a single set of configs", + }, + { + input: mapstr.M{ + "metrics": mapstr.M{ + "1": mapstr.M{ + "module": "prometheus", + "period": "15s", + }, + "2": mapstr.M{ + "module": "dropwizard", + "period": "20s", + }, + }, + }, + output: []mapstr.M{ + { + "module": "prometheus", + "period": "15s", + }, + { + "module": "dropwizard", + "period": "20s", + }, + }, + message: "Multiple hints with numeric prefix should return configs in numeric ordering", + }, + { + input: mapstr.M{ + "metrics": mapstr.M{ + "1": mapstr.M{ + "module": "prometheus", + "period": "15s", + }, + "module": "dropwizard", + "period": "20s", + }, + }, + output: []mapstr.M{ + { + "module": "prometheus", + "period": "15s", + }, + { + "module": "dropwizard", + "period": "20s", + }, + }, + message: "Multiple hints with numeric prefix and default should return configs with defaults at the last", + }, + } + + for _, test := range tests { + t.Run(test.message, func(t *testing.T) { + assert.Equal(t, test.output, GetHintsAsList(test.input, "metrics")) + }) + } +} diff --git a/pkg/autodiscover/utils/key.go b/pkg/autodiscover/utils/key.go new file mode 100644 index 000000000000..a54fdcf38bb2 --- /dev/null +++ b/pkg/autodiscover/utils/key.go @@ -0,0 +1,26 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package utils + +import "strings" + +// DeDot a string by replacing all . with _ +// This helps when sending data to Elasticsearch to prevent object and key collisions. +func DeDot(s string) string { + return strings.ReplaceAll(s, ".", "_") +} diff --git a/x-pack/dockerlogbeat/pipelinemanager/clientLogReader.go b/x-pack/dockerlogbeat/pipelinemanager/clientLogReader.go index 1826ed0eee12..901bf93ddc73 100644 --- a/x-pack/dockerlogbeat/pipelinemanager/clientLogReader.go +++ b/x-pack/dockerlogbeat/pipelinemanager/clientLogReader.go @@ -18,8 +18,8 @@ import ( "github.com/elastic/beats/v7/libbeat/beat" "github.com/elastic/beats/v7/libbeat/common/acker" + helper "github.com/elastic/beats/v7/pkg/autodiscover/docker" "github.com/elastic/beats/v7/x-pack/dockerlogbeat/pipereader" - helper "github.com/elastic/elastic-agent-autodiscover/docker" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/mapstr" ) diff --git a/x-pack/filebeat/input/streaming/config.go b/x-pack/filebeat/input/streaming/config.go index eb604d438db8..45779e76ca7b 100644 --- a/x-pack/filebeat/input/streaming/config.go +++ b/x-pack/filebeat/input/streaming/config.go @@ -39,7 +39,7 @@ type config struct { // program. If it has a cursor field, that field will // be overwritten by any stored cursor, but will be // available if no stored cursor exists. - State map[string]interface{} `config:"state"` + State map[string]any `config:"state"` // Auth is the authentication config for connection. Auth authConfig `config:"auth"` // URL is the websocket url to connect to. diff --git a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go index 0489c62da3ec..6cce0849f3aa 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go +++ b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider.go @@ -13,9 +13,9 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/autodiscover/template" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" awsauto "github.com/elastic/beats/v7/x-pack/libbeat/autodiscover/providers/aws" awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" - "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" diff --git a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider_test.go b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider_test.go index f72f600bc542..244069655cb7 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/ec2/provider_test.go +++ b/x-pack/libbeat/autodiscover/providers/aws/ec2/provider_test.go @@ -13,9 +13,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" awsauto "github.com/elastic/beats/v7/x-pack/libbeat/autodiscover/providers/aws" "github.com/elastic/beats/v7/x-pack/libbeat/autodiscover/providers/aws/test" - "github.com/elastic/elastic-agent-autodiscover/bus" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/logp/logptest" @@ -26,7 +26,7 @@ func Test_internalBuilder(t *testing.T) { instance := fakeEC2Instance() instances := []*ec2Instance{instance} fetcher := newMockFetcher(instances, nil) - log := logp.NewLogger("ec2") + log := logp.NewLogger("ec2") //nolint:forbidigo // test setup pBus := bus.New(log, "test") cfg := &awsauto.Config{ diff --git a/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go b/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go index 4012460b5dbd..e3afa7ba1f39 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go +++ b/x-pack/libbeat/autodiscover/providers/aws/elb/provider.go @@ -13,9 +13,9 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/autodiscover/template" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" awsauto "github.com/elastic/beats/v7/x-pack/libbeat/autodiscover/providers/aws" awscommon "github.com/elastic/beats/v7/x-pack/libbeat/common/aws" - "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" diff --git a/x-pack/libbeat/autodiscover/providers/aws/elb/provider_test.go b/x-pack/libbeat/autodiscover/providers/aws/elb/provider_test.go index a2f151a5ece9..4f720c7baffc 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/elb/provider_test.go +++ b/x-pack/libbeat/autodiscover/providers/aws/elb/provider_test.go @@ -14,8 +14,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" awsauto "github.com/elastic/beats/v7/x-pack/libbeat/autodiscover/providers/aws" - "github.com/elastic/elastic-agent-autodiscover/bus" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" "github.com/elastic/elastic-agent-libs/logp/logptest" @@ -64,7 +64,7 @@ func (tea *testEventAccumulator) waitForNumEvents(t *testing.T, targetLen int, t } func Test_internalBuilder(t *testing.T) { - log := logp.NewLogger("elb") + log := logp.NewLogger("elb") //nolint:forbidigo // test setup lbl := fakeLbl() lbls := []*lbListener{lbl} fetcher := newMockFetcher(lbls, nil) diff --git a/x-pack/libbeat/autodiscover/providers/aws/test/provider.go b/x-pack/libbeat/autodiscover/providers/aws/test/provider.go index 8779c99b7e66..d07183f07f51 100644 --- a/x-pack/libbeat/autodiscover/providers/aws/test/provider.go +++ b/x-pack/libbeat/autodiscover/providers/aws/test/provider.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/elastic/elastic-agent-autodiscover/bus" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" ) // TestEventAccumulator defined a list of events for testing @@ -48,7 +48,7 @@ func (tea *TestEventAccumulator) Get() []bus.Event { func (tea *TestEventAccumulator) WaitForNumEvents(t *testing.T, targetLen int, timeout time.Duration) { start := time.Now() - for time.Now().Sub(start) < timeout { + for time.Since(start) < timeout { if tea.Len() >= targetLen { return } diff --git a/x-pack/libbeat/autodiscover/providers/nomad/nomad.go b/x-pack/libbeat/autodiscover/providers/nomad/nomad.go index 417e01fe7507..803b5452a353 100644 --- a/x-pack/libbeat/autodiscover/providers/nomad/nomad.go +++ b/x-pack/libbeat/autodiscover/providers/nomad/nomad.go @@ -11,13 +11,13 @@ import ( "github.com/gofrs/uuid/v5" - "github.com/elastic/elastic-agent-autodiscover/utils" + "github.com/elastic/beats/v7/pkg/autodiscover/utils" "github.com/elastic/beats/v7/libbeat/autodiscover" "github.com/elastic/beats/v7/libbeat/autodiscover/template" "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" "github.com/elastic/beats/v7/x-pack/libbeat/common/nomad" - "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/keystore" "github.com/elastic/elastic-agent-libs/logp" diff --git a/x-pack/libbeat/autodiscover/providers/nomad/nomad_test.go b/x-pack/libbeat/autodiscover/providers/nomad/nomad_test.go index a465aef0364f..59c18bb2c048 100644 --- a/x-pack/libbeat/autodiscover/providers/nomad/nomad_test.go +++ b/x-pack/libbeat/autodiscover/providers/nomad/nomad_test.go @@ -18,8 +18,8 @@ import ( "github.com/elastic/beats/v7/libbeat/autodiscover/template" "github.com/elastic/beats/v7/libbeat/tests/resources" + "github.com/elastic/beats/v7/pkg/autodiscover/bus" "github.com/elastic/beats/v7/x-pack/libbeat/common/nomad" - "github.com/elastic/elastic-agent-autodiscover/bus" conf "github.com/elastic/elastic-agent-libs/config" "github.com/elastic/elastic-agent-libs/logp/logptest" "github.com/elastic/elastic-agent-libs/mapstr" diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/container.go b/x-pack/metricbeat/module/awsfargate/task_stats/container.go index 9c4d7d0d0e61..6f827c0a03b8 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/container.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/container.go @@ -5,8 +5,8 @@ package task_stats import ( - helpers "github.com/elastic/elastic-agent-autodiscover/docker" - "github.com/elastic/elastic-agent-autodiscover/utils" + helpers "github.com/elastic/beats/v7/pkg/autodiscover/docker" + "github.com/elastic/beats/v7/pkg/autodiscover/utils" ) // container is a struct representation of a container diff --git a/x-pack/winlogbeat/module/testing.go b/x-pack/winlogbeat/module/testing.go index 0add079c4343..c4a0f4ab23f6 100644 --- a/x-pack/winlogbeat/module/testing.go +++ b/x-pack/winlogbeat/module/testing.go @@ -207,7 +207,7 @@ func testIngestPipeline(t *testing.T, pipeline, pattern string, p *params) { // assertEqual asserts that the two objects are deeply equal. If not it will // error the test and output a diff of the two objects' JSON representation. -func assertEqual(t testing.TB, expected, actual interface{}) bool { +func assertEqual(t testing.TB, expected, actual any) bool { t.Helper() if reflect.DeepEqual(expected, actual) { diff --git a/x-pack/winlogbeat/module/wintest/docker_test.go b/x-pack/winlogbeat/module/wintest/docker_test.go index 46687a6d1d36..96c57926051d 100644 --- a/x-pack/winlogbeat/module/wintest/docker_test.go +++ b/x-pack/winlogbeat/module/wintest/docker_test.go @@ -83,8 +83,7 @@ func TestDocker(t *testing.T) { } defer conn.Close() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() err = conn.Connect(ctx) if err != nil { t.Fatalf("unexpected error making connection: %v", err) diff --git a/x-pack/winlogbeat/module/wintest/simulate_test.go b/x-pack/winlogbeat/module/wintest/simulate_test.go index a5e79f7cc00d..a22c0c30c0ce 100644 --- a/x-pack/winlogbeat/module/wintest/simulate_test.go +++ b/x-pack/winlogbeat/module/wintest/simulate_test.go @@ -11,7 +11,6 @@ package wintest_test import ( - "context" "encoding/json" "fmt" "os" @@ -74,8 +73,7 @@ func TestSimulate(t *testing.T) { } defer conn.Close() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx := t.Context() err = conn.Connect(ctx) if err != nil { t.Fatalf("unexpected error making connection: %v", err)