From acedfd0a0aaee0b58fa090fe24e5824212404803 Mon Sep 17 00:00:00 2001 From: Jaime Soriano Pastor Date: Fri, 1 Aug 2025 17:45:40 +0200 Subject: [PATCH 01/24] Support OTel config in policy tests --- internal/testrunner/runners/policy/policy.go | 49 +++++++++++++- .../testrunner/runners/policy/policy_test.go | 66 +++++++++++++++++++ 2 files changed, 113 insertions(+), 2 deletions(-) diff --git a/internal/testrunner/runners/policy/policy.go b/internal/testrunner/runners/policy/policy.go index 39ac7d60e0..9510add971 100644 --- a/internal/testrunner/runners/policy/policy.go +++ b/internal/testrunner/runners/policy/policy.go @@ -97,6 +97,7 @@ func expectedPathFor(testPath string) string { type policyEntryFilter struct { name string elementsEntries []policyEntryFilter + elementsReplace *policyEntryReplace memberReplace *policyEntryReplace onlyIfEmpty bool ignoreValues []any @@ -164,6 +165,29 @@ var policyEntryFilters = []policyEntryFilter{ {name: "data_stream.elasticsearch", onlyIfEmpty: true}, }}, }}, + + // OTel Collector IDs are relevant, but just check that they are there. + {name: "extensions", memberReplace: &otelComponentIDReplace}, + {name: "receivers", memberReplace: &otelComponentIDReplace}, + {name: "processors", memberReplace: &otelComponentIDReplace}, + {name: "exporters", memberReplace: &otelComponentIDReplace}, + {name: "service.extensions", elementsReplace: &otelComponentIDReplace}, + + // TODO: The signals here will need patterns at some moment, as they can also contain ids. + {name: "service.pipelines.logs.receivers", elementsReplace: &otelComponentIDReplace}, + {name: "service.pipelines.logs.processors", elementsReplace: &otelComponentIDReplace}, + {name: "service.pipelines.logs.exporters", elementsReplace: &otelComponentIDReplace}, + {name: "service.pipelines.metrics.receivers", elementsReplace: &otelComponentIDReplace}, + {name: "service.pipelines.metrics.processors", elementsReplace: &otelComponentIDReplace}, + {name: "service.pipelines.metrics.exporters", elementsReplace: &otelComponentIDReplace}, + {name: "service.pipelines.traces.receivers", elementsReplace: &otelComponentIDReplace}, + {name: "service.pipelines.traces.processors", elementsReplace: &otelComponentIDReplace}, + {name: "service.pipelines.traces.exporters", elementsReplace: &otelComponentIDReplace}, +} + +var otelComponentIDReplace = policyEntryReplace{ + regexp: regexp.MustCompile(`^([^/]+)/.*$`), + replace: "$1/componentid", } // cleanPolicy prepares a policy YAML as returned by the download API to be compared with other @@ -218,10 +242,31 @@ func cleanPolicyMap(policyMap common.MapStr, entries []policyEntryFilter) (commo if !ok { return nil, fmt.Errorf("expected map, found %T", v) } + regexp := entry.memberReplace.regexp + replacement := entry.memberReplace.replace for k, e := range m { - if entry.memberReplace.regexp.MatchString(k) { + key := k + if regexp.MatchString(k) { delete(m, k) - m[entry.memberReplace.replace] = e + key = regexp.ReplaceAllString(k, replacement) + m[key] = e + } + strElement, ok := e.(string) + if ok && regexp.MatchString(strElement) { + m[key] = regexp.ReplaceAllString(strElement, replacement) + } + } + case entry.elementsReplace != nil: + list, ok := v.([]any) + if !ok { + return nil, fmt.Errorf("expected list, found %T", v) + } + regexp := entry.elementsReplace.regexp + replacement := entry.elementsReplace.replace + for i, e := range list { + strElement, ok := e.(string) + if ok && regexp.MatchString(strElement) { + list[i] = regexp.ReplaceAllString(strElement, replacement) } } default: diff --git a/internal/testrunner/runners/policy/policy_test.go b/internal/testrunner/runners/policy/policy_test.go index 130a9a0535..207cc43496 100644 --- a/internal/testrunner/runners/policy/policy_test.go +++ b/internal/testrunner/runners/policy/policy_test.go @@ -285,6 +285,72 @@ secret_references: `, equal: false, }, + { + title: "otel ids", + expected: ` +inputs: [] +output_permissions: + default: + _elastic_agent_checks: + cluster: + - monitor + _elastic_agent_monitoring: + indices: [] + uuid-for-permissions-on-related-indices: + indices: + - names: + - logs-*-* + privileges: + - auto_configure + - create_doc +receivers: + httpcheck/componentid: + collection_interval: 1m + targets: + - endpoints: + - https://epr.elastic.co + method: GET +secret_references: [] +service: + pipelines: + logs: + receivers: + - httpcheck/componentid + +`, + found: ` +inputs: [] +output_permissions: + default: + _elastic_agent_checks: + cluster: + - monitor + _elastic_agent_monitoring: + indices: [] + uuid-for-permissions-on-related-indices: + indices: + - names: + - logs-*-* + privileges: + - auto_configure + - create_doc +receivers: + httpcheck/b0f518d6-4e2d-4c5d-bda7-f9808df537b7: + collection_interval: 1m + targets: + - endpoints: + - https://epr.elastic.co + method: GET +secret_references: [] +service: + pipelines: + logs: + receivers: + - httpcheck/b0f518d6-4e2d-4c5d-bda7-f9808df537b7 + +`, + equal: true, + }, } for _, c := range cases { From 2c5c553151b722fbfe8bf0b17e09f902ec7a92fd Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Sep 2025 13:31:31 +0200 Subject: [PATCH 02/24] Add test package --- test/packages/parallel/httpcheck/LICENSE.txt | 93 ++++++++++++++++++ .../_dev/deploy/docker/docker-compose.yml | 6 ++ .../_dev/test/policy/test-eprcheck.expected | 28 ++++++ .../_dev/test/policy/test-eprcheck.yml | 3 + .../_dev/test/system/test-local-config.yml | 4 + .../httpcheck/agent/input/input.yml.hbs | 20 ++++ .../packages/parallel/httpcheck/changelog.yml | 6 ++ .../parallel/httpcheck/docs/README.md | 84 ++++++++++++++++ .../parallel/httpcheck/fields/base-fields.yml | 12 +++ .../parallel/httpcheck/img/sample-logo.svg | 1 + .../httpcheck/img/sample-screenshot.png | Bin 0 -> 18849 bytes test/packages/parallel/httpcheck/manifest.yml | 76 ++++++++++++++ 12 files changed, 333 insertions(+) create mode 100644 test/packages/parallel/httpcheck/LICENSE.txt create mode 100644 test/packages/parallel/httpcheck/_dev/deploy/docker/docker-compose.yml create mode 100644 test/packages/parallel/httpcheck/_dev/test/policy/test-eprcheck.expected create mode 100644 test/packages/parallel/httpcheck/_dev/test/policy/test-eprcheck.yml create mode 100644 test/packages/parallel/httpcheck/_dev/test/system/test-local-config.yml create mode 100644 test/packages/parallel/httpcheck/agent/input/input.yml.hbs create mode 100644 test/packages/parallel/httpcheck/changelog.yml create mode 100644 test/packages/parallel/httpcheck/docs/README.md create mode 100644 test/packages/parallel/httpcheck/fields/base-fields.yml create mode 100644 test/packages/parallel/httpcheck/img/sample-logo.svg create mode 100644 test/packages/parallel/httpcheck/img/sample-screenshot.png create mode 100644 test/packages/parallel/httpcheck/manifest.yml diff --git a/test/packages/parallel/httpcheck/LICENSE.txt b/test/packages/parallel/httpcheck/LICENSE.txt new file mode 100644 index 0000000000..809108b857 --- /dev/null +++ b/test/packages/parallel/httpcheck/LICENSE.txt @@ -0,0 +1,93 @@ +Elastic License 2.0 + +URL: https://www.elastic.co/licensing/elastic-license + +## Acceptance + +By using the software, you agree to all of the terms and conditions below. + +## Copyright License + +The licensor grants you a non-exclusive, royalty-free, worldwide, +non-sublicensable, non-transferable license to use, copy, distribute, make +available, and prepare derivative works of the software, in each case subject to +the limitations and conditions below. + +## Limitations + +You may not provide the software to third parties as a hosted or managed +service, where the service provides users with access to any substantial set of +the features or functionality of the software. + +You may not move, change, disable, or circumvent the license key functionality +in the software, and you may not remove or obscure any functionality in the +software that is protected by the license key. + +You may not alter, remove, or obscure any licensing, copyright, or other notices +of the licensor in the software. Any use of the licensor’s trademarks is subject +to applicable law. + +## Patents + +The licensor grants you a license, under any patent claims the licensor can +license, or becomes able to license, to make, have made, use, sell, offer for +sale, import and have imported the software, in each case subject to the +limitations and conditions in this license. This license does not cover any +patent claims that you cause to be infringed by modifications or additions to +the software. If you or your company make any written claim that the software +infringes or contributes to infringement of any patent, your patent license for +the software granted under these terms ends immediately. If your company makes +such a claim, your patent license ends immediately for work on behalf of your +company. + +## Notices + +You must ensure that anyone who gets a copy of any part of the software from you +also gets a copy of these terms. + +If you modify the software, you must include in any modified copies of the +software prominent notices stating that you have modified the software. + +## No Other Rights + +These terms do not imply any licenses other than those expressly granted in +these terms. + +## Termination + +If you use the software in violation of these terms, such use is not licensed, +and your licenses will automatically terminate. If the licensor provides you +with a notice of your violation, and you cease all violation of this license no +later than 30 days after you receive that notice, your licenses will be +reinstated retroactively. However, if you violate these terms after such +reinstatement, any additional violation of these terms will cause your licenses +to terminate automatically and permanently. + +## No Liability + +*As far as the law allows, the software comes as is, without any warranty or +condition, and the licensor will not be liable to you for any damages arising +out of these terms or the use or nature of the software, under any kind of +legal claim.* + +## Definitions + +The **licensor** is the entity offering these terms, and the **software** is the +software the licensor makes available under these terms, including any portion +of it. + +**you** refers to the individual or entity agreeing to these terms. + +**your company** is any legal entity, sole proprietorship, or other kind of +organization that you work for, plus all organizations that have control over, +are under the control of, or are under common control with that +organization. **control** means ownership of substantially all the assets of an +entity, or the power to direct its management and policies by vote, contract, or +otherwise. Control can be direct or indirect. + +**your licenses** are all the licenses granted to you for the software under +these terms. + +**use** means anything you do with the software requiring one of your licenses. + +**trademark** means trademarks, service marks, and similar rights. diff --git a/test/packages/parallel/httpcheck/_dev/deploy/docker/docker-compose.yml b/test/packages/parallel/httpcheck/_dev/deploy/docker/docker-compose.yml new file mode 100644 index 0000000000..61ae2ded6a --- /dev/null +++ b/test/packages/parallel/httpcheck/_dev/deploy/docker/docker-compose.yml @@ -0,0 +1,6 @@ +version: '2.3' +services: + web: + image: nginx:1.29.0-alpine-slim + ports: + - 80 diff --git a/test/packages/parallel/httpcheck/_dev/test/policy/test-eprcheck.expected b/test/packages/parallel/httpcheck/_dev/test/policy/test-eprcheck.expected new file mode 100644 index 0000000000..aa549f1e32 --- /dev/null +++ b/test/packages/parallel/httpcheck/_dev/test/policy/test-eprcheck.expected @@ -0,0 +1,28 @@ +inputs: [] +output_permissions: + default: + _elastic_agent_checks: + cluster: + - monitor + _elastic_agent_monitoring: + indices: [] + uuid-for-permissions-on-related-indices: + indices: + - names: + - metrics-*-* + privileges: + - auto_configure + - create_doc +receivers: + httpcheck/componentid: + collection_interval: 1m + targets: + - endpoints: + - https://epr.elastic.co + method: GET +secret_references: [] +service: + pipelines: + metrics: + receivers: + - httpcheck/componentid diff --git a/test/packages/parallel/httpcheck/_dev/test/policy/test-eprcheck.yml b/test/packages/parallel/httpcheck/_dev/test/policy/test-eprcheck.yml new file mode 100644 index 0000000000..25fa4992fb --- /dev/null +++ b/test/packages/parallel/httpcheck/_dev/test/policy/test-eprcheck.yml @@ -0,0 +1,3 @@ +vars: + endpoints: + - https://epr.elastic.co diff --git a/test/packages/parallel/httpcheck/_dev/test/system/test-local-config.yml b/test/packages/parallel/httpcheck/_dev/test/system/test-local-config.yml new file mode 100644 index 0000000000..90b58daf76 --- /dev/null +++ b/test/packages/parallel/httpcheck/_dev/test/system/test-local-config.yml @@ -0,0 +1,4 @@ +service: web +vars: + endpoints: + - http://{{Hostname}}:{{Port}} diff --git a/test/packages/parallel/httpcheck/agent/input/input.yml.hbs b/test/packages/parallel/httpcheck/agent/input/input.yml.hbs new file mode 100644 index 0000000000..a127b79176 --- /dev/null +++ b/test/packages/parallel/httpcheck/agent/input/input.yml.hbs @@ -0,0 +1,20 @@ +receivers: + httpcheck: + collection_interval: {{period}} + targets: + - method: {{method}} + endpoints: +{{#each endpoints}} + - {{this}} +{{/each}} +{{#if headers }} + headers: +{{#each headers}} + {{this}} +{{/each}} +{{/if}} + +service: + pipelines: + metrics: + receivers: [httpcheck] diff --git a/test/packages/parallel/httpcheck/changelog.yml b/test/packages/parallel/httpcheck/changelog.yml new file mode 100644 index 0000000000..bb0320a524 --- /dev/null +++ b/test/packages/parallel/httpcheck/changelog.yml @@ -0,0 +1,6 @@ +# newer versions go on top +- version: "0.0.1" + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/1 # FIXME Replace with the real PR link diff --git a/test/packages/parallel/httpcheck/docs/README.md b/test/packages/parallel/httpcheck/docs/README.md new file mode 100644 index 0000000000..d5eafab45c --- /dev/null +++ b/test/packages/parallel/httpcheck/docs/README.md @@ -0,0 +1,84 @@ + + + +# Simple HTTP Check + + + +## Data streams + + + + + + + + + + + +## Requirements + +You need Elasticsearch for storing and searching your data and Kibana for visualizing and managing it. +You can use our hosted Elasticsearch Service on Elastic Cloud, which is recommended, or self-manage the Elastic Stack on your own hardware. + + + +## Setup + + + +For step-by-step instructions on how to set up an integration, see the +[Getting started](https://www.elastic.co/guide/en/welcome-to-elastic/current/getting-started-observability.html) guide. + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/packages/parallel/httpcheck/fields/base-fields.yml b/test/packages/parallel/httpcheck/fields/base-fields.yml new file mode 100644 index 0000000000..7c798f4534 --- /dev/null +++ b/test/packages/parallel/httpcheck/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/test/packages/parallel/httpcheck/img/sample-logo.svg b/test/packages/parallel/httpcheck/img/sample-logo.svg new file mode 100644 index 0000000000..6268dd88f3 --- /dev/null +++ b/test/packages/parallel/httpcheck/img/sample-logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/packages/parallel/httpcheck/img/sample-screenshot.png b/test/packages/parallel/httpcheck/img/sample-screenshot.png new file mode 100644 index 0000000000000000000000000000000000000000..d7a56a3ecc078c38636698cefba33f86291dd178 GIT binary patch literal 18849 zcmeEu^S~#!E#4Tq;}?6chqwB{?k=6jc5D4>l%v(rleJ2Y%tW zDj9g7px}|*e;{M?LDwiK3@FNS(lDRTd-MJYIyUJCN948~OJk1M(DrJyI#iV;P4k~& zFZo35IfQt0RwlUN`48^6(1dv_wm(y1xhEdMld=Y?!%u=fPT_*{3( zwBwz3#qR}_)t>C*jp5@U)Ti~B)Y;qq*TRxZJ7ZRN_^A3TDAEM*@7Ve%(Ro7=1%1B< zVj6GBUTxXev>_^SFA zgKZ=g4aTS}9>Ofj7cSB0WO?gQ)x=+!hs_)b$6#>ScFZ>XAoIX)%Bc|BDC~JFBk0f0 z0NY}6gb)&!qx^FWC(!ji+Kl$V$2|ocA=vN0TM0Y`U?tX+T)c*C zA!IL(T2Vm%MCLa85^if@J@Kkprx8QN5!6eCR@4Oa5S?4-4|ou?90mFCM8D!;n(5xz zO}-*t!TntN>|a$s(kGQg1P-U?hqvGF2_fGvd&~yZ_l3Qf&j~XWa=;>N3#-~#zjzcc z*m18L`A-K2o!d@J>a8SRbm4P&-q1(H>|JgIymDbnJF&@008`=X!P?4DGgZb>voUl^ zNJKgPR4S={)3vuk_{n@=M8q;;aJL>q+VLdTnO=}`&x;1DKjJA3*f*idS{jP5?+;!W zn-^7021Z4zv`Aq`hmX1aid997RNh3fa-@PG(W7TzKa1W&5^y3|lPeETP7j9qXpo4)7%(W0_2 z^Nmq;t@rb1eP3?%kOkH`P%!zTC7ZHjSfNN3*Sb#=3#jB*KpNGNfnRZ{N(6DrW(;B2Bwom<%m?VQP%K+ zsFeF1-(DY}oP@)w^Kw~gPg03q?N;)Ec6^|nikA34T~RynX*z}H>R~qgT$`Zbhn8wzZs$j2fsGN&rOK-mIBBvzD@a8FgbLpL!h5N^u&0wG} zq!#md3MHITv?3@$37J?lc_5*LWJTTjel;IiU-Yq;(g9I^D&KN_NKVS0O~GvB~FzPM6}=4d%fG4Nw4pZshcyLqK@`b8?RhD38haIyr@+8+0r5TC1*C7^WleJ zZN3_ngTD#RQvNL*;qD2H@cBWJbCC#d!}=oKfod5SE9a?!?j%DVt1z@inN}Iy$r+96 zM@P?AC+(`cM;z6J94BYGJ;+P-N#yj$?`G26ydS&OVH?~JY(N4l()Fh+x+DoJ@r<+i zhm^ck@QP`=fLApr62@KyOef~}zuG;(VbDQmw|Wb+oSHSw=%w9R)=et0cY*~ytX)#M zEXlK^p;zM@vTnXn+C1vwP)~TJv|TvDE2($;;EzC5_5IL#H;u z)#CO8)TSzbt8)wHB8$I8KcIojx&GoE)3QNu{CQ+_xBmQ&`mL5-u=BX(hs^hMY^ zae!!*Q;Tr$@(0~GoBJAohGw*d{l8~!aXop87aaSUb2jm)Tk>#$1*cdo5Sl+?oD!l4Og~yX+soottl4 zp4OartUuAN(dD~yLJ}`A1*!D4-|L^hM;`_DM^1KYs-VF(}h(BjRO``b+xV~%O=-)?p z7ciJH7Fnl?V&=ay_AB{oQoa2iR;6$^tiE|-eRCFy|3F@%j#6gUxkZX@?K`F$u#;T< z4IZORpUthmB?U`;zrOkp?P(Rvd5TFRWrBJmVg;KEZvJ+;Q}FRY%QZ?c^&$oPXW+C5 zdN#c>v%U?QuE+hMQdzxS1Q(BT90;29qu#^A?a^)Ui;{TJ;%`nLgm2ew$J4NvREjCJ z$`C7&?tH$CrVG@M3J1-KJw_*9BKeL*JX{ zN+Vg_TXb9^jJO$ZGkXO6BBFDjt~w5`w2TB*z$&1W5Il3IiDs=ZMDt|9iRtKET*wF6 z0Z+|N87p-5Fh)^(*l>OVr5^aY5LW(@PuM>Qo@&)yj6XRkPm1>eTF#Y_c*aRF^ZY5A z9FAU7lKEHG@i{wJMPg;n6z2|69d-)q9@<7t()d-zPy&X zdXG7{Uw{k23)CzzQAXw#iqj<1u~W@K_Ljc#?ukh;fRKHeJ2l~Z+52b2n^bGiDF2oX zm25FLx|4AP8>rAi@koY03lrtS#X?zK591c?2iZ_jjc>0y>q9>fU<08o6zG%z9WK+S zDwZMW4~28wu#ye#V*@#5t^S@NiAA`3{SF$xINmc_WW^u-C9M=H>RQ1>WM=|R!660{ z6E6%DwX`eu<3pkmz7Z=FCRd$(vhDkc3yMnSr)5C*aho)DZ<12$`$TXj<8Z70)|rK7 zXFD8QzksfWZU`qL2K8X{C~TcF{KVW`3Y{IMb&)T9%1V`tv(HY1 z+LXkLyM|3mtLD{x-#hOw-U?sr-iLeHFA|=-sGZ4#hX)atL!a91(tWJc+og&5W}VfZ zpgE7`{5D`~?yGR++y7~xA&eU0N*ZezDjF$> zUeK&1aTFQRg*?v^Z2e7u<`lk$czR6}b6Cl-qA9%A`#A6q0*zyTu)X`3rhjR86NK3= zLdw{+-F}+b2gxd-qF7>Rla}dFkj|L#c|pg5Ni+MRA|BZH(@ME*o<1ijKcoXb%PVfJ ztp_uf=G%kvU((pHcw90Xut=}atA!giM-5By)f40nKp zv7Wdb{;^<}VRvruH~rYr~wEuYY2ov-5Q|p@u3Da9+z7PeIpBAwi?RxnxN3Kt+N9L(LUS%wxY` z>e&1VV;{CYw8DNRlvBH)>!I49SU4R!t3I4=y;mCevPZh!-}~G+F>6hcL_Rli4r zC4(WN)`j$>^S=~GMGR=^)A6wrqi(-x{xK37&Vx!OS6t=KQ2JVZo#GrSODtTe=TVh%*qfF%91nqsMNLNL^Gp|_ zz%I*HUkMQGqb!1eh{{bp|0GSCDbkG_D_d)8<(0r<6-%Qi7qDa7xZjcdZ$?Rth9L!f z$erCcs3<~mtupywbaT8NWZF#v?iZkvqSz3@p`RiXs7P!GUa~-U9hEG(NgI#3BzO-# z!9JWf(;r!*A=@g$f}>wi|6Q@9z8AmYf~x8G%sp>C5cfuJY;hs1o3Ozu^{pH0AFbs%yU)Xy5>Cf?qXiHn*-PAfKDRiy`U0sFSKFsgEZ6_ z9#ma!<#Izr^}_z*>PRSt564u6We*XmZUx^jv*dK; z4zyFZ*ZFSE!00<6!|+#33&R)@RA8V9YRjp$HS9?CGq*xDSDRbX#i;}mateEF{fqTI zt?X}Efkq_Ap*_ETgaikOBbQ|;47}hwX44K`(DUI@C)QiG&6UJ1UmRn*Q@6%e`+x(gpQp74O{;yli8YLCV}qD z4gIyZd_(8ED~WWaeXOb0^r=9=AiDT}by~+$KVF~M{ywbQl zng-h?a_E;yX?DCr4|_h7JMc7>xgWf7Ek-VmH^hCYunVp3{(d{---&%-GZ=rK#V5Jo zJvP8b!2AA5?9)G8gwzB6ze3TU<5*Pqms^Q-?C9-CN~4hb-`U0D@kAkTWn23``cao^ z8IWAp8h7`%ZA+eI?w$sJktq5m>e&0@mQn>2BdpKAxbj1$m$8Z;`!iFvl9($Lb9Ff? zT^6cTZ~HgIeR6R*;G(rzpgsJP41Fx9Df;G6{;k6T(i}&8hX(jHSC@~#X@70h#)g(( z*9vUC+a*b%oAdf1$}Z3NR;|c5nY4^Z51pfqk(tmJbB;Q#ka#tf5eae;-kq$I{xO3<(TI$0lSe-JQzJ*es;il=Kn_?&?E zfLbs{qErPqm)-*ZfwbA*D-shgb|1;X;cH*yA|q8gS=HiosF=-kbdk6--SR+`F^H_` z0*i`J==@XSe=HT;_``G}ulE=H@*3GU*?gVd@h*`eT^GKjI;C@8+h~;(u3bA#b&bN{ zYw>dJ$(;RfHDLlndS`CWOE=g0jOocCc&;w(dOzrLf4-DK*MD@P_;u&CbfMw=#Q-B` zDq8hGwKN-O7(hQA_bP3f5XrZH+@*FGw~ppmDgNWcf|Lf*Pc%e5dw1DcJ1BWm!z7z3 zr^toEU*P(>G#;_1X}Rz(5lbDtCui%hY^d3lm)kw0vyk zX~K4$AG#7cG`6s2%9g9zsaQ9o?;3yzW4Pt!;NlS zzI#G7tiq&@eV&}qDtY(e$1JwscAfle%Al{3>Nr%``n?`Jac^CdOXUbFgI3;m{RkA~ zokl+lxuw9=%W&MmzA+G%ZdFMMP&N2^6BWjG2Lt|xKx)lMCR@b0n+xgw<)&Dwi?}>- z+$_e|@M;uW@3z6)q&L7bYitZ%huzGqH_qHOr&G5o!?(8TJv_MN1ka|&c6_!Q>#PgHSFoPWiLg|k_{ zQd#Zy&BPkU(0OE5S35!B5qb6%T3Wd#J(zBl8dw6I#xIDDF-LBPi-jXv1E?!gE|1OIdTejK)+U3ooC^otSIRsWZf-`&K}6}s!407Y58zH zK(oYx*7sN1O|Z_1YIJS_H$E@DH(hB4QKNCGQT3PTvwYoe2&8WKi5`5tU-r4!>_V3XUT}N)>8V;+z-!@-IGCKiD>E9RC(K`NMx=;Qp zf$2g^t?)zpU0L!BZi(oE#)^Z_biT*Svh>r#%1=O+Wo37G`Q)4@k#Pe?^mgBIugC)8 zyEICH=`{A~^x#X&%tr-$j|(nXrIrGQYNY+C3M+LO;yUU4-|v>a5#P)XYp>_|C0f0n{_p0mvwWmghfd%!Cm}$qBDxOqA3htLs~ghSA1>6^dVgd~ zVHHBBy6;Pp=El;dkTE=ttp~BoOJ$L@EB3Z37T1kTNG3tm4PY5O-7hP5DA$-k=vV&6 z?RiAm;W~*o)R7!x9>u$&@|&D4xMmJ*y+^-6t!F0u8G~78t&Bs#W>w_NbW>W9M3tXWXRf zI86FWVx%iXXh6MJ>dg#?lNu{K@S#nzMIG4PXQd%!Bvc*H0c7F_Y=adptJr*cHevMQ z%?Xu~q8CFw>^L*S_83kVhq=)hf0%_Lq}SE*g(Da_A{kXVZfAd*YCwp~bG32wi&SNM z#QZ7}Ug5-=+s^uqAh_|}gzya<(&E?XAZ%0ybd9nraj?|z1YfPr*{N?Q{ji}YG`T#| z=uwJZHIMlsmevnenT#-)t$L*=2wh|1EYXW?_36TR?L!sUItJVxaC0$Gb|gq4{|4gA z(v0ODFj!T)jc5>65ys)* z7$aBHfbKdz@QJq1b`NT`344*g()$>5*Ey`TPB7WI;|_8o8t9-_4ikFub|I{66>ge> zHA+6onzFKY*eaiA!77SD*^&LyumAR6gSvxY6Q?;!AvI{rZ##!G$%ZfIgce4F`aF;e z?jVh%+B-vj69ei~bh_zA9w}S4B4rzRKQ1~u$gwVu_x5PlRKDXX2(_2Mm7fs%6{SS7Qh1gWT8xaxc=f8`mW38ukIZxwU;lmHABwFSg50*o zrj%f%j~IKR?N5Dxwrq|sTa?!pd{b3sFM&~{4~_^YH4$bI^Fq2W4-y`))^|7fS?i0) zJ&Z9wY!8%l7@gAr`2{fqA;L;ptQR*X2|xUtrT47KK%XN+dydN$*M?65LuXTRabgERR{n>;E;(&vS0_@COY!p<%5LsRqGpER%~YjkSK zwBo9-2|-ZFiU3TT&S+@}3gDT35t0IXTzX@yHA(v>Y8;-mZNySQ&fE7RJ1^tzJfvdApX& z*!+tE)Y{oR%jk8A)3EiI3i*(TOwP!;B3hAOj?KQ6^h-q~1V^166uYS~mH*2Hh*0}r z`R3u1#^LG9IW|^QT^|61H(T1Jz?n;(Z>52lU0BO>Q6*zgpP*gTFk2Uw)!3zt>3F~_ ztil4!R*-j}wjh%&(kSB%}X=u4RbFRp@^l+$SmM@nW9B;yGbf@nasjFMEE{m9Oe

}qal5$moSACwfNXLXG5|3R0AtBcN` z?%yS)&>O>sqxU64U~C3&Q^>z-Zt}WuX4Wh3dKj9EO zfSbV!c3e;EOeKHQmWEw#NM4;*tw-2o@x&kKT?rsmy-F|$jw-F>WgA7?C@{O1qPg*J zf92|RTBMh&ptHADFc{T+cB?+mOj>h2HKgwkxq6w&XBxPc?>=JKvU2K9aU93@vp-R% z{5T=P$9U}AYZ5QU{3%7}YZ+ACWXw#-U zWyxU(OP#Q9-2AeGmCwcp`zWghf2hvsOjWjDQbU?U`v0&a--f1`v0Bd8HLiLmo)PKz5!A1|XVO+89 zm3h2~6yI~cpWor!_yt-?Lt>z`c0a7cJAW)#d8N8nNIf0H<+v;s4{0guDD(?T7Z<~$ zd`$vpZ_QQgFaMT0_d5&+(jwGU?M1FqUu6wjA-9z?mRM}(CmSdK;2e$Na}F-8jbhgN z9)@AIQeghf{xCC^{9P%VdYW1PP#}2BJwWt z0Hd8%st1NK5%h+)UB^mVwh{e#8TIm$xxgGo6I5;e{~VUeeMGRpM_Z%=eH5$X1}?Z5 z`|*_Vp~K&ziz45-Ih9y>EOr(Buy0&n$dbQ4$5eSr=Ti z#~7^n8dmem;$0D4+6eV7&G2D~d@ z+R#u8+nw_N%7_U_1e53P?~&10^m|ZUXrZhVp04lQLsGos%0fRDhS=@>8TOAAxK;Cy z9GZw_1pfSxD5~xoR!INI?tU0wrKDd6^Tv{jL>`Xb49kBaNPlhMaIfh_nq_)zB7NcX z05XeQKz`@BDUx7*i!V~%dc8XQ#ngBw0A2tSr(npSCrNy5Z7>48v&Zz?0{%FRElh_h zN2|?#EhJL5HQMIu6m1=ypTR?tVymHK)xQvS9ir7FzMp?CjlND39PK`od#GytVhZWp zQ1@>MTE1*Ip>hnXSWa?XbMH#708@j12yPbm`JfcqIgmJepn$5YgkJn_%5I)mr`Q(k z-a0yFR3A`houhvf&|wNpIsV{2p%MqhR@`@R(l6`}iufEgI*UxWq~26?WTpZCV{JtG zYL?&#I98fyf_;2S0?_V{=Aa4t^x%vy$pF$_Lh7W2f*~5uPvGYh;vZhMv|u+Z?2t0~ zcYPXdxbg6OS*LUjR_=jLDt)ab6;?g1IuySLG@UE;jLpt-wjLX&RlY>fnd@f&?0NyT zht5vhP^};k6`U76$%&I)iWPNxG6KPjdh`S6>g9GN@;KObQsLG zKyjfrPR0PU1B0a0=)3@9eCDl?mB9rFdlTMtTAeZv2}F*|@JWleq2+H1bt>>x!^wTk z+I)cgsZwzCMwoRpW_*!3IySTQu!`HWugAXe(Ai(a9Rsu;*0#o6torxwNMxPzEAjt` z>70Vw;HCQ?AnP`RKQ;2R8h%;LI#tx^(MO*lMWJe4_?)Q571P`kTmN#(ez21V!<6+S z@Uap+y%#8&cGgdf+E@y$dUx3g#)=#5k31Vqv0p!%L`*=-PiQAiSg-d9lKRZQDuJ-| zA96zwwomG+4}X$vR*IU=NC!vL<`rUTbf_uRJC4FS;k&HtV<=<)p(qymH)=MDV^aqK z#%sid7K|~!H`J!7hRr~Z!emxgWq6#GpQs%c#BM+scvNGz|Gi4G`;8Z~dP8)+51iB8 zw)0fazNz5(iK$LJeC_4e^8&@wT(DZ~~>SStz3P(>V8CLNlZqgv=2K-|Lu~si@XFwMN>QE^k zVS2U_A?Q$?M`NkU}^!M8m%O&T=kW>dG}1s2I~hxp9Y=a=1XX-(fB5) zej3`e5Et~R^r%?CZK0)UZsF_+tSOGIBMdrtMf#oJjGF9U`*P8t>i*TWed$Z2WNUZ* z_1Qw4Yr+Q0@bD?hD0P-^v}?FpPBg~zz5~g@J#J76C695|P>1l;OS8%~hZh5&-9Ji# z50%&56ZK4FC9}{jHL0!=qo9Yd(GGHCEX2|-F(f}q6@NMT4P3rQd{Q!=bz-8N(Z^!N;;ZzAWRf@C?X>mG=_NgyQX_?Jv$m(9$W>P;+e}O|&w&DjbsJPdWp0A2$yLr*!BY73Z z5d*BCaTI)w=sTlofc>n}@v_tSXIK?8(g`G_06u>SD*fOZJ~visq3lBVS2+cf-r$UQ zZ(8A0g&5M$IV7w5nqL(m$VS0X?=yy-e6>S>Ca3wZNT)b{GF39_gJdONflqc-j$b~o z2l@@h{$KVfC)V?#We*)@xYC;L^<@cHo>8axRMbSzw|eYTl|8pkabsQJ(3`z{>5H}c z`psz_Y6t)hvzL^=}P#++XUl6v`-j)SuXd6BynjNZ!&c2hnyE&4*K$nXn31Zk)cm+lx;> zya{T?{MRtSu?^3Y9bS&O$*mW^vRUpv!J3Tz12?3&Y62b_oiZ$24O(75Z)JWb+Rj)ACbK`f<&tSwtT$|Sy z$41kRPiM-jnPY9PKrLyI`pHm6LusMsrO*HpmE){Kp1^u2t%6nW^;GB|!4k!Ik8oav zjM?DBKh9G@W0gEwiU-M}0B)}olvoM71RccgiZBCs)L?q_GX&JDhegx4k2&cNatr5w zU)1#2USb8&`etO5Vk z?0}K+*2*@a5yt*X{qg0@8jEz~jcylVj>-042p1PBnabI#xUiCRD!ouw3?u-wwsqwF z8(@m8-Lk7q@v154g6yvx_tRDa>}oqpVda)wfI9(;ZVGt1v^{<|X?vC_(i@IJC+2I_lusrT=$h zF1lPc*Neb`;Xgrdf`p$w)~MzQW0M3_FYRKu{2$VU82J^B=X1#^<&P$_`=S$Ey04WU zTxG;hrFNLhWC*p+sH3x=JVcBJ9*7>eO20)n671SxQhZQlHMRP8FyO}yai~OTsbms0 zQ3b$C1Cn!>jMHDq{VX1ab^~_Q!z+f75+_AuwiN0*wA_#M#0|rU{+NlB%>Y+TNT0Gj z`3^LKMSJjz2(?lwg~ixDl_5%rzzZ}o_6Fj9e)T7gpH4=BgT1zmwJpC@g(f%&0`}8B z%7Y&qlP3aFmI#nmT`|R3+Lwzp+PLXt|5g%vlY_$fvse7zjus0D0fA##r+i4G4K-2Y zC#H95NGoYfWP#ZF_v$^Li{PZpm}fc&)aL?5doPcb835Cr6`T+EzzcEvLtmXcbAb<^ zw!_Zgk6Az7YA@*vb)(G{_W-B|zrf76z^`X%jOgqIIaqi~5nUup3vugzzg&rA^w(zR z+qCzvIV~nGR=47pDOcNTzuBw#5a=<=DMvGa)g zPw$^pmq9Fg&b#BZrPSoml(149rZS!fioV*Dy$z440U3MXDJmI?RZqLy0}IKSxN)o( z8+8wIZs#q(|KTg6y;Z(=96>xfpUsr@SP}I^v zN^R;ZVrDaWmNrM5-<X@k6JyjvA3;jHhma|Y|7!Vk& zgf(UK_6~cC;!|b!YTjke=nBiUqQdb#I9TY}!s5P)H+^c;9cW(QO8O%n5J^8Xfktd*qrn)+?-gP`m%B&q zi^}7jKm`yMW8ITFOMN#!QIB6$SWx*75tnCMaNg*_J*WuwBh~AT>0($nS8%&zmFQDp z$dL65niDtTV%!Kg1`6epWoQGNG`$`doy;Zjaa`keyL0F6iJMae6FIgnhAfzU%m@V+ zm5rQihLwS~b6{-bVR1ZSzBI7(Yj+V6T-8V*7I`ptWArGdy~8pnV>fALpi~NQLZ7;^ zpaj35=md<~-(tNmF69UX3?ua}A7UIn)q5i1iPYEGlhYSbkfeX`5epkxtzk3Qbu| zlgA`7ts%IvF4HJ}-98akyRnjCo{u-`A4&b+r?s|o`4wdYAHs-yh91p$7C_|+EdYH5 z10`!*=n+W9g>V&dfU1H!J}ASZi&-?`2IlDOAHnu306rD`y>jT)4^@S(X4XhN2{g9i zj-ym98+RT|d0ejIFJCM5>S{mT-8uGmRRqkJ3sMO_AQDrv77Q zv$t>zaVpVF6eBguE%9M2u?E-Oleft8z5+~W`G}KXD(Yc;7m4{Op>Le(k`g1UK7(1# zt6g}$n=Tdn{T4pu>v!c;xRCd_WI$Ali13x=U_0T!Ga-U~9W88q-lU+RLn2`N8Ouho z^0@SvC>$DguHWx)?^*ms-{PVq%dn(U3vrLj9zITDqQZ`H>Wsp@Gf%}SG=m)Vh}F$ztQAbwVGdDgd!28j&yX9wLW&s! zNR~6`nYg;ULAq8zi<;gUchAV5ib67Y##l2 zy+%gaD(|~G4@||{A;TYDSoS>q2o{t23t-^!NDSDEm8j3ao7Ei>KYLEpb$jz}7ciAM zD}trDN+AVVT_lXW<++~>8>Cj8fzJo@R;>%nGq)6+w?(#mNc#1J4W+!hA}?g$0Xqo? zn67qJmss)e%k(xO*&K@z6+}nHA(lCkb6n-|{pSztys$8HiOWTVR)tCO*Q9~if%3n7`uxGzE+OCu zwcVV|tgQdq60952$>85-GHk$lwM(uI+CU1?i{sVnKd0+UNq#eSSKjUKfDDgLnBG1y z^v?f#MRFkph~TgkoKBvM`L_~we8__xpLcjh`GwV|87q`vazJq?SX=mXhdvK>VqUf~ z4sYoTIpt5S)KrE-?>&=cRoBumD7;b5pq!Y07)#I$`)<@U+mo*dE*P~773p*u^6waO z2#thJahX_ySlYMpjx%h<)i43ao~Is`^Ya zMNZkuChEA7+ZJe6$>-C*dzTYf3#1SY82yFG?S&Q)5rTbKS-XLjckTLEc7>^sFcntQ zBeNXCSg&q1N3Bi^4zlQ%mcEBQ%2ab$?(;t-$HYd2%cnX$uuwU#I_6D3($m zR(>gHzM9ODf;r8b0l5LuEIQVZiQ0-|3Y_xzJkZc*CD=bPJ+&J+>>se%D4uTq?Ny{l z0Z5~og*Wa1O&anlcRWu_%o)(x?IZ0CfUNk_R-ik>GyvdFmpu1wHZaKTDGhL zqxsji)n<+)VKbV0_BRq9E;Kb`f=&vn(BK0Ba-gL?ZN;^^b3YFg6R=!q#zM;tcX0dM zdy5PPx@6pJPXHzH7$dGjM|6@6777nXPWV;CIQdNf(*Znv)sMy&Xcq> zhCq+6h6&v8<0}vd2(sKqU3j>fr7&#Xy%qZHcMU3m{wld^Nstkz8GagB?Y=SI&H z&{&BSA-|(i35$9(l6LpFyLm$0M0fK`Dz!~ezL?yEInsXAFR!bHe;ZL>Gd(#Hv?<$%`^b)oi?x%(jkylCPb=juPlF znMo&o961=NZ_$gd{xp1ZY2dNDOS!=XVj!M^A z+$z`EK4v=m{Bs{&I4W)({`&<5*^BV#z{IBAI_d+9Qx;~ zby?2zEjzUUeZWBDo5cz>%;z||z)<+6UtC)y60yD5J5`oo_zSM;l21@CY<0_|)NME5 zs)kHCMBa5YzB#N=W2aR?y9((~WuYwwf+HAc2mvU>NYlxOTvGf^Ye3za?*f-qUs^`a zT3>RPh9*Jf%3*bf|kqtnD_Buxv!<9N>BbuD#uYv-q^ z%RDnd7a3O4M9Y~TNISS@9K}JDkdg@>x8E6@n8jF=6qiDV+}{!V)(o?ykcr0sxBGEx zo!X;pc=r{H^vw6ztV5VZXBa4~(ujB$rZQ|AaGN@J7#q%2nU9gJ)g6dcj}zYB1& z@iFE0vMQVxa|v7tDHS$gwX$Ihc#M^DXRC>J@Zk?dC(3uB_s~*W&m-01DFMQGWjj5x z5po1@1gPl!v1Yra@qPG{D;$bYLM3qOwpl~7f~l)#n< zP+6`!NYe3EE~4RFR#_e=7YctPRBt6$He@`%e5m}f$M%yzC2S0<1}hRPjO>HJY~ z*dx(nbMbjv*;o&k{qzBdF|lS;UNVKziV=gbLq}UOCwr8GT5E9oRYQ}+>DhbQ1R=lj zgcNJN8|D)$Mx3#c+t@lhqcDUnHGVt0&EyQ{b5)=52B(VTzw=pQ^ba3`JB@BU^lS`_ zJEiLzgU#Acd_!}FMxCWC**FP^i#P}bYzNs78)#uSejEtYLbG>JJ7Igtho2oKQ;XW~ z4eMGO+t!_;G^V6c&R`5Tg+Pz2ToN(aybq4Q0ssie_{`t*DO%V7FaZ`{MBobFc9|pV z70o5ayHGJo9$$&Pgbs)pWNzduAcbh?~U?_P)(ve0S*3H%eNF&a5XR=!J#4c z;t992n7ZJr{*%`^dU1d-ALE8!3i#v;3r4r%j+JFCe=%3Vj=8{aXe zs)jrcUBZ=;LudcTUXj2ub>K5!{HHFHJ}Trx(PYugbQ8yK7&sqX;(;|UWjk3tGs3zuceeX)i4i_jA8Qz2Bc%DxN8 zXw!$+9jBtEHd1y90bYG4f8DcJM)Ab!M39tH5zz94*MAvnhA377@buNupSOUU3j8~> zd6&hk^ENRCp9T?_QUHk<=(&9Q^MJ^pi;nKOYNR@?L=RCSmKMJ5UQJQ`X!i~(gD*P! zs`RobzJG3Ra_Pg+WZUXUmMU$ilpwfcEti6)mw(~MZ0q!^sza>#jv!-+7B6F3QuMWg zVO!rXwD+lF1BBTito?ml-CV3vxuek~TKuOX^N6sol$v*{_%nAuD7i81eXm^Lz(Z~I z2Xj_Dts#G0&C;PV_Wkq*1QvB7+Post4={v;gk7b9u%#DC_bh(iJm$rqog^{JEx6NE zrs5^2SEL$|98#2WV#iG@L6cq|)SuTMSfGocPl65wUd^|5Lbpnb(;t>-Qu2jvANLgv zdte0vED-3C@^BdyHWLL(7{G$WA02z@JG!T-U^Q7HZ(7Bs&vchkh(p&}KvnS{MG^i6 z4r){gJp9p7WyWOEiKA2Cm6EXIn&&gk|Fc6^78OpPrX4ExCFE=SD$xcH;C2eB^{XTI zaxz_Cef*Yj==w_i_BTGXP;8C&f? z*QEM>={jFM8)lWAR870pG4XEWsl%%K|82S5b=9hVz7p_6i-d(Iyvq76&a#PV zR;VbQV|n?mg}&(ehClg%tK%IjgtnTR-u)lxH06XxXqH0soAZbB_Rm)XX=6Nge1uoG7 z9vQM_S~2h53n|W`y{{R9+=08rv~MohI_v4-BU^7fZ0-A}#b5{AOSTJm+(J;9yw%pD zX6u62GJ&@HKX5zQwq~j8T!Hrv-Mk^QSB5cu09L03{ToDO7jikM0WAcsjW>D}^jqCF zT0DEZ@K^KO_MD*%M!+V)lGVU6?LpX)eQVXEmq}R`NIJv;kBitJ!nW?0OxTVlu2ADf zE{A!*0g3%nwVcBD+AgT5bGx@WOnQk{zRpiZ4HhP`3BF%N|HdqPbbiV5)7x)kzC3ID zZ;27>0^mrMgWc7evsbQY`l`l})wr+e;=8U_!2&B77;1qL!N8y)eTJ2lf#CvhR~!Qa mc;sM|90DP5A*JW%f2r=u1xt!e4gwD_V(@hJb6Mw<&;$SznOm^{ literal 0 HcmV?d00001 diff --git a/test/packages/parallel/httpcheck/manifest.yml b/test/packages/parallel/httpcheck/manifest.yml new file mode 100644 index 0000000000..7d69d7ec00 --- /dev/null +++ b/test/packages/parallel/httpcheck/manifest.yml @@ -0,0 +1,76 @@ +format_version: 3.4.0 +name: httpcheck +title: "Simple HTTP Check" +version: 0.0.1 +source: + license: "Elastic-2.0" +description: "Perform HTTP checks using the HTTP Check receiver of the OTel Collector." +type: input +categories: + - monitoring + - network + - web +conditions: + kibana: + version: "^9.2.0" + elastic: + subscription: "basic" +screenshots: + - src: /img/sample-screenshot.png + title: Sample screenshot + size: 600x600 + type: image/png +icons: + - src: /img/sample-logo.svg + title: Sample logo + size: 32x32 + type: image/svg+xml +policy_templates: + - name: check + type: metrics + title: HTTP Check + description: Perform HTTP checks and collect its results. + input: otelcol + template_path: input.yml.hbs + vars: + - name: period + type: text + title: Collection Interval + multi: false + required: true + show_user: true + default: 1m + - name: method + type: select + title: "HTTP method to use" + multi: false + options: + - value: GET + text: GET + - value: HEAD + text: HEAD + - value: POST + text: POST + required: true + show_user: true + default: GET + - name: endpoints + type: text + title: HTTP endpoints to check + multi: true + required: true + show_user: true + - name: headers + type: text + title: HTTP headers + description: HTTP headers, defined as YAML key-values. + multi: true + required: false + show_user: false +elasticsearch: + index_template: + mappings: + subobjects: false +owner: + github: elastic/ecosystem + type: elastic From 6f8de382f5161ef7cdb63849253118063f2aff12 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Sep 2025 16:08:53 +0200 Subject: [PATCH 03/24] Add kibana feature flag --- internal/stack/_static/kibana.yml.tmpl | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/internal/stack/_static/kibana.yml.tmpl b/internal/stack/_static/kibana.yml.tmpl index a86be3d45c..60344d4f3c 100644 --- a/internal/stack/_static/kibana.yml.tmpl +++ b/internal/stack/_static/kibana.yml.tmpl @@ -41,9 +41,16 @@ xpack.fleet.registryUrl: "https://package-registry:8080" xpack.fleet.agents.enabled: true xpack.fleet.agents.fleet_server.hosts: ["{{ fact "fleet_url" }}"] -{{ if and (not (semverLessThan $version "8.7.0")) (semverLessThan $version "8.10.0-SNAPSHOT") }} -xpack.fleet.enableExperimental: ["experimentalDataStreamSettings"] # Enable experimental toggles in Fleet UI -{{ end }} +{{/* Feature flags introduced in 8.3.0 https://github.com/elastic/kibana/pull/130253 */}} +{{- if (not (semverLessThan $version "8.3.0")) }} +xpack.fleet.enableExperimental: [ +{{- if and (not (semverLessThan $version "8.7.0")) (semverLessThan $version "8.10.0-SNAPSHOT") }} + "experimentalDataStreamSettings" # Enable experimental toggles in Fleet UI +{{- else if (not (semverLessThan $version "9.2.0-SNAPSHOT")) }} + "enableOtelIntegrations" # Support for otelcol input +{{- end }} +] +{{- end }} xpack.encryptedSavedObjects.encryptionKey: "12345678901234567890123456789012" From b19a6d87e0073cb6445cd923d9be4cf61602937c Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Sep 2025 16:10:05 +0200 Subject: [PATCH 04/24] Remove debug level for fleet plugin --- internal/stack/_static/kibana.yml.tmpl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/internal/stack/_static/kibana.yml.tmpl b/internal/stack/_static/kibana.yml.tmpl index 60344d4f3c..f877181c76 100644 --- a/internal/stack/_static/kibana.yml.tmpl +++ b/internal/stack/_static/kibana.yml.tmpl @@ -168,7 +168,3 @@ xpack.fleet.outputs: {{- if eq $version "9.0.0-SNAPSHOT" }} xpack.fleet.internal.registry.kibanaVersionCheckEnabled: false {{- end }} - -logging.loggers: - - name: plugins.fleet - level: debug From 304ef01e8bf677c4c06f9be4d5edf86ecc56217e Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Sep 2025 16:21:10 +0200 Subject: [PATCH 05/24] Fix error in kibana config server.ssl for stack versions < 8.15.0 --- internal/stack/_static/kibana.yml.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/stack/_static/kibana.yml.tmpl b/internal/stack/_static/kibana.yml.tmpl index f877181c76..d119d6a0d5 100644 --- a/internal/stack/_static/kibana.yml.tmpl +++ b/internal/stack/_static/kibana.yml.tmpl @@ -4,7 +4,7 @@ server.host: "0.0.0.0" {{- $kibana_http2_enabled := fact "kibana_http2_enabled" -}} {{- if (and (eq $kibana_http2_enabled "true") (not (semverLessThan $version "8.15.0-SNAPSHOT"))) }} server.protocol: http2 -{{ end -}} +{{- end }} server.ssl.enabled: true server.ssl.certificate: "/usr/share/kibana/config/certs/cert.pem" server.ssl.key: "/usr/share/kibana/config/certs/key.pem" From 6b25a6056719deb84d2ab8ce3b2e8ac337f4b184 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Sep 2025 16:21:56 +0200 Subject: [PATCH 06/24] Move comments --- internal/stack/_static/kibana.yml.tmpl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/stack/_static/kibana.yml.tmpl b/internal/stack/_static/kibana.yml.tmpl index d119d6a0d5..7cd96e60d0 100644 --- a/internal/stack/_static/kibana.yml.tmpl +++ b/internal/stack/_static/kibana.yml.tmpl @@ -45,9 +45,11 @@ xpack.fleet.agents.fleet_server.hosts: ["{{ fact "fleet_url" }}"] {{- if (not (semverLessThan $version "8.3.0")) }} xpack.fleet.enableExperimental: [ {{- if and (not (semverLessThan $version "8.7.0")) (semverLessThan $version "8.10.0-SNAPSHOT") }} - "experimentalDataStreamSettings" # Enable experimental toggles in Fleet UI + # Enable experimental toggles in Fleet UI + "experimentalDataStreamSettings" {{- else if (not (semverLessThan $version "9.2.0-SNAPSHOT")) }} - "enableOtelIntegrations" # Support for otelcol input + # Support for otelcol input + "enableOtelIntegrations" {{- end }} ] {{- end }} From b649672b0a40fffbf6df8342da4049f0989d1d61 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Sep 2025 16:56:17 +0200 Subject: [PATCH 07/24] Add namespaces field to test package --- .../parallel/httpcheck/_dev/test/policy/test-eprcheck.expected | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/packages/parallel/httpcheck/_dev/test/policy/test-eprcheck.expected b/test/packages/parallel/httpcheck/_dev/test/policy/test-eprcheck.expected index aa549f1e32..d85004838f 100644 --- a/test/packages/parallel/httpcheck/_dev/test/policy/test-eprcheck.expected +++ b/test/packages/parallel/httpcheck/_dev/test/policy/test-eprcheck.expected @@ -1,4 +1,6 @@ inputs: [] +namespaces: + - default output_permissions: default: _elastic_agent_checks: From 0ad6db6e58327f327e760c0ce82ca28494543862 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Sep 2025 16:59:17 +0200 Subject: [PATCH 08/24] Set specific version --- test/packages/parallel/httpcheck.stack_version | 1 + 1 file changed, 1 insertion(+) create mode 100644 test/packages/parallel/httpcheck.stack_version diff --git a/test/packages/parallel/httpcheck.stack_version b/test/packages/parallel/httpcheck.stack_version new file mode 100644 index 0000000000..69ff65ae4b --- /dev/null +++ b/test/packages/parallel/httpcheck.stack_version @@ -0,0 +1 @@ +9.2.0-SNAPSHOT From 37b53c999c04283179b9530cb3db60a8047ccc99 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Sep 2025 17:17:46 +0200 Subject: [PATCH 09/24] Do not replace values in memberReplace case --- internal/testrunner/runners/policy/policy.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/testrunner/runners/policy/policy.go b/internal/testrunner/runners/policy/policy.go index 9510add971..f84d87258c 100644 --- a/internal/testrunner/runners/policy/policy.go +++ b/internal/testrunner/runners/policy/policy.go @@ -251,10 +251,10 @@ func cleanPolicyMap(policyMap common.MapStr, entries []policyEntryFilter) (commo key = regexp.ReplaceAllString(k, replacement) m[key] = e } - strElement, ok := e.(string) - if ok && regexp.MatchString(strElement) { - m[key] = regexp.ReplaceAllString(strElement, replacement) - } + // strElement, ok := e.(string) + // if ok && regexp.MatchString(strElement) { + // m[key] = regexp.ReplaceAllString(strElement, replacement) + // } } case entry.elementsReplace != nil: list, ok := v.([]any) From 5fe45aac241c57bbbf9b68a9be6bca3aadcea3f5 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Sep 2025 17:20:41 +0200 Subject: [PATCH 10/24] Revert "Add namespaces field to test package" This reverts commit b649672b0a40fffbf6df8342da4049f0989d1d61. --- .../parallel/httpcheck/_dev/test/policy/test-eprcheck.expected | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/packages/parallel/httpcheck/_dev/test/policy/test-eprcheck.expected b/test/packages/parallel/httpcheck/_dev/test/policy/test-eprcheck.expected index d85004838f..aa549f1e32 100644 --- a/test/packages/parallel/httpcheck/_dev/test/policy/test-eprcheck.expected +++ b/test/packages/parallel/httpcheck/_dev/test/policy/test-eprcheck.expected @@ -1,6 +1,4 @@ inputs: [] -namespaces: - - default output_permissions: default: _elastic_agent_checks: From 94ed7143154458399e8848bd65a118898fec2603 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Sep 2025 17:58:00 +0200 Subject: [PATCH 11/24] Skip system test for httpcheck test package --- .../parallel/httpcheck/_dev/test/system/test-local-config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/packages/parallel/httpcheck/_dev/test/system/test-local-config.yml b/test/packages/parallel/httpcheck/_dev/test/system/test-local-config.yml index 90b58daf76..1710aec23e 100644 --- a/test/packages/parallel/httpcheck/_dev/test/system/test-local-config.yml +++ b/test/packages/parallel/httpcheck/_dev/test/system/test-local-config.yml @@ -1,3 +1,6 @@ +skip: + reason: Not supported system tests with input type otelcol. + link: https://github.com/elastic/elastic-package/issues/2835 service: web vars: endpoints: From ad580aea7f4344380f1e093f9575106397d8d4ae Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Sep 2025 18:48:49 +0200 Subject: [PATCH 12/24] Update spec version for httpcheck --- test/packages/parallel/httpcheck/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/packages/parallel/httpcheck/manifest.yml b/test/packages/parallel/httpcheck/manifest.yml index 7d69d7ec00..20af460097 100644 --- a/test/packages/parallel/httpcheck/manifest.yml +++ b/test/packages/parallel/httpcheck/manifest.yml @@ -1,4 +1,4 @@ -format_version: 3.4.0 +format_version: 3.5.0 name: httpcheck title: "Simple HTTP Check" version: 0.0.1 From b5ac0b50ec1b142ee933abaede491c68ce81d185 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Thu, 4 Sep 2025 18:50:50 +0200 Subject: [PATCH 13/24] Do not replace strings in memberReplace --- internal/testrunner/runners/policy/policy.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/internal/testrunner/runners/policy/policy.go b/internal/testrunner/runners/policy/policy.go index f84d87258c..fe7dd737db 100644 --- a/internal/testrunner/runners/policy/policy.go +++ b/internal/testrunner/runners/policy/policy.go @@ -251,10 +251,6 @@ func cleanPolicyMap(policyMap common.MapStr, entries []policyEntryFilter) (commo key = regexp.ReplaceAllString(k, replacement) m[key] = e } - // strElement, ok := e.(string) - // if ok && regexp.MatchString(strElement) { - // m[key] = regexp.ReplaceAllString(strElement, replacement) - // } } case entry.elementsReplace != nil: list, ok := v.([]any) From a5e135a37cc5fb3e772f7c1fba9b885cbf86ecde Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Fri, 5 Sep 2025 17:28:49 +0200 Subject: [PATCH 14/24] Update tests including more than one OTEL processor --- .../testrunner/runners/policy/policy_test.go | 118 +++++++++++++++++- 1 file changed, 112 insertions(+), 6 deletions(-) diff --git a/internal/testrunner/runners/policy/policy_test.go b/internal/testrunner/runners/policy/policy_test.go index 207cc43496..f8451fb4e5 100644 --- a/internal/testrunner/runners/policy/policy_test.go +++ b/internal/testrunner/runners/policy/policy_test.go @@ -121,7 +121,7 @@ output_permissions: - monitor _elastic_agent_monitoring: indices: [] - uuid-for-permissions-on-related-indices: + 8d024b11-4e82-4192-8e7f-be71d1b13aac: indices: - names: - metrics-*-* @@ -221,7 +221,7 @@ output_permissions: - monitor _elastic_agent_monitoring: indices: [] - uuid-for-permissions-on-related-indices: + bfe4f402-df02-4673-8a71-fd5b29f1e2f3: indices: - names: - metrics-*-* @@ -296,15 +296,22 @@ output_permissions: - monitor _elastic_agent_monitoring: indices: [] - uuid-for-permissions-on-related-indices: + 05c98f91-203c-44a9-bee7-dd621c9bd37e: indices: - names: - logs-*-* privileges: - auto_configure - create_doc +processors: + batch/11c35ad0-4351-49d4-9c78-fa679ce9d950: + send_batch_size: 10 + timeout: 1s + batch/e6e379c5-6446-4090-af10-a9e5f8fc4640: + send_batch_size: 10000 + timeout: 10s receivers: - httpcheck/componentid: + httpcheck/4bae34b3-8f66-49c1-b04f-d58af1b5f743: collection_interval: 1m targets: - endpoints: @@ -315,7 +322,10 @@ service: pipelines: logs: receivers: - - httpcheck/componentid + - httpcheck/4bae34b3-8f66-49c1-b04f-d58af1b5f743 + processors: + - batch/11c35ad0-4351-49d4-9c78-fa679ce9d950 + - batch/e6e379c5-6446-4090-af10-a9e5f8fc4640 `, found: ` @@ -325,15 +335,108 @@ output_permissions: _elastic_agent_checks: cluster: - monitor + aeb4d606-2d90-4b41-b231-27bfad6dea09: + indices: + - names: + - logs-*-* + privileges: + - auto_configure + - create_doc _elastic_agent_monitoring: indices: [] - uuid-for-permissions-on-related-indices: +processors: + batch/567fce7a-ff2e-4a6c-a32a-0abb4671b39b: + send_batch_size: 10 + timeout: 1s + batch/8ec6ee99-2176-4231-9668-908069c77784: + send_batch_size: 10000 + timeout: 10s +receivers: + httpcheck/b0f518d6-4e2d-4c5d-bda7-f9808df537b7: + collection_interval: 1m + targets: + - endpoints: + - https://epr.elastic.co + method: GET +secret_references: [] +service: + pipelines: + logs: + receivers: + - httpcheck/b0f518d6-4e2d-4c5d-bda7-f9808df537b7 + processors: + - batch/567fce7a-ff2e-4a6c-a32a-0abb4671b39b + - batch/8ec6ee99-2176-4231-9668-908069c77784 + +`, + equal: true, + }, + { + title: "otel hardcode expected ids", + expected: ` +inputs: [] +output_permissions: + default: + _elastic_agent_checks: + cluster: + - monitor + _elastic_agent_monitoring: + indices: [] + 05c98f91-203c-44a9-bee7-dd621c9bd37e: + indices: + - names: + - logs-*-* + privileges: + - auto_configure + - create_doc +processors: + batch/compomentid-0: + send_batch_size: 10 + timeout: 1s + batch/compomentid-1: + send_batch_size: 10000 + timeout: 10s +receivers: + httpcheck/compomentid-0: + collection_interval: 1m + targets: + - endpoints: + - https://epr.elastic.co + method: GET +secret_references: [] +service: + pipelines: + logs: + receivers: + - httpcheck/componentid-0 + processors: + - batch/componentid-0 + - batch/componentid-1 + +`, + found: ` +inputs: [] +output_permissions: + default: + _elastic_agent_checks: + cluster: + - monitor + aeb4d606-2d90-4b41-b231-27bfad6dea09: indices: - names: - logs-*-* privileges: - auto_configure - create_doc + _elastic_agent_monitoring: + indices: [] +processors: + batch/567fce7a-ff2e-4a6c-a32a-0abb4671b39b: + send_batch_size: 10 + timeout: 1s + batch/8ec6ee99-2176-4231-9668-908069c77784: + send_batch_size: 10000 + timeout: 10s receivers: httpcheck/b0f518d6-4e2d-4c5d-bda7-f9808df537b7: collection_interval: 1m @@ -347,6 +450,9 @@ service: logs: receivers: - httpcheck/b0f518d6-4e2d-4c5d-bda7-f9808df537b7 + processors: + - batch/567fce7a-ff2e-4a6c-a32a-0abb4671b39b + - batch/8ec6ee99-2176-4231-9668-908069c77784 `, equal: true, From 2e85b428735920003722d20ec2b933dd174f19a1 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 8 Sep 2025 15:50:41 +0200 Subject: [PATCH 15/24] Update otel IDs in test --- internal/testrunner/runners/policy/policy_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/testrunner/runners/policy/policy_test.go b/internal/testrunner/runners/policy/policy_test.go index f8451fb4e5..0d60a55a1c 100644 --- a/internal/testrunner/runners/policy/policy_test.go +++ b/internal/testrunner/runners/policy/policy_test.go @@ -390,14 +390,14 @@ output_permissions: - auto_configure - create_doc processors: - batch/compomentid-0: + batch/componentid-0: send_batch_size: 10 timeout: 1s - batch/compomentid-1: + batch/componentid-1: send_batch_size: 10000 timeout: 10s receivers: - httpcheck/compomentid-0: + httpcheck/componentid-0: collection_interval: 1m targets: - endpoints: From 50de2aa39b1a559b759a9cfd8c890cede50be780 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 8 Sep 2025 16:16:40 +0200 Subject: [PATCH 16/24] Replace otel IDs definitions and usages using regexes --- internal/testrunner/runners/policy/policy.go | 93 ++++++++++++++++---- 1 file changed, 74 insertions(+), 19 deletions(-) diff --git a/internal/testrunner/runners/policy/policy.go b/internal/testrunner/runners/policy/policy.go index fe7dd737db..ac4ca49022 100644 --- a/internal/testrunner/runners/policy/policy.go +++ b/internal/testrunner/runners/policy/policy.go @@ -13,6 +13,7 @@ import ( "path/filepath" "regexp" "slices" + "strconv" "strings" "github.com/pmezard/go-difflib/difflib" @@ -20,6 +21,7 @@ import ( "github.com/elastic/elastic-package/internal/common" "github.com/elastic/elastic-package/internal/kibana" + "github.com/elastic/elastic-package/internal/logger" ) func dumpExpectedAgentPolicy(ctx context.Context, kibanaClient *kibana.Client, testPath string, policyID string) error { @@ -71,6 +73,9 @@ func comparePolicies(expected, found []byte) (string, error) { if err != nil { return "", fmt.Errorf("failed to prepare found policy: %w", err) } + logger.Tracef("expected policy:\n%s", want) + logger.Tracef("found policy:\n%s", got) + if bytes.Equal(want, got) { return "", nil } @@ -167,33 +172,42 @@ var policyEntryFilters = []policyEntryFilter{ }}, // OTel Collector IDs are relevant, but just check that they are there. - {name: "extensions", memberReplace: &otelComponentIDReplace}, - {name: "receivers", memberReplace: &otelComponentIDReplace}, - {name: "processors", memberReplace: &otelComponentIDReplace}, - {name: "exporters", memberReplace: &otelComponentIDReplace}, - {name: "service.extensions", elementsReplace: &otelComponentIDReplace}, - - // TODO: The signals here will need patterns at some moment, as they can also contain ids. - {name: "service.pipelines.logs.receivers", elementsReplace: &otelComponentIDReplace}, - {name: "service.pipelines.logs.processors", elementsReplace: &otelComponentIDReplace}, - {name: "service.pipelines.logs.exporters", elementsReplace: &otelComponentIDReplace}, - {name: "service.pipelines.metrics.receivers", elementsReplace: &otelComponentIDReplace}, - {name: "service.pipelines.metrics.processors", elementsReplace: &otelComponentIDReplace}, - {name: "service.pipelines.metrics.exporters", elementsReplace: &otelComponentIDReplace}, - {name: "service.pipelines.traces.receivers", elementsReplace: &otelComponentIDReplace}, - {name: "service.pipelines.traces.processors", elementsReplace: &otelComponentIDReplace}, - {name: "service.pipelines.traces.exporters", elementsReplace: &otelComponentIDReplace}, + // {name: "extensions", memberUniqueReplace: &uniqueOtelComponentIDReplace}, + // {name: "receivers", memberUniqueReplace: &uniqueOtelComponentIDReplace}, + // {name: "processors", memberUniqueReplace: &uniqueOtelComponentIDReplace}, + // {name: "exporters", memberUniqueReplace: &uniqueOtelComponentIDReplace}, + // {name: "service.extensions", elementsReplace: &otelComponentIDReplace}, + + // // TODO: The signals here will need patterns at some moment, as they can also contain ids. + // {name: "service.pipelines.logs.receivers", elementsReplace: &otelComponentIDReplace}, + // {name: "service.pipelines.logs.processors", elementsReplace: &otelComponentIDReplace}, + // {name: "service.pipelines.logs.exporters", elementsReplace: &otelComponentIDReplace}, + // {name: "service.pipelines.metrics.receivers", elementsReplace: &otelComponentIDReplace}, + // {name: "service.pipelines.metrics.processors", elementsReplace: &otelComponentIDReplace}, + // {name: "service.pipelines.metrics.exporters", elementsReplace: &otelComponentIDReplace}, + // {name: "service.pipelines.traces.receivers", elementsReplace: &otelComponentIDReplace}, + // {name: "service.pipelines.traces.processors", elementsReplace: &otelComponentIDReplace}, + // {name: "service.pipelines.traces.exporters", elementsReplace: &otelComponentIDReplace}, } -var otelComponentIDReplace = policyEntryReplace{ - regexp: regexp.MustCompile(`^([^/]+)/.*$`), - replace: "$1/componentid", +// var otelComponentIDReplace = policyEntryReplace{ +// regexp: regexp.MustCompile(`^([^/]+)/.*$`), +// replace: "$1/componentid", +// } + +var uniqueOtelComponentIDReplace = policyEntryReplace{ + regexp: regexp.MustCompile(`^(\s{2,})([^/]+)/([^:]+):(\s*)$`), + replace: "$1$2/componentid-%s:$4", } +var otelComponentsIDsRegexp = regexp.MustCompile(`(?m)^(?:extensions|receivers|processors|exporters):\n(?:\s{2,}.+\n)+`) + // cleanPolicy prepares a policy YAML as returned by the download API to be compared with other // policies. This preparation is based on removing contents that are generated, or replace them // by controlled values. func cleanPolicy(policy []byte, entriesToClean []policyEntryFilter) ([]byte, error) { + policy = replaceOtelComponentIDs(policy) + var policyMap common.MapStr err := yaml.Unmarshal(policy, &policyMap) if err != nil { @@ -208,6 +222,47 @@ func cleanPolicy(policy []byte, entriesToClean []policyEntryFilter) ([]byte, err return yaml.Marshal(policyMap) } +func replaceOtelComponentIDs(policy []byte) []byte { + // Keep track of all the replacements performed to later replace the references in services.pipelines + replacementsDone := map[string]string{} + + // regex to find the otel components sections + // otelIDsRegexp := regexp.MustCompile(`(?m)^(?:extensions|receivers|processors|exporters):\n(?:\s{2,}.+\n)+`) + policy = otelComponentsIDsRegexp.ReplaceAllFunc(policy, func(match []byte) []byte { + logger.Tracef("found otel components section:\n%s", string(match)) + count := 0 + lines := bytes.Split(match, []byte("\n")) + for i, line := range lines { + line = bytes.TrimRight(line, "\r\n") + stringLine := string(line) + if strings.Contains(stringLine, ":") { + if uniqueOtelComponentIDReplace.regexp.MatchString(stringLine) { + replacement := fmt.Sprintf(uniqueOtelComponentIDReplace.replace, strconv.Itoa(count)) + count++ + lines[i] = []byte(uniqueOtelComponentIDReplace.regexp.ReplaceAllString(stringLine, replacement)) + logger.Tracef("matching line for unique replacement: %s - replacement: %s", stringLine, string(lines[i])) + + // store the otel ID found with the space indentation and the colon to be replaced later + otelID := strings.TrimSuffix(strings.TrimSpace(stringLine), ":") + replacementsDone[otelID] = strings.TrimSuffix(strings.TrimSpace(string(lines[i])), ":") + } + } + } + return bytes.Join(lines, []byte("\n")) + }) + + // Replace other references to the otel component IDs replaced before: + // service.extensions + // service.pipelines..(receivers|processors|exporters) + for original, replacement := range replacementsDone { + originalArrayItem := fmt.Sprintf("- %s", original) + replacedArrayItem := fmt.Sprintf("- %s", replacement) + + policy = bytes.ReplaceAll(policy, []byte(originalArrayItem), []byte(replacedArrayItem)) + } + return policy +} + func cleanPolicyMap(policyMap common.MapStr, entries []policyEntryFilter) (common.MapStr, error) { for _, entry := range entries { v, err := policyMap.GetValue(entry.name) From 075be2a5214ac9f459376bdac2751d0a7adbb547 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 8 Sep 2025 16:39:46 +0200 Subject: [PATCH 17/24] Allow to be otel IDs with empty maps --- internal/testrunner/runners/policy/policy.go | 29 ++++++++++++++----- .../testrunner/runners/policy/policy_test.go | 16 ++++++++++ 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/internal/testrunner/runners/policy/policy.go b/internal/testrunner/runners/policy/policy.go index ac4ca49022..48076223ca 100644 --- a/internal/testrunner/runners/policy/policy.go +++ b/internal/testrunner/runners/policy/policy.go @@ -196,11 +196,23 @@ var policyEntryFilters = []policyEntryFilter{ // } var uniqueOtelComponentIDReplace = policyEntryReplace{ - regexp: regexp.MustCompile(`^(\s{2,})([^/]+)/([^:]+):(\s*)$`), + regexp: regexp.MustCompile(`^(\s{2,})([^/]+)/([^:]+):(\s\{\}|\s*)$`), replace: "$1$2/componentid-%s:$4", } -var otelComponentsIDsRegexp = regexp.MustCompile(`(?m)^(?:extensions|receivers|processors|exporters):\n(?:\s{2,}.+\n)+`) +// otelComponentIDsRegexp is the regex to find otel components sections and their IDs to replace them with controlled values. +// It matches sections like: +// extensions: +// +// health_check/4391d954-1ffe-4014-a256-5eda78a71828: {} +// +// receivers: +// +// otlp/123e4567-e89b-12d3-a456-426614174000: +// protocols: +// grpc: {} +// http: {} +var otelComponentIDsRegexp = regexp.MustCompile(`(?m)^(?:extensions|receivers|processors|exporters):(?:\s\{\}\n|\n(?:\s{2,}.+\n)+)`) // cleanPolicy prepares a policy YAML as returned by the download API to be compared with other // policies. This preparation is based on removing contents that are generated, or replace them @@ -227,8 +239,7 @@ func replaceOtelComponentIDs(policy []byte) []byte { replacementsDone := map[string]string{} // regex to find the otel components sections - // otelIDsRegexp := regexp.MustCompile(`(?m)^(?:extensions|receivers|processors|exporters):\n(?:\s{2,}.+\n)+`) - policy = otelComponentsIDsRegexp.ReplaceAllFunc(policy, func(match []byte) []byte { + policy = otelComponentIDsRegexp.ReplaceAllFunc(policy, func(match []byte) []byte { logger.Tracef("found otel components section:\n%s", string(match)) count := 0 lines := bytes.Split(match, []byte("\n")) @@ -240,18 +251,20 @@ func replaceOtelComponentIDs(policy []byte) []byte { replacement := fmt.Sprintf(uniqueOtelComponentIDReplace.replace, strconv.Itoa(count)) count++ lines[i] = []byte(uniqueOtelComponentIDReplace.regexp.ReplaceAllString(stringLine, replacement)) - logger.Tracef("matching line for unique replacement: %s - replacement: %s", stringLine, string(lines[i])) // store the otel ID found with the space indentation and the colon to be replaced later - otelID := strings.TrimSuffix(strings.TrimSpace(stringLine), ":") - replacementsDone[otelID] = strings.TrimSuffix(strings.TrimSpace(string(lines[i])), ":") + otelID := strings.SplitN(strings.TrimSpace(stringLine), ":", 2)[0] + replacementsDone[otelID] = strings.SplitN(strings.TrimSpace(string(lines[i])), ":", 2)[0] + + logger.Tracef("replaced otel component ID %q with %q", otelID, replacementsDone[otelID]) } } } return bytes.Join(lines, []byte("\n")) }) - // Replace other references to the otel component IDs replaced before: + // Replace references in arrays to the otel component IDs replaced before. + // These references can be in: // service.extensions // service.pipelines..(receivers|processors|exporters) for original, replacement := range replacementsDone { diff --git a/internal/testrunner/runners/policy/policy_test.go b/internal/testrunner/runners/policy/policy_test.go index 0d60a55a1c..d1456995f8 100644 --- a/internal/testrunner/runners/policy/policy_test.go +++ b/internal/testrunner/runners/policy/policy_test.go @@ -303,6 +303,8 @@ output_permissions: privileges: - auto_configure - create_doc +extensions: + health_check/31c94f44-214a-4778-8a36-acc2634096f7: {} processors: batch/11c35ad0-4351-49d4-9c78-fa679ce9d950: send_batch_size: 10 @@ -319,6 +321,8 @@ receivers: method: GET secret_references: [] service: + extensions: + - health_check/31c94f44-214a-4778-8a36-acc2634096f7 pipelines: logs: receivers: @@ -344,6 +348,8 @@ output_permissions: - create_doc _elastic_agent_monitoring: indices: [] +extensions: + health_check/4391d954-1ffe-4014-a256-5eda78a71829: {} processors: batch/567fce7a-ff2e-4a6c-a32a-0abb4671b39b: send_batch_size: 10 @@ -360,6 +366,8 @@ receivers: method: GET secret_references: [] service: + extensions: + - health_check/4391d954-1ffe-4014-a256-5eda78a71829 pipelines: logs: receivers: @@ -389,6 +397,8 @@ output_permissions: privileges: - auto_configure - create_doc +extensions: + health_check/componentid-0: {} processors: batch/componentid-0: send_batch_size: 10 @@ -405,6 +415,8 @@ receivers: method: GET secret_references: [] service: + extensions: + - health_check/componentid-0 pipelines: logs: receivers: @@ -430,6 +442,8 @@ output_permissions: - create_doc _elastic_agent_monitoring: indices: [] +extensions: + health_check/4391d954-1ffe-4014-a256-5eda78a71828: {} processors: batch/567fce7a-ff2e-4a6c-a32a-0abb4671b39b: send_batch_size: 10 @@ -446,6 +460,8 @@ receivers: method: GET secret_references: [] service: + extensions: + - health_check/4391d954-1ffe-4014-a256-5eda78a71828 pipelines: logs: receivers: From 68f0e090463058c6bb7b15b3711a769b3a78aae4 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 8 Sep 2025 18:01:03 +0200 Subject: [PATCH 18/24] Add comments and remove unused code --- internal/testrunner/runners/policy/policy.go | 50 ++++---------------- 1 file changed, 8 insertions(+), 42 deletions(-) diff --git a/internal/testrunner/runners/policy/policy.go b/internal/testrunner/runners/policy/policy.go index 48076223ca..728cb8f979 100644 --- a/internal/testrunner/runners/policy/policy.go +++ b/internal/testrunner/runners/policy/policy.go @@ -73,8 +73,8 @@ func comparePolicies(expected, found []byte) (string, error) { if err != nil { return "", fmt.Errorf("failed to prepare found policy: %w", err) } - logger.Tracef("expected policy:\n%s", want) - logger.Tracef("found policy:\n%s", got) + logger.Tracef("expected policy after cleaning:\n%s", want) + logger.Tracef("found policy after cleaning:\n%s", got) if bytes.Equal(want, got) { return "", nil @@ -102,7 +102,6 @@ func expectedPathFor(testPath string) string { type policyEntryFilter struct { name string elementsEntries []policyEntryFilter - elementsReplace *policyEntryReplace memberReplace *policyEntryReplace onlyIfEmpty bool ignoreValues []any @@ -170,31 +169,8 @@ var policyEntryFilters = []policyEntryFilter{ {name: "data_stream.elasticsearch", onlyIfEmpty: true}, }}, }}, - - // OTel Collector IDs are relevant, but just check that they are there. - // {name: "extensions", memberUniqueReplace: &uniqueOtelComponentIDReplace}, - // {name: "receivers", memberUniqueReplace: &uniqueOtelComponentIDReplace}, - // {name: "processors", memberUniqueReplace: &uniqueOtelComponentIDReplace}, - // {name: "exporters", memberUniqueReplace: &uniqueOtelComponentIDReplace}, - // {name: "service.extensions", elementsReplace: &otelComponentIDReplace}, - - // // TODO: The signals here will need patterns at some moment, as they can also contain ids. - // {name: "service.pipelines.logs.receivers", elementsReplace: &otelComponentIDReplace}, - // {name: "service.pipelines.logs.processors", elementsReplace: &otelComponentIDReplace}, - // {name: "service.pipelines.logs.exporters", elementsReplace: &otelComponentIDReplace}, - // {name: "service.pipelines.metrics.receivers", elementsReplace: &otelComponentIDReplace}, - // {name: "service.pipelines.metrics.processors", elementsReplace: &otelComponentIDReplace}, - // {name: "service.pipelines.metrics.exporters", elementsReplace: &otelComponentIDReplace}, - // {name: "service.pipelines.traces.receivers", elementsReplace: &otelComponentIDReplace}, - // {name: "service.pipelines.traces.processors", elementsReplace: &otelComponentIDReplace}, - // {name: "service.pipelines.traces.exporters", elementsReplace: &otelComponentIDReplace}, } -// var otelComponentIDReplace = policyEntryReplace{ -// regexp: regexp.MustCompile(`^([^/]+)/.*$`), -// replace: "$1/componentid", -// } - var uniqueOtelComponentIDReplace = policyEntryReplace{ regexp: regexp.MustCompile(`^(\s{2,})([^/]+)/([^:]+):(\s\{\}|\s*)$`), replace: "$1$2/componentid-%s:$4", @@ -218,6 +194,10 @@ var otelComponentIDsRegexp = regexp.MustCompile(`(?m)^(?:extensions|receivers|pr // policies. This preparation is based on removing contents that are generated, or replace them // by controlled values. func cleanPolicy(policy []byte, entriesToClean []policyEntryFilter) ([]byte, error) { + // OTel Collector IDs are relevant, but it is needed to just check that they are there. + // This function replaces them with controlled values. + // This replacement needs to be done before unmarshalling the YAML, as the IDs are keys + // in maps, and using the policyEntryFilter with memberReplace will not keep the same ordering. policy = replaceOtelComponentIDs(policy) var policyMap common.MapStr @@ -234,13 +214,14 @@ func cleanPolicy(policy []byte, entriesToClean []policyEntryFilter) ([]byte, err return yaml.Marshal(policyMap) } +// replaceOtelComponentIDs finds OTel Collector component IDs in the policy and replaces them with controlled values. +// It also replaces references to those IDs in service.extensions and service.pipelines. func replaceOtelComponentIDs(policy []byte) []byte { // Keep track of all the replacements performed to later replace the references in services.pipelines replacementsDone := map[string]string{} // regex to find the otel components sections policy = otelComponentIDsRegexp.ReplaceAllFunc(policy, func(match []byte) []byte { - logger.Tracef("found otel components section:\n%s", string(match)) count := 0 lines := bytes.Split(match, []byte("\n")) for i, line := range lines { @@ -255,8 +236,6 @@ func replaceOtelComponentIDs(policy []byte) []byte { // store the otel ID found with the space indentation and the colon to be replaced later otelID := strings.SplitN(strings.TrimSpace(stringLine), ":", 2)[0] replacementsDone[otelID] = strings.SplitN(strings.TrimSpace(string(lines[i])), ":", 2)[0] - - logger.Tracef("replaced otel component ID %q with %q", otelID, replacementsDone[otelID]) } } } @@ -320,19 +299,6 @@ func cleanPolicyMap(policyMap common.MapStr, entries []policyEntryFilter) (commo m[key] = e } } - case entry.elementsReplace != nil: - list, ok := v.([]any) - if !ok { - return nil, fmt.Errorf("expected list, found %T", v) - } - regexp := entry.elementsReplace.regexp - replacement := entry.elementsReplace.replace - for i, e := range list { - strElement, ok := e.(string) - if ok && regexp.MatchString(strElement) { - list[i] = regexp.ReplaceAllString(strElement, replacement) - } - } default: if entry.onlyIfEmpty && !isEmpty(v, entry.ignoreValues) { continue From ea59a6b3d40a40502fd8511dd57475296464352b Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Mon, 8 Sep 2025 19:11:11 +0200 Subject: [PATCH 19/24] Update function documentation --- internal/testrunner/runners/policy/policy.go | 29 ++++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/internal/testrunner/runners/policy/policy.go b/internal/testrunner/runners/policy/policy.go index 728cb8f979..954875bf29 100644 --- a/internal/testrunner/runners/policy/policy.go +++ b/internal/testrunner/runners/policy/policy.go @@ -178,26 +178,26 @@ var uniqueOtelComponentIDReplace = policyEntryReplace{ // otelComponentIDsRegexp is the regex to find otel components sections and their IDs to replace them with controlled values. // It matches sections like: -// extensions: // -// health_check/4391d954-1ffe-4014-a256-5eda78a71828: {} +// extensions: +// health_check/4391d954-1ffe-4014-a256-5eda78a71828: {} // -// receivers: -// -// otlp/123e4567-e89b-12d3-a456-426614174000: -// protocols: -// grpc: {} -// http: {} +// receivers: +// httpcheck/b0f518d6-4e2d-4c5d-bda7-f9808df537b7: +// collection_interval: 1m +// targets: +// - endpoints: +// - https://epr.elastic.co +// method: GET var otelComponentIDsRegexp = regexp.MustCompile(`(?m)^(?:extensions|receivers|processors|exporters):(?:\s\{\}\n|\n(?:\s{2,}.+\n)+)`) // cleanPolicy prepares a policy YAML as returned by the download API to be compared with other // policies. This preparation is based on removing contents that are generated, or replace them // by controlled values. func cleanPolicy(policy []byte, entriesToClean []policyEntryFilter) ([]byte, error) { - // OTel Collector IDs are relevant, but it is needed to just check that they are there. - // This function replaces them with controlled values. - // This replacement needs to be done before unmarshalling the YAML, as the IDs are keys - // in maps, and using the policyEntryFilter with memberReplace will not keep the same ordering. + // Replacement of the OTEL component IDs needs to be done before unmarshalling the YAML. + // The OTEL IDs are keys in maps, and using the policyEntryFilter with memberReplace does + // not ensure to keep the same ordering. policy = replaceOtelComponentIDs(policy) var policyMap common.MapStr @@ -217,10 +217,8 @@ func cleanPolicy(policy []byte, entriesToClean []policyEntryFilter) ([]byte, err // replaceOtelComponentIDs finds OTel Collector component IDs in the policy and replaces them with controlled values. // It also replaces references to those IDs in service.extensions and service.pipelines. func replaceOtelComponentIDs(policy []byte) []byte { - // Keep track of all the replacements performed to later replace the references in services.pipelines replacementsDone := map[string]string{} - // regex to find the otel components sections policy = otelComponentIDsRegexp.ReplaceAllFunc(policy, func(match []byte) []byte { count := 0 lines := bytes.Split(match, []byte("\n")) @@ -233,7 +231,8 @@ func replaceOtelComponentIDs(policy []byte) []byte { count++ lines[i] = []byte(uniqueOtelComponentIDReplace.regexp.ReplaceAllString(stringLine, replacement)) - // store the otel ID found with the space indentation and the colon to be replaced later + // store the otel ID replaced without the space indentation and the colon to be replaced later + // (e.g. http_check/4391d954-1ffe-4014-a256-5eda78a71828 replaced by http_check/componentid-0) otelID := strings.SplitN(strings.TrimSpace(stringLine), ":", 2)[0] replacementsDone[otelID] = strings.SplitN(strings.TrimSpace(string(lines[i])), ":", 2)[0] } From 3750339edff11e00efc91a1b9899de8dca130790 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 9 Sep 2025 13:37:41 +0200 Subject: [PATCH 20/24] Add connectors to OTEL components regex Co-authored-by: Jaime Soriano Pastor --- internal/testrunner/runners/policy/policy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/testrunner/runners/policy/policy.go b/internal/testrunner/runners/policy/policy.go index 954875bf29..b4d1ef46ab 100644 --- a/internal/testrunner/runners/policy/policy.go +++ b/internal/testrunner/runners/policy/policy.go @@ -189,7 +189,7 @@ var uniqueOtelComponentIDReplace = policyEntryReplace{ // - endpoints: // - https://epr.elastic.co // method: GET -var otelComponentIDsRegexp = regexp.MustCompile(`(?m)^(?:extensions|receivers|processors|exporters):(?:\s\{\}\n|\n(?:\s{2,}.+\n)+)`) +var otelComponentIDsRegexp = regexp.MustCompile(`(?m)^(?:extensions|receivers|processors|connectors|exporters):(?:\s\{\}\n|\n(?:\s{2,}.+\n)+)`) // cleanPolicy prepares a policy YAML as returned by the download API to be compared with other // policies. This preparation is based on removing contents that are generated, or replace them From c68ae5ce5c76047e340f6ad77d317c0d837767cf Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 9 Sep 2025 13:51:45 +0200 Subject: [PATCH 21/24] Remove unnecessary if --- internal/testrunner/runners/policy/policy.go | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/internal/testrunner/runners/policy/policy.go b/internal/testrunner/runners/policy/policy.go index b4d1ef46ab..9da93e82a7 100644 --- a/internal/testrunner/runners/policy/policy.go +++ b/internal/testrunner/runners/policy/policy.go @@ -225,17 +225,15 @@ func replaceOtelComponentIDs(policy []byte) []byte { for i, line := range lines { line = bytes.TrimRight(line, "\r\n") stringLine := string(line) - if strings.Contains(stringLine, ":") { - if uniqueOtelComponentIDReplace.regexp.MatchString(stringLine) { - replacement := fmt.Sprintf(uniqueOtelComponentIDReplace.replace, strconv.Itoa(count)) - count++ - lines[i] = []byte(uniqueOtelComponentIDReplace.regexp.ReplaceAllString(stringLine, replacement)) - - // store the otel ID replaced without the space indentation and the colon to be replaced later - // (e.g. http_check/4391d954-1ffe-4014-a256-5eda78a71828 replaced by http_check/componentid-0) - otelID := strings.SplitN(strings.TrimSpace(stringLine), ":", 2)[0] - replacementsDone[otelID] = strings.SplitN(strings.TrimSpace(string(lines[i])), ":", 2)[0] - } + if uniqueOtelComponentIDReplace.regexp.MatchString(stringLine) { + replacement := fmt.Sprintf(uniqueOtelComponentIDReplace.replace, strconv.Itoa(count)) + count++ + lines[i] = []byte(uniqueOtelComponentIDReplace.regexp.ReplaceAllString(stringLine, replacement)) + + // store the otel ID replaced without the space indentation and the colon to be replaced later + // (e.g. http_check/4391d954-1ffe-4014-a256-5eda78a71828 replaced by http_check/componentid-0) + otelID := strings.SplitN(strings.TrimSpace(stringLine), ":", 2)[0] + replacementsDone[otelID] = strings.SplitN(strings.TrimSpace(string(lines[i])), ":", 2)[0] } } return bytes.Join(lines, []byte("\n")) From 640a70017f019919a59cfaf3414f0c68ebf9a619 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 9 Sep 2025 13:53:46 +0200 Subject: [PATCH 22/24] Use strings.Cut instead of strings.SplitN --- internal/testrunner/runners/policy/policy.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/testrunner/runners/policy/policy.go b/internal/testrunner/runners/policy/policy.go index 9da93e82a7..3ec598f5ce 100644 --- a/internal/testrunner/runners/policy/policy.go +++ b/internal/testrunner/runners/policy/policy.go @@ -232,8 +232,8 @@ func replaceOtelComponentIDs(policy []byte) []byte { // store the otel ID replaced without the space indentation and the colon to be replaced later // (e.g. http_check/4391d954-1ffe-4014-a256-5eda78a71828 replaced by http_check/componentid-0) - otelID := strings.SplitN(strings.TrimSpace(stringLine), ":", 2)[0] - replacementsDone[otelID] = strings.SplitN(strings.TrimSpace(string(lines[i])), ":", 2)[0] + otelID, _, _ := strings.Cut(strings.TrimSpace(stringLine), ":") + replacementsDone[otelID], _, _ = strings.Cut(strings.TrimSpace(string(lines[i])), ":") } } return bytes.Join(lines, []byte("\n")) From 0c5483c9ba7c95b7d7bc5b0f147ba8dfa65ad326 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 9 Sep 2025 14:24:59 +0200 Subject: [PATCH 23/24] Use scanner to read matching lines of the given policy --- internal/testrunner/runners/policy/policy.go | 22 ++++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/internal/testrunner/runners/policy/policy.go b/internal/testrunner/runners/policy/policy.go index 3ec598f5ce..1d10e67075 100644 --- a/internal/testrunner/runners/policy/policy.go +++ b/internal/testrunner/runners/policy/policy.go @@ -5,6 +5,7 @@ package policy import ( + "bufio" "bytes" "context" "errors" @@ -221,22 +222,25 @@ func replaceOtelComponentIDs(policy []byte) []byte { policy = otelComponentIDsRegexp.ReplaceAllFunc(policy, func(match []byte) []byte { count := 0 - lines := bytes.Split(match, []byte("\n")) - for i, line := range lines { - line = bytes.TrimRight(line, "\r\n") - stringLine := string(line) - if uniqueOtelComponentIDReplace.regexp.MatchString(stringLine) { + scanner := bufio.NewScanner(bytes.NewReader(match)) + var section strings.Builder + for scanner.Scan() { + line := scanner.Text() + if uniqueOtelComponentIDReplace.regexp.MatchString(line) { + originalOtelID, _, _ := strings.Cut(strings.TrimSpace(line), ":") + replacement := fmt.Sprintf(uniqueOtelComponentIDReplace.replace, strconv.Itoa(count)) count++ - lines[i] = []byte(uniqueOtelComponentIDReplace.regexp.ReplaceAllString(stringLine, replacement)) + line = uniqueOtelComponentIDReplace.regexp.ReplaceAllString(line, replacement) // store the otel ID replaced without the space indentation and the colon to be replaced later // (e.g. http_check/4391d954-1ffe-4014-a256-5eda78a71828 replaced by http_check/componentid-0) - otelID, _, _ := strings.Cut(strings.TrimSpace(stringLine), ":") - replacementsDone[otelID], _, _ = strings.Cut(strings.TrimSpace(string(lines[i])), ":") + replacementsDone[originalOtelID], _, _ = strings.Cut(strings.TrimSpace(string(line)), ":") } + section.WriteString(line + "\n") } - return bytes.Join(lines, []byte("\n")) + + return []byte(section.String()) }) // Replace references in arrays to the otel component IDs replaced before. From 7a7dc37dd4ea0ab1a95fe1cfdc9ecb3311996233 Mon Sep 17 00:00:00 2001 From: Mario Rodriguez Molins Date: Tue, 9 Sep 2025 16:22:14 +0200 Subject: [PATCH 24/24] Add variable to system test - httpcheck package Co-authored-by: Jaime Soriano Pastor --- .../parallel/httpcheck/_dev/test/system/test-local-config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/test/packages/parallel/httpcheck/_dev/test/system/test-local-config.yml b/test/packages/parallel/httpcheck/_dev/test/system/test-local-config.yml index 1710aec23e..918e1b323a 100644 --- a/test/packages/parallel/httpcheck/_dev/test/system/test-local-config.yml +++ b/test/packages/parallel/httpcheck/_dev/test/system/test-local-config.yml @@ -3,5 +3,6 @@ skip: link: https://github.com/elastic/elastic-package/issues/2835 service: web vars: + period: 1s endpoints: - http://{{Hostname}}:{{Port}}