From 3431f6dc6d6c50a1207e8fe8795bc4cdeb76f02d Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Fri, 7 Feb 2025 12:20:56 +0100 Subject: [PATCH 01/17] feat(opensearch): Add OpenSearch image --- conf.py | 2 + opensearch/Dockerfile | 77 +++++++++++++++++++ .../patches/2.18.0/01-opensearch_home.patch | 24 ++++++ opensearch/stackable/patches/apply_patches.sh | 44 +++++++++++ opensearch/versions.py | 7 ++ 5 files changed, 154 insertions(+) create mode 100644 opensearch/Dockerfile create mode 100644 opensearch/stackable/patches/2.18.0/01-opensearch_home.patch create mode 100755 opensearch/stackable/patches/apply_patches.sh create mode 100644 opensearch/versions.py diff --git a/conf.py b/conf.py index 23c5e54ce..77562ccf7 100644 --- a/conf.py +++ b/conf.py @@ -24,6 +24,7 @@ nifi = importlib.import_module("nifi.versions") omid = importlib.import_module("omid.versions") opa = importlib.import_module("opa.versions") +opensearch = importlib.import_module("opensearch.versions") spark_k8s = importlib.import_module("spark-k8s.versions") stackable_base = importlib.import_module("stackable-base.versions") superset = importlib.import_module("superset.versions") @@ -51,6 +52,7 @@ {"name": "nifi", "versions": nifi.versions}, {"name": "omid", "versions": omid.versions}, {"name": "opa", "versions": opa.versions}, + {"name": "opensearch", "versions": opensearch.versions}, {"name": "spark-k8s", "versions": spark_k8s.versions}, {"name": "stackable-base", "versions": stackable_base.versions}, {"name": "superset", "versions": superset.versions}, diff --git a/opensearch/Dockerfile b/opensearch/Dockerfile new file mode 100644 index 000000000..47c1aae40 --- /dev/null +++ b/opensearch/Dockerfile @@ -0,0 +1,77 @@ +ARG PRODUCT + +FROM opensearchproject/opensearch:${PRODUCT} AS source + +FROM stackable/image/java-devel AS builder + +ARG PRODUCT +ARG STACKABLE_USER_UID + +RUN < Date: Wed, 12 Feb 2025 17:56:41 +0100 Subject: [PATCH 02/17] feat(opensearch): Change JDK to version 17 --- opensearch/versions.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/opensearch/versions.py b/opensearch/versions.py index 1356fb3f9..33de093d6 100644 --- a/opensearch/versions.py +++ b/opensearch/versions.py @@ -1,7 +1,9 @@ versions = [ { "product": "2.18.0", - "java-base": "21", - "java-devel": "21", + # The performance analyzer works with JDK 17, but not with 21. + # https://github.com/opensearch-project/performance-analyzer-rca/issues/545 + "java-base": "17", + "java-devel": "17", }, ] From be90d10ff906bd76074c910a0227ed59699a985f Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Fri, 14 Feb 2025 11:14:08 +0100 Subject: [PATCH 03/17] feat(jdk-base): Create a jdk-base image for OpenSearch --- conf.py | 2 + jdk-base/Dockerfile | 59 +++++++++++++++++++++++++++++ jdk-base/licenses/STACKABLE_LICENSE | 43 +++++++++++++++++++++ jdk-base/versions.py | 26 +++++++++++++ opensearch/Dockerfile | 41 ++++++++++++++++---- opensearch/versions.py | 2 +- 6 files changed, 165 insertions(+), 8 deletions(-) create mode 100644 jdk-base/Dockerfile create mode 100644 jdk-base/licenses/STACKABLE_LICENSE create mode 100644 jdk-base/versions.py diff --git a/conf.py b/conf.py index 77562ccf7..7b0eaeea4 100644 --- a/conf.py +++ b/conf.py @@ -18,6 +18,7 @@ hive = importlib.import_module("hive.versions") java_base = importlib.import_module("java-base.versions") java_devel = importlib.import_module("java-devel.versions") +jdk_base = importlib.import_module("jdk-base.versions") kafka = importlib.import_module("kafka.versions") krb5 = importlib.import_module("krb5.versions") vector = importlib.import_module("vector.versions") @@ -46,6 +47,7 @@ {"name": "hive", "versions": hive.versions}, {"name": "java-base", "versions": java_base.versions}, {"name": "java-devel", "versions": java_devel.versions}, + {"name": "jdk-base", "versions": jdk_base.versions}, {"name": "kafka", "versions": kafka.versions}, {"name": "krb5", "versions": krb5.versions}, {"name": "vector", "versions": vector.versions}, diff --git a/jdk-base/Dockerfile b/jdk-base/Dockerfile new file mode 100644 index 000000000..582b65768 --- /dev/null +++ b/jdk-base/Dockerfile @@ -0,0 +1,59 @@ +# syntax=docker/dockerfile:1.10.0@sha256:865e5dd094beca432e8c0a1d5e1c465db5f998dca4e439981029b3b81fb39ed5 +# check=error=true + +# +# Provides the common Java Development Kit for SDP products +# +FROM stackable/image/vector + +ARG PRODUCT +ARG RELEASE="1" + +LABEL name="Stackable image for OpenJDK" \ + maintainer="info@stackable.tech" \ + vendor="Stackable GmbH" \ + version="${PRODUCT}" \ + release="${RELEASE}" \ + summary="The Stackable OpenJDK base image." \ + description="This image is the base image for all Stackable Java product images which require a JDK." + +# See: https://adoptium.net/en-gb/installation/linux/#_centosrhelfedora_instructions +RUN cat < /etc/yum.repos.d/adoptium.repo +[Adoptium] +name=Adoptium +# The adoptium mirror times-out often, so we have created a pull-through cache +# baseurl=https://packages.adoptium.net/artifactory/rpm/rhel/\$releasever/\$basearch +baseurl=https://build-repo.stackable.tech/repository/Adoptium/\$releasever/\$basearch +enabled=1 +gpgcheck=1 +gpgkey=https://packages.adoptium.net/artifactory/api/gpg/key/public +EOF + +RUN microdnf update && \ + microdnf install \ + # Needed to run Java programs + "temurin-${PRODUCT}-jdk" \ + # Needed, because otherwise e.g. Zookeeper fails with + # Caused by: java.io.FileNotFoundException: /usr/lib/jvm/java-11-openjdk-11.0.20.0.8-2.el8.x86_64/lib/tzdb.dat (No such file or directory) + tzdata-java \ + # Most of the Java tools needs at least "klist" and "kinit" for Kerberos integration + krb5-workstation \ + --nodocs && \ + microdnf clean all + +COPY java-base/licenses /licenses + +ENV JAVA_HOME="/usr/lib/jvm/temurin-${PRODUCT}-jdk" + +# This image doesn't include the development packages for Java. +# For images that need the devel package (ex. Spark) use this env variable to +# install the proper javac version. +# +# microdnf install java-${JAVA_VERSION}-openjdk-devel +# +ENV JAVA_VERSION=$PRODUCT + +# Mitigation for CVE-2021-44228 (Log4Shell) +# This variable is supported as of Log4j version 2.10 and +# disables the vulnerable feature +ENV LOG4J_FORMAT_MSG_NO_LOOKUPS=true diff --git a/jdk-base/licenses/STACKABLE_LICENSE b/jdk-base/licenses/STACKABLE_LICENSE new file mode 100644 index 000000000..1b9535c9e --- /dev/null +++ b/jdk-base/licenses/STACKABLE_LICENSE @@ -0,0 +1,43 @@ +Licensed under the Open Software License version 3.0 + +1) Grant of Copyright License. Licensor grants You a worldwide, royalty-free, non-exclusive, sublicensable license, for the duration of the copyright, to do the following: + +a) to reproduce the Original Work in copies, either alone or as part of a collective work; + +b) to translate, adapt, alter, transform, modify, or arrange the Original Work, thereby creating derivative works ("Derivative Works") based upon the Original Work; + +c) to distribute or communicate copies of the Original Work and Derivative Works to the public, with the proviso that copies of Original Work or Derivative Works that You distribute or communicate shall be licensed under this Open Software License; + +d) to perform the Original Work publicly; and + +e) to display the Original Work publicly. + +2) Grant of Patent License. Licensor grants You a worldwide, royalty-free, non-exclusive, sublicensable license, under patent claims owned or controlled by the Licensor that are embodied in the Original Work as furnished by the Licensor, for the duration of the patents, to make, use, sell, offer for sale, have made, and import the Original Work and Derivative Works. + +3) Grant of Source Code License. The term "Source Code" means the preferred form of the Original Work for making modifications to it and all available documentation describing how to modify the Original Work. Licensor agrees to provide a machine-readable copy of the Source Code of the Original Work along with each copy of the Original Work that Licensor distributes. Licensor reserves the right to satisfy this obligation by placing a machine-readable copy of the Source Code in an information repository reasonably calculated to permit inexpensive and convenient access by You for as long as Licensor continues to distribute the Original Work. + +4) Exclusions From License Grant. Neither the names of Licensor, nor the names of any contributors to the Original Work, nor any of their trademarks or service marks, may be used to endorse or promote products derived from this Original Work without express prior permission of the Licensor. Except as expressly stated herein, nothing in this License grants any license to Licensor's trademarks, copyrights, patents, trade secrets or any other intellectual property. No patent license is granted to make, use, sell, offer for sale, have made, or import embodiments of any patent claims other than the licensed claims defined in Section 2. No license is granted to the trademarks of Licensor even if such marks are included in the Original Work. Nothing in this License shall be interpreted to prohibit Licensor from licensing under terms different from this License any Original Work that Licensor otherwise would have a right to license. + +5) External Deployment. The term "External Deployment" means the use, distribution, or communication of the Original Work or Derivative Works in any way such that the Original Work or Derivative Works may be used by anyone other than You, whether those works are distributed or communicated to those persons or made available as an application intended for use over a network. As an express condition for the grants of license hereunder, You must treat any External Deployment by You of the Original Work or a Derivative Work as a distribution under section 1(c). + +6) Attribution Rights. You must retain, in the Source Code of any Derivative Works that You create, all copyright, patent, or trademark notices from the Source Code of the Original Work, as well as any notices of licensing and any descriptive text identified therein as an "Attribution Notice." You must cause the Source Code for any Derivative Works that You create to carry a prominent Attribution Notice reasonably calculated to inform recipients that You have modified the Original Work. + +7) Warranty of Provenance and Disclaimer of Warranty. Licensor warrants that the copyright in and to the Original Work and the patent rights granted herein by Licensor are owned by the Licensor or are sublicensed to You under the terms of this License with the permission of the contributor(s) of those copyrights and patent rights. Except as expressly stated in the immediately preceding sentence, the Original Work is provided under this License on an "AS IS" BASIS and WITHOUT WARRANTY, either express or implied, including, without limitation, the warranties of non-infringement, merchantability or fitness for a particular purpose. THE ENTIRE RISK AS TO THE QUALITY OF THE ORIGINAL WORK IS WITH YOU. This DISCLAIMER OF WARRANTY constitutes an essential part of this License. No license to the Original Work is granted by this License except under this disclaimer. + +8) Limitation of Liability. Under no circumstances and under no legal theory, whether in tort (including negligence), contract, or otherwise, shall the Licensor be liable to anyone for any indirect, special, incidental, or consequential damages of any character arising as a result of this License or the use of the Original Work including, without limitation, damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses. This limitation of liability shall not apply to the extent applicable law prohibits such limitation. + +9) Acceptance and Termination. If, at any time, You expressly assented to this License, that assent indicates your clear and irrevocable acceptance of this License and all of its terms and conditions. If You distribute or communicate copies of the Original Work or a Derivative Work, You must make a reasonable effort under the circumstances to obtain the express assent of recipients to the terms of this License. This License conditions your rights to undertake the activities listed in Section 1, including your right to create Derivative Works based upon the Original Work, and doing so without honoring these terms and conditions is prohibited by copyright law and international treaty. Nothing in this License is intended to affect copyright exceptions and limitations (including "fair use" or "fair dealing"). This License shall terminate immediately and You may no longer exercise any of the rights granted to You by this License upon your failure to honor the conditions in Section 1(c). + +10) Termination for Patent Action. This License shall terminate automatically and You may no longer exercise any of the rights granted to You by this License as of the date You commence an action, including a cross-claim or counterclaim, against Licensor or any licensee alleging that the Original Work infringes a patent. This termination provision shall not apply for an action alleging patent infringement by combinations of the Original Work with other software or hardware. + +11) Jurisdiction, Venue and Governing Law. Any action or suit relating to this License may be brought only in the courts of a jurisdiction wherein the Licensor resides or in which Licensor conducts its primary business, and under the laws of that jurisdiction excluding its conflict-of-law provisions. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. Any use of the Original Work outside the scope of this License or after its termination shall be subject to the requirements and penalties of copyright or patent law in the appropriate jurisdiction. This section shall survive the termination of this License. + +12) Attorneys' Fees. In any action to enforce the terms of this License or seeking damages relating thereto, the prevailing party shall be entitled to recover its costs and expenses, including, without limitation, reasonable attorneys' fees and costs incurred in connection with such action, including any appeal of such action. This section shall survive the termination of this License. + +13) Miscellaneous. If any provision of this License is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. + +14) Definition of "You" in This License. "You" throughout this License, whether in upper or lower case, means an individual or a legal entity exercising rights under, and complying with all of the terms of, this License. For legal entities, "You" includes any entity that controls, is controlled by, or is under common control with you. For purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +15) Right to Use. You may use the Original Work in all ways not otherwise restricted or conditioned by this License or by law, and Licensor promises not to interfere with or be responsible for such uses by You. + +16) Modification of This License. This License is Copyright © 2005 Lawrence Rosen. Permission is granted to copy, distribute, or communicate this License without modification. Nothing in this License permits You to modify this License as applied to the Original Work or to Derivative Works. However, You may modify the text of this License and copy, distribute or communicate your modified version (the "Modified License") and apply it to other original works of authorship subject to the following conditions: (i) You may not indicate in any way that your Modified License is the "Open Software License" or "OSL" and you may not use those names in the name of your Modified License; (ii) You must replace the notice specified in the first paragraph above with the notice "Licensed under " or with a notice of your own that is not confusingly similar to the notice in this License; and (iii) You may not claim that your original works are open source software unless your Modified License has been approved by Open Source Initiative (OSI) and You comply with its license review and certification process. diff --git a/jdk-base/versions.py b/jdk-base/versions.py new file mode 100644 index 000000000..58e38ba05 --- /dev/null +++ b/jdk-base/versions.py @@ -0,0 +1,26 @@ +versions = [ + { + "product": "8", + "vector": "0.43.1", + }, + { + "product": "11", + "vector": "0.43.1", + }, + { + "product": "17", + "vector": "0.43.1", + }, + { + "product": "21", + "vector": "0.43.1", + }, + { + "product": "22", + "vector": "0.43.1", + }, + { + "product": "23", + "vector": "0.43.1", + }, +] diff --git a/opensearch/Dockerfile b/opensearch/Dockerfile index 47c1aae40..49158c054 100644 --- a/opensearch/Dockerfile +++ b/opensearch/Dockerfile @@ -1,7 +1,9 @@ ARG PRODUCT + FROM opensearchproject/opensearch:${PRODUCT} AS source + FROM stackable/image/java-devel AS builder ARG PRODUCT @@ -11,8 +13,7 @@ RUN < Date: Fri, 14 Feb 2025 11:33:29 +0100 Subject: [PATCH 04/17] feat(opensearch): Upgrade to version 2.19.0 --- .../patches/2.18.0/01-opensearch_home.patch | 24 ---- .../patches/2.19.0/01-opensearch_home.patch | 103 ++++++++++++++++++ opensearch/versions.py | 2 +- 3 files changed, 104 insertions(+), 25 deletions(-) delete mode 100644 opensearch/stackable/patches/2.18.0/01-opensearch_home.patch create mode 100644 opensearch/stackable/patches/2.19.0/01-opensearch_home.patch diff --git a/opensearch/stackable/patches/2.18.0/01-opensearch_home.patch b/opensearch/stackable/patches/2.18.0/01-opensearch_home.patch deleted file mode 100644 index 1db48831c..000000000 --- a/opensearch/stackable/patches/2.18.0/01-opensearch_home.patch +++ /dev/null @@ -1,24 +0,0 @@ -diff --git a/config/jvm.options b/config/jvm.options -index bcaa277..df9c34e 100644 ---- a/config/jvm.options -+++ b/config/jvm.options -@@ -89,5 +89,5 @@ - ## OpenDistro Performance Analyzer - -Dclk.tck=100 - -Djdk.attach.allowAttachSelf=true ---Djava.security.policy=/usr/share/opensearch/config/opensearch-performance-analyzer/opensearch_security.policy -+-Djava.security.policy=/stackable/opensearch/config/opensearch-performance-analyzer/opensearch_security.policy - --add-opens=jdk.attach/sun.tools.attach=ALL-UNNAMED -diff --git a/opensearch-docker-entrypoint.sh b/opensearch-docker-entrypoint.sh -index d998663..e1795ce 100755 ---- a/opensearch-docker-entrypoint.sh -+++ b/opensearch-docker-entrypoint.sh -@@ -9,7 +9,7 @@ - # in favor of a newly started container - - # Export OpenSearch Home --export OPENSEARCH_HOME=/usr/share/opensearch -+export OPENSEARCH_HOME=/stackable/opensearch - export OPENSEARCH_PATH_CONF=$OPENSEARCH_HOME/config - - # The virtual file /proc/self/cgroup should list the current cgroup diff --git a/opensearch/stackable/patches/2.19.0/01-opensearch_home.patch b/opensearch/stackable/patches/2.19.0/01-opensearch_home.patch new file mode 100644 index 000000000..f35a9155e --- /dev/null +++ b/opensearch/stackable/patches/2.19.0/01-opensearch_home.patch @@ -0,0 +1,103 @@ +diff --git a/config/jvm.options b/config/jvm.options +index bcaa277..df9c34e 100644 +--- a/config/jvm.options ++++ b/config/jvm.options +@@ -89,5 +89,5 @@ + ## OpenDistro Performance Analyzer + -Dclk.tck=100 + -Djdk.attach.allowAttachSelf=true +--Djava.security.policy=/usr/share/opensearch/config/opensearch-performance-analyzer/opensearch_security.policy ++-Djava.security.policy=/stackable/opensearch/config/opensearch-performance-analyzer/opensearch_security.policy + --add-opens=jdk.attach/sun.tools.attach=ALL-UNNAMED +diff --git a/config/opensearch-performance-analyzer/supervisord.conf b/config/opensearch-performance-analyzer/supervisord.conf +index e4c061c..b67886f 100644 +--- a/config/opensearch-performance-analyzer/supervisord.conf ++++ b/config/opensearch-performance-analyzer/supervisord.conf +@@ -1,13 +1,13 @@ + ; supervisor config file + + [unix_http_server] +-file=/usr/share/supervisor/performance_analyzer/supervisord.sock ++file=/stackable/supervisor/performance_analyzer/supervisord.sock + chmod=0770 + + [supervisord] +-logfile=/usr/share/supervisor/performance_analyzer/supervisord.log ; (main log file;default $CWD/supervisord.log) +-pidfile=/usr/share/supervisor/performance_analyzer/supervisord.pid ; (supervisord pidfile;default supervisord.pid) +-childlogdir=/usr/share/supervisor/performance_analyzer ; ('AUTO' child log dir, default $TEMP) ++logfile=/stackable/supervisor/performance_analyzer/supervisord.log ; (main log file;default $CWD/supervisord.log) ++pidfile=/stackable/supervisor/performance_analyzer/supervisord.pid ; (supervisord pidfile;default supervisord.pid) ++childlogdir=/stackable/supervisor/performance_analyzer ; ('AUTO' child log dir, default $TEMP) + + ; the below section must remain in the config file for RPC + ; (supervisorctl/web interface) to work, additional interfaces may be +@@ -16,7 +16,7 @@ childlogdir=/usr/share/supervisor/performance_analyzer ; ('AUTO' chi + supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + + [supervisorctl] +-serverurl=/usr/share/supervisord.sock ++serverurl=/stackable/supervisord.sock + + ; The [include] section can just contain the "files" setting. This + ; setting can list multiple files (separated by whitespace or +@@ -27,7 +27,7 @@ serverurl=/usr/share/supervisord.sock + files = /etc/supervisor/conf.d/*.conf + + [program:performance_analyzer] +-command=/usr/share/opensearch/performance-analyzer-rca/bin/performance-analyzer-agent /usr/share/opensearch ++command=/stackable/opensearch/performance-analyzer-rca/bin/performance-analyzer-agent /stackable/opensearch + user=1000 + autostart=true ; start at supervisord start (default: true) + autorestart=unexpected ; autorestart if exited after running (def: unexpected) +diff --git a/opensearch-docker-entrypoint.sh b/opensearch-docker-entrypoint.sh +index d998663..45ef018 100755 +--- a/opensearch-docker-entrypoint.sh ++++ b/opensearch-docker-entrypoint.sh +@@ -9,7 +9,6 @@ + # in favor of a newly started container + + # Export OpenSearch Home +-export OPENSEARCH_HOME=/usr/share/opensearch + export OPENSEARCH_PATH_CONF=$OPENSEARCH_HOME/config + + # The virtual file /proc/self/cgroup should list the current cgroup +diff --git a/performance-analyzer-rca/config/supervisord.conf b/performance-analyzer-rca/config/supervisord.conf +index e4c061c..b67886f 100644 +--- a/performance-analyzer-rca/config/supervisord.conf ++++ b/performance-analyzer-rca/config/supervisord.conf +@@ -1,13 +1,13 @@ + ; supervisor config file + + [unix_http_server] +-file=/usr/share/supervisor/performance_analyzer/supervisord.sock ++file=/stackable/supervisor/performance_analyzer/supervisord.sock + chmod=0770 + + [supervisord] +-logfile=/usr/share/supervisor/performance_analyzer/supervisord.log ; (main log file;default $CWD/supervisord.log) +-pidfile=/usr/share/supervisor/performance_analyzer/supervisord.pid ; (supervisord pidfile;default supervisord.pid) +-childlogdir=/usr/share/supervisor/performance_analyzer ; ('AUTO' child log dir, default $TEMP) ++logfile=/stackable/supervisor/performance_analyzer/supervisord.log ; (main log file;default $CWD/supervisord.log) ++pidfile=/stackable/supervisor/performance_analyzer/supervisord.pid ; (supervisord pidfile;default supervisord.pid) ++childlogdir=/stackable/supervisor/performance_analyzer ; ('AUTO' child log dir, default $TEMP) + + ; the below section must remain in the config file for RPC + ; (supervisorctl/web interface) to work, additional interfaces may be +@@ -16,7 +16,7 @@ childlogdir=/usr/share/supervisor/performance_analyzer ; ('AUTO' chi + supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + + [supervisorctl] +-serverurl=/usr/share/supervisord.sock ++serverurl=/stackable/supervisord.sock + + ; The [include] section can just contain the "files" setting. This + ; setting can list multiple files (separated by whitespace or +@@ -27,7 +27,7 @@ serverurl=/usr/share/supervisord.sock + files = /etc/supervisor/conf.d/*.conf + + [program:performance_analyzer] +-command=/usr/share/opensearch/performance-analyzer-rca/bin/performance-analyzer-agent /usr/share/opensearch ++command=/stackable/opensearch/performance-analyzer-rca/bin/performance-analyzer-agent /stackable/opensearch + user=1000 + autostart=true ; start at supervisord start (default: true) + autorestart=unexpected ; autorestart if exited after running (def: unexpected) diff --git a/opensearch/versions.py b/opensearch/versions.py index a007f1bed..d1024dcd2 100644 --- a/opensearch/versions.py +++ b/opensearch/versions.py @@ -1,6 +1,6 @@ versions = [ { - "product": "2.18.0", + "product": "2.19.0", # The performance analyzer works with JDK 17, but not with 21. # https://github.com/opensearch-project/performance-analyzer-rca/issues/545 "java-devel": "17", From bc4abeea4c20c07b56fac2b910f1f02da85c860e Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Mon, 31 Mar 2025 09:28:08 +0200 Subject: [PATCH 05/17] feat(opensearch): Upgrade to version 2.19.1 and install the plugin repository-s3 --- opensearch/Dockerfile | 6 +++++- .../patches/{2.19.0 => 2.19.1}/01-opensearch_home.patch | 0 opensearch/versions.py | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) rename opensearch/stackable/patches/{2.19.0 => 2.19.1}/01-opensearch_home.patch (100%) diff --git a/opensearch/Dockerfile b/opensearch/Dockerfile index 49158c054..fce9c5ad0 100644 --- a/opensearch/Dockerfile +++ b/opensearch/Dockerfile @@ -19,7 +19,7 @@ microdnf clean all rm -rf /var/cache/yum EOF -USER ${STACKABLE_USER_UID} +USER ${STACKABLE_USER_UID}:0 WORKDIR /stackable COPY \ @@ -38,6 +38,10 @@ COPY \ WORKDIR /stackable/opensearch +RUN < Date: Wed, 16 Jul 2025 19:10:18 +0200 Subject: [PATCH 06/17] wip: build from source --- conf.py | 7 + jdk-base/versions.py | 16 +- opensearch/Dockerfile | 75 +++--- opensearch/security-plugin/Dockerfile | 25 ++ .../stackable/patches/3.1.0.0/patchable.toml | 2 + .../stackable/patches/patchable.toml | 2 + opensearch/security-plugin/versions.py | 6 + .../patches/3.1.0/0001-opensearch-home.patch | 228 ++++++++++++++++++ .../stackable/patches/3.1.0/patchable.toml | 2 + opensearch/stackable/patches/patchable.toml | 2 + opensearch/versions.py | 9 +- 11 files changed, 332 insertions(+), 42 deletions(-) create mode 100644 opensearch/security-plugin/Dockerfile create mode 100644 opensearch/security-plugin/stackable/patches/3.1.0.0/patchable.toml create mode 100644 opensearch/security-plugin/stackable/patches/patchable.toml create mode 100644 opensearch/security-plugin/versions.py create mode 100644 opensearch/stackable/patches/3.1.0/0001-opensearch-home.patch create mode 100644 opensearch/stackable/patches/3.1.0/patchable.toml create mode 100644 opensearch/stackable/patches/patchable.toml diff --git a/conf.py b/conf.py index 914bc17fe..1adc195ae 100644 --- a/conf.py +++ b/conf.py @@ -30,6 +30,9 @@ omid = importlib.import_module("omid.versions") opa = importlib.import_module("opa.versions") opensearch = importlib.import_module("opensearch.versions") +opensearch_security_plugin = importlib.import_module( + "opensearch.security-plugin.versions" +) spark_k8s = importlib.import_module("spark-k8s.versions") stackable_base = importlib.import_module("stackable-base.versions") stackable_devel = importlib.import_module("stackable-devel.versions") @@ -68,6 +71,10 @@ {"name": "omid", "versions": omid.versions}, {"name": "opa", "versions": opa.versions}, {"name": "opensearch", "versions": opensearch.versions}, + { + "name": "opensearch/security-plugin", + "versions": opensearch_security_plugin.versions, + }, {"name": "spark-k8s", "versions": spark_k8s.versions}, {"name": "stackable-base", "versions": stackable_base.versions}, {"name": "stackable-devel", "versions": stackable_devel.versions}, diff --git a/jdk-base/versions.py b/jdk-base/versions.py index 58e38ba05..036d18203 100644 --- a/jdk-base/versions.py +++ b/jdk-base/versions.py @@ -1,26 +1,30 @@ versions = [ { "product": "8", - "vector": "0.43.1", + "vector": "0.47.0", }, { "product": "11", - "vector": "0.43.1", + "vector": "0.47.0", }, { "product": "17", - "vector": "0.43.1", + "vector": "0.47.0", }, { "product": "21", - "vector": "0.43.1", + "vector": "0.47.0", }, { "product": "22", - "vector": "0.43.1", + "vector": "0.47.0", }, { "product": "23", - "vector": "0.43.1", + "vector": "0.47.0", + }, + { + "product": "24", + "vector": "0.47.0", }, ] diff --git a/opensearch/Dockerfile b/opensearch/Dockerfile index fce9c5ad0..77bb7f1b6 100644 --- a/opensearch/Dockerfile +++ b/opensearch/Dockerfile @@ -1,50 +1,50 @@ -ARG PRODUCT - - -FROM opensearchproject/opensearch:${PRODUCT} AS source - -FROM stackable/image/java-devel AS builder +FROM stackable/image/opensearch/security-plugin AS opensearch-security-plugin +FROM stackable/image/java-devel AS opensearch-builder ARG PRODUCT +ARG RELEASE +ARG OPENSEARCH_SECURITY_PLUGIN ARG STACKABLE_USER_UID -RUN < +Date: Wed, 16 Jul 2025 14:29:10 +0200 +Subject: opensearch home + +--- + .../docker/src/docker/bin/docker-entrypoint.sh | 12 ++++++------ + distribution/packages/build.gradle | 18 +++++++++--------- + .../packages/src/common/env/opensearch | 2 +- + .../packages/src/common/scripts/postinst | 6 +++--- + .../packages/src/common/scripts/postrm | 8 ++++---- + .../src/common/systemd/opensearch.service | 4 ++-- + 6 files changed, 25 insertions(+), 25 deletions(-) + +diff --git a/distribution/docker/src/docker/bin/docker-entrypoint.sh b/distribution/docker/src/docker/bin/docker-entrypoint.sh +index 59603462ac9..2e5cc42be34 100644 +--- a/distribution/docker/src/docker/bin/docker-entrypoint.sh ++++ b/distribution/docker/src/docker/bin/docker-entrypoint.sh +@@ -46,7 +46,7 @@ fi + # This is also sourced in opensearch-env, and is only needed here + # as well because we use ELASTIC_PASSWORD below. Sourcing this script + # is idempotent. +-source /usr/share/opensearch/bin/opensearch-env-from-file ++source /stackable/opensearch/bin/opensearch-env-from-file + + if [[ -f bin/opensearch-users ]]; then + # Check for the ELASTIC_PASSWORD environment variable to set the +@@ -56,7 +56,7 @@ if [[ -f bin/opensearch-users ]]; then + # enabled, but we have no way of knowing which node we are yet. We'll just + # honor the variable if it's present. + if [[ -n "$ELASTIC_PASSWORD" ]]; then +- [[ -f /usr/share/opensearch/config/opensearch.keystore ]] || (run_as_other_user_if_needed opensearch-keystore create) ++ [[ -f /stackable/opensearch/config/opensearch.keystore ]] || (run_as_other_user_if_needed opensearch-keystore create) + if ! (run_as_other_user_if_needed opensearch-keystore has-passwd --silent) ; then + # keystore is unencrypted + if ! (run_as_other_user_if_needed opensearch-keystore list | grep -q '^bootstrap.password$'); then +@@ -73,7 +73,7 @@ if [[ -f bin/opensearch-users ]]; then + fi + fi + +-if ls "/usr/share/opensearch/lib" | grep -E -q "bc-fips.*\.jar"; then ++if ls "/stackable/opensearch/lib" | grep -E -q "bc-fips.*\.jar"; then + # If BouncyCastle FIPS is detected - enforcing keystore password policy. + + if [[ -z "$KEYSTORE_PASSWORD" ]]; then +@@ -81,7 +81,7 @@ if ls "/usr/share/opensearch/lib" | grep -E -q "bc-fips.*\.jar"; then + exit 1 + fi + +- if [[ ! -f /usr/share/opensearch/config/opensearch.keystore ]]; then ++ if [[ ! -f /stackable/opensearch/config/opensearch.keystore ]]; then + # Keystore not found - creating with password. + COMMANDS="$(printf "%s\n%s" "$KEYSTORE_PASSWORD" "$KEYSTORE_PASSWORD")" + echo "$COMMANDS" | run_as_other_user_if_needed opensearch-keystore create -p +@@ -99,8 +99,8 @@ fi + if [[ "$(id -u)" == "0" ]]; then + # If requested and running as root, mutate the ownership of bind-mounts + if [[ -n "$TAKE_FILE_OWNERSHIP" ]]; then +- chown -R 1000:0 /usr/share/opensearch/{data,logs} ++ chown -R 1000:0 /stackable/opensearch/{data,logs} + fi + fi + +-run_as_other_user_if_needed /usr/share/opensearch/bin/opensearch <<<"$KEYSTORE_PASSWORD" ++run_as_other_user_if_needed /stackable/opensearch/bin/opensearch <<<"$KEYSTORE_PASSWORD" +diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle +index a02f907f86a..eb9c048f32c 100644 +--- a/distribution/packages/build.gradle ++++ b/distribution/packages/build.gradle +@@ -88,7 +88,7 @@ void addProcessFilesTask(String type, boolean jdk) { + // create empty dirs, we set the permissions when configuring the packages + mkdir "${packagingFiles}/var/log/opensearch" + mkdir "${packagingFiles}/var/lib/opensearch" +- mkdir "${packagingFiles}/usr/share/opensearch/plugins" ++ mkdir "${packagingFiles}/stackable/opensearch/plugins" + + // bare empty dir for /etc/opensearch and /etc/opensearch/jvm.options.d + mkdir "${packagingFiles}/opensearch" +@@ -154,7 +154,7 @@ Closure commonPackageConfig(String type, boolean jdk, String architecture) { + + // top level "into" directive is not inherited from ospackage for some reason, so we must + // specify it again explicitly for copying common files +- into('/usr/share/opensearch') { ++ into('/stackable/opensearch') { + into('bin') { + with binFiles(type, jdk) + } +@@ -198,7 +198,7 @@ Closure commonPackageConfig(String type, boolean jdk, String architecture) { + + // license files + if (type == 'deb') { +- into("/usr/share/doc/${packageName}") { ++ into("/stackable/doc/${packageName}") { + from "${packagingFiles}/copyright" + filePermissions { + unix 0644 +@@ -206,7 +206,7 @@ Closure commonPackageConfig(String type, boolean jdk, String architecture) { + } + } else { + assert type == 'rpm' +- into('/usr/share/opensearch') { ++ into('/stackable/opensearch') { + from(rootProject.file('licenses')) { + include 'APACHE-LICENSE-2.0.txt' + rename { 'LICENSE.txt' } +@@ -277,7 +277,7 @@ Closure commonPackageConfig(String type, boolean jdk, String architecture) { + unix 0644 + } + } +- into('/usr/share/opensearch/bin') { ++ into('/stackable/opensearch/bin') { + from "${packagingFiles}/systemd/systemd-entrypoint" + filePermissions { + unix 0755 +@@ -312,9 +312,9 @@ Closure commonPackageConfig(String type, boolean jdk, String architecture) { + } + copyEmptyDir('/var/log/opensearch', 'opensearch', 'opensearch', 0750) + copyEmptyDir('/var/lib/opensearch', 'opensearch', 'opensearch', 0750) +- copyEmptyDir('/usr/share/opensearch/plugins', 'root', 'root', 0755) ++ copyEmptyDir('/stackable/opensearch/plugins', 'root', 'root', 0755) + +- into '/usr/share/opensearch' ++ into '/stackable/opensearch' + with noticeFile(jdk) + } + } +@@ -348,7 +348,7 @@ ospackage { + user = 'root' + permissionGroup = 'root' + +- into '/usr/share/opensearch' ++ into '/stackable/opensearch' + } + + Closure commonDebConfig(boolean jdk, String architecture) { +@@ -368,7 +368,7 @@ Closure commonDebConfig(boolean jdk, String architecture) { + requires 'libc6' + requires 'adduser' + +- into('/usr/share/lintian/overrides') { ++ into('/stackable/lintian/overrides') { + from('src/deb/lintian/opensearch') + filePermissions { + unix 0644 +diff --git a/distribution/packages/src/common/env/opensearch b/distribution/packages/src/common/env/opensearch +index a8b68297669..bd5ad77ded1 100644 +--- a/distribution/packages/src/common/env/opensearch ++++ b/distribution/packages/src/common/env/opensearch +@@ -3,7 +3,7 @@ + ################################ + + # OpenSearch home directory +-OPENSEARCH_HOME=/usr/share/opensearch ++OPENSEARCH_HOME=/stackable/opensearch + + # OpenSearch Java path + #OPENSEARCH_JAVA_HOME=/usr/lib/jvm/java-11-amazon-corretto +diff --git a/distribution/packages/src/common/scripts/postinst b/distribution/packages/src/common/scripts/postinst +index 308e86b8502..54baa373520 100644 +--- a/distribution/packages/src/common/scripts/postinst ++++ b/distribution/packages/src/common/scripts/postinst +@@ -103,16 +103,16 @@ fi + # the equivalent code for rpm is in posttrans + if [ "$PACKAGE" = "deb" ]; then + if [ ! -f "${OPENSEARCH_PATH_CONF}"/opensearch.keystore ]; then +- /usr/share/opensearch/bin/opensearch-keystore create ++ /stackable/opensearch/bin/opensearch-keystore create + chown root:opensearch "${OPENSEARCH_PATH_CONF}"/opensearch.keystore + chmod 660 "${OPENSEARCH_PATH_CONF}"/opensearch.keystore + md5sum "${OPENSEARCH_PATH_CONF}"/opensearch.keystore > "${OPENSEARCH_PATH_CONF}"/.opensearch.keystore.initial_md5sum + else +- if /usr/share/opensearch/bin/opensearch-keystore has-passwd --silent ; then ++ if /stackable/opensearch/bin/opensearch-keystore has-passwd --silent ; then + echo "### Warning: unable to upgrade encrypted keystore" 1>&2 + echo " Please run opensearch-keystore upgrade and enter password" 1>&2 + else +- /usr/share/opensearch/bin/opensearch-keystore upgrade ++ /stackable/opensearch/bin/opensearch-keystore upgrade + fi + fi + fi +diff --git a/distribution/packages/src/common/scripts/postrm b/distribution/packages/src/common/scripts/postrm +index 75eded92a8e..7d494d5bf19 100644 +--- a/distribution/packages/src/common/scripts/postrm ++++ b/distribution/packages/src/common/scripts/postrm +@@ -59,16 +59,16 @@ if [ "$REMOVE_DIRS" = "true" ]; then + echo " OK" + fi + +- if [ -d /usr/share/opensearch/plugins ]; then ++ if [ -d /stackable/opensearch/plugins ]; then + echo -n "Deleting plugins directory..." +- rm -rf /usr/share/opensearch/plugins ++ rm -rf /stackable/opensearch/plugins + echo " OK" + fi + + # plugins may have contained bin files +- if [ -d /usr/share/opensearch/bin ]; then ++ if [ -d /stackable/opensearch/bin ]; then + echo -n "Deleting plugin bin directories..." +- rm -rf /usr/share/opensearch/bin ++ rm -rf /stackable/opensearch/bin + echo " OK" + fi + +diff --git a/distribution/packages/src/common/systemd/opensearch.service b/distribution/packages/src/common/systemd/opensearch.service +index 760fc39723f..c1781a24945 100644 +--- a/distribution/packages/src/common/systemd/opensearch.service ++++ b/distribution/packages/src/common/systemd/opensearch.service +@@ -21,7 +21,7 @@ PrivateTmp=true + EnvironmentFile=-/etc/default/opensearch + EnvironmentFile=-/etc/sysconfig/opensearch + +-WorkingDirectory=/usr/share/opensearch ++WorkingDirectory=/stackable/opensearch + + User=opensearch + Group=opensearch +@@ -29,7 +29,7 @@ Group=opensearch + ExecStartPre=/bin/mkdir -p /dev/shm/performanceanalyzer + ExecStartPre=/bin/chown opensearch:opensearch /dev/shm/performanceanalyzer + +-ExecStart=/usr/share/opensearch/bin/systemd-entrypoint -p ${PID_DIR}/opensearch.pid --quiet ++ExecStart=/stackable/opensearch/bin/systemd-entrypoint -p ${PID_DIR}/opensearch.pid --quiet + + # StandardOutput is configured to redirect to journalctl since + # some error messages may be logged in standard output before diff --git a/opensearch/stackable/patches/3.1.0/patchable.toml b/opensearch/stackable/patches/3.1.0/patchable.toml new file mode 100644 index 000000000..aaa2b34b4 --- /dev/null +++ b/opensearch/stackable/patches/3.1.0/patchable.toml @@ -0,0 +1,2 @@ +mirror = "https://github.com/stackabletech/opensearch" +base = "8ff7c6ee924a49f0f59f80a6e1c73073c8904214" diff --git a/opensearch/stackable/patches/patchable.toml b/opensearch/stackable/patches/patchable.toml new file mode 100644 index 000000000..8bea42c4f --- /dev/null +++ b/opensearch/stackable/patches/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/opensearch-project/opensearch" +default-mirror = "https://github.com/stackabletech/opensearch" diff --git a/opensearch/versions.py b/opensearch/versions.py index f0f30be15..2aae116b8 100644 --- a/opensearch/versions.py +++ b/opensearch/versions.py @@ -1,9 +1,8 @@ versions = [ { - "product": "2.19.1", - # The performance analyzer works with JDK 17, but not with 21. - # https://github.com/opensearch-project/performance-analyzer-rca/issues/545 - "java-devel": "17", - "jdk-base": "17", + "product": "3.1.0", + "java-devel": "24", + "jdk-base": "24", + "opensearch/security-plugin": "3.1.0.0", }, ] From 9ee4c4016fe0fc07b0fdc1e672a43ea27adf1ed6 Mon Sep 17 00:00:00 2001 From: Lukas Krug Date: Tue, 22 Jul 2025 11:05:02 +0200 Subject: [PATCH 07/17] feat: SBOMs for OpenSearch and opensearch-security-plugin (#1211) feat: SBOMs for opensearch --- opensearch/Dockerfile | 5 ++- opensearch/security-plugin/Dockerfile | 1 + .../3.1.0.0/0001-Add-CycloneDX-plugin.patch | 38 +++++++++++++++++++ .../3.1.0/0002-Add-CycloneDX-plugin.patch | 38 +++++++++++++++++++ 4 files changed, 81 insertions(+), 1 deletion(-) create mode 100644 opensearch/security-plugin/stackable/patches/3.1.0.0/0001-Add-CycloneDX-plugin.patch create mode 100644 opensearch/stackable/patches/3.1.0/0002-Add-CycloneDX-plugin.patch diff --git a/opensearch/Dockerfile b/opensearch/Dockerfile index 77bb7f1b6..f91c5289e 100644 --- a/opensearch/Dockerfile +++ b/opensearch/Dockerfile @@ -11,7 +11,6 @@ WORKDIR /stackable COPY --chown=${STACKABLE_USER_UID}:0 opensearch/stackable/patches/patchable.toml /stackable/src/opensearch/stackable/patches/patchable.toml COPY --chown=${STACKABLE_USER_UID}:0 opensearch/stackable/patches/${PRODUCT} /stackable/src/opensearch/stackable/patches/${PRODUCT} -COPY --chown=${STACKABLE_USER_UID}:0 --from=opensearch-security-plugin /stackable/src/opensearch/security-plugin/patchable-work/worktree/${OPENSEARCH_SECURITY_PLUGIN}/build/distributions/opensearch-security-${OPENSEARCH_SECURITY_PLUGIN}-SNAPSHOT.zip /stackable/opensearch-security-plugin/opensearch-security-${OPENSEARCH_SECURITY_PLUGIN}-SNAPSHOT.zip RUN < +Date: Tue, 22 Jul 2025 09:52:55 +0200 +Subject: Add CycloneDX plugin + +--- + build.gradle | 11 +++++++++++ + 1 file changed, 11 insertions(+) + +diff --git a/build.gradle b/build.gradle +index bb2e65ab..314bb7be 100644 +--- a/build.gradle ++++ b/build.gradle +@@ -71,6 +71,7 @@ plugins { + id 'eclipse' + id "com.github.spotbugs" version "6.2.0" + id "com.google.osdetector" version "1.7.3" ++ id "org.cyclonedx.bom" version "2.3.1" + } + + allprojects { +@@ -87,6 +88,16 @@ apply from: 'gradle/formatting.gradle' + licenseFile = rootProject.file('LICENSE.txt') + noticeFile = rootProject.file('NOTICE.txt') + ++cyclonedxBom { ++ includeConfigs = ["runtimeClasspath"] ++ includeLicenseText = false ++ skipConfigs = ["compileClasspath", "testCompileClasspath"] ++ projectType = "application" ++ schemaVersion = "1.6" ++ outputFormat = "json" ++ componentVersion = opensearch_build ++} ++ + spotbugs { + includeFilter = file('spotbugs-include.xml') + } diff --git a/opensearch/stackable/patches/3.1.0/0002-Add-CycloneDX-plugin.patch b/opensearch/stackable/patches/3.1.0/0002-Add-CycloneDX-plugin.patch new file mode 100644 index 000000000..5948e0492 --- /dev/null +++ b/opensearch/stackable/patches/3.1.0/0002-Add-CycloneDX-plugin.patch @@ -0,0 +1,38 @@ +From b216d37795bbf81a5ebad73101d8081a8f93068c Mon Sep 17 00:00:00 2001 +From: dervoeti +Date: Mon, 21 Jul 2025 12:45:43 +0200 +Subject: Add CycloneDX plugin + +--- + build.gradle | 11 +++++++++++ + 1 file changed, 11 insertions(+) + +diff --git a/build.gradle b/build.gradle +index e7988cb852f..4c2f2374a99 100644 +--- a/build.gradle ++++ b/build.gradle +@@ -58,6 +58,7 @@ plugins { + id "org.gradle.test-retry" version "1.6.2" apply false + id "test-report-aggregation" + id 'jacoco-report-aggregation' ++ id 'org.cyclonedx.bom' version '1.8.2' + } + + apply from: 'gradle/build-complete.gradle' +@@ -77,6 +78,16 @@ allprojects { + description = "OpenSearch subproject ${project.path}" + } + ++cyclonedxBom { ++ includeConfigs = ["runtimeClasspath"] ++ includeLicenseText = false ++ skipConfigs = ["compileClasspath", "testCompileClasspath"] ++ projectType = "application" ++ schemaVersion = "1.6" ++ outputFormat = "json" ++ componentVersion = VersionProperties.getOpenSearch() ++} ++ + configure(allprojects - project(':distribution:archives:integ-test-zip')) { + project.pluginManager.withPlugin('nebula.maven-base-publish') { + if (project.pluginManager.hasPlugin('opensearch.build') == false) { From aff059e2970b28d84ffb9397a8f961e2af15fdb7 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Tue, 22 Jul 2025 16:36:13 +0200 Subject: [PATCH 08/17] fix file structure in image and make multi-arch compatible --- opensearch/Dockerfile | 42 +++++-- opensearch/security-plugin/Dockerfile | 2 +- .../patches/2.19.1/01-opensearch_home.patch | 103 ------------------ 3 files changed, 32 insertions(+), 115 deletions(-) delete mode 100644 opensearch/stackable/patches/2.19.1/01-opensearch_home.patch diff --git a/opensearch/Dockerfile b/opensearch/Dockerfile index f91c5289e..544001824 100644 --- a/opensearch/Dockerfile +++ b/opensearch/Dockerfile @@ -6,6 +6,7 @@ ARG PRODUCT ARG RELEASE ARG OPENSEARCH_SECURITY_PLUGIN ARG STACKABLE_USER_UID +ARG TARGETARCH WORKDIR /stackable @@ -29,14 +30,17 @@ NEW_VERSION="${PRODUCT}-stackable${RELEASE}" tar -czf /stackable/opensearch-${NEW_VERSION}-src.tar.gz . ./gradlew localDistro ./gradlew cyclonedxBom --warning-mode=summary -cp -r ./distribution/archives/linux-tar/build/install/opensearch-${PRODUCT}-SNAPSHOT /stackable/opensearch -cp ./distribution/docker/src/docker/bin/docker-entrypoint.sh /stackable/opensearch/opensearch-docker-entrypoint.sh -cp build/reports/bom.json /stackable/opensearch/opensearch-${PRODUCT}-SNAPSHOT.cdx.json +cp -r build/distribution/local/opensearch-${PRODUCT}-SNAPSHOT /stackable/opensearch +cp distribution/docker/src/docker/bin/docker-entrypoint.sh /stackable/opensearch/opensearch-docker-entrypoint.sh +sed -i -e 's/OPENSEARCH_DISTRIBUTION_TYPE=tar/OPENSEARCH_DISTRIBUTION_TYPE=docker/' /stackable/opensearch/bin/opensearch-env EOF -WORKDIR /stackable/opensearch-security-plugin -COPY --chown=${STACKABLE_USER_UID}:0 --from=opensearch-security-plugin /stackable/src/opensearch/security-plugin/patchable-work/worktree/${OPENSEARCH_SECURITY_PLUGIN}/build/distributions/opensearch-security-${OPENSEARCH_SECURITY_PLUGIN}-SNAPSHOT.zip /stackable/opensearch-security-plugin/opensearch-security-${OPENSEARCH_SECURITY_PLUGIN}-SNAPSHOT.zip -COPY --chown=${STACKABLE_USER_UID}:0 --from=opensearch-security-plugin /stackable/src/opensearch/security-plugin/patchable-work/worktree/${OPENSEARCH_SECURITY_PLUGIN}/build/reports/bom.json /stackable/opensearch-security-plugin/opensearch-security-${OPENSEARCH_SECURITY_PLUGIN}-SNAPSHOT.cdx.json +WORKDIR /stackable/opensearch-security +COPY \ + --chown=${STACKABLE_USER_UID}:0 \ + --from=opensearch-security-plugin \ + /stackable/src/opensearch/security-plugin/patchable-work/worktree/${OPENSEARCH_SECURITY_PLUGIN}/build/distributions/opensearch-security-${OPENSEARCH_SECURITY_PLUGIN}-SNAPSHOT.zip \ + /stackable/opensearch-security/opensearch-security-${OPENSEARCH_SECURITY_PLUGIN}-SNAPSHOT.zip RUN < Date: Wed, 23 Jul 2025 11:36:35 +0200 Subject: [PATCH 09/17] install performance analyzer plugin --- opensearch/Dockerfile | 3 ++- opensearch/versions.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/opensearch/Dockerfile b/opensearch/Dockerfile index 544001824..9c3a20f3a 100644 --- a/opensearch/Dockerfile +++ b/opensearch/Dockerfile @@ -5,8 +5,8 @@ FROM stackable/image/java-devel AS opensearch-builder ARG PRODUCT ARG RELEASE ARG OPENSEARCH_SECURITY_PLUGIN +ARG OPENSEARCH_PERFORMANCE_ANALYZER ARG STACKABLE_USER_UID -ARG TARGETARCH WORKDIR /stackable @@ -52,6 +52,7 @@ WORKDIR /stackable/opensearch RUN < Date: Wed, 23 Jul 2025 14:48:17 +0200 Subject: [PATCH 10/17] remove legacy apply-patches file --- opensearch/stackable/patches/apply_patches.sh | 44 ------------------- 1 file changed, 44 deletions(-) delete mode 100755 opensearch/stackable/patches/apply_patches.sh diff --git a/opensearch/stackable/patches/apply_patches.sh b/opensearch/stackable/patches/apply_patches.sh deleted file mode 100755 index 833b3e9c7..000000000 --- a/opensearch/stackable/patches/apply_patches.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -# Enable error handling and unset variable checking -set -eu -set -o pipefail - -# Check if $1 (VERSION) is provided -if [ -z "${1-}" ]; then - echo "Please provide a value for VERSION as the first argument." - exit 1 -fi - -VERSION="$1" -PATCH_DIR="patches/$VERSION" - -# Check if version-specific patches directory exists -if [ ! -d "$PATCH_DIR" ]; then - echo "Patches directory '$PATCH_DIR' does not exist." - exit 1 -fi - -# Create an array to hold the patches in sorted order -declare -a patch_files=() - -echo "Applying patches from ${PATCH_DIR}" now - -# Read the patch files into the array -while IFS= read -r -d $'\0' file; do - patch_files+=("$file") -done < <(find "$PATCH_DIR" -name "*.patch" -print0 | sort -zV) - -echo "Found ${#patch_files[@]} patches, applying now" - -# Iterate through sorted patch files -for patch_file in "${patch_files[@]}"; do - echo "Applying $patch_file" - # We can not use Git here, as we are not within a Git repo - patch --directory "." --strip=1 < "$patch_file" || { - echo "Failed to apply $patch_file" - exit 1 - } -done - -echo "All patches applied successfully." From 61d50cfe1344c0b6aa212d03e16adb1abf816b15 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 23 Jul 2025 14:51:32 +0200 Subject: [PATCH 11/17] add changelog entry --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 86c8e909c..117552d63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,7 @@ All notable changes to this project will be documented in this file. - zookeeper: bump jetty version for CVE-2024-13009 in 3.9.3 ([#1179]) - zookeeper: bump netty version for CVE-2025-24970 in 3.9.3 ([#1180]) - hadoop: backport HADOOP-19352, HADOOP-19335, HADOOP-19465, HADOOP-19456 and HADOOP-19225 to fix vulnerabilities in Hadoop `3.4.1` ([#1184]) +- opensearch: Add Opensearch as new product with version `3.1.0` ([#1215]). ### Changed @@ -217,6 +218,7 @@ All notable changes to this project will be documented in this file. [#1185]: https://github.com/stackabletech/docker-images/pull/1185 [#1188]: https://github.com/stackabletech/docker-images/pull/1188 [#1189]: https://github.com/stackabletech/docker-images/pull/1189 +[#1215]: https://github.com/stackabletech/docker-images/pull/1215 ## [25.3.0] - 2025-03-21 From 841b3a071df88d30b16cfb16150aeabed6d061c6 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Fri, 25 Jul 2025 14:36:42 +0200 Subject: [PATCH 12/17] build plugins from source and stop building snapshots --- conf.py | 7 +++ opensearch/Dockerfile | 62 +++++++++++++++---- opensearch/performance-analyzer/Dockerfile | 26 ++++++++ .../3.1.0.0/0001-Add-CycloneDX-plugin.patch | 39 ++++++++++++ .../stackable/patches/3.1.0.0/patchable.toml | 1 + .../stackable/patches/patchable.toml | 2 + opensearch/performance-analyzer/versions.py | 6 ++ opensearch/security-plugin/Dockerfile | 4 +- opensearch/security-plugin/versions.py | 2 +- opensearch/versions.py | 6 +- 10 files changed, 138 insertions(+), 17 deletions(-) create mode 100644 opensearch/performance-analyzer/Dockerfile create mode 100644 opensearch/performance-analyzer/stackable/patches/3.1.0.0/0001-Add-CycloneDX-plugin.patch create mode 100644 opensearch/performance-analyzer/stackable/patches/3.1.0.0/patchable.toml create mode 100644 opensearch/performance-analyzer/stackable/patches/patchable.toml create mode 100644 opensearch/performance-analyzer/versions.py diff --git a/conf.py b/conf.py index bda278f91..b98223e0c 100644 --- a/conf.py +++ b/conf.py @@ -33,6 +33,9 @@ opensearch_security_plugin = importlib.import_module( "opensearch.security-plugin.versions" ) +opensearch_performance_analyzer = importlib.import_module( + "opensearch.performance-analyzer.versions" +) spark_k8s = importlib.import_module("spark-k8s.versions") stackable_base = importlib.import_module("stackable-base.versions") stackable_devel = importlib.import_module("stackable-devel.versions") @@ -75,6 +78,10 @@ "name": "opensearch/security-plugin", "versions": opensearch_security_plugin.versions, }, + { + "name": "opensearch/performance-analyzer", + "versions": opensearch_performance_analyzer.versions, + }, {"name": "spark-k8s", "versions": spark_k8s.versions}, {"name": "stackable-base", "versions": stackable_base.versions}, {"name": "stackable-devel", "versions": stackable_devel.versions}, diff --git a/opensearch/Dockerfile b/opensearch/Dockerfile index 9c3a20f3a..86a724a14 100644 --- a/opensearch/Dockerfile +++ b/opensearch/Dockerfile @@ -1,5 +1,5 @@ - FROM stackable/image/opensearch/security-plugin AS opensearch-security-plugin +FROM stackable/image/opensearch/performance-analyzer AS opensearch-performance-analyzer FROM stackable/image/java-devel AS opensearch-builder ARG PRODUCT @@ -28,32 +28,56 @@ cd "$(/stackable/patchable --images-repo-root=src checkout opensearch ${PRODUCT} NEW_VERSION="${PRODUCT}-stackable${RELEASE}" # Create snapshot of the source code including custom patches tar -czf /stackable/opensearch-${NEW_VERSION}-src.tar.gz . -./gradlew localDistro -./gradlew cyclonedxBom --warning-mode=summary -cp -r build/distribution/local/opensearch-${PRODUCT}-SNAPSHOT /stackable/opensearch +./gradlew clean localDistro -Dbuild.snapshot=false +./gradlew cyclonedxBom --warning-mode=summary -Dbuild.snapshot=false +cp -r build/distribution/local/opensearch-${PRODUCT} /stackable/opensearch cp distribution/docker/src/docker/bin/docker-entrypoint.sh /stackable/opensearch/opensearch-docker-entrypoint.sh +# repository-s3 plugin +cd plugins/repository-s3 +../../gradlew clean assemble -Dbuild.snapshot=false +mkdir /stackable/opensearch/plugins/repository-s3 +cp build/distributions/repository-s3-${PRODUCT}.zip /stackable/opensearch/plugins/repository-s3/repository-s3-${PRODUCT}.zip +cp -r config/repository-s3 /stackable/opensearch/config/repository-s3 +cd /stackable/opensearch/plugins/repository-s3 +unzip repository-s3-${PRODUCT}.zip +rm repository-s3-${PRODUCT}.zip +# Fix an error where OpenSearch running in a container is denied access to swap space size sed -i -e 's/OPENSEARCH_DISTRIBUTION_TYPE=tar/OPENSEARCH_DISTRIBUTION_TYPE=docker/' /stackable/opensearch/bin/opensearch-env EOF +WORKDIR /stackable/opensearch-performance-analyzer +COPY \ + --chown=${STACKABLE_USER_UID}:0 \ + --from=opensearch-performance-analyzer \ + /stackable/src/opensearch/performance-analyzer/patchable-work/worktree/${OPENSEARCH_PERFORMANCE_ANALYZER}/build/distributions/opensearch-performance-analyzer-${OPENSEARCH_PERFORMANCE_ANALYZER}.zip \ + opensearch-performance-analyzer-${OPENSEARCH_PERFORMANCE_ANALYZER}.zip +COPY \ + --chown=${STACKABLE_USER_UID}:0 \ + --from=opensearch-performance-analyzer \ + /stackable/src/opensearch/performance-analyzer/patchable-work/worktree/${OPENSEARCH_PERFORMANCE_ANALYZER}/config \ + /stackable/opensearch/config/opensearch-performance-analyzer + +RUN < +Date: Thu, 24 Jul 2025 14:39:31 +0200 +Subject: Add-CycloneDX-plugin + +--- + build.gradle | 12 +++++++++++- + 1 file changed, 11 insertions(+), 1 deletion(-) + +diff --git a/build.gradle b/build.gradle +index cb784cd..91a6a1e 100644 +--- a/build.gradle ++++ b/build.gradle +@@ -54,7 +54,7 @@ plugins { + id 'checkstyle' + id 'org.ajoberstar.grgit' version '5.0.0' + id 'org.gradle.test-retry' version '1.6.1' +- ++ id "org.cyclonedx.bom" version "2.3.1" + } + + checkstyle { +@@ -121,6 +121,16 @@ apply plugin: 'opensearch.java-agent' + licenseFile = rootProject.file('LICENSE.txt') + noticeFile = rootProject.file('NOTICE.txt') + ++cyclonedxBom { ++ includeConfigs = ["runtimeClasspath"] ++ includeLicenseText = false ++ skipConfigs = ["compileClasspath", "testCompileClasspath"] ++ projectType = "application" ++ schemaVersion = "1.6" ++ outputFormat = "json" ++ componentVersion = opensearch_build ++} ++ + spotless { + java { + licenseHeaderFile(file('license-header')) diff --git a/opensearch/performance-analyzer/stackable/patches/3.1.0.0/patchable.toml b/opensearch/performance-analyzer/stackable/patches/3.1.0.0/patchable.toml new file mode 100644 index 000000000..ca3f27e26 --- /dev/null +++ b/opensearch/performance-analyzer/stackable/patches/3.1.0.0/patchable.toml @@ -0,0 +1 @@ +base = "269779067cf2b9cc37de26ce092dc5d7a73d1383" diff --git a/opensearch/performance-analyzer/stackable/patches/patchable.toml b/opensearch/performance-analyzer/stackable/patches/patchable.toml new file mode 100644 index 000000000..053864a41 --- /dev/null +++ b/opensearch/performance-analyzer/stackable/patches/patchable.toml @@ -0,0 +1,2 @@ +upstream = "https://github.com/opensearch-project/performance-analyzer" +default-mirror = "https://github.com/stackabletech/opensearch-performance-analyzer" diff --git a/opensearch/performance-analyzer/versions.py b/opensearch/performance-analyzer/versions.py new file mode 100644 index 000000000..a46fa4c9f --- /dev/null +++ b/opensearch/performance-analyzer/versions.py @@ -0,0 +1,6 @@ +versions = [ + { + "product": "3.1.0.0", + "java-devel": "21", + }, +] diff --git a/opensearch/security-plugin/Dockerfile b/opensearch/security-plugin/Dockerfile index e04feaf65..7f89c66d9 100644 --- a/opensearch/security-plugin/Dockerfile +++ b/opensearch/security-plugin/Dockerfile @@ -15,8 +15,8 @@ cd "$(/stackable/patchable --images-repo-root=src checkout opensearch/security-p # Create snapshot of the source code including custom patches tar -czf /stackable/opensearch-security-${PRODUCT}-stackable${RELEASE}-src.tar.gz . -./gradlew clean assemble -./gradlew cyclonedxBom --warning-mode=summary +./gradlew clean assemble -Dbuild.snapshot=false +./gradlew cyclonedxBom --warning-mode=summary -Dbuild.snapshot=false EOF RUN < Date: Tue, 29 Jul 2025 14:17:55 +0200 Subject: [PATCH 13/17] replace performance analyzer with telemetry-otel & address PR feedback --- CHANGELOG.md | 8 +- conf.py | 8 +- jdk-base/Dockerfile | 2 +- opensearch/Dockerfile | 106 ++++++++---------- opensearch/performance-analyzer/Dockerfile | 26 ----- .../3.1.0.0/0001-Add-CycloneDX-plugin.patch | 39 ------- .../stackable/patches/3.1.0.0/patchable.toml | 1 - .../stackable/patches/patchable.toml | 2 - opensearch/performance-analyzer/versions.py | 6 - opensearch/security-plugin/Dockerfile | 3 + opensearch/versions.py | 1 - 11 files changed, 56 insertions(+), 146 deletions(-) delete mode 100644 opensearch/performance-analyzer/Dockerfile delete mode 100644 opensearch/performance-analyzer/stackable/patches/3.1.0.0/0001-Add-CycloneDX-plugin.patch delete mode 100644 opensearch/performance-analyzer/stackable/patches/3.1.0.0/patchable.toml delete mode 100644 opensearch/performance-analyzer/stackable/patches/patchable.toml delete mode 100644 opensearch/performance-analyzer/versions.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 474d90d28..ada195201 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. ## [Unreleased] +### Added + +- opensearch: Add Opensearch as new product with version `3.1.0` ([#1215]). + +[#1215]: https://github.com/stackabletech/docker-images/pull/1215 + ## [25.7.0] - 2025-07-23 ## [25.7.0-rc1] - 2025-07-18 @@ -65,7 +71,6 @@ All notable changes to this project will be documented in this file. - hadoop: backport HADOOP-19352, HADOOP-19335, HADOOP-19465, HADOOP-19456 and HADOOP-19225 to fix vulnerabilities in Hadoop `3.4.1` ([#1184]) - hadoop: Backport HADOOP-18583 to make OpenSSL 3.x work with the native hadoop libraries ([#1209]). - spark: backport [SPARK-51311] Promote bcprov-jdk18on to compile scope ([#1212]). -- opensearch: Add Opensearch as new product with version `3.1.0` ([#1215]). ### Changed @@ -230,7 +235,6 @@ All notable changes to this project will be documented in this file. [#1197]: https://github.com/stackabletech/docker-images/pull/1197 [#1209]: https://github.com/stackabletech/docker-images/pull/1209 [#1212]: https://github.com/stackabletech/docker-images/pull/1212 -[#1215]: https://github.com/stackabletech/docker-images/pull/1215 ## [25.3.0] - 2025-03-21 diff --git a/conf.py b/conf.py index b98223e0c..fa4c83017 100644 --- a/conf.py +++ b/conf.py @@ -33,9 +33,6 @@ opensearch_security_plugin = importlib.import_module( "opensearch.security-plugin.versions" ) -opensearch_performance_analyzer = importlib.import_module( - "opensearch.performance-analyzer.versions" -) spark_k8s = importlib.import_module("spark-k8s.versions") stackable_base = importlib.import_module("stackable-base.versions") stackable_devel = importlib.import_module("stackable-devel.versions") @@ -78,10 +75,6 @@ "name": "opensearch/security-plugin", "versions": opensearch_security_plugin.versions, }, - { - "name": "opensearch/performance-analyzer", - "versions": opensearch_performance_analyzer.versions, - }, {"name": "spark-k8s", "versions": spark_k8s.versions}, {"name": "stackable-base", "versions": stackable_base.versions}, {"name": "stackable-devel", "versions": stackable_devel.versions}, @@ -109,6 +102,7 @@ "kafka": {"id": "625ff25b91bdcd4b49c823a4"}, "nifi": {"id": "625586a32e9e14bc8118e203"}, "opa": {"id": "6255838bea1feb8bec4aaaa3"}, + "opensearch": {"id": "6880fe690db664aa303d3a28"}, "spark-k8s": {"id": "62613e81f8ce82a2f247dda5"}, "superset": {"id": "62557e5fea1feb8bec4aaaa0"}, "tools": {"id": "62557cd575ab7e30884aaaa0"}, diff --git a/jdk-base/Dockerfile b/jdk-base/Dockerfile index 582b65768..776804c1d 100644 --- a/jdk-base/Dockerfile +++ b/jdk-base/Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:1.10.0@sha256:865e5dd094beca432e8c0a1d5e1c465db5f998dca4e439981029b3b81fb39ed5 +# syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 # check=error=true # diff --git a/opensearch/Dockerfile b/opensearch/Dockerfile index 86a724a14..6f8a8bc8d 100644 --- a/opensearch/Dockerfile +++ b/opensearch/Dockerfile @@ -1,11 +1,12 @@ +# syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 +# check=error=true + FROM stackable/image/opensearch/security-plugin AS opensearch-security-plugin -FROM stackable/image/opensearch/performance-analyzer AS opensearch-performance-analyzer FROM stackable/image/java-devel AS opensearch-builder ARG PRODUCT ARG RELEASE ARG OPENSEARCH_SECURITY_PLUGIN -ARG OPENSEARCH_PERFORMANCE_ANALYZER ARG STACKABLE_USER_UID WORKDIR /stackable @@ -28,38 +29,22 @@ cd "$(/stackable/patchable --images-repo-root=src checkout opensearch ${PRODUCT} NEW_VERSION="${PRODUCT}-stackable${RELEASE}" # Create snapshot of the source code including custom patches tar -czf /stackable/opensearch-${NEW_VERSION}-src.tar.gz . -./gradlew clean localDistro -Dbuild.snapshot=false +if [[ $(uname -m) == "arm64" ]] +then + ./scripts/build.sh -v ${PRODUCT} -s false -a arm64 + tar -xzf artifacts/dist/opensearch-min-${PRODUCT}-linux-arm64.tar.gz -C /stackable +else + ./scripts/build.sh -v ${PRODUCT} -s false -a x64 + tar -xzf artifacts/dist/opensearch-min-${PRODUCT}-linux-x64.tar.gz -C /stackable +fi +unzip artifacts/core-plugins/repository-s3-${PRODUCT}.zip -d /stackable/opensearch-${PRODUCT}/plugins/repository-s3/ +mv /stackable/opensearch-${PRODUCT}/plugins/repository-s3/config /stackable/opensearch-${PRODUCT}/config/repository-s3 +unzip artifacts/core-plugins/telemetry-otel-${PRODUCT}.zip -d /stackable/opensearch-${PRODUCT}/plugins/telemetry-otel/ +mv /stackable/opensearch-${PRODUCT}/plugins/telemetry-otel/config /stackable/opensearch-${PRODUCT}/config/telemetry-otel ./gradlew cyclonedxBom --warning-mode=summary -Dbuild.snapshot=false -cp -r build/distribution/local/opensearch-${PRODUCT} /stackable/opensearch -cp distribution/docker/src/docker/bin/docker-entrypoint.sh /stackable/opensearch/opensearch-docker-entrypoint.sh -# repository-s3 plugin -cd plugins/repository-s3 -../../gradlew clean assemble -Dbuild.snapshot=false -mkdir /stackable/opensearch/plugins/repository-s3 -cp build/distributions/repository-s3-${PRODUCT}.zip /stackable/opensearch/plugins/repository-s3/repository-s3-${PRODUCT}.zip -cp -r config/repository-s3 /stackable/opensearch/config/repository-s3 -cd /stackable/opensearch/plugins/repository-s3 -unzip repository-s3-${PRODUCT}.zip -rm repository-s3-${PRODUCT}.zip -# Fix an error where OpenSearch running in a container is denied access to swap space size -sed -i -e 's/OPENSEARCH_DISTRIBUTION_TYPE=tar/OPENSEARCH_DISTRIBUTION_TYPE=docker/' /stackable/opensearch/bin/opensearch-env -EOF - -WORKDIR /stackable/opensearch-performance-analyzer -COPY \ - --chown=${STACKABLE_USER_UID}:0 \ - --from=opensearch-performance-analyzer \ - /stackable/src/opensearch/performance-analyzer/patchable-work/worktree/${OPENSEARCH_PERFORMANCE_ANALYZER}/build/distributions/opensearch-performance-analyzer-${OPENSEARCH_PERFORMANCE_ANALYZER}.zip \ - opensearch-performance-analyzer-${OPENSEARCH_PERFORMANCE_ANALYZER}.zip -COPY \ - --chown=${STACKABLE_USER_UID}:0 \ - --from=opensearch-performance-analyzer \ - /stackable/src/opensearch/performance-analyzer/patchable-work/worktree/${OPENSEARCH_PERFORMANCE_ANALYZER}/config \ - /stackable/opensearch/config/opensearch-performance-analyzer - -RUN < -Date: Thu, 24 Jul 2025 14:39:31 +0200 -Subject: Add-CycloneDX-plugin - ---- - build.gradle | 12 +++++++++++- - 1 file changed, 11 insertions(+), 1 deletion(-) - -diff --git a/build.gradle b/build.gradle -index cb784cd..91a6a1e 100644 ---- a/build.gradle -+++ b/build.gradle -@@ -54,7 +54,7 @@ plugins { - id 'checkstyle' - id 'org.ajoberstar.grgit' version '5.0.0' - id 'org.gradle.test-retry' version '1.6.1' -- -+ id "org.cyclonedx.bom" version "2.3.1" - } - - checkstyle { -@@ -121,6 +121,16 @@ apply plugin: 'opensearch.java-agent' - licenseFile = rootProject.file('LICENSE.txt') - noticeFile = rootProject.file('NOTICE.txt') - -+cyclonedxBom { -+ includeConfigs = ["runtimeClasspath"] -+ includeLicenseText = false -+ skipConfigs = ["compileClasspath", "testCompileClasspath"] -+ projectType = "application" -+ schemaVersion = "1.6" -+ outputFormat = "json" -+ componentVersion = opensearch_build -+} -+ - spotless { - java { - licenseHeaderFile(file('license-header')) diff --git a/opensearch/performance-analyzer/stackable/patches/3.1.0.0/patchable.toml b/opensearch/performance-analyzer/stackable/patches/3.1.0.0/patchable.toml deleted file mode 100644 index ca3f27e26..000000000 --- a/opensearch/performance-analyzer/stackable/patches/3.1.0.0/patchable.toml +++ /dev/null @@ -1 +0,0 @@ -base = "269779067cf2b9cc37de26ce092dc5d7a73d1383" diff --git a/opensearch/performance-analyzer/stackable/patches/patchable.toml b/opensearch/performance-analyzer/stackable/patches/patchable.toml deleted file mode 100644 index 053864a41..000000000 --- a/opensearch/performance-analyzer/stackable/patches/patchable.toml +++ /dev/null @@ -1,2 +0,0 @@ -upstream = "https://github.com/opensearch-project/performance-analyzer" -default-mirror = "https://github.com/stackabletech/opensearch-performance-analyzer" diff --git a/opensearch/performance-analyzer/versions.py b/opensearch/performance-analyzer/versions.py deleted file mode 100644 index a46fa4c9f..000000000 --- a/opensearch/performance-analyzer/versions.py +++ /dev/null @@ -1,6 +0,0 @@ -versions = [ - { - "product": "3.1.0.0", - "java-devel": "21", - }, -] diff --git a/opensearch/security-plugin/Dockerfile b/opensearch/security-plugin/Dockerfile index 7f89c66d9..0ee1adcdb 100644 --- a/opensearch/security-plugin/Dockerfile +++ b/opensearch/security-plugin/Dockerfile @@ -1,3 +1,6 @@ +# syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7 +# check=error=true + FROM stackable/image/java-devel AS security-plugin-builder ARG PRODUCT diff --git a/opensearch/versions.py b/opensearch/versions.py index d252d77f6..708e430d7 100644 --- a/opensearch/versions.py +++ b/opensearch/versions.py @@ -4,6 +4,5 @@ "java-devel": "21", "jdk-base": "21", "opensearch/security-plugin": "3.1.0.0", - "opensearch/performance-analyzer": "3.1.0.0", }, ] From 9162b578f37808026a42797a09aa8e74419f26e7 Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Wed, 30 Jul 2025 16:01:33 +0200 Subject: [PATCH 14/17] simplify build process --- opensearch/Dockerfile | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/opensearch/Dockerfile b/opensearch/Dockerfile index 6f8a8bc8d..d8c0aec30 100644 --- a/opensearch/Dockerfile +++ b/opensearch/Dockerfile @@ -8,6 +8,7 @@ ARG PRODUCT ARG RELEASE ARG OPENSEARCH_SECURITY_PLUGIN ARG STACKABLE_USER_UID +ARG TARGETARCH WORKDIR /stackable @@ -26,17 +27,12 @@ USER ${STACKABLE_USER_UID} RUN < Date: Wed, 30 Jul 2025 18:03:35 +0200 Subject: [PATCH 15/17] address feedback in PR --- .github/ISSUE_TEMPLATE/early-pre-release.md | 1 + .../update-product-opensearch.md | 79 +++++++++++++++++++ .github/workflows/build_opensearch.yaml | 35 ++++++++ .github/workflows/preflight.yaml | 1 + .scripts/enumerate-product-versions.py | 1 + .scripts/update_feature_tracker_db.sh | 1 + README.md | 8 +- opensearch/Dockerfile | 18 +++-- opensearch/README.md | 12 +++ opensearch/security-plugin/Dockerfile | 2 +- 10 files changed, 147 insertions(+), 11 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/update-product-opensearch.md create mode 100644 .github/workflows/build_opensearch.yaml create mode 100644 opensearch/README.md diff --git a/.github/ISSUE_TEMPLATE/early-pre-release.md b/.github/ISSUE_TEMPLATE/early-pre-release.md index 9028d996c..223eb3a05 100644 --- a/.github/ISSUE_TEMPLATE/early-pre-release.md +++ b/.github/ISSUE_TEMPLATE/early-pre-release.md @@ -45,6 +45,7 @@ Part of stackabletech/issues#xxx. - [ ] [Create issue from template: update-product-kafka.md](https://github.com/stackabletech/docker-images/issues/new?template=update-product-kafka.md) - [ ] [Create issue from template: update-product-nifi.md](https://github.com/stackabletech/docker-images/issues/new?template=update-product-nifi.md) - [ ] [Create issue from template: update-product-opa.md](https://github.com/stackabletech/docker-images/issues/new?template=update-product-opa.md) +- [ ] [Create issue from template: update-product-opensearch.md](https://github.com/stackabletech/docker-images/issues/new?template=update-product-opensearch.md) - [ ] [Create issue from template: update-product-spark.md](https://github.com/stackabletech/docker-images/issues/new?template=update-product-spark.md) - [ ] [Create issue from template: update-product-superset.md](https://github.com/stackabletech/docker-images/issues/new?template=update-product-superset.md) - [ ] [Create issue from template: update-product-trino.md](https://github.com/stackabletech/docker-images/issues/new?template=update-product-trino.md) diff --git a/.github/ISSUE_TEMPLATE/update-product-opensearch.md b/.github/ISSUE_TEMPLATE/update-product-opensearch.md new file mode 100644 index 000000000..c2af763c8 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/update-product-opensearch.md @@ -0,0 +1,79 @@ +--- +name: Update OpenSearch +about: >- + This template contains instructions specific to updating this product and/or + container image(s). +title: >- + chore(opensearch): Update container images ahead of Stackable Release YY.M.X +labels: [] +# Currently, projects cannot be assigned via front-matter. +projects: ['stackabletech/10'] +assignees: '' +--- + +Part of #xxx. + + +- Add: `x.x.x` +- Remove: `y.y.y` + +> [!TIP] +> Please add the `scheduled-for/20XX-XX` label, and add to the [Stackable Engineering][1] project. +> +> [1]: https://github.com/orgs/stackabletech/projects/10 + +## Update tasks + +- [ ] Update `versions.py` to reflect the agreed upon versions in the spreadsheet (including the removal of old versions). +- [ ] Update `versions.py` to the latest supported version of JVM (base and devel). +- [ ] Update other dependencies if applicable (eg: security-plugin, etc). +- [ ] Check other operators (getting_started / kuttl / supported-versions) for usage of the versions. Add the PR(s) to the list below. +- [ ] Update the version in demos. Add the PR(s) to the list below. + +## Related Pull Requests + +> [!TIP] +> Delete any items that do not apply so that all applicable items can be checked. +> For example, if you add release notes to the documentation repository, you do not need the latter two criteria. + +- _Link to the docker-images PR (product update)_ +- _Link to the operator PR (getting_started / kuttl / supported-versions)_ +- _Link to any other operator PRs (getting_started / kuttl)_ +- _Link to demo PR (raise against the `main` branch)_ +- _Link to the Release Notes PR in the documentation repo (if not a comment below)_ + +## Acceptance + +> [!TIP] +> This list should be completed by the assignee(s), once respective PRs have been merged. Once all items have been +> checked, the issue can be moved into _Development: Done_. + +- [ ] Can build image (either locally, or in CI) +- [ ] Kuttl smoke tests passes (either locally, or in CI) +- [ ] Release notes added to documentation and linked as a PR above +- [ ] Release notes written in a comment below +- [ ] Applicable `release-note` label added to this issue + +
+Testing instructions + +```shell +# See the latest version at https://pypi.org/project/image-tools-stackabletech/ +pip install image-tools-stackabletech==0.0.16 + +bake --product opensearch=x.y.z # where x.y.z is the new version added in this PR + +kind load docker-image oci.stackable.tech/sdp/opensearch:x.y.z-stackable0.0.0-dev + +# Change directory into the opa-operator repository and update the +# product version in tests/test-definition.yaml +./scripts/run-tests --test-suite smoke-latest # or similar +``` + +
+ +_Please consider updating this template if these instructions are wrong, or +could be made clearer._ diff --git a/.github/workflows/build_opensearch.yaml b/.github/workflows/build_opensearch.yaml new file mode 100644 index 000000000..764b151fb --- /dev/null +++ b/.github/workflows/build_opensearch.yaml @@ -0,0 +1,35 @@ +--- +name: Build OpenSearch +run-name: | + Build OpenSearch (attempt #${{ github.run_attempt }}) + +on: + workflow_dispatch: + schedule: + - cron: '0 0 2/2 * *' # https://crontab.guru/#0_0_2/2_*_* + push: + branches: [main] + tags: ['*'] + paths: + # To check dependencies, run this ( you will need to consider transitive dependencies) + # bake --product PRODUCT -d | grep -v 'docker buildx bake' | jq '.target | keys[]' + - opensearch/** + - vector/** + - stackable-base/** + - java-base/** + - java-devel/** + - .github/actions/** + - .github/workflows/build_opensearch.yaml + - .github/workflows/reusable_build_image.yaml + +jobs: + build_image: + name: Reusable Workflow + uses: ./.github/workflows/reusable_build_image.yaml + secrets: + harbor-robot-secret: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }} + slack-token: ${{ secrets.SLACK_CONTAINER_IMAGE_TOKEN }} + with: + product-name: kafka + sdp-version: ${{ github.ref_type == 'tag' && github.ref_name || '0.0.0-dev' }} + registry-namespace: sdp diff --git a/.github/workflows/preflight.yaml b/.github/workflows/preflight.yaml index 488b1f345..3c3b5e97a 100644 --- a/.github/workflows/preflight.yaml +++ b/.github/workflows/preflight.yaml @@ -66,6 +66,7 @@ jobs: - nifi - omid - opa + - opensearch - spark-k8s - superset - trino diff --git a/.scripts/enumerate-product-versions.py b/.scripts/enumerate-product-versions.py index 7c3180c26..6beaa085c 100644 --- a/.scripts/enumerate-product-versions.py +++ b/.scripts/enumerate-product-versions.py @@ -18,6 +18,7 @@ "krb5", "nifi", "opa", + "opensearch", "omid", "spark-k8s", "superset", diff --git a/.scripts/update_feature_tracker_db.sh b/.scripts/update_feature_tracker_db.sh index 8531d54bb..56ae3a446 100755 --- a/.scripts/update_feature_tracker_db.sh +++ b/.scripts/update_feature_tracker_db.sh @@ -33,6 +33,7 @@ PRODUCT_CODE_NAMES=( hive kafka nifi + opensearch spark-k8s superset trino diff --git a/README.md b/README.md index 1d8ee7a44..a5f2602ea 100644 --- a/README.md +++ b/README.md @@ -8,9 +8,9 @@ This repository contains Dockerfiles and scripts to build base images for use wi | [![Build Airflow]][build_airflow.yaml] | [![Build Druid]][build_druid.yaml] | [![Build Hadoop]][build_hadoop.yaml] | [![Build HBase]][build_hbase.yaml] | | [![Build Hive]][build_hive.yaml] | [![Build Java Base]][build_java-base.yaml] | [![Build Java Development]][build_java-devel.yaml] | [![Build Kafka Testing Tools]][build_kafka-testing-tools.yaml] | | [![Build Kafka]][build_kafka.yaml] | [![Build Krb5]][build_krb5.yaml] | [![Build NiFi]][build_nifi.yaml] | [![Build Omid]][build_omid.yaml] | -| [![Build OPA]][build_opa.yaml] | [![Build Spark Connect Client]][build_spark-connect-client.yaml] | [![Build Spark K8s]][build_spark-k8s.yaml] | [![Build Stackable Base]][build_stackable-base.yaml] | -| [![Build Superset]][build_superset.yaml] | [![Build Testing Tools]][build_testing-tools.yaml] | [![Build Tools]][build_tools.yaml] | [![Build Trino CLI]][build_trino-cli.yaml] | -| [![Build Trino]][build_trino.yaml] | [![Build Vector]][build_vector.yaml] | [![Build ZooKeeper]][build_zookeeper.yaml] | | +| [![Build OPA]][build_opa.yaml] | [![Build OpenSearch]][build_opensearch.yaml] | [![Build Spark Connect Client]][build_spark-connect-client.yaml] | [![Build Spark K8s]][build_spark-k8s.yaml] | +| [![Build Stackable Base]][build_stackable-base.yaml] | [![Build Superset]][build_superset.yaml] | [![Build Testing Tools]][build_testing-tools.yaml] | [![Build Tools]][build_tools.yaml] | +| [![Build Trino CLI]][build_trino-cli.yaml] | [![Build Trino]][build_trino.yaml] | [![Build Vector]][build_vector.yaml] | [![Build ZooKeeper]][build_zookeeper.yaml] | ## Prerequisites @@ -239,6 +239,8 @@ ENTRYPOINT ["/stackable-zookeeper-operator"] [build_omid.yaml]: https://github.com/stackabletech/docker-images/actions/workflows/build_omid.yaml [Build OPA]: https://github.com/stackabletech/docker-images/actions/workflows/build_opa.yaml/badge.svg [build_opa.yaml]: https://github.com/stackabletech/docker-images/actions/workflows/build_opa.yaml +[Build OpenSearch]: https://github.com/stackabletech/docker-images/actions/workflows/build_opensearch.yaml/badge.svg +[build_opensearch.yaml]: https://github.com/stackabletech/docker-images/actions/workflows/build_opensearch.yaml [Build Spark Connect Client]: https://github.com/stackabletech/docker-images/actions/workflows/build_spark-connect-client.yaml/badge.svg [build_spark-connect-client.yaml]: https://github.com/stackabletech/docker-images/actions/workflows/build_spark-connect-client.yaml [Build Spark K8s]: https://github.com/stackabletech/docker-images/actions/workflows/build_spark-k8s.yaml/badge.svg diff --git a/opensearch/Dockerfile b/opensearch/Dockerfile index d8c0aec30..532213397 100644 --- a/opensearch/Dockerfile +++ b/opensearch/Dockerfile @@ -32,7 +32,7 @@ NEW_VERSION="${PRODUCT}-stackable${RELEASE}" # Create snapshot of the source code including custom patches tar -czf /stackable/opensearch-${NEW_VERSION}-src.tar.gz . ./scripts/build.sh -v "${PRODUCT}" -s false -a "${ARCH}" -tar -xzf artifacts/dist/opensearch-min-${PRODUCT}-linux-${ARCH}.tar.gz -C /stackable +tar -xzf "artifacts/dist/opensearch-min-${PRODUCT}-linux-${ARCH}.tar.gz" -C /stackable unzip artifacts/core-plugins/repository-s3-${PRODUCT}.zip -d /stackable/opensearch-${PRODUCT}/plugins/repository-s3/ mv /stackable/opensearch-${PRODUCT}/plugins/repository-s3/config /stackable/opensearch-${PRODUCT}/config/repository-s3 unzip artifacts/core-plugins/telemetry-otel-${PRODUCT}.zip -d /stackable/opensearch-${PRODUCT}/plugins/telemetry-otel/ @@ -65,14 +65,14 @@ rm -r jdk # Change the group permissions already in the builder image to reduce # the size of the final image. # see https://github.com/stackabletech/docker-images/issues/961 -chmod -R g=u "${HOME}" chmod +x /stackable/opensearch-${PRODUCT}/opensearch-docker-entrypoint.sh +chmod -R g=u /stackable EOF # The OpenSearch Performance Analyzer needs a JDK, not just a JRE. # With a JRE, the following exception is thrown: # java.lang.ClassNotFoundException: com.sun.tools.attach.VirtualMachine -FROM stackable/image/jdk-base +FROM stackable/image/jdk-base AS final ARG PRODUCT ARG RELEASE @@ -129,6 +129,7 @@ COPY \ --from=opensearch-security-plugin \ /stackable/src/opensearch/security-plugin/patchable-work/worktree/${OPENSEARCH_SECURITY_PLUGIN}/build/reports/bom.json \ /stackable/opensearch-security-${OPENSEARCH_SECURITY_PLUGIN}-stackable${RELEASE}.cdx.json + RUN < Date: Thu, 31 Jul 2025 13:54:44 +0200 Subject: [PATCH 16/17] fix copypaste errors --- .github/ISSUE_TEMPLATE/update-product-opensearch.md | 2 +- .github/workflows/build_opensearch.yaml | 4 ++-- CHANGELOG.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/update-product-opensearch.md b/.github/ISSUE_TEMPLATE/update-product-opensearch.md index c2af763c8..dbfde143b 100644 --- a/.github/ISSUE_TEMPLATE/update-product-opensearch.md +++ b/.github/ISSUE_TEMPLATE/update-product-opensearch.md @@ -68,7 +68,7 @@ bake --product opensearch=x.y.z # where x.y.z is the new version added in this P kind load docker-image oci.stackable.tech/sdp/opensearch:x.y.z-stackable0.0.0-dev -# Change directory into the opa-operator repository and update the +# Change directory into the opensearch-operator repository and update the # product version in tests/test-definition.yaml ./scripts/run-tests --test-suite smoke-latest # or similar ``` diff --git a/.github/workflows/build_opensearch.yaml b/.github/workflows/build_opensearch.yaml index 764b151fb..a1d40c3c3 100644 --- a/.github/workflows/build_opensearch.yaml +++ b/.github/workflows/build_opensearch.yaml @@ -16,7 +16,7 @@ on: - opensearch/** - vector/** - stackable-base/** - - java-base/** + - jdk-base/** - java-devel/** - .github/actions/** - .github/workflows/build_opensearch.yaml @@ -30,6 +30,6 @@ jobs: harbor-robot-secret: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }} slack-token: ${{ secrets.SLACK_CONTAINER_IMAGE_TOKEN }} with: - product-name: kafka + product-name: opensearch sdp-version: ${{ github.ref_type == 'tag' && github.ref_name || '0.0.0-dev' }} registry-namespace: sdp diff --git a/CHANGELOG.md b/CHANGELOG.md index 58c2de590..17487400f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,8 +14,8 @@ All notable changes to this project will be documented in this file. - all: Use our build-repo to cache NPM dependencies ([#1219]) [#1207]: https://github.com/stackabletech/docker-images/pull/1207 -[#1219]: https://github.com/stackabletech/docker-images/pull/1219 [#1215]: https://github.com/stackabletech/docker-images/pull/1215 +[#1219]: https://github.com/stackabletech/docker-images/pull/1219 ## [25.7.0] - 2025-07-23 From e5e8a2c5034613bda8ea5249a92b0ae2933a678f Mon Sep 17 00:00:00 2001 From: Benedikt Labrenz Date: Thu, 31 Jul 2025 20:16:47 +0200 Subject: [PATCH 17/17] add stackable-devel to build dependencies --- .github/workflows/build_opensearch.yaml | 1 + opensearch/Dockerfile | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_opensearch.yaml b/.github/workflows/build_opensearch.yaml index a1d40c3c3..4c9503981 100644 --- a/.github/workflows/build_opensearch.yaml +++ b/.github/workflows/build_opensearch.yaml @@ -16,6 +16,7 @@ on: - opensearch/** - vector/** - stackable-base/** + - stackable-devel/** - jdk-base/** - java-devel/** - .github/actions/** diff --git a/opensearch/Dockerfile b/opensearch/Dockerfile index 532213397..7fb712445 100644 --- a/opensearch/Dockerfile +++ b/opensearch/Dockerfile @@ -145,7 +145,6 @@ chmod g=u /stackable/*.cdx.json ln -s /stackable/opensearch-${PRODUCT}-stackable${RELEASE} ${OPENSEARCH_HOME} chown -h ${STACKABLE_USER_UID}:0 ${OPENSEARCH_HOME} - # ---------------------------------------- # Checks # This section is to run final checks to ensure the created final images