diff --git a/tests/soak/Dockerfile b/tests/soak/Dockerfile deleted file mode 100644 index eb89ebb0e..000000000 --- a/tests/soak/Dockerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM debian:bookworm -ARG LK_VERSION -ARG CKPY_VERSION -RUN test -n "${LK_VERSION}" || (echo "LK_VERSION env variable required" && exit 1) -RUN test -n "${CKPY_VERSION}" || (echo "CKPY_VERSION env variable required" && exit 1) -ENV DEBIAN_FRONTEND=noninteractive -RUN apt update && apt install -y sudo -RUN mkdir -p /soaktests -COPY bootstrap.sh /soaktests -WORKDIR /soaktests -RUN /soaktests/bootstrap.sh ${CKPY_VERSION} ${LK_VERSION} -ENTRYPOINT [ "/soaktests/confluent-kafka-python/tests/soak/run.sh" ] diff --git a/tests/soak/README.md b/tests/soak/README.md index 29adebc2e..e8d89aef9 100644 --- a/tests/soak/README.md +++ b/tests/soak/README.md @@ -10,10 +10,24 @@ OpenTelemetry reporting supported through OTLP. # Installation -TESTID=normal \ -LK_VERSION=v2.2.0 \ -CKPY_VERSION=v2.2.0 \ -CC_BOOSTRAP_SERVERS=_ \ -CC_USERNAME=_ \ -CC_PASSWORD=_ \ -DOCKER_REPOSITORY=_ ./install.sh +1. Edit `ccloud.config` + +2. Edit `otel-config.yaml` + +3. the first time: +```bash +./bootstrap.sh +``` +4. next times: +```bash +./build.sh +``` + +5. +```bash +. venv/bin/activate + +5. Run some tests +```bash +TESTID= ./run.sh ccloud.config +``` \ No newline at end of file diff --git a/tests/soak/bootstrap.sh b/tests/soak/bootstrap.sh index 1900c97ec..3e9790fc3 100755 --- a/tests/soak/bootstrap.sh +++ b/tests/soak/bootstrap.sh @@ -2,7 +2,7 @@ # # -# Bootstrap EC2 instance (Ubuntu 18.04) for soak client use +# Bootstrap EC2 instance (Ubuntu) for soak client use # # Usage: # $0 @@ -10,50 +10,32 @@ set -e if [[ $# != 2 ]]; then - echo "Usage: $0 " + echo "Usage: $0 " exit 1 fi -python_branch=$1 -librdkafka_branch=$2 -venv=$PWD/venv - -sudo apt update -sudo apt install -y git curl make gcc g++ zlib1g-dev libssl-dev libzstd-dev \ - python3-dev python3-pip python3-venv - -if [[ ! -d confluent-kafka-python ]]; then - git clone https://github.com/confluentinc/confluent-kafka-python -fi - -pushd confluent-kafka-python - -git checkout $python_branch - -echo "Installing librdkafka $librdkafka_branch" -tools/bootstrap-librdkafka.sh --require-ssl $librdkafka_branch /usr -rm -rf tmp-build +librdkafka_branch=$1 +python_branch=$2 +otel_collector_version=0.130.0 +otel_collector_package_url="https://github.com/open-telemetry/"\ +"opentelemetry-collector-releases/releases/download/"\ +"v${otel_collector_version}/otelcol-contrib_${otel_collector_version}_linux_amd64.deb" -# echo "Installing interceptors" -# tools/install-interceptors.sh -echo "Setting up virtualenv in $venv" -if [[ ! -d $venv ]]; then - python3 -m venv $venv -fi -source $venv/bin/activate - -pip install -U pip - -pip install -v .[soaktest] - -popd # ..python - -echo "Verifying python client installation" -python -c "import confluent_kafka; print(confluent_kafka.version(), confluent_kafka.libversion())" - -deactivate +sudo apt update +sudo apt install -y git curl wget make gcc g++ zlib1g-dev libssl-dev \ + libzstd-dev python3-dev python3-pip python3-venv +wget -O otel_collector_package.deb $otel_collector_package_url +sudo dpkg -i otel_collector_package.deb +rm otel_collector_package.deb +sudo cp otel-config.yaml /etc/otelcol-contrib/config.yaml +sudo systemctl restart otelcol-contrib +cp setup_all_versions.py $HOME/setup_all_versions.py +chmod +x $HOME/setup_all_versions.py + +./build.sh $librdkafka_branch $python_branch +venv=$PWD/venv echo "All done, activate the virtualenv in $venv before running the client:" echo "source $venv/bin/activate" diff --git a/tests/soak/build.sh b/tests/soak/build.sh index 279ecf458..7f990d25c 100755 --- a/tests/soak/build.sh +++ b/tests/soak/build.sh @@ -11,25 +11,50 @@ fi set -eu +testdir=$PWD +mkdir -p $testdir/librdkafka-installation +if [[ ! -d confluent-kafka-python ]]; then + git clone https://github.com/confluentinc/confluent-kafka-python +fi + +venv=$PWD/venv +if [[ ! -d $venv ]]; then + echo "Setting up virtualenv in $venv" + python3 -m venv $venv + source $venv/bin/activate + pip install -U pip + pip install -r $testdir/../../requirements/requirements-soaktest.txt + deactivate +fi echo "Building and installing librdkafka $librdkafka_version" +if [[ ! -d librdkafka ]]; then + git clone https://github.com/confluentinc/librdkafka.git +fi pushd librdkafka -sudo make uninstall git fetch --tags git checkout $librdkafka_version -./configure --reconfigure +echo "Configuring librdkafka $librdkafka_version with prefix $testdir/librdkafka-installation" +./configure --prefix=$testdir/librdkafka-installation +sudo make uninstall make clean make -j -sudo make install +make install popd +export LIBRARY_PATH=$testdir/librdkafka-installation/lib +export LD_LIBRARY_PATH=$testdir/librdkafka-installation/lib +export CPLUS_INCLUDE_PATH=$testdir/librdkafka-installation/include +export C_INCLUDE_PATH=$testdir/librdkafka-installation/include echo "Building confluent-kafka-python $cflpy_version" set +u source venv/bin/activate +python3 -m pip uninstall -y confluent-kafka set -u pushd confluent-kafka-python +rm -rf ./build git fetch --tags git checkout $cflpy_version python3 -m pip install . @@ -38,4 +63,3 @@ popd echo "" echo "==============================================================================" (cd / ; python3 -c 'import confluent_kafka as c; print("python", c.version(), "librdkafka", c.libversion())') - diff --git a/tests/soak/ccloud.config.example b/tests/soak/ccloud.config.example new file mode 100644 index 000000000..85e0b9c04 --- /dev/null +++ b/tests/soak/ccloud.config.example @@ -0,0 +1,9 @@ +bootstrap.servers= +sasl.mechanisms=PLAIN +security.protocol=SASL_SSL +sasl.username= +sasl.password= +enable.idempotence=true +debug=eos,generic,broker,security,consumer +linger.ms=2 +compression.type=lz4 diff --git a/tests/soak/install.sh b/tests/soak/install.sh deleted file mode 100755 index 0bd5b7b8a..000000000 --- a/tests/soak/install.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash -set -e - -DOCKER_REPOSITORY_DEFAULT=${DOCKER_REPOSITORY:-docker.io/library/njc-py-soak-tests} -NAMESPACE=njc-soak-tests -NOCACHE=${NOCACHE:---no-cache} - -for var in LK_VERSION CKPY_VERSION CC_BOOSTRAP_SERVERS CC_USERNAME CC_PASSWORD TESTID \ -DOCKER_REPOSITORY_DEFAULT; do - VAR_VALUE=$(eval echo \$$var) - if [ -z "$VAR_VALUE" ]; then - echo "env variable $var is required" - exit 1 - fi -done - -TAG=${LK_VERSION}-${CKPY_VERSION} - -COMMAND="docker build . $NOCACHE --build-arg LK_VERSION=${LK_VERSION} \ ---build-arg CKPY_VERSION=${CKPY_VERSION} \ --t ${DOCKER_REPOSITORY_DEFAULT}:${TAG}" -echo $COMMAND -$COMMAND - -if [ ! -z "$DOCKER_REPOSITORY" ]; then - COMMAND="docker push ${DOCKER_REPOSITORY}:${TAG}" - echo $COMMAND - $COMMAND -fi - -if [ "$(uname -p)" = "x86_64" ]; then - NODE_ARCH="amd64" -else - NODE_ARCH="arm64" -fi - -COMMAND="helm upgrade --install njc-py-soak-tests-${TESTID} ./njc-py-soak-tests \ ---set "cluster.bootstrapServers=${CC_BOOSTRAP_SERVERS}" \ ---set "cluster.username=${CC_USERNAME}" \ ---set "cluster.password=${CC_PASSWORD}" \ ---set "image.repository=${DOCKER_REPOSITORY_DEFAULT}" \ ---set "testid=${TESTID}" \ ---set "fullnameOverride=njc-py-soak-tests-${TESTID}" \ ---set "image.tag=${TAG}" \ ---set "nodeSelector.kubernetes\\.io/arch=${NODE_ARCH}" \ ---namespace "${NAMESPACE}" --create-namespace" -echo $COMMAND -$COMMAND diff --git a/tests/soak/njc-py-soak-tests/.helmignore b/tests/soak/njc-py-soak-tests/.helmignore deleted file mode 100644 index 0e8a0eb36..000000000 --- a/tests/soak/njc-py-soak-tests/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/tests/soak/njc-py-soak-tests/Chart.yaml b/tests/soak/njc-py-soak-tests/Chart.yaml deleted file mode 100644 index 71e9d4248..000000000 --- a/tests/soak/njc-py-soak-tests/Chart.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: v2 -name: njc-py-soak-tests -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.0.0" diff --git a/tests/soak/njc-py-soak-tests/templates/NOTES.txt b/tests/soak/njc-py-soak-tests/templates/NOTES.txt deleted file mode 100644 index fb58cd2ce..000000000 --- a/tests/soak/njc-py-soak-tests/templates/NOTES.txt +++ /dev/null @@ -1 +0,0 @@ -NJC Python soak tests installed! diff --git a/tests/soak/njc-py-soak-tests/templates/_helpers.tpl b/tests/soak/njc-py-soak-tests/templates/_helpers.tpl deleted file mode 100644 index e1b849e97..000000000 --- a/tests/soak/njc-py-soak-tests/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "njc-py-soak-tests.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "njc-py-soak-tests.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "njc-py-soak-tests.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "njc-py-soak-tests.labels" -}} -helm.sh/chart: {{ include "njc-py-soak-tests.chart" . }} -{{ include "njc-py-soak-tests.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "njc-py-soak-tests.selectorLabels" -}} -app.kubernetes.io/name: {{ include "njc-py-soak-tests.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "njc-py-soak-tests.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "njc-py-soak-tests.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/tests/soak/njc-py-soak-tests/templates/deployment.yaml b/tests/soak/njc-py-soak-tests/templates/deployment.yaml deleted file mode 100644 index 837aab835..000000000 --- a/tests/soak/njc-py-soak-tests/templates/deployment.yaml +++ /dev/null @@ -1,87 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "njc-py-soak-tests.fullname" . }} - labels: - {{- include "njc-py-soak-tests.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.replicaCount }} - selector: - matchLabels: - {{- include "njc-py-soak-tests.selectorLabels" . | nindent 6 }} - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: 0 - maxUnavailable: 1 - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "njc-py-soak-tests.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "njc-py-soak-tests.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - volumes: - - name: secret - secret: - secretName: {{ include "njc-py-soak-tests.fullname" $ }}-secret - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - env: - - name: NODEIP - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: status.hostIP - - name: OTEL_METRICS_EXPORTER - value: "otlp" - - name: OTEL_RESOURCE_ATTRIBUTES - value: "service.name={{ include "njc-py-soak-tests.fullname" . }},service.version={{ .Values.image.tag | default .Chart.AppVersion }}" - - name: OTEL_TRACES_EXPORTER - value: "none" - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: "http://$(NODEIP):14317" - - name: OTEL_METRIC_EXPORT_INTERVAL - value: "10000" - - name: TESTID - value: "{{ .Values.testid }}" - volumeMounts: - - name: "secret" - mountPath: "/soaktests/confluent-kafka-python/ccloud.config" - subPath: ccloud.config - readOnly: true - # livenessProbe: - # httpGet: - # path: / - # port: http - # readinessProbe: - # httpGet: - # path: / - # port: http - resources: - {{- toYaml .Values.resources | nindent 12 }} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/tests/soak/njc-py-soak-tests/templates/secrets.yaml b/tests/soak/njc-py-soak-tests/templates/secrets.yaml deleted file mode 100644 index 16eefd49b..000000000 --- a/tests/soak/njc-py-soak-tests/templates/secrets.yaml +++ /dev/null @@ -1,35 +0,0 @@ -{{- range $nameSuffix, $values := .Values.secrets }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: {{ include "njc-py-soak-tests.fullname" $ }}-{{ $nameSuffix }} - {{- with $values.annotations }} - annotations: - {{- range $key, $value := . }} - {{- printf "%s: %s" $key (tpl $value $ | quote) | nindent 4 }} - {{- end }} - {{- end }} - labels: - {{- range $key, $value := $values.labels }} - {{- printf "%s: %s" $key (tpl $value $ | quote) | nindent 4 }} - {{- end }} -type: {{ default "Opaque" $values.type }} -{{- with $values.data }} -data: - {{- toYaml . | nindent 2 }} -{{- end }} -stringData: - ccloud.config: |- - bootstrap.servers={{ $.Values.cluster.bootstrapServers }} - sasl.mechanisms=PLAIN - security.protocol=SASL_SSL - sasl.username={{ $.Values.cluster.username }} - sasl.password={{ $.Values.cluster.password }} - {{- $.Values.properties | nindent 4 -}} -{{- with $values.stringData }} - {{- range $key, $value := . }} - {{- printf "%s: %s" $key (tpl $value $ | quote) | nindent 2 }} - {{- end }} -{{- end }} -{{- end -}} diff --git a/tests/soak/njc-py-soak-tests/templates/serviceaccount.yaml b/tests/soak/njc-py-soak-tests/templates/serviceaccount.yaml deleted file mode 100644 index 88e498c8a..000000000 --- a/tests/soak/njc-py-soak-tests/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "njc-py-soak-tests.serviceAccountName" . }} - labels: - {{- include "njc-py-soak-tests.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/tests/soak/njc-py-soak-tests/values.yaml b/tests/soak/njc-py-soak-tests/values.yaml deleted file mode 100644 index 0640bfc86..000000000 --- a/tests/soak/njc-py-soak-tests/values.yaml +++ /dev/null @@ -1,66 +0,0 @@ -# Default values for njc-py-soak-tests. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 - -image: - # To use in minikube - repository: docker.io/library/njc-py-soak-tests - pullPolicy: Always - # Overrides the image tag whose default is the chart appVersion. - tag: "" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "njc-py-soak-tests" - -serviceAccount: - # Specifies whether a service account should be created - create: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -podAnnotations: {} - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -resources: - limits: - cpu: 100m - memory: 512Mi - requests: - cpu: 100m - memory: 128Mi - -cluster: - bootstrapServers: "" - username: "" - password: "" - -properties: |- - enable.idempotence=true - debug=eos,generic,broker,security,consumer - linger.ms=2 - compression.type=lz4 - -secrets: - secret: {} - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/tests/soak/otel-config.yaml b/tests/soak/otel-config.yaml new file mode 100644 index 000000000..091aaa78d --- /dev/null +++ b/tests/soak/otel-config.yaml @@ -0,0 +1,34 @@ +receivers: + otlp: + protocols: + grpc: + +processors: + batch: + send_batch_max_size: 100 + send_batch_size: 10 + timeout: 10s + +extensions: + sigv4auth: + region: us-west-2 + service: aps + assume_role: + arn: FILL_IN_ROLE_ARN_HERE + +exporters: + prometheusremotewrite: + endpoint: FILL_IN_REMOTE_WRITE_ENDPOINT_HERE + auth: + authenticator: sigv4auth + +service: + extensions: [sigv4auth] + telemetry: + logs: + level: "debug" + pipelines: + metrics: + receivers: [otlp] + processors: [batch] + exporters: [prometheusremotewrite] \ No newline at end of file diff --git a/tests/soak/run.sh b/tests/soak/run.sh index 47826e590..103daabc1 100755 --- a/tests/soak/run.sh +++ b/tests/soak/run.sh @@ -1,9 +1,9 @@ #!/bin/bash -# -set -e source venv/bin/activate +testdir=$PWD +export LD_LIBRARY_PATH=$testdir/librdkafka-installation/lib librdkafka_version=$(python3 -c 'from confluent_kafka import libversion; print(libversion()[0])') if [[ -z $librdkafka_version ]]; then @@ -13,12 +13,23 @@ fi set -u topic="pysoak-$TESTID-$librdkafka_version" - -echo "Starting soak client using topic $topic" +logfile="${TESTID}.log.bz2" +export HOSTNAME=$(hostname) +echo "Starting soak client using topic $topic. Logging to $logfile." set +x -time opentelemetry-instrument confluent-kafka-python/tests/soak/soakclient.py -i $TESTID -t $topic -r 80 -f confluent-kafka-python/ccloud.config 2>&1 +# Ignore SIGINT in children (inherited) +trap "" SIGINT +time opentelemetry-instrument $testdir/soakclient.py -i $TESTID -t $topic -r 80 -f $1 |& tee /dev/tty | bzip2 > $logfile & +PID=$! +# On SIGINT kill only the first process of the pipe +onsigint() { + # List children of $PID only + ps --ppid $PID -f | grep soakclient.py | grep -v grep | awk '{print $2}' | xargs kill +} +trap onsigint SIGINT +# Await the result +wait $PID ret=$? echo "Python client exited with status $ret" +echo "Ending soak client using topic $topic. Logging to $logfile." exit $ret - - diff --git a/tests/soak/setup_all_versions.py b/tests/soak/setup_all_versions.py new file mode 100755 index 000000000..fbec78394 --- /dev/null +++ b/tests/soak/setup_all_versions.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +import os +import subprocess + +PYTHON_SOAK_TEST_BRANCH = 'dev_otel_agent_soak_test' + +LIBRDKAFKA_VERSIONS=[ + #'2.11.1' + #'2.10.1', + '2.8.0', + '2.6.1', + '2.5.3', + '2.4.0', + '2.3.0', + '2.2.0', + '2.1.1', + '2.0.2' +] + +PYTHON_VERSIONS=[ + #'2.11.0' + #'2.10.1', + '2.8.0', + '2.6.1', + '2.5.3', + '2.4.0', + '2.3.0', + '2.2.0', + '2.1.1', + '2.0.2' +] + +def run(command): + print(f'Running command: {command}') + env = os.environ.copy() + env['MAKEFLAGS'] = 'SHELL=/bin/bash -j8' # For parallel make jobs + result = subprocess.run(command, shell=True, env=env) + if result.returncode != 0: + raise Exception(f'Command failed: {command}, result: {result.returncode}') + +if __name__ == '__main__': + for i, librdkafka_version in enumerate(LIBRDKAFKA_VERSIONS): + python_version = PYTHON_VERSIONS[i] + folder_name = f'confluent-kafka-python_{librdkafka_version.replace('.', '_')}' + if os.path.exists(folder_name): + print(f'Skipping \'{folder_name}\', already exists.') + continue + + run(f'git clone https://github.com/confluentinc/confluent-kafka-python.git') + run(f'mv confluent-kafka-python {folder_name}') + run(f'cd {folder_name} && git checkout {PYTHON_SOAK_TEST_BRANCH}') + run(f'cd {folder_name}/tests/soak && ./build.sh v{librdkafka_version} v{python_version}') + run(f'cp config/*.config {folder_name}/tests/soak') diff --git a/tests/soak/soakclient.py b/tests/soak/soakclient.py index 027cc2e83..f43ab0f5f 100755 --- a/tests/soak/soakclient.py +++ b/tests/soak/soakclient.py @@ -26,12 +26,11 @@ # from confluent_kafka import KafkaError, KafkaException, version -from confluent_kafka import Producer +from confluent_kafka import Producer, Consumer from confluent_kafka.admin import AdminClient, NewTopic from collections import defaultdict from builtins import int from opentelemetry import metrics -from common import TestConsumer import argparse import threading import time @@ -429,6 +428,7 @@ def filter_config(conf, filter_out, strip_prefix): # Create topic (might already exist) aconf = filter_config(conf, ["consumer.", "producer."], "admin.") + aconf['client.id'] = self.testid self.create_topic(self.topic, aconf) # @@ -440,6 +440,7 @@ def filter_config(conf, filter_out, strip_prefix): # Producer pconf = filter_config(conf, ["consumer.", "admin."], "producer.") pconf['error_cb'] = self.producer_error_cb + pconf['client.id'] = self.testid self.producer = Producer(pconf) # Consumer @@ -447,7 +448,13 @@ def filter_config(conf, filter_out, strip_prefix): cconf['error_cb'] = self.consumer_error_cb cconf['on_commit'] = self.consumer_commit_cb self.logger.info("consumer: using group.id {}".format(cconf['group.id'])) - self.consumer = TestConsumer(cconf) + cconf['client.id'] = self.testid + self.consumer = Consumer(cconf) + + # Initialize some counters to zero to make them appear in the metrics + self.incr_counter("consumer.error", 0) + self.incr_counter("consumer.msgdup", 0) + self.incr_counter("producer.errorcb", 0) # Create and start producer thread self.producer_thread = threading.Thread(target=self.producer_thread_main)