Skip to content

Commit dcd430a

Browse files
pawelpaszkipawelpaszki
authored
RHOAIENG-34272: add support for building codeflare-sdk tests image (#939)
* RHOAIENG-34272: add support for building codeflare-sdk tests image
1 parent f336a2d commit dcd430a

13 files changed

+878
-9
lines changed

‎Makefile‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Image tag for image containing e2e tests
2+
E2E_TEST_IMAGE_VERSION ?= latest
3+
E2E_TEST_IMAGE ?= quay.io/opendatahub/codeflare-sdk-tests:${E2E_TEST_IMAGE_VERSION}
4+
5+
# Build the test image
6+
.PHONY: build-test-image
7+
build-test-image:
8+
@echo "Building test image: $(E2E_TEST_IMAGE)"
9+
# Build the Docker image using podman
10+
podman build -f images/tests/Dockerfile -t $(E2E_TEST_IMAGE) .
11+
12+
# Push the test image
13+
.PHONY: push-test-image
14+
push-test-image:
15+
@echo "Pushing test image: $(E2E_TEST_IMAGE)"
16+
podman push $(E2E_TEST_IMAGE)

‎images/tests/Dockerfile‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
# Multi-stage build for Python tests with oc CLI
2+
FROM python:3.12-slim AS builder
3+
4+
# Install system dependencies needed for building
5+
RUN apt-get update && \
6+
apt-get install -y --no-install-recommends \
7+
curl \
8+
build-essential \
9+
&& rm -rf /var/lib/apt/lists/*
10+
11+
# Install Poetry
12+
ENV POETRY_VERSION=1.8.3 \
13+
POETRY_HOME="/opt/poetry" \
14+
POETRY_VIRTUALENVS_IN_PROJECT=true \
15+
POETRY_NO_INTERACTION=1 \
16+
POETRY_CACHE_DIR=/tmp/poetry_cache
17+
18+
ENV PATH="$POETRY_HOME/bin:$PATH"
19+
20+
RUN pip install --no-cache-dir "poetry==$POETRY_VERSION"
21+
22+
# ============================================================================
23+
# Base Directory: /codeflare-sdk
24+
# This is the root directory where all project files will be located.
25+
# Similar to kuberay structure, all source code and tests are under /codeflare-sdk
26+
# ============================================================================
27+
WORKDIR /codeflare-sdk
28+
29+
# Copy dependency files
30+
COPY pyproject.toml poetry.lock* ./
31+
32+
# Install dependencies (including test dependencies)
33+
RUN poetry install --no-root --with test && rm -rf $POETRY_CACHE_DIR
34+
35+
# Runtime stage
36+
FROM python:3.12-slim
37+
38+
# Install system dependencies for runtime
39+
RUN apt-get update && \
40+
apt-get install -y --no-install-recommends \
41+
curl \
42+
ca-certificates \
43+
&& rm -rf /var/lib/apt/lists/*
44+
45+
# Install OpenShift CLI (oc)
46+
RUN curl -L https://mirror.openshift.com/pub/openshift-v4/clients/oc/latest/linux/oc.tar.gz | \
47+
tar -xz -C /usr/local/bin && \
48+
chmod +x /usr/local/bin/oc && \
49+
oc version --client
50+
51+
# Install Poetry for runtime (needed for poetry run command)
52+
ENV POETRY_VERSION=1.8.3 \
53+
POETRY_HOME="/opt/poetry" \
54+
POETRY_VIRTUALENVS_IN_PROJECT=true \
55+
POETRY_NO_INTERACTION=1
56+
57+
ENV PATH="$POETRY_HOME/bin:/codeflare-sdk/.venv/bin:$PATH"
58+
59+
RUN pip install --no-cache-dir "poetry==$POETRY_VERSION"
60+
61+
# ============================================================================
62+
# Base Directory: /codeflare-sdk
63+
# This is the root directory where all project files will be located.
64+
# Similar to kuberay structure, all source code and tests are under /codeflare-sdk
65+
# ============================================================================
66+
WORKDIR /codeflare-sdk
67+
68+
# Copy virtual environment from builder
69+
COPY --from=builder /codeflare-sdk/.venv /codeflare-sdk/.venv
70+
71+
# Copy project files
72+
COPY pyproject.toml poetry.lock* ./
73+
COPY README.md ./
74+
COPY src/ ./src/
75+
COPY tests/ ./tests/
76+
77+
# Copy test runner script, entrypoint, and RBAC file
78+
COPY images/tests/run-tests.sh /codeflare-sdk/run-tests.sh
79+
COPY images/tests/entrypoint.sh /codeflare-sdk/entrypoint.sh
80+
COPY images/tests/rbac-test-user-permissions.yaml /codeflare-sdk/images/tests/rbac-test-user-permissions.yaml
81+
RUN chmod +x /codeflare-sdk/run-tests.sh /codeflare-sdk/entrypoint.sh
82+
83+
# Install the codeflare_sdk package in editable mode so it can be imported
84+
# This is needed because we used --no-root in the builder stage
85+
WORKDIR /codeflare-sdk
86+
RUN /codeflare-sdk/.venv/bin/pip install -e .
87+
88+
# ============================================================================
89+
# Working Directory: /codeflare-sdk
90+
# Tests expect to run from the project root so relative paths like ./tests/e2e/ work correctly
91+
# ============================================================================
92+
WORKDIR /codeflare-sdk
93+
94+
ENV KUBECONFIG=/codeflare-sdk/tests/.kube/config
95+
96+
# ============================================================================
97+
# Test Results Directory: /codeflare-sdk/tests/results
98+
# This directory will contain pytest output files (JUnit XML, coverage, etc.)
99+
# Mount this as a volume when running the container to access test results
100+
# ============================================================================
101+
RUN mkdir -p /codeflare-sdk/tests/results
102+
103+
# ============================================================================
104+
# Environment File Setup
105+
# The containerEnvFile should be passed via --env-file parameter when running
106+
# the container. Expected environment variables:
107+
# TEST_USER_USERNAME, TEST_USER_PASSWORD
108+
# OCP_ADMIN_USER_USERNAME, OCP_ADMIN_USER_PASSWORD
109+
# Example: podman run --env-file containerEnvFile codeflare-sdk-tests
110+
# ============================================================================
111+
112+
# ============================================================================
113+
# Default Command (can be overridden when running the container)
114+
# This CMD runs the test wrapper script which handles:
115+
# - Extracting OpenShift API URL from kubeconfig
116+
# - Applying RBAC policies
117+
# - Logging in with TEST_USER
118+
# - Running tests
119+
# - Logging in with OCP_ADMIN_USER after tests
120+
# - Cleaning up RBAC
121+
# To override it, just provide your command after the image name:
122+
# podman run --env-file containerEnvFile codeflare-sdk-tests poetry run pytest tests/e2e/specific_test.py
123+
# ============================================================================
124+
# Set entrypoint to handle -- separator arguments
125+
ENTRYPOINT ["/codeflare-sdk/entrypoint.sh"]
126+
# Default command (can be overridden, but entrypoint will handle it)
127+
CMD []

‎images/tests/entrypoint.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/bin/sh
2+
# Entrypoint script that handles -- separator in podman commands
3+
# Passes all arguments to run-tests.sh which will forward them to pytest
4+
5+
exec /codeflare-sdk/run-tests.sh "$@"

‎images/tests/rbac-test-user-permissions.yaml‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
---
2+
# RBAC permissions for test user to run e2e tests
3+
# Apply this as cluster-admin before running tests:
4+
# oc apply -f images/tests/rbac-test-user-permissions.yaml
5+
# OR
6+
# kubectl apply -f images/tests/rbac-test-user-permissions.yaml
7+
#
8+
# The username "TEST_USER_USERNAME_PLACEHOLDER" will be replaced at runtime with the actual test username
9+
10+
# For OpenShift: Grant self-provisioner role (allows namespace creation)
11+
# This is the recommended approach for OpenShift
12+
apiVersion: rbac.authorization.k8s.io/v1
13+
kind: ClusterRoleBinding
14+
metadata:
15+
name: test-user-self-provisioner
16+
# For OpenShift, you can also use: oc adm policy add-cluster-role-to-user self-provisioner TEST_USER_USERNAME_PLACEHOLDER
17+
subjects:
18+
- kind: User
19+
name: TEST_USER_USERNAME_PLACEHOLDER
20+
apiGroup: rbac.authorization.k8s.io
21+
roleRef:
22+
kind: ClusterRole
23+
name: self-provisioner
24+
apiGroup: rbac.authorization.k8s.io
25+
---
26+
# Alternative: Grant admin role (more permissive, use if self-provisioner doesn't work)
27+
# For OpenShift, you can also use: oc adm policy add-cluster-role-to-user admin TEST_USER_USERNAME_PLACEHOLDER
28+
apiVersion: rbac.authorization.k8s.io/v1
29+
kind: ClusterRoleBinding
30+
metadata:
31+
name: test-user-admin
32+
subjects:
33+
- kind: User
34+
name: TEST_USER_USERNAME_PLACEHOLDER
35+
apiGroup: rbac.authorization.k8s.io
36+
roleRef:
37+
kind: ClusterRole
38+
name: admin
39+
apiGroup: rbac.authorization.k8s.io
40+
---
41+
# For Kubernetes: Grant cluster-admin role (allows all operations including namespace creation)
42+
# This is more permissive but ensures all test operations work
43+
apiVersion: rbac.authorization.k8s.io/v1
44+
kind: ClusterRoleBinding
45+
metadata:
46+
name: test-user-cluster-admin
47+
subjects:
48+
- kind: User
49+
name: TEST_USER_USERNAME_PLACEHOLDER
50+
apiGroup: rbac.authorization.k8s.io
51+
roleRef:
52+
kind: ClusterRole
53+
name: cluster-admin
54+
apiGroup: rbac.authorization.k8s.io
55+
---
56+
# Additional permissions for Kueue resources (if needed)
57+
# This allows the user to create/manage Kueue CustomResources
58+
apiVersion: rbac.authorization.k8s.io/v1
59+
kind: ClusterRole
60+
metadata:
61+
name: test-user-kueue-admin
62+
rules:
63+
- apiGroups:
64+
- kueue.x-k8s.io
65+
resources:
66+
- clusterqueues
67+
- resourceflavors
68+
- localqueues
69+
- workloads
70+
verbs:
71+
- get
72+
- list
73+
- watch
74+
- create
75+
- update
76+
- patch
77+
- delete
78+
---
79+
apiVersion: rbac.authorization.k8s.io/v1
80+
kind: ClusterRoleBinding
81+
metadata:
82+
name: test-user-kueue-admin
83+
subjects:
84+
- kind: User
85+
name: TEST_USER_USERNAME_PLACEHOLDER
86+
apiGroup: rbac.authorization.k8s.io
87+
roleRef:
88+
kind: ClusterRole
89+
name: test-user-kueue-admin
90+
apiGroup: rbac.authorization.k8s.io
91+
---
92+
# Permissions for RayCluster and RayJob CustomResources
93+
apiVersion: rbac.authorization.k8s.io/v1
94+
kind: ClusterRole
95+
metadata:
96+
name: test-user-ray-admin
97+
rules:
98+
- apiGroups:
99+
- ray.io
100+
resources:
101+
- rayclusters
102+
- rayjobs
103+
verbs:
104+
- get
105+
- list
106+
- watch
107+
- create
108+
- update
109+
- patch
110+
- delete
111+
---
112+
apiVersion: rbac.authorization.k8s.io/v1
113+
kind: ClusterRoleBinding
114+
metadata:
115+
name: test-user-ray-admin
116+
subjects:
117+
- kind: User
118+
name: TEST_USER_USERNAME_PLACEHOLDER
119+
apiGroup: rbac.authorization.k8s.io
120+
roleRef:
121+
kind: ClusterRole
122+
name: test-user-ray-admin
123+
apiGroup: rbac.authorization.k8s.io
124+
---
125+
# Comprehensive RBAC role for Kueue batch operations
126+
# This role provides permissions for namespaces, Kueue, Ray, and core Kubernetes resources
127+
apiVersion: rbac.authorization.k8s.io/v1
128+
kind: ClusterRole
129+
metadata:
130+
name: kueue-batch-user-role
131+
rules:
132+
- apiGroups: [""]
133+
resources: ["namespaces"]
134+
verbs: ["create", "get", "list", "watch"]
135+
- apiGroups: ["kueue.x-k8s.io"]
136+
resources: ["clusterqueues", "resourceflavors", "localqueues"]
137+
verbs: ["create", "get", "list", "watch", "update", "patch", "delete"]
138+
- apiGroups: ["ray.io"]
139+
resources: ["rayclusters", "rayjobs"]
140+
verbs: ["create", "get", "list", "watch", "update", "patch", "delete"]
141+
- apiGroups: ["apps"]
142+
resources: ["deployments"]
143+
verbs: ["create", "get", "list", "watch", "update", "patch", "delete"]
144+
- apiGroups: [""]
145+
resources: ["pods", "services", "configmaps", "secrets"]
146+
verbs: ["create", "get", "list", "watch", "update", "patch", "delete"]
147+
---
148+
# ClusterRoleBinding for authenticated users (group-based)
149+
apiVersion: rbac.authorization.k8s.io/v1
150+
kind: ClusterRoleBinding
151+
metadata:
152+
name: kueue-batch-user-rolebinding
153+
subjects:
154+
- kind: Group
155+
apiGroup: rbac.authorization.k8s.io
156+
name: 'system:authenticated'
157+
roleRef:
158+
apiGroup: rbac.authorization.k8s.io
159+
kind: ClusterRole
160+
name: kueue-batch-user-role
161+
---
162+
# ClusterRoleBinding for specific test user
163+
apiVersion: rbac.authorization.k8s.io/v1
164+
kind: ClusterRoleBinding
165+
metadata:
166+
name: kueue-batch-user-specific-rolebinding
167+
subjects:
168+
- kind: User
169+
apiGroup: rbac.authorization.k8s.io
170+
name: 'TEST_USER_USERNAME_PLACEHOLDER'
171+
roleRef:
172+
apiGroup: rbac.authorization.k8s.io
173+
kind: ClusterRole
174+
name: kueue-batch-user-role

0 commit comments

Comments
 (0)