From c9be60820352db7de953b729f3653825c535c661 Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Tue, 29 Jul 2025 21:33:53 +0000 Subject: [PATCH 1/4] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20ch?= =?UTF-8?q?anges=20to=20main=20this=20commit=20is=20based=20on?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Created using spr 1.3.6 [skip ci] --- premerge/buildbot/Dockerfile | 10 ++ premerge/buildbot/dispatch_job.py | 179 ++++++++++++++++++++++++ premerge/buildbot/dispatch_job_test.py | 86 ++++++++++++ premerge/buildbot/requirements.lock.txt | 92 ++++++++++++ premerge/buildbot/requirements.txt | 2 + premerge/buildbot/startup.sh | 20 +++ 6 files changed, 389 insertions(+) create mode 100644 premerge/buildbot/Dockerfile create mode 100644 premerge/buildbot/dispatch_job.py create mode 100644 premerge/buildbot/dispatch_job_test.py create mode 100644 premerge/buildbot/requirements.lock.txt create mode 100644 premerge/buildbot/requirements.txt create mode 100644 premerge/buildbot/startup.sh diff --git a/premerge/buildbot/Dockerfile b/premerge/buildbot/Dockerfile new file mode 100644 index 000000000..96ee435cc --- /dev/null +++ b/premerge/buildbot/Dockerfile @@ -0,0 +1,10 @@ +FROM ubuntu:24.04 +RUN apt-get update && apt-get install -y python3 python3-pip +COPY requirements.lock.txt /requirements.lock.txt +RUN pip3 install --break-system-packages -r /requirements.lock.txt && rm /requirements.lock.txt +RUN mkdir /app +WORKDIR /app +COPY dispatch_job.py . +COPY startup.sh . +RUN chmod +x startup.sh +ENTRYPOINT /app/startup.sh diff --git a/premerge/buildbot/dispatch_job.py b/premerge/buildbot/dispatch_job.py new file mode 100644 index 000000000..de62b8673 --- /dev/null +++ b/premerge/buildbot/dispatch_job.py @@ -0,0 +1,179 @@ +"""Dispatches a job to the k8s cluster. + +This script takes in a commit SHA to test along with the platform, spawns a job +to test it, and then streams the logs from the job. We read logs from the job +every so often using the kuberntes logging API rather than directly executing +commands inside the container and streaming the output. This is to work +around https://github.com/kubernetes-sigs/apiserver-network-proxy/issues/748. +""" + +import sys +import logging +import time +import dateutil +import datetime +import json + +import kubernetes + +PLATFORM_TO_NAMESPACE = {"Linux": "llvm-premerge-linux-builbot"} +LOG_SECONDS_TO_QUERY = 10 +SECONDS_QUERY_LOGS_EVERY = 5 + + +def start_build_linux(commit_sha: str, k8s_client) -> str: + """Spawns a pod to build/test LLVM at the specified SHA. + + Args: + commit_sha: The commit SHA to build/run the tests at. + k8s_client: The kubernetes client instance to use for spawning the pod. + + Returns: + A string containing the name of the pod. + """ + pod_name = f"build-{commit_sha}" + commands = [ + "git clone --depth 100 https://github.com/llvm/llvm-project", + "cd llvm-project", + f"git checkout ${commit_sha}", + "export CC=clang", + "export CXX=clang++", + './.ci/monolithic-linux.sh "bolt;clang;clang-tools-extra;flang;libclc;lld;lldb;llvm;mlir;polly" "check-bolt check-clang check-clang-cir check-clang-tools check-flang check-lld check-lldb check-llvm check-mlir check-polly" "compiler-rt;libc;libcxx;libcxxabi;libunwind" "check-compiler-rt check-libc" "check-cxx check-cxxabi check-unwind" "OFF"' + "echo BUILD FINISHED", + ] + pod_definition = { + "apiVersion": "v1", + "kind": "Pod", + "metadata": { + "name": pod_name, + "namespace": PLATFORM_TO_NAMESPACE["Linux"], + }, + "spec": { + "containers": [ + { + "name": "build", + "image": "ghcr.io/llvm/ci-ubuntu-24.04", + "command": ["/bin/bash", "-c", ";".join(commands)], + } + ], + "restartPolicy": "Never", + }, + } + kubernetes.utils.create_from_dict(k8s_client, pod_definition) + return pod_name + + +def read_logs(pod_name: str, namespace: str, v1_api) -> list[str]: + """Reads logs from the specified pod. + + Reads logs using the k8s API and returns a nicely formatted list of + strings. + + Args: + pod_name: The name of the pod to read logs from. + namespace: The namespace the pod is in. + v1_api: The kubernetes API instance to use for querying logs. + + Returns: + A list of strings representing the log lines. + """ + logs = v1_api.read_namespaced_pod_log( + name=pod_name, + namespace=namespace, + timestamps=True, + since_seconds=LOG_SECONDS_TO_QUERY, + ) + return logs.split("\n")[:-1] + + +def get_logs_to_print( + logs: list[str], latest_time: datetime.datetime +) -> tuple[datetime.datetime, list[str]]: + """Get the logs that we should be printing. + + This function takes in a raw list of logs along with the timestamp of the + last log line to be printed and returns the new log lines that should be + printed. + + Args: + logs: The raw list of log lines. + latest_time: The timestamp from the last log line that was printed. + + Returns: + A tuple containing the timestamp of the last log line returned and a list + of strings containing the log lines that should be printed. + """ + first_new_index = 0 + time_stamp = latest_time + for log_line in logs: + time_stamp_str = log_line.split(" ")[0] + time_stamp = dateutil.parser.parse(time_stamp_str[:-1]) + if time_stamp > latest_time: + break + first_new_index += 1 + last_time_stamp = latest_time + if logs: + last_time_stamp_str = logs[-1].split(" ")[0] + last_time_stamp = dateutil.parser.parse(last_time_stamp_str[:-1]) + return (last_time_stamp, logs[first_new_index:]) + + +def print_logs( + pod_name: str, namespace: str, v1_api, lastest_time: datetime.datetime +) -> tuple[bool, datetime.datetime]: + """Queries the pod and prints the relevant log lines. + + Args: + pod_name: The pod to print the logs for. + namespace: The namespace the log is in. + v1_api: The kubernetes API client instance to use for querying the logs. + latest_time: The timestamp of the last log line to be printed. + + Returns: + A tuple containing a boolean representing whether or not the pod has + finished executing and the timestamp of the last log line printed. + """ + logs = read_logs(pod_name, namespace, v1_api) + new_time_stamp, logs_to_print = get_logs_to_print(logs, lastest_time) + pod_finished = False + for log_line in logs_to_print: + print(log_line.split("\r")[-1]) + if "BUILD FINISHED" in log_line: + pod_finished = True + + return (pod_finished, new_time_stamp) + + +def main(commit_sha: str, platform: str): + kubernetes.config.load_kube_config() + k8s_client = kubernetes.client.ApiClient() + if platform == "Linux": + pod_name = start_build_linux(commit_sha, k8s_client) + else: + raise ValueError("Unrecognized platform.") + namespace = PLATFORM_TO_NAMESPACE[platform] + latest_time = datetime.datetime.min + v1_api = kubernetes.client.CoreV1Api() + while True: + try: + pod_finished, latest_time = print_logs( + pod_name, namespace, v1_api, latest_time + ) + if pod_finished: + break + except kubernetes.client.exceptions.ApiException as log_exception: + if "ContainerCreating" in json.loads(log_exception.body)["message"]: + logging.warning( + "Cannot yet read logs from the pod: waiting for the container to start." + ) + else: + logging.warning(f"Failed to get logs from the pod: {log_exception}") + time.sleep(SECONDS_QUERY_LOGS_EVERY) + v1_api.delete_namespaced_pod(pod_name, namespace) + + +if __name__ == "__main__": + if len(sys.argv) != 3: + logging.fatal("Expected usage is dispatch_job.py {commit SHA} {platform}") + sys.exit(1) + main(sys.argv[1], sys.argv[2]) diff --git a/premerge/buildbot/dispatch_job_test.py b/premerge/buildbot/dispatch_job_test.py new file mode 100644 index 000000000..8a7dc311a --- /dev/null +++ b/premerge/buildbot/dispatch_job_test.py @@ -0,0 +1,86 @@ +"""Tests for the dispatch_job.py script.""" + +import unittest +import datetime +import dateutil + +import dispatch_job + + +class TestDispatchJobs(unittest.TestCase): + def test_get_logs_first_time(self): + """Test we return the correct logs if we have not seen any before.""" + log_lines = [ + "2025-07-29T15:48:00.259595535Z test1", + "2025-07-29T15:48:00.383251277Z test2", + ] + current_timestamp = datetime.datetime.min + latest_timestamp, lines_to_print = dispatch_job.get_logs_to_print( + log_lines, current_timestamp + ) + self.assertSequenceEqual( + lines_to_print, + [ + "2025-07-29T15:48:00.259595535Z test1", + "2025-07-29T15:48:00.383251277Z test2", + ], + ) + self.assertEqual( + latest_timestamp, dateutil.parser.parse("2025-07-29T15:48:00.383251277") + ) + + def test_get_logs_nonoverlapping(self): + """Test we return the correct logs for non-overlapping ranges. + + Test that if the timestamp of the last log that we have printed is + less than the current set returned by kubernetes, we return the correct + lines. + """ + log_lines = [ + "2025-07-29T15:48:01.787177054Z test1", + "2025-07-29T15:48:03.074715108Z test2", + ] + current_timestamp = dateutil.parser.parse("2025-07-29T15:48:00.383251277") + latest_timestamp, lines_to_print = dispatch_job.get_logs_to_print( + log_lines, current_timestamp + ) + self.assertSequenceEqual( + lines_to_print, + [ + "2025-07-29T15:48:01.787177054Z test1", + "2025-07-29T15:48:03.074715108Z test2", + ], + ) + self.assertEqual( + latest_timestamp, dateutil.parser.parse("2025-07-29T15:48:03.074715108") + ) + + def test_get_logs_overlapping(self): + """Test we return the correct logs for overlapping ranges. + + Test that if the last line to be printed is contained within the logs + kubernetes returned, we skip the lines that have already been printed. + """ + log_lines = [ + "2025-07-29T15:48:00.383251277Z test1", + "2025-07-29T15:48:01.787177054Z test2", + "2025-07-29T15:48:03.074715108Z test3", + ] + current_timestamp = dateutil.parser.parse("2025-07-29T15:48:00.383251277") + latest_timestamp, lines_to_print = dispatch_job.get_logs_to_print( + log_lines, current_timestamp + ) + self.assertSequenceEqual( + lines_to_print, + [ + "2025-07-29T15:48:01.787177054Z test2", + "2025-07-29T15:48:03.074715108Z test3", + ], + ) + self.assertEqual( + latest_timestamp, dateutil.parser.parse("2025-07-29T15:48:03.074715108") + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/premerge/buildbot/requirements.lock.txt b/premerge/buildbot/requirements.lock.txt new file mode 100644 index 000000000..a49faeed5 --- /dev/null +++ b/premerge/buildbot/requirements.lock.txt @@ -0,0 +1,92 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --output-file=requirements.lock.txt requirements.txt +# +attrs==25.3.0 + # via twisted +autobahn==24.4.2 + # via buildbot-worker +automat==25.4.16 + # via twisted +buildbot-worker==3.11.7 + # via -r requirements.txt +cachetools==5.5.2 + # via google-auth +certifi==2025.7.14 + # via + # kubernetes + # requests +cffi==1.17.1 + # via cryptography +charset-normalizer==3.4.2 + # via requests +constantly==23.10.4 + # via twisted +cryptography==45.0.5 + # via autobahn +durationpy==0.10 + # via kubernetes +google-auth==2.40.3 + # via kubernetes +hyperlink==21.0.0 + # via + # autobahn + # twisted +idna==3.10 + # via + # hyperlink + # requests +incremental==24.7.2 + # via twisted +kubernetes==33.1.0 + # via -r requirements.txt +msgpack==1.1.1 + # via buildbot-worker +oauthlib==3.3.1 + # via + # kubernetes + # requests-oauthlib +pyasn1==0.6.1 + # via + # pyasn1-modules + # rsa +pyasn1-modules==0.4.2 + # via google-auth +pycparser==2.22 + # via cffi +python-dateutil==2.9.0.post0 + # via kubernetes +pyyaml==6.0.2 + # via kubernetes +requests==2.32.4 + # via + # kubernetes + # requests-oauthlib +requests-oauthlib==2.0.0 + # via kubernetes +rsa==4.9.1 + # via google-auth +six==1.17.0 + # via + # buildbot-worker + # kubernetes + # python-dateutil +twisted==25.5.0 + # via buildbot-worker +txaio==25.6.1 + # via autobahn +typing-extensions==4.14.1 + # via twisted +urllib3==2.5.0 + # via + # kubernetes + # requests +websocket-client==1.8.0 + # via kubernetes +zope-interface==7.2 + # via twisted + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/premerge/buildbot/requirements.txt b/premerge/buildbot/requirements.txt new file mode 100644 index 000000000..dfd1b8447 --- /dev/null +++ b/premerge/buildbot/requirements.txt @@ -0,0 +1,2 @@ +kubernetes==33.1.0 +buildbot-worker==3.11.7 diff --git a/premerge/buildbot/startup.sh b/premerge/buildbot/startup.sh new file mode 100644 index 000000000..7f39b6463 --- /dev/null +++ b/premerge/buildbot/startup.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# This script performs all the necessary setup and then starts the buildbot +# worker. + +mkdir /worker +buildbot-worker create-worker /worker \ + lab.llvm.org:9994 \ + $BUILDBOT_USERNAME \ + $BUILDBOT_PASSWORD + +echo "Google LLVM Premerge Infra Rotation " \ + > /worker/info/admin + +{ + echo "Premerge container (https://github.com/llvm/llvm-project/pkgs/container/ci-ubuntu-24.04)" + echo "GCP n2/n2d standard instances." +} > /worker/info/host + +buildbot-worker start /worker From 128356c7fcd1127da585de5f6593fee9f56483fe Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Tue, 29 Jul 2025 21:34:38 +0000 Subject: [PATCH 2/4] fix Created using spr 1.3.6 --- .github/workflows/build-premerge-buildbot-container.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build-premerge-buildbot-container.yml b/.github/workflows/build-premerge-buildbot-container.yml index f80d56f3f..786b9fc02 100644 --- a/.github/workflows/build-premerge-buildbot-container.yml +++ b/.github/workflows/build-premerge-buildbot-container.yml @@ -11,8 +11,6 @@ on: - .github/workflows/build-premerge-builtbot-container.yml - 'premerge/builtbot/**' pull_request: - branches: - - main paths: - .github/workflows/build-premerge-builtbot-container.yml - 'premerge/buildbot/**' From c0a5cb15e15cc47c0fe6611b26886da4b4004709 Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Wed, 30 Jul 2025 16:48:15 +0000 Subject: [PATCH 3/4] feedback Created using spr 1.3.6 --- .github/workflows/build-premerge-buildbot-container.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-premerge-buildbot-container.yml b/.github/workflows/build-premerge-buildbot-container.yml index 786b9fc02..cfb2a0f49 100644 --- a/.github/workflows/build-premerge-buildbot-container.yml +++ b/.github/workflows/build-premerge-buildbot-container.yml @@ -8,11 +8,11 @@ on: branches: - main paths: - - .github/workflows/build-premerge-builtbot-container.yml - - 'premerge/builtbot/**' + - .github/workflows/build-premerge-buildbot-container.yml + - 'premerge/buildbot/**' pull_request: paths: - - .github/workflows/build-premerge-builtbot-container.yml + - .github/workflows/build-premerge-buildbot-container.yml - 'premerge/buildbot/**' jobs: From 00f3f1c6df79fe7da5d3379fda849daf41f2ab9a Mon Sep 17 00:00:00 2001 From: Aiden Grossman Date: Thu, 31 Jul 2025 00:42:34 +0000 Subject: [PATCH 4/4] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20ch?= =?UTF-8?q?anges=20introduced=20through=20rebase?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Created using spr 1.3.6 [skip ci] --- premerge/buildbot/dispatch_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/premerge/buildbot/dispatch_job.py b/premerge/buildbot/dispatch_job.py index de62b8673..31fd4b152 100644 --- a/premerge/buildbot/dispatch_job.py +++ b/premerge/buildbot/dispatch_job.py @@ -16,7 +16,7 @@ import kubernetes -PLATFORM_TO_NAMESPACE = {"Linux": "llvm-premerge-linux-builbot"} +PLATFORM_TO_NAMESPACE = {"Linux": "llvm-premerge-linux-buildbot"} LOG_SECONDS_TO_QUERY = 10 SECONDS_QUERY_LOGS_EVERY = 5