Merge pull request #4 from elastic/docker-manual-upload

Pipeline and scripts for custom node 18 build (with glibc 2.17)

Author: delanni

.buildkite/pipeline.yml

@@ -1,17 +1,38 @@
-agents:
-  provider: aws
-  instanceType: m6g.xlarge
-  imagePrefix: drivah-ubuntu-2204-aarch64
+env:
+  TARGET_VERSION: 18.16.1
 
 steps:
-  - label: ":buildah: Building Container Images for ARM64"
-    branches: "*"
-    command: |
-      buildah --version
-      drivah build --changed-since=main ./containers/arm64
+  - label: Annotate build with node version
+    command: buildkite-agent annotate "node.js v$${OVERRIDE_TARGET_VERSION:-$$TARGET_VERSION} with glibc 2.17" --style 'info'
+    agents:
+      queue: kibana-default
 
-  # - label: ":buildah: Building Container Images for AMD64"
-  #   branches: "*"
-  #   command: |
-  #     buildah --version
-  #     drivah build --changed-since=main ./.buildkite/containers/amd64
+  - label: Build custom node.js artifacts with glibc 2.17 for x64
+    command:
+      - scripts/create_build_images.sh
+      - scripts/build_nodejs.sh
+      - scripts/upload_nodejs_artifacts.sh
+    env:
+      ARCH: amd64
+    agents:
+      queue: c2-16
+    timeout_in_minutes: 60 # ideally runs in ~30m
+
+  - label: Build custom node.js artifacts with glibc 2.17 for arm64
+    command:
+      - scripts/create_build_images.sh
+      - scripts/build_nodejs.sh
+      - scripts/upload_nodejs_artifacts.sh
+    env:
+      ARCH: arm64
+    agents:
+      queue: c2-60 # cross-compiling takes a while
+    timeout_in_minutes: 180 # ideally runs in ~2hr
+
+  - wait
+
+  - label: Fix SHASUMS256.txt with newly built files' hashes
+    command:
+      - scripts/replace_sha_hashes.sh
+    agents:
+      queue: kibana-default

.gitignore

@@ -1,4 +1,4 @@
 .vscode
 .idea
 
-workdir
+workdir/

README.md

@@ -1,2 +1,46 @@
 # kibana-custom-nodejs-builds
-Contains configuration and sources to build node.js for custom platforms
+Contains configuration and sources to build node.js
+
+The main usecase right now is building `node.js@18+` with `[email protected]`, which is required for some older platforms. (More context: https://github.com/nodejs/unofficial-builds/pull/69)
+
+## Running locally
+You can run most scripts locally on Mac/Linux. You'll need a few of the build/infra tools:
+ - Docker
+ - node.js
+ - gsutil (`brew install google-cloud-sdk`)
+
+Export some env variables required for the builds
+```sh
+  export ARCH="arm64"
+  export TARGET_VERSION="18.16.1"
+```
+
+Then run individual scripts locally:
+```sh
+  ./scripts/create_build_images.sh
+  ./scripts/build_nodejs.sh
+```
+
+## Docker image for node.js builds
+One of the main components we need to create in this step is a docker image for an environment that's set up for building `node.js`.
+
+The bits for this component are in the [build-image-config](./build-image-config/) folder.
+
+The docker image uses mounted directories as working directories, as well as outputting the artifacts in these directories.
+
+
+## Scripts for running the builds
+Most of the `buildkite` logic is sheltered in the [scripts](./scripts/) directory.
+
+
+
+## Context
+During development, we found some more information that can be helpful as context, should anyone find this repo again
+
+ - This repository is only needed while
+   - centos:7 / RHEL7 is supported by Elastic, and we ship node.js with Kibana
+   - the unofficial-builds repo accepts a linux/arm64 build (https://github.com/nodejs/unofficial-builds/pull/83)
+ - The created Docker images needn't be pushed
+   - they can be used once for the build, then rebuilt in case we need to run it again
+ - I decided to remove the `VARIATION` attribute on the node.js:
+   - build would result in the variation showing up in the file and folder names, making the logistics more difficult, if we want to keep this mostly transparent for Kibana

containers/custom-nodejs-builder-amd64/Dockerfile renamed to build-image-config/Dockerfile

@@ -22,7 +22,6 @@ RUN ulimit -n 1024 \
          glibc-devel
 
 COPY --chown=node:node entrypoint.sh /home/node/entrypoint.sh
-COPY --chown=node:node re2_entrypoint.sh /home/node/re2_entrypoint.sh
 
 USER node
 

containers/custom-nodejs-builder-arm64/entrypoint.sh renamed to build-image-config/entrypoint.sh

@@ -5,14 +5,17 @@ set -x
 
 release_url_base="$1"
 full_version="$2"
-config_flags=${3:-""} #"--without-dtrace --without-npm --without-etw"
+config_flags=${3:-""}
 
 if [[ $(arch) == x86_64 ]]; then
   architecture="x64";
 else
   architecture="arm64"
 fi
 
+ls -la  "/home/node/workdir/src"
+ls -la  "/home/node/workdir/src/node-${full_version}"
+
 cd "/home/node/workdir/src/node-${full_version}"
 
 # Compile from source
@@ -25,10 +28,11 @@ export CXX="ccache g++"
 make -j"$(getconf _NPROCESSORS_ONLN)" binary V= \
   DESTCPU="$architecture" \
   ARCH="$architecture" \
-  VARIATION="glibc-217" \
   DISTTYPE="release" \
   RELEASE_URLBASE="$release_url_base" \
   CONFIG_FLAGS="$config_flags"
 
 mkdir -p /home/node/workdir/dist/
+chmod a+w /home/node/workdir/dist
 mv node-*.tar.?z /home/node/workdir/dist/
+chmod a+rwx /home/node/workdir/dist/*

catalog-info.yaml

@@ -8,21 +8,3 @@ metadata:
   links:
     - title: Pipeline
       url: https://buildkite.com/elastic/kibana-custom-nodejs-builds
-
-spec:
-  type: buildkite-pipeline
-  owner: group:kibana-operations
-  system: buildkite
-  implementation:
-    apiVersion: buildkite.elastic.dev/v1
-    kind: Pipeline
-    metadata:
-      name: kibana-custom-nodejs-builds
-    spec:
-      repository: elastic/kibana-custom-nodejs-builds
-      pipeline_file: ".buildkite/pipeline.yml"
-      teams:
-        kibana-operations:
-          access_level: MANAGE_BUILD_AND_READ
-        everyone:
-          access_level: READ_ONLY