From 29de89975f58631db7a3fb451691f95bd60f9bf9 Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Tue, 10 Jun 2025 14:17:09 +0200 Subject: [PATCH 1/7] compile-options: Fixed shellcheck warnings Signed-off-by: Lars Erik Wik --- build-scripts/compile-options | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/build-scripts/compile-options b/build-scripts/compile-options index 52ec21e3c..73e46040b 100644 --- a/build-scripts/compile-options +++ b/build-scripts/compile-options @@ -5,7 +5,7 @@ # Autodect PROJECT if not set -if [ x"$PROJECT" = x ] +if [ -z "$PROJECT" ] then case x"$JOB_NAME" in *-community-*) PROJECT=community;; @@ -18,9 +18,9 @@ fi # If still not set, then either we are running outside Jenkins, or this # is not a main "build" type job (it could be the bootstrap job). # Do directory-based auto-detection. -if [ x"$PROJECT" = x ] +if [ -z "$PROJECT" ] then - if [ -d $BASEDIR/nova ] + if [ -d "$BASEDIR"/nova ] then PROJECT=nova else @@ -34,7 +34,7 @@ export PROJECT # When running manually, you can just export this variable. # It's a flag: if it's set to 1 - then we use system OpenSSL. # Otherwise, we build it. -if [ x"$SYSTEM_SSL" = x ] +if [ -z "$SYSTEM_SSL" ] then # We don't bundle OpenSSL on RHEL 8 (and newer in the future) if [ "$OS" = "rhel" ] && expr "$OS_VERSION" ">=" "8" >/dev/null @@ -82,7 +82,7 @@ case "$OS_FAMILY" in esac # When we don't bundle OpenSSL, then we need to pull it from /usr/lib64. -if [ x"$SYSTEM_SSL" = x1 ] +if [ "$SYSTEM_SSL" = 1 ] then LDFLAGS="$LDFLAGS -L/usr/lib64" fi @@ -100,7 +100,7 @@ EMBEDDED_DB="lmdb" ############### Fill in build dependencies in DEPS variable ################ DEPS= -[ $OS_FAMILY = mingw ] && var_append DEPS "pthreads-w32 libgnurx" +[ "$OS_FAMILY" = mingw ] && var_append DEPS "pthreads-w32 libgnurx" # libgcc_s.so is needed before we compile any other dependency # on some platforms! @@ -110,7 +110,7 @@ esac var_append DEPS "$EMBEDDED_DB pcre2" -if ! [ x"$SYSTEM_SSL" = x1 ] +if ! [ "$SYSTEM_SSL" = 1 ] then # FIXME: Why do we need zlib? # ANSWER: Openssl uses it optionally, TODO DISABLE @@ -160,7 +160,7 @@ case "$EXPLICIT_ROLE" in hub) ROLE=hub;; *) # Not running under Jenkins? - if [ x"$JENKINS_SERVER_COOKIE" = x ] + if [ -z "$JENKINS_SERVER_COOKIE" ] then case "$PROJECT-$ARCH-$OS-${OS_VERSION}" in community-*) ROLE=agent;; From b4f173137f3ca8596d1cfd97fe08ebf390fbd3b1 Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Tue, 10 Jun 2025 14:56:07 +0200 Subject: [PATCH 2/7] compile-options: Remove option to specify embedded DB We don't really build with something else then lmdb. Hence, we don't need the option to build with something else. Signed-off-by: Lars Erik Wik --- build-scripts/compile-options | 6 +----- build-scripts/configure | 6 +----- scripts/deptool.py | 3 --- 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/build-scripts/compile-options b/build-scripts/compile-options index 73e46040b..512985a7f 100644 --- a/build-scripts/compile-options +++ b/build-scripts/compile-options @@ -92,10 +92,6 @@ export LDFLAGS DEB_LDFLAGS_APPEND="$LDFLAGS" export DEB_LDFLAGS_APPEND -# Embedded DB selection -EMBEDDED_DB="lmdb" - - ############### Fill in build dependencies in DEPS variable ################ @@ -108,7 +104,7 @@ case "$OS_FAMILY" in solaris|aix) var_append DEPS "libgcc" ;; esac -var_append DEPS "$EMBEDDED_DB pcre2" +var_append DEPS "lmdb pcre2" if ! [ "$SYSTEM_SSL" = 1 ] then diff --git a/build-scripts/configure b/build-scripts/configure index 931fa0319..be875eac3 100755 --- a/build-scripts/configure +++ b/build-scripts/configure @@ -15,12 +15,8 @@ esac P=$BUILDPREFIX -ARGS="--prefix=$P --libdir=$P/lib --with-workdir=$P --sysconfdir=/etc --with-openssl=$P --with-pcre2=$P --with-librsync=$P --with-init-script" +ARGS="--prefix=$P --libdir=$P/lib --with-workdir=$P --sysconfdir=/etc --with-lmdb=$P --with-openssl=$P --with-pcre2=$P --with-librsync=$P --with-init-script" -if [ $EMBEDDED_DB = lmdb ] -then - var_append ARGS "--with-lmdb=$P" -fi case "$DEPS" in *pthreads-w32*) var_append ARGS "--with-pthreads=$P" ;; esac diff --git a/scripts/deptool.py b/scripts/deptool.py index d884c9cad..607d1ed29 100644 --- a/scripts/deptool.py +++ b/scripts/deptool.py @@ -236,8 +236,6 @@ def deps_list(self, ref="master"): """Returns a sorted list of dependencies for given ref, for example: `["lcov", "libgnurx", "pthreads-w32"]`. Assumes the proper ref is checked out by `self.buildscripts_repo`. """ - # TODO: get value of $EMBEDDED_DB from file - embedded_db = "lmdb" if ref == "3.7.x": options_file = self.buildscripts_repo.get_file( "build-scripts/install-dependencies" @@ -264,7 +262,6 @@ def deps_list(self, ref="master"): # in the middle we also do some clean-ups only_deps = ( " ".join(only_deps) - .replace("$EMBEDDED_DB", embedded_db) .replace("libgcc ", "") .split(" ") ) From e33850545b86c86a5dc347b8d11ed07aa58acb69 Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Tue, 10 Jun 2025 16:37:35 +0200 Subject: [PATCH 3/7] compile-options: Document script Ticket: ENT-12600 Signed-off-by: Lars Erik Wik --- build-scripts/compile-options | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/build-scripts/compile-options b/build-scripts/compile-options index 512985a7f..6f3979ca1 100644 --- a/build-scripts/compile-options +++ b/build-scripts/compile-options @@ -62,6 +62,13 @@ case "$OS_FAMILY" in ;; hpux) LDFLAGS="-L$BUILDPREFIX/lib -Wl,+b$BUILDPREFIX/lib" + + # Use ‘gcc’ when building things on HP-UX + # + # HP-UX ships with ‘cc’ which invokes the ‘HP-UX bundled C compiler’ which + # lacks some functionality we need to build things. Among the other things, + # it doesn’t know how to link ‘.so’ files directly and it doesn’t recognize + # them as valid input type. CC=gcc export CC ;; From 8ef45267ea03284bd4ff9e4289d23a6644be5266 Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Fri, 13 Jun 2025 15:55:00 +0200 Subject: [PATCH 4/7] compile-options: Refactor/document dependencies Ticket: ENT-12600 Signed-off-by: Lars Erik Wik --- build-scripts/compile-options | 142 +++++++++++++++++++++++----------- 1 file changed, 95 insertions(+), 47 deletions(-) diff --git a/build-scripts/compile-options b/build-scripts/compile-options index 6f3979ca1..5fa4a834f 100644 --- a/build-scripts/compile-options +++ b/build-scripts/compile-options @@ -103,51 +103,84 @@ export DEB_LDFLAGS_APPEND ############### Fill in build dependencies in DEPS variable ################ DEPS= -[ "$OS_FAMILY" = mingw ] && var_append DEPS "pthreads-w32 libgnurx" -# libgcc_s.so is needed before we compile any other dependency -# on some platforms! -case "$OS_FAMILY" in - solaris|aix) var_append DEPS "libgcc" ;; -esac +# Windows specific dependencies +if [ "$OS_FAMILY" = mingw ]; then + # Win32 does not support pthreads natively. The pthreads-w32 project provide + # solution to this problem. + var_append DEPS "pthreads-w32" -var_append DEPS "lmdb pcre2" - -if ! [ "$SYSTEM_SSL" = 1 ] -then - # FIXME: Why do we need zlib? - # ANSWER: Openssl uses it optionally, TODO DISABLE - var_append DEPS "zlib openssl" + # A port of the regex functionality from the glibc Library for use on + # Windows platforms + var_append DEPS "libgnurx" fi -# libsasl needed for solaris +# AIX / Solaris specific dependencies case "$OS_FAMILY" in - solaris|hpux) var_append DEPS "sasl2" ;; -esac + solaris|aix) -# iconv is needed for libxml2 on some platforms -case "$OS_FAMILY" in - aix|solaris) var_append DEPS "libiconv" ;; + # libgcc_s.so is needed before we compile any other dependency + # on some platforms! + var_append DEPS "libgcc" + + # iconv is needed for libxml2 on some platforms + var_append DEPS "libiconv" + ;; esac -var_append DEPS "libxml2 libyaml" -var_append DEPS "diffutils" -var_append DEPS "librsync" +# We use system bundled SSL on RHEL >= 8 +if ! [ "$SYSTEM_SSL" = 1 ] +then + # zlib is a compression library witch is a dependency of OpenSSL. However, + # can we remove it (CFE-4013)? + var_append DEPS "zlib" -# LDAP functions in the agent -# and LDAP authentication functionality in Mission Portal -case "$PROJECT" in - nova) var_append DEPS "openldap" -esac + # Toolkit for TLS + var_append DEPS "openssl" +fi -case "$PROJECT" in - nova) var_append DEPS "leech" +# Solaris / HP-UX specific dependencies +case "$OS_FAMILY" in + solaris|hpux) + # Generic library that implements the Simple Authentication and Security + # Layer (SASL) framework + var_append DEPS "sasl2" + ;; esac -# libacl & libattr - not for the exotics (linux only?) +# Common dependencies +var_append DEPS "libxml2" # Library for parsing XML +var_append DEPS "libyaml" # Library for parsing YAML +var_append DEPS "diffutils" # Library for comparing files +var_append DEPS "librsync" # Library for synchronization of files +var_append DEPS "lmdb" # Library for key-value store, used extensively by + # CFEngine components (cf-agent, etc) +var_append DEPS "pcre2" # Library for compiling/matching regex + +# Enterprise only dependencies +if [ "$PROJECT" = nova ]; then + # openldap is used for storing accounts, passwords and group memberships + # - Agent has LDAP related policy functions + # - Mission Portal uses LDAP for authentication functionality + var_append DEPS "openldap" + + # leech is used for efficient synchronization of tabular data. + # - cf-hub requests state changes from cf-serverd + # - state changes are recorded by cf-agent + var_append DEPS "leech" +fi + +# Non-exotics dependencies case "$OS_FAMILY" in - hpux|aix|solaris|freebsd|mingw) ;; - *) var_append DEPS "libattr libacl" ;; + hpux|aix|solaris|freebsd|mingw) + ;; + *) + # Library for managing Extended Attributes (xattrs) on filesystems + var_append DEPS "libattr" + + # POSIX Access Control Lists (ACLs) on filesystems + var_append DEPS "libacl" + ;; esac # ROLE @@ -215,32 +248,47 @@ export ROLE case "$ROLE" in # HUB-ONLY dependencies hub) - var_append DEPS "libcurl-hub" - var_append DEPS "nghttp2 libexpat apr apr-util apache git rsync" - var_append DEPS "postgresql php" + # Note that we make a separate curl package for the hub. This is because + # the hub will include the curl binary, but we don't want that for the + # clients. + var_append DEPS "libcurl-hub" # Provides API for performing network requests + var_append DEPS "nghttp2" # Provides implementation of the HTTP/2 protocol. + var_append DEPS "libexpat" # Provides stream-oriented XML parser + var_append DEPS "apr apr-util" # Provides a common interface to underlying OS features (used by Apache) + var_append DEPS "apache" # Provides HTTP server + var_append DEPS "git" # Provides a version control system + var_append DEPS "rsync" # Binary to efficiently synchronize files + var_append DEPS "postgresql" # Relational database + var_append DEPS "php" # Scripting language for web development ;; # AGENT-ONLY dependencies agent) - var_append DEPS "libcurl" + var_append DEPS "libcurl" # Provides API for performing network requests ;; esac +# Make sure init.d script and systemd service is used where needed. +# +# We install in all Linux platforms, because it does no harm on platforms +# without systemd, and it has the advantage of working out of the box on +# platforms that adopt systemd later. +if [ "$OS_FAMILY" = linux ]; then + WITH_SYSTEMD=yes +else + WITH_SYSTEMD=no +fi -# systemd is not a build dependency, but should we generate -# unit files for it? -case "$OS_FAMILY" in - linux) WITH_SYSTEMD=yes;; - *) WITH_SYSTEMD=no;; -esac - - +# Determine whether or not to run tests case "$OS_FAMILY" in - mingw|freebsd) TESTS=no ;; - *) TESTS=all ;; + mingw|freebsd) + TESTS=no + ;; + *) + TESTS=all + ;; esac export TESTS - # Don't let existing LD_LIBRARY_PATH variables disturb the build. # Java sets this when Jenkins launches it. # PS! Solaris shell returns false if the var is already unset. From f92051fd599505c60883cfb07071dd7dc76cbc56 Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Tue, 17 Jun 2025 13:10:10 +0200 Subject: [PATCH 5/7] compile-options: Consistent case statement format Ticket: ENT-12600 Signed-off-by: Lars Erik Wik --- build-scripts/compile-options | 68 ++++++++++++++++++++++++++--------- 1 file changed, 51 insertions(+), 17 deletions(-) diff --git a/build-scripts/compile-options b/build-scripts/compile-options index 5fa4a834f..1ef3ad4e7 100644 --- a/build-scripts/compile-options +++ b/build-scripts/compile-options @@ -8,10 +8,18 @@ if [ -z "$PROJECT" ] then case x"$JOB_NAME" in - *-community-*) PROJECT=community;; - *-enterprise-*) PROJECT=nova;; - *-hub-*) PROJECT=nova;; - *-agent-*) PROJECT=nova;; + *-community-*) + PROJECT=community + ;; + *-enterprise-*) + PROJECT=nova + ;; + *-hub-*) + PROJECT=nova + ;; + *-agent-*) + PROJECT=nova + ;; esac fi @@ -192,25 +200,51 @@ esac # and build according to the role specified by it. case "$EXPLICIT_ROLE" in - agent) ROLE=agent;; - hub) ROLE=hub;; + agent) + ROLE=agent + ;; + hub) + ROLE=hub + ;; *) # Not running under Jenkins? if [ -z "$JENKINS_SERVER_COOKIE" ] then case "$PROJECT-$ARCH-$OS-${OS_VERSION}" in - community-*) ROLE=agent;; + community-*) + ROLE=agent + ;; # We do not support 32 bits hubs anymore - nova-i386-*-*) ROLE=agent;; - nova-s390*-*-*) ROLE=agent;; - nova-*-centos-*) ROLE=hub;; - nova-*-debian-*) ROLE=hub;; - nova-*-opensuse-*) ROLE=hub;; - nova-*-rhel-*) ROLE=hub;; - nova-*-sles-*) ROLE=hub;; - nova-*-ubuntu-*) ROLE=hub;; - nova-*-mingw-*) ROLE=agent;; - nova-*) ROLE=agent;; + nova-i386-*-*) + ROLE=agent + ;; + nova-s390*-*-*) + ROLE=agent + ;; + nova-*-centos-*) + ROLE=hub + ;; + nova-*-debian-*) + ROLE=hub + ;; + nova-*-opensuse-*) + ROLE=hub + ;; + nova-*-rhel-*) + ROLE=hub + ;; + nova-*-sles-*) + ROLE=hub + ;; + nova-*-ubuntu-*) + ROLE=hub + ;; + nova-*-mingw-*) + ROLE=agent + ;; + nova-*) + ROLE=agent + ;; *) echo "Unknown project: $PROJECT" exit 42 From 7a5ab4cd5554091cc850f41cb3b0cba5cd2b52f9 Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Thu, 24 Jul 2025 11:06:57 +0200 Subject: [PATCH 6/7] compile-option: Remove auto detect role code This auto detect role code is weird. It tries to detect whether to build a hub or agent package based on the platform it's being built for. In many places it even decides to build hub packages for platforms where it's not supported. I wonder if life will just go on if I remove it. Signed-off-by: Lars Erik Wik --- build-scripts/compile-options | 74 +---------------------------------- 1 file changed, 2 insertions(+), 72 deletions(-) diff --git a/build-scripts/compile-options b/build-scripts/compile-options index 1ef3ad4e7..75278051a 100644 --- a/build-scripts/compile-options +++ b/build-scripts/compile-options @@ -191,14 +191,6 @@ case "$OS_FAMILY" in ;; esac -# ROLE -# We can autodetect if we are building a hub or an agent, but -# that is suboptimal since the list might change at any time or -# we might want to build an agent package for a platform that is a hub -# or viceversa. By default we assume the old behavior but if the -# environment variable EXPLICIT_ROLE is defined, then we change our behavior -# and build according to the role specified by it. - case "$EXPLICIT_ROLE" in agent) ROLE=agent @@ -207,70 +199,8 @@ case "$EXPLICIT_ROLE" in ROLE=hub ;; *) - # Not running under Jenkins? - if [ -z "$JENKINS_SERVER_COOKIE" ] - then - case "$PROJECT-$ARCH-$OS-${OS_VERSION}" in - community-*) - ROLE=agent - ;; - # We do not support 32 bits hubs anymore - nova-i386-*-*) - ROLE=agent - ;; - nova-s390*-*-*) - ROLE=agent - ;; - nova-*-centos-*) - ROLE=hub - ;; - nova-*-debian-*) - ROLE=hub - ;; - nova-*-opensuse-*) - ROLE=hub - ;; - nova-*-rhel-*) - ROLE=hub - ;; - nova-*-sles-*) - ROLE=hub - ;; - nova-*-ubuntu-*) - ROLE=hub - ;; - nova-*-mingw-*) - ROLE=agent - ;; - nova-*) - ROLE=agent - ;; - *) - echo "Unknown project: $PROJECT" - exit 42 - ;; - esac - echo "Autodetected $ROLE role based on missing Jenkins label and OS." - - else # we are running under Jenkins - - # The "label" variable is set in multi-matrix jobs; - # thus it is *not set* in the bootstrap job. - case x"$label" in - x) - echo "label is not set, assuming that this is not the main build job; Setting ROLE=agent." - ROLE=agent - ;; - *_HUB_*) - echo "Autodetected hub role based on '_HUB_' in Jenkins label." - ROLE=hub - ;; - *) - echo "Autodetected agent role based on missing '_HUB_' in Jenkins label." - ROLE=agent - ;; - esac - fi + echo "$(basename "$0"): Error: Bad EXPLICIT_ROLE '$EXPLICIT_ROLE': defaulting to 'agent'" + ROLE=agent ;; esac From 61170ddccc47b608d2f7e40e1715a6734404d7a2 Mon Sep 17 00:00:00 2001 From: Lars Erik Wik Date: Mon, 4 Aug 2025 13:23:49 +0200 Subject: [PATCH 7/7] compile-options: split comments on dots and commas It makes it easier to read diffs. Ticket: ENT-12600 Signed-off-by: Lars Erik Wik --- build-scripts/compile-options | 46 +++++++++++++++-------------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/build-scripts/compile-options b/build-scripts/compile-options index 75278051a..0caf6140f 100644 --- a/build-scripts/compile-options +++ b/build-scripts/compile-options @@ -23,8 +23,8 @@ then esac fi -# If still not set, then either we are running outside Jenkins, or this -# is not a main "build" type job (it could be the bootstrap job). +# If still not set, then either we are running outside Jenkins, +# or this is not a main "build" type job (it could be the bootstrap job). # Do directory-based auto-detection. if [ -z "$PROJECT" ] then @@ -59,13 +59,11 @@ export SYSTEM_SSL case "$OS_FAMILY" in aix) - # This flag is needed because AIX defaults to producing .a shared libraries, - # but we need it to make .so libraries. It doesn't work correctly when - # specified in the Makefile, so we specify it in the environment instead. + # This flag is needed because AIX defaults to producing .a shared libraries, but we need it to make .so libraries. + # It doesn't work correctly when specified in the Makefile, so we specify it in the environment instead. LDFLAGS="-Wl,-brtl" - # AIX needs default RPATH (libpath on AIX) specified as well, otherwise it - # won't find libc. + # AIX needs default RPATH (libpath on AIX) specified as well, otherwise it won't find libc. LDFLAGS="$LDFLAGS -L$BUILDPREFIX/lib -Wl,-blibpath:$BUILDPREFIX/lib:/usr/lib:/lib" ;; hpux) @@ -73,10 +71,8 @@ case "$OS_FAMILY" in # Use ‘gcc’ when building things on HP-UX # - # HP-UX ships with ‘cc’ which invokes the ‘HP-UX bundled C compiler’ which - # lacks some functionality we need to build things. Among the other things, - # it doesn’t know how to link ‘.so’ files directly and it doesn’t recognize - # them as valid input type. + # HP-UX ships with ‘cc’ which invokes the ‘HP-UX bundled C compiler’ which lacks some functionality we need to build things. + # Among the other things, it doesn’t know how to link ‘.so’ files directly and it doesn’t recognize them as valid input type. CC=gcc export CC ;; @@ -114,12 +110,11 @@ DEPS= # Windows specific dependencies if [ "$OS_FAMILY" = mingw ]; then - # Win32 does not support pthreads natively. The pthreads-w32 project provide - # solution to this problem. + # Win32 does not support pthreads natively. + # The pthreads-w32 project provides a solution to this problem. var_append DEPS "pthreads-w32" - # A port of the regex functionality from the glibc Library for use on - # Windows platforms + # A port of the regex functionality from the glibc Library for use on Windows platforms var_append DEPS "libgnurx" fi @@ -127,8 +122,7 @@ fi case "$OS_FAMILY" in solaris|aix) - # libgcc_s.so is needed before we compile any other dependency - # on some platforms! + # libgcc_s.so is needed before we compile any other dependency on some platforms! var_append DEPS "libgcc" # iconv is needed for libxml2 on some platforms @@ -139,8 +133,8 @@ esac # We use system bundled SSL on RHEL >= 8 if ! [ "$SYSTEM_SSL" = 1 ] then - # zlib is a compression library witch is a dependency of OpenSSL. However, - # can we remove it (CFE-4013)? + # zlib is a compression library witch is a dependency of OpenSSL. + # However, can we remove it (CFE-4013)? var_append DEPS "zlib" # Toolkit for TLS @@ -150,8 +144,7 @@ fi # Solaris / HP-UX specific dependencies case "$OS_FAMILY" in solaris|hpux) - # Generic library that implements the Simple Authentication and Security - # Layer (SASL) framework + # Generic library that implements the Simple Authentication and Security Layer (SASL) framework var_append DEPS "sasl2" ;; esac @@ -212,9 +205,9 @@ export ROLE case "$ROLE" in # HUB-ONLY dependencies hub) - # Note that we make a separate curl package for the hub. This is because - # the hub will include the curl binary, but we don't want that for the - # clients. + # Note that we make a separate curl package for the hub. + # This is because the hub will include the curl binary, + # but we don't want that for the clients. var_append DEPS "libcurl-hub" # Provides API for performing network requests var_append DEPS "nghttp2" # Provides implementation of the HTTP/2 protocol. var_append DEPS "libexpat" # Provides stream-oriented XML parser @@ -233,9 +226,8 @@ esac # Make sure init.d script and systemd service is used where needed. # -# We install in all Linux platforms, because it does no harm on platforms -# without systemd, and it has the advantage of working out of the box on -# platforms that adopt systemd later. +# We install in all Linux platforms, because it does no harm on platforms without systemd, +# and it has the advantage of working out of the box on platforms that adopt systemd later. if [ "$OS_FAMILY" = linux ]; then WITH_SYSTEMD=yes else