Skip to content

Commit 8f20c2c

Browse files
nwang92nwang92
authored
Converting redhat + debian platforms to py3 (#444)
* Converting redhat + debian platforms to py3 * Fixing py2 installation in py23 images * Feature/bump rhel version (#448) * Bumping ubi8 base from 8.3-201 to 8.3-230 * Patching security release for python3-apt * Updating p11-kit for CVE-2020-29363 * Adding CVE to whitelist due to false positives * Making python3 default in py23 images
1 parent ee72f39 commit 8f20c2c

File tree

18 files changed

+191
-157
lines changed

18 files changed

+191
-157
lines changed

‎.circleci/config.yml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,10 @@ executors:
44
py3:
55
docker:
66
- image: cimg/python:3.7
7-
circleci_medium:
7+
circleci_large:
88
machine:
99
image: ubuntu-1604:201903-01
10-
resource_class: medium
10+
resource_class: large
1111
circleci_xlarge:
1212
machine:
1313
image: ubuntu-1604:201903-01
@@ -72,7 +72,7 @@ jobs:
7272
whitelist: /root/project/clair-whitelist.yml
7373

7474
test_redhat_8_small:
75-
executor: circleci_medium
75+
executor: circleci_large
7676
steps:
7777
- checkout
7878
- attach_workspace:
@@ -136,7 +136,7 @@ jobs:
136136
path: test-results
137137

138138
test_debian_10_small:
139-
executor: circleci_medium
139+
executor: circleci_large
140140
steps:
141141
- checkout
142142
- attach_workspace:

‎base/centos-7/Dockerfile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright 2018-2020 Splunk
1+
# Copyright 2018-2021 Splunk
22
#
33
# Licensed under the Apache License, Version 2.0 (the "License");
44
# you may not use this file except in compliance with the License.
@@ -16,7 +16,7 @@ FROM centos:7
1616
LABEL maintainer="[email protected]"
1717

1818
ARG SCLOUD_URL
19-
ENV SCLOUD_URL ${SCLOUD_URL}
19+
ENV SCLOUD_URL=${SCLOUD_URL}
2020

2121
COPY install.sh /install.sh
2222
RUN /install.sh && rm -rf /install.sh

‎base/centos-7/install.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/bin/bash
2-
# Copyright 2018-2020 Splunk
2+
# Copyright 2018-2021 Splunk
33
#
44
# Licensed under the Apache License, Version 2.0 (the "License");
55
# you may not use this file except in compliance with the License.
@@ -46,4 +46,4 @@ rm -rf /etc/security/limits.d/20-nproc.conf
4646

4747
# Clean
4848
yum clean all
49-
rm -rf /install.sh /anaconda-post.log /var/log/anaconda/*
49+
rm -rf /anaconda-post.log /var/log/anaconda/*

‎base/centos-8/Dockerfile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright 2018-2020 Splunk
1+
# Copyright 2018-2021 Splunk
22
#
33
# Licensed under the Apache License, Version 2.0 (the "License");
44
# you may not use this file except in compliance with the License.
@@ -16,7 +16,7 @@ FROM centos:8
1616
LABEL maintainer="[email protected]"
1717

1818
ARG SCLOUD_URL
19-
ENV SCLOUD_URL ${SCLOUD_URL}
19+
ENV SCLOUD_URL=${SCLOUD_URL}
2020

2121
COPY install.sh /install.sh
2222
RUN /install.sh && rm -rf /install.sh

‎base/centos-8/install.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/bin/bash
2-
# Copyright 2018-2020 Splunk
2+
# Copyright 2018-2021 Splunk
33
#
44
# Licensed under the Apache License, Version 2.0 (the "License");
55
# you may not use this file except in compliance with the License.
@@ -52,4 +52,4 @@ ln -s /bin/python3 /bin/python
5252

5353
# Clean
5454
yum clean all
55-
rm -rf /install.sh /anaconda-post.log /var/log/anaconda/*
55+
rm -rf /anaconda-post.log /var/log/anaconda/*

‎base/debian-10/Dockerfile

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright 2018 Splunk
1+
# Copyright 2018-2021 Splunk
22
#
33
# Licensed under the Apache License, Version 2.0 (the "License");
44
# you may not use this file except in compliance with the License.
@@ -16,9 +16,9 @@ FROM debian:buster-slim
1616
LABEL maintainer="[email protected]"
1717

1818
ARG SCLOUD_URL
19-
ENV SCLOUD_URL ${SCLOUD_URL}
20-
21-
ENV DEBIAN_FRONTEND=noninteractive
19+
ENV SCLOUD_URL=${SCLOUD_URL} \
20+
DEBIAN_FRONTEND=noninteractive \
21+
PYTHON_VERSION=3.7.9
2222

2323
COPY install.sh /install.sh
2424
RUN /install.sh && rm -rf /install.sh

‎base/debian-10/install.sh

Lines changed: 50 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/bin/bash
2-
# Copyright 2018 Splunk
2+
# Copyright 2018-2021 Splunk
33
#
44
# Licensed under the Apache License, Version 2.0 (the "License");
55
# you may not use this file except in compliance with the License.
@@ -14,8 +14,10 @@
1414
# limitations under the License.
1515

1616
set -e
17-
apt update
18-
apt install -y locales wget gnupg
17+
18+
# Generate UTF-8 char map and locale
19+
apt-get update -y
20+
apt-get install -y --no-install-recommends locales wget gnupg
1921
echo "en_US.UTF-8 UTF-8" > /etc/locale.gen
2022
rm -f /usr/share/locale/locale.alias
2123
ln -s /etc/locale.alias /usr/share/locale/locale.alias
@@ -27,35 +29,60 @@ export LANG=en_US.utf8
2729
ln -sf /usr/share/zoneinfo/UTC /etc/localtime
2830
/usr/sbin/dpkg-reconfigure -f noninteractive tzdata
2931

30-
# Install additional dependencies
31-
apt update
32+
# Install utility packages
33+
apt-get install -y --no-install-recommends curl sudo libgssapi-krb5-2 busybox procps acl gcc make \
34+
libffi-dev libssl-dev make build-essential libbz2-dev \
35+
wget xz-utils ca-certificates zlib1g-dev python3-apt \
36+
p11-kit
37+
38+
# Install Python and necessary packages
39+
PY_SHORT=${PYTHON_VERSION%.*}
40+
wget -O /tmp/python.tgz https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz
41+
mkdir -p /tmp/pyinstall
42+
tar -xzC /tmp/pyinstall/ --strip-components=1 -f /tmp/python.tgz
43+
rm /tmp/python.tgz
44+
cd /tmp/pyinstall
45+
./configure --enable-optimizations --prefix=/usr --with-ensurepip=install
46+
make altinstall LDFLAGS="-Wl,--strip-all"
47+
rm -rf /tmp/pyinstall
48+
ln -sf /usr/bin/python${PY_SHORT} /usr/bin/python
49+
ln -sf /usr/bin/pip${PY_SHORT} /usr/bin/pip
50+
# For ansible apt module
51+
cd /tmp
52+
apt-get download python3-apt=1.8.4.3
53+
dpkg -x python3-apt_1.8.4.3_amd64.deb python3-apt
54+
rm python3-apt_1.8.4.3_amd64.deb
55+
cp -r /tmp/python3-apt/usr/lib/python3/dist-packages/* /usr/lib/python${PY_SHORT}/site-packages/
56+
cd /usr/lib/python${PY_SHORT}/site-packages/
57+
cp apt_pkg.cpython-37m-x86_64-linux-gnu.so apt_pkg.so
58+
cp apt_inst.cpython-37m-x86_64-linux-gnu.so apt_inst.so
59+
rm -rf /tmp/python3-apt
60+
# Install splunk-ansible dependencies
61+
cd /
62+
pip -q --no-cache-dir install wheel requests ansible jmespath --upgrade
63+
# Remove tests packaged in python libs
64+
find /usr/lib/ -depth \( -type d -a -not -wholename '*/ansible/plugins/test' -a \( -name test -o -name tests -o -name idle_test \) \) -exec rm -rf '{}' \;
65+
find /usr/lib/ -depth \( -type f -a -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \) -exec rm -rf '{}' \;
66+
find /usr/lib/ -depth \( -type f -a -name 'wininst-*.exe' \) -exec rm -rf '{}' \;
67+
ldconfig
3268

33-
# put back tools for customer support
34-
apt-get install -y --no-install-recommends curl sudo libgssapi-krb5-2 busybox procps acl gcc libpython-dev libffi-dev libssl-dev
35-
apt-get install -y --no-install-recommends python-pip python-setuptools python-requests python-yaml
36-
pip --no-cache-dir install ansible jmespath
37-
apt-get remove -y gcc libffi-dev libssl-dev libpython-dev
38-
apt-get autoremove -y
69+
apt-get remove -y --allow-remove-essential gcc libffi-dev libssl-dev make build-essential libbz2-dev xz-utils zlib1g-dev
70+
apt-get autoremove -y --allow-remove-essential
3971

4072
# Install scloud
4173
wget -O /usr/bin/scloud.tar.gz ${SCLOUD_URL}
4274
tar -xf /usr/bin/scloud.tar.gz -C /usr/bin/
4375
rm /usr/bin/scloud.tar.gz
4476

77+
# Enable busybox symlinks
4578
cd /bin
46-
ln -s busybox clear
47-
ln -s busybox find
48-
ln -s busybox killall
49-
ln -s busybox netstat
50-
ln -s busybox nslookup
51-
ln -s busybox ping
52-
ln -s busybox ping6
53-
ln -s busybox readline
54-
ln -s busybox route
55-
ln -s busybox syslogd
56-
ln -s busybox traceroute
57-
ln -s busybox vi
79+
BBOX_LINKS=( clear find diff hostname killall netstat nslookup ping ping6 readline route syslogd tail traceroute vi )
80+
for item in "${BBOX_LINKS[@]}"
81+
do
82+
ln -s busybox $item || true
83+
done
5884
chmod u+s /bin/ping
5985

86+
# Clean
6087
apt clean autoclean
6188
rm -rf /var/lib/apt/lists/*

‎base/debian-9/Dockerfile

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright 2018 Splunk
1+
# Copyright 2018-2021 Splunk
22
#
33
# Licensed under the Apache License, Version 2.0 (the "License");
44
# you may not use this file except in compliance with the License.
@@ -16,9 +16,9 @@ FROM debian:stretch-slim
1616
LABEL maintainer="[email protected]"
1717

1818
ARG SCLOUD_URL
19-
ENV SCLOUD_URL ${SCLOUD_URL}
20-
21-
ENV DEBIAN_FRONTEND=noninteractive
19+
ENV SCLOUD_URL=${SCLOUD_URL} \
20+
DEBIAN_FRONTEND=noninteractive \
21+
PYTHON_VERSION=3.7.9
2222

2323
COPY install.sh /install.sh
2424
RUN /install.sh && rm -rf /install.sh

‎base/debian-9/install.sh

Lines changed: 50 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/bin/bash
2-
# Copyright 2018 Splunk
2+
# Copyright 2018-2021 Splunk
33
#
44
# Licensed under the Apache License, Version 2.0 (the "License");
55
# you may not use this file except in compliance with the License.
@@ -14,8 +14,10 @@
1414
# limitations under the License.
1515

1616
set -e
17-
apt-get update
18-
apt-get install -y locales wget gnupg apt-utils
17+
18+
# Generate UTF-8 char map and locale
19+
apt-get update -y
20+
apt-get install -y --no-install-recommends locales wget gnupg apt-utils
1921
echo "en_US.UTF-8 UTF-8" > /etc/locale.gen
2022
rm -f /usr/share/locale/locale.alias
2123
ln -s /etc/locale.alias /usr/share/locale/locale.alias
@@ -27,37 +29,59 @@ export LANG=en_US.utf8
2729
ln -sf /usr/share/zoneinfo/UTC /etc/localtime
2830
/usr/sbin/dpkg-reconfigure -f noninteractive tzdata
2931

30-
# Install additional dependencies
31-
echo "deb http://ppa.launchpad.net/ansible/ansible/ubuntu xenial main" >> /etc/apt/sources.list
32-
apt-key adv --keyserver https://keyserver.ubuntu.com --recv-keys 93C4A3FD7BB9C367
33-
apt-get update
32+
# Install utility packages
33+
apt-get install -y --no-install-recommends curl sudo libgssapi-krb5-2 busybox procps acl gcc make \
34+
libffi-dev libssl-dev make build-essential libbz2-dev \
35+
wget xz-utils ca-certificates zlib1g-dev
36+
37+
# Install Python and necessary packages
38+
PY_SHORT=${PYTHON_VERSION%.*}
39+
wget -O /tmp/python.tgz https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz
40+
mkdir -p /tmp/pyinstall
41+
tar -xzC /tmp/pyinstall/ --strip-components=1 -f /tmp/python.tgz
42+
rm /tmp/python.tgz
43+
cd /tmp/pyinstall
44+
./configure --enable-optimizations --prefix=/usr --with-ensurepip=install
45+
make altinstall LDFLAGS="-Wl,--strip-all"
46+
rm -rf /tmp/pyinstall
47+
ln -sf /usr/bin/python${PY_SHORT} /usr/bin/python
48+
ln -sf /usr/bin/pip${PY_SHORT} /usr/bin/pip
49+
# For ansible apt module
50+
cd /tmp
51+
apt-get download python3-apt=1.4.2
52+
dpkg -x python3-apt_1.4.2_amd64.deb python3-apt
53+
rm python3-apt_1.4.2_amd64.deb
54+
cp -r /tmp/python3-apt/usr/lib/python3/dist-packages/* /usr/lib/python${PY_SHORT}/site-packages/
55+
cd /usr/lib/python${PY_SHORT}/site-packages/
56+
cp apt_pkg.cpython-35m-x86_64-linux-gnu.so apt_pkg.so
57+
cp apt_inst.cpython-35m-x86_64-linux-gnu.so apt_inst.so
58+
rm -rf /tmp/python3-apt
59+
# Install splunk-ansible dependencies
60+
cd /
61+
pip -q --no-cache-dir install wheel requests ansible jmespath --upgrade
62+
# Remove tests packaged in python libs
63+
find /usr/lib/ -depth \( -type d -a -not -wholename '*/ansible/plugins/test' -a \( -name test -o -name tests -o -name idle_test \) \) -exec rm -rf '{}' \;
64+
find /usr/lib/ -depth \( -type f -a -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \) -exec rm -rf '{}' \;
65+
find /usr/lib/ -depth \( -type f -a -name 'wininst-*.exe' \) -exec rm -rf '{}' \;
66+
ldconfig
3467

35-
# put back tools for customer support
36-
apt-cache show ansible
37-
apt-get install -y --no-install-recommends ansible curl sudo libgssapi-krb5-2 busybox procps acl
38-
apt-get install -y --no-install-recommends python-requests python-jmespath
68+
apt-get remove -y --allow-remove-essential gcc libffi-dev libssl-dev make build-essential libbz2-dev xz-utils zlib1g-dev
69+
apt-get autoremove -y --allow-remove-essential
3970

4071
# Install scloud
4172
wget -O /usr/bin/scloud.tar.gz ${SCLOUD_URL}
4273
tar -xf /usr/bin/scloud.tar.gz -C /usr/bin/
4374
rm /usr/bin/scloud.tar.gz
4475

76+
# Enable busybox symlinks
4577
cd /bin
46-
ln -s busybox clear
47-
ln -s busybox find
48-
ln -s busybox diff
49-
ln -s busybox killall
50-
ln -s busybox netstat
51-
ln -s busybox nslookup
52-
ln -s busybox ping
53-
ln -s busybox ping6
54-
ln -s busybox readline
55-
ln -s busybox route
56-
ln -s busybox syslogd
57-
ln -s busybox tail
58-
ln -s busybox traceroute
59-
ln -s busybox vi
78+
BBOX_LINKS=( clear find diff hostname killall netstat nslookup ping ping6 readline route syslogd tail traceroute vi )
79+
for item in "${BBOX_LINKS[@]}"
80+
do
81+
ln -s busybox $item || true
82+
done
6083
chmod u+s /bin/ping
6184

62-
apt-get clean autoclean
85+
# Clean
86+
apt clean autoclean
6387
rm -rf /var/lib/apt/lists/*

‎base/redhat-8/Dockerfile

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright 2018-2020 Splunk
1+
# Copyright 2018-2021 Splunk
22
#
33
# Licensed under the Apache License, Version 2.0 (the "License");
44
# you may not use this file except in compliance with the License.
@@ -16,7 +16,8 @@
1616
# the container catalog moved from registry.access.redhat.com to registry.redhat.io
1717
# So at some point before they deprecate the old registry we have to make sure that
1818
# we have access to the new registry and change where we pull the ubi image from.
19-
FROM registry.access.redhat.com/ubi8/ubi-minimal:8.2-267
19+
FROM registry.access.redhat.com/ubi8/ubi-minimal:8.3-230
20+
2021
LABEL name="splunk" \
2122
maintainer="[email protected]" \
2223
vendor="splunk" \
@@ -25,7 +26,8 @@ LABEL name="splunk" \
2526
description="Splunk Enterprise is a platform for operational intelligence. Our software lets you collect, analyze, and act upon the untapped value of big data that your technology infrastructure, security systems, and business applications generate. It gives you insights to drive operational performance and business results."
2627

2728
ARG SCLOUD_URL
28-
ENV SCLOUD_URL ${SCLOUD_URL}
29+
ENV SCLOUD_URL=${SCLOUD_URL} \
30+
PYTHON_VERSION=3.7.9
2931

3032
COPY install.sh /install.sh
3133

0 commit comments

Comments
 (0)