diff --git a/.github/workflows/ci-enterprise.yaml b/.github/workflows/ci-enterprise.yaml index 10afd7a00e..00eca74571 100644 --- a/.github/workflows/ci-enterprise.yaml +++ b/.github/workflows/ci-enterprise.yaml @@ -184,12 +184,15 @@ jobs: scanners: "vuln" test-container: + strategy: + matrix: + deployment_type: ["docker", "podman"] runs-on: ubuntu-latest + continue-on-error: true needs: - meta - build_action - container: - image: python:3.9-buster + services: splunk: image: splunk/splunk:${{ fromJson(needs.meta.outputs.matrix_supportedSplunk)[0].version }} @@ -202,41 +205,68 @@ jobs: SPLUNK_PASSWORD: Changed@11 SPLUNK_START_ARGS: --accept-license SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - sc4s: - image: ${{ needs.meta.outputs.container_base }} - ports: - - 514:514 - - 601:601 - - 5614:5514 - - 5601:5601 - - 6000:6000 - - 6002:6002 - - 9000:9000 - env: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" - SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION: "yes" - SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT: 6000 - SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT: 5514 - SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT: 5514 - SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT: 5601 - SC4S_LISTEN_SPECTRACOM_NTP_TCP_PORT: 6002 - SC4S_LISTEN_CISCO_ESA_TCP_PORT: 9000 - SC4S_LISTEN_RARITAN_DSX_TCP_PORT: 9001 - SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL: "yes" - SC4S_SOURCE_RICOH_SYSLOG_FIXHOST: "yes" - TEST_SC4S_ACTIVATE_EXAMPLES: "yes" - SC4S_DEBUG_CONTAINER: "yes" - SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG: "yes" - SC4S_NETAPP_ONTAP_NEW_FORMAT: "yes" - SC4S_USE_VPS_CACHE: "yes" + steps: - name: Checkout uses: actions/checkout@v4 with: submodules: false persist-credentials: false + - name: Install Ansible and other dependencies as python package + run: | + pip install ansible~=6.1.0 --no-cache-dir \ + && pip install pywinrm>=0.4.2 --no-cache-dir \ + && pip install ansible-lint>=6.0.0 --no-cache-dir \ + && pip install docker --no-cache-dir + - name: Configure Ansible Environment Variables + env: + ANSIBLE_CONFIG: ansible.cfg + ANSIBLE_HOST_KEY_CHECKING: False + run: | + echo "ANSIBLE_CONFIG is set to: $ANSIBLE_CONFIG" + echo "ANSIBLE_HOST_KEY_CHECKING is set to: $ANSIBLE_HOST_KEY_CHECKING" + sudo systemctl set-environment SC4S_IMAGE=${{ needs.meta.outputs.container_base }} + - name: Update inventory file + run: | + cat << EOF > ansible/inventory/inventory.yaml + --- + all: + hosts: + children: + node: + hosts: + node_1: + ansible_host: 127.0.0.1 + ansible_connection: local + ansible_user: root + - name: Update env_file + run: | + echo "Updating ansible/inventory/inventory.yaml" + cat << EOF > ansible/resources/env_file + SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://127.0.0.1:8088 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no + SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION=yes + SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT=6000 + SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT=5514 + SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT=5514 + SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT=5601 + SC4S_LISTEN_SPECTRACOM_NTP_TCP_PORT=6002 + SC4S_LISTEN_CISCO_ESA_TCP_PORT=9000 + SC4S_LISTEN_RARITAN_DSX_TCP_PORT=9001 + SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL=yes + SC4S_SOURCE_RICOH_SYSLOG_FIXHOST=yes + TEST_SC4S_ACTIVATE_EXAMPLES=yes + SC4S_DEBUG_CONTAINER=yes + SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG=yes + SC4S_NETAPP_ONTAP_NEW_FORMAT=yes + SC4S_USE_VPS_CACHE=yes + - name: Update current SC4S image in unit file + run: | + sed -i 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service + - name: Run Ansible Playbook + run: | + ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml - name: Run tests run: | pip3 install poetry @@ -245,63 +275,28 @@ jobs: poetry run pytest -v --tb=long \ --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ - --sc4s_host=sc4s \ + --splunk_host=127.0.0.1 \ + --sc4s_host=127.0.0.1 \ --junitxml=test-results/test.xml \ -n 14 \ -k 'not lite and not name_cache' - test-ipv4-name-cache: - runs-on: ubuntu-latest - needs: - - meta - - build_action - container: - image: python:3.9-buster - services: - splunk: - image: splunk/splunk:${{ fromJson(needs.meta.outputs.matrix_supportedSplunk)[0].version }} - ports: - - 8088:8088 - - 8089:8089 - env: - SPLUNK_HEC_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SPLUNK_PASSWORD: Changed@11 - SPLUNK_START_ARGS: --accept-license - SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - sc4s: - image: ${{ needs.meta.outputs.container_base }} - ports: - - 514:514 - env: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" - SC4S_USE_NAME_CACHE: "yes" - SC4S_CLEAR_NAME_CACHE: "yes" - steps: - - name: Checkout - uses: actions/checkout@v4 + - name: artifact-test-results + uses: actions/upload-artifact@v4 with: - submodules: false - persist-credentials: false - - name: Run tests - run: | - pip3 install poetry - poetry install - mkdir -p test-results || true - poetry run pytest -v --tb=long \ - --splunk_type=external \ - --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ - --sc4s_host=sc4s \ - --junitxml=test-results/test.xml \ - -n 1 \ - -m 'name_cache' - test-ipv6-name-cache: + name: test-container-results_${{ matrix.deployment_type }}.xml + path: test-results/test.xml + if: ${{ !cancelled() }} + + test-name-cache: + strategy: + matrix: + SC4S_IPV6_ENABLE: ["yes","no"] runs-on: ubuntu-latest needs: - meta - build_action + # runs all of the steps inside the specified container rather than on the VM host. + # Because of this the network configuration changes from host based network to a container network. container: image: python:3.9-buster services: @@ -315,6 +310,7 @@ jobs: SPLUNK_PASSWORD: Changed@11 SPLUNK_START_ARGS: --accept-license SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz + sc4s: image: ${{ needs.meta.outputs.container_base }} ports: @@ -325,7 +321,8 @@ jobs: SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" SC4S_USE_NAME_CACHE: "yes" SC4S_CLEAR_NAME_CACHE: "yes" - SC4S_IPV6_ENABLE: "yes" + SC4S_IPV6_ENABLE: "${{ matrix.SC4S_IPV6_ENABLE }}" + steps: - name: Checkout uses: actions/checkout@v4 @@ -345,6 +342,12 @@ jobs: --junitxml=test-results/test.xml \ -n 1 \ -m 'name_cache' + - name: artifact-test-results + uses: actions/upload-artifact@v4 + with: + name: test-name-cache-results_IPv6_${{ matrix.SC4S_IPV6_ENABLE }}.xml + path: test-results/test.xml + if: ${{ !cancelled() }} release: name: Release runs-on: ubuntu-latest @@ -352,7 +355,7 @@ jobs: - meta - build_action - test-container - - test-ipv4-name-cache + - test-name-cache steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/ci-lite.yaml b/.github/workflows/ci-lite.yaml index 79c5066140..b254734ee7 100644 --- a/.github/workflows/ci-lite.yaml +++ b/.github/workflows/ci-lite.yaml @@ -178,14 +178,14 @@ jobs: scanners: "vuln" test-container: + strategy: + matrix: + deployment_type: ["docker", "podman"] runs-on: ubuntu-latest + continue-on-error: true needs: - meta - build_action - # runs all of the steps inside the specified container rather than on the VM host. - # Because of this the network configuration changes from host based network to a container network. - container: - image: python:3.9-buster services: splunk: @@ -200,92 +200,67 @@ jobs: SPLUNK_START_ARGS: --accept-license SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - sc4s: - image: ${{ needs.meta.outputs.container_base }} - ports: - - 514:514 - - 601:601 - - 5614:5514 - - 5601:5601 - - 6000:6000 - - 6002:6002 - - 9000:9000 - env: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" - SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION: "yes" - SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT: 6000 - SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT: 5514 - SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT: 5514 - SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT: 5601 - SC4S_LISTEN_SPECTRACOM_NTP_TCP_PORT: 6002 - SC4S_LISTEN_CISCO_ESA_TCP_PORT: 9000 - SC4S_LISTEN_RARITAN_DSX_TCP_PORT: 9001 - SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL: "yes" - SC4S_SOURCE_RICOH_SYSLOG_FIXHOST: "yes" - TEST_SC4S_ACTIVATE_EXAMPLES: "yes" - SC4S_DEBUG_CONTAINER: "yes" - SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG: "yes" - SC4S_NETAPP_ONTAP_NEW_FORMAT: "yes" - SC4S_USE_VPS_CACHE: "yes" steps: - name: Checkout uses: actions/checkout@v4 with: submodules: false persist-credentials: false - - name: Run tests + - name: Install Ansible and other dependencies as python package run: | - pip3 install poetry - poetry install - mkdir -p test-results || true - poetry run pytest -v --tb=long \ - --splunk_type=external \ - --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ - --sc4s_host=sc4s \ - --junitxml=test-results/test.xml \ - -n 14 -m "lite or addons" - - test-ipv4-name-cache: - runs-on: ubuntu-latest - needs: - - meta - - build_action - # runs all of the steps inside the specified container rather than on the VM host. - # Because of this the network configuration changes from host based network to a container network. - container: - image: python:3.9-buster - services: - splunk: - image: splunk/splunk:${{ fromJson(needs.meta.outputs.matrix_supportedSplunk)[0].version }} - ports: - - 8088:8088 - - 8089:8089 + pip install ansible~=6.1.0 --no-cache-dir \ + && pip install pywinrm>=0.4.2 --no-cache-dir \ + && pip install ansible-lint>=6.0.0 --no-cache-dir \ + && pip install docker + - name: Configure Ansible Environment Variables env: - SPLUNK_HEC_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SPLUNK_PASSWORD: Changed@11 - SPLUNK_START_ARGS: --accept-license - SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - - sc4s: - image: ${{ needs.meta.outputs.container_base }} - ports: - - 514:514 - env: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" - SC4S_USE_NAME_CACHE: "yes" - SC4S_CLEAR_NAME_CACHE: "yes" - - steps: - - name: Checkout - uses: actions/checkout@v4 - with: - submodules: false - persist-credentials: false + ANSIBLE_CONFIG: ansible.cfg + ANSIBLE_HOST_KEY_CHECKING: False + run: | + echo "ANSIBLE_CONFIG is set to: $ANSIBLE_CONFIG" + echo "ANSIBLE_HOST_KEY_CHECKING is set to: $ANSIBLE_HOST_KEY_CHECKING" + sudo systemctl set-environment SC4S_IMAGE=${{ needs.meta.outputs.container_base }} + - name: Update inventory file + run: | + cat << EOF > ansible/inventory/inventory.yaml + --- + all: + hosts: + children: + node: + hosts: + node_1: + ansible_host: 127.0.0.1 + ansible_connection: local + ansible_user: root + - name: Update env_file + run: | + echo "Updating ansible/inventory/inventory.yaml" + cat << EOF > ansible/resources/env_file + SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://127.0.0.1:8088 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no + SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION=yes + SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT=6000 + SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT=5514 + SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT=5514 + SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT=5601 + SC4S_LISTEN_SPECTRACOM_NTP_TCP_PORT=6002 + SC4S_LISTEN_CISCO_ESA_TCP_PORT=9000 + SC4S_LISTEN_RARITAN_DSX_TCP_PORT=9001 + SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL=yes + SC4S_SOURCE_RICOH_SYSLOG_FIXHOST=yes + TEST_SC4S_ACTIVATE_EXAMPLES=yes + SC4S_DEBUG_CONTAINER=yes + SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG=yes + SC4S_NETAPP_ONTAP_NEW_FORMAT=yes + SC4S_USE_VPS_CACHE=yes + - name: Update current SC4S image in unit file + run: | + sed -i 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service + - name: Run Ansible Playbook + run: | + ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml - name: Run tests run: | pip3 install poetry @@ -294,13 +269,21 @@ jobs: poetry run pytest -v --tb=long \ --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ - --sc4s_host=sc4s \ + --splunk_host=127.0.0.1 \ + --sc4s_host=127.0.0.1 \ --junitxml=test-results/test.xml \ - -n 1 \ - -m 'name_cache' - - test-ipv6-name-cache: + -n 14 -m "lite or addons" + - name: artifact-test-results + uses: actions/upload-artifact@v4 + with: + name: test-container-results_${{ matrix.deployment_type }}.xml + path: test-results/test.xml + if: ${{ !cancelled() }} + + test-name-cache: + strategy: + matrix: + SC4S_IPV6_ENABLE: ["yes","no"] runs-on: ubuntu-latest needs: - meta @@ -331,7 +314,7 @@ jobs: SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" SC4S_USE_NAME_CACHE: "yes" SC4S_CLEAR_NAME_CACHE: "yes" - SC4S_IPV6_ENABLE: "yes" + SC4S_IPV6_ENABLE: "${{ matrix.SC4S_IPV6_ENABLE }}" steps: - name: Checkout @@ -352,6 +335,12 @@ jobs: --junitxml=test-results/test.xml \ -n 1 \ -m 'name_cache' + - name: artifact-test-results + uses: actions/upload-artifact@v4 + with: + name: test-name-cache-results_IPv6_${{ matrix.SC4S_IPV6_ENABLE }}.xml + path: test-results/test.xml + if: ${{ !cancelled() }} test-healthcheck: runs-on: ubuntu-latest @@ -412,8 +401,7 @@ jobs: - meta - build_action - test-container - - test-ipv4-name-cache - - test-ipv6-name-cache + - test-name-cache - test-healthcheck - test-healthcheck-unit-tests steps: diff --git a/.github/workflows/ci-main.yaml b/.github/workflows/ci-main.yaml index a2cdbb3280..8f80566db1 100644 --- a/.github/workflows/ci-main.yaml +++ b/.github/workflows/ci-main.yaml @@ -178,14 +178,14 @@ jobs: scanners: "vuln" test-container: + strategy: + matrix: + deployment_type: ["docker", "podman"] runs-on: ubuntu-latest + continue-on-error: true needs: - meta - build_action - # runs all of the steps inside the specified container rather than on the VM host. - # Because of this the network configuration changes from host based network to a container network. - container: - image: python:3.9-buster services: splunk: @@ -200,42 +200,67 @@ jobs: SPLUNK_START_ARGS: --accept-license SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - sc4s: - image: ${{ needs.meta.outputs.container_base }} - ports: - - 514:514 - - 601:601 - - 5614:5514 - - 5601:5601 - - 6000:6000 - - 6002:6002 - - 9000:9000 - env: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" - SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION: "yes" - SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT: 6000 - SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT: 5514 - SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT: 5514 - SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT: 5601 - SC4S_LISTEN_SPECTRACOM_NTP_TCP_PORT: 6002 - SC4S_LISTEN_CISCO_ESA_TCP_PORT: 9000 - SC4S_LISTEN_RARITAN_DSX_TCP_PORT: 9001 - SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL: "yes" - SC4S_SOURCE_RICOH_SYSLOG_FIXHOST: "yes" - TEST_SC4S_ACTIVATE_EXAMPLES: "yes" - SC4S_DEBUG_CONTAINER: "yes" - SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG: "yes" - SC4S_NETAPP_ONTAP_NEW_FORMAT: "yes" - SC4S_USE_VPS_CACHE: "yes" - steps: - name: Checkout uses: actions/checkout@v4 with: submodules: false persist-credentials: false + - name: Install Ansible and other dependencies as python package + run: | + pip install ansible~=6.1.0 --no-cache-dir \ + && pip install pywinrm>=0.4.2 --no-cache-dir \ + && pip install ansible-lint>=6.0.0 --no-cache-dir \ + && pip install docker + - name: Configure Ansible Environment Variables + env: + ANSIBLE_CONFIG: ansible.cfg + ANSIBLE_HOST_KEY_CHECKING: False + run: | + echo "ANSIBLE_CONFIG is set to: $ANSIBLE_CONFIG" + echo "ANSIBLE_HOST_KEY_CHECKING is set to: $ANSIBLE_HOST_KEY_CHECKING" + sudo systemctl set-environment SC4S_IMAGE=${{ needs.meta.outputs.container_base }} + - name: Update inventory file + run: | + cat << EOF > ansible/inventory/inventory.yaml + --- + all: + hosts: + children: + node: + hosts: + node_1: + ansible_host: 127.0.0.1 + ansible_connection: local + ansible_user: root + - name: Update env_file + run: | + echo "Updating ansible/inventory/inventory.yaml" + cat << EOF > ansible/resources/env_file + SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://127.0.0.1:8088 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=70b6ae71-76b3-4c38-9597-0c5b37ad9630 + SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no + SC4S_DEST_SPLUNK_HEC_DEFAULT_HTTP_COMPRESSION=yes + SC4S_LISTEN_PFSENSE_FIREWALL_TCP_PORT=6000 + SC4S_LISTEN_SIMPLE_TEST_ONE_TCP_PORT=5514 + SC4S_LISTEN_SIMPLE_TEST_ONE_UDP_PORT=5514 + SC4S_LISTEN_SIMPLE_TEST_TWO_TCP_PORT=5601 + SC4S_LISTEN_SPECTRACOM_NTP_TCP_PORT=6002 + SC4S_LISTEN_CISCO_ESA_TCP_PORT=9000 + SC4S_LISTEN_RARITAN_DSX_TCP_PORT=9001 + SC4S_LISTEN_CHECKPOINT_SPLUNK_NOISE_CONTROL=yes + SC4S_SOURCE_RICOH_SYSLOG_FIXHOST=yes + TEST_SC4S_ACTIVATE_EXAMPLES=yes + SC4S_DEBUG_CONTAINER=yes + SC4S_SOURCE_VMWARE_VSPHERE_GROUPMSG=yes + SC4S_NETAPP_ONTAP_NEW_FORMAT=yes + SC4S_USE_VPS_CACHE=yes + - name: Update current SC4S image in unit file + run: | + sed -i 's|Environment="SC4S_IMAGE=ghcr.io/splunk/splunk-connect-for-syslog/container3:latest"|Environment="SC4S_IMAGE=${{ needs.meta.outputs.container_base }}"|' ansible/resources/${{ matrix.deployment_type }}_sc4s.service + - name: Run Ansible Playbook + run: | + ansible-playbook --connection=local -i ansible/inventory/inventory.yaml ansible/playbooks/${{ matrix.deployment_type }}.yml - name: Run tests run: | pip3 install poetry @@ -244,13 +269,22 @@ jobs: poetry run pytest -v --tb=long \ --splunk_type=external \ --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ - --sc4s_host=sc4s \ + --splunk_host=127.0.0.1 \ + --sc4s_host=127.0.0.1 \ --junitxml=test-results/test.xml \ -n 14 \ - -k 'not lite and not name_cache' + -k 'checkpoint or not lite and not name_cache' + - name: artifact-test-results + uses: actions/upload-artifact@v4 + with: + name: test-container-results_${{ matrix.deployment_type }}.xml + path: test-results/test.xml + if: ${{ !cancelled() }} - test-ipv4-name-cache: + test-name-cache: + strategy: + matrix: + SC4S_IPV6_ENABLE: ["yes","no"] runs-on: ubuntu-latest needs: - meta @@ -281,6 +315,7 @@ jobs: SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" SC4S_USE_NAME_CACHE: "yes" SC4S_CLEAR_NAME_CACHE: "yes" + SC4S_IPV6_ENABLE: "${{ matrix.SC4S_IPV6_ENABLE }}" steps: - name: Checkout @@ -301,59 +336,12 @@ jobs: --junitxml=test-results/test.xml \ -n 1 \ -m 'name_cache' - - test-ipv6-name-cache: - runs-on: ubuntu-latest - needs: - - meta - - build_action - # runs all of the steps inside the specified container rather than on the VM host. - # Because of this the network configuration changes from host based network to a container network. - container: - image: python:3.9-buster - services: - splunk: - image: splunk/splunk:${{ fromJson(needs.meta.outputs.matrix_supportedSplunk)[0].version }} - ports: - - 8088:8088 - - 8089:8089 - env: - SPLUNK_HEC_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SPLUNK_PASSWORD: Changed@11 - SPLUNK_START_ARGS: --accept-license - SPLUNK_APPS_URL: https://github.com/splunk/splunk-configurations-base-indexes/releases/download/v1.0.0/splunk_configurations_base_indexes-1.0.0.tar.gz - - sc4s: - image: ${{ needs.meta.outputs.container_base }} - ports: - - 514:514 - env: - SC4S_DEST_SPLUNK_HEC_DEFAULT_URL: https://splunk:8088 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN: 70b6ae71-76b3-4c38-9597-0c5b37ad9630 - SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY: "no" - SC4S_USE_NAME_CACHE: "yes" - SC4S_CLEAR_NAME_CACHE: "yes" - SC4S_IPV6_ENABLE: "yes" - - steps: - - name: Checkout - uses: actions/checkout@v4 + - name: artifact-test-results + uses: actions/upload-artifact@v4 with: - submodules: false - persist-credentials: false - - name: Run tests - run: | - pip3 install poetry - poetry install - mkdir -p test-results || true - poetry run pytest -v --tb=long \ - --splunk_type=external \ - --splunk_hec_token=70b6ae71-76b3-4c38-9597-0c5b37ad9630 \ - --splunk_host=splunk \ - --sc4s_host=sc4s \ - --junitxml=test-results/test.xml \ - -n 1 \ - -m 'name_cache' + name: test-name-cache-results_IPv6_${{ matrix.SC4S_IPV6_ENABLE }}.xml + path: test-results/test.xml + if: ${{ !cancelled() }} test-healthcheck: runs-on: ubuntu-latest @@ -437,8 +425,7 @@ jobs: - meta - build_action - test-container - - test-ipv4-name-cache - - test-ipv6-name-cache + - test-name-cache - test-healthcheck - test-healthcheck-unit-tests - mike diff --git a/ansible/resources/docker_sc4s.service b/ansible/resources/docker_sc4s.service index 6b18a46cfd..667fff51d3 100644 --- a/ansible/resources/docker_sc4s.service +++ b/ansible/resources/docker_sc4s.service @@ -43,4 +43,4 @@ ExecStart=/usr/bin/docker run \ --name SC4S \ --rm $SC4S_IMAGE -Restart=on-failure \ No newline at end of file +Restart=on-failure diff --git a/ansible/roles/install_docker/tasks/install_docker_debian.yml b/ansible/roles/install_docker/tasks/install_docker_debian.yml index 21a517907d..d3d434a774 100644 --- a/ansible/roles/install_docker/tasks/install_docker_debian.yml +++ b/ansible/roles/install_docker/tasks/install_docker_debian.yml @@ -23,10 +23,16 @@ repo: deb https://download.docker.com/linux/ubuntu focal stable state: present +- name: Check if 'docker' command exists + ansible.builtin.command: which docker + register: docker_command_check + ignore_errors: true + - name: Install docker-ce apt: name: docker-ce state: latest + when: docker_command_check.rc != 0 - name: Install Docker Module for Python pip: diff --git a/ansible/tasks/docker/deploy_app.yml b/ansible/tasks/docker/deploy_app.yml index 50462fac88..85ad163724 100644 --- a/ansible/tasks/docker/deploy_app.yml +++ b/ansible/tasks/docker/deploy_app.yml @@ -12,7 +12,7 @@ - name: Copying unit file on the server copy: - src: "/opt/ansible/resources/docker_sc4s.service" + src: "../../../ansible/resources/docker_sc4s.service" dest: "/lib/systemd/system/sc4s.service" owner: "{{ ansible_user }}" group: "{{ ansible_user }}" @@ -20,7 +20,7 @@ - name: Copying env_file file on the server copy: - src: "/opt/ansible/resources/env_file" + src: "../../../ansible/resources/env_file" dest: "/opt/sc4s/env_file" owner: "{{ ansible_user }}" group: "{{ ansible_user }}" diff --git a/ansible/tasks/podman/deploy_app.yml b/ansible/tasks/podman/deploy_app.yml index 07632d8eaa..f74fe85807 100644 --- a/ansible/tasks/podman/deploy_app.yml +++ b/ansible/tasks/podman/deploy_app.yml @@ -12,7 +12,7 @@ - name: Copying unit file on the server copy: - src: "/opt/ansible/resources/podman_sc4s.service" + src: "../../../ansible/resources/podman_sc4s.service" dest: "/lib/systemd/system/sc4s.service" owner: "{{ ansible_user }}" group: "{{ ansible_user }}" @@ -20,7 +20,7 @@ - name: Copying env_file file on the server copy: - src: "/opt/ansible/resources/env_file" + src: "../../../ansible/resources/env_file" dest: "/opt/sc4s/env_file" owner: "{{ ansible_user }}" group: "{{ ansible_user }}" diff --git a/tests/conftest.py b/tests/conftest.py index 7141695854..79862ba1e4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -32,7 +32,7 @@ def pytest_addoption(parser): "--sc4s_host", action="store", dest="sc4s_host", - default="127.0.0.1", + default="127.0.0.0", help="Address of the sc4s Server", ) diff --git a/tests/splunkutils.py b/tests/splunkutils.py index dd8f92190f..362c35bd18 100644 --- a/tests/splunkutils.py +++ b/tests/splunkutils.py @@ -4,7 +4,7 @@ # license that can be found in the LICENSE-BSD2 file or at # https://opensource.org/licenses/BSD-2-Clause from time import sleep - +import splunklib.results as results def splunk_single(service, search, attempt_limit=10): kwargs_normalsearch = {"exec_mode": "normal"} @@ -34,6 +34,10 @@ def splunk_single(service, search, attempt_limit=10): result_count = stats["resultCount"] event_count = stats["eventCount"] if result_count > 0 or tried > attempt_limit: + event_stream = job.events(output_mode="json") + print("Events from the search job:") + for event in results.JSONResultsReader(event_stream): + print(event) break else: tried += 1 diff --git a/tests/test_checkpoint.py b/tests/test_checkpoint.py index 4cc6514c21..5d18bf1747 100644 --- a/tests/test_checkpoint.py +++ b/tests/test_checkpoint.py @@ -325,7 +325,7 @@ def test_checkpoint_splunk_smartconsole(record_property, setup_splunk, setup_sc4 # <6>kernel: sd 2:0:0:0: SCSI error: return code = 0x00040000 @pytest.mark.addons("checkpoint") -def test_checkpoint_splunk_os(record_property, setup_splunk, setup_sc4s, get_pid): +def test_checkpoint_os(record_property, setup_splunk, setup_sc4s, get_pid): host = f"{shortuuid.ShortUUID().random(length=5).lower()}-{shortuuid.ShortUUID().random(length=5).lower()}" pid = get_pid