Merge branch 'main' into fix/graceful-elasticsearch

Author: widhalmt

.github/workflows/kics.yml

@@ -0,0 +1,43 @@
+---
+name: KICS Security Scan
+on:
+  workflow_dispatch:
+    inputs:
+      logLevel:
+        description: 'Log level'
+        required: true
+        default: 'warning'
+        type: choice
+        options:
+          - info
+          - warning
+          - debug
+  pull_request:
+  push:
+    branches:
+      - 'main'
+  merge_group:
+  schedule:
+    - cron: '15 6 * * 4'
+jobs:
+  kics:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Mkdir results-dir
+        # make sure results dir is created
+        run: mkdir -p results-dir
+      - name: run kics Scan
+        uses: Checkmarx/kics-github-action@5a6152ef88416063435cebadfec9de28bcfd041d # v2.1.4
+        with:
+          # path: 'roles,plugins'
+          path: '.'
+          # fail_on: high
+          ignore_on_exit: results
+          output_formats: 'json,sarif'
+          output_path: results-dir
+      - name: Upload SARIF file
+        uses: github/codeql-action/upload-sarif@af56b044b5d41c317aef5d19920b3183cb4fbbec # v3
+        with:
+          sarif_file: results-dir/results.sarif

docs/role-elasticsearch.md

@@ -15,7 +15,7 @@ Role Variables
 * *elasticsearch_node_types*: List of types of this very node. Please refer to [official docs](https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html) for details. (default: not set. allowed value: array of types)
 + *elasticsearch_nodename*': Node name of the Elasticsearch node. (default: value of `ansible_hostname`)
 * *elasticsearch_clustername*: Name the Elasticsearch Cluster (default: `elasticsearch`)
-* *elasticsearch_heap*: Heapsize for Elasticsearch. (Half of free memory on host. Maximum 30GB. (default: Half of hosts memory. Min 1GB, Max 30GB)
+* *elasticsearch_heap*: Heapsize for Elasticsearch. Set to `false` to follow Elastic recommendations for elasticsearch 8.x (default: Half of hosts memory. Min 1GB, Max 30GB)
 * *elasticsearch_tls_key_passphrase*: Passphrase for elasticsearch certificates (default: `PleaseChangeMeIndividually`)
 * *elasticsearch_cert_validity_period*: number of days that the generated certificates are valid (default: 1095).
 * *elasticsearch_cert_expiration_buffer*: Ansible will renew the elasticsearch certificate if its validity is shorter than this value, which should be number of days. (default: 30)

molecule/elasticsearch_no-security/verify.yml

@@ -9,8 +9,10 @@
   tasks:
 
 # Remember, this is the no-security scenario. So no https
+# The comment below will create an exception for KICS security scan
     - name: Health check
       ansible.builtin.uri:
+# kics-scan ignore-line
         url: http://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health
         method: GET
         return_content: yes
@@ -24,6 +26,7 @@
 
     - name: Node check
       ansible.builtin.uri:
+# kics-scan ignore-line
         url: http://localhost:{{ elasticstack_elasticsearch_http_port }}/_cat/nodes
         method: GET
         return_content: yes

molecule/elasticsearch_test_modules/converge.yml

@@ -50,7 +50,7 @@
       netways.elasticstack.elasticsearch_user:
         name: new-user1
         fullname: New User
-        password: changeMe123!
+        password: "{{ lookup('community.general.random_string', length=12, min_lower=1, min_upper=1, min_numeric=1, min_special=1, override_special='-_=!') }}"
         email: [email protected]
         roles:
           - new-role1

molecule/elasticstack_default/converge.yml

@@ -18,10 +18,9 @@
     elasticstack_full_stack: true
     elasticstack_no_log: false
     logstash_pipeline_unsafe_shutdown: true
-    logstash_redis_password: "ThisIsMyRedisTest"
+    logstash_redis_password: "{{ lookup('ansible.builtin.password', '/tmp/redispassword', chars=['ascii_letters'], length=15) }}"
     redis_requirepass: "{{ logstash_redis_password }}"
-    beats_filebeat_syslog_udp: true
-    beats_filebeat_syslog_tcp: true
+    beats_filebeat_journald: true
     beats_filebeat_modules:
       - system
     beats_fields:
@@ -51,19 +50,3 @@
     - name: Include Beats
       ansible.builtin.include_role:
         name: beats
-    - name: Install rsyslog
-      ansible.builtin.package:
-        name: rsyslog
-    - name: Remove cache # noqa: risky-shell-pipe
-      ansible.builtin.shell: >
-        if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
-        rm -rf /var/cache/*
-      changed_when: false
-    - name: Configure rsyslog
-      ansible.builtin.lineinfile:
-        line: "*.* @@localhost:514"
-        path: /etc/rsyslog.conf
-    - name: Start rsyslog
-      ansible.builtin.service:
-        name: rsyslog
-        state: started

molecule/elasticstack_default/molecule.yml

@@ -32,6 +32,11 @@ platforms:
     pre_build_image: true
 provisioner:
   name: ansible
+  inventory:
+    host_vars:
+      elasticstack${ELASTIC_RELEASE}-cluster2-${MOLECULE_DISTRO}:
+        elasticsearch_tls_key_passphrase: UniqueHostPassword
+
   # Just enable temporarily. Sometimes it's useful, but most of the time it's
   # overwhelming
   #env:

molecule/elasticstack_default/prepare.yml

@@ -35,9 +35,13 @@
           - unzip
           - systemd
 
+# KICS complains about packages being updated.
+# In this case, a mere test scenario it is OK, though.
+
     - name: Update all installed packages RHEL
       ansible.builtin.yum:
         name: '*'
+# kics-scan ignore-line
         state: latest
         update_cache: yes
         update_only: yes
@@ -46,6 +50,7 @@
     - name: Update all installed packages Debian
       ansible.builtin.apt:
         name: '*'
+# kics-scan ignore-line
         state: latest
         update_cache: yes
       when: ansible_os_family == "Debian"

molecule/elasticstack_default/verify.yml

@@ -17,10 +17,6 @@
         port: 5044
       when: "'logstash' in group_names"
 
-    - name: Wait for syslog port to open
-      ansible.builtin.wait_for:
-        port: 514
-
     - name: Set elasticsearch_ca variable if not already done by user
       ansible.builtin.set_fact:
         elasticsearch_ca: "{{ groups[elasticstack_elasticsearch_group_name][0] }}"

molecule/logstash_full_stack-oss/converge.yml

@@ -19,22 +19,14 @@
     logstash_security: false
     logstash_pipeline_unsafe_shutdown: true
     elasticstack_security: false
-    beats_filebeat_syslog_udp: true
-    beats_filebeat_syslog_tcp: true
+    beats_filebeat_journald: true
     logstash_beats_tls: false
     elasticstack_release: 7
     elasticstack_no_log: false
   tasks:
     - name: "Include Elastics repos role"
       ansible.builtin.include_role:
         name: repos
-    - name: Install rsyslog
-      ansible.builtin.package:
-        name: rsyslog
-    - name: Start rsyslog
-      ansible.builtin.service:
-        name: rsyslog
-        state: started
     - name: "Include Elasticsearch role"
       ansible.builtin.include_role:
         name: elasticsearch
@@ -47,12 +39,3 @@
     - name: "Include Logstash"
       ansible.builtin.include_role:
         name: logstash
-    - name: Configure rsyslog
-      ansible.builtin.lineinfile:
-        line: "*.* @@localhost:514"
-        path: /etc/rsyslog.conf
-    - name: Restart rsyslog
-      ansible.builtin.service:
-        name: rsyslog
-        state: restarted
-      changed_when: false

molecule/logstash_full_stack-oss/verify.yml

@@ -13,10 +13,6 @@
   - name: Run syntax check
     ansible.builtin.command: "/usr/share/logstash/bin/logstash --path.settings=/etc/logstash -t"
     when: "'logstash' in group_names"
-  - name: Check for open port tcp {{ elasticstack_beats_port }}
-    ansible.builtin.wait_for:
-      port: "{{ elasticstack_beats_port }}"
-    when: "'logstash' in group_names"
   - name: Query for Logstasch indices
     ansible.builtin.shell: >
       curl -s http://localhost:{{ elasticstack_elasticsearch_http_port }}/_cat/indices |