Merge branch 'master' of github.com:StackStorm/st2 into workflow_engine_graceful_shutdown

Author: Khushboo

.github/workflows/microbenchmarks.yaml

@@ -87,8 +87,10 @@ jobs:
             virtualenv
             ~/virtualenv
           key: ${{ runner.os }}-v4-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-v4-python-${{ matrix.python }}-
+          # Don't use alternative key as if requirements.txt has altered we
+          # don't want to retrieve previous cache
+          #restore-keys: |
+          #  ${{ runner.os }}-v4-python-${{ matrix.python }}-
       - name: Cache APT Dependencies
         id: cache-apt-deps
         uses: actions/cache@v2

.github/workflows/orquesta-integration-tests.yaml

@@ -143,8 +143,10 @@ jobs:
           # !virtualenv/lib/python*/site-packages/st2*
           # !virtualenv/bin/st2*
           key: ${{ runner.os }}-v4-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-v4-python-${{ matrix.python }}-
+          # Don't use alternative key as if requirements.txt has altered we
+          # don't want to retrieve previous cache
+          #restore-keys: |
+          #  ${{ runner.os }}-v4-python-${{ matrix.python }}-
       - name: Cache APT Dependencies
         id: cache-apt-deps
         uses: actions/cache@v2

CHANGELOG.rst

@@ -7,7 +7,6 @@ in development
 Fixed
 ~~~~~
 
-
 * Fix deserialization bug in st2 API for url encoded payloads. #5536
 
   Contributed by @sravs-dev
@@ -41,6 +40,12 @@ Fixed
 
   Contributed by Amanda McGuinness (@amanda11 intive) #5581
 
+* Downgrade tenacity as tooz dependency on tenacity has always been < 7.0.0 #5607
+
+  Contributed by @khushboobhatia01
+
+* Updated paramiko version to 2.10.3 to add support for more key verification algorithms. #5600
+
 Added
 ~~~~~
 
@@ -110,6 +115,59 @@ Added
 
   Contributed by Amanda McGuinness (@amanda11 Intive)
 
+* Add new ``api.auth_cookie_secure`` and ``api.auth_cookie_same_site`` config options which
+  specify values which are set for ``secure`` and ``SameSite`` attribute for the auth cookie
+  we set when authenticating via token / api key in query parameter value (e.g. via st2web).
+
+  For security reasons, ``api.auth_cookie_secure`` defaults to ``True``. This should only be
+  changed to ``False`` if you have a valid reason to not run StackStorm behind HTTPs proxy.
+
+  Default value for ``api.auth_cookie_same_site`` is ``lax``. If you want to disable this
+  functionality so it behaves the same as in the previous releases, you can set that option
+  to ``None``.
+
+  #5248
+
+  Contributed by @Kami.
+
+* Add new ``st2 action-alias test <message string>`` CLI command which allows users to easily
+  test action alias matching and result formatting.
+
+  This command will first try to find a matching alias (same as ``st2 action-alias match``
+  command) and if a match is found, trigger an execution (same as ``st2 action-alias execute``
+  command) and format the execution result.
+
+  This means it uses exactly the same flow as commands on chat, but the interaction avoids
+  chat and hubot which should make testing and developing aliases easier and faster. #5143
+
+  #5143
+
+  Contributed by @Kami.
+
+* Add new ``credentials.basic_auth = username:password`` CLI configuration option.
+
+  This argument allows client to use additional set of basic auth credentials when talking to the
+  StackStorm API endpoints (api, auth, stream) - that is, in addition to the token / api key
+  native StackStorm auth.
+
+  This allows for simple basic auth based multi factor authentication implementation for
+  installations which don't utilize SSO.
+
+  #5152
+
+  Contributed by @Kami.
+
+* Add new audit message when a user has decrypted a key whether manually in the container (st2 key get [] --decrypt)
+  or through a workflow with a defined config. #5594
+  Contributed by @dmork123
+
+* Added garbage collection for rule_enforcement and trace models #5596/5602
+  Contributed by Amanda McGuinness (@amanda11 intive)
+
+* Added garbage collection for workflow execution and task execution objects #4924
+  Contributed by @srimandaleeka01 and @amanda11
+
+
 Fixed
 ~~~~~
 
@@ -119,10 +177,31 @@ Fixed
 
 * Fix ``st2-self-check`` script reporting falsey success when the nested workflows runs failed. #5487
 
+* Fix actions from the contrib/linux pack that fail on CentOS-8 but work on other operating systems and distributions. (bug fix) #4999 #5004
+
+  Reported by @blag and @dove-young contributed by @winem.
+
 * Use byte type lock name which is supported by all tooz drivers. #5529
 
   Contributed by @khushboobhatia01
 
+* Fixed issue where pack index searches are ignoring no_proxy #5497
+
+  Contributed by @minsis
+
+* Fixed trigger references emitted by ``linux.file_watch.line``. #5467
+
+  Prior to this patch multiple files could be watched but the rule reference of last registered file
+  would be used for all trigger emissions causing rule enforcement to fail.  References are now tracked
+  on a per file basis and used in trigger emissions.
+
+  Contributed by @nzlosh
+
+Changed
+~~~~~~~
+
+* Bump black to v22.3.0 - This is  used internally to reformat our python code. #5606
+
 3.6.0 - October 29, 2021
 ------------------------
 

Makefile

@@ -381,15 +381,19 @@ black: requirements .black-check
 		echo "Running black on" $$component; \
 		echo "==========================================================="; \
 		. $(VIRTUALENV_DIR)/bin/activate ; black --check --config pyproject.toml $$component/ || exit 1; \
-		. $(VIRTUALENV_DIR)/bin/activate ; black $$(grep -rl '^#!/.*python' $$component/bin) || exit 1; \
+		if [ -d "$$component/bin" ]; then \
+			. $(VIRTUALENV_DIR)/bin/activate ; black $$(grep -rl '^#!/.*python' $$component/bin) || exit 1; \
+		fi \
 	done
 	# runner modules and packages
 	@for component in $(COMPONENTS_RUNNERS); do\
 		echo "==========================================================="; \
 		echo "Running black on" $$component; \
 		echo "==========================================================="; \
 		. $(VIRTUALENV_DIR)/bin/activate ; black --check --config pyproject.toml $$component/ || exit 1; \
-		. $(VIRTUALENV_DIR)/bin/activate ; black $$(grep -rl '^#!/.*python' $$component/bin) || exit 1; \
+		if [ -d "$$component/bin" ]; then \
+			. $(VIRTUALENV_DIR)/bin/activate ; black $$(grep -rl '^#!/.*python' $$component/bin) || exit 1; \
+		fi \
 	done
 	. $(VIRTUALENV_DIR)/bin/activate; black --check --config pyproject.toml contrib/ || exit 1;
 	. $(VIRTUALENV_DIR)/bin/activate; black --check --config pyproject.toml scripts/*.py || exit 1;
@@ -411,15 +415,19 @@ black: requirements .black-format
 		echo "Running black on" $$component; \
 		echo "==========================================================="; \
 		. $(VIRTUALENV_DIR)/bin/activate ; black --config pyproject.toml $$component/ || exit 1; \
-		. $(VIRTUALENV_DIR)/bin/activate ; black --config pyproject.toml $$(grep -rl '^#!/.*python' $$component/bin) || exit 1; \
+		if [ -d "$$component/bin" ]; then \
+			. $(VIRTUALENV_DIR)/bin/activate ; black --config pyproject.toml $$(grep -rl '^#!/.*python' $$component/bin) || exit 1; \
+		fi \
 	done
 	# runner modules and packages
 	@for component in $(COMPONENTS_RUNNERS); do\
 		echo "==========================================================="; \
 		echo "Running black on" $$component; \
 		echo "==========================================================="; \
 		. $(VIRTUALENV_DIR)/bin/activate ; black --config pyproject.toml  $$component/ || exit 1; \
-		. $(VIRTUALENV_DIR)/bin/activate ; black --config pyproject.toml $$(grep -rl '^#!/.*python' $$component/bin) || exit 1; \
+		if [ -d "$$component/bin" ]; then \
+			. $(VIRTUALENV_DIR)/bin/activate ; black --config pyproject.toml $$(grep -rl '^#!/.*python' $$component/bin) || exit 1; \
+		fi \
 	done
 	. $(VIRTUALENV_DIR)/bin/activate; black --config pyproject.toml contrib/ || exit 1;
 	. $(VIRTUALENV_DIR)/bin/activate; black --config pyproject.toml scripts/*.py || exit 1;

conf/st2.conf.sample

@@ -38,6 +38,11 @@ workflows_pool_size = 40
 [api]
 # List of origins allowed for api, auth and stream
 allow_origin = http://127.0.0.1:3000 # comma separated list allowed here.
+# SameSite attribute value for the auth-token cookie we set on successful authentication from st2web. If you don't have a specific reason (e.g. supporting old browsers) we recommend you set this value to strict. Setting it to "unset" will default to the behavior in previous releases and not set this SameSite header value.
+# Valid values: strict, lax, none, unset
+auth_cookie_same_site = lax
+# True if secure flag should be set for "auth-token" cookie which is set on successful authentication via st2web. You should only set this to False if you have a good reason to not run and access StackStorm behind https proxy.
+auth_cookie_secure = True
 # None
 debug = False
 # StackStorm API server host
@@ -142,6 +147,7 @@ ssl = False
 # ca_certs file contains a set of concatenated CA certificates, which are used to validate certificates passed from MongoDB.
 ssl_ca_certs = None
 # Specifies whether a certificate is required from the other side of the connection, and whether it will be validated if provided
+# Valid values: none, optional, required
 ssl_cert_reqs = None
 # Certificate file used to identify the localconnection
 ssl_certfile = None
@@ -151,7 +157,7 @@ ssl_keyfile = None
 ssl_match_hostname = True
 # username for db login
 username = None
-# Compression level when compressors is set to zlib. Valid calues are -1 to 9. Defaults to 6.
+# Compression level when compressors is set to zlib. Valid values are -1 to 9. Defaults to 6.
 zlib_compression_level =
 
 [exporter]
@@ -161,20 +167,28 @@ dump_dir = /opt/stackstorm/exports/
 logging = /etc/st2/logging.exporter.conf
 
 [garbagecollector]
-# Action execution output objects (ones generated by action output streaming) older than this value (days) will be automatically deleted.
+# Action execution output objects (ones generated by action output streaming) older than this value (days) will be automatically deleted. Defaults to 7.
 action_executions_output_ttl = 7
-# Action executions and related objects (live actions, action output objects) older than this value (days) will be automatically deleted.
+# Action executions and related objects (live actions, action output objects) older than this value (days) will be automatically deleted. Defaults to None (disabled).
 action_executions_ttl = None
 # How often to check database for old data and perform garbage collection.
 collection_interval = 600
 # Location of the logging configuration file.
 logging = /etc/st2/logging.garbagecollector.conf
 # Set to True to perform garbage collection on Inquiries (based on the TTL value per Inquiry)
 purge_inquiries = False
+# Rule enforcements older than this value (days) will be automatically deleted. Defaults to None (disabled).
+rule_enforcements_ttl = None
 # How long to wait / sleep (in seconds) between collection of different object types.
 sleep_delay = 2
-# Trigger instances older than this value (days) will be automatically deleted.
+# Workflow task execution output objects (generated by action output streaming) older than this value (days) will be automatically deleted. Defaults to None (disabled).
+task_executions_ttl = None
+# Trace objects older than this value (days) will be automatically deleted. Defaults to None (disabled).
+traces_ttl = None
+# Trigger instances older than this value (days) will be automatically deleted. Defaults to None (disabled).
 trigger_instances_ttl = None
+# Workflow execution output objects (generated by action output streaming) older than this value (days) will be automatically deleted. Defaults to None (disabled).
+workflow_executions_ttl = None
 
 [keyvalue]
 # Allow encryption of values in key value stored qualified as "secret".
@@ -195,7 +209,8 @@ redirect_stderr = False
 [messaging]
 # URL of all the nodes in a messaging service cluster.
 cluster_urls =  # comma separated list allowed here.
-# Compression algorithm to use for compressing the payloads which are sent over the message bus. Valid values include: zstd, lzma, bz2, gzip. Defaults to no compression.
+# Compression algorithm to use for compressing the payloads which are sent over the message bus. Defaults to no compression.
+# Valid values: zstd, lzma, bz2, gzip, None
 compression = None
 # How many times should we retry connection before failing.
 connection_retries = 10
@@ -208,6 +223,7 @@ ssl = False
 # ca_certs file contains a set of concatenated CA certificates, which are used to validate certificates passed from RabbitMQ.
 ssl_ca_certs = None
 # Specifies whether a certificate is required from the other side of the connection, and whether it will be validated if provided.
+# Valid values: none, optional, required
 ssl_cert_reqs = None
 # Certificate file used to identify the local connection (client).
 ssl_certfile = None

conf/st2rc.sample.ini

@@ -21,6 +21,11 @@ username = test1
 password = testpassword
 # or authenticate with an api key.
 # api_key = <key>
+# Optional additional http basic auth credentials which are sent with each HTTP
+# request except the auth request to /v1/auth/tokens endpoint.
+# Available in StackStorm >= v3.4.0
+# NOTE: Username can't contain colon (:) character.
+# basic_auth = username:password
 
 [api]
 url = http://127.0.0.1:9101/v1

contrib/linux/README.md

@@ -55,4 +55,5 @@ Example trigger payload:
 
 ## Troubleshooting
 
-* On CentOS7/RHEL7, dig is not installed by default. Run ``sudo yum install bind-utils`` to install.
+* On CentOS7/RHEL7, dig is not installed by default. Run ``sudo yum install bind-utils`` to install.
+* On CentOS8/RHEL8, lsof is not installed by default. Run ``sudo yum install lsof`` to install.

contrib/linux/actions/checks/check_loadavg.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/opt/stackstorm/st2/bin/python
 
 # Copyright 2020 The StackStorm Authors.
 # Copyright 2019 Extreme Networks, Inc.

contrib/linux/actions/checks/check_processes.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/opt/stackstorm/st2/bin/python
 
 # Copyright 2020 The StackStorm Authors.
 # Copyright 2019 Extreme Networks, Inc.

contrib/linux/actions/service.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/opt/stackstorm/st2/bin/python
 
 # Copyright 2020 The StackStorm Authors.
 # Copyright 2019 Extreme Networks, Inc.