Skip to content

Commit 5e2ced4

Browse files
nwang92nwang92
authored
Adding tests for splunkd certificates (#365)
* Adding tests for splunkd certificates * Adding distributed test for splunktcp-ssl
1 parent 0d4f5e1 commit 5e2ced4

File tree

1 file changed

+265
-0
lines changed

1 file changed

+265
-0
lines changed

‎tests/test_docker_splunk.py

Lines changed: 265 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
FIXTURES_DIR = os.path.join(FILE_DIR, "fixtures")
3030
REPO_DIR = os.path.join(FILE_DIR, "..")
3131
SCENARIOS_DIR = os.path.join(FILE_DIR, "..", "test_scenarios")
32+
DEFAULTS_DIR = os.path.join(SCENARIOS_DIR, "defaults")
3233
EXAMPLE_APP = os.path.join(FIXTURES_DIR, "splunk_app_example")
3334
EXAMPLE_APP_TGZ = os.path.join(FIXTURES_DIR, "splunk_app_example.tgz")
3435
# Setup logging
@@ -1968,6 +1969,168 @@ def test_adhoc_1uf_splunktcp_ssl(self):
19681969
except Exception as e:
19691970
self.logger.error(e)
19701971
raise e
1972+
finally:
1973+
if cid:
1974+
self.client.remove_container(cid, v=True, force=True)
1975+
files = [
1976+
os.path.join(FIXTURES_DIR, "ca.key"),
1977+
os.path.join(FIXTURES_DIR, "ca.csr"),
1978+
os.path.join(FIXTURES_DIR, "ca.pem"),
1979+
os.path.join(FIXTURES_DIR, "server.key"),
1980+
os.path.join(FIXTURES_DIR, "server.csr"),
1981+
os.path.join(FIXTURES_DIR, "server.pem"),
1982+
os.path.join(FIXTURES_DIR, "cert.pem"),
1983+
os.path.join(FIXTURES_DIR, "cacert.pem"),
1984+
os.path.join(FIXTURES_DIR, "default.yml")
1985+
]
1986+
self.cleanup_files(files)
1987+
1988+
def test_adhoc_1so_splunkd_custom_ssl(self):
1989+
# Generate default.yml
1990+
cid = self.client.create_container(self.SPLUNK_IMAGE_NAME, tty=True, command="create-defaults")
1991+
self.client.start(cid.get("Id"))
1992+
output = self.get_container_logs(cid.get("Id"))
1993+
self.client.remove_container(cid.get("Id"), v=True, force=True)
1994+
# Get the password
1995+
password = re.search(r"^ password: (.*?)\n", output, flags=re.MULTILINE|re.DOTALL).group(1).strip()
1996+
assert password and password != "null"
1997+
# Commands to generate self-signed certificates for Splunk here: https://docs.splunk.com/Documentation/Splunk/latest/Security/ConfigureSplunkforwardingtousesignedcertificates
1998+
passphrase = "heyallyoucoolcatsandkittens"
1999+
cmds = [
2000+
"openssl genrsa -aes256 -passout pass:{pw} -out {path}/ca.key 2048".format(pw=passphrase, path=FIXTURES_DIR),
2001+
"openssl req -new -key {path}/ca.key -passin pass:{pw} -out {path}/ca.csr -subj /CN=localhost".format(pw=passphrase, path=FIXTURES_DIR),
2002+
"openssl x509 -req -in {path}/ca.csr -sha512 -passin pass:{pw} -signkey {path}/ca.key -CAcreateserial -out {path}/ca.pem -days 3".format(pw=passphrase, path=FIXTURES_DIR),
2003+
"openssl genrsa -aes256 -passout pass:{pw} -out {path}/server.key 2048".format(pw=passphrase, path=FIXTURES_DIR),
2004+
"openssl req -new -passin pass:{pw} -key {path}/server.key -out {path}/server.csr -subj /CN=localhost".format(pw=passphrase, path=FIXTURES_DIR),
2005+
"openssl x509 -req -passin pass:{pw} -in {path}/server.csr -SHA256 -CA {path}/ca.pem -CAkey {path}/ca.key -CAcreateserial -out {path}/server.pem -days 3".format(pw=passphrase, path=FIXTURES_DIR),
2006+
"cat {path}/server.pem {path}/server.key {path}/ca.pem > {path}/cert.pem".format(path=FIXTURES_DIR),
2007+
"cat {path}/server.pem {path}/ca.pem > {path}/cacert.pem".format(path=FIXTURES_DIR)
2008+
]
2009+
for cmd in cmds:
2010+
execute_cmd = subprocess.check_output(["/bin/sh", "-c", cmd])
2011+
# Update s2s ssl settings
2012+
output = re.sub(r'''^ ssl:.*?password: null''', r''' ssl:
2013+
ca: /tmp/defaults/ca.pem
2014+
cert: /tmp/defaults/cert.pem
2015+
enable: true
2016+
password: {}'''.format(passphrase), output, flags=re.MULTILINE|re.DOTALL)
2017+
# Write the default.yml to a file
2018+
with open(os.path.join(FIXTURES_DIR, "default.yml"), "w") as f:
2019+
f.write(output)
2020+
# Create the container and mount the default.yml
2021+
cid = None
2022+
try:
2023+
splunk_container_name = generate_random_string()
2024+
cid = self.client.create_container(self.SPLUNK_IMAGE_NAME, tty=True, ports=[8000, 8089],
2025+
volumes=["/tmp/defaults/"], name=splunk_container_name,
2026+
environment={"DEBUG": "true",
2027+
"SPLUNK_START_ARGS": "--accept-license",
2028+
"SPLUNK_PASSWORD": password},
2029+
host_config=self.client.create_host_config(binds=[FIXTURES_DIR + ":/tmp/defaults/"],
2030+
port_bindings={8089: ("0.0.0.0",), 8000: ("0.0.0.0",)})
2031+
)
2032+
cid = cid.get("Id")
2033+
self.client.start(cid)
2034+
# Poll for the container to be ready
2035+
assert self.wait_for_containers(1, name=splunk_container_name)
2036+
# Check splunkd
2037+
assert self.check_splunkd("admin", password)
2038+
# Check if the created file exists
2039+
exec_command = self.client.exec_create(cid, "cat /opt/splunk/etc/system/local/server.conf", user="splunk")
2040+
std_out = self.client.exec_start(exec_command)
2041+
assert "enableSplunkdSSL = 1" in std_out
2042+
assert "sslRootCAPath = /tmp/defaults/ca.pem" in std_out
2043+
assert "serverCert = /tmp/defaults/cert.pem" in std_out
2044+
# Check splunkd using the custom certs
2045+
mgmt_port = self.client.port(cid, 8089)[0]["HostPort"]
2046+
url = "https://localhost:{}/services/server/info".format(mgmt_port)
2047+
kwargs = {"auth": ("admin", password), "verify": "{}/cacert.pem".format(FIXTURES_DIR)}
2048+
status, content = self.handle_request_retry("GET", url, kwargs)
2049+
assert status == 200
2050+
except Exception as e:
2051+
self.logger.error(e)
2052+
raise e
2053+
finally:
2054+
if cid:
2055+
self.client.remove_container(cid, v=True, force=True)
2056+
files = [
2057+
os.path.join(FIXTURES_DIR, "ca.key"),
2058+
os.path.join(FIXTURES_DIR, "ca.csr"),
2059+
os.path.join(FIXTURES_DIR, "ca.pem"),
2060+
os.path.join(FIXTURES_DIR, "server.key"),
2061+
os.path.join(FIXTURES_DIR, "server.csr"),
2062+
os.path.join(FIXTURES_DIR, "server.pem"),
2063+
os.path.join(FIXTURES_DIR, "cert.pem"),
2064+
os.path.join(FIXTURES_DIR, "cacert.pem"),
2065+
os.path.join(FIXTURES_DIR, "default.yml")
2066+
]
2067+
self.cleanup_files(files)
2068+
2069+
def test_adhoc_1uf_splunkd_custom_ssl(self):
2070+
# Generate default.yml
2071+
cid = self.client.create_container(self.UF_IMAGE_NAME, tty=True, command="create-defaults")
2072+
self.client.start(cid.get("Id"))
2073+
output = self.get_container_logs(cid.get("Id"))
2074+
self.client.remove_container(cid.get("Id"), v=True, force=True)
2075+
# Get the password
2076+
password = re.search(r"^ password: (.*?)\n", output, flags=re.MULTILINE|re.DOTALL).group(1).strip()
2077+
assert password and password != "null"
2078+
# Commands to generate self-signed certificates for Splunk here: https://docs.splunk.com/Documentation/Splunk/latest/Security/ConfigureSplunkforwardingtousesignedcertificates
2079+
passphrase = "heyallyoucoolcatsandkittens"
2080+
cmds = [
2081+
"openssl genrsa -aes256 -passout pass:{pw} -out {path}/ca.key 2048".format(pw=passphrase, path=FIXTURES_DIR),
2082+
"openssl req -new -key {path}/ca.key -passin pass:{pw} -out {path}/ca.csr -subj /CN=localhost".format(pw=passphrase, path=FIXTURES_DIR),
2083+
"openssl x509 -req -in {path}/ca.csr -sha512 -passin pass:{pw} -signkey {path}/ca.key -CAcreateserial -out {path}/ca.pem -days 3".format(pw=passphrase, path=FIXTURES_DIR),
2084+
"openssl genrsa -aes256 -passout pass:{pw} -out {path}/server.key 2048".format(pw=passphrase, path=FIXTURES_DIR),
2085+
"openssl req -new -passin pass:{pw} -key {path}/server.key -out {path}/server.csr -subj /CN=localhost".format(pw=passphrase, path=FIXTURES_DIR),
2086+
"openssl x509 -req -passin pass:{pw} -in {path}/server.csr -SHA256 -CA {path}/ca.pem -CAkey {path}/ca.key -CAcreateserial -out {path}/server.pem -days 3".format(pw=passphrase, path=FIXTURES_DIR),
2087+
"cat {path}/server.pem {path}/server.key {path}/ca.pem > {path}/cert.pem".format(path=FIXTURES_DIR),
2088+
"cat {path}/server.pem {path}/ca.pem > {path}/cacert.pem".format(path=FIXTURES_DIR)
2089+
]
2090+
for cmd in cmds:
2091+
execute_cmd = subprocess.check_output(["/bin/sh", "-c", cmd])
2092+
# Update s2s ssl settings
2093+
output = re.sub(r'''^ ssl:.*?password: null''', r''' ssl:
2094+
ca: /tmp/defaults/ca.pem
2095+
cert: /tmp/defaults/cert.pem
2096+
enable: true
2097+
password: {}'''.format(passphrase), output, flags=re.MULTILINE|re.DOTALL)
2098+
# Write the default.yml to a file
2099+
with open(os.path.join(FIXTURES_DIR, "default.yml"), "w") as f:
2100+
f.write(output)
2101+
# Create the container and mount the default.yml
2102+
cid = None
2103+
try:
2104+
splunk_container_name = generate_random_string()
2105+
cid = self.client.create_container(self.UF_IMAGE_NAME, tty=True, ports=[8000, 8089],
2106+
volumes=["/tmp/defaults/"], name=splunk_container_name,
2107+
environment={"DEBUG": "true",
2108+
"SPLUNK_START_ARGS": "--accept-license",
2109+
"SPLUNK_PASSWORD": password},
2110+
host_config=self.client.create_host_config(binds=[FIXTURES_DIR + ":/tmp/defaults/"],
2111+
port_bindings={8089: ("0.0.0.0",), 8000: ("0.0.0.0",)})
2112+
)
2113+
cid = cid.get("Id")
2114+
self.client.start(cid)
2115+
# Poll for the container to be ready
2116+
assert self.wait_for_containers(1, name=splunk_container_name)
2117+
# Check splunkd
2118+
assert self.check_splunkd("admin", password)
2119+
# Check if the created file exists
2120+
exec_command = self.client.exec_create(cid, "cat /opt/splunkforwarder/etc/system/local/server.conf", user="splunk")
2121+
std_out = self.client.exec_start(exec_command)
2122+
assert "enableSplunkdSSL = 1" in std_out
2123+
assert "sslRootCAPath = /tmp/defaults/ca.pem" in std_out
2124+
assert "serverCert = /tmp/defaults/cert.pem" in std_out
2125+
# Check splunkd using the custom certs
2126+
mgmt_port = self.client.port(cid, 8089)[0]["HostPort"]
2127+
url = "https://localhost:{}/services/server/info".format(mgmt_port)
2128+
kwargs = {"auth": ("admin", password), "verify": "{}/cacert.pem".format(FIXTURES_DIR)}
2129+
status, content = self.handle_request_retry("GET", url, kwargs)
2130+
assert status == 200
2131+
except Exception as e:
2132+
self.logger.error(e)
2133+
raise e
19712134
finally:
19722135
if cid:
19732136
self.client.remove_container(cid, v=True, force=True)
@@ -2932,6 +3095,108 @@ def test_compose_1uf1so(self):
29323095
assert search_providers[0] == "so1"
29333096
assert distinct_hosts == 2
29343097

3098+
def test_compose_3idx1cm_splunktcp_ssl(self):
3099+
# Generate default.yml
3100+
cid = self.client.create_container(self.SPLUNK_IMAGE_NAME, tty=True, command="create-defaults")
3101+
self.client.start(cid.get("Id"))
3102+
output = self.get_container_logs(cid.get("Id"))
3103+
self.client.remove_container(cid.get("Id"), v=True, force=True)
3104+
# Get the password
3105+
password = re.search(r"^ password: (.*?)\n", output, flags=re.MULTILINE|re.DOTALL).group(1).strip()
3106+
assert password and password != "null"
3107+
# Commands to generate self-signed certificates for Splunk here: https://docs.splunk.com/Documentation/Splunk/latest/Security/ConfigureSplunkforwardingtousesignedcertificates
3108+
passphrase = "carolebaskindidit"
3109+
cmds = [
3110+
"openssl genrsa -aes256 -passout pass:{pw} -out {path}/ca.key 2048".format(pw=passphrase, path=DEFAULTS_DIR),
3111+
"openssl req -new -key {path}/ca.key -passin pass:{pw} -out {path}/ca.csr -subj /CN=localhost".format(pw=passphrase, path=DEFAULTS_DIR),
3112+
"openssl x509 -req -in {path}/ca.csr -sha512 -passin pass:{pw} -signkey {path}/ca.key -CAcreateserial -out {path}/ca.pem -days 3".format(pw=passphrase, path=DEFAULTS_DIR),
3113+
"openssl genrsa -aes256 -passout pass:{pw} -out {path}/server.key 2048".format(pw=passphrase, path=DEFAULTS_DIR),
3114+
"openssl req -new -passin pass:{pw} -key {path}/server.key -out {path}/server.csr -subj /CN=localhost".format(pw=passphrase, path=DEFAULTS_DIR),
3115+
"openssl x509 -req -passin pass:{pw} -in {path}/server.csr -SHA256 -CA {path}/ca.pem -CAkey {path}/ca.key -CAcreateserial -out {path}/server.pem -days 3".format(pw=passphrase, path=DEFAULTS_DIR),
3116+
"cat {path}/server.pem {path}/server.key {path}/ca.pem > {path}/cert.pem".format(path=DEFAULTS_DIR)
3117+
]
3118+
for cmd in cmds:
3119+
execute_cmd = subprocess.check_output(["/bin/sh", "-c", cmd])
3120+
# Update s2s ssl settings
3121+
output = re.sub(r''' s2s:.*?ssl: false''', r''' s2s:
3122+
ca: /tmp/defaults/ca.pem
3123+
cert: /tmp/defaults/cert.pem
3124+
enable: true
3125+
password: {}
3126+
port: 9997
3127+
ssl: true'''.format(passphrase), output, flags=re.DOTALL)
3128+
# Write the default.yml to a file
3129+
with open(os.path.join(DEFAULTS_DIR, "default.yml"), "w") as f:
3130+
f.write(output)
3131+
# Standup deployment
3132+
try:
3133+
self.compose_file_name = "3idx1cm.yaml"
3134+
self.project_name = generate_random_string()
3135+
container_count, rc = self.compose_up()
3136+
assert rc == 0
3137+
# Wait for containers to come up
3138+
assert self.wait_for_containers(container_count, label="com.docker.compose.project={}".format(self.project_name), timeout=600)
3139+
# Get container logs
3140+
container_mapping = {"cm1": "cm", "idx1": "idx", "idx2": "idx", "idx3": "idx"}
3141+
for container in container_mapping:
3142+
# Check ansible version & configs
3143+
ansible_logs = self.get_container_logs(container)
3144+
self.check_ansible(ansible_logs)
3145+
# Check values in log output
3146+
inventory_json = self.extract_json(container)
3147+
self.check_common_keys(inventory_json, container_mapping[container])
3148+
try:
3149+
assert inventory_json["splunk_indexer"]["hosts"] == ["idx1", "idx2", "idx3"]
3150+
assert inventory_json["splunk_cluster_master"]["hosts"] == ["cm1"]
3151+
except KeyError as e:
3152+
self.logger.error(e)
3153+
raise e
3154+
# Check Splunkd on all the containers
3155+
assert self.check_splunkd("admin", self.password)
3156+
# Make sure apps are installed, and shcluster is setup properly
3157+
containers = self.client.containers(filters={"label": "com.docker.compose.project={}".format(self.project_name)})
3158+
assert len(containers) == 4
3159+
for container in containers:
3160+
container_name = container["Names"][0].strip("/")
3161+
cid = container["Id"]
3162+
exec_command = self.client.exec_create(cid, "cat /opt/splunk/etc/system/local/inputs.conf", user="splunk")
3163+
std_out = self.client.exec_start(exec_command)
3164+
assert "[splunktcp-ssl:9997]" in std_out
3165+
assert "disabled = 0" in std_out
3166+
assert "[SSL]" in std_out
3167+
assert "serverCert = /tmp/defaults/cert.pem" in std_out
3168+
assert "[sslConfig]" in std_out
3169+
assert "rootCA = /tmp/defaults/ca.pem" in std_out
3170+
if container_name == "cm1":
3171+
exec_command = self.client.exec_create(cid, "cat /opt/splunk/etc/system/local/outputs.conf", user="splunk")
3172+
std_out = self.client.exec_start(exec_command)
3173+
assert "clientCert = /tmp/defaults/cert.pem" in std_out
3174+
assert "sslPassword" in std_out
3175+
assert "useClientSSLCompression = true" in std_out
3176+
# Check that data is being forwarded properly
3177+
time.sleep(15)
3178+
search_providers, distinct_hosts = self.search_internal_distinct_hosts("cm1", password=self.password)
3179+
assert len(search_providers) == 4
3180+
assert "idx1" in search_providers
3181+
assert "idx2" in search_providers
3182+
assert "idx3" in search_providers
3183+
assert distinct_hosts == 4
3184+
except Exception as e:
3185+
self.logger.error(e)
3186+
raise e
3187+
finally:
3188+
files = [
3189+
os.path.join(DEFAULTS_DIR, "ca.key"),
3190+
os.path.join(DEFAULTS_DIR, "ca.csr"),
3191+
os.path.join(DEFAULTS_DIR, "ca.pem"),
3192+
os.path.join(DEFAULTS_DIR, "server.key"),
3193+
os.path.join(DEFAULTS_DIR, "server.csr"),
3194+
os.path.join(DEFAULTS_DIR, "server.pem"),
3195+
os.path.join(DEFAULTS_DIR, "cert.pem"),
3196+
os.path.join(DEFAULTS_DIR, "default.yml")
3197+
]
3198+
self.cleanup_files(files)
3199+
29353200
def test_compose_3idx1cm_default_repl_factor(self):
29363201
# Generate default.yml
29373202
cid = self.client.create_container(self.SPLUNK_IMAGE_NAME, tty=True, command="create-defaults")

0 commit comments

Comments
 (0)