Merge remote-tracking branch 'origin/develop' into SDK-30/nosemgrep

Author: aciesielczyk-splunk

.github/workflows/test.yml

@@ -1,7 +1,7 @@
 name: Python CI
 
 on:
-  [ push, pull_request, workflow_dispatch ]
+  [ push, workflow_dispatch ]
 
 jobs:
   build:
@@ -11,11 +11,22 @@ jobs:
       matrix:
         os:
           - ubuntu-latest
-        python: [ 3.7, 3.9, 3.13]
+        python: [ 3.9, 3.13 ]
         splunk-version:
           - "8.1"
           - "8.2"
           - "latest"
+        include:
+          - os: ubuntu-22.04
+            python: 3.7
+            splunk-version: "8.1"
+          - os: ubuntu-22.04
+            python: 3.7
+            splunk-version: "8.2"
+          - os: ubuntu-22.04
+            python: 3.7
+            splunk-version: "latest"
+
       fail-fast: false
 
     steps:

splunklib/binding.py

@@ -39,8 +39,8 @@
 from http import client
 from http.cookies import SimpleCookie
 from xml.etree.ElementTree import XML, ParseError
-from splunklib.data import record
-from splunklib import __version__
+from .data import record
+from . import __version__
 
 
 logger = logging.getLogger(__name__)

splunklib/client.py

@@ -68,9 +68,9 @@
 from time import sleep
 from urllib import parse
 
-from splunklib import data
-from splunklib.data import record
-from splunklib.binding import (AuthenticationError, Context, HTTPError, UrlEncoded,
+from . import data
+from .data import record
+from .binding import (AuthenticationError, Context, HTTPError, UrlEncoded,
                                _encode, _make_cookie_header, _NoAuthenticationToken,
                                namespace)
 
@@ -779,7 +779,7 @@ def get_api_version(self, path):
         # For example, "/services/search/jobs" is using API v1
         api_version = 1
 
-        versionSearch = re.search('(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path)
+        versionSearch = re.search(r'(?:servicesNS\/[^/]+\/[^/]+|services)\/[^/]+\/v(\d+)\/', path)
         if versionSearch:
             api_version = int(versionSearch.group(1))
 
@@ -3999,4 +3999,4 @@ def batch_save(self, *documents):
         data = json.dumps(documents)
 
         return json.loads(
-            self._post('batch_save', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8'))
+            self._post('batch_save', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8'))

splunklib/modularinput/event.py

@@ -15,7 +15,7 @@
 from io import TextIOBase
 import xml.etree.ElementTree as ET
 
-from splunklib.utils import ensure_str
+from ..utils import ensure_str
 
 
 class Event:

splunklib/modularinput/event_writer.py

@@ -15,7 +15,7 @@
 import sys
 import traceback
 
-from splunklib.utils import ensure_str
+from ..utils import ensure_str
 from .event import ET
 
 

splunklib/searchcommands/internals.py

@@ -554,7 +554,7 @@ def write_record(self, record):
 
     def write_records(self, records):
         self._ensure_validity()
-        records = list(records)
+        records = [] if records is NotImplemented else list(records)
         write_record = self._write_record
         for record in records:
             write_record(record)

splunklib/searchcommands/reporting_command.py

@@ -77,21 +77,26 @@ def map(self, records):
         """
         return NotImplemented
 
-    def prepare(self):
-
-        phase = self.phase
+    def _has_custom_method(self, method_name):
+        method = getattr(self.__class__, method_name, None)
+        base_method = getattr(ReportingCommand, method_name, None)
+        return callable(method) and (method is not base_method)
 
-        if phase == 'map':
-            # noinspection PyUnresolvedReferences
-            self._configuration = self.map.ConfigurationSettings(self)
+    def prepare(self):
+        if self.phase == 'map':
+            if self._has_custom_method('map'):
+                phase_method = getattr(self.__class__, 'map')
+                self._configuration = phase_method.ConfigurationSettings(self)
+            else:
+                self._configuration = self.ConfigurationSettings(self)
             return
 
-        if phase == 'reduce':
+        if self.phase == 'reduce':
             streaming_preop = chain((self.name, 'phase="map"', str(self._options)), self.fieldnames)
             self._configuration.streaming_preop = ' '.join(streaming_preop)
             return
 
-        raise RuntimeError(f'Unrecognized reporting command phase: {json_encode_string(str(phase))}')
+        raise RuntimeError(f'Unrecognized reporting command phase: {json_encode_string(str(self.phase))}')
 
     def reduce(self, records):
         """ Override this method to produce a reporting data structure.

splunklib/searchcommands/search_command.py

@@ -34,11 +34,8 @@
 from urllib.parse import urlsplit
 from warnings import warn
 from xml.etree import ElementTree
-from splunklib.utils import ensure_str
-
 
 # Relative imports
-import splunklib
 from . import Boolean, Option, environment
 from .internals import (
     CommandLineParser,
@@ -53,6 +50,7 @@
     RecordWriterV2,
     json_encode_string)
 from ..client import Service
+from ..utils import ensure_str
 
 
 # ----------------------------------------------------------------------------------------------------------------------
@@ -342,24 +340,29 @@ def service(self):
             of :code:`None` is returned.
 
         """
+
         if self._service is not None:
             return self._service
 
         metadata = self._metadata
 
         if metadata is None:
+            self.logger.warning("Missing metadata for service creation.")
             return None
 
         try:
             searchinfo = self._metadata.searchinfo
         except AttributeError:
+            self.logger.warning("Missing searchinfo in metadata for service creation.")
             return None
 
         splunkd_uri = searchinfo.splunkd_uri
 
-        if splunkd_uri is None:
+        if splunkd_uri is None or splunkd_uri == "" or splunkd_uri == " ":
+            self.logger.warning(f"Incorrect value for Splunkd URI: {splunkd_uri!r} in metadata")
             return None
 
+
         uri = urlsplit(splunkd_uri, allow_fragments=False)
 
         self._service = Service(

tests/searchcommands/test_reporting_command.py

@@ -32,3 +32,42 @@ def reduce(self, records):
     data = list(data_chunk.data)
     assert len(data) == 1
     assert int(data[0]['sum']) == sum(range(0, 10))
+
+
+def test_simple_reporting_command_with_map():
+    @searchcommands.Configuration()
+    class MapAndReduceReportingCommand(searchcommands.ReportingCommand):
+        def map(self, records):
+            for record in records:
+                record["value"] = str(int(record["value"]) * 2)
+                yield record
+
+        def reduce(self, records):
+            total = 0
+            for record in records:
+                total += int(record["value"])
+            yield {"sum": total}
+
+    cmd = MapAndReduceReportingCommand()
+    ifile = io.BytesIO()
+
+    input_data = [{"value": str(i)} for i in range(5)]
+
+    mapped_data = list(cmd.map(input_data))
+
+    ifile.write(chunky.build_getinfo_chunk())
+    ifile.write(chunky.build_data_chunk(mapped_data))
+    ifile.seek(0)
+
+    ofile = io.BytesIO()
+    cmd._process_protocol_v2([], ifile, ofile)
+
+    ofile.seek(0)
+    chunk_stream = chunky.ChunkedDataStream(ofile)
+    chunk_stream.read_chunk()
+    data_chunk = chunk_stream.read_chunk()
+    assert data_chunk.meta['finished'] is True
+
+    result = list(data_chunk.data)
+    expected_sum = sum(i * 2 for i in range(5))
+    assert int(result[0]["sum"]) == expected_sum

tests/searchcommands/test_search_command.py

@@ -22,14 +22,17 @@
 import codecs
 import os
 import re
+import logging
 
 from io import TextIOWrapper
+from unittest.mock import MagicMock, patch
 
 import pytest
 
 import splunklib
 from splunklib.searchcommands import Configuration, StreamingCommand
 from splunklib.searchcommands.decorators import ConfigurationSetting, Option
+from splunklib.searchcommands.internals import ObjectView
 from splunklib.searchcommands.search_command import SearchCommand
 from splunklib.client import Service
 from splunklib.utils import ensure_binary
@@ -265,6 +268,52 @@ def test_process_scpv2(self):
 
     _package_directory = os.path.dirname(os.path.abspath(__file__))
 
+class TestSearchCommandService(TestCase):
+    def setUp(self):
+        TestCase.setUp(self)
+        self.command = SearchCommand()
+        console_handler = logging.StreamHandler()
+        console_handler.setLevel(logging.WARNING)
+        self.command.logger.addHandler(console_handler)
+
+    def test_service_exists(self):
+        self.command._service = Service()
+        self.assertIsNotNone(self.command.service)
+
+    def test_service_not_exists(self):
+        self.assertIsNone(self.command.service)
+
+    def test_missing_metadata(self):
+        with self.assertLogs(self.command.logger, level='WARNING') as log:
+            service = self.command.service
+            self.assertIsNone(service)
+        self.assertTrue(any("Missing metadata for service creation." in message for message in log.output))
+
+    def test_missing_searchinfo(self):
+        with self.assertLogs(self.command.logger, level='WARNING') as log:
+            self.command._metadata = ObjectView({})
+            self.assertIsNone(self.command.service)
+        self.assertTrue(any("Missing searchinfo in metadata for service creation." in message for message in log.output))
+
+
+    def test_missing_splunkd_uri(self):
+        with self.assertLogs(self.command.logger, level='WARNING') as log:
+            metadata = ObjectView({"searchinfo": ObjectView({"splunkd_uri": ""})})
+            self.command._metadata = metadata
+            self.assertIsNone(self.command.service)
+        self.assertTrue(any("Incorrect value for Splunkd URI: '' in metadata" in message for message in log.output))
+
+
+
+    def test_service_returns_valid_service_object(self):
+        metadata = ObjectView({"searchinfo":ObjectView({"splunkd_uri":"https://127.0.0.1:8089",
+                                                        "session_key":"mock_session_key",
+                                                        "app":"search",
+                                                        })})
+        self.command._metadata = metadata
+        self.assertIsInstance(self.command.service, Service)
+
+
 
 if __name__ == "__main__":
     main()