Attachments upload with 429

eccles · eccles · commit 3c6fc43aa793 · 2023-02-15T15:08:21.000Z
Problem:
if an attachments upload encounters a 429 then the connection is closed
resulting in a zero length file.

Solution:
Make sure that Verify is True which will enable keep-alive.
Additionally upgrade all dependencies and reformat code to latest
standard.

Signed-off-by: Paul Hewlett &lt;phewlett76@gmail.com&gt;
diff --git a/archivist/applications.py b/archivist/applications.py
@@ -162,7 +162,6 @@ def __params(
         display_name: Optional[str] = None,
         custom_claims: Optional[dict[str, str]] = None,
     ) -> dict[str, Any]:
-
         params = {}
 
         if display_name is not None:
diff --git a/archivist/archivist.py b/archivist/archivist.py
@@ -222,7 +222,6 @@ def __copy__(self) -> Archivist:
         )
 
     def _add_headers(self, headers: dict[str, str] | None) -> dict[str, Any]:
-
         if isinstance(headers, dict):
             newheaders = {**headers}
         else:
diff --git a/archivist/archivistpublic.py b/archivist/archivistpublic.py
@@ -83,7 +83,6 @@ def __init__(
         verify: bool = True,
         max_time: float = MAX_TIME,
     ):
-
         self._verify = verify
         self._response_ring_buffer = deque(maxlen=self.RING_BUFFER_MAX_LEN)
         self._session = None
diff --git a/archivist/cmds/runner/run.py b/archivist/cmds/runner/run.py
@@ -17,7 +17,6 @@
 
 
 def run(arch: archivist.Archivist, args):
-
     LOGGER.info("Using version %s of rkvst-archivist", about.__version__)
     LOGGER.info("Namespace %s", args.namespace)
 
diff --git a/archivist/cmds/template/run.py b/archivist/cmds/template/run.py
@@ -18,7 +18,6 @@
 
 
 def run(arch: "type_helper.Archivist", args):
-
     LOGGER.info("Using version %s of rkvst-archivist", about.__version__)
     LOGGER.info("Namespace %s", args.namespace)
 
diff --git a/archivist/compliance.py b/archivist/compliance.py
@@ -106,7 +106,6 @@ def compliant_at_report(self, compliance: dict[str, Any]):
 
         LOGGER.info("Compliant %s", compliance["compliant"])
         for outcome in compliance["compliance"]:
-
             if outcome["compliant"]:
                 continue
 
diff --git a/archivist/parser.py b/archivist/parser.py
@@ -145,7 +145,6 @@ def common_parser(description: str):
 
 
 def endpoint(args):
-
     if args.verbose:
         set_logger("DEBUG")
     else:
@@ -190,7 +189,7 @@ def endpoint(args):
         LOGGER.error("Critical error.  Aborting.")
         sys_exit(1)
 
-    arch = Archivist(args.url, auth, verify=False, fixtures=fixtures)
+    arch = Archivist(args.url, auth, fixtures=fixtures)
     if arch is None:
         LOGGER.error("Critical error.  Aborting.")
         sys_exit(1)
diff --git a/archivist/runner.py b/archivist/runner.py
@@ -280,7 +280,6 @@ def args(self, identity_method, step):
             label = self.get(f"{noun}_label")
             func = self.label("use", noun)
             if label is not None and func:
-
                 identity = self.identity_from_label(noun, identity_method)
                 if identity is None:
                     raise ArchivistInvalidOperationError(f"unknown {noun} '{label}'")
diff --git a/archivist/subjects.py b/archivist/subjects.py
@@ -267,7 +267,6 @@ def __params(
         wallet_pub_key: Optional[list[str]] = None,
         tessera_pub_key: Optional[list[str]] = None,
     ) -> dict[str, Any]:
-
         params = {}
 
         if display_name is not None:
diff --git a/examples/access_policies_filter.py b/examples/access_policies_filter.py
@@ -38,7 +38,6 @@ def main():
         "https://app.rkvst.io",
         auth,
     ) as arch:
-
         # count access_policies...
         print(
             "no.of access_policies",
diff --git a/examples/access_policy_create.py b/examples/access_policy_create.py
@@ -38,7 +38,6 @@ def main():
         "https://app.rkvst.io",
         (client_id, client_secret),
     ) as arch:
-
         props = {
             "display_name": "Friendly name of the policy",
             "description": "Description of the policy",
diff --git a/examples/applications_registration.py b/examples/applications_registration.py
@@ -33,7 +33,6 @@ def main():
         "https://app.rkvst.io",
         authtoken,
     ) as arch:
-
         # create application
         application = arch.applications.create(
             f"Application display name {uuid4()}",
diff --git a/examples/create_event.py b/examples/create_event.py
@@ -156,7 +156,6 @@ def main():
         "https://app.rkvst.io",
         (client_id, client_secret),
     ) as arch:
-
         # Create a new asset
         asset = create_asset(arch)
         print("Asset", json_dumps(asset, sort_keys=True, indent=4))
diff --git a/examples/filter_assets.py b/examples/filter_assets.py
@@ -39,7 +39,6 @@ def main():
         "https://app.rkvst.io",
         (client_id, client_secret),
     ) as arch:
-
         # list all assets with required attributes and properties
         props = {"confirmation_status": "CONFIRMED"}
         attrs = {"arc_display_type": "Traffic light"}
diff --git a/examples/filter_publicevents.py b/examples/filter_publicevents.py
@@ -15,7 +15,6 @@ def main():
     """
     # Initialize connection to ArchivistPublic
     with ArchivistPublic() as public:
-
         # Get all public events with required attributes and properties
         props = {"confirmation_status": "CONFIRMED"}
         attrs = {"arc_display_type": "Traffic light"}
diff --git a/examples/get_publicevent.py b/examples/get_publicevent.py
@@ -13,7 +13,6 @@ def main():
     """Main function of getting public events."""
     # Initialize connection to ArchivistPublic
     with ArchivistPublic() as public:
-
         # URL is the fully-attested URL returned by archivist
         event = public.events.read(
             "https://app.rkvst.io/archivist/"
diff --git a/examples/runner.py b/examples/runner.py
@@ -17,7 +17,6 @@
 
 
 def run(arch: Archivist, args):
-
     LOGGER.info("Using version %s of rkvst-archivist", about.__version__)
     LOGGER.info("Namespace %s", args.namespace)
 
diff --git a/examples/sbom_release.py b/examples/sbom_release.py
@@ -98,8 +98,7 @@ def main():
         client_secret_filename=getenv("RKVST_APPREG_SECRET_FILENAME"),
     )
 
-    with Archivist(rkvst_url, auth, verify=False, max_time=300) as arch:
-
+    with Archivist(rkvst_url, auth, max_time=300) as arch:
         asset, event = sbom_release(
             arch, getenv("BUILD_BUILDNUMBER"), getenv("SBOM_FILEPATH")
         )
diff --git a/examples/scan_test.py b/examples/scan_test.py
@@ -137,8 +137,7 @@ def main():
         client_secret_filename=getenv("RKVST_APPREG_SECRET_FILENAME"),
     )
 
-    with Archivist(getenv("RKVST_URL"), auth, verify=False, max_time=300) as arch:
-
+    with Archivist(getenv("RKVST_URL"), auth, max_time=300) as arch:
         print("##[group]Today")
         today = date.today()
         scan_test(arch, today.strftime("%Y-%m-%d"))
diff --git a/examples/subject_create.py b/examples/subject_create.py
@@ -38,7 +38,6 @@ def main():
         "https://app.rkvst.io",
         (client_id, client_secret),
     ) as arch:
-
         # subject_string is the base64 encoding of the self subject of the other organization
         subject_string = (
             "eyJpZGVudGl0eSI6ICJzdWJqZWN0cy8wMDAwMDAwMC0wMDAwLTAwMDAtMDA"
diff --git a/examples/subjects_filter.py b/examples/subjects_filter.py
@@ -32,7 +32,6 @@ def main():
         "https://app.rkvst.io",
         (client_id, client_secret),
     ) as arch:
-
         # count subjects...
         print("no.of subjects", arch.subjects.count(display_name="Some display name"))
 
diff --git a/functests/execaccess_policies.py b/functests/execaccess_policies.py
@@ -120,7 +120,7 @@ def setUp(self):
             client_secret=getenv("RKVST_APPREG_SECRET"),
             client_secret_filename=getenv("RKVST_APPREG_SECRET_FILENAME"),
         )
-        self.arch = Archivist(getenv("RKVST_URL"), auth, verify=False)
+        self.arch = Archivist(getenv("RKVST_URL"), auth)
 
         # these are for access_policies
         self.ac_props = deepcopy(PROPS)
@@ -301,7 +301,7 @@ def setUp(self):
         super().setUp()
         with open(getenv("RKVST_AUTHTOKEN_FILENAME_2"), encoding="utf-8") as fd:
             auth_2 = fd.read().strip()
-        self.arch_2 = Archivist(getenv("RKVST_URL"), auth_2, verify=False)
+        self.arch_2 = Archivist(getenv("RKVST_URL"), auth_2)
 
         # creates reciprocal subjects for arch 1 and arch 2.
         # subject 1 contains details of subject 2 to be shared
diff --git a/functests/execapplications.py b/functests/execapplications.py
@@ -55,7 +55,7 @@ def setUp(self):
             client_secret=getenv("RKVST_APPREG_SECRET"),
             client_secret_filename=getenv("RKVST_APPREG_SECRET_FILENAME"),
         )
-        self.arch = Archivist(getenv("RKVST_URL"), auth, verify=False)
+        self.arch = Archivist(getenv("RKVST_URL"), auth)
         self.display_name = f"{DISPLAY_NAME} {uuid4()}"
 
     def tearDown(self):
@@ -231,9 +231,7 @@ def test_archivist_token(self):
         with Archivist(
             getenv("RKVST_URL"),
             (application["client_id"], application["credentials"][0]["secret"]),
-            verify=False,
         ) as new_arch:
-
             # now we create an asset and add events 10 times with a 60s sleep
             # this should trigger a token refresh
             traffic_light = deepcopy(ATTRS)
diff --git a/functests/execassets.py b/functests/execassets.py
@@ -85,7 +85,7 @@ def setUp(self):
             client_secret=getenv("RKVST_APPREG_SECRET"),
             client_secret_filename=getenv("RKVST_APPREG_SECRET_FILENAME"),
         )
-        self.arch = Archivist(getenv("RKVST_URL"), auth, verify=False, max_time=600)
+        self.arch = Archivist(getenv("RKVST_URL"), auth, max_time=600)
         self.attrs = deepcopy(ATTRS)
         self.traffic_light = deepcopy(ATTRS)
         self.traffic_light["arc_display_type"] = "Traffic light with violation camera"
@@ -248,7 +248,7 @@ def setUp(self):
             client_secret=getenv("RKVST_APPREG_SECRET"),
             client_secret_filename=getenv("RKVST_APPREG_SECRET_FILENAME"),
         )
-        self.arch = Archivist(getenv("RKVST_URL"), auth, verify=False, max_time=600)
+        self.arch = Archivist(getenv("RKVST_URL"), auth, max_time=600)
 
     def tearDown(self):
         self.arch.close()
diff --git a/functests/execattachments.py b/functests/execattachments.py
@@ -44,7 +44,7 @@ def setUp(self):
             client_secret=getenv("RKVST_APPREG_SECRET"),
             client_secret_filename=getenv("RKVST_APPREG_SECRET_FILENAME"),
         )
-        self.arch = Archivist(getenv("RKVST_URL"), auth, verify=False)
+        self.arch = Archivist(getenv("RKVST_URL"), auth)
         self.file_uuid: str = ""
 
         with suppress(FileNotFoundError):
@@ -195,7 +195,7 @@ def setUp(self):
             client_id=getenv("RKVST_APPREG_CLIENT"),
             client_secret_filename=getenv("RKVST_APPREG_SECRET_FILENAME"),
         )
-        self.arch = Archivist(getenv("RKVST_URL"), auth, verify=False)
+        self.arch = Archivist(getenv("RKVST_URL"), auth)
 
     def tearDown(self):
         self.arch.close()
diff --git a/functests/execcompliance_policies.py b/functests/execcompliance_policies.py
@@ -98,7 +98,7 @@ def setUp(self):
             client_secret=getenv("RKVST_APPREG_SECRET"),
             client_secret_filename=getenv("RKVST_APPREG_SECRET_FILENAME"),
         )
-        self.arch = Archivist(getenv("RKVST_URL"), auth, verify=False)
+        self.arch = Archivist(getenv("RKVST_URL"), auth)
 
     def tearDown(self):
         self.arch.close()
diff --git a/functests/execnotebooks.py b/functests/execnotebooks.py
@@ -32,9 +32,7 @@ def setUp(self):
         self.client_id = getenv("RKVST_APPREG_CLIENT")
         self.client_secret = getenv("RKVST_APPREG_SECRET")
         self.url = getenv("RKVST_URL")
-        self.arch = Archivist(
-            self.url, (self.client_id, self.client_secret), verify=False
-        )
+        self.arch = Archivist(self.url, (self.client_id, self.client_secret))
 
     def tearDown(self):
         self.arch.close()
diff --git a/functests/execpublicassets.py b/functests/execpublicassets.py
@@ -87,7 +87,7 @@ def setUp(self):
             client_secret_filename=getenv("RKVST_APPREG_SECRET_FILENAME"),
         )
         self.url = getenv("RKVST_URL")
-        self.arch = Archivist(self.url, auth, verify=False, max_time=600)
+        self.arch = Archivist(self.url, auth, max_time=600)
         self.attrs = deepcopy(ATTRS)
         self.traffic_light = deepcopy(ATTRS)
         self.traffic_light["arc_display_type"] = "Traffic light with violation camera"
diff --git a/functests/execrunner.py b/functests/execrunner.py
@@ -40,7 +40,7 @@ def setUp(self):
             client_secret=getenv("RKVST_APPREG_SECRET"),
             client_secret_filename=getenv("RKVST_APPREG_SECRET_FILENAME"),
         )
-        self.arch = Archivist(getenv("RKVST_URL"), auth, verify=False, max_time=300)
+        self.arch = Archivist(getenv("RKVST_URL"), auth, max_time=300)
 
     def tearDown(self):
         self.arch.close()
diff --git a/functests/execsboms.py b/functests/execsboms.py
@@ -52,7 +52,7 @@ def setUp(self):
             client_secret=getenv("RKVST_APPREG_SECRET"),
             client_secret_filename=getenv("RKVST_APPREG_SECRET_FILENAME"),
         )
-        self.arch = Archivist(getenv("RKVST_URL"), auth, verify=False)
+        self.arch = Archivist(getenv("RKVST_URL"), auth)
         self.file_uuid: str = ""
         self.title = "TestSBOM"
 
diff --git a/functests/execsubjects.py b/functests/execsubjects.py
@@ -59,7 +59,7 @@ def setUp(self):
             auth_token_filename=getenv("RKVST_AUTHTOKEN_FILENAME"),
             client_secret_filename=getenv("RKVST_APPREG_SECRET_FILENAME"),
         )
-        self.arch = Archivist(getenv("RKVST_URL"), auth, verify=False)
+        self.arch = Archivist(getenv("RKVST_URL"), auth)
         self.display_name = f"{DISPLAY_NAME} {uuid4()}"
 
     def tearDown(self):
diff --git a/requirements-dev.txt b/requirements-dev.txt
@@ -1,21 +1,21 @@
 -r requirements.txt
 
 # code quality
-autopep8~=1.6
-black[jupyter]~=22.6
-coverage[toml]~=6.4
+autopep8~=2.0
+black[jupyter]~=23.1
+coverage[toml]~=7.1
 pip-audit~=2.4
-pycodestyle~=2.9
-pylint~=2.14
+pycodestyle~=2.10
+pylint~=2.16
 pyright~=1.1
-unittest-xml-reporting~=3.2.0
+unittest-xml-reporting~=3.2
 testbook~=0.4
 
 # analyze dependencies
-pipdeptree~=2.2
+pipdeptree~=2.3
 
 # uploading to pypi
-build~=0.8
+build~=0.10
 twine~=4.0
 
 # for sbom.xml file
diff --git a/requirements.txt b/requirements.txt
@@ -2,14 +2,14 @@
 # for the published wheel - the file docs/requirements.txt
 # must be kept in sync with this file.
 #
+# We can only upgrade to 2.2 of backoff when support for py3.7 is dropped
 backoff~=1.11
 certifi
 flatten-dict~=0.4
-iso8601~=1.0
+iso8601~=1.1
 Jinja2~=3.0
 pyaml-env~=1.1
 requests~=2.28
-requests-toolbelt~=0.9
 rfc3339~=6.2
 xmltodict~=0.13
 
diff --git a/scripts/builder.sh b/scripts/builder.sh
@@ -14,7 +14,7 @@ then
 fi
 
 docker run \
-    --rm \
+    --rm -it \
     -v $(pwd):/home/builder \
     -u $(id -u):$(id -g) \
     -e FUNCTEST \
diff --git a/unittests/testassets.py b/unittests/testassets.py
@@ -321,7 +321,6 @@ def common_assets_create_if_not_exists_nonexistent_asset(
         ) as mock_location, mock.patch.object(
             self.arch.attachments, "create"
         ) as mock_attachments:
-
             mock_get.side_effect = ArchivistNotFoundError
             mock_post.return_value = MockResponse(200, **resp)
             if loc_resp is not None:
diff --git a/unittests/testeventsconfirm.py b/unittests/testeventsconfirm.py
@@ -41,7 +41,6 @@ def test_events_create_with_confirmation(self):
         with mock.patch.object(
             self.arch.session, "post"
         ) as mock_post, mock.patch.object(self.arch.session, "get") as mock_get:
-
             mock_post.return_value = MockResponse(200, **RESPONSE)
             mock_get.return_value = MockResponse(200, **RESPONSE)
 
diff --git a/unittests/testeventscreate.py b/unittests/testeventscreate.py
@@ -376,7 +376,6 @@ def test_events_create_with_explicit_confirmation(self):
         with mock.patch.object(
             self.arch.session, "post"
         ) as mock_post, mock.patch.object(self.arch.session, "get") as mock_get:
-
             mock_post.return_value = MockResponse(200, **RESPONSE)
             mock_get.return_value = MockResponse(200, **RESPONSE)
 
diff --git a/unittests/testpublic.py b/unittests/testpublic.py
@@ -59,7 +59,7 @@ def test_public(self):
             self.assertEqual(
                 public.public,
                 True,
-                msg="verify must be True",
+                msg="public must be True",
             )
             with self.assertRaises(AttributeError):
                 e = public.Illegal_endpoint

Original file line number	Diff line number	Diff line change
`@@ -222,7 +222,6 @@ def __copy__(self) -> Archivist:`
`222`	`222`	`)`
`223`	`223`
`224`	`224`	`def _add_headers(self, headers: dict[str, str] \| None) -> dict[str, Any]:`
`225`		`-`
`226`	`225`	`if isinstance(headers, dict):`
`227`	`226`	`newheaders = {**headers}`
`228`	`227`	`else:`