Skip to content

Commit 41c3702

Browse files
balsa-sarenacbalsa-sarenac
committed
feat(logging): make runs observable and failure-tolerant
Operators could not reliably tell whether scheduled upgrades ran, failed, or actually changed package state from server logs. This consolidates logging behavior across entrypoints, records run lifecycle and upgrade outcomes, and separates regular vs error streams for faster incident triage. It also keeps execution resilient when file logging cannot be initialized by falling back to stderr logging, so automation can still produce status output instead of failing early on log path or permission drift.
1 parent 58f2fca commit 41c3702

5 files changed

Lines changed: 242 additions & 55 deletions

File tree

‎upgrade/scripts/find_compatible_versions.py‎

Lines changed: 37 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
from packaging.utils import parse_wheel_filename
1111
from packaging.version import Version
1212

13+
from upgrade.scripts.logging_config import configure_logging
1314
from upgrade.scripts.requirements import (
1415
filter_versions,
1516
parse_requirements_txt,
@@ -116,18 +117,32 @@ def find_compatible_versions(
116117
or if requirements or requirements_file is not provided.
117118
"""
118119
response_status = {}
120+
regular_log_path = None
121+
error_log_path = None
122+
default_log_location = "/var/log/find_compatible_versions.log"
123+
try:
124+
regular_log_path, error_log_path = configure_logging(
125+
log_location=log_location,
126+
default_log_location=default_log_location,
127+
test=bool(test),
128+
level=logging.DEBUG if test else logging.INFO,
129+
)
130+
except Exception as logging_config_error:
131+
configure_logging(
132+
log_location=None,
133+
default_log_location=default_log_location,
134+
test=True,
135+
level=logging.INFO,
136+
)
137+
logging.warning(
138+
"Failed to configure file logging at %s; falling back to stderr logging: %s",
139+
log_location or default_log_location,
140+
logging_config_error,
141+
)
142+
logging.info("Starting find_compatible_versions run venv_path=%s", venv_path)
119143
try:
120144
if requirements is None and requirements_file is None:
121145
raise Exception("Either requirements or requirements_file is required.")
122-
if test:
123-
logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(message)s")
124-
else:
125-
log_location = log_location or "/var/log/manage_venv.log"
126-
logging.basicConfig(
127-
filename=log_location,
128-
level=logging.WARNING,
129-
format="%(asctime)s %(message)s",
130-
)
131146
if cloudsmith_url:
132147
is_cloudsmith_url_valid(cloudsmith_url)
133148

@@ -137,18 +152,24 @@ def find_compatible_versions(
137152
)
138153
if upgrade_version:
139154
response_status["responseStatus"] = CompatibleUpgradeStatus.AVAILABLE.value
140-
logging.info(f"Found compatible upgrade version: {upgrade_version}")
155+
logging.info("Found compatible upgrade version: %s", upgrade_version)
141156
else:
142-
response_status[
143-
"responseStatus"
144-
] = CompatibleUpgradeStatus.AT_LATEST_VERSION.value
157+
response_status["responseStatus"] = (
158+
CompatibleUpgradeStatus.AT_LATEST_VERSION.value
159+
)
145160
logging.info("At latest upgrade version")
146-
except Exception as e:
161+
except Exception:
147162
response_status["responseStatus"] = CompatibleUpgradeStatus.ERROR.value
148-
logging.error(e)
149-
raise e
163+
logging.exception("find_compatible_versions run failed")
164+
raise
150165
finally:
151166
response = json.dumps(response_status)
167+
logging.info(
168+
"Completed find_compatible_versions run status=%s regular_log=%s error_log=%s",
169+
response_status.get("responseStatus"),
170+
regular_log_path,
171+
error_log_path,
172+
)
152173
print(response)
153174

154175

‎upgrade/scripts/logging_config.py‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
import logging
2+
import sys
3+
from logging.handlers import WatchedFileHandler
4+
from pathlib import Path
5+
from typing import Optional, Tuple
6+
7+
DEFAULT_LOG_FORMAT = "%(asctime)s %(levelname)s %(name)s %(message)s"
8+
DEFAULT_DATE_FORMAT = "%Y-%m-%dT%H:%M:%S%z"
9+
10+
11+
class _MaxLevelFilter(logging.Filter):
12+
def __init__(self, max_level: int):
13+
super().__init__()
14+
self.max_level = max_level
15+
16+
def filter(self, record: logging.LogRecord) -> bool:
17+
return record.levelno < self.max_level
18+
19+
20+
def get_error_log_path(log_location: str) -> str:
21+
log_path = Path(log_location)
22+
if log_path.suffix:
23+
return str(log_path.with_name(f"{log_path.stem}.error{log_path.suffix}"))
24+
return f"{log_location}.error.log"
25+
26+
27+
def configure_logging(
28+
*,
29+
log_location: Optional[str],
30+
default_log_location: str,
31+
test: bool,
32+
level: int = logging.INFO,
33+
) -> Tuple[Optional[str], Optional[str]]:
34+
root = logging.getLogger()
35+
for handler in list(root.handlers):
36+
root.removeHandler(handler)
37+
handler.close()
38+
39+
formatter = logging.Formatter(DEFAULT_LOG_FORMAT, datefmt=DEFAULT_DATE_FORMAT)
40+
41+
if test:
42+
stream_handler = logging.StreamHandler(sys.stderr)
43+
stream_handler.setLevel(level)
44+
stream_handler.setFormatter(formatter)
45+
root.addHandler(stream_handler)
46+
root.setLevel(level)
47+
return None, None
48+
49+
regular_log_path = log_location or default_log_location
50+
error_log_path = get_error_log_path(regular_log_path)
51+
52+
regular_handler = WatchedFileHandler(regular_log_path)
53+
regular_handler.setLevel(level)
54+
regular_handler.addFilter(_MaxLevelFilter(logging.ERROR))
55+
regular_handler.setFormatter(formatter)
56+
57+
error_handler = WatchedFileHandler(error_log_path)
58+
error_handler.setLevel(logging.ERROR)
59+
error_handler.setFormatter(formatter)
60+
61+
root.addHandler(regular_handler)
62+
root.addHandler(error_handler)
63+
root.setLevel(level)
64+
return regular_log_path, error_log_path

‎upgrade/scripts/manage_venv.py‎

Lines changed: 38 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
from pathlib import Path
1111
from typing import Any, List, Optional
1212

13+
from upgrade.scripts.logging_config import configure_logging
1314
from upgrade.scripts.requirements import parse_requirements_txt, to_requirements_obj
1415
from upgrade.scripts.utils import (
1516
is_development_cloudsmith,
@@ -322,19 +323,38 @@ def manage_venv(
322323
local_installation_path: Optional[str] = None,
323324
):
324325
response_status = {}
326+
regular_log_path = None
327+
error_log_path = None
328+
default_log_location = "/var/log/manage_venv.log"
329+
try:
330+
regular_log_path, error_log_path = configure_logging(
331+
log_location=log_location,
332+
default_log_location=default_log_location,
333+
test=bool(test),
334+
level=logging.DEBUG if test else logging.INFO,
335+
)
336+
except Exception as logging_config_error:
337+
configure_logging(
338+
log_location=None,
339+
default_log_location=default_log_location,
340+
test=True,
341+
level=logging.INFO,
342+
)
343+
logging.warning(
344+
"Failed to configure file logging at %s; falling back to stderr logging: %s",
345+
log_location or default_log_location,
346+
logging_config_error,
347+
)
348+
logging.info(
349+
"Starting manage_venv run envs_home=%s auto_upgrade=%s local_wheels=%s",
350+
envs_home,
351+
auto_upgrade,
352+
update_from_local_wheels,
353+
)
325354
try:
326355
if requirements is None and requirements_file is None:
327356
raise Exception("Either requirements or requirements_file is required.")
328357

329-
if test:
330-
logging.basicConfig(level=logging.DEBUG, format="%(asctime)s %(message)s")
331-
else:
332-
log_location = log_location or "/var/log/manage_venv.log"
333-
logging.basicConfig(
334-
filename=log_location,
335-
level=logging.WARNING,
336-
format="%(asctime)s %(message)s",
337-
)
338358
if cloudsmith_url:
339359
is_cloudsmith_url_valid(cloudsmith_url)
340360

@@ -354,12 +374,18 @@ def manage_venv(
354374
local_installation_path,
355375
)
356376
response_status["responseStatus"] = VenvUpgradeStatus.UPGRADED.value
357-
except Exception as e:
358-
logging.error(e)
377+
except Exception:
378+
logging.exception("manage_venv run failed")
359379
response_status["responseStatus"] = VenvUpgradeStatus.ERROR.value
360-
raise e
380+
raise
361381
finally:
362382
response = json.dumps(response_status)
383+
logging.info(
384+
"Completed manage_venv run status=%s regular_log=%s error_log=%s",
385+
response_status.get("responseStatus"),
386+
regular_log_path,
387+
error_log_path,
388+
)
363389
print(response)
364390

365391

0 commit comments

Comments
 (0)