Skip to content

Commit e698375

Browse files
cbeauchesnecbeauchesne
authored
Adds OTEL_COLLECTOR_E2E in CI (#5688)
1 parent f1b1b5f commit e698375

File tree

7 files changed

+80
-72
lines changed

7 files changed

+80
-72
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,6 @@ jobs:
6666
strategy:
6767
matrix:
6868
include: ${{ fromJson(needs.compute_libraries_and_scenarios.outputs.library_matrix) }}
69-
7069
fail-fast: false
7170
uses: ./.github/workflows/system-tests.yml
7271
permissions:
@@ -87,6 +86,7 @@ jobs:
8786
library: ${{ matrix.library }}
8887
scenarios: ${{ needs.compute_libraries_and_scenarios.outputs.scenarios }}
8988
scenarios_groups: ${{ needs.compute_libraries_and_scenarios.outputs.scenarios_groups }}
89+
excluded_scenarios: OTEL_COLLECTOR_E2E # rely on real backend
9090
parametric_job_count: ${{ matrix.version == 'dev' && 2 || 1 }} # test both use cases
9191
skip_empty_scenarios: true
9292
display_summary: true

‎.github/workflows/run-end-to-end.yml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -472,6 +472,11 @@ jobs:
472472
- name: Run OTEL_COLLECTOR scenario
473473
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"OTEL_COLLECTOR"')
474474
run: ./run.sh OTEL_COLLECTOR
475+
- name: Run OTEL_COLLECTOR_E2E scenario
476+
if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"OTEL_COLLECTOR_E2E"')
477+
run: ./run.sh OTEL_COLLECTOR_E2E
478+
env:
479+
DD_API_KEY: ${{ secrets.DD_API_KEY }}
475480

476481
- name: Run all scenarios in replay mode
477482
if: success() && steps.build.outcome == 'success' && inputs._enable_replay_scenarios

‎tests/otel_postgres_metrics_e2e/test_postgres_metrics.py‎

Lines changed: 36 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -22,46 +22,46 @@
2222
"postgresql.rollbacks": {"data_type": "Sum", "description": "The number of rollbacks"},
2323
"postgresql.db_size": {"data_type": "Sum", "description": "The database disk usage"},
2424
"postgresql.table.count": {"data_type": "Sum", "description": "Number of user tables in a database"},
25-
"postgresql.backends": {"data_type": "Sum", "description": "The number of backends"},
26-
"postgresql.bgwriter.buffers.allocated": {"data_type": "Sum", "description": "Number of buffers allocated"},
27-
"postgresql.bgwriter.buffers.writes": {"data_type": "Sum", "description": "Number of buffers written"},
28-
"postgresql.bgwriter.checkpoint.count": {"data_type": "Sum", "description": "The number of checkpoints performed"},
29-
"postgresql.bgwriter.duration": {
30-
"data_type": "Sum",
31-
"description": "Total time spent writing and syncing files to disk by checkpoints",
32-
},
33-
"postgresql.bgwriter.maxwritten": {
34-
"data_type": "Sum",
35-
"description": "Number of times the background writer stopped a cleaning scan because it had written too many buffers",
36-
},
25+
# "postgresql.backends": {"data_type": "Sum", "description": "The number of backends"},
26+
# "postgresql.bgwriter.buffers.allocated": {"data_type": "Sum", "description": "Number of buffers allocated"},
27+
# "postgresql.bgwriter.buffers.writes": {"data_type": "Sum", "description": "Number of buffers written"},
28+
# "postgresql.bgwriter.checkpoint.count": {"data_type": "Sum", "description": "The number of checkpoints performed"},
29+
# "postgresql.bgwriter.duration": {
30+
# "data_type": "Sum",
31+
# "description": "Total time spent writing and syncing files to disk by checkpoints",
32+
# },
33+
# "postgresql.bgwriter.maxwritten": {
34+
# "data_type": "Sum",
35+
# "description": "Number of times the background writer stopped a cleaning scan because it had written too many buffers",
36+
# },
3737
# Optional metrics (enabled in otelcol-config-with-postgres.yaml)
38-
"postgresql.blks_hit": {
39-
"data_type": "Sum",
40-
"description": "Number of times disk blocks were found already in the buffer cache",
41-
},
42-
"postgresql.blks_read": {"data_type": "Sum", "description": "Number of disk blocks read in this database"},
38+
# "postgresql.blks_hit": {
39+
# "data_type": "Sum",
40+
# "description": "Number of times disk blocks were found already in the buffer cache",
41+
# },
42+
# "postgresql.blks_read": {"data_type": "Sum", "description": "Number of disk blocks read in this database"},
4343
"postgresql.database.locks": {"data_type": "Gauge", "description": "The number of database locks"},
4444
"postgresql.deadlocks": {"data_type": "Sum", "description": "The number of deadlocks"},
45-
"postgresql.temp.io": {
46-
"data_type": "Sum",
47-
"description": "Total amount of data written to temporary files by queries",
48-
},
45+
# "postgresql.temp.io": {
46+
# "data_type": "Sum",
47+
# "description": "Total amount of data written to temporary files by queries",
48+
# },
4949
"postgresql.temp_files": {"data_type": "Sum", "description": "The number of temp files"},
50-
"postgresql.tup_deleted": {"data_type": "Sum", "description": "Number of rows deleted by queries in the database"},
51-
"postgresql.tup_fetched": {"data_type": "Sum", "description": "Number of rows fetched by queries in the database"},
52-
"postgresql.tup_inserted": {
53-
"data_type": "Sum",
54-
"description": "Number of rows inserted by queries in the database",
55-
},
56-
"postgresql.tup_returned": {
57-
"data_type": "Sum",
58-
"description": "Number of rows returned by queries in the database",
59-
},
60-
"postgresql.tup_updated": {"data_type": "Sum", "description": "Number of rows updated by queries in the database"},
61-
"postgresql.function.calls": {
62-
"data_type": "Sum",
63-
"description": "The number of calls made to a function. Requires `track_functions=pl|all` in Postgres config.",
64-
},
50+
# "postgresql.tup_deleted": {"data_type": "Sum", "description": "Number of rows deleted by queries in the database"},
51+
# "postgresql.tup_fetched": {"data_type": "Sum", "description": "Number of rows fetched by queries in the database"},
52+
# "postgresql.tup_inserted": {
53+
# "data_type": "Sum",
54+
# "description": "Number of rows inserted by queries in the database",
55+
# },
56+
# "postgresql.tup_returned": {
57+
# "data_type": "Sum",
58+
# "description": "Number of rows returned by queries in the database",
59+
# },
60+
# "postgresql.tup_updated": {"data_type": "Sum", "description": "Number of rows updated by queries in the database"},
61+
# "postgresql.function.calls": {
62+
# "data_type": "Sum",
63+
# "description": "The number of calls made to a function. Requires `track_functions=pl|all` in Postgres config.",
64+
# },
6565
"postgresql.sequential_scans": {"data_type": "Sum", "description": "The number of sequential scans"},
6666
"postgresql.table.size": {"data_type": "Sum", "description": "Disk space used by a table."},
6767
"postgresql.rows": {"data_type": "Sum", "description": "The number of rows in the database"},

‎utils/_context/_scenarios/core.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ def _create_log_subfolder(self, subfolder: str, *, remove_if_exists: bool = Fals
129129
if remove_if_exists:
130130
shutil.rmtree(path, ignore_errors=True)
131131

132-
Path(path).mkdir(parents=True, exist_ok=True)
132+
Path(path).mkdir(mode=0o777, parents=True, exist_ok=True)
133133

134134
def __call__(self, test_object): # noqa: ANN001 (tes_object can be a class or a class method)
135135
"""Handles @scenarios.scenario_name"""

‎utils/_context/_scenarios/otel_collector.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,12 @@ def __init__(self, name: str, *, use_proxy: bool = True, mocked_backend: bool =
1919
name,
2020
github_workflow="endtoend",
2121
doc="TODO",
22-
scenario_groups=[scenario_groups.end_to_end],
22+
scenario_groups=[scenario_groups.end_to_end, scenario_groups.all],
2323
include_postgres_db=True,
2424
use_proxy=use_proxy,
2525
mocked_backend=mocked_backend,
2626
)
27+
2728
self.library = ComponentVersion("otel_collector", "0.0.0")
2829

2930
self.collector_container = OpenTelemetryCollectorContainer(

‎utils/_context/containers.py‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -139,8 +139,8 @@ def configure(self, *, host_log_folder: str, replay: bool):
139139
self.stop_previous_container()
140140
self._starting_lock = RLock()
141141

142-
Path(self.log_folder_path).mkdir(exist_ok=True, parents=True)
143-
Path(f"{self.log_folder_path}/logs").mkdir(exist_ok=True, parents=True)
142+
Path(self.log_folder_path).mkdir(mode=0o777, exist_ok=True, parents=True)
143+
Path(f"{self.log_folder_path}/logs").mkdir(mode=0o777, exist_ok=True, parents=True)
144144

145145
self.image.load()
146146
self.image.save_image_info(self.log_folder_path)
@@ -1279,14 +1279,15 @@ def __init__(
12791279
environment=environment,
12801280
volumes=volumes,
12811281
ports={"13133/tcp": ("0.0.0.0", 13133)}, # noqa: S104
1282+
user=f"{os.getuid()}:{os.getgid()}",
12821283
)
12831284

12841285
def configure(self, *, host_log_folder: str, replay: bool) -> None:
1285-
self.volumes[f"./{host_log_folder}/docker/collector/logs"] = {"bind": "/var/log/system-tests", "mode": "rw"}
1286-
self.volumes[self.config_file] = {"bind": "/etc/otelcol-config.yml", "mode": "ro"}
1287-
12881286
super().configure(host_log_folder=host_log_folder, replay=replay)
12891287

1288+
self.volumes[f"{self.log_folder_path}/logs"] = {"bind": "/var/log/system-tests", "mode": "rw"}
1289+
self.volumes[self.config_file] = {"bind": "/etc/otelcol-config.yml", "mode": "ro"}
1290+
12901291
# Override wait_for_health because we cannot do docker exec for container opentelemetry-collector-contrib
12911292
def wait_for_health(self) -> bool:
12921293
time.sleep(20) # It takes long for otel collector to start
@@ -1312,6 +1313,7 @@ def start(self, network: Network) -> Container:
13121313
new_mode = prev_mode | stat.S_IROTH
13131314
if prev_mode != new_mode:
13141315
Path(self.config_file).chmod(new_mode)
1316+
13151317
return super().start(network)
13161318

13171319

‎utils/build/docker/otelcol-config-with-postgres.yaml‎

Lines changed: 28 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -13,43 +13,43 @@ receivers:
1313
collection_interval: 10s
1414
tls:
1515
insecure: true
16-
events: # from the opentelemetry-collector-contrib/receiver/postgresqlreceiver/README.md
17-
db.server.query_sample:
18-
enabled: true
19-
db.server.top_query:
20-
enabled: true
21-
query_sample_collection:
22-
max_rows_per_query: 100
23-
top_query_collection:
24-
max_rows_per_query: 100
25-
top_n_query: 100
16+
# events: # from the opentelemetry-collector-contrib/receiver/postgresqlreceiver/README.md
17+
# db.server.query_sample:
18+
# enabled: true
19+
# db.server.top_query:
20+
# enabled: true
21+
# # query_sample_collection:
22+
# # max_rows_per_query: 100
23+
# # top_query_collection:
24+
# # max_rows_per_query: 100
25+
# # top_n_query: 100
2626
metrics:
27-
postgresql.blks_read:
28-
enabled: true
29-
postgresql.blks_hit:
30-
enabled: true
27+
# postgresql.blks_read:
28+
# enabled: true
29+
# postgresql.blks_hit:
30+
# enabled: true
3131
postgresql.database.locks:
3232
enabled: true
3333
postgresql.deadlocks:
3434
enabled: true
35-
postgresql.function.calls:
36-
enabled: true
35+
# postgresql.function.calls:
36+
# enabled: true
3737
postgresql.sequential_scans:
3838
enabled: true
39-
postgresql.temp.io:
40-
enabled: true
39+
# postgresql.temp.io:
40+
# enabled: true
4141
postgresql.temp_files:
4242
enabled: true
43-
postgresql.tup_deleted:
44-
enabled: true
45-
postgresql.tup_fetched:
46-
enabled: true
47-
postgresql.tup_inserted:
48-
enabled: true
49-
postgresql.tup_returned:
50-
enabled: true
51-
postgresql.tup_updated:
52-
enabled: true
43+
# postgresql.tup_deleted:
44+
# enabled: true
45+
# postgresql.tup_fetched:
46+
# enabled: true
47+
# postgresql.tup_inserted:
48+
# enabled: true
49+
# postgresql.tup_returned:
50+
# enabled: true
51+
# postgresql.tup_updated:
52+
# enabled: true
5353
postgresql.wal.delay:
5454
enabled: true
5555
postgresql.wal.age:

0 commit comments

Comments
 (0)