generated from amazon-archives/__template_Custom
-
Notifications
You must be signed in to change notification settings - Fork 176
Open
Labels
PPLPiped processing languagePiped processing languagebugSomething isn't workingSomething isn't working
Description
Query Information
PPL Command/Query:
source=opensearch-sql_test_index_otel_logs | timechart limit=1 span=1d max(severityNumber) by severityText
Expected Result:
{
"schema": [
{
"name": "@timestamp",
"type": "timestamp"
},
{
"name": "severityText",
"type": "string"
},
{
"name": "max(severityNumber)",
"type": "bigint"
}
],
"datarows": [
[
"2024-01-15 00:00:00",
"FATAL4",
24
],
[
"2024-01-15 00:00:00",
"OTHER",
23
]
],
"total": 2,
"size": 2
}Actual Result:
{
"schema": [
{
"name": "@timestamp",
"type": "timestamp"
},
{
"name": "severityText",
"type": "string"
},
{
"name": "max(severityNumber)",
"type": "bigint"
}
],
"datarows": [
[
"2024-01-15 00:00:00",
"FATAL4",
24
],
[
"2024-01-15 00:00:00",
"OTHER",
276
]
],
"total": 2,
"size": 2
}Dataset Information
Dataset/Schema Type
- OpenTelemetry (OTEL)
- Simple Schema for Observability (SS4O)
- Open Cybersecurity Schema Framework (OCSF)
- Custom (details below)
Index Mapping
{"mappings":{"dynamic_templates":[{"long_resource_attributes":{"path_match":"resource.attributes.*","match_mapping_type":"long","mapping":{"type":"long"}}},{"double_resource_attributes":{"path_match":"resource.attributes.*","match_mapping_type":"double","mapping":{"type":"double"}}},{"string_resource_attributes":{"path_match":"resource.attributes.*","match_mapping_type":"string","mapping":{"type":"keyword"}}},{"long_attributes":{"path_match":"attributes.*","match_mapping_type":"long","mapping":{"type":"long"}}},{"double_attributes":{"path_match":"attributes.*","match_mapping_type":"double","mapping":{"type":"double"}}},{"string_attributes":{"path_match":"attributes.*","match_mapping_type":"string","mapping":{"type":"keyword"}}}],"date_detection":false,"properties":{"@timestamp":{"type":"date_nanos"},"time":{"type":"date_nanos"},"body":{"type":"text"},"severityNumber":{"type":"long"},"severityText":{"type":"keyword"},"traceId":{"type":"keyword"},"spanId":{"type":"keyword"},"flags":{"type":"long"},"droppedAttributesCount":{"type":"integer"},"instrumentationScope":{"properties":{"name":{"type":"keyword"},"version":{"type":"keyword"},"droppedAttributesCount":{"type":"integer"}}},"resource":{"properties":{"attributes":{"properties":{"service.name":{"type":"keyword"},"service.namespace":{"type":"keyword"},"service.version":{"type":"keyword"},"host.name":{"type":"keyword"},"container.id":{"type":"keyword"},"k8s.namespace":{"type":"keyword"},"k8s.pod.name":{"type":"keyword"}}},"droppedAttributesCount":{"type":"integer"}}},"attributes":{"properties":{"user.id":{"type":"keyword"},"user.email":{"type":"keyword"},"product.id":{"type":"keyword"},"order.id":{"type":"keyword"},"error.code":{"type":"keyword"},"error.type":{"type":"keyword"},"error.message":{"type":"text"},"http.method":{"type":"keyword"},"http.status_code":{"type":"long"},"http.url":{"type":"keyword"},"client.ip":{"type":"ip","fields":{"keyword":{"type":"keyword"}}},"db.statement":{"type":"text"},"db.operation":{"type":"keyword"},"db.table":{"type":"keyword"},"email.to":{"type":"keyword"},"email.subject":{"type":"text"},"query.raw":{"type":"text"},"query.type":{"type":"keyword"},"security.threat":{"type":"keyword"},"payment.amount":{"type":"double"},"quantity":{"type":"long"},"kafka.topic":{"type":"keyword"},"kafka.partition":{"type":"long"},"kafka.offset":{"type":"long"},"redis.command":{"type":"keyword"},"redis.key":{"type":"keyword"},"redis.ttl":{"type":"long"},"jwt.algorithm":{"type":"keyword"},"jwt.issuer":{"type":"keyword"},"rate_limit.current":{"type":"long"},"rate_limit.max":{"type":"long"},"graphql.operation":{"type":"keyword"},"graphql.field":{"type":"keyword"},"webhook.url":{"type":"keyword"},"webhook.status":{"type":"keyword"},"mongodb.collection":{"type":"keyword"},"mongodb.operation":{"type":"keyword"},"mongodb.filter":{"type":"text"},"elasticsearch.query":{"type":"text"},"grpc.method":{"type":"keyword"},"grpc.status_code":{"type":"long"},"ssl.domain":{"type":"keyword"},"ssl.days_until_expiry":{"type":"long"},"memory.usage_percent":{"type":"double"},"memory.total_gb":{"type":"long"},"batch.total_records":{"type":"long"},"batch.matched_records":{"type":"long"},"k8s.pod.name":{"type":"keyword"},"k8s.container.restart_count":{"type":"long"},"health.status":{"type":"keyword"},"cors.origin":{"type":"keyword"},"cors.method":{"type":"keyword"},"duration_ms":{"type":"long"}}}}}}Sample Data
{"@timestamp": "2024-01-15T10:30:04.567890123Z", "time": "2024-01-15T10:30:04.567890123Z", "severityNumber": 9, "severityText": "INFO", "body": "Email notification sent to [email protected] with subject: 'Welcome! Your order #12345 is confirmed'", "traceId": "", "spanId": "", "flags": 0, "resource": {"attributes": {"service.name": "notification-service"}}, "attributes": {"email.to": "[email protected]", "email.subject": "Welcome! Your order #12345 is confirmed", "order.id": "12345"}}Bug Description
Issue Summary:
When the limit parameter presents, timechart categorize the categories that exceed the limit to an OTHER category. The max(severityNumber) field of all the other categories should be 23 instead of 276.
The error may result from a wrong assumption of timechart's imlementation: all aggregations are accumulative. It seems that all the max(severityNumber) of the rest groups are summed together to get the 276 number. However, it should take the maximum instead of the sum.
Environment Information
OpenSearch Version:
3.3.0
Metadata
Metadata
Assignees
Labels
PPLPiped processing languagePiped processing languagebugSomething isn't workingSomething isn't working
Type
Projects
Status
In progress