diff --git a/packages/kubernetes/changelog.yml b/packages/kubernetes/changelog.yml index 34d469051ae..fb1979cc28d 100644 --- a/packages/kubernetes/changelog.yml +++ b/packages/kubernetes/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.81.1" + changes: + - description: Fix processing of Azure AKS audit logs. + type: bugfix + link: https://github.com/elastic/integrations/pull/15585 - version: "1.81.0" changes: - description: Support for collecting audit logs from cloud providers. diff --git a/packages/kubernetes/data_stream/audit_logs/_dev/test/pipeline/test-audit.log b/packages/kubernetes/data_stream/audit_logs/_dev/test/pipeline/test-audit.log index c8206b49fec..381559aafdc 100644 --- a/packages/kubernetes/data_stream/audit_logs/_dev/test/pipeline/test-audit.log +++ b/packages/kubernetes/data_stream/audit_logs/_dev/test/pipeline/test-audit.log @@ -1,4 +1,5 @@ {"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"abcde12345","stage":"ResponseComplete","requestURI":"/api/v1/namespaces/default/pods","verb":"get","user":{"username":"system:serviceaccounts:default:default","uid":"12345678","groups":["system:authenticated"]},"sourceIPs":["67.43.156.1"],"userAgent":"kubectl/v1.26.1","objectRef":{"resource":"pods","namespace":"default","name":"my-pod","apiGroup":"","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":200},"requestReceivedTimestamp":"2025-03-04T06:22:18.819232Z","stageTimestamp":"2025-03-04T06:22:18.822532Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":""}} {"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"abcde12345","stage":"ResponseComplete","requestURI":"/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/elastic-agent-cluster-test","verb":"get","user":{"username":"system:serviceaccount:kube-system:elastic-agent","uid":"12345678","groups":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"],"extra":{}},"sourceIPs":["67.43.156.1"],"userAgent":"elastic-agent/v0.0.0 (linux/amd64) kubernetes/$Format","objectRef":{"resource":"leases","namespace":"kube-system","name":"elastic-agent-cluster-test","apiGroup":"coordination.k8s.io","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":200},"requestReceivedTimestamp":"2025-07-16T10:12:56.525137Z","stageTimestamp":"2025-07-16T10:12:56.563177Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by RoleBinding \"elastic-agent/kube-system\" of Role \"elastic-agent\" to ServiceAccount \"elastic-agent/kube-system\""}} {"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"abcde12345","stage":"ResponseComplete","requestURI":"/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/elastic-agent-cluster-test","verb":"get","user":{"username":"system:serviceaccount:kube-system:elastic-agent","uid":"12345678","groups":["system:serviceaccounts","system:serviceaccounts:kube-system","system:authenticated"],"extra":{}},"sourceIPs":["67.43.156.1"],"userAgent":"elastic-agent/v0.0.0 (linux/amd64) kubernetes/$Format","objectRef":{"resource":"leases","namespace":"kube-system","name":"elastic-agent-cluster-test","apiGroup":"coordination.k8s.io","apiVersion":"v1"},"responseStatus":{"metadata":{},"code":200},"requestReceivedTimestamp":"2025-07-16T10:12:56.525137Z","stageTimestamp":"2025-07-16T10:12:56.563177Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by RoleBinding \"elastic-agent/kube-system\" of Role \"elastic-agent\" to ServiceAccount \"elastic-agent/kube-system\""}} -{"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"system:gke-master-healthcheck"},"authorizationInfo":[{"granted":true,"permission":"io.k8s.get","resource":"readyz"}],"methodName":"io.k8s.get","requestMetadata":{"callerIp":"67.43.156.1","callerSuppliedUserAgent":"gke-master-healthcheck"},"resourceName":"readyz","serviceName":"k8s.io","status":{"code":0}},"insertId":"1234abcd","resource":{"type":"k8s_cluster","labels":{"cluster_name":"test-cluster","location":"us-central1","project_id":"elastic-siem"}},"timestamp":"2025-07-13T08:38:39.127266Z","labels":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"system:public-info-viewer\" of ClusterRole \"system:public-info-viewer\" to Group \"system:authenticated\""},"logName":"projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fdata_access","operation":{"id":"1234abcd","producer":"k8s.io","first":true,"last":true},"receiveTimestamp":"2025-07-13T08:38:41.005864307Z"} \ No newline at end of file +{"protoPayload":{"@type":"type.googleapis.com/google.cloud.audit.AuditLog","authenticationInfo":{"principalEmail":"system:gke-master-healthcheck"},"authorizationInfo":[{"granted":true,"permission":"io.k8s.get","resource":"readyz"}],"methodName":"io.k8s.get","requestMetadata":{"callerIp":"67.43.156.1","callerSuppliedUserAgent":"gke-master-healthcheck"},"resourceName":"readyz","serviceName":"k8s.io","status":{"code":0}},"insertId":"1234abcd","resource":{"type":"k8s_cluster","labels":{"cluster_name":"test-cluster","location":"us-central1","project_id":"elastic-siem"}},"timestamp":"2025-07-13T08:38:39.127266Z","labels":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":"RBAC: allowed by ClusterRoleBinding \"system:public-info-viewer\" of ClusterRole \"system:public-info-viewer\" to Group \"system:authenticated\""},"logName":"projects/elastic-siem/logs/cloudaudit.googleapis.com%2Fdata_access","operation":{"id":"1234abcd","producer":"k8s.io","first":true,"last":true},"receiveTimestamp":"2025-07-13T08:38:41.005864307Z"} +{"category":"kube-audit-admin","operationName":"Microsoft.ContainerService/managedClusters/diagnosticLogs/Read","properties":{"containerID":"aaaa1111","log":{"kind":"Event","apiVersion":"audit.k8s.io/v1","level":"Metadata","auditID":"abcde12345","stage":"ResponseComplete","requestURI":"/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/elastic-agent-cluster-test","verb":"update","user":{"username":"aksService","groups":["system:masters","system:authenticated"],"extra":{}},"sourceIPs":["67.43.156.1"],"userAgent":"elastic-agent/v0.0.0 (linux/amd64) kubernetes/$Format","objectRef":{"resource":"leases","namespace":"kube-system","name":"elastic-agent-cluster-test","uid":"12345678","apiGroup":"coordination.k8s.io","apiVersion":"v1","resourceVersion":"12345678"},"responseStatus":{"metadata":{},"code":200},"requestReceivedTimestamp":"2025-09-30T06:23:35.091134Z","stageTimestamp":"2025-09-30T06:23:35.101182Z","annotations":{"authorization.k8s.io/decision":"allow","authorization.k8s.io/reason":""}},"pod":"kube-apiserver-1234567890","stream":"stdout"},"resourceId":"/SUBSCRIPTIONS/1234567890/RESOURCEGROUPS/TEST-RG/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/TEST-AKS","serviceBuild":"na","time":"2025-09-30T06:23:35.101355367Z"} diff --git a/packages/kubernetes/data_stream/audit_logs/_dev/test/pipeline/test-audit.log-expected.json b/packages/kubernetes/data_stream/audit_logs/_dev/test/pipeline/test-audit.log-expected.json index a5283d0b262..a6a1ea03ffe 100644 --- a/packages/kubernetes/data_stream/audit_logs/_dev/test/pipeline/test-audit.log-expected.json +++ b/packages/kubernetes/data_stream/audit_logs/_dev/test/pipeline/test-audit.log-expected.json @@ -323,6 +323,100 @@ "tags": [ "preserve_original_event" ] + }, + { + "@timestamp": "2025-09-30T06:23:35.101Z", + "client": { + "ip": [ + "67.43.156.1" + ] + }, + "event": { + "action": "update", + "kind": "event", + "original": "{\"category\":\"kube-audit-admin\",\"operationName\":\"Microsoft.ContainerService/managedClusters/diagnosticLogs/Read\",\"properties\":{\"containerID\":\"aaaa1111\",\"log\":{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"abcde12345\",\"stage\":\"ResponseComplete\",\"requestURI\":\"/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/elastic-agent-cluster-test\",\"verb\":\"update\",\"user\":{\"username\":\"aksService\",\"groups\":[\"system:masters\",\"system:authenticated\"],\"extra\":{}},\"sourceIPs\":[\"67.43.156.1\"],\"userAgent\":\"elastic-agent/v0.0.0 (linux/amd64) kubernetes/$Format\",\"objectRef\":{\"resource\":\"leases\",\"namespace\":\"kube-system\",\"name\":\"elastic-agent-cluster-test\",\"uid\":\"12345678\",\"apiGroup\":\"coordination.k8s.io\",\"apiVersion\":\"v1\",\"resourceVersion\":\"12345678\"},\"responseStatus\":{\"metadata\":{},\"code\":200},\"requestReceivedTimestamp\":\"2025-09-30T06:23:35.091134Z\",\"stageTimestamp\":\"2025-09-30T06:23:35.101182Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"\"}},\"pod\":\"kube-apiserver-1234567890\",\"stream\":\"stdout\"},\"resourceId\":\"/SUBSCRIPTIONS/1234567890/RESOURCEGROUPS/TEST-RG/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/TEST-AKS\",\"serviceBuild\":\"na\",\"time\":\"2025-09-30T06:23:35.101355367Z\"}", + "outcome": "success" + }, + "kubernetes": { + "audit": { + "aks_metadata": { + "category": "kube-audit-admin", + "container_id": "aaaa1111", + "operation_name": "Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", + "pod": "kube-apiserver-1234567890", + "resource_id": "/SUBSCRIPTIONS/1234567890/RESOURCEGROUPS/TEST-RG/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/TEST-AKS", + "service_build": "na", + "stream": "stdout", + "time": "2025-09-30T06:23:35.101355367Z" + }, + "annotations": { + "authorization_k8s_io/decision": "allow" + }, + "apiVersion": "audit.k8s.io/v1", + "auditID": "abcde12345", + "kind": "Event", + "level": "Metadata", + "objectRef": { + "apiGroup": "coordination.k8s.io", + "apiVersion": "v1", + "name": "elastic-agent-cluster-test", + "namespace": "kube-system", + "resource": "leases", + "resourceVersion": "12345678", + "uid": "12345678" + }, + "requestReceivedTimestamp": "2025-09-30T06:23:35.091134Z", + "requestURI": "/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/elastic-agent-cluster-test", + "responseStatus": { + "code": 200 + }, + "sourceIPs": [ + "67.43.156.1" + ], + "stage": "ResponseComplete", + "stageTimestamp": "2025-09-30T06:23:35.101182Z", + "user": { + "groups": [ + "system:masters", + "system:authenticated" + ], + "username": "aksService" + }, + "userAgent": "elastic-agent/v0.0.0 (linux/amd64) kubernetes/$Format", + "verb": "update" + } + }, + "orchestrator": { + "api_version": "audit.k8s.io/v1", + "namespace": "kube-system", + "resource": { + "name": "elastic-agent-cluster-test", + "type": "leases" + }, + "type": "kubernetes" + }, + "related": { + "ip": [ + "67.43.156.1" + ], + "user": [ + "aksService" + ] + }, + "source": { + "ip": [ + "67.43.156.1" + ] + }, + "tags": [ + "preserve_original_event" + ], + "user": { + "name": "aksService" + }, + "user_agent": { + "original": "elastic-agent/v0.0.0 (linux/amd64) kubernetes/$Format" + } } ] } diff --git a/packages/kubernetes/data_stream/audit_logs/_dev/test/pipeline/test-kube-audit.log b/packages/kubernetes/data_stream/audit_logs/_dev/test/pipeline/test-kube-audit.log new file mode 100644 index 00000000000..f81303fc916 --- /dev/null +++ b/packages/kubernetes/data_stream/audit_logs/_dev/test/pipeline/test-kube-audit.log @@ -0,0 +1 @@ +{"category":"kube-audit","operationName":"Microsoft.ContainerService/managedClusters/diagnosticLogs/Read","properties":{"log":"{\"kind\":\"Event\",\"apiVersion\":\"audit.k8s.io/v1\",\"level\":\"Metadata\",\"auditID\":\"ad26f7bc-f1c6-4097-90f1-e0924e12f257\",\"stage\":\"ResponseComplete\",\"requestURI\":\"/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/kubelet-serving-csr-approver\",\"verb\":\"update\",\"user\":{\"username\":\"aksService\",\"groups\":[\"system:masters\",\"system:authenticated\"]},\"sourceIPs\":[\"172.31.51.172\"],\"userAgent\":\"approver/v0.0.0 (linux/amd64) kubernetes/$Format/leader-election\",\"objectRef\":{\"resource\":\"leases\",\"namespace\":\"kube-system\",\"name\":\"kubelet-serving-csr-approver\",\"uid\":\"14be99f8-ebb7-47b9-a194-43e63d9386af\",\"apiGroup\":\"coordination.k8s.io\",\"apiVersion\":\"v1\",\"resourceVersion\":\"93076016\"},\"responseStatus\":{\"metadata\":{},\"code\":200},\"requestReceivedTimestamp\":\"2025-10-07T16:19:19.643609Z\",\"stageTimestamp\":\"2025-10-07T16:19:19.647762Z\",\"annotations\":{\"authorization.k8s.io/decision\":\"allow\",\"authorization.k8s.io/reason\":\"\"}}","containerID":"a64cba7fefbf5020788dc29d9247157585bbb64826bf7209623ca7bb49b15fe7","stream":"stdout","pod":"kube-apiserver-869d7bb754-kkg69"},"resourceId":"/SUBSCRIPTIONS/ae2861b3-e901-49bd-99f3-660eb5747107/RESOURCEGROUPS/TEST/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/TEST-CLUSTER","serviceBuild":"na","time":"2025-10-07T16:19:19.647880072Z"} diff --git a/packages/kubernetes/data_stream/audit_logs/_dev/test/pipeline/test-kube-audit.log-expected.json b/packages/kubernetes/data_stream/audit_logs/_dev/test/pipeline/test-kube-audit.log-expected.json new file mode 100644 index 00000000000..5121d104d03 --- /dev/null +++ b/packages/kubernetes/data_stream/audit_logs/_dev/test/pipeline/test-kube-audit.log-expected.json @@ -0,0 +1,95 @@ +{ + "expected": [ + { + "@timestamp": "2025-10-07T16:19:19.647Z", + "client": { + "ip": [ + "172.31.51.172" + ] + }, + "event": { + "action": "update", + "kind": "event", + "original": "{\"category\":\"kube-audit\",\"operationName\":\"Microsoft.ContainerService/managedClusters/diagnosticLogs/Read\",\"properties\":{\"log\":\"{\\\"kind\\\":\\\"Event\\\",\\\"apiVersion\\\":\\\"audit.k8s.io/v1\\\",\\\"level\\\":\\\"Metadata\\\",\\\"auditID\\\":\\\"ad26f7bc-f1c6-4097-90f1-e0924e12f257\\\",\\\"stage\\\":\\\"ResponseComplete\\\",\\\"requestURI\\\":\\\"/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/kubelet-serving-csr-approver\\\",\\\"verb\\\":\\\"update\\\",\\\"user\\\":{\\\"username\\\":\\\"aksService\\\",\\\"groups\\\":[\\\"system:masters\\\",\\\"system:authenticated\\\"]},\\\"sourceIPs\\\":[\\\"172.31.51.172\\\"],\\\"userAgent\\\":\\\"approver/v0.0.0 (linux/amd64) kubernetes/$Format/leader-election\\\",\\\"objectRef\\\":{\\\"resource\\\":\\\"leases\\\",\\\"namespace\\\":\\\"kube-system\\\",\\\"name\\\":\\\"kubelet-serving-csr-approver\\\",\\\"uid\\\":\\\"14be99f8-ebb7-47b9-a194-43e63d9386af\\\",\\\"apiGroup\\\":\\\"coordination.k8s.io\\\",\\\"apiVersion\\\":\\\"v1\\\",\\\"resourceVersion\\\":\\\"93076016\\\"},\\\"responseStatus\\\":{\\\"metadata\\\":{},\\\"code\\\":200},\\\"requestReceivedTimestamp\\\":\\\"2025-10-07T16:19:19.643609Z\\\",\\\"stageTimestamp\\\":\\\"2025-10-07T16:19:19.647762Z\\\",\\\"annotations\\\":{\\\"authorization.k8s.io/decision\\\":\\\"allow\\\",\\\"authorization.k8s.io/reason\\\":\\\"\\\"}}\",\"containerID\":\"a64cba7fefbf5020788dc29d9247157585bbb64826bf7209623ca7bb49b15fe7\",\"stream\":\"stdout\",\"pod\":\"kube-apiserver-869d7bb754-kkg69\"},\"resourceId\":\"/SUBSCRIPTIONS/ae2861b3-e901-49bd-99f3-660eb5747107/RESOURCEGROUPS/TEST/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/TEST-CLUSTER\",\"serviceBuild\":\"na\",\"time\":\"2025-10-07T16:19:19.647880072Z\"}", + "outcome": "success" + }, + "kubernetes": { + "audit": { + "aks_metadata": { + "category": "kube-audit", + "container_id": "a64cba7fefbf5020788dc29d9247157585bbb64826bf7209623ca7bb49b15fe7", + "operation_name": "Microsoft.ContainerService/managedClusters/diagnosticLogs/Read", + "pod": "kube-apiserver-869d7bb754-kkg69", + "resource_id": "/SUBSCRIPTIONS/ae2861b3-e901-49bd-99f3-660eb5747107/RESOURCEGROUPS/TEST/PROVIDERS/MICROSOFT.CONTAINERSERVICE/MANAGEDCLUSTERS/TEST-CLUSTER", + "service_build": "na", + "stream": "stdout", + "time": "2025-10-07T16:19:19.647880072Z" + }, + "annotations": { + "authorization_k8s_io/decision": "allow" + }, + "apiVersion": "audit.k8s.io/v1", + "auditID": "ad26f7bc-f1c6-4097-90f1-e0924e12f257", + "kind": "Event", + "level": "Metadata", + "objectRef": { + "apiGroup": "coordination.k8s.io", + "apiVersion": "v1", + "name": "kubelet-serving-csr-approver", + "namespace": "kube-system", + "resource": "leases", + "resourceVersion": "93076016", + "uid": "14be99f8-ebb7-47b9-a194-43e63d9386af" + }, + "requestReceivedTimestamp": "2025-10-07T16:19:19.643609Z", + "requestURI": "/apis/coordination.k8s.io/v1/namespaces/kube-system/leases/kubelet-serving-csr-approver", + "responseStatus": { + "code": 200 + }, + "sourceIPs": [ + "172.31.51.172" + ], + "stage": "ResponseComplete", + "stageTimestamp": "2025-10-07T16:19:19.647762Z", + "user": { + "groups": [ + "system:masters", + "system:authenticated" + ], + "username": "aksService" + }, + "userAgent": "approver/v0.0.0 (linux/amd64) kubernetes/$Format/leader-election", + "verb": "update" + } + }, + "orchestrator": { + "api_version": "audit.k8s.io/v1", + "namespace": "kube-system", + "resource": { + "name": "kubelet-serving-csr-approver", + "type": "leases" + }, + "type": "kubernetes" + }, + "related": { + "ip": [ + "172.31.51.172" + ], + "user": [ + "aksService" + ] + }, + "source": { + "ip": [ + "172.31.51.172" + ] + }, + "user": { + "name": "aksService" + }, + "user_agent": { + "original": "approver/v0.0.0 (linux/amd64) kubernetes/$Format/leader-election" + } + } + ] +} diff --git a/packages/kubernetes/data_stream/audit_logs/elasticsearch/ingest_pipeline/default.yml b/packages/kubernetes/data_stream/audit_logs/elasticsearch/ingest_pipeline/default.yml index 3c74bb6bc77..78af39617a5 100644 --- a/packages/kubernetes/data_stream/audit_logs/elasticsearch/ingest_pipeline/default.yml +++ b/packages/kubernetes/data_stream/audit_logs/elasticsearch/ingest_pipeline/default.yml @@ -14,6 +14,73 @@ processors: field: message if: ctx.event?.original != null ignore_missing: true + # For Azure AKS logs + - rename: + field: kubernetes.audit + target_field: tmp.aks_audit + ignore_missing: true + if: > + ctx.kubernetes?.audit.category != null && + ctx.kubernetes?.audit.operationName != null && + ctx.kubernetes?.audit.resourceId != null && + ctx.kubernetes?.audit.time != null + tag: rename_azure_aks_audit + - json: + field: tmp.aks_audit.properties.log + target_field: kubernetes.audit + if: ctx.tmp?.aks_audit?.properties?.log instanceof String && ctx.kubernetes?.audit == null + tag: json_azure_aks_audit_log + - rename: + field: tmp.aks_audit.properties.log + target_field: kubernetes.audit + if: ctx.tmp?.aks_audit?.properties?.log != null && ctx.kubernetes?.audit == null + tag: rename_azure_aks_audit_log + - remove: + field: tmp.aks_audit.properties.log + ignore_missing: true + - rename: + field: tmp.aks_audit + target_field: kubernetes.audit.aks_metadata + ignore_missing: true + tag: rename_azure_aks_metadata + - date: + field: kubernetes.audit.aks_metadata.time + if: ctx.kubernetes?.audit?.aks_metadata?.time != null + tag: date_azure_aks_time + formats: + - ISO8601 + - rename: + field: kubernetes.audit.aks_metadata.operationName + target_field: kubernetes.audit.aks_metadata.operation_name + ignore_missing: true + tag: rename_aks_metadata_operation_name + - rename: + field: kubernetes.audit.aks_metadata.resourceId + target_field: kubernetes.audit.aks_metadata.resource_id + ignore_missing: true + tag: rename_aks_metadata_resource_id + - rename: + field: kubernetes.audit.aks_metadata.serviceBuild + target_field: kubernetes.audit.aks_metadata.service_build + ignore_missing: true + tag: rename_aks_metadata_service_build + - rename: + field: kubernetes.audit.aks_metadata.properties.containerID + target_field: kubernetes.audit.aks_metadata.container_id + ignore_missing: true + tag: rename_aks_metadata_container_id + - rename: + field: kubernetes.audit.aks_metadata.properties.pod + target_field: kubernetes.audit.aks_metadata.pod + ignore_missing: true + tag: rename_aks_metadata_pod + - rename: + field: kubernetes.audit.aks_metadata.properties.stream + target_field: kubernetes.audit.aks_metadata.stream + ignore_missing: true + tag: rename_aks_metadata_stream + + # General processors - remove: if: "ctx.kubernetes?.audit?.responseObject != null" field: ["kubernetes.audit.responseObject.metadata"] diff --git a/packages/kubernetes/data_stream/audit_logs/fields/azure.yml b/packages/kubernetes/data_stream/audit_logs/fields/azure.yml new file mode 100644 index 00000000000..fcf26d0146e --- /dev/null +++ b/packages/kubernetes/data_stream/audit_logs/fields/azure.yml @@ -0,0 +1,27 @@ +- name: azure + type: group + fields: + - name: consumer_group + type: keyword + description: | + Consumer group. + - name: enqueued_time + type: keyword + description: | + The enqueued time. + - name: eventhub + type: keyword + description: | + Event hub name. + - name: offset + type: long + description: | + Offset. + - name: partition_id + type: keyword + description: | + Partition ID. + - name: sequence_number + type: long + description: |- + Sequence number. \ No newline at end of file diff --git a/packages/kubernetes/data_stream/audit_logs/fields/ecs.yml b/packages/kubernetes/data_stream/audit_logs/fields/ecs.yml index cab6764ae11..5851f98316f 100644 --- a/packages/kubernetes/data_stream/audit_logs/fields/ecs.yml +++ b/packages/kubernetes/data_stream/audit_logs/fields/ecs.yml @@ -4,6 +4,8 @@ name: error.message - external: ecs name: event.ingested +- external: ecs + name: event.original - external: ecs name: agent.name - external: ecs @@ -32,7 +34,23 @@ name: event.action - external: ecs name: event.outcome +- external: ecs + name: orchestrator.api_version +- external: ecs + name: orchestrator.cluster.id +- external: ecs + name: orchestrator.cluster.name +- external: ecs + name: orchestrator.namespace +- external: ecs + name: orchestrator.resource.name +- external: ecs + name: orchestrator.resource.type +- external: ecs + name: orchestrator.type - external: ecs name: related.ip - external: ecs name: related.user +- external: ecs + name: tags diff --git a/packages/kubernetes/data_stream/audit_logs/fields/fields.yml b/packages/kubernetes/data_stream/audit_logs/fields/fields.yml index 56c61423132..83f17bc6e2c 100644 --- a/packages/kubernetes/data_stream/audit_logs/fields/fields.yml +++ b/packages/kubernetes/data_stream/audit_logs/fields/fields.yml @@ -1,6 +1,32 @@ - name: kubernetes.audit type: group fields: + - name: aks_metadata + type: group + fields: + - name: category + description: The log category of the event being logged. Only for Azure AKS logs. + type: keyword + - name: container_id + description: The ID of the container that the event is logging. Only for Azure AKS logs. + type: keyword + - name: operation_name + description: The name of the operation that the event is logging. Only for Azure AKS logs. + type: keyword + - name: pod + description: The name of the pod that emitted the event. Only for Azure AKS logs. + type: keyword + - name: resource_id + description: The resource ID of the resource that emitted the event. Only for Azure AKS logs. + type: keyword + - name: service_build + description: The build of the service that the event is logging. Only for Azure AKS logs. + type: keyword + - name: stream + type: keyword + - name: time + description: The timestamp (UTC) of the event being logged. Only for Azure AKS logs. + type: date - name: apiVersion type: keyword description: Audit event api version diff --git a/packages/kubernetes/docs/audit-logs.md b/packages/kubernetes/docs/audit-logs.md index 687b82ee0bb..e7c6b3126d8 100644 --- a/packages/kubernetes/docs/audit-logs.md +++ b/packages/kubernetes/docs/audit-logs.md @@ -96,6 +96,12 @@ An example event for `audit` looks as following: | agent.name | Custom name of the agent. This is a name that can be given to an agent. This can be helpful if for example two Filebeat instances are running on the same host but a human readable separation is needed on which Filebeat instance data is coming from. | keyword | | agent.type | Type of the agent. The agent type always stays the same and should be given by the agent used. In case of Filebeat the agent would always be Filebeat also if two Filebeat instances are run on the same machine. | keyword | | agent.version | Version of the agent. | keyword | +| azure.consumer_group | Consumer group. | keyword | +| azure.enqueued_time | The enqueued time. | keyword | +| azure.eventhub | Event hub name. | keyword | +| azure.offset | Offset. | long | +| azure.partition_id | Partition ID. | keyword | +| azure.sequence_number | Sequence number. | long | | client.ip | IP address of the client (IPv4 or IPv6). | ip | | cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | | cloud.availability_zone | Availability zone in which this host, resource, or service is located. | keyword | @@ -115,6 +121,7 @@ An example event for `audit` looks as following: | event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | | event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | +| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | | event.outcome | This is one of four ECS Categorization Fields, and indicates the lowest level in the ECS category hierarchy. `event.outcome` simply denotes whether the event represents a success or a failure from the perspective of the entity that produced the event. Note that when a single transaction is described in multiple events, each event may populate different values of `event.outcome`, according to their perspective. Also note that in the case of a compound event (a single event that contains multiple logical events), this field should be populated with the value that best captures the overall success or failure from the perspective of the event producer. Further note that not all events will have an associated outcome. For example, this field is generally not populated for metric events, events with `event.type:info`, or any events for which an outcome does not make logical sense. | keyword | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | @@ -134,6 +141,14 @@ An example event for `audit` looks as following: | host.os.version | Operating system version as a raw string. | keyword | | host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | | input.type | Type of input. | keyword | +| kubernetes.audit.aks_metadata.category | The log category of the event being logged. Only for Azure AKS logs. | keyword | +| kubernetes.audit.aks_metadata.container_id | The ID of the container that the event is logging. Only for Azure AKS logs. | keyword | +| kubernetes.audit.aks_metadata.operation_name | The name of the operation that the event is logging. Only for Azure AKS logs. | keyword | +| kubernetes.audit.aks_metadata.pod | The name of the pod that emitted the event. Only for Azure AKS logs. | keyword | +| kubernetes.audit.aks_metadata.resource_id | The resource ID of the resource that emitted the event. Only for Azure AKS logs. | keyword | +| kubernetes.audit.aks_metadata.service_build | The build of the service that the event is logging. Only for Azure AKS logs. | keyword | +| kubernetes.audit.aks_metadata.stream | | keyword | +| kubernetes.audit.aks_metadata.time | The timestamp (UTC) of the event being logged. Only for Azure AKS logs. | date | | kubernetes.audit.annotations.authorization_k8s_io/decision | | keyword | | kubernetes.audit.annotations.authorization_k8s_io/reason | | text | | kubernetes.audit.annotations.pod-security_kubernetes_io/audit-violations | | text | @@ -231,9 +246,17 @@ An example event for `audit` looks as following: | log.file.vol | The serial number of the volume that contains a file. (Windows-only) | keyword | | log.offset | Offset of the entry in the log file. | long | | message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | +| orchestrator.api_version | API version being used to carry out the action | keyword | +| orchestrator.cluster.id | Unique ID of the cluster. | keyword | +| orchestrator.cluster.name | Name of the cluster. | keyword | +| orchestrator.namespace | Namespace in which the action is taking place. | keyword | +| orchestrator.resource.name | Name of the resource being acted upon. | keyword | +| orchestrator.resource.type | Type of resource being acted upon. | keyword | +| orchestrator.type | Orchestrator cluster type (e.g. kubernetes, nomad or cloudfoundry). | keyword | | related.ip | All of the IPs seen on your event. | ip | | related.user | All the user names or other user identifiers seen on the event. | keyword | | source.ip | IP address of the source (IPv4 or IPv6). | ip | +| tags | List of keywords used to tag each event. | keyword | | user.id | Unique identifier of the user. | keyword | | user.name | Short name or login of the user. | keyword | | user.name.text | Multi-field of `user.name`. | match_only_text | diff --git a/packages/kubernetes/manifest.yml b/packages/kubernetes/manifest.yml index a730675e3e9..ecc4594a7ff 100644 --- a/packages/kubernetes/manifest.yml +++ b/packages/kubernetes/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.1.2 name: kubernetes title: Kubernetes -version: 1.81.0 +version: 1.81.1 description: Collect logs and metrics from Kubernetes clusters with Elastic Agent. type: integration categories: