diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1f0980f829d..1c57ba97841 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -499,4 +499,5 @@ /packages/cisco_meraki_metrics @elastic/obs-infraobs-integrations /packages/panw_metrics @elastic/obs-infraobs-integrations /packages/o365_metrics @elastic/obs-infraobs-integrations +/packages/chargeback @elastic/customer-architects /packages/aws_billing @elastic/obs-infraobs-integrations diff --git a/packages/chargeback/_dev/build/build.yml b/packages/chargeback/_dev/build/build.yml new file mode 100755 index 00000000000..97fc8aa10cd --- /dev/null +++ b/packages/chargeback/_dev/build/build.yml @@ -0,0 +1,3 @@ +dependencies: + ecs: + reference: git@v8.17.0 diff --git a/packages/chargeback/_dev/build/docs/README.md b/packages/chargeback/_dev/build/docs/README.md new file mode 100755 index 00000000000..d4ee6fda934 --- /dev/null +++ b/packages/chargeback/_dev/build/docs/README.md @@ -0,0 +1,61 @@ +# Chargeback + +_Technical preview: This integration is being developed by Elastic's Customer Engineering team. Please report any issues to the Elastician who shared this integration with you._ + +The Chargeback integration provides FinOps visibility into Elastic usage across tenants. By integrating data from the [**Elasticsearch Service Billing**](https://www.elastic.co/docs/reference/integrations/ess_billing/) and [**Elasticsearch**](https://www.elastic.co/docs/reference/integrations/elasticsearch/) integrations, it enables the determination of value provided by each deployment, data stream, and tier accross the organisation. This allows Centre of Excellence (CoE) teams to accurately allocate costs back to the appropriate tenant. + +## What is FinOps? + +FinOps is an operational framework and cultural practice aimed at maximizing the business value of cloud usage. It facilitates timely, data-driven decision-making and promotes financial accountability through collaboration among engineering, finance, and business teams. + +## Purpose + +The Chargeback integration assists organisations in addressing a crucial question: + +> **"How is my organisation consuming the Elastic solution, and to which tenants can I allocate these costs?"** + +The integration provides a breakdown of Elastic Consumption Units (ECUs) per: + +- Deployment +- Data tier +- Data stream +- Day + +Currently, Chargeback calculations consider only Elasticsearch data nodes. Contributions from other assets, like Kibana or ML nodes, are assumed to be shared proportionally among tenants. To incorporate indexing, querying, and storage in a weighted manner, a blended value is created using the following default weights (modifiable): +- Indexing: `20` (applicable only to the hot tier) +- Querying: `20` +- Storage: `40` + +This default weighting means storage contributes most to the blended cost calculation, with indexing considered only on the hot tier. Adjust these weights based on your organisation's needs and best judgment. + +Chargeback is also present based on a configured rate and unit. These are used to display cost in the local currency, for instance `EUR`, with a rate of `0.85`. + +All configuration values can be updated, as follows: + +``` +POST chargeback_conf_lookup/_update/config +{ + "doc": { + "conf_ecu_rate": 0.85, + "conf_ecu_rate_unit": "EUR", + "conf_indexing_weight": 50, + "conf_query_weight": 20, + "conf_storage_weight": 40 + } +} +``` + +Chargeback data can be viewed in the `[Chargeback] Cost and Consumption breakdown` dashboard. + +![Cost and Consumption breakdown](../img/chargeback.png) + +## Requirements + +To use this integration, the following prerequisites must be met: + +- The monitoring cluster, where this integration is installed, must be on version 8.18.0+ due to its use of [ES|QL LOOKUP JOIN](https://www.elastic.co/docs/reference/query-languages/esql/esql-lookup-join). +- The [**Elasticsearch Service Billing**](https://www.elastic.co/docs/reference/integrations/ess_billing/) integration (v1.4.1+) must be installed and running. +- The [**Elasticsearch**](https://www.elastic.co/docs/reference/integrations/elasticsearch/) integration (v1.16.0+) must be installed and collecting [usage data](https://www.elastic.co/docs/reference/integrations/elasticsearch/#indices-and-data-streams-usage-analysis) from all relevant deployments. +- The Transform named `logs-elasticsearch.index_pivot-default-{VERSION}` must be running, which is an asset of the **Elasticsearch** integration. + +This integration must be installed on the **Monitoring cluster** where the above mentioned relevant usage and billing data is collected. \ No newline at end of file diff --git a/packages/chargeback/changelog.yml b/packages/chargeback/changelog.yml new file mode 100644 index 00000000000..72b037f5518 --- /dev/null +++ b/packages/chargeback/changelog.yml @@ -0,0 +1,61 @@ +# newer versions go on top +- version: 0.1.7 + changes: + - description: "Swap the use of deployment_id or deployment name to a concatenation of both, to make it easier to identify the deployment in the dashboard." + type: enhancement + link: https://github.com/elastic/integrations/pull/14545 +- version: 0.1.6 + changes: + - description: "Remove the use of usage alias, and stick to using `monitoring-indices` as usage sorce. ES Integration transform should be run regardless of wether the ES integration has been installed on an agent or not. This fix will increase performance, when relying on Stack Monitoring data. Also, use `metrics-ess_billing.billing-*` to be able to use not only the default namespace." + type: enhancement + link: https://github.com/elastic/integrations/pull/14545 +- version: 0.1.5 + changes: + - description: "Fixing the control error in the dashboard by adding a data view." + type: bugfix + link: https://github.com/elastic/integrations/pull/14545 +- version: 0.1.4 + changes: + - description: "Consistent naming of `datastream`. Add `| LIMIT 5000` to ESQL top query to cater for large organisations." + type: bugfix + link: https://github.com/elastic/integrations/pull/14545 +- version: 0.1.3 + changes: + - description: "Made sure the colour palette is predictable by using the eui_amsterdam_color_blind palate. Add ECU rate to the dashboard." + type: bugfix + link: https://github.com/elastic/integrations/pull/14545 +- version: 0.1.2 + changes: + - description: "Set transforms to not auto start to allow integration to be uploaded successfully." + type: bugfix + link: https://github.com/elastic/integrations/pull/14545 +- version: 0.1.1 + changes: + - description: "Aligned fields returned to field names used in visualisation." + type: bugfix + link: https://github.com/elastic/integrations/pull/14545 +- version: 0.1.0 + changes: + - description: "Adding ECU rate unit to the configuration lookup index." + type: enhancement + link: https://github.com/elastic/integrations/pull/14545 +- version: 0.0.4 + changes: + - description: "Fix sorting on `Blended value: % ECU per data stream per day` visualisation." + type: bugfix + link: https://github.com/elastic/integrations/pull/14545 +- version: 0.0.3 + changes: + - description: Added support to work with the new `chargeback-monitoring-read` alias, which is used to read from either the ouput of the Elasticsearch Integration, or from the Stack Monitoring indices. + type: enhancement + link: https://github.com/elastic/integrations/pull/14545 +- version: 0.0.2 + changes: + - description: Making use of config lookup values for ecu rate, indexing weight, querying weight, and storage weight. + type: enhancement + link: https://github.com/elastic/integrations/pull/14545 +- version: 0.0.1 + changes: + - description: Initial draft of the package + type: enhancement + link: https://github.com/elastic/integrations/pull/14545 diff --git a/packages/chargeback/data_stream/billing/elasticsearch/ingest_pipeline/default.yml b/packages/chargeback/data_stream/billing/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..f83f6a55dfb --- /dev/null +++ b/packages/chargeback/data_stream/billing/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,11 @@ +--- +description: "Chargeback: Set composite_key from @timestamp and deployment_id." +processors: + - script: + lang: painless + source: > + if (ctx['@timestamp'] != null) { + ctx.composite_key = ZonedDateTime.parse(ctx['@timestamp']).toLocalDate().toString() + '_' + ctx.deployment_id; + } + ignore_failure: true + tag: ess_billing \ No newline at end of file diff --git a/packages/chargeback/data_stream/billing/fields/base-fields.yml b/packages/chargeback/data_stream/billing/fields/base-fields.yml new file mode 100644 index 00000000000..7c798f4534c --- /dev/null +++ b/packages/chargeback/data_stream/billing/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/chargeback/data_stream/billing/manifest.yml b/packages/chargeback/data_stream/billing/manifest.yml new file mode 100644 index 00000000000..79e1a432759 --- /dev/null +++ b/packages/chargeback/data_stream/billing/manifest.yml @@ -0,0 +1,2 @@ +title: "Billing Metrics Aggregation" +type: metrics \ No newline at end of file diff --git a/packages/chargeback/data_stream/usage/elasticsearch/ingest_pipeline/default.yml b/packages/chargeback/data_stream/usage/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..1528515a681 --- /dev/null +++ b/packages/chargeback/data_stream/usage/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,25 @@ +--- +description: "Chargeback: Set correlation keys and blended calculation weights." +processors: + - script: + lang: painless + source: > + if (ctx.cluster_name != null) { + ctx.deployment_id = ctx.cluster_name; + } else if (ctx.elasticsearch?.cluster?.name != null) { + ctx.deployment_id = ctx.elasticsearch.cluster.name; + } + + if (ctx['@timestamp'] != null && ctx.deployment_id != null) { + def date = ZonedDateTime.parse(ctx['@timestamp']).toLocalDate().toString(); + ctx.composite_key = date + '_' + ctx.deployment_id; + + if (ctx.tier != null) { + ctx.composite_tier_key = ctx.composite_key + '_' + ctx.tier.replace('/', '_'); + } + + if (ctx.datastream != null) { + ctx.composite_datastream_key = ctx.composite_key + '_' + ctx.datastream; + } + } + ignore_failure: true \ No newline at end of file diff --git a/packages/chargeback/data_stream/usage/fields/base-fields.yml b/packages/chargeback/data_stream/usage/fields/base-fields.yml new file mode 100644 index 00000000000..7c798f4534c --- /dev/null +++ b/packages/chargeback/data_stream/usage/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/chargeback/data_stream/usage/manifest.yml b/packages/chargeback/data_stream/usage/manifest.yml new file mode 100644 index 00000000000..b131403b007 --- /dev/null +++ b/packages/chargeback/data_stream/usage/manifest.yml @@ -0,0 +1,2 @@ +title: Usage Metrics Aggregation +type: metrics \ No newline at end of file diff --git a/packages/chargeback/docs/README.md b/packages/chargeback/docs/README.md new file mode 100644 index 00000000000..d4ee6fda934 --- /dev/null +++ b/packages/chargeback/docs/README.md @@ -0,0 +1,61 @@ +# Chargeback + +_Technical preview: This integration is being developed by Elastic's Customer Engineering team. Please report any issues to the Elastician who shared this integration with you._ + +The Chargeback integration provides FinOps visibility into Elastic usage across tenants. By integrating data from the [**Elasticsearch Service Billing**](https://www.elastic.co/docs/reference/integrations/ess_billing/) and [**Elasticsearch**](https://www.elastic.co/docs/reference/integrations/elasticsearch/) integrations, it enables the determination of value provided by each deployment, data stream, and tier accross the organisation. This allows Centre of Excellence (CoE) teams to accurately allocate costs back to the appropriate tenant. + +## What is FinOps? + +FinOps is an operational framework and cultural practice aimed at maximizing the business value of cloud usage. It facilitates timely, data-driven decision-making and promotes financial accountability through collaboration among engineering, finance, and business teams. + +## Purpose + +The Chargeback integration assists organisations in addressing a crucial question: + +> **"How is my organisation consuming the Elastic solution, and to which tenants can I allocate these costs?"** + +The integration provides a breakdown of Elastic Consumption Units (ECUs) per: + +- Deployment +- Data tier +- Data stream +- Day + +Currently, Chargeback calculations consider only Elasticsearch data nodes. Contributions from other assets, like Kibana or ML nodes, are assumed to be shared proportionally among tenants. To incorporate indexing, querying, and storage in a weighted manner, a blended value is created using the following default weights (modifiable): +- Indexing: `20` (applicable only to the hot tier) +- Querying: `20` +- Storage: `40` + +This default weighting means storage contributes most to the blended cost calculation, with indexing considered only on the hot tier. Adjust these weights based on your organisation's needs and best judgment. + +Chargeback is also present based on a configured rate and unit. These are used to display cost in the local currency, for instance `EUR`, with a rate of `0.85`. + +All configuration values can be updated, as follows: + +``` +POST chargeback_conf_lookup/_update/config +{ + "doc": { + "conf_ecu_rate": 0.85, + "conf_ecu_rate_unit": "EUR", + "conf_indexing_weight": 50, + "conf_query_weight": 20, + "conf_storage_weight": 40 + } +} +``` + +Chargeback data can be viewed in the `[Chargeback] Cost and Consumption breakdown` dashboard. + +![Cost and Consumption breakdown](../img/chargeback.png) + +## Requirements + +To use this integration, the following prerequisites must be met: + +- The monitoring cluster, where this integration is installed, must be on version 8.18.0+ due to its use of [ES|QL LOOKUP JOIN](https://www.elastic.co/docs/reference/query-languages/esql/esql-lookup-join). +- The [**Elasticsearch Service Billing**](https://www.elastic.co/docs/reference/integrations/ess_billing/) integration (v1.4.1+) must be installed and running. +- The [**Elasticsearch**](https://www.elastic.co/docs/reference/integrations/elasticsearch/) integration (v1.16.0+) must be installed and collecting [usage data](https://www.elastic.co/docs/reference/integrations/elasticsearch/#indices-and-data-streams-usage-analysis) from all relevant deployments. +- The Transform named `logs-elasticsearch.index_pivot-default-{VERSION}` must be running, which is an asset of the **Elasticsearch** integration. + +This integration must be installed on the **Monitoring cluster** where the above mentioned relevant usage and billing data is collected. \ No newline at end of file diff --git a/packages/chargeback/elasticsearch/transform/billing_cluster_cost/fields/fields.yml b/packages/chargeback/elasticsearch/transform/billing_cluster_cost/fields/fields.yml new file mode 100644 index 00000000000..ca91ffec28c --- /dev/null +++ b/packages/chargeback/elasticsearch/transform/billing_cluster_cost/fields/fields.yml @@ -0,0 +1,15 @@ +- name: "@timestamp" + type: date + description: The timestamp representing the day of the billing data. +- name: deployment_id + type: keyword + description: Unique ID of the deployment. +- name: deployment_name + type: keyword + description: Human-readable name of the deployment. +- name: total_ecu + type: double + description: Total ECU usage aggregated for the deployment per day. +- name: composite_key + type: keyword + description: Composite key used for billing attribution (set by ingest pipeline) consisting of date and deployment ID. diff --git a/packages/chargeback/elasticsearch/transform/billing_cluster_cost/manifest.yml b/packages/chargeback/elasticsearch/transform/billing_cluster_cost/manifest.yml new file mode 100644 index 00000000000..765066063d4 --- /dev/null +++ b/packages/chargeback/elasticsearch/transform/billing_cluster_cost/manifest.yml @@ -0,0 +1,8 @@ +start: true +destination_index_template: + settings: + index: +# mode: "lookup" + codec: best_compression + mappings: + dynamic: false \ No newline at end of file diff --git a/packages/chargeback/elasticsearch/transform/billing_cluster_cost/transform.yml b/packages/chargeback/elasticsearch/transform/billing_cluster_cost/transform.yml new file mode 100644 index 00000000000..5e42db853fb --- /dev/null +++ b/packages/chargeback/elasticsearch/transform/billing_cluster_cost/transform.yml @@ -0,0 +1,42 @@ +description: Aggregates daily total ECU usage per deployment from billing metrics, using ingested timestamps with a 1-hour sync delay and running every 60 minutes. +source: + index: + - metrics-ess_billing.billing-* + query: + range: + ess.billing.total_ecu: + gt: 0 +dest: + index: billing_cluster_cost_lookup + pipeline: metrics-chargeback.billing-0.1.7 +frequency: 60m +sync: + time: + field: event.ingested + delay: 1h +pivot: + group_by: + "@timestamp": + date_histogram: + field: "@timestamp" + calendar_interval: 1d + deployment_id: + terms: + field: ess.billing.deployment_id + deployment_name: + terms: + field: ess.billing.deployment_name + aggregations: + total_ecu: + sum: + field: ess.billing.total_ecu +settings: + # This is required to prevent the transform from clobbering the Fleet-managed mappings. + deduce_mappings: false + unattended: true +_meta: + managed: true + run_as_kibana_system: false + # Bump this version to delete, reinstall, and restart the transform during package. + # Version bump is needed if there is any code change in transform. + fleet_transform_version: 0.1.7 diff --git a/packages/chargeback/elasticsearch/transform/cluster_datastream_contribution/fields/fields.yml b/packages/chargeback/elasticsearch/transform/cluster_datastream_contribution/fields/fields.yml new file mode 100644 index 00000000000..f489ab2bbf6 --- /dev/null +++ b/packages/chargeback/elasticsearch/transform/cluster_datastream_contribution/fields/fields.yml @@ -0,0 +1,24 @@ +- name: "@timestamp" + type: date + description: Daily timestamp representing when the metric was recorded. +- name: cluster_name + type: keyword + description: Name of the Elasticsearch cluster. +- name: datastream + type: keyword + description: The name of the data stream. +- name: datastream_sum_indexing_time + type: long + description: Total indexing time in milliseconds for the data stream. +- name: datastream_sum_query_time + type: long + description: Total query time in milliseconds for the data stream. +- name: datastream_sum_store_size + type: long + description: Total primary shard store size in bytes for the data stream. +- name: datastream_sum_data_set_store_size + type: long + description: Total dataset size in bytes for the data stream. +- name: composite_key + type: keyword + description: Composite key used for billing attribution (set by ingest pipeline) consisting of date and deployment ID. diff --git a/packages/chargeback/elasticsearch/transform/cluster_datastream_contribution/manifest.yml b/packages/chargeback/elasticsearch/transform/cluster_datastream_contribution/manifest.yml new file mode 100644 index 00000000000..f6a8a36f8ef --- /dev/null +++ b/packages/chargeback/elasticsearch/transform/cluster_datastream_contribution/manifest.yml @@ -0,0 +1,8 @@ +start: false +destination_index_template: + settings: + index: +# mode: "lookup" + codec: best_compression + mappings: + dynamic: false \ No newline at end of file diff --git a/packages/chargeback/elasticsearch/transform/cluster_datastream_contribution/transform.yml b/packages/chargeback/elasticsearch/transform/cluster_datastream_contribution/transform.yml new file mode 100644 index 00000000000..3c19665a8e7 --- /dev/null +++ b/packages/chargeback/elasticsearch/transform/cluster_datastream_contribution/transform.yml @@ -0,0 +1,63 @@ +description: Aggregates daily total ECU usage per datastream from billing metrics, using ingested timestamps with a 1-hour sync delay and running every 60 minutes. +source: + index: + - monitoring-indices # Dependency: ES Integration added, but not neceserily installed. Need the output of the transform only. +dest: + index: cluster_datastream_contribution_lookup + pipeline: metrics-chargeback.usage-0.1.7 +frequency: 60m +sync: + time: + field: "@timestamp" + delay: 1h +pivot: + group_by: + "@timestamp": + date_histogram: + field: "@timestamp" + calendar_interval: 1d + cluster_name: + terms: + field: elasticsearch.cluster.name + datastream: + terms: + script: + lang: painless + source: > + if (doc.containsKey('elasticsearch.index.datastream') && !doc['elasticsearch.index.datastream'].empty) { + return doc['elasticsearch.index.datastream'].value; + } + + def name = doc.containsKey('elasticsearch.index.name') && !doc['elasticsearch.index.name'].empty + ? doc['elasticsearch.index.name'].value + : null; + if (name == null) return 'unknown'; + + def matcher = /^(?:partial-)?(?:restored-)?(?:shrink-.{4}-)?(?:\.ds-)?([a-z_0-9\-.]+?)(?:-\d{4}\.\d{2}(?:\.\d{2})?)?(?:-\d+)?$/.matcher(name); + if (matcher.matches()) { + return matcher.group(1); + } + return 'unknown'; + aggregations: + datastream_sum_indexing_time: + sum: + field: elasticsearch.index.total.indexing.index_time_in_millis + datastream_sum_query_time: + sum: + field: elasticsearch.index.total.search.query_time_in_millis + datastream_sum_store_size: + sum: + field: elasticsearch.index.total.store.size_in_bytes + datastream_sum_data_set_store_size: + sum: + field: elasticsearch.index.primaries.store.total_data_set_size_in_bytes +settings: + # This is required to prevent the transform from clobbering the Fleet-managed mappings. + deduce_mappings: false + unattended: true +_meta: + managed: true + run_as_kibana_system: false + # Bump this version to delete, reinstall, and restart the transform during package. + # Version bump is needed if there is any code change in transform. + fleet_transform_version: 0.1.7 diff --git a/packages/chargeback/elasticsearch/transform/cluster_deployment_contribution/fields/fields.yml b/packages/chargeback/elasticsearch/transform/cluster_deployment_contribution/fields/fields.yml new file mode 100644 index 00000000000..63866dda2e0 --- /dev/null +++ b/packages/chargeback/elasticsearch/transform/cluster_deployment_contribution/fields/fields.yml @@ -0,0 +1,21 @@ +- name: "@timestamp" + type: date + description: Daily timestamp representing when the metric was recorded. +- name: cluster_name + type: keyword + description: Name of the Elasticsearch cluster. +- name: deployment_sum_indexing_time + type: long + description: Total indexing time in milliseconds for the deployment. +- name: deployment_sum_query_time + type: long + description: Total query time in milliseconds for the deployment. +- name: deployment_sum_store_size + type: long + description: Total primary shard store size in bytes for the deployment. +- name: deployment_sum_data_set_store_size + type: long + description: Total dataset size in bytes for the deployment. +- name: composite_key + type: keyword + description: Composite key used for billing attribution (set by ingest pipeline) consisting of date and deployment ID. diff --git a/packages/chargeback/elasticsearch/transform/cluster_deployment_contribution/manifest.yml b/packages/chargeback/elasticsearch/transform/cluster_deployment_contribution/manifest.yml new file mode 100644 index 00000000000..485504dce48 --- /dev/null +++ b/packages/chargeback/elasticsearch/transform/cluster_deployment_contribution/manifest.yml @@ -0,0 +1,8 @@ +start: false +destination_index_template: + settings: + index: +# mode: "lookup" + codec: best_compression + mappings: + dynamic: false \ No newline at end of file diff --git a/packages/chargeback/elasticsearch/transform/cluster_deployment_contribution/transform.yml b/packages/chargeback/elasticsearch/transform/cluster_deployment_contribution/transform.yml new file mode 100644 index 00000000000..6bdc90442ba --- /dev/null +++ b/packages/chargeback/elasticsearch/transform/cluster_deployment_contribution/transform.yml @@ -0,0 +1,44 @@ +description: Aggregates daily total ECU usage per deployment from billing metrics, using ingested timestamps with a 1-hour sync delay and running every 60 minutes. +source: + index: + - monitoring-indices # Dependency: ES Integration added, but not neceserily installed. Need the output of the transform only. +dest: + index: cluster_deployment_contribution_lookup + pipeline: metrics-chargeback.usage-0.1.7 +frequency: 60m +sync: + time: + field: "@timestamp" + delay: 1h +pivot: + group_by: + "@timestamp": + date_histogram: + field: "@timestamp" + calendar_interval: 1d + cluster_name: + terms: + field: elasticsearch.cluster.name + aggregations: + deployment_sum_indexing_time: + sum: + field: elasticsearch.index.total.indexing.index_time_in_millis + deployment_sum_query_time: + sum: + field: elasticsearch.index.total.search.query_time_in_millis + deployment_sum_store_size: + sum: + field: elasticsearch.index.total.store.size_in_bytes + deployment_sum_data_set_store_size: + sum: + field: elasticsearch.index.primaries.store.total_data_set_size_in_bytes +settings: + # This is required to prevent the transform from clobbering the Fleet-managed mappings. + deduce_mappings: false + unattended: true +_meta: + managed: true + run_as_kibana_system: false + # Bump this version to delete, reinstall, and restart the transform during package. + # Version bump is needed if there is any code change in transform. + fleet_transform_version: 0.1.7 \ No newline at end of file diff --git a/packages/chargeback/elasticsearch/transform/cluster_tier_and_ds_contribution/fields/fields.yml b/packages/chargeback/elasticsearch/transform/cluster_tier_and_ds_contribution/fields/fields.yml new file mode 100644 index 00000000000..54cc1c6bb6c --- /dev/null +++ b/packages/chargeback/elasticsearch/transform/cluster_tier_and_ds_contribution/fields/fields.yml @@ -0,0 +1,27 @@ +- name: "@timestamp" + type: date + description: Daily timestamp representing when the metric was recorded. +- name: cluster_name + type: keyword + description: Name of the Elasticsearch cluster. +- name: tier + type: keyword + description: The data tier where the index resides (e.g. hot, warm, cold, frozen). +- name: datastream + type: keyword + description: The name of the data stream. +- name: tier_and_datastream_sum_indexing_time + type: long + description: Total indexing time in milliseconds for the tier and data stream. +- name: tier_and_datastream_sum_query_time + type: long + description: Total query time in milliseconds for the tier and data stream. +- name: tier_and_datastream_sum_store_size + type: long + description: Total primary shard store size in bytes for the tier and data stream. +- name: tier_and_datastream_sum_data_set_store_size + type: long + description: Total dataset size in bytes for the tier and data stream. +- name: composite_key + type: keyword + description: Composite key used for billing attribution (set by ingest pipeline) consisting of date and data tier and data stream name. diff --git a/packages/chargeback/elasticsearch/transform/cluster_tier_and_ds_contribution/manifest.yml b/packages/chargeback/elasticsearch/transform/cluster_tier_and_ds_contribution/manifest.yml new file mode 100644 index 00000000000..6ad56abcd38 --- /dev/null +++ b/packages/chargeback/elasticsearch/transform/cluster_tier_and_ds_contribution/manifest.yml @@ -0,0 +1,8 @@ +start: false +destination_index_template: + settings: + index: +# mode: "lookup" + codec: best_compression + mappings: + dynamic: false \ No newline at end of file diff --git a/packages/chargeback/elasticsearch/transform/cluster_tier_and_ds_contribution/transform.yml b/packages/chargeback/elasticsearch/transform/cluster_tier_and_ds_contribution/transform.yml new file mode 100644 index 00000000000..a460610b774 --- /dev/null +++ b/packages/chargeback/elasticsearch/transform/cluster_tier_and_ds_contribution/transform.yml @@ -0,0 +1,78 @@ +description: Aggregates daily total ECU usage per tier and datastream from billing metrics, using ingested timestamps with a 1-hour sync delay and running every 60 minutes. +source: + index: + - monitoring-indices # Dependency: ES Integration added, but not neceserily installed. Need the output of the transform only. +dest: + index: cluster_tier_and_datastream_contribution_lookup + pipeline: metrics-chargeback.usage-0.1.7 +frequency: 60m +sync: + time: + field: "@timestamp" + delay: 1h +pivot: + group_by: + "@timestamp": + date_histogram: + field: "@timestamp" + calendar_interval: 1d + cluster_name: + terms: + field: elasticsearch.cluster.name + tier: + terms: + script: + lang: painless + source: > + if (doc.containsKey('elasticsearch.index.tier') && !doc['elasticsearch.index.tier'].empty) { + return doc['elasticsearch.index.tier'].value; + } + def pref = doc.containsKey('elasticsearch.index.tier_preference') && !doc['elasticsearch.index.tier_preference'].empty ? doc['elasticsearch.index.tier_preference'].value : null; + if (pref == null) return 'unknown'; + if (pref.contains('data_hot') || pref.contains('data_content')) return 'hot/content'; + if (pref.contains('data_warm')) return 'warm'; + if (pref.contains('data_cold')) return 'cold'; + if (pref.contains('data_frozen')) return 'frozen'; + return 'unknown'; + datastream: + terms: + script: + lang: painless + source: > + if (doc.containsKey('elasticsearch.index.datastream') && !doc['elasticsearch.index.datastream'].empty) { + return doc['elasticsearch.index.datastream'].value; + } + + def name = doc.containsKey('elasticsearch.index.name') && !doc['elasticsearch.index.name'].empty + ? doc['elasticsearch.index.name'].value + : null; + if (name == null) return 'unknown'; + + def matcher = /^(?:partial-)?(?:restored-)?(?:shrink-.{4}-)?(?:\.ds-)?([a-z_0-9\-.]+?)(?:-\d{4}\.\d{2}(?:\.\d{2})?)?(?:-\d+)?$/.matcher(name); + if (matcher.matches()) { + return matcher.group(1); + } + return 'unknown'; + aggregations: + tier_and_datastream_sum_indexing_time: + sum: + field: elasticsearch.index.total.indexing.index_time_in_millis + tier_and_datastream_sum_query_time: + sum: + field: elasticsearch.index.total.search.query_time_in_millis + tier_and_datastream_sum_store_size: + sum: + field: elasticsearch.index.total.store.size_in_bytes + tier_and_datastream_sum_data_set_store_size: + sum: + field: elasticsearch.index.primaries.store.total_data_set_size_in_bytes +settings: + # This is required to prevent the transform from clobbering the Fleet-managed mappings. + deduce_mappings: false + unattended: true +_meta: + managed: true + run_as_kibana_system: false + # Bump this version to delete, reinstall, and restart the transform during package. + # Version bump is needed if there is any code change in transform. + fleet_transform_version: 0.1.7 \ No newline at end of file diff --git a/packages/chargeback/elasticsearch/transform/cluster_tier_contribution/fields/fields.yml b/packages/chargeback/elasticsearch/transform/cluster_tier_contribution/fields/fields.yml new file mode 100644 index 00000000000..548f3e8f290 --- /dev/null +++ b/packages/chargeback/elasticsearch/transform/cluster_tier_contribution/fields/fields.yml @@ -0,0 +1,24 @@ +- name: "@timestamp" + type: date + description: Daily timestamp representing when the metric was recorded. +- name: cluster_name + type: keyword + description: Name of the Elasticsearch cluster. +- name: tier + type: keyword + description: The data tier where the index resides (e.g. hot, warm, cold, frozen). +- name: tier_sum_indexing_time + type: long + description: Total indexing time in milliseconds for the tier. +- name: tier_sum_query_time + type: long + description: Total query time in milliseconds for the tier. +- name: tier_sum_store_size + type: long + description: Total primary shard store size in bytes for the tier. +- name: tier_sum_data_set_store_size + type: long + description: Total dataset size in bytes for the tier. +- name: composite_key + type: keyword + description: Composite key used for billing attribution (set by ingest pipeline) consisting of date and data tier. diff --git a/packages/chargeback/elasticsearch/transform/cluster_tier_contribution/manifest.yml b/packages/chargeback/elasticsearch/transform/cluster_tier_contribution/manifest.yml new file mode 100644 index 00000000000..665a52b597c --- /dev/null +++ b/packages/chargeback/elasticsearch/transform/cluster_tier_contribution/manifest.yml @@ -0,0 +1,8 @@ +start: false +destination_index_template: + settings: + index: +# mode: "lookup" + codec: best_compression + mappings: + dynamic: false diff --git a/packages/chargeback/elasticsearch/transform/cluster_tier_contribution/transform.yml b/packages/chargeback/elasticsearch/transform/cluster_tier_contribution/transform.yml new file mode 100644 index 00000000000..74745bbe2c1 --- /dev/null +++ b/packages/chargeback/elasticsearch/transform/cluster_tier_contribution/transform.yml @@ -0,0 +1,60 @@ +description: Aggregates daily total ECU usage per tier from billing metrics, using ingested timestamps with a 1-hour sync delay and running every 60 minutes. +source: + index: + - monitoring-indices # Dependency: ES Integration added, but not neceserily installed. Need the output of the transform only. +dest: + index: cluster_tier_contribution_lookup + pipeline: metrics-chargeback.usage-0.1.7 +frequency: 60m +sync: + time: + field: "@timestamp" + delay: 1h +pivot: + group_by: + "@timestamp": + date_histogram: + field: "@timestamp" + calendar_interval: 1d + cluster_name: + terms: + field: elasticsearch.cluster.name + tier: + terms: + script: + lang: painless + source: > + if (doc.containsKey('elasticsearch.index.tier') && !doc['elasticsearch.index.tier'].empty) { + return doc['elasticsearch.index.tier'].value; + } + + def pref = doc.containsKey('elasticsearch.index.tier_preference') && !doc['elasticsearch.index.tier_preference'].empty ? doc['elasticsearch.index.tier_preference'].value : null; + if (pref == null) return 'unknown'; + if (pref.contains('data_hot') || pref.contains('data_content')) return 'hot/content'; + if (pref.contains('data_warm')) return 'warm'; + if (pref.contains('data_cold')) return 'cold'; + if (pref.contains('data_frozen')) return 'frozen'; + return 'unknown'; + aggregations: + tier_sum_indexing_time: + sum: + field: elasticsearch.index.total.indexing.index_time_in_millis + tier_sum_query_time: + sum: + field: elasticsearch.index.total.search.query_time_in_millis + tier_sum_store_size: + sum: + field: elasticsearch.index.total.store.size_in_bytes + tier_sum_data_set_store_size: + sum: + field: elasticsearch.index.primaries.store.total_data_set_size_in_bytes +settings: + # This is required to prevent the transform from clobbering the Fleet-managed mappings. + deduce_mappings: false + unattended: true +_meta: + managed: true + run_as_kibana_system: false + # Bump this version to delete, reinstall, and restart the transform during package. + # Version bump is needed if there is any code change in transform. + fleet_transform_version: 0.1.7 \ No newline at end of file diff --git a/packages/chargeback/img/chargeback.png b/packages/chargeback/img/chargeback.png new file mode 100644 index 00000000000..971f878e988 Binary files /dev/null and b/packages/chargeback/img/chargeback.png differ diff --git a/packages/chargeback/img/elastic-logo.png b/packages/chargeback/img/elastic-logo.png new file mode 100644 index 00000000000..d7a56a3ecc0 Binary files /dev/null and b/packages/chargeback/img/elastic-logo.png differ diff --git a/packages/chargeback/img/elastic-logo.svg b/packages/chargeback/img/elastic-logo.svg new file mode 100644 index 00000000000..6268dd88f3b --- /dev/null +++ b/packages/chargeback/img/elastic-logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/chargeback/kibana/dashboard/chargeback-39a39857-746c-4a29-adca-3c2fcb6bcfb6.json b/packages/chargeback/kibana/dashboard/chargeback-39a39857-746c-4a29-adca-3c2fcb6bcfb6.json new file mode 100644 index 00000000000..860797f6d49 --- /dev/null +++ b/packages/chargeback/kibana/dashboard/chargeback-39a39857-746c-4a29-adca-3c2fcb6bcfb6.json @@ -0,0 +1,4414 @@ +{ + "attributes": { + "controlGroupInput": { + "chainingSystem": "HIERARCHICAL", + "controlStyle": "oneLine", + "ignoreParentSettingsJSON": { + "ignoreFilters": false, + "ignoreQuery": false, + "ignoreTimerange": false, + "ignoreValidations": false + }, + "panelsJSON": { + "ab7831d2-e2cb-4ece-a4a6-b22c17f30aec": { + "explicitInput": { + "dataViewId": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "exclude": false, + "existsSelected": false, + "fieldName": "deployment_id", + "id": "ab7831d2-e2cb-4ece-a4a6-b22c17f30aec", + "searchTechnique": "prefix", + "selectedOptions": [], + "sort": { + "by": "_count", + "direction": "desc" + }, + "title": "Deployment ID" + }, + "grow": false, + "order": 0, + "type": "optionsListControl", + "width": "large" + } + }, + "showApplySelections": false + }, + "description": "Provides visual insights into cluster-level cost and usage metrics, helping track Elastic consumption across tiers, deployments, and data streams. Built from transformed billing and usage data, this dashboard enables transparent chargeback calculations and trend analysis.", + "kibanaSavedObjectMeta": { + "searchSourceJSON": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "optionsJSON": { + "hidePanelTitles": false, + "syncColors": false, + "syncCursor": true, + "syncTooltips": false, + "useMargins": true + }, + "panelsJSON": [ + { + "embeddableConfig": { + "attributes": { + "references": [], + "state": { + "adHocDataViews": { + "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "name": "billing_cluster_cost_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_cluster_cost_lookup", + "type": "esql" + } + }, + "datasourceStates": { + "textBased": { + "indexPatternRefs": [ + { + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "timeField": "@timestamp", + "title": "billing_cluster_cost_lookup" + } + ], + "layers": { + "c2dbde04-0769-4485-b4fa-4cb05a1d975e": { + "columns": [ + { + "columnId": "@timestamp", + "customLabel": true, + "fieldName": "@timestamp", + "label": "Date", + "meta": { + "esType": "date", + "type": "date" + } + }, + { + "columnId": "deployment", + "customLabel": true, + "fieldName": "deployment", + "label": "Deployment", + "meta": { + "esType": "keyword", + "type": "string" + } + }, + { + "columnId": "total_ecu_value", + "customLabel": true, + "fieldName": "total_ecu_value", + "inMetricDimension": true, + "label": "Normalised cost", + "meta": { + "esType": "double", + "type": "number" + }, + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + } + } + ], + "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| EVAL total_ecu_value = total_ecu * conf_ecu_rate, deployment = CONCAT(deployment_name, \" (\", LEFT(deployment_id, 6), \")\")\n| KEEP @timestamp, deployment, total_ecu_value | SORT @timestamp desc | LIMIT 5000" + + }, + "timeField": "@timestamp" + } + } + } + }, + "filters": [], + "needsRefresh": false, + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| EVAL total_ecu_value = total_ecu * conf_ecu_rate, deployment = CONCAT(deployment_name, \" (\", LEFT(deployment_id, 6), \")\")\n| KEEP @timestamp, deployment, total_ecu_value | SORT @timestamp desc | LIMIT 5000" + + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "total_ecu_value" + ], + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "layerId": "c2dbde04-0769-4485-b4fa-4cb05a1d975e", + "layerType": "data", + "seriesType": "bar_stacked", + "splitAccessor": "deployment", + "xAccessor": "@timestamp" + } + ], + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "bar_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "Normalised cost over time", + "visualizationType": "lnsXY" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| EVAL total_ecu_value = total_ecu * conf_ecu_rate, deployment = CONCAT(deployment_name, \" (\", LEFT(deployment_id, 6), \")\")\n| KEEP @timestamp, deployment, total_ecu_value | SORT @timestamp desc | LIMIT 5000" + + }, + "searchSessionId": "48b157e3-14ff-4c71-8178-4a69f0592d85", + "syncColors": false, + "syncCursor": true, + "syncTooltips": false, + "timeRange": { + "from": "now-6M", + "to": "now" + } + }, + "gridData": { + "h": 9, + "i": "d9069c7e-bfaf-4045-aedf-ca1042c21010", + "w": 34, + "x": 0, + "y": 0 + }, + "panelIndex": "d9069c7e-bfaf-4045-aedf-ca1042c21010", + "title": "Normalised cost over the last 6 months", + "type": "lens" + }, + { + "embeddableConfig": { + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "id": "", + "params": { + "fontSize": 12, + "markdown": "In this dashboard, normalised cost is the ECU (Elastic Consumption Units) * conversion rate (`conf_ecu_rate`).", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 5, + "i": "27ce47e9-a75f-4a58-9dd5-da735ef85ab2", + "w": 8, + "x": 34, + "y": 0 + }, + "panelIndex": "27ce47e9-a75f-4a58-9dd5-da735ef85ab2", + "title": "Normalised cost", + "type": "visualization" + }, + { + "type": "lens", + "title": "Blended rate weighting percentages", + "embeddableConfig": { + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "syncColors": false, + "syncCursor": true, + "syncTooltips": false, + "searchSessionId": "62d2fe44-dd84-4cd0-ac53-38b090b0eff6", + "filters": [], + "query": { + "esql": "FROM chargeback_conf_lookup | KEEP *weight" + }, + "attributes": { + "title": "conf_indexing_weight & conf_query_weight & conf_storage_weight of (empty)", + "references": [], + "state": { + "datasourceStates": { + "textBased": { + "layers": { + "005d9023-92e0-4783-a349-4a2e05b0bce4": { + "index": "8f632b34627f697e9702333f8ef042ab73e3436b01f9a29710a96cc9dad323b9", + "query": { + "esql": "FROM chargeback_conf_lookup | KEEP *weight" + }, + "columns": [ + { + "columnId": "conf_indexing_weight", + "fieldName": "conf_indexing_weight", + "label": "Weight", + "customLabel": true, + "meta": { + "type": "number", + "esType": "integer" + }, + "inMetricDimension": true + }, + { + "columnId": "conf_query_weight", + "fieldName": "conf_query_weight", + "label": "Weight", + "customLabel": true, + "meta": { + "type": "number", + "esType": "integer" + }, + "inMetricDimension": true + }, + { + "columnId": "conf_storage_weight", + "fieldName": "conf_storage_weight", + "label": "Weight", + "customLabel": false, + "meta": { + "type": "number", + "esType": "integer" + }, + "inMetricDimension": true + } + ] + } + }, + "indexPatternRefs": [ + { + "id": "8f632b34627f697e9702333f8ef042ab73e3436b01f9a29710a96cc9dad323b9", + "title": "chargeback_conf_lookup" + } + ] + } + }, + "filters": [], + "query": { + "esql": "FROM chargeback_conf_lookup | KEEP *weight" + }, + "visualization": { + "legend": { + "isVisible": false, + "position": "right", + "showSingleSeries": false, + "isInside": true, + "verticalAlignment": "bottom", + "horizontalAlignment": "right" + }, + "valueLabels": "hide", + "fittingFunction": "Linear", + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "preferredSeriesType": "bar_stacked", + "layers": [ + { + "layerId": "005d9023-92e0-4783-a349-4a2e05b0bce4", + "seriesType": "bar_percentage_stacked", + "accessors": [ + "conf_indexing_weight", + "conf_query_weight", + "conf_storage_weight" + ], + "layerType": "data", + "colorMapping": { + "assignments": [], + "specialAssignments": [ + { + "rule": { + "type": "other" + }, + "color": { + "type": "loop" + }, + "touched": false + } + ], + "paletteId": "eui_amsterdam_color_blind", + "colorMode": { + "type": "categorical" + } + } + } + ] + }, + "adHocDataViews": { + "8f632b34627f697e9702333f8ef042ab73e3436b01f9a29710a96cc9dad323b9": { + "id": "8f632b34627f697e9702333f8ef042ab73e3436b01f9a29710a96cc9dad323b9", + "title": "chargeback_conf_lookup", + "sourceFilters": [], + "type": "esql", + "fieldFormats": {}, + "runtimeFieldMap": {}, + "allowNoIndex": false, + "name": "chargeback_conf_lookup", + "allowHidden": false + } + }, + "needsRefresh": false + }, + "visualizationType": "lnsXY" + } + }, + "panelIndex": "a3a091e7-6bff-4edf-bc65-993e8a3648de", + "gridData": { + "i": "a3a091e7-6bff-4edf-bc65-993e8a3648de", + "y": 0, + "x": 42, + "w": 6, + "h": 9 + } + }, + { + "type": "lens", + "embeddableConfig": { + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "syncColors": false, + "syncCursor": true, + "syncTooltips": false, + "filters": [], + "query": { + "esql": "FROM chargeback_conf_lookup | keep conf_ecu_rate_unit, conf_ecu_rate" + }, + "attributes": { + "references": [], + "state": { + "datasourceStates": { + "textBased": { + "layers": { + "4dba9fd5-8ec5-498a-b710-1512bd5dc18f": { + "index": "8f632b34627f697e9702333f8ef042ab73e3436b01f9a29710a96cc9dad323b9", + "query": { + "esql": "FROM chargeback_conf_lookup | keep conf_ecu_rate_unit, conf_ecu_rate" + }, + "columns": [ + { + "columnId": "conf_ecu_rate_unit", + "fieldName": "conf_ecu_rate_unit", + "label": "Unit", + "customLabel": true, + "meta": { + "type": "string", + "esType": "keyword" + } + }, + { + "columnId": "conf_ecu_rate", + "fieldName": "conf_ecu_rate", + "label": "Rate", + "customLabel": true, + "meta": { + "type": "number", + "esType": "double" + }, + "inMetricDimension": true, + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + } + } + ] + } + }, + "indexPatternRefs": [ + { + "id": "8f632b34627f697e9702333f8ef042ab73e3436b01f9a29710a96cc9dad323b9", + "title": "chargeback_conf_lookup" + } + ] + } + }, + "filters": [], + "query": { + "esql": "FROM chargeback_conf_lookup | keep conf_ecu_rate_unit, conf_ecu_rate" + }, + "visualization": { + "layerId": "4dba9fd5-8ec5-498a-b710-1512bd5dc18f", + "layerType": "data", + "columns": [ + { + "columnId": "conf_ecu_rate_unit" + }, + { + "columnId": "conf_ecu_rate" + } + ] + }, + "adHocDataViews": { + "8f632b34627f697e9702333f8ef042ab73e3436b01f9a29710a96cc9dad323b9": { + "id": "8f632b34627f697e9702333f8ef042ab73e3436b01f9a29710a96cc9dad323b9", + "title": "chargeback_conf_lookup", + "sourceFilters": [], + "type": "esql", + "fieldFormats": {}, + "runtimeFieldMap": {}, + "allowNoIndex": false, + "name": "chargeback_conf_lookup", + "allowHidden": false + } + }, + "needsRefresh": false + }, + "visualizationType": "lnsDatatable" + } + }, + "panelIndex": "54ff8784-cfb6-41dc-ad15-50547f49f9e7", + "gridData": { + "h": 4, + "i": "54ff8784-cfb6-41dc-ad15-50547f49f9e7", + "w": 8, + "x": 34, + "y": 5 + } + }, + { + "embeddableConfig": { + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "id": "", + "params": { + "fontSize": 12, + "markdown": "# Deployment statistics", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 3, + "i": "08bdbcda-fa42-4fb2-bb50-b4ed2620ef6b", + "w": 48, + "x": 0, + "y": 8 + }, + "panelIndex": "08bdbcda-fa42-4fb2-bb50-b4ed2620ef6b", + "type": "visualization" + }, + { + "embeddableConfig": { + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 12, + "markdown": "# Data tier and data stream overview", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 3, + "i": "784ed861-b61f-4c71-a3bc-02cb513a8019", + "w": 48, + "x": 0, + "y": 26 + }, + "panelIndex": "784ed861-b61f-4c71-a3bc-02cb513a8019", + "title": "", + "type": "visualization" + }, + { + "embeddableConfig": { + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 12, + "markdown": "## Indexing", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 3, + "i": "4d8d5ff0-b63f-4d39-be03-e6f85a8b60ce", + "w": 12, + "x": 0, + "y": 39 + }, + "panelIndex": "4d8d5ff0-b63f-4d39-be03-e6f85a8b60ce", + "title": "", + "type": "visualization" + }, + { + "embeddableConfig": { + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 12, + "markdown": "## Querying", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 3, + "i": "40a91be7-f01d-47ce-ba7f-683bcee42626", + "w": 12, + "x": 12, + "y": 39 + }, + "panelIndex": "40a91be7-f01d-47ce-ba7f-683bcee42626", + "title": "", + "type": "visualization" + }, + { + "embeddableConfig": { + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 12, + "markdown": "## Storage", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 3, + "i": "d0c647b4-4b26-49d3-b54a-1ddbdf995143", + "w": 12, + "x": 24, + "y": 39 + }, + "panelIndex": "d0c647b4-4b26-49d3-b54a-1ddbdf995143", + "title": "", + "type": "visualization" + }, + { + "embeddableConfig": { + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 12, + "markdown": "## Blended", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 3, + "i": "540d6135-d996-43ed-a192-30d51e2e5454", + "w": 12, + "x": 36, + "y": 39 + }, + "panelIndex": "540d6135-d996-43ed-a192-30d51e2e5454", + "title": "", + "type": "visualization" + }, + { + "embeddableConfig": { + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 12, + "markdown": "# Data tier and data stream per day", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 3, + "i": "b1e12ff9-83d2-4dd7-a16e-ec4597d14dbc", + "w": 48, + "x": 0, + "y": 72 + }, + "panelIndex": "b1e12ff9-83d2-4dd7-a16e-ec4597d14dbc", + "title": "", + "type": "visualization" + }, + { + "embeddableConfig": { + "attributes": { + "references": [], + "state": { + "adHocDataViews": { + "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "name": "billing_cluster_cost_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_cluster_cost_lookup", + "type": "esql" + } + }, + "datasourceStates": { + "textBased": { + "indexPatternRefs": [ + { + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "timeField": "@timestamp", + "title": "billing_cluster_cost_lookup" + } + ], + "layers": { + "1a2f8269-f56f-4c02-8506-76b723acc828": { + "columns": [ + { + "columnId": "agg_querying", + "customLabel": true, + "fieldName": "agg_querying", + "inMetricDimension": true, + "label": "Normalised cost", + "meta": { + "esType": "double", + "type": "number" + }, + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + } + }, + { + "columnId": "@timestamp", + "customLabel": true, + "fieldName": "@timestamp", + "label": "Date", + "meta": { + "esType": "date", + "type": "date" + } + }, + { + "columnId": "datastream", + "customLabel": true, + "fieldName": "datastream", + "label": "Data stream", + "meta": { + "esType": "keyword", + "type": "string" + } + } + ], + "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| EVAL \n querying = CASE (\n deployment_sum_query_time > 0,\n datastream_sum_query_time / deployment_sum_query_time * total_ecu\n ) * conf_ecu_rate\n| STATS \n agg_querying = sum(querying) \n BY \n @timestamp, \n datastream\n| WHERE agg_querying > 0\n| SORT agg_querying DESC" + }, + "timeField": "@timestamp" + } + } + } + }, + "filters": [], + "needsRefresh": false, + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| EVAL \n querying = CASE (\n deployment_sum_query_time > 0,\n datastream_sum_query_time / deployment_sum_query_time * total_ecu\n ) * conf_ecu_rate\n| STATS \n agg_querying = sum(querying) \n BY \n @timestamp, \n datastream\n| WHERE agg_querying > 0\n| SORT agg_querying DESC" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "agg_querying" + ], + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "layerId": "1a2f8269-f56f-4c02-8506-76b723acc828", + "layerType": "data", + "seriesType": "bar_stacked", + "splitAccessor": "datastream", + "xAccessor": "@timestamp" + } + ], + "legend": { + "isVisible": false, + "position": "right", + "showSingleSeries": false + }, + "preferredSeriesType": "bar_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "agg_querying over @timestamp", + "visualizationType": "lnsXY" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| EVAL \n querying = CASE (\n deployment_sum_query_time > 0,\n datastream_sum_query_time / deployment_sum_query_time * total_ecu\n ) * conf_ecu_rate\n| STATS \n agg_querying = sum(querying) \n BY \n @timestamp, \n datastream\n| WHERE agg_querying > 0\n| SORT agg_querying DESC" + }, + "searchSessionId": "48b157e3-14ff-4c71-8178-4a69f0592d85", + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 14, + "i": "83bbd574-571b-4401-af7f-d0998cd9af2e", + "w": 16, + "x": 0, + "y": 95 + }, + "panelIndex": "83bbd574-571b-4401-af7f-d0998cd9af2e", + "title": "Querying: Normalised cost per data stream per day", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [], + "state": { + "adHocDataViews": { + "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "name": "billing_cluster_cost_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_cluster_cost_lookup", + "type": "esql" + } + }, + "datasourceStates": { + "textBased": { + "indexPatternRefs": [ + { + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "timeField": "@timestamp", + "title": "billing_cluster_cost_lookup" + } + ], + "layers": { + "d24af063-ec41-4736-bf4e-70f1fe4da4cf": { + "columns": [ + { + "columnId": "agg_indexing", + "customLabel": true, + "fieldName": "agg_indexing", + "inMetricDimension": true, + "label": "Normalised cost", + "meta": { + "esType": "double", + "type": "number" + }, + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + } + }, + { + "columnId": "@timestamp", + "customLabel": true, + "fieldName": "@timestamp", + "label": "Date", + "meta": { + "esType": "date", + "type": "date" + } + }, + { + "columnId": "tier", + "customLabel": true, + "fieldName": "tier", + "label": "Data tier", + "meta": { + "esType": "keyword", + "type": "string" + } + } + ], + "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| EVAL \n indexing = CASE (\n deployment_sum_indexing_time > 0,\n tier_sum_indexing_time / deployment_sum_indexing_time * total_ecu\n ) * conf_ecu_rate\n| STATS \n agg_indexing = sum(indexing)\n BY \n @timestamp,\n tier\n| WHERE agg_indexing > 0" + }, + "timeField": "@timestamp" + } + } + } + }, + "filters": [], + "needsRefresh": false, + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| EVAL \n indexing = CASE (\n deployment_sum_indexing_time > 0,\n tier_sum_indexing_time / deployment_sum_indexing_time * total_ecu\n ) * conf_ecu_rate\n| STATS \n agg_indexing = sum(indexing)\n BY \n @timestamp,\n tier\n| WHERE agg_indexing > 0" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "agg_indexing" + ], + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "layerId": "d24af063-ec41-4736-bf4e-70f1fe4da4cf", + "layerType": "data", + "seriesType": "bar_stacked", + "splitAccessor": "tier", + "xAccessor": "@timestamp" + } + ], + "legend": { + "isVisible": false, + "position": "right", + "showSingleSeries": false + }, + "preferredSeriesType": "bar_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "agg_indexing over @timestamp", + "visualizationType": "lnsXY" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| EVAL \n indexing = CASE (\n deployment_sum_indexing_time > 0,\n tier_sum_indexing_time / deployment_sum_indexing_time * total_ecu\n ) * conf_ecu_rate\n| STATS \n agg_indexing = sum(indexing)\n BY \n @timestamp,\n tier\n| WHERE agg_indexing > 0" + }, + "searchSessionId": "48b157e3-14ff-4c71-8178-4a69f0592d85", + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 14, + "i": "45a29dd3-1990-4717-bf6b-25a5d85d2344", + "w": 16, + "x": 16, + "y": 78 + }, + "panelIndex": "45a29dd3-1990-4717-bf6b-25a5d85d2344", + "title": "Indexing: Normalised cost per tier per day", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [], + "state": { + "adHocDataViews": { + "3db0d6b31d8afd75067c7a445bb4f066a52d9d451e8237e51956bd0d7e1ecb52": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "3db0d6b31d8afd75067c7a445bb4f066a52d9d451e8237e51956bd0d7e1ecb52", + "name": "monitoring-indices", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "monitoring-indices", + "type": "esql" + } + }, + "datasourceStates": { + "textBased": { + "indexPatternRefs": [ + { + "id": "3db0d6b31d8afd75067c7a445bb4f066a52d9d451e8237e51956bd0d7e1ecb52", + "timeField": "@timestamp", + "title": "monitoring-indices" + } + ], + "layers": { + "d5520b0e-567f-4f0a-b424-fc79eb79e2cc": { + "columns": [ + { + "columnId": "@timestamp", + "customLabel": true, + "fieldName": "@timestamp", + "label": "Date", + "meta": { + "esType": "date", + "type": "date" + } + }, + { + "columnId": "datastream", + "customLabel": true, + "fieldName": "datastream", + "label": "Data stream", + "meta": { + "esType": "keyword", + "type": "string" + } + }, + { + "columnId": "agg_indexing", + "customLabel": true, + "fieldName": "agg_indexing", + "inMetricDimension": true, + "label": "% Normalised cost", + "meta": { + "esType": "long", + "type": "number" + }, + "params": { + "format": { + "id": "percent", + "params": { + "decimals": 0 + } + } + } + } + ], + "index": "3db0d6b31d8afd75067c7a445bb4f066a52d9d451e8237e51956bd0d7e1ecb52", + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| EVAL \n indexing = CASE (\n deployment_sum_indexing_time > 0,\n datastream_sum_indexing_time / deployment_sum_indexing_time * total_ecu\n ) * conf_ecu_rate\n| STATS \n agg_indexing = sum(indexing)\n BY \n @timestamp,\n datastream\n| WHERE agg_indexing > 0\n| SORT agg_indexing DESC" + }, + "timeField": "@timestamp" + } + } + } + }, + "filters": [], + "needsRefresh": false, + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| EVAL \n indexing = CASE (\n deployment_sum_indexing_time > 0,\n datastream_sum_indexing_time / deployment_sum_indexing_time * total_ecu\n ) * conf_ecu_rate\n| STATS \n agg_indexing = sum(indexing)\n BY \n @timestamp,\n datastream\n| WHERE agg_indexing > 0\n| SORT agg_indexing DESC" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "agg_indexing" + ], + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "layerId": "d5520b0e-567f-4f0a-b424-fc79eb79e2cc", + "layerType": "data", + "seriesType": "bar_percentage_stacked", + "splitAccessor": "datastream", + "xAccessor": "@timestamp" + } + ], + "legend": { + "isVisible": false, + "position": "right", + "showSingleSeries": false + }, + "preferredSeriesType": "bar_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "agg_indexing over @timestamp", + "visualizationType": "lnsXY" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| EVAL \n indexing = CASE (\n deployment_sum_indexing_time > 0,\n datastream_sum_indexing_time / deployment_sum_indexing_time * total_ecu\n ) * conf_ecu_rate\n| STATS \n agg_indexing = sum(indexing)\n BY \n @timestamp,\n datastream\n| WHERE agg_indexing > 0\n| SORT agg_indexing DESC" + }, + "searchSessionId": "48b157e3-14ff-4c71-8178-4a69f0592d85", + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 14, + "i": "9bc8fa5a-d47a-4a79-ab6a-032d726f085f", + "w": 16, + "x": 32, + "y": 78 + }, + "panelIndex": "9bc8fa5a-d47a-4a79-ab6a-032d726f085f", + "title": "Indexing: % Normalised cost per data stream per day", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [], + "state": { + "adHocDataViews": { + "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "name": "billing_cluster_cost_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_cluster_cost_lookup", + "type": "esql" + } + }, + "datasourceStates": { + "textBased": { + "indexPatternRefs": [ + { + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "timeField": "@timestamp", + "title": "billing_cluster_cost_lookup" + } + ], + "layers": { + "61842c44-afd9-429c-bf2e-18a4a079e5e8": { + "columns": [ + { + "columnId": "agg_querying", + "customLabel": true, + "fieldName": "agg_querying", + "inMetricDimension": true, + "label": "Normalised cost", + "meta": { + "esType": "double", + "type": "number" + }, + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + } + }, + { + "columnId": "@timestamp", + "customLabel": true, + "fieldName": "@timestamp", + "label": "Date", + "meta": { + "esType": "date", + "type": "date" + } + }, + { + "columnId": "tier", + "customLabel": false, + "fieldName": "tier", + "label": "tier", + "meta": { + "esType": "keyword", + "type": "string" + } + } + ], + "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| EVAL \n querying = CASE (\n deployment_sum_query_time > 0,\n tier_sum_query_time / deployment_sum_query_time * total_ecu\n ) * conf_ecu_rate\n| STATS \n agg_querying = sum(querying)\n BY \n @timestamp,\n tier\n| WHERE agg_querying > 0" + }, + "timeField": "@timestamp" + } + } + } + }, + "filters": [], + "needsRefresh": false, + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| EVAL \n querying = CASE (\n deployment_sum_query_time > 0,\n tier_sum_query_time / deployment_sum_query_time * total_ecu\n ) * conf_ecu_rate\n| STATS \n agg_querying = sum(querying)\n BY \n @timestamp,\n tier\n| WHERE agg_querying > 0" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "agg_querying" + ], + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "layerId": "61842c44-afd9-429c-bf2e-18a4a079e5e8", + "layerType": "data", + "seriesType": "bar_stacked", + "splitAccessor": "tier", + "xAccessor": "@timestamp" + } + ], + "legend": { + "isVisible": false, + "position": "right", + "showSingleSeries": false + }, + "preferredSeriesType": "bar_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "agg_querying over @timestamp", + "visualizationType": "lnsXY" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| EVAL \n querying = CASE (\n deployment_sum_query_time > 0,\n tier_sum_query_time / deployment_sum_query_time * total_ecu\n ) * conf_ecu_rate\n| STATS \n agg_querying = sum(querying)\n BY \n @timestamp,\n tier\n| WHERE agg_querying > 0" + }, + "searchSessionId": "48b157e3-14ff-4c71-8178-4a69f0592d85", + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 14, + "i": "b0df8c79-87d4-4a7a-8c7b-4752862fb2dd", + "w": 16, + "x": 16, + "y": 95 + }, + "panelIndex": "b0df8c79-87d4-4a7a-8c7b-4752862fb2dd", + "title": "Querying: Noralised cost per tier per day", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [], + "state": { + "adHocDataViews": { + "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "name": "billing_cluster_cost_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_cluster_cost_lookup", + "type": "esql" + } + }, + "datasourceStates": { + "textBased": { + "indexPatternRefs": [ + { + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "timeField": "@timestamp", + "title": "billing_cluster_cost_lookup" + } + ], + "layers": { + "ba345378-8ee0-40ec-9fb0-d26588d09107": { + "columns": [ + { + "columnId": "c33c8ff5-8a22-4f4f-9b92-5e16485f5d9a", + "customLabel": true, + "fieldName": "deployment", + "label": "Deployment", + "meta": { + "esType": "keyword", + "sourceParams": { + "indexPattern": "billing_cluster_cost_lookup" + }, + "type": "string" + } + }, + { + "columnId": "2ab259ce-29c3-4c5c-817b-b7f036c62be8", + "customLabel": true, + "fieldName": "agg_blended", + "label": "Normalised cost", + "meta": { + "esType": "double", + "sourceParams": { + "indexPattern": "billing_cluster_cost_lookup" + }, + "type": "number" + }, + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + } + }, + { + "columnId": "af3de6ff-d14d-4cea-bb56-4ee9012054b6", + "customLabel": true, + "fieldName": "tier", + "label": "Data tier", + "meta": { + "esType": "keyword", + "sourceParams": { + "indexPattern": "billing_cluster_cost_lookup" + }, + "type": "string" + } + } + ], + "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| EVAL indexing = CASE (deployment_sum_indexing_time > 0, tier_sum_indexing_time / deployment_sum_indexing_time * total_ecu), querying = CASE (deployment_sum_query_time > 0, tier_sum_query_time / deployment_sum_query_time * total_ecu), data_set = CASE (deployment_sum_data_set_store_size > 0, tier_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu), store = CASE (deployment_sum_store_size > 0, tier_sum_store_size / deployment_sum_store_size * total_ecu), storage = CASE (store == 0, data_set, store), total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight, total_weight_cold = conf_storage_weight + conf_query_weight, blended = CASE (tier == \"hot/content\", ((storage * conf_storage_weight) + (querying * conf_query_weight) + (indexing * conf_indexing_weight)) / total_weight_hot, ((storage * conf_storage_weight) + (querying * conf_query_weight)) / total_weight_cold) * conf_ecu_rate, deployment = CONCAT(deployment_name, \" (\", LEFT(deployment_id, 6), \")\")\n| STATS agg_blended = sum(blended) BY deployment, tier\n| KEEP deployment, tier, agg_blended\n| WHERE agg_blended > 0" + }, + "timeField": "@timestamp" + } + } + } + }, + "filters": [], + "needsRefresh": false, + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| EVAL indexing = CASE (deployment_sum_indexing_time > 0, tier_sum_indexing_time / deployment_sum_indexing_time * total_ecu), querying = CASE (deployment_sum_query_time > 0, tier_sum_query_time / deployment_sum_query_time * total_ecu), data_set = CASE (deployment_sum_data_set_store_size > 0, tier_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu), store = CASE (deployment_sum_store_size > 0, tier_sum_store_size / deployment_sum_store_size * total_ecu), storage = CASE (store == 0, data_set, store), total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight, total_weight_cold = conf_storage_weight + conf_query_weight, blended = CASE (tier == \"hot/content\", ((storage * conf_storage_weight) + (querying * conf_query_weight) + (indexing * conf_indexing_weight)) / total_weight_hot, ((storage * conf_storage_weight) + (querying * conf_query_weight)) / total_weight_cold) * conf_ecu_rate, deployment = CONCAT(deployment_name, \" (\", LEFT(deployment_id, 6), \")\")\n| STATS agg_blended = sum(blended) BY deployment, tier\n| KEEP deployment, tier, agg_blended\n| WHERE agg_blended > 0" + }, + "visualization": { + "columns": [ + { + "columnId": "c33c8ff5-8a22-4f4f-9b92-5e16485f5d9a", + "isMetric": false, + "isTransposed": false + }, + { + "columnId": "2ab259ce-29c3-4c5c-817b-b7f036c62be8", + "isMetric": true, + "isTransposed": false + }, + { + "columnId": "af3de6ff-d14d-4cea-bb56-4ee9012054b6", + "isMetric": false, + "isTransposed": true + } + ], + "layerId": "ba345378-8ee0-40ec-9fb0-d26588d09107", + "layerType": "data" + } + }, + "title": "Table ", + "visualizationType": "lnsDatatable" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| EVAL indexing = CASE (deployment_sum_indexing_time > 0, tier_sum_indexing_time / deployment_sum_indexing_time * total_ecu), querying = CASE (deployment_sum_query_time > 0, tier_sum_query_time / deployment_sum_query_time * total_ecu), data_set = CASE (deployment_sum_data_set_store_size > 0, tier_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu), store = CASE (deployment_sum_store_size > 0, tier_sum_store_size / deployment_sum_store_size * total_ecu), storage = CASE (store == 0, data_set, store), total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight, total_weight_cold = conf_storage_weight + conf_query_weight, blended = CASE (tier == \"hot/content\", ((storage * conf_storage_weight) + (querying * conf_query_weight) + (indexing * conf_indexing_weight)) / total_weight_hot, ((storage * conf_storage_weight) + (querying * conf_query_weight)) / total_weight_cold) * conf_ecu_rate, deployment = CONCAT(deployment_name, \" (\", LEFT(deployment_id, 6), \")\")\n| STATS agg_blended = sum(blended) BY deployment, tier\n| KEEP deployment, tier, agg_blended\n| WHERE agg_blended > 0" + }, + "searchSessionId": "48b157e3-14ff-4c71-8178-4a69f0592d85", + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 15, + "i": "8049f13b-c56d-4a6e-a4e3-5cf4274fef60", + "w": 24, + "x": 0, + "y": 11 + }, + "panelIndex": "8049f13b-c56d-4a6e-a4e3-5cf4274fef60", + "title": "Normalised cost per deployment and tier", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [], + "state": { + "adHocDataViews": { + "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "name": "billing_cluster_cost_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_cluster_cost_lookup", + "type": "esql" + } + }, + "datasourceStates": { + "textBased": { + "indexPatternRefs": [ + { + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "timeField": "@timestamp", + "title": "billing_cluster_cost_lookup" + } + ], + "layers": { + "91b4be84-6eb5-4b77-ad13-b85d992f3bbf": { + "columns": [ + { + "columnId": "deployment", + "customLabel": true, + "fieldName": "deployment", + "label": "Deployment", + "meta": { + "esType": "keyword", + "type": "string" + } + }, + { + "columnId": "tier", + "customLabel": true, + "fieldName": "tier", + "label": "Data tier", + "meta": { + "esType": "keyword", + "type": "string" + } + }, + { + "columnId": "agg_blended", + "customLabel": true, + "fieldName": "agg_blended", + "inMetricDimension": true, + "label": "Normalised cost", + "meta": { + "esType": "double", + "type": "number" + }, + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + } + } + ], + "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| EVAL indexing = CASE (deployment_sum_indexing_time > 0, tier_sum_indexing_time / deployment_sum_indexing_time * total_ecu), querying = CASE (deployment_sum_query_time > 0, tier_sum_query_time / deployment_sum_query_time * total_ecu), data_set = CASE (deployment_sum_data_set_store_size > 0, tier_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu), store = CASE (deployment_sum_store_size > 0, tier_sum_store_size / deployment_sum_store_size * total_ecu), storage = CASE (store == 0, data_set, store), total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight, total_weight_cold = conf_storage_weight + conf_query_weight, blended = CASE (tier == \"hot/content\", ((storage * conf_storage_weight) + (querying * conf_query_weight) + (indexing * conf_indexing_weight)) / total_weight_hot, ((storage * conf_storage_weight) + (querying * conf_query_weight)) / total_weight_cold) * conf_ecu_rate, deployment = CONCAT(deployment_name, \" (\", LEFT(deployment_id, 6), \")\")\n| STATS agg_blended = sum(blended) BY deployment, tier\n| KEEP deployment, tier, agg_blended\n| WHERE agg_blended > 0" + }, + "timeField": "@timestamp" + } + } + } + }, + "filters": [], + "needsRefresh": false, + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| EVAL indexing = CASE (deployment_sum_indexing_time > 0, tier_sum_indexing_time / deployment_sum_indexing_time * total_ecu), querying = CASE (deployment_sum_query_time > 0, tier_sum_query_time / deployment_sum_query_time * total_ecu), data_set = CASE (deployment_sum_data_set_store_size > 0, tier_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu), store = CASE (deployment_sum_store_size > 0, tier_sum_store_size / deployment_sum_store_size * total_ecu), storage = CASE (store == 0, data_set, store), total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight, total_weight_cold = conf_storage_weight + conf_query_weight, blended = CASE (tier == \"hot/content\", ((storage * conf_storage_weight) + (querying * conf_query_weight) + (indexing * conf_indexing_weight)) / total_weight_hot, ((storage * conf_storage_weight) + (querying * conf_query_weight)) / total_weight_cold) * conf_ecu_rate, deployment = CONCAT(deployment_name, \" (\", LEFT(deployment_id, 6), \")\")\n| STATS agg_blended = sum(blended) BY deployment, tier\n| KEEP deployment, tier, agg_blended\n| WHERE agg_blended > 0" + }, + "visualization": { + "layers": [ + { + "categoryDisplay": "default", + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "layerId": "91b4be84-6eb5-4b77-ad13-b85d992f3bbf", + "layerType": "data", + "legendDisplay": "default", + "metrics": [ + "agg_blended" + ], + "nestedLegend": false, + "numberDisplay": "percent", + "primaryGroups": [ + "deployment", + "tier" + ] + } + ], + "shape": "treemap" + } + }, + "title": "Pie", + "visualizationType": "lnsPie" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| EVAL indexing = CASE (deployment_sum_indexing_time > 0, tier_sum_indexing_time / deployment_sum_indexing_time * total_ecu), querying = CASE (deployment_sum_query_time > 0, tier_sum_query_time / deployment_sum_query_time * total_ecu), data_set = CASE (deployment_sum_data_set_store_size > 0, tier_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu), store = CASE (deployment_sum_store_size > 0, tier_sum_store_size / deployment_sum_store_size * total_ecu), storage = CASE (store == 0, data_set, store), total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight, total_weight_cold = conf_storage_weight + conf_query_weight, blended = CASE (tier == \"hot/content\", ((storage * conf_storage_weight) + (querying * conf_query_weight) + (indexing * conf_indexing_weight)) / total_weight_hot, ((storage * conf_storage_weight) + (querying * conf_query_weight)) / total_weight_cold) * conf_ecu_rate, deployment = CONCAT(deployment_name, \" (\", LEFT(deployment_id, 6), \")\")\n| STATS agg_blended = sum(blended) BY deployment, tier\n| KEEP deployment, tier, agg_blended\n| WHERE agg_blended > 0" + }, + "searchSessionId": "48b157e3-14ff-4c71-8178-4a69f0592d85", + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 15, + "i": "normalised-cost-per-deployment-and-tier", + "w": 24, + "x": 24, + "y": 11 + }, + "panelIndex": "normalised-cost-per-deployment-and-tier", + "title": "Normalised cost per deployment and tier", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [], + "state": { + "adHocDataViews": { + "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "name": "billing_cluster_cost_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_cluster_cost_lookup", + "type": "esql" + } + }, + "datasourceStates": { + "textBased": { + "indexPatternRefs": [ + { + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "timeField": "@timestamp", + "title": "billing_cluster_cost_lookup" + } + ], + "layers": { + "a21abe71-d247-47eb-8c2f-81f2cf30b387": { + "columns": [ + { + "columnId": "agg_blended", + "customLabel": false, + "fieldName": "agg_blended", + "inMetricDimension": true, + "label": "Blended", + "meta": { + "esType": "double", + "sourceParams": { + "indexPattern": "billing_cluster_cost_lookup" + }, + "type": "number" + }, + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + } + }, + { + "columnId": "datastream", + "customLabel": true, + "fieldName": "datastream", + "inMetricDimension": true, + "label": "Data stream", + "meta": { + "esType": "keyword", + "type": "string" + } + } + ], + "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, datastream_sum_indexing_time / deployment_sum_indexing_time * total_ecu),\n querying = CASE (deployment_sum_query_time > 0, datastream_sum_query_time / deployment_sum_query_time * total_ecu),\n data_set = CASE (deployment_sum_data_set_store_size > 0, datastream_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE (deployment_sum_store_size > 0, datastream_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE (store == 0, data_set, store),\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE (\n tier == \"hot/content\",\n ((storage * conf_storage_weight) + (querying * conf_query_weight) + (indexing * conf_indexing_weight)) / total_weight_hot,\n ((storage * conf_storage_weight) + (querying * conf_query_weight)) / total_weight_cold\n ) * conf_ecu_rate\n| STATS\n agg_blended = sum(blended)\n BY\n datastream,\n tier\n| WHERE agg_blended > 0\n| SORT agg_blended DESC" + }, + "timeField": "@timestamp" + } + } + } + }, + "filters": [], + "needsRefresh": false, + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, datastream_sum_indexing_time / deployment_sum_indexing_time * total_ecu),\n querying = CASE (deployment_sum_query_time > 0, datastream_sum_query_time / deployment_sum_query_time * total_ecu),\n data_set = CASE (deployment_sum_data_set_store_size > 0, datastream_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE (deployment_sum_store_size > 0, datastream_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE (store == 0, data_set, store),\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE (\n tier == \"hot/content\",\n ((storage * conf_storage_weight) + (querying * conf_query_weight) + (indexing * conf_indexing_weight)) / total_weight_hot,\n ((storage * conf_storage_weight) + (querying * conf_query_weight)) / total_weight_cold\n ) * conf_ecu_rate\n| STATS\n agg_blended = sum(blended)\n BY\n datastream,\n tier\n| WHERE agg_blended > 0\n| SORT agg_blended DESC" + }, + "visualization": { + "layers": [ + { + "categoryDisplay": "default", + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "emptySizeRatio": 0.3, + "layerId": "a21abe71-d247-47eb-8c2f-81f2cf30b387", + "layerType": "data", + "legendDisplay": "default", + "metrics": [ + "agg_blended" + ], + "nestedLegend": false, + "numberDisplay": "percent", + "primaryGroups": [ + "datastream" + ] + } + ], + "shape": "donut" + } + }, + "title": "agg_blended of datastream", + "visualizationType": "lnsPie" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, datastream_sum_indexing_time / deployment_sum_indexing_time * total_ecu),\n querying = CASE (deployment_sum_query_time > 0, datastream_sum_query_time / deployment_sum_query_time * total_ecu),\n data_set = CASE (deployment_sum_data_set_store_size > 0, datastream_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE (deployment_sum_store_size > 0, datastream_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE (store == 0, data_set, store),\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE (\n tier == \"hot/content\",\n ((storage * conf_storage_weight) + (querying * conf_query_weight) + (indexing * conf_indexing_weight)) / total_weight_hot,\n ((storage * conf_storage_weight) + (querying * conf_query_weight)) / total_weight_cold\n ) * conf_ecu_rate\n| STATS\n agg_blended = sum(blended)\n BY\n datastream,\n tier\n| WHERE agg_blended > 0\n| SORT agg_blended DESC" + }, + "searchSessionId": "48b157e3-14ff-4c71-8178-4a69f0592d85", + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 15, + "i": "cb467fdf-5cb1-41b5-8235-8b823cbd9310", + "w": 12, + "x": 36, + "y": 42 + }, + "panelIndex": "cb467fdf-5cb1-41b5-8235-8b823cbd9310", + "title": "Blended: Normalised cost totals by data stream", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [], + "state": { + "adHocDataViews": { + "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "name": "billing_cluster_cost_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_cluster_cost_lookup", + "type": "esql" + } + }, + "datasourceStates": { + "textBased": { + "indexPatternRefs": [ + { + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "timeField": "@timestamp", + "title": "billing_cluster_cost_lookup" + } + ], + "layers": { + "0f6d5278-b325-49bc-9eb7-2febd2a5de0f": { + "columns": [ + { + "columnId": "agg_indexing", + "customLabel": true, + "fieldName": "agg_indexing", + "inMetricDimension": true, + "label": "Indexing", + "meta": { + "esType": "double", + "type": "number" + }, + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + } + }, + { + "columnId": "datastream", + "customLabel": true, + "fieldName": "datastream", + "label": "Data stream", + "meta": { + "esType": "keyword", + "type": "string" + } + } + ], + "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL \n indexing = CASE (deployment_sum_indexing_time > 0, datastream_sum_indexing_time / deployment_sum_indexing_time * total_ecu) * conf_ecu_rate\n| STATS \n agg_indexing = sum(indexing)\n BY \n datastream\n| WHERE agg_indexing > 0\n| SORT agg_indexing DESC" + }, + "timeField": "@timestamp" + } + } + } + }, + "filters": [], + "needsRefresh": false, + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL \n indexing = CASE (deployment_sum_indexing_time > 0, datastream_sum_indexing_time / deployment_sum_indexing_time * total_ecu) * conf_ecu_rate\n| STATS \n agg_indexing = sum(indexing)\n BY \n datastream\n| WHERE agg_indexing > 0\n| SORT agg_indexing DESC" + }, + "visualization": { + "layers": [ + { + "categoryDisplay": "default", + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "emptySizeRatio": 0.3, + "layerId": "0f6d5278-b325-49bc-9eb7-2febd2a5de0f", + "layerType": "data", + "legendDisplay": "default", + "metrics": [ + "agg_indexing" + ], + "nestedLegend": false, + "numberDisplay": "percent", + "primaryGroups": [ + "datastream" + ] + } + ], + "shape": "donut" + } + }, + "title": "Bar vertical stacked", + "visualizationType": "lnsPie" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL \n indexing = CASE (deployment_sum_indexing_time > 0, datastream_sum_indexing_time / deployment_sum_indexing_time * total_ecu) * conf_ecu_rate\n| STATS \n agg_indexing = sum(indexing)\n BY \n datastream\n| WHERE agg_indexing > 0\n| SORT agg_indexing DESC" + }, + "searchSessionId": "48b157e3-14ff-4c71-8178-4a69f0592d85", + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 15, + "i": "ed65354e-848c-4d93-bcd6-b9c50793e242", + "w": 12, + "x": 0, + "y": 42 + }, + "panelIndex": "ed65354e-848c-4d93-bcd6-b9c50793e242", + "title": "Indexing: Normalised cost totals by data stream", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [], + "state": { + "adHocDataViews": { + "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "name": "billing_cluster_cost_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_cluster_cost_lookup", + "type": "esql" + } + }, + "datasourceStates": { + "textBased": { + "indexPatternRefs": [ + { + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "timeField": "@timestamp", + "title": "billing_cluster_cost_lookup" + } + ], + "layers": { + "3b03b6a8-35ae-4554-8bd6-a887d8b36748": { + "columns": [ + { + "columnId": "agg_querying", + "customLabel": true, + "fieldName": "agg_querying", + "inMetricDimension": true, + "label": "Querying", + "meta": { + "esType": "double", + "type": "number" + }, + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + } + }, + { + "columnId": "datastream", + "customLabel": true, + "fieldName": "datastream", + "label": "Data stream", + "meta": { + "esType": "keyword", + "type": "string" + } + } + ], + "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL \n querying = CASE (deployment_sum_query_time > 0, datastream_sum_query_time / deployment_sum_query_time * total_ecu) * conf_ecu_rate\n| STATS \n agg_querying = sum(querying)\n BY \n datastream\n| WHERE agg_querying > 0\n| SORT agg_querying DESC" + }, + "timeField": "@timestamp" + } + } + } + }, + "filters": [], + "needsRefresh": false, + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL \n querying = CASE (deployment_sum_query_time > 0, datastream_sum_query_time / deployment_sum_query_time * total_ecu) * conf_ecu_rate\n| STATS \n agg_querying = sum(querying)\n BY \n datastream\n| WHERE agg_querying > 0\n| SORT agg_querying DESC" + }, + "visualization": { + "layers": [ + { + "categoryDisplay": "default", + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "emptySizeRatio": 0.3, + "layerId": "3b03b6a8-35ae-4554-8bd6-a887d8b36748", + "layerType": "data", + "legendDisplay": "default", + "metrics": [ + "agg_querying" + ], + "nestedLegend": false, + "numberDisplay": "percent", + "primaryGroups": [ + "datastream" + ] + } + ], + "shape": "donut" + } + }, + "title": "Bar vertical stacked", + "visualizationType": "lnsPie" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL \n querying = CASE (deployment_sum_query_time > 0, datastream_sum_query_time / deployment_sum_query_time * total_ecu) * conf_ecu_rate\n| STATS \n agg_querying = sum(querying)\n BY \n datastream\n| WHERE agg_querying > 0\n| SORT agg_querying DESC" + }, + "searchSessionId": "48b157e3-14ff-4c71-8178-4a69f0592d85", + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 15, + "i": "4a2ecf27-21c7-4fff-986b-05b25e0ce8f8", + "w": 12, + "x": 12, + "y": 42 + }, + "panelIndex": "4a2ecf27-21c7-4fff-986b-05b25e0ce8f8", + "title": "Querying: Normalised cost totals by data stream", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [], + "state": { + "adHocDataViews": { + "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "name": "billing_cluster_cost_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_cluster_cost_lookup", + "type": "esql" + } + }, + "datasourceStates": { + "textBased": { + "indexPatternRefs": [ + { + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "timeField": "@timestamp", + "title": "billing_cluster_cost_lookup" + } + ], + "layers": { + "2813628b-2a20-484c-9a19-11955d86e9ff": { + "columns": [ + { + "columnId": "agg_storage", + "customLabel": true, + "fieldName": "agg_storage", + "inMetricDimension": true, + "label": "Storage", + "meta": { + "esType": "double", + "type": "number" + }, + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + } + }, + { + "columnId": "datastream", + "customLabel": true, + "fieldName": "datastream", + "label": "Data stream", + "meta": { + "esType": "keyword", + "type": "string" + } + } + ], + "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, datastream_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE (deployment_sum_store_size > 0, datastream_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE (store == 0, data_set, store) * conf_ecu_rate\n| STATS \n agg_storage = sum(storage)\n BY \n datastream\n| WHERE agg_storage > 0\n| SORT agg_storage DESC" + }, + "timeField": "@timestamp" + } + } + } + }, + "filters": [], + "needsRefresh": false, + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, datastream_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE (deployment_sum_store_size > 0, datastream_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE (store == 0, data_set, store) * conf_ecu_rate\n| STATS \n agg_storage = sum(storage)\n BY \n datastream\n| WHERE agg_storage > 0\n| SORT agg_storage DESC" + }, + "visualization": { + "layers": [ + { + "categoryDisplay": "default", + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "emptySizeRatio": 0.3, + "layerId": "2813628b-2a20-484c-9a19-11955d86e9ff", + "layerType": "data", + "legendDisplay": "default", + "metrics": [ + "agg_storage" + ], + "nestedLegend": false, + "numberDisplay": "percent", + "primaryGroups": [ + "datastream" + ] + } + ], + "shape": "donut" + } + }, + "title": "Bar vertical stacked", + "visualizationType": "lnsPie" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, datastream_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE (deployment_sum_store_size > 0, datastream_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE (store == 0, data_set, store) * conf_ecu_rate\n| STATS \n agg_storage = sum(storage)\n BY \n datastream\n| WHERE agg_storage > 0\n| SORT agg_storage DESC" + }, + "searchSessionId": "48b157e3-14ff-4c71-8178-4a69f0592d85", + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 15, + "i": "3269b898-7190-40b6-ad0e-d29fcf98e762", + "w": 12, + "x": 24, + "y": 42 + }, + "panelIndex": "3269b898-7190-40b6-ad0e-d29fcf98e762", + "title": "Storage: Normalised cost totals by data stream", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [], + "state": { + "adHocDataViews": { + "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "allowHidden": false, + "allowNoIndex": false, + "fieldAttrs": null, + "fieldFormats": {}, + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "name": "billing_cluster_cost_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_cluster_cost_lookup", + "type": "esql" + } + }, + "datasourceStates": { + "textBased": { + "indexPatternRefs": [ + { + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "timeField": "@timestamp", + "title": "billing_cluster_cost_lookup" + } + ], + "initialContext": null, + "layers": { + "e7e008d5-4100-482b-af1b-01db94a4274c": { + "columns": [ + { + "columnId": "3d98e1e4-e4df-47a2-9c7f-1efd2ecf0098", + "customLabel": true, + "fieldName": "datastream", + "label": "Data stream", + "meta": { + "esType": "keyword", + "sourceParams": { + "indexPattern": "billing_cluster_cost_lookup" + }, + "type": "string" + }, + "variable": null + }, + { + "columnId": "93aa5ff0-235f-4d8f-b926-748804244c15", + "customLabel": true, + "fieldName": "tier", + "label": "Data tier", + "meta": { + "esType": "keyword", + "sourceParams": { + "indexPattern": "billing_cluster_cost_lookup" + }, + "type": "string" + }, + "variable": null + }, + { + "columnId": "4db503d0-ae68-4f07-9b2f-42a2ef99af1f", + "customLabel": true, + "fieldName": "ecu_indexing", + "label": "Indexing", + "meta": { + "esType": "double", + "sourceParams": { + "indexPattern": "billing_cluster_cost_lookup" + }, + "type": "number" + }, + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + }, + "variable": null + }, + { + "columnId": "18a2dacc-eb7c-450d-8eee-8cc9a09a6362", + "customLabel": true, + "fieldName": "ecu_querying", + "label": "Querying", + "meta": { + "esType": "double", + "sourceParams": { + "indexPattern": "billing_cluster_cost_lookup" + }, + "type": "number" + }, + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + }, + "variable": null + }, + { + "columnId": "6cd474f2-dda5-4131-b0d9-9f99b0f64232", + "customLabel": true, + "fieldName": "ecu_storage", + "label": "Storage", + "meta": { + "esType": "double", + "sourceParams": { + "indexPattern": "billing_cluster_cost_lookup" + }, + "type": "number" + }, + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + }, + "variable": null + }, + { + "columnId": "59ed77e7-b251-475a-8f30-b905e253f944", + "customLabel": true, + "fieldName": "ecu_blended", + "label": "Blended", + "meta": { + "esType": "double", + "sourceParams": { + "indexPattern": "billing_cluster_cost_lookup" + }, + "type": "number" + }, + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + }, + "variable": null + } + ], + "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| EVAL\n indexing = CASE(deployment_sum_indexing_time > 0, tier_and_datastream_sum_indexing_time / deployment_sum_indexing_time * total_ecu) * conf_ecu_rate,\n querying = CASE(deployment_sum_query_time > 0, tier_and_datastream_sum_query_time / deployment_sum_query_time * total_ecu) * conf_ecu_rate,\n data_set = CASE(deployment_sum_data_set_store_size > 0, tier_and_datastream_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE(deployment_sum_store_size > 0, tier_and_datastream_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE(store == 0, data_set, store) * conf_ecu_rate,\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE(\n tier == \"hot/content\",\n ((storage / conf_ecu_rate * conf_storage_weight) + (querying / conf_ecu_rate * conf_query_weight) + (indexing / conf_ecu_rate * conf_indexing_weight)) / total_weight_hot,\n ((storage / conf_ecu_rate * conf_storage_weight) + (querying / conf_ecu_rate * conf_query_weight)) / total_weight_cold\n ) * conf_ecu_rate\n| STATS\n ecu_indexing = sum(indexing),\n ecu_querying = sum(querying),\n ecu_storage = sum(storage),\n ecu_blended = sum(blended)\n BY datastream, tier\n| SORT ecu_blended DESC" + }, + "timeField": "@timestamp" + } + } + } + }, + "filters": [], + "needsRefresh": false, + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| EVAL\n indexing = CASE(deployment_sum_indexing_time > 0, tier_and_datastream_sum_indexing_time / deployment_sum_indexing_time * total_ecu) * conf_ecu_rate,\n querying = CASE(deployment_sum_query_time > 0, tier_and_datastream_sum_query_time / deployment_sum_query_time * total_ecu) * conf_ecu_rate,\n data_set = CASE(deployment_sum_data_set_store_size > 0, tier_and_datastream_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE(deployment_sum_store_size > 0, tier_and_datastream_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE(store == 0, data_set, store) * conf_ecu_rate,\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE(\n tier == \"hot/content\",\n ((storage / conf_ecu_rate * conf_storage_weight) + (querying / conf_ecu_rate * conf_query_weight) + (indexing / conf_ecu_rate * conf_indexing_weight)) / total_weight_hot,\n ((storage / conf_ecu_rate * conf_storage_weight) + (querying / conf_ecu_rate * conf_query_weight)) / total_weight_cold\n ) * conf_ecu_rate\n| STATS\n ecu_indexing = sum(indexing),\n ecu_querying = sum(querying),\n ecu_storage = sum(storage),\n ecu_blended = sum(blended)\n BY datastream, tier\n| SORT ecu_blended DESC" + }, + "visualization": { + "columns": [ + { + "columnId": "3d98e1e4-e4df-47a2-9c7f-1efd2ecf0098", + "isMetric": false, + "isTransposed": false, + "width": 536.6666666666667 + }, + { + "columnId": "93aa5ff0-235f-4d8f-b926-748804244c15", + "isMetric": false, + "isTransposed": false + }, + { + "columnId": "4db503d0-ae68-4f07-9b2f-42a2ef99af1f", + "isMetric": true, + "isTransposed": false + }, + { + "columnId": "18a2dacc-eb7c-450d-8eee-8cc9a09a6362", + "isMetric": true, + "isTransposed": false + }, + { + "columnId": "6cd474f2-dda5-4131-b0d9-9f99b0f64232", + "isMetric": true, + "isTransposed": false + }, + { + "columnId": "59ed77e7-b251-475a-8f30-b905e253f944", + "isMetric": true, + "isTransposed": false + } + ], + "layerId": "e7e008d5-4100-482b-af1b-01db94a4274c", + "layerType": "data", + "sorting": null + } + }, + "title": "ecu_indexing \u0026 ecu_querying \u0026 ecu_storage \u0026 ecu_blended of datastream", + "visualizationType": "lnsDatatable" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_tier_and_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| EVAL\n indexing = CASE(deployment_sum_indexing_time > 0, tier_and_datastream_sum_indexing_time / deployment_sum_indexing_time * total_ecu) * conf_ecu_rate,\n querying = CASE(deployment_sum_query_time > 0, tier_and_datastream_sum_query_time / deployment_sum_query_time * total_ecu) * conf_ecu_rate,\n data_set = CASE(deployment_sum_data_set_store_size > 0, tier_and_datastream_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE(deployment_sum_store_size > 0, tier_and_datastream_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE(store == 0, data_set, store) * conf_ecu_rate,\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE(\n tier == \"hot/content\",\n ((storage / conf_ecu_rate * conf_storage_weight) + (querying / conf_ecu_rate * conf_query_weight) + (indexing / conf_ecu_rate * conf_indexing_weight)) / total_weight_hot,\n ((storage / conf_ecu_rate * conf_storage_weight) + (querying / conf_ecu_rate * conf_query_weight)) / total_weight_cold\n ) * conf_ecu_rate\n| STATS\n ecu_indexing = sum(indexing),\n ecu_querying = sum(querying),\n ecu_storage = sum(storage),\n ecu_blended = sum(blended)\n BY datastream, tier\n| SORT ecu_blended DESC" + }, + "searchSessionId": "48b157e3-14ff-4c71-8178-4a69f0592d85", + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 15, + "i": "964a716b-bb61-4cc0-ba77-c1e73e2da836", + "w": 48, + "x": 0, + "y": 57 + }, + "panelIndex": "964a716b-bb61-4cc0-ba77-c1e73e2da836", + "title": "Normalised cost per data stream per tier", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [], + "state": { + "adHocDataViews": { + "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "name": "billing_cluster_cost_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_cluster_cost_lookup", + "type": "esql" + } + }, + "datasourceStates": { + "textBased": { + "indexPatternRefs": [ + { + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "timeField": "@timestamp", + "title": "billing_cluster_cost_lookup" + } + ], + "layers": { + "52906f25-a114-4d45-b425-f0ff154f4ac1": { + "columns": [ + { + "columnId": "agg_blended", + "customLabel": true, + "fieldName": "agg_blended", + "inMetricDimension": true, + "label": "Normalised cost", + "meta": { + "esType": "double", + "type": "number" + }, + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + } + }, + { + "columnId": "@timestamp", + "customLabel": true, + "fieldName": "@timestamp", + "label": "Date", + "meta": { + "esType": "date", + "type": "date" + } + }, + { + "columnId": "datastream", + "customLabel": false, + "fieldName": "datastream", + "label": "datastream", + "meta": { + "esType": "keyword", + "type": "string" + } + } + ], + "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, datastream_sum_indexing_time / deployment_sum_indexing_time * total_ecu),\n querying = CASE (deployment_sum_query_time > 0, datastream_sum_query_time / deployment_sum_query_time * total_ecu),\n data_set = CASE (deployment_sum_data_set_store_size > 0, datastream_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE (deployment_sum_store_size > 0, datastream_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE (store == 0, data_set, store),\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n blended = ((storage * conf_storage_weight) + (querying * conf_query_weight) + (indexing * conf_indexing_weight)) / total_weight_hot * conf_ecu_rate\n| STATS\n agg_blended = sum(blended)\n BY\n @timestamp,\n datastream\n| WHERE agg_blended > 0\n| SORT agg_blended DESC" + }, + "timeField": "@timestamp" + } + } + } + }, + "filters": [], + "needsRefresh": false, + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, datastream_sum_indexing_time / deployment_sum_indexing_time * total_ecu),\n querying = CASE (deployment_sum_query_time > 0, datastream_sum_query_time / deployment_sum_query_time * total_ecu),\n data_set = CASE (deployment_sum_data_set_store_size > 0, datastream_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE (deployment_sum_store_size > 0, datastream_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE (store == 0, data_set, store),\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n blended = ((storage * conf_storage_weight) + (querying * conf_query_weight) + (indexing * conf_indexing_weight)) / total_weight_hot * conf_ecu_rate\n| STATS\n agg_blended = sum(blended)\n BY\n @timestamp,\n datastream\n| WHERE agg_blended > 0\n| SORT agg_blended DESC" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "agg_blended" + ], + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "layerId": "52906f25-a114-4d45-b425-f0ff154f4ac1", + "layerType": "data", + "seriesType": "bar_stacked", + "splitAccessor": "datastream", + "xAccessor": "@timestamp" + } + ], + "legend": { + "isVisible": false, + "position": "right", + "showSingleSeries": false + }, + "preferredSeriesType": "bar_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "agg_blended over @timestamp", + "visualizationType": "lnsXY" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, datastream_sum_indexing_time / deployment_sum_indexing_time * total_ecu),\n querying = CASE (deployment_sum_query_time > 0, datastream_sum_query_time / deployment_sum_query_time * total_ecu),\n data_set = CASE (deployment_sum_data_set_store_size > 0, datastream_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE (deployment_sum_store_size > 0, datastream_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE (store == 0, data_set, store),\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n blended = ((storage * conf_storage_weight) + (querying * conf_query_weight) + (indexing * conf_indexing_weight)) / total_weight_hot * conf_ecu_rate\n| STATS\n agg_blended = sum(blended)\n BY\n @timestamp,\n datastream\n| WHERE agg_blended > 0\n| SORT agg_blended DESC" + }, + "searchSessionId": "48b157e3-14ff-4c71-8178-4a69f0592d85", + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 14, + "i": "4c97109a-2048-470f-ae18-8c424ba2c12e", + "w": 16, + "x": 0, + "y": 130 + }, + "panelIndex": "4c97109a-2048-470f-ae18-8c424ba2c12e", + "title": "Blended value: Normalised cost per data stream per day", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [], + "state": { + "adHocDataViews": { + "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "name": "billing_cluster_cost_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_cluster_cost_lookup", + "type": "esql" + } + }, + "datasourceStates": { + "textBased": { + "indexPatternRefs": [ + { + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "timeField": "@timestamp", + "title": "billing_cluster_cost_lookup" + } + ], + "layers": { + "b72a0ab5-dbf5-4fd2-bdef-c855fe354bee": { + "columns": [ + { + "columnId": "agg_storage", + "customLabel": true, + "fieldName": "agg_storage", + "inMetricDimension": true, + "label": "Normalised cost", + "meta": { + "esType": "double", + "type": "number" + }, + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + } + }, + { + "columnId": "@timestamp", + "customLabel": true, + "fieldName": "@timestamp", + "label": "Date", + "meta": { + "esType": "date", + "type": "date" + } + }, + { + "columnId": "datastream", + "customLabel": true, + "fieldName": "datastream", + "label": "Data stream", + "meta": { + "esType": "keyword", + "type": "string" + } + } + ], + "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, datastream_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE (deployment_sum_store_size > 0, datastream_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE (store == 0, data_set, store) * conf_ecu_rate\n| STATS \n agg_storage = sum(storage)\n BY \n @timestamp, \n datastream\n| WHERE agg_storage > 0\n| SORT agg_storage DESC" + }, + "timeField": "@timestamp" + } + } + } + }, + "filters": [], + "needsRefresh": false, + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, datastream_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE (deployment_sum_store_size > 0, datastream_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE (store == 0, data_set, store) * conf_ecu_rate\n| STATS \n agg_storage = sum(storage)\n BY \n @timestamp, \n datastream\n| WHERE agg_storage > 0\n| SORT agg_storage DESC" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "agg_storage" + ], + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "layerId": "b72a0ab5-dbf5-4fd2-bdef-c855fe354bee", + "layerType": "data", + "seriesType": "bar_stacked", + "splitAccessor": "datastream", + "xAccessor": "@timestamp" + } + ], + "legend": { + "isVisible": false, + "position": "right", + "showSingleSeries": false + }, + "preferredSeriesType": "bar_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "agg_storage over @timestamp", + "visualizationType": "lnsXY" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, datastream_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE (deployment_sum_store_size > 0, datastream_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE (store == 0, data_set, store) * conf_ecu_rate\n| STATS \n agg_storage = sum(storage)\n BY \n @timestamp, \n datastream\n| WHERE agg_storage > 0\n| SORT agg_storage DESC" + }, + "searchSessionId": "48b157e3-14ff-4c71-8178-4a69f0592d85", + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 15, + "i": "6430d172-9b7f-45a0-a836-c46af0276b19", + "w": 16, + "x": 0, + "y": 112 + }, + "panelIndex": "6430d172-9b7f-45a0-a836-c46af0276b19", + "title": "Storage: Nomalised cost per data stream per day", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [], + "state": { + "adHocDataViews": { + "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "name": "billing_cluster_cost_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_cluster_cost_lookup", + "type": "esql" + } + }, + "datasourceStates": { + "textBased": { + "indexPatternRefs": [ + { + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "timeField": "@timestamp", + "title": "billing_cluster_cost_lookup" + } + ], + "layers": { + "91bcc41e-3a2c-434d-8df7-29c01340e712": { + "columns": [ + { + "columnId": "agg_querying", + "customLabel": true, + "fieldName": "agg_querying", + "inMetricDimension": true, + "label": "% Normalised cost", + "meta": { + "esType": "double", + "type": "number" + }, + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + } + }, + { + "columnId": "@timestamp", + "customLabel": true, + "fieldName": "@timestamp", + "label": "Date", + "meta": { + "esType": "date", + "type": "date" + } + }, + { + "columnId": "datastream", + "customLabel": true, + "fieldName": "datastream", + "label": "Data stream", + "meta": { + "esType": "keyword", + "type": "string" + } + } + ], + "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL \n querying = CASE (deployment_sum_query_time > 0, datastream_sum_query_time / deployment_sum_query_time * total_ecu) * conf_ecu_rate\n| STATS \n agg_querying = sum(querying)\n BY \n @timestamp, \n datastream\n| WHERE agg_querying > 0\n| SORT agg_querying DESC" + }, + "timeField": "@timestamp" + } + } + } + }, + "filters": [], + "needsRefresh": false, + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL \n querying = CASE (deployment_sum_query_time > 0, datastream_sum_query_time / deployment_sum_query_time * total_ecu) * conf_ecu_rate\n| STATS \n agg_querying = sum(querying)\n BY \n @timestamp, \n datastream\n| WHERE agg_querying > 0\n| SORT agg_querying DESC" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "agg_querying" + ], + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "layerId": "91bcc41e-3a2c-434d-8df7-29c01340e712", + "layerType": "data", + "seriesType": "bar_percentage_stacked", + "splitAccessor": "datastream", + "xAccessor": "@timestamp" + } + ], + "legend": { + "isVisible": false, + "position": "right", + "showSingleSeries": false + }, + "preferredSeriesType": "bar_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "agg_querying over @timestamp", + "visualizationType": "lnsXY" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL \n querying = CASE (deployment_sum_query_time > 0, datastream_sum_query_time / deployment_sum_query_time * total_ecu) * conf_ecu_rate\n| STATS \n agg_querying = sum(querying)\n BY \n @timestamp, \n datastream\n| WHERE agg_querying > 0\n| SORT agg_querying DESC" + }, + "searchSessionId": "48b157e3-14ff-4c71-8178-4a69f0592d85", + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 14, + "i": "b3e62c24-abb9-431e-a228-cbc5e7b8b02a", + "w": 16, + "x": 32, + "y": 95 + }, + "panelIndex": "b3e62c24-abb9-431e-a228-cbc5e7b8b02a", + "title": "Querying: % Normalised cost per data stream per day", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [], + "state": { + "adHocDataViews": { + "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "name": "billing_cluster_cost_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_cluster_cost_lookup", + "type": "esql" + } + }, + "datasourceStates": { + "textBased": { + "indexPatternRefs": [ + { + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "timeField": "@timestamp", + "title": "billing_cluster_cost_lookup" + } + ], + "layers": { + "b72a0ab5-dbf5-4fd2-bdef-c855fe354bee": { + "columns": [ + { + "columnId": "agg_storage", + "customLabel": true, + "fieldName": "agg_storage", + "inMetricDimension": true, + "label": "% Normalised cost", + "meta": { + "esType": "double", + "type": "number" + }, + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + } + }, + { + "columnId": "@timestamp", + "customLabel": true, + "fieldName": "@timestamp", + "label": "Date", + "meta": { + "esType": "date", + "type": "date" + } + }, + { + "columnId": "datastream", + "customLabel": true, + "fieldName": "datastream", + "label": "Data stream", + "meta": { + "esType": "keyword", + "type": "string" + } + } + ], + "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, datastream_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE (deployment_sum_store_size > 0, datastream_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE (store == 0, data_set, store) * conf_ecu_rate\n| STATS \n agg_storage = sum(storage)\n BY \n @timestamp, \n datastream\n| WHERE agg_storage > 0\n| SORT agg_storage DESC" + }, + "timeField": "@timestamp" + } + } + } + }, + "filters": [], + "needsRefresh": false, + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, datastream_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE (deployment_sum_store_size > 0, datastream_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE (store == 0, data_set, store) * conf_ecu_rate\n| STATS \n agg_storage = sum(storage)\n BY \n @timestamp, \n datastream\n| WHERE agg_storage > 0\n| SORT agg_storage DESC" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "agg_storage" + ], + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "layerId": "b72a0ab5-dbf5-4fd2-bdef-c855fe354bee", + "layerType": "data", + "seriesType": "bar_percentage_stacked", + "splitAccessor": "datastream", + "xAccessor": "@timestamp" + } + ], + "legend": { + "isVisible": false, + "position": "right", + "showSingleSeries": false + }, + "preferredSeriesType": "bar_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "agg_storage over @timestamp", + "visualizationType": "lnsXY" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, datastream_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE (deployment_sum_store_size > 0, datastream_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE (store == 0, data_set, store) * conf_ecu_rate\n| STATS \n agg_storage = sum(storage)\n BY \n @timestamp, \n datastream\n| WHERE agg_storage > 0\n| SORT agg_storage DESC" + }, + "searchSessionId": "48b157e3-14ff-4c71-8178-4a69f0592d85", + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 15, + "i": "a672f0fa-cdf6-4e27-a2c8-f753eef30a06", + "w": 16, + "x": 32, + "y": 112 + }, + "panelIndex": "a672f0fa-cdf6-4e27-a2c8-f753eef30a06", + "title": "Storage: % Normalised cost per data stream per day", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [], + "state": { + "adHocDataViews": { + "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "name": "billing_cluster_cost_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_cluster_cost_lookup", + "type": "esql" + } + }, + "datasourceStates": { + "textBased": { + "indexPatternRefs": [ + { + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "timeField": "@timestamp", + "title": "billing_cluster_cost_lookup" + } + ], + "layers": { + "8c2f087d-e623-4f6f-b232-97ab4284e4db": { + "columns": [ + { + "columnId": "@timestamp", + "customLabel": true, + "fieldName": "@timestamp", + "inMetricDimension": true, + "label": "Date", + "meta": { + "esType": "date", + "type": "date" + } + }, + { + "columnId": "30c47aa5-b389-479c-a2b6-d91c77a952b9", + "customLabel": true, + "fieldName": "agg_blended", + "label": "% Normalised cost", + "meta": { + "esType": "double", + "sourceParams": { + "indexPattern": "billing_cluster_cost_lookup" + }, + "type": "number" + }, + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + } + }, + { + "columnId": "57d5389c-1311-460d-afe2-cecc4a8d16f4", + "customLabel": true, + "fieldName": "datastream", + "label": "Data stream", + "meta": { + "esType": "keyword", + "sourceParams": { + "indexPattern": "billing_cluster_cost_lookup" + }, + "type": "string" + } + } + ], + "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, datastream_sum_indexing_time / deployment_sum_indexing_time * total_ecu) * conf_ecu_rate,\n querying = CASE (deployment_sum_query_time > 0, datastream_sum_query_time / deployment_sum_query_time * total_ecu) * conf_ecu_rate,\n data_set = CASE (deployment_sum_data_set_store_size > 0, datastream_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE (deployment_sum_store_size > 0, datastream_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE (store == 0, data_set, store) * conf_ecu_rate,\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE (\n tier == \"hot/content\",\n ((storage / conf_ecu_rate * conf_storage_weight) + (querying / conf_ecu_rate * conf_query_weight) + (indexing / conf_ecu_rate * conf_indexing_weight)) / total_weight_hot,\n ((storage / conf_ecu_rate * conf_storage_weight) + (querying / conf_ecu_rate * conf_query_weight)) / total_weight_cold\n ) * conf_ecu_rate\n| STATS\n agg_blended = sum(blended)\n BY\n @timestamp,\n datastream,\n tier\n| WHERE agg_blended > 0\n| SORT agg_blended DESC" + }, + "timeField": "@timestamp" + } + } + } + }, + "filters": [], + "needsRefresh": false, + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, datastream_sum_indexing_time / deployment_sum_indexing_time * total_ecu) * conf_ecu_rate,\n querying = CASE (deployment_sum_query_time > 0, datastream_sum_query_time / deployment_sum_query_time * total_ecu) * conf_ecu_rate,\n data_set = CASE (deployment_sum_data_set_store_size > 0, datastream_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE (deployment_sum_store_size > 0, datastream_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE (store == 0, data_set, store) * conf_ecu_rate,\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE (\n tier == \"hot/content\",\n ((storage / conf_ecu_rate * conf_storage_weight) + (querying / conf_ecu_rate * conf_query_weight) + (indexing / conf_ecu_rate * conf_indexing_weight)) / total_weight_hot,\n ((storage / conf_ecu_rate * conf_storage_weight) + (querying / conf_ecu_rate * conf_query_weight)) / total_weight_cold\n ) * conf_ecu_rate\n| STATS\n agg_blended = sum(blended)\n BY\n @timestamp,\n datastream,\n tier\n| WHERE agg_blended > 0\n| SORT agg_blended DESC" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "30c47aa5-b389-479c-a2b6-d91c77a952b9" + ], + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "layerId": "8c2f087d-e623-4f6f-b232-97ab4284e4db", + "layerType": "data", + "seriesType": "bar_percentage_stacked", + "splitAccessor": "57d5389c-1311-460d-afe2-cecc4a8d16f4", + "xAccessor": "@timestamp" + } + ], + "legend": { + "isVisible": false, + "position": "right", + "showSingleSeries": false + }, + "preferredSeriesType": "bar_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "agg_indexing \u0026 agg_querying \u0026 agg_storage \u0026 agg_blended over @timestamp", + "visualizationType": "lnsXY" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, datastream_sum_indexing_time / deployment_sum_indexing_time * total_ecu) * conf_ecu_rate,\n querying = CASE (deployment_sum_query_time > 0, datastream_sum_query_time / deployment_sum_query_time * total_ecu) * conf_ecu_rate,\n data_set = CASE (deployment_sum_data_set_store_size > 0, datastream_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE (deployment_sum_store_size > 0, datastream_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE (store == 0, data_set, store) * conf_ecu_rate,\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE (\n tier == \"hot/content\",\n ((storage / conf_ecu_rate * conf_storage_weight) + (querying / conf_ecu_rate * conf_query_weight) + (indexing / conf_ecu_rate * conf_indexing_weight)) / total_weight_hot,\n ((storage / conf_ecu_rate * conf_storage_weight) + (querying / conf_ecu_rate * conf_query_weight)) / total_weight_cold\n ) * conf_ecu_rate\n| STATS\n agg_blended = sum(blended)\n BY\n @timestamp,\n datastream,\n tier\n| WHERE agg_blended > 0\n| SORT agg_blended DESC" + }, + "searchSessionId": "48b157e3-14ff-4c71-8178-4a69f0592d85", + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 14, + "i": "4dbbad0a-7ceb-443b-912b-0c1ae9fb3a6a", + "w": 16, + "x": 32, + "y": 130 + }, + "panelIndex": "4dbbad0a-7ceb-443b-912b-0c1ae9fb3a6a", + "title": "Blended value: % Normalised cost per data stream per day", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [], + "state": { + "adHocDataViews": { + "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "name": "billing_cluster_cost_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_cluster_cost_lookup", + "type": "esql" + } + }, + "datasourceStates": { + "textBased": { + "indexPatternRefs": [ + { + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "timeField": "@timestamp", + "title": "billing_cluster_cost_lookup" + } + ], + "layers": { + "43f9cfe6-d828-4773-ab65-129f92cf7f4d": { + "columns": [ + { + "columnId": "@timestamp", + "customLabel": true, + "fieldName": "@timestamp", + "inMetricDimension": true, + "label": "Date", + "meta": { + "esType": "date", + "type": "date" + } + }, + { + "columnId": "50a91b89-2e68-4a69-90a0-d2ab79115b96", + "customLabel": true, + "fieldName": "tier", + "label": "Data tier", + "meta": { + "esType": "keyword", + "sourceParams": { + "indexPattern": "billing_cluster_cost_lookup" + }, + "type": "string" + } + }, + { + "columnId": "1501af72-2509-4ae0-88a0-c1481e438377", + "customLabel": true, + "fieldName": "agg_blended", + "label": "Normalised cost", + "meta": { + "esType": "double", + "sourceParams": { + "indexPattern": "billing_cluster_cost_lookup" + }, + "type": "number" + }, + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + } + } + ], + "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, tier_sum_indexing_time / deployment_sum_indexing_time * total_ecu) * conf_ecu_rate,\n querying = CASE (deployment_sum_query_time > 0, tier_sum_query_time / deployment_sum_query_time * total_ecu) * conf_ecu_rate,\n data_set = CASE (deployment_sum_data_set_store_size > 0, tier_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE (deployment_sum_store_size > 0, tier_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE (store == 0, data_set, store) * conf_ecu_rate,\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE (\n tier == \"hot/content\",\n ((storage / conf_ecu_rate * conf_storage_weight) + (querying / conf_ecu_rate * conf_query_weight) + (indexing / conf_ecu_rate * conf_indexing_weight)) / total_weight_hot,\n ((storage / conf_ecu_rate * conf_storage_weight) + (querying / conf_ecu_rate * conf_query_weight)) / total_weight_cold\n ) * conf_ecu_rate\n| STATS\n agg_blended = sum(blended)\n BY\n @timestamp,\n tier\n| WHERE agg_blended > 0" + }, + "timeField": "@timestamp" + } + } + } + }, + "filters": [], + "needsRefresh": false, + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, tier_sum_indexing_time / deployment_sum_indexing_time * total_ecu) * conf_ecu_rate,\n querying = CASE (deployment_sum_query_time > 0, tier_sum_query_time / deployment_sum_query_time * total_ecu) * conf_ecu_rate,\n data_set = CASE (deployment_sum_data_set_store_size > 0, tier_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE (deployment_sum_store_size > 0, tier_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE (store == 0, data_set, store) * conf_ecu_rate,\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE (\n tier == \"hot/content\",\n ((storage / conf_ecu_rate * conf_storage_weight) + (querying / conf_ecu_rate * conf_query_weight) + (indexing / conf_ecu_rate * conf_indexing_weight)) / total_weight_hot,\n ((storage / conf_ecu_rate * conf_storage_weight) + (querying / conf_ecu_rate * conf_query_weight)) / total_weight_cold\n ) * conf_ecu_rate\n| STATS\n agg_blended = sum(blended)\n BY\n @timestamp,\n tier\n| WHERE agg_blended > 0" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "1501af72-2509-4ae0-88a0-c1481e438377" + ], + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "layerId": "43f9cfe6-d828-4773-ab65-129f92cf7f4d", + "layerType": "data", + "seriesType": "bar_stacked", + "splitAccessor": "50a91b89-2e68-4a69-90a0-d2ab79115b96", + "xAccessor": "@timestamp" + } + ], + "legend": { + "isVisible": false, + "position": "right", + "showSingleSeries": false + }, + "preferredSeriesType": "bar_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "agg_indexing \u0026 agg_querying \u0026 agg_storage \u0026 agg_blended over @timestamp", + "visualizationType": "lnsXY" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, tier_sum_indexing_time / deployment_sum_indexing_time * total_ecu) * conf_ecu_rate,\n querying = CASE (deployment_sum_query_time > 0, tier_sum_query_time / deployment_sum_query_time * total_ecu) * conf_ecu_rate,\n data_set = CASE (deployment_sum_data_set_store_size > 0, tier_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE (deployment_sum_store_size > 0, tier_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE (store == 0, data_set, store) * conf_ecu_rate,\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE (\n tier == \"hot/content\",\n ((storage / conf_ecu_rate * conf_storage_weight) + (querying / conf_ecu_rate * conf_query_weight) + (indexing / conf_ecu_rate * conf_indexing_weight)) / total_weight_hot,\n ((storage / conf_ecu_rate * conf_storage_weight) + (querying / conf_ecu_rate * conf_query_weight)) / total_weight_cold\n ) * conf_ecu_rate\n| STATS\n agg_blended = sum(blended)\n BY\n @timestamp,\n tier\n| WHERE agg_blended > 0" + }, + "searchSessionId": "48b157e3-14ff-4c71-8178-4a69f0592d85", + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 14, + "i": "5732f95c-3cc0-46c8-a8ab-14f8c6937f9f", + "w": 16, + "x": 16, + "y": 130 + }, + "panelIndex": "5732f95c-3cc0-46c8-a8ab-14f8c6937f9f", + "title": "Blended value: Normalised cost per tier per day", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [], + "state": { + "adHocDataViews": { + "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "name": "billing_cluster_cost_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_cluster_cost_lookup", + "type": "esql" + } + }, + "datasourceStates": { + "textBased": { + "indexPatternRefs": [ + { + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "timeField": "@timestamp", + "title": "billing_cluster_cost_lookup" + } + ], + "layers": { + "0ee24895-1496-4b23-96ec-ec09a8f49332": { + "columns": [ + { + "columnId": "agg_indexing", + "customLabel": true, + "fieldName": "agg_indexing", + "inMetricDimension": true, + "label": "Normalised cost", + "meta": { + "esType": "double", + "type": "number" + }, + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + } + }, + { + "columnId": "@timestamp", + "customLabel": true, + "fieldName": "@timestamp", + "label": "Date", + "meta": { + "esType": "date", + "type": "date" + } + }, + { + "columnId": "datastream", + "customLabel": true, + "fieldName": "datastream", + "label": "Data stream", + "meta": { + "esType": "keyword", + "type": "string" + } + } + ], + "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL \n indexing = CASE (deployment_sum_indexing_time > 0, datastream_sum_indexing_time / deployment_sum_indexing_time * total_ecu) * conf_ecu_rate\n| STATS \n agg_indexing = sum(indexing)\n BY \n @timestamp, \n datastream\n| WHERE agg_indexing > 0\n| SORT agg_indexing DESC" + }, + "timeField": "@timestamp" + } + } + } + }, + "filters": [], + "needsRefresh": false, + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL \n indexing = CASE (deployment_sum_indexing_time > 0, datastream_sum_indexing_time / deployment_sum_indexing_time * total_ecu) * conf_ecu_rate\n| STATS \n agg_indexing = sum(indexing)\n BY \n @timestamp, \n datastream\n| WHERE agg_indexing > 0\n| SORT agg_indexing DESC" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "agg_indexing" + ], + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "layerId": "0ee24895-1496-4b23-96ec-ec09a8f49332", + "layerType": "data", + "seriesType": "bar_stacked", + "splitAccessor": "datastream", + "xAccessor": "@timestamp" + } + ], + "legend": { + "isVisible": false, + "position": "right", + "showSingleSeries": false + }, + "preferredSeriesType": "bar_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "agg_indexing over @timestamp", + "visualizationType": "lnsXY" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_datastream_contribution_lookup ON composite_key\n| EVAL \n indexing = CASE (deployment_sum_indexing_time > 0, datastream_sum_indexing_time / deployment_sum_indexing_time * total_ecu) * conf_ecu_rate\n| STATS \n agg_indexing = sum(indexing)\n BY \n @timestamp, \n datastream\n| WHERE agg_indexing > 0\n| SORT agg_indexing DESC" + }, + "searchSessionId": "48b157e3-14ff-4c71-8178-4a69f0592d85", + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 14, + "i": "dafb7c36-3a1d-4cbb-9944-51beedf535f6", + "w": 16, + "x": 0, + "y": 78 + }, + "panelIndex": "dafb7c36-3a1d-4cbb-9944-51beedf535f6", + "title": "Indexing: Normalised per data stream per day", + "type": "lens" + }, + { + "embeddableConfig": { + "attributes": { + "references": [], + "state": { + "adHocDataViews": { + "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "name": "billing_cluster_cost_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_cluster_cost_lookup", + "type": "esql" + } + }, + "datasourceStates": { + "textBased": { + "indexPatternRefs": [ + { + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "timeField": "@timestamp", + "title": "billing_cluster_cost_lookup" + } + ], + "layers": { + "93fd5f12-463f-4e8e-a9b8-0ac77d0522a9": { + "columns": [ + { + "columnId": "agg_storage", + "customLabel": true, + "fieldName": "agg_storage", + "inMetricDimension": true, + "label": "Normalised cost", + "meta": { + "esType": "double", + "type": "number" + }, + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + } + }, + { + "columnId": "@timestamp", + "customLabel": true, + "fieldName": "@timestamp", + "label": "Date", + "meta": { + "esType": "date", + "type": "date" + } + }, + { + "columnId": "tier", + "customLabel": true, + "fieldName": "tier", + "label": "Data tier", + "meta": { + "esType": "keyword", + "type": "string" + } + } + ], + "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, tier_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE (deployment_sum_store_size > 0, tier_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE (store == 0, data_set, store) * conf_ecu_rate\n| STATS \n agg_storage = sum(storage)\n BY \n @timestamp,\n tier\n| WHERE agg_storage > 0" + }, + "timeField": "@timestamp" + } + } + } + }, + "filters": [], + "needsRefresh": false, + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, tier_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE (deployment_sum_store_size > 0, tier_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE (store == 0, data_set, store) * conf_ecu_rate\n| STATS \n agg_storage = sum(storage)\n BY \n @timestamp,\n tier\n| WHERE agg_storage > 0" + }, + "visualization": { + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "fittingFunction": "Linear", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "labelsOrientation": { + "x": 0, + "yLeft": 0, + "yRight": 0 + }, + "layers": [ + { + "accessors": [ + "agg_storage" + ], + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "layerId": "93fd5f12-463f-4e8e-a9b8-0ac77d0522a9", + "layerType": "data", + "seriesType": "bar_stacked", + "splitAccessor": "tier", + "xAccessor": "@timestamp" + } + ], + "legend": { + "isVisible": false, + "position": "right", + "showSingleSeries": false + }, + "preferredSeriesType": "bar_stacked", + "tickLabelsVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "valueLabels": "hide" + } + }, + "title": "agg_storage over @timestamp", + "visualizationType": "lnsXY" + }, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| EVAL \n data_set = CASE (deployment_sum_data_set_store_size > 0, tier_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE (deployment_sum_store_size > 0, tier_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE (store == 0, data_set, store) * conf_ecu_rate\n| STATS \n agg_storage = sum(storage)\n BY \n @timestamp,\n tier\n| WHERE agg_storage > 0" + }, + "searchSessionId": "48b157e3-14ff-4c71-8178-4a69f0592d85", + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 15, + "i": "ba7efeab-de6c-4301-8e97-7adb98dae58b", + "w": 16, + "x": 16, + "y": 112 + }, + "panelIndex": "ba7efeab-de6c-4301-8e97-7adb98dae58b", + "title": "Storage: Normalised cost per tier per day", + "type": "lens" + }, + { + "embeddableConfig": { + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 12, + "markdown": "## Indexing details", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 3, + "i": "9cd4730b-d3f5-438e-a816-4d5aebf1eae6", + "w": 48, + "x": 0, + "y": 75 + }, + "panelIndex": "9cd4730b-d3f5-438e-a816-4d5aebf1eae6", + "title": "", + "type": "visualization" + }, + { + "embeddableConfig": { + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 12, + "markdown": "## Querying details", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 3, + "i": "8de95af1-8e9a-424a-baaf-8cfcb2577f59", + "w": 48, + "x": 0, + "y": 92 + }, + "panelIndex": "8de95af1-8e9a-424a-baaf-8cfcb2577f59", + "title": "", + "type": "visualization" + }, + { + "embeddableConfig": { + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 12, + "markdown": "## Storage details", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 3, + "i": "4d3f1bfb-ea89-4978-ab6f-9033a507785f", + "w": 48, + "x": 0, + "y": 109 + }, + "panelIndex": "4d3f1bfb-ea89-4978-ab6f-9033a507785f", + "title": "", + "type": "visualization" + }, + { + "embeddableConfig": { + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "savedVis": { + "data": { + "aggs": [], + "searchSource": { + "filter": [], + "query": { + "language": "kuery", + "query": "" + } + } + }, + "description": "", + "params": { + "fontSize": 12, + "markdown": "## Blended details", + "openLinksInNewTab": false + }, + "title": "", + "type": "markdown", + "uiState": {} + } + }, + "gridData": { + "h": 3, + "i": "a2cb12f7-9b9e-4bf7-864e-9101107e0c3e", + "w": 48, + "x": 0, + "y": 127 + }, + "panelIndex": "a2cb12f7-9b9e-4bf7-864e-9101107e0c3e", + "title": "", + "type": "visualization" + }, + { + "embeddableConfig": { + "attributes": { + "references": [], + "state": { + "adHocDataViews": { + "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f": { + "allowHidden": false, + "allowNoIndex": false, + "fieldFormats": {}, + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "name": "billing_cluster_cost_lookup", + "runtimeFieldMap": {}, + "sourceFilters": [], + "timeFieldName": "@timestamp", + "title": "billing_cluster_cost_lookup", + "type": "esql" + } + }, + "datasourceStates": { + "textBased": { + "indexPatternRefs": [ + { + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "timeField": "@timestamp", + "title": "billing_cluster_cost_lookup" + } + ], + "layers": { + "9d199172-f9fa-4fc4-82af-c7e2672a3f54": { + "columns": [ + { + "columnId": "a76076ce-f372-4a48-857d-75010bbf5970", + "fieldName": "tier", + "label": "tier", + "meta": { + "esType": "keyword", + "sourceParams": { + "indexPattern": "billing_cluster_cost_lookup" + }, + "type": "string" + } + }, + { + "columnId": "6e881cac-4719-493c-95f8-e3bfa38c3219", + "customLabel": true, + "fieldName": "agg_indexing", + "label": "Indexing", + "meta": { + "esType": "double", + "sourceParams": { + "indexPattern": "billing_cluster_cost_lookup" + }, + "type": "number" + }, + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + } + }, + { + "columnId": "97a86307-08fc-4b2d-a82a-36a2ecc9c75d", + "customLabel": true, + "fieldName": "agg_querying", + "label": "Querying", + "meta": { + "esType": "double", + "sourceParams": { + "indexPattern": "billing_cluster_cost_lookup" + }, + "type": "number" + }, + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + } + }, + { + "columnId": "2b086331-8a38-4df2-b042-7827d3519d63", + "customLabel": true, + "fieldName": "agg_storage", + "label": "Storage", + "meta": { + "esType": "double", + "sourceParams": { + "indexPattern": "billing_cluster_cost_lookup" + }, + "type": "number" + }, + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + } + }, + { + "columnId": "307273f7-1b4c-450c-80cd-ef0326b38f6e", + "customLabel": true, + "fieldName": "agg_blended", + "label": "Blended", + "meta": { + "esType": "double", + "sourceParams": { + "indexPattern": "billing_cluster_cost_lookup" + }, + "type": "number" + }, + "params": { + "format": { + "id": "number", + "params": { + "decimals": 2 + } + } + } + } + ], + "index": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, tier_sum_indexing_time / deployment_sum_indexing_time * total_ecu) * conf_ecu_rate,\n querying = CASE (deployment_sum_query_time > 0, tier_sum_query_time / deployment_sum_query_time * total_ecu) * conf_ecu_rate,\n data_set = CASE (deployment_sum_data_set_store_size > 0, tier_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE (deployment_sum_store_size > 0, tier_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE (store == 0, data_set, store) * conf_ecu_rate,\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE (\n tier == \"hot/content\",\n ((storage / conf_ecu_rate * conf_storage_weight) + (querying / conf_ecu_rate * conf_query_weight) + (indexing / conf_ecu_rate * conf_indexing_weight)) / total_weight_hot,\n ((storage / conf_ecu_rate * conf_storage_weight) + (querying / conf_ecu_rate * conf_query_weight)) / total_weight_cold\n ) * conf_ecu_rate\n| STATS\n agg_indexing = sum(indexing),\n agg_querying = sum(querying),\n agg_storage = sum(storage),\n agg_blended = sum(blended)\n BY\n tier\n| WHERE agg_blended > 0" + }, + "timeField": "@timestamp" + } + } + } + }, + "filters": [], + "needsRefresh": false, + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, tier_sum_indexing_time / deployment_sum_indexing_time * total_ecu) * conf_ecu_rate,\n querying = CASE (deployment_sum_query_time > 0, tier_sum_query_time / deployment_sum_query_time * total_ecu) * conf_ecu_rate,\n data_set = CASE (deployment_sum_data_set_store_size > 0, tier_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE (deployment_sum_store_size > 0, tier_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE (store == 0, data_set, store) * conf_ecu_rate,\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE (\n tier == \"hot/content\",\n ((storage / conf_ecu_rate * conf_storage_weight) + (querying / conf_ecu_rate * conf_query_weight) + (indexing / conf_ecu_rate * conf_indexing_weight)) / total_weight_hot,\n ((storage / conf_ecu_rate * conf_storage_weight) + (querying / conf_ecu_rate * conf_query_weight)) / total_weight_cold\n ) * conf_ecu_rate\n| STATS\n agg_indexing = sum(indexing),\n agg_querying = sum(querying),\n agg_storage = sum(storage),\n agg_blended = sum(blended)\n BY\n tier\n| WHERE agg_blended > 0" + }, + "visualization": { + "columns": [ + { + "columnId": "a76076ce-f372-4a48-857d-75010bbf5970", + "isMetric": false, + "isTransposed": false + }, + { + "columnId": "6e881cac-4719-493c-95f8-e3bfa38c3219", + "isMetric": true, + "isTransposed": false + }, + { + "columnId": "97a86307-08fc-4b2d-a82a-36a2ecc9c75d", + "isMetric": true, + "isTransposed": false + }, + { + "columnId": "2b086331-8a38-4df2-b042-7827d3519d63", + "isMetric": true, + "isTransposed": false + }, + { + "columnId": "307273f7-1b4c-450c-80cd-ef0326b38f6e", + "isMetric": true, + "isTransposed": false + } + ], + "layerId": "9d199172-f9fa-4fc4-82af-c7e2672a3f54", + "layerType": "data" + } + }, + "title": "agg_indexing \u0026 agg_querying \u0026 agg_storage \u0026 agg_blended of tier", + "visualizationType": "lnsDatatable" + }, + "description": "Normalised cost breakdown per tier, based on Indexing, Querying, Storage and a blended ratio of all of these.", + "enhancements": { + "dynamicActions": { + "events": [] + } + }, + "filters": [], + "query": { + "esql": "FROM billing_cluster_cost_lookup\n| EVAL config_join_key = \"chargeback_config\"\n| LOOKUP JOIN chargeback_conf_lookup ON config_join_key\n| LOOKUP JOIN cluster_deployment_contribution_lookup ON composite_key\n| LOOKUP JOIN cluster_tier_contribution_lookup ON composite_key\n| EVAL\n indexing = CASE (deployment_sum_indexing_time > 0, tier_sum_indexing_time / deployment_sum_indexing_time * total_ecu) * conf_ecu_rate,\n querying = CASE (deployment_sum_query_time > 0, tier_sum_query_time / deployment_sum_query_time * total_ecu) * conf_ecu_rate,\n data_set = CASE (deployment_sum_data_set_store_size > 0, tier_sum_data_set_store_size / deployment_sum_data_set_store_size * total_ecu),\n store = CASE (deployment_sum_store_size > 0, tier_sum_store_size / deployment_sum_store_size * total_ecu),\n storage = CASE (store == 0, data_set, store) * conf_ecu_rate,\n total_weight_hot = conf_storage_weight + conf_query_weight + conf_indexing_weight,\n total_weight_cold = conf_storage_weight + conf_query_weight,\n blended = CASE (\n tier == \"hot/content\",\n ((storage / conf_ecu_rate * conf_storage_weight) + (querying / conf_ecu_rate * conf_query_weight) + (indexing / conf_ecu_rate * conf_indexing_weight)) / total_weight_hot,\n ((storage / conf_ecu_rate * conf_storage_weight) + (querying / conf_ecu_rate * conf_query_weight)) / total_weight_cold\n ) * conf_ecu_rate\n| STATS\n agg_indexing = sum(indexing),\n agg_querying = sum(querying),\n agg_storage = sum(storage),\n agg_blended = sum(blended)\n BY\n tier\n| WHERE agg_blended > 0" + }, + "searchSessionId": "48b157e3-14ff-4c71-8178-4a69f0592d85", + "syncColors": false, + "syncCursor": true, + "syncTooltips": false + }, + "gridData": { + "h": 10, + "i": "18643569-92d3-4a30-aafe-6ae11acc4c58", + "w": 48, + "x": 0, + "y": 29 + }, + "panelIndex": "18643569-92d3-4a30-aafe-6ae11acc4c58", + "title": "Normalised cost per tier", + "type": "lens" + } + ], + "timeRestore": false, + "title": "[Chargeback] Cost and Consumption breakdown", + "version": 3 + }, + "coreMigrationVersion": "8.8.0", + "created_at": "2025-06-06T13:26:53.339Z", + "created_by": "u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0", + "id": "chargeback-39a39857-746c-4a29-adca-3c2fcb6bcfb6", + "managed": false, + "references": [ + { + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f", + "name": "controlGroup_ad78c22a-f7b7-4810-8af5-30fbbca8e93e:optionsListDataView", + "type": "index-pattern" + } + ], + "type": "dashboard", + "typeMigrationVersion": "10.2.0", + "updated_by": "u_mGBROF_q5bmFCATbLXAcCwKa0k8JvONAwSruelyKA5E_0" +} \ No newline at end of file diff --git a/packages/chargeback/manifest.yml b/packages/chargeback/manifest.yml new file mode 100644 index 00000000000..013952d4aad --- /dev/null +++ b/packages/chargeback/manifest.yml @@ -0,0 +1,27 @@ +format_version: 3.3.2 +name: chargeback +title: "Chargeback" +version: 0.1.7 +description: "This package calculates chargeback based on billing and consumption data" +type: integration +categories: + - cloud + - custom +conditions: + kibana: + version: "8.18.0" + elastic: + subscription: "basic" +screenshots: + - src: /img/elastic-logo.png + title: Elastic logo + size: 600x600 + type: image/png +icons: + - src: /img/elastic-logo.svg + title: Elastic logo + size: 32x32 + type: image/svg+xml +owner: + github: elastic/customer-architects + type: elastic \ No newline at end of file diff --git a/packages/chargeback/pre-setup.sh b/packages/chargeback/pre-setup.sh new file mode 100644 index 00000000000..9b5192f235c --- /dev/null +++ b/packages/chargeback/pre-setup.sh @@ -0,0 +1,186 @@ +# Create the lookup indices for chargeback configuration and billing metrics +# These indices are used to store configuration and billing data for chargeback calculations. + +PUT chargeback_conf_lookup +{ + "settings": { + "index.mode": "lookup", + "index.hidden": true + }, + "mappings": { + "_meta": { + "managed": true, + "package": { "name": "chargeback", "version": "0.1.7" } + }, + "properties": { + "config_join_key": { "type": "keyword" }, + "conf_ecu_rate": { "type": "float" }, + "conf_ecu_rate_unit": { "type": "keyword"}, + "conf_indexing_weight": { "type": "integer" }, + "conf_query_weight": { "type": "integer" }, + "conf_storage_weight": { "type": "integer" } + } + } +} + +# Add the default configuration to the chargeback_conf_lookup index. +POST chargeback_conf_lookup/_doc/config +{ + "config_join_key": "chargeback_config", + "conf_ecu_rate": 0.85, + "conf_ecu_rate_unit": "EUR", + "conf_indexing_weight": 20, + "conf_query_weight": 20, + "conf_storage_weight": 40 +} + +# Create the lookup indices for billing and cluster contributions. +PUT billing_cluster_cost_lookup +{ + "settings": { + "index.mode": "lookup", + "index.hidden": true + }, + "mappings": { + "_meta": { + "managed": true, + "package": { "name": "chargeback", "version": "0.1.7" } + }, + "properties": { + "@timestamp": { "type": "date" }, + "billing_name": { + "type": "text", + "fields": { "keyword": { "type": "keyword", "ignore_above": 256 } } + }, + "billing_type": { + "type": "text", + "fields": { "keyword": { "type": "keyword", "ignore_above": 256 } } + }, + "composite_key": { "type": "keyword" }, + "config_join_key": { "type": "keyword" }, + "deployment_id": { "type": "keyword" }, + "deployment_name": { + "type": "text", + "fields": { "keyword": { "type": "keyword", "ignore_above": 256 } } + }, + "total_ecu": { "type": "float" } + } + } +} + +PUT cluster_datastream_contribution_lookup +{ + "settings": { + "index.mode": "lookup", + "index.hidden": true + }, + "mappings": { + "_meta": { + "managed": true, + "package": { "name": "chargeback", "version": "0.1.7" } + }, + "properties": { + "@timestamp": { "type": "date" }, + "composite_key": { "type": "keyword" }, + "composite_datastream_key": { "type": "keyword" }, + "config_join_key": { "type": "keyword" }, + "cluster_name": { "type": "keyword" }, + "deployment_id": { "type": "keyword" }, + "datastream": { "type": "keyword" }, + "datastream_sum_indexing_time": { "type": "double" }, + "datastream_sum_query_time": { "type": "double" }, + "datastream_sum_store_size": { "type": "double" }, + "datastream_sum_data_set_store_size": { "type": "double" } + } + } +} + +PUT cluster_deployment_contribution_lookup +{ + "settings": { + "index.mode": "lookup", + "index.hidden": true + }, + "mappings": { + "_meta": { + "managed": true, + "package": { "name": "chargeback", "version": "0.1.7" } + }, + "properties": { + "@timestamp": { "type": "date" }, + "composite_key": { "type": "keyword" }, + "config_join_key": { "type": "keyword" }, + "cluster_name": { "type": "keyword" }, + "deployment_id": { "type": "keyword" }, + "deployment_sum_indexing_time": { "type": "double" }, + "deployment_sum_query_time": { "type": "double" }, + "deployment_sum_store_size": { "type": "double" }, + "deployment_sum_data_set_store_size": { "type": "double" } + } + } +} + +PUT cluster_tier_and_datastream_contribution_lookup +{ + "settings": { + "index.mode": "lookup", + "index.hidden": true + }, + "mappings": { + "_meta": { + "managed": true, + "package": { "name": "chargeback", "version": "0.1.7" } + }, + "properties": { + "@timestamp": { "type": "date" }, + "composite_key": { "type": "keyword" }, + "composite_tier_key": { "type": "keyword" }, + "config_join_key": { "type": "keyword" }, + "cluster_name": { "type": "keyword" }, + "deployment_id": { "type": "keyword" }, + "tier": { "type": "keyword" }, + "datastream": { "type": "keyword" }, + "tier_and_datastream_sum_indexing_time": { "type": "double" }, + "tier_and_datastream_sum_query_time": { "type": "double" }, + "tier_and_datastream_sum_store_size": { "type": "double" }, + "tier_and_datastream_sum_data_set_store_size": { "type": "double" } + } + } +} + +PUT cluster_tier_contribution_lookup +{ + "settings": { + "index.mode": "lookup", + "index.hidden": true + }, + "mappings": { + "_meta": { + "managed": true, + "package": { "name": "chargeback", "version": "0.1.7" } + }, + "properties": { + "@timestamp": { "type": "date" }, + "composite_key": { "type": "keyword" }, + "composite_tier_key": { "type": "keyword" }, + "config_join_key": { "type": "keyword" }, + "cluster_name": { "type": "keyword" }, + "deployment_id": { "type": "keyword" }, + "tier": { "type": "keyword" }, + "tier_sum_indexing_time": { "type": "double" }, + "tier_sum_query_time": { "type": "double" }, + "tier_sum_store_size": { "type": "double" }, + "tier_sum_data_set_store_size": { "type": "double" } + } + } +} + +# Create data view used for control. +POST kbn:/api/data_views/data_view +{ + "data_view": { + "name": "[Chargeback] Billing Cluster Cost", + "title": "billing_cluster_cost_lookup", + "id": "2bf6c0d816ef0a2d56d03ede549c16c08c35db2cf02d78c12756a98a33f50e4f" + } +} \ No newline at end of file diff --git a/packages/chargeback/validation.yml b/packages/chargeback/validation.yml new file mode 100644 index 00000000000..bcc8f74ac3a --- /dev/null +++ b/packages/chargeback/validation.yml @@ -0,0 +1,3 @@ +errors: + exclude_checks: + - SVR00002