From a3590d876b3f6b936981d0361b645882a127a0ac Mon Sep 17 00:00:00 2001 From: Tom Myers Date: Fri, 11 Sep 2026 23:55:24 +0100 Subject: [PATCH 1/4] increase cloudwatch autodiscover limit and add section to readme explaining the consequence --- .../_dev/build/docs/README.md | 8 ++++++++ packages/aws_cloudwatch_input_otel/changelog.yml | 5 +++++ packages/aws_cloudwatch_input_otel/docs/README.md | 8 ++++++++ packages/aws_cloudwatch_input_otel/manifest.yml | 12 ++++++------ 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/packages/aws_cloudwatch_input_otel/_dev/build/docs/README.md b/packages/aws_cloudwatch_input_otel/_dev/build/docs/README.md index 90d6bb8da66..df78f907484 100644 --- a/packages/aws_cloudwatch_input_otel/_dev/build/docs/README.md +++ b/packages/aws_cloudwatch_input_otel/_dev/build/docs/README.md @@ -83,6 +83,14 @@ Guidance: | AWS Application ELB | 1m | 1m | | AWS ECS | 1m | 1m | +### Autodiscover Limit and impact on cost + +This integration automatically discovers and collects all CloudWatch metrics published for the configured namespaces. The CloudWatch API bills per metric requested, so the more resources that exist in your account, the more metrics are discovered and the higher the collection cost. + +To avoid unexpectedly large bills, the number of metrics collected is capped by the **Autodiscover Limit**, which defaults to 10,000, high enough to capture all metrics in a typical deployment. The limit applies to each namespace separately. It is a ceiling, not a fixed value - you are billed only for the metrics that actually exist, so smaller accounts cost proportionally less. With the 10,000 limit, the maximum cost per namespace is ~$3,500 per month at a 5-minute collection interval. + +If you need to reduce cost, you can lower the **Autodiscover Limit** in the integration's advanced settings. Reducing the limit below the number of metrics in your account means some metrics will not be collected, and which ones are dropped is not predictable. We recommend changing this value only if you understand the metric volume in your AWS account, and lowering it gradually while verifying that the metrics you rely on are still present. + ## Authentication The integration supports the following ways to authenticate to AWS: diff --git a/packages/aws_cloudwatch_input_otel/changelog.yml b/packages/aws_cloudwatch_input_otel/changelog.yml index d2aafced844..6dd0b5710dd 100644 --- a/packages/aws_cloudwatch_input_otel/changelog.yml +++ b/packages/aws_cloudwatch_input_otel/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.7.0" + changes: + - description: Increase default metric autodiscover_limit to 10,000 series + type: enhancement + link: https://github.com/elastic/integrations/pull/99999 - version: "0.6.0" changes: - description: Publish Agentless to Elastic Managed integrations name change. diff --git a/packages/aws_cloudwatch_input_otel/docs/README.md b/packages/aws_cloudwatch_input_otel/docs/README.md index ddae93b4cc0..da0c8d98a59 100644 --- a/packages/aws_cloudwatch_input_otel/docs/README.md +++ b/packages/aws_cloudwatch_input_otel/docs/README.md @@ -84,6 +84,14 @@ Guidance: | AWS Application ELB | 1m | 1m | | AWS ECS | 1m | 1m | +### Autodiscover Limit and impact on cost + +This integration automatically discovers and collects all CloudWatch metrics published for the configured namespaces. The CloudWatch API bills per metric requested, so the more resources that exist in your account, the more metrics are discovered and the higher the collection cost. + +To avoid unexpectedly large bills, the number of metrics collected is capped by the **Autodiscover Limit**, which defaults to 10,000, high enough to capture all metrics in a typical deployment. The limit applies to each namespace separately. It is a ceiling, not a fixed value - you are billed only for the metrics that actually exist, so smaller accounts cost proportionally less. With the 10,000 limit, the maximum cost per namespace is ~$3,500 per month at a 5-minute collection interval. + +If you need to reduce cost, you can lower the **Autodiscover Limit** in the integration's advanced settings. Reducing the limit below the number of metrics in your account means some metrics will not be collected, and which ones are dropped is not predictable. We recommend changing this value only if you understand the metric volume in your AWS account, and lowering it gradually while verifying that the metrics you rely on are still present. + ## Authentication The integration supports the following ways to authenticate to AWS: diff --git a/packages/aws_cloudwatch_input_otel/manifest.yml b/packages/aws_cloudwatch_input_otel/manifest.yml index 60d9d13d807..207dd46eafc 100644 --- a/packages/aws_cloudwatch_input_otel/manifest.yml +++ b/packages/aws_cloudwatch_input_otel/manifest.yml @@ -1,7 +1,7 @@ format_version: 3.5.0 name: aws_cloudwatch_input_otel title: "AWS CloudWatch (OpenTelemetry)" -version: 0.6.0 +version: 0.7.0 source: license: "Elastic-2.0" description: "Collect AWS CloudWatch metrics for selected AWS services (EC2, RDS, SQS, ELB — Application, Classic, Network, Gateway — Lambda, Fargate) using the OpenTelemetry Collector awscloudwatch receiver." @@ -102,7 +102,7 @@ policy_templates: type: integer title: Autodiscover Limit description: Maximum number of metrics to enumerate for this namespace. Higher values pick up more metrics but cost more API calls. - default: 100 + default: 10000 required: false show_user: false - name: collection_interval @@ -163,7 +163,7 @@ policy_templates: type: integer title: Autodiscover Limit description: Maximum number of metrics to enumerate for this namespace. Higher values pick up more metrics but cost more API calls. - default: 100 + default: 10000 required: false show_user: false - name: collection_interval @@ -223,7 +223,7 @@ policy_templates: type: integer title: Autodiscover Limit description: Maximum number of metrics to enumerate for this namespace. Higher values pick up more metrics but cost more API calls. - default: 100 + default: 10000 required: false show_user: false - name: collection_interval @@ -282,7 +282,7 @@ policy_templates: type: integer title: Autodiscover Limit description: Maximum number of metrics to enumerate for this namespace. Higher values pick up more metrics but cost more API calls. - default: 100 + default: 10000 required: false show_user: false - name: collection_interval @@ -583,7 +583,7 @@ policy_templates: type: integer title: Autodiscover Limit description: Maximum number of metrics to enumerate for this namespace. Higher values pick up more metrics but cost more API calls. - default: 100 + default: 10000 required: false show_user: false - name: collection_interval From c553f97d18256ae42ed36b8d2cf01e25be32522c Mon Sep 17 00:00:00 2001 From: Tom Myers Date: Fri, 11 Sep 2026 23:59:13 +0100 Subject: [PATCH 2/4] update PR link in changelog --- packages/aws_cloudwatch_input_otel/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/aws_cloudwatch_input_otel/changelog.yml b/packages/aws_cloudwatch_input_otel/changelog.yml index 6dd0b5710dd..f1cd1f7b25f 100644 --- a/packages/aws_cloudwatch_input_otel/changelog.yml +++ b/packages/aws_cloudwatch_input_otel/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Increase default metric autodiscover_limit to 10,000 series type: enhancement - link: https://github.com/elastic/integrations/pull/99999 + link: https://github.com/elastic/integrations/pull/21199 - version: "0.6.0" changes: - description: Publish Agentless to Elastic Managed integrations name change. From 68a07a6a77e3f9aa6df66f069a846601b62ad163 Mon Sep 17 00:00:00 2001 From: Tom Myers Date: Wed, 16 Sep 2026 14:03:11 +0100 Subject: [PATCH 3/4] update default collection period to 5m for all data streams to reduce cost ceiling; clarify README. --- .../_dev/build/docs/README.md | 27 ++++++++++--------- .../aws_cloudwatch_input_otel/changelog.yml | 2 +- .../aws_cloudwatch_input_otel/docs/README.md | 27 ++++++++++--------- .../aws_cloudwatch_input_otel/manifest.yml | 24 ++++++++--------- 4 files changed, 43 insertions(+), 37 deletions(-) diff --git a/packages/aws_cloudwatch_input_otel/_dev/build/docs/README.md b/packages/aws_cloudwatch_input_otel/_dev/build/docs/README.md index df78f907484..22e306275b5 100644 --- a/packages/aws_cloudwatch_input_otel/_dev/build/docs/README.md +++ b/packages/aws_cloudwatch_input_otel/_dev/build/docs/README.md @@ -71,23 +71,26 @@ Each service template exposes two settings that control how metrics are polled f Guidance: - Set **Period** to the metric's native publishing resolution — typically **5 minutes** for services on 5-minute resolution (for example, EC2 basic monitoring) and **1 minute** for services that publish at 1-minute resolution (for example, RDS and Application ELB). -- In most cases, set **Collection Interval** equal to **Period**. Polling more frequently than the period just re-reads the same data point; polling less frequently can miss points. +- Set **Collection Interval** to **Period**, or to a multiple of it. Each poll returns every data point in the elapsed window, so polling less often reduces cost proportionally without losing data — it only delays how soon the data arrives. Polling more often than Period re-reads the same data point and multiplies cost for no benefit. - The defaults below are pre-tuned per service, so you typically don't need to change them. -| Service | Collection Interval | Period | -|---|---------------------|-----------------| -| AWS EC2 | 5m | 5m | -| AWS Lambda | 1m | 1m | -| AWS RDS | 1m | 1m | -| AWS SQS | 1m | 1m | -| AWS Application ELB | 1m | 1m | -| AWS ECS | 1m | 1m | +| Service | Collection Interval | Period | +|---------------------|---------------------|--------| +| AWS EC2 | 5m | 5m | +| AWS Lambda | 5m | 1m | +| AWS RDS | 5m | 1m | +| AWS SQS | 5m | 1m | +| AWS Application ELB | 5m | 1m | +| AWS Classic ELB | 5m | 1m | +| AWS Network ELB | 5m | 1m | +| AWS Gateway ELB | 5m | 1m | +| AWS ECS | 5m | 1m | -### Autodiscover Limit and impact on cost +### Autodiscover Limit and impact on cost -This integration automatically discovers and collects all CloudWatch metrics published for the configured namespaces. The CloudWatch API bills per metric requested, so the more resources that exist in your account, the more metrics are discovered and the higher the collection cost. +This integration automatically discovers and collects CloudWatch metrics published for the configured namespaces. The CloudWatch API bills per metric requested, so the more resources that exist in your account, the more metrics are discovered and the higher the collection cost. -To avoid unexpectedly large bills, the number of metrics collected is capped by the **Autodiscover Limit**, which defaults to 10,000, high enough to capture all metrics in a typical deployment. The limit applies to each namespace separately. It is a ceiling, not a fixed value - you are billed only for the metrics that actually exist, so smaller accounts cost proportionally less. With the 10,000 limit, the maximum cost per namespace is ~$3,500 per month at a 5-minute collection interval. +To avoid unexpectedly large bills, the number of metrics collected is capped by the **Autodiscover Limit**, which defaults to 10,000, high enough to capture all metrics in a typical deployment. The limit applies to each namespace separately. It is a ceiling, not a fixed value - you are billed only for the metrics that actually exist, so smaller accounts cost proportionally less. With the 10,000 limit, the maximum cost per namespace is ~$2,600 per month at the default 5-minute collection interval. If you need to reduce cost, you can lower the **Autodiscover Limit** in the integration's advanced settings. Reducing the limit below the number of metrics in your account means some metrics will not be collected, and which ones are dropped is not predictable. We recommend changing this value only if you understand the metric volume in your AWS account, and lowering it gradually while verifying that the metrics you rely on are still present. diff --git a/packages/aws_cloudwatch_input_otel/changelog.yml b/packages/aws_cloudwatch_input_otel/changelog.yml index f1cd1f7b25f..08c3986e3f3 100644 --- a/packages/aws_cloudwatch_input_otel/changelog.yml +++ b/packages/aws_cloudwatch_input_otel/changelog.yml @@ -1,7 +1,7 @@ # newer versions go on top - version: "0.7.0" changes: - - description: Increase default metric autodiscover_limit to 10,000 series + - description: Increase the default metric autodiscover limit to 10,000 metrics and reduce the default collection interval to 5m for every service, retaining 1m metric resolution where the service publishes at 1m. type: enhancement link: https://github.com/elastic/integrations/pull/21199 - version: "0.6.0" diff --git a/packages/aws_cloudwatch_input_otel/docs/README.md b/packages/aws_cloudwatch_input_otel/docs/README.md index da0c8d98a59..0f6293648bd 100644 --- a/packages/aws_cloudwatch_input_otel/docs/README.md +++ b/packages/aws_cloudwatch_input_otel/docs/README.md @@ -72,23 +72,26 @@ Each service template exposes two settings that control how metrics are polled f Guidance: - Set **Period** to the metric's native publishing resolution — typically **5 minutes** for services on 5-minute resolution (for example, EC2 basic monitoring) and **1 minute** for services that publish at 1-minute resolution (for example, RDS and Application ELB). -- In most cases, set **Collection Interval** equal to **Period**. Polling more frequently than the period just re-reads the same data point; polling less frequently can miss points. +- Set **Collection Interval** to **Period**, or to a multiple of it. Each poll returns every data point in the elapsed window, so polling less often reduces cost proportionally without losing data — it only delays how soon the data arrives. Polling more often than Period re-reads the same data point and multiplies cost for no benefit. - The defaults below are pre-tuned per service, so you typically don't need to change them. -| Service | Collection Interval | Period | -|---|---------------------|-----------------| -| AWS EC2 | 5m | 5m | -| AWS Lambda | 1m | 1m | -| AWS RDS | 1m | 1m | -| AWS SQS | 1m | 1m | -| AWS Application ELB | 1m | 1m | -| AWS ECS | 1m | 1m | +| Service | Collection Interval | Period | +|---------------------|---------------------|--------| +| AWS EC2 | 5m | 5m | +| AWS Lambda | 5m | 1m | +| AWS RDS | 5m | 1m | +| AWS SQS | 5m | 1m | +| AWS Application ELB | 5m | 1m | +| AWS Classic ELB | 5m | 1m | +| AWS Network ELB | 5m | 1m | +| AWS Gateway ELB | 5m | 1m | +| AWS ECS | 5m | 1m | -### Autodiscover Limit and impact on cost +### Autodiscover Limit and impact on cost -This integration automatically discovers and collects all CloudWatch metrics published for the configured namespaces. The CloudWatch API bills per metric requested, so the more resources that exist in your account, the more metrics are discovered and the higher the collection cost. +This integration automatically discovers and collects CloudWatch metrics published for the configured namespaces. The CloudWatch API bills per metric requested, so the more resources that exist in your account, the more metrics are discovered and the higher the collection cost. -To avoid unexpectedly large bills, the number of metrics collected is capped by the **Autodiscover Limit**, which defaults to 10,000, high enough to capture all metrics in a typical deployment. The limit applies to each namespace separately. It is a ceiling, not a fixed value - you are billed only for the metrics that actually exist, so smaller accounts cost proportionally less. With the 10,000 limit, the maximum cost per namespace is ~$3,500 per month at a 5-minute collection interval. +To avoid unexpectedly large bills, the number of metrics collected is capped by the **Autodiscover Limit**, which defaults to 10,000, high enough to capture all metrics in a typical deployment. The limit applies to each namespace separately. It is a ceiling, not a fixed value - you are billed only for the metrics that actually exist, so smaller accounts cost proportionally less. With the 10,000 limit, the maximum cost per namespace is ~$2,600 per month at the default 5-minute collection interval. If you need to reduce cost, you can lower the **Autodiscover Limit** in the integration's advanced settings. Reducing the limit below the number of metrics in your account means some metrics will not be collected, and which ones are dropped is not predictable. We recommend changing this value only if you understand the metric volume in your AWS account, and lowering it gradually while verifying that the metrics you rely on are still present. diff --git a/packages/aws_cloudwatch_input_otel/manifest.yml b/packages/aws_cloudwatch_input_otel/manifest.yml index 207dd46eafc..ecd827f7499 100644 --- a/packages/aws_cloudwatch_input_otel/manifest.yml +++ b/packages/aws_cloudwatch_input_otel/manifest.yml @@ -170,7 +170,7 @@ policy_templates: type: duration title: Collection Interval description: How often the receiver polls CloudWatch. - default: 1m + default: 5m required: false show_user: false - name: period @@ -230,7 +230,7 @@ policy_templates: type: duration title: Collection Interval description: How often the receiver polls CloudWatch. - default: 1m + default: 5m required: false show_user: false - name: period @@ -289,7 +289,7 @@ policy_templates: type: duration title: Collection Interval description: How often the receiver polls CloudWatch. - default: 1m + default: 5m required: false show_user: false - name: period @@ -341,14 +341,14 @@ policy_templates: type: integer title: Autodiscover Limit description: Maximum number of metrics to enumerate for this namespace. Higher values pick up more metrics but cost more API calls. - default: 250 + default: 10000 required: false show_user: false - name: collection_interval type: duration title: Collection Interval description: How often the receiver polls CloudWatch. - default: 1m + default: 5m required: false show_user: false - name: period @@ -401,14 +401,14 @@ policy_templates: type: integer title: Autodiscover Limit description: Maximum number of metrics to enumerate for this namespace. Higher values pick up more metrics but cost more API calls. - default: 250 + default: 10000 required: false show_user: false - name: collection_interval type: duration title: Collection Interval description: How often the receiver polls CloudWatch. - default: 1m + default: 5m required: false show_user: false - name: period @@ -463,14 +463,14 @@ policy_templates: description: >- Maximum number of metrics to enumerate for this namespace. NLB can have 200+ metric/dimension combinations — set this higher than the default if metrics are missing. - default: 250 + default: 10000 required: false show_user: false - name: collection_interval type: duration title: Collection Interval description: How often the receiver polls CloudWatch. - default: 1m + default: 5m required: false show_user: false - name: period @@ -523,14 +523,14 @@ policy_templates: type: integer title: Autodiscover Limit description: Maximum number of metrics to enumerate for this namespace. Higher values pick up more metrics but cost more API calls. - default: 250 + default: 10000 required: false show_user: false - name: collection_interval type: duration title: Collection Interval description: How often the receiver polls CloudWatch. - default: 1m + default: 5m required: false show_user: false - name: period @@ -590,7 +590,7 @@ policy_templates: type: duration title: Collection Interval description: How often the receiver polls CloudWatch. - default: 1m + default: 5m required: false show_user: false - name: period From d774fb3f8cb18266746194d973f102831e5c5614 Mon Sep 17 00:00:00 2001 From: Tom Myers Date: Thu, 17 Sep 2026 12:53:55 +0100 Subject: [PATCH 4/4] update 'services' table in docs to match available policy templates --- packages/aws_cloudwatch_input_otel/_dev/build/docs/README.md | 3 +++ packages/aws_cloudwatch_input_otel/docs/README.md | 3 +++ 2 files changed, 6 insertions(+) diff --git a/packages/aws_cloudwatch_input_otel/_dev/build/docs/README.md b/packages/aws_cloudwatch_input_otel/_dev/build/docs/README.md index 22e306275b5..98b65a5a662 100644 --- a/packages/aws_cloudwatch_input_otel/_dev/build/docs/README.md +++ b/packages/aws_cloudwatch_input_otel/_dev/build/docs/README.md @@ -36,6 +36,9 @@ Each AWS service is available as a separate option when you add the integration. | AWS RDS | `AWS/RDS` | | AWS SQS | `AWS/SQS` | | AWS Application ELB | `AWS/ApplicationELB` | +| AWS Classic ELB | `AWS/ELB` | +| AWS Network ELB | `AWS/NetworkELB` | +| AWS Gateway ELB | `AWS/GatewayELB` | | AWS ECS / Fargate | `AWS/ECS` | Each service collects a set of statistics chosen to suit that service's metrics (for example, averages for utilization metrics and sums for counters). These defaults work well out of the box and require no configuration. diff --git a/packages/aws_cloudwatch_input_otel/docs/README.md b/packages/aws_cloudwatch_input_otel/docs/README.md index 0f6293648bd..fb03f03e264 100644 --- a/packages/aws_cloudwatch_input_otel/docs/README.md +++ b/packages/aws_cloudwatch_input_otel/docs/README.md @@ -37,6 +37,9 @@ Each AWS service is available as a separate option when you add the integration. | AWS RDS | `AWS/RDS` | | AWS SQS | `AWS/SQS` | | AWS Application ELB | `AWS/ApplicationELB` | +| AWS Classic ELB | `AWS/ELB` | +| AWS Network ELB | `AWS/NetworkELB` | +| AWS Gateway ELB | `AWS/GatewayELB` | | AWS ECS / Fargate | `AWS/ECS` | Each service collects a set of statistics chosen to suit that service's metrics (for example, averages for utilization metrics and sums for counters). These defaults work well out of the box and require no configuration.