[aws.cloudtrail] Ingest Pipeline Improvments and Bugfixes

packages/aws/changelog.yml

@@ -1,4 +1,23 @@
 # newer versions go on top
+- version: "7.4.0"
+  changes:
+    - description: |
+        The changes below apply to the CloudTrail data stream only.
+        Replace every `ignore_failure: true` with an `ignore_missing` flag, a condition guard, or an explicit `on_failure` handler reporting to `error.message`, so real faults surface instead of being silently swallowed.
+        Add support for CloudTrail fields that were previously dropped.
+        Populate the `cloud.provider`, `cloud.service.name` and `related.*` ECS fields.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/20403
+    - description: |
+        The fixes below apply to the CloudTrail data stream only.
+        Stop populating `file.hash.sha256` on digest records, where it held the hash of the previous digest file while `file.path` pointed at the current one. The value is still available as `aws.cloudtrail.digest.previous_hash_value` and in `related.hash`.
+        Fix `DeleteObject` using `event.type: delete`, which is not a valid ECS value. It is now `deletion`.
+        Fix `Publish`, `SendCommand` and `Converse` producing no `event.type`. They now use `info`, `start` and `info` respectively.
+        Fix `retain: none` behaving like `retain: all`. It now drops both the keyword and the flattened copies as documented.
+        Fix `retain: keyword` and `retain: minimal` discarding digest and Insights records. The setting governs API call payloads and no longer affects either record type.
+        Fix `aws.cloudtrail.service_event_details` surviving `retain: flattened` and `retain: none`. It is now governed like the other payload fields.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/20403
 - version: "7.3.0"
   changes:
     - description: Add tags to ingest pipeline processors and add `preserve_original_event` to pipeline-level `on_failure` handlers.

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-add-permission-json.log-expected.json

@@ -41,7 +41,11 @@
                 "account": {
                     "id": "000000000"
                 },
-                "region": "us-east-1"
+                "provider": "aws",
+                "region": "us-east-1",
+                "service": {
+                    "name": "lambda"
+                }
             },
             "ecs": {
                 "version": "8.11.0"
@@ -72,6 +76,9 @@
                     "arn:aws:iam::000000000:user/test@elastic.co",
                     "cloudtrail-events-test"
                 ],
+                "ip": [
+                    "216.160.83.56"
+                ],
                 "user": [
                     "PRINCIPALID",
                     "test@elastic.co"

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-add-user-to-group-json.log-expected.json

@@ -34,7 +34,11 @@
                 "account": {
                     "id": "123456789012"
                 },
-                "region": "us-east-2"
+                "provider": "aws",
+                "region": "us-east-2",
+                "service": {
+                    "name": "iam"
+                }
             },
             "ecs": {
                 "version": "8.11.0"
@@ -64,6 +68,9 @@
                     "EXAMPLE_KEY_ID",
                     "Alice"
                 ],
+                "ip": [
+                    "127.0.0.1"
+                ],
                 "user": [
                     "Bob",
                     "EX_PRINCIPAL_ID",

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-email-session-json.log-expected.json

@@ -56,7 +56,11 @@
                 "account": {
                     "id": "111111111111"
                 },
-                "region": "us-east-2"
+                "provider": "aws",
+                "region": "us-east-2",
+                "service": {
+                    "name": "sts"
+                }
             },
             "ecs": {
                 "version": "8.11.0"
@@ -85,6 +89,9 @@
                     "AWSReservedSSO_SecurityAudit_8df5ccd37b",
                     "arn:aws:iam::111111111111:role/JohnRole2"
                 ],
+                "ip": [
+                    "81.2.69.144"
+                ],
                 "user": [
                     "firstname.lastname@example.com",
                     "firstname.lastname",

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-assume-role-json.log-expected.json

@@ -80,7 +80,11 @@
                 "account": {
                     "id": "111111111111"
                 },
-                "region": "us-east-2"
+                "provider": "aws",
+                "region": "us-east-2",
+                "service": {
+                    "name": "sts"
+                }
             },
             "ecs": {
                 "version": "8.11.0"
@@ -111,6 +115,9 @@
                     "arn:aws:iam::111111111111:role/JohnRole1",
                     "arn:aws:iam::111111111111:role/JohnRole2"
                 ],
+                "ip": [
+                    "81.2.69.144"
+                ],
                 "user": [
                     "JohnRole1",
                     "AROAIN5ATK5U7KEXAMPLE:JohnRole1",
@@ -259,7 +266,11 @@
                 "account": {
                     "id": "111111111111"
                 },
-                "region": "us-east-2"
+                "provider": "aws",
+                "region": "us-east-2",
+                "service": {
+                    "name": "sts"
+                }
             },
             "ecs": {
                 "version": "8.11.0"
@@ -289,6 +300,12 @@
                     "arn:aws:iam::111111111111:role/JohnRole1",
                     "arn:aws:iam::111111111111:role/JohnRole2"
                 ],
+                "hosts": [
+                    "i-1234567890abcdef0"
+                ],
+                "ip": [
+                    "81.2.69.144"
+                ],
                 "user": [
                     "JohnRole1",
                     "AROAIN5ATK5U7KEXAMPLE:JohnRole1",

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-attach-group-policy-json.log-expected.json

@@ -35,7 +35,11 @@
                 "account": {
                     "id": "000000000"
                 },
-                "region": "us-east-1"
+                "provider": "aws",
+                "region": "us-east-1",
+                "service": {
+                    "name": "iam"
+                }
             },
             "ecs": {
                 "version": "8.11.0"
@@ -74,6 +78,9 @@
                     "ACCESSKEYID",
                     "arn:aws:iam::000000000:user/test@elastic.co"
                 ],
+                "ip": [
+                    "216.160.83.56"
+                ],
                 "user": [
                     "PRINCIPALID",
                     "test@elastic.co"
@@ -138,6 +145,9 @@
         },
         {
             "@timestamp": "2021-11-11T01:02:03.123456789Z",
+            "cloud": {
+                "provider": "aws"
+            },
             "ecs": {
                 "version": "8.11.0"
             },

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-attach-role-policy-json.log-expected.json

@@ -35,7 +35,11 @@
                 "account": {
                     "id": "000000000"
                 },
-                "region": "us-east-1"
+                "provider": "aws",
+                "region": "us-east-1",
+                "service": {
+                    "name": "iam"
+                }
             },
             "ecs": {
                 "version": "8.11.0"
@@ -70,6 +74,9 @@
                     "arn:aws:iam::000000000:user/test@elastic.co",
                     "cloudtrail-role"
                 ],
+                "ip": [
+                    "216.160.83.56"
+                ],
                 "user": [
                     "PRINCIPALID",
                     "test@elastic.co"

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-attach-user-policy-json.log-expected.json

@@ -29,6 +29,7 @@
                         "arn": "arn:aws:sts::000000000:assumed-role/ec2-instance-role/i-06815aa7cf7d21f8f",
                         "session_context": {
                             "creation_date": "2024-10-30T19:14:35.000Z",
+                            "ec2_role_delivery": "2.0",
                             "mfa_authenticated": "false",
                             "session_issuer": {
                                 "account_id": "000000000",
@@ -45,7 +46,11 @@
                 "account": {
                     "id": "000000000"
                 },
-                "region": "us-east-1"
+                "provider": "aws",
+                "region": "us-east-1",
+                "service": {
+                    "name": "iam"
+                }
             },
             "ecs": {
                 "version": "8.11.0"
@@ -82,6 +87,9 @@
                     "arn:aws:iam::aws:policy/AdministratorAccess",
                     "arn:aws:iam::000000000:role/ec2-instance-role"
                 ],
+                "ip": [
+                    "216.160.83.56"
+                ],
                 "user": [
                     "i-06815aa7cf7d21f8f",
                     "pwncloud-backdoor-user",

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-attach-user-role-json.log-expected.json

@@ -35,7 +35,11 @@
                 "account": {
                     "id": "00000000000"
                 },
-                "region": "us-east-1"
+                "provider": "aws",
+                "region": "us-east-1",
+                "service": {
+                    "name": "iam"
+                }
             },
             "ecs": {
                 "version": "8.11.0"
@@ -71,6 +75,9 @@
                     "ACCESSKEYID",
                     "test-cloudtrail"
                 ],
+                "ip": [
+                    "216.160.83.56"
+                ],
                 "user": [
                     "test-cloudtrail",
                     "PRINCIPALID",

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-authorize-security-group-egress-json.log-expected.json

@@ -64,6 +64,7 @@
                         "arn": "arn:aws:sts::000000000:assumed-role/ec2-instance-role/i-06815aa7cf7d21f8f",
                         "session_context": {
                             "creation_date": "2024-10-30T19:14:35.000Z",
+                            "ec2_role_delivery": "2.0",
                             "mfa_authenticated": "false",
                             "session_issuer": {
                                 "account_id": "000000000",
@@ -80,7 +81,11 @@
                 "account": {
                     "id": "000000000"
                 },
-                "region": "us-east-1"
+                "provider": "aws",
+                "region": "us-east-1",
+                "service": {
+                    "name": "ec2"
+                }
             },
             "ecs": {
                 "version": "8.11.0"
@@ -106,6 +111,9 @@
                     "arn:aws:iam::000000000:role/ec2-instance-role",
                     "sg-038ccc3a1f7b05f42"
                 ],
+                "ip": [
+                    "216.160.83.56"
+                ],
                 "user": [
                     "i-06815aa7cf7d21f8f",
                     "PRINCIPALID:i-06815aa7cf7d21f8f",

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-authorize-security-group-ingress-json.log-expected.json

@@ -70,7 +70,11 @@
                 "account": {
                     "id": "000000000"
                 },
-                "region": "us-east-1"
+                "provider": "aws",
+                "region": "us-east-1",
+                "service": {
+                    "name": "ec2"
+                }
             },
             "ecs": {
                 "version": "8.11.0"
@@ -98,6 +102,9 @@
                     "sg-0f63fecfa17ef4fee",
                     "sgr-027eb7c59241731e0"
                 ],
+                "ip": [
+                    "216.160.83.56"
+                ],
                 "user": [
                     "PRINCIPALID",
                     "test@elastic.co"

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-batch-get-secret-value-json.log-expected.json

@@ -37,7 +37,11 @@
                 "account": {
                     "id": "000000000"
                 },
-                "region": "us-east-1"
+                "provider": "aws",
+                "region": "us-east-1",
+                "service": {
+                    "name": "secretsmanager"
+                }
             },
             "ecs": {
                 "version": "8.11.0"
@@ -62,6 +66,9 @@
                     "arn:aws:secretsmanager:us-east-1:000000000:secret:DdApiKeySecret-aUFEgO4SmSGN-xqgqXR",
                     "arn:aws:iam::000000000:user/test@elastic.co"
                 ],
+                "ip": [
+                    "216.160.83.56"
+                ],
                 "user": [
                     "PRINCIPALID",
                     "test@elastic.co"

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-change-password-json.log-expected.json

@@ -28,7 +28,11 @@
                 "account": {
                     "id": "0123456789012"
                 },
-                "region": "us-east-1"
+                "provider": "aws",
+                "region": "us-east-1",
+                "service": {
+                    "name": "iam"
+                }
             },
             "ecs": {
                 "version": "8.11.0"
@@ -55,6 +59,9 @@
                     "Alice",
                     "arn:aws:iam::0123456789012:user/Alice"
                 ],
+                "ip": [
+                    "127.0.0.1"
+                ],
                 "user": [
                     "0123456789012",
                     "Alice"
@@ -112,7 +119,11 @@
                 "account": {
                     "id": "0123456789012"
                 },
-                "region": "us-east-1"
+                "provider": "aws",
+                "region": "us-east-1",
+                "service": {
+                    "name": "iam"
+                }
             },
             "ecs": {
                 "version": "8.11.0"
@@ -139,6 +150,9 @@
                     "Alice",
                     "arn:aws:iam::0123456789012:user/Alice"
                 ],
+                "ip": [
+                    "127.0.0.1"
+                ],
                 "user": [
                     "0123456789012",
                     "Alice"

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-cloudtrail-digest-json-keyword.log

@@ -0,0 +1 @@
+{"awsAccountId":"123456789123","digestStartTime":"2020-09-11T18:36:49Z","digestEndTime":"2020-09-11T19:36:49Z","digestS3Bucket":"alice-bucket","digestS3Object":"AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T193649Z.json.gz","digestPublicKeyFingerprint":"47aaa19f7eec22e9bd0b5e58cfade8cb","digestSignatureAlgorithm":"SHA256withRSA","newestEventTime":"2020-09-11T19:26:24Z","oldestEventTime":"2020-09-11T18:32:04Z","previousDigestS3Bucket":"alice-bucket","previousDigestS3Object":"AWSLogs/123456789123/CloudTrail-Digest/us-west-2/2020/09/11/123456789123_CloudTrail-Digest_us-west-2_leh-ct-test_us-west-2_20200911T183649Z.json.gz","previousDigestHashValue":"531914fcfa0dbacf0c9dd1475a1fdcb5dea6e85921409f3c3ec0ba39063c860","previousDigestHashAlgorithm":"SHA-256","previousDigestSignature":"10e0872f32fa1d299d0cc98e94d4c88a6a2eada9d9fc3ae6d53dfe8d54c7caf807072f1e1eec47efdeecfcc22483887f8fddfc954ae587fba43e7676b5547f432fa8722ba1c5baa6b233bcb528ce7c01e3748aab8f28c16c024de79da820128b4c9e5ce65e98a9c4e631687ecc89c224a11bb3df06ce441ff740e4ac9fbd41159e77f5863550118284121f193e357866fbd0463faffb56e194af196e35a7675c3bbd0a398f43159343c3f59129d6339a281a8fdb3192f3fffea9bd21dbb0a705ebfae1921f2133aab0ad29522aea6df0828c1780d3f3ed6b8270ab3ba24459916b0fbbe82fba6ff9677bafe7306e0f5edcc0f1508cdb4e36f3e3b30e653e9987","logFiles":[{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1930Z_l2pGqVS53QcGdAkp.json.gz","hashValue":"420784a5bbc12e9ac442451e8ec1356744fdeabf4fee0d2222508db6d448139c","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:26:24Z","oldestEventTime":"2020-09-11T19:26:24Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1915Z_TIKlbLnJ6IwUxqxw.json.gz","hashValue":"4e1eb2a8b41d032cbb16e5449fc8f3eac304e7d43017a391b37c788c77336196","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T19:11:18Z","oldestEventTime":"2020-09-11T19:11:18Z"},{"s3Bucket":"alice-bucket","s3Object":"AWSLogs/123456789123/CloudTrail/us-west-2/2020/09/11/123456789123_CloudTrail_us-west-2_20200911T1835Z_OPJhVNodH1gY760s.json.gz","hashValue":"2695aeb3b4c1f021fe76e0b36f5ac15e557c41c58af6eef282d77ef056210d70","hashAlgorithm":"SHA-256","newestEventTime":"2020-09-11T18:32:04Z","oldestEventTime":"2020-09-11T18:32:04Z"}]}

packages/aws/data_stream/cloudtrail/_dev/test/pipeline/test-cloudtrail-digest-json-keyword.log-config.yml

@@ -0,0 +1,13 @@
+dynamic_fields:
+  # This can be removed after ES 8.16.2 is set as the minimum version supported in the manifest.
+  # Once removed, it requires to update pipeline tests to remove the trailing dot where required.
+  # Relates:  https://github.com/elastic/elasticsearch/pull/117213
+  "user_agent.version": '^\d+\.\d+(\.|\..*)?$'
+fields:
+  # Simulate @timestamp value from Filebeat.
+  '@timestamp': '2021-11-11T01:02:03.123456789Z'
+  tags:
+    - preserve_original_event
+    - actor_target_mapping
+  _conf:
+    retain: keyword