From e8768ca5e211e7bb571974bc26771cd4843e86c9 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 6 Aug 2026 09:48:28 +0200 Subject: [PATCH 1/6] [ti_opencti] Fix service DLL handling and other minor pipeline issues The original service DLL handling had two broken renames: - To `_ingest.node._value.additional_names` rather than `_ingest._value....` (from the foreach processor). - From `_ingest._value.node` to `_ingest._value`, which failed because the target exists. It also produced `service_dlls`, while the mappings and downstream processors expect `service_dll`. It's fixed and simplified by replacing the four nested foreach processors with a single Painless script that unwraps each `serviceDlls.edges[].node` into `service_dll` and renames the OpenCTI-specific fields at the same time. Invalid `?.` syntax is removed from Mustache templates (it's for Painless, not Mustache). The `threat.indicator.file.size` value is set with `copy_from` rather than a template to avoid unnecessary stringification. --- packages/ti_opencti/changelog.yml | 11 ++ .../test-process-with-service-dll.json | 86 ++++++++++++ ...rocess-with-service-dll.json-expected.json | 124 ++++++++++++++++++ .../elasticsearch/ingest_pipeline/default.yml | 80 +++++------ .../ingest_pipeline/ecs_from_file.yml | 2 +- packages/ti_opencti/manifest.yml | 2 +- 6 files changed, 254 insertions(+), 51 deletions(-) create mode 100644 packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-process-with-service-dll.json create mode 100644 packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-process-with-service-dll.json-expected.json diff --git a/packages/ti_opencti/changelog.yml b/packages/ti_opencti/changelog.yml index 7ae0c620a61..3d1d8af2521 100644 --- a/packages/ti_opencti/changelog.yml +++ b/packages/ti_opencti/changelog.yml @@ -1,4 +1,15 @@ # newer versions go on top +- version: "2.15.4" + changes: + - description: Fix handling of service DLL observables to avoid invalid and incorrect renames, and simplify. + type: bugfix + link: https://github.com/elastic/integrations/pull/1 + - description: Fix invalid `?.` syntax in a Mustache templates. + type: bugfix + link: https://github.com/elastic/integrations/pull/1 + - description: Copy `threat.indicator.file.size` value without unnecessary stringification. + type: bugfix + link: https://github.com/elastic/integrations/pull/1 - version: "2.15.3" changes: - description: Set agentless deployment mode `release` field to `ga`. diff --git a/packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-process-with-service-dll.json b/packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-process-with-service-dll.json new file mode 100644 index 00000000000..cecc50ec440 --- /dev/null +++ b/packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-process-with-service-dll.json @@ -0,0 +1,86 @@ +{ + "events": [ + { + "id": "abc123ab-c123-abc1-23ab-c123abc123ab", + "standard_id": "indicator--123abc12-3abc-123a-bc12-3abc123abc12", + "is_inferred": false, + "revoked": true, + "confidence": 30, + "lang": "en", + "created": "2023-01-17T06:44:58.868Z", + "modified": "2023-01-17T09:29:44.973Z", + "pattern_type": "stix", + "pattern_version": "2.1", + "pattern": "[process:command_line = './gedit-bin --new-window']", + "name": "Example process ./gedit-bin --new-window", + "description": null, + "valid_from": "2017-08-23T14:00:51.000Z", + "valid_until": "2017-10-22T14:00:51.000Z", + "x_opencti_score": 50, + "x_opencti_detection": false, + "x_opencti_main_observable_type": "Process", + "createdBy": { + "identity_class": "tester", + "name": "Manual" + }, + "objectMarking": [], + "objectLabel": [], + "killChainPhases": [], + "externalReferences": { + "edges": [] + }, + "observables": { + "edges": [ + { + "node": { + "id": "def123de-123d-f123-ef12-def123def123", + "standard_id": "process--a12b3a12-3a12-3a12-3a12-3a12b3a12b3a", + "entity_type": "Process", + "observable_value": "1221", + "pid": 1221, + "command_line": "./gedit-bin --new-window", + "serviceDlls": { + "edges": [ + { + "node": { + "hashes": [ + { + "algorithm": "MD5", + "hash": "6ad06d0d468f76fdc23e561054eab063" + }, + { + "algorithm": "SHA-256", + "hash": "841514e050d5c7d0e9b431de1c1379bcbcc0d0bb290290b1cfbd57e0f21fcc36" + } + ], + "size": 44544, + "name": "evil-service.dll", + "name_enc": null, + "magic_number_hex": null, + "mime_type": "application/vnd.microsoft.portable-executable", + "ctime": "2023-01-10T00:00:00.000Z", + "mtime": "2023-01-11T00:00:00.000Z", + "atime": null, + "x_opencti_additional_names": [ + "innocent-name.dll" + ], + "obsContent": { + "payload_bin": null, + "url": "https://malware.example.test/evil-service.dll", + "encryption_algorithm": null, + "decryption_key": null + } + } + } + ] + } + } + } + ], + "pageInfo": { + "globalCount": 1 + } + } + } + ] +} diff --git a/packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-process-with-service-dll.json-expected.json b/packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-process-with-service-dll.json-expected.json new file mode 100644 index 00000000000..489aebf210c --- /dev/null +++ b/packages/ti_opencti/data_stream/indicator/_dev/test/pipeline/test-process-with-service-dll.json-expected.json @@ -0,0 +1,124 @@ +{ + "expected": [ + { + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "threat" + ], + "created": "2023-01-17T06:44:58.868Z", + "id": "abc123ab-c123-abc1-23ab-c123abc123ab", + "kind": "enrichment", + "type": [ + "indicator" + ] + }, + "opencti": { + "indicator": { + "creator_identity_class": "tester", + "detection": false, + "invalid_or_revoked_from": "2017-10-22T14:00:51.000Z", + "is_inferred": false, + "lang": "en", + "observables_count": 1, + "pattern": "[process:command_line = './gedit-bin --new-window']", + "pattern_type": "stix", + "pattern_version": "2.1", + "revoked": true, + "score": 50, + "standard_id": "indicator--123abc12-3abc-123a-bc12-3abc123abc12", + "valid_from": "2017-08-23T14:00:51.000Z", + "valid_until": "2017-10-22T14:00:51.000Z" + }, + "observable": { + "process": { + "command_line": "./gedit-bin --new-window", + "entity_type": "Process", + "id": "def123de-123d-f123-ef12-def123def123", + "pid": 1221, + "service_dll": [ + { + "additional_names": [ + "innocent-name.dll" + ], + "content": { + "url": "https://malware.example.test/evil-service.dll" + }, + "ctime": "2023-01-10T00:00:00.000Z", + "hash": { + "md5": "6ad06d0d468f76fdc23e561054eab063", + "sha256": "841514e050d5c7d0e9b431de1c1379bcbcc0d0bb290290b1cfbd57e0f21fcc36" + }, + "mime_type": "application/vnd.microsoft.portable-executable", + "mtime": "2023-01-11T00:00:00.000Z", + "name": "evil-service.dll", + "size": 44544 + } + ], + "standard_id": "process--a12b3a12-3a12-3a12-3a12-3a12b3a12b3a", + "value": "1221" + } + } + }, + "related": { + "hash": [ + "841514e050d5c7d0e9b431de1c1379bcbcc0d0bb290290b1cfbd57e0f21fcc36", + "6ad06d0d468f76fdc23e561054eab063" + ], + "hosts": [ + "malware.example.test" + ] + }, + "tags": [ + "forwarded", + "opencti-indicator", + "ecs-indicator-detail" + ], + "threat": { + "feed": { + "dashboard_id": "ti_opencti-83b2bef0-591c-11ee-ba5f-49a63bb985cd", + "description": "Indicator data from OpenCTI", + "name": "OpenCTI", + "reference": "https://docs.opencti.io/latest/usage/overview/" + }, + "indicator": { + "confidence": "Medium", + "file": { + "created": "2023-01-10T00:00:00.000Z", + "extension": [ + "dll", + "dll" + ], + "hash": { + "md5": "6ad06d0d468f76fdc23e561054eab063", + "sha256": "841514e050d5c7d0e9b431de1c1379bcbcc0d0bb290290b1cfbd57e0f21fcc36" + }, + "mime_type": "application/vnd.microsoft.portable-executable", + "mtime": "2023-01-11T00:00:00.000Z", + "name": [ + "evil-service.dll", + "innocent-name.dll" + ], + "size": 44544, + "type": "file" + }, + "modified_at": "2023-01-17T09:29:44.973Z", + "name": "Example process ./gedit-bin --new-window", + "provider": "Manual", + "reference": "https://demo.opencti.io/dashboard/observations/indicators/abc123ab-c123-abc1-23ab-c123abc123ab", + "type": "process", + "url": { + "domain": "malware.example.test", + "extension": "dll", + "full": "https://malware.example.test/evil-service.dll", + "original": "https://malware.example.test/evil-service.dll", + "path": "/evil-service.dll", + "scheme": "https" + } + } + } + } + ] +} diff --git a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml index bc6b8726481..35736769194 100644 --- a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml @@ -285,50 +285,32 @@ processors: target_field: _ingest._value.node.content ignore_missing: true - - foreach: - field: observables.edges - ignore_missing: true - processor: - foreach: - field: _ingest._value.node.serviceDlls.edges - ignore_missing: true - processor: - rename: - field: _ingest._value.node.x_opencti_additional_names - target_field: _ingest.node._value.additional_names - - - foreach: - field: observables.edges - ignore_missing: true - processor: - foreach: - field: _ingest._value.node.serviceDlls.edges - ignore_missing: true - processor: - rename: - field: _ingest._value.node.obsContent - target_field: _ingest._value.node.content - - - foreach: - field: observables.edges - ignore_missing: true - processor: - foreach: - field: _ingest._value.node.serviceDlls.edges - ignore_missing: true - processor: - rename: - field: _ingest._value.node - target_field: _ingest._value - - - foreach: - field: observables.edges - ignore_missing: true - processor: - rename: - field: _ingest._value.node.serviceDlls.edges - target_field: _ingest._value.node.service_dlls - ignore_missing: true + - script: + description: Unwrap service DLL nodes from the GraphQL connection into service_dll + lang: painless + source: | + if (ctx.observables?.edges instanceof List) { + for (def edge : ctx.observables.edges) { + def dllEdges = edge.node?.serviceDlls?.edges; + if (dllEdges instanceof List) { + def dlls = []; + for (def dllEdge : dllEdges) { + def dll = dllEdge.node; + if (dll != null) { + if (dll.containsKey('x_opencti_additional_names')) { + dll.additional_names = dll.remove('x_opencti_additional_names'); + } + if (dll.containsKey('obsContent')) { + dll.content = dll.remove('obsContent'); + } + dlls.add(dll); + } + } + edge.node.service_dll = dlls; + } + edge.node?.remove('serviceDlls'); + } + } - script: description: Merge array of objects into a single object for startup_info @@ -371,9 +353,9 @@ processors: } observable.remove('hashes'); } - if (observable.containsKey('service_dlls')) { - for (int ii = 0; ii < observable.service_dlls.length; ii++) { - Map serviceDll = observable.service_dlls[ii]; + if (observable.containsKey('service_dll')) { + for (int ii = 0; ii < observable.service_dll.length; ii++) { + Map serviceDll = observable.service_dll[ii]; if (serviceDll.containsKey('hashes')) { def ecsHash = hashesToECS(serviceDll.hashes); if (ecsHash.size() > 0) { @@ -561,7 +543,7 @@ processors: processor: append: field: _tmp_found_urls - value: "{{{_ingest._value.content?.url}}}" + value: "{{{_ingest._value.content.url}}}" allow_duplicates: false - foreach: field: opencti.observable.media_content @@ -581,7 +563,7 @@ processors: processor: append: field: _tmp_found_urls - value: "{{{_ingest._value.content?.url}}}" + value: "{{{_ingest._value.content.url}}}" allow_duplicates: false - script: description: Remove null or empty items diff --git a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_file.yml b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_file.yml index 39b1f8f5c2e..a8e68cf7ad5 100644 --- a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_file.yml +++ b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_file.yml @@ -77,7 +77,7 @@ processors: - set: field: _tmp_file.size - value: "{{{_ingest._value.size}}}" + copy_from: _ingest._value.size ignore_empty_value: true - set: diff --git a/packages/ti_opencti/manifest.yml b/packages/ti_opencti/manifest.yml index 4d082f785db..1eb5f43c2cd 100644 --- a/packages/ti_opencti/manifest.yml +++ b/packages/ti_opencti/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.4.0" name: ti_opencti title: OpenCTI -version: "2.15.3" +version: "2.15.4" description: "Ingest threat intelligence indicators from OpenCTI with Elastic Agent." type: integration source: From 97d4c3f142449e8de2cadd4f1786c87fcb4ba03f Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 6 Aug 2026 11:00:09 +0200 Subject: [PATCH 2/6] Update links in changelog entry. --- packages/ti_opencti/changelog.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/ti_opencti/changelog.yml b/packages/ti_opencti/changelog.yml index 3d1d8af2521..311b1b19417 100644 --- a/packages/ti_opencti/changelog.yml +++ b/packages/ti_opencti/changelog.yml @@ -3,13 +3,13 @@ changes: - description: Fix handling of service DLL observables to avoid invalid and incorrect renames, and simplify. type: bugfix - link: https://github.com/elastic/integrations/pull/1 + link: https://github.com/elastic/integrations/pull/20569 - description: Fix invalid `?.` syntax in a Mustache templates. type: bugfix - link: https://github.com/elastic/integrations/pull/1 + link: https://github.com/elastic/integrations/pull/20569 - description: Copy `threat.indicator.file.size` value without unnecessary stringification. type: bugfix - link: https://github.com/elastic/integrations/pull/1 + link: https://github.com/elastic/integrations/pull/20569 - version: "2.15.3" changes: - description: Set agentless deployment mode `release` field to `ga`. From 1d00510462c286b91a41ace85282d8162ddfb786 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 6 Aug 2026 11:01:22 +0200 Subject: [PATCH 3/6] Fix typo in changelog entry. --- packages/ti_opencti/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ti_opencti/changelog.yml b/packages/ti_opencti/changelog.yml index 311b1b19417..b2a554177b3 100644 --- a/packages/ti_opencti/changelog.yml +++ b/packages/ti_opencti/changelog.yml @@ -4,7 +4,7 @@ - description: Fix handling of service DLL observables to avoid invalid and incorrect renames, and simplify. type: bugfix link: https://github.com/elastic/integrations/pull/20569 - - description: Fix invalid `?.` syntax in a Mustache templates. + - description: Fix invalid `?.` syntax in Mustache templates. type: bugfix link: https://github.com/elastic/integrations/pull/20569 - description: Copy `threat.indicator.file.size` value without unnecessary stringification. From 95980537360ac43de2d4b1fcbe290e84da9f4dcb Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 6 Aug 2026 12:07:55 +0200 Subject: [PATCH 4/6] Add generated pipeline tags. --- .../elasticsearch/ingest_pipeline/default.yml | 125 ++++++++++++++++-- .../ingest_pipeline/ecs_from_artifact.yml | 7 + .../ecs_from_autonomous_system.yml | 4 + .../ingest_pipeline/ecs_from_directory.yml | 7 + .../ecs_from_domain_name_or_hostname.yml | 4 + .../ingest_pipeline/ecs_from_file.yml | 13 ++ .../ingest_pipeline/ecs_from_pattern.yml | 9 ++ .../ingest_pipeline/ecs_from_url_field.yml | 7 + .../ecs_from_windows_registry_key.yml | 4 + .../ecs_from_windows_registry_value_type.yml | 5 + .../ecs_from_x509_certificate.yml | 12 ++ 11 files changed, 187 insertions(+), 10 deletions(-) diff --git a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml index 35736769194..f210807b38b 100644 --- a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml @@ -11,69 +11,86 @@ processors: if: ctx.error?.message != null description: error message set and no data to process. + #################### # Event ECS fields # #################### - - set: + tag: set_ecs_version_0f7334bb field: ecs.version value: '8.11.0' - set: + tag: set_event_kind_a131107f field: event.kind value: enrichment - set: + tag: set_event_category_cceb7c22 field: event.category value: ['threat'] - set: + tag: set_event_type_97bb78b9 field: event.type value: ['indicator'] - rename: + tag: rename_created_to_event_created_0f9cf48d field: created target_field: event.created - rename: + tag: rename_id_to_event_id_4f49abf1 field: id target_field: event.id + ###################### # Threat feed fields # ###################### - - set: + tag: set_threat_feed_dashboard_id_9340fe1d field: threat.feed.dashboard_id value: ti_opencti-83b2bef0-591c-11ee-ba5f-49a63bb985cd - set: + tag: set_threat_feed_description_8bd01795 field: threat.feed.description value: Indicator data from OpenCTI - set: + tag: set_threat_feed_name_60b56abf field: threat.feed.name value: OpenCTI - set: + tag: set_threat_feed_reference_99cb7716 field: threat.feed.reference value: https://docs.opencti.io/latest/usage/overview/ + ############################ # Generic indicator fields # ############################ - - set: + tag: set_threat_indicator_reference_1f855278 field: threat.indicator.reference value: "{{{_conf.url}}}/dashboard/observations/indicators/{{{event.id}}}" - remove: + tag: remove__conf_8e93c735 field: _conf ignore_missing: true - rename: + tag: rename_standard_id_to_opencti_indicator_standard_id_5c8532d0 field: standard_id target_field: opencti.indicator.standard_id - rename: + tag: rename_is_inferred_to_opencti_indicator_is_inferred_edd5adba field: is_inferred target_field: opencti.indicator.is_inferred - rename: + tag: rename_revoked_to_opencti_indicator_revoked_1df3098a field: revoked target_field: opencti.indicator.revoked - set: + tag: set_threat_indicator_confidence_a674fe52 field: threat.indicator.confidence value: 'Not Specified' - script: + tag: script_2fa9aea9 # based on https://stix2.readthedocs.io/en/latest/api/confidence/stix2.confidence.scales.html#stix2.confidence.scales.value_to_none_low_medium_high lang: painless source: | @@ -91,47 +108,61 @@ processors: ctx.threat.indicator.confidence = 'Not Specified'; } - remove: + tag: remove_confidence_05d7236d field: confidence - rename: + tag: rename_lang_to_opencti_indicator_lang_a557a930 field: lang target_field: opencti.indicator.lang - rename: + tag: rename_modified_to_threat_indicator_modified_at_890a6f22 field: modified target_field: threat.indicator.modified_at - rename: + tag: rename_updated_at_to_opencti_indicator_updated_at_fe4f5733 field: updated_at target_field: opencti.indicator.updated_at ignore_missing: true - rename: + tag: rename_pattern_type_to_opencti_indicator_pattern_type_ca5b7dc0 field: pattern_type target_field: opencti.indicator.pattern_type - rename: + tag: rename_pattern_version_to_opencti_indicator_pattern_version_fa977761 field: pattern_version target_field: opencti.indicator.pattern_version if: ctx.pattern_version != null ignore_missing: true - remove: + tag: remove_pattern_version_7fadaa07 field: pattern_version ignore_missing: true - rename: + tag: rename_pattern_to_opencti_indicator_pattern_96a05bd2 field: pattern target_field: opencti.indicator.pattern - rename: + tag: rename_name_to_threat_indicator_name_f77818ba field: name target_field: threat.indicator.name - rename: + tag: rename_description_to_threat_indicator_description_fbb019ae field: description target_field: threat.indicator.description - remove: + tag: remove_threat_indicator_description_bc5b2134 field: threat.indicator.description if: ctx.threat.indicator.description == null || ctx.threat.indicator.description.isEmpty() - rename: + tag: rename_valid_from_to_opencti_indicator_valid_from_c8653800 field: valid_from target_field: opencti.indicator.valid_from - rename: + tag: rename_valid_until_to_opencti_indicator_valid_until_74ae5a8c field: valid_until target_field: opencti.indicator.valid_until - script: + tag: script_8bfd3177 lang: painless source: | if (ctx.opencti.indicator.revoked == true && @@ -142,15 +173,19 @@ processors: ctx.opencti.indicator.invalid_or_revoked_from = ctx.opencti.indicator.valid_until; } - rename: + tag: rename_x_opencti_score_to_opencti_indicator_score_70107380 field: x_opencti_score target_field: opencti.indicator.score - rename: + tag: rename_x_opencti_detection_to_opencti_indicator_detection_728ab91a field: x_opencti_detection target_field: opencti.indicator.detection - rename: + tag: rename_x_opencti_main_observable_type_to_threat_indicator_type_8d88887a field: x_opencti_main_observable_type target_field: threat.indicator.type - script: + tag: script_30110556 description: Reformat indicator type lang: painless source: | @@ -159,18 +194,22 @@ processors: type = type.replace('stixfile', 'file'); ctx.threat.indicator.type = type; - rename: + tag: rename_createdBy_name_to_threat_indicator_provider_48fa0454 field: createdBy.name target_field: threat.indicator.provider ignore_missing: true - rename: + tag: rename_createdBy_identity_class_to_opencti_indicator_creator_identity_class_c28b0503 field: createdBy.identity_class target_field: opencti.indicator.creator_identity_class ignore_missing: true - remove: + tag: remove_createdBy_ba8401db field: createdBy ignore_missing: true - gsub: + tag: gsub_objectMarking_0_definition_to_threat_indicator_marking_tlp_902801f9 if: | ctx.objectMarking != null && !ctx.objectMarking.isEmpty() && @@ -180,9 +219,11 @@ processors: replacement: '' target_field: threat.indicator.marking.tlp - remove: + tag: remove_objectMarking_335e3bbd field: objectMarking - foreach: + tag: foreach_objectLabel_a5be1ce4 field: objectLabel ignore_missing: true processor: @@ -191,9 +232,11 @@ processors: value: "{{{_ingest._value.value}}}" allow_duplicates: false - remove: + tag: remove_objectLabel_fabc71d6 field: objectLabel - foreach: + tag: foreach_killChainPhases_180ffef6 field: killChainPhases ignore_missing: true processor: @@ -201,9 +244,11 @@ processors: field: opencti.indicator.kill_chain_phase value: "[{{{_ingest._value.kill_chain_name}}}] {{{_ingest._value.phase_name}}}" - remove: + tag: remove_killChainPhases_a8d0f04c field: killChainPhases - script: + tag: script_55962473 description: Move external reference nodes to their target location, without nulls lang: painless source: | @@ -232,18 +277,21 @@ processors: ctx.opencti.indicator.external_reference.add(newNode); } - remove: + tag: remove_externalReferences_8bdc8f12 field: externalReferences + ##################### # Observable fields # ##################### - - rename: + tag: rename_observables_pageInfo_globalCount_to_opencti_indicator_observables_count_f9217ef4 field: observables.pageInfo.globalCount target_field: opencti.indicator.observables_count ignore_missing: true - foreach: + tag: foreach_observables_edges_3db59843 field: observables.edges ignore_missing: true processor: @@ -251,6 +299,7 @@ processors: field: _ingest._value.node.value ignore_missing: true - foreach: + tag: foreach_observables_edges_83957232 field: observables.edges ignore_missing: true processor: @@ -259,6 +308,7 @@ processors: target_field: _ingest._value.node.value - foreach: + tag: foreach_observables_edges_be319b43 field: observables.edges ignore_missing: true processor: @@ -268,6 +318,7 @@ processors: ignore_missing: true - foreach: + tag: foreach_observables_edges_e71fe9bb field: observables.edges ignore_missing: true processor: @@ -277,6 +328,7 @@ processors: ignore_missing: true - foreach: + tag: foreach_observables_edges_9e3da177 field: observables.edges ignore_missing: true processor: @@ -286,6 +338,7 @@ processors: ignore_missing: true - script: + tag: script_7d05410e description: Unwrap service DLL nodes from the GraphQL connection into service_dll lang: painless source: | @@ -313,6 +366,7 @@ processors: } - script: + tag: script_14cfaa7a description: Merge array of objects into a single object for startup_info lang: painless source: | @@ -329,6 +383,7 @@ processors: } - script: + tag: script_7251d63c description: Restructure hashes to ECS format lang: painless source: | @@ -368,6 +423,7 @@ processors: } - script: + tag: script_95fdd14d description: Move observable nodes to their target location lang: painless source: | @@ -389,10 +445,12 @@ processors: ctx.opencti.observable[entity].add(observables[i]['node']); } - remove: + tag: remove_observables_ec3e1a2e field: observables ignore_missing: true - script: + tag: script_0c64bfed description: Remove null values and empty fieldsets from observables lang: painless source: | @@ -435,11 +493,12 @@ processors: } } + ######################################### # ECS indicator fields from observables # ######################################### - - foreach: + tag: foreach_opencti_observable_email_addr_ecaeab6a field: opencti.observable.email_addr ignore_missing: true processor: @@ -449,6 +508,7 @@ processors: allow_duplicates: false - foreach: + tag: foreach_opencti_observable_ipv4_addr_3987f58a field: opencti.observable.ipv4_addr ignore_missing: true processor: @@ -458,6 +518,7 @@ processors: allow_duplicates: false - foreach: + tag: foreach_opencti_observable_ipv6_addr_c949b11c field: opencti.observable.ipv6_addr ignore_missing: true processor: @@ -468,6 +529,7 @@ processors: # For ECS, remove any subnet mask from IP values arriving in CIDR notation - foreach: + tag: foreach_threat_indicator_ip_d127dfdc field: threat.indicator.ip ignore_missing: true processor: @@ -477,6 +539,7 @@ processors: replacement: "" - foreach: + tag: foreach_opencti_observable_network_traffic_b91bd1a0 field: opencti.observable.network_traffic ignore_missing: true processor: @@ -485,6 +548,7 @@ processors: value: "{{{_ingest._value.src_port}}}" allow_duplicates: false - foreach: + tag: foreach_opencti_observable_network_traffic_e4916ba3 field: opencti.observable.network_traffic ignore_missing: true processor: @@ -494,6 +558,7 @@ processors: allow_duplicates: false - foreach: + tag: foreach_opencti_observable_autonomous_system_9eddba75 field: opencti.observable.autonomous_system ignore_missing: true processor: @@ -501,6 +566,7 @@ processors: name: '{{ IngestPipeline "ecs_from_autonomous_system" }}' - foreach: + tag: foreach_opencti_observable_windows_registry_key_72d20dbf field: opencti.observable.windows_registry_key ignore_missing: true processor: @@ -508,6 +574,7 @@ processors: name: '{{ IngestPipeline "ecs_from_windows_registry_key" }}' - foreach: + tag: foreach_opencti_observable_windows_registry_value_type_3e52ad11 field: opencti.observable.windows_registry_value_type ignore_missing: true processor: @@ -515,6 +582,7 @@ processors: name: '{{ IngestPipeline "ecs_from_windows_registry_value_type" }}' - foreach: + tag: foreach_opencti_observable_x509_certificate_6435f22f field: opencti.observable.x509_certificate ignore_missing: true processor: @@ -522,6 +590,7 @@ processors: name: '{{ IngestPipeline "ecs_from_x509_certificate" }}' - foreach: + tag: foreach_opencti_observable_url_dc833672 field: opencti.observable.url ignore_missing: true processor: @@ -530,6 +599,7 @@ processors: value: "{{{_ingest._value.value}}}" allow_duplicates: false - foreach: + tag: foreach_opencti_observable_artifact_6aaa5375 field: opencti.observable.artifact ignore_missing: true processor: @@ -538,6 +608,7 @@ processors: value: "{{{_ingest._value.url}}}" allow_duplicates: false - foreach: + tag: foreach_opencti_observable_file_a5c9ebce field: opencti.observable.file ignore_missing: true processor: @@ -546,6 +617,7 @@ processors: value: "{{{_ingest._value.content.url}}}" allow_duplicates: false - foreach: + tag: foreach_opencti_observable_media_content_90bd22cd field: opencti.observable.media_content ignore_missing: true processor: @@ -554,6 +626,7 @@ processors: value: "{{{_ingest._value.url}}}" allow_duplicates: false - foreach: + tag: foreach_opencti_observable_process_d0066551 field: opencti.observable.process ignore_missing: true processor: @@ -566,6 +639,7 @@ processors: value: "{{{_ingest._value.content.url}}}" allow_duplicates: false - script: + tag: script_d8c725a6 description: Remove null or empty items lang: painless source: | @@ -579,16 +653,19 @@ processors: ctx._tmp_found_urls = result; } - foreach: + tag: foreach__tmp_found_urls_5bcbbb30 field: _tmp_found_urls ignore_missing: true processor: pipeline: name: '{{ IngestPipeline "ecs_from_url_field" }}' - remove: + tag: remove__tmp_found_urls_420f0d34 field: _tmp_found_urls ignore_missing: true - foreach: + tag: foreach_opencti_observable_domain_name_64bd73b7 field: opencti.observable.domain_name ignore_missing: true processor: @@ -597,6 +674,7 @@ processors: value: "{{{_ingest._value.value}}}" allow_duplicates: false - foreach: + tag: foreach_opencti_observable_hostname_56f0eb3c field: opencti.observable.hostname ignore_missing: true processor: @@ -605,16 +683,19 @@ processors: value: "{{{_ingest._value.value}}}" allow_duplicates: false - foreach: + tag: foreach__tmp_found_domain_names_and_hostnames_558c3efe field: _tmp_found_domain_names_and_hostnames ignore_missing: true processor: pipeline: name: '{{ IngestPipeline "ecs_from_domain_name_or_hostname" }}' - remove: + tag: remove__tmp_found_domain_names_and_hostnames_ebaf9c4a field: _tmp_found_domain_names_and_hostnames ignore_missing: true - foreach: + tag: foreach_opencti_observable_file_8011320f field: opencti.observable.file ignore_missing: true processor: @@ -622,6 +703,7 @@ processors: name: '{{ IngestPipeline "ecs_from_file" }}' - foreach: + tag: foreach_opencti_observable_process_79a8f024 field: opencti.observable.process ignore_missing: true processor: @@ -633,6 +715,7 @@ processors: name: '{{ IngestPipeline "ecs_from_file" }}' - foreach: + tag: foreach_opencti_observable_directory_95bb2cf7 field: opencti.observable.directory ignore_missing: true processor: @@ -640,6 +723,7 @@ processors: name: '{{ IngestPipeline "ecs_from_directory" }}' - foreach: + tag: foreach_opencti_observable_artifact_77e14cb3 field: opencti.observable.artifact ignore_missing: true processor: @@ -660,6 +744,7 @@ processors: ignore_missing: true - foreach: + tag: foreach_opencti_indicator__patterns_83e78c47 field: opencti.indicator._patterns ignore_missing: true processor: @@ -667,14 +752,16 @@ processors: name: '{{ IngestPipeline "ecs_from_pattern" }}' - remove: + tag: remove_opencti_indicator__patterns_18e67427 field: opencti.indicator._patterns ignore_missing: true + ###################### # ECS related fields # ###################### - - foreach: + tag: foreach_opencti_observable_x509_certificate_ba403611 field: opencti.observable.x509_certificate ignore_missing: true processor: @@ -687,6 +774,7 @@ processors: value: "{{{_ingest._value}}}" allow_duplicates: false - script: + tag: script_d3f7fd43 description: Extract file hashes to related.hash, handling both array and single object cases lang: painless source: | @@ -715,6 +803,7 @@ processors: } } - foreach: + tag: foreach_opencti_observable_artifact_4c1ad4e3 field: opencti.observable.artifact ignore_missing: true processor: @@ -727,6 +816,7 @@ processors: value: "{{{_ingest._value}}}" allow_duplicates: false - foreach: + tag: foreach_opencti_observable_process_d502ee1e field: opencti.observable.process ignore_missing: true processor: @@ -744,6 +834,7 @@ processors: allow_duplicates: false - foreach: + tag: foreach_opencti_observable_hostname_55369f20 field: opencti.observable.hostname ignore_missing: true processor: @@ -752,6 +843,7 @@ processors: value: "{{{_ingest._value.value}}}" allow_duplicates: false - foreach: + tag: foreach_threat_indicator_url_441074f7 field: threat.indicator.url ignore_missing: true processor: @@ -761,11 +853,13 @@ processors: allow_duplicates: false - set: + tag: set_related_ip_776c7669 field: related.ip copy_from: threat.indicator.ip if: ctx.threat.indicator.ip instanceof List - foreach: + tag: foreach_threat_indicator_email_address_81fcc02c field: threat.indicator.email.address ignore_missing: true processor: @@ -774,6 +868,7 @@ processors: value: "{{{_ingest._value}}}" allow_duplicates: false - foreach: + tag: foreach_threat_indicator_url_6544aad7 field: threat.indicator.url ignore_missing: true processor: @@ -782,6 +877,7 @@ processors: value: "{{{_ingest._value.username}}}" allow_duplicates: false - foreach: + tag: foreach_opencti_observable_user_account_e4428b4e field: opencti.observable.user_account ignore_missing: true processor: @@ -790,6 +886,7 @@ processors: value: "{{{_ingest._value.user_id}}}" allow_duplicates: false - foreach: + tag: foreach_opencti_observable_user_account_31dd5216 field: opencti.observable.user_account ignore_missing: true processor: @@ -799,6 +896,7 @@ processors: allow_duplicates: false - script: + tag: script_38ed5043 description: Remove null or empty items, remove list if empty lang: painless source: | @@ -825,6 +923,7 @@ processors: cleanUp(ctx.related, 'user'); } + ############################################## # Restructure to avoid non-leaf lists, under # # - opencti.observable.* # @@ -835,7 +934,6 @@ processors: # - threat.indicator.url # # - threat.indicator.x509 # ############################################## - - script: description: Restructure some lists of maps to avoid non-leaf lists tag: merge_maps @@ -905,7 +1003,7 @@ processors: # This prevents duplicates when multiple agents fetch the same data - fingerprint: fields: - - opencti.indicator.standard_id # STIX ID is globally unique and consistent + - opencti.indicator.standard_id # STIX ID is globally unique and consistent - threat.indicator.modified_at - opencti.indicator.updated_at target_field: _id @@ -913,11 +1011,12 @@ processors: description: Generate consistent document ID for deduplication tag: fingerprint_id + ########################################################### # Tag indicators suitable for Security rule creation # ########################################################### - - set: + tag: set_opencti_indicator_rule_compatible_6efdbb7d field: opencti.indicator.rule_compatible value: true if: >- @@ -930,36 +1029,42 @@ processors: description: Mark indicators that can be converted to detection rules - append: + tag: append_tags_bcf1b90e field: tags value: detection-rule-candidate if: ctx.opencti?.indicator?.rule_compatible == true allow_duplicates: false - set: + tag: set_opencti_indicator_detection_rule_type_c5e9a6fa field: opencti.indicator.detection_rule.type value: query if: ctx.opencti?.indicator?.pattern_type == 'kql' || ctx.opencti?.indicator?.pattern_type == 'lucene' - set: + tag: set_opencti_indicator_detection_rule_type_fec943c5 field: opencti.indicator.detection_rule.type value: eql if: ctx.opencti?.indicator?.pattern_type == 'eql' - set: + tag: set_opencti_indicator_detection_rule_type_33211ec5 field: opencti.indicator.detection_rule.type value: esql if: ctx.opencti?.indicator?.pattern_type == 'esql' - set: + tag: set_opencti_indicator_detection_rule_query_7d52745b field: opencti.indicator.detection_rule.query copy_from: opencti.indicator.pattern if: ctx.opencti?.indicator?.rule_compatible == true + ####################################################### # Tag to show if ECS indicator details were populated # ####################################################### - - append: + tag: append_tags_5d8c4c89 field: tags value: ecs-indicator-detail if: > diff --git a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_artifact.yml b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_artifact.yml index 69301c340b7..36b6b34249b 100644 --- a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_artifact.yml +++ b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_artifact.yml @@ -5,6 +5,7 @@ processors: # The url field is processed elsewhere. - foreach: + tag: foreach__ingest__value_additional_names_5f2fc97a field: _ingest._value.additional_names ignore_missing: true processor: @@ -13,6 +14,7 @@ processors: value: "{{{_ingest._value}}}" allow_duplicates: false - script: + tag: script_873e6b90 description: Remove null or empty names, remove name field if no names lang: painless source: | @@ -33,22 +35,26 @@ processors: } - set: + tag: set__tmp_file_mime_type_087066e0 field: _tmp_file.mime_type value: "{{{_ingest._value.mime_type}}}" ignore_empty_value: true - set: + tag: set__tmp_file_type_68f1b6c3 field: _tmp_file.type value: "file" ignore_empty_value: true - set: + tag: set__tmp_file_hash_f8450041 field: _tmp_file.hash copy_from: _ingest._value.hash ignore_empty_value: true # append object - script: + tag: script_af218bb0 description: Append to the destination lang: painless source: | @@ -58,4 +64,5 @@ processors: ctx.threat.indicator.file.add(ctx._tmp_file); - remove: + tag: remove__tmp_file_0fcb07f0 field: _tmp_file diff --git a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_autonomous_system.yml b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_autonomous_system.yml index 0b01913586a..9b9909caee5 100644 --- a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_autonomous_system.yml +++ b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_autonomous_system.yml @@ -3,15 +3,18 @@ description: Build ECS fields from an OpenCTI autonomous system observable processors: - set: + tag: set__tmp_as_number_8a3bcf5e field: _tmp_as.number value: "{{{_ingest._value.number}}}" - set: + tag: set__tmp_as_organization_name_15a2780f field: _tmp_as.organization.name value: "{{{_ingest._value.name}}}" # append object - script: + tag: script_0b2e52c8 description: Append to the destination lang: painless source: | @@ -21,4 +24,5 @@ processors: ctx.threat.indicator.as.add(ctx._tmp_as); - remove: + tag: remove__tmp_as_d25e9972 field: _tmp_as diff --git a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_directory.yml b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_directory.yml index ebe6af2ad94..a5023ab073d 100644 --- a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_directory.yml +++ b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_directory.yml @@ -3,32 +3,38 @@ description: Build ECS fields from an OpenCTI directory observable processors: - set: + tag: set__tmp_file_accessed_6f0852d5 field: _tmp_file.accessed value: "{{{_ingest._value.atime}}}" ignore_empty_value: true - set: + tag: set__tmp_file_created_b498f898 field: _tmp_file.created value: "{{{_ingest._value.ctime}}}" ignore_empty_value: true - set: + tag: set__tmp_file_mtime_78697244 field: _tmp_file.mtime value: "{{{_ingest._value.mtime}}}" ignore_empty_value: true - append: + tag: append__tmp_file_path_c5672fdd field: _tmp_file.path value: "{{{_ingest._value.path}}}" allow_duplicates: false - set: + tag: set__tmp_file_type_f45e3912 field: _tmp_file.type value: "dir" ignore_empty_value: true # append object - script: + tag: script_af218bb0 description: Append to the destination lang: painless source: | @@ -38,4 +44,5 @@ processors: ctx.threat.indicator.file.add(ctx._tmp_file); - remove: + tag: remove__tmp_file_0fcb07f0 field: _tmp_file diff --git a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_domain_name_or_hostname.yml b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_domain_name_or_hostname.yml index 785d41486e4..42e9782e953 100644 --- a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_domain_name_or_hostname.yml +++ b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_domain_name_or_hostname.yml @@ -3,16 +3,19 @@ description: Build ECS fields from an OpenCTI domain name or hostname observable processors: - rename: + tag: rename__ingest__value_to__tmp_url_domain_4edfb54c field: _ingest._value target_field: _tmp_url.domain - registered_domain: + tag: registered_domain__tmp_url_domain_to__tmp_url_2141d2a3 field: _tmp_url.domain target_field: _tmp_url ignore_missing: true # append object - script: + tag: script_de8b81a4 description: Append to the destination lang: painless source: | @@ -22,4 +25,5 @@ processors: ctx.threat.indicator.url.add(ctx._tmp_url); - remove: + tag: remove__tmp_url_049933dd field: _tmp_url diff --git a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_file.yml b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_file.yml index a8e68cf7ad5..9214667d83f 100644 --- a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_file.yml +++ b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_file.yml @@ -3,25 +3,30 @@ description: Build ECS fields from an OpenCTI file observable processors: - set: + tag: set__tmp_file_accessed_6f0852d5 field: _tmp_file.accessed value: "{{{_ingest._value.atime}}}" ignore_empty_value: true - set: + tag: set__tmp_file_created_b498f898 field: _tmp_file.created value: "{{{_ingest._value.ctime}}}" ignore_empty_value: true - set: + tag: set__tmp_file_mtime_78697244 field: _tmp_file.mtime value: "{{{_ingest._value.mtime}}}" ignore_empty_value: true - append: + tag: append__tmp_file_name_e5f1dc6d field: _tmp_file.name value: "{{{_ingest._value.name}}}" allow_duplicates: false - foreach: + tag: foreach__ingest__value_additional_names_5f2fc97a field: _ingest._value.additional_names ignore_missing: true processor: @@ -30,6 +35,7 @@ processors: value: "{{{_ingest._value}}}" allow_duplicates: false - script: + tag: script_3509eda3 description: Remove null or empty names, remove name field if no names lang: painless source: | @@ -48,6 +54,7 @@ processors: } - script: + tag: script_0c964542 description: Get file extensions from file names lang: painless source: | @@ -71,27 +78,32 @@ processors: } - set: + tag: set__tmp_file_mime_type_087066e0 field: _tmp_file.mime_type value: "{{{_ingest._value.mime_type}}}" ignore_empty_value: true - set: + tag: set__tmp_file_size_d51520a3 field: _tmp_file.size copy_from: _ingest._value.size ignore_empty_value: true - set: + tag: set__tmp_file_type_68f1b6c3 field: _tmp_file.type value: "file" ignore_empty_value: true - set: + tag: set__tmp_file_hash_f8450041 field: _tmp_file.hash copy_from: _ingest._value.hash ignore_empty_value: true # append object - script: + tag: script_af218bb0 description: Append to the destination lang: painless source: | @@ -101,4 +113,5 @@ processors: ctx.threat.indicator.file.add(ctx._tmp_file); - remove: + tag: remove__tmp_file_0fcb07f0 field: _tmp_file diff --git a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_pattern.yml b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_pattern.yml index f67778b0d5e..86596d82ac3 100644 --- a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_pattern.yml +++ b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_pattern.yml @@ -3,6 +3,7 @@ description: Build ECS indicator fields from an STIX pattern processors: - grok: + tag: grok__ingest__value_826928eb field: _ingest._value patterns: - "file:hashes.'?MD5'?%{SPACE}=%{SPACE}'%{DATA:threat.indicator.file.hash.md5}'" @@ -19,6 +20,7 @@ processors: ignore_failure: true - set: + tag: set_threat_indicator_file_type_25e26339 if: ctx.threat?.indicator?.file?.name != null field: threat.indicator.file.type value: "file" @@ -45,28 +47,33 @@ processors: # For ECS, remove any subnet mask from IP values arriving in CIDR notation - gsub: + tag: gsub_threat_indicator_ip_66a02181 field: threat.indicator.ip pattern: '/\d+$' replacement: "" ignore_missing: true - uri_parts: + tag: uri_parts_threat_indicator_url_original_to_threat_indicator_url_37de733d field: threat.indicator.url.original target_field: threat.indicator.url keep_original: true ignore_failure: true - registered_domain: + tag: registered_domain_threat_indicator_url_domain_to_threat_indicator_url_139f4153 field: threat.indicator.url.domain target_field: threat.indicator.url ignore_missing: true - set: + tag: set_threat_indicator_url_full_947438bd field: threat.indicator.url.full copy_from: threat.indicator.url.original ignore_empty_value: true - gsub: + tag: gsub__tmp_registry_cc160753 field: _tmp_registry pattern: '\\\\' replacement: '\\' @@ -97,11 +104,13 @@ processors: if: ctx.tmp_registry?.hive != null - rename: + tag: rename_tmp_registry_to_threat_indicator_registry_b7fb3695 field: tmp_registry target_field: threat.indicator.registry ignore_missing: true - remove: + tag: remove__tmp_registry_e95582c0 field: _tmp_registry ignore_missing: true diff --git a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_url_field.yml b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_url_field.yml index b57309cf816..d76e79398b7 100644 --- a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_url_field.yml +++ b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_url_field.yml @@ -3,30 +3,36 @@ description: Build ECS fields from a url field processors: - rename: + tag: rename__ingest__value_to__tmp_url_original_4d8b49ad field: _ingest._value target_field: _tmp_url.original - uri_parts: + tag: uri_parts__tmp_url_original_to__tmp_url_3f118dbd field: _tmp_url.original target_field: _tmp_url keep_original: true ignore_failure: true - remove: + tag: remove__tmp_url_user_info_563e99c8 field: _tmp_url.user_info ignore_missing: true - registered_domain: + tag: registered_domain__tmp_url_domain_to__tmp_url_2141d2a3 field: _tmp_url.domain target_field: _tmp_url ignore_missing: true - set: + tag: set__tmp_url_full_b946e429 field: _tmp_url.full copy_from: _tmp_url.original # append object - script: + tag: script_de8b81a4 description: Append to the destination lang: painless source: | @@ -36,4 +42,5 @@ processors: ctx.threat.indicator.url.add(ctx._tmp_url); - remove: + tag: remove__tmp_url_049933dd field: _tmp_url diff --git a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_windows_registry_key.yml b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_windows_registry_key.yml index a09eda6e03b..6a2755e53f7 100644 --- a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_windows_registry_key.yml +++ b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_windows_registry_key.yml @@ -3,6 +3,7 @@ description: Build ECS fields from an OpenCTI windows registry key observable processors: - grok: + tag: grok__ingest__value_attribute_key_af8a3014 field: _ingest._value.attribute_key patterns: - '^(%{HIVE_NAMES:_tmp_registry.hive}\\)?%{GREEDYDATA:_tmp_registry.key}$' @@ -10,6 +11,7 @@ processors: HIVE_NAMES: "(?i:HKEY_CLASSES_ROOT|HKCR|HKEY_CURRENT_USER|HKCU|HKEY_LOCAL_MACHINE|HKLM|HKEY_USERS|HKU|HKEY_CURRENT_CONFIG|HKCC)" - script: + tag: script_7b7767c2 description: Normalize hive names (ECS uses abbreviated names) lang: painless params: @@ -25,6 +27,7 @@ processors: # append object - script: + tag: script_caa36794 description: Append to the destination lang: painless source: | @@ -34,4 +37,5 @@ processors: ctx.threat.indicator.registry.add(ctx._tmp_registry); - remove: + tag: remove__tmp_registry_cc89fb9b field: _tmp_registry diff --git a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_windows_registry_value_type.yml b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_windows_registry_value_type.yml index 66bf3805c78..ad506b51c61 100644 --- a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_windows_registry_value_type.yml +++ b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_windows_registry_value_type.yml @@ -3,19 +3,23 @@ description: Build ECS fields from an OpenCTI windows registry value type observ processors: - set: + tag: set__tmp_registry_value_3cdafa31 field: _tmp_registry.value value: "{{{_ingest._value.name}}}" - set: + tag: set__tmp_registry_data_type_3f0f21ea field: _tmp_registry.data.type value: "{{{_ingest._value.data_type}}}" - append: + tag: append__tmp_registry_data_strings_a2f63f1f field: _tmp_registry.data.strings value: "{{{_ingest._value.data}}}" # append object - script: + tag: script_caa36794 description: Append to the destination lang: painless source: | @@ -25,4 +29,5 @@ processors: ctx.threat.indicator.registry.add(ctx._tmp_registry); - remove: + tag: remove__tmp_registry_cc89fb9b field: _tmp_registry diff --git a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_x509_certificate.yml b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_x509_certificate.yml index 42cdcf1e0e8..89230e01405 100644 --- a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_x509_certificate.yml +++ b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/ecs_from_x509_certificate.yml @@ -3,47 +3,58 @@ description: Build ECS fields from an OpenCTI x509 certificate observable processors: - set: + tag: set__tmp_x509_alternative_names_1c4ecca2 field: _tmp_x509.alternative_names value: "{{{_ingest._value.subject_alternative_name}}}" - set: + tag: set__tmp_x509_issuer_common_name_2e66ceaf field: _tmp_x509.issuer.common_name value: "{{{_ingest._value.issuer}}}" - set: + tag: set__tmp_x509_not_after_7b284eed field: _tmp_x509.not_after value: "{{{_ingest._value.validity_not_after}}}" - set: + tag: set__tmp_x509_not_before_279166b1 field: _tmp_x509.not_before value: "{{{_ingest._value.validity_not_before}}}" - set: + tag: set__tmp_x509_public_key_algorithm_4f5ebbf3 field: _tmp_x509.public_key_algorithm value: "{{{_ingest._value.subject_public_key_algorithm}}}" - set: + tag: set__tmp_x509_public_key_exponent_054b5c55 field: _tmp_x509.public_key_exponent value: "{{{_ingest._value.subject_public_key_exponent}}}" - set: + tag: set__tmp_x509_serial_number_dcabc406 field: _tmp_x509.serial_number value: "{{{_ingest._value.serial_number}}}" - set: + tag: set__tmp_x509_signature_algorithm_532d3b7a field: _tmp_x509.signature_algorithm value: "{{{_ingest._value.signature_algorithm}}}" - set: + tag: set__tmp_x509_subject_common_name_6cd8d489 field: _tmp_x509.subject.common_name value: "{{{_ingest._value.subject}}}" - set: + tag: set__tmp_x509_version_number_594004ca field: _tmp_x509.version_number value: "{{{_ingest._value.version}}}" # append object - script: + tag: script_64ec2064 description: Append to the destination lang: painless source: | @@ -53,4 +64,5 @@ processors: ctx.threat.indicator.x509.add(ctx._tmp_x509); - remove: + tag: remove__tmp_x509_ae712662 field: _tmp_x509 From 85ece162ae29687d4d56adb8795cc9d9a5181378 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 6 Aug 2026 12:08:32 +0200 Subject: [PATCH 5/6] Update changelog regarding tags fix. --- packages/ti_opencti/changelog.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/ti_opencti/changelog.yml b/packages/ti_opencti/changelog.yml index b2a554177b3..90ac1a27f50 100644 --- a/packages/ti_opencti/changelog.yml +++ b/packages/ti_opencti/changelog.yml @@ -1,5 +1,5 @@ # newer versions go on top -- version: "2.15.4" +- version: "2.16.0" changes: - description: Fix handling of service DLL observables to avoid invalid and incorrect renames, and simplify. type: bugfix @@ -10,6 +10,9 @@ - description: Copy `threat.indicator.file.size` value without unnecessary stringification. type: bugfix link: https://github.com/elastic/integrations/pull/20569 + - description: Add tags to each pipeline processor. + type: enhancement + link: https://github.com/elastic/integrations/pull/20569 - version: "2.15.3" changes: - description: Set agentless deployment mode `release` field to `ga`. From 94582b3635d9e7e3721d0d12e64566ed7bdb7536 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Thu, 13 Aug 2026 09:12:35 +0200 Subject: [PATCH 6/6] Invert the conditions and dedent the code. --- .../elasticsearch/ingest_pipeline/default.yml | 41 ++++++++++--------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml index f210807b38b..874a5174181 100644 --- a/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_opencti/data_stream/indicator/elasticsearch/ingest_pipeline/default.yml @@ -342,27 +342,30 @@ processors: description: Unwrap service DLL nodes from the GraphQL connection into service_dll lang: painless source: | - if (ctx.observables?.edges instanceof List) { - for (def edge : ctx.observables.edges) { - def dllEdges = edge.node?.serviceDlls?.edges; - if (dllEdges instanceof List) { - def dlls = []; - for (def dllEdge : dllEdges) { - def dll = dllEdge.node; - if (dll != null) { - if (dll.containsKey('x_opencti_additional_names')) { - dll.additional_names = dll.remove('x_opencti_additional_names'); - } - if (dll.containsKey('obsContent')) { - dll.content = dll.remove('obsContent'); - } - dlls.add(dll); - } - } - edge.node.service_dll = dlls; + if (!(ctx.observables?.edges instanceof List)) { + return; + } + for (def edge : ctx.observables.edges) { + def serviceDlls = edge.node?.remove('serviceDlls'); + def dllEdges = serviceDlls?.edges; + if (!(dllEdges instanceof List)) { + continue; + } + def dlls = []; + for (def dllEdge : dllEdges) { + def dll = dllEdge.node; + if (dll == null) { + continue; + } + if (dll.containsKey('x_opencti_additional_names')) { + dll.additional_names = dll.remove('x_opencti_additional_names'); + } + if (dll.containsKey('obsContent')) { + dll.content = dll.remove('obsContent'); } - edge.node?.remove('serviceDlls'); + dlls.add(dll); } + edge.node.service_dll = dlls; } - script: