[ti_abusech] Add YARAify data stream and commercial API for URLhaus

packages/ti_abusech/_dev/benchmark/rally/yaraify-benchmark.yml

@@ -0,0 +1,14 @@
+---
+description: Benchmark 100000 abusech.yaraify events ingested
+data_stream:
+  name: yaraify
+corpora:
+  generator:
+    total_events: 100000
+    template:
+      type: gotext
+      path: ./yaraify-benchmark/template.ndjson
+    config:
+      path: ./yaraify-benchmark/config.yml
+    fields:
+      path: ./yaraify-benchmark/fields.yml

packages/ti_abusech/_dev/benchmark/rally/yaraify-benchmark/config.yml

@@ -0,0 +1,47 @@
+fields:
+  - name: agent.id
+    value: "da6cb4c8-c84c-4c5f-97c7-f8586a098af4"
+  - name: host.name
+    cardinality: 100
+  - name: timestamp
+    period: -24h # one day
+  - name: duration_start
+    enum:
+      - "-1h"
+      - "-2h"
+      - "-3h"
+  - name: expire_dur
+    range:
+      min: 1
+      max: 365
+  - name: author
+    cardinality: 1000
+  - name: description
+    cardinality: 10000
+  - name: malpedia_family
+    cardinality: 1000
+  - name: rule_name
+    cardinality: 100000
+  - name: yarahub_author_twitter
+    cardinality: 1000
+  - name: yarahub_license
+    enum:
+      - "CC BY-SA 4.0"
+      - "CC0 1.0"
+      - "MIT"
+  - name: yarahub_reference_link
+    cardinality: 10000
+  - name: yarahub_reference_md5
+    cardinality: 100000
+  - name: yarahub_rule_matching_tlp
+    enum:
+      - "TLP:WHITE"
+      - "TLP:GREEN"
+      - "TLP:AMBER"
+  - name: yarahub_rule_sharing_tlp
+    enum:
+      - "TLP:WHITE"
+      - "TLP:GREEN"
+      - "TLP:AMBER"
+  - name: yarahub_uuid
+    cardinality: 100000

packages/ti_abusech/_dev/benchmark/rally/yaraify-benchmark/fields.yml

@@ -0,0 +1,32 @@
+- name: host.name
+  type: keyword
+- name: agent.id
+  type: keyword
+- name: timestamp
+  type: date
+- name: duration_start
+  type: keyword
+- name: expire_dur
+  type: integer
+- name: author
+  type: keyword
+- name: description
+  type: keyword
+- name: malpedia_family
+  type: keyword
+- name: rule_name
+  type: keyword
+- name: yarahub_author_twitter
+  type: keyword
+- name: yarahub_license
+  type: keyword
+- name: yarahub_reference_link
+  type: keyword
+- name: yarahub_reference_md5
+  type: keyword
+- name: yarahub_rule_matching_tlp
+  type: keyword
+- name: yarahub_rule_sharing_tlp
+  type: keyword
+- name: yarahub_uuid
+  type: keyword

packages/ti_abusech/_dev/benchmark/rally/yaraify-benchmark/template.ndjson

@@ -0,0 +1,53 @@
+{{- $timestamp := generate "timestamp" -}}
+{{- $expireDur := generate "expire_dur" -}}
+{{- $hostname := generate "host.name" -}}
+{{- $agentId := generate "agent.id" -}}
+{{- $duration_start := generate "duration_start" -}}
+{{- $firstSeen := ($timestamp | date_modify (print $duration_start)).Format "2006-01-02T15:04:05Z" -}}
+{{- $dateWritten := ($timestamp | date_modify "-240h").Format "2006-01-02" -}}
+{{- $author := generate "author" -}}
+{{- $description := generate "description" -}}
+{{- $malpedia := generate "malpedia_family" -}}
+{{- $ruleName := generate "rule_name" -}}
+{{- $twitter := generate "yarahub_author_twitter" -}}
+{{- $license := generate "yarahub_license" -}}
+{{- $refLink := generate "yarahub_reference_link" -}}
+{{- $refMd5 := generate "yarahub_reference_md5" -}}
+{{- $matchTlp := generate "yarahub_rule_matching_tlp" -}}
+{{- $shareTlp := generate "yarahub_rule_sharing_tlp" -}}
+{{- $uuid := generate "yarahub_uuid" -}}
+{
+  "@timestamp": "{{ $timestamp.Format "2006-01-02T15:04:05.999999Z07:00" }}",
+  "_conf": {
+    "ioc_expiration_duration": "{{ $expireDur }}d"
+  },
+  "agent": {
+    "id": "{{$agentId}}",
+    "name": "{{$hostname}}",
+    "type": "filebeat",
+    "version": "8.18.0",
+    "ephemeral_id": "{{$agentId}}"
+  },
+  "data_stream": {
+    "namespace": "ep",
+    "type": "logs",
+    "dataset": "ti_abusech.yaraify"
+  },
+  "elastic_agent": {
+    "id": "60a22c30-4c80-4e95-b5b1-87ff540d8afd",
+    "snapshot": false,
+    "version": "8.18.0"
+  },
+  "message": "{\"author\": \"{{ $author }}\",\"date_written\": \"{{ $dateWritten }}\",\"description\": \"detects {{ $description }}\",\"first_seen\": \"{{ $firstSeen }}\",\"malpedia_family\": \"win.{{ $malpedia }}\",\"rule_name\": \"win_{{ $ruleName }}\",\"yarahub_author_twitter\": \"@{{ $twitter }}\",\"yarahub_license\": \"{{ $license }}\",\"yarahub_reference_link\": \"https://example.com/research/{{ $refLink }}\",\"yarahub_reference_md5\": \"{{ $refMd5 }}\",\"yarahub_rule_matching_tlp\": \"{{ $matchTlp }}\",\"yarahub_rule_sharing_tlp\": \"{{ $shareTlp }}\",\"yarahub_uuid\": \"{{ $uuid }}\"}",
+  "event": {
+    "dataset": "ti_abusech.yaraify"
+  },
+  "input": {
+    "type": "cel"
+  },
+  "tags": [
+    "preserve_original_event",
+    "forwarded",
+    "abusech-yaraify"
+  ]
+}

packages/ti_abusech/_dev/benchmark/system/deploy/docker/files/config.yml

@@ -1,4 +1,43 @@
 rules:
+  # yaraify data stream (commercial API)
+  - path: /v1/login
+    methods: ["POST"]
+    request_headers:
+      Content-Type: "application/json"
+    responses:
+      - status_code: 200
+        body: |-
+          {
+            "code": 200,
+            "token": "test-commercial-token",
+            "expires": 4102444800
+          }
+  - path: /yaraify/v1/rules
+    methods: ["GET"]
+    query_params:
+      since: ["{since:.*}"]
+      page_size: ["{page_size:.*}"]
+    request_headers:
+      Authorization:
+        - "Bearer test-commercial-token"
+    responses:
+      - status_code: 200
+        body: |-
+          {
+            "items": [
+              {{/* Comma is added at the end of each line inside the template to preserve JSON format */}}
+              {{- $g := glob "/var/log/corpus-*" -}}
+              {{- range $g -}}
+                {{- file . -}}
+              {{- end -}}
+              {{/* A last line of hard-coded data is required to properly close the JSON body */}}
+              {"author":"Example Author","date_written":"2024-01-10","description":"detects ExampleBot","first_seen":"2024-01-15T11:16:44Z","malpedia_family":"win.examplebot","rule_name":"win_examplebot","yarahub_author_twitter":"@example_author","yarahub_license":"CC BY-SA 4.0","yarahub_reference_link":"https://example.com/research/examplebot","yarahub_reference_md5":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","yarahub_rule_matching_tlp":"TLP:WHITE","yarahub_rule_sharing_tlp":"TLP:WHITE","yarahub_uuid":"11111111-1111-4111-8111-111111111111"}
+            ],
+            "meta": {
+              "page_size": 100,
+              "total_items": 100001
+            }
+          }
   # ja3_fingerprints data stream
   - path: /blacklist/ja3_fingerprints.csv
     methods: ["GET"]

packages/ti_abusech/_dev/benchmark/system/yaraify-benchmark.yml

@@ -0,0 +1,27 @@
+---
+description: Benchmark 100000 yaraify events ingested
+input: cel
+vars:
+  enable_request_tracer: true
+  api_type: commercial
+  username: test@example.com
+  password: test-password
+data_stream:
+  name: yaraify
+  vars:
+    url: "http://svc-abusech:8080"
+    preserve_original_event: true
+    ioc_expiration_duration: 5d
+warmup_time_period: 2s
+corpora:
+  input_service:
+    name: abusech
+  generator:
+    total_events: 100000
+    template:
+      path: ./yaraify-benchmark/template.ndjson
+      type: gotext
+    config:
+      path: ./yaraify-benchmark/config.yml
+    fields:
+      path: ./yaraify-benchmark/fields.yml

packages/ti_abusech/_dev/benchmark/system/yaraify-benchmark/config.yml

@@ -0,0 +1,39 @@
+fields:
+  - name: timestamp
+    period: -24h # one day
+  - name: duration_start
+    enum:
+      - "-1h"
+      - "-2h"
+      - "-3h"
+  - name: author
+    cardinality: 1000
+  - name: description
+    cardinality: 10000
+  - name: malpedia_family
+    cardinality: 1000
+  - name: rule_name
+    cardinality: 100000
+  - name: yarahub_author_twitter
+    cardinality: 1000
+  - name: yarahub_license
+    enum:
+      - "CC BY-SA 4.0"
+      - "CC0 1.0"
+      - "MIT"
+  - name: yarahub_reference_link
+    cardinality: 10000
+  - name: yarahub_reference_md5
+    cardinality: 100000
+  - name: yarahub_rule_matching_tlp
+    enum:
+      - "TLP:WHITE"
+      - "TLP:GREEN"
+      - "TLP:AMBER"
+  - name: yarahub_rule_sharing_tlp
+    enum:
+      - "TLP:WHITE"
+      - "TLP:GREEN"
+      - "TLP:AMBER"
+  - name: yarahub_uuid
+    cardinality: 100000

packages/ti_abusech/_dev/benchmark/system/yaraify-benchmark/fields.yml

@@ -0,0 +1,26 @@
+- name: timestamp
+  type: date
+- name: duration_start
+  type: keyword
+- name: author
+  type: keyword
+- name: description
+  type: keyword
+- name: malpedia_family
+  type: keyword
+- name: rule_name
+  type: keyword
+- name: yarahub_author_twitter
+  type: keyword
+- name: yarahub_license
+  type: keyword
+- name: yarahub_reference_link
+  type: keyword
+- name: yarahub_reference_md5
+  type: keyword
+- name: yarahub_rule_matching_tlp
+  type: keyword
+- name: yarahub_rule_sharing_tlp
+  type: keyword
+- name: yarahub_uuid
+  type: keyword

packages/ti_abusech/_dev/benchmark/system/yaraify-benchmark/template.ndjson

@@ -0,0 +1,16 @@
+{{- $timestamp := generate "timestamp" }}
+{{- $duration_start := generate "duration_start" }}
+{{- $firstSeen := ($timestamp | date_modify (print $duration_start)).Format "2006-01-02T15:04:05Z" }}
+{{- $dateWritten := ($timestamp | date_modify "-240h").Format "2006-01-02" }}
+{{- $author := generate "author" }}
+{{- $description := generate "description" }}
+{{- $malpedia := generate "malpedia_family" }}
+{{- $ruleName := generate "rule_name" }}
+{{- $twitter := generate "yarahub_author_twitter" }}
+{{- $license := generate "yarahub_license" }}
+{{- $refLink := generate "yarahub_reference_link" }}
+{{- $refMd5 := generate "yarahub_reference_md5" }}
+{{- $matchTlp := generate "yarahub_rule_matching_tlp" }}
+{{- $shareTlp := generate "yarahub_rule_sharing_tlp" }}
+{{- $uuid := generate "yarahub_uuid" }}
+{"author": "{{ $author }}","date_written": "{{ $dateWritten }}","description": "detects {{ $description }}","first_seen": "{{ $firstSeen }}","malpedia_family": "win.{{ $malpedia }}","rule_name": "win_{{ $ruleName }}","yarahub_author_twitter": "@{{ $twitter }}","yarahub_license": "{{ $license }}","yarahub_reference_link": "https://example.com/research/{{ $refLink }}","yarahub_reference_md5": "{{ $refMd5 }}","yarahub_rule_matching_tlp": "{{ $matchTlp }}","yarahub_rule_sharing_tlp": "{{ $shareTlp }}","yarahub_uuid": "{{ $uuid }}"},