Plugin builder and documentation

.github/workflows/ci.yaml

@@ -22,7 +22,9 @@ on:
   workflow_dispatch:
 
 env:
-  DOCKER_REGISTRY_REPOSITORY: sysflowtelemetry/sf-processor  
+  DOCKER_REGISTRY_REPOSITORY: sysflowtelemetry/sf-processor
+  PLUGIN_BUILDER_DOCKER_REGISTRY_REPOSITORY: sysflowtelemetry/plugin-builder
+  GH_ORGANIZATION: sysflow-telemetry
 
 jobs:
   lint-core:
@@ -60,7 +62,7 @@ jobs:
       - name: Read manifest        
         shell: bash        
         run: |             
-          echo "UBI_VERSION=$(awk -F'=' '/UBI_VERSION/{print $2}' makefile.manifest.inc)" >> $GITHUB_ENV      
+          echo "UBI_VERSION=$(awk -F'=' '/UBI_VERSION/{print $2}' makefile.manifest.inc)" >> $GITHUB_ENV            
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v3
@@ -100,13 +102,29 @@ jobs:
           echo "##[set-output name=ubi_version;]$(awk -F'=' '/UBI_VERSION/{print $2}' makefile.manifest.inc)"
           echo "##[set-output name=branch;]$(echo ${GITHUB_REF##refs/*/})"
           echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"    
+      - name: Build and push plugin builder image
+        id: docker_plugin_builder        
+        uses: docker/build-push-action@v2        
+        with:
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: |
+            ${{ env.PLUGIN_BUILDER_DOCKER_REGISTRY_REPOSITORY }}:${{ steps.args.outputs.branch }}
+            ghcr.io/${{ env.GH_ORGANIZATION }}/plugin-builder:${{ steps.args.outputs.branch }}        
+          target: base    
+          labels: "${{ steps.meta.outputs.labels }}"
+          build-args: |            
+            BUILD_NUMBER=${{ steps.args.outputs.sha_short }}
+            VERSION=${{ steps.args.outputs.branch }}
+            RELEASE=${{ steps.args.outputs.sha_short }}
+            UBI_VER=${{ steps.args.outputs.ubi_version }} 
       - name: Build and push
         id: docker_build        
         uses: docker/build-push-action@v2        
         with:
           push: ${{ github.event_name != 'pull_request' }}
           tags: "${{ steps.meta.outputs.tags }}"
           labels: "${{ steps.meta.outputs.labels }}"
+          cache-from: type=registry,ref=ghcr.io/${{ env.GH_ORGANIZATION }}/plugin-builder:${{ steps.args.outputs.branch }}
           build-args: |            
             BUILD_NUMBER=${{ steps.args.outputs.sha_short }}
             VERSION=${{ steps.args.outputs.branch }}

CHANGELOG.md

@@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 - Packaging in deb, rpm, and targz formats
 - Added 14 new MITRE TTP tagging rules
 - Added support for quiet logging mode
+- Added plugin builder image to support plugin development and releases
 
 ### Changed
 

Makefile

@@ -12,7 +12,7 @@ include ./makefile.manifest.inc
 # Basic go commands
 PATH=$(shell printenv PATH):/usr/local/go/bin
 GOCMD=go
-GOBUILD=$(GOCMD) build -tags exclude_graphdriver_btrfs
+GOBUILD=$(GOCMD) build -trimpath -tags exclude_graphdriver_btrfs
 GOCLEAN=$(GOCMD) clean
 GOTEST=$(GOCMD) test -tags exclude_graphdriver_btrfs
 GOGET=$(GOCMD) get -tags exclude_graphdriver_btrfs
@@ -66,11 +66,11 @@ install: build
 
 .PHONY: docker-build
 docker-build:
-	( DOCKER_BUILDKIT=1 docker build -t sysflowtelemetry/sf-processor:${SYSFLOW_VERSION} --build-arg UBI_VER=$(UBI_VERSION) --target=runtime -f Dockerfile . )
+	( DOCKER_BUILDKIT=1 docker build --cache-from=sysflowtelemetry/plugin-builder:${SYSFLOW_VERSION} -t sysflowtelemetry/sf-processor:${SYSFLOW_VERSION} --build-arg UBI_VER=$(UBI_VERSION) --target=runtime -f Dockerfile . )
 
-.PHONY: docker-build-base
-docker-build-base:
-	( DOCKER_BUILDKIT=1 docker build -t sysflowtelemetry/sf-processor:base --build-arg UBI_VER=$(UBI_VERSION) --target=base -f Dockerfile . )
+.PHONY: docker-plugin-builder
+docker-plugin-builder:
+	( DOCKER_BUILDKIT=1 docker build -t sysflowtelemetry/plugin-builder:${SYSFLOW_VERSION} --build-arg UBI_VER=$(UBI_VERSION) --target=base -f Dockerfile . )
 
 .PHONY: pull
 pull:

core/exporter/encoders/ecs.go

@@ -28,7 +28,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/cespare/xxhash"
+	"github.com/cespare/xxhash/v2"
 	"github.com/satta/gommunityid"
 	"github.com/sysflow-telemetry/sf-apis/go/sfgo"
 	"github.com/sysflow-telemetry/sf-processor/core/exporter/commons"
@@ -210,10 +210,10 @@ func (ecs *ECSRecord) encodeFileFlow(rec *engine.Record) {
 	category := ECS_CAT_FILE
 	eventType := ECS_TYPE_ACCESS
 	action := category + "-" + eventType
-	if opFlags&sfgo.OP_READ_RECV == sfgo.OP_READ_RECV && (rbytes > 0 || rops > 0 ) {
+	if opFlags&sfgo.OP_READ_RECV == sfgo.OP_READ_RECV && (rbytes > 0 || rops > 0) {
 		action = action + "-" + ECS_ACTION_READ
 	}
-	if opFlags&sfgo.OP_WRITE_SEND == sfgo.OP_WRITE_SEND && ( wbytes > 0 || wops > 0 ) {
+	if opFlags&sfgo.OP_WRITE_SEND == sfgo.OP_WRITE_SEND && (wbytes > 0 || wops > 0) {
 		eventType = ECS_TYPE_CHANGE
 		action = action + "-" + ECS_ACTION_WRITE
 	}
@@ -311,15 +311,15 @@ func encodeContainer(rec *engine.Record) JSONData {
 func encodeUser(rec *engine.Record) JSONData {
 	gname := engine.Mapper.MapStr(engine.SF_PROC_GROUP)(rec)
 	group := JSONData{
-		ECS_GROUP_ID:   engine.Mapper.MapInt(engine.SF_PROC_GID)(rec),
+		ECS_GROUP_ID: engine.Mapper.MapInt(engine.SF_PROC_GID)(rec),
 	}
 	if gname != sfgo.Zeros.String {
 		group[ECS_GROUP_NAME] = gname
 	}
 	uname := engine.Mapper.MapStr(engine.SF_PROC_USER)(rec)
 	user := JSONData{
-		ECS_GROUP:     group,
-		ECS_USER_ID:   engine.Mapper.MapInt(engine.SF_PROC_UID)(rec),
+		ECS_GROUP:   group,
+		ECS_USER_ID: engine.Mapper.MapInt(engine.SF_PROC_UID)(rec),
 	}
 	if uname != sfgo.Zeros.String {
 		user[ECS_USER_NAME] = uname

core/exporter/encoders/occurrence.go

@@ -31,7 +31,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/cespare/xxhash"
+	"github.com/cespare/xxhash/v2"
 	"github.com/linkedin/goavro"
 	cmap "github.com/orcaman/concurrent-map"
 	"github.com/steakknife/bloomfilter"

core/go.mod

@@ -27,7 +27,7 @@ require (
 	github.com/RackSec/srslog v0.0.0-20180709174129-a4725f04ec91
 	github.com/actgardner/gogen-avro/v7 v7.3.1
 	github.com/antlr/antlr4 v0.0.0-20200417160354-8c50731894e0
-	github.com/cespare/xxhash v1.1.0
+	github.com/cespare/xxhash/v2 v2.1.2
 	github.com/elastic/go-elasticsearch/v8 v8.0.0-20210427093042-01613f93a7ae
 	github.com/fsnotify/fsnotify v1.4.9
 	github.com/linkedin/goavro v2.1.0+incompatible
@@ -37,7 +37,7 @@ require (
 	github.com/satta/gommunityid v0.0.0-20210315182841-1cdcb73ce408
 	github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570
 	github.com/stretchr/testify v1.7.0
-	github.com/sysflow-telemetry/sf-apis/go v0.0.0-20220204025248-25d44ab7fe1f
+	github.com/sysflow-telemetry/sf-apis/go v0.0.0-20220221182316-8e41f56e373e
 )
 
 require (

core/go.sum

@@ -64,7 +64,6 @@ github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfy
 github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU=
 github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
-github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
 github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
@@ -112,6 +111,8 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
+github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw=
 github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M=
 github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E=
@@ -698,7 +699,6 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1
 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
 github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
@@ -735,8 +735,8 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69
 github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
-github.com/sysflow-telemetry/sf-apis/go v0.0.0-20220204025248-25d44ab7fe1f h1:quICy9nfEUaCRZpn0SDI3q6iRVfETcGq3LD/Bw1DGjc=
-github.com/sysflow-telemetry/sf-apis/go v0.0.0-20220204025248-25d44ab7fe1f/go.mod h1:6gzcis2lTDCL37xUZg+6u1JU0gAodMIvqp5wDl4ddms=
+github.com/sysflow-telemetry/sf-apis/go v0.0.0-20220221182316-8e41f56e373e h1:VR8J80DbrpjOBdgc5qGZLuxqo1PJYJcdbLgyZ+9yQL0=
+github.com/sysflow-telemetry/sf-apis/go v0.0.0-20220221182316-8e41f56e373e/go.mod h1:0xb+dFYM8a3FGTjB6s5w7Yt58HxdJat3ZRXr/dTndG8=
 github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
 github.com/tchap/go-patricia v2.3.0+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
 github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=

core/policyengine/engine/actionhandler.go

@@ -53,7 +53,7 @@ func (ah *ActionHandler) loadUserActions(dir string) {
 	if paths, err := ioutils.ListFilePaths(dir, ".so"); err == nil {
 		var plug *plugin.Plugin
 		for _, path := range paths {
-			logger.Info.Println("Loading user-defined actions from file " + path)
+			logger.Info.Println("Loading user-defined action from file " + path)
 			if plug, err = plugin.Open(path); err != nil {
 				logger.Error.Println(err.Error())
 				continue

core/policyengine/engine/config.go

@@ -54,7 +54,7 @@ type Config struct {
 
 // CreateConfig creates a new config object from config dictionary.
 func CreateConfig(conf map[string]interface{}) (Config, error) {
-	var c Config = Config{Mode: AlertMode, Concurrency: 5, Monitor: NoneType, MonitorInterval: 30 * time.Second} // default values
+	var c Config = Config{Mode: AlertMode, Concurrency: 5, Monitor: NoneType, MonitorInterval: 30 * time.Second, ActionDir: "../resources/actions"} // default values
 	var err error
 
 	if v, ok := conf[PoliciesConfigKey].(string); ok {

core/policyengine/engine/fieldmapper.go

@@ -27,7 +27,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/cespare/xxhash"
+	"github.com/cespare/xxhash/v2"
 	"github.com/sysflow-telemetry/sf-apis/go/logger"
 	"github.com/sysflow-telemetry/sf-apis/go/sfgo"
 )

core/policyengine/lang/Sfpl.g4

@@ -9,7 +9,7 @@ NAME: 'name';
 ITEMS: 'items';
 COND: 'condition';
 DESC: 'desc' ;
-ACTION: 'action';
+ACTIONS: 'actions';
 OUTPUT: 'output';
 PRIORITY: 'priority';
 TAGS: 'tags';
@@ -29,11 +29,11 @@ defs
 	;
 
 prule			
-	: DECL RULE DEF text DESC DEF text COND DEF expression (OUTPUT DEF text | ACTION DEF actions | PRIORITY DEF severity | TAGS DEF tags | PREFILTER DEF prefilter | ENABLED DEF enabled | WARNEVTTYPE DEF warnevttype | SKIPUNKNOWN DEF skipunknown)*
+	: DECL RULE DEF text DESC DEF text COND DEF expression (OUTPUT DEF text | ACTIONS DEF actions | PRIORITY DEF severity | TAGS DEF tags | PREFILTER DEF prefilter | ENABLED DEF enabled | WARNEVTTYPE DEF warnevttype | SKIPUNKNOWN DEF skipunknown)*
 	;
 
 srule
-	: DECL RULE DEF text DESC DEF text COND DEF expression (OUTPUT DEF text | ACTION DEF actions | PRIORITY DEF severity | TAGS DEF tags | PREFILTER DEF prefilter | ENABLED DEF enabled | WARNEVTTYPE DEF warnevttype | SKIPUNKNOWN DEF skipunknown)*
+	: DECL RULE DEF text DESC DEF text COND DEF expression (OUTPUT DEF text | ACTIONS DEF actions | PRIORITY DEF severity | TAGS DEF tags | PREFILTER DEF prefilter | ENABLED DEF enabled | WARNEVTTYPE DEF warnevttype | SKIPUNKNOWN DEF skipunknown)*
 	;
 
 pfilter
@@ -134,7 +134,7 @@ atom
 text
 	: ({!(p.GetCurrentToken().GetText() == "desc" ||
 	      p.GetCurrentToken().GetText() == "condition" ||
-	      p.GetCurrentToken().GetText() == "action" ||
+	      p.GetCurrentToken().GetText() == "actions" ||
 	      p.GetCurrentToken().GetText() == "output" ||
 	      p.GetCurrentToken().GetText() == "priority" ||
 	      p.GetCurrentToken().GetText() == "tags" ||

core/policyengine/lang/parser/Sfpl.interp

@@ -9,7 +9,7 @@ null
 'items'
 'condition'
 'desc'
-'action'
+'actions'
 'output'
 'priority'
 'tags'
@@ -66,7 +66,7 @@ NAME
 ITEMS
 COND
 DESC
-ACTION
+ACTIONS
 OUTPUT
 PRIORITY
 TAGS

core/policyengine/lang/parser/Sfpl.tokens

@@ -7,7 +7,7 @@ NAME=6
 ITEMS=7
 COND=8
 DESC=9
-ACTION=10
+ACTIONS=10
 OUTPUT=11
 PRIORITY=12
 TAGS=13
@@ -61,7 +61,7 @@ ANY=54
 'items'=7
 'condition'=8
 'desc'=9
-'action'=10
+'actions'=10
 'output'=11
 'priority'=12
 'tags'=13

core/policyengine/lang/parser/SfplLexer.tokens

@@ -7,7 +7,7 @@ NAME=6
 ITEMS=7
 COND=8
 DESC=9
-ACTION=10
+ACTIONS=10
 OUTPUT=11
 PRIORITY=12
 TAGS=13
@@ -61,7 +61,7 @@ ANY=54
 'items'=7
 'condition'=8
 'desc'=9
-'action'=10
+'actions'=10
 'output'=11
 'priority'=12
 'tags'=13