diff --git a/Dockerfiles/arkime.Dockerfile b/Dockerfiles/arkime.Dockerfile
index 430a10f9c..efa3875c5 100644
--- a/Dockerfiles/arkime.Dockerfile
+++ b/Dockerfiles/arkime.Dockerfile
@@ -4,7 +4,7 @@ FROM debian:11-slim AS build
ENV DEBIAN_FRONTEND noninteractive
-ENV ARKIME_VERSION "v4.1.0"
+ENV ARKIME_VERSION "v4.2.0"
ENV ARKIME_DIR "/opt/arkime"
ENV ARKIME_URL "https://github.com/arkime/arkime.git"
ENV ARKIME_LOCALELASTICSEARCH no
diff --git a/Dockerfiles/dashboards-helper.Dockerfile b/Dockerfiles/dashboards-helper.Dockerfile
index 34ab245ce..514120bda 100644
--- a/Dockerfiles/dashboards-helper.Dockerfile
+++ b/Dockerfiles/dashboards-helper.Dockerfile
@@ -47,10 +47,10 @@ ENV DASHBOARDS_URL $DASHBOARDS_URL
ENV DASHBOARDS_DARKMODE $DASHBOARDS_DARKMODE
ENV PATH="/data:${PATH}"
-ENV SUPERCRONIC_VERSION "0.2.1"
+ENV SUPERCRONIC_VERSION "0.2.2"
ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64"
ENV SUPERCRONIC "supercronic-linux-amd64"
-ENV SUPERCRONIC_SHA1SUM "d7f4c0886eb85249ad05ed592902fa6865bb9d70"
+ENV SUPERCRONIC_SHA1SUM "2319da694833c7a147976b8e5f337cd83397d6be"
ENV SUPERCRONIC_CRONTAB "/etc/crontab"
ENV ECS_RELEASES_URL "https://api.github.com/repos/elastic/ecs/releases/latest"
diff --git a/Dockerfiles/dashboards.Dockerfile b/Dockerfiles/dashboards.Dockerfile
index a1d19eb9c..842d448d9 100644
--- a/Dockerfiles/dashboards.Dockerfile
+++ b/Dockerfiles/dashboards.Dockerfile
@@ -14,10 +14,10 @@ ENV PGROUP "dashboarder"
ENV TERM xterm
-ARG OPENSEARCH_VERSION="2.5.0"
+ARG OPENSEARCH_VERSION="2.6.0"
ENV OPENSEARCH_VERSION $OPENSEARCH_VERSION
-ARG OPENSEARCH_DASHBOARDS_VERSION="2.5.0"
+ARG OPENSEARCH_DASHBOARDS_VERSION="2.6.0"
ENV OPENSEARCH_DASHBOARDS_VERSION $OPENSEARCH_DASHBOARDS_VERSION
# base system dependencies for checking out and building plugins
@@ -68,7 +68,7 @@ RUN eval "$(nodenv init -)" && \
# runtime ##################################################################
-FROM opensearchproject/opensearch-dashboards:2.5.0
+FROM opensearchproject/opensearch-dashboards:2.6.0
LABEL maintainer="malcolm@inl.gov"
LABEL org.opencontainers.image.authors='malcolm@inl.gov'
@@ -122,7 +122,7 @@ RUN yum upgrade -y && \
/usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin remove securityDashboards --allow-root && \
cd /usr/share/opensearch-dashboards/plugins && \
/usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install file:///tmp/kbnSankeyVis.zip --allow-root && \
- /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install https://github.com/lguillaud/osd_transform_vis/releases/download/$OSD_TRANSFORM_VIS_VERSION/transformVis-$OSD_TRANSFORM_VIS_VERSION.zip --allow-root && \
+ # TODO: when 2.6.0 is released /usr/share/opensearch-dashboards/bin/opensearch-dashboards-plugin install https://github.com/lguillaud/osd_transform_vis/releases/download/$OSD_TRANSFORM_VIS_VERSION/transformVis-$OSD_TRANSFORM_VIS_VERSION.zip --allow-root && \
# trying to see if things still work if these are owned by root (to avoid a costly chown on container startup)
chown --silent -R root:root /usr/share/opensearch-dashboards/plugins/* \
/usr/share/opensearch-dashboards/node_modules/* \
diff --git a/Dockerfiles/file-monitor.Dockerfile b/Dockerfiles/file-monitor.Dockerfile
index 29fe5c138..47d18ee59 100644
--- a/Dockerfiles/file-monitor.Dockerfile
+++ b/Dockerfiles/file-monitor.Dockerfile
@@ -79,7 +79,7 @@ ENV YARA_VERSION "4.2.3"
ENV YARA_URL "https://github.com/VirusTotal/yara/archive/v${YARA_VERSION}.tar.gz"
ENV YARA_RULES_SRC_DIR "/yara-rules-src"
ENV YARA_RULES_DIR "/yara-rules"
-ENV CAPA_VERSION "4.0.1"
+ENV CAPA_VERSION "5.0.0"
ENV CAPA_URL "https://github.com/fireeye/capa/releases/download/v${CAPA_VERSION}/capa-v${CAPA_VERSION}-linux.zip"
ENV CAPA_DIR "/opt/capa"
ENV CAPA_BIN "${CAPA_DIR}/capa"
@@ -89,10 +89,10 @@ ENV EXTRACTED_FILE_HTTP_SERVER_ENCRYPT $EXTRACTED_FILE_HTTP_SERVER_ENCRYPT
ENV EXTRACTED_FILE_HTTP_SERVER_KEY $EXTRACTED_FILE_HTTP_SERVER_KEY
ENV EXTRACTED_FILE_HTTP_SERVER_PORT $EXTRACTED_FILE_HTTP_SERVER_PORT
-ENV SUPERCRONIC_VERSION "0.2.1"
+ENV SUPERCRONIC_VERSION "0.2.2"
ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64"
ENV SUPERCRONIC "supercronic-linux-amd64"
-ENV SUPERCRONIC_SHA1SUM "d7f4c0886eb85249ad05ed592902fa6865bb9d70"
+ENV SUPERCRONIC_SHA1SUM "2319da694833c7a147976b8e5f337cd83397d6be"
ENV SUPERCRONIC_CRONTAB "/etc/crontab"
COPY --chmod=755 shared/bin/yara_rules_setup.sh /usr/local/bin/
diff --git a/Dockerfiles/filebeat.Dockerfile b/Dockerfiles/filebeat.Dockerfile
index 656d10106..552aa8a53 100644
--- a/Dockerfiles/filebeat.Dockerfile
+++ b/Dockerfiles/filebeat.Dockerfile
@@ -1,4 +1,4 @@
-FROM docker.elastic.co/beats/filebeat-oss:8.6.1
+FROM docker.elastic.co/beats/filebeat-oss:8.6.2
# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved.
LABEL maintainer="malcolm@inl.gov"
@@ -57,10 +57,10 @@ ARG FILEBEAT_TCP_PARSE_TARGET_FIELD=""
ARG FILEBEAT_TCP_PARSE_DROP_FIELD=""
ARG FILEBEAT_TCP_TAG="_malcolm_beats"
-ENV SUPERCRONIC_VERSION "0.2.1"
+ENV SUPERCRONIC_VERSION "0.2.2"
ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64"
ENV SUPERCRONIC "supercronic-linux-amd64"
-ENV SUPERCRONIC_SHA1SUM "d7f4c0886eb85249ad05ed592902fa6865bb9d70"
+ENV SUPERCRONIC_SHA1SUM "2319da694833c7a147976b8e5f337cd83397d6be"
ENV SUPERCRONIC_CRONTAB "/etc/crontab"
ENV TINI_VERSION v0.19.0
diff --git a/Dockerfiles/logstash.Dockerfile b/Dockerfiles/logstash.Dockerfile
index aea8e55ab..7ced0f3ea 100644
--- a/Dockerfiles/logstash.Dockerfile
+++ b/Dockerfiles/logstash.Dockerfile
@@ -1,4 +1,4 @@
-FROM opensearchproject/logstash-oss-with-opensearch-output-plugin:8.4.0
+FROM opensearchproject/logstash-oss-with-opensearch-output-plugin:8.6.1
LABEL maintainer="malcolm@inl.gov"
LABEL org.opencontainers.image.authors='malcolm@inl.gov'
@@ -46,7 +46,8 @@ USER root
ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /usr/bin/tini
-RUN apt-get -q update && \
+RUN set -x && \
+ apt-get -q update && \
apt-get -y -q --no-install-recommends upgrade && \
apt-get -y --no-install-recommends install \
gettext \
@@ -57,6 +58,8 @@ RUN apt-get -q update && \
tini && \
chmod +x /usr/bin/tini && \
pip3 install ipaddress supervisor manuf pyyaml && \
+ export JAVA_HOME=/usr/share/logstash/jdk && \
+ /usr/share/logstash/vendor/jruby/bin/jruby -S gem install bundler && \
echo "gem 'lru_cache'" >> /usr/share/logstash/Gemfile && \
/usr/share/logstash/bin/ruby -S bundle install && \
logstash-plugin install --preserve logstash-filter-translate logstash-filter-cidr logstash-filter-dns \
diff --git a/Dockerfiles/netbox.Dockerfile b/Dockerfiles/netbox.Dockerfile
index beea788dd..9cdc51318 100644
--- a/Dockerfiles/netbox.Dockerfile
+++ b/Dockerfiles/netbox.Dockerfile
@@ -22,10 +22,10 @@ ENV PUSER "boxer"
ENV PGROUP "boxer"
ENV PUSER_PRIV_DROP true
-ENV SUPERCRONIC_VERSION "0.2.1"
+ENV SUPERCRONIC_VERSION "0.2.2"
ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64"
ENV SUPERCRONIC "supercronic-linux-amd64"
-ENV SUPERCRONIC_SHA1SUM "d7f4c0886eb85249ad05ed592902fa6865bb9d70"
+ENV SUPERCRONIC_SHA1SUM "2319da694833c7a147976b8e5f337cd83397d6be"
ENV SUPERCRONIC_CRONTAB "/etc/crontab"
ENV NETBOX_DEVICETYPE_LIBRARY_URL "https://codeload.github.com/netbox-community/devicetype-library/tar.gz/master"
@@ -62,6 +62,8 @@ RUN apt-get -q update && \
usermod -a -G tty ${PUSER} && \
mkdir -p /opt/unit "${NETBOX_DEVICETYPE_LIBRARY_PATH}" && \
chown -R $PUSER:$PGROUP /etc/netbox /opt/unit /opt/netbox && \
+ # trying to see if things still work if these are owned by root (to avoid a costly chown on container startup)
+ chown --silent -R root:root /opt/netbox/venv/* && \
cd "$(dirname "${NETBOX_DEVICETYPE_LIBRARY_PATH}")" && \
curl -sSL "$NETBOX_DEVICETYPE_LIBRARY_URL" | tar xzvf - -C ./"$(basename "${NETBOX_DEVICETYPE_LIBRARY_PATH}")" --strip-components 1 && \
mkdir -p /opt/netbox/netbox/$BASE_PATH && \
diff --git a/Dockerfiles/opensearch.Dockerfile b/Dockerfiles/opensearch.Dockerfile
index 98dac3235..f6d168fd1 100644
--- a/Dockerfiles/opensearch.Dockerfile
+++ b/Dockerfiles/opensearch.Dockerfile
@@ -1,4 +1,4 @@
-FROM opensearchproject/opensearch:2.5.0
+FROM opensearchproject/opensearch:2.6.0
# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved.
LABEL maintainer="malcolm@inl.gov"
diff --git a/Dockerfiles/pcap-monitor.Dockerfile b/Dockerfiles/pcap-monitor.Dockerfile
index 02f2d2ffb..263016c0c 100644
--- a/Dockerfiles/pcap-monitor.Dockerfile
+++ b/Dockerfiles/pcap-monitor.Dockerfile
@@ -59,7 +59,7 @@ RUN apt-get -q update && \
vim-tiny && \
apt-get clean && \
rm -rf /var/lib/apt/lists/* && \
- pip3 install --no-cache-dir opensearch-py opensearch-dsl pyzmq pyinotify python-magic requests && \
+ pip3 install --no-cache-dir opensearch-py pyzmq pyinotify python-magic requests && \
groupadd --gid ${DEFAULT_GID} ${PGROUP} && \
useradd -M --uid ${DEFAULT_UID} --gid ${DEFAULT_GID} ${PUSER}
diff --git a/Dockerfiles/suricata.Dockerfile b/Dockerfiles/suricata.Dockerfile
index 7a2723ca8..42dde7a4f 100644
--- a/Dockerfiles/suricata.Dockerfile
+++ b/Dockerfiles/suricata.Dockerfile
@@ -27,10 +27,10 @@ ENV PGROUP "suricata"
# a final check in docker_entrypoint.sh before startup
ENV PUSER_PRIV_DROP false
-ENV SUPERCRONIC_VERSION "0.2.1"
+ENV SUPERCRONIC_VERSION "0.2.2"
ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64"
ENV SUPERCRONIC "supercronic-linux-amd64"
-ENV SUPERCRONIC_SHA1SUM "d7f4c0886eb85249ad05ed592902fa6865bb9d70"
+ENV SUPERCRONIC_SHA1SUM "2319da694833c7a147976b8e5f337cd83397d6be"
ENV SUPERCRONIC_CRONTAB "/etc/crontab"
ENV YQ_VERSION "4.24.2"
diff --git a/Dockerfiles/zeek.Dockerfile b/Dockerfiles/zeek.Dockerfile
index 69291e3b2..e0c59c022 100644
--- a/Dockerfiles/zeek.Dockerfile
+++ b/Dockerfiles/zeek.Dockerfile
@@ -31,15 +31,15 @@ ENV PUSER_PRIV_DROP false
# for download and install
ARG ZEEK_LTS=true
-ARG ZEEK_VERSION=5.0.6-0
+ARG ZEEK_VERSION=5.0.7-0
ENV ZEEK_LTS $ZEEK_LTS
ENV ZEEK_VERSION $ZEEK_VERSION
-ENV SUPERCRONIC_VERSION "0.2.1"
+ENV SUPERCRONIC_VERSION "0.2.2"
ENV SUPERCRONIC_URL "https://github.com/aptible/supercronic/releases/download/v$SUPERCRONIC_VERSION/supercronic-linux-amd64"
ENV SUPERCRONIC "supercronic-linux-amd64"
-ENV SUPERCRONIC_SHA1SUM "d7f4c0886eb85249ad05ed592902fa6865bb9d70"
+ENV SUPERCRONIC_SHA1SUM "2319da694833c7a147976b8e5f337cd83397d6be"
ENV SUPERCRONIC_CRONTAB "/etc/crontab"
# for build
@@ -165,8 +165,8 @@ ADD shared/bin/nic-capture-setup.sh /usr/local/bin/
# these ENVs should match the number of third party scripts/plugins installed by zeek_install_plugins.sh
ENV ZEEK_THIRD_PARTY_PLUGINS_COUNT 22
ENV ZEEK_THIRD_PARTY_PLUGINS_GREP "(Zeek::Spicy|ANALYZER_SPICY_DHCP|ANALYZER_SPICY_DNS|ANALYZER_SPICY_HTTP|ANALYZER_SPICY__OSPF|ANALYZER_SPICY_OPENVPN_UDP\b|ANALYZER_SPICY_IPSEC_UDP\b|ANALYZER_SPICY_TFTP|ANALYZER_SPICY_WIREGUARD|ANALYZER_SPICY_LDAP_TCP|ANALYZER_SPICY_GENISYS_TCP|ANALYZER_S7COMM_TCP|Corelight::CommunityID|Corelight::PE_XOR|ICSNPP::BACnet|ICSNPP::BSAP|ICSNPP::ENIP|ICSNPP::ETHERCAT|ICSNPP::OPCUA_Binary|Salesforce::GQUIC|Zeek::PROFINET|Zeek::TDS)"
-ENV ZEEK_THIRD_PARTY_SCRIPTS_COUNT 23
-ENV ZEEK_THIRD_PARTY_SCRIPTS_GREP "(bzar/main|callstranger-detector/callstranger|cve-2020-0601/cve-2020-0601|cve-2020-13777/cve-2020-13777|CVE-2020-16898/CVE-2020-16898|CVE-2021-38647/omigod|CVE-2021-31166/detect|CVE-2021-41773/CVE_2021_41773|CVE-2021-42292/main|cve-2021-44228/CVE_2021_44228|cve-2022-22954/main|cve-2022-26809/main|CVE-2022-3602/__load__|hassh/hassh|http-more-files-names/main|ja3/ja3|pingback/detect|ripple20/ripple20|SIGRed/CVE-2020-1350|zeek-EternalSafety/main|zeek-httpattacks/main|zeek-sniffpass/__load__|zerologon/main)\.(zeek|bro)"
+ENV ZEEK_THIRD_PARTY_SCRIPTS_COUNT 25
+ENV ZEEK_THIRD_PARTY_SCRIPTS_GREP "(bro-is-darknet/main|bro-simple-scan/scan|bzar/main|callstranger-detector/callstranger|cve-2020-0601/cve-2020-0601|cve-2020-13777/cve-2020-13777|CVE-2020-16898/CVE-2020-16898|CVE-2021-38647/omigod|CVE-2021-31166/detect|CVE-2021-41773/CVE_2021_41773|CVE-2021-42292/main|cve-2021-44228/CVE_2021_44228|cve-2022-22954/main|cve-2022-26809/main|CVE-2022-3602/__load__|hassh/hassh|http-more-files-names/main|ja3/ja3|pingback/detect|ripple20/ripple20|SIGRed/CVE-2020-1350|zeek-EternalSafety/main|zeek-httpattacks/main|zeek-sniffpass/__load__|zerologon/main)\.(zeek|bro)"
RUN mkdir -p /tmp/logs && \
cd /tmp/logs && \
diff --git a/_config.yml b/_config.yml
index 94758981e..ada0f7e65 100644
--- a/_config.yml
+++ b/_config.yml
@@ -4,7 +4,7 @@ description: A powerful, easily deployable network traffic analysis tool suite
logo: docs/images/logo/Malcolm_outline_banner_dark.png
remote_theme: pages-themes/minimal@v0.2.0
external_download_url: https://malcolm.fyi/docs/download.html
-youtube_url: https://www.youtube.com/c/MalcolmNetworkTrafficAnalysisToolSuite
+youtube_url: https://www.youtube.com/@MalcolmNetworkTrafficAnalysis
mastodon:
id:
url:
diff --git a/api/project/__init__.py b/api/project/__init__.py
index b18119bc7..df906f6f6 100644
--- a/api/project/__init__.py
+++ b/api/project/__init__.py
@@ -1,7 +1,6 @@
import dateparser
import json
import malcolm_common
-import opensearch_dsl
import opensearchpy
import os
import pytz
@@ -167,15 +166,15 @@
else defaultdict(lambda: None)
)
if opensearchCreds['user'] is not None:
- opensearchDslHttpAuth = f"{opensearchCreds['user']}:{opensearchCreds['password']}"
+ opensearchHttpAuth = f"{opensearchCreds['user']}:{opensearchCreds['password']}"
opensearchReqHttpAuth = HTTPBasicAuth(opensearchCreds['user'], opensearchCreds['password'])
else:
- opensearchDslHttpAuth = None
+ opensearchHttpAuth = None
opensearchReqHttpAuth = None
-opensearch_dsl.connections.create_connection(
+opensearchClient = opensearchpy.OpenSearch(
hosts=[opensearchUrl],
- http_auth=opensearchDslHttpAuth,
+ http_auth=opensearchHttpAuth,
verify_certs=opensearchSslVerify,
ssl_assert_hostname=False,
ssl_show_warn=False,
@@ -333,7 +332,7 @@ def filtertime(search, args, default_from="1 day ago", default_to="now"):
Parameters
----------
- search : opensearch_dsl.Search
+ search : opensearchpy.Search
The object representing the OpenSearch Search query
args : dict
The dictionary which should contain 'from' and 'to' times (see gettimes)
@@ -377,7 +376,7 @@ def filtervalues(search, args):
Parameters
----------
- search : opensearch_dsl.Search
+ search : opensearchpy.Search
The object representing the OpenSearch Search query
args : dict
The dictionary which should contain 'filter' (see getfilters)
@@ -413,7 +412,7 @@ def filtervalues(search, args):
)
else:
# field does not exist ("is null")
- s = s.filter('bool', must_not=opensearch_dsl.Q('exists', field=fieldname))
+ s = s.filter('bool', must_not=opensearchpy.helpers.query.Q('exists', field=fieldname))
if debugApi:
print(f'filtervalues: {json.dumps(s.to_dict())}')
@@ -442,8 +441,11 @@ def bucketfield(fieldname, current_request, urls=None):
fields
the name of the field(s) on which the aggregation was performed
"""
- s = opensearch_dsl.Search(
- using=opensearch_dsl.connections.get_connection(), index=app.config["ARKIME_INDEX_PATTERN"]
+ global opensearchClient
+
+ s = opensearchpy.Search(
+ using=opensearchClient,
+ index=app.config["ARKIME_INDEX_PATTERN"],
).extra(size=0)
args = get_request_arguments(current_request)
start_time_ms, end_time_ms, s = filtertime(s, args)
@@ -523,10 +525,13 @@ def document(index):
results
array of the documents retrieved (up to 'limit')
"""
+ global opensearchClient
+
args = get_request_arguments(request)
- s = opensearch_dsl.Search(using=opensearch_dsl.connections.get_connection(), index=index).extra(
- size=int(deep_get(args, ["limit"], app.config["RESULT_SET_LIMIT"]))
- )
+ s = opensearchpy.Search(
+ using=opensearchClient,
+ index=index,
+ ).extra(size=int(deep_get(args, ["limit"], app.config["RESULT_SET_LIMIT"])))
start_time_ms, end_time_ms, s = filtertime(s, args, default_from="1970-1-1", default_to="now")
filters, s = filtervalues(s, args)
return jsonify(
@@ -574,6 +579,8 @@ def fields():
fields
A dict of dicts where key is the field name and value may contain 'description' and 'type'
"""
+ global opensearchClient
+
args = get_request_arguments(request)
templateName = args['template'] if 'template' in args else app.config["MALCOLM_TEMPLATE"]
@@ -585,8 +592,9 @@ def fields():
if arkimeFields:
try:
# get fields from Arkime's field's table
- s = opensearch_dsl.Search(
- using=opensearch_dsl.connections.get_connection(), index=app.config["ARKIME_FIELDS_INDEX"]
+ s = opensearchpy.Search(
+ using=opensearchClient,
+ index=app.config["ARKIME_FIELDS_INDEX"],
).extra(size=5000)
for hit in [x['_source'] for x in s.execute().to_dict().get('hits', {}).get('hits', [])]:
if (fieldname := deep_get(hit, ['dbField2'])) and (fieldname not in fields):
@@ -697,6 +705,8 @@ def version():
opensearch_health
a JSON structure containing OpenSearch cluster health
"""
+ global opensearchClient
+
return jsonify(
version=app.config["MALCOLM_VERSION"],
built=app.config["BUILD_DATE"],
@@ -706,7 +716,7 @@ def version():
auth=opensearchReqHttpAuth,
verify=opensearchSslVerify,
).json(),
- opensearch_health=opensearch_dsl.connections.get_connection().cluster.health(),
+ opensearch_health=opensearchClient.cluster.health(),
)
@@ -783,6 +793,8 @@ def event():
status
the JSON-formatted OpenSearch response from indexing/updating the alert record
"""
+ global opensearchClient
+
alert = {}
idxResponse = {}
data = get_request_arguments(request)
@@ -880,7 +892,7 @@ def event():
alert['event']['hits'] = hitCount
docDateStr = dateparser.parse(alert['@timestamp']).strftime('%y%m%d')
- idxResponse = opensearch_dsl.connections.get_connection().index(
+ idxResponse = opensearchClient.index(
index=f"{app.config['ARKIME_INDEX_PATTERN'].rstrip('*')}{docDateStr}",
id=f"{docDateStr}-{alert['event']['id']}",
body=alert,
diff --git a/api/requirements.txt b/api/requirements.txt
index 3c3b90f4a..77c4ee103 100644
--- a/api/requirements.txt
+++ b/api/requirements.txt
@@ -1,8 +1,7 @@
pytz==2021.3
Flask==2.0.2
gunicorn==20.1.0
-opensearch-py==2.1.1
-opensearch-dsl==2.0.1
+opensearch-py==2.2.0
requests==2.26.0
regex==2022.3.2
dateparser==1.1.1
\ No newline at end of file
diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml
index 7d52feba7..ccdc723e3 100644
--- a/docker-compose-standalone.yml
+++ b/docker-compose-standalone.yml
@@ -359,7 +359,7 @@ x-pcap-capture-variables: &pcap-capture-variables
services:
opensearch:
- image: malcolmnetsec/opensearch:23.02.0
+ image: malcolmnetsec/opensearch:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -400,7 +400,7 @@ services:
retries: 3
start_period: 180s
dashboards-helper:
- image: malcolmnetsec/dashboards-helper:23.02.0
+ image: malcolmnetsec/dashboards-helper:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -431,7 +431,7 @@ services:
retries: 3
start_period: 30s
dashboards:
- image: malcolmnetsec/dashboards:23.02.0
+ image: malcolmnetsec/dashboards:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -456,7 +456,7 @@ services:
retries: 3
start_period: 210s
logstash:
- image: malcolmnetsec/logstash-oss:23.02.0
+ image: malcolmnetsec/logstash-oss:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -499,7 +499,7 @@ services:
retries: 3
start_period: 600s
filebeat:
- image: malcolmnetsec/filebeat-oss:23.02.0
+ image: malcolmnetsec/filebeat-oss:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -538,7 +538,7 @@ services:
retries: 3
start_period: 60s
arkime:
- image: malcolmnetsec/arkime:23.02.0
+ image: malcolmnetsec/arkime:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -576,7 +576,7 @@ services:
retries: 3
start_period: 210s
zeek:
- image: malcolmnetsec/zeek:23.02.0
+ image: malcolmnetsec/zeek:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -615,7 +615,7 @@ services:
retries: 3
start_period: 60s
zeek-live:
- image: malcolmnetsec/zeek:23.02.0
+ image: malcolmnetsec/zeek:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -647,7 +647,7 @@ services:
- ./zeek-logs/extract_files:/zeek/extract_files
- ./zeek/intel:/opt/zeek/share/zeek/site/intel
suricata:
- image: malcolmnetsec/suricata:23.02.0
+ image: malcolmnetsec/suricata:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -684,7 +684,7 @@ services:
retries: 3
start_period: 120s
suricata-live:
- image: malcolmnetsec/suricata:23.02.0
+ image: malcolmnetsec/suricata:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -711,7 +711,7 @@ services:
- ./suricata-logs:/var/log/suricata
- ./suricata/rules:/opt/suricata/rules:ro
file-monitor:
- image: malcolmnetsec/file-monitor:23.02.0
+ image: malcolmnetsec/file-monitor:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -735,7 +735,7 @@ services:
retries: 3
start_period: 60s
pcap-capture:
- image: malcolmnetsec/pcap-capture:23.02.0
+ image: malcolmnetsec/pcap-capture:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -757,7 +757,7 @@ services:
- ./nginx/ca-trust:/var/local/ca-trust:ro
- ./pcap/upload:/pcap
pcap-monitor:
- image: malcolmnetsec/pcap-monitor:23.02.0
+ image: malcolmnetsec/pcap-monitor:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -783,7 +783,7 @@ services:
retries: 3
start_period: 90s
upload:
- image: malcolmnetsec/file-upload:23.02.0
+ image: malcolmnetsec/file-upload:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -811,7 +811,7 @@ services:
retries: 3
start_period: 60s
htadmin:
- image: malcolmnetsec/htadmin:23.02.0
+ image: malcolmnetsec/htadmin:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -835,7 +835,7 @@ services:
retries: 3
start_period: 60s
freq:
- image: malcolmnetsec/freq:23.02.0
+ image: malcolmnetsec/freq:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -856,7 +856,7 @@ services:
retries: 3
start_period: 60s
name-map-ui:
- image: malcolmnetsec/name-map-ui:23.02.0
+ image: malcolmnetsec/name-map-ui:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -877,7 +877,7 @@ services:
retries: 3
start_period: 60s
netbox:
- image: malcolmnetsec/netbox:23.02.0
+ image: malcolmnetsec/netbox:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -908,7 +908,7 @@ services:
retries: 3
start_period: 120s
netbox-postgres:
- image: malcolmnetsec/postgresql:23.02.0
+ image: malcolmnetsec/postgresql:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -931,7 +931,7 @@ services:
retries: 3
start_period: 45s
netbox-redis:
- image: malcolmnetsec/redis:23.02.0
+ image: malcolmnetsec/redis:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -958,7 +958,7 @@ services:
retries: 3
start_period: 45s
netbox-redis-cache:
- image: malcolmnetsec/redis:23.02.0
+ image: malcolmnetsec/redis:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -984,7 +984,7 @@ services:
retries: 3
start_period: 45s
api:
- image: malcolmnetsec/api:23.02.0
+ image: malcolmnetsec/api:23.03.0
command: gunicorn --bind 0:5000 manage:app
restart: "no"
stdin_open: false
@@ -1007,7 +1007,7 @@ services:
retries: 3
start_period: 60s
nginx-proxy:
- image: malcolmnetsec/nginx-proxy:23.02.0
+ image: malcolmnetsec/nginx-proxy:23.03.0
restart: "no"
stdin_open: false
tty: true
diff --git a/docker-compose.yml b/docker-compose.yml
index 9b4b2753d..25cb415e6 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -362,7 +362,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/opensearch.Dockerfile
- image: malcolmnetsec/opensearch:23.02.0
+ image: malcolmnetsec/opensearch:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -406,7 +406,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/dashboards-helper.Dockerfile
- image: malcolmnetsec/dashboards-helper:23.02.0
+ image: malcolmnetsec/dashboards-helper:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -440,7 +440,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/dashboards.Dockerfile
- image: malcolmnetsec/dashboards:23.02.0
+ image: malcolmnetsec/dashboards:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -468,7 +468,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/logstash.Dockerfile
- image: malcolmnetsec/logstash-oss:23.02.0
+ image: malcolmnetsec/logstash-oss:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -518,7 +518,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/filebeat.Dockerfile
- image: malcolmnetsec/filebeat-oss:23.02.0
+ image: malcolmnetsec/filebeat-oss:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -560,7 +560,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/arkime.Dockerfile
- image: malcolmnetsec/arkime:23.02.0
+ image: malcolmnetsec/arkime:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -604,7 +604,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/zeek.Dockerfile
- image: malcolmnetsec/zeek:23.02.0
+ image: malcolmnetsec/zeek:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -647,7 +647,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/zeek.Dockerfile
- image: malcolmnetsec/zeek:23.02.0
+ image: malcolmnetsec/zeek:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -683,7 +683,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/suricata.Dockerfile
- image: malcolmnetsec/suricata:23.02.0
+ image: malcolmnetsec/suricata:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -723,7 +723,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/suricata.Dockerfile
- image: malcolmnetsec/suricata:23.02.0
+ image: malcolmnetsec/suricata:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -753,7 +753,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/file-monitor.Dockerfile
- image: malcolmnetsec/file-monitor:23.02.0
+ image: malcolmnetsec/file-monitor:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -780,7 +780,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/pcap-capture.Dockerfile
- image: malcolmnetsec/pcap-capture:23.02.0
+ image: malcolmnetsec/pcap-capture:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -805,7 +805,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/pcap-monitor.Dockerfile
- image: malcolmnetsec/pcap-monitor:23.02.0
+ image: malcolmnetsec/pcap-monitor:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -834,7 +834,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/file-upload.Dockerfile
- image: malcolmnetsec/file-upload:23.02.0
+ image: malcolmnetsec/file-upload:23.03.0
restart: "no"
stdin_open: false
tty: true
@@ -862,7 +862,7 @@ services:
retries: 3
start_period: 60s
htadmin:
- image: malcolmnetsec/htadmin:23.02.0
+ image: malcolmnetsec/htadmin:23.03.0
build:
context: .
dockerfile: Dockerfiles/htadmin.Dockerfile
@@ -889,7 +889,7 @@ services:
retries: 3
start_period: 60s
freq:
- image: malcolmnetsec/freq:23.02.0
+ image: malcolmnetsec/freq:23.03.0
build:
context: .
dockerfile: Dockerfiles/freq.Dockerfile
@@ -913,7 +913,7 @@ services:
retries: 3
start_period: 60s
name-map-ui:
- image: malcolmnetsec/name-map-ui:23.02.0
+ image: malcolmnetsec/name-map-ui:23.03.0
build:
context: .
dockerfile: Dockerfiles/name-map-ui.Dockerfile
@@ -937,7 +937,7 @@ services:
retries: 3
start_period: 60s
netbox:
- image: malcolmnetsec/netbox:23.02.0
+ image: malcolmnetsec/netbox:23.03.0
build:
context: .
dockerfile: Dockerfiles/netbox.Dockerfile
@@ -972,7 +972,7 @@ services:
retries: 3
start_period: 120s
netbox-postgres:
- image: malcolmnetsec/postgresql:23.02.0
+ image: malcolmnetsec/postgresql:23.03.0
build:
context: .
dockerfile: Dockerfiles/postgresql.Dockerfile
@@ -998,7 +998,7 @@ services:
retries: 3
start_period: 45s
netbox-redis:
- image: malcolmnetsec/redis:23.02.0
+ image: malcolmnetsec/redis:23.03.0
build:
context: .
dockerfile: Dockerfiles/redis.Dockerfile
@@ -1028,7 +1028,7 @@ services:
retries: 3
start_period: 45s
netbox-redis-cache:
- image: malcolmnetsec/redis:23.02.0
+ image: malcolmnetsec/redis:23.03.0
build:
context: .
dockerfile: Dockerfiles/redis.Dockerfile
@@ -1057,7 +1057,7 @@ services:
retries: 3
start_period: 45s
api:
- image: malcolmnetsec/api:23.02.0
+ image: malcolmnetsec/api:23.03.0
build:
context: .
dockerfile: Dockerfiles/api.Dockerfile
@@ -1086,7 +1086,7 @@ services:
build:
context: .
dockerfile: Dockerfiles/nginx.Dockerfile
- image: malcolmnetsec/nginx-proxy:23.02.0
+ image: malcolmnetsec/nginx-proxy:23.03.0
restart: "no"
stdin_open: false
tty: true
diff --git a/docs/download.md b/docs/download.md
index 79d274a95..f0ae7097e 100644
--- a/docs/download.md
+++ b/docs/download.md
@@ -16,7 +16,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno
| ISO | SHA256 |
|---|---|
-| [malcolm-23.02.0.iso](/iso/malcolm-23.02.0.iso) (5.3GiB) | [`2d43ad3023e19460852ba5bf28e4b6d35072308bef07945baa4284bbf77fbb09`](/iso/malcolm-23.02.0.iso.sha256.txt) |
+| [malcolm-23.03.0.iso](/iso/malcolm-23.03.0.iso) (5.3GiB) | [`9459fb0ce61fba8c7a9a9457b24d42182519dbb62247111471e38f8c190113eb`](/iso/malcolm-23.03.0.iso.sha256.txt) |
## Hedgehog Linux
@@ -26,7 +26,7 @@ While official downloads of the Malcolm installer ISO are not provided, an **uno
| ISO | SHA256 |
|---|---|
-| [hedgehog-23.02.0.iso](/iso/hedgehog-23.02.0.iso) (2.3GiB) | [`21f0a0ae746c993853c26bbe88995f25fc1784192155616c14f2b4645c14ef44`](/iso/hedgehog-23.02.0.iso.sha256.txt) |
+| [hedgehog-23.03.0.iso](/iso/hedgehog-23.03.0.iso) (2.3GiB) | [`3cdba91e417f6ada83130aabc3be38dd0a8b12b6bda227859a546ace198680bc`](/iso/hedgehog-23.03.0.iso.sha256.txt) |
## Warning
diff --git a/docs/hedgehog-iso-build.md b/docs/hedgehog-iso-build.md
index 29b0e1dbd..c37ed0b23 100644
--- a/docs/hedgehog-iso-build.md
+++ b/docs/hedgehog-iso-build.md
@@ -29,7 +29,7 @@ Building the ISO may take 90 minutes or more depending on your system. As the bu
```
…
-Finished, created "/sensor-build/hedgehog-23.02.0.iso"
+Finished, created "/sensor-build/hedgehog-23.03.0.iso"
…
```
diff --git a/docs/malcolm-iso.md b/docs/malcolm-iso.md
index e5009970d..edeb32810 100644
--- a/docs/malcolm-iso.md
+++ b/docs/malcolm-iso.md
@@ -41,7 +41,7 @@ Building the ISO may take 30 minutes or more depending on your system. As the bu
```
…
-Finished, created "/malcolm-build/malcolm-iso/malcolm-23.02.0.iso"
+Finished, created "/malcolm-build/malcolm-iso/malcolm-23.03.0.iso"
…
```
diff --git a/docs/quickstart.md b/docs/quickstart.md
index 96278aa73..01dbc0a35 100644
--- a/docs/quickstart.md
+++ b/docs/quickstart.md
@@ -53,26 +53,26 @@ You can then observe that the images have been retrieved by running `docker imag
```
$ docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
-malcolmnetsec/api 23.02.0 xxxxxxxxxxxx 3 days ago 158MB
-malcolmnetsec/arkime 23.02.0 xxxxxxxxxxxx 3 days ago 816MB
-malcolmnetsec/dashboards 23.02.0 xxxxxxxxxxxx 3 days ago 1.02GB
-malcolmnetsec/dashboards-helper 23.02.0 xxxxxxxxxxxx 3 days ago 184MB
-malcolmnetsec/file-monitor 23.02.0 xxxxxxxxxxxx 3 days ago 588MB
-malcolmnetsec/file-upload 23.02.0 xxxxxxxxxxxx 3 days ago 259MB
-malcolmnetsec/filebeat-oss 23.02.0 xxxxxxxxxxxx 3 days ago 624MB
-malcolmnetsec/freq 23.02.0 xxxxxxxxxxxx 3 days ago 132MB
-malcolmnetsec/htadmin 23.02.0 xxxxxxxxxxxx 3 days ago 242MB
-malcolmnetsec/logstash-oss 23.02.0 xxxxxxxxxxxx 3 days ago 1.35GB
-malcolmnetsec/name-map-ui 23.02.0 xxxxxxxxxxxx 3 days ago 143MB
-malcolmnetsec/netbox 23.02.0 xxxxxxxxxxxx 3 days ago 1.01GB
-malcolmnetsec/nginx-proxy 23.02.0 xxxxxxxxxxxx 3 days ago 121MB
-malcolmnetsec/opensearch 23.02.0 xxxxxxxxxxxx 3 days ago 1.17GB
-malcolmnetsec/pcap-capture 23.02.0 xxxxxxxxxxxx 3 days ago 121MB
-malcolmnetsec/pcap-monitor 23.02.0 xxxxxxxxxxxx 3 days ago 213MB
-malcolmnetsec/postgresql 23.02.0 xxxxxxxxxxxx 3 days ago 268MB
-malcolmnetsec/redis 23.02.0 xxxxxxxxxxxx 3 days ago 34.2MB
-malcolmnetsec/suricata 23.02.0 xxxxxxxxxxxx 3 days ago 278MB
-malcolmnetsec/zeek 23.02.0 xxxxxxxxxxxx 3 days ago 1GB
+malcolmnetsec/api 23.03.0 xxxxxxxxxxxx 3 days ago 158MB
+malcolmnetsec/arkime 23.03.0 xxxxxxxxxxxx 3 days ago 816MB
+malcolmnetsec/dashboards 23.03.0 xxxxxxxxxxxx 3 days ago 1.02GB
+malcolmnetsec/dashboards-helper 23.03.0 xxxxxxxxxxxx 3 days ago 184MB
+malcolmnetsec/file-monitor 23.03.0 xxxxxxxxxxxx 3 days ago 588MB
+malcolmnetsec/file-upload 23.03.0 xxxxxxxxxxxx 3 days ago 259MB
+malcolmnetsec/filebeat-oss 23.03.0 xxxxxxxxxxxx 3 days ago 624MB
+malcolmnetsec/freq 23.03.0 xxxxxxxxxxxx 3 days ago 132MB
+malcolmnetsec/htadmin 23.03.0 xxxxxxxxxxxx 3 days ago 242MB
+malcolmnetsec/logstash-oss 23.03.0 xxxxxxxxxxxx 3 days ago 1.35GB
+malcolmnetsec/name-map-ui 23.03.0 xxxxxxxxxxxx 3 days ago 143MB
+malcolmnetsec/netbox 23.03.0 xxxxxxxxxxxx 3 days ago 1.01GB
+malcolmnetsec/nginx-proxy 23.03.0 xxxxxxxxxxxx 3 days ago 121MB
+malcolmnetsec/opensearch 23.03.0 xxxxxxxxxxxx 3 days ago 1.17GB
+malcolmnetsec/pcap-capture 23.03.0 xxxxxxxxxxxx 3 days ago 121MB
+malcolmnetsec/pcap-monitor 23.03.0 xxxxxxxxxxxx 3 days ago 213MB
+malcolmnetsec/postgresql 23.03.0 xxxxxxxxxxxx 3 days ago 268MB
+malcolmnetsec/redis 23.03.0 xxxxxxxxxxxx 3 days ago 34.2MB
+malcolmnetsec/suricata 23.03.0 xxxxxxxxxxxx 3 days ago 278MB
+malcolmnetsec/zeek 23.03.0 xxxxxxxxxxxx 3 days ago 1GB
```
### Import from pre-packaged tarballs
diff --git a/docs/ubuntu-install-example.md b/docs/ubuntu-install-example.md
index d09d25bdb..61785dcde 100644
--- a/docs/ubuntu-install-example.md
+++ b/docs/ubuntu-install-example.md
@@ -261,26 +261,26 @@ Pulling zeek ... done
user@host:~/Malcolm$ docker images
REPOSITORY TAG IMAGE ID CREATED SIZE
-malcolmnetsec/api 23.02.0 xxxxxxxxxxxx 3 days ago 158MB
-malcolmnetsec/arkime 23.02.0 xxxxxxxxxxxx 3 days ago 816MB
-malcolmnetsec/dashboards 23.02.0 xxxxxxxxxxxx 3 days ago 1.02GB
-malcolmnetsec/dashboards-helper 23.02.0 xxxxxxxxxxxx 3 days ago 184MB
-malcolmnetsec/file-monitor 23.02.0 xxxxxxxxxxxx 3 days ago 588MB
-malcolmnetsec/file-upload 23.02.0 xxxxxxxxxxxx 3 days ago 259MB
-malcolmnetsec/filebeat-oss 23.02.0 xxxxxxxxxxxx 3 days ago 624MB
-malcolmnetsec/freq 23.02.0 xxxxxxxxxxxx 3 days ago 132MB
-malcolmnetsec/htadmin 23.02.0 xxxxxxxxxxxx 3 days ago 242MB
-malcolmnetsec/logstash-oss 23.02.0 xxxxxxxxxxxx 3 days ago 1.35GB
-malcolmnetsec/name-map-ui 23.02.0 xxxxxxxxxxxx 3 days ago 143MB
-malcolmnetsec/netbox 23.02.0 xxxxxxxxxxxx 3 days ago 1.01GB
-malcolmnetsec/nginx-proxy 23.02.0 xxxxxxxxxxxx 3 days ago 121MB
-malcolmnetsec/opensearch 23.02.0 xxxxxxxxxxxx 3 days ago 1.17GB
-malcolmnetsec/pcap-capture 23.02.0 xxxxxxxxxxxx 3 days ago 121MB
-malcolmnetsec/pcap-monitor 23.02.0 xxxxxxxxxxxx 3 days ago 213MB
-malcolmnetsec/postgresql 23.02.0 xxxxxxxxxxxx 3 days ago 268MB
-malcolmnetsec/redis 23.02.0 xxxxxxxxxxxx 3 days ago 34.2MB
-malcolmnetsec/suricata 23.02.0 xxxxxxxxxxxx 3 days ago 278MB
-malcolmnetsec/zeek 23.02.0 xxxxxxxxxxxx 3 days ago 1GB
+malcolmnetsec/api 23.03.0 xxxxxxxxxxxx 3 days ago 158MB
+malcolmnetsec/arkime 23.03.0 xxxxxxxxxxxx 3 days ago 816MB
+malcolmnetsec/dashboards 23.03.0 xxxxxxxxxxxx 3 days ago 1.02GB
+malcolmnetsec/dashboards-helper 23.03.0 xxxxxxxxxxxx 3 days ago 184MB
+malcolmnetsec/file-monitor 23.03.0 xxxxxxxxxxxx 3 days ago 588MB
+malcolmnetsec/file-upload 23.03.0 xxxxxxxxxxxx 3 days ago 259MB
+malcolmnetsec/filebeat-oss 23.03.0 xxxxxxxxxxxx 3 days ago 624MB
+malcolmnetsec/freq 23.03.0 xxxxxxxxxxxx 3 days ago 132MB
+malcolmnetsec/htadmin 23.03.0 xxxxxxxxxxxx 3 days ago 242MB
+malcolmnetsec/logstash-oss 23.03.0 xxxxxxxxxxxx 3 days ago 1.35GB
+malcolmnetsec/name-map-ui 23.03.0 xxxxxxxxxxxx 3 days ago 143MB
+malcolmnetsec/netbox 23.03.0 xxxxxxxxxxxx 3 days ago 1.01GB
+malcolmnetsec/nginx-proxy 23.03.0 xxxxxxxxxxxx 3 days ago 121MB
+malcolmnetsec/opensearch 23.03.0 xxxxxxxxxxxx 3 days ago 1.17GB
+malcolmnetsec/pcap-capture 23.03.0 xxxxxxxxxxxx 3 days ago 121MB
+malcolmnetsec/pcap-monitor 23.03.0 xxxxxxxxxxxx 3 days ago 213MB
+malcolmnetsec/postgresql 23.03.0 xxxxxxxxxxxx 3 days ago 268MB
+malcolmnetsec/redis 23.03.0 xxxxxxxxxxxx 3 days ago 34.2MB
+malcolmnetsec/suricata 23.03.0 xxxxxxxxxxxx 3 days ago 278MB
+malcolmnetsec/zeek 23.03.0 xxxxxxxxxxxx 3 days ago 1GB
```
Finally, we can start Malcolm. When Malcolm starts it will stream informational and debug messages to the console. If you wish, you can safely close the console or use `Ctrl+C` to stop these messages; Malcolm will continue running in the background.
diff --git a/logstash/pipelines/zeek/10_zeek_prep.conf b/logstash/pipelines/zeek/10_zeek_prep.conf
index 7f92bbeb2..15d1d17fa 100644
--- a/logstash/pipelines/zeek/10_zeek_prep.conf
+++ b/logstash/pipelines/zeek/10_zeek_prep.conf
@@ -6,14 +6,18 @@ filter {
drop { id => "drop_zeek_invalid_logs" }
}
- # tags may have been specified, like: conn(tagA,tagB,tagC).log, extract the log type (conn) and the tags (tagA,tagB,tagC)
- # also normalize log types with - in their names to _ (e.g., opcua-binary -> opcua_binary)
+ # - Tags may have been specified, like: conn(tagA,tagB,tagC).log, extract the log type (conn) and the tags (tagA,tagB,tagC).
+ # - Normalize log types with - in their names to _ (e.g., opcua-binary -> opcua_binary).
+ # - Zeek log files might be caught by filebeat right in the middle of being renamed/moved as
+ # part of log rotation (ie., renamed from conn.log to conn.2020-01-16-14-00-00.log or
+ # conn.2020_01_16_14_00_00.log). We don't care about that, ignore the date part and just process
+ # the log source as we normally would.
ruby {
- id => "ruby_zeek_source_extract"
- #↓Type ↓Tags
+ id => "ruby_zeek_log_source_extract"
+ # ↓Type ↓Tags ↓Rotate Timestamp (discard) ↓.log (discard)
code => "
- if fileParts = event.get('[log][file][path]').split('/').last.match(/^(.*?)(?:\((.*)\))?\.log/i) then
- logType, tags = fileParts.captures
+ if fileParts = event.get('[log][file][path]').split('/').last.match(/^(.*?)(?:\((.*)\))?(?:\.\d{4}[_:-]?\d{2}[_:-]?\d{2}[_:-]?\d{2}[_:-]?\d{2}[_:-]?\d{2})?\.log/i) then
+ logType, tags = fileParts.captures
event.set('[log_source]', logType.gsub('-', '_')) unless logType.nil?
event.set('[@metadata][zeek_log_tags]', tags) unless tags.nil?
end"
diff --git a/logstash/pipelines/zeek/11_zeek_parse.conf b/logstash/pipelines/zeek/11_zeek_parse.conf
index e720c3afb..9314d04b9 100644
--- a/logstash/pipelines/zeek/11_zeek_parse.conf
+++ b/logstash/pipelines/zeek/11_zeek_parse.conf
@@ -5077,10 +5077,6 @@ filter {
add_tag => [ "ics" ]
}
- } else if ([log_source] =~ /\.\d{4}[_:-]?\d{2}[_:-]?\d{2}[_:-]?\d{2}[_:-]?\d{2}[_:-]?\d{2}$/) {
- # filebeat caught a file right in the middle of being renamed/moved
- drop { id => "drop_renamed_logfile_opcua" }
-
} else {
# some other unknown zeek opcua- log file. should start with ts at least!
csv {
@@ -5105,12 +5101,6 @@ filter {
} # if / else if for opcua log types
- } else if ([log_source] =~ /\.\d{4}[_:-]?\d{2}[_:-]?\d{2}[_:-]?\d{2}[_:-]?\d{2}[_:-]?\d{2}$/) {
- # filebeat caught a file right in the middle of being renamed/moved
- # (ie., renamed from conn.log to conn.2020-01-16-14-00-00.log or conn.2020_01_16_14_00_00.log).
- # this has actually already been processed, so ignore this event.
- drop { id => "drop_renamed_logfile" }
-
} else {
# some other unknown zeek log file. should start with ts at least!
csv {
diff --git a/malcolm-iso/build.sh b/malcolm-iso/build.sh
index 7eaf72adb..e9e6990ad 100755
--- a/malcolm-iso/build.sh
+++ b/malcolm-iso/build.sh
@@ -131,6 +131,8 @@ if [ -d "$WORKDIR" ]; then
pushd "$MALCOLM_DEST_DIR/scripts/" >/dev/null 2>&1
ln -s ./control.py auth_setup
ln -s ./control.py logs
+ ln -s ./control.py netbox-backup
+ ln -s ./control.py netbox-restore
ln -s ./control.py restart
ln -s ./control.py start
ln -s ./control.py status
diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/autostart/set-malcolm-gtk-bookmark.desktop b/malcolm-iso/config/includes.chroot/etc/skel/.config/autostart/set-malcolm-gtk-bookmark.desktop
new file mode 100644
index 000000000..0f31f1ae1
--- /dev/null
+++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/autostart/set-malcolm-gtk-bookmark.desktop
@@ -0,0 +1,7 @@
+[Desktop Entry]
+Encoding=UTF-8
+Name=set-malcolm-gtk-bookmark
+Comment=Add Malcolm directory to GTK-3.0 bookmarks
+Exec=/usr/local/bin/set-malcolm-gtk-bookmark.sh
+Terminal=false
+Type=Application
diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-29/16782182331.desktop b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-29/16782182331.desktop
new file mode 100644
index 000000000..91947e698
--- /dev/null
+++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-29/16782182331.desktop
@@ -0,0 +1,10 @@
+[Desktop Entry]
+Name=Configure Malcolm
+Exec=tilix --title="Malcolm Configuration" --maximize -e /bin/bash -l -c "/usr/bin/python3 ~/Malcolm/scripts/install.py --configure"
+Comment=install.py --configure
+Terminal=false
+Type=Application
+Icon=org.xfce.settings.manager
+Categories=Network;
+StartupNotify=true
+X-XFCE-Source=file:///usr/share/applications/malcolm-configure.desktop
\ No newline at end of file
diff --git a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml
index 6c0a032a4..9d0f3e6b6 100644
--- a/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml
+++ b/malcolm-iso/config/includes.chroot/etc/skel/.config/xfce4/xfconf/xfce-perchannel-xml/xfce4-panel.xml
@@ -22,6 +22,7 @@
+
@@ -63,6 +64,7 @@
+
@@ -150,5 +152,10 @@
+
+
+
+
+
diff --git a/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-configure.desktop b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-configure.desktop
new file mode 100644
index 000000000..253286161
--- /dev/null
+++ b/malcolm-iso/config/includes.chroot/usr/share/applications/malcolm-configure.desktop
@@ -0,0 +1,9 @@
+[Desktop Entry]
+Name=Configure Malcolm
+Exec=tilix --title="Malcolm Configuration" --maximize -e /bin/bash -l -c "/usr/bin/python3 ~/Malcolm/scripts/install.py --configure"
+Comment=install.py --configure
+Terminal=false
+Type=Application
+Icon=org.xfce.settings.manager
+Categories=Network;
+StartupNotify=true
\ No newline at end of file
diff --git a/scripts/control.py b/scripts/control.py
index 415e04eda..596ce0349 100755
--- a/scripts/control.py
+++ b/scripts/control.py
@@ -24,7 +24,7 @@
from malcolm_common import *
from base64 import b64encode
from collections import defaultdict, namedtuple
-from subprocess import PIPE, STDOUT, Popen, check_call, CalledProcessError
+from subprocess import PIPE, DEVNULL, Popen, TimeoutExpired
from urllib.parse import urlparse
try:
@@ -63,6 +63,7 @@ def __exit__(self, *args):
except:
coloramaImported = False
+
###################################################################################################
# perform a service-keystore operation in a Docker container
#
@@ -103,7 +104,6 @@ def keystore_op(service, dropPriv=False, *keystore_args, **run_process_kwargs):
uidGidDict = None
try:
-
uidGidDict = GetUidGidFromComposeFile(args.composeFile)
composeFileLines = list()
@@ -213,7 +213,6 @@ def keystore_op(service, dropPriv=False, *keystore_args, **run_process_kwargs):
raise Exception(f'Unable to identify docker image for {service} in {args.composeFile}')
if dockerCmd is not None:
-
# append whatever other arguments to pass to the executable filespec
if keystore_args:
dockerCmd.extend(list(keystore_args))
@@ -326,7 +325,6 @@ def netboxRestore(backupFileName=None):
global dockerComposeBin
if backupFileName and os.path.isfile(backupFileName):
-
# docker-compose use local temporary path
osEnv = os.environ.copy()
osEnv['TMPDIR'] = MalcolmTmpPath
@@ -345,6 +343,12 @@ def netboxRestore(backupFileName=None):
f'{uidGidDict["PUID"]}:{uidGidDict["PGID"]}',
]
+ # if the netbox_init.py process is happening, interrupt it
+ dockerCmd = dockerCmdBase + ['netbox', 'bash', '-c', 'pgrep -f /usr/local/bin/netbox_init.py | xargs -r kill']
+ err, results = run_process(dockerCmd, env=osEnv, debug=args.debug)
+ if (err != 0) and args.debug:
+ eprint(f'Error interrupting netbox_init.py: {results}')
+
# drop the existing netbox database
dockerCmd = dockerCmdBase + ['netbox-postgres', 'dropdb', '-U', 'netbox', 'netbox', '--force']
err, results = run_process(dockerCmd, env=osEnv, debug=args.debug)
@@ -456,6 +460,9 @@ def logs():
r'^(-?(?:[1-9][0-9]*)?[0-9]{4})-(1[0-2]|0[1-9])-(3[01]|0[1-9]|[12][0-9])T(2[0-3]|[01][0-9]):([0-5][0-9]):([0-5][0-9])(\.[0-9]+)?(Z|[+-](?:2[0-3]|[01][0-9]):[0-5][0-9])?$'
)
+ finishedStartingRegEx = re.compile(r'.+Pipelines\s+running\s+\{.*:non_running_pipelines=>\[\]\}')
+ finishedStarting = False
+
# increase COMPOSE_HTTP_TIMEOUT to be ridiculously large so docker-compose never times out the TTY doing debug output
osEnv = os.environ.copy()
osEnv['COMPOSE_HTTP_TIMEOUT'] = '100000000'
@@ -485,6 +492,7 @@ def logs():
][: 8 if args.service is not None else -1],
env=osEnv,
stdout=PIPE,
+ stderr=None if args.debug else DEVNULL,
)
while True:
output = process.stdout.readline()
@@ -495,6 +503,12 @@ def logs():
outputStrEscaped = EscapeAnsi(outputStr)
if ignoreRegEx.match(outputStrEscaped):
pass ### print(f'!!!!!!!: {outputStr}')
+ elif (
+ (args.cmdStart or args.cmdRestart)
+ and (not args.cmdLogs)
+ and finishedStartingRegEx.match(outputStrEscaped)
+ ):
+ finishedStarting = True
else:
serviceMatch = serviceRegEx.search(outputStrEscaped)
serviceMatchFmt = serviceRegEx.search(outputStr) if coloramaImported else serviceMatch
@@ -510,7 +524,6 @@ def logs():
outputJson = LoadStrIfJson(messageStrToTestJson)
if isinstance(outputJson, dict):
-
# if there's a timestamp, move it outside of the JSON to the beginning of the log string
timeKey = None
if 'time' in outputJson:
@@ -531,7 +544,6 @@ def logs():
and ('job.position' in outputJson)
and ('job.command' in outputJson)
):
-
# this is a status output line from supercronic, let's format and clean it up so it fits in better with the rest of the logs
# remove some clutter for the display
@@ -615,6 +627,27 @@ def logs():
else:
time.sleep(0.5)
+
+ if finishedStarting:
+ process.terminate()
+ try:
+ process.wait(timeout=5.0)
+ except TimeoutExpired:
+ process.kill()
+ # # TODO: Replace 'localhost' with an outwards-facing IP since I doubt anybody is
+ # accessing these from the Malcolm server.
+ print("\nStarted Malcolm\n\n")
+ print("Malcolm services can be accessed via the following URLs:")
+ print("------------------------------------------------------------------------------")
+ print(" - Arkime: https://localhost/")
+ print(" - OpenSearch Dashboards: https://localhost/dashboards/")
+ print(" - PCAP upload (web): https://localhost/upload/")
+ print(" - PCAP upload (sftp): sftp://username@127.0.0.1:8022/files/")
+ print(" - Host and subnet name mapping editor: https://localhost/name-map-ui/")
+ print(" - NetBox: https://localhost/netbox/\n")
+ print(" - Account management: https://localhost:488/\n")
+ print(" - Documentation: https://localhost/readme/\n")
+
process.poll()
@@ -818,19 +851,7 @@ def start():
# start docker
err, out = run_process([dockerComposeBin, '-f', args.composeFile, 'up', '--detach'], env=osEnv, debug=args.debug)
- if err == 0:
- eprint("Started Malcolm\n\n")
- eprint("In a few minutes, Malcolm services will be accessible via the following URLs:")
- eprint("------------------------------------------------------------------------------")
- eprint(" - Arkime: https://localhost/")
- eprint(" - OpenSearch Dashboards: https://localhost/dashboards/")
- eprint(" - PCAP upload (web): https://localhost/upload/")
- eprint(" - PCAP upload (sftp): sftp://username@127.0.0.1:8022/files/")
- eprint(" - Host and subnet name mapping editor: https://localhost/name-map-ui/")
- eprint(" - NetBox: https://localhost/netbox/\n")
- eprint(" - Account management: https://localhost:488/\n")
- eprint(" - Documentation: https://localhost/readme/\n")
- else:
+ if err != 0:
eprint("Malcolm failed to start\n")
eprint("\n".join(out))
exit(err)
@@ -844,7 +865,6 @@ def authSetup(wipe=False):
global opensslBin
if YesOrNo('Store administrator username/password for local Malcolm access?', default=True):
-
# prompt username and password
usernamePrevious = None
password = None
@@ -1023,7 +1043,6 @@ def authSetup(wipe=False):
filebeatPath = os.path.join(MalcolmPath, os.path.join('filebeat', 'certs'))
if YesOrNo('(Re)generate self-signed certificates for a remote log forwarder', default=True):
with pushd(logstashPath):
-
# make clean to clean previous files
for pat in ['*.srl', '*.csr', '*.key', '*.crt', '*.pem']:
for oldfile in glob.glob(pat):
@@ -1262,7 +1281,6 @@ def authSetup(wipe=False):
'Store username/password for email alert sender account? (see https://opensearch.org/docs/latest/monitoring-plugins/alerting/monitors/#authenticate-sender-account)',
default=False,
):
-
# prompt username and password
emailPassword = None
emailPasswordConfirm = None
@@ -1539,7 +1557,6 @@ def main():
exit(2)
with pushd(MalcolmPath):
-
# don't run this as root
if (pyPlatform != PLATFORM_WINDOWS) and (
(os.getuid() == 0) or (os.geteuid() == 0) or (getpass.getuser() == 'root')
diff --git a/scripts/demo/reset_and_auto_populate.sh b/scripts/demo/reset_and_auto_populate.sh
index 7b2c38113..dd1997303 100755
--- a/scripts/demo/reset_and_auto_populate.sh
+++ b/scripts/demo/reset_and_auto_populate.sh
@@ -68,7 +68,6 @@ FULL_PWD="$($REALPATH "$(pwd)" | head -n 1)"
# script options
set -e
set -u
-set -o pipefail
ENCODING="utf-8"
# parse command-line options
@@ -86,7 +85,8 @@ PCAP_RELATIVE_ADJUST="false"
PCAP_PROCESS_PRE_WAIT=120
PCAP_PROCESS_IDLE_SECONDS=180
PCAP_PROCESS_IDLE_MAX_SECONDS=3600
-while getopts 'vwronlb:m:x:s:d:' OPTION; do
+NETBOX_INIT_MAX_SECONDS=300
+while getopts 'vwronlb:m:i:x:s:d:' OPTION; do
case "$OPTION" in
v)
VERBOSE_FLAG="-v"
@@ -132,6 +132,12 @@ while getopts 'vwronlb:m:x:s:d:' OPTION; do
fi
;;
+ i)
+ if [[ $OPTARG =~ $NUMERIC_REGEX ]] ; then
+ NETBOX_INIT_MAX_SECONDS=$OPTARG
+ fi
+ ;;
+
d)
PCAP_DATE="$OPTARG"
;;
@@ -274,6 +280,20 @@ if [[ -f "$MALCOLM_DOCKER_COMPOSE" ]] && \
if [[ -n "$NETBOX_BACKUP_FILE" ]] && [[ -f "$NETBOX_BACKUP_FILE" ]]; then
# restore the netbox backup
[[ -n $VERBOSE_FLAG ]] && echo "Restoring NetBox database backup" >&2
+ # wait for NetBox to be ready with the initial startup before we go mucking around
+ CURRENT_TIME=$(date -u +%s)
+ FIRST_NETBOX_INIT_CHECK_TIME=$CURRENT_TIME
+ until docker-compose -f "$MALCOLM_FILE" logs netbox 2>/dev/null | grep -q "Unit configuration loaded successfully"; do
+ [[ -n $VERBOSE_FLAG ]] && echo "waiting for NetBox initialization to complete..." >&2
+ sleep 10
+ # if it's been more than the maximum wait time, bail
+ CURRENT_TIME=$(date -u +%s)
+ if (( ($CURRENT_TIME - $FIRST_NETBOX_INIT_CHECK_TIME) >= $NETBOX_INIT_MAX_SECONDS )); then
+ [[ -n $VERBOSE_FLAG ]] && echo "Max wait time expired waiting for netbox_init" >&2
+ break
+ fi
+ done
+ sleep 20
./scripts/netbox-restore $VERBOSE_FLAG -f "$MALCOLM_FILE" --netbox-restore "$NETBOX_BACKUP_FILE" || true
fi
diff --git a/scripts/install.py b/scripts/install.py
index e0f4109ec..3d2c878f9 100755
--- a/scripts/install.py
+++ b/scripts/install.py
@@ -785,11 +785,11 @@ def tweak_malcolm_runtime(
'Enter AES-256-CBC encryption password for downloaded preserved files (or leave blank for unencrypted)'
)
if fileCarveMode is not None:
- if InstallerYesOrNo('Scan extracted files with ClamAV?', default=False):
+ if InstallerYesOrNo('Scan extracted files with ClamAV?', default=True):
clamAvScan = True
- if InstallerYesOrNo('Scan extracted files with Yara?', default=False):
+ if InstallerYesOrNo('Scan extracted files with Yara?', default=True):
yaraScan = True
- if InstallerYesOrNo('Scan extracted PE files with Capa?', default=False):
+ if InstallerYesOrNo('Scan extracted PE files with Capa?', default=True):
capaScan = True
if InstallerYesOrNo('Lookup extracted file hashes with VirusTotal?', default=False):
while len(vtotApiKey) <= 1:
diff --git a/scripts/malcolm_appliance_packager.sh b/scripts/malcolm_appliance_packager.sh
index 84083b54c..53db37ca0 100755
--- a/scripts/malcolm_appliance_packager.sh
+++ b/scripts/malcolm_appliance_packager.sh
@@ -106,6 +106,8 @@ if mkdir "$DESTDIR"; then
pushd "./scripts" >/dev/null 2>&1
ln -s ./control.py auth_setup
ln -s ./control.py logs
+ ln -s ./control.py netbox-backup
+ ln -s ./control.py netbox-restore
ln -s ./control.py restart
ln -s ./control.py start
ln -s ./control.py status
diff --git a/scripts/third-party-logs/fluent-bit-setup.ps1 b/scripts/third-party-logs/fluent-bit-setup.ps1
index f635e44e7..0b8bee2dd 100644
--- a/scripts/third-party-logs/fluent-bit-setup.ps1
+++ b/scripts/third-party-logs/fluent-bit-setup.ps1
@@ -9,7 +9,7 @@
###############################################################################
$fluent_bit_version = '2.0'
-$fluent_bit_full_version = '2.0.6'
+$fluent_bit_full_version = '2.0.9'
###############################################################################
# select an item from a menu provided in an array
diff --git a/sensor-iso/arkime/Dockerfile b/sensor-iso/arkime/Dockerfile
index df5b76424..32b8f1bb8 100644
--- a/sensor-iso/arkime/Dockerfile
+++ b/sensor-iso/arkime/Dockerfile
@@ -6,7 +6,7 @@ LABEL maintainer="malcolm@inl.gov"
ENV DEBIAN_FRONTEND noninteractive
-ENV ARKIME_VERSION "4.1.0"
+ENV ARKIME_VERSION "4.2.0"
ENV ARKIME_DIR "/opt/arkime"
RUN sed -i "s/bullseye main/bullseye main contrib non-free/g" /etc/apt/sources.list && \
diff --git a/sensor-iso/beats/Dockerfile b/sensor-iso/beats/Dockerfile
index 648bf6f18..5126c1bb3 100644
--- a/sensor-iso/beats/Dockerfile
+++ b/sensor-iso/beats/Dockerfile
@@ -41,7 +41,7 @@ RUN set -x && \
go run bootstrap.go
ENV BEATS=filebeat
-ENV BEATS_VERSION=8.6.1
+ENV BEATS_VERSION=8.6.2
ADD ./build.sh /build.sh
RUN [ "chmod", "+x", "/build.sh" ]
diff --git a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot b/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot
index 6a821436d..af459e612 100755
--- a/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot
+++ b/sensor-iso/config/hooks/normal/0910-sensor-build.hook.chroot
@@ -13,14 +13,14 @@ GITHUB_API_CURL_ARGS+=( -H )
GITHUB_API_CURL_ARGS+=( "Accept: application/vnd.github.v3+json" )
[[ -n "$GITHUB_TOKEN" ]] && GITHUB_API_CURL_ARGS+=( -H ) && GITHUB_API_CURL_ARGS+=( "Authorization: token $GITHUB_TOKEN" )
-ZEEK_VER=5.0.6-0
+ZEEK_VER=5.0.7-0
ZEEK_LTS=true
ZEEK_DIR="/opt/zeek"
export PATH="${ZEEK_DIR}"/bin:$PATH
SURICATA_RULES_DIR="/etc/suricata/rules"
-BEATS_VER="8.6.1"
+BEATS_VER="8.6.2"
BEATS_OSS="-oss"
BEATS_DEB_URL_TEMPLATE_REPLACER="XXXXX"
BEATS_DEB_URL_TEMPLATE="https://artifacts.elastic.co/downloads/beats/$BEATS_DEB_URL_TEMPLATE_REPLACER/$BEATS_DEB_URL_TEMPLATE_REPLACER$BEATS_OSS-$BEATS_VER-amd64.deb"
diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/gtk-3.0/bookmarks b/sensor-iso/config/includes.chroot/etc/skel/.config/gtk-3.0/bookmarks
new file mode 100644
index 000000000..5b6b943a6
--- /dev/null
+++ b/sensor-iso/config/includes.chroot/etc/skel/.config/gtk-3.0/bookmarks
@@ -0,0 +1 @@
+file:///opt/sensor/sensor_ctl
\ No newline at end of file
diff --git a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-15/16346759461.desktop b/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-15/16346759461.desktop
index fcaabc15c..8210ef836 100644
--- a/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-15/16346759461.desktop
+++ b/sensor-iso/config/includes.chroot/etc/skel/.config/xfce4/panel/launcher-15/16346759461.desktop
@@ -3,7 +3,7 @@ Version=1.0
Name=Tilix
Comment=A tiling terminal for Gnome
Keywords=shell;prompt;command;commandline;cmd;
-Exec=tilix
+Exec=tilix --working-directory=/opt/sensor/sensor_ctl
Terminal=false
Type=Application
StartupNotify=true
@@ -15,11 +15,11 @@ X-XFCE-Source=file:///usr/share/applications/com.gexperts.Tilix.desktop
[Desktop Action new-window]
Name=New Window
-Exec=tilix --action=app-new-window
+Exec=tilix --action=app-new-window --working-directory=/opt/sensor/sensor_ctl
[Desktop Action new-session]
Name=New Session
-Exec=tilix --action=app-new-session
+Exec=tilix --action=app-new-session --working-directory=/opt/sensor/sensor_ctl
[Desktop Action preferences]
Name=Preferences
diff --git a/sensor-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek b/sensor-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek
index 29ef4f319..917c55401 100644
--- a/sensor-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek
+++ b/sensor-iso/config/includes.chroot/usr/local/etc/zeek/local.zeek
@@ -25,7 +25,6 @@ redef Broker::default_listen_address = "127.0.0.1";
redef ignore_checksums = T;
@load tuning/defaults
-@load misc/scan
@load frameworks/software/vulnerable
@load frameworks/software/version-changes
@load frameworks/software/windows-version-detection
diff --git a/sensor-iso/interface/requirements.txt b/sensor-iso/interface/requirements.txt
index 7742a82e3..ab47c4010 100644
--- a/sensor-iso/interface/requirements.txt
+++ b/sensor-iso/interface/requirements.txt
@@ -1,16 +1,16 @@
certifi==2022.12.7
-chardet==3.0.4
-click==7.1.2
-Flask==1.1.2
-Flask-Cors==3.0.9
-gunicorn==20.0.4
-idna==2.10
-itsdangerous==1.1.0
-Jinja2==2.11.3
-MarkupSafe==1.1.1
-psutil
-python-dotenv==0.14.0
-requests==2.26.0
-six==1.15.0
-urllib3==1.26.7
-Werkzeug==1.0.1
+chardet==5.1.0
+click==8.1.3
+Flask==2.2.3
+Flask-Cors==3.0.10
+gunicorn==20.1.0
+idna==3.4
+itsdangerous==2.1.2
+Jinja2==3.1.2
+MarkupSafe==2.1.2
+psutil==5.9.4
+python-dotenv==0.21.1
+requests==2.28.2
+six==1.16.0
+urllib3==1.26.14
+Werkzeug==2.2.3
diff --git a/shared/bin/pcap_watcher.py b/shared/bin/pcap_watcher.py
index a55708785..34846d2dd 100755
--- a/shared/bin/pcap_watcher.py
+++ b/shared/bin/pcap_watcher.py
@@ -26,8 +26,9 @@
from pcap_utils import *
from collections import defaultdict
-import opensearchpy
-import opensearch_dsl
+from opensearchpy import OpenSearch, Search
+from opensearchpy.exceptions import ConnectionError, ConnectionTimeout
+from urllib3.exceptions import NewConnectionError
###################################################################################################
MINIMUM_CHECKED_FILE_SIZE_DEFAULT = 24
@@ -44,59 +45,85 @@
verboseDebug = False
pdbFlagged = False
args = None
-opensearchDslHttpAuth = None
+opensearchHttpAuth = None
scriptName = os.path.basename(__file__)
scriptPath = os.path.dirname(os.path.realpath(__file__))
origPath = os.getcwd()
shuttingDown = False
DEFAULT_NODE_NAME = os.getenv('PCAP_NODE_NAME', 'malcolm')
+
###################################################################################################
# watch files written to and moved to this directory
class EventWatcher(pyinotify.ProcessEvent):
-
# notify on files written in-place then closed (IN_CLOSE_WRITE), and moved into this directory (IN_MOVED_TO)
_methods = ["IN_CLOSE_WRITE", "IN_MOVED_TO"]
def __init__(self):
global args
- global opensearchDslHttpAuth
+ global opensearchHttpAuth
global debug
global verboseDebug
super().__init__()
self.useOpenSearch = False
+ self.openSearchClient = None
# if we're going to be querying OpenSearch for past PCAP file status, connect now
if args.opensearchUrl is not None:
-
connected = False
healthy = False
# create the connection to OpenSearch
while (not connected) and (not shuttingDown):
try:
- if debug:
- eprint(f"{scriptName}:\tconnecting to OpenSearch {args.opensearchUrl}...")
- opensearch_dsl.connections.create_connection(
- hosts=[args.opensearchUrl],
- http_auth=opensearchDslHttpAuth,
- verify_certs=args.opensearchSslVerify,
- ssl_assert_hostname=False,
- ssl_show_warn=False,
- )
- if verboseDebug:
- eprint(f"{scriptName}:\t{opensearch_dsl.connections.get_connection().cluster.health()}")
- connected = opensearch_dsl.connections.get_connection() is not None
+ try:
+ if debug:
+ eprint(f"{scriptName}:\tconnecting to OpenSearch {args.opensearchUrl}...")
- except opensearchpy.exceptions.ConnectionError as connError:
+ self.openSearchClient = OpenSearch(
+ hosts=[args.opensearchUrl],
+ http_auth=opensearchHttpAuth,
+ verify_certs=args.opensearchSslVerify,
+ ssl_assert_hostname=False,
+ ssl_show_warn=False,
+ request_timeout=1,
+ )
+
+ if verboseDebug:
+ eprint(f"{scriptName}:\t{self.openSearchClient.cluster.health()}")
+
+ self.openSearchClient.cluster.health(
+ wait_for_status='red',
+ request_timeout=1,
+ )
+
+ if verboseDebug:
+ eprint(f"{scriptName}:\t{self.openSearchClient.cluster.health()}")
+
+ connected = self.openSearchClient is not None
+ if not connected:
+ time.sleep(1)
+
+ except (
+ ConnectionError,
+ ConnectionTimeout,
+ ConnectionRefusedError,
+ NewConnectionError,
+ ) as connError:
+ if debug:
+ eprint(f"{scriptName}:\tOpenSearch connection error: {connError}")
+
+ except Exception as genericError:
if debug:
- eprint(f"{scriptName}:\tOpenSearch connection error: {connError}")
+ eprint(f"{scriptName}:\tUnexpected exception while connecting to OpenSearch: {genericError}")
if (not connected) and args.opensearchWaitForHealth:
time.sleep(1)
else:
+ if args.opensearchWaitForHealth:
+ time.sleep(1)
break
# if requested, wait for at least "yellow" health in the cluster for the "files" index
@@ -104,14 +131,20 @@ def __init__(self):
try:
if debug:
eprint(f"{scriptName}:\twaiting for OpenSearch to be healthy")
- opensearch_dsl.connections.get_connection().cluster.health(
- index=ARKIME_FILES_INDEX, wait_for_status='yellow'
+ self.openSearchClient.cluster.health(
+ index=ARKIME_FILES_INDEX,
+ wait_for_status='yellow',
)
if verboseDebug:
- eprint(f"{scriptName}:\t{opensearch_dsl.connections.get_connection().cluster.health()}")
+ eprint(f"{scriptName}:\t{self.openSearchClient.cluster.health()}")
healthy = True
- except opensearchpy.exceptions.ConnectionTimeout as connError:
+ except (
+ ConnectionError,
+ ConnectionTimeout,
+ ConnectionRefusedError,
+ NewConnectionError,
+ ) as connError:
if verboseDebug:
eprint(f"{scriptName}:\tOpenSearch health check: {connError}")
@@ -140,10 +173,8 @@ def __init__(self):
###################################################################################################
# set up event processor to append processed events from to the event queue
def event_process_generator(cls, method):
-
# actual method called when we are notified of a file
def _method_name(self, event):
-
global args
global debug
global verboseDebug
@@ -153,7 +184,6 @@ def _method_name(self, event):
# the entity must be a regular PCAP file and actually exist
if (not event.dir) and os.path.isfile(event.pathname):
-
# get the file magic description and mime type
fileMime = magic.from_file(event.pathname, mime=True)
fileType = magic.from_file(event.pathname)
@@ -163,14 +193,13 @@ def _method_name(self, event):
if (args.minBytes <= fileSize <= args.maxBytes) and (
(fileMime in PCAP_MIME_TYPES) or ('pcap-ng' in fileType)
):
-
relativePath = remove_prefix(event.pathname, os.path.join(args.baseDir, ''))
# check with Arkime's files index in OpenSearch and make sure it's not a duplicate
fileIsDuplicate = False
if self.useOpenSearch:
s = (
- opensearch_dsl.Search(index=ARKIME_FILES_INDEX)
+ Search(using=self.openSearchClient, index=ARKIME_FILES_INDEX)
.filter("term", node=args.nodeName)
.query("wildcard", name=f"*{os.path.sep}{relativePath}")
)
@@ -243,7 +272,7 @@ def debug_toggle_handler(signum, frame):
# main
def main():
global args
- global opensearchDslHttpAuth
+ global opensearchHttpAuth
global debug
global verboseDebug
global debugToggled
@@ -423,7 +452,7 @@ def main():
args.opensearchUrl = 'http://opensearch:9200'
elif 'url' in opensearchCreds:
args.opensearchUrl = opensearchCreds['url']
- opensearchDslHttpAuth = (
+ opensearchHttpAuth = (
f"{opensearchCreds['user']}:{opensearchCreds['password']}" if opensearchCreds['user'] is not None else None
)
diff --git a/shared/bin/set-malcolm-gtk-bookmark.sh b/shared/bin/set-malcolm-gtk-bookmark.sh
new file mode 100755
index 000000000..f324b94dc
--- /dev/null
+++ b/shared/bin/set-malcolm-gtk-bookmark.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# Copyright (c) 2023 Battelle Energy Alliance, LLC. All rights reserved.
+
+if ! grep -q Malcolm$ "$HOME"/.config/gtk-3.0/bookmarks && [[ -d "$HOME"/Malcolm ]]; then
+ mkdir -p "$HOME"/.config/gtk-3.0/
+ echo -e "\nfile://$HOME/Malcolm" >> "$HOME"/.config/gtk-3.0/bookmarks
+fi
diff --git a/shared/bin/zeek_install_plugins.sh b/shared/bin/zeek_install_plugins.sh
index bf7523163..f3840098c 100755
--- a/shared/bin/zeek_install_plugins.sh
+++ b/shared/bin/zeek_install_plugins.sh
@@ -108,6 +108,8 @@ ZKG_GITHUB_URLS=(
"https://github.com/corelight/zerologon"
"https://github.com/cybera/zeek-sniffpass"
"https://github.com/mmguero-dev/bzar"
+ "https://github.com/ncsa/bro-is-darknet"
+ "https://github.com/ncsa/bro-simple-scan"
"https://github.com/precurse/zeek-httpattacks"
"https://github.com/salesforce/GQUIC_Protocol_Analyzer"
"https://github.com/salesforce/hassh"
diff --git a/zeek/config/local.zeek b/zeek/config/local.zeek
index 709cbcf2a..42df7ff2e 100644
--- a/zeek/config/local.zeek
+++ b/zeek/config/local.zeek
@@ -25,7 +25,6 @@ redef Broker::default_listen_address = "127.0.0.1";
redef ignore_checksums = T;
@load tuning/defaults
-@load misc/scan
@load frameworks/software/vulnerable
@load frameworks/software/version-changes
@load frameworks/software/windows-version-detection