From 2069ce45f46f3f1f6ac03b4c07e37674efef9dda Mon Sep 17 00:00:00 2001 From: Karthikeyan Singaravelan Date: Sun, 14 Apr 2024 18:27:28 +0530 Subject: [PATCH 1/3] Add color to logs in UI based on given keywords for error and warning. --- airflow/config_templates/config.yml | 16 ++++++++++ .../js/dag/details/taskInstance/Logs/utils.ts | 9 ++++++ airflow/www/static/js/utils/index.ts | 29 +++++++++++++++++++ airflow/www/templates/airflow/grid.html | 2 ++ airflow/www/views.py | 5 ++++ 5 files changed, 61 insertions(+) diff --git a/airflow/config_templates/config.yml b/airflow/config_templates/config.yml index 60a9d3c7dc6b7..995f9f0393561 100644 --- a/airflow/config_templates/config.yml +++ b/airflow/config_templates/config.yml @@ -979,6 +979,22 @@ logging: type: boolean example: ~ default: "True" + color_log_error_keywords: + description: | + A comma separated list of keywords related to errors whose presence should display the line in red + color in UI + version_added: 2.10.0 + type: string + example: ~ + default: "error,exception" + color_log_warning_keywords: + description: | + A comma separated list of keywords related to warning whose presence should display the line in yellow + color in UI + version_added: 2.10.0 + type: string + example: ~ + default: "warn" metrics: description: | StatsD (https://github.com/etsy/statsd) integration settings. diff --git a/airflow/www/static/js/dag/details/taskInstance/Logs/utils.ts b/airflow/www/static/js/dag/details/taskInstance/Logs/utils.ts index 1d41b3dccd5a1..c5543284bc694 100644 --- a/airflow/www/static/js/dag/details/taskInstance/Logs/utils.ts +++ b/airflow/www/static/js/dag/details/taskInstance/Logs/utils.ts @@ -20,6 +20,7 @@ /* global moment */ import { AnsiUp } from "ansi_up"; +import { getMetaValue, addColorKeyword } from "src/utils"; import { defaultFormatWithTZ } from "src/datetime_utils"; export enum LogLevel { @@ -38,6 +39,13 @@ export const logLevelColorMapping = { [LogLevel.CRITICAL]: "red.400", }; +const errorKeywords = getMetaValue("color_log_error_keywords") + .split(",") + .map((keyword) => keyword.toLowerCase()); +const warningKeywords = getMetaValue("color_log_warning_keywords") + .split(",") + .map((keyword) => keyword.toLowerCase()); + export const parseLogs = ( data: string | undefined, timezone: string | null, @@ -112,6 +120,7 @@ export const parseLogs = ( line.includes(fileSourceFilter) ) ) { + parsedLine = addColorKeyword(parsedLine, errorKeywords, warningKeywords); // for lines with color convert to nice HTML const coloredLine = ansiUp.ansi_to_html(parsedLine); diff --git a/airflow/www/static/js/utils/index.ts b/airflow/www/static/js/utils/index.ts index 37dc5cb022c88..f6a1bb5e4d835 100644 --- a/airflow/www/static/js/utils/index.ts +++ b/airflow/www/static/js/utils/index.ts @@ -185,6 +185,34 @@ const toSentenceCase = (camelCase: string): string => { return ""; }; +const addColorKeyword = ( + parsedLine: string, + errorKeywords: string[], + warningKeywords: string[] +): string => { + const lowerParsedLine = parsedLine.toLowerCase(); + const containsError = errorKeywords.some((keyword) => + lowerParsedLine.includes(keyword) + ); + const bold = (line: string) => `\x1b[1m${line}\x1b[0m`; + const red = (line: string) => `\x1b[31m${line}\x1b[39m`; + const yellow = (line: string) => `\x1b[33m${line}\x1b[39m`; + + if (containsError) { + return bold(red(parsedLine)); + } + + const containsWarning = warningKeywords.some((keyword) => + lowerParsedLine.includes(keyword) + ); + + if (containsWarning) { + return bold(yellow(parsedLine)); + } + + return parsedLine; +}; + export { hoverDelay, finalStatesMap, @@ -197,4 +225,5 @@ export { getStatusBackgroundColor, useOffsetTop, toSentenceCase, + addColorKeyword, }; diff --git a/airflow/www/templates/airflow/grid.html b/airflow/www/templates/airflow/grid.html index 3c7bb236e178e..e90389212b14f 100644 --- a/airflow/www/templates/airflow/grid.html +++ b/airflow/www/templates/airflow/grid.html @@ -28,6 +28,8 @@ + + {% endblock %} {% block content %} diff --git a/airflow/www/views.py b/airflow/www/views.py index 328312658bd18..f26a5337ff548 100644 --- a/airflow/www/views.py +++ b/airflow/www/views.py @@ -2786,6 +2786,9 @@ def legacy_tree(self): @provide_session def grid(self, dag_id: str, session: Session = NEW_SESSION): """Get Dag's grid view.""" + color_log_error_keywords = conf.get("logging", "color_log_error_keywords", fallback="") + color_log_warning_keywords = conf.get("logging", "color_log_warning_keywords", fallback="") + dag = get_airflow_app().dag_bag.get_dag(dag_id, session=session) dag_model = DagModel.get_dagmodel(dag_id, session=session) if not dag: @@ -2843,6 +2846,8 @@ def grid(self, dag_id: str, session: Session = NEW_SESSION): ), included_events_raw=included_events_raw, excluded_events_raw=excluded_events_raw, + color_log_error_keywords=color_log_error_keywords, + color_log_warning_keywords=color_log_warning_keywords, ) @expose("/calendar") From d4bda92ed119eead04efac44be0022962ef00dfa Mon Sep 17 00:00:00 2001 From: Karthikeyan Singaravelan Date: Sun, 14 Apr 2024 18:33:46 +0530 Subject: [PATCH 2/3] Filter empty keywords when color_log_error_keywords is unset. --- airflow/www/static/js/dag/details/taskInstance/Logs/utils.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/airflow/www/static/js/dag/details/taskInstance/Logs/utils.ts b/airflow/www/static/js/dag/details/taskInstance/Logs/utils.ts index c5543284bc694..2503e11de602d 100644 --- a/airflow/www/static/js/dag/details/taskInstance/Logs/utils.ts +++ b/airflow/www/static/js/dag/details/taskInstance/Logs/utils.ts @@ -41,9 +41,11 @@ export const logLevelColorMapping = { const errorKeywords = getMetaValue("color_log_error_keywords") .split(",") + .filter((keyword) => keyword.length > 0) .map((keyword) => keyword.toLowerCase()); const warningKeywords = getMetaValue("color_log_warning_keywords") .split(",") + .filter((keyword) => keyword.length > 0) .map((keyword) => keyword.toLowerCase()); export const parseLogs = ( From c56f50c7dd85eed2e4d872415aa17c7e43d5816b Mon Sep 17 00:00:00 2001 From: Karthikeyan Singaravelan Date: Fri, 19 Apr 2024 21:59:22 +0530 Subject: [PATCH 3/3] Add tests and rename function. --- .../js/dag/details/taskInstance/Logs/utils.ts | 8 ++- airflow/www/static/js/utils/index.test.ts | 49 ++++++++++++++++++- airflow/www/static/js/utils/index.ts | 14 +++--- 3 files changed, 61 insertions(+), 10 deletions(-) diff --git a/airflow/www/static/js/dag/details/taskInstance/Logs/utils.ts b/airflow/www/static/js/dag/details/taskInstance/Logs/utils.ts index 2503e11de602d..0de5676916bbb 100644 --- a/airflow/www/static/js/dag/details/taskInstance/Logs/utils.ts +++ b/airflow/www/static/js/dag/details/taskInstance/Logs/utils.ts @@ -20,7 +20,7 @@ /* global moment */ import { AnsiUp } from "ansi_up"; -import { getMetaValue, addColorKeyword } from "src/utils"; +import { getMetaValue, highlightByKeywords } from "src/utils"; import { defaultFormatWithTZ } from "src/datetime_utils"; export enum LogLevel { @@ -122,7 +122,11 @@ export const parseLogs = ( line.includes(fileSourceFilter) ) ) { - parsedLine = addColorKeyword(parsedLine, errorKeywords, warningKeywords); + parsedLine = highlightByKeywords( + parsedLine, + errorKeywords, + warningKeywords + ); // for lines with color convert to nice HTML const coloredLine = ansiUp.ansi_to_html(parsedLine); diff --git a/airflow/www/static/js/utils/index.test.ts b/airflow/www/static/js/utils/index.test.ts index 4c8be82cfd470..a76ee62c867dd 100644 --- a/airflow/www/static/js/utils/index.test.ts +++ b/airflow/www/static/js/utils/index.test.ts @@ -19,7 +19,12 @@ import { isEmpty } from "lodash"; import type { DagRun } from "src/types"; -import { getDagRunLabel, getTask, getTaskSummary } from "."; +import { + getDagRunLabel, + getTask, + getTaskSummary, + highlightByKeywords, +} from "."; const sampleTasks = { id: null, @@ -148,3 +153,45 @@ describe("Test getDagRunLabel", () => { expect(runLabel).toBe(dagRun.executionDate); }); }); + +describe("Test highlightByKeywords", () => { + test("Highlight error line by red color", async () => { + const originalLine = "line with Error"; + const expected = `\x1b[1m\x1b[31mline with Error\x1b[39m\x1b[0m`; + const highlightedLine = highlightByKeywords( + originalLine, + ["error"], + ["warn"] + ); + expect(highlightedLine).toBe(expected); + }); + test("Highlight warning line by yellow color", async () => { + const originalLine = "line with Warning"; + const expected = `\x1b[1m\x1b[33mline with Warning\x1b[39m\x1b[0m`; + const highlightedLine = highlightByKeywords( + originalLine, + ["error"], + ["warn"] + ); + expect(highlightedLine).toBe(expected); + }); + test("Highlight line by red color when line has both error and warning", async () => { + const originalLine = "line with error Warning"; + const expected = `\x1b[1m\x1b[31mline with error Warning\x1b[39m\x1b[0m`; + const highlightedLine = highlightByKeywords( + originalLine, + ["error"], + ["warn"] + ); + expect(highlightedLine).toBe(expected); + }); + test("No highlight", async () => { + const originalLine = "sample line"; + const highlightedLine = highlightByKeywords( + originalLine, + ["error"], + ["warn"] + ); + expect(highlightedLine).toBe(originalLine); + }); +}); diff --git a/airflow/www/static/js/utils/index.ts b/airflow/www/static/js/utils/index.ts index f6a1bb5e4d835..9eb8af9638c42 100644 --- a/airflow/www/static/js/utils/index.ts +++ b/airflow/www/static/js/utils/index.ts @@ -185,21 +185,21 @@ const toSentenceCase = (camelCase: string): string => { return ""; }; -const addColorKeyword = ( +const highlightByKeywords = ( parsedLine: string, errorKeywords: string[], warningKeywords: string[] ): string => { const lowerParsedLine = parsedLine.toLowerCase(); + const red = (line: string) => `\x1b[1m\x1b[31m${line}\x1b[39m\x1b[0m`; + const yellow = (line: string) => `\x1b[1m\x1b[33m${line}\x1b[39m\x1b[0m`; + const containsError = errorKeywords.some((keyword) => lowerParsedLine.includes(keyword) ); - const bold = (line: string) => `\x1b[1m${line}\x1b[0m`; - const red = (line: string) => `\x1b[31m${line}\x1b[39m`; - const yellow = (line: string) => `\x1b[33m${line}\x1b[39m`; if (containsError) { - return bold(red(parsedLine)); + return red(parsedLine); } const containsWarning = warningKeywords.some((keyword) => @@ -207,7 +207,7 @@ const addColorKeyword = ( ); if (containsWarning) { - return bold(yellow(parsedLine)); + return yellow(parsedLine); } return parsedLine; @@ -225,5 +225,5 @@ export { getStatusBackgroundColor, useOffsetTop, toSentenceCase, - addColorKeyword, + highlightByKeywords, };