FIX: Reduce logs panel noise and make execution messages readable (OR-1457)

backend/visitran/events/printer.py

@@ -86,13 +86,13 @@ def parse_and_fire_reports() -> None:
                 status=iterate_result.status,
             )
         )
-        if iterate_result.end_status == str(ExecStatus.Success):
+        if iterate_result.end_status == ExecStatus.OK.value:
             pass_count += 1
-        if iterate_result.end_status == str(ExecStatus.Warn):
+        elif iterate_result.end_status == ExecStatus.Warn.value:
             warn_count += 1
-        if iterate_result.end_status == str(ExecStatus.Error):
+        elif iterate_result.end_status == ExecStatus.Fail.value:
             error_count += 1
-        if iterate_result.end_status == str(ExecStatus.Skipped):
+        elif iterate_result.end_status == ExecStatus.Skipped.value:
             skip_count += 1
 
     functions.fire_event(

backend/visitran/events/types.py

@@ -469,7 +469,7 @@ def message(self) -> str:
 
 
 @dataclass
-class UsingCachedObject(InfoLevel, proto_type.UsingCachedObject):
+class UsingCachedObject(DebugLevel, proto_type.UsingCachedObject):
     def code(self) -> str:
         return "Z009"
 

backend/visitran/visitran.py

@@ -260,7 +260,7 @@ def execute_graph(self) -> None:
                         destination_table_obj=(
                             str(node.destination_table_obj) if hasattr(node, "destination_table_obj") else ""
                         ),
-                        materialization=str(node.materialization),
+                        materialization=node.materialization.value if hasattr(node.materialization, "value") else str(node.materialization),
                         select_statement=(str(node.select_statement) if hasattr(node, "select_statement") else ""),
                         source_schema_name=node.source_schema_name,
                         source_table_name=node.source_table_name,
@@ -276,8 +276,8 @@ def execute_graph(self) -> None:
                     ending_time=datetime.datetime.now(),
                     failures=False,
                     info_message=f"Running {node_name}",
-                    status=str(ExecStatus.Success),
-                    end_status=str(ExecStatus.OK),
+                    status=ExecStatus.Success.value,
+                    end_status=ExecStatus.OK.value,
                 )
                 sequence_number += 1
                 BASE_RESULT.append(base_result)
@@ -291,8 +291,8 @@ def execute_graph(self) -> None:
                     node_name=str(node_name),
                     sequence_num=sequence_number,
                     ending_time=datetime.datetime.now(),
-                    status=str(ExecStatus.Error),
-                    end_status=str(ExecStatus.Fail),
+                    status=ExecStatus.Error.value,
+                    end_status=ExecStatus.Fail.value,
                     info_message=f"Error occurred while running {node_name}",
                     failures=True,
                 )
@@ -438,7 +438,7 @@ def search_n_run_models(self, model_name: str = None, model_names: list = None)
             )
 
             for cls in cls_set:
-                fire_event(ProcessingModel(cls=str(cls)))
+                fire_event(ProcessingModel(cls=getattr(cls, "__name__", "") or file_name))
                 # process each model only once
                 # they might be imported in several places!
 
@@ -550,9 +550,9 @@ def search_n_run_tests(self) -> None:
                     sequence_num=test_files.index(tf),
                     failures=True,
                     ending_time=datetime.datetime.now(),
-                    status=str(ExecStatus.Error),
+                    status=ExecStatus.Error.value,
                     info_message=f"Error occured in {tf} execution",
-                    end_status=str(ExecStatus.Fail),
+                    end_status=ExecStatus.Fail.value,
                 )
                 BASE_RESULT.append(base_result)
                 parse_and_fire_reports()
@@ -611,8 +611,8 @@ def _run_tests(self, tf: str, tst_cls: Any, tst_obj: Any, test_methods: list[str
                     info_message=f"Test assertion error: \
                     {repr(err)} in {test_func.__name__}",
                     ending_time=datetime.datetime.now(),
-                    status=str(ExecStatus.Error),
-                    end_status=str(ExecStatus.Fail),
+                    status=ExecStatus.Error.value,
+                    end_status=ExecStatus.Fail.value,
                     sequence_num=sequence_num,
                 )
                 BASE_RESULT.append(base_result)
@@ -624,8 +624,8 @@ def _run_tests(self, tf: str, tst_cls: Any, tst_obj: Any, test_methods: list[str
                 ending_time=datetime.datetime.now(),
                 failures=False,
                 info_message=f"Running {tf}",
-                status=str(ExecStatus.Success),
-                end_status=str(ExecStatus.OK),
+                status=ExecStatus.Success.value,
+                end_status=ExecStatus.OK.value,
                 sequence_num=sequence_num,
             )
             BASE_RESULT.append(base_result)
@@ -727,14 +727,14 @@ def validate_and_run_seed(self, seed_file, session_id: str) -> dict[str, str]:
                 seed_result = SeedResult(
                     schema_name=schema,
                     seed_path=file_name,
-                    status=str(ExecStatus.START),
+                    status=ExecStatus.START.value,
                 )
                 SEED_RESULT.append(seed_result)
                 seed_obj: BaseSeed = self.db_adapter.run_seeds(schema=schema, abs_path=file_path)
                 seed_result = SeedResult(
                     schema_name=schema,
                     seed_path=file_name,
-                    status=str(ExecStatus.COMPLETED),
+                    status=ExecStatus.COMPLETED.value,
                 )
                 SEED_RESULT.append(seed_result)
                 parse_and_fire_seed_report()
@@ -789,14 +789,14 @@ def run_snapshot(self) -> None:
                     snapshot_result = SnapshotResult(
                         source_table=obj.source_table_name,
                         unique_key=obj.unique_key,
-                        status=str(ExecStatus.START),
+                        status=ExecStatus.START.value,
                     )
                     SNAPSHOT_RESULT.append(snapshot_result)
                     self.db_adapter.run_scd(visitran_snapshot=obj)
                     snapshot_result = SnapshotResult(
                         source_table=obj.source_table_name,
                         unique_key=obj.unique_key,
-                        status=str(ExecStatus.COMPLETED),
+                        status=ExecStatus.COMPLETED.value,
                     )
                     SNAPSHOT_RESULT.append(snapshot_result)
                     parse_and_fire_snapshot_report()

frontend/src/ide/editor/no-code-model/no-code-model.jsx

@@ -4,13 +4,15 @@ import {
   Input,
   Modal,
   Pagination,
+  Select,
   Space,
   Table,
   Tabs,
+  theme,
   Tooltip,
   Typography,
 } from "antd";
-import { useEffect, useRef, useState } from "react";
+import { useEffect, useMemo, useRef, useState } from "react";
 import { Resizable } from "react-resizable";
 import Cookies from "js-cookie";
 import AnsiToHtml from "ansi-to-html";
@@ -146,7 +148,17 @@ ResizableTitle.propTypes = {
   width: PropTypes.number,
 };
 
+const LOG_LEVEL_RANK = { debug: 0, info: 1, warn: 2, warning: 2, error: 3 };
+const LOG_LEVEL_COLOR = {
+  debug: "#8c8c8c",
+  info: undefined,
+  warn: "#d48806",
+  warning: "#d48806",
+  error: "#cf1322",
+};
+
 function NoCodeModel({ nodeData }) {
+  const { token } = theme.useToken();
   const axios = useAxiosPrivate();
   const csrfToken = Cookies.get("csrftoken");
   const sessionId = Cookies.get("sessionid");
@@ -206,7 +218,9 @@ function NoCodeModel({ nodeData }) {
   const [seqEdges, setSeqEdges, onSeqEdgesChange] = useEdgesState();
   const [reactFlowInstance, setReactFlowInstance] = useState(null);
   const [logsInfo, setLogsInfo] = useState([]);
-  // const logsInfo = [];
+  const [logsLevel, setLogsLevel] = useState(
+    () => localStorage.getItem("visitran.logsLevel") || "info"
+  );
   const [reveal, setReveal] = useState(false);
   const [seqOrder, setSeqOrder] = useState({});
   const [specRevert, setSpecRevert] = useState(false);
@@ -366,6 +380,19 @@ function NoCodeModel({ nodeData }) {
       ALLOWED_ATTR: ["style"],
     });
 
+  const filteredLogs = useMemo(
+    () =>
+      logsInfo.filter(
+        (entry) =>
+          (LOG_LEVEL_RANK[entry.level] ?? 1) >= (LOG_LEVEL_RANK[logsLevel] ?? 1)
+      ),
+    [logsInfo, logsLevel]
+  );
+  const handleLogsLevelChange = (value) => {
+    setLogsLevel(value);
+    localStorage.setItem("visitran.logsLevel", value);
+  };
+
   const hideGenAIAndTimeTravelTabs = true;
   const BOTTOM_TABS = [
     {
@@ -643,21 +670,43 @@ function NoCodeModel({ nodeData }) {
       ),
       key: "logs",
       children: (
-        <div
-          className="logsSection"
-          style={{
-            height: `calc(${bottomSectionRef.current.height} - 70px)`,
-          }}
-        >
-          {logsInfo?.map((el, index) => {
-            return (
+        <>
+          <div
+            style={{
+              display: "flex",
+              justifyContent: "flex-end",
+              padding: "4px 8px",
+              borderBottom: `1px solid ${token.colorBorderSecondary}`,
+            }}
+          >
+            <Select
+              size="small"
+              value={logsLevel}
+              onChange={handleLogsLevelChange}
+              style={{ width: 140 }}
+              options={[
+                { value: "debug", label: "All logs" },
+                { value: "info", label: "Info & above" },
+                { value: "warn", label: "Warnings & errors" },
+                { value: "error", label: "Errors only" },
+              ]}
+            />
+          </div>
+          <div
+            className="logsSection"
+            style={{
+              height: `calc(${bottomSectionRef.current.height} - 100px)`,
+            }}
+          >
+            {filteredLogs.map((el, index) => (
               <div
                 key={index}
-                dangerouslySetInnerHTML={{ __html: parseLog(el) }}
-              ></div>
-            );
-          })}
-        </div>
+                style={{ color: LOG_LEVEL_COLOR[el.level] }}
+                dangerouslySetInnerHTML={{ __html: parseLog(el.message) }}
+              />
+            ))}
+          </div>
+        </>
       ),
     },
   ].filter((tab) => {
@@ -1762,28 +1811,32 @@ function NoCodeModel({ nodeData }) {
     };
     const newSocket = io(getBaseUrl(), body);
 
+    let socketSessionId = null;
     newSocket.on("connect", () => {
       // Listen for the session ID sent by the server
       newSocket.on("session_id", (data) => {
-        const sessionId = data.session_id;
+        socketSessionId = data.session_id;
         // Listen for messages in the specific room (session ID)
-        newSocket.on(`logs:${sessionId}`, (data) => {
-          const temp = data?.data?.message;
+        newSocket.on(`logs:${socketSessionId}`, (data) => {
+          const message = data?.data?.message;
+          const level = (data?.data?.level || "info").toLowerCase();
+          if (!message) return;
           const doc = document.getElementsByClassName("logsSection");
           if (doc[0]) {
             setTimeout(() => {
               doc[0].scrollTop = doc[0].scrollHeight;
             }, 800);
           }
-          setLogsInfo((old) => {
-            return [...old, temp];
-          });
+          setLogsInfo((old) => [...old, { level, message }]);
         });
       });
     });
     return () => {
       // unsubscribe to the channel to stop listening the socket messages for the logId
-      newSocket.off(`logs:${sessionId}`);
+      if (socketSessionId) {
+        newSocket.off(`logs:${socketSessionId}`);
+      }
+      newSocket.disconnect();
     };
   }, [sessionId]);