diff --git a/README.md b/README.md index fa47341..ef75438 100644 --- a/README.md +++ b/README.md @@ -1 +1,38 @@ -# CogniOS +# CogniOS - Archietecture diagram +```mermaid +graph TD + subgraph System Environment + proc[/proc Filesystem/] + psutil[psutil wrapper] + end + + subgraph CogniOS Core + DB[(Central SQLite DB)] + TC[Module 1: Telemetry Collector\nBackground Daemon] + + TC -->|Writes real-time data| DB + proc --> TC + psutil --> TC + end + + subgraph Intelligent Modules + OD[Module 2: OS Doctor\nIsolation Forest] + FO[Module 3: FocusOS\nCNN Classifier] + BB[Module 4: BlackBox\nRolling Window / Replay] + RE[Module 5: Research Engine\nRL & Simulators] + end + + DB <-->|Reads Data / Writes Alerts| OD + DB <-->|Reads Heatmap / Writes Configs| FO + DB <-->|Reads Trace / Writes Narrative| BB + DB <-->|Reads Traces for Simulation| RE + + subgraph User Interface + DASH[Streamlit Dashboard\nUnified GUI] + end + + OD --> DASH + FO --> DASH + BB --> DASH + RE --> DASH +``` diff --git a/cognios_telemetry.db b/cognios_telemetry.db new file mode 100644 index 0000000..b3ec599 Binary files /dev/null and b/cognios_telemetry.db differ diff --git a/collectors/layer1_system.py b/collectors/layer1_system.py index 291e147..1760bd0 100644 --- a/collectors/layer1_system.py +++ b/collectors/layer1_system.py @@ -1 +1,223 @@ -"""Layer 1 system telemetry collection.""" +import psutil +import time +from datetime import datetime,timezone +import sys +import os +#adding path to locate the utils +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import utils +from utils.helpers import rate_mb_s + +# a dictionary to store previous values +_last = { + "time": None, + "disk_read_bytes": None, + "disk_write_bytes": None, + "net_bytes_sent": None, + "net_bytes_recv": None, +} + + +# collects metriics +def collect_layer1_metrics(): + now = time.time() + timestamp = datetime.now(timezone.utc).isoformat() + #process + try: + procs = {} + for p in psutil.process_iter(["name", "memory_percent"]): + try: + p.cpu_percent() + procs[p.pid] = p + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + time.sleep(0.1) + process_data = [] + for pid, p in procs.items(): + try: + cpu = round(p.cpu_percent(), 2) + mem = round(p.info.get("memory_percent") or 0.0, 2) + if cpu > 0.5 or mem > 0.5: + process_data.append((p.info.get("name"), cpu, mem)) + except (psutil.NoSuchProcess, psutil.AccessDenied): + pass + except Exception: + process_data = [] + # CPU Metrics + cpu_usage_percent = psutil.cpu_percent(interval=None) + cpu_times = psutil.cpu_times() + cpu_user_time = cpu_times.user + cpu_system_time = cpu_times.system + cpu_idle_time = cpu_times.idle + cpu_iowait_time = getattr(cpu_times, 'iowait', None) # iowait for Linux only + cpu_busy_time=cpu_user_time + cpu_system_time + + # freq , ctx_switches are not available on all systems + try: + freq=psutil.cpu_freq() + cpu_current_freq=freq.current if freq else None + except Exception: + cpu_current_freq=None + + try: + cpu_ctx_switches=psutil.cpu_stats().ctx_switches + except Exception: + cpu_ctx_switches=None + + # Memory Metrics + + vmem=psutil.virtual_memory() + memory_percent=vmem.percent + memory_used=vmem.used + memory_available=vmem.available + memory_cached=getattr(vmem, 'cached', None) #cached and buffers not available in the mac and windows + memory_buffers=getattr(vmem, 'buffers', None) # None- like value nahi mili aur 0- ek valid value hai + + swap=psutil.swap_memory() + swap_percent=swap.percent + swap_sin=swap.sin + swap_sout=swap.sout + + # Disk Metrics + + # same reason, in windows "/" is not valid + try: + disk_usage=psutil.disk_usage('/').percent + except Exception: + disk_usage=None + + try: + disk_io = psutil.disk_io_counters() + except Exception: + disk_io=None + + disk_read=None + disk_write=None + disk_read_time=None + disk_write_time=None + + if disk_io is not None: + elapsed_time = (now - _last["time"]) if _last["time"] else 0 + disk_read = rate_mb_s(disk_io.read_bytes, _last["disk_read_bytes"], elapsed_time) + disk_write = rate_mb_s(disk_io.write_bytes, _last["disk_write_bytes"], elapsed_time) + disk_read_time = getattr(disk_io, 'read_time', None) + disk_write_time = getattr(disk_io, 'write_time', None) + + _last["disk_read_bytes"] = disk_io.read_bytes + _last["disk_write_bytes"] = disk_io.write_bytes + + + # Network Metrics + net_io=psutil.net_io_counters() + net_bytes_sent=net_io.bytes_sent if net_io else 0 + net_bytes_received=net_io.bytes_recv if net_io else 0 + net_packets_sent=net_io.packets_sent if net_io else 0 + net_packets_received=net_io.packets_recv if net_io else 0 + net_errs=(net_io.errin + net_io.errout) if net_io else 0 + net_drops=(net_io.dropin + net_io.dropout) if net_io else 0 + + net_rate_mb_s=None + if _last["time"]: + elapsed_time = now - _last["time"] + sent_rate = rate_mb_s(net_bytes_sent, _last["net_bytes_sent"], elapsed_time) + recv_rate = rate_mb_s(net_bytes_received, _last["net_bytes_recv"], elapsed_time) + if sent_rate is not None and recv_rate is not None: + net_rate_mb_s = sent_rate + recv_rate + + _last["net_bytes_sent"] = net_bytes_sent + _last["net_bytes_recv"] = net_bytes_received + _last["time"] = now + + + #b Load Average Metrics + try: + load_avg1, load_avg5, load_avg15=psutil.getloadavg() + + # except AttributeError: # catches specific error when getloadavg is not support + except Exception: + load_avg1=load_avg5=load_avg15=None + + #initialising the process_type_count variables + total_processes, running_processes, sleeping_processes, zombie_processes = 0, 0, 0, 0 + for p in psutil.process_iter(['status']): + total_processes+= 1 + try: + status=p.info['status'] + if status==psutil.STATUS_RUNNING: + running_processes+=1 + elif status==psutil.STATUS_SLEEPING: + sleeping_processes+=1 + elif status==psutil.STATUS_ZOMBIE: + zombie_processes+=1 + except (psutil.NoSuchProcess,psutil.AccessDenied): + pass + + + # Temperature and Battery Metrics (if available) + temp_avg, temp_max = None, None + try: + temps=psutil.sensors_temperatures() + all_temps=[t.current for sensors in temps.values() for t in sensors] + if all_temps: + temp_avg = sum(all_temps) / len(all_temps) + temp_max = max(all_temps) + except Exception: + pass + battery_percent=None + try: + battery=psutil.sensors_battery() + if battery: + battery_percent=battery.percent + except Exception: + pass + + return { + "timestamp": timestamp, + "cpu_usage_percent": cpu_usage_percent, + "cpu_current_freq": cpu_current_freq, + "cpu_user_time": cpu_user_time, + "cpu_system_time": cpu_system_time, + "cpu_idle_time": cpu_idle_time, + "cpu_iowait_time": cpu_iowait_time, + "cpu_busy_time": cpu_busy_time, + "cpu_ctx_switches": cpu_ctx_switches, + + "memory_percent": memory_percent, + "memory_used":memory_used, + "memory_available":memory_available, + "memory_cached":memory_cached, + "memory_buffers":memory_buffers, + "swap_percent":swap_percent, + "swap_sin":swap_sin, + "swap_sout":swap_sout, + + "disk_usage_percent":disk_usage, + "disk_read":disk_read, + "disk_write":disk_write, + "disk_read_time":disk_read_time, + "disk_write_time":disk_write_time, + + "net_bytes_sent":net_bytes_sent, + "net_bytes_received":net_bytes_received, + "net_packets_sent":net_packets_sent, + "net_packets_received":net_packets_received, + "net_errs":net_errs, + "net_drops":net_drops, + + "load_avg1":load_avg1, + "load_avg5":load_avg5, + "load_avg15":load_avg15, + "total_processes":total_processes, + "running_processes":running_processes, + "sleeping_processes":sleeping_processes, + "zombie_processes":zombie_processes, + "avg_temp":temp_avg, + "max_temp":temp_max, + "battery_percent":battery_percent, + "process_data":process_data + } + +if __name__ == "__main__": + import json + metrics = collect_layer1_metrics() + print(json.dumps(metrics, indent=4)) \ No newline at end of file diff --git a/collectors/layer2_process.py b/collectors/layer2_process.py index 87632ca..2414894 100644 --- a/collectors/layer2_process.py +++ b/collectors/layer2_process.py @@ -1 +1,85 @@ """Layer 2 process telemetry collection.""" +import time +import psutil + +def collect_process_telemetry(prev_states=None): + if prev_states is None: + prev_states = {} + + current_states = {} + processed_snapshots = [] + current_time = time.time() + + for proc in psutil.process_iter(['pid', 'name', 'num_threads', 'status']): + try: + info = proc.info + pid = info['pid'] + + cpu_percent = proc.cpu_percent(interval=None) + mem_info = proc.memory_info() + mem_percent = proc.memory_percent() + + try: + io_counters = proc.io_counters() + read_bytes = io_counters.read_bytes + write_bytes = io_counters.write_bytes + except (psutil.AccessDenied, AttributeError): + read_bytes, write_bytes = 0, 0 + + current_states[pid] = { + 'read_bytes': read_bytes, + 'write_bytes': write_bytes, + 'timestamp': current_time + } + + calc_read_rate = 0.0 + calc_write_rate = 0.0 + if pid in prev_states: + prev = prev_states[pid] + time_delta = current_time - prev['timestamp'] + if time_delta > 0: + calc_read_rate = max(0.0, (read_bytes - prev['read_bytes']) / time_delta) + calc_write_rate = max(0.0, (write_bytes - prev['write_bytes']) / time_delta) + + processed_snapshots.append({ + 'pid': pid, + 'name': info['name'] or 'Unknown', + 'cpu_percent': round(cpu_percent, 2), + 'memory_percent': round(mem_percent, 2), + 'rss_memory': mem_info.rss / (1024 * 1024), # MB + 'vms_memory': mem_info.vms / (1024 * 1024 * 1024), # GB + 'thread_count': info['num_threads'] or 1, + 'read_bytes_sec': round(calc_read_rate, 2), + 'write_bytes_sec': round(calc_write_rate, 2), + 'status': info['status'] or 'unknown' + }) + + except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): + continue + + # Slice out the two lists independently + top_cpu = sorted(processed_snapshots, key=lambda x: x['cpu_percent'], reverse=True)[:5] + top_mem = sorted(processed_snapshots, key=lambda x: x['rss_memory'], reverse=True)[:5] + + # Return as a structured tuple + return top_cpu, top_mem, current_states + + +if __name__ == '__main__': + from db import init_db, insert_separated_telemetry + + init_db() + baselines = {} + print("CogniOS Separated Telemetry Daemon started...") + + try: + while True: + top_cpu, top_mem, baselines = collect_process_telemetry(baselines) + + # Pass both lists cleanly to our data layer + insert_separated_telemetry(top_cpu, top_mem) + + print(f"Committed Top 5 CPU and Top 5 Memory snapshots.") + time.sleep(5) + except KeyboardInterrupt: + print("\nDaemon safely terminated.") \ No newline at end of file diff --git a/config.py b/config.py index be02e34..c4c8d21 100644 --- a/config.py +++ b/config.py @@ -1 +1 @@ -"""CogniOS configuration.""" +DB_PATH = "cognios_telemetry.db" \ No newline at end of file diff --git a/db.py b/db.py index e5c2395..369f6bc 100644 --- a/db.py +++ b/db.py @@ -1 +1,212 @@ """Shared database schema and read/write interface.""" +import sqlite3 # for both layers +import time # for layer 2 +from config import DB_PATH # for layer 1 + +# layer 2 db code starts here + +DB_NAME = "cognios_telemetry.db" + +def init_db(): + """Initializes separate structural tables for CPU and Memory metrics.""" + with sqlite3.connect(DB_NAME) as conn: + cursor = conn.cursor() + + # Table 1: Dedicated CPU telemetry + cursor.execute(""" + CREATE TABLE IF NOT EXISTS top_cpu_telemetry ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp REAL NOT NULL, + pid INTEGER NOT NULL, + name TEXT NOT NULL, + cpu_percent REAL NOT NULL, + memory_percent REAL NOT NULL, + rss_memory INTEGER NOT NULL, + vms_memory INTEGER NOT NULL, + thread_count INTEGER NOT NULL, + read_bytes_sec REAL NOT NULL, + write_bytes_sec REAL NOT NULL, + status TEXT NOT NULL + ) + """) + + # Table 2: Dedicated RAM telemetry + cursor.execute(""" + CREATE TABLE IF NOT EXISTS top_ram_telemetry ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp REAL NOT NULL, + pid INTEGER NOT NULL, + name TEXT NOT NULL, + cpu_percent REAL NOT NULL, + memory_percent REAL NOT NULL, + rss_memory INTEGER NOT NULL, + vms_memory INTEGER NOT NULL, + thread_count INTEGER NOT NULL, + read_bytes_sec REAL NOT NULL, + write_bytes_sec REAL NOT NULL, + status TEXT NOT NULL + ) + """) + conn.commit() + + +def _map_to_rows(process_list, current_timestamp): + """Helper function to transform list of dicts to flat tuples for SQLite.""" + return [ + ( + current_timestamp, + p['pid'], + p['name'], + p['cpu_percent'], + p['memory_percent'], + p['rss_memory'], + p['vms_memory'], + p['thread_count'], + p['read_bytes_sec'], + p['write_bytes_sec'], + p['status'] + ) + for p in process_list + ] + + +def insert_separated_telemetry(top_cpu, top_mem): + """ + Inserts data cleanly into their respective tables. + Even if a process exists in both lists, it is safely recorded + in both metrics tables under the same time block window. + """ + current_timestamp = time.time() + + # Map the dictionaries into raw tuple rows + cpu_rows = _map_to_rows(top_cpu, current_timestamp) + ram_rows = _map_to_rows(top_mem, current_timestamp) + + insert_query = """ + INSERT INTO {} ( + timestamp, pid, name, cpu_percent, memory_percent, + rss_memory, vms_memory, thread_count, read_bytes_sec, + write_bytes_sec, status + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """ + + with sqlite3.connect(DB_NAME) as conn: + cursor = conn.cursor() + + + if cpu_rows: + cursor.executemany(insert_query.format("top_cpu_telemetry"), cpu_rows) + + if ram_rows: + cursor.executemany(insert_query.format("top_ram_telemetry"), ram_rows) + + conn.commit() +# layer 2 db code ends here + +# layer 1 db code starts here + + +db_path = DB_PATH + +# Creating table for all the metrics collected from the system + +def create_connection(db_path): + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + cursor.execute('''CREATE TABLE IF NOT EXISTS layer1_sys ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp TEXT NOT NULL, + + --CPU Metrics + cpu_usage_percent REAL, + cpu_freq REAL, + cpu_user_time REAL, + cpu_system_time REAL, + cpu_idle_time REAL, + cpu_iowait_time REAL, + cpu_busy_time REAL, + cpu_ctx_switches REAL, + + --Memory Metrics + memory_percent REAL, + memory_used INTEGER, + memory_available INTEGER, + memory_cached INTEGER, + memory_buffers INTEGER, + swap_percent REAL, + swap_sin INTEGER, + swap_sout INTEGER, + + --Disk Metrics + disk_usage_percent REAL, + disk_read_mb_s REAL, + disk_write_mb_s REAL, + disk_read_time INTEGER, + disk_write_time INTEGER, + + --Network Metrics + net_rate_mb_s REAL, + net_bytes_sent INTEGER, + net_bytes_recv INTEGER, + net_packets_sent INTEGER, + net_packets_recv INTEGER, + net_errs INTEGER, + net_drops INTEGER, + + --System Metrics + load_avg_1 REAL, + load_avg_5 REAL, + load_avg_15 REAL, + total_processes INTEGER, + running_processes INTEGER, + sleeping_processes INTEGER, + zombie_processes INTEGER, + + --Hardware Metrics + avg_temp REAL, + max_temp REAL, + battery_percent REAL + ) + ''') + + conn.commit() + return conn + +# Function to write the collected metrics into the database + +def write_layer1(conn, timestamp, cpu_usage_percent, cpu_freq, cpu_user_time, cpu_system_time, cpu_idle_time, cpu_iowait_time, cpu_busy_time, cpu_ctx_switches, memory_percent, memory_used, memory_available, memory_cached, memory_buffers, swap_percent, swap_sin, swap_sout, disk_usage_percent, disk_read_mb_s, disk_write_mb_s, disk_read_time, disk_write_time, load_avg_1, load_avg_5, load_avg_15, total_processes, running_processes, sleeping_processes, zombie_processes, avg_temp, max_temp, battery_percent): + cursor = conn.cursor() + cursor.execute(''' + INSERT INTO layer1_sys (timestamp, cpu_usage_percent, cpu_freq, cpu_user_time, cpu_system_time, cpu_idle_time, cpu_iowait_time, cpu_busy_time, cpu_ctx_switches, memory_percent, memory_used, memory_available, memory_cached, memory_buffers, swap_percent, swap_sin, swap_sout, disk_usage_percent, disk_read_mb_s, disk_write_mb_s, disk_read_time, disk_write_time, load_avg_1, load_avg_5, load_avg_15, total_processes, running_processes, sleeping_processes, zombie_processes, avg_temp, max_temp, battery_percent) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ''', (timestamp, + cpu_usage_percent, + cpu_freq, cpu_user_time, + cpu_system_time, + cpu_idle_time, + cpu_iowait_time, + cpu_busy_time, + cpu_ctx_switches, + memory_percent, + memory_used, + memory_available, + memory_cached, + memory_buffers, + swap_percent, + swap_sin, + swap_sout, + disk_usage_percent, + disk_read_mb_s, + disk_write_mb_s, + disk_read_time, + disk_write_time, + load_avg_1, + load_avg_5, + load_avg_15, + total_processes, + running_processes, + sleeping_processes, + zombie_processes, + avg_temp, + max_temp, + battery_percent)) diff --git a/test.py b/test.py new file mode 100644 index 0000000..ec4aad2 --- /dev/null +++ b/test.py @@ -0,0 +1,29 @@ +"Testing Document" +import time +from collectors.layer2_process import collect_process_telemetry +from db import init_db, insert_separated_telemetry + +def test_layer2_pipeline(): + print("Initializing CogniOS Test Database...") + init_db() + + baselines = {} + print("Running initial telemetry sweep (Baseline generation)...") + top_cpu, top_mem, baselines = collect_process_telemetry(baselines) + + # Wait 5 seconds to simulate a real monitoring pulse + print("Pacing for 5 seconds to calculate real delta rates...") + time.sleep(5) + + print("Running second telemetry sweep...") + top_cpu, top_mem, baselines = collect_process_telemetry(baselines) + + print(f"Top CPU Process Count: {len(top_cpu)}") + print(f"Top RAM Process Count: {len(top_mem)}") + + print("Writing captured data to separate SQLite tables...") + insert_separated_telemetry(top_cpu, top_mem) + print("Success! Check your root directory for 'cognios_telemetry.db'.") + +if __name__ == "__main__": + test_layer2_pipeline() \ No newline at end of file diff --git a/utils/helpers.py b/utils/helpers.py new file mode 100644 index 0000000..7a92516 --- /dev/null +++ b/utils/helpers.py @@ -0,0 +1,9 @@ + + +def rate_mb_s(current_bytes, last_bytes, elapsed_sec): + + if last_bytes is None or elapsed_sec <= 0: + return None + # how many bytes have been sent/received since the last check, and convert to MB/s by dividing by 1 MB + delta_bytes = current_bytes - last_bytes + return (delta_bytes / elapsed_sec) / (1024 * 1024) \ No newline at end of file