diff --git a/.gemini/skills/initiative-start/reasearch-skill.md b/.gemini/skills/initiative-start/reasearch-skill.md index c6195d6b5..b9eac514d 100644 --- a/.gemini/skills/initiative-start/reasearch-skill.md +++ b/.gemini/skills/initiative-start/reasearch-skill.md @@ -21,8 +21,9 @@ Ensure all work begins with **documentation-first understanding**, **PRD validat 1. `/CONTRIBUTING.md` 2. `/README.md` 3. `/.github/skills/code-standards/SKILL.md` - 4. `/docs/**` - 5. Related module/code context if referenced + 4. `/.github/skills/database-patterns/SKILL.md` — read when the feature touches the `Devices` table or requires audit/history logging + 5. `/docs/**` + 6. Related module/code context if referenced * Extract: @@ -69,6 +70,13 @@ If anything is unclear: * Breaking assumptions * Plugin or API mismatches +* **If the feature writes to the `Devices` table:** + + * Load `/.github/skills/database-patterns/SKILL.md` + * Audit all write paths listed there — confirm which are affected + * Determine if a SQLite trigger or Python hook is the right approach + * Check if `*Source` fields already solve attribution requirements + * Clearly report inconsistencies before proceeding --- diff --git a/.gemini/skills/plugin-development/plugin-skill.md b/.gemini/skills/plugin-development/plugin-skill.md index 215d9bf22..619495f28 100644 --- a/.gemini/skills/plugin-development/plugin-skill.md +++ b/.gemini/skills/plugin-development/plugin-skill.md @@ -1,16 +1,86 @@ -# Plugin development skill +--- +name: netalertx-plugin-development +description: Create and run NetAlertX plugins. Use this when asked to create plugin, run plugin, test plugin, or develop plugin functionality. +--- -1. Assess requirements -2. Read `docs/PLUGINS_DEV.md` -3. Confirm field mapping -4. Ask clarifying questions -5. code placed in `front/plugins/` +# Plugin Development +## Expected Workflow -Plugin prefix: +1. Read this skill and `docs/PLUGINS_DEV.md` for full context. +2. Find or create the plugin in `front/plugins//`. +3. Read the plugin's `config.json` and `script.py` to understand its functionality. +4. Run: `python3 front/plugins//script.py` +5. Retrieve the result from `/tmp/log/plugins/last_result..log` quickly — the backend deletes it after processing. -- has to be uppercase letters only (no underscores) -- must be unique -- keep short but readable if possible +## Run a Plugin Manually + +```bash +python3 front/plugins//script.py +``` + +Ensure `sys.path` includes `/app/front/plugins` and `/app/server` (as in the template). + +## Plugin Structure + +```text +front/plugins// +├── config.json # Manifest with settings +├── script.py # Main script +└── ... +``` + +## Plugin Prefix Rules + +- Uppercase letters only (no underscores, no numbers) +- Must be unique across all plugins +- Short but readable (e.g., `ARPSCAN`, `MQTT`, `UNIFI`) + +## Settings Pattern + +- `_RUN`: execution phase +- `_RUN_SCHD`: cron-like schedule +- `_CMD`: script path +- `_RUN_TIMEOUT`: timeout in seconds +- `_WATCH`: columns to watch for changes + +## Data Contract + +Scripts write to `/tmp/log/plugins/last_result..log` using `plugin_helper.py`: + +```python +from plugin_helper import Plugin_Objects + +plugin_objects = Plugin_Objects() +plugin_objects.add_object(...) # During processing +plugin_objects.write_result_file() # Exactly once at end +``` + +**Important:** The backend processes and deletes the result file almost immediately. Retrieve it quickly if inspecting output. + +## Execution Phases + +| Phase | Trigger | +|-------|---------| +| `once` | Once at startup | +| `schedule` | On cron schedule | +| `always_after_scan` | After every scan | +| `before_name_updates` | Before name resolution | +| `on_new_device` | When new device detected | +| `on_notification` | When notification triggered | + +## Plugin Formats + +| Format | Purpose | Phase | +|--------|---------|-------| +| publisher | Send notifications | `on_notification` | +| dev scanner | Create/manage devices | `schedule` | +| name discovery | Discover device names | `before_name_updates` | +| importer | Import from services | `schedule` | +| system | Core functionality | `schedule` | + +## Starting Point + +Copy from `front/plugins/__template` and customize. Read `docs/PLUGINS_DEV.md` for the full development guide. diff --git a/.gemini/skills/project-navigation/SKILL.md b/.gemini/skills/project-navigation/SKILL.md index 24ef258a7..a3c273105 100644 --- a/.gemini/skills/project-navigation/SKILL.md +++ b/.gemini/skills/project-navigation/SKILL.md @@ -3,13 +3,56 @@ name: project-navigation description: Reference for the NetAlertX codebase structure, key file paths, and configuration locations. Use this when exploring the codebase or looking for specific components like the backend entry point, frontend files, or database location. --- -# Project Navigation & Structure +# Project Navigation -## Codebase Structure & Key Paths +## Key Paths -- **Source Code:** `/workspaces/NetAlertX` (mapped to `/app` in container via symlink). -- **Backend Entry:** `server/api_server/api_server_start.py` (Flask) and `server/__main__.py`. -- **Frontend:** `front/` (PHP/JS). -- **Plugins:** `front/plugins/`. -- **Config:** `/data/config/app.conf` (runtime) or `back/app.conf` (default). -- **Database:** `/data/db/app.db` (SQLite). +| Component | Path | +|-----------|------| +| Workspace root | `/workspaces/NetAlertX` | +| Backend entry | `server/__main__.py` | +| API server | `server/api_server/api_server_start.py` | +| Plugin system | `server/plugin.py` | +| Initialization | `server/initialise.py` | +| Frontend | `front/` | +| Frontend JS | `front/js/common.js` | +| Frontend PHP | `front/php/server/*.php` | +| Plugins | `front/plugins/` | +| Plugin template | `front/plugins/__template` | +| Database helpers | `server/db/db_helper.py` | +| Device model | `server/models/device_instance.py` | +| Messaging | `server/messaging/` | +| Workflows | `server/workflows/` | + +## Architecture + +NetAlertX uses a frontend–backend architecture: the frontend runs on **PHP + Nginx** (see `front/`), the backend is implemented in **Python** (see `server/`), and scheduled tasks are managed by a **supercronic** scheduler. + +## Runtime Paths + +| Data | Path | +|------|------| +| Config (runtime) | `/data/config/app.conf` | +| Config (default) | `back/app.conf` | +| Database | `/data/db/app.db` | +| API JSON cache | `/tmp/api/*.json` | +| Logs | `/tmp/log/` | +| Plugin logs | `/tmp/log/plugins/` | + +## Environment Variables + +Use these instead of hardcoding paths: +- `NETALERTX_DB` +- `NETALERTX_LOG` +- `NETALERTX_CONFIG` +- `NETALERTX_DATA` +- `NETALERTX_APP` + +## Documentation + +| Topic | Path | +|-------|------| +| Plugin development | `docs/PLUGINS_DEV.md` | +| System settings | `docs/SETTINGS_SYSTEM.md` | +| API docs | `docs/API_*.md` | +| Debug guides | `docs/DEBUG_*.md` | diff --git a/.gemini/skills/settings/SKILL.md b/.gemini/skills/settings/SKILL.md new file mode 100644 index 000000000..0c21ce339 --- /dev/null +++ b/.gemini/skills/settings/SKILL.md @@ -0,0 +1,412 @@ +--- +name: nax-settings-development +description: Best practices for creating and maintaining NetAlertX settings (config.json), including UI, validation, localization, runtime behavior, and implementation consistency. +--- + +# NetAlertX Settings Development + +This skill defines the conventions and best practices for adding or modifying settings in NetAlertX. + +The goal is for every setting to be: + +- Declarative +- Self-documenting +- Automatically rendered by the Settings UI +- Fully localized +- Backwards compatible +- Easy to maintain + +--- + +## Quick Reference + +**Read a setting (backend):** +```python +from helper import get_setting_value +value = get_setting_value('SETTING_NAME') +``` +Never read `app.conf` directly. Always use `get_setting_value()`. + +**Read a setting (frontend):** +```javascript +getSetting("SETTING_NAME") +``` + +**Add a core setting** — use `ccd()` in `server/initialise.py`: +```python +ccd('SETTING_NAME', 'default_value', 'description') +``` + +**Add a plugin setting** — define in the plugin's `config.json` under the `settings` key. + +| File | Purpose | +|------|---------| +| `/data/config/app.conf` | Runtime config (source of truth) | +| `back/app.conf` | Default config (template) | + +Use `APP_CONF_OVERRIDE` for settings that must be set before startup. + +--- + +# 1. Follow existing patterns + +Before creating a new setting: + +- Search for similar settings. +- Reuse existing naming conventions. +- Reuse existing setting types. +- Reuse existing validation patterns. +- Follow the style already used by similar plugins. + +Avoid introducing new setting types unless absolutely necessary. + +--- + +# 2. Setting naming + +Setting keys use uppercase snake case. + +Good: + +```text +UI_DEV_SECTIONS +SCAN_INTERVAL +MQTT_HOST +``` + +Avoid: + +```text +uiDevSections +scanInterval +ScanInterval +``` + +Names should clearly describe the purpose. + +--- + +# 3. Categories + +Place settings into the most appropriate category. + +Keep related settings together. + +Avoid creating new categories unless there is a clear need. + +--- + +# 4. Choose the correct type + +Use the simplest type that correctly models the value. + +Common types include: + +- string +- integer +- float +- boolean +- password +- select +- array +- textarea + +Avoid encoding structured JSON inside string settings. + +--- + +# 5. Validation + +Always define validation whenever appropriate. + +Examples include: + +- minimum values +- maximum values +- regex validation +- allowed options + +Reject invalid configuration rather than silently accepting it. + +--- + +# 6. Defaults + +Provide sensible defaults that work for a fresh installation. + +Avoid defaults that require external services or additional configuration. + +A default value should exist in exactly one place. + +Use the `default_value` defined in `config.json`. + +Do not duplicate the same default value elsewhere in Python or JavaScript. + +--- + +# 7. Names and descriptions + +Setting names should be concise. + +Examples: + +- Scan interval +- MQTT Host +- Hide device sections + +Descriptions should explain: + +- what the setting does +- when it should be used +- important side effects + +Avoid implementation details. + +Good: + +> Interval between network scans in seconds. + +Bad: + +> Calls scheduler.py every X seconds. + +--- + +# 8. Boolean settings + +Boolean settings should read naturally. + +Prefer verbs such as: + +- Enable +- Disable +- Require +- Allow +- Ignore +- Show +- Hide + +Examples: + +- Enable MQTT +- Require NICs Online +- Hide Offline Devices + +--- + +# 9. Select settings + +When users must choose from predefined values, use a select setting. + +Never require users to remember internal values. + +Each option should have a meaningful label. + +--- + +# 10. Array settings + +Use arrays only when multiple independent values are expected. + +Examples: + +- ignored MAC addresses +- subnet lists +- plugin lists + +--- + +# 11. Localization + +Every setting must have localized language strings. + +If the language strings are **not** defined directly inside `config.json`, they must exist in `en_us.json`. + +For example, for: + +```text +UI_DEV_SECTIONS +``` + +add: + +```json +"UI_DEV_SECTIONS_name": "Hide device sections", +"UI_DEV_SECTIONS_description": "Select which UI elements to hide on the Devices page." +``` + +When using external language files, `config.json` must reference: + +```json +"name": [ + { + "string": "_GLOBAL_LANG_FILES_" + } +] +``` + +This tells NetAlertX to resolve the display name from the language files. + +--- + +# 12. Source of truth + +`app.conf` is the source of truth. + +Keep the following in mind: + +- User-defined values always come from `app.conf`. +- Default values from `config.json` are only used when the setting is missing from `app.conf`. +- Never assume settings are permanently stored in the database. + +--- + +# 13. Plugin-first settings + +Whenever possible, define new settings inside a plugin's `config.json`. + +Avoid adding hardcoded application settings unless there is a compelling architectural reason. + +Plugin settings are the preferred and future-proof approach. + +--- + +# 14. Runtime lifecycle + +Understand how settings flow through the application. + +```text +config.json + │ + │ default values + ▼ +app.conf (source of truth) + │ + ▼ +Settings database table + │ + ▼ +table_settings.json API + │ + ▼ +Frontend (getSetting()) +``` + +The database and API are runtime representations only. + +They are regenerated from `app.conf` during initialization. + +--- + +# 15. Access + +Code should never read `app.conf` directly. + +In the Frontend, always retrieve settings through: + +```javascript +getSetting("SETTING_NAME") +``` + +And in the backend, always retrieve settings through: + +```python +get_setting_value("SETTING_NAME") +``` + +This guarantees the value comes from the generated settings API. + +--- + +# 16. Metadata + +Each setting automatically has a corresponding `__metadata` entry generated in `app.conf`. + +Do not manually create or modify metadata entries unless working on the settings framework itself. + +--- + +# 17. Backend + +Backend code should: + +- use the existing configuration helpers +- avoid duplicated parsing logic +- gracefully handle missing values +- avoid hardcoded defaults + +--- + +# 18. Frontend + +A correctly defined setting should render automatically in the existing Settings UI. + +Avoid writing custom JavaScript unless absolutely necessary. + +--- + +# 19. Prefer configuration over code + +If a behaviour can reasonably be controlled through an existing setting type (boolean, select, integer, array, etc.), introduce a setting instead of hardcoding special-case logic. + +Configuration is preferred over implementation-specific behaviour whenever practical. + +--- + +# 20. Backwards compatibility + +Avoid: + +- renaming settings +- changing data types +- changing semantics + +If unavoidable: + +- provide migration logic +- preserve compatibility where possible + +--- + +# 21. Documentation + +Every user-facing setting should eventually be documented. + +Include: + +- purpose +- accepted values +- default value +- examples when useful + +--- + +# 22. Pull request checklist + +Before submitting a PR, verify: + +- [ ] Setting follows existing conventions. +- [ ] Correct category used. +- [ ] Appropriate type selected. +- [ ] Validation defined. +- [ ] Sensible default provided. +- [ ] Default defined only once. +- [ ] Name is concise. +- [ ] Description is clear. +- [ ] Localization strings added. +- [ ] `_GLOBAL_LANG_FILES_` used where appropriate. +- [ ] Plugin-first approach followed where applicable. +- [ ] Frontend renders automatically. +- [ ] Backend uses existing configuration helpers. +- [ ] No unnecessary frontend code added. +- [ ] No duplicated parsing logic. +- [ ] Backwards compatibility maintained. +- [ ] Documentation updated if required. + +--- + +# References + +- https://docs.netalertx.com/PLUGINS_DEV_SETTINGS/ +- https://docs.netalertx.com/SETTINGS_SYSTEM/ +- https://docs.netalertx.com/PLUGINS/ \ No newline at end of file diff --git a/.gemini/skills/skills-index/SKILL.md b/.gemini/skills/skills-index/SKILL.md new file mode 100644 index 000000000..d5272a5d9 --- /dev/null +++ b/.gemini/skills/skills-index/SKILL.md @@ -0,0 +1,64 @@ +--- +name: skills-index +description: Index of all available skills across both Gemini CLI (.gemini/skills/) and GitHub Copilot (.github/skills/). Load this to find the right skill for a task, or to locate the counterpart skill in the other AI system. +--- + +# Skills Index — Cross-Reference + +Two AI assistants are configured for this project, each with their own skill directory: + +- **Gemini CLI** → `.gemini/skills/` +- **GitHub Copilot** → `.github/skills/` + +Skills with the same purpose exist in both, sometimes under different names and with different depth. This index maps them so you can find the richer version when needed. + +--- + +## Shared Skills (exist in both) + +| Topic | Gemini Skill | Copilot Skill | Notes | +|-------|-------------|--------------|-------| +| Testing | `testing-workflow` | `testing-workflow` | Gemini version emphasises full suite preference and container detection; Copilot version covers `testFailure` tool and PYTHONPATH | +| Settings & config | `settings` | `settings-management` | Gemini version is more comprehensive (22-point guide + PR checklist); Copilot version covers `ccd()` and `get_setting_value()` usage | +| MCP activation | `mcp-activation` | `mcp-activation` | Gemini version covers Gemini CLI session restart; Copilot version covers VS Code window reload | +| Project navigation | `project-navigation` | `project-navigation` | Copilot version has full path tables and env vars; Gemini version is a brief reference | +| Plugin dev | `plugin-development` | `plugin-run-development` | Copilot version is comprehensive (data contract, phases, formats); Gemini version is a brief checklist pointing to `docs/PLUGINS_DEV.md` | +| Devcontainer | `devcontainer-management` | `devcontainer-services` + `devcontainer-setup` + `devcontainer-configs` | Gemini combines into one (uses `docker exec`); Copilot splits into 3 focused skills | + +--- + +## Copilot-Only Skills + +No Gemini equivalent yet: + +| Copilot Skill | Purpose | +|--------------|---------| +| `api-development` | Creating REST API endpoints | +| `authentication` | API tokens and 401/403 debugging | +| `code-standards` | Coding conventions and style rules | +| `database-patterns` | Device table write paths, SQLite triggers, audit logging, `*Source` attribution | +| `database-reset` | Wipe and regenerate the database and config | +| `docker-build` | Build Docker images for testing or production | +| `docker-prune` | Clean unused Docker resources (destructive — requires confirmation) | +| `sample-data` | Load synthetic device data into the devcontainer | + +--- + +## Gemini-Only Skills + +No Copilot equivalent yet: + +| Gemini Skill | Purpose | +|-------------|---------| +| `initiative-start` | Research methodology and structured approach for new tasks | + +--- + +## Adding a New Skill + +When adding a skill, create it in **both** directories to keep both AI systems current: + +- `.gemini/skills//SKILL.md` — auto-discovered by Gemini CLI via YAML frontmatter +- `.github/skills//SKILL.md` — add an entry to the skills table in `.github/copilot-instructions.md` + +Keep the body content identical between both files. Only the frontmatter `name`/`description` may differ slightly to match each system's discovery heuristics. diff --git a/.gemini/skills/testing-workflow/SKILL.md b/.gemini/skills/testing-workflow/SKILL.md index debf79839..0b7ac7f78 100644 --- a/.gemini/skills/testing-workflow/SKILL.md +++ b/.gemini/skills/testing-workflow/SKILL.md @@ -75,4 +75,34 @@ The retrieved token MUST be used in all subsequent API or test calls requiring a If tests fail with 403 Forbidden or empty tokens: 1. Verify server is running and use the setup script (`/workspaces/NetAlertX/.devcontainer/scripts/setup.sh`) if required. 2. Verify `app.conf` inside the container: `cat /data/config/app.conf` + +## Check for Pre-Existing Failures + +Before attributing failures to your changes, check what was already broken: + +```bash +cd /workspaces/NetAlertX; pytest test/ --tb=no -q 2>&1 | tail -20 +``` + +Do not fix pre-existing failures unless that is the explicit goal. + +## PYTHONPATH + +The test environment is pre-configured with: +- `/app` — primary location where Python runs in production +- `/app/server` — symlink to `/workspaces/NetAlertX/server` +- `/app/front/plugins` — symlink to `/workspaces/NetAlertX/front/plugins` +- `/workspaces/NetAlertX/test` +- `/workspaces/NetAlertX/server` +- `/workspaces/NetAlertX` + +## Docker Test Image + +If the Dockerfile or dependencies changed, rebuild the test image before running: + +```bash +docker buildx build -t netalertx-test . +``` + +Takes ~30 seconds; ~90 seconds if the venv stage changed. 3. Verify Python can read it: `python3 -c "from helper import get_setting_value; print(get_setting_value('API_TOKEN'))"` \ No newline at end of file diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 187da1a82..0dd53bc02 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -27,6 +27,7 @@ Procedural knowledge lives in `.github/skills/`. Load the appropriate skill when | Task | Skill | |------|-------| +| Find or cross-reference any skill | `skills-overview` | | Run tests, check failures | `testing-workflow` | | Start/stop/restart services | `devcontainer-services` | | Wipe database, fresh start | `database-reset` | @@ -42,6 +43,7 @@ Procedural knowledge lives in `.github/skills/`. Load the appropriate skill when | Settings and config | `settings-management` | | Find files and paths | `project-navigation` | | Coding standards | `code-standards` | +| Devices table write paths, SQLite triggers, audit logging, `*Source` attribution | `database-patterns` | ## Execution Protocol diff --git a/.github/skills/database-patterns/SKILL.md b/.github/skills/database-patterns/SKILL.md new file mode 100644 index 000000000..f6125158b --- /dev/null +++ b/.github/skills/database-patterns/SKILL.md @@ -0,0 +1,142 @@ +--- +name: netalertx-database-patterns +description: NetAlertX database architecture patterns. Use this when designing features that write to the Devices table, implementing audit/history logging, or choosing between trigger-based vs Python-hook approaches. +--- + +# Database Patterns + +## Devices Table — Write-Path Inventory + +Before implementing any feature that reads or writes the `Devices` table, audit ALL write paths. The table is modified from many locations — missing one path is a correctness bug. + +**Known production write paths (as of 2026-07-04):** + +| File | Function | Fields written | +|---|---|---| +| `server/models/device_instance.py` | `setDeviceData()` | All user-editable fields | +| `server/models/device_instance.py` | `updateField()` | Any single field (workflows) | +| `server/models/device_instance.py` | `updateDeviceColumn()` | Any single column | +| `server/models/device_instance.py` | `deleteDevices()` etc. | DELETE operations | +| `server/scan/device_handling.py` | `update_devices_data_from_scan()` | Scan-derived fields | +| `server/scan/device_handling.py` | `update_vendors_from_mac()` | `devVendor`, `devVendorSource` | +| `server/scan/device_handling.py` | Name resolution block | `devName`, `devFQDN`, `*Source` | +| `server/scan/device_handling.py` | `update_ipv4_ipv6()` | `devPrimaryIPv4`, `devPrimaryIPv6` | +| `server/scan/device_handling.py` | `update_icons_and_types()` | `devIcon`, `devType` | +| `server/scan/device_handling.py` | `update_presence_from_CurrentScan()` | `devPresentLastScan` | +| `server/scan/device_handling.py` | `update_devLastConnection_from_CurrentScan()` | `devLastConnection` | +| `server/scan/device_handling.py` | `update_devPresentLastScan_based_on_*()` | `devPresentLastScan` | +| `server/db/authoritative_handler.py` | `enforce_source_on_user_update()` | `*Source` columns | +| `server/db/authoritative_handler.py` | `lock_field()` / `unlock_field()` | `*Source` columns | +| `server/models/notification_instance.py` | `clearPendingEmailFlag()` | `devLastNotification` | +| `front/plugins/db_cleanup/script.py` | `cleanup_database()` | DELETE operations | + +**Key insight:** Most scan functions use `sql.executemany()` — there is no per-row Python state available. Python hooks before/after executemany require a pre-fetch+diff pattern that is expensive and error-prone. + +--- + +## `*Source` Fields — Attribution System + +The `FIELD_SOURCE_MAP` in `server/db/authoritative_handler.py` defines 10 fields that carry write attribution via paired `*Source` columns: + +```python +FIELD_SOURCE_MAP = { + "devMac": "devMacSource", + "devName": "devNameSource", + "devFQDN": "devFQDNSource", + "devLastIP": "devLastIPSource", + "devVendor": "devVendorSource", + "devSSID": "devSSIDSource", + "devParentMAC": "devParentMACSource", + "devParentPort": "devParentPortSource", + "devParentRelType": "devParentRelTypeSource", + "devVlan": "devVlanSource", +} +``` + +`*Source` values: `'USER'`, `'LOCKED'`, `'NEWDEV'`, or a plugin prefix (e.g., `'ARPSCAN'`, `'NSLOOKUP'`). + +These fields are updated **in the same transaction** as the primary field. A SQLite `AFTER UPDATE` trigger can read `NEW.devNameSource` to obtain correct attribution without any extra context-passing. + +**Attribution rules for features that need `changedBy`:** + +| Field category | Attribution | +|---|---| +| In `FIELD_SOURCE_MAP` | `COALESCE(NULLIF(NEW.Source, ''), 'system')` | +| User-only fields (`devGroup`, `devComments`, `devFavorite`, `devOwner`, `devLocation`, etc.) | `'user:api'` — only `setDeviceData()` writes these | +| Auto-computed fields (`devIcon`, `devType`, `devPrimaryIPv4`, `devPrimaryIPv6`) | `'system'` | +| `*Source` fields themselves | `'system'` | + +--- + +## Cross-Cutting Concerns — Prefer SQLite Triggers Over Python Hooks + +When a feature needs to intercept **every write** to the `Devices` table (audit logging, computed columns, cascading logic), prefer a **SQLite `AFTER UPDATE` / `AFTER INSERT` trigger** over Python-layer hooks. + +**Why:** +- Triggers catch all 14+ write paths automatically, including `executemany()` bulk updates +- Zero modifications to existing write-path functions (DRY) +- Self-healing: future write paths are automatically covered +- Attribution is available via `NEW.*Source` fields (see above) + +**When Python hooks are still appropriate:** +- The logic needs access to Python objects, settings, or services not available in SQL +- The feature only fires from one or two known write paths +- The logic is too complex to express in SQL (multi-table joins with app-layer business logic) + +### Trigger Performance Pattern + +```sql +CREATE TRIGGER trg_example +AFTER UPDATE ON Devices +FOR EACH ROW +-- Guard: short-circuit entire body when feature is disabled (zero cost) +WHEN (SELECT CAST(setValue AS INTEGER) FROM Settings WHERE setKey = 'FEATURE_ENABLED') > 0 +BEGIN + -- Per-field conditional insert + INSERT INTO SomeTable (devGUID, column, oldVal, newVal, changedBy, ts) + SELECT NEW.devGUID, 'devName', OLD.devName, NEW.devName, + COALESCE(NULLIF(NEW.devNameSource, ''), 'system'), + datetime('now', 'utc') + WHERE OLD.devName IS NOT NEW.devName + AND instr(',' || (SELECT setValue FROM Settings WHERE setKey = 'TRACKED_FIELDS') || ',', ',devName,') > 0; + -- Repeat for each tracked field... +END; +``` + +**Performance:** The Settings table is tiny (~100 rows) and stays in SQLite's page cache. Per-row Settings reads inside triggers are effectively in-memory lookups. The `WHEN` guard makes the disabled state zero-cost. + +--- + +## Snapshot vs Event-Sourced Audit Logging + +When implementing change history, always use **event-sourced (per-field rows)** not **snapshots (full row copies)**. + +| | Event-sourced | Snapshot | +|---|---|---| +| Storage | Small — only changed fields | Large — all 40+ columns every mutation | +| Filter by field | O(log n) via index | O(n) — must diff every adjacent pair | +| Filter by source | O(log n) via index | Not possible without diffing | +| `changedBy` attribution | Embedded at write time | Not available without extra context | +| Retention calculation | Simple timestamp DELETE | Same, but much higher storage | + +At 1000 devices, 5-min scan interval, 14-day retention: snapshot storage ≈ 280 MB/day. Event-sourced storage for the same workload is typically <1 MB/day (most scans produce no tracked field changes). + +--- + +## `DevicesHistory` Table — Reference Schema + +```sql +CREATE TABLE IF NOT EXISTS DevicesHistory ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + devGUID TEXT NOT NULL, + timestamp DATETIME DEFAULT CURRENT_TIMESTAMP, + changedBy TEXT NOT NULL, + changedColumn TEXT NOT NULL, + oldValue TEXT, + newValue TEXT, + FOREIGN KEY (devGUID) REFERENCES Devices(devGUID) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS idx_devhist_guid_column ON DevicesHistory(devGUID, changedColumn); +CREATE INDEX IF NOT EXISTS idx_devhist_timestamp ON DevicesHistory(timestamp); +``` diff --git a/.github/skills/mcp-activation/SKILL.md b/.github/skills/mcp-activation/SKILL.md index abbdb317f..106190abe 100644 --- a/.github/skills/mcp-activation/SKILL.md +++ b/.github/skills/mcp-activation/SKILL.md @@ -7,9 +7,14 @@ description: Enables live interaction with the NetAlertX runtime. This skill con This skill configures the environment to expose the Model Context Protocol (MCP) server to AI agents running inside the devcontainer. +## Prerequisites + +1. **Devcontainer connected:** You must be working inside the NetAlertX devcontainer. +2. **Backend running:** The Python backend must be running — it generates `app.conf` containing the API token. Use the `devcontainer-services` skill if it's not running. + ## Usage -This skill assumes you are already running within the NetAlertX devcontainer. +This skill assumes you are already running within the NetAlertX devcontainer. 1. **Generate Configurations:** Run the configuration generation script to extract the API Token and update the VS Code MCP settings. diff --git a/.github/skills/settings-management/SKILL.md b/.github/skills/settings-management/SKILL.md index 36d5adf64..1d41acbea 100644 --- a/.github/skills/settings-management/SKILL.md +++ b/.github/skills/settings-management/SKILL.md @@ -5,35 +5,175 @@ description: Manage NetAlertX configuration settings. Use this when asked to add # Settings Management -## Reading Settings +## Quick Reference +**Read a setting (backend):** ```python from helper import get_setting_value - value = get_setting_value('SETTING_NAME') ``` - Never hardcode ports, secrets, or configuration values. Always use `get_setting_value()`. -## Adding Core Settings - -Use `ccd()` in `server/initialise.py`: +**Read a setting (frontend):** +```javascript +getSetting("SETTING_NAME") +``` +**Add a core setting** — use `ccd()` in `server/initialise.py`: ```python ccd('SETTING_NAME', 'default_value', 'description') ``` -## Adding Plugin Settings - -Define in plugin's `config.json` manifest under the settings section. - -## Config Files +**Add a plugin setting** — define in the plugin's `config.json` under the `settings` key. | File | Purpose | |------|---------| -| `/data/config/app.conf` | Runtime config (modified by app) | +| `/data/config/app.conf` | Runtime config (source of truth, modified by app) | | `back/app.conf` | Default config (template) | -## Environment Override +Use `APP_CONF_OVERRIDE` for settings that must be set before startup. + +--- + +# Development Guide + +The goal is for every setting to be: + +- Declarative and self-documenting +- Automatically rendered by the Settings UI +- Fully localized +- Backwards compatible +- Easy to maintain + +--- + +## 1. Follow existing patterns + +Before creating a new setting, search for similar ones. Reuse existing naming conventions, types, and validation patterns. Follow the style already used by similar plugins. Avoid introducing new setting types unless absolutely necessary. + +--- + +## 2. Setting naming + +Setting keys use uppercase snake case: `UI_DEV_SECTIONS`, `SCAN_INTERVAL`, `MQTT_HOST`. +Names should clearly describe the purpose. + +--- + +## 3. Categories + +Place settings into the most appropriate category. Keep related settings together. Avoid creating new categories unless there is a clear need. + +--- + +## 4. Choose the correct type + +Use the simplest type that correctly models the value: `string`, `integer`, `float`, `boolean`, `password`, `select`, `array`, `textarea`. Avoid encoding structured JSON inside string settings. + +--- + +## 5. Validation + +Always define validation whenever appropriate (min/max values, regex, allowed options). Reject invalid configuration rather than silently accepting it. + +--- + +## 6. Defaults + +Provide sensible defaults that work for a fresh installation. The `default_value` in `config.json` is the single source of truth — do not duplicate it in Python or JavaScript. + +--- + +## 7. Names and descriptions + +Names should be concise. Descriptions should explain what the setting does, when to use it, and any important side effects — not implementation details. + +--- + +## 8. Boolean settings + +Prefer verbs: Enable, Disable, Require, Allow, Ignore, Show, Hide. Examples: *Enable MQTT*, *Require NICs Online*, *Hide Offline Devices*. + +--- + +## 9. Select settings + +Use when users must choose from predefined values. Each option must have a meaningful label — never require users to remember internal values. + +--- + +## 10. Array settings + +Use only when multiple independent values are expected (e.g., ignored MACs, subnet lists, plugin lists). + +--- + +## 11. Localization + +Every setting must have localized language strings. If not defined directly in `config.json`, they must exist in `en_us.json`. + +Example for `UI_DEV_SECTIONS`: +```json +"UI_DEV_SECTIONS_name": "Hide device sections", +"UI_DEV_SECTIONS_description": "Select which UI elements to hide on the Devices page." +``` + +When using external language files, `config.json` must reference: +```json +"name": [{ "string": "_GLOBAL_LANG_FILES_" }] +``` + +--- + +## 12. Source of truth + +`app.conf` is the source of truth. User-defined values always come from `app.conf`; defaults from `config.json` apply only when a setting is absent. Never read `app.conf` directly — always go through `get_setting_value()`. + +--- + +## 13. Plugin-first settings + +Whenever possible, define new settings inside a plugin's `config.json`. Avoid adding hardcoded application settings unless there is a compelling architectural reason. + +--- + +## 14. Runtime lifecycle + +```text +config.json (defaults) + → app.conf (source of truth) + → Settings database table + → table_settings.json API + → Frontend getSetting() / Backend get_setting_value() +``` + +The database and API are runtime representations only, regenerated from `app.conf` at init. + +--- + +## 15. Backwards compatibility + +Avoid renaming settings, changing data types, or changing semantics. If unavoidable, provide migration logic and preserve compatibility where possible. + +--- + +## 16. Pull request checklist + +- [ ] Follows existing naming conventions (uppercase snake case) +- [ ] Correct category used +- [ ] Appropriate type selected, validation defined +- [ ] Sensible default — defined only once (in `config.json`) +- [ ] Name concise, description clear (no implementation details) +- [ ] Localization strings added; `_GLOBAL_LANG_FILES_` used where appropriate +- [ ] Plugin-first approach followed where applicable +- [ ] Frontend renders automatically without custom JavaScript +- [ ] Backend uses `get_setting_value()` — no direct `app.conf` reads +- [ ] Backwards compatibility maintained +- [ ] Documentation updated if required + +--- + +## References -Use `APP_CONF_OVERRIDE` environment variable for settings that must be set before startup. +- https://docs.netalertx.com/PLUGINS_DEV_SETTINGS/ +- https://docs.netalertx.com/SETTINGS_SYSTEM/ diff --git a/.github/skills/skills-overview/SKILL.md b/.github/skills/skills-overview/SKILL.md new file mode 100644 index 000000000..f526459ec --- /dev/null +++ b/.github/skills/skills-overview/SKILL.md @@ -0,0 +1,64 @@ +--- +name: skills-overview +description: Index of all available skills across both GitHub Copilot (.github/skills/) and Gemini CLI (.gemini/skills/). Load this to find the right skill for a task, or to locate the counterpart skill in the other AI system. +--- + +# Skills Index — Cross-Reference + +Two AI assistants are configured for this project, each with their own skill directory: + +- **GitHub Copilot** → `.github/skills/` +- **Gemini CLI** → `.gemini/skills/` + +Skills with the same purpose exist in both, sometimes under different names and with different depth. This index maps them so you can find the richer version when needed. + +--- + +## Shared Skills (exist in both) + +| Topic | Copilot Skill | Gemini Skill | Notes | +|-------|--------------|--------------|-------| +| Testing | `testing-workflow` | `testing-workflow` | Copilot version covers `testFailure` tool and PYTHONPATH; Gemini version emphasises full suite preference and container detection | +| Settings & config | `settings-management` | `settings` | Gemini version is more comprehensive (22-point guide + PR checklist); Copilot version covers `ccd()` and `get_setting_value()` usage | +| MCP activation | `mcp-activation` | `mcp-activation` | Copilot version covers VS Code window reload; Gemini version covers Gemini CLI session restart | +| Project navigation | `project-navigation` | `project-navigation` | Copilot version has full path tables and env vars; Gemini version is a brief reference | +| Plugin dev | `plugin-run-development` | `plugin-development` | Copilot version is comprehensive (data contract, phases, formats); Gemini version is a brief checklist pointing to `docs/PLUGINS_DEV.md` | +| Devcontainer | `devcontainer-services` + `devcontainer-setup` + `devcontainer-configs` | `devcontainer-management` | Copilot splits into 3 focused skills; Gemini combines into one (uses `docker exec`) | + +--- + +## Copilot-Only Skills + +No Gemini equivalent yet: + +| Skill | Purpose | +|-------|---------| +| `api-development` | Creating REST API endpoints | +| `authentication` | API tokens and 401/403 debugging | +| `code-standards` | Coding conventions and style rules | +| `database-patterns` | Device table write paths, SQLite triggers, audit logging, `*Source` attribution | +| `database-reset` | Wipe and regenerate the database and config | +| `docker-build` | Build Docker images for testing or production | +| `docker-prune` | Clean unused Docker resources (destructive — requires confirmation) | +| `sample-data` | Load synthetic device data into the devcontainer | + +--- + +## Gemini-Only Skills + +No Copilot equivalent yet: + +| Skill | Purpose | +|-------|---------| +| `initiative-start` | Research methodology and structured approach for new tasks | + +--- + +## Adding a New Skill + +When adding a skill, create it in **both** directories to keep both AI systems current: + +- `.github/skills//SKILL.md` — add an entry to the skills table in `.github/copilot-instructions.md` +- `.gemini/skills//SKILL.md` — auto-discovered by Gemini CLI via YAML frontmatter + +Keep the body content identical between both files. Only the frontmatter `name`/`description` may differ slightly to match each system's discovery heuristics. diff --git a/back/app.conf b/back/app.conf index 4add3c212..105e4159e 100755 --- a/back/app.conf +++ b/back/app.conf @@ -23,6 +23,8 @@ TIMEZONE='Europe/Berlin' LOADED_PLUGINS=['ARPSCAN', 'AVAHISCAN', 'CSVBCKP','DBCLNP', 'DIGSCAN', 'INTRNT', 'MAINT', 'NEWDEV', 'NBTSCAN', 'NSLOOKUP','NTFPRCS', 'SETPWD', 'SMTP', 'SYNC', 'VNDRPDT', 'WORKFLOWS', 'UI'] DAYS_TO_KEEP_EVENTS=90 +DEV_HIST_DAYS=1 +DEV_HIST_TRACKED=['devMac','devName','devOwner','devType','devVendor','devFavorite','devGroup','devComments','devLastIP','devFQDN','devPrimaryIPv4','devPrimaryIPv6','devVlan','devForceStatus','devStaticIP','devScan','devAlertDown','devCanSleep','devSkipRepeated','devLocation','devIsArchived','devParentMAC','devParentPort','devParentRelType','devReqNicsOnline','devIcon','devSite','devSSID','devSyncHubNode'] # Used for generating links in emails. Make sure not to add a trailing slash! REPORT_DASHBOARD_URL='update_REPORT_DASHBOARD_URL_setting' diff --git a/docs/API_GRAPHQL.md b/docs/API_GRAPHQL.md index cd176f577..ff36a3f2d 100755 --- a/docs/API_GRAPHQL.md +++ b/docs/API_GRAPHQL.md @@ -3,6 +3,7 @@ GraphQL queries are **read-optimized for speed**. Data may be slightly out of date until the file system cache refreshes. The GraphQL endpoints allow you to access the following objects: * Devices +* Device History (change audit log) * Settings * Events * PluginsObjects @@ -415,3 +416,101 @@ curl 'http://host:GRAPHQL_PORT/graphql' \ * Plugin queries scope `dbCount` to the requested `plugin`/`foreignKey` so badge counts reflect per-plugin totals. * The schema is **read-only** — updates must be performed through other APIs or configuration management. See the other [API](API.md) endpoints for details. +--- + +## Device History Queries + +Device field change history is stored in `DevicesHistory` and exposed via two queries. History tracking is controlled by `DEV_HIST_DAYS` (retention window) and `DEV_HIST_TRACKED` (list of audited columns). Set `DEV_HIST_DAYS = 0` to disable tracking entirely. + +### `deviceHistoryGrouped` — Single Device + +Returns grouped change events for one device. Events are grouped by `(timestamp, changedBy)` so that simultaneous field changes are returned as a single object. + +```graphql +query DeviceHistory($devGUID: String!, $changedColumn: String, $changedBy: String, $limit: Int, $offset: Int) { + deviceHistoryGrouped( + devGUID: $devGUID + changedColumn: $changedColumn + changedBy: $changedBy + limit: $limit + offset: $offset + ) { + count + history { + devGUID + timestamp + changedBy + changes { + changedColumn + oldValue + newValue + } + } + } +} +``` + +#### Parameters + +| Parameter | Type | Required | Description | +|-----------------|----------|----------|----------------------------------------------------------------------| +| `devGUID` | `String` | No | GUID of the device to fetch history for | +| `changedColumn` | `String` | No | Filter to groups containing a change to this specific column | +| `changedBy` | `String` | No | Filter to a specific attribution source (`USER`, `ARPSCAN`, etc.) | +| `limit` | `Int` | No | Max grouped events to return (default `50`) | +| `offset` | `Int` | No | Grouped-event offset for pagination (default `0`) | + +#### Response fields + +| Field | Description | +|----------------------------|-----------------------------------------------------------------------------| +| `count` | Total number of grouped events matching the filters (use for pagination) | +| `history[].devGUID` | Device GUID | +| `history[].timestamp` | UTC timestamp of the change event | +| `history[].changedBy` | Attribution: `USER`, plugin prefix (e.g. `ARPSCAN`), or `system` | +| `history[].changes[].changedColumn` | Column name that changed | +| `history[].changes[].oldValue` | Value before the change (`null` for new devices) | +| `history[].changes[].newValue` | Value after the change | + +#### `curl` Example + +```sh +curl -X POST http://localhost:20212/graphql \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer YOUR_API_TOKEN" \ + -d '{ + "query": "query($devGUID:String!,$limit:Int,$offset:Int){deviceHistoryGrouped(devGUID:$devGUID,limit:$limit,offset:$offset){count history{timestamp changedBy changes{changedColumn oldValue newValue}}}}", + "variables": { + "devGUID": "your-device-guid-here", + "limit": 25, + "offset": 0 + } + }' +``` + +--- + + + +--- + +### Filter Values Endpoint + +To populate dynamic filter dropdowns, fetch available `changedBy` and `changedColumn` values via the REST endpoint: + +```sh +GET /devices/history/filters?devGUID= # scoped to one device +GET /devices/history/filters # all devices +``` + +Response: +```json +{ + "success": true, + "data": { + "changedBy": ["ARPSCAN", "USER", "VNDRPDT", "system"], + "changedColumn": ["devLastIP", "devName", "devNameSource", "devVendor"] + } +} +``` + diff --git a/docs/DEVICE_CHANGE_LOG.md b/docs/DEVICE_CHANGE_LOG.md new file mode 100644 index 000000000..c2f3b25f1 --- /dev/null +++ b/docs/DEVICE_CHANGE_LOG.md @@ -0,0 +1,73 @@ +# Change log + +The **_Change log_** provides a historical record of changes made to your devices. It allows you to see **what changed, when it changed, and what caused the change**, making it easier to troubleshoot unexpected updates, audit configuration changes, and understand how device information evolves over time. + +![Device Change log](./img/DEVICE_MANAGEMENT/device_change_log.png) + +## What is recorded? + +Whenever a tracked device property changes, NetAlertX records: + +* **Time** the change occurred +* **Source** that made the change +* **Device** that was updated +* **Field** that changed +* **Previous value** +* **New value** + +Related changes that occur at the same time are grouped together into a single event, making it easy to see all updates from one scan or user action. + +## Understanding the Source column + +The **Source** column identifies what caused the change. + +Depending on the field, the source may be: + +* A discovery plugin such as **ARPSCAN**, **NSLOOKUP**, **UNIFIAPI**, or another plugin that supplied the information. +* **user:api** for changes made manually through the NetAlertX interface or API. +* **system** for values calculated or maintained internally by NetAlertX. + +This helps distinguish between information discovered automatically and changes made by users. + +## Filtering the history + +The _Change log_ includes filters to help locate specific events. + +You can filter by: + +* **Source** to view changes made by a specific plugin or user. +* **Changed Field** to display changes to a particular device property, such as IP address or hostname. + +You can also use the search box to quickly find matching devices by GUID, values, or fields. + +## Viewing grouped changes + +A single action often updates multiple properties on a device. + +Instead of showing each field as a separate record, the _Change log_ groups related updates together. For example, a network scan might update both a device's IP address and hostname at the same time. These changes appear together in one entry with each modified field listed underneath. + +## Retention + +The amount of history retained is controlled by the **Device History Days (`DEV_HIST_DAYS`)** setting. + +Older history entries are automatically removed once they exceed the configured retention period. this is maintained by the `DBCLNP` plugin. + +Setting this value to **0** disables _Change log_ recording entirely. + +## Choosing what to track + +Not every device property needs to be recorded. + +The **Tracked Device History Fields (`DEV_HIST_TRACKED`)** setting lets you choose exactly which device fields should be monitored. Tracking only the fields that are important for your environment can significantly reduce database growth while keeping the most useful audit information. + +## Performance considerations + +Recording history introduces additional database writes whenever tracked fields change. For most installations the overhead is minimal, but on larger networks it can generate a substantial amount of historical data. + +If you want to reduce storage usage or improve performance, consider: + +* Tracking only the device fields that matter to you. +* Reducing the history retention period. +* Disabling the _Change log_ entirely by setting **`DEV_HIST_DAYS`** to **0**. + +These settings allow you to balance historical visibility with database size and performance. For more tips on how to optimize system resource use check the [performance guide](./PERFORMANCE.md). diff --git a/docs/DEVICE_PRESENCE.md b/docs/DEVICE_PRESENCE.md new file mode 100644 index 000000000..89133678e --- /dev/null +++ b/docs/DEVICE_PRESENCE.md @@ -0,0 +1,45 @@ +## Presence View + +![Device presence](./img/DEVICE_MANAGEMENT/device_presence.png) + +The **Presence View** provides a visual timeline of when each device was detected on your network, making it easy to understand activity over time instead of relying only on the current status. + +Use this view to answer questions such as: + +* When was this device first discovered? +* When was it last seen online? +* How often does this device connect to the network? +* Has the device been continuously online or only appearing intermittently? +* When did the device disappear or return? + +The timeline makes it easy to identify recurring activity patterns, unexpected outages, or devices that only appear at certain times of the day or week. + +### Timeline Views + +The timeline can be displayed at different time scales depending on the level of detail you need: + +* **Week** – View recent activity with the highest level of detail. +* **Month** – Review activity across the past month. +* **Year** – Get a long-term overview of device presence and seasonal patterns. + +Switch between these views using the **Week**, **Month**, and **Year** buttons above the timeline. + +### Timeline Legend + +The timeline uses color coding to indicate device presence: + +*
**Now online** – The device was detected during the most recent scan and is currently online. +*
**Past online** – The device was online previously but is currently offline. +*
**Past online (mismatch)** – Historical activity was detected, but the beginning of the session could not be determined or conflicting data was encountered. + +### Typical Uses + +The Presence View is especially useful for: + +* Troubleshooting intermittent connectivity issues. +* Identifying devices that frequently disconnect and reconnect. +* Understanding occupancy or usage patterns for phones, laptops, IoT devices, and other network equipment. +* Verifying when visitors' devices or temporary equipment were present on the network. +* Reviewing historical presence without searching through individual events or logs. + +Unlike the Device Details page, which shows the current state of a device, the Presence View focuses on **when** devices were active, providing valuable historical context at a glance. To investigate exactly **what** changed during a device's lifetime, see the [Change Log](./DEVICE_CHANGE_LOG.md). diff --git a/docs/PERFORMANCE.md b/docs/PERFORMANCE.md index 8abd4d349..a6c74f6dd 100755 --- a/docs/PERFORMANCE.md +++ b/docs/PERFORMANCE.md @@ -132,6 +132,14 @@ When **any** name-resolution plugin has `devName` in its `SET_ALWAYS` list, the The actual number of DB rows updated is logged at `verbose` level under `[Update Device Name] SET_ALWAYS re-resolve - DB rows updated`. + +--- + +## Disable change log tracking + +To improve performance, you can reduce or disable change log tracking. Use the `DEV_HIST_TRACKED` setting to limit tracking to specific columns, and adjust `DEV_HIST_DAYS` to control how long history is retained. Setting `DEV_HIST_DAYS` to `0` disables history tracking entirely. + + --- ## Storing Temporary Files in Memory diff --git a/docs/SETTINGS_SYSTEM.md b/docs/SETTINGS_SYSTEM.md index bcc0f46c8..0991da46d 100755 --- a/docs/SETTINGS_SYSTEM.md +++ b/docs/SETTINGS_SYSTEM.md @@ -36,6 +36,37 @@ Plugin settings are loaded dynamically from the `config.json` of individual plug ![Screen 1][screen1] +### Localization + +Every setting must have localized language strings. + +If the language strings are **not** defined directly inside `config.json`, they must exist in `en_us.json`. + +For example, for: + +```text +UI_DEV_SECTIONS +``` + +add: + +```json +"UI_DEV_SECTIONS_name": "Hide device sections", +"UI_DEV_SECTIONS_description": "Select which UI elements to hide on the Devices page." +``` + +When using external language files, `config.json` must reference: + +```json +"name": [ + { + "string": "_GLOBAL_LANG_FILES_" + } +] +``` + +This tells NetAlertX to resolve the display name from the language files. + ### Settings Process flow The process flow is mostly managed by the [initialise.py](/server/initialise.py) file. diff --git a/docs/WHERE_NETALERTX_FITS.md b/docs/WHERE_NETALERTX_FITS.md new file mode 100644 index 000000000..5f3dd7150 --- /dev/null +++ b/docs/WHERE_NETALERTX_FITS.md @@ -0,0 +1,314 @@ +# Where NetAlertX Fits + +Understanding where NetAlertX fits in your network management toolkit makes it easier to get the most from it. + +Modern networks are typically managed using several specialized tools. One application may monitor uptime, another may capture packets, another may enforce firewall rules, while another documents the network. + +NetAlertX fills a different role: it provides **continuous network visibility** by discovering devices, maintaining an accurate asset inventory, detecting changes, and notifying you when something important happens. + +It answers one of the most common questions in network administration: + +> **What is on my network, and what has changed?** + +--- + +## What NetAlertX Does + +NetAlertX continuously discovers devices using one or more discovery plugins and builds a living inventory of your network. + +Depending on your configuration, it can collect information including: + +* IP addresses +* MAC addresses +* Hostnames +* Vendors +* Device types +* Open ports +* Network interfaces +* Parent/child relationships +* First and last seen timestamps +* Custom metadata + +Because discovery runs continuously, your inventory stays up to date automatically. + +Learn more about the available discovery methods in the [Plugins](./PLUGINS.md) documentation. + +--- + +## Detecting Change + +Knowing what exists on your network is useful. + +Knowing **what changed** is often even more valuable. + +NetAlertX continuously records changes such as: + +* New devices +* Devices going offline or coming back online +* IP address changes +* Hostname changes +* Vendor identification +* Open port changes (via the `NMAP` [plugin](./PLUGINS.md)) +* Devices returning after long periods offline +* Configuration updates + +These changes become events that can be viewed historically, filtered, searched, or used to trigger automation. + +--- + +## Notifications + +![Global notification settings](./img/NOTIFICATIONS/Global-notification-settings.png) + +Not every event deserves immediate attention. + +NetAlertX lets you choose which events [generate notifications](./NOTIFICATIONS.md) so that important alerts stand out without creating unnecessary noise. + +Examples include: + +* Unknown devices joining the network +* Critical infrastructure becoming unavailable +* IP address changes +* Port changes +* Custom workflow events + +Notification publishers can be enabled independently, allowing NetAlertX to integrate with your existing communication channels. + +--- + +## Automation + +![Workflow Action target](./img/WORKFLOWS/actions_target.png) + +As networks grow, manually responding to every event quickly becomes impractical. + +NetAlertX includes a [workflow engine](./WORKFLOWS.md) that can automatically react to events. + +Typical automations include: + +* Mark trusted devices automatically +* Assign devices to groups +* Update fields +* Archive old devices with the same IP +* Assign devices to the right parent node + +Automation helps keep device management consistent while reducing repetitive administrative work. + +--- + +## Device Relationships + +![Hover relationships detail](./img/NETWORK_TREE/Network_tree_setup_hover.png) + +Networks are more than collections of independent devices. + +Devices often depend on one another. + +Examples include: + +* Access points connected to switches +* Virtual machines hosted by physical servers +* Containers running on hosts +* Network interfaces belonging to a parent device +* IoT gateways connected to sensors + +NetAlertX supports [device relationships](./NETWORK_TREE.md) to better represent how devices are connected and how they depend on each other. + +--- + +## Historical Visibility + +![Device presence](./img/DEVICE_MANAGEMENT/device_presence.png) + +Many monitoring systems focus only on the current state. + +NetAlertX also records historical information, helping you understand how devices behave over time. + +You can answer questions like: + +* When was this device first discovered? +* When was it last online? +* How does its network presence change over time? +* Which configuration attributes have changed? + +Historical data provides operational context without requiring a separate logging platform. + +Two key views help explore this history: + +* **Presence View** – when a device was active on the network +* **Change log** – what changed in the device’s configuration over time + +--- + +## Change log + +![Device change log](./img/DEVICE_MANAGEMENT/device_change_log.png) + +The **Change log** records all tracked changes to device attributes, showing what changed and when it happened. + +Each entry includes: + +* Timestamp of the change +* Previous and new values +* Source of the change (plugin, workflow, or user action) + +You can filter, search, and sort changes to quickly investigate events such as IP changes, hostname updates, VLAN changes, or device classification updates. + +| View | Answers | +| ----------------- | ------------------- | +| **Presence View** | When was it active? | +| **Change log** | What changed? | + + +--- + +## Plugin Architecture + +![Action target](./img/plugins_json_settings.png) + +Every environment is different. + +Some users prefer simple ARP scanning. + +Others rely on: + +* SNMP +* DHCP leases +* Router integrations +* Pi-hole +* Virtualization platforms +* Cloud services +* External asset inventories + +NetAlertX is built around a plugin architecture that allows multiple discovery sources to work together, improving accuracy while remaining flexible. + +See the complete list in [Plugins](./PLUGINS.md). + +--- + +## Integrations + +NetAlertX is designed to complement your existing infrastructure rather than replace it. + +It integrates with many popular platforms, including: + +* Home Assistant +* Pi-hole +* MQTT +* Notification services +* REST APIs +* Custom workflows + +This allows NetAlertX to become part of your existing automation and monitoring ecosystem. + +--- + +## How NetAlertX Compares + +The following diagram illustrates how NetAlertX complements other common network tools. + +| Need | Typical Tool | NetAlertX Role | +| --- | --- | --- | +| Device discovery | ✅ | **Core capability** | +| Asset inventory | ✅ | **Core capability** | +| Detect network changes | ✅ | **Core capability** | +| Historical device tracking | ✅ | **Core capability** | +| Notifications | ✅ | **Core capability** | +| Automation | ✅ | **Core capability** | +| Packet inspection | Wireshark | Not intended | +| Firewall enforcement | pfSense, OPNsense | Not intended | +| IDS/IPS | Suricata, Snort | Not intended | +| Network documentation | NetBox, NetMap | Complements them | +| Uptime monitoring | Uptime Kuma, Nagios | Complements them | + +NetAlertX works best alongside these tools, providing the continuously updated device inventory that many monitoring platforms lack. + +--- + +## Typical Deployments + +### 🏠 Home Networks + +Monitor: + +* Family devices +* Smart home equipment +* NAS systems +* TVs and media devices +* Game consoles +* IoT devices + +Receive alerts whenever new or unknown devices join your network. + +--- + +### 🧪 Homelabs + +Track constantly changing environments including: + +* Docker containers +* Virtual machines +* Kubernetes nodes +* Development systems +* Test networks + +Maintain an accurate inventory without manual documentation. + +--- + +### 🏢 Small Businesses + +Maintain visibility across office networks by tracking: + +* Unauthorized devices and shadow IT +* Office infrastructure and critical core systems +* Real-time asset inventory drift + +--- + +### 🌍 Remote Sites + +Deploy NetAlertX at branch offices or remote locations to maintain local visibility while monitoring multiple sites centrally. + +--- + +## What NetAlertX Is Designed To Do + +NetAlertX is designed to excel at: + +* Continuous device discovery +* Asset inventory +* Network visibility +* Change detection +* Historical tracking +* Notifications +* Workflow automation +* Plugin extensibility + +By focusing on these capabilities, NetAlertX remains lightweight, flexible, and easy to integrate into existing environments. + +--- + +## What NetAlertX Is Not + +NetAlertX is **not** intended to replace: + +* Firewalls +* Packet analyzers +* IDS/IPS platforms +* Configuration management systems +* Network documentation platforms +* Dedicated uptime monitoring systems + +Instead, it complements these tools by providing continuous awareness of the devices connected to your network. + +--- + +## Next Steps + +Now that you understand where NetAlertX fits, continue with one of the following guides: + +* **[Installation](INSTALLATION.md)** — Install NetAlertX on your platform. +* **[Plugins](./PLUGINS.md)** — Configure device discovery. +* **[Features](FEATURES.md)** — Explore everything NetAlertX can do. +* **[API](API.md)** — Integrate NetAlertX with your own applications. \ No newline at end of file diff --git a/docs/img/DEVICE_MANAGEMENT/device_change_log.png b/docs/img/DEVICE_MANAGEMENT/device_change_log.png new file mode 100644 index 000000000..7166d7d92 Binary files /dev/null and b/docs/img/DEVICE_MANAGEMENT/device_change_log.png differ diff --git a/docs/img/DEVICE_MANAGEMENT/device_presence.png b/docs/img/DEVICE_MANAGEMENT/device_presence.png new file mode 100644 index 000000000..493a31d3c Binary files /dev/null and b/docs/img/DEVICE_MANAGEMENT/device_presence.png differ diff --git a/docs/index.md b/docs/index.md index 720842a29..dc8e80e98 100755 --- a/docs/index.md +++ b/docs/index.md @@ -56,6 +56,23 @@ hide: --- +## What is NetAlertX? + +NetAlertX continuously discovers the devices on your network, builds a living asset inventory, detects changes, and alerts you when something important happens. + +Whether you're monitoring a home network, homelab, business, or remote site, NetAlertX helps answer questions like: + +- What devices are currently connected? +- Has a new or unknown device appeared? +- Which servers or services are offline? +- Has a device changed IP address, hostname, or open ports? +- When was a device first or last seen? + +Unlike packet analyzers or firewall appliances, NetAlertX focuses on **network visibility**, **asset discovery**, **change detection**, **historical tracking**, and **automation**. + +> [!TIP] +> **New to NetAlertX?** Start with **[Where NetAlertX Fits](./WHERE_NETALERTX_FITS)** to understand how it complements the rest of your network tools. + ## Help and Support If you need help or run into issues, here are some resources to guide you: diff --git a/docs/stylesheets/custom.css b/docs/stylesheets/custom.css index a8585d21a..f7105106c 100644 --- a/docs/stylesheets/custom.css +++ b/docs/stylesheets/custom.css @@ -14,6 +14,19 @@ * limitations under the License. */ + :root { + --color-white: #fff; + --color-aqua: #00c0ef; + --color-lightblue: #3c8dbc; + --color-blue: #0060df; + --color-green: #00a65a; + --color-yellow: #f39c12; + --color-red: #dd4b39; + --color-gray: #8c8c8c; + --color-darkgray: #3f3e3e; + --color-lightgray: #bec5cb; + --color-black: #000; +} /* --- HERO SECTION --- */ .hero-wrapper { @@ -143,3 +156,25 @@ p.hero-description { .learn-more-btn:hover { opacity: 0.8; } + +/* key presence styles */ +.presenceOnlineNow{ + background-color: var(--color-green); +} + +.presenceOnlinePast{ + background-color: var(--color-blue); +} + +.presenceOnlinePastMiss{ + background-color: var(--color-yellow); +} + +.presenceKeyBoxes +{ + width: 10px; + height: 10px; + display: inline-block; + /* background: #fff; */ + opacity: .75; +} diff --git a/front/changeLog.php b/front/changeLog.php new file mode 100644 index 000000000..92aa75822 --- /dev/null +++ b/front/changeLog.php @@ -0,0 +1,11 @@ + +
+ +
+ + + diff --git a/front/changeLogCore.php b/front/changeLogCore.php new file mode 100644 index 000000000..4b47f52a6 --- /dev/null +++ b/front/changeLogCore.php @@ -0,0 +1,409 @@ + + + + +
+ + + + + + + +
+
+
+ + +
+ + + + + +
+ + +
+ + + + + + + + + + + + + + + +
+ +
+
+ +
+
+
+
+ + \ No newline at end of file diff --git a/front/css/app.css b/front/css/app.css index b851c5ed6..facd9d994 100755 --- a/front/css/app.css +++ b/front/css/app.css @@ -1001,6 +1001,19 @@ height: 50px; opacity: .75; } +#presencePage .helpIcon{ + top: 2px; + display: initial; + font-size: x-small; +} + +.calendarHelpIcon{ + display: block; + position: absolute; + font-size: x-small; + top: -5px; +} + /* --------------------------------------------------------- */ /* report */ /* --------------------------------------------------------- */ @@ -2206,6 +2219,12 @@ textarea[readonly], padding-top: 20px; } +.pluginsCore .helpIcon +{ + margin-left: -20px; + font-size: x-small; +} + /*Hidden special button*/ @media (max-width: 365px) { @@ -2636,7 +2655,29 @@ table.dataTable tbody > tr.selected background-color: rgba(140, 140, 140, 0.05); } -/* ===== Loading Skeleton ===== */ +/* ----------------------------------------------------------------------------- + Change log +----------------------------------------------------------------------------- */ + +.change-log .helpIcon{ + left: 15px; + font-size: x-small; +} + +#panHistory +{ + min-height:850px +} + +#panHistory .change-log-skeleton-class +{ + margin-left: 0px; + width: calc(100% - 40px); +} + +/* ----------------------------------------------------------------------------- + Loading Skeletons +----------------------------------------------------------------------------- */ /* Custom properties (overridden in dark-patch.css for Dark/System themes) */ :root { @@ -2677,7 +2718,7 @@ table.dataTable tbody > tr.selected /* Page-level overlay base */ #settings-skeleton, #devices-skeleton, #device-details-skeleton, -#events-skeleton, #presence-skeleton, #report-skeleton, #skel-app-events, +#events-skeleton, #presence-skeleton, #report-skeleton, #skel-app-events, #change-log-skeleton #notifications-skeleton, #workflows-skeleton, #plugins-skeleton, .skel-tab-pane { position: absolute; top: 0; @@ -2699,6 +2740,7 @@ table.dataTable tbody > tr.selected #notifications-skeleton { margin: 15px; top: 50px; min-height: 400px; } #workflows-skeleton { top: 50px; width: 770px; margin: 0 auto; min-height: 300px; } #plugins-skeleton { top: 50px; padding: 0; min-height: 400px; } +#change-log-skeleton { display: block; min-height: 400px; z-index: 1; position: absolute; margin-left: 15px; padding-top: 15px; width:100%;} #skel-tab-sessions, #skel-tab-events, #skel-tab-presence {margin: -10px; padding: 20px;} /* Hide scrollbars */ diff --git a/front/deviceDetails.php b/front/deviceDetails.php index e6236527d..5e0b76e43 100755 --- a/front/deviceDetails.php +++ b/front/deviceDetails.php @@ -111,6 +111,14 @@ +
  • + + + + + + +
  • - +
    + +
    @@ -596,9 +609,17 @@ function hideDeviceDetailsSkeleton() { if (typeof hideSessionsTabSkeleton === 'function') hideSessionsTabSkeleton(); if (typeof hidePresenceTabSkeleton === 'function') hidePresenceTabSkeleton(); if (typeof hideEventsTabSkeleton === 'function') hideEventsTabSkeleton(); + $('#skel-tab-history').hide(); }, 15000); } + +// Load history when the tab becomes visible +$(document).on('shown.bs.tab', 'a[id="tabHistory"]', function() { + _histPage = 0; + // loadHistoryData(); +}); + diff --git a/front/deviceDetailsPresence.php b/front/deviceDetailsPresence.php index 31d4741f6..36d5915cd 100755 --- a/front/deviceDetailsPresence.php +++ b/front/deviceDetailsPresence.php @@ -21,6 +21,11 @@ + + + + +
    diff --git a/front/js/cache.js b/front/js/cache.js index dcae0d188..8fb675e64 100644 --- a/front/js/cache.js +++ b/front/js/cache.js @@ -509,6 +509,45 @@ function getDevDataByMac(macAddress, dbColumn) { return null; // Return a default value if MAC address is not found } + +// ----------------------------------------------------------------------------- +// Get a device property using GUID as key and DB column name as parameter +function getDevDataByGuid(devGUID, dbColumn) { + + const sessionDataKey = CACHE_KEYS.DEVICES_ALL; + const devicesCache = getCache(sessionDataKey); + + if (!devicesCache || devicesCache == "") { + console.warn(`[getDevDataByGuid] Cache key "${sessionDataKey}" is empty — cache may not be initialized yet.`); + return null; + } + + const devices = parseDeviceCache(devicesCache); + + if (devices.length === 0) { + return null; + } + + const guid = devGUID?.toLowerCase(); + + + for (const device of devices) { + + if ((device["devGUID"] || "").toLowerCase() === guid) { + + if (dbColumn) { + return device[dbColumn]; + } + + return device; + } + } + + console.error("⚠ Device with GUID not found: " + devGUID); + return null; +} + + // ----------------------------------------------------------------------------- /** * Fetches the full device list from table_devices.json and stores it in diff --git a/front/js/common.js b/front/js/common.js index 006bad8fe..765d34387 100755 --- a/front/js/common.js +++ b/front/js/common.js @@ -1175,6 +1175,82 @@ function isSQLQuery(query) { return sqlRegex.test(query); } +/** + * Lazy tab initializer + * - Ensures a tab is only initialized once + * - Waits until tab is visible in DOM + * - Uses polling (compatible with your existing architecture) + */ + +function waitForAppReady(cb) { + + if (window.__appReady) { + cb(); + return; + } + + const t = setInterval(() => { + + if (window.__appReady) { + clearInterval(t); + cb(); + } + + }, 50); +} + +function initLazyTab(selector, initFn, intervalMs = 200) { + + const stateKey = `__lazy_init_${selector}`; + let initialized = false; + + function runner() { + + // prevent duplicate execution + if (initialized || window[stateKey]) return; + + const $el = $(selector); + + // --------------------------------------------------------- + // CASE 1: element does not exist → standalone page variant + // run immediately (no DOM dependency) + // --------------------------------------------------------- + if (!$el.length) { + initialized = true; + window[stateKey] = true; + + try { + initFn(); + } catch (e) { + console.error("Lazy init (no-element) failed:", selector, e); + } + return; + } + + // --------------------------------------------------------- + // CASE 2: element exists AND is visible → run immediately + // --------------------------------------------------------- + if ($el.is(':visible')) { + initialized = true; + window[stateKey] = true; + + try { + initFn(); + } catch (e) { + console.error("Lazy init (visible) failed:", selector, e); + } + return; + } + + // --------------------------------------------------------- + // CASE 3: element exists but hidden → wait for visibility + // (tab mode) + // --------------------------------------------------------- + setTimeout(runner, intervalMs); + } + + runner(); +} // ------------------------------------------------------------------- // Get corresponding plugin setting object diff --git a/front/js/ui_components.js b/front/js/ui_components.js index 917ff8607..8d3bdc1c3 100755 --- a/front/js/ui_components.js +++ b/front/js/ui_components.js @@ -1164,5 +1164,5 @@ function getLengthMenu(newEntry) { return [values, labels]; } - +window.__appReady = true; console.log("init ui_components.js") \ No newline at end of file diff --git a/front/php/templates/header.php b/front/php/templates/header.php index 67d453874..577ba986e 100755 --- a/front/php/templates/header.php +++ b/front/php/templates/header.php @@ -332,14 +332,14 @@ function update_servertime() { -