From acfcd295c491d2d6bba5f5c5784301eac9147677 Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Wed, 27 May 2026 21:25:49 -0500 Subject: [PATCH 01/40] Add Huntress.io integration --- huntress/CHANGELOG.md | 10 + huntress/README.md | 171 +++ huntress/assets/configuration/spec.yaml | 78 ++ .../dashboards/huntress_siem_overview.json | 1038 +++++++++++++++++ .../monitors/huntress_siem_check_failed.json | 39 + .../monitors/huntress_siem_errors_high.json | 36 + .../huntress_siem_no_logs_collected.json | 35 + huntress/assets/service_checks.json | 14 + huntress/checks.d/huntress.py | 549 +++++++++ huntress/conf.d/huntress.yaml.example | 56 + huntress/datadog_checks/__init__.py | 1 + huntress/datadog_checks/huntress/__about__.py | 1 + huntress/datadog_checks/huntress/__init__.py | 4 + .../huntress/data/conf.yaml.example | 56 + huntress/datadog_checks/huntress/huntress.py | 549 +++++++++ huntress/hatch.toml | 7 + huntress/images/huntress-logo.png | Bin 0 -> 28877 bytes huntress/manifest.json | 77 ++ huntress/metadata.csv | 7 + huntress/pyproject.toml | 65 ++ huntress/tests/__init__.py | 0 huntress/tests/conftest.py | 49 + huntress/tests/data/huntress_mockoon.json | 549 +++++++++ huntress/tests/docker/Dockerfile | 10 + huntress/tests/docker/docker-compose.yml | 10 + huntress/tests/fixtures/siem_query_empty.json | 4 + huntress/tests/fixtures/siem_query_page1.json | 29 + huntress/tests/fixtures/siem_query_page2.json | 16 + huntress/tests/test_docker.py | 59 + huntress/tests/test_huntress.py | 660 +++++++++++ huntress/tests/test_unit.py | 37 + 31 files changed, 4216 insertions(+) create mode 100644 huntress/CHANGELOG.md create mode 100644 huntress/README.md create mode 100644 huntress/assets/configuration/spec.yaml create mode 100644 huntress/assets/dashboards/huntress_siem_overview.json create mode 100644 huntress/assets/monitors/huntress_siem_check_failed.json create mode 100644 huntress/assets/monitors/huntress_siem_errors_high.json create mode 100644 huntress/assets/monitors/huntress_siem_no_logs_collected.json create mode 100644 huntress/assets/service_checks.json create mode 100644 huntress/checks.d/huntress.py create mode 100644 huntress/conf.d/huntress.yaml.example create mode 100644 huntress/datadog_checks/__init__.py create mode 100644 huntress/datadog_checks/huntress/__about__.py create mode 100644 huntress/datadog_checks/huntress/__init__.py create mode 100644 huntress/datadog_checks/huntress/data/conf.yaml.example create mode 100644 huntress/datadog_checks/huntress/huntress.py create mode 100644 huntress/hatch.toml create mode 100644 huntress/images/huntress-logo.png create mode 100644 huntress/manifest.json create mode 100644 huntress/metadata.csv create mode 100644 huntress/pyproject.toml create mode 100644 huntress/tests/__init__.py create mode 100644 huntress/tests/conftest.py create mode 100644 huntress/tests/data/huntress_mockoon.json create mode 100644 huntress/tests/docker/Dockerfile create mode 100644 huntress/tests/docker/docker-compose.yml create mode 100644 huntress/tests/fixtures/siem_query_empty.json create mode 100644 huntress/tests/fixtures/siem_query_page1.json create mode 100644 huntress/tests/fixtures/siem_query_page2.json create mode 100644 huntress/tests/test_docker.py create mode 100644 huntress/tests/test_huntress.py create mode 100644 huntress/tests/test_unit.py diff --git a/huntress/CHANGELOG.md b/huntress/CHANGELOG.md new file mode 100644 index 0000000000..461b8c6ce0 --- /dev/null +++ b/huntress/CHANGELOG.md @@ -0,0 +1,10 @@ +# CHANGELOG - Huntress + +## Unreleased + +**_Added_**: + +- Initial release of the Huntress SIEM integration for Datadog. Polls the Huntress + Managed SIEM API via ES|QL, enriches logs with organization metadata, and forwards + all events to Datadog Logs. Emits collection health metrics and a pre-built overview + dashboard with monitor templates. diff --git a/huntress/README.md b/huntress/README.md new file mode 100644 index 0000000000..76592d4c0f --- /dev/null +++ b/huntress/README.md @@ -0,0 +1,171 @@ +# Huntress + +## Overview + +[Huntress](https://www.huntress.com/) is a managed security platform providing endpoint detection and response (EDR), antivirus, security awareness training, and a Managed SIEM product that continuously collects and analyzes endpoint telemetry. + +This integration polls the Huntress Managed SIEM API using ES|QL queries and forwards all security events to Datadog as logs. Each collection run: + +1. Loads a checkpoint — the timestamp of the last successful collection +2. Executes a configurable ES|QL query for the elapsed time window +3. Paginates through all result pages +4. Optionally enriches each log with Huntress organization metadata (org name, key, account ID) +5. Forwards logs to Datadog preserving all Elastic Common Schema (ECS) field names +6. Advances the checkpoint only after all pages are successfully sent + +This integration is designed for managed security providers (MSPs) and enterprise teams who want to correlate Huntress threat detections alongside infrastructure and application telemetry in Datadog. + +## Setup + +### Prerequisites + +- Datadog Agent 7.x or later +- A Huntress account with the Managed SIEM feature enabled +- Huntress API credentials (public API key + secret key) from the Huntress Partner Portal under **Settings > API Credentials** + +### Installation + +Install the integration package from the Agent: + +```bash +datadog-agent integration install -t datadog-huntress==1.0.0 +``` + +### Configuration + +1. Create the configuration file at `/etc/datadog-agent/conf.d/huntress.yaml` (Linux/macOS) or `C:\ProgramData\Datadog\conf.d\huntress.yaml` (Windows). A fully-annotated example is at `datadog_checks/huntress/data/conf.yaml.example`. + +2. Edit `huntress.yaml` with your credentials: + + ```yaml + init_config: {} + + instances: + - huntress_api_key: "" + huntress_secret_key: "" + esql_query: "FROM logs" + tags: + - "source:huntress" + - "service:huntress-siem" + - "env:production" + ``` + +3. Restart the Agent: + + ```bash + # Linux (systemd) + sudo systemctl restart datadog-agent + + # macOS + sudo launchctl stop com.datadoghq.agent && sudo launchctl start com.datadoghq.agent + ``` + +4. Verify the check: + + ```bash + sudo datadog-agent check huntress + ``` + + Logs appear in Datadog Log Explorer filtered by `source:huntress` within one collection interval (default: 15 minutes). + +### Multiple Huntress accounts + +Add additional blocks under `instances:` — each runs independently with its own checkpoint, org metadata cache, and metrics: + +```yaml +instances: + - huntress_api_key: "" + huntress_secret_key: "" + esql_query: "FROM logs" + tags: ["source:huntress", "env:production"] + + - huntress_api_key: "" + huntress_secret_key: "" + esql_query: "FROM logs" + tags: ["source:huntress", "env:staging"] +``` + +### Configuration reference + +| Field | Required | Default | Description | +| ------------------------- | -------- | ------------------------- | ----------------------------------------- | +| `huntress_api_key` | Yes | — | Huntress public API key | +| `huntress_secret_key` | Yes | — | Huntress secret API key | +| `esql_query` | Yes | — | ES\|QL query; must begin with `FROM logs` | +| `enrich_with_org_tags` | No | `true` | Fetch and attach org metadata as log tags | +| `org_cache_ttl_seconds` | No | `3600` | How long to cache org metadata (seconds) | +| `min_collection_interval` | No | `900` | Seconds between runs (minimum 60) | +| `max_pages_per_run` | No | `100` | Page cap per run (~20,000 logs maximum) | +| `huntress_base_url` | No | `https://api.huntress.io` | Override for sandbox environments | +| `tags` | No | `[]` | Extra tags on every forwarded log | + +## Data Collected + +### Logs + +All logs collected from the Huntress Managed SIEM API are forwarded to Datadog with: + +- `ddsource: huntress` — enables automatic log pipeline processing +- ECS field names preserved as top-level log attributes (for example, `event.category`, `host.hostname`, `user.name`) +- Organization metadata tags when `enrich_with_org_tags: true` (for example, `huntress_org_name`, `huntress_org_key`, `huntress_account_id`) + +### Metrics + +| Metric | Type | Description | +| ------------------------------------ | ----- | ------------------------------------------------------- | +| `huntress.siem.logs_collected` | Gauge | Log events collected per run | +| `huntress.siem.pages_fetched` | Gauge | API pages fetched per run | +| `huntress.siem.run_duration_seconds` | Gauge | Wall time of the collection run | +| `huntress.siem.errors` | Count | Errors by type (`error_type` tag) | +| `huntress.siem.api_call_limit` | Gauge | Total API requests allowed per minute (from Huntress) | +| `huntress.siem.api_call_remaining` | Gauge | API requests remaining in the current minute | + +See [metadata.csv][1] for a full list of metrics. + +### Events + +The Huntress integration does not emit Datadog events. + +### Service Checks + +**`huntress.siem.check_status`** +Returns `CRITICAL` if the collection run fails for any reason; `OK` otherwise. + +## Troubleshooting + +**No logs in Datadog after first run** + +- Run `sudo datadog-agent check huntress` and inspect the output +- Verify the API key pair is valid by checking the Huntress Partner Portal +- Confirm the Managed SIEM feature is enabled on the account +- Check that `esql_query` begins with `FROM logs` + +**`huntress.siem.errors` count is increasing** + +Inspect the `error_type` tag to identify the root cause: + +| `error_type` | Cause | Resolution | +| ------------------ | ---------------------------------- | ------------------------------------------------------------ | +| `auth_failure` | Invalid or rotated API credentials | Update `huntress_api_key` / `huntress_secret_key` | +| `timeout` | ES\|QL query too broad | Add a `KEEP` or `WHERE` clause to the query | +| `invalid_query` | Malformed ES\|QL | Fix the `esql_query` value | +| `server_error` | Transient Huntress API error | Check [Huntress status page](https://status.huntress.com) | +| `connection_error` | Network issue | Verify connectivity from the Agent host to `api.huntress.io` | +| `run_failure` | Unexpected error during collection | Check Agent logs for the full stack trace | + +**`huntress.siem.api_call_remaining` is very low or zero** + +The Huntress API allows 60 requests per minute. The integration logs a warning when fewer than 10 requests remain in a given minute. If this happens regularly, reduce `max_pages_per_run` or increase `min_collection_interval` to spread out collection runs. + +**Duplicate logs after Agent restart** + +This is expected on the first restart after a failed run — the checkpoint is only advanced when all pages are successfully sent. Subsequent runs resume from the last successful checkpoint. + +Need help? Contact [Datadog support][2]. + +## Support + +For questions and support, [contact Datadog support][2]. + +[1]: https://github.com/DataDog/integrations-extras/blob/master/huntress/metadata.csv +[2]: https://docs.datadoghq.com/help/ diff --git a/huntress/assets/configuration/spec.yaml b/huntress/assets/configuration/spec.yaml new file mode 100644 index 0000000000..f0b3e57bc9 --- /dev/null +++ b/huntress/assets/configuration/spec.yaml @@ -0,0 +1,78 @@ +name: Huntress +files: +- name: huntress.yaml + options: + - template: init_config + options: + - template: init_config/default + - template: instances + options: + - name: huntress_api_key + required: true + description: | + Huntress public API key. Obtain from the Huntress Partner Portal under + Settings > API Credentials. Each instance uses its own key pair, enabling + multiple Huntress accounts to feed into the same Datadog organization. + value: + type: string + example: + - name: huntress_secret_key + required: true + description: | + Huntress private (secret) API key paired with huntress_api_key. + value: + type: string + example: + - name: esql_query + required: true + description: | + ES|QL query to execute against the Huntress SIEM API. Must begin with + "FROM logs" (case-insensitive). Use KEEP to select specific ECS columns + or omit for all fields. Do not add time range filters — range_start and + range_end are managed automatically by the check. + value: + type: string + example: "FROM logs" + - name: enrich_with_org_tags + description: | + When true, the check fetches Huntress organization metadata (org name, + org key, account ID) and attaches it as log tags. The org cache is + refreshed according to org_cache_ttl_seconds. + value: + type: boolean + example: true + default: true + - name: org_cache_ttl_seconds + description: | + How long (in seconds) to cache organization metadata before re-fetching + from the Huntress API. Set to 0 to refresh on every run. + value: + type: integer + example: 3600 + default: 3600 + - name: min_collection_interval + description: | + Seconds between check runs. Must be >= 60 to stay within Huntress API + rate limits (60 requests per minute). Defaults to 900 (15 minutes). + value: + type: integer + example: 900 + default: 900 + - name: max_pages_per_run + description: | + Maximum number of result pages to fetch per run. Each page contains up + to 200 log events. Protects against runaway pagination on large backlogs. + If this cap is hit, the checkpoint does not advance and remaining pages + are fetched on the next run. + value: + type: integer + example: 100 + default: 100 + - name: huntress_base_url + description: | + Huntress API base URL. Override for sandbox or testing environments. + value: + type: string + example: "https://api.huntress.io" + default: "https://api.huntress.io" + - template: instances/default diff --git a/huntress/assets/dashboards/huntress_siem_overview.json b/huntress/assets/dashboards/huntress_siem_overview.json new file mode 100644 index 0000000000..07dae11a93 --- /dev/null +++ b/huntress/assets/dashboards/huntress_siem_overview.json @@ -0,0 +1,1038 @@ +{ + "title": "Huntress SIEM Overview", + "description": "Monitor Huntress Managed SIEM log collection health and security event activity forwarded to Datadog.", + "widgets": [ + { + "id": 1001, + "definition": { + "type": "note", + "content": "## Huntress SIEM\n\n[Huntress](https://www.huntress.com/) Managed SIEM continuously collects endpoint security telemetry and threat detections. This dashboard shows the health of the Datadog log collection pipeline and a summary of security events forwarded from Huntress.\n\n**Tips**\n- Use the timeframe selector in the upper-right corner to change the observation window.\n- Clone this dashboard to add custom widgets, filters, or team-specific views.", + "background_color": "blue", + "font_size": "14", + "text_align": "left", + "vertical_align": "center", + "show_tick": false, + "has_padding": true + }, + "layout": { + "x": 0, + "y": 0, + "width": 12, + "height": 3 + } + }, + { + "id": 1010, + "definition": { + "title": "Collection Health", + "background_color": "vivid_blue", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 1011, + "definition": { + "title": "Check Status", + "title_size": "16", + "title_align": "left", + "type": "check_status", + "check": "huntress.siem.check_status", + "grouping": "cluster", + "group_by": [], + "tags": ["*"] + }, + "layout": { + "x": 0, + "y": 0, + "width": 2, + "height": 4 + } + }, + { + "id": 1012, + "definition": { + "title": "Logs Collected (Last Run)", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "sum:huntress.siem.logs_collected{*}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "query1" + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 2, + "y": 0, + "width": 2, + "height": 2 + } + }, + { + "id": 1013, + "definition": { + "title": "Pages Fetched (Last Run)", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "sum:huntress.siem.pages_fetched{*}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "query1" + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 4, + "y": 0, + "width": 2, + "height": 2 + } + }, + { + "id": 1014, + "definition": { + "title": "Last Run Duration (s)", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "avg:huntress.siem.run_duration_seconds{*}", + "aggregator": "last" + } + ], + "formulas": [ + { + "formula": "query1" + } + ] + } + ], + "autoscale": true, + "precision": 1 + }, + "layout": { + "x": 6, + "y": 0, + "width": 2, + "height": 2 + } + }, + { + "id": 1015, + "definition": { + "title": "Errors (This Window)", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "sum:huntress.siem.errors{*}.as_count()", + "aggregator": "sum" + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "red_on_white" + }, + { + "comparator": "<=", + "value": 0, + "palette": "green_on_white" + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 8, + "y": 0, + "width": 2, + "height": 2 + } + }, + { + "id": 1016, + "definition": { + "title": "Errors by Type", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "sum:huntress.siem.errors{*} by {error_type}.as_count()", + "aggregator": "sum" + } + ], + "formulas": [ + { + "formula": "query1", + "limit": { + "count": 10, + "order": "desc" + } + } + ] + } + ] + }, + "layout": { + "x": 10, + "y": 0, + "width": 2, + "height": 4 + } + }, + { + "id": 1017, + "definition": { + "title": "Logs Collected Per Run", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "type": "timeseries", + "requests": [ + { + "response_format": "timeseries", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "sum:huntress.siem.logs_collected{*} by {huntress_account_id}" + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "display_type": "bars", + "style": { + "palette": "datadog16", + "line_type": "solid", + "line_width": "normal" + } + } + ] + }, + "layout": { + "x": 2, + "y": 2, + "width": 4, + "height": 2 + } + }, + { + "id": 1018, + "definition": { + "title": "Collection Errors Over Time", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "type": "timeseries", + "requests": [ + { + "response_format": "timeseries", + "queries": [ + { + "name": "query1", + "data_source": "metrics", + "query": "sum:huntress.siem.errors{*} by {error_type}.as_count()" + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "display_type": "bars", + "style": { + "palette": "warm", + "line_type": "solid", + "line_width": "normal" + } + } + ] + }, + "layout": { + "x": 6, + "y": 2, + "width": 4, + "height": 2 + } + }, + { + "id": 1019, + "definition": { + "title": "API Requests Remaining (Rate Limit Window)", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "type": "timeseries", + "requests": [ + { + "response_format": "timeseries", + "queries": [ + { + "name": "limit", + "data_source": "metrics", + "query": "avg:huntress.siem.api_call_limit{*}" + }, + { + "name": "remaining", + "data_source": "metrics", + "query": "avg:huntress.siem.api_call_remaining{*}" + } + ], + "formulas": [ + { + "formula": "limit", + "alias": "Limit" + }, + { + "formula": "remaining", + "alias": "Remaining" + } + ], + "display_type": "line", + "style": { + "palette": "cool", + "line_type": "solid", + "line_width": "normal" + } + } + ], + "markers": [ + { + "value": "y = 10", + "display_type": "error dashed", + "label": "Low remaining (< 10)" + } + ] + }, + "layout": { + "x": 10, + "y": 2, + "width": 2, + "height": 2 + } + } + ] + }, + "layout": { + "x": 0, + "y": 3, + "width": 12, + "height": 1 + } + }, + { + "id": 1030, + "definition": { + "title": "Security Events", + "background_color": "vivid_orange", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 1031, + "definition": { + "title": "Total Events", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:huntress" + }, + "indexes": ["*"], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "formulas": [ + { + "formula": "query1" + } + ] + } + ], + "autoscale": true, + "precision": 0 + }, + "layout": { + "x": 0, + "y": 0, + "width": 3, + "height": 4 + } + }, + { + "id": 1032, + "definition": { + "title": "Events by Category", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:huntress" + }, + "indexes": ["*"], + "group_by": [ + { + "facet": "@event.category", + "sort": { + "aggregation": "count", + "order": "desc" + }, + "limit": 10 + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "formulas": [ + { + "formula": "query1", + "limit": { + "count": 10, + "order": "desc" + } + } + ] + } + ] + }, + "layout": { + "x": 3, + "y": 0, + "width": 3, + "height": 4 + } + }, + { + "id": 1033, + "definition": { + "title": "Events by Outcome", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:huntress" + }, + "indexes": ["*"], + "group_by": [ + { + "facet": "@event.outcome", + "sort": { + "aggregation": "count", + "order": "desc" + }, + "limit": 10 + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "formulas": [ + { + "formula": "query1", + "limit": { + "count": 10, + "order": "desc" + } + } + ] + } + ] + }, + "layout": { + "x": 6, + "y": 0, + "width": 3, + "height": 4 + } + }, + { + "id": 1034, + "definition": { + "title": "Events by Action", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:huntress" + }, + "indexes": ["*"], + "group_by": [ + { + "facet": "@event.action", + "sort": { + "aggregation": "count", + "order": "desc" + }, + "limit": 10 + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "formulas": [ + { + "formula": "query1", + "limit": { + "count": 10, + "order": "desc" + } + } + ] + } + ] + }, + "layout": { + "x": 9, + "y": 0, + "width": 3, + "height": 4 + } + }, + { + "id": 1035, + "definition": { + "title": "Events Over Time by Category", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "type": "timeseries", + "requests": [ + { + "response_format": "timeseries", + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:huntress" + }, + "indexes": ["*"], + "group_by": [ + { + "facet": "@event.category", + "sort": { + "aggregation": "count", + "order": "desc" + }, + "limit": 10 + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "display_type": "bars", + "style": { + "palette": "datadog16", + "line_type": "solid", + "line_width": "normal" + } + } + ] + }, + "layout": { + "x": 0, + "y": 4, + "width": 12, + "height": 3 + } + } + ] + }, + "layout": { + "x": 0, + "y": 12, + "width": 12, + "height": 1 + } + }, + { + "id": 1050, + "definition": { + "title": "Top Activity", + "background_color": "vivid_green", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 1051, + "definition": { + "title": "Top Hosts", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:huntress" + }, + "indexes": ["*"], + "group_by": [ + { + "facet": "@host.hostname", + "sort": { + "aggregation": "count", + "order": "desc" + }, + "limit": 10 + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "formulas": [ + { + "formula": "query1", + "limit": { + "count": 10, + "order": "desc" + } + } + ] + } + ] + }, + "layout": { + "x": 0, + "y": 0, + "width": 4, + "height": 4 + } + }, + { + "id": 1052, + "definition": { + "title": "Top Users", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:huntress" + }, + "indexes": ["*"], + "group_by": [ + { + "facet": "@user.name", + "sort": { + "aggregation": "count", + "order": "desc" + }, + "limit": 10 + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "formulas": [ + { + "formula": "query1", + "limit": { + "count": 10, + "order": "desc" + } + } + ] + } + ] + }, + "layout": { + "x": 4, + "y": 0, + "width": 4, + "height": 4 + } + }, + { + "id": 1053, + "definition": { + "title": "Top Organizations", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:huntress" + }, + "indexes": ["*"], + "group_by": [ + { + "facet": "@huntress_org_name", + "sort": { + "aggregation": "count", + "order": "desc" + }, + "limit": 10 + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "formulas": [ + { + "formula": "query1", + "limit": { + "count": 10, + "order": "desc" + } + } + ] + } + ] + }, + "layout": { + "x": 8, + "y": 0, + "width": 4, + "height": 4 + } + }, + { + "id": 1054, + "definition": { + "title": "Top Event Providers", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:huntress" + }, + "indexes": ["*"], + "group_by": [ + { + "facet": "@event.provider", + "sort": { + "aggregation": "count", + "order": "desc" + }, + "limit": 10 + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "formulas": [ + { + "formula": "query1", + "limit": { + "count": 10, + "order": "desc" + } + } + ] + } + ] + }, + "layout": { + "x": 0, + "y": 4, + "width": 4, + "height": 4 + } + }, + { + "id": 1055, + "definition": { + "title": "Events by OS Platform", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:huntress" + }, + "indexes": ["*"], + "group_by": [ + { + "facet": "@host.os.platform", + "sort": { + "aggregation": "count", + "order": "desc" + }, + "limit": 10 + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "formulas": [ + { + "formula": "query1", + "limit": { + "count": 10, + "order": "desc" + } + } + ] + } + ] + }, + "layout": { + "x": 4, + "y": 4, + "width": 4, + "height": 4 + } + }, + { + "id": 1056, + "definition": { + "title": "Authentication Events — Failed Logins", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "type": "timeseries", + "requests": [ + { + "response_format": "timeseries", + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:huntress @event.category:authentication @event.outcome:failure" + }, + "indexes": ["*"], + "group_by": [ + { + "facet": "@host.hostname", + "sort": { + "aggregation": "count", + "order": "desc" + }, + "limit": 10 + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "display_type": "bars", + "style": { + "palette": "warm", + "line_type": "solid", + "line_width": "normal" + } + } + ] + }, + "layout": { + "x": 8, + "y": 4, + "width": 4, + "height": 4 + } + } + ] + }, + "layout": { + "x": 0, + "y": 21, + "width": 12, + "height": 1 + } + }, + { + "id": 1070, + "definition": { + "title": "Recent Huntress SIEM Events", + "title_size": "16", + "title_align": "left", + "type": "log_stream", + "indexes": [], + "query": "source:huntress", + "sort": { + "column": "time", + "order": "desc" + }, + "columns": [ + "@event.category", + "@event.action", + "@event.outcome", + "@host.hostname", + "@user.name", + "@huntress_org_name" + ], + "show_date_column": true, + "show_message_column": true, + "message_display": "expanded-md" + }, + "layout": { + "x": 0, + "y": 32, + "width": 12, + "height": 6 + } + } + ], + "template_variables": [], + "layout_type": "ordered", + "notify_list": [], + "reflow_type": "fixed" +} diff --git a/huntress/assets/monitors/huntress_siem_check_failed.json b/huntress/assets/monitors/huntress_siem_check_failed.json new file mode 100644 index 0000000000..4414cbdf10 --- /dev/null +++ b/huntress/assets/monitors/huntress_siem_check_failed.json @@ -0,0 +1,39 @@ +{ + "version": 2, + "created_at": "2026-05-27", + "last_updated_at": "2026-05-27", + "title": "Huntress SIEM collection run failed", + "tags": [ + "integration:huntress" + ], + "description": "The Huntress SIEM check emits a CRITICAL service check whenever a collection run fails — for example due to an authentication error, network timeout, or invalid ES|QL query. This monitor alerts on any such failure so operators can investigate and restore log collection before data gaps accumulate.", + "definition": { + "message": "The Huntress SIEM Agent check on {{host.name}} has entered a CRITICAL state, meaning the last collection run failed. Logs may no longer be flowing from Huntress to Datadog.\n\nCommon causes:\n- Invalid or rotated API credentials (`huntress_api_key` / `huntress_secret_key`)\n- Network connectivity issue between the Agent host and `api.huntress.io`\n- Invalid `esql_query` in the instance configuration\n- Huntress SIEM feature not enabled on this account\n\nCheck the Agent logs for details:\n```\nsudo datadog-agent check huntress\n```\n\n@pagerduty @slack-security-alerts", + "name": "Huntress SIEM collection run failed on {{host.name}}", + "options": { + "include_tags": true, + "new_host_delay": 300, + "no_data_timeframe": 60, + "notify_audit": false, + "notify_no_data": false, + "renotify_interval": 60, + "renotify_statuses": [ + "alert" + ], + "require_full_window": false, + "silenced": {}, + "thresholds": { + "critical": 1, + "ok": 1 + }, + "timeout_h": 0 + }, + "priority": 2, + "query": "\"huntress.siem.check_status\".over(\"*\").last(2).count_by_status()", + "restricted_roles": null, + "tags": [ + "integration:huntress" + ], + "type": "service check" + } +} diff --git a/huntress/assets/monitors/huntress_siem_errors_high.json b/huntress/assets/monitors/huntress_siem_errors_high.json new file mode 100644 index 0000000000..a83e4b3d81 --- /dev/null +++ b/huntress/assets/monitors/huntress_siem_errors_high.json @@ -0,0 +1,36 @@ +{ + "version": 2, + "created_at": "2026-05-27", + "last_updated_at": "2026-05-27", + "title": "Huntress SIEM collection errors detected", + "tags": [ + "integration:huntress" + ], + "description": "The Huntress SIEM check emits `huntress.siem.errors` with an `error_type` tag whenever an API call fails. A small number of transient errors (rate limits, brief timeouts) is normal and automatically retried. A sustained elevated error count indicates a persistent problem such as repeated auth failures, server errors, or a consistently invalid query that is preventing log collection.", + "definition": { + "message": "The Huntress SIEM integration has recorded {{value}} errors in the last 30 minutes (threshold: {{threshold}}).\n\nInspect the error type using the `error_type` tag in Datadog Metrics Explorer:\n- `error_type:auth_failure` — check API credentials\n- `error_type:timeout` — query may be too broad; add a KEEP or WHERE clause\n- `error_type:invalid_query` — `esql_query` is malformed\n- `error_type:server_error` — transient Huntress API issue\n- `error_type:connection_error` — network connectivity problem\n\nAgent check logs:\n```\nsudo datadog-agent check huntress\n```\n\n@pagerduty @slack-security-alerts", + "name": "Huntress SIEM collection errors detected", + "options": { + "include_tags": true, + "new_group_delay": 60, + "notify_audit": false, + "on_missing_data": "show_no_data", + "renotify_interval": 60, + "renotify_statuses": [ + "alert" + ], + "require_full_window": false, + "thresholds": { + "critical": 5, + "warning": 2 + } + }, + "priority": 3, + "query": "sum(last_30m):sum:huntress.siem.errors{*}.as_count() > 5", + "restricted_roles": null, + "tags": [ + "integration:huntress" + ], + "type": "query alert" + } +} diff --git a/huntress/assets/monitors/huntress_siem_no_logs_collected.json b/huntress/assets/monitors/huntress_siem_no_logs_collected.json new file mode 100644 index 0000000000..7fd51f1adc --- /dev/null +++ b/huntress/assets/monitors/huntress_siem_no_logs_collected.json @@ -0,0 +1,35 @@ +{ + "version": 2, + "created_at": "2026-05-27", + "last_updated_at": "2026-05-27", + "title": "Huntress SIEM no logs collected", + "tags": [ + "integration:huntress" + ], + "description": "When the Huntress SIEM integration is healthy, it collects at least some logs each run — even in quiet environments, the Huntress API typically returns a small number of heartbeat or audit events. Zero logs collected over an extended period may indicate that the SIEM query window is returning no results, the Huntress account has no active log sources, or the check is silently failing without triggering the service check.", + "definition": { + "message": "The Huntress SIEM integration collected 0 logs in the last 2 hours.\n\nThis may be expected in a brand-new deployment or a quiet environment, but if you expect ongoing log flow, investigate:\n\n1. Verify the Agent check is running: `sudo datadog-agent check huntress`\n2. Confirm `esql_query` returns results in the Huntress Partner Portal\n3. Check that the Huntress account has active endpoints and SIEM log sources\n4. Review `huntress.siem.check_status` — if CRITICAL, the check itself is failing\n\n@slack-security-alerts", + "name": "Huntress SIEM no logs collected in 2 hours", + "options": { + "include_tags": true, + "new_group_delay": 300, + "notify_audit": false, + "on_missing_data": "show_no_data", + "renotify_interval": 120, + "renotify_statuses": [ + "alert" + ], + "require_full_window": true, + "thresholds": { + "critical": 1 + } + }, + "priority": 3, + "query": "sum(last_2h):sum:huntress.siem.logs_collected{*}.as_count() < 1", + "restricted_roles": null, + "tags": [ + "integration:huntress" + ], + "type": "query alert" + } +} diff --git a/huntress/assets/service_checks.json b/huntress/assets/service_checks.json new file mode 100644 index 0000000000..5a8e160a3c --- /dev/null +++ b/huntress/assets/service_checks.json @@ -0,0 +1,14 @@ +[ + { + "agent_version": "7.0.0", + "integration": "huntress", + "check": "huntress.siem.check_status", + "statuses": [ + "ok", + "critical" + ], + "groups": [], + "name": "Huntress SIEM Check Status", + "description": "Returns `CRITICAL` if the Huntress SIEM collection run fails for any reason (auth error, network error, invalid query, etc.), returns `OK` otherwise." + } +] diff --git a/huntress/checks.d/huntress.py b/huntress/checks.d/huntress.py new file mode 100644 index 0000000000..640cf2c232 --- /dev/null +++ b/huntress/checks.d/huntress.py @@ -0,0 +1,549 @@ +import base64 +import gzip +import hashlib +import json +import time +from datetime import datetime, timedelta, timezone + +import requests +from datadog_checks.base import AgentCheck, ConfigurationError + +CHECK_NAME = "huntress" + + +class HuntressCheck(AgentCheck): + """Datadog Agent check: polls Huntress SIEM via ES|QL and forwards logs.""" + + HUNTRESS_SIEM_ENDPOINT = "/v1/siem/query" + HUNTRESS_ACCOUNT_ENDPOINT = "/v1/account" + HUNTRESS_ORGS_ENDPOINT = "/v1/accounts/{account_id}/organizations" + + CHECKPOINT_CACHE_KEY_PREFIX = "huntress_last_collected_at_" + ORG_CACHE_KEY_PREFIX = "huntress_org_cache_" + + MAX_LOGS_PER_BATCH = 1000 + MAX_BATCH_SIZE_BYTES = 4 * 1024 * 1024 # 4 MB headroom under 5 MB limit + + DEFAULT_BASE_URL = "https://api.huntress.io" + DEFAULT_MIN_COLLECTION_INTERVAL = 900 + DEFAULT_MAX_PAGES_PER_RUN = 100 + DEFAULT_ORG_CACHE_TTL = 3600 + + SERVICE_CHECK_NAME = "huntress.siem.check_status" + + def check(self, instance): + start_time = time.time() + + # Reset per-run rate limit state + self._last_api_call_limit = None + self._last_api_call_remaining = None + + api_key = instance.get("huntress_api_key", "").strip() + secret_key = instance.get("huntress_secret_key", "").strip() + esql_query = instance.get("esql_query", "").strip() + base_url = instance.get("huntress_base_url", self.DEFAULT_BASE_URL).rstrip("/") + max_pages = int(instance.get("max_pages_per_run", self.DEFAULT_MAX_PAGES_PER_RUN)) + min_interval = int(instance.get("min_collection_interval", self.DEFAULT_MIN_COLLECTION_INTERVAL)) + enrich_orgs = instance.get("enrich_with_org_tags", True) + org_ttl = int(instance.get("org_cache_ttl_seconds", self.DEFAULT_ORG_CACHE_TTL)) + extra_tags = list(instance.get("tags", [])) + + if not api_key: + raise ConfigurationError("huntress_api_key is required") + if not secret_key: + raise ConfigurationError("huntress_secret_key is required") + if not esql_query: + raise ConfigurationError("esql_query is required") + if not esql_query.lower().lstrip().startswith("from logs"): + raise ConfigurationError("esql_query must begin with 'FROM logs' (case-insensitive)") + + instance_hash = self._instance_hash(instance) + headers = self._get_auth_header(api_key, secret_key) + + # Org enrichment + org_cache = None + if enrich_orgs: + org_cache = self._get_or_refresh_org_cache( + base_url, headers, instance_hash, org_ttl + ) + + # Checkpoint + range_start = self._load_checkpoint(instance_hash) + now = datetime.now(timezone.utc) + range_end = now.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + + if range_start is None: + default_start = now - timedelta(seconds=min_interval) + range_start = default_start.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + else: + # Add 1ms offset to avoid re-fetching the last millisecond from previous run + try: + dt = datetime.fromisoformat(range_start.replace("Z", "+00:00")) + dt = dt + timedelta(milliseconds=1) + range_start = dt.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + except Exception: + pass + + self.log.debug( + "Huntress SIEM collection: range_start=%s range_end=%s", range_start, range_end + ) + + page_token = None + total_logs = 0 + pages_fetched = 0 + hit_page_cap = False + success = False + + try: + while True: + logs, next_token = self._query_page( + base_url, headers, esql_query, range_start, range_end, page_token + ) + + if logs: + service_tag = self._extract_service(extra_tags) + batch = [] + for raw in logs: + org_tags = self._get_org_tags(raw, org_cache) if org_cache else [] + all_tags = extra_tags + org_tags + payload = self._transform_log(raw, all_tags, service_tag) + batch.append(payload) + if len(batch) >= self.MAX_LOGS_PER_BATCH: + self._send_logs_batch(batch) + batch = [] + if batch: + self._send_logs_batch(batch) + + total_logs += len(logs) + pages_fetched += 1 + + if next_token and pages_fetched < max_pages: + page_token = next_token + else: + if next_token and pages_fetched >= max_pages: + hit_page_cap = True + self.log.warning( + "Huntress SIEM: hit max_pages_per_run=%d cap; " + "remaining pages will be collected next run", + max_pages, + ) + break + + if not hit_page_cap: + self._save_checkpoint(instance_hash, range_end) + + success = True + + except Exception as exc: + self.log.error("Huntress SIEM run failed: %s", exc) + self.count( + "huntress.siem.errors", + 1, + tags=extra_tags + ["error_type:run_failure"], + ) + self.service_check(self.SERVICE_CHECK_NAME, self.CRITICAL, tags=extra_tags) + raise + + finally: + duration = time.time() - start_time + self.gauge("huntress.siem.run_duration_seconds", duration, tags=extra_tags) + self.gauge("huntress.siem.logs_collected", total_logs, tags=extra_tags) + self.gauge("huntress.siem.pages_fetched", pages_fetched, tags=extra_tags) + if self._last_api_call_limit is not None: + self.gauge("huntress.siem.api_call_limit", self._last_api_call_limit, tags=extra_tags) + if self._last_api_call_remaining is not None: + self.gauge("huntress.siem.api_call_remaining", self._last_api_call_remaining, tags=extra_tags) + + if success: + self.service_check(self.SERVICE_CHECK_NAME, self.OK, tags=extra_tags) + + # ------------------------------------------------------------------ # + # Auth # + # ------------------------------------------------------------------ # + + def _get_auth_header(self, api_key, secret_key): + token = base64.b64encode(f"{api_key}:{secret_key}".encode()).decode() + return { + "Authorization": f"Basic {token}", + "Content-Type": "application/json", + "Accept": "application/json", + } + + def _parse_rate_limit_headers(self, resp): + """Parse x-huntress-api-call-limit / x-huntress-api-call-remaining headers.""" + try: + limit = resp.headers.get("x-huntress-api-call-limit") + if limit is not None: + self._last_api_call_limit = int(limit) + except (ValueError, TypeError, AttributeError): + pass + try: + remaining = resp.headers.get("x-huntress-api-call-remaining") + if remaining is not None: + self._last_api_call_remaining = int(remaining) + if self._last_api_call_remaining < 10: + self.log.warning( + "Huntress API rate limit nearly exhausted: %d/%d requests remaining this minute", + self._last_api_call_remaining, + self._last_api_call_limit or 60, + ) + except (ValueError, TypeError, AttributeError): + pass + + # ------------------------------------------------------------------ # + # SIEM query # + # ------------------------------------------------------------------ # + + def _query_page(self, base_url, headers, esql, range_start, range_end, page_token=None): + """POST /v1/siem/query; return (logs_list, next_page_token | None).""" + url = base_url + self.HUNTRESS_SIEM_ENDPOINT + body = {"esql": esql, "range_start": range_start, "range_end": range_end} + if page_token: + body["page_token"] = page_token + + response = self._request_with_retry( + method="POST", + url=url, + headers=headers, + json_body=body, + ) + + data = response.json() + logs = data.get("logs", []) + pagination = data.get("pagination", {}) + next_token = pagination.get("next_page_token") + return logs, next_token + + # ------------------------------------------------------------------ # + # HTTP with retry / error handling # + # ------------------------------------------------------------------ # + + def _request_with_retry(self, method, url, headers, json_body=None, params=None): + """Execute an HTTP request with retry logic per PRD §7.""" + max_retries_5xx = 3 + max_retries_408 = 2 + backoff_5xx = [5, 10, 20] + backoff_408 = [2, 4] + + attempt = 0 + last_exc = None + + while True: + try: + resp = requests.request( + method, + url, + headers=headers, + json=json_body, + params=params, + timeout=30, + ) + + if resp.status_code == 200: + self._parse_rate_limit_headers(resp) + return resp + + if resp.status_code == 400: + self.count( + "huntress.siem.errors", 1, + tags=["error_type:bad_request"] + ) + raise Exception( + f"Huntress API 400 Bad Request: {resp.text}" + ) + + if resp.status_code == 401: + self.count( + "huntress.siem.errors", 1, + tags=["error_type:auth_failure"] + ) + raise Exception( + "Huntress API 401 Unauthorized — check huntress_api_key and huntress_secret_key" + ) + + if resp.status_code == 404: + raise Exception( + "Huntress API 404 — SIEM feature may not be enabled on this account" + ) + + if resp.status_code == 408: + if attempt < max_retries_408: + wait = backoff_408[attempt] + self.log.warning( + "Huntress API 408 Query Timeout; retrying in %ds (attempt %d/%d)", + wait, attempt + 1, max_retries_408, + ) + time.sleep(wait) + attempt += 1 + continue + self.count( + "huntress.siem.errors", 1, + tags=["error_type:timeout"] + ) + raise Exception("Huntress API 408 Query Timeout — query may be too broad") + + if resp.status_code == 413: + raise Exception( + "Huntress API 413 Memory Limit — narrow the query with KEEP or WHERE clauses" + ) + + if resp.status_code == 422: + self.count( + "huntress.siem.errors", 1, + tags=["error_type:invalid_query"] + ) + raise Exception( + f"Huntress API 422 Invalid ES|QL query: {resp.text}" + ) + + if resp.status_code == 429: + self.log.warning( + "Huntress API 429 Rate Limited — sleeping 60s then retrying" + ) + time.sleep(60) + continue + + if 500 <= resp.status_code < 600: + if attempt < max_retries_5xx: + wait = backoff_5xx[attempt] + self.log.warning( + "Huntress API %d Server Error; retrying in %ds (attempt %d/%d)", + resp.status_code, wait, attempt + 1, max_retries_5xx, + ) + time.sleep(wait) + attempt += 1 + continue + self.count( + "huntress.siem.errors", 1, + tags=["error_type:server_error"] + ) + raise Exception( + f"Huntress API {resp.status_code} Server Error after {max_retries_5xx} retries" + ) + + raise Exception( + f"Huntress API unexpected status {resp.status_code}: {resp.text}" + ) + + except requests.exceptions.RequestException as exc: + last_exc = exc + if attempt < max_retries_5xx: + wait = backoff_5xx[attempt] + self.log.warning( + "Huntress API network error (%s); retrying in %ds (attempt %d/%d)", + exc, wait, attempt + 1, max_retries_5xx, + ) + time.sleep(wait) + attempt += 1 + continue + self.count( + "huntress.siem.errors", 1, + tags=["error_type:connection_error"] + ) + raise Exception( + f"Huntress API connection error after {max_retries_5xx} retries: {exc}" + ) from exc + + # ------------------------------------------------------------------ # + # Log transformation # + # ------------------------------------------------------------------ # + + def _extract_service(self, tags): + for tag in tags: + if tag.startswith("service:"): + return tag.split(":", 1)[1] + return "huntress-siem" + + def _transform_log(self, raw_log, tags, service): + message = ( + raw_log.get("log.original") + or raw_log.get("message") + or json.dumps(raw_log) + ) + + timestamp = raw_log.get("@timestamp") + date_ms = None + if timestamp: + try: + if isinstance(timestamp, (int, float)): + date_ms = int(timestamp) + else: + dt = datetime.fromisoformat(str(timestamp).replace("Z", "+00:00")) + date_ms = int(dt.timestamp() * 1000) + except Exception: + pass + + payload = { + "message": message, + "ddsource": "huntress", + "ddtags": ",".join(tags), + "service": service, + } + if date_ms is not None: + payload["date"] = date_ms + + # Preserve all ECS fields as top-level log attributes + for key, value in raw_log.items(): + if key not in ("log.original", "message", "@timestamp"): + payload[key] = value + + return payload + + # ------------------------------------------------------------------ # + # Log sending # + # ------------------------------------------------------------------ # + + def _send_logs_batch(self, logs_batch): + """Send a batch of up to MAX_LOGS_PER_BATCH log payloads via self.send_log().""" + for log_payload in logs_batch: + self.send_log(log_payload) + + # ------------------------------------------------------------------ # + # Checkpoint # + # ------------------------------------------------------------------ # + + def _load_checkpoint(self, instance_hash): + key = self.CHECKPOINT_CACHE_KEY_PREFIX + instance_hash + raw = self.read_persistent_cache(key) + if not raw: + return None + try: + data = json.loads(raw) + return data.get("last_collected_at") + except Exception: + return None + + def _save_checkpoint(self, instance_hash, timestamp_iso): + key = self.CHECKPOINT_CACHE_KEY_PREFIX + instance_hash + payload = json.dumps({"last_collected_at": timestamp_iso, "schema_version": 1}) + self.write_persistent_cache(key, payload) + + # ------------------------------------------------------------------ # + # Org metadata enrichment # + # ------------------------------------------------------------------ # + + def _get_or_refresh_org_cache(self, base_url, headers, instance_hash, ttl_seconds): + """Return the org cache dict, refreshing if stale or absent.""" + cached = self._load_org_cache(instance_hash) + if cached: + try: + fetched_at_str = cached.get("fetched_at", "") + fetched_at = datetime.fromisoformat(fetched_at_str.replace("Z", "+00:00")) + age = (datetime.now(timezone.utc) - fetched_at).total_seconds() + if ttl_seconds == 0 or age < ttl_seconds: + return cached + except Exception: + pass + + try: + account_id = self._get_account_id(base_url, headers) + orgs = self._fetch_org_cache(base_url, headers, account_id) + cache_payload = { + "fetched_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z", + "account_id": account_id, + "orgs": orgs, + } + self._save_org_cache(instance_hash, cache_payload) + return cache_payload + except Exception as exc: + self.log.warning( + "Huntress: failed to fetch org metadata — logs will be collected without org tags: %s", + exc, + ) + return None + + def _get_account_id(self, base_url, headers): + url = base_url + self.HUNTRESS_ACCOUNT_ENDPOINT + resp = self._request_with_retry("GET", url, headers) + data = resp.json() + account = data.get("account", data) + return account["id"] + + def _fetch_org_cache(self, base_url, headers, account_id): + """Paginate /v1/accounts/{id}/organizations; return {org_id_str: {...}} dict.""" + orgs = {} + page = 1 + while True: + url = base_url + self.HUNTRESS_ORGS_ENDPOINT.format(account_id=account_id) + resp = self._request_with_retry( + "GET", url, headers, params={"limit": 500, "page": page} + ) + data = resp.json() + org_list = data.get("organizations", []) + for org in org_list: + orgs[str(org["id"])] = { + "name": org.get("name", ""), + "key": org.get("key", ""), + "account_id": account_id, + } + pagination = data.get("pagination", {}) + if pagination.get("next_page_token") or ( + pagination.get("current_page", 1) < pagination.get("total_pages", 1) + ): + page += 1 + else: + break + return orgs + + def _load_org_cache(self, instance_hash): + key = self.ORG_CACHE_KEY_PREFIX + instance_hash + raw = self.read_persistent_cache(key) + if not raw: + return None + try: + return json.loads(raw) + except Exception: + return None + + def _save_org_cache(self, instance_hash, cache_payload): + key = self.ORG_CACHE_KEY_PREFIX + instance_hash + self.write_persistent_cache(key, json.dumps(cache_payload)) + + def _get_org_tags(self, raw_log, org_cache): + """Match a log record to an org in the cache; return list of tag strings.""" + if not org_cache: + return [] + + account_id = org_cache.get("account_id") + orgs = org_cache.get("orgs", {}) + + base_tags = [] + if account_id is not None: + base_tags.append(f"huntress_account_id:{account_id}") + + # Strategy 1: match by organization.id + org_id = raw_log.get("organization.id") or ( + raw_log.get("organization", {}).get("id") if isinstance(raw_log.get("organization"), dict) else None + ) + if org_id is not None: + org = orgs.get(str(org_id)) + if org: + return base_tags + [ + f"huntress_org_id:{org_id}", + f"huntress_org_name:{org['name']}", + f"huntress_org_key:{org['key']}", + ] + + # Strategy 2: match by organization.name (reverse lookup) + org_name = raw_log.get("organization.name") or ( + raw_log.get("organization", {}).get("name") if isinstance(raw_log.get("organization"), dict) else None + ) + if org_name: + for oid, org in orgs.items(): + if org.get("name") == org_name: + return base_tags + [ + f"huntress_org_id:{oid}", + f"huntress_org_name:{org['name']}", + f"huntress_org_key:{org['key']}", + ] + + # Strategy 3: account-level tag only + return base_tags + + # ------------------------------------------------------------------ # + # Instance hash # + # ------------------------------------------------------------------ # + + def _instance_hash(self, instance): + key = f"{instance.get('huntress_api_key', '')}:{instance.get('esql_query', '')}" + return hashlib.md5(key.encode()).hexdigest()[:12] diff --git a/huntress/conf.d/huntress.yaml.example b/huntress/conf.d/huntress.yaml.example new file mode 100644 index 0000000000..48832eef85 --- /dev/null +++ b/huntress/conf.d/huntress.yaml.example @@ -0,0 +1,56 @@ +init_config: {} + +instances: + ## --- Account 1 --- + - ## Required: Huntress API credentials (Basic Auth). + ## Each instance has its own key pair, allowing multiple Huntress accounts + ## to feed into the same Datadog org simultaneously. + huntress_api_key: "" + huntress_secret_key: "" + + ## Required: ES|QL query to execute. Must begin with "FROM logs" (case-insensitive). + ## The query determines which log fields are returned. Use KEEP to select specific + ## columns, or omit for all ECS fields. Do NOT add time range filters — range_start + ## and range_end are managed automatically by the check. + esql_query: "FROM logs" + + ## Optional: Whether to enrich logs with Huntress organization metadata + ## (org name, org key, account ID). Fetches /v1/account and + ## /v1/accounts/{id}/organizations on first run and caches for org_cache_ttl_seconds. + ## Default: true + enrich_with_org_tags: true + + ## Optional: How long (seconds) to cache the org metadata before re-fetching. + ## Default: 3600 (1 hour). Set to 0 to refresh on every run. + org_cache_ttl_seconds: 3600 + + ## Optional: Collection interval in seconds. Must be >= 60 to stay within Huntress + ## rate limits (60 req/min). Default: 900 (15 minutes). + min_collection_interval: 900 + + ## Optional: Maximum number of pages to fetch per run (200 logs/page). + ## Protects against runaway pagination on large backlogs. + ## Default: 100 (up to 20,000 logs per run). + max_pages_per_run: 100 + + ## Optional: Huntress API base URL. Override for testing or sandbox environments. + huntress_base_url: "https://api.huntress.io" + + ## Optional: Additional tags attached to every log event forwarded to Datadog. + ## huntress_account_id, huntress_org_name, and huntress_org_key are added + ## automatically when enrich_with_org_tags is true. + tags: + - "source:huntress" + - "service:huntress-siem" + - "env:production" + + ## --- Account 2 (optional second Huntress account) --- + ## Each instance runs independently with its own checkpoint, org cache, and metrics. + ## Use multiple instances to fan in logs from multiple Huntress accounts. + - huntress_api_key: "" + huntress_secret_key: "" + esql_query: "FROM logs" + tags: + - "source:huntress" + - "service:huntress-siem" + - "env:staging" diff --git a/huntress/datadog_checks/__init__.py b/huntress/datadog_checks/__init__.py new file mode 100644 index 0000000000..69e3be50da --- /dev/null +++ b/huntress/datadog_checks/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/huntress/datadog_checks/huntress/__about__.py b/huntress/datadog_checks/huntress/__about__.py new file mode 100644 index 0000000000..5becc17c04 --- /dev/null +++ b/huntress/datadog_checks/huntress/__about__.py @@ -0,0 +1 @@ +__version__ = "1.0.0" diff --git a/huntress/datadog_checks/huntress/__init__.py b/huntress/datadog_checks/huntress/__init__.py new file mode 100644 index 0000000000..d4eb6b4b60 --- /dev/null +++ b/huntress/datadog_checks/huntress/__init__.py @@ -0,0 +1,4 @@ +from .__about__ import __version__ +from .huntress import HuntressCheck + +__all__ = ['__version__', 'HuntressCheck'] diff --git a/huntress/datadog_checks/huntress/data/conf.yaml.example b/huntress/datadog_checks/huntress/data/conf.yaml.example new file mode 100644 index 0000000000..48832eef85 --- /dev/null +++ b/huntress/datadog_checks/huntress/data/conf.yaml.example @@ -0,0 +1,56 @@ +init_config: {} + +instances: + ## --- Account 1 --- + - ## Required: Huntress API credentials (Basic Auth). + ## Each instance has its own key pair, allowing multiple Huntress accounts + ## to feed into the same Datadog org simultaneously. + huntress_api_key: "" + huntress_secret_key: "" + + ## Required: ES|QL query to execute. Must begin with "FROM logs" (case-insensitive). + ## The query determines which log fields are returned. Use KEEP to select specific + ## columns, or omit for all ECS fields. Do NOT add time range filters — range_start + ## and range_end are managed automatically by the check. + esql_query: "FROM logs" + + ## Optional: Whether to enrich logs with Huntress organization metadata + ## (org name, org key, account ID). Fetches /v1/account and + ## /v1/accounts/{id}/organizations on first run and caches for org_cache_ttl_seconds. + ## Default: true + enrich_with_org_tags: true + + ## Optional: How long (seconds) to cache the org metadata before re-fetching. + ## Default: 3600 (1 hour). Set to 0 to refresh on every run. + org_cache_ttl_seconds: 3600 + + ## Optional: Collection interval in seconds. Must be >= 60 to stay within Huntress + ## rate limits (60 req/min). Default: 900 (15 minutes). + min_collection_interval: 900 + + ## Optional: Maximum number of pages to fetch per run (200 logs/page). + ## Protects against runaway pagination on large backlogs. + ## Default: 100 (up to 20,000 logs per run). + max_pages_per_run: 100 + + ## Optional: Huntress API base URL. Override for testing or sandbox environments. + huntress_base_url: "https://api.huntress.io" + + ## Optional: Additional tags attached to every log event forwarded to Datadog. + ## huntress_account_id, huntress_org_name, and huntress_org_key are added + ## automatically when enrich_with_org_tags is true. + tags: + - "source:huntress" + - "service:huntress-siem" + - "env:production" + + ## --- Account 2 (optional second Huntress account) --- + ## Each instance runs independently with its own checkpoint, org cache, and metrics. + ## Use multiple instances to fan in logs from multiple Huntress accounts. + - huntress_api_key: "" + huntress_secret_key: "" + esql_query: "FROM logs" + tags: + - "source:huntress" + - "service:huntress-siem" + - "env:staging" diff --git a/huntress/datadog_checks/huntress/huntress.py b/huntress/datadog_checks/huntress/huntress.py new file mode 100644 index 0000000000..640cf2c232 --- /dev/null +++ b/huntress/datadog_checks/huntress/huntress.py @@ -0,0 +1,549 @@ +import base64 +import gzip +import hashlib +import json +import time +from datetime import datetime, timedelta, timezone + +import requests +from datadog_checks.base import AgentCheck, ConfigurationError + +CHECK_NAME = "huntress" + + +class HuntressCheck(AgentCheck): + """Datadog Agent check: polls Huntress SIEM via ES|QL and forwards logs.""" + + HUNTRESS_SIEM_ENDPOINT = "/v1/siem/query" + HUNTRESS_ACCOUNT_ENDPOINT = "/v1/account" + HUNTRESS_ORGS_ENDPOINT = "/v1/accounts/{account_id}/organizations" + + CHECKPOINT_CACHE_KEY_PREFIX = "huntress_last_collected_at_" + ORG_CACHE_KEY_PREFIX = "huntress_org_cache_" + + MAX_LOGS_PER_BATCH = 1000 + MAX_BATCH_SIZE_BYTES = 4 * 1024 * 1024 # 4 MB headroom under 5 MB limit + + DEFAULT_BASE_URL = "https://api.huntress.io" + DEFAULT_MIN_COLLECTION_INTERVAL = 900 + DEFAULT_MAX_PAGES_PER_RUN = 100 + DEFAULT_ORG_CACHE_TTL = 3600 + + SERVICE_CHECK_NAME = "huntress.siem.check_status" + + def check(self, instance): + start_time = time.time() + + # Reset per-run rate limit state + self._last_api_call_limit = None + self._last_api_call_remaining = None + + api_key = instance.get("huntress_api_key", "").strip() + secret_key = instance.get("huntress_secret_key", "").strip() + esql_query = instance.get("esql_query", "").strip() + base_url = instance.get("huntress_base_url", self.DEFAULT_BASE_URL).rstrip("/") + max_pages = int(instance.get("max_pages_per_run", self.DEFAULT_MAX_PAGES_PER_RUN)) + min_interval = int(instance.get("min_collection_interval", self.DEFAULT_MIN_COLLECTION_INTERVAL)) + enrich_orgs = instance.get("enrich_with_org_tags", True) + org_ttl = int(instance.get("org_cache_ttl_seconds", self.DEFAULT_ORG_CACHE_TTL)) + extra_tags = list(instance.get("tags", [])) + + if not api_key: + raise ConfigurationError("huntress_api_key is required") + if not secret_key: + raise ConfigurationError("huntress_secret_key is required") + if not esql_query: + raise ConfigurationError("esql_query is required") + if not esql_query.lower().lstrip().startswith("from logs"): + raise ConfigurationError("esql_query must begin with 'FROM logs' (case-insensitive)") + + instance_hash = self._instance_hash(instance) + headers = self._get_auth_header(api_key, secret_key) + + # Org enrichment + org_cache = None + if enrich_orgs: + org_cache = self._get_or_refresh_org_cache( + base_url, headers, instance_hash, org_ttl + ) + + # Checkpoint + range_start = self._load_checkpoint(instance_hash) + now = datetime.now(timezone.utc) + range_end = now.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + + if range_start is None: + default_start = now - timedelta(seconds=min_interval) + range_start = default_start.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + else: + # Add 1ms offset to avoid re-fetching the last millisecond from previous run + try: + dt = datetime.fromisoformat(range_start.replace("Z", "+00:00")) + dt = dt + timedelta(milliseconds=1) + range_start = dt.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + except Exception: + pass + + self.log.debug( + "Huntress SIEM collection: range_start=%s range_end=%s", range_start, range_end + ) + + page_token = None + total_logs = 0 + pages_fetched = 0 + hit_page_cap = False + success = False + + try: + while True: + logs, next_token = self._query_page( + base_url, headers, esql_query, range_start, range_end, page_token + ) + + if logs: + service_tag = self._extract_service(extra_tags) + batch = [] + for raw in logs: + org_tags = self._get_org_tags(raw, org_cache) if org_cache else [] + all_tags = extra_tags + org_tags + payload = self._transform_log(raw, all_tags, service_tag) + batch.append(payload) + if len(batch) >= self.MAX_LOGS_PER_BATCH: + self._send_logs_batch(batch) + batch = [] + if batch: + self._send_logs_batch(batch) + + total_logs += len(logs) + pages_fetched += 1 + + if next_token and pages_fetched < max_pages: + page_token = next_token + else: + if next_token and pages_fetched >= max_pages: + hit_page_cap = True + self.log.warning( + "Huntress SIEM: hit max_pages_per_run=%d cap; " + "remaining pages will be collected next run", + max_pages, + ) + break + + if not hit_page_cap: + self._save_checkpoint(instance_hash, range_end) + + success = True + + except Exception as exc: + self.log.error("Huntress SIEM run failed: %s", exc) + self.count( + "huntress.siem.errors", + 1, + tags=extra_tags + ["error_type:run_failure"], + ) + self.service_check(self.SERVICE_CHECK_NAME, self.CRITICAL, tags=extra_tags) + raise + + finally: + duration = time.time() - start_time + self.gauge("huntress.siem.run_duration_seconds", duration, tags=extra_tags) + self.gauge("huntress.siem.logs_collected", total_logs, tags=extra_tags) + self.gauge("huntress.siem.pages_fetched", pages_fetched, tags=extra_tags) + if self._last_api_call_limit is not None: + self.gauge("huntress.siem.api_call_limit", self._last_api_call_limit, tags=extra_tags) + if self._last_api_call_remaining is not None: + self.gauge("huntress.siem.api_call_remaining", self._last_api_call_remaining, tags=extra_tags) + + if success: + self.service_check(self.SERVICE_CHECK_NAME, self.OK, tags=extra_tags) + + # ------------------------------------------------------------------ # + # Auth # + # ------------------------------------------------------------------ # + + def _get_auth_header(self, api_key, secret_key): + token = base64.b64encode(f"{api_key}:{secret_key}".encode()).decode() + return { + "Authorization": f"Basic {token}", + "Content-Type": "application/json", + "Accept": "application/json", + } + + def _parse_rate_limit_headers(self, resp): + """Parse x-huntress-api-call-limit / x-huntress-api-call-remaining headers.""" + try: + limit = resp.headers.get("x-huntress-api-call-limit") + if limit is not None: + self._last_api_call_limit = int(limit) + except (ValueError, TypeError, AttributeError): + pass + try: + remaining = resp.headers.get("x-huntress-api-call-remaining") + if remaining is not None: + self._last_api_call_remaining = int(remaining) + if self._last_api_call_remaining < 10: + self.log.warning( + "Huntress API rate limit nearly exhausted: %d/%d requests remaining this minute", + self._last_api_call_remaining, + self._last_api_call_limit or 60, + ) + except (ValueError, TypeError, AttributeError): + pass + + # ------------------------------------------------------------------ # + # SIEM query # + # ------------------------------------------------------------------ # + + def _query_page(self, base_url, headers, esql, range_start, range_end, page_token=None): + """POST /v1/siem/query; return (logs_list, next_page_token | None).""" + url = base_url + self.HUNTRESS_SIEM_ENDPOINT + body = {"esql": esql, "range_start": range_start, "range_end": range_end} + if page_token: + body["page_token"] = page_token + + response = self._request_with_retry( + method="POST", + url=url, + headers=headers, + json_body=body, + ) + + data = response.json() + logs = data.get("logs", []) + pagination = data.get("pagination", {}) + next_token = pagination.get("next_page_token") + return logs, next_token + + # ------------------------------------------------------------------ # + # HTTP with retry / error handling # + # ------------------------------------------------------------------ # + + def _request_with_retry(self, method, url, headers, json_body=None, params=None): + """Execute an HTTP request with retry logic per PRD §7.""" + max_retries_5xx = 3 + max_retries_408 = 2 + backoff_5xx = [5, 10, 20] + backoff_408 = [2, 4] + + attempt = 0 + last_exc = None + + while True: + try: + resp = requests.request( + method, + url, + headers=headers, + json=json_body, + params=params, + timeout=30, + ) + + if resp.status_code == 200: + self._parse_rate_limit_headers(resp) + return resp + + if resp.status_code == 400: + self.count( + "huntress.siem.errors", 1, + tags=["error_type:bad_request"] + ) + raise Exception( + f"Huntress API 400 Bad Request: {resp.text}" + ) + + if resp.status_code == 401: + self.count( + "huntress.siem.errors", 1, + tags=["error_type:auth_failure"] + ) + raise Exception( + "Huntress API 401 Unauthorized — check huntress_api_key and huntress_secret_key" + ) + + if resp.status_code == 404: + raise Exception( + "Huntress API 404 — SIEM feature may not be enabled on this account" + ) + + if resp.status_code == 408: + if attempt < max_retries_408: + wait = backoff_408[attempt] + self.log.warning( + "Huntress API 408 Query Timeout; retrying in %ds (attempt %d/%d)", + wait, attempt + 1, max_retries_408, + ) + time.sleep(wait) + attempt += 1 + continue + self.count( + "huntress.siem.errors", 1, + tags=["error_type:timeout"] + ) + raise Exception("Huntress API 408 Query Timeout — query may be too broad") + + if resp.status_code == 413: + raise Exception( + "Huntress API 413 Memory Limit — narrow the query with KEEP or WHERE clauses" + ) + + if resp.status_code == 422: + self.count( + "huntress.siem.errors", 1, + tags=["error_type:invalid_query"] + ) + raise Exception( + f"Huntress API 422 Invalid ES|QL query: {resp.text}" + ) + + if resp.status_code == 429: + self.log.warning( + "Huntress API 429 Rate Limited — sleeping 60s then retrying" + ) + time.sleep(60) + continue + + if 500 <= resp.status_code < 600: + if attempt < max_retries_5xx: + wait = backoff_5xx[attempt] + self.log.warning( + "Huntress API %d Server Error; retrying in %ds (attempt %d/%d)", + resp.status_code, wait, attempt + 1, max_retries_5xx, + ) + time.sleep(wait) + attempt += 1 + continue + self.count( + "huntress.siem.errors", 1, + tags=["error_type:server_error"] + ) + raise Exception( + f"Huntress API {resp.status_code} Server Error after {max_retries_5xx} retries" + ) + + raise Exception( + f"Huntress API unexpected status {resp.status_code}: {resp.text}" + ) + + except requests.exceptions.RequestException as exc: + last_exc = exc + if attempt < max_retries_5xx: + wait = backoff_5xx[attempt] + self.log.warning( + "Huntress API network error (%s); retrying in %ds (attempt %d/%d)", + exc, wait, attempt + 1, max_retries_5xx, + ) + time.sleep(wait) + attempt += 1 + continue + self.count( + "huntress.siem.errors", 1, + tags=["error_type:connection_error"] + ) + raise Exception( + f"Huntress API connection error after {max_retries_5xx} retries: {exc}" + ) from exc + + # ------------------------------------------------------------------ # + # Log transformation # + # ------------------------------------------------------------------ # + + def _extract_service(self, tags): + for tag in tags: + if tag.startswith("service:"): + return tag.split(":", 1)[1] + return "huntress-siem" + + def _transform_log(self, raw_log, tags, service): + message = ( + raw_log.get("log.original") + or raw_log.get("message") + or json.dumps(raw_log) + ) + + timestamp = raw_log.get("@timestamp") + date_ms = None + if timestamp: + try: + if isinstance(timestamp, (int, float)): + date_ms = int(timestamp) + else: + dt = datetime.fromisoformat(str(timestamp).replace("Z", "+00:00")) + date_ms = int(dt.timestamp() * 1000) + except Exception: + pass + + payload = { + "message": message, + "ddsource": "huntress", + "ddtags": ",".join(tags), + "service": service, + } + if date_ms is not None: + payload["date"] = date_ms + + # Preserve all ECS fields as top-level log attributes + for key, value in raw_log.items(): + if key not in ("log.original", "message", "@timestamp"): + payload[key] = value + + return payload + + # ------------------------------------------------------------------ # + # Log sending # + # ------------------------------------------------------------------ # + + def _send_logs_batch(self, logs_batch): + """Send a batch of up to MAX_LOGS_PER_BATCH log payloads via self.send_log().""" + for log_payload in logs_batch: + self.send_log(log_payload) + + # ------------------------------------------------------------------ # + # Checkpoint # + # ------------------------------------------------------------------ # + + def _load_checkpoint(self, instance_hash): + key = self.CHECKPOINT_CACHE_KEY_PREFIX + instance_hash + raw = self.read_persistent_cache(key) + if not raw: + return None + try: + data = json.loads(raw) + return data.get("last_collected_at") + except Exception: + return None + + def _save_checkpoint(self, instance_hash, timestamp_iso): + key = self.CHECKPOINT_CACHE_KEY_PREFIX + instance_hash + payload = json.dumps({"last_collected_at": timestamp_iso, "schema_version": 1}) + self.write_persistent_cache(key, payload) + + # ------------------------------------------------------------------ # + # Org metadata enrichment # + # ------------------------------------------------------------------ # + + def _get_or_refresh_org_cache(self, base_url, headers, instance_hash, ttl_seconds): + """Return the org cache dict, refreshing if stale or absent.""" + cached = self._load_org_cache(instance_hash) + if cached: + try: + fetched_at_str = cached.get("fetched_at", "") + fetched_at = datetime.fromisoformat(fetched_at_str.replace("Z", "+00:00")) + age = (datetime.now(timezone.utc) - fetched_at).total_seconds() + if ttl_seconds == 0 or age < ttl_seconds: + return cached + except Exception: + pass + + try: + account_id = self._get_account_id(base_url, headers) + orgs = self._fetch_org_cache(base_url, headers, account_id) + cache_payload = { + "fetched_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z", + "account_id": account_id, + "orgs": orgs, + } + self._save_org_cache(instance_hash, cache_payload) + return cache_payload + except Exception as exc: + self.log.warning( + "Huntress: failed to fetch org metadata — logs will be collected without org tags: %s", + exc, + ) + return None + + def _get_account_id(self, base_url, headers): + url = base_url + self.HUNTRESS_ACCOUNT_ENDPOINT + resp = self._request_with_retry("GET", url, headers) + data = resp.json() + account = data.get("account", data) + return account["id"] + + def _fetch_org_cache(self, base_url, headers, account_id): + """Paginate /v1/accounts/{id}/organizations; return {org_id_str: {...}} dict.""" + orgs = {} + page = 1 + while True: + url = base_url + self.HUNTRESS_ORGS_ENDPOINT.format(account_id=account_id) + resp = self._request_with_retry( + "GET", url, headers, params={"limit": 500, "page": page} + ) + data = resp.json() + org_list = data.get("organizations", []) + for org in org_list: + orgs[str(org["id"])] = { + "name": org.get("name", ""), + "key": org.get("key", ""), + "account_id": account_id, + } + pagination = data.get("pagination", {}) + if pagination.get("next_page_token") or ( + pagination.get("current_page", 1) < pagination.get("total_pages", 1) + ): + page += 1 + else: + break + return orgs + + def _load_org_cache(self, instance_hash): + key = self.ORG_CACHE_KEY_PREFIX + instance_hash + raw = self.read_persistent_cache(key) + if not raw: + return None + try: + return json.loads(raw) + except Exception: + return None + + def _save_org_cache(self, instance_hash, cache_payload): + key = self.ORG_CACHE_KEY_PREFIX + instance_hash + self.write_persistent_cache(key, json.dumps(cache_payload)) + + def _get_org_tags(self, raw_log, org_cache): + """Match a log record to an org in the cache; return list of tag strings.""" + if not org_cache: + return [] + + account_id = org_cache.get("account_id") + orgs = org_cache.get("orgs", {}) + + base_tags = [] + if account_id is not None: + base_tags.append(f"huntress_account_id:{account_id}") + + # Strategy 1: match by organization.id + org_id = raw_log.get("organization.id") or ( + raw_log.get("organization", {}).get("id") if isinstance(raw_log.get("organization"), dict) else None + ) + if org_id is not None: + org = orgs.get(str(org_id)) + if org: + return base_tags + [ + f"huntress_org_id:{org_id}", + f"huntress_org_name:{org['name']}", + f"huntress_org_key:{org['key']}", + ] + + # Strategy 2: match by organization.name (reverse lookup) + org_name = raw_log.get("organization.name") or ( + raw_log.get("organization", {}).get("name") if isinstance(raw_log.get("organization"), dict) else None + ) + if org_name: + for oid, org in orgs.items(): + if org.get("name") == org_name: + return base_tags + [ + f"huntress_org_id:{oid}", + f"huntress_org_name:{org['name']}", + f"huntress_org_key:{org['key']}", + ] + + # Strategy 3: account-level tag only + return base_tags + + # ------------------------------------------------------------------ # + # Instance hash # + # ------------------------------------------------------------------ # + + def _instance_hash(self, instance): + key = f"{instance.get('huntress_api_key', '')}:{instance.get('esql_query', '')}" + return hashlib.md5(key.encode()).hexdigest()[:12] diff --git a/huntress/hatch.toml b/huntress/hatch.toml new file mode 100644 index 0000000000..f62423b83c --- /dev/null +++ b/huntress/hatch.toml @@ -0,0 +1,7 @@ +[env.collectors.datadog-checks] + +[[envs.default.matrix]] +python = ["3.12"] + +[envs.default.env-vars] +DDEV_SKIP_GENERIC_TAGS_CHECK = "true" diff --git a/huntress/images/huntress-logo.png b/huntress/images/huntress-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..14d2fa3080582929d9c9f46a668fe394ad9622c3 GIT binary patch literal 28877 zcmXtf1yCGK*ELRXN$}tn+}+(4NN{%z?iSqL-9m6*+}#}(cXxM(f1kJhZ>zQ_nwh>$ zPtTQe&JF*mAcca6j|c$)fg&Rj^>#c{P++M zBoH#TjMfrf*>WbE5i3!y=Cu88#`)zE%8R{=>r=R4M#ASddH2i{s-Q9Wz?-$tU#VV|A zd;~(rR5A^ME6;aR?=g4td|^*e6;}i=1qdD!d>1jeBqUzRh_C<~#*4490aoerDD3o# zOvy8;2+K=LOQQ}p_GGjlXLtJq=0r~H<9x-$Oo*(p0*+?^BSU3tyhOM$ z32xthI7zYe0`Sg08Jyu&NJLVl4{4G&vbfSDg0Dg+juHq=#`|>b@5fjT6BsR^qM+!9 zH=Z;=*vOQs6Od!10RPTqJZ8WEAVe4+$=_GOrvGAxC?;AK#bOG~vfisJKhLQBBz;X% z0Z2=S*$u|@uqK8#9jtE{dm_w(jehH1V$rysW9f!`cAKV?E25;JC?FY(>Ty*1D5vWV z$14P3Vqz-c8uuBbj{vz|PoQR$M!>s86IEjf#4a@3wc`7-=az{JzD9B|q=%KS)KBlV zrkSgq1{tB0Y%r)N;kZA5w}{ONI_1B4U>fXznBmvLR;0SwnqJ+%!Fv_)_W=d_YfNBo zKst&Z7BCIGGh`(mf%_Fd^4TIMDFdB4LxX};f-<-?LP8YhQVH^g@%+R8TS3GMr-m6S zds`Z*Nju9A9`G&?{91*xnpqDGkf;5P>;Of$ZrGY07f1WI5hcY#3YJ-xaQ9>i!655D z3#FEU`}DYWE*|ZCVGjSTBEJYF1uqniHO_);#>tt)I2gP&0C@s>sPX1401g}n_{2#Y zmVlsSC8eaY@r!JOx_=`VMUVINAMk&lb_W|5EM3P^>&t;h&#Hkcal%GW*CZvyDsV-? zPVZ(5$#C~QdM3hD;`zO3kp95XQ?g5VQAdf(1%7&H;Gw+ILfrX++i)hN+m6s66~7D) z4FxsY%R9JjU9EpLVxppwf6Y*^lg14o+x^9Fk}zY4F)>+TIIMy7UNTZ#KGH*F0XNqF ztyppbpFS`IQ_-P>;PDMdj$aCV)LrK)dZ5S(bGoAxAm?;?%=O@Ek0N@^SpUU1mq82@e+>npVzqlaetAqoN>8`#LrXj|_}uYZL2hoqfd zsV*(p3O0@`8}KIIa8;l$3I3T4U9_y;AKHED~7prhk|a}YFWKjU?e)5Lqi{g8ppCc8!!6s7m{C=5=DicxL**<|<@S8apa6|Zc5 zm#q0~_(yRg3ie<3o6Bwj+DcCaIYa1L>Gh7gs{k&1p-E{_5lmd~5DJjudv(u}q-A>SK_TK|q@DyCb@Iw7i;Z{RW zJgRYzWPq=fYVJkJ!`l8sh_2Q8c=&ZIMOqs{h2)*}n$5rY$5zq@upO$+ZFiL^b4%O2 zSTo7$!kf*^u{q|D$I04Q@h6ZWIxuCXlrc3EeC!a;>lRhDG^Cm5I(wkLDON$DR@BGv zYuLCKV`Q#>Ju1nTd+3`V$kNOV#~rl({n2X*PxP%{;97|Fq@dElQq`_UEW{LmOOK0SnYP?!Vbxi*g>fORP^?GITepZ?4_!!4%ZINS*}NX5eSL-W1*je>Dn{DVHI_a*YhF8X8=0rmUbVK*S>e z2aJ3L6y;olyGr&C%Xs-qgNwDd1dNREsyRNBKg3J<^`PG=?1R1&4jg*x4fPXc78(EI z2*LM7{%T@a%Slcd$?9=%x)^C2+S%}46Mo@?yZn(?Rpaj*o31OLvFZ57S+#IK$_4Ca z$!6HJizt&*e&Kp<>~%yg99Vg~oB}%3CVe7UI>dViF^Q%v?Xk*Ukt*vJ)U>8GOiThB zm_1q(v(I{FhXwr26)|gy?LD*BPjZNC1eqeJQ=ul37k-gY-9g*wd=^_;;tjW{yFJ1* zwQM6%wWS%XW4|;-ak+Nh>HSl@a+_LX3MX{1a^KYubi}Qgi+5ZQ5RK&MChw-jE9b94 zoD~zVkC8#Ty3H}_U#4ToVMOpMK zuo5t3G~PIKFC3Vo<=ND!Vsi-|*&yD^f)-{y9WLh9<5gJA8iM}rE3VUixEdm;S@kR% zW`3orw)v=fFg5jb(qgf^!MOY0y>(_JF?{D#U3m=My@2&h*=;!x!#ApFbTkAAf3vJ7 z%v9lP=|b6+)2CkRlC3aG9>2Jn?gBb+CC{N9#F#RvXFo#Z3n81*5vJg|^9?odZ?~DZ zA5!Fv(M;Nnu`Tm!W#V+PUx4yWF6hFPBXFCrHEfFHJZ7r)_KqTxv6YebI5ooi-95hM ziU8;XiTnho3C(oikF8{DUL3FIm{9nbjvVNQXU#6Yui)D~th2@AURv)Qq6muQN$R{U z9&k*E)e`0mDG!aD5rk=o>@J{AWh|REzvKorMGnL-dIa(+FOql}mkuC{;5YXa=!2#| z?ev?B$sME|TThx}S2Rr_-2?1A->sPPIt0$!+{~xwWz->Tf&sR*Tqvim82T@~dDVgc zf}?aPnxLY)^{~>s%sGee#OzW(U^(|TQk2&T2neC%=DpAuPZCNm|X$FLb#sq%a|V!Zk2I z^uon7+<#R-fBOe;Ixs5Rk{gZE7im8vcXnF@$#xa&dBdLX2>3Fq$a@UK!*t!BoOX84 z)aB5dK;T?Hm*q{tGgQbj8*bjeLV%!9x=dV-^$)?EIu3nBzBDqb`_#20omcn}WDC!kHEFF1(n9QW zpXq;`j@*1W-%P$A(?V@;XWuDC73Ffj*Abv|Lwyq*52${-1UJC)$+|e#F?)E5tgjZ^Js3^sGaqL7^Eq*(4h?h; z+GV!*YusU;5_3x*y$aL(BM)jxy?GhRxTp`~LetO;Xo=`nNSNE*DIL?Ow@@maS0V3g zPtfJ}Xz78Mqon!*<9of8-~aD9xpd0&b`z?-tbUX>M&{4dsrKt~KvS3*S^S?=wwSvD zjq6b|G#_-TTs&VGL6aei~MN~O{Hs@P5)Vi2b zO~fBdM;W8PT5tAmRrde$C7fFXRMB~qrZduJ>iJr;&mM+zd{|VNhg=bt;45Y`osD8z z&|t-8wz%!T*)wXO&(w=uATwu*H0bP5Ctr3%Z@#HTZY_M)@t0fg+<<;Iyr>}STNbE+s=w1*5-;QVkk%8~7xA_HX( z{iTGIAmmQvL>DV}qP1H53Aq>Dhcac#`hQQ>(XE65u}uCDFBLL>rnPB!856U#33Zih z;mu0KC-j~p5H-bIw?EN5*|as}gAM~FKzlyD)vTW>UKTy_V*+6Q6CWqi517avr-1cs zhB<=9y}C;L#xrUWj;1K<3*+MpwJyJj+FXU$FY9aC68>rvPV9YzK`?E-q3_6JphemAz%0=xB5g>~HtZNC`P} zWY3yH(|nHyY=d!$IRc%E5P=VQH#fr%h3DFr&T{{KZ6AE<0=nVsV ze5DVp$kAA2!7`WhNuymWDQJoHkRLfZP~)i5a)nCBiV!x=actVDl!~N^LCkfhAMoXR5sIDnBqMjin z?!)`vAeFdvir2tM>S~JkC5L0*v$Iu{SK(l1)eJnxlL9NtG z2bS=hlV?_ItBX6MndcBb=g2O-XIs@S6tyOF{RWr~80eQ*TcyGi)!&EoE~-3EwD%kUpr)eImLUR{NPRZ67Xo z_l8@yv&ptPyOG0rJ@I`uH!tixR$?pzD%LzUVY^S0Y3fiBdcp_ z8c^6^*)kHKGjc(_OxUT+q)OBCM|;w$9kFDf!1#6SOf6N~J3S_bS0Sd;B3`~2I+B*5 zLJ2pk^5eyKdR>4NC}KWw!gYO{bzZBtfuuurzYij$U4_QH4(Pa2%$||5noZ{X$t8A$ zLUNg@Ks`qF?ab@K>u(!khVPGl@o+2Y0>N)fX;@^qfs`IpANnN5s>B#aVw)*?dwD2zp-zVlX z1*{{?nF?)<S?SNf*4!mNp<~q))U?Q%G@h_86?`u(tz==-k&rt&?xAc-Z;~{ z<*^P0tQ*!yru9wyV#ZJxP& zl|YCXo0nV@Av({=Jdcir;!mRfF71{tKE$iMR+T-${;}HG;^R+I)}pD0WS{0ovr&`v zd;{cq`5}j&YImGs_r3+dz5_9=QnWz})3E^tf}P}dtnjtT-nM*+sdxr@ufjr0qs;Os z6iwH0r%WNKUmv>U2JLFQIb9*A>M?X9PmszPflV5Bw9~k>h@~a08%BF0KsP7?y2=vK ztdo}$U#+<>;8F^)iUIgIUNNI}Q~Qq{0tCdNWl9`dlRT_VF*=0UdOD)DviZ&BKx!an zTX_gR+JIbSlg>P5y>c3YYC;srauY6Cvv?#wltTP2L_F$Qu3$!HcQhk_8V?J1u6XSa z9Gt1rIGb%Xvoc0U-5MVwA}!wDF=fh2?444Nf$lk zzSffWxu)giJ(1;va8N?3)7WytW~fIE~YStpE$4Q2ppTfhqOdD zK?bLivc>%JeOHxeAx~S+%^xr^h z(pl^6?sHsZa(r*YeQ+3mfM`pvajV)lc^C?q?UT+iu;*bE!YyulH^c4}aM=j-s9%x4 z`4}J^Ifk9u4Q;7YFY6CyL*W~8q4ZT5<%!6vhm&Jy(XQBAXomHf#E{HCF9$zm!2 zNIGs6Me7kb#L~T9GPx{fdkzoa6>7ctb#vRksqLcE#qhJU-+dRZ6O|u457;O2CaSjZ z(Y}N1z?eda8?82ap2r<_X1ny7*K*)XM=Q(gWsD%Lsw5ata{VX2Lh`4g))(Icw6CI>ntxKk3|l zf`ID^VJpUQLcK7otddrTH^@6Wc*OYa;WmrTQ@c61!<4qWeDx0{GCOx<)n4)2#4lWM z$qn*-o|k)AtX-%W7kRdY28WP}&I|g8mUjfa%DJ&?zRhjE+n6IV?$D7I+jqZe04yzu zT^m4uJw>cCsh*Pw`=K~tDo{RQSty+Pk@j7uh34{pA6$*F->Kd3jU$&bpgN%7`8Fvg zAhgl^fHZr-y`FOrgsEUK$l%5EWgLQElRHI9I;?a4yG1;Y-mC268>yk3n)k@|FzGnR zACfKg=lJyP?s_`9MSDN`>qGgUY*_)7t%<60y6ec^mzV$EgdQe2LB*7oD#O0dWXmeU z(x^aB#Lci*P)S#^i|U`z?OGXFg4GVZ*K^OXk5gZn z*sfGZzag3}@$@GW(+}zN{o;vTXgsKvHhp#dQc9$^?b^U}u-@8Xit=MR>n9v>UR^Om zS5Ywe*(*4!R2j$R;i=k32;bS^@+Nu0$R05s3nSz?D9-uV1ULYFZ7~G>d1Ntak+CYGC;KM&Z#1u3jD~9kA{Cx)1`P^sP%r}Jtg3V zJDCR}R?jD)JX-b;Hk6b1kq^%yky&{mIq$xMMy54qbpaZg+!CjJe z-jzUk#%!{7XWp!6H8xeSv?uz7zEACEh5?1JIz8_c?{VpetLpLLHb|bF{#zD&-qil= zWM2RseJlBHHb@NaD*^T#B2xFr8rs_se(SaNCzUf^WWwv=8EB<_r9jz_!$$(Un?1UG zN*9KUq-@Lpvc4YKy;A-#>i8e3F@ghYwm_X@V@NluU(X+M?G<`xw zcyB&pwea}K6NsfqM0!Sjqw@%Sj?l^%Vt2p?cDC<`j)r@*2gVdC-&knXs;2aZ!r@`= zc4Gou(5Fl{Qhl9JWc_|zceI}}H*)COY4EU015!ZGl^+p*S>eXUN1CB5ARgkJzlP6+ zhFGuSck~prbVoSKBGZr?k=>8Bs3xe$9uB{23=0Jf!zXwPZjNJ_At%?k zM0^|RQZuZ0Rx(sSSnssglV7Y&cxhA}G;m>fB{A|M&f-mf?;r5;5iEFgu$pzOb6^N4 zAEjn3$ddLnuj0%(yiZXHysohknrl%yu-wdafX3OMR}uqrXp3JJ54?#EqUC&>zf1fw zc{txfTtO{*12ZYL9i|WWd3vmOg7b8!7T<$eT%T%ur!B{KErM3qL$cv?s z^h`A&JgW)w9!d4UAOKE{#Y_Qz2I}iJj!sIGoQT|*pTxQ>(1>D0&};^pL$EIxy0@3eA4s40vOj$D;2oHSX){rH)@ajE_kF<@ei`AcXA`Kx~F z>p?BIE46^n-dKpZ5kL8+O6NES`y0t6UM$aRR?9}6lcBIRE!HqMd6mrMt+>j)!m@aWl!!`|nHvf9S#2*Qd( zI=9m}$~Pk=gzRc@e}d|w<7yq8FX$m?8{i*5C_J_N5(wxG6!BSckYCF{VwbG0zbTv7 z)CY1Fis;6+vfdFF)p6WQfj4*+t9EWwjLc^yU+vgtrN+*;g9pt1ZjkX~59F4=i zyrx7_GZnYB`3yT~XX30j`!9zG>S9y3#3}9I;<}$pqXZ@M@Aaoo!pdGK!;qZq_7xGO zppH4EiKVxXsJ}~8Z?^eihF;>nP#}4sA~=qpRBnN0ehobdJa(taQq{qy5BGiFd^!sA z-jPE^k0*wtTowvVA`0L1tekN}d2Brecn_H9|EP(Y5MI7XCGaG!DH1sj1w%Z^J?S2( zbtCrC{rJSgUOSUHoThfDa_LkD`q+^%?emW!_xKe?(iHj(yKTaTg&;B7Pz}?{ne94f zDljajxoosJ+V5y(lM!6s7aYKwC?l@Ck&nxzq3zs{bu!uPesc{Ogw3OXE4qJF1H31(_@OoMBw>iEu~*^8;Olr z2W`7ymPpqOzXB0zPu+PYQ}%FrVnvf4JeLW8QxX?hY%hjc_aQPu=bEn zXa29mr4;LQKTPPu3ZL7n1a1?`Tf1^bE12I1lxM<*{GS#eI0cbyw+@`-AD1ihKmQUL z`7Y=`4~{+PT3QjK-?h_vSivraKFT6_xDiwU&(u z`Z{adGTVATBXp#NB_3|={=dDC0v_!h8<;yuL_EB{4UvjxwErfrjZM(Kn#0}c;;gnL zN_alOR{Vf=_YPMPFnv!%E%0u=WdEf^_RJN+T$8r5T>%AIv$>BW%&F|i~3f7I6rfQZ(Ci^^B>w8D(xwlO`L4z4FteqJY+8e z`d6|r*tjuq5%pXU`Ymn1h+I5%p^K8_b?m*@9!W zND1%dtW;Zs|DbER$D}eP>F8T1jnutX{)$LJyP3aMt6)!r|GXY#4t{ zUCr?FAi_l9iI!5Rt|kmB83GU&SJHeYjY1H1w3P{VV8)Zz82M+II9b@}W!SwEz5{mz zF413fx5@^Ii0rHXj`d+S50f1bIc)xYe*vEhleE)mel|Wh2a0V{L5KuwZ~_>o+}IknZ%Gl~ew3FU^AkdxG3 z_YHcZmQc-Tef=JJjw`{i@v9T^L7e=&_L!iQm~rdEN(S7f-jJ69CL%=>NTYwtxc^`3 z^NI8Os6$5kHKc4J``=A^w(Lfp*uzow@V7oOVc8wQDUIMW5!MQ$ zzWz`i7fF3rIvy7)qnJ7y$%^z9q3O@efq!;*ckDAxIe^aUKC#AYU^GD*OU2Z-eZLnY z_KO}3%<~8%i}(y^L4F}Ui>Wzs<@2WsG&~f1hRyQ>v#Iw9wcTA9wK{Y;g__48E!ssN zy-0s0<*qy|;T8}YkRh`WHy>pc0h|!?7@c1-;Px<@`gb^TBBtzgWf9-bRscVeEuO_k4x zhMTD3zof&M)~0z9qyEV02S@pth%LT;4>v~jQu_&Jpx7oil#V^c9nDCtwDNN?bg-f- z;^N+}*u`tY-5slyo}r^fV)EH|7VW5G$I<-Ocn;p!@G{ft{3I`uXc9bj;WolY^sh{$ zr5%tT%+%9BywZqTrUM83$gO!Wj&bhbr1g2<{kJ6^Q1@j@Tt~USEH3U3^Wz!1qF5l- z4^IJIIkdagvGFQ;e_Un-qL%*5+yZBCnY6%ckA1ahS^*_7lDQ&$|>je zlH%jZ`xA}`Ne?w9Dq7=sMIU=2B?aWf72#b=Af&vHcae6WqIi}n%A!BfB-P!rc zNjF1C$e5D%4-P(y<=@ZxuJdzCrUmDio>U|?EKF9`Q}}&ixjQ0&hGOlvGpRnJ>f3d? zUZx)t+g?#I7XqPPH9-FT1dK>Mqa_bNA)L>+xHW$6ry1eqR$7dJTW_}v0&nE2dFJiy0GkNB8(7lmPKkGpR9Jo$2c5(fHUgB%u=fYof-&K~ti+s|!s(H_Kil^V7~> zFnq!SlK-0tMa*YHKRv{FoGM$vpn=Y!tPxU$%#EAnOOLY)S5*uHIb-H2U>5i_3Qhnc zZk?==Gsmd*5dt>302Qxptxe3a#ga_{@}WE{t0vIUj?qH<%O|aWmOA)&*WUJllJ!%g z77p~&P0fNG!bqRVW>HhQ5^jBQzbpNC#ZVl0bETbcMCW!rgfC>%(6-FHZEoP`7#Y`t zO`(7OLz<6|cDnK)#@~X&d8a8UZD~(ypgqjUr3~c@pe8Wkm$L&lf)>^c%hkVIni0b+ zB-w1l%>=L722=8(l~iY?)r9(g+8C~mqoB8IC-(f6^Xo-claeCL9(e?dYPQNZ-=6VS zGOB?N(C`F0qVAnowd7MQ)z^X~+NA?)QgjvO^sLd7bH9A}(w8TC z+Ltu|`boc|uJ-mKryMG*-86Lsd&_y>qk+=jV6w;F_rGm!*pCQ8^J~TB#AS3xX+vrN zNyyH=Vx_Q|#Up+R)av-da-}orD(JvxgM`GY0s7l6?j`p}>cn?^z^hZ@pIccqq_)<= z?8+Mh_-%)^X6c5 zu4olHwRTqC(2$N^`IeapzL)`Z!1 zIo;X;cSbsHxV4w&xAGnD>2&#@$uR{!ZgB}$?}V7J-{UHCfw?`}BfQ0h-mzaezHGA2eKKqk8nNUT}71nd+Q5rS~4tP$(ysdoHQ*h!FHcie=zRzrOc7^2O#+-5_`co7I5y`a^C4Qmxze=pDGRU^H*h2|NTv49}`qu{72(`4)%c`>kQ)QX>8TPLAfjlAq(g=y_MQ}r+ z!O{rU^YczujtFWT)}T)nmM@IAMl-QC&uv2gUp!L~5;dJGAknEa7h&avSt>cs_X}jj ztmQxAAI0~g3ID)sA!MM4_?$p1pc^8{P!Z_J%chQ+cK^V_F2}%)|-<$JEw%RFN30>iWGj6gUUDW>#}lp6AC^jXBa~2!kbXiSTi-EhtuT z?e8}@{*h0m6iOb+qga{JG0z{g^2uD_?-lIgbp#1NOaE$STNvT%W~k5ROvT~Ab_BgE zAtUri?@eRlM8x#F;=dy8+=Zk7ZE&_nfwOPRU<;zQXSeM&h1v@Z%tznM)8z2WU72?| z?Hfc7{9_6fMONcaCwqFahwk+`m$;i}X!#d-IJhTH1^TjZWT zu~~7@5$!_kWZ*W3az`P_+?*uFygsM_my1N9v_1-hS+tucz0@$G2N;WK^-&`fm@G}mRJRJPgwD0QTF>!r zrTwPSa-_i<+!}UOPd9lIF8)0|s_Fb*+JrD2pf>yTf#WLuLtS?KyY7g0JYiUi`WDOj z2r8E|F7wwboH{&Am{={VSxCRRhBLKsg%ZDzhJZ}|v(39=yT2WUWp&xj?o(4J9ZbV4 zrY|jv3wMydo09{4cQzY0mDge`8D5d!t(dbrs=HCXha8ibe>d3?HXyDYo9*yB7Y#vP z!{)O*CX6j;`fRBt+L3`?@aGJ)fm$GcxnA-!79K=0c<+uBt1NNtpNCdP&O`=?0D6`P z(*R@~u-^==%ryL?TM6}SNSh~X^-IjIDK9ZyXn-=NDJ=6_^m%m_C68g$i3tM&Iyia- ziC`W)NpmX9-ClqCQfrK8=4nk~*o!A{mj190&M?*M=oLFsPZC zXt`G~3WTm$HCV4v>m^{#)BGC$_$9W|y_QT{#%T<}bGE>G-cBEGEM4zHS#sfL=?C8e zZ+UZC_r(Rm-D1aRx&j9`f;*+z_Mt#Q60TV|VNU-)b()+ktE(DBOZ*7S;SGx`Nt;(} zEC3(782>ZH5>3IKZdjUoOcEdWYY&xQ5wC}`fH7Xp zf&QV2F+Eeypn>m|B@l3yn;Oilm-Lj!WxLh308(;~ngjC~XGL7YXVx$7C_p)jzoMhV zvLQKZpy+V@9vs@N(N9FX5zzV@%arA8Q(hYofwtbV`){6myU@beQ z;WcR&$?EFt!VI52O8V;X;9E2Tzot*_@$)TPf1;wWg4cHJ$Mx%T2PU!LX+ZPfxHky# zUgPp22h0@lB}?le(*B_!gXzF`qkILCW22?khy>r~dZ^m67c+?;_}O2og;wsQ*B>@G)-B(leM z<%@!9qF8Tq4IwdOVLSJJnrMAoo5uUUyPdqZ8~z7Dl3UPqYq&57Oj9NZC6b}pBBUTk zt4h0Uo2DFEJ2~D-c=_~Nb_>ZemGp8S9;Q)0q4RC`8EBfd?PuH8*2j=6ym@abB!hM1 z>0kkdMvk`OAp936tzZ>uD{k9KHjf5=7fG;Y&^xl9tJG8qYs!oBvIud*fMZ?}<56 zZI#P)J)sRN1#jvawNewN|LQq)k5@GY67AIV-@P&QtNl{#!QB!nM=3>>$3obKoYQy} z{~~6kUhgJ6n$)haoZ~XnJ`w2_{v=)^fECE5~1qs7B^v#!RN5*EZ-*rpf ztZ8_!Yqx?_k$pdsY7YZYxmfCSK``~QCAFwSKMJ5jbr5*yG;oF z!j->Ze+=t;-hpY2fM6eNdJy)gLqDYrRS&~yD=kfV_01YFb?K~)e!)3|M~hi^?1)Y0 zBjGn{)NK>l7(CMNU@Uz{Kg`nA;7+N5QOw{Vn6NsK$!ivlu91zrT`l0?OHZs{yDltk zA<*?5uYfm2GN%bnZO_L*_8b=ttWzsN7(z`%@&~)LD@)62yaVr%eJgyDRApV z)(SV_kDA8DlUtknjhpiw+*dW%alu#@-BZ`{R=h)CHSa4E$nUl6p7VV?dS|a0Dm<{J0DhKj-U9jBL z;nH#wJg4)5Oda$Q^KdI-^yaNSKVPp0dxix}CPMpn3tY>Rc4ocrHTWqNeoVagfw1mH zmG%DxCU0poA%pt4HiArC`!*S&J}0`kM?zD!dNo3`7rdpxGM#)ckEB|*#U@JihaSfh zlAO2H!gW|M9&fSbSxdRAEc`o^!qbeb(^+4|4dUP&X@zj;Vsgn_4z@To#tJ|XfA6s) z0k9H|CBH;?v4-6xyET2d-G(iJ$UnOMD;Rry!=};EfnuxsLT%>i1B4QV+T+3Ny5B<7 z(H+!0kUL;CS2oq|K3@vECfL)eEoCf^3cFc4Sf0$IEiRqp)4R?g_Q-h^9O8t%jI3u{ zW^Ng3dRq>;{LIf5c>NpS$x{(0Av+C!3EeUUywJH}6-e!zAYoUVmlC>COgzy3EO_TP z^*Q>U8&`PAarV7zQG?XM8qs-DhObJjpS zlBOZ7EfDk{(tFtx(F{X|ebrAIY%E-y?mey2R1Jo4x>@OT`$BRfh7s8|`hNVVIx&4B zh+8jGq4cN#^4k?WwbGb9NSL;Vg7rrPU;##tB^~4_MzRrXb|UOKJ}dJZ0pE;MJ-s(! z$=rU6m}6p(hDl7h!&`LA%n-AabCF*0Z- z#|(dryMqZmTg1?|H*E5eN8LaSwUtzZI$(Pe$eUQ5dTrjT z(Jc#tU;91CfsFiStlh9FG4dWxEmSOi^3JWNrx_0G)Yqs&ImN|4uy?|yp|J76Gv&X) z!9TiJKNMr?F13Shs4wv++1;@P=<5zTci-KqpIG~uwPRFs&4Ukk5Hg#FoW?f_PL|Fl zt?N=6I3#X}TxZ&qoN|CBnClstskk*DW62Rev0pfnB z5>H61zNmL6PTv0p=9CRyv6%+|Vr-^=5bsUHs8eMdr`g=#(?15jX}OfyJ9ZQ!GHX2O zbkkLQ^uU!j9A7p95ADAfBXyu!kUJayz+xid9dCwQ`;Qe`YW%_F)UQ0m)83SM45%$U zw}K2DM&T3iilAlFkbbVzGmSO)W&~`chN@31l~|GxKOxQegKe44U%lu=-WJ8jRC1b zc0DJGZZKu%-H%W{r^tBE0gExd9r(PeY+s?C6WX`z$A93~nikqrxkuab933Jm2q_4u zw8o0?MtrtGFm{3M{^dFjfac+jwtY;hLnUE7+1D5I+XMHX^gEI8hU9SIyuZ(SM&-5d z`#R?-##R&28@3-`+oQxi19!}~?=laoMc7KcFNP47h2tNG;h&`UJ141*V3&#ds&;Xf z9(=dJU`P|HvJKxz;cgZ#f+qjCTL%!vL(e?LABoRg?`xBrZ$QjJ1Pr7ZHS%8&F{fV){-KNXX{nJfOT(u26Lf!|r zob}Z{lqkoQ)b@~ZQ+R4WO^~V>H*d`CE0njNi|Ot4Jy!rn`ry_s4VM0e(v&J$azxm6`d9 zb9)l%vcxEamZfoDv1NQfO#6o7T)M#4bjzU@(!fmKgR>*@ghL}1w8Cwn%7-hEbEG**Ot8I&MEaTuMc{i>0_?~srl*F60 zq^fMz)qFOb{|kGH!vcKhpA}v!hU-22G6vIl8d*0t@2Zo7kPFO*1ILk@Koj4FKPO?l zL=*R}9AmTd#-R%IQjmD`#T=^5=)|VZj8+cUd;4GrA0eFq!i#X8cnsW~tgftQ96YXu z^7-#ZXq*AZh%ECoc6`~kBr=G$=eC)#IwZXvafs;5%_rRJ?E%k+IR_O*jr6}3i^;u~ z2!C&aWjcjp7A3eQ`0T`IZLz5bo9bzpLE43?eQ^V%Vq{PI6g?xDUtqnwe2fl zKRJiJK|ZEMu&mzcvO4y-<)L09Y;EOPN;eX5s|F2C#VqS;a6CR9X4=Rn56*6AUt5e? z)WmjMZf$clttwabUkyDVMsC49xx3jBb3#?(Ws8)aK7fCR(G;G=a13u`{ie5u~ z+EzQeEdyq+kY7$_`xDW~)xKz7cvN&-QbO|&_xl&W!TeMUnlxo_+`wT`5X+~Xnu`hh zOxyLAm11N&i%w1nBsC8QFD1d%sxgTq9>}ESsN^tF2FW&74=Zrmz3OWjO+sU zP1EdFJr*!Q{p@-*U#5KT$Zp-^zjRdi)z$Qn;F4Kd(ppHw57B8?cjnLE_4m|I-KZTj zZADqj_2we1|Goe*X^yUF#BZFlp_%fw*z$DQ;nnQ?iGy<4w^X7dZ$oKw+jt{kF|^A;!7JxoJW8;c4W%r z@G@<-p752&IMU#i8SREsB|kRwlt-;yxz(}_YsY$|^~=s*)MWr#>Gwd+=MEa=y)A)V zECXPzCfOyE=Z%PaGMh&DUitjyH2Lx)%?}yPdy=!o1${S)Nz|m}e|}1kHdH^L83~@P z>pqv;Zkn~KkuBXMCpm{EJmAhYO@Ag*|Ax)>zeY|lIe`O)@#VBQo zvZgn+iQnMdaeVJ|DoHZm7zXzS@4PWEQ$R=mf#YWVe=S{gTvW~12bAt^P$}t@4r%Ew z=~%iuq@=q$C6|J(V@+&gFH6)lX*#vN9YXV?b`u>@VOE*|3XuE5(dUmuxn5@NRd;Agq;do{==e zxitDDEZFXC5Pf7Hgr$?4S>12tC02^ZErX;xp}|hs^G6zi!d*U=TU6u%Z}becZ|iGm z@cnAZHg9e@LsKv!S~!B?$Tc|D?Cd2<326n2I2a(M$v=4;;mZ-r$S{>3!vDh1F@4y8 z`(vae99x6EWC|7*$&f-Q^n(!*j%};^CAFusPEK17RNtEJW%49kKfhY-@r*msWHs+= zo$a?ERXGeBbG-^eWT0MbfbM$2~K)M<0^!^`01uYuDGI4Bvw;Hr&Ci8Vm8(eoI7q<@MS{9^XP7&?WmF6)!DWH&Ns4|&pDJW-`dXr1a+NIf9B>Az9- z!4rMVNEm$?+pzqx?mI1Fps(NRZ1ckMDL9OU*U6+U;( z*|KLY;EHn$65IuXS;N8tiX>?gG#4o=`?I8rH~Cx#{P(%5c56FzqxTW##!o%YuB%1v zi{iYOb%q)mAG3`JZaB-X2adw2$_b(<3J({?Gg2fQAlI4%)D505rIo*@$$RdsaDKWy zuF|j^>7mN_ZD1K)?4bITE^KHM%#q52?;1^8{FOFEI|x}PVbvYmnWFXq4qP9Ix&WN>Dw7QoyaD{68{C*upZ zvg(YIaHk(0FDM*qXBIhsLxoJnzCxw@}Spup<~nEMwjtULs1I zq?Gis$C3GgF8-)rEW*l=V({hngD*pNQzXNCi*LiHnm%Q+N%FL2zJ}Ro~oG4 z9LudI8mkDr)L0;W%fvbw0db1 z>k7I*eFzM47&qTE`+;}Zt7hV*&5&`-Hcfsp+&an^zgSfL?B(}T%6-xxo3rUrLeB$x zrzB!(cjR`Tni5<&n*+yBY|XvB-2@^7r#Anb9`Mq#4=gHZ`}nYFMeAdJM{pW8YRj)$ z*JWw%L^U2Gic-6!Oh&x1QqXjc;vse`7W{cKZ5^VYrPo1?ju_Mc6MN}jb4;RgDa77Ba%p9e zs=fq%%}l28=R^L&%Bhbg5`;KRY8-V4zjq{>O>FkKhrpk;pqgeFP-Y_k{WQ##7i->W zTNKsgQNCL91<8;GV=8iD!!jk^ zL~@_}40#f{Qi~ZA_G!03n?}>l3G|XBpUKqgriMrNfnXmP9F56qT!G%D zePngTn4t)js&dw$Xi6x&o-(SYZ#txv-9$I-rF1iTFf z0sj8MF2T7U^FmA>8wNYHO73-(!|3Ok-SF;g&P}|0O(7`Tx|{uo1`5(+{=GzNCN;<+ z2nMtzy|^GD0gh{#+}tqnRD{?0gfqU4emDhKGGy)oR@Rc|ze&zp7GZCswfeQ=xl<%y zO5&>>)vfVu0TmB5Cu>Ne2t;`yMqFkfg~`Wfhr86b2JRwph$()%fj(&=L-nuhR4w^x zu{ft&9nbf6`IlNTv%IhF^8HL(e3J9^Z;>1<6M3!%!{EJ+M6@arX6i(`nFf>v(@Ny<6)qo>>nwG?ntDr3jRKTE|G9ns8dCPjW{;ynma_X0UFV z%vv5zzFxQ8A}O_(YJRUibb_(8)yT8h?i2_!*+duIduN_=nA4!PeW6e+C-bD#dxo~0 zc~oL!bu7bnmmg^0V7K@BV`RurV`&>%wxQ%lG!HBN=`d}QPP1)>5l&I1LHFMZlFpju zV~mi8*>yN)x-4#kmyQ;=H3QDS3V+HBiA?i~;(F6$F|L zJ{h8pG1||R-XXUp4;dHnJTi|}@pYS7e+;8o0$#8HJWlt8(e!MU;O=wM=Y!J?{TJni zbAkk>yNZstPhsBkDTii`wVi9nsSTpmu<1w*0FoO6mg?6ztlxgNY<3W%|ym;-7D=Mj&SBsuwJ>uyfJE|&?h zZO_}%U6v@fRLh>p8*;@*=jcLo(+q9LMw=^ah_fmE^1iYA9D=cOu3Fb?4o((_j*hEt zhG5q0$P&ujCgXk>te%|51ggo|Wp0D9f>PkKhQV3^SCkhuGxw%TcUrq1fN6e`Emc!f zhLUv!FO#jWVEHh#IJIhJgT7k8rY6-YmfWw5qQy!ZI;`LBHDwVG@}dscdmn_pO*@e< zu8AsH^BjVGaV+<8G?%-=W=%(cbwrK5=Z^5~)I&6!&owE0&lC zF?ASs1*{#dNmO+_f7nO`YB6+L97%CqfJh?IXzaC0PZfV8zij@Fsm}q*9)C|{|8th=dHr$x=!jTP-!asW3}eMZJ=TE#J| zIdbVJE$yX=VY3tFzH*(B;=?aC8yRDS*{**az2aBxGCRODl3S4fjb^M<)_GUZ;sUF(Z3kuO5+dY8UM^ zu<$aAGA`vZnk{3VyknXn6Fe!p}|RUzck zdl8a+xRN1i%YTp$&`-XWy;HYcmjx&Q#nGvU8 zm+0BDBlVcHNav`o;|nIp5W@VP!Lq0oVR^X>DkS2}8B5 zK(eN4itKKmNY%|&wV-nw|NBRnC-nwdTf_+>AYPi5qWPJSe0Ub4y{0RDYToM>_AEJ9 zCK8D!uP1)7SK7z=n%8g--!^xt3BJRFY~3yNTJ~i&CaKP(jM3%W^Ig0p#akd3*#-ZMO;TCO|O8E+WBfs1?ws`yFvLc7&my?UnN%|9289s8D!TatLp z)wvM`cSs=&A$3Nc^DVBCd${tH!gP6K3tV>Q2&2Q!JaLX)3^u|X@+9y|=`*JagoZToi#`&KCUnMdK_h(<1aUQ83y`j;LGn`)Zq$+@$X zPbbPkN8}4D9xxT6S{+HdrRj@r2#4B@T1T7u6W%-HA}|?_W~AWbEilN~!^k{(yGd~N ztE-lOib3+qnF{OS^fWf}D$0;(;VjWbiV$ul`TniZoLz6w7xlB8=lf3g#%V~v-DA1O z)!%Rj{*T`+aSd2)Has>0HMG%OMPoYi z`i)zYqg{~e@?g}L96eKjxc-`H!2if7kD$|<$L~ry)a1o-^YF=Q6|yZpA*|kraNhl0 zMp#EpmU3vEA2WwS@n>J?Sj}bDq39LIT?O(nsTZY3O9XtEZwQBJ-7Q@R2U|$0l+quE zJJeT;O*VdYQ4Va8y1-=fhBeQJ(0AGy)c#& z+D{pCiND#ddwAbfupNQ)#dU7(4JpTzI?2f1KO@|W$@RU#8 zV`7-ax1nIbQw_Ry3Vv0HZY@#{qmbfS*S!vtJ=cJhgvU{kBDf{y!hOq$f4{aj9+QUM z3ghmr=zz%ASL3ZQJ+1OQcH|=$BlR1-y2V^O2h(-U^eNtWa=JE&XQoDr0ZYjA?5rw~ zP!yLdbv{!8vKQy18BhOa+O+#}&5Z+5To?eAxj6xR!9Txd@|2E3{~22wxZ}f#Yxeq3 zdG>SGp;wL`)zt!pjm?V)n@oi7VjWG(?!mX8mlGs0W`yA$8M zW@kueYt}qiG@OeXp&X`en6nzm4Be7_uLIF})D~2ql)!YEq_DV|Iy4>u7LSOkWmJ`L zNS=1+Gv$X;ufVt7@O9`rIPPvV>ek;3lPr?E)M3yGS+rcqs=Uo98R5P~l#IuyT2psQ zxoo=wf8r*jY@tjD|rf7UGzw|~6`)yTPe$BH6kiScTsdxF4sVm0yjr!LRHLSbgU1M=%?b^1YbcSy{TOD*+?)sDJ z`RP_~xqn(Ts30OyJ?FWFh14XCWxizsNzcUnQNn133wJ}e$*$d|oMY9s0%0dagfznr z%U`ZfqW{66ybizYhL8;A3D2okOl~4a(CFy3R_lLA_O$t5B;a-m?oI5PIPOnRv%i+u z(LIQUbtW5jxAn^Kg)9us*w%Q8QH=T5$CrO#m~1=)d1u{c=?gLK`1nm#5+~h5w!5j`kOgjl03Cr zB}}`yH6UT~s>U68Y2mFrEw*FJwqWF_lVpQ}*_+3Szi9lqD4)>ch<+|N??&~X2pn5k*l@#qLvuvw+8?}a z7aC;!@r$WLXjVbJhb;@NI=u;^DzkPi{Sv9e1s#vu$-w?xV=wZlqK~fcUH1IHF zMb??JXIR2gb8EPNLwhFy;TiU9DKewiUbdGDm03WV6peQh?0FDkX?zH}5gNNCvv%Cm zXcF_1pw~F5(d)71XB&c5^*3B{D}W95N14H5zP3wlzf$?nqiHaV9&k)-2_!a)Mo{k2gmzL;BOg-gIOZM5B4XxxtrD|^n zmG^=i&0doC(ez?%0$_;c@9T|xSk?L+OkG1xdAA*C5}shEj(3)PzS{ASwvZ`wQVHtF zzrE&3_C)8ZiYRE-jX$Tvaves6PY91)6TTw7KKS#2V*Dgye?@#bIg|0K`SDb5eeP^k z+)*o`@Q!|9LHxqQ$)$|v2wq)jEIZk_5#J`^)Cy)z(Q#dHJCH_Qi*u8P!7m9tbG*3C zB^N=SVp&7GaZ?xh@2h4<0go;3Lt50Wr8MZW4Kcyx9s3J%1$Z%hQp=>Ik({icekg!0 zP}^p;o)yz|q?mQE_ZKJD+;0z$zLjnN+;h6L=10cGwMKY;d0K1ouQ_k=i)M9>;_RnG zmEoR$uE|WITM;g<9PbNyV~aZKhb%#F<0JPsp6i{6Uu`HE{O0!HsraZa93m^^8h|KD z&_8AuI8SJ-mRUr}r!;+Z`w6#WVR+9@m%qI){SnTKY)NT9NrX4uKQ zqG<~!-xUk*Ys(>i=cpQig=B&)$j}#c(4#47{xwuE&0B)G7XI`yhuz5(ghhu;X6+1s zc@^}Qz_z{t|VAoJ8W1@=L^bGbkNtv%d>?v4Fxh_a9xLEhVY1@`W6o* zT%#6O1j6Ry$<=HZM~kZ@eBtJDbYkMSK%f3TZqsKO*)hni=Yq-?~9c#7ZbF>+Yz!XVb|2YucxK&{sge?y_FDf5a3)J?gN zQQAaE_B&fHhf3KH3-{%$L}?N!ItRk@h4~JxgXafUHt%%~XFf=+-HWZ#U_wc`CAT3o zf}r(sMSDPN@;QMkgE3}H&8~YBv%J7!gZ8w%OC$kWV?FA8@zo}}x^C_F2T7Dx0-o-b zwE! zh<8ketP&veS2R_NoFUtZ2$h%}0WS}Iu$)Pa_c{Ba##cF?40+1;X0h^_dJv`gmnv3m z?MLye*jRJ=CnSxpGkftDY5+!*c$lv$6+nEgw`@DAAMxiz-kCQ=&Z)&QCEXcSspc=d z2aBe|{7h$wGa|ujbLo?q4GjZ9JxPjpM>qwU_0nU1m~x6T+u~Y4Mb}}9gU-S)S*GhB zlhijZt?C~!-hiWgQwL=;KHV@Y&eMm&!*q7tK69xZ*_BT(3`Zy_W7%1mvgdYkuGmVm zEpNSAKVC3$;>}htJ_%bg}7b@;W+3bY=37y3+}AG(m#$wHKearfm29OZ zIn4hh1pt#D!>xA)1wCkH5g&4xeOOG?r`2{k{bVhaFC%);O!%qQyJ5@~81QZv>0(-< z(kdyIpVH`Jw29umLp5>IuZjERi_rcJb$(Hf_vknx#jshu3?x?e-pu^_sOJce5H~#ssf`R>!_ZdUcVDgzRk3U?M?V=Ext|shX1e2|=5i zTSJ3!rH-2;eCPk0mrB&mrKJ+KGtV>HX}_ zynIdA3PoOi)H(HHeTmQnrJ%Pv+#$=(QaN*Awci= z7JrYkpmRuA+VV0FPMMy)ON!n8oH2yd zjP@A{3WNIJHv(#t32ug4QOyx)e@gDOkQe1YjK%hc0Pfz_Od)BBt_0EKh{?TjI@I8` zjq(sXGUp$(69b}>U~Z6-$s=XSj$La{kkFcOp$?rf zRi^r*!7jXB&EPK;1A)tD+wu-tlj-q~hGtXT&YX;lyGIT10|zI^?|{?@fm2w#M$o~> z!UQi|t6jiAS5OtA5T1WZ`fU69okDi(C90Wmyx8zQD^xaAiGzT5dTtIq_OJq%yycCp zahDL__t|W2MfE#g!hv_b?<|>{M<6MZeerDbi;1ItxdnY8kO5dq=uqXY?y$dQ<3`pbbk)R$ zncos$<}ULp{-sqOff`6EN-H}iP%aloVkUL=OsBt1>*vTO|1qgN*ur}E8`|m%Ja?r$ z#&7gukDgH$7U${=dAzm{c8fY(rHFU?MM>l1Mq?**(wu7jJK2V9;c3Ugt7 zn+4R$H9r|a!Sx>(+l^%(Q5ES;<3wC5jBZhv(vt!4mojlW=eShcz-Z{bo7!t4a&$AX zy+Vf7cG>+tT6{}7BhJ=7yiI>rPFomO0lau1RgQ1ZjWt4%b+6L7ts2G1 zWeB~JxxCb{sk|`=5hcW~<=b^cI|LKcD1TuC)ZTAy$HcIN5(leWD0iKbd*bj{Uzc6p z5ltME52nwZa#ZbPI|Q?)!}6MtF*Wd;RdBoH=~2Lq%yZ&qOIa%4)h;C9a^ztB)?RcS z&NPHnzCN}`+d>9sE-Rj7`1a+UyUJHXesKS$x8;_6S&TJdQD6`YHh|W2uW9vQG0K_h z7y+0PciIYw;m}KC0oudp+U`lC5Zf-v7#=J7z2dBcj(3we1-k|~;%B&Ue+;#D%}?{- zdRBn)xy%Jzg8Ceck+Xy*ln7lB%RMKxEEq?CzBy%}GW zH-++$oh2sc-iTHu1(v+PdRoBSO>G~S3LY~t!ui$?vqHUXnb~D!VZ(U#m4p+Jn|BBPW9WE~8<2_U+!wU^X)e`73nc%f2F8<1NI5Y7AcLVo z(wk=fs^rwjo_W?ZZy%gh>ZTa7r>JhRwKdJ65b8ry_oGz5C%Lanajm>Y?iiRm5SWmX z;^?6h1E8`o`WyT~Zt4@oKO49kXoQU{@f}G3wt~!;>!7vL*tXqNgz$tkf#J$6$Xbwk z1;;!l?PFyk^~iEM^x$)&AtQ!3jQn2&iixdVeMK7}Ub$$qvG$ct%L_lf)scPVgkMqV z>Sg~e`p1U?*J2o0)Bvjm*R z9m{WBbSF1os*_@$h&SKC?Na>+~|Rd21+5|G#(Bf_x-JDcRW-n%Zw)0;W?thsDAu z%oTJr%Clmm3z$D=U2JTyvxZ@*^EkWN9RtZ}+IyG;{qOhU20?_wx{IQn+o47F?j!@* zXe%!*021bLm*Pi56}Z%6tHkSLKWQrSxCW}Ut#IeQw5z9~UQRjm1g3*1<5)YoE3$EF z7=ralv~!w3!M_3qH62K_QB8a>s*k7*RGoGK=`+WZu)XNBP2c%%C12DBnYei2w@nCo zetgtdZC7{KTgtVQj>dw72XqN(ijw&RrSfZe+% z{tiJH`;f)mrpX5cKS3K;xM++H5U zkv`&cZH@aMa@bVmx!?MpRso#~!I>a*<^B?J$>enCz}Ec0aOUMCHBOePc?#an(c#Jf zt}bbZQRsvGl$MjDso%1yGbzE#f81Y(+6yL?M!gCj8gnQ@d4ZT%B>o35G{dD5%9dXKu$w@4H&Y`9`?Vf`Q!V=uRilU}KG-hxUqt&xV*zY&LE%RHLl z*z785qPG)pI-Kj_QAlgncLUy4!IsA7tOb2mx8Ps3{zW}#Wx?j0i|a!>@@B6;?C!X< zGah%ENWY#E%(PfCuNB)8;(A*wetQ_7<5cP{6$fGpX8_X!Ao@zjldl2m6zUZpol z7o-`2#?|{+p$&VIJKu?Z*3bM64~Q2hj~HY+ua>%cVQ8kG0P@ba?@sE~r;h{!Ptxgn)kB+7 zqTG7O7>b+{bme4*?Rm!K-{*BthRN@Da?YTW@1(YAU)XNk7>20Ypj0{+&3Z?sv4@0n zo*kCB&CTgE@#`cZBG3x86|)#|-hr@%n5vHrwN|ym@rvXP@&dms4(^!xqj8=iFb8mea%d}j+g%GgPS&Eq>pJlG53sc5sK`q^Hv-5 zQbhySB5)X}tLk0r%eb{u4yaNIImNhl=EwnlE1-)F3*(GnwUvWKtwl5&&NWtj5n>`TaKwQ&0+=Buvp%P z(T@zZbA#ld3tpBWS1PZvF(&C&;g%lhvZU%CZ< zIU9xLV9Md-sdR^iYDiI4VgBH{_uu(qXrq~TnF&z=_*-h~l95eII1%#c6DKw98`Hs( zpXB9Eh543#r?ChPEoT!S;Ol5FVcDzXQ5lN$2B%784GY?$&dJ(mTx3%ton1H$ykl`Y zOZIVd7MKKkiS=p%&g#fPfxDLQp{yg{!-ufsKuk>I$!tyQ19<;yo5}r3ojf5Ov0m;b z(eWH;XZQ)1x*FPq&xYb+kpnBu6!C(v_l)y3h0-u;fQ)_mgfy8=!m_QC<3(E5PS^cT zsXILM^UvWUc7Nad1yEG;gDgp2ATFj-B(V2j=KJ|yfQh93*Xcrp&V&6zL#T!Agh%Lc z5ev|lVu6yB-tT-#3ay6d9Yipx%0m&V01Q)BN6&(biB3(9!-(|1$3xHmU5BG<9XD$b z4?TtbZ)v1I3Op$#_^2t7dvK*2CQ1F9nRM@$I}*CZVaMZVhPZ{iP~Zh(B?V?e;pP1J z7S-uIwRv)PZ2w*>MmV_nTtmj8exE1W5d`Sn&``&3efThKg@UGAuTXi5M*odX8@#Sn z&T*Iw|6sJJ{9j9RlS(BVoQ!9`p#@8qHK7bYM0!Qk^XfzWnu6DrzxBLE^2vuEOIcrmwj7nPGu0ha$y+m{z z${Ld!xEH=r3;my6Sz{;xr2MhoKK9Hbv}A@*;5o;x$MVX6dsw$a6C0tGStEw_-n;nBO!K}}}D zN8+o!hhMNAeSif9+6h(_ecj7bbew)TEQkH0Juqj5tA89$iz>$DJ7(X1EefeoagfMm zAic*MXirV`-zFn!+;)P65@bA-|6m&bTUmtl!8fx6aulU?d%4fQWR?G|&L)24)riC! z@F@enTW`(+^Zy?kp}8tCBn1xF&7J+kZWJ?E{%2SF!>= z{(JIk4c`2N=z_2q@WbgF+G9fD3m%f?=mFRilgq9H3HUei22V;U7=0.13.0", +] +build-backend = "hatchling.build" + +[project] +name = "datadog-huntress" +description = "The Huntress SIEM integration for Datadog" +readme = "README.md" +license = "BSD-3-Clause" +requires-python = ">=3.12" +keywords = [ + "datadog", + "datadog agent", + "datadog check", + "huntress", + "siem", + "security", +] +authors = [ + { name = "Datadog", email = "packages@datadoghq.com" }, +] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: BSD License", + "Private :: Do Not Upload", + "Programming Language :: Python :: 3.12", + "Topic :: System :: Monitoring", +] +dependencies = [ + "datadog-checks-base>=37.11.0", +] +dynamic = [ + "version", +] + +[project.optional-dependencies] +deps = [] + +[project.urls] +Source = "https://github.com/DataDog/integrations-extras" + +[tool.hatch.version] +path = "datadog_checks/huntress/__about__.py" + +[tool.hatch.build.targets.sdist] +include = [ + "/datadog_checks", + "/tests", + "/manifest.json", +] + +[tool.pytest.ini_options] +addopts = "-m 'not integration'" + +[tool.hatch.build.targets.wheel] +include = [ + "/datadog_checks/huntress", +] +dev-mode-dirs = [ + ".", +] diff --git a/huntress/tests/__init__.py b/huntress/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/huntress/tests/conftest.py b/huntress/tests/conftest.py new file mode 100644 index 0000000000..2e7067ab9c --- /dev/null +++ b/huntress/tests/conftest.py @@ -0,0 +1,49 @@ +import os + +import pytest + +try: + from datadog_checks.dev import docker_run, get_docker_hostname, get_here + + HERE = get_here() + HOST = get_docker_hostname() + MOCKOON_PORT = 3002 + HAS_DATADOG_DEV = True +except ImportError: + HAS_DATADOG_DEV = False + + +@pytest.fixture(scope='session') +def dd_environment(): + if not HAS_DATADOG_DEV: + yield + return + + compose_file = os.path.join(HERE, "docker", "docker-compose.yml") + with docker_run( + compose_file, + log_patterns=["Server started"], + ): + yield { + "huntress_api_key": "hk_testpublickey", + "huntress_secret_key": "hs_testsecretkey", + "esql_query": "FROM logs", + "huntress_base_url": f"http://{HOST}:{MOCKOON_PORT}", + "enrich_with_org_tags": True, + "org_cache_ttl_seconds": 3600, + "min_collection_interval": 60, + "max_pages_per_run": 10, + "tags": ["source:huntress", "env:test"], + } + + +@pytest.fixture +def instance(): + return { + "huntress_api_key": "test_api_key", + "huntress_secret_key": "test_secret_key", + "esql_query": "FROM logs", + "enrich_with_org_tags": False, + "min_collection_interval": 900, + "max_pages_per_run": 100, + } diff --git a/huntress/tests/data/huntress_mockoon.json b/huntress/tests/data/huntress_mockoon.json new file mode 100644 index 0000000000..5d5d07a86f --- /dev/null +++ b/huntress/tests/data/huntress_mockoon.json @@ -0,0 +1,549 @@ +{ + "uuid": "d79d60a4-f853-41f1-b1bc-394540ba0418", + "lastMigration": 20, + "name": "Huntress API MOCK", + "endpointPrefix": "", + "latency": 0, + "port": 3002, + "hostname": "0.0.0.0", + "routes": [ + { + "uuid": "6cc05227-ed63-494f-a215-c06fa89640f0", + "documentation": "Account Endpoint", + "method": "get", + "endpoint": "v1/account", + "responses": [ + { + "uuid": "16142531-e3c2-4e50-9cf8-2f2bed17c0bc", + "body": "{\n \"account\": {\n \"id\": 1,\n \"name\": \"Test Account\",\n \"status\": \"active\"\n }\n}", + "latency": 0, + "statusCode": 200, + "label": "Account Endpoint", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-huntress-api-call-limit", + "value": "60" + }, + { + "key": "x-huntress-api-call-remaining", + "value": "55" + } + ], + "filePath": "", + "sendFileAsBody": false, + "rules": [], + "rulesOperator": "OR", + "disableTemplating": false, + "fallbackTo404": false, + "default": true + } + ], + "enabled": true, + "randomResponse": false, + "sequentialResponse": false + }, + { + "uuid": "fe19593d-a249-4077-ab07-b5d2239606d2", + "documentation": "Organizations for a specific account (paginated)", + "method": "get", + "endpoint": "v1/accounts/:id/organizations", + "responses": [ + { + "uuid": "bbd0b95c-7e9a-4446-bb7d-34b2827a05f9", + "body": "{\n \"organizations\": [\n {\n \"id\": 1,\n \"name\": \"Acme Inc.\",\n \"key\": \"acme\",\n \"account_id\": 1\n },\n {\n \"id\": 2,\n \"name\": \"Globex Corp\",\n \"key\": \"globex\",\n \"account_id\": 1\n },\n {\n \"id\": 3,\n \"name\": \"Initech LLC\",\n \"key\": \"initech\",\n \"account_id\": 1\n }\n ],\n \"pagination\": {\n \"next_page_token\": null\n }\n}", + "latency": 0, + "statusCode": 200, + "label": "Account Organizations", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-huntress-api-call-limit", + "value": "60" + }, + { + "key": "x-huntress-api-call-remaining", + "value": "55" + } + ], + "filePath": "", + "sendFileAsBody": false, + "rules": [], + "rulesOperator": "OR", + "disableTemplating": false, + "fallbackTo404": false, + "default": true + } + ], + "enabled": true, + "randomResponse": false, + "sequentialResponse": false + }, + { + "uuid": "95ad0b89-aabe-4e44-95e6-9ad78ca56f1d", + "documentation": "SIEM ES|QL Query Endpoint", + "method": "post", + "endpoint": "v1/siem/query", + "responses": [ + { + "uuid": "d3c533fb-4595-4a19-bebf-1ff2a2daf6ce", + "body": "{\n \"logs\": [\n {\n \"@timestamp\": \"2026-05-27T14:00:00.000Z\",\n \"message\": \"An account was successfully logged on.\",\n \"log.original\": \"An account was successfully logged on.\",\n \"event.provider\": \"Microsoft-Windows-Security-Auditing\",\n \"event.category\": \"authentication\",\n \"event.outcome\": \"success\",\n \"event.action\": \"logged-on\",\n \"host.hostname\": \"DESKTOP-ABC123\",\n \"host.os.platform\": \"windows\",\n \"user.name\": \"alice\",\n \"organization.id\": 1,\n \"organization.name\": \"Acme Inc.\"\n },\n {\n \"@timestamp\": \"2026-05-27T14:01:30.000Z\",\n \"message\": \"A logon was attempted using explicit credentials.\",\n \"log.original\": \"A logon was attempted using explicit credentials.\",\n \"event.provider\": \"Microsoft-Windows-Security-Auditing\",\n \"event.category\": \"authentication\",\n \"event.outcome\": \"unknown\",\n \"event.action\": \"logged-on\",\n \"host.hostname\": \"SERVER-XYZ789\",\n \"host.os.platform\": \"windows\",\n \"user.name\": \"bob\",\n \"organization.id\": 2,\n \"organization.name\": \"Globex Corp\"\n },\n {\n \"@timestamp\": \"2026-05-27T14:02:00.000Z\",\n \"message\": \"An attempt was made to reset an account's password.\",\n \"log.original\": \"An attempt was made to reset an account's password.\",\n \"event.provider\": \"Microsoft-Windows-Security-Auditing\",\n \"event.category\": \"iam\",\n \"event.outcome\": \"success\",\n \"event.action\": \"user-password-reset\",\n \"host.hostname\": \"DC-PRIMARY\",\n \"host.os.platform\": \"windows\",\n \"user.name\": \"charlie\",\n \"organization.id\": 3,\n \"organization.name\": \"Initech LLC\"\n }\n ],\n \"pagination\": {\n \"next_page_token\": null\n }\n}", + "latency": 0, + "statusCode": 200, + "label": "SIEM Query Response", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-huntress-api-call-limit", + "value": "60" + }, + { + "key": "x-huntress-api-call-remaining", + "value": "55" + } + ], + "filePath": "", + "sendFileAsBody": false, + "rules": [], + "rulesOperator": "OR", + "disableTemplating": true, + "fallbackTo404": false, + "default": true + } + ], + "enabled": true, + "randomResponse": false, + "sequentialResponse": false + }, + { + "uuid": "b3bea0dc-39a2-442d-81de-a11f918c6cd2", + "documentation": "Organizations Endpoint", + "method": "get", + "endpoint": "v1/organizations", + "responses": [ + { + "uuid": "ad456172-d25c-4799-b1d7-245c88fa1879", + "body": "{\n \"organizations\": [\n {{#repeat 9}}\n {\n \"id\": {{int 1 9}},\n \"name\": \"{{company}}\",\n \"created_at\": \"{{date '2022-05-01' '2022-05-30' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"updated_at\": \"{{date '2022-06-01' '2022-06-30' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"account_id\": 1,\n \"key\": \"{{faker 'internet.domainWord'}}\",\n \"notify_emails\": [{{email}}, {{email}}],\n \"incident_reports_count\": {{int 0 8}}\n },\n {{/repeat}}\n {\n \"id\": 10,\n \"name\": \"{{company}}\",\n \"created_at\": \"{{date '2022-05-01' '2022-05-30' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"updated_at\": \"{{date '2022-06-01' '2022-06-30' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"account_id\": 1,\n \"key\": \"{{faker 'word.adjective'}}\",\n \"notify_emails\": [{{email}}, {{email}}],\n \"incident_reports_count\": {{int 0 8}}\n }\n ],\n \"pagination\": {\n \"current_page\": 1,\n \"current_page_count\": 1,\n \"limit\": 10,\n \"total_count\": 20,\n \"next_page\": 2,\n \"next_page_url\": \"https://api.huntress.io/v1/organizations?page=2&limit=10\"\n }\n}", + "latency": 0, + "statusCode": 200, + "label": "Organizations Endpoint", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-huntress-api-call-limit", + "value": "60" + }, + { + "key": "x-huntress-api-call-remaining", + "value": "55" + } + ], + "filePath": "", + "sendFileAsBody": false, + "rules": [], + "rulesOperator": "OR", + "disableTemplating": false, + "fallbackTo404": false, + "default": true + } + ], + "enabled": true, + "randomResponse": false, + "sequentialResponse": false + }, + { + "uuid": "a221cee0-b6ab-4b43-afa8-313188214d38", + "documentation": "Returns data for a specific organization.", + "method": "get", + "endpoint": "v1/organizations/:id", + "responses": [ + { + "uuid": "51e9cab3-7780-4ce2-bfb7-245bf8d8daa7", + "body": "{\n \"organization\": {\n \"id\": {{int 1 10}},\n \"name\": \"{{company}}\",\n \"created_at\": \"{{date '2022-05-01' '2022-05-30' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"updated_at\": \"{{date '2022-06-01' '2022-06-30' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"account_id\": 1,\n \"key\": \"{{faker 'internet.domainWord'}}\",\n \"notify_emails\": [{{email}}, {{email}}],\n \"incident_reports_count\": {{int 0 8}}\n }\n}", + "latency": 0, + "statusCode": 200, + "label": "Returns data for a specific organization.", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-huntress-api-call-limit", + "value": "60" + }, + { + "key": "x-huntress-api-call-remaining", + "value": "55" + } + ], + "filePath": "", + "sendFileAsBody": false, + "rules": [], + "rulesOperator": "OR", + "disableTemplating": false, + "fallbackTo404": false, + "default": true + } + ], + "enabled": true, + "randomResponse": false, + "sequentialResponse": false + }, + { + "uuid": "05f22b2a-81f5-46d1-ac8a-7a171472c216", + "documentation": "Agents Endpoint", + "method": "get", + "endpoint": "v1/agents", + "responses": [ + { + "uuid": "37418ddf-bd35-404d-beea-8add484c2098", + "body": "{\n \"agents\": [\n {{#repeat 9}}\n {\n \"id\": {{int 1 9}},\n \"version\": \"{{oneOf (array '0.13.12' '0.13.10' '0.13.8')}}\",\n \"arch\": \"x86_64\",\n \"win_build_number\": {{oneOf (array '19043' '19044' '22000' '17763' '20348')}},\n \"domain_name\": \"{{faker 'internet.domainName'}}\",\n \"created_at\": \"{{date '2022-05-01' '2022-05-30' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"hostname\": \"{{oneOf (array 'Laptop' 'Workstation' 'Desktop' 'Server')}}-{{faker 'word.adjective'}}\",\n \"ipv4_address\": \"{{ipv4}}\",\n \"external_ip\": \"{{ipv4}}\",\n \"mac_addresses\": [\"{{faker 'internet.mac'}}\", \"{{faker 'internet.mac'}}\"],\n \"updated_at\": \"{{date '2022-06-01' '2022-06-30' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"last_survey_at\": \"{{faker 'date.recent'}}\",\n \"last_callback_at\": \"{{faker 'date.recent'}}\",\n \"account_id\": 1,\n \"organization_id\": {{int 1 10}},\n \"os\": \"{{oneOf (array 'Windows 10 Pro' 'Windows 11 Pro' 'Windows Server 2019' 'Windows Server 2022')}}\",\n \"service_pack_major\": 0,\n \"service_pack_minor\": 0,\n \"tags\": {{{someOf (array 'SQL' 'VIP' 'Critical' 'Excluded' 'temp') 1 2 true}}},\n \"os_major\": 10,\n \"os_minor\": 0,\n \"version_number\": null\n },\n {{/repeat}}\n {\n \"id\": 10,\n \"version\": \"{{oneOf (array '0.13.12' '0.13.10' '0.13.8')}}\",\n \"arch\": \"x86_64\",\n \"win_build_number\": {{oneOf (array '19043' '19044' '22000' '17763' '20348')}},\n \"domain_name\": \"{{faker 'internet.domainName'}}\",\n \"created_at\": \"{{date '2022-05-01' '2022-05-30' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"hostname\": \"{{oneOf (array 'Laptop' 'Workstation' 'Desktop' 'Server')}}-{{faker 'word.adjective'}}\",\n \"ipv4_address\": \"{{ipv4}}\",\n \"external_ip\": \"{{ipv4}}\",\n \"mac_addresses\": [\"{{faker 'internet.mac'}}\", \"{{faker 'internet.mac'}}\"],\n \"updated_at\": \"{{date '2022-06-01' '2022-06-30' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"last_survey_at\": \"{{faker 'date.recent'}}\",\n \"last_callback_at\": \"{{faker 'date.recent'}}\",\n \"account_id\": {{int 1 4}},\n \"organization_id\": {{int 1 20}},\n \"os\": \"{{oneOf (array 'Windows 10 Pro' 'Windows 11 Pro' 'Windows Server 2019' 'Windows Server 2022')}}\",\n \"service_pack_major\": 0,\n \"service_pack_minor\": 0,\n \"tags\": {{{someOf (array 'SQL' 'VIP' 'Critical' 'Excluded' 'temp') 1 2 true}}},\n \"os_major\": 10,\n \"os_minor\": 0,\n \"version_number\": null\n }\n ],\n \"pagination\": {\n \"pagination\": {\n \"current_page\": 1,\n \"current_page_count\": 1,\n \"limit\": 10,\n \"total_count\": 20,\n \"next_page\": 2,\n \"next_page_url\": \"https://api.huntress.io/v1/agents?page=2&limit=10\"\n }\n}", + "latency": 0, + "statusCode": 200, + "label": "Agents Endpoint", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-huntress-api-call-limit", + "value": "60" + }, + { + "key": "x-huntress-api-call-remaining", + "value": "55" + } + ], + "filePath": "", + "sendFileAsBody": false, + "rules": [], + "rulesOperator": "OR", + "disableTemplating": false, + "fallbackTo404": false, + "default": true + } + ], + "enabled": true, + "randomResponse": false, + "sequentialResponse": false + }, + { + "uuid": "296e1e0f-19c0-4b99-9538-587670fdad66", + "documentation": "Returns data for a specific agent.", + "method": "get", + "endpoint": "v1/agents/:id", + "responses": [ + { + "uuid": "1e05ac6c-75f1-4bd8-861e-aa44910cf780", + "body": "{\n \"agent\": {\n \"id\": {{int 1 20}},\n \"version\": \"{{oneOf (array '0.13.12' '0.13.10' '0.13.8')}}\",\n \"arch\": \"x86_64\",\n \"win_build_number\": {{oneOf (array '19043' '19044' '22000' '17763' '20348')}},\n \"domain_name\": \"{{faker 'internet.domainName'}}\",\n \"created_at\": \"{{date '2022-05-01' '2022-05-30' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"hostname\": \"{{oneOf (array 'Laptop' 'Workstation' 'Desktop' 'Server')}}-{{faker 'word.adjective'}}\",\n \"ipv4_address\": \"{{ipv4}}\",\n \"external_ip\": \"{{ipv4}}\",\n \"mac_addresses\": [\"{{faker 'internet.mac'}}\", \"{{faker 'internet.mac'}}\"],\n \"updated_at\": \"{{date '2022-06-01' '2022-06-30' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"last_survey_at\": \"{{faker 'date.recent'}}\",\n \"last_callback_at\": \"{{faker 'date.recent'}}\",\n \"account_id\": 1,\n \"organization_id\": {{int 1 9}},\n \"os\": \"{{oneOf (array 'Windows 10 Pro' 'Windows 11 Pro' 'Windows Server 2019' 'Windows Server 2022')}}\",\n \"service_pack_major\": 0,\n \"service_pack_minor\": 0,\n \"tags\": {{{someOf (array 'SQL' 'VIP' 'Critical' 'Excluded' 'temp') 1 2 true}}},\n \"os_major\": 10,\n \"os_minor\": 0,\n \"version_number\": null\n }\n}", + "latency": 0, + "statusCode": 200, + "label": "Returns data for a specific agent.", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-huntress-api-call-limit", + "value": "60" + }, + { + "key": "x-huntress-api-call-remaining", + "value": "55" + } + ], + "filePath": "", + "sendFileAsBody": false, + "rules": [], + "rulesOperator": "OR", + "disableTemplating": false, + "fallbackTo404": false, + "default": true + } + ], + "enabled": true, + "randomResponse": false, + "sequentialResponse": false + }, + { + "uuid": "3e117dbc-fed1-457d-8eae-22b8885b407e", + "documentation": "Incident Reports Endpoint", + "method": "get", + "endpoint": "v1/incident_reports", + "responses": [ + { + "uuid": "aa51588a-abbc-477b-8c16-f453c12d7f12", + "body": "{\n \"incident_reports\": [\n {{#repeat 9}}\n {\n \"id\": {{int 1 100}},\n \"status\": \"{{oneOf (array 'sent' 'closed' 'dismissed')}}\",\n \"summary\": null,\n \"body\": \"\",\n \"updated_at\": \"{{date '2022-06-01' '2022-06-30' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"agent_id\": {{int 1 20}},\n \"status_updated_at\": \"{{date '2022-06-01' '2022-06-30' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"organization_id\": {{int 1 10}},\n \"sent_at\": \"{{date '2022-05-28' '2022-05-30' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"account_id\": 1,\n \"subject\": \"{{oneOf (array 'LOW' 'HIGH' 'CRITICAL')}} - Incident on {{oneOf (array 'Laptop' 'Workstation' 'Desktop' 'Server')}}-{{faker 'word.adjective'}}\",\n \"created_by_id\": null,\n \"remediation_instructions\": null,\n \"footholds\": \"\",\n \"notes\": \"\",\n \"severity\": \"{{oneOf (array 'low' 'high' 'critical')}}\",\n \"assigned_to_id\": null,\n \"closed_at\": null,\n \"indicator_types\": {{{someOf (array 'footholds' 'monitored_files' 'process_detections' 'ransomware_canaries' 'antivirus_detections') 1 4 true}}},\n \"indicator_counts\": {\n \"footholds\": {{int 0 2}},\n \"monitored_files\": {{int 0 2}},\n \"process_detections\": {{int 0 2}},\n \"ransomware_canaries\": {{int 0 2}},\n \"antivirus_detections\": {{int 0 2}}\n },\n \"details\": {}\n },\n {{/repeat}}\n {\n \"id\": 10,\n \"status\": \"{{oneOf (array 'sent' 'closed' 'dismissed')}}\",\n \"summary\": null,\n \"body\": \"\",\n \"updated_at\": \"{{date '2022-06-01' '2022-06-30' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"agent_id\": {{int 1 20}},\n \"status_updated_at\": \"{{date '2022-06-01' '2022-06-30' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"organization_id\": {{int 1 10}},\n \"sent_at\": \"{{date '2022-05-28' '2022-05-30' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"account_id\": 1,\n \"subject\": \"{{oneOf (array 'LOW' 'HIGH' 'CRITICAL')}} - Incident on {{oneOf (array 'Laptop' 'Workstation' 'Desktop' 'Server')}}-{{faker 'word.adjective'}}\",\n \"created_by_id\": null,\n \"remediation_instructions\": null,\n \"footholds\": \"\",\n \"notes\": \"\",\n \"severity\": \"{{oneOf (array 'low' 'high' 'critical')}}\",\n \"assigned_to_id\": null,\n \"closed_at\": null,\n \"indicator_types\": {{{someOf (array 'footholds' 'monitored_files' 'process_detections' 'ransomware_canaries' 'antivirus_detections') 1 4 true}}},\n \"indicator_counts\": {\n \"footholds\": {{int 0 2}},\n \"monitored_files\": {{int 0 2}},\n \"process_detections\": {{int 0 2}},\n \"ransomware_canaries\": {{int 0 2}},\n \"antivirus_detections\": {{int 0 2}}\n },\n \"details\": {}\n }\n ],\n \"pagination\": {\n \"pagination\": {\n \"current_page\": 1,\n \"current_page_count\": 1,\n \"limit\": 10,\n \"total_count\": 20,\n \"next_page\": 2,\n \"next_page_url\": \"https://api.huntress.io/v1/incident_reports?page=2&limit=10\"\n }\n}", + "latency": 0, + "statusCode": 200, + "label": "Incident Reports Endpoint", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-huntress-api-call-limit", + "value": "60" + }, + { + "key": "x-huntress-api-call-remaining", + "value": "55" + } + ], + "filePath": "", + "sendFileAsBody": false, + "rules": [], + "rulesOperator": "OR", + "disableTemplating": false, + "fallbackTo404": false, + "default": true + } + ], + "enabled": true, + "randomResponse": false, + "sequentialResponse": false + }, + { + "uuid": "7898545a-cd88-4311-9caf-e06d286b0550", + "documentation": "Returns data for a specific incident report.", + "method": "get", + "endpoint": "v1/incident_reports/:id", + "responses": [ + { + "uuid": "253432c2-3a0e-4907-b23c-116a59c79295", + "body": "{\n \"incident_report\": {\n \"id\": {{int 1 100}},\n \"status\": \"{{oneOf (array 'sent' 'closed' 'dismissed')}}\",\n \"summary\": null,\n \"body\": \"\",\n \"updated_at\": \"{{date '2022-06-01' '2022-06-30' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"agent_id\": {{int 1 20}},\n \"status_updated_at\": \"{{date '2022-06-01' '2022-06-30' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"organization_id\": {{int 1 10}},\n \"sent_at\": \"{{date '2022-05-28' '2022-05-30' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"account_id\": 1,\n \"subject\": \"{{oneOf (array 'LOW' 'HIGH' 'CRITICAL')}} - Incident on {{oneOf (array 'Laptop' 'Workstation' 'Desktop' 'Server')}}-{{faker 'word.adjective'}}\",\n \"created_by_id\": null,\n \"remediation_instructions\": null,\n \"footholds\": \"\",\n \"notes\": \"\",\n \"severity\": \"{{oneOf (array 'low' 'high' 'critical')}}\",\n \"assigned_to_id\": null,\n \"closed_at\": null,\n \"{{oneOf (array null 'low' 'high' 'critical')}}\",\n \"indicator_types\": {{{someOf (array 'footholds' 'monitored_files' 'process_detections' 'ransomware_canaries' 'antivirus_detections') 1 4 true}}},\n \"indicator_counts\": {\n \"footholds\": {{int 0 2}},\n \"monitored_files\": {{int 0 2}},\n \"process_detections\": {{int 0 2}},\n \"ransomware_canaries\": {{int 0 2}},\n \"antivirus_detections\": {{int 0 2}}\n },\n \"details\": {}\n }\n}", + "latency": 0, + "statusCode": 200, + "label": "Returns data for a specific incident report.", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-huntress-api-call-limit", + "value": "60" + }, + { + "key": "x-huntress-api-call-remaining", + "value": "55" + } + ], + "filePath": "", + "sendFileAsBody": false, + "rules": [], + "rulesOperator": "OR", + "disableTemplating": false, + "fallbackTo404": false, + "default": true + } + ], + "enabled": true, + "randomResponse": false, + "sequentialResponse": false + }, + { + "uuid": "100d3f08-49ea-42b9-87ca-580ae07a2177", + "documentation": "Reports Endpoint", + "method": "get", + "endpoint": "v1/reports", + "responses": [ + { + "uuid": "64b3e476-7302-45ff-8098-360548da5394", + "body": "{\n \"reports\": [\n {{#repeat 9}}\n {\n \"id\": {{int 1 100}},\n \"type\": \"{{oneOf (array 'monthly_summary' 'quarterly_summary' 'yearly_summary')}}\",\n \"period\": \"{{date '2022-05-28' '2022-05-30' \"yyyy-MM-dd\"}}...{{date '2022-06-28' '2022-06-30' \"yyyy-MM-dd\"}}\",\n \"organization_id\": {{int 1 10}},\n \"created_at\": \"{{date '2022-05-27' '2022-05-28' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"updated_at\": \"{{date '2022-07-01' '2022-07-03' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"url\": \"https://huntress.io/rails/active_storage/blobs/redirect/{{guid}}.pdf?disposition=download\"\n },\n {{/repeat}}\n {\n \"id\": 10,\n \"type\": \"{{oneOf (array 'monthly_summary' 'quarterly_summary' 'yearly_summary')}}\",\n \"period\": \"{{date '2022-05-28' '2022-05-30' \"yyyy-MM-dd\"}}...{{date '2022-06-28' '2022-06-30' \"yyyy-MM-dd\"}}\",\n \"organization_id\": {{int 1 10}},\n \"created_at\": \"{{date '2022-05-27' '2022-05-28' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"updated_at\": \"{{date '2022-07-01' '2022-07-03' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"url\": \"https://huntress.io/rails/active_storage/blobs/redirect/{{guid}}.pdf?disposition=download\"\n }\n ],\n \"pagination\": {\n \"pagination\": {\n \"current_page\": 1,\n \"current_page_count\": 1,\n \"limit\": 10,\n \"total_count\": 20,\n \"next_page\": 2,\n \"next_page_url\": \"https://api.huntress.io/v1/reports?page=2&limit=10\"\n }\n}", + "latency": 0, + "statusCode": 200, + "label": "Reports Endpoint", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-huntress-api-call-limit", + "value": "60" + }, + { + "key": "x-huntress-api-call-remaining", + "value": "55" + } + ], + "filePath": "", + "sendFileAsBody": false, + "rules": [], + "rulesOperator": "OR", + "disableTemplating": false, + "fallbackTo404": false, + "default": true + } + ], + "enabled": true, + "randomResponse": false, + "sequentialResponse": false + }, + { + "uuid": "5e000c10-c3c3-4d34-ace9-4bfea1a3fa5c", + "documentation": "Returns data for a specific report.", + "method": "get", + "endpoint": "v1/reports/:id", + "responses": [ + { + "uuid": "5f7f7dca-ae67-4c06-812e-4db991c26d23", + "body": "{\n \"report\": {\n \"id\": {{int 1 100}},\n \"type\": \"{{oneOf (array 'monthly_summary' 'quarterly_summary' 'yearly_summary')}}\",\n \"period\": \"{{date '2022-05-28' '2022-05-30' \"yyyy-MM-dd\"}}...{{date '2022-06-28' '2022-06-30' \"yyyy-MM-dd\"}}\",\n \"organization_id\": {{int 1 10}},\n \"created_at\": \"{{date '2022-05-27' '2022-05-28' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"updated_at\": \"{{date '2022-07-01' '2022-07-03' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"url\": \"https://huntress.io/rails/active_storage/blobs/redirect/{{guid}}.pdf?disposition=download\"\n }\n}", + "latency": 0, + "statusCode": 200, + "label": "Returns data for a specific report.", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-huntress-api-call-limit", + "value": "60" + }, + { + "key": "x-huntress-api-call-remaining", + "value": "55" + } + ], + "filePath": "", + "sendFileAsBody": false, + "rules": [], + "rulesOperator": "OR", + "disableTemplating": false, + "fallbackTo404": false, + "default": true + } + ], + "enabled": true, + "randomResponse": false, + "sequentialResponse": false + }, + { + "uuid": "05614def-07e9-49b3-aaa9-5f7eff45274e", + "documentation": "Billing Reports Endpoint", + "method": "get", + "endpoint": "v1/billing_reports", + "responses": [ + { + "uuid": "4cf2997a-64eb-4ac2-a2a0-a38149a1dc5a", + "body": "{\n \"billing_reports\": [\n {{#repeat 9}}\n {\n \"id\": {{int 1 9}},\n \"plan\": \"{{oneOf (array 'Huntress Partner Pay As You Go' 'Huntress Partner 100 Agents' 'Huntress Partner 200 Agents' 'Huntress Partner 300 Agents')}}\",\n \"quantity\": {{int 50 400}},\n \"amount\": {{int 150 900}},\n \"currency_type\": \"{{oneOf (array 'USD' 'AUD' 'CAD')}}\",\n \"receipt\": \"https://pay.stripe/com/invoice/invst_{{guid}}\",\n \"status\": \"{{oneOf (array 'open' 'paid' 'failed' 'partial_refund' 'full_refund')}}\",\n \"created_at\": \"{{date '2022-05-27' '2022-05-28' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"updated_at\": \"{{date '2022-05-29' '2022-05-30' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\"\n },\n {{/repeat}}\n {\n \"id\": 10,\n \"plan\": \"{{oneOf (array 'Huntress Partner Pay As You Go' 'Huntress Partner 100 Agents' 'Huntress Partner 200 Agents' 'Huntress Partner 300 Agents')}}\",\n \"quantity\": {{int 50 400}},\n \"amount\": {{int 150 900}},\n \"currency_type\": \"{{oneOf (array 'USD' 'AUD' 'CAD')}}\",\n \"receipt\": \"https://pay.stripe/com/invoice/invst_{{guid}}\",\n \"status\": \"{{oneOf (array 'open' 'paid' 'failed' 'partial_refund' 'full_refund')}}\",\n \"created_at\": \"{{date '2022-05-27' '2022-05-28' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"updated_at\": \"{{date '2022-05-29' '2022-05-30' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\"\n }\n ],\n \"pagination\": {\n \"pagination\": {\n \"current_page\": 1,\n \"current_page_count\": 1,\n \"limit\": 10,\n \"total_count\": 20,\n \"next_page\": 2,\n \"next_page_url\": \"https://api.huntress.io/v1/billing_reports?page=2&limit=10\"\n }\n}", + "latency": 0, + "statusCode": 200, + "label": "Billing Reports Endpoint", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-huntress-api-call-limit", + "value": "60" + }, + { + "key": "x-huntress-api-call-remaining", + "value": "55" + } + ], + "filePath": "", + "sendFileAsBody": false, + "rules": [], + "rulesOperator": "OR", + "disableTemplating": false, + "fallbackTo404": false, + "default": true + } + ], + "enabled": true, + "randomResponse": false, + "sequentialResponse": false + }, + { + "uuid": "72142a57-71d5-4a8a-bb17-179c00da4248", + "documentation": "Returns data for a specific billing report.", + "method": "get", + "endpoint": "v1/billing_reports/:id", + "responses": [ + { + "uuid": "2d1c37f2-f441-44a4-a560-f51e1b25baff", + "body": "{\n \"billing_report\": {\n \"id\": {{int 1 9}},\n \"plan\": \"{{oneOf (array 'Huntress Partner Pay As You Go' 'Huntress Partner 100 Agents' 'Huntress Partner 200 Agents' 'Huntress Partner 300 Agents')}}\",\n \"quantity\": {{int 50 400}},\n \"amount\": {{int 150 900}},\n \"currency_type\": \"{{oneOf (array 'USD' 'AUD' 'CAD')}}\",\n \"receipt\": \"https://pay.stripe/com/invoice/invst_{{guid}}\",\n \"status\": \"{{oneOf (array 'open' 'paid' 'failed' 'partial_refund' 'full_refund')}}\",\n \"created_at\": \"{{date '2022-05-27' '2022-05-28' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\",\n \"updated_at\": \"{{date '2022-05-29' '2022-05-30' \"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\"}}\"\n }\n}\n", + "latency": 0, + "statusCode": 200, + "label": "Returns data for a specific billing report.", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" + }, + { + "key": "x-huntress-api-call-limit", + "value": "60" + }, + { + "key": "x-huntress-api-call-remaining", + "value": "55" + } + ], + "filePath": "", + "sendFileAsBody": false, + "rules": [], + "rulesOperator": "OR", + "disableTemplating": false, + "fallbackTo404": false, + "default": true + } + ], + "enabled": true, + "randomResponse": false, + "sequentialResponse": false + } + ], + "proxyMode": false, + "proxyHost": "", + "proxyRemovePrefix": false, + "tlsOptions": { + "enabled": false, + "type": "CERT", + "pfxPath": "", + "certPath": "", + "keyPath": "", + "caPath": "", + "passphrase": "" + }, + "cors": true, + "headers": [ + { + "key": "Authorization", + "value": "Basic {{base64 'hk_1kjlkn112lknlk2222lk:hs_4343fffdjvnekjve44444fdvdfvdf222frrf4'}}" + } + ], + "proxyReqHeaders": [ + { + "key": "", + "value": "" + } + ], + "proxyResHeaders": [ + { + "key": "", + "value": "" + } + ] +} \ No newline at end of file diff --git a/huntress/tests/docker/Dockerfile b/huntress/tests/docker/Dockerfile new file mode 100644 index 0000000000..9cf0c1a52c --- /dev/null +++ b/huntress/tests/docker/Dockerfile @@ -0,0 +1,10 @@ +FROM mockoon/cli:latest + +# Embed the Huntress mock data for standalone use (docker build). +# The docker-compose.yml mounts the file directly for development, +# so rebuilds are not required when huntress_mockoon.json changes. +COPY huntress_mockoon.json /data/huntress_mockoon.json + +EXPOSE 3002 + +ENTRYPOINT ["mockoon-cli", "start", "--data", "/data/huntress_mockoon.json", "--port", "3002", "--hostname", "0.0.0.0"] diff --git a/huntress/tests/docker/docker-compose.yml b/huntress/tests/docker/docker-compose.yml new file mode 100644 index 0000000000..d790e30128 --- /dev/null +++ b/huntress/tests/docker/docker-compose.yml @@ -0,0 +1,10 @@ +services: + mockoon: + build: + context: ../.. + dockerfile: tests/docker/Dockerfile + container_name: dd-test-huntress-mockoon + volumes: + - ../data/huntress_mockoon.json:/data/huntress_mockoon.json:ro + ports: + - "3002:3002" diff --git a/huntress/tests/fixtures/siem_query_empty.json b/huntress/tests/fixtures/siem_query_empty.json new file mode 100644 index 0000000000..2977178ef8 --- /dev/null +++ b/huntress/tests/fixtures/siem_query_empty.json @@ -0,0 +1,4 @@ +{ + "logs": [], + "pagination": {} +} diff --git a/huntress/tests/fixtures/siem_query_page1.json b/huntress/tests/fixtures/siem_query_page1.json new file mode 100644 index 0000000000..6cacd3a42d --- /dev/null +++ b/huntress/tests/fixtures/siem_query_page1.json @@ -0,0 +1,29 @@ +{ + "logs": [ + { + "@timestamp": "2026-05-27T14:00:00.000Z", + "message": "An account was successfully logged on.", + "log.original": "An account was successfully logged on.", + "event.provider": "Microsoft-Windows-Security-Auditing", + "event.category": "authentication", + "event.outcome": "success", + "host.hostname": "DESKTOP-ABC123", + "organization.id": 101, + "organization.name": "Acme Inc." + }, + { + "@timestamp": "2026-05-27T14:01:00.000Z", + "message": "A logon was attempted using explicit credentials.", + "log.original": "A logon was attempted using explicit credentials.", + "event.provider": "Microsoft-Windows-Security-Auditing", + "event.category": "authentication", + "event.outcome": "unknown", + "host.hostname": "SERVER-XYZ", + "organization.id": 102, + "organization.name": "Globex Corp" + } + ], + "pagination": { + "next_page_token": "page2_token_abc123" + } +} diff --git a/huntress/tests/fixtures/siem_query_page2.json b/huntress/tests/fixtures/siem_query_page2.json new file mode 100644 index 0000000000..31ab07a952 --- /dev/null +++ b/huntress/tests/fixtures/siem_query_page2.json @@ -0,0 +1,16 @@ +{ + "logs": [ + { + "@timestamp": "2026-05-27T14:02:00.000Z", + "message": "Special privileges assigned to new logon.", + "log.original": "Special privileges assigned to new logon.", + "event.provider": "Microsoft-Windows-Security-Auditing", + "event.category": "iam", + "event.outcome": "success", + "host.hostname": "DESKTOP-ABC123", + "organization.id": 101, + "organization.name": "Acme Inc." + } + ], + "pagination": {} +} diff --git a/huntress/tests/test_docker.py b/huntress/tests/test_docker.py new file mode 100644 index 0000000000..ff4dbf07d3 --- /dev/null +++ b/huntress/tests/test_docker.py @@ -0,0 +1,59 @@ +""" +Integration tests for the Huntress SIEM check against a live Mockoon mock server. + +Run with Docker available: + ddev test huntress -m integration +""" +import pytest + +from datadog_checks.base import AgentCheck +from datadog_checks.huntress import HuntressCheck + +pytestmark = pytest.mark.integration + + +def test_integration_check_runs(dd_environment, aggregator): + """Full check run against the Mockoon server — verifies happy path end-to-end.""" + instance = dd_environment + check = HuntressCheck("huntress", {}, [instance]) + check.check(instance) + + aggregator.assert_metric("huntress.siem.logs_collected", at_least=0) + aggregator.assert_metric("huntress.siem.pages_fetched", at_least=1) + aggregator.assert_metric("huntress.siem.run_duration_seconds", at_least=0) + aggregator.assert_metric("huntress.siem.api_call_limit", value=60) + aggregator.assert_metric("huntress.siem.api_call_remaining", value=55) + aggregator.assert_service_check("huntress.siem.check_status", status=AgentCheck.OK) + aggregator.assert_all_metrics_covered() + + +def test_integration_org_enrichment(dd_environment, aggregator): + """Verify org metadata is fetched from the mock and applied to logs.""" + instance = dict(dd_environment, enrich_with_org_tags=True, org_cache_ttl_seconds=0) + check = HuntressCheck("huntress", {}, [instance]) + + sent_logs = [] + original_send = check._send_logs_batch + + def capture(batch): + sent_logs.extend(batch) + original_send(batch) + + check._send_logs_batch = capture + check.check(instance) + + assert sent_logs, "Expected at least one log to be forwarded" + for log in sent_logs: + assert "huntress_account_id" in log.get("ddtags", ""), ( + f"Missing huntress_account_id in ddtags: {log.get('ddtags')}" + ) + + +def test_integration_rate_limit_metrics(dd_environment, aggregator): + """Verify rate limit headers are parsed and emitted as metrics.""" + instance = dict(dd_environment, enrich_with_org_tags=False) + check = HuntressCheck("huntress", {}, [instance]) + check.check(instance) + + aggregator.assert_metric("huntress.siem.api_call_limit", value=60) + aggregator.assert_metric("huntress.siem.api_call_remaining", value=55) diff --git a/huntress/tests/test_huntress.py b/huntress/tests/test_huntress.py new file mode 100644 index 0000000000..401216677c --- /dev/null +++ b/huntress/tests/test_huntress.py @@ -0,0 +1,660 @@ +""" +Unit tests for the Huntress SIEM → Datadog Logs integration. +Covers all 17 scenarios from PRD §12. +""" +import json +import os +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch + +import pytest + +from datadog_checks.huntress import HuntressCheck + + +FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures") + + +def _load_fixture(name): + with open(os.path.join(FIXTURES, name)) as f: + return json.load(f) + + +def _make_instance(**kwargs): + base = { + "huntress_api_key": "pub_key", + "huntress_secret_key": "secret_key", + "esql_query": "FROM logs", + "enrich_with_org_tags": False, + "tags": ["source:huntress", "env:test"], + } + base.update(kwargs) + return base + + +def _make_check(**kwargs): + inst = kwargs.pop("instance", None) or _make_instance(**kwargs) + check = HuntressCheck("huntress", {}, [inst]) + check.log = MagicMock() + # Use an in-memory dict to prevent persistent cache from bleeding between tests + _cache = {} + check.read_persistent_cache = lambda key: _cache.get(key) + check.write_persistent_cache = lambda key, value: _cache.__setitem__(key, value) + return check, inst + + +def _mock_response(status_code, body, headers=None): + resp = MagicMock() + resp.status_code = status_code + resp.json.return_value = body + resp.text = json.dumps(body) + # Use a real dict so header parsing in _parse_rate_limit_headers works correctly. + resp.headers = headers if headers is not None else { + "x-huntress-api-call-limit": "60", + "x-huntress-api-call-remaining": "55", + } + return resp + + +# =========================================================================== +# 1. Happy path — single page +# =========================================================================== + +def test_happy_path_single_page(): + check, instance = _make_check() + fixture = _load_fixture("siem_query_empty.json") + fixture["logs"] = _load_fixture("siem_query_page1.json")["logs"][:1] + + with patch.object(check, "_request_with_retry", return_value=_mock_response(200, fixture)) as mock_req, \ + patch.object(check, "_send_logs_batch") as mock_send, \ + patch.object(check, "_save_checkpoint") as mock_save, \ + patch("time.time", side_effect=[1000.0, 1001.0]): + + check.check(instance) + + mock_send.assert_called_once() + mock_save.assert_called_once() + batch = mock_send.call_args[0][0] + assert len(batch) == 1 + assert batch[0]["ddsource"] == "huntress" + assert batch[0]["message"] == "An account was successfully logged on." + + +# =========================================================================== +# 2. Happy path — multi-page +# =========================================================================== + +def test_happy_path_multi_page(): + check, instance = _make_check() + page1 = _load_fixture("siem_query_page1.json") + page2 = _load_fixture("siem_query_page2.json") + + responses = [ + _mock_response(200, page1), + _mock_response(200, page2), + ] + + with patch.object(check, "_request_with_retry", side_effect=responses), \ + patch.object(check, "_send_logs_batch") as mock_send, \ + patch.object(check, "_save_checkpoint") as mock_save, \ + patch("time.time", side_effect=[1000.0, 1002.0]): + + check.check(instance) + + assert mock_send.call_count == 2 + all_logs = mock_send.call_args_list[0][0][0] + mock_send.call_args_list[1][0][0] + assert len(all_logs) == 3 + mock_save.assert_called_once() + + +# =========================================================================== +# 3. Auth failure (401) +# =========================================================================== + +def test_auth_failure_401(): + check, instance = _make_check() + + with patch.object(check, "_request_with_retry", + side_effect=Exception("Huntress API 401 Unauthorized")), \ + patch.object(check, "_save_checkpoint") as mock_save, \ + patch.object(check, "count") as mock_count, \ + patch("time.time", side_effect=[1000.0, 1001.0]): + with pytest.raises(Exception, match="401"): + check.check(instance) + + mock_save.assert_not_called() + + +def test_auth_failure_401_raises_via_retry(): + """Verify _request_with_retry itself raises on 401.""" + check, instance = _make_check() + resp = _mock_response(401, {"error": "unauthorized"}) + + with patch("requests.request", return_value=resp), \ + patch.object(check, "count") as mock_count: + with pytest.raises(Exception, match="401"): + check._request_with_retry("POST", "https://api.huntress.io/v1/siem/query", + {}, json_body={}) + + mock_count.assert_called_with("huntress.siem.errors", 1, tags=["error_type:auth_failure"]) + + +# =========================================================================== +# 4. Query timeout (408) — retried twice then aborts +# =========================================================================== + +def test_query_timeout_408(): + check, instance = _make_check() + resp_408 = _mock_response(408, {"error": "timeout"}) + + with patch("requests.request", return_value=resp_408), \ + patch("time.sleep") as mock_sleep, \ + patch.object(check, "count") as mock_count: + with pytest.raises(Exception, match="408"): + check._request_with_retry("POST", "https://x/q", {}, json_body={}) + + # Two retries → sleeps of 2s and 4s + assert mock_sleep.call_count == 2 + sleep_args = [c[0][0] for c in mock_sleep.call_args_list] + assert sleep_args == [2, 4] + mock_count.assert_called_with("huntress.siem.errors", 1, tags=["error_type:timeout"]) + + +# =========================================================================== +# 5. Rate limit (429) — sleeps 60s and retries +# =========================================================================== + +def test_rate_limit_429_then_success(): + check, instance = _make_check() + resp_429 = _mock_response(429, {"error": "rate_limited"}) + resp_200 = _mock_response(200, _load_fixture("siem_query_empty.json")) + + with patch("requests.request", side_effect=[resp_429, resp_200]), \ + patch("time.sleep") as mock_sleep: + result = check._request_with_retry("POST", "https://x/q", {}, json_body={}) + + mock_sleep.assert_called_once_with(60) + assert result.status_code == 200 + + +# =========================================================================== +# 6. Invalid ES|QL (422) +# =========================================================================== + +def test_invalid_esql_422(): + check, instance = _make_check() + resp_422 = _mock_response(422, {"error": "invalid query"}) + + with patch("requests.request", return_value=resp_422), \ + patch.object(check, "count") as mock_count: + with pytest.raises(Exception, match="422"): + check._request_with_retry("POST", "https://x/q", {}, json_body={}) + + mock_count.assert_called_with("huntress.siem.errors", 1, tags=["error_type:invalid_query"]) + + +# =========================================================================== +# 7. Checkpoint persistence — second run uses previous range_end as range_start +# =========================================================================== + +def test_checkpoint_persistence(): + check, instance = _make_check() + saved_ts = "2026-05-27T13:00:00.000Z" + instance_hash = check._instance_hash(instance) + check.write_persistent_cache( + check.CHECKPOINT_CACHE_KEY_PREFIX + instance_hash, + json.dumps({"last_collected_at": saved_ts, "schema_version": 1}), + ) + + captured_bodies = [] + + def capture_request(method, url, headers, json_body=None, params=None): + if json_body: + captured_bodies.append(json_body) + return _mock_response(200, _load_fixture("siem_query_empty.json")) + + with patch.object(check, "_request_with_retry", side_effect=capture_request), \ + patch("time.time", side_effect=[1000.0, 1001.0]): + check.check(instance) + + assert captured_bodies, "No SIEM query was made" + # range_start should be 1ms after saved_ts + assert captured_bodies[0]["range_start"] == "2026-05-27T13:00:00.001Z" + + +# =========================================================================== +# 8. No checkpoint (first run) — range_start defaults to now - interval +# =========================================================================== + +def test_no_checkpoint_first_run(): + check, instance = _make_check(min_collection_interval=900) + captured_bodies = [] + + def capture_request(method, url, headers, json_body=None, params=None): + if json_body: + captured_bodies.append(json_body) + return _mock_response(200, _load_fixture("siem_query_empty.json")) + + fixed_now = datetime(2026, 5, 27, 14, 0, 0, tzinfo=timezone.utc) + with patch.object(check, "_request_with_retry", side_effect=capture_request), \ + patch("datadog_checks.huntress.huntress.datetime") as mock_dt, \ + patch("time.time", side_effect=[1000.0, 1001.0]): + mock_dt.now.return_value = fixed_now + mock_dt.fromisoformat = datetime.fromisoformat + check.check(instance) + + assert captured_bodies + body = captured_bodies[0] + range_start = datetime.fromisoformat(body["range_start"].replace("Z", "+00:00")) + range_end = datetime.fromisoformat(body["range_end"].replace("Z", "+00:00")) + diff_seconds = (range_end - range_start).total_seconds() + assert abs(diff_seconds - 900) < 2 + + +# =========================================================================== +# 9. ES|QL validation — query not starting with FROM logs raises ConfigurationError +# =========================================================================== + +def test_esql_validation_rejects_bad_query(): + check, instance = _make_check(esql_query="SELECT * FROM logs") + with pytest.raises(Exception, match="FROM logs"): + check.check(instance) + + +def test_esql_validation_accepts_from_logs(): + check, instance = _make_check(esql_query="FROM logs | KEEP @timestamp, message") + with patch.object(check, "_request_with_retry", + return_value=_mock_response(200, _load_fixture("siem_query_empty.json"))), \ + patch("time.time", side_effect=[1000.0, 1001.0]): + check.check(instance) # should not raise + + +def test_esql_validation_case_insensitive(): + check, instance = _make_check(esql_query="from logs | limit 100") + with patch.object(check, "_request_with_retry", + return_value=_mock_response(200, _load_fixture("siem_query_empty.json"))), \ + patch("time.time", side_effect=[1000.0, 1001.0]): + check.check(instance) # should not raise + + +# =========================================================================== +# 10. Log transformation — ECS fields correctly mapped to Datadog payload +# =========================================================================== + +def test_log_transformation(): + check, instance = _make_check() + raw = { + "@timestamp": "2026-05-27T14:00:00.000Z", + "log.original": "An account was successfully logged on.", + "message": "fallback message", + "event.provider": "Microsoft-Windows-Security-Auditing", + "host.hostname": "DESKTOP-ABC123", + "event.category": "authentication", + } + tags = ["source:huntress", "env:test", "huntress_account_id:42"] + payload = check._transform_log(raw, tags, "huntress-siem") + + assert payload["message"] == "An account was successfully logged on." + assert payload["ddsource"] == "huntress" + assert payload["service"] == "huntress-siem" + assert payload["ddtags"] == "source:huntress,env:test,huntress_account_id:42" + assert payload["date"] == 1779890400000 + assert payload["event.provider"] == "Microsoft-Windows-Security-Auditing" + assert payload["host.hostname"] == "DESKTOP-ABC123" + assert "@timestamp" not in payload + assert "log.original" not in payload + + +def test_log_transformation_fallback_message(): + check, instance = _make_check() + raw = {"message": "fallback used", "@timestamp": "2026-05-27T14:00:00.000Z"} + payload = check._transform_log(raw, [], "huntress-siem") + assert payload["message"] == "fallback used" + + +def test_log_transformation_json_fallback(): + check, instance = _make_check() + raw = {"event.category": "network", "@timestamp": "2026-05-27T14:00:00.000Z"} + payload = check._transform_log(raw, [], "huntress-siem") + assert "event.category" in payload["message"] + + +# =========================================================================== +# 11. Batching — >1,000 logs split into multiple batches +# =========================================================================== + +def test_batching_over_1000_logs(): + check, instance = _make_check() + + large_log_list = [ + { + "@timestamp": "2026-05-27T14:00:00.000Z", + "message": f"log {i}", + } + for i in range(1500) + ] + fixture = {"logs": large_log_list, "pagination": {}} + + with patch.object(check, "_request_with_retry", return_value=_mock_response(200, fixture)), \ + patch.object(check, "_send_logs_batch") as mock_send, \ + patch("time.time", side_effect=[1000.0, 1001.0]): + check.check(instance) + + assert mock_send.call_count == 2 + first_batch = mock_send.call_args_list[0][0][0] + second_batch = mock_send.call_args_list[1][0][0] + assert len(first_batch) == 1000 + assert len(second_batch) == 500 + + +# =========================================================================== +# 12. max_pages_per_run cap — stops after N pages; checkpoint does NOT advance +# =========================================================================== + +def test_max_pages_per_run_cap(): + check, instance = _make_check(max_pages_per_run=1) + page1 = _load_fixture("siem_query_page1.json") + + with patch.object(check, "_request_with_retry", return_value=_mock_response(200, page1)), \ + patch.object(check, "_save_checkpoint") as mock_save, \ + patch("time.time", side_effect=[1000.0, 1001.0]): + check.check(instance) + + mock_save.assert_not_called() + + +# =========================================================================== +# 13. Org enrichment — cache hit (no extra API calls) +# =========================================================================== + +def test_org_enrichment_cache_hit(): + check, instance = _make_check(enrich_with_org_tags=True) + instance_hash = check._instance_hash(instance) + + # Pre-populate cache (fresh) + from datetime import datetime, timezone + cache = { + "fetched_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z", + "account_id": 42, + "orgs": { + "101": {"name": "Acme Inc.", "key": "acme", "account_id": 42}, + }, + } + check.write_persistent_cache(check.ORG_CACHE_KEY_PREFIX + instance_hash, json.dumps(cache)) + + page1 = _load_fixture("siem_query_page1.json") + page1["pagination"] = {} + + api_calls = [] + + def side_effect(method, url, headers, json_body=None, params=None): + api_calls.append(url) + return _mock_response(200, page1) + + with patch.object(check, "_request_with_retry", side_effect=side_effect), \ + patch.object(check, "_send_logs_batch") as mock_send, \ + patch("time.time", side_effect=[1000.0, 1001.0]): + check.check(instance) + + # Only the SIEM query should have been called — no org API calls + assert all("/siem/query" in u for u in api_calls) + + # Verify org tags applied + batch = mock_send.call_args_list[0][0][0] + first_log_tags = batch[0]["ddtags"] + assert "huntress_org_name:Acme Inc." in first_log_tags + assert "huntress_org_key:acme" in first_log_tags + assert "huntress_account_id:42" in first_log_tags + + +# =========================================================================== +# 14. Org enrichment — cache miss (fetches account + orgs) +# =========================================================================== + +def test_org_enrichment_cache_miss(): + check, instance = _make_check(enrich_with_org_tags=True) + + account_resp = _mock_response(200, {"account": {"id": 42}}) + orgs_resp = _mock_response(200, { + "organizations": [ + {"id": 101, "name": "Acme Inc.", "key": "acme"}, + ], + "pagination": {}, + }) + siem_resp = _mock_response(200, { + "logs": _load_fixture("siem_query_page1.json")["logs"][:1], + "pagination": {}, + }) + + call_urls = [] + + def side_effect(method, url, headers, json_body=None, params=None): + call_urls.append(url) + if "/account" in url and "organization" not in url: + return account_resp + elif "organizations" in url: + return orgs_resp + else: + return siem_resp + + with patch.object(check, "_request_with_retry", side_effect=side_effect), \ + patch.object(check, "_send_logs_batch") as mock_send, \ + patch("time.time", side_effect=[1000.0, 1001.0]): + check.check(instance) + + assert any("/account" in u for u in call_urls) + assert any("organizations" in u for u in call_urls) + + batch = mock_send.call_args_list[0][0][0] + assert "huntress_account_id:42" in batch[0]["ddtags"] + + +# =========================================================================== +# 15. Org enrichment — fetch failure → warning, logs collected without org tags +# =========================================================================== + +def test_org_enrichment_fetch_failure(): + check, instance = _make_check(enrich_with_org_tags=True) + siem_resp = _mock_response(200, { + "logs": _load_fixture("siem_query_page1.json")["logs"][:1], + "pagination": {}, + }) + + def side_effect(method, url, headers, json_body=None, params=None): + if "/account" in url and "organization" not in url: + raise Exception("Network error fetching account") + return siem_resp + + with patch.object(check, "_request_with_retry", side_effect=side_effect), \ + patch.object(check, "_send_logs_batch") as mock_send, \ + patch("time.time", side_effect=[1000.0, 1001.0]): + check.check(instance) # must not raise + + check.log.warning.assert_called() + # Logs should still be sent + mock_send.assert_called() + batch = mock_send.call_args_list[0][0][0] + # No org tags on logs + assert "huntress_org_name" not in batch[0]["ddtags"] + + +# =========================================================================== +# 16. Org enrichment — no org match → only huntress_account_id tag +# =========================================================================== + +def test_org_enrichment_no_org_match(): + check, instance = _make_check() + org_cache = { + "fetched_at": "2026-05-27T14:00:00.000Z", + "account_id": 42, + "orgs": {"101": {"name": "Acme Inc.", "key": "acme", "account_id": 42}}, + } + raw_log = {"@timestamp": "2026-05-27T14:00:00.000Z", "message": "no org field"} + tags = check._get_org_tags(raw_log, org_cache) + + assert "huntress_account_id:42" in tags + assert not any("org_name" in t or "org_key" in t for t in tags) + + +# =========================================================================== +# 17. Multi-instance isolation — independent checkpoints and org caches +# =========================================================================== + +def test_multi_instance_isolation(): + instance_a = _make_instance( + huntress_api_key="key_a", + huntress_secret_key="secret_a", + esql_query="FROM logs", + ) + instance_b = _make_instance( + huntress_api_key="key_b", + huntress_secret_key="secret_b", + esql_query="FROM logs", + ) + + check_a = HuntressCheck("huntress", {}, [instance_a]) + check_a.log = MagicMock() + _cache_a = {} + check_a.read_persistent_cache = lambda key: _cache_a.get(key) + check_a.write_persistent_cache = lambda key, value: _cache_a.__setitem__(key, value) + + check_b = HuntressCheck("huntress", {}, [instance_b]) + check_b.log = MagicMock() + _cache_b = {} + check_b.read_persistent_cache = lambda key: _cache_b.get(key) + check_b.write_persistent_cache = lambda key, value: _cache_b.__setitem__(key, value) + + hash_a = check_a._instance_hash(instance_a) + hash_b = check_b._instance_hash(instance_b) + + assert hash_a != hash_b + + # Checkpoints stored under different keys must return different values + check_a._save_checkpoint(hash_a, "2026-05-27T14:00:00.000Z") + check_b._save_checkpoint(hash_b, "2026-05-27T15:00:00.000Z") + + assert check_a._load_checkpoint(hash_a) == "2026-05-27T14:00:00.000Z" + assert check_b._load_checkpoint(hash_b) == "2026-05-27T15:00:00.000Z" + # Each check's cache is isolated — A cannot see B's checkpoint + assert check_a._load_checkpoint(hash_b) is None + + +# =========================================================================== +# Bonus: _get_org_tags strategies +# =========================================================================== + +def test_org_tags_by_id(): + check, _ = _make_check() + org_cache = { + "account_id": 42, + "orgs": {"101": {"name": "Acme Inc.", "key": "acme", "account_id": 42}}, + } + raw = {"organization.id": 101} + tags = check._get_org_tags(raw, org_cache) + assert "huntress_org_id:101" in tags + assert "huntress_org_name:Acme Inc." in tags + assert "huntress_org_key:acme" in tags + assert "huntress_account_id:42" in tags + + +def test_org_tags_by_name_reverse_lookup(): + check, _ = _make_check() + org_cache = { + "account_id": 42, + "orgs": {"101": {"name": "Acme Inc.", "key": "acme", "account_id": 42}}, + } + raw = {"organization.name": "Acme Inc."} + tags = check._get_org_tags(raw, org_cache) + assert "huntress_org_name:Acme Inc." in tags + assert "huntress_org_key:acme" in tags + + +def test_org_tags_nested_org_field(): + """organization field as a nested dict (e.g. from ECS nested structure).""" + check, _ = _make_check() + org_cache = { + "account_id": 42, + "orgs": {"101": {"name": "Acme Inc.", "key": "acme", "account_id": 42}}, + } + raw = {"organization": {"id": 101, "name": "Acme Inc."}} + tags = check._get_org_tags(raw, org_cache) + assert "huntress_org_name:Acme Inc." in tags + + +# =========================================================================== +# Rate limit header parsing +# =========================================================================== + +def test_rate_limit_headers_parsed_and_stored(): + """_parse_rate_limit_headers populates _last_api_call_* attributes.""" + check, _ = _make_check() + check._last_api_call_limit = None + check._last_api_call_remaining = None + + resp = _mock_response(200, {}, headers={ + "x-huntress-api-call-limit": "60", + "x-huntress-api-call-remaining": "42", + }) + check._parse_rate_limit_headers(resp) + + assert check._last_api_call_limit == 60 + assert check._last_api_call_remaining == 42 + + +def test_rate_limit_warning_when_low(): + """Warning is logged when remaining drops below 10.""" + check, _ = _make_check() + check._last_api_call_limit = None + check._last_api_call_remaining = None + + resp = _mock_response(200, {}, headers={ + "x-huntress-api-call-limit": "60", + "x-huntress-api-call-remaining": "3", + }) + check._parse_rate_limit_headers(resp) + + check.log.warning.assert_called() + assert check._last_api_call_remaining == 3 + + +def test_rate_limit_headers_missing_gracefully(): + """Missing rate limit headers do not raise; attributes stay None.""" + check, _ = _make_check() + check._last_api_call_limit = None + check._last_api_call_remaining = None + + resp = _mock_response(200, {}, headers={"Content-Type": "application/json"}) + check._parse_rate_limit_headers(resp) + + assert check._last_api_call_limit is None + assert check._last_api_call_remaining is None + + +def test_rate_limit_metrics_emitted_in_check(): + """huntress.siem.api_call_* gauges are emitted when headers were present.""" + check, instance = _make_check() + fixture = _load_fixture("siem_query_empty.json") + fixture["logs"] = [] + + emitted_gauges = {} + + def capture_gauge(name, value, tags=None): + emitted_gauges[name] = value + + real_request_with_retry = check._request_with_retry + + def fake_retry(method, url, headers, json_body=None, params=None): + resp = _mock_response(200, fixture, headers={ + "x-huntress-api-call-limit": "60", + "x-huntress-api-call-remaining": "48", + }) + check._parse_rate_limit_headers(resp) + return resp + + with patch.object(check, "_request_with_retry", side_effect=fake_retry), \ + patch.object(check, "gauge", side_effect=capture_gauge), \ + patch("time.time", side_effect=[1000.0, 1001.0]): + check.check(instance) + + assert emitted_gauges.get("huntress.siem.api_call_limit") == 60 + assert emitted_gauges.get("huntress.siem.api_call_remaining") == 48 diff --git a/huntress/tests/test_unit.py b/huntress/tests/test_unit.py new file mode 100644 index 0000000000..a14d0fdd48 --- /dev/null +++ b/huntress/tests/test_unit.py @@ -0,0 +1,37 @@ +from typing import Any, Callable, Dict # noqa: F401 + +import pytest +from datadog_checks.base import AgentCheck # noqa: F401 +from datadog_checks.base.stubs.aggregator import AggregatorStub # noqa: F401 +from datadog_checks.huntress import HuntressCheck +from unittest.mock import MagicMock, patch + + +@pytest.fixture +def huntress_instance(): + return { + "huntress_api_key": "test_key", + "huntress_secret_key": "test_secret", + "esql_query": "FROM logs", + "enrich_with_org_tags": False, + } + + +def test_check_initializes(huntress_instance): + check = HuntressCheck('huntress', {}, [huntress_instance]) + assert check is not None + assert check.SERVICE_CHECK_NAME == 'huntress.siem.check_status' + + +def test_check_runs(dd_run_check, aggregator, huntress_instance): + check = HuntressCheck('huntress', {}, [huntress_instance]) + check.log = MagicMock() + + empty_response = {"logs": [], "pagination": {}} + + with patch.object(check, "_request_with_retry", return_value=MagicMock( + status_code=200, + json=lambda: empty_response, + headers={"x-huntress-api-call-limit": "60", "x-huntress-api-call-remaining": "60"}, + )): + dd_run_check(check) From d29312fc5abe682300e29c0417b187504bd89b62 Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Wed, 27 May 2026 23:32:58 -0500 Subject: [PATCH 02/40] Update codeowners for huntress --- .github/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3412e2a8b5..e2bfa118fa 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1184,6 +1184,8 @@ code-coverage.datadog.yml @DataDog/agent-integr /aerospike_enterprise/ @support-aerospike @DataDog/ecosystems-review /scamalytics/ @ScamalyticsDev @DataDog/ecosystems-review +/huntress/ @kyletaylored kyle.taylor@datadoghq.com + # LEAVE THE FOLLOWING LOG OWNERSHIP LAST IN THE FILE # Make sure logs team is the full owner for all logs related files **/assets/logs/ @DataDog/logs-integrations-reviewers From 04bf61cd863c281c4a995804fb755f3874a5fcdc Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Wed, 27 May 2026 23:38:04 -0500 Subject: [PATCH 03/40] Fix ddev formatting issues --- huntress/checks.d/huntress.py | 109 +++----- huntress/datadog_checks/huntress/huntress.py | 109 +++----- huntress/tests/fixtures/siem_query_empty.json | 2 +- huntress/tests/fixtures/siem_query_page1.json | 2 +- huntress/tests/fixtures/siem_query_page2.json | 2 +- huntress/tests/test_docker.py | 1 + huntress/tests/test_huntress.py | 242 +++++++++++------- huntress/tests/test_unit.py | 17 +- 8 files changed, 235 insertions(+), 249 deletions(-) diff --git a/huntress/checks.d/huntress.py b/huntress/checks.d/huntress.py index 640cf2c232..9ba9d54ca9 100644 --- a/huntress/checks.d/huntress.py +++ b/huntress/checks.d/huntress.py @@ -1,11 +1,11 @@ import base64 -import gzip import hashlib import json import time from datetime import datetime, timedelta, timezone import requests + from datadog_checks.base import AgentCheck, ConfigurationError CHECK_NAME = "huntress" @@ -63,9 +63,7 @@ def check(self, instance): # Org enrichment org_cache = None if enrich_orgs: - org_cache = self._get_or_refresh_org_cache( - base_url, headers, instance_hash, org_ttl - ) + org_cache = self._get_or_refresh_org_cache(base_url, headers, instance_hash, org_ttl) # Checkpoint range_start = self._load_checkpoint(instance_hash) @@ -84,9 +82,7 @@ def check(self, instance): except Exception: pass - self.log.debug( - "Huntress SIEM collection: range_start=%s range_end=%s", range_start, range_end - ) + self.log.debug("Huntress SIEM collection: range_start=%s range_end=%s", range_start, range_end) page_token = None total_logs = 0 @@ -96,9 +92,7 @@ def check(self, instance): try: while True: - logs, next_token = self._query_page( - base_url, headers, esql_query, range_start, range_end, page_token - ) + logs, next_token = self._query_page(base_url, headers, esql_query, range_start, range_end, page_token) if logs: service_tag = self._extract_service(extra_tags) @@ -123,8 +117,7 @@ def check(self, instance): if next_token and pages_fetched >= max_pages: hit_page_cap = True self.log.warning( - "Huntress SIEM: hit max_pages_per_run=%d cap; " - "remaining pages will be collected next run", + "Huntress SIEM: hit max_pages_per_run=%d cap; remaining pages will be collected next run", max_pages, ) break @@ -226,7 +219,6 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) backoff_408 = [2, 4] attempt = 0 - last_exc = None while True: try: @@ -244,62 +236,40 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) return resp if resp.status_code == 400: - self.count( - "huntress.siem.errors", 1, - tags=["error_type:bad_request"] - ) - raise Exception( - f"Huntress API 400 Bad Request: {resp.text}" - ) + self.count("huntress.siem.errors", 1, tags=["error_type:bad_request"]) + raise Exception(f"Huntress API 400 Bad Request: {resp.text}") if resp.status_code == 401: - self.count( - "huntress.siem.errors", 1, - tags=["error_type:auth_failure"] - ) - raise Exception( - "Huntress API 401 Unauthorized — check huntress_api_key and huntress_secret_key" - ) + self.count("huntress.siem.errors", 1, tags=["error_type:auth_failure"]) + raise Exception("Huntress API 401 Unauthorized — check huntress_api_key and huntress_secret_key") if resp.status_code == 404: - raise Exception( - "Huntress API 404 — SIEM feature may not be enabled on this account" - ) + raise Exception("Huntress API 404 — SIEM feature may not be enabled on this account") if resp.status_code == 408: if attempt < max_retries_408: wait = backoff_408[attempt] self.log.warning( "Huntress API 408 Query Timeout; retrying in %ds (attempt %d/%d)", - wait, attempt + 1, max_retries_408, + wait, + attempt + 1, + max_retries_408, ) time.sleep(wait) attempt += 1 continue - self.count( - "huntress.siem.errors", 1, - tags=["error_type:timeout"] - ) + self.count("huntress.siem.errors", 1, tags=["error_type:timeout"]) raise Exception("Huntress API 408 Query Timeout — query may be too broad") if resp.status_code == 413: - raise Exception( - "Huntress API 413 Memory Limit — narrow the query with KEEP or WHERE clauses" - ) + raise Exception("Huntress API 413 Memory Limit — narrow the query with KEEP or WHERE clauses") if resp.status_code == 422: - self.count( - "huntress.siem.errors", 1, - tags=["error_type:invalid_query"] - ) - raise Exception( - f"Huntress API 422 Invalid ES|QL query: {resp.text}" - ) + self.count("huntress.siem.errors", 1, tags=["error_type:invalid_query"]) + raise Exception(f"Huntress API 422 Invalid ES|QL query: {resp.text}") if resp.status_code == 429: - self.log.warning( - "Huntress API 429 Rate Limited — sleeping 60s then retrying" - ) + self.log.warning("Huntress API 429 Rate Limited — sleeping 60s then retrying") time.sleep(60) continue @@ -308,41 +278,34 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) wait = backoff_5xx[attempt] self.log.warning( "Huntress API %d Server Error; retrying in %ds (attempt %d/%d)", - resp.status_code, wait, attempt + 1, max_retries_5xx, + resp.status_code, + wait, + attempt + 1, + max_retries_5xx, ) time.sleep(wait) attempt += 1 continue - self.count( - "huntress.siem.errors", 1, - tags=["error_type:server_error"] - ) - raise Exception( - f"Huntress API {resp.status_code} Server Error after {max_retries_5xx} retries" - ) + self.count("huntress.siem.errors", 1, tags=["error_type:server_error"]) + raise Exception(f"Huntress API {resp.status_code} Server Error after {max_retries_5xx} retries") - raise Exception( - f"Huntress API unexpected status {resp.status_code}: {resp.text}" - ) + raise Exception(f"Huntress API unexpected status {resp.status_code}: {resp.text}") except requests.exceptions.RequestException as exc: - last_exc = exc if attempt < max_retries_5xx: wait = backoff_5xx[attempt] self.log.warning( "Huntress API network error (%s); retrying in %ds (attempt %d/%d)", - exc, wait, attempt + 1, max_retries_5xx, + exc, + wait, + attempt + 1, + max_retries_5xx, ) time.sleep(wait) attempt += 1 continue - self.count( - "huntress.siem.errors", 1, - tags=["error_type:connection_error"] - ) - raise Exception( - f"Huntress API connection error after {max_retries_5xx} retries: {exc}" - ) from exc + self.count("huntress.siem.errors", 1, tags=["error_type:connection_error"]) + raise Exception(f"Huntress API connection error after {max_retries_5xx} retries: {exc}") from exc # ------------------------------------------------------------------ # # Log transformation # @@ -355,11 +318,7 @@ def _extract_service(self, tags): return "huntress-siem" def _transform_log(self, raw_log, tags, service): - message = ( - raw_log.get("log.original") - or raw_log.get("message") - or json.dumps(raw_log) - ) + message = raw_log.get("log.original") or raw_log.get("message") or json.dumps(raw_log) timestamp = raw_log.get("@timestamp") date_ms = None @@ -465,9 +424,7 @@ def _fetch_org_cache(self, base_url, headers, account_id): page = 1 while True: url = base_url + self.HUNTRESS_ORGS_ENDPOINT.format(account_id=account_id) - resp = self._request_with_retry( - "GET", url, headers, params={"limit": 500, "page": page} - ) + resp = self._request_with_retry("GET", url, headers, params={"limit": 500, "page": page}) data = resp.json() org_list = data.get("organizations", []) for org in org_list: diff --git a/huntress/datadog_checks/huntress/huntress.py b/huntress/datadog_checks/huntress/huntress.py index 640cf2c232..9ba9d54ca9 100644 --- a/huntress/datadog_checks/huntress/huntress.py +++ b/huntress/datadog_checks/huntress/huntress.py @@ -1,11 +1,11 @@ import base64 -import gzip import hashlib import json import time from datetime import datetime, timedelta, timezone import requests + from datadog_checks.base import AgentCheck, ConfigurationError CHECK_NAME = "huntress" @@ -63,9 +63,7 @@ def check(self, instance): # Org enrichment org_cache = None if enrich_orgs: - org_cache = self._get_or_refresh_org_cache( - base_url, headers, instance_hash, org_ttl - ) + org_cache = self._get_or_refresh_org_cache(base_url, headers, instance_hash, org_ttl) # Checkpoint range_start = self._load_checkpoint(instance_hash) @@ -84,9 +82,7 @@ def check(self, instance): except Exception: pass - self.log.debug( - "Huntress SIEM collection: range_start=%s range_end=%s", range_start, range_end - ) + self.log.debug("Huntress SIEM collection: range_start=%s range_end=%s", range_start, range_end) page_token = None total_logs = 0 @@ -96,9 +92,7 @@ def check(self, instance): try: while True: - logs, next_token = self._query_page( - base_url, headers, esql_query, range_start, range_end, page_token - ) + logs, next_token = self._query_page(base_url, headers, esql_query, range_start, range_end, page_token) if logs: service_tag = self._extract_service(extra_tags) @@ -123,8 +117,7 @@ def check(self, instance): if next_token and pages_fetched >= max_pages: hit_page_cap = True self.log.warning( - "Huntress SIEM: hit max_pages_per_run=%d cap; " - "remaining pages will be collected next run", + "Huntress SIEM: hit max_pages_per_run=%d cap; remaining pages will be collected next run", max_pages, ) break @@ -226,7 +219,6 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) backoff_408 = [2, 4] attempt = 0 - last_exc = None while True: try: @@ -244,62 +236,40 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) return resp if resp.status_code == 400: - self.count( - "huntress.siem.errors", 1, - tags=["error_type:bad_request"] - ) - raise Exception( - f"Huntress API 400 Bad Request: {resp.text}" - ) + self.count("huntress.siem.errors", 1, tags=["error_type:bad_request"]) + raise Exception(f"Huntress API 400 Bad Request: {resp.text}") if resp.status_code == 401: - self.count( - "huntress.siem.errors", 1, - tags=["error_type:auth_failure"] - ) - raise Exception( - "Huntress API 401 Unauthorized — check huntress_api_key and huntress_secret_key" - ) + self.count("huntress.siem.errors", 1, tags=["error_type:auth_failure"]) + raise Exception("Huntress API 401 Unauthorized — check huntress_api_key and huntress_secret_key") if resp.status_code == 404: - raise Exception( - "Huntress API 404 — SIEM feature may not be enabled on this account" - ) + raise Exception("Huntress API 404 — SIEM feature may not be enabled on this account") if resp.status_code == 408: if attempt < max_retries_408: wait = backoff_408[attempt] self.log.warning( "Huntress API 408 Query Timeout; retrying in %ds (attempt %d/%d)", - wait, attempt + 1, max_retries_408, + wait, + attempt + 1, + max_retries_408, ) time.sleep(wait) attempt += 1 continue - self.count( - "huntress.siem.errors", 1, - tags=["error_type:timeout"] - ) + self.count("huntress.siem.errors", 1, tags=["error_type:timeout"]) raise Exception("Huntress API 408 Query Timeout — query may be too broad") if resp.status_code == 413: - raise Exception( - "Huntress API 413 Memory Limit — narrow the query with KEEP or WHERE clauses" - ) + raise Exception("Huntress API 413 Memory Limit — narrow the query with KEEP or WHERE clauses") if resp.status_code == 422: - self.count( - "huntress.siem.errors", 1, - tags=["error_type:invalid_query"] - ) - raise Exception( - f"Huntress API 422 Invalid ES|QL query: {resp.text}" - ) + self.count("huntress.siem.errors", 1, tags=["error_type:invalid_query"]) + raise Exception(f"Huntress API 422 Invalid ES|QL query: {resp.text}") if resp.status_code == 429: - self.log.warning( - "Huntress API 429 Rate Limited — sleeping 60s then retrying" - ) + self.log.warning("Huntress API 429 Rate Limited — sleeping 60s then retrying") time.sleep(60) continue @@ -308,41 +278,34 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) wait = backoff_5xx[attempt] self.log.warning( "Huntress API %d Server Error; retrying in %ds (attempt %d/%d)", - resp.status_code, wait, attempt + 1, max_retries_5xx, + resp.status_code, + wait, + attempt + 1, + max_retries_5xx, ) time.sleep(wait) attempt += 1 continue - self.count( - "huntress.siem.errors", 1, - tags=["error_type:server_error"] - ) - raise Exception( - f"Huntress API {resp.status_code} Server Error after {max_retries_5xx} retries" - ) + self.count("huntress.siem.errors", 1, tags=["error_type:server_error"]) + raise Exception(f"Huntress API {resp.status_code} Server Error after {max_retries_5xx} retries") - raise Exception( - f"Huntress API unexpected status {resp.status_code}: {resp.text}" - ) + raise Exception(f"Huntress API unexpected status {resp.status_code}: {resp.text}") except requests.exceptions.RequestException as exc: - last_exc = exc if attempt < max_retries_5xx: wait = backoff_5xx[attempt] self.log.warning( "Huntress API network error (%s); retrying in %ds (attempt %d/%d)", - exc, wait, attempt + 1, max_retries_5xx, + exc, + wait, + attempt + 1, + max_retries_5xx, ) time.sleep(wait) attempt += 1 continue - self.count( - "huntress.siem.errors", 1, - tags=["error_type:connection_error"] - ) - raise Exception( - f"Huntress API connection error after {max_retries_5xx} retries: {exc}" - ) from exc + self.count("huntress.siem.errors", 1, tags=["error_type:connection_error"]) + raise Exception(f"Huntress API connection error after {max_retries_5xx} retries: {exc}") from exc # ------------------------------------------------------------------ # # Log transformation # @@ -355,11 +318,7 @@ def _extract_service(self, tags): return "huntress-siem" def _transform_log(self, raw_log, tags, service): - message = ( - raw_log.get("log.original") - or raw_log.get("message") - or json.dumps(raw_log) - ) + message = raw_log.get("log.original") or raw_log.get("message") or json.dumps(raw_log) timestamp = raw_log.get("@timestamp") date_ms = None @@ -465,9 +424,7 @@ def _fetch_org_cache(self, base_url, headers, account_id): page = 1 while True: url = base_url + self.HUNTRESS_ORGS_ENDPOINT.format(account_id=account_id) - resp = self._request_with_retry( - "GET", url, headers, params={"limit": 500, "page": page} - ) + resp = self._request_with_retry("GET", url, headers, params={"limit": 500, "page": page}) data = resp.json() org_list = data.get("organizations", []) for org in org_list: diff --git a/huntress/tests/fixtures/siem_query_empty.json b/huntress/tests/fixtures/siem_query_empty.json index 2977178ef8..e190ca9244 100644 --- a/huntress/tests/fixtures/siem_query_empty.json +++ b/huntress/tests/fixtures/siem_query_empty.json @@ -1,4 +1,4 @@ { "logs": [], "pagination": {} -} +} \ No newline at end of file diff --git a/huntress/tests/fixtures/siem_query_page1.json b/huntress/tests/fixtures/siem_query_page1.json index 6cacd3a42d..90e2dd2b61 100644 --- a/huntress/tests/fixtures/siem_query_page1.json +++ b/huntress/tests/fixtures/siem_query_page1.json @@ -26,4 +26,4 @@ "pagination": { "next_page_token": "page2_token_abc123" } -} +} \ No newline at end of file diff --git a/huntress/tests/fixtures/siem_query_page2.json b/huntress/tests/fixtures/siem_query_page2.json index 31ab07a952..0349317c55 100644 --- a/huntress/tests/fixtures/siem_query_page2.json +++ b/huntress/tests/fixtures/siem_query_page2.json @@ -13,4 +13,4 @@ } ], "pagination": {} -} +} \ No newline at end of file diff --git a/huntress/tests/test_docker.py b/huntress/tests/test_docker.py index ff4dbf07d3..e46b2a430d 100644 --- a/huntress/tests/test_docker.py +++ b/huntress/tests/test_docker.py @@ -4,6 +4,7 @@ Run with Docker available: ddev test huntress -m integration """ + import pytest from datadog_checks.base import AgentCheck diff --git a/huntress/tests/test_huntress.py b/huntress/tests/test_huntress.py index 401216677c..b53b0de05f 100644 --- a/huntress/tests/test_huntress.py +++ b/huntress/tests/test_huntress.py @@ -2,6 +2,7 @@ Unit tests for the Huntress SIEM → Datadog Logs integration. Covers all 17 scenarios from PRD §12. """ + import json import os from datetime import datetime, timezone @@ -11,7 +12,6 @@ from datadog_checks.huntress import HuntressCheck - FIXTURES = os.path.join(os.path.dirname(__file__), "fixtures") @@ -49,10 +49,14 @@ def _mock_response(status_code, body, headers=None): resp.json.return_value = body resp.text = json.dumps(body) # Use a real dict so header parsing in _parse_rate_limit_headers works correctly. - resp.headers = headers if headers is not None else { - "x-huntress-api-call-limit": "60", - "x-huntress-api-call-remaining": "55", - } + resp.headers = ( + headers + if headers is not None + else { + "x-huntress-api-call-limit": "60", + "x-huntress-api-call-remaining": "55", + } + ) return resp @@ -60,16 +64,18 @@ def _mock_response(status_code, body, headers=None): # 1. Happy path — single page # =========================================================================== + def test_happy_path_single_page(): check, instance = _make_check() fixture = _load_fixture("siem_query_empty.json") fixture["logs"] = _load_fixture("siem_query_page1.json")["logs"][:1] - with patch.object(check, "_request_with_retry", return_value=_mock_response(200, fixture)) as mock_req, \ - patch.object(check, "_send_logs_batch") as mock_send, \ - patch.object(check, "_save_checkpoint") as mock_save, \ - patch("time.time", side_effect=[1000.0, 1001.0]): - + with ( + patch.object(check, "_request_with_retry", return_value=_mock_response(200, fixture)), + patch.object(check, "_send_logs_batch") as mock_send, + patch.object(check, "_save_checkpoint") as mock_save, + patch("time.time", side_effect=[1000.0, 1001.0]), + ): check.check(instance) mock_send.assert_called_once() @@ -84,6 +90,7 @@ def test_happy_path_single_page(): # 2. Happy path — multi-page # =========================================================================== + def test_happy_path_multi_page(): check, instance = _make_check() page1 = _load_fixture("siem_query_page1.json") @@ -94,11 +101,12 @@ def test_happy_path_multi_page(): _mock_response(200, page2), ] - with patch.object(check, "_request_with_retry", side_effect=responses), \ - patch.object(check, "_send_logs_batch") as mock_send, \ - patch.object(check, "_save_checkpoint") as mock_save, \ - patch("time.time", side_effect=[1000.0, 1002.0]): - + with ( + patch.object(check, "_request_with_retry", side_effect=responses), + patch.object(check, "_send_logs_batch") as mock_send, + patch.object(check, "_save_checkpoint") as mock_save, + patch("time.time", side_effect=[1000.0, 1002.0]), + ): check.check(instance) assert mock_send.call_count == 2 @@ -111,14 +119,16 @@ def test_happy_path_multi_page(): # 3. Auth failure (401) # =========================================================================== + def test_auth_failure_401(): check, instance = _make_check() - with patch.object(check, "_request_with_retry", - side_effect=Exception("Huntress API 401 Unauthorized")), \ - patch.object(check, "_save_checkpoint") as mock_save, \ - patch.object(check, "count") as mock_count, \ - patch("time.time", side_effect=[1000.0, 1001.0]): + with ( + patch.object(check, "_request_with_retry", side_effect=Exception("Huntress API 401 Unauthorized")), + patch.object(check, "_save_checkpoint") as mock_save, + patch.object(check, "count"), + patch("time.time", side_effect=[1000.0, 1001.0]), + ): with pytest.raises(Exception, match="401"): check.check(instance) @@ -130,11 +140,9 @@ def test_auth_failure_401_raises_via_retry(): check, instance = _make_check() resp = _mock_response(401, {"error": "unauthorized"}) - with patch("requests.request", return_value=resp), \ - patch.object(check, "count") as mock_count: + with patch("requests.request", return_value=resp), patch.object(check, "count") as mock_count: with pytest.raises(Exception, match="401"): - check._request_with_retry("POST", "https://api.huntress.io/v1/siem/query", - {}, json_body={}) + check._request_with_retry("POST", "https://api.huntress.io/v1/siem/query", {}, json_body={}) mock_count.assert_called_with("huntress.siem.errors", 1, tags=["error_type:auth_failure"]) @@ -143,13 +151,16 @@ def test_auth_failure_401_raises_via_retry(): # 4. Query timeout (408) — retried twice then aborts # =========================================================================== + def test_query_timeout_408(): check, instance = _make_check() resp_408 = _mock_response(408, {"error": "timeout"}) - with patch("requests.request", return_value=resp_408), \ - patch("time.sleep") as mock_sleep, \ - patch.object(check, "count") as mock_count: + with ( + patch("requests.request", return_value=resp_408), + patch("time.sleep") as mock_sleep, + patch.object(check, "count") as mock_count, + ): with pytest.raises(Exception, match="408"): check._request_with_retry("POST", "https://x/q", {}, json_body={}) @@ -164,13 +175,13 @@ def test_query_timeout_408(): # 5. Rate limit (429) — sleeps 60s and retries # =========================================================================== + def test_rate_limit_429_then_success(): check, instance = _make_check() resp_429 = _mock_response(429, {"error": "rate_limited"}) resp_200 = _mock_response(200, _load_fixture("siem_query_empty.json")) - with patch("requests.request", side_effect=[resp_429, resp_200]), \ - patch("time.sleep") as mock_sleep: + with patch("requests.request", side_effect=[resp_429, resp_200]), patch("time.sleep") as mock_sleep: result = check._request_with_retry("POST", "https://x/q", {}, json_body={}) mock_sleep.assert_called_once_with(60) @@ -181,12 +192,12 @@ def test_rate_limit_429_then_success(): # 6. Invalid ES|QL (422) # =========================================================================== + def test_invalid_esql_422(): check, instance = _make_check() resp_422 = _mock_response(422, {"error": "invalid query"}) - with patch("requests.request", return_value=resp_422), \ - patch.object(check, "count") as mock_count: + with patch("requests.request", return_value=resp_422), patch.object(check, "count") as mock_count: with pytest.raises(Exception, match="422"): check._request_with_retry("POST", "https://x/q", {}, json_body={}) @@ -197,6 +208,7 @@ def test_invalid_esql_422(): # 7. Checkpoint persistence — second run uses previous range_end as range_start # =========================================================================== + def test_checkpoint_persistence(): check, instance = _make_check() saved_ts = "2026-05-27T13:00:00.000Z" @@ -213,8 +225,10 @@ def capture_request(method, url, headers, json_body=None, params=None): captured_bodies.append(json_body) return _mock_response(200, _load_fixture("siem_query_empty.json")) - with patch.object(check, "_request_with_retry", side_effect=capture_request), \ - patch("time.time", side_effect=[1000.0, 1001.0]): + with ( + patch.object(check, "_request_with_retry", side_effect=capture_request), + patch("time.time", side_effect=[1000.0, 1001.0]), + ): check.check(instance) assert captured_bodies, "No SIEM query was made" @@ -226,6 +240,7 @@ def capture_request(method, url, headers, json_body=None, params=None): # 8. No checkpoint (first run) — range_start defaults to now - interval # =========================================================================== + def test_no_checkpoint_first_run(): check, instance = _make_check(min_collection_interval=900) captured_bodies = [] @@ -236,9 +251,11 @@ def capture_request(method, url, headers, json_body=None, params=None): return _mock_response(200, _load_fixture("siem_query_empty.json")) fixed_now = datetime(2026, 5, 27, 14, 0, 0, tzinfo=timezone.utc) - with patch.object(check, "_request_with_retry", side_effect=capture_request), \ - patch("datadog_checks.huntress.huntress.datetime") as mock_dt, \ - patch("time.time", side_effect=[1000.0, 1001.0]): + with ( + patch.object(check, "_request_with_retry", side_effect=capture_request), + patch("datadog_checks.huntress.huntress.datetime") as mock_dt, + patch("time.time", side_effect=[1000.0, 1001.0]), + ): mock_dt.now.return_value = fixed_now mock_dt.fromisoformat = datetime.fromisoformat check.check(instance) @@ -255,6 +272,7 @@ def capture_request(method, url, headers, json_body=None, params=None): # 9. ES|QL validation — query not starting with FROM logs raises ConfigurationError # =========================================================================== + def test_esql_validation_rejects_bad_query(): check, instance = _make_check(esql_query="SELECT * FROM logs") with pytest.raises(Exception, match="FROM logs"): @@ -263,17 +281,23 @@ def test_esql_validation_rejects_bad_query(): def test_esql_validation_accepts_from_logs(): check, instance = _make_check(esql_query="FROM logs | KEEP @timestamp, message") - with patch.object(check, "_request_with_retry", - return_value=_mock_response(200, _load_fixture("siem_query_empty.json"))), \ - patch("time.time", side_effect=[1000.0, 1001.0]): + with ( + patch.object( + check, "_request_with_retry", return_value=_mock_response(200, _load_fixture("siem_query_empty.json")) + ), + patch("time.time", side_effect=[1000.0, 1001.0]), + ): check.check(instance) # should not raise def test_esql_validation_case_insensitive(): check, instance = _make_check(esql_query="from logs | limit 100") - with patch.object(check, "_request_with_retry", - return_value=_mock_response(200, _load_fixture("siem_query_empty.json"))), \ - patch("time.time", side_effect=[1000.0, 1001.0]): + with ( + patch.object( + check, "_request_with_retry", return_value=_mock_response(200, _load_fixture("siem_query_empty.json")) + ), + patch("time.time", side_effect=[1000.0, 1001.0]), + ): check.check(instance) # should not raise @@ -281,6 +305,7 @@ def test_esql_validation_case_insensitive(): # 10. Log transformation — ECS fields correctly mapped to Datadog payload # =========================================================================== + def test_log_transformation(): check, instance = _make_check() raw = { @@ -323,6 +348,7 @@ def test_log_transformation_json_fallback(): # 11. Batching — >1,000 logs split into multiple batches # =========================================================================== + def test_batching_over_1000_logs(): check, instance = _make_check() @@ -335,9 +361,11 @@ def test_batching_over_1000_logs(): ] fixture = {"logs": large_log_list, "pagination": {}} - with patch.object(check, "_request_with_retry", return_value=_mock_response(200, fixture)), \ - patch.object(check, "_send_logs_batch") as mock_send, \ - patch("time.time", side_effect=[1000.0, 1001.0]): + with ( + patch.object(check, "_request_with_retry", return_value=_mock_response(200, fixture)), + patch.object(check, "_send_logs_batch") as mock_send, + patch("time.time", side_effect=[1000.0, 1001.0]), + ): check.check(instance) assert mock_send.call_count == 2 @@ -351,13 +379,16 @@ def test_batching_over_1000_logs(): # 12. max_pages_per_run cap — stops after N pages; checkpoint does NOT advance # =========================================================================== + def test_max_pages_per_run_cap(): check, instance = _make_check(max_pages_per_run=1) page1 = _load_fixture("siem_query_page1.json") - with patch.object(check, "_request_with_retry", return_value=_mock_response(200, page1)), \ - patch.object(check, "_save_checkpoint") as mock_save, \ - patch("time.time", side_effect=[1000.0, 1001.0]): + with ( + patch.object(check, "_request_with_retry", return_value=_mock_response(200, page1)), + patch.object(check, "_save_checkpoint") as mock_save, + patch("time.time", side_effect=[1000.0, 1001.0]), + ): check.check(instance) mock_save.assert_not_called() @@ -367,12 +398,14 @@ def test_max_pages_per_run_cap(): # 13. Org enrichment — cache hit (no extra API calls) # =========================================================================== + def test_org_enrichment_cache_hit(): check, instance = _make_check(enrich_with_org_tags=True) instance_hash = check._instance_hash(instance) # Pre-populate cache (fresh) from datetime import datetime, timezone + cache = { "fetched_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z", "account_id": 42, @@ -391,9 +424,11 @@ def side_effect(method, url, headers, json_body=None, params=None): api_calls.append(url) return _mock_response(200, page1) - with patch.object(check, "_request_with_retry", side_effect=side_effect), \ - patch.object(check, "_send_logs_batch") as mock_send, \ - patch("time.time", side_effect=[1000.0, 1001.0]): + with ( + patch.object(check, "_request_with_retry", side_effect=side_effect), + patch.object(check, "_send_logs_batch") as mock_send, + patch("time.time", side_effect=[1000.0, 1001.0]), + ): check.check(instance) # Only the SIEM query should have been called — no org API calls @@ -411,20 +446,27 @@ def side_effect(method, url, headers, json_body=None, params=None): # 14. Org enrichment — cache miss (fetches account + orgs) # =========================================================================== + def test_org_enrichment_cache_miss(): check, instance = _make_check(enrich_with_org_tags=True) account_resp = _mock_response(200, {"account": {"id": 42}}) - orgs_resp = _mock_response(200, { - "organizations": [ - {"id": 101, "name": "Acme Inc.", "key": "acme"}, - ], - "pagination": {}, - }) - siem_resp = _mock_response(200, { - "logs": _load_fixture("siem_query_page1.json")["logs"][:1], - "pagination": {}, - }) + orgs_resp = _mock_response( + 200, + { + "organizations": [ + {"id": 101, "name": "Acme Inc.", "key": "acme"}, + ], + "pagination": {}, + }, + ) + siem_resp = _mock_response( + 200, + { + "logs": _load_fixture("siem_query_page1.json")["logs"][:1], + "pagination": {}, + }, + ) call_urls = [] @@ -437,9 +479,11 @@ def side_effect(method, url, headers, json_body=None, params=None): else: return siem_resp - with patch.object(check, "_request_with_retry", side_effect=side_effect), \ - patch.object(check, "_send_logs_batch") as mock_send, \ - patch("time.time", side_effect=[1000.0, 1001.0]): + with ( + patch.object(check, "_request_with_retry", side_effect=side_effect), + patch.object(check, "_send_logs_batch") as mock_send, + patch("time.time", side_effect=[1000.0, 1001.0]), + ): check.check(instance) assert any("/account" in u for u in call_urls) @@ -453,21 +497,27 @@ def side_effect(method, url, headers, json_body=None, params=None): # 15. Org enrichment — fetch failure → warning, logs collected without org tags # =========================================================================== + def test_org_enrichment_fetch_failure(): check, instance = _make_check(enrich_with_org_tags=True) - siem_resp = _mock_response(200, { - "logs": _load_fixture("siem_query_page1.json")["logs"][:1], - "pagination": {}, - }) + siem_resp = _mock_response( + 200, + { + "logs": _load_fixture("siem_query_page1.json")["logs"][:1], + "pagination": {}, + }, + ) def side_effect(method, url, headers, json_body=None, params=None): if "/account" in url and "organization" not in url: raise Exception("Network error fetching account") return siem_resp - with patch.object(check, "_request_with_retry", side_effect=side_effect), \ - patch.object(check, "_send_logs_batch") as mock_send, \ - patch("time.time", side_effect=[1000.0, 1001.0]): + with ( + patch.object(check, "_request_with_retry", side_effect=side_effect), + patch.object(check, "_send_logs_batch") as mock_send, + patch("time.time", side_effect=[1000.0, 1001.0]), + ): check.check(instance) # must not raise check.log.warning.assert_called() @@ -482,6 +532,7 @@ def side_effect(method, url, headers, json_body=None, params=None): # 16. Org enrichment — no org match → only huntress_account_id tag # =========================================================================== + def test_org_enrichment_no_org_match(): check, instance = _make_check() org_cache = { @@ -500,6 +551,7 @@ def test_org_enrichment_no_org_match(): # 17. Multi-instance isolation — independent checkpoints and org caches # =========================================================================== + def test_multi_instance_isolation(): instance_a = _make_instance( huntress_api_key="key_a", @@ -543,6 +595,7 @@ def test_multi_instance_isolation(): # Bonus: _get_org_tags strategies # =========================================================================== + def test_org_tags_by_id(): check, _ = _make_check() org_cache = { @@ -585,16 +638,21 @@ def test_org_tags_nested_org_field(): # Rate limit header parsing # =========================================================================== + def test_rate_limit_headers_parsed_and_stored(): """_parse_rate_limit_headers populates _last_api_call_* attributes.""" check, _ = _make_check() check._last_api_call_limit = None check._last_api_call_remaining = None - resp = _mock_response(200, {}, headers={ - "x-huntress-api-call-limit": "60", - "x-huntress-api-call-remaining": "42", - }) + resp = _mock_response( + 200, + {}, + headers={ + "x-huntress-api-call-limit": "60", + "x-huntress-api-call-remaining": "42", + }, + ) check._parse_rate_limit_headers(resp) assert check._last_api_call_limit == 60 @@ -607,10 +665,14 @@ def test_rate_limit_warning_when_low(): check._last_api_call_limit = None check._last_api_call_remaining = None - resp = _mock_response(200, {}, headers={ - "x-huntress-api-call-limit": "60", - "x-huntress-api-call-remaining": "3", - }) + resp = _mock_response( + 200, + {}, + headers={ + "x-huntress-api-call-limit": "60", + "x-huntress-api-call-remaining": "3", + }, + ) check._parse_rate_limit_headers(resp) check.log.warning.assert_called() @@ -641,19 +703,23 @@ def test_rate_limit_metrics_emitted_in_check(): def capture_gauge(name, value, tags=None): emitted_gauges[name] = value - real_request_with_retry = check._request_with_retry - def fake_retry(method, url, headers, json_body=None, params=None): - resp = _mock_response(200, fixture, headers={ - "x-huntress-api-call-limit": "60", - "x-huntress-api-call-remaining": "48", - }) + resp = _mock_response( + 200, + fixture, + headers={ + "x-huntress-api-call-limit": "60", + "x-huntress-api-call-remaining": "48", + }, + ) check._parse_rate_limit_headers(resp) return resp - with patch.object(check, "_request_with_retry", side_effect=fake_retry), \ - patch.object(check, "gauge", side_effect=capture_gauge), \ - patch("time.time", side_effect=[1000.0, 1001.0]): + with ( + patch.object(check, "_request_with_retry", side_effect=fake_retry), + patch.object(check, "gauge", side_effect=capture_gauge), + patch("time.time", side_effect=[1000.0, 1001.0]), + ): check.check(instance) assert emitted_gauges.get("huntress.siem.api_call_limit") == 60 diff --git a/huntress/tests/test_unit.py b/huntress/tests/test_unit.py index a14d0fdd48..e25a8a7f70 100644 --- a/huntress/tests/test_unit.py +++ b/huntress/tests/test_unit.py @@ -1,10 +1,11 @@ from typing import Any, Callable, Dict # noqa: F401 +from unittest.mock import MagicMock, patch import pytest + from datadog_checks.base import AgentCheck # noqa: F401 from datadog_checks.base.stubs.aggregator import AggregatorStub # noqa: F401 from datadog_checks.huntress import HuntressCheck -from unittest.mock import MagicMock, patch @pytest.fixture @@ -29,9 +30,13 @@ def test_check_runs(dd_run_check, aggregator, huntress_instance): empty_response = {"logs": [], "pagination": {}} - with patch.object(check, "_request_with_retry", return_value=MagicMock( - status_code=200, - json=lambda: empty_response, - headers={"x-huntress-api-call-limit": "60", "x-huntress-api-call-remaining": "60"}, - )): + with patch.object( + check, + "_request_with_retry", + return_value=MagicMock( + status_code=200, + json=lambda: empty_response, + headers={"x-huntress-api-call-limit": "60", "x-huntress-api-call-remaining": "60"}, + ), + ): dd_run_check(check) From f054ec8355506e8b46d49413959bfbb7457fd83e Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Wed, 27 May 2026 23:50:53 -0500 Subject: [PATCH 04/40] Fix changelog formatting --- huntress/CHANGELOG.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/huntress/CHANGELOG.md b/huntress/CHANGELOG.md index 461b8c6ce0..b9a9569a19 100644 --- a/huntress/CHANGELOG.md +++ b/huntress/CHANGELOG.md @@ -2,9 +2,8 @@ ## Unreleased -**_Added_**: +***Added***: -- Initial release of the Huntress SIEM integration for Datadog. Polls the Huntress - Managed SIEM API via ES|QL, enriches logs with organization metadata, and forwards - all events to Datadog Logs. Emits collection health metrics and a pre-built overview - dashboard with monitor templates. +* Initial release of the Huntress SIEM integration for Datadog. +* Polls the Huntress Managed SIEM API via ES|QL, enriches logs with organization metadata, and forwards all events to Datadog Logs. +* Emits collection health metrics and a pre-built overview dashboard with monitor templates. From d52a491216cdfdc17667b7cc1cb198213ce6eb84 Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Thu, 28 May 2026 00:13:06 -0500 Subject: [PATCH 05/40] Remove media, use unclaimed source id --- huntress/{images => }/huntress-logo.png | Bin huntress/manifest.json | 10 ++-------- 2 files changed, 2 insertions(+), 8 deletions(-) rename huntress/{images => }/huntress-logo.png (100%) diff --git a/huntress/images/huntress-logo.png b/huntress/huntress-logo.png similarity index 100% rename from huntress/images/huntress-logo.png rename to huntress/huntress-logo.png diff --git a/huntress/manifest.json b/huntress/manifest.json index 7b3a8d6c94..8c0d207a22 100644 --- a/huntress/manifest.json +++ b/huntress/manifest.json @@ -11,13 +11,7 @@ "changelog": "CHANGELOG.md", "description": "Collect and forward Huntress Managed SIEM logs to Datadog for security monitoring.", "title": "Huntress", - "media": [ - { - "media_type": "image", - "caption": "Huntress Managed SIEM integration for Datadog", - "image_url": "images/huntress-logo.png" - } - ], + "media": [], "resources": [ { "resource_type": "documentation", @@ -59,7 +53,7 @@ "service_checks": { "metadata_path": "assets/service_checks.json" }, - "source_type_id": 10350, + "source_type_id": 10450, "auto_install": true }, "dashboards": { From 9f0f722f0559e64d9299bfba2e603584a6995727 Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Thu, 28 May 2026 00:13:26 -0500 Subject: [PATCH 06/40] Fix validation issues --- huntress/README.md | 45 +++--- huntress/assets/configuration/spec.yaml | 146 ++++++++--------- huntress/assets/service_checks.json | 4 +- huntress/conf.d/huntress.yaml.example | 11 -- .../huntress/data/conf.yaml.example | 153 +++++++++++------- 5 files changed, 194 insertions(+), 165 deletions(-) diff --git a/huntress/README.md b/huntress/README.md index 76592d4c0f..22343f4c30 100644 --- a/huntress/README.md +++ b/huntress/README.md @@ -6,7 +6,7 @@ This integration polls the Huntress Managed SIEM API using ES|QL queries and forwards all security events to Datadog as logs. Each collection run: -1. Loads a checkpoint — the timestamp of the last successful collection +1. Loads a checkpoint - the timestamp of the last successful collection 2. Executes a configurable ES|QL query for the elapsed time window 3. Paginates through all result pages 4. Optionally enriches each log with Huntress organization metadata (org name, key, account ID) @@ -70,7 +70,7 @@ datadog-agent integration install -t datadog-huntress==1.0.0 ### Multiple Huntress accounts -Add additional blocks under `instances:` — each runs independently with its own checkpoint, org metadata cache, and metrics: +Add additional blocks under `instances:` - each runs independently with its own checkpoint, org metadata cache, and metrics: ```yaml instances: @@ -87,17 +87,16 @@ instances: ### Configuration reference -| Field | Required | Default | Description | -| ------------------------- | -------- | ------------------------- | ----------------------------------------- | -| `huntress_api_key` | Yes | — | Huntress public API key | -| `huntress_secret_key` | Yes | — | Huntress secret API key | -| `esql_query` | Yes | — | ES\|QL query; must begin with `FROM logs` | -| `enrich_with_org_tags` | No | `true` | Fetch and attach org metadata as log tags | -| `org_cache_ttl_seconds` | No | `3600` | How long to cache org metadata (seconds) | -| `min_collection_interval` | No | `900` | Seconds between runs (minimum 60) | -| `max_pages_per_run` | No | `100` | Page cap per run (~20,000 logs maximum) | -| `huntress_base_url` | No | `https://api.huntress.io` | Override for sandbox environments | -| `tags` | No | `[]` | Extra tags on every forwarded log | +| Field | Required | Default | Description | +| ----------------------- | -------- | ------------------------- | ----------------------------------------- | +| `huntress_api_key` | Yes | - | Huntress public API key | +| `huntress_secret_key` | Yes | - | Huntress secret API key | +| `esql_query` | Yes | - | ES\|QL query; must begin with `FROM logs` | +| `enrich_with_org_tags` | No | `true` | Fetch and attach org metadata as log tags | +| `org_cache_ttl_seconds` | No | `3600` | How long to cache org metadata (seconds) | +| `max_pages_per_run` | No | `100` | Page cap per run (~20,000 logs maximum) | +| `huntress_base_url` | No | `https://api.huntress.io` | Override for sandbox environments | +| `tags` | No | `[]` | Extra tags on every forwarded log | ## Data Collected @@ -105,20 +104,20 @@ instances: All logs collected from the Huntress Managed SIEM API are forwarded to Datadog with: -- `ddsource: huntress` — enables automatic log pipeline processing +- `ddsource: huntress` - enables automatic log pipeline processing - ECS field names preserved as top-level log attributes (for example, `event.category`, `host.hostname`, `user.name`) - Organization metadata tags when `enrich_with_org_tags: true` (for example, `huntress_org_name`, `huntress_org_key`, `huntress_account_id`) ### Metrics -| Metric | Type | Description | -| ------------------------------------ | ----- | ------------------------------------------------------- | -| `huntress.siem.logs_collected` | Gauge | Log events collected per run | -| `huntress.siem.pages_fetched` | Gauge | API pages fetched per run | -| `huntress.siem.run_duration_seconds` | Gauge | Wall time of the collection run | -| `huntress.siem.errors` | Count | Errors by type (`error_type` tag) | -| `huntress.siem.api_call_limit` | Gauge | Total API requests allowed per minute (from Huntress) | -| `huntress.siem.api_call_remaining` | Gauge | API requests remaining in the current minute | +| Metric | Type | Description | +| ------------------------------------ | ----- | ----------------------------------------------------- | +| `huntress.siem.logs_collected` | Gauge | Log events collected per run | +| `huntress.siem.pages_fetched` | Gauge | API pages fetched per run | +| `huntress.siem.run_duration_seconds` | Gauge | Wall time of the collection run | +| `huntress.siem.errors` | Count | Errors by type (`error_type` tag) | +| `huntress.siem.api_call_limit` | Gauge | Total API requests allowed per minute (from Huntress) | +| `huntress.siem.api_call_remaining` | Gauge | API requests remaining in the current minute | See [metadata.csv][1] for a full list of metrics. @@ -159,7 +158,7 @@ The Huntress API allows 60 requests per minute. The integration logs a warning w **Duplicate logs after Agent restart** -This is expected on the first restart after a failed run — the checkpoint is only advanced when all pages are successfully sent. Subsequent runs resume from the last successful checkpoint. +This is expected on the first restart after a failed run - the checkpoint is only advanced when all pages are successfully sent. Subsequent runs resume from the last successful checkpoint. Need help? Contact [Datadog support][2]. diff --git a/huntress/assets/configuration/spec.yaml b/huntress/assets/configuration/spec.yaml index f0b3e57bc9..015b6c7b1d 100644 --- a/huntress/assets/configuration/spec.yaml +++ b/huntress/assets/configuration/spec.yaml @@ -1,78 +1,74 @@ name: Huntress files: -- name: huntress.yaml - options: - - template: init_config + - name: huntress.yaml options: - - template: init_config/default - - template: instances - options: - - name: huntress_api_key - required: true - description: | - Huntress public API key. Obtain from the Huntress Partner Portal under - Settings > API Credentials. Each instance uses its own key pair, enabling - multiple Huntress accounts to feed into the same Datadog organization. - value: - type: string - example: - - name: huntress_secret_key - required: true - description: | - Huntress private (secret) API key paired with huntress_api_key. - value: - type: string - example: - - name: esql_query - required: true - description: | - ES|QL query to execute against the Huntress SIEM API. Must begin with - "FROM logs" (case-insensitive). Use KEEP to select specific ECS columns - or omit for all fields. Do not add time range filters — range_start and - range_end are managed automatically by the check. - value: - type: string - example: "FROM logs" - - name: enrich_with_org_tags - description: | - When true, the check fetches Huntress organization metadata (org name, - org key, account ID) and attaches it as log tags. The org cache is - refreshed according to org_cache_ttl_seconds. - value: - type: boolean - example: true - default: true - - name: org_cache_ttl_seconds - description: | - How long (in seconds) to cache organization metadata before re-fetching - from the Huntress API. Set to 0 to refresh on every run. - value: - type: integer - example: 3600 - default: 3600 - - name: min_collection_interval - description: | - Seconds between check runs. Must be >= 60 to stay within Huntress API - rate limits (60 requests per minute). Defaults to 900 (15 minutes). - value: - type: integer - example: 900 - default: 900 - - name: max_pages_per_run - description: | - Maximum number of result pages to fetch per run. Each page contains up - to 200 log events. Protects against runaway pagination on large backlogs. - If this cap is hit, the checkpoint does not advance and remaining pages - are fetched on the next run. - value: - type: integer - example: 100 - default: 100 - - name: huntress_base_url - description: | - Huntress API base URL. Override for sandbox or testing environments. - value: - type: string - example: "https://api.huntress.io" - default: "https://api.huntress.io" - - template: instances/default + - template: init_config + options: + - template: init_config/default + - template: instances + options: + - template: instances/default + overrides: + min_collection_interval.value.default: 900 + - name: huntress_api_key + required: true + secret: true + description: | + Huntress public API key. Obtain from the Huntress Partner Portal under + Settings > API Credentials. Each instance uses its own key pair, enabling + multiple Huntress accounts to feed into the same Datadog organization. + value: + type: string + example: + - name: huntress_secret_key + required: true + secret: true + description: | + Huntress private (secret) API key paired with huntress_api_key. + value: + type: string + example: + - name: esql_query + required: true + description: | + ES|QL query to execute against the Huntress SIEM API. Must begin with + "FROM logs" (case-insensitive). Use KEEP to select specific ECS columns + or omit for all fields. Do not add time range filters — range_start and + range_end are managed automatically by the check. + value: + type: string + example: "FROM logs" + - name: enrich_with_org_tags + description: | + When true, the check fetches Huntress organization metadata (org name, + org key, account ID) and attaches it as log tags. The org cache is + refreshed according to org_cache_ttl_seconds. + value: + type: boolean + example: true + default: true + - name: org_cache_ttl_seconds + description: | + How long (in seconds) to cache organization metadata before re-fetching + from the Huntress API. Set to 0 to refresh on every run. + value: + type: integer + example: 3600 + default: 3600 + - name: max_pages_per_run + description: | + Maximum number of result pages to fetch per run. Each page contains up + to 200 log events. Protects against runaway pagination on large backlogs. + If this cap is hit, the checkpoint does not advance and remaining pages + are fetched on the next run. + value: + type: integer + example: 100 + default: 100 + - name: huntress_base_url + description: | + Huntress API base URL. Override for sandbox or testing environments. + value: + type: string + example: "https://api.huntress.io" + default: "https://api.huntress.io" diff --git a/huntress/assets/service_checks.json b/huntress/assets/service_checks.json index 5a8e160a3c..f5b604d8c2 100644 --- a/huntress/assets/service_checks.json +++ b/huntress/assets/service_checks.json @@ -1,7 +1,7 @@ [ { "agent_version": "7.0.0", - "integration": "huntress", + "integration": "Huntress", "check": "huntress.siem.check_status", "statuses": [ "ok", @@ -11,4 +11,4 @@ "name": "Huntress SIEM Check Status", "description": "Returns `CRITICAL` if the Huntress SIEM collection run fails for any reason (auth error, network error, invalid query, etc.), returns `OK` otherwise." } -] +] \ No newline at end of file diff --git a/huntress/conf.d/huntress.yaml.example b/huntress/conf.d/huntress.yaml.example index 48832eef85..95086765cf 100644 --- a/huntress/conf.d/huntress.yaml.example +++ b/huntress/conf.d/huntress.yaml.example @@ -43,14 +43,3 @@ instances: - "source:huntress" - "service:huntress-siem" - "env:production" - - ## --- Account 2 (optional second Huntress account) --- - ## Each instance runs independently with its own checkpoint, org cache, and metrics. - ## Use multiple instances to fan in logs from multiple Huntress accounts. - - huntress_api_key: "" - huntress_secret_key: "" - esql_query: "FROM logs" - tags: - - "source:huntress" - - "service:huntress-siem" - - "env:staging" diff --git a/huntress/datadog_checks/huntress/data/conf.yaml.example b/huntress/datadog_checks/huntress/data/conf.yaml.example index 48832eef85..b1148aaaeb 100644 --- a/huntress/datadog_checks/huntress/data/conf.yaml.example +++ b/huntress/datadog_checks/huntress/data/conf.yaml.example @@ -1,56 +1,101 @@ -init_config: {} +## All options defined here are available to all instances. +# +init_config: + ## @param service - string - optional + ## Attach the tag `service:` to every metric, event, and service check emitted by this integration. + ## + ## Additionally, this sets the default `service` for every log source. + # + # service: + +## Every instance is scheduled independently of the others. +# instances: - ## --- Account 1 --- - - ## Required: Huntress API credentials (Basic Auth). - ## Each instance has its own key pair, allowing multiple Huntress accounts - ## to feed into the same Datadog org simultaneously. - huntress_api_key: "" - huntress_secret_key: "" - - ## Required: ES|QL query to execute. Must begin with "FROM logs" (case-insensitive). - ## The query determines which log fields are returned. Use KEEP to select specific - ## columns, or omit for all ECS fields. Do NOT add time range filters — range_start - ## and range_end are managed automatically by the check. - esql_query: "FROM logs" - - ## Optional: Whether to enrich logs with Huntress organization metadata - ## (org name, org key, account ID). Fetches /v1/account and - ## /v1/accounts/{id}/organizations on first run and caches for org_cache_ttl_seconds. - ## Default: true - enrich_with_org_tags: true - - ## Optional: How long (seconds) to cache the org metadata before re-fetching. - ## Default: 3600 (1 hour). Set to 0 to refresh on every run. - org_cache_ttl_seconds: 3600 - - ## Optional: Collection interval in seconds. Must be >= 60 to stay within Huntress - ## rate limits (60 req/min). Default: 900 (15 minutes). - min_collection_interval: 900 - - ## Optional: Maximum number of pages to fetch per run (200 logs/page). - ## Protects against runaway pagination on large backlogs. - ## Default: 100 (up to 20,000 logs per run). - max_pages_per_run: 100 - - ## Optional: Huntress API base URL. Override for testing or sandbox environments. - huntress_base_url: "https://api.huntress.io" - - ## Optional: Additional tags attached to every log event forwarded to Datadog. - ## huntress_account_id, huntress_org_name, and huntress_org_key are added - ## automatically when enrich_with_org_tags is true. - tags: - - "source:huntress" - - "service:huntress-siem" - - "env:production" - - ## --- Account 2 (optional second Huntress account) --- - ## Each instance runs independently with its own checkpoint, org cache, and metrics. - ## Use multiple instances to fan in logs from multiple Huntress accounts. - - huntress_api_key: "" - huntress_secret_key: "" - esql_query: "FROM logs" - tags: - - "source:huntress" - - "service:huntress-siem" - - "env:staging" + + - + ## @param tags - list of strings - optional + ## A list of tags to attach to every metric and service check emitted by this instance. + ## + ## Learn more about tagging at https://docs.datadoghq.com/tagging + # + # tags: + # - : + # - : + + ## @param service - string - optional + ## Attach the tag `service:` to every metric, event, and service check emitted by this integration. + ## + ## Overrides any `service` defined in the `init_config` section. + # + # service: + + ## @param min_collection_interval - number - optional - default: 15 + ## This changes the collection interval of the check. For more information, see: + ## https://docs.datadoghq.com/developers/write_agent_check/#collection-interval + # + # min_collection_interval: 15 + + ## @param empty_default_hostname - boolean - optional - default: false + ## This forces the check to send metrics with no hostname. + ## + ## This is useful for cluster-level checks. + # + # empty_default_hostname: false + + ## @param metric_patterns - mapping - optional + ## A mapping of metrics to include or exclude, with each entry being a regular expression. + ## + ## Metrics defined in `exclude` will take precedence in case of overlap. + # + # metric_patterns: + # include: + # - + # exclude: + # - + + ## @param huntress_api_key - string - required + ## Huntress public API key. Obtain from the Huntress Partner Portal under + ## Settings > API Credentials. Each instance uses its own key pair, enabling + ## multiple Huntress accounts to feed into the same Datadog organization. + # + huntress_api_key: + + ## @param huntress_secret_key - string - required + ## Huntress private (secret) API key paired with huntress_api_key. + # + huntress_secret_key: + + ## @param esql_query - string - required + ## ES|QL query to execute against the Huntress SIEM API. Must begin with + ## "FROM logs" (case-insensitive). Use KEEP to select specific ECS columns + ## or omit for all fields. Do not add time range filters — range_start and + ## range_end are managed automatically by the check. + # + esql_query: FROM logs + + ## @param enrich_with_org_tags - boolean - optional - default: true + ## When true, the check fetches Huntress organization metadata (org name, + ## org key, account ID) and attaches it as log tags. The org cache is + ## refreshed according to org_cache_ttl_seconds. + # + # enrich_with_org_tags: true + + ## @param org_cache_ttl_seconds - integer - optional - default: 3600 + ## How long (in seconds) to cache organization metadata before re-fetching + ## from the Huntress API. Set to 0 to refresh on every run. + # + # org_cache_ttl_seconds: 3600 + + ## @param max_pages_per_run - integer - optional - default: 100 + ## Maximum number of result pages to fetch per run. Each page contains up + ## to 200 log events. Protects against runaway pagination on large backlogs. + ## If this cap is hit, the checkpoint does not advance and remaining pages + ## are fetched on the next run. + # + # max_pages_per_run: 100 + + ## @param huntress_base_url - string - optional - default: https://api.huntress.io + ## Huntress API base URL. Override for sandbox or testing environments. + # + # huntress_base_url: https://api.huntress.io From c1b2128bb8082566affd7f214a2973a5f62b9815 Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Thu, 28 May 2026 01:36:35 -0500 Subject: [PATCH 07/40] Add multiple log query support per API key --- huntress/README.md | 41 ++-- huntress/assets/configuration/spec.yaml | 43 +++- .../huntress/data/conf.yaml.example | 30 ++- huntress/datadog_checks/huntress/huntress.py | 206 ++++++++++-------- huntress/tests/conftest.py | 4 +- huntress/tests/test_huntress.py | 25 ++- huntress/tests/test_unit.py | 2 +- 7 files changed, 210 insertions(+), 141 deletions(-) diff --git a/huntress/README.md b/huntress/README.md index 22343f4c30..1dd55dfd26 100644 --- a/huntress/README.md +++ b/huntress/README.md @@ -43,7 +43,8 @@ datadog-agent integration install -t datadog-huntress==1.0.0 instances: - huntress_api_key: "" huntress_secret_key: "" - esql_query: "FROM logs" + log_queries: + - esql_query: "FROM logs" tags: - "source:huntress" - "service:huntress-siem" @@ -76,27 +77,37 @@ Add additional blocks under `instances:` - each runs independently with its own instances: - huntress_api_key: "" huntress_secret_key: "" - esql_query: "FROM logs" + log_queries: + - esql_query: "FROM logs" tags: ["source:huntress", "env:production"] - huntress_api_key: "" huntress_secret_key: "" - esql_query: "FROM logs" + log_queries: + - esql_query: "FROM logs" tags: ["source:huntress", "env:staging"] ``` ### Configuration reference -| Field | Required | Default | Description | -| ----------------------- | -------- | ------------------------- | ----------------------------------------- | -| `huntress_api_key` | Yes | - | Huntress public API key | -| `huntress_secret_key` | Yes | - | Huntress secret API key | -| `esql_query` | Yes | - | ES\|QL query; must begin with `FROM logs` | -| `enrich_with_org_tags` | No | `true` | Fetch and attach org metadata as log tags | -| `org_cache_ttl_seconds` | No | `3600` | How long to cache org metadata (seconds) | -| `max_pages_per_run` | No | `100` | Page cap per run (~20,000 logs maximum) | -| `huntress_base_url` | No | `https://api.huntress.io` | Override for sandbox environments | -| `tags` | No | `[]` | Extra tags on every forwarded log | +**`init_config` options** (apply to all instances): + +| Field | Required | Default | Description | +| ----------------- | -------- | ------- | ----------------------------------------------- | +| `request_timeout` | No | `30` | HTTP request timeout in seconds for all API calls | + +**`instances` options** (per Huntress account): + +| Field | Required | Default | Description | +| ----------------------- | -------- | ------------------------- | ------------------------------------------------------------------------------ | +| `huntress_api_key` | Yes | - | Huntress public API key | +| `huntress_secret_key` | Yes | - | Huntress secret API key | +| `log_queries` | Yes | - | List of query objects; each has `esql_query` (required) and `tags` (optional). `esql_query` must begin with `FROM logs` | +| `enrich_with_org_tags` | No | `true` | Fetch and attach org metadata as log tags | +| `org_cache_ttl_seconds` | No | `3600` | How long to cache org metadata (seconds) | +| `max_pages_per_run` | No | `100` | Page cap per query per run (~20,000 logs maximum) | +| `huntress_base_url` | No | `https://api.huntress.io` | Override for sandbox environments | +| `tags` | No | `[]` | Extra tags on every forwarded log | ## Data Collected @@ -137,7 +148,7 @@ Returns `CRITICAL` if the collection run fails for any reason; `OK` otherwise. - Run `sudo datadog-agent check huntress` and inspect the output - Verify the API key pair is valid by checking the Huntress Partner Portal - Confirm the Managed SIEM feature is enabled on the account -- Check that `esql_query` begins with `FROM logs` +- Check that each `log_queries[].esql_query` begins with `FROM logs` **`huntress.siem.errors` count is increasing** @@ -147,7 +158,7 @@ Inspect the `error_type` tag to identify the root cause: | ------------------ | ---------------------------------- | ------------------------------------------------------------ | | `auth_failure` | Invalid or rotated API credentials | Update `huntress_api_key` / `huntress_secret_key` | | `timeout` | ES\|QL query too broad | Add a `KEEP` or `WHERE` clause to the query | -| `invalid_query` | Malformed ES\|QL | Fix the `esql_query` value | +| `invalid_query` | Malformed ES\|QL | Fix the `esql_query` value in the failing `log_queries` entry | | `server_error` | Transient Huntress API error | Check [Huntress status page](https://status.huntress.com) | | `connection_error` | Network issue | Verify connectivity from the Agent host to `api.huntress.io` | | `run_failure` | Unexpected error during collection | Check Agent logs for the full stack trace | diff --git a/huntress/assets/configuration/spec.yaml b/huntress/assets/configuration/spec.yaml index 015b6c7b1d..d5af6c8d8a 100644 --- a/huntress/assets/configuration/spec.yaml +++ b/huntress/assets/configuration/spec.yaml @@ -5,6 +5,14 @@ files: - template: init_config options: - template: init_config/default + - name: request_timeout + description: | + HTTP request timeout in seconds applied to all Huntress API calls. + Increase this if the Huntress SIEM API is slow to respond on large queries. + value: + type: number + example: 30 + default: 30 - template: instances options: - template: instances/default @@ -28,16 +36,29 @@ files: value: type: string example: - - name: esql_query + - name: log_queries required: true description: | - ES|QL query to execute against the Huntress SIEM API. Must begin with - "FROM logs" (case-insensitive). Use KEEP to select specific ECS columns - or omit for all fields. Do not add time range filters — range_start and - range_end are managed automatically by the check. + List of ES|QL queries to execute against the Huntress SIEM API. Each entry + requires an esql_query string (must begin with "FROM logs", case-insensitive) + and an optional tags list attached only to logs produced by that query. + Multiple queries run independently with separate checkpoints, enabling + targeted collection for different log types within the same account. + Do not add time range filters — range_start and range_end are managed + automatically by the check. value: - type: string - example: "FROM logs" + type: array + items: + type: object + properties: + - name: esql_query + type: string + - name: tags + type: array + items: + type: string + example: + - esql_query: "FROM logs" - name: enrich_with_org_tags description: | When true, the check fetches Huntress organization metadata (org name, @@ -57,10 +78,10 @@ files: default: 3600 - name: max_pages_per_run description: | - Maximum number of result pages to fetch per run. Each page contains up - to 200 log events. Protects against runaway pagination on large backlogs. - If this cap is hit, the checkpoint does not advance and remaining pages - are fetched on the next run. + Maximum number of result pages to fetch per query per run. Each page + contains up to 200 log events. Protects against runaway pagination on + large backlogs. If this cap is hit, the checkpoint does not advance and + remaining pages are fetched on the next run. value: type: integer example: 100 diff --git a/huntress/datadog_checks/huntress/data/conf.yaml.example b/huntress/datadog_checks/huntress/data/conf.yaml.example index b1148aaaeb..4e99f4420a 100644 --- a/huntress/datadog_checks/huntress/data/conf.yaml.example +++ b/huntress/datadog_checks/huntress/data/conf.yaml.example @@ -9,6 +9,12 @@ init_config: # # service: + ## @param request_timeout - number - optional - default: 30 + ## HTTP request timeout in seconds applied to all Huntress API calls. + ## Increase this if the Huntress SIEM API is slow to respond on large queries. + # + # request_timeout: 30 + ## Every instance is scheduled independently of the others. # instances: @@ -66,13 +72,17 @@ instances: # huntress_secret_key: - ## @param esql_query - string - required - ## ES|QL query to execute against the Huntress SIEM API. Must begin with - ## "FROM logs" (case-insensitive). Use KEEP to select specific ECS columns - ## or omit for all fields. Do not add time range filters — range_start and - ## range_end are managed automatically by the check. + ## @param log_queries - list of mappings - required + ## List of ES|QL queries to execute against the Huntress SIEM API. Each entry + ## requires an esql_query string (must begin with "FROM logs", case-insensitive) + ## and an optional tags list attached only to logs produced by that query. + ## Multiple queries run independently with separate checkpoints, enabling + ## targeted collection for different log types within the same account. + ## Do not add time range filters — range_start and range_end are managed + ## automatically by the check. # - esql_query: FROM logs + log_queries: + - esql_query: FROM logs ## @param enrich_with_org_tags - boolean - optional - default: true ## When true, the check fetches Huntress organization metadata (org name, @@ -88,10 +98,10 @@ instances: # org_cache_ttl_seconds: 3600 ## @param max_pages_per_run - integer - optional - default: 100 - ## Maximum number of result pages to fetch per run. Each page contains up - ## to 200 log events. Protects against runaway pagination on large backlogs. - ## If this cap is hit, the checkpoint does not advance and remaining pages - ## are fetched on the next run. + ## Maximum number of result pages to fetch per query per run. Each page + ## contains up to 200 log events. Protects against runaway pagination on + ## large backlogs. If this cap is hit, the checkpoint does not advance and + ## remaining pages are fetched on the next run. # # max_pages_per_run: 100 diff --git a/huntress/datadog_checks/huntress/huntress.py b/huntress/datadog_checks/huntress/huntress.py index 9ba9d54ca9..796358129f 100644 --- a/huntress/datadog_checks/huntress/huntress.py +++ b/huntress/datadog_checks/huntress/huntress.py @@ -25,6 +25,7 @@ class HuntressCheck(AgentCheck): MAX_BATCH_SIZE_BYTES = 4 * 1024 * 1024 # 4 MB headroom under 5 MB limit DEFAULT_BASE_URL = "https://api.huntress.io" + DEFAULT_REQUEST_TIMEOUT = 30 DEFAULT_MIN_COLLECTION_INTERVAL = 900 DEFAULT_MAX_PAGES_PER_RUN = 100 DEFAULT_ORG_CACHE_TTL = 3600 @@ -40,100 +41,60 @@ def check(self, instance): api_key = instance.get("huntress_api_key", "").strip() secret_key = instance.get("huntress_secret_key", "").strip() - esql_query = instance.get("esql_query", "").strip() base_url = instance.get("huntress_base_url", self.DEFAULT_BASE_URL).rstrip("/") max_pages = int(instance.get("max_pages_per_run", self.DEFAULT_MAX_PAGES_PER_RUN)) min_interval = int(instance.get("min_collection_interval", self.DEFAULT_MIN_COLLECTION_INTERVAL)) enrich_orgs = instance.get("enrich_with_org_tags", True) org_ttl = int(instance.get("org_cache_ttl_seconds", self.DEFAULT_ORG_CACHE_TTL)) extra_tags = list(instance.get("tags", [])) + log_queries = instance.get("log_queries") or [] if not api_key: raise ConfigurationError("huntress_api_key is required") if not secret_key: raise ConfigurationError("huntress_secret_key is required") - if not esql_query: - raise ConfigurationError("esql_query is required") - if not esql_query.lower().lstrip().startswith("from logs"): - raise ConfigurationError("esql_query must begin with 'FROM logs' (case-insensitive)") + if not log_queries: + raise ConfigurationError("log_queries must contain at least one query") instance_hash = self._instance_hash(instance) headers = self._get_auth_header(api_key, secret_key) - # Org enrichment + # Org enrichment cache is shared across all queries for this instance org_cache = None if enrich_orgs: org_cache = self._get_or_refresh_org_cache(base_url, headers, instance_hash, org_ttl) - # Checkpoint - range_start = self._load_checkpoint(instance_hash) - now = datetime.now(timezone.utc) - range_end = now.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" - - if range_start is None: - default_start = now - timedelta(seconds=min_interval) - range_start = default_start.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" - else: - # Add 1ms offset to avoid re-fetching the last millisecond from previous run - try: - dt = datetime.fromisoformat(range_start.replace("Z", "+00:00")) - dt = dt + timedelta(milliseconds=1) - range_start = dt.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" - except Exception: - pass - - self.log.debug("Huntress SIEM collection: range_start=%s range_end=%s", range_start, range_end) - - page_token = None total_logs = 0 - pages_fetched = 0 - hit_page_cap = False + total_pages = 0 success = False try: - while True: - logs, next_token = self._query_page(base_url, headers, esql_query, range_start, range_end, page_token) - - if logs: - service_tag = self._extract_service(extra_tags) - batch = [] - for raw in logs: - org_tags = self._get_org_tags(raw, org_cache) if org_cache else [] - all_tags = extra_tags + org_tags - payload = self._transform_log(raw, all_tags, service_tag) - batch.append(payload) - if len(batch) >= self.MAX_LOGS_PER_BATCH: - self._send_logs_batch(batch) - batch = [] - if batch: - self._send_logs_batch(batch) - - total_logs += len(logs) - pages_fetched += 1 + for query_def in log_queries: + esql = query_def.get("esql_query", "").strip() + query_tags = list(query_def.get("tags", [])) + + if not esql: + raise ConfigurationError("Each entry in log_queries must have a non-empty esql_query") + if not esql.lower().lstrip().startswith("from logs"): + raise ConfigurationError( + f"esql_query must begin with 'FROM logs' (case-insensitive): {esql!r}" + ) - if next_token and pages_fetched < max_pages: - page_token = next_token - else: - if next_token and pages_fetched >= max_pages: - hit_page_cap = True - self.log.warning( - "Huntress SIEM: hit max_pages_per_run=%d cap; remaining pages will be collected next run", - max_pages, - ) - break + # Each query tracks its own checkpoint so queries are independently resumable + checkpoint_key = instance_hash + "_" + self._query_hash(esql) + all_tags = extra_tags + query_tags - if not hit_page_cap: - self._save_checkpoint(instance_hash, range_end) + logs, pages = self._run_query( + base_url, headers, esql, checkpoint_key, max_pages, min_interval, org_cache, all_tags + ) + total_logs += logs + total_pages += pages success = True except Exception as exc: self.log.error("Huntress SIEM run failed: %s", exc) - self.count( - "huntress.siem.errors", - 1, - tags=extra_tags + ["error_type:run_failure"], - ) + self.count("huntress.siem.errors", 1, tags=extra_tags + ["error_type:run_failure"]) self.service_check(self.SERVICE_CHECK_NAME, self.CRITICAL, tags=extra_tags) raise @@ -141,7 +102,7 @@ def check(self, instance): duration = time.time() - start_time self.gauge("huntress.siem.run_duration_seconds", duration, tags=extra_tags) self.gauge("huntress.siem.logs_collected", total_logs, tags=extra_tags) - self.gauge("huntress.siem.pages_fetched", pages_fetched, tags=extra_tags) + self.gauge("huntress.siem.pages_fetched", total_pages, tags=extra_tags) if self._last_api_call_limit is not None: self.gauge("huntress.siem.api_call_limit", self._last_api_call_limit, tags=extra_tags) if self._last_api_call_remaining is not None: @@ -150,6 +111,73 @@ def check(self, instance): if success: self.service_check(self.SERVICE_CHECK_NAME, self.OK, tags=extra_tags) + # ------------------------------------------------------------------ # + # Per-query execution # + # ------------------------------------------------------------------ # + + def _run_query(self, base_url, headers, esql, checkpoint_key, max_pages, min_interval, org_cache, all_tags): + """Paginate a single ES|QL query. Returns (logs_collected, pages_fetched).""" + range_start = self._load_checkpoint(checkpoint_key) + now = datetime.now(timezone.utc) + range_end = now.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + + if range_start is None: + default_start = now - timedelta(seconds=min_interval) + range_start = default_start.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + else: + # Add 1ms to avoid re-fetching the boundary event from the previous run + try: + dt = datetime.fromisoformat(range_start.replace("Z", "+00:00")) + dt = dt + timedelta(milliseconds=1) + range_start = dt.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + except Exception: + pass + + self.log.debug( + "Huntress SIEM query: esql=%r range_start=%s range_end=%s", + esql[:80], range_start, range_end, + ) + + service_tag = self._extract_service(all_tags) + page_token = None + total_logs = 0 + pages_fetched = 0 + hit_page_cap = False + + while True: + logs, next_token = self._query_page(base_url, headers, esql, range_start, range_end, page_token) + + if logs: + batch = [] + for raw in logs: + org_tags = self._get_org_tags(raw, org_cache) if org_cache else [] + payload = self._transform_log(raw, all_tags + org_tags, service_tag) + batch.append(payload) + if len(batch) >= self.MAX_LOGS_PER_BATCH: + self._send_logs_batch(batch) + batch = [] + if batch: + self._send_logs_batch(batch) + + total_logs += len(logs) + pages_fetched += 1 + + if next_token and pages_fetched < max_pages: + page_token = next_token + else: + if next_token and pages_fetched >= max_pages: + hit_page_cap = True + self.log.warning( + "Huntress SIEM: hit max_pages_per_run=%d for query %r; remaining pages collected next run", + max_pages, esql[:60], + ) + break + + if not hit_page_cap: + self._save_checkpoint(checkpoint_key, range_end) + + return total_logs, pages_fetched + # ------------------------------------------------------------------ # # Auth # # ------------------------------------------------------------------ # @@ -163,7 +191,6 @@ def _get_auth_header(self, api_key, secret_key): } def _parse_rate_limit_headers(self, resp): - """Parse x-huntress-api-call-limit / x-huntress-api-call-remaining headers.""" try: limit = resp.headers.get("x-huntress-api-call-limit") if limit is not None: @@ -194,13 +221,7 @@ def _query_page(self, base_url, headers, esql, range_start, range_end, page_toke if page_token: body["page_token"] = page_token - response = self._request_with_retry( - method="POST", - url=url, - headers=headers, - json_body=body, - ) - + response = self._request_with_retry(method="POST", url=url, headers=headers, json_body=body) data = response.json() logs = data.get("logs", []) pagination = data.get("pagination", {}) @@ -213,6 +234,7 @@ def _query_page(self, base_url, headers, esql, range_start, range_end, page_toke def _request_with_retry(self, method, url, headers, json_body=None, params=None): """Execute an HTTP request with retry logic per PRD §7.""" + timeout = self.init_config.get("request_timeout", self.DEFAULT_REQUEST_TIMEOUT) max_retries_5xx = 3 max_retries_408 = 2 backoff_5xx = [5, 10, 20] @@ -228,7 +250,7 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) headers=headers, json=json_body, params=params, - timeout=30, + timeout=timeout, ) if resp.status_code == 200: @@ -251,9 +273,7 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) wait = backoff_408[attempt] self.log.warning( "Huntress API 408 Query Timeout; retrying in %ds (attempt %d/%d)", - wait, - attempt + 1, - max_retries_408, + wait, attempt + 1, max_retries_408, ) time.sleep(wait) attempt += 1 @@ -278,10 +298,7 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) wait = backoff_5xx[attempt] self.log.warning( "Huntress API %d Server Error; retrying in %ds (attempt %d/%d)", - resp.status_code, - wait, - attempt + 1, - max_retries_5xx, + resp.status_code, wait, attempt + 1, max_retries_5xx, ) time.sleep(wait) attempt += 1 @@ -296,10 +313,7 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) wait = backoff_5xx[attempt] self.log.warning( "Huntress API network error (%s); retrying in %ds (attempt %d/%d)", - exc, - wait, - attempt + 1, - max_retries_5xx, + exc, wait, attempt + 1, max_retries_5xx, ) time.sleep(wait) attempt += 1 @@ -353,7 +367,6 @@ def _transform_log(self, raw_log, tags, service): # ------------------------------------------------------------------ # def _send_logs_batch(self, logs_batch): - """Send a batch of up to MAX_LOGS_PER_BATCH log payloads via self.send_log().""" for log_payload in logs_batch: self.send_log(log_payload) @@ -361,8 +374,8 @@ def _send_logs_batch(self, logs_batch): # Checkpoint # # ------------------------------------------------------------------ # - def _load_checkpoint(self, instance_hash): - key = self.CHECKPOINT_CACHE_KEY_PREFIX + instance_hash + def _load_checkpoint(self, checkpoint_key): + key = self.CHECKPOINT_CACHE_KEY_PREFIX + checkpoint_key raw = self.read_persistent_cache(key) if not raw: return None @@ -372,8 +385,8 @@ def _load_checkpoint(self, instance_hash): except Exception: return None - def _save_checkpoint(self, instance_hash, timestamp_iso): - key = self.CHECKPOINT_CACHE_KEY_PREFIX + instance_hash + def _save_checkpoint(self, checkpoint_key, timestamp_iso): + key = self.CHECKPOINT_CACHE_KEY_PREFIX + checkpoint_key payload = json.dumps({"last_collected_at": timestamp_iso, "schema_version": 1}) self.write_persistent_cache(key, payload) @@ -498,9 +511,18 @@ def _get_org_tags(self, raw_log, org_cache): return base_tags # ------------------------------------------------------------------ # - # Instance hash # + # Hashing # # ------------------------------------------------------------------ # def _instance_hash(self, instance): - key = f"{instance.get('huntress_api_key', '')}:{instance.get('esql_query', '')}" + """Stable hash for a Huntress account (api key + secret + base url).""" + key = "|".join([ + instance.get("huntress_api_key", ""), + instance.get("huntress_secret_key", ""), + instance.get("huntress_base_url", self.DEFAULT_BASE_URL), + ]) return hashlib.md5(key.encode()).hexdigest()[:12] + + def _query_hash(self, esql_query): + """Short hash to build a per-query checkpoint key.""" + return hashlib.md5(esql_query.encode()).hexdigest()[:8] diff --git a/huntress/tests/conftest.py b/huntress/tests/conftest.py index 2e7067ab9c..052649b65d 100644 --- a/huntress/tests/conftest.py +++ b/huntress/tests/conftest.py @@ -27,7 +27,7 @@ def dd_environment(): yield { "huntress_api_key": "hk_testpublickey", "huntress_secret_key": "hs_testsecretkey", - "esql_query": "FROM logs", + "log_queries": [{"esql_query": "FROM logs"}], "huntress_base_url": f"http://{HOST}:{MOCKOON_PORT}", "enrich_with_org_tags": True, "org_cache_ttl_seconds": 3600, @@ -42,7 +42,7 @@ def instance(): return { "huntress_api_key": "test_api_key", "huntress_secret_key": "test_secret_key", - "esql_query": "FROM logs", + "log_queries": [{"esql_query": "FROM logs"}], "enrich_with_org_tags": False, "min_collection_interval": 900, "max_pages_per_run": 100, diff --git a/huntress/tests/test_huntress.py b/huntress/tests/test_huntress.py index b53b0de05f..2c5e577564 100644 --- a/huntress/tests/test_huntress.py +++ b/huntress/tests/test_huntress.py @@ -21,10 +21,11 @@ def _load_fixture(name): def _make_instance(**kwargs): + esql = kwargs.pop("esql_query", "FROM logs") base = { "huntress_api_key": "pub_key", "huntress_secret_key": "secret_key", - "esql_query": "FROM logs", + "log_queries": [{"esql_query": esql}], "enrich_with_org_tags": False, "tags": ["source:huntress", "env:test"], } @@ -213,8 +214,10 @@ def test_checkpoint_persistence(): check, instance = _make_check() saved_ts = "2026-05-27T13:00:00.000Z" instance_hash = check._instance_hash(instance) + query_hash = check._query_hash("FROM logs") + checkpoint_key = instance_hash + "_" + query_hash check.write_persistent_cache( - check.CHECKPOINT_CACHE_KEY_PREFIX + instance_hash, + check.CHECKPOINT_CACHE_KEY_PREFIX + checkpoint_key, json.dumps({"last_collected_at": saved_ts, "schema_version": 1}), ) @@ -556,12 +559,10 @@ def test_multi_instance_isolation(): instance_a = _make_instance( huntress_api_key="key_a", huntress_secret_key="secret_a", - esql_query="FROM logs", ) instance_b = _make_instance( huntress_api_key="key_b", huntress_secret_key="secret_b", - esql_query="FROM logs", ) check_a = HuntressCheck("huntress", {}, [instance_a]) @@ -578,17 +579,21 @@ def test_multi_instance_isolation(): hash_a = check_a._instance_hash(instance_a) hash_b = check_b._instance_hash(instance_b) + query_hash = check_a._query_hash("FROM logs") assert hash_a != hash_b - # Checkpoints stored under different keys must return different values - check_a._save_checkpoint(hash_a, "2026-05-27T14:00:00.000Z") - check_b._save_checkpoint(hash_b, "2026-05-27T15:00:00.000Z") + # Each query gets its own checkpoint key: instance_hash + "_" + query_hash + ck_a = hash_a + "_" + query_hash + ck_b = hash_b + "_" + query_hash - assert check_a._load_checkpoint(hash_a) == "2026-05-27T14:00:00.000Z" - assert check_b._load_checkpoint(hash_b) == "2026-05-27T15:00:00.000Z" + check_a._save_checkpoint(ck_a, "2026-05-27T14:00:00.000Z") + check_b._save_checkpoint(ck_b, "2026-05-27T15:00:00.000Z") + + assert check_a._load_checkpoint(ck_a) == "2026-05-27T14:00:00.000Z" + assert check_b._load_checkpoint(ck_b) == "2026-05-27T15:00:00.000Z" # Each check's cache is isolated — A cannot see B's checkpoint - assert check_a._load_checkpoint(hash_b) is None + assert check_a._load_checkpoint(ck_b) is None # =========================================================================== diff --git a/huntress/tests/test_unit.py b/huntress/tests/test_unit.py index e25a8a7f70..e34907e5d2 100644 --- a/huntress/tests/test_unit.py +++ b/huntress/tests/test_unit.py @@ -13,7 +13,7 @@ def huntress_instance(): return { "huntress_api_key": "test_key", "huntress_secret_key": "test_secret", - "esql_query": "FROM logs", + "log_queries": [{"esql_query": "FROM logs"}], "enrich_with_org_tags": False, } From 785969dac6dd315b1f8d61923fa9d93b416b9eb3 Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Thu, 28 May 2026 02:04:52 -0500 Subject: [PATCH 08/40] Run formatter --- huntress/datadog_checks/huntress/huntress.py | 37 +++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/huntress/datadog_checks/huntress/huntress.py b/huntress/datadog_checks/huntress/huntress.py index 796358129f..d806eb0300 100644 --- a/huntress/datadog_checks/huntress/huntress.py +++ b/huntress/datadog_checks/huntress/huntress.py @@ -76,9 +76,7 @@ def check(self, instance): if not esql: raise ConfigurationError("Each entry in log_queries must have a non-empty esql_query") if not esql.lower().lstrip().startswith("from logs"): - raise ConfigurationError( - f"esql_query must begin with 'FROM logs' (case-insensitive): {esql!r}" - ) + raise ConfigurationError(f"esql_query must begin with 'FROM logs' (case-insensitive): {esql!r}") # Each query tracks its own checkpoint so queries are independently resumable checkpoint_key = instance_hash + "_" + self._query_hash(esql) @@ -135,7 +133,9 @@ def _run_query(self, base_url, headers, esql, checkpoint_key, max_pages, min_int self.log.debug( "Huntress SIEM query: esql=%r range_start=%s range_end=%s", - esql[:80], range_start, range_end, + esql[:80], + range_start, + range_end, ) service_tag = self._extract_service(all_tags) @@ -169,7 +169,8 @@ def _run_query(self, base_url, headers, esql, checkpoint_key, max_pages, min_int hit_page_cap = True self.log.warning( "Huntress SIEM: hit max_pages_per_run=%d for query %r; remaining pages collected next run", - max_pages, esql[:60], + max_pages, + esql[:60], ) break @@ -273,7 +274,9 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) wait = backoff_408[attempt] self.log.warning( "Huntress API 408 Query Timeout; retrying in %ds (attempt %d/%d)", - wait, attempt + 1, max_retries_408, + wait, + attempt + 1, + max_retries_408, ) time.sleep(wait) attempt += 1 @@ -298,7 +301,10 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) wait = backoff_5xx[attempt] self.log.warning( "Huntress API %d Server Error; retrying in %ds (attempt %d/%d)", - resp.status_code, wait, attempt + 1, max_retries_5xx, + resp.status_code, + wait, + attempt + 1, + max_retries_5xx, ) time.sleep(wait) attempt += 1 @@ -313,7 +319,10 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) wait = backoff_5xx[attempt] self.log.warning( "Huntress API network error (%s); retrying in %ds (attempt %d/%d)", - exc, wait, attempt + 1, max_retries_5xx, + exc, + wait, + attempt + 1, + max_retries_5xx, ) time.sleep(wait) attempt += 1 @@ -516,11 +525,13 @@ def _get_org_tags(self, raw_log, org_cache): def _instance_hash(self, instance): """Stable hash for a Huntress account (api key + secret + base url).""" - key = "|".join([ - instance.get("huntress_api_key", ""), - instance.get("huntress_secret_key", ""), - instance.get("huntress_base_url", self.DEFAULT_BASE_URL), - ]) + key = "|".join( + [ + instance.get("huntress_api_key", ""), + instance.get("huntress_secret_key", ""), + instance.get("huntress_base_url", self.DEFAULT_BASE_URL), + ] + ) return hashlib.md5(key.encode()).hexdigest()[:12] def _query_hash(self, esql_query): From ba274ff4e2aab21eb4c28866f40163d5627a1cbf Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Thu, 28 May 2026 02:16:47 -0500 Subject: [PATCH 09/40] Already moved mock data file --- huntress/tests/docker/Dockerfile | 5 ----- 1 file changed, 5 deletions(-) diff --git a/huntress/tests/docker/Dockerfile b/huntress/tests/docker/Dockerfile index 9cf0c1a52c..d141da6a6d 100644 --- a/huntress/tests/docker/Dockerfile +++ b/huntress/tests/docker/Dockerfile @@ -1,10 +1,5 @@ FROM mockoon/cli:latest -# Embed the Huntress mock data for standalone use (docker build). -# The docker-compose.yml mounts the file directly for development, -# so rebuilds are not required when huntress_mockoon.json changes. -COPY huntress_mockoon.json /data/huntress_mockoon.json - EXPOSE 3002 ENTRYPOINT ["mockoon-cli", "start", "--data", "/data/huntress_mockoon.json", "--port", "3002", "--hostname", "0.0.0.0"] From 9cfceb8d78e11c4b6f3b171b46522ff8fff24d90 Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Thu, 28 May 2026 03:17:45 -0500 Subject: [PATCH 10/40] Add query_name, remove md5, fix monitors --- huntress/README.md | 11 +- huntress/assets/configuration/spec.yaml | 5 +- .../monitors/huntress_siem_check_failed.json | 55 ++--- .../monitors/huntress_siem_errors_high.json | 51 ++--- .../huntress_siem_no_logs_collected.json | 49 ++--- huntress/checks.d/huntress.py | 202 +++++++++++------- .../huntress/data/conf.yaml.example | 3 +- huntress/datadog_checks/huntress/huntress.py | 13 +- huntress/tests/conftest.py | 4 +- huntress/tests/test_huntress.py | 2 +- huntress/tests/test_unit.py | 2 +- 11 files changed, 207 insertions(+), 190 deletions(-) diff --git a/huntress/README.md b/huntress/README.md index 1dd55dfd26..e9e338a12a 100644 --- a/huntress/README.md +++ b/huntress/README.md @@ -44,7 +44,8 @@ datadog-agent integration install -t datadog-huntress==1.0.0 - huntress_api_key: "" huntress_secret_key: "" log_queries: - - esql_query: "FROM logs" + - name: "all-logs" + esql_query: "FROM logs" tags: - "source:huntress" - "service:huntress-siem" @@ -78,13 +79,15 @@ instances: - huntress_api_key: "" huntress_secret_key: "" log_queries: - - esql_query: "FROM logs" + - name: "all-logs" + esql_query: "FROM logs" tags: ["source:huntress", "env:production"] - huntress_api_key: "" huntress_secret_key: "" log_queries: - - esql_query: "FROM logs" + - name: "all-logs" + esql_query: "FROM logs" tags: ["source:huntress", "env:staging"] ``` @@ -102,7 +105,7 @@ instances: | ----------------------- | -------- | ------------------------- | ------------------------------------------------------------------------------ | | `huntress_api_key` | Yes | - | Huntress public API key | | `huntress_secret_key` | Yes | - | Huntress secret API key | -| `log_queries` | Yes | - | List of query objects; each has `esql_query` (required) and `tags` (optional). `esql_query` must begin with `FROM logs` | +| `log_queries` | Yes | - | List of query objects; each has `name` (required), `esql_query` (required, must begin with `FROM logs`), and `tags` (optional). `name` is used as the `query_name` tag on metrics | | `enrich_with_org_tags` | No | `true` | Fetch and attach org metadata as log tags | | `org_cache_ttl_seconds` | No | `3600` | How long to cache org metadata (seconds) | | `max_pages_per_run` | No | `100` | Page cap per query per run (~20,000 logs maximum) | diff --git a/huntress/assets/configuration/spec.yaml b/huntress/assets/configuration/spec.yaml index d5af6c8d8a..d335237261 100644 --- a/huntress/assets/configuration/spec.yaml +++ b/huntress/assets/configuration/spec.yaml @@ -51,6 +51,8 @@ files: items: type: object properties: + - name: name + type: string - name: esql_query type: string - name: tags @@ -58,7 +60,8 @@ files: items: type: string example: - - esql_query: "FROM logs" + - name: "all-logs" + esql_query: "FROM logs" - name: enrich_with_org_tags description: | When true, the check fetches Huntress organization metadata (org name, diff --git a/huntress/assets/monitors/huntress_siem_check_failed.json b/huntress/assets/monitors/huntress_siem_check_failed.json index 4414cbdf10..96d78f2538 100644 --- a/huntress/assets/monitors/huntress_siem_check_failed.json +++ b/huntress/assets/monitors/huntress_siem_check_failed.json @@ -1,39 +1,28 @@ { - "version": 2, - "created_at": "2026-05-27", - "last_updated_at": "2026-05-27", - "title": "Huntress SIEM collection run failed", + "id": 19780007, + "name": "Huntress SIEM collection run failed on {{host.name}}", + "type": "service check", + "query": "\"huntress.siem.check_status\".over(\"*\").by(\"error_type\").last(2).count_by_status()", + "message": "The Huntress SIEM Agent check on {{host.name}} has entered a CRITICAL state, meaning the last collection run failed. Logs may no longer be flowing from Huntress to Datadog.\n\nCommon causes:\n- Invalid or rotated API credentials (`huntress_api_key` / `huntress_secret_key`)\n- Network connectivity issue between the Agent host and `api.huntress.io`\n- Invalid `esql_query` in the instance configuration\n- Huntress SIEM feature not enabled on this account\n\nCheck the Agent logs for details:\n```\nsudo datadog-agent check huntress\n```", "tags": [ "integration:huntress" ], - "description": "The Huntress SIEM check emits a CRITICAL service check whenever a collection run fails — for example due to an authentication error, network timeout, or invalid ES|QL query. This monitor alerts on any such failure so operators can investigate and restore log collection before data gaps accumulate.", - "definition": { - "message": "The Huntress SIEM Agent check on {{host.name}} has entered a CRITICAL state, meaning the last collection run failed. Logs may no longer be flowing from Huntress to Datadog.\n\nCommon causes:\n- Invalid or rotated API credentials (`huntress_api_key` / `huntress_secret_key`)\n- Network connectivity issue between the Agent host and `api.huntress.io`\n- Invalid `esql_query` in the instance configuration\n- Huntress SIEM feature not enabled on this account\n\nCheck the Agent logs for details:\n```\nsudo datadog-agent check huntress\n```\n\n@pagerduty @slack-security-alerts", - "name": "Huntress SIEM collection run failed on {{host.name}}", - "options": { - "include_tags": true, - "new_host_delay": 300, - "no_data_timeframe": 60, - "notify_audit": false, - "notify_no_data": false, - "renotify_interval": 60, - "renotify_statuses": [ - "alert" - ], - "require_full_window": false, - "silenced": {}, - "thresholds": { - "critical": 1, - "ok": 1 - }, - "timeout_h": 0 + "options": { + "thresholds": { + "critical": 1, + "ok": 1 }, - "priority": 2, - "query": "\"huntress.siem.check_status\".over(\"*\").last(2).count_by_status()", - "restricted_roles": null, - "tags": [ - "integration:huntress" + "notify_audit": false, + "renotify_interval": 60, + "timeout_h": 0, + "include_tags": true, + "renotify_statuses": [ + "alert" ], - "type": "service check" - } -} + "require_full_window": false, + "escalation_message": "" + }, + "priority": 2, + "draft_status": "published", + "assets": [] +} \ No newline at end of file diff --git a/huntress/assets/monitors/huntress_siem_errors_high.json b/huntress/assets/monitors/huntress_siem_errors_high.json index a83e4b3d81..d78bf36023 100644 --- a/huntress/assets/monitors/huntress_siem_errors_high.json +++ b/huntress/assets/monitors/huntress_siem_errors_high.json @@ -1,36 +1,25 @@ { - "version": 2, - "created_at": "2026-05-27", - "last_updated_at": "2026-05-27", - "title": "Huntress SIEM collection errors detected", + "name": "Huntress SIEM collection errors detected", + "type": "query alert", + "query": "sum(last_30m):sum:huntress.siem.errors{*} by {error_type}.as_count() > 5", + "message": "The Huntress SIEM integration has recorded {{value}} errors in the last 30 minutes (threshold: {{threshold}}).\n\nInspect the error type using the `error_type` tag in Datadog Metrics Explorer:\n- `error_type:auth_failure` — check API credentials\n- `error_type:timeout` — query may be too broad; add a KEEP or WHERE clause\n- `error_type:invalid_query` — `esql_query` is malformed\n- `error_type:server_error` — transient Huntress API issue\n- `error_type:connection_error` — network connectivity problem\n\nAgent check logs:\n```\nsudo datadog-agent check huntress\n```", "tags": [ "integration:huntress" ], - "description": "The Huntress SIEM check emits `huntress.siem.errors` with an `error_type` tag whenever an API call fails. A small number of transient errors (rate limits, brief timeouts) is normal and automatically retried. A sustained elevated error count indicates a persistent problem such as repeated auth failures, server errors, or a consistently invalid query that is preventing log collection.", - "definition": { - "message": "The Huntress SIEM integration has recorded {{value}} errors in the last 30 minutes (threshold: {{threshold}}).\n\nInspect the error type using the `error_type` tag in Datadog Metrics Explorer:\n- `error_type:auth_failure` — check API credentials\n- `error_type:timeout` — query may be too broad; add a KEEP or WHERE clause\n- `error_type:invalid_query` — `esql_query` is malformed\n- `error_type:server_error` — transient Huntress API issue\n- `error_type:connection_error` — network connectivity problem\n\nAgent check logs:\n```\nsudo datadog-agent check huntress\n```\n\n@pagerduty @slack-security-alerts", - "name": "Huntress SIEM collection errors detected", - "options": { - "include_tags": true, - "new_group_delay": 60, - "notify_audit": false, - "on_missing_data": "show_no_data", - "renotify_interval": 60, - "renotify_statuses": [ - "alert" - ], - "require_full_window": false, - "thresholds": { - "critical": 5, - "warning": 2 - } + "options": { + "thresholds": { + "critical": 5, + "warning": 2 }, - "priority": 3, - "query": "sum(last_30m):sum:huntress.siem.errors{*}.as_count() > 5", - "restricted_roles": null, - "tags": [ - "integration:huntress" - ], - "type": "query alert" - } -} + "notify_audit": false, + "include_tags": true, + "on_missing_data": "show_no_data", + "renotify_interval": 0, + "require_full_window": false, + "escalation_message": "", + "new_group_delay": 0 + }, + "priority": 3, + "draft_status": "published", + "assets": [] +} \ No newline at end of file diff --git a/huntress/assets/monitors/huntress_siem_no_logs_collected.json b/huntress/assets/monitors/huntress_siem_no_logs_collected.json index 7fd51f1adc..17ef12d2d9 100644 --- a/huntress/assets/monitors/huntress_siem_no_logs_collected.json +++ b/huntress/assets/monitors/huntress_siem_no_logs_collected.json @@ -1,35 +1,24 @@ { - "version": 2, - "created_at": "2026-05-27", - "last_updated_at": "2026-05-27", - "title": "Huntress SIEM no logs collected", + "name": "Huntress SIEM no logs collected in 2 hours", + "type": "query alert", + "query": "sum(last_2h):sum:huntress.siem.logs_collected{*} by {query_name}.as_count() < 1", + "message": "The Huntress SIEM integration collected 0 logs in the last 2 hours.\n\nThis may be expected in a brand-new deployment or a quiet environment, but if you expect ongoing log flow, investigate:\n\n1. Verify the Agent check is running: `sudo datadog-agent check huntress`\n2. Confirm `esql_query` returns results in the Huntress Partner Portal\n3. Check that the Huntress account has active endpoints and SIEM log sources\n4. Review `huntress.siem.check_status` — if CRITICAL, the check itself is failing", "tags": [ "integration:huntress" ], - "description": "When the Huntress SIEM integration is healthy, it collects at least some logs each run — even in quiet environments, the Huntress API typically returns a small number of heartbeat or audit events. Zero logs collected over an extended period may indicate that the SIEM query window is returning no results, the Huntress account has no active log sources, or the check is silently failing without triggering the service check.", - "definition": { - "message": "The Huntress SIEM integration collected 0 logs in the last 2 hours.\n\nThis may be expected in a brand-new deployment or a quiet environment, but if you expect ongoing log flow, investigate:\n\n1. Verify the Agent check is running: `sudo datadog-agent check huntress`\n2. Confirm `esql_query` returns results in the Huntress Partner Portal\n3. Check that the Huntress account has active endpoints and SIEM log sources\n4. Review `huntress.siem.check_status` — if CRITICAL, the check itself is failing\n\n@slack-security-alerts", - "name": "Huntress SIEM no logs collected in 2 hours", - "options": { - "include_tags": true, - "new_group_delay": 300, - "notify_audit": false, - "on_missing_data": "show_no_data", - "renotify_interval": 120, - "renotify_statuses": [ - "alert" - ], - "require_full_window": true, - "thresholds": { - "critical": 1 - } + "options": { + "thresholds": { + "critical": 1 }, - "priority": 3, - "query": "sum(last_2h):sum:huntress.siem.logs_collected{*}.as_count() < 1", - "restricted_roles": null, - "tags": [ - "integration:huntress" - ], - "type": "query alert" - } -} + "notify_audit": false, + "include_tags": true, + "on_missing_data": "show_no_data", + "renotify_interval": 0, + "require_full_window": true, + "escalation_message": "", + "new_host_delay": 300 + }, + "priority": 3, + "draft_status": "published", + "assets": [] +} \ No newline at end of file diff --git a/huntress/checks.d/huntress.py b/huntress/checks.d/huntress.py index 9ba9d54ca9..9d2367f8e6 100644 --- a/huntress/checks.d/huntress.py +++ b/huntress/checks.d/huntress.py @@ -25,6 +25,7 @@ class HuntressCheck(AgentCheck): MAX_BATCH_SIZE_BYTES = 4 * 1024 * 1024 # 4 MB headroom under 5 MB limit DEFAULT_BASE_URL = "https://api.huntress.io" + DEFAULT_REQUEST_TIMEOUT = 30 DEFAULT_MIN_COLLECTION_INTERVAL = 900 DEFAULT_MAX_PAGES_PER_RUN = 100 DEFAULT_ORG_CACHE_TTL = 3600 @@ -40,33 +41,86 @@ def check(self, instance): api_key = instance.get("huntress_api_key", "").strip() secret_key = instance.get("huntress_secret_key", "").strip() - esql_query = instance.get("esql_query", "").strip() base_url = instance.get("huntress_base_url", self.DEFAULT_BASE_URL).rstrip("/") max_pages = int(instance.get("max_pages_per_run", self.DEFAULT_MAX_PAGES_PER_RUN)) min_interval = int(instance.get("min_collection_interval", self.DEFAULT_MIN_COLLECTION_INTERVAL)) enrich_orgs = instance.get("enrich_with_org_tags", True) org_ttl = int(instance.get("org_cache_ttl_seconds", self.DEFAULT_ORG_CACHE_TTL)) extra_tags = list(instance.get("tags", [])) + log_queries = instance.get("log_queries") or [] if not api_key: raise ConfigurationError("huntress_api_key is required") if not secret_key: raise ConfigurationError("huntress_secret_key is required") - if not esql_query: - raise ConfigurationError("esql_query is required") - if not esql_query.lower().lstrip().startswith("from logs"): - raise ConfigurationError("esql_query must begin with 'FROM logs' (case-insensitive)") + if not log_queries: + raise ConfigurationError("log_queries must contain at least one query") instance_hash = self._instance_hash(instance) headers = self._get_auth_header(api_key, secret_key) - # Org enrichment + # Org enrichment cache is shared across all queries for this instance org_cache = None if enrich_orgs: org_cache = self._get_or_refresh_org_cache(base_url, headers, instance_hash, org_ttl) - # Checkpoint - range_start = self._load_checkpoint(instance_hash) + total_logs = 0 + total_pages = 0 + success = False + + try: + for query_def in log_queries: + query_name = query_def.get("name", "").strip() + esql = query_def.get("esql_query", "").strip() + query_tags = list(query_def.get("tags", [])) + + if not query_name: + raise ConfigurationError("Each entry in log_queries must have a non-empty 'name'") + if not esql: + raise ConfigurationError("Each entry in log_queries must have a non-empty esql_query") + if not esql.lower().lstrip().startswith("from logs"): + raise ConfigurationError(f"esql_query must begin with 'FROM logs' (case-insensitive): {esql!r}") + + # Each query tracks its own checkpoint so queries are independently resumable + checkpoint_key = instance_hash + "_" + self._query_hash(esql) + all_tags = extra_tags + query_tags + query_metric_tags = extra_tags + [f"query_name:{query_name}"] + + logs, pages = self._run_query( + base_url, headers, esql, checkpoint_key, max_pages, min_interval, org_cache, all_tags + ) + total_logs += logs + total_pages += pages + + self.gauge("huntress.siem.logs_collected", logs, tags=query_metric_tags) + self.gauge("huntress.siem.pages_fetched", pages, tags=query_metric_tags) + + success = True + + except Exception as exc: + self.log.error("Huntress SIEM run failed: %s", exc) + self.count("huntress.siem.errors", 1, tags=extra_tags + ["error_type:run_failure"]) + self.service_check(self.SERVICE_CHECK_NAME, self.CRITICAL, tags=extra_tags) + raise + + finally: + duration = time.time() - start_time + self.gauge("huntress.siem.run_duration_seconds", duration, tags=extra_tags) + if self._last_api_call_limit is not None: + self.gauge("huntress.siem.api_call_limit", self._last_api_call_limit, tags=extra_tags) + if self._last_api_call_remaining is not None: + self.gauge("huntress.siem.api_call_remaining", self._last_api_call_remaining, tags=extra_tags) + + if success: + self.service_check(self.SERVICE_CHECK_NAME, self.OK, tags=extra_tags) + + # ------------------------------------------------------------------ # + # Per-query execution # + # ------------------------------------------------------------------ # + + def _run_query(self, base_url, headers, esql, checkpoint_key, max_pages, min_interval, org_cache, all_tags): + """Paginate a single ES|QL query. Returns (logs_collected, pages_fetched).""" + range_start = self._load_checkpoint(checkpoint_key) now = datetime.now(timezone.utc) range_end = now.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" @@ -74,7 +128,7 @@ def check(self, instance): default_start = now - timedelta(seconds=min_interval) range_start = default_start.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" else: - # Add 1ms offset to avoid re-fetching the last millisecond from previous run + # Add 1ms to avoid re-fetching the boundary event from the previous run try: dt = datetime.fromisoformat(range_start.replace("Z", "+00:00")) dt = dt + timedelta(milliseconds=1) @@ -82,73 +136,53 @@ def check(self, instance): except Exception: pass - self.log.debug("Huntress SIEM collection: range_start=%s range_end=%s", range_start, range_end) + self.log.debug( + "Huntress SIEM query: esql=%r range_start=%s range_end=%s", + esql[:80], + range_start, + range_end, + ) + service_tag = self._extract_service(all_tags) page_token = None total_logs = 0 pages_fetched = 0 hit_page_cap = False - success = False - try: - while True: - logs, next_token = self._query_page(base_url, headers, esql_query, range_start, range_end, page_token) - - if logs: - service_tag = self._extract_service(extra_tags) - batch = [] - for raw in logs: - org_tags = self._get_org_tags(raw, org_cache) if org_cache else [] - all_tags = extra_tags + org_tags - payload = self._transform_log(raw, all_tags, service_tag) - batch.append(payload) - if len(batch) >= self.MAX_LOGS_PER_BATCH: - self._send_logs_batch(batch) - batch = [] - if batch: + while True: + logs, next_token = self._query_page(base_url, headers, esql, range_start, range_end, page_token) + + if logs: + batch = [] + for raw in logs: + org_tags = self._get_org_tags(raw, org_cache) if org_cache else [] + payload = self._transform_log(raw, all_tags + org_tags, service_tag) + batch.append(payload) + if len(batch) >= self.MAX_LOGS_PER_BATCH: self._send_logs_batch(batch) + batch = [] + if batch: + self._send_logs_batch(batch) - total_logs += len(logs) - pages_fetched += 1 - - if next_token and pages_fetched < max_pages: - page_token = next_token - else: - if next_token and pages_fetched >= max_pages: - hit_page_cap = True - self.log.warning( - "Huntress SIEM: hit max_pages_per_run=%d cap; remaining pages will be collected next run", - max_pages, - ) - break - - if not hit_page_cap: - self._save_checkpoint(instance_hash, range_end) - - success = True + total_logs += len(logs) + pages_fetched += 1 - except Exception as exc: - self.log.error("Huntress SIEM run failed: %s", exc) - self.count( - "huntress.siem.errors", - 1, - tags=extra_tags + ["error_type:run_failure"], - ) - self.service_check(self.SERVICE_CHECK_NAME, self.CRITICAL, tags=extra_tags) - raise + if next_token and pages_fetched < max_pages: + page_token = next_token + else: + if next_token and pages_fetched >= max_pages: + hit_page_cap = True + self.log.warning( + "Huntress SIEM: hit max_pages_per_run=%d for query %r; remaining pages collected next run", + max_pages, + esql[:60], + ) + break - finally: - duration = time.time() - start_time - self.gauge("huntress.siem.run_duration_seconds", duration, tags=extra_tags) - self.gauge("huntress.siem.logs_collected", total_logs, tags=extra_tags) - self.gauge("huntress.siem.pages_fetched", pages_fetched, tags=extra_tags) - if self._last_api_call_limit is not None: - self.gauge("huntress.siem.api_call_limit", self._last_api_call_limit, tags=extra_tags) - if self._last_api_call_remaining is not None: - self.gauge("huntress.siem.api_call_remaining", self._last_api_call_remaining, tags=extra_tags) + if not hit_page_cap: + self._save_checkpoint(checkpoint_key, range_end) - if success: - self.service_check(self.SERVICE_CHECK_NAME, self.OK, tags=extra_tags) + return total_logs, pages_fetched # ------------------------------------------------------------------ # # Auth # @@ -163,7 +197,6 @@ def _get_auth_header(self, api_key, secret_key): } def _parse_rate_limit_headers(self, resp): - """Parse x-huntress-api-call-limit / x-huntress-api-call-remaining headers.""" try: limit = resp.headers.get("x-huntress-api-call-limit") if limit is not None: @@ -194,13 +227,7 @@ def _query_page(self, base_url, headers, esql, range_start, range_end, page_toke if page_token: body["page_token"] = page_token - response = self._request_with_retry( - method="POST", - url=url, - headers=headers, - json_body=body, - ) - + response = self._request_with_retry(method="POST", url=url, headers=headers, json_body=body) data = response.json() logs = data.get("logs", []) pagination = data.get("pagination", {}) @@ -213,6 +240,7 @@ def _query_page(self, base_url, headers, esql, range_start, range_end, page_toke def _request_with_retry(self, method, url, headers, json_body=None, params=None): """Execute an HTTP request with retry logic per PRD §7.""" + timeout = self.init_config.get("request_timeout", self.DEFAULT_REQUEST_TIMEOUT) max_retries_5xx = 3 max_retries_408 = 2 backoff_5xx = [5, 10, 20] @@ -228,7 +256,7 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) headers=headers, json=json_body, params=params, - timeout=30, + timeout=timeout, ) if resp.status_code == 200: @@ -353,7 +381,6 @@ def _transform_log(self, raw_log, tags, service): # ------------------------------------------------------------------ # def _send_logs_batch(self, logs_batch): - """Send a batch of up to MAX_LOGS_PER_BATCH log payloads via self.send_log().""" for log_payload in logs_batch: self.send_log(log_payload) @@ -361,8 +388,8 @@ def _send_logs_batch(self, logs_batch): # Checkpoint # # ------------------------------------------------------------------ # - def _load_checkpoint(self, instance_hash): - key = self.CHECKPOINT_CACHE_KEY_PREFIX + instance_hash + def _load_checkpoint(self, checkpoint_key): + key = self.CHECKPOINT_CACHE_KEY_PREFIX + checkpoint_key raw = self.read_persistent_cache(key) if not raw: return None @@ -372,8 +399,8 @@ def _load_checkpoint(self, instance_hash): except Exception: return None - def _save_checkpoint(self, instance_hash, timestamp_iso): - key = self.CHECKPOINT_CACHE_KEY_PREFIX + instance_hash + def _save_checkpoint(self, checkpoint_key, timestamp_iso): + key = self.CHECKPOINT_CACHE_KEY_PREFIX + checkpoint_key payload = json.dumps({"last_collected_at": timestamp_iso, "schema_version": 1}) self.write_persistent_cache(key, payload) @@ -498,9 +525,20 @@ def _get_org_tags(self, raw_log, org_cache): return base_tags # ------------------------------------------------------------------ # - # Instance hash # + # Hashing # # ------------------------------------------------------------------ # def _instance_hash(self, instance): - key = f"{instance.get('huntress_api_key', '')}:{instance.get('esql_query', '')}" - return hashlib.md5(key.encode()).hexdigest()[:12] + """Stable hash for a Huntress account (api key + secret + base url).""" + key = "|".join( + [ + instance.get("huntress_api_key", ""), + instance.get("huntress_secret_key", ""), + instance.get("huntress_base_url", self.DEFAULT_BASE_URL), + ] + ) + return hashlib.sha256(key.encode()).hexdigest()[:12] + + def _query_hash(self, esql_query): + """Short hash to build a per-query checkpoint key.""" + return hashlib.sha256(esql_query.encode()).hexdigest()[:8] diff --git a/huntress/datadog_checks/huntress/data/conf.yaml.example b/huntress/datadog_checks/huntress/data/conf.yaml.example index 4e99f4420a..61159be111 100644 --- a/huntress/datadog_checks/huntress/data/conf.yaml.example +++ b/huntress/datadog_checks/huntress/data/conf.yaml.example @@ -82,7 +82,8 @@ instances: ## automatically by the check. # log_queries: - - esql_query: FROM logs + - name: all-logs + esql_query: FROM logs ## @param enrich_with_org_tags - boolean - optional - default: true ## When true, the check fetches Huntress organization metadata (org name, diff --git a/huntress/datadog_checks/huntress/huntress.py b/huntress/datadog_checks/huntress/huntress.py index d806eb0300..9d2367f8e6 100644 --- a/huntress/datadog_checks/huntress/huntress.py +++ b/huntress/datadog_checks/huntress/huntress.py @@ -70,9 +70,12 @@ def check(self, instance): try: for query_def in log_queries: + query_name = query_def.get("name", "").strip() esql = query_def.get("esql_query", "").strip() query_tags = list(query_def.get("tags", [])) + if not query_name: + raise ConfigurationError("Each entry in log_queries must have a non-empty 'name'") if not esql: raise ConfigurationError("Each entry in log_queries must have a non-empty esql_query") if not esql.lower().lstrip().startswith("from logs"): @@ -81,6 +84,7 @@ def check(self, instance): # Each query tracks its own checkpoint so queries are independently resumable checkpoint_key = instance_hash + "_" + self._query_hash(esql) all_tags = extra_tags + query_tags + query_metric_tags = extra_tags + [f"query_name:{query_name}"] logs, pages = self._run_query( base_url, headers, esql, checkpoint_key, max_pages, min_interval, org_cache, all_tags @@ -88,6 +92,9 @@ def check(self, instance): total_logs += logs total_pages += pages + self.gauge("huntress.siem.logs_collected", logs, tags=query_metric_tags) + self.gauge("huntress.siem.pages_fetched", pages, tags=query_metric_tags) + success = True except Exception as exc: @@ -99,8 +106,6 @@ def check(self, instance): finally: duration = time.time() - start_time self.gauge("huntress.siem.run_duration_seconds", duration, tags=extra_tags) - self.gauge("huntress.siem.logs_collected", total_logs, tags=extra_tags) - self.gauge("huntress.siem.pages_fetched", total_pages, tags=extra_tags) if self._last_api_call_limit is not None: self.gauge("huntress.siem.api_call_limit", self._last_api_call_limit, tags=extra_tags) if self._last_api_call_remaining is not None: @@ -532,8 +537,8 @@ def _instance_hash(self, instance): instance.get("huntress_base_url", self.DEFAULT_BASE_URL), ] ) - return hashlib.md5(key.encode()).hexdigest()[:12] + return hashlib.sha256(key.encode()).hexdigest()[:12] def _query_hash(self, esql_query): """Short hash to build a per-query checkpoint key.""" - return hashlib.md5(esql_query.encode()).hexdigest()[:8] + return hashlib.sha256(esql_query.encode()).hexdigest()[:8] diff --git a/huntress/tests/conftest.py b/huntress/tests/conftest.py index 052649b65d..e8cc327f90 100644 --- a/huntress/tests/conftest.py +++ b/huntress/tests/conftest.py @@ -27,7 +27,7 @@ def dd_environment(): yield { "huntress_api_key": "hk_testpublickey", "huntress_secret_key": "hs_testsecretkey", - "log_queries": [{"esql_query": "FROM logs"}], + "log_queries": [{"name": "all-logs", "esql_query": "FROM logs"}], "huntress_base_url": f"http://{HOST}:{MOCKOON_PORT}", "enrich_with_org_tags": True, "org_cache_ttl_seconds": 3600, @@ -42,7 +42,7 @@ def instance(): return { "huntress_api_key": "test_api_key", "huntress_secret_key": "test_secret_key", - "log_queries": [{"esql_query": "FROM logs"}], + "log_queries": [{"name": "all-logs", "esql_query": "FROM logs"}], "enrich_with_org_tags": False, "min_collection_interval": 900, "max_pages_per_run": 100, diff --git a/huntress/tests/test_huntress.py b/huntress/tests/test_huntress.py index 2c5e577564..ece745a58d 100644 --- a/huntress/tests/test_huntress.py +++ b/huntress/tests/test_huntress.py @@ -25,7 +25,7 @@ def _make_instance(**kwargs): base = { "huntress_api_key": "pub_key", "huntress_secret_key": "secret_key", - "log_queries": [{"esql_query": esql}], + "log_queries": [{"name": "test-query", "esql_query": esql}], "enrich_with_org_tags": False, "tags": ["source:huntress", "env:test"], } diff --git a/huntress/tests/test_unit.py b/huntress/tests/test_unit.py index e34907e5d2..2ad322e049 100644 --- a/huntress/tests/test_unit.py +++ b/huntress/tests/test_unit.py @@ -13,7 +13,7 @@ def huntress_instance(): return { "huntress_api_key": "test_key", "huntress_secret_key": "test_secret", - "log_queries": [{"esql_query": "FROM logs"}], + "log_queries": [{"name": "all-logs", "esql_query": "FROM logs"}], "enrich_with_org_tags": False, } From a9ec4afe9423bf9d37c46fb141b01240bb7b126e Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Thu, 28 May 2026 03:32:55 -0500 Subject: [PATCH 11/40] Add escape hatch for hitting rate limit --- huntress/README.md | 65 +++++-- huntress/assets/configuration/spec.yaml | 22 ++- huntress/checks.d/huntress.py | 87 +++++++++- .../huntress/data/conf.yaml.example | 20 ++- huntress/datadog_checks/huntress/huntress.py | 87 +++++++++- huntress/metadata.csv | 5 + huntress/tests/test_huntress.py | 159 ++++++++++++++++++ 7 files changed, 408 insertions(+), 37 deletions(-) diff --git a/huntress/README.md b/huntress/README.md index e9e338a12a..755d8a45cf 100644 --- a/huntress/README.md +++ b/huntress/README.md @@ -105,12 +105,37 @@ instances: | ----------------------- | -------- | ------------------------- | ------------------------------------------------------------------------------ | | `huntress_api_key` | Yes | - | Huntress public API key | | `huntress_secret_key` | Yes | - | Huntress secret API key | -| `log_queries` | Yes | - | List of query objects; each has `name` (required), `esql_query` (required, must begin with `FROM logs`), and `tags` (optional). `name` is used as the `query_name` tag on metrics | -| `enrich_with_org_tags` | No | `true` | Fetch and attach org metadata as log tags | -| `org_cache_ttl_seconds` | No | `3600` | How long to cache org metadata (seconds) | -| `max_pages_per_run` | No | `100` | Page cap per query per run (~20,000 logs maximum) | -| `huntress_base_url` | No | `https://api.huntress.io` | Override for sandbox environments | -| `tags` | No | `[]` | Extra tags on every forwarded log | +| `log_queries` | No\* | - | List of query objects; each has `name` (required), `esql_query` (required, must begin with `FROM logs`), and `tags` (optional). `name` is used as the `query_name` tag on metrics | +| `metrics.agents.enabled` | No\* | `false` | Collect agent fleet metrics (total, by platform, by Defender/firewall status) | +| `metrics.agents.max_pages` | No | `20` | Max pages of agents to fetch per run (500 agents/page) | +| `enrich_with_org_tags` | No | `true` | Fetch and attach org metadata as log tags | +| `org_cache_ttl_seconds` | No | `3600` | How long to cache org metadata (seconds) | +| `max_pages_per_run` | No | `100` | Page cap per query per run (~20,000 logs maximum) | +| `huntress_base_url` | No | `https://api.huntress.io` | Override for sandbox environments | +| `tags` | No | `[]` | Extra tags on every forwarded metric and log | + +\* At least one of `log_queries` or `metrics.agents.enabled: true` must be configured per instance. + +### Rate limit considerations + +The Huntress API allows 60 requests per minute per API key pair. A typical run with 3 SIEM queries and agent metrics uses roughly 5–25 requests, well within this budget. For large accounts with high log volume or thousands of agents, consider splitting concerns across two instances using separate API key pairs: + +```yaml +instances: + # Instance 1: SIEM log collection only + - huntress_api_key: "" + huntress_secret_key: "" + log_queries: + - name: "all-logs" + esql_query: "FROM logs" + + # Instance 2: agent metrics only (isolated rate limit budget) + - huntress_api_key: "" + huntress_secret_key: "" + metrics: + agents: + enabled: true +``` ## Data Collected @@ -124,14 +149,26 @@ All logs collected from the Huntress Managed SIEM API are forwarded to Datadog w ### Metrics -| Metric | Type | Description | -| ------------------------------------ | ----- | ----------------------------------------------------- | -| `huntress.siem.logs_collected` | Gauge | Log events collected per run | -| `huntress.siem.pages_fetched` | Gauge | API pages fetched per run | -| `huntress.siem.run_duration_seconds` | Gauge | Wall time of the collection run | -| `huntress.siem.errors` | Count | Errors by type (`error_type` tag) | -| `huntress.siem.api_call_limit` | Gauge | Total API requests allowed per minute (from Huntress) | -| `huntress.siem.api_call_remaining` | Gauge | API requests remaining in the current minute | +**SIEM metrics** (always emitted per collection run): + +| Metric | Type | Tags | Description | +| ------------------------------------ | ----- | -------------------------- | ----------------------------------------------------- | +| `huntress.siem.logs_collected` | Gauge | `query_name:` | Log events collected per query per run | +| `huntress.siem.pages_fetched` | Gauge | `query_name:` | API pages fetched per query per run | +| `huntress.siem.run_duration_seconds` | Gauge | | Wall time of the full collection run | +| `huntress.siem.errors` | Count | `error_type:` | Errors by type | +| `huntress.siem.api_call_limit` | Gauge | | Total API requests allowed per minute (from Huntress) | +| `huntress.siem.api_call_remaining` | Gauge | | API requests remaining in the current minute | + +**Agent metrics** (emitted when `metrics.agents.enabled: true`): + +| Metric | Type | Tags | Description | +| --------------------------------- | ----- | ------------------- | ---------------------------------------------- | +| `huntress.agents.total` | Gauge | | Total agent count | +| `huntress.agents.count` | Gauge | `platform:` | Agent count by platform (windows/darwin/linux) | +| `huntress.agents.defender_status` | Gauge | `defender_status:` | Agent count by Defender AV status | +| `huntress.agents.firewall_status` | Gauge | `firewall_status:` | Agent count by firewall status | +| `huntress.agents.pages_fetched` | Gauge | | API pages used to fetch the agent list | See [metadata.csv][1] for a full list of metrics. diff --git a/huntress/assets/configuration/spec.yaml b/huntress/assets/configuration/spec.yaml index d335237261..ad3f55cb90 100644 --- a/huntress/assets/configuration/spec.yaml +++ b/huntress/assets/configuration/spec.yaml @@ -37,7 +37,6 @@ files: type: string example: - name: log_queries - required: true description: | List of ES|QL queries to execute against the Huntress SIEM API. Each entry requires an esql_query string (must begin with "FROM logs", case-insensitive) @@ -96,3 +95,24 @@ files: type: string example: "https://api.huntress.io" default: "https://api.huntress.io" + - name: metrics + description: | + Controls which Huntress data is collected as Datadog metrics beyond the + standard SIEM run metrics. Omit this section to disable additional metric + collection (logs-only instance). Configure a separate instance with only + this section and different API credentials to isolate the agent metrics + API call budget from SIEM log collection. + value: + type: object + properties: + - name: agents + type: object + properties: + - name: enabled + type: boolean + - name: max_pages + type: integer + example: + agents: + enabled: true + max_pages: 20 diff --git a/huntress/checks.d/huntress.py b/huntress/checks.d/huntress.py index 9d2367f8e6..edaf8fab64 100644 --- a/huntress/checks.d/huntress.py +++ b/huntress/checks.d/huntress.py @@ -17,6 +17,7 @@ class HuntressCheck(AgentCheck): HUNTRESS_SIEM_ENDPOINT = "/v1/siem/query" HUNTRESS_ACCOUNT_ENDPOINT = "/v1/account" HUNTRESS_ORGS_ENDPOINT = "/v1/accounts/{account_id}/organizations" + HUNTRESS_AGENTS_ENDPOINT = "/v1/agents" CHECKPOINT_CACHE_KEY_PREFIX = "huntress_last_collected_at_" ORG_CACHE_KEY_PREFIX = "huntress_org_cache_" @@ -29,6 +30,7 @@ class HuntressCheck(AgentCheck): DEFAULT_MIN_COLLECTION_INTERVAL = 900 DEFAULT_MAX_PAGES_PER_RUN = 100 DEFAULT_ORG_CACHE_TTL = 3600 + DEFAULT_AGENT_MAX_PAGES = 20 # 20 × 500/page = up to 10k agents SERVICE_CHECK_NAME = "huntress.siem.check_status" @@ -49,23 +51,29 @@ def check(self, instance): extra_tags = list(instance.get("tags", [])) log_queries = instance.get("log_queries") or [] + metrics_config = instance.get("metrics") or {} + agents_config = metrics_config.get("agents") or {} + collect_agents = bool(agents_config.get("enabled", False)) + agents_max_pages = int(agents_config.get("max_pages", self.DEFAULT_AGENT_MAX_PAGES)) + if not api_key: raise ConfigurationError("huntress_api_key is required") if not secret_key: raise ConfigurationError("huntress_secret_key is required") - if not log_queries: - raise ConfigurationError("log_queries must contain at least one query") + if not log_queries and not collect_agents: + raise ConfigurationError( + "Configure at least one of: log_queries (for SIEM log collection) " + "or metrics.agents.enabled: true (for agent metrics)" + ) instance_hash = self._instance_hash(instance) headers = self._get_auth_header(api_key, secret_key) # Org enrichment cache is shared across all queries for this instance org_cache = None - if enrich_orgs: + if enrich_orgs and log_queries: org_cache = self._get_or_refresh_org_cache(base_url, headers, instance_hash, org_ttl) - total_logs = 0 - total_pages = 0 success = False try: @@ -89,16 +97,17 @@ def check(self, instance): logs, pages = self._run_query( base_url, headers, esql, checkpoint_key, max_pages, min_interval, org_cache, all_tags ) - total_logs += logs - total_pages += pages self.gauge("huntress.siem.logs_collected", logs, tags=query_metric_tags) self.gauge("huntress.siem.pages_fetched", pages, tags=query_metric_tags) + if collect_agents: + self._collect_agent_metrics(base_url, headers, extra_tags, agents_max_pages) + success = True except Exception as exc: - self.log.error("Huntress SIEM run failed: %s", exc) + self.log.error("Huntress check run failed: %s", exc) self.count("huntress.siem.errors", 1, tags=extra_tags + ["error_type:run_failure"]) self.service_check(self.SERVICE_CHECK_NAME, self.CRITICAL, tags=extra_tags) raise @@ -115,7 +124,7 @@ def check(self, instance): self.service_check(self.SERVICE_CHECK_NAME, self.OK, tags=extra_tags) # ------------------------------------------------------------------ # - # Per-query execution # + # Per-query SIEM execution # # ------------------------------------------------------------------ # def _run_query(self, base_url, headers, esql, checkpoint_key, max_pages, min_interval, org_cache, all_tags): @@ -184,6 +193,66 @@ def _run_query(self, base_url, headers, esql, checkpoint_key, max_pages, min_int return total_logs, pages_fetched + # ------------------------------------------------------------------ # + # Agent metrics # + # ------------------------------------------------------------------ # + + def _collect_agent_metrics(self, base_url, headers, extra_tags, max_pages): + """Fetch all agents and emit platform/status counts as Datadog metrics.""" + agents = [] + page_token = None + pages_fetched = 0 + + while True: + params = {"limit": 500} + if page_token: + params["page_token"] = page_token + + url = base_url + self.HUNTRESS_AGENTS_ENDPOINT + resp = self._request_with_retry("GET", url, headers, params=params) + data = resp.json() + batch = data.get("agents", []) + agents.extend(batch) + pages_fetched += 1 + + pagination = data.get("pagination", {}) + next_token = pagination.get("next_page_token") + if next_token and pages_fetched < max_pages: + page_token = next_token + else: + if next_token and pages_fetched >= max_pages: + self.log.warning( + "Huntress agents: hit max_pages=%d; some agents may be excluded from this run's metrics", + max_pages, + ) + break + + by_platform = {} + by_defender_status = {} + by_firewall_status = {} + + for agent in agents: + platform = (agent.get("platform") or "unknown").lower() + by_platform[platform] = by_platform.get(platform, 0) + 1 + + def_status = (agent.get("defender_status") or "unknown").lower().replace(" ", "_") + by_defender_status[def_status] = by_defender_status.get(def_status, 0) + 1 + + fw_status = (agent.get("firewall_status") or "unknown").lower().replace(" ", "_") + by_firewall_status[fw_status] = by_firewall_status.get(fw_status, 0) + 1 + + self.gauge("huntress.agents.total", len(agents), tags=extra_tags) + self.gauge("huntress.agents.pages_fetched", pages_fetched, tags=extra_tags) + + for platform, count in by_platform.items(): + self.gauge("huntress.agents.count", count, tags=extra_tags + [f"platform:{platform}"]) + + for status, count in by_defender_status.items(): + self.gauge("huntress.agents.defender_status", count, tags=extra_tags + [f"defender_status:{status}"]) + + for status, count in by_firewall_status.items(): + self.gauge("huntress.agents.firewall_status", count, tags=extra_tags + [f"firewall_status:{status}"]) + # ------------------------------------------------------------------ # # Auth # # ------------------------------------------------------------------ # diff --git a/huntress/datadog_checks/huntress/data/conf.yaml.example b/huntress/datadog_checks/huntress/data/conf.yaml.example index 61159be111..d360300ffa 100644 --- a/huntress/datadog_checks/huntress/data/conf.yaml.example +++ b/huntress/datadog_checks/huntress/data/conf.yaml.example @@ -72,7 +72,7 @@ instances: # huntress_secret_key: - ## @param log_queries - list of mappings - required + ## @param log_queries - list of mappings - optional ## List of ES|QL queries to execute against the Huntress SIEM API. Each entry ## requires an esql_query string (must begin with "FROM logs", case-insensitive) ## and an optional tags list attached only to logs produced by that query. @@ -81,9 +81,9 @@ instances: ## Do not add time range filters — range_start and range_end are managed ## automatically by the check. # - log_queries: - - name: all-logs - esql_query: FROM logs + # log_queries: + # - name: all-logs + # esql_query: FROM logs ## @param enrich_with_org_tags - boolean - optional - default: true ## When true, the check fetches Huntress organization metadata (org name, @@ -110,3 +110,15 @@ instances: ## Huntress API base URL. Override for sandbox or testing environments. # # huntress_base_url: https://api.huntress.io + + ## @param metrics - mapping - optional + ## Controls which Huntress data is collected as Datadog metrics beyond the + ## standard SIEM run metrics. Omit this section to disable additional metric + ## collection (logs-only instance). Configure a separate instance with only + ## this section and different API credentials to isolate the agent metrics + ## API call budget from SIEM log collection. + # + # metrics: + # agents: + # enabled: true + # max_pages: 20 diff --git a/huntress/datadog_checks/huntress/huntress.py b/huntress/datadog_checks/huntress/huntress.py index 9d2367f8e6..edaf8fab64 100644 --- a/huntress/datadog_checks/huntress/huntress.py +++ b/huntress/datadog_checks/huntress/huntress.py @@ -17,6 +17,7 @@ class HuntressCheck(AgentCheck): HUNTRESS_SIEM_ENDPOINT = "/v1/siem/query" HUNTRESS_ACCOUNT_ENDPOINT = "/v1/account" HUNTRESS_ORGS_ENDPOINT = "/v1/accounts/{account_id}/organizations" + HUNTRESS_AGENTS_ENDPOINT = "/v1/agents" CHECKPOINT_CACHE_KEY_PREFIX = "huntress_last_collected_at_" ORG_CACHE_KEY_PREFIX = "huntress_org_cache_" @@ -29,6 +30,7 @@ class HuntressCheck(AgentCheck): DEFAULT_MIN_COLLECTION_INTERVAL = 900 DEFAULT_MAX_PAGES_PER_RUN = 100 DEFAULT_ORG_CACHE_TTL = 3600 + DEFAULT_AGENT_MAX_PAGES = 20 # 20 × 500/page = up to 10k agents SERVICE_CHECK_NAME = "huntress.siem.check_status" @@ -49,23 +51,29 @@ def check(self, instance): extra_tags = list(instance.get("tags", [])) log_queries = instance.get("log_queries") or [] + metrics_config = instance.get("metrics") or {} + agents_config = metrics_config.get("agents") or {} + collect_agents = bool(agents_config.get("enabled", False)) + agents_max_pages = int(agents_config.get("max_pages", self.DEFAULT_AGENT_MAX_PAGES)) + if not api_key: raise ConfigurationError("huntress_api_key is required") if not secret_key: raise ConfigurationError("huntress_secret_key is required") - if not log_queries: - raise ConfigurationError("log_queries must contain at least one query") + if not log_queries and not collect_agents: + raise ConfigurationError( + "Configure at least one of: log_queries (for SIEM log collection) " + "or metrics.agents.enabled: true (for agent metrics)" + ) instance_hash = self._instance_hash(instance) headers = self._get_auth_header(api_key, secret_key) # Org enrichment cache is shared across all queries for this instance org_cache = None - if enrich_orgs: + if enrich_orgs and log_queries: org_cache = self._get_or_refresh_org_cache(base_url, headers, instance_hash, org_ttl) - total_logs = 0 - total_pages = 0 success = False try: @@ -89,16 +97,17 @@ def check(self, instance): logs, pages = self._run_query( base_url, headers, esql, checkpoint_key, max_pages, min_interval, org_cache, all_tags ) - total_logs += logs - total_pages += pages self.gauge("huntress.siem.logs_collected", logs, tags=query_metric_tags) self.gauge("huntress.siem.pages_fetched", pages, tags=query_metric_tags) + if collect_agents: + self._collect_agent_metrics(base_url, headers, extra_tags, agents_max_pages) + success = True except Exception as exc: - self.log.error("Huntress SIEM run failed: %s", exc) + self.log.error("Huntress check run failed: %s", exc) self.count("huntress.siem.errors", 1, tags=extra_tags + ["error_type:run_failure"]) self.service_check(self.SERVICE_CHECK_NAME, self.CRITICAL, tags=extra_tags) raise @@ -115,7 +124,7 @@ def check(self, instance): self.service_check(self.SERVICE_CHECK_NAME, self.OK, tags=extra_tags) # ------------------------------------------------------------------ # - # Per-query execution # + # Per-query SIEM execution # # ------------------------------------------------------------------ # def _run_query(self, base_url, headers, esql, checkpoint_key, max_pages, min_interval, org_cache, all_tags): @@ -184,6 +193,66 @@ def _run_query(self, base_url, headers, esql, checkpoint_key, max_pages, min_int return total_logs, pages_fetched + # ------------------------------------------------------------------ # + # Agent metrics # + # ------------------------------------------------------------------ # + + def _collect_agent_metrics(self, base_url, headers, extra_tags, max_pages): + """Fetch all agents and emit platform/status counts as Datadog metrics.""" + agents = [] + page_token = None + pages_fetched = 0 + + while True: + params = {"limit": 500} + if page_token: + params["page_token"] = page_token + + url = base_url + self.HUNTRESS_AGENTS_ENDPOINT + resp = self._request_with_retry("GET", url, headers, params=params) + data = resp.json() + batch = data.get("agents", []) + agents.extend(batch) + pages_fetched += 1 + + pagination = data.get("pagination", {}) + next_token = pagination.get("next_page_token") + if next_token and pages_fetched < max_pages: + page_token = next_token + else: + if next_token and pages_fetched >= max_pages: + self.log.warning( + "Huntress agents: hit max_pages=%d; some agents may be excluded from this run's metrics", + max_pages, + ) + break + + by_platform = {} + by_defender_status = {} + by_firewall_status = {} + + for agent in agents: + platform = (agent.get("platform") or "unknown").lower() + by_platform[platform] = by_platform.get(platform, 0) + 1 + + def_status = (agent.get("defender_status") or "unknown").lower().replace(" ", "_") + by_defender_status[def_status] = by_defender_status.get(def_status, 0) + 1 + + fw_status = (agent.get("firewall_status") or "unknown").lower().replace(" ", "_") + by_firewall_status[fw_status] = by_firewall_status.get(fw_status, 0) + 1 + + self.gauge("huntress.agents.total", len(agents), tags=extra_tags) + self.gauge("huntress.agents.pages_fetched", pages_fetched, tags=extra_tags) + + for platform, count in by_platform.items(): + self.gauge("huntress.agents.count", count, tags=extra_tags + [f"platform:{platform}"]) + + for status, count in by_defender_status.items(): + self.gauge("huntress.agents.defender_status", count, tags=extra_tags + [f"defender_status:{status}"]) + + for status, count in by_firewall_status.items(): + self.gauge("huntress.agents.firewall_status", count, tags=extra_tags + [f"firewall_status:{status}"]) + # ------------------------------------------------------------------ # # Auth # # ------------------------------------------------------------------ # diff --git a/huntress/metadata.csv b/huntress/metadata.csv index 3326743cf0..d289e4ddc8 100644 --- a/huntress/metadata.csv +++ b/huntress/metadata.csv @@ -5,3 +5,8 @@ huntress.siem.run_duration_seconds,gauge,,second,,Wall time of the Huntress SIEM huntress.siem.errors,count,,,,Number of errors encountered during the Huntress SIEM collection run.,0,huntress,siem errors, huntress.siem.api_call_limit,gauge,,,,Huntress API rate limit (requests per minute) as reported by the x-huntress-api-call-limit response header.,0,huntress,api call limit, huntress.siem.api_call_remaining,gauge,,,,Remaining Huntress API requests in the current rate limit window as reported by the x-huntress-api-call-remaining response header.,0,huntress,api call remaining, +huntress.agents.total,gauge,,,,Total number of Huntress agents registered in the account.,0,huntress,agents total, +huntress.agents.count,gauge,,,,Number of Huntress agents broken down by platform tag (windows/darwin/linux).,0,huntress,agents count, +huntress.agents.defender_status,gauge,,,,Number of Huntress agents broken down by defender_status tag (healthy/unhealthy/etc).,0,huntress,agents defender status, +huntress.agents.firewall_status,gauge,,,,Number of Huntress agents broken down by firewall_status tag (enabled/disabled/isolated/etc).,0,huntress,agents firewall status, +huntress.agents.pages_fetched,gauge,,,,Number of API pages fetched to retrieve the full agent list in the last run.,0,huntress,agents pages fetched, diff --git a/huntress/tests/test_huntress.py b/huntress/tests/test_huntress.py index ece745a58d..f7be6c5de2 100644 --- a/huntress/tests/test_huntress.py +++ b/huntress/tests/test_huntress.py @@ -729,3 +729,162 @@ def fake_retry(method, url, headers, json_body=None, params=None): assert emitted_gauges.get("huntress.siem.api_call_limit") == 60 assert emitted_gauges.get("huntress.siem.api_call_remaining") == 48 + + +# =========================================================================== +# Configuration validation — neither log_queries nor metrics.agents +# =========================================================================== + + +def test_validation_neither_configured_raises(): + instance = { + "huntress_api_key": "pub_key", + "huntress_secret_key": "secret_key", + } + check = HuntressCheck("huntress", {}, [instance]) + check.log = MagicMock() + _cache = {} + check.read_persistent_cache = lambda key: _cache.get(key) + check.write_persistent_cache = lambda key, value: _cache.__setitem__(key, value) + with pytest.raises(Exception, match="at least one"): + check.check(instance) + + +def test_validation_metrics_only_instance(): + """An instance with no log_queries but metrics.agents.enabled=True is valid.""" + instance = { + "huntress_api_key": "pub_key", + "huntress_secret_key": "secret_key", + "metrics": {"agents": {"enabled": True}}, + } + check = HuntressCheck("huntress", {}, [instance]) + check.log = MagicMock() + _cache = {} + check.read_persistent_cache = lambda key: _cache.get(key) + check.write_persistent_cache = lambda key, value: _cache.__setitem__(key, value) + + agents_resp = _mock_response(200, {"agents": [], "pagination": {}}) + with ( + patch.object(check, "_request_with_retry", return_value=agents_resp), + patch("time.time", side_effect=[1000.0, 1001.0]), + ): + check.check(instance) # must not raise + + +# =========================================================================== +# Agent metrics +# =========================================================================== + + +def _make_agents_instance(**kwargs): + base = { + "huntress_api_key": "pub_key", + "huntress_secret_key": "secret_key", + "metrics": {"agents": {"enabled": True}}, + "tags": ["source:huntress"], + } + base.update(kwargs) + return base + + +def test_agent_metrics_emitted(): + """_collect_agent_metrics emits total, per-platform, per-status gauges.""" + instance = _make_agents_instance() + check = HuntressCheck("huntress", {}, [instance]) + check.log = MagicMock() + _cache = {} + check.read_persistent_cache = lambda key: _cache.get(key) + check.write_persistent_cache = lambda key, value: _cache.__setitem__(key, value) + + agents = [ + {"platform": "windows", "defender_status": "Healthy", "firewall_status": "Enabled"}, + {"platform": "windows", "defender_status": "Healthy", "firewall_status": "Disabled"}, + {"platform": "darwin", "defender_status": None, "firewall_status": "Enabled"}, + ] + agents_resp = _mock_response(200, {"agents": agents, "pagination": {}}) + + emitted = {} + + def capture_gauge(name, value, tags=None): + emitted[(name, tuple(sorted(tags or [])))] = value + + with ( + patch.object(check, "_request_with_retry", return_value=agents_resp), + patch.object(check, "gauge", side_effect=capture_gauge), + patch("time.time", side_effect=[1000.0, 1001.0]), + ): + check.check(instance) + + assert emitted[("huntress.agents.total", ("source:huntress",))] == 3 + assert emitted[("huntress.agents.count", ("platform:windows", "source:huntress"))] == 2 + assert emitted[("huntress.agents.count", ("platform:darwin", "source:huntress"))] == 1 + assert emitted[("huntress.agents.defender_status", ("defender_status:healthy", "source:huntress"))] == 2 + assert emitted[("huntress.agents.defender_status", ("defender_status:unknown", "source:huntress"))] == 1 + assert emitted[("huntress.agents.firewall_status", ("firewall_status:enabled", "source:huntress"))] == 2 + assert emitted[("huntress.agents.firewall_status", ("firewall_status:disabled", "source:huntress"))] == 1 + + +def test_agent_metrics_pagination(): + """Agent collection paginates until no next_page_token.""" + instance = _make_agents_instance(metrics={"agents": {"enabled": True, "max_pages": 5}}) + check = HuntressCheck("huntress", {}, [instance]) + check.log = MagicMock() + _cache = {} + check.read_persistent_cache = lambda key: _cache.get(key) + check.write_persistent_cache = lambda key, value: _cache.__setitem__(key, value) + + page1 = { + "agents": [{"platform": "windows", "defender_status": "Healthy", "firewall_status": "Enabled"}], + "pagination": {"next_page_token": "tok2"}, + } + page2 = { + "agents": [{"platform": "linux", "defender_status": "Healthy", "firewall_status": "Enabled"}], + "pagination": {}, + } + + responses = [_mock_response(200, page1), _mock_response(200, page2)] + call_urls = [] + + def side_effect(method, url, headers, json_body=None, params=None): + call_urls.append(url) + return responses.pop(0) + + emitted = {} + + def capture_gauge(name, value, tags=None): + emitted[(name, tuple(sorted(tags or [])))] = value + + with ( + patch.object(check, "_request_with_retry", side_effect=side_effect), + patch.object(check, "gauge", side_effect=capture_gauge), + patch("time.time", side_effect=[1000.0, 1001.0]), + ): + check.check(instance) + + assert emitted[("huntress.agents.total", ("source:huntress",))] == 2 + assert emitted[("huntress.agents.count", ("platform:windows", "source:huntress"))] == 1 + assert emitted[("huntress.agents.count", ("platform:linux", "source:huntress"))] == 1 + assert len(call_urls) == 2 + + +def test_agent_metrics_max_pages_cap(): + """Agent collection stops at max_pages even when more pages exist.""" + instance = _make_agents_instance(metrics={"agents": {"enabled": True, "max_pages": 1}}) + check = HuntressCheck("huntress", {}, [instance]) + check.log = MagicMock() + _cache = {} + check.read_persistent_cache = lambda key: _cache.get(key) + check.write_persistent_cache = lambda key, value: _cache.__setitem__(key, value) + + page_with_more = { + "agents": [{"platform": "windows", "defender_status": "Healthy", "firewall_status": "Enabled"}], + "pagination": {"next_page_token": "there_is_more"}, + } + + with ( + patch.object(check, "_request_with_retry", return_value=_mock_response(200, page_with_more)), + patch("time.time", side_effect=[1000.0, 1001.0]), + ): + check.check(instance) + + check.log.warning.assert_called() From f1c4d69431ab39e4e44cf7edb46091661787fb07 Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Fri, 29 May 2026 17:05:58 -0500 Subject: [PATCH 12/40] Fix monitor schema --- .../monitors/huntress_siem_check_failed.json | 53 +++++++++++-------- .../monitors/huntress_siem_errors_high.json | 48 ++++++++++------- .../huntress_siem_no_logs_collected.json | 46 +++++++++------- 3 files changed, 88 insertions(+), 59 deletions(-) diff --git a/huntress/assets/monitors/huntress_siem_check_failed.json b/huntress/assets/monitors/huntress_siem_check_failed.json index 96d78f2538..4e255f6694 100644 --- a/huntress/assets/monitors/huntress_siem_check_failed.json +++ b/huntress/assets/monitors/huntress_siem_check_failed.json @@ -1,28 +1,37 @@ { - "id": 19780007, - "name": "Huntress SIEM collection run failed on {{host.name}}", - "type": "service check", - "query": "\"huntress.siem.check_status\".over(\"*\").by(\"error_type\").last(2).count_by_status()", - "message": "The Huntress SIEM Agent check on {{host.name}} has entered a CRITICAL state, meaning the last collection run failed. Logs may no longer be flowing from Huntress to Datadog.\n\nCommon causes:\n- Invalid or rotated API credentials (`huntress_api_key` / `huntress_secret_key`)\n- Network connectivity issue between the Agent host and `api.huntress.io`\n- Invalid `esql_query` in the instance configuration\n- Huntress SIEM feature not enabled on this account\n\nCheck the Agent logs for details:\n```\nsudo datadog-agent check huntress\n```", + "version": 2, + "created_at": "2026-05-29", + "last_updated_at": "2026-05-29", + "title": "Huntress SIEM collection run failed", "tags": [ "integration:huntress" ], - "options": { - "thresholds": { - "critical": 1, - "ok": 1 - }, - "notify_audit": false, - "renotify_interval": 60, - "timeout_h": 0, - "include_tags": true, - "renotify_statuses": [ - "alert" + "description": "The Huntress SIEM Agent check has entered a CRITICAL state, meaning the last collection run failed. Logs may no longer be flowing from Huntress to Datadog.", + "definition": { + "name": "Huntress SIEM collection run failed on {{host.name}}", + "type": "service check", + "query": "\"huntress.siem.check_status\".over(\"*\").by(\"error_type\").last(2).count_by_status()", + "message": "The Huntress SIEM Agent check on {{host.name}} has entered a CRITICAL state, meaning the last collection run failed. Logs may no longer be flowing from Huntress to Datadog.\n\nCommon causes:\n- Invalid or rotated API credentials (`huntress_api_key` / `huntress_secret_key`)\n- Network connectivity issue between the Agent host and `api.huntress.io`\n- Invalid `esql_query` in the instance configuration\n- Huntress SIEM feature not enabled on this account\n\nCheck the Agent logs for details:\n```\nsudo datadog-agent check huntress\n```", + "tags": [ + "integration:huntress" ], - "require_full_window": false, - "escalation_message": "" - }, - "priority": 2, - "draft_status": "published", - "assets": [] + "options": { + "thresholds": { + "critical": 1, + "ok": 1 + }, + "notify_audit": false, + "renotify_interval": 60, + "timeout_h": 0, + "include_tags": true, + "renotify_statuses": [ + "alert" + ], + "require_full_window": false, + "escalation_message": "" + }, + "priority": 2, + "draft_status": "published", + "assets": [] + } } \ No newline at end of file diff --git a/huntress/assets/monitors/huntress_siem_errors_high.json b/huntress/assets/monitors/huntress_siem_errors_high.json index d78bf36023..3c9250330c 100644 --- a/huntress/assets/monitors/huntress_siem_errors_high.json +++ b/huntress/assets/monitors/huntress_siem_errors_high.json @@ -1,25 +1,35 @@ { - "name": "Huntress SIEM collection errors detected", - "type": "query alert", - "query": "sum(last_30m):sum:huntress.siem.errors{*} by {error_type}.as_count() > 5", - "message": "The Huntress SIEM integration has recorded {{value}} errors in the last 30 minutes (threshold: {{threshold}}).\n\nInspect the error type using the `error_type` tag in Datadog Metrics Explorer:\n- `error_type:auth_failure` — check API credentials\n- `error_type:timeout` — query may be too broad; add a KEEP or WHERE clause\n- `error_type:invalid_query` — `esql_query` is malformed\n- `error_type:server_error` — transient Huntress API issue\n- `error_type:connection_error` — network connectivity problem\n\nAgent check logs:\n```\nsudo datadog-agent check huntress\n```", + "version": 2, + "created_at": "2026-05-29", + "last_updated_at": "2026-05-29", + "title": "Huntress SIEM collection errors detected", "tags": [ "integration:huntress" ], - "options": { - "thresholds": { - "critical": 5, - "warning": 2 + "description": "The Huntress SIEM integration has recorded errors in the last 30 minutes.", + "definition": { + "name": "Huntress SIEM collection errors detected", + "type": "query alert", + "query": "sum(last_30m):sum:huntress.siem.errors{*} by {error_type}.as_count() > 5", + "message": "The Huntress SIEM integration has recorded {{value}} errors in the last 30 minutes (threshold: {{threshold}}).\n\nInspect the error type using the `error_type` tag in Datadog Metrics Explorer:\n- `error_type:auth_failure` — check API credentials\n- `error_type:timeout` — query may be too broad; add a KEEP or WHERE clause\n- `error_type:invalid_query` — `esql_query` is malformed\n- `error_type:server_error` — transient Huntress API issue\n- `error_type:connection_error` — network connectivity problem\n\nAgent check logs:\n```\nsudo datadog-agent check huntress\n```", + "tags": [ + "integration:huntress" + ], + "options": { + "thresholds": { + "critical": 5, + "warning": 2 + }, + "notify_audit": false, + "include_tags": true, + "on_missing_data": "show_no_data", + "renotify_interval": 0, + "require_full_window": false, + "escalation_message": "", + "new_group_delay": 0 }, - "notify_audit": false, - "include_tags": true, - "on_missing_data": "show_no_data", - "renotify_interval": 0, - "require_full_window": false, - "escalation_message": "", - "new_group_delay": 0 - }, - "priority": 3, - "draft_status": "published", - "assets": [] + "priority": 3, + "draft_status": "published", + "assets": [] + } } \ No newline at end of file diff --git a/huntress/assets/monitors/huntress_siem_no_logs_collected.json b/huntress/assets/monitors/huntress_siem_no_logs_collected.json index 17ef12d2d9..35cab53dab 100644 --- a/huntress/assets/monitors/huntress_siem_no_logs_collected.json +++ b/huntress/assets/monitors/huntress_siem_no_logs_collected.json @@ -1,24 +1,34 @@ { - "name": "Huntress SIEM no logs collected in 2 hours", - "type": "query alert", - "query": "sum(last_2h):sum:huntress.siem.logs_collected{*} by {query_name}.as_count() < 1", - "message": "The Huntress SIEM integration collected 0 logs in the last 2 hours.\n\nThis may be expected in a brand-new deployment or a quiet environment, but if you expect ongoing log flow, investigate:\n\n1. Verify the Agent check is running: `sudo datadog-agent check huntress`\n2. Confirm `esql_query` returns results in the Huntress Partner Portal\n3. Check that the Huntress account has active endpoints and SIEM log sources\n4. Review `huntress.siem.check_status` — if CRITICAL, the check itself is failing", + "version": 2, + "created_at": "2026-05-29", + "last_updated_at": "2026-05-29", + "title": "Huntress SIEM no logs collected in 2 hours", "tags": [ "integration:huntress" ], - "options": { - "thresholds": { - "critical": 1 + "description": "The Huntress SIEM integration collected 0 logs in the last 2 hours.", + "definition": { + "name": "Huntress SIEM no logs collected in 2 hours", + "type": "query alert", + "query": "sum(last_2h):sum:huntress.siem.logs_collected{*} by {query_name}.as_count() < 1", + "message": "The Huntress SIEM integration collected 0 logs in the last 2 hours.\n\nThis may be expected in a brand-new deployment or a quiet environment, but if you expect ongoing log flow, investigate:\n\n1. Verify the Agent check is running: `sudo datadog-agent check huntress`\n2. Confirm `esql_query` returns results in the Huntress Partner Portal\n3. Check that the Huntress account has active endpoints and SIEM log sources\n4. Review `huntress.siem.check_status` — if CRITICAL, the check itself is failing", + "tags": [ + "integration:huntress" + ], + "options": { + "thresholds": { + "critical": 1 + }, + "notify_audit": false, + "include_tags": true, + "on_missing_data": "show_no_data", + "renotify_interval": 0, + "require_full_window": true, + "escalation_message": "", + "new_host_delay": 300 }, - "notify_audit": false, - "include_tags": true, - "on_missing_data": "show_no_data", - "renotify_interval": 0, - "require_full_window": true, - "escalation_message": "", - "new_host_delay": 300 - }, - "priority": 3, - "draft_status": "published", - "assets": [] + "priority": 3, + "draft_status": "published", + "assets": [] + } } \ No newline at end of file From 5d97e9d629d683fd0778775f86e7bc06c224ceaa Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Fri, 29 May 2026 17:17:37 -0500 Subject: [PATCH 13/40] Fix ascii validation --- huntress/README.md | 74 +++++++++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/huntress/README.md b/huntress/README.md index 755d8a45cf..e717faa456 100644 --- a/huntress/README.md +++ b/huntress/README.md @@ -95,30 +95,30 @@ instances: **`init_config` options** (apply to all instances): -| Field | Required | Default | Description | -| ----------------- | -------- | ------- | ----------------------------------------------- | +| Field | Required | Default | Description | +| ----------------- | -------- | ------- | ------------------------------------------------- | | `request_timeout` | No | `30` | HTTP request timeout in seconds for all API calls | **`instances` options** (per Huntress account): -| Field | Required | Default | Description | -| ----------------------- | -------- | ------------------------- | ------------------------------------------------------------------------------ | -| `huntress_api_key` | Yes | - | Huntress public API key | -| `huntress_secret_key` | Yes | - | Huntress secret API key | -| `log_queries` | No\* | - | List of query objects; each has `name` (required), `esql_query` (required, must begin with `FROM logs`), and `tags` (optional). `name` is used as the `query_name` tag on metrics | -| `metrics.agents.enabled` | No\* | `false` | Collect agent fleet metrics (total, by platform, by Defender/firewall status) | -| `metrics.agents.max_pages` | No | `20` | Max pages of agents to fetch per run (500 agents/page) | -| `enrich_with_org_tags` | No | `true` | Fetch and attach org metadata as log tags | -| `org_cache_ttl_seconds` | No | `3600` | How long to cache org metadata (seconds) | -| `max_pages_per_run` | No | `100` | Page cap per query per run (~20,000 logs maximum) | -| `huntress_base_url` | No | `https://api.huntress.io` | Override for sandbox environments | -| `tags` | No | `[]` | Extra tags on every forwarded metric and log | +| Field | Required | Default | Description | +| -------------------------- | -------- | ------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `huntress_api_key` | Yes | - | Huntress public API key | +| `huntress_secret_key` | Yes | - | Huntress secret API key | +| `log_queries` | No\* | - | List of query objects; each has `name` (required), `esql_query` (required, must begin with `FROM logs`), and `tags` (optional). `name` is used as the `query_name` tag on metrics | +| `metrics.agents.enabled` | No\* | `false` | Collect agent fleet metrics (total, by platform, by Defender/firewall status) | +| `metrics.agents.max_pages` | No | `20` | Max pages of agents to fetch per run (500 agents/page) | +| `enrich_with_org_tags` | No | `true` | Fetch and attach org metadata as log tags | +| `org_cache_ttl_seconds` | No | `3600` | How long to cache org metadata (seconds) | +| `max_pages_per_run` | No | `100` | Page cap per query per run (~20,000 logs maximum) | +| `huntress_base_url` | No | `https://api.huntress.io` | Override for sandbox environments | +| `tags` | No | `[]` | Extra tags on every forwarded metric and log | \* At least one of `log_queries` or `metrics.agents.enabled: true` must be configured per instance. ### Rate limit considerations -The Huntress API allows 60 requests per minute per API key pair. A typical run with 3 SIEM queries and agent metrics uses roughly 5–25 requests, well within this budget. For large accounts with high log volume or thousands of agents, consider splitting concerns across two instances using separate API key pairs: +The Huntress API allows 60 requests per minute per API key pair. A typical run with 3 SIEM queries and agent metrics uses roughly 5 - 25 requests, well within this budget. For large accounts with high log volume or thousands of agents, consider splitting concerns across two instances using separate API key pairs: ```yaml instances: @@ -151,24 +151,24 @@ All logs collected from the Huntress Managed SIEM API are forwarded to Datadog w **SIEM metrics** (always emitted per collection run): -| Metric | Type | Tags | Description | -| ------------------------------------ | ----- | -------------------------- | ----------------------------------------------------- | -| `huntress.siem.logs_collected` | Gauge | `query_name:` | Log events collected per query per run | -| `huntress.siem.pages_fetched` | Gauge | `query_name:` | API pages fetched per query per run | -| `huntress.siem.run_duration_seconds` | Gauge | | Wall time of the full collection run | -| `huntress.siem.errors` | Count | `error_type:` | Errors by type | -| `huntress.siem.api_call_limit` | Gauge | | Total API requests allowed per minute (from Huntress) | -| `huntress.siem.api_call_remaining` | Gauge | | API requests remaining in the current minute | +| Metric | Type | Tags | Description | +| ------------------------------------ | ----- | ------------- | ----------------------------------------------------- | +| `huntress.siem.logs_collected` | Gauge | `query_name:` | Log events collected per query per run | +| `huntress.siem.pages_fetched` | Gauge | `query_name:` | API pages fetched per query per run | +| `huntress.siem.run_duration_seconds` | Gauge | | Wall time of the full collection run | +| `huntress.siem.errors` | Count | `error_type:` | Errors by type | +| `huntress.siem.api_call_limit` | Gauge | | Total API requests allowed per minute (from Huntress) | +| `huntress.siem.api_call_remaining` | Gauge | | API requests remaining in the current minute | **Agent metrics** (emitted when `metrics.agents.enabled: true`): -| Metric | Type | Tags | Description | -| --------------------------------- | ----- | ------------------- | ---------------------------------------------- | -| `huntress.agents.total` | Gauge | | Total agent count | -| `huntress.agents.count` | Gauge | `platform:` | Agent count by platform (windows/darwin/linux) | -| `huntress.agents.defender_status` | Gauge | `defender_status:` | Agent count by Defender AV status | -| `huntress.agents.firewall_status` | Gauge | `firewall_status:` | Agent count by firewall status | -| `huntress.agents.pages_fetched` | Gauge | | API pages used to fetch the agent list | +| Metric | Type | Tags | Description | +| --------------------------------- | ----- | ------------------ | ---------------------------------------------- | +| `huntress.agents.total` | Gauge | | Total agent count | +| `huntress.agents.count` | Gauge | `platform:` | Agent count by platform (windows/darwin/linux) | +| `huntress.agents.defender_status` | Gauge | `defender_status:` | Agent count by Defender AV status | +| `huntress.agents.firewall_status` | Gauge | `firewall_status:` | Agent count by firewall status | +| `huntress.agents.pages_fetched` | Gauge | | API pages used to fetch the agent list | See [metadata.csv][1] for a full list of metrics. @@ -194,14 +194,14 @@ Returns `CRITICAL` if the collection run fails for any reason; `OK` otherwise. Inspect the `error_type` tag to identify the root cause: -| `error_type` | Cause | Resolution | -| ------------------ | ---------------------------------- | ------------------------------------------------------------ | -| `auth_failure` | Invalid or rotated API credentials | Update `huntress_api_key` / `huntress_secret_key` | -| `timeout` | ES\|QL query too broad | Add a `KEEP` or `WHERE` clause to the query | +| `error_type` | Cause | Resolution | +| ------------------ | ---------------------------------- | ------------------------------------------------------------- | +| `auth_failure` | Invalid or rotated API credentials | Update `huntress_api_key` / `huntress_secret_key` | +| `timeout` | ES\|QL query too broad | Add a `KEEP` or `WHERE` clause to the query | | `invalid_query` | Malformed ES\|QL | Fix the `esql_query` value in the failing `log_queries` entry | -| `server_error` | Transient Huntress API error | Check [Huntress status page](https://status.huntress.com) | -| `connection_error` | Network issue | Verify connectivity from the Agent host to `api.huntress.io` | -| `run_failure` | Unexpected error during collection | Check Agent logs for the full stack trace | +| `server_error` | Transient Huntress API error | Check [Huntress status page](https://status.huntress.com) | +| `connection_error` | Network issue | Verify connectivity from the Agent host to `api.huntress.io` | +| `run_failure` | Unexpected error during collection | Check Agent logs for the full stack trace | **`huntress.siem.api_call_remaining` is very low or zero** From 157d3d6d6913ff751ccef4a5360e97c07b5c041d Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Fri, 29 May 2026 17:25:14 -0500 Subject: [PATCH 14/40] Drop secret from hash --- huntress/checks.d/huntress.py | 174 ++++++++++++------- huntress/datadog_checks/huntress/huntress.py | 174 ++++++++++++------- 2 files changed, 230 insertions(+), 118 deletions(-) diff --git a/huntress/checks.d/huntress.py b/huntress/checks.d/huntress.py index edaf8fab64..391652fc90 100644 --- a/huntress/checks.d/huntress.py +++ b/huntress/checks.d/huntress.py @@ -43,18 +43,23 @@ def check(self, instance): api_key = instance.get("huntress_api_key", "").strip() secret_key = instance.get("huntress_secret_key", "").strip() - base_url = instance.get("huntress_base_url", self.DEFAULT_BASE_URL).rstrip("/") - max_pages = int(instance.get("max_pages_per_run", self.DEFAULT_MAX_PAGES_PER_RUN)) - min_interval = int(instance.get("min_collection_interval", self.DEFAULT_MIN_COLLECTION_INTERVAL)) + base_url = instance.get("huntress_base_url", + self.DEFAULT_BASE_URL).rstrip("/") + max_pages = int(instance.get("max_pages_per_run", + self.DEFAULT_MAX_PAGES_PER_RUN)) + min_interval = int(instance.get( + "min_collection_interval", self.DEFAULT_MIN_COLLECTION_INTERVAL)) enrich_orgs = instance.get("enrich_with_org_tags", True) - org_ttl = int(instance.get("org_cache_ttl_seconds", self.DEFAULT_ORG_CACHE_TTL)) + org_ttl = int(instance.get("org_cache_ttl_seconds", + self.DEFAULT_ORG_CACHE_TTL)) extra_tags = list(instance.get("tags", [])) log_queries = instance.get("log_queries") or [] metrics_config = instance.get("metrics") or {} agents_config = metrics_config.get("agents") or {} collect_agents = bool(agents_config.get("enabled", False)) - agents_max_pages = int(agents_config.get("max_pages", self.DEFAULT_AGENT_MAX_PAGES)) + agents_max_pages = int(agents_config.get( + "max_pages", self.DEFAULT_AGENT_MAX_PAGES)) if not api_key: raise ConfigurationError("huntress_api_key is required") @@ -72,7 +77,8 @@ def check(self, instance): # Org enrichment cache is shared across all queries for this instance org_cache = None if enrich_orgs and log_queries: - org_cache = self._get_or_refresh_org_cache(base_url, headers, instance_hash, org_ttl) + org_cache = self._get_or_refresh_org_cache( + base_url, headers, instance_hash, org_ttl) success = False @@ -83,11 +89,14 @@ def check(self, instance): query_tags = list(query_def.get("tags", [])) if not query_name: - raise ConfigurationError("Each entry in log_queries must have a non-empty 'name'") + raise ConfigurationError( + "Each entry in log_queries must have a non-empty 'name'") if not esql: - raise ConfigurationError("Each entry in log_queries must have a non-empty esql_query") + raise ConfigurationError( + "Each entry in log_queries must have a non-empty esql_query") if not esql.lower().lstrip().startswith("from logs"): - raise ConfigurationError(f"esql_query must begin with 'FROM logs' (case-insensitive): {esql!r}") + raise ConfigurationError( + f"esql_query must begin with 'FROM logs' (case-insensitive): {esql!r}") # Each query tracks its own checkpoint so queries are independently resumable checkpoint_key = instance_hash + "_" + self._query_hash(esql) @@ -98,30 +107,39 @@ def check(self, instance): base_url, headers, esql, checkpoint_key, max_pages, min_interval, org_cache, all_tags ) - self.gauge("huntress.siem.logs_collected", logs, tags=query_metric_tags) - self.gauge("huntress.siem.pages_fetched", pages, tags=query_metric_tags) + self.gauge("huntress.siem.logs_collected", + logs, tags=query_metric_tags) + self.gauge("huntress.siem.pages_fetched", + pages, tags=query_metric_tags) if collect_agents: - self._collect_agent_metrics(base_url, headers, extra_tags, agents_max_pages) + self._collect_agent_metrics( + base_url, headers, extra_tags, agents_max_pages) success = True except Exception as exc: self.log.error("Huntress check run failed: %s", exc) - self.count("huntress.siem.errors", 1, tags=extra_tags + ["error_type:run_failure"]) - self.service_check(self.SERVICE_CHECK_NAME, self.CRITICAL, tags=extra_tags) + self.count("huntress.siem.errors", 1, + tags=extra_tags + ["error_type:run_failure"]) + self.service_check(self.SERVICE_CHECK_NAME, + self.CRITICAL, tags=extra_tags) raise finally: duration = time.time() - start_time - self.gauge("huntress.siem.run_duration_seconds", duration, tags=extra_tags) + self.gauge("huntress.siem.run_duration_seconds", + duration, tags=extra_tags) if self._last_api_call_limit is not None: - self.gauge("huntress.siem.api_call_limit", self._last_api_call_limit, tags=extra_tags) + self.gauge("huntress.siem.api_call_limit", + self._last_api_call_limit, tags=extra_tags) if self._last_api_call_remaining is not None: - self.gauge("huntress.siem.api_call_remaining", self._last_api_call_remaining, tags=extra_tags) + self.gauge("huntress.siem.api_call_remaining", + self._last_api_call_remaining, tags=extra_tags) if success: - self.service_check(self.SERVICE_CHECK_NAME, self.OK, tags=extra_tags) + self.service_check(self.SERVICE_CHECK_NAME, + self.OK, tags=extra_tags) # ------------------------------------------------------------------ # # Per-query SIEM execution # @@ -135,7 +153,8 @@ def _run_query(self, base_url, headers, esql, checkpoint_key, max_pages, min_int if range_start is None: default_start = now - timedelta(seconds=min_interval) - range_start = default_start.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + range_start = default_start.strftime( + "%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" else: # Add 1ms to avoid re-fetching the boundary event from the previous run try: @@ -159,13 +178,16 @@ def _run_query(self, base_url, headers, esql, checkpoint_key, max_pages, min_int hit_page_cap = False while True: - logs, next_token = self._query_page(base_url, headers, esql, range_start, range_end, page_token) + logs, next_token = self._query_page( + base_url, headers, esql, range_start, range_end, page_token) if logs: batch = [] for raw in logs: - org_tags = self._get_org_tags(raw, org_cache) if org_cache else [] - payload = self._transform_log(raw, all_tags + org_tags, service_tag) + org_tags = self._get_org_tags( + raw, org_cache) if org_cache else [] + payload = self._transform_log( + raw, all_tags + org_tags, service_tag) batch.append(payload) if len(batch) >= self.MAX_LOGS_PER_BATCH: self._send_logs_batch(batch) @@ -235,23 +257,31 @@ def _collect_agent_metrics(self, base_url, headers, extra_tags, max_pages): platform = (agent.get("platform") or "unknown").lower() by_platform[platform] = by_platform.get(platform, 0) + 1 - def_status = (agent.get("defender_status") or "unknown").lower().replace(" ", "_") - by_defender_status[def_status] = by_defender_status.get(def_status, 0) + 1 + def_status = (agent.get("defender_status") + or "unknown").lower().replace(" ", "_") + by_defender_status[def_status] = by_defender_status.get( + def_status, 0) + 1 - fw_status = (agent.get("firewall_status") or "unknown").lower().replace(" ", "_") - by_firewall_status[fw_status] = by_firewall_status.get(fw_status, 0) + 1 + fw_status = (agent.get("firewall_status") + or "unknown").lower().replace(" ", "_") + by_firewall_status[fw_status] = by_firewall_status.get( + fw_status, 0) + 1 self.gauge("huntress.agents.total", len(agents), tags=extra_tags) - self.gauge("huntress.agents.pages_fetched", pages_fetched, tags=extra_tags) + self.gauge("huntress.agents.pages_fetched", + pages_fetched, tags=extra_tags) for platform, count in by_platform.items(): - self.gauge("huntress.agents.count", count, tags=extra_tags + [f"platform:{platform}"]) + self.gauge("huntress.agents.count", count, + tags=extra_tags + [f"platform:{platform}"]) for status, count in by_defender_status.items(): - self.gauge("huntress.agents.defender_status", count, tags=extra_tags + [f"defender_status:{status}"]) + self.gauge("huntress.agents.defender_status", count, + tags=extra_tags + [f"defender_status:{status}"]) for status, count in by_firewall_status.items(): - self.gauge("huntress.agents.firewall_status", count, tags=extra_tags + [f"firewall_status:{status}"]) + self.gauge("huntress.agents.firewall_status", count, + tags=extra_tags + [f"firewall_status:{status}"]) # ------------------------------------------------------------------ # # Auth # @@ -296,7 +326,8 @@ def _query_page(self, base_url, headers, esql, range_start, range_end, page_toke if page_token: body["page_token"] = page_token - response = self._request_with_retry(method="POST", url=url, headers=headers, json_body=body) + response = self._request_with_retry( + method="POST", url=url, headers=headers, json_body=body) data = response.json() logs = data.get("logs", []) pagination = data.get("pagination", {}) @@ -309,7 +340,8 @@ def _query_page(self, base_url, headers, esql, range_start, range_end, page_toke def _request_with_retry(self, method, url, headers, json_body=None, params=None): """Execute an HTTP request with retry logic per PRD §7.""" - timeout = self.init_config.get("request_timeout", self.DEFAULT_REQUEST_TIMEOUT) + timeout = self.init_config.get( + "request_timeout", self.DEFAULT_REQUEST_TIMEOUT) max_retries_5xx = 3 max_retries_408 = 2 backoff_5xx = [5, 10, 20] @@ -333,15 +365,20 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) return resp if resp.status_code == 400: - self.count("huntress.siem.errors", 1, tags=["error_type:bad_request"]) - raise Exception(f"Huntress API 400 Bad Request: {resp.text}") + self.count("huntress.siem.errors", 1, + tags=["error_type:bad_request"]) + raise Exception( + f"Huntress API 400 Bad Request: {resp.text}") if resp.status_code == 401: - self.count("huntress.siem.errors", 1, tags=["error_type:auth_failure"]) - raise Exception("Huntress API 401 Unauthorized — check huntress_api_key and huntress_secret_key") + self.count("huntress.siem.errors", 1, tags=[ + "error_type:auth_failure"]) + raise Exception( + "Huntress API 401 Unauthorized — check huntress_api_key and huntress_secret_key") if resp.status_code == 404: - raise Exception("Huntress API 404 — SIEM feature may not be enabled on this account") + raise Exception( + "Huntress API 404 — SIEM feature may not be enabled on this account") if resp.status_code == 408: if attempt < max_retries_408: @@ -355,18 +392,24 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) time.sleep(wait) attempt += 1 continue - self.count("huntress.siem.errors", 1, tags=["error_type:timeout"]) - raise Exception("Huntress API 408 Query Timeout — query may be too broad") + self.count("huntress.siem.errors", 1, + tags=["error_type:timeout"]) + raise Exception( + "Huntress API 408 Query Timeout — query may be too broad") if resp.status_code == 413: - raise Exception("Huntress API 413 Memory Limit — narrow the query with KEEP or WHERE clauses") + raise Exception( + "Huntress API 413 Memory Limit — narrow the query with KEEP or WHERE clauses") if resp.status_code == 422: - self.count("huntress.siem.errors", 1, tags=["error_type:invalid_query"]) - raise Exception(f"Huntress API 422 Invalid ES|QL query: {resp.text}") + self.count("huntress.siem.errors", 1, tags=[ + "error_type:invalid_query"]) + raise Exception( + f"Huntress API 422 Invalid ES|QL query: {resp.text}") if resp.status_code == 429: - self.log.warning("Huntress API 429 Rate Limited — sleeping 60s then retrying") + self.log.warning( + "Huntress API 429 Rate Limited — sleeping 60s then retrying") time.sleep(60) continue @@ -383,10 +426,13 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) time.sleep(wait) attempt += 1 continue - self.count("huntress.siem.errors", 1, tags=["error_type:server_error"]) - raise Exception(f"Huntress API {resp.status_code} Server Error after {max_retries_5xx} retries") + self.count("huntress.siem.errors", 1, tags=[ + "error_type:server_error"]) + raise Exception( + f"Huntress API {resp.status_code} Server Error after {max_retries_5xx} retries") - raise Exception(f"Huntress API unexpected status {resp.status_code}: {resp.text}") + raise Exception( + f"Huntress API unexpected status {resp.status_code}: {resp.text}") except requests.exceptions.RequestException as exc: if attempt < max_retries_5xx: @@ -401,8 +447,10 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) time.sleep(wait) attempt += 1 continue - self.count("huntress.siem.errors", 1, tags=["error_type:connection_error"]) - raise Exception(f"Huntress API connection error after {max_retries_5xx} retries: {exc}") from exc + self.count("huntress.siem.errors", 1, tags=[ + "error_type:connection_error"]) + raise Exception( + f"Huntress API connection error after {max_retries_5xx} retries: {exc}") from exc # ------------------------------------------------------------------ # # Log transformation # @@ -415,7 +463,8 @@ def _extract_service(self, tags): return "huntress-siem" def _transform_log(self, raw_log, tags, service): - message = raw_log.get("log.original") or raw_log.get("message") or json.dumps(raw_log) + message = raw_log.get("log.original") or raw_log.get( + "message") or json.dumps(raw_log) timestamp = raw_log.get("@timestamp") date_ms = None @@ -424,7 +473,8 @@ def _transform_log(self, raw_log, tags, service): if isinstance(timestamp, (int, float)): date_ms = int(timestamp) else: - dt = datetime.fromisoformat(str(timestamp).replace("Z", "+00:00")) + dt = datetime.fromisoformat( + str(timestamp).replace("Z", "+00:00")) date_ms = int(dt.timestamp() * 1000) except Exception: pass @@ -470,7 +520,8 @@ def _load_checkpoint(self, checkpoint_key): def _save_checkpoint(self, checkpoint_key, timestamp_iso): key = self.CHECKPOINT_CACHE_KEY_PREFIX + checkpoint_key - payload = json.dumps({"last_collected_at": timestamp_iso, "schema_version": 1}) + payload = json.dumps( + {"last_collected_at": timestamp_iso, "schema_version": 1}) self.write_persistent_cache(key, payload) # ------------------------------------------------------------------ # @@ -483,7 +534,8 @@ def _get_or_refresh_org_cache(self, base_url, headers, instance_hash, ttl_second if cached: try: fetched_at_str = cached.get("fetched_at", "") - fetched_at = datetime.fromisoformat(fetched_at_str.replace("Z", "+00:00")) + fetched_at = datetime.fromisoformat( + fetched_at_str.replace("Z", "+00:00")) age = (datetime.now(timezone.utc) - fetched_at).total_seconds() if ttl_seconds == 0 or age < ttl_seconds: return cached @@ -519,8 +571,10 @@ def _fetch_org_cache(self, base_url, headers, account_id): orgs = {} page = 1 while True: - url = base_url + self.HUNTRESS_ORGS_ENDPOINT.format(account_id=account_id) - resp = self._request_with_retry("GET", url, headers, params={"limit": 500, "page": page}) + url = base_url + \ + self.HUNTRESS_ORGS_ENDPOINT.format(account_id=account_id) + resp = self._request_with_retry("GET", url, headers, params={ + "limit": 500, "page": page}) data = resp.json() org_list = data.get("organizations", []) for org in org_list: @@ -531,7 +585,8 @@ def _fetch_org_cache(self, base_url, headers, account_id): } pagination = data.get("pagination", {}) if pagination.get("next_page_token") or ( - pagination.get("current_page", 1) < pagination.get("total_pages", 1) + pagination.get("current_page", 1) < pagination.get( + "total_pages", 1) ): page += 1 else: @@ -566,7 +621,8 @@ def _get_org_tags(self, raw_log, org_cache): # Strategy 1: match by organization.id org_id = raw_log.get("organization.id") or ( - raw_log.get("organization", {}).get("id") if isinstance(raw_log.get("organization"), dict) else None + raw_log.get("organization", {}).get("id") if isinstance( + raw_log.get("organization"), dict) else None ) if org_id is not None: org = orgs.get(str(org_id)) @@ -579,7 +635,8 @@ def _get_org_tags(self, raw_log, org_cache): # Strategy 2: match by organization.name (reverse lookup) org_name = raw_log.get("organization.name") or ( - raw_log.get("organization", {}).get("name") if isinstance(raw_log.get("organization"), dict) else None + raw_log.get("organization", {}).get("name") if isinstance( + raw_log.get("organization"), dict) else None ) if org_name: for oid, org in orgs.items(): @@ -598,11 +655,10 @@ def _get_org_tags(self, raw_log, org_cache): # ------------------------------------------------------------------ # def _instance_hash(self, instance): - """Stable hash for a Huntress account (api key + secret + base url).""" + """Stable hash for a Huntress account (api key + base url).""" key = "|".join( [ instance.get("huntress_api_key", ""), - instance.get("huntress_secret_key", ""), instance.get("huntress_base_url", self.DEFAULT_BASE_URL), ] ) diff --git a/huntress/datadog_checks/huntress/huntress.py b/huntress/datadog_checks/huntress/huntress.py index edaf8fab64..391652fc90 100644 --- a/huntress/datadog_checks/huntress/huntress.py +++ b/huntress/datadog_checks/huntress/huntress.py @@ -43,18 +43,23 @@ def check(self, instance): api_key = instance.get("huntress_api_key", "").strip() secret_key = instance.get("huntress_secret_key", "").strip() - base_url = instance.get("huntress_base_url", self.DEFAULT_BASE_URL).rstrip("/") - max_pages = int(instance.get("max_pages_per_run", self.DEFAULT_MAX_PAGES_PER_RUN)) - min_interval = int(instance.get("min_collection_interval", self.DEFAULT_MIN_COLLECTION_INTERVAL)) + base_url = instance.get("huntress_base_url", + self.DEFAULT_BASE_URL).rstrip("/") + max_pages = int(instance.get("max_pages_per_run", + self.DEFAULT_MAX_PAGES_PER_RUN)) + min_interval = int(instance.get( + "min_collection_interval", self.DEFAULT_MIN_COLLECTION_INTERVAL)) enrich_orgs = instance.get("enrich_with_org_tags", True) - org_ttl = int(instance.get("org_cache_ttl_seconds", self.DEFAULT_ORG_CACHE_TTL)) + org_ttl = int(instance.get("org_cache_ttl_seconds", + self.DEFAULT_ORG_CACHE_TTL)) extra_tags = list(instance.get("tags", [])) log_queries = instance.get("log_queries") or [] metrics_config = instance.get("metrics") or {} agents_config = metrics_config.get("agents") or {} collect_agents = bool(agents_config.get("enabled", False)) - agents_max_pages = int(agents_config.get("max_pages", self.DEFAULT_AGENT_MAX_PAGES)) + agents_max_pages = int(agents_config.get( + "max_pages", self.DEFAULT_AGENT_MAX_PAGES)) if not api_key: raise ConfigurationError("huntress_api_key is required") @@ -72,7 +77,8 @@ def check(self, instance): # Org enrichment cache is shared across all queries for this instance org_cache = None if enrich_orgs and log_queries: - org_cache = self._get_or_refresh_org_cache(base_url, headers, instance_hash, org_ttl) + org_cache = self._get_or_refresh_org_cache( + base_url, headers, instance_hash, org_ttl) success = False @@ -83,11 +89,14 @@ def check(self, instance): query_tags = list(query_def.get("tags", [])) if not query_name: - raise ConfigurationError("Each entry in log_queries must have a non-empty 'name'") + raise ConfigurationError( + "Each entry in log_queries must have a non-empty 'name'") if not esql: - raise ConfigurationError("Each entry in log_queries must have a non-empty esql_query") + raise ConfigurationError( + "Each entry in log_queries must have a non-empty esql_query") if not esql.lower().lstrip().startswith("from logs"): - raise ConfigurationError(f"esql_query must begin with 'FROM logs' (case-insensitive): {esql!r}") + raise ConfigurationError( + f"esql_query must begin with 'FROM logs' (case-insensitive): {esql!r}") # Each query tracks its own checkpoint so queries are independently resumable checkpoint_key = instance_hash + "_" + self._query_hash(esql) @@ -98,30 +107,39 @@ def check(self, instance): base_url, headers, esql, checkpoint_key, max_pages, min_interval, org_cache, all_tags ) - self.gauge("huntress.siem.logs_collected", logs, tags=query_metric_tags) - self.gauge("huntress.siem.pages_fetched", pages, tags=query_metric_tags) + self.gauge("huntress.siem.logs_collected", + logs, tags=query_metric_tags) + self.gauge("huntress.siem.pages_fetched", + pages, tags=query_metric_tags) if collect_agents: - self._collect_agent_metrics(base_url, headers, extra_tags, agents_max_pages) + self._collect_agent_metrics( + base_url, headers, extra_tags, agents_max_pages) success = True except Exception as exc: self.log.error("Huntress check run failed: %s", exc) - self.count("huntress.siem.errors", 1, tags=extra_tags + ["error_type:run_failure"]) - self.service_check(self.SERVICE_CHECK_NAME, self.CRITICAL, tags=extra_tags) + self.count("huntress.siem.errors", 1, + tags=extra_tags + ["error_type:run_failure"]) + self.service_check(self.SERVICE_CHECK_NAME, + self.CRITICAL, tags=extra_tags) raise finally: duration = time.time() - start_time - self.gauge("huntress.siem.run_duration_seconds", duration, tags=extra_tags) + self.gauge("huntress.siem.run_duration_seconds", + duration, tags=extra_tags) if self._last_api_call_limit is not None: - self.gauge("huntress.siem.api_call_limit", self._last_api_call_limit, tags=extra_tags) + self.gauge("huntress.siem.api_call_limit", + self._last_api_call_limit, tags=extra_tags) if self._last_api_call_remaining is not None: - self.gauge("huntress.siem.api_call_remaining", self._last_api_call_remaining, tags=extra_tags) + self.gauge("huntress.siem.api_call_remaining", + self._last_api_call_remaining, tags=extra_tags) if success: - self.service_check(self.SERVICE_CHECK_NAME, self.OK, tags=extra_tags) + self.service_check(self.SERVICE_CHECK_NAME, + self.OK, tags=extra_tags) # ------------------------------------------------------------------ # # Per-query SIEM execution # @@ -135,7 +153,8 @@ def _run_query(self, base_url, headers, esql, checkpoint_key, max_pages, min_int if range_start is None: default_start = now - timedelta(seconds=min_interval) - range_start = default_start.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + range_start = default_start.strftime( + "%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" else: # Add 1ms to avoid re-fetching the boundary event from the previous run try: @@ -159,13 +178,16 @@ def _run_query(self, base_url, headers, esql, checkpoint_key, max_pages, min_int hit_page_cap = False while True: - logs, next_token = self._query_page(base_url, headers, esql, range_start, range_end, page_token) + logs, next_token = self._query_page( + base_url, headers, esql, range_start, range_end, page_token) if logs: batch = [] for raw in logs: - org_tags = self._get_org_tags(raw, org_cache) if org_cache else [] - payload = self._transform_log(raw, all_tags + org_tags, service_tag) + org_tags = self._get_org_tags( + raw, org_cache) if org_cache else [] + payload = self._transform_log( + raw, all_tags + org_tags, service_tag) batch.append(payload) if len(batch) >= self.MAX_LOGS_PER_BATCH: self._send_logs_batch(batch) @@ -235,23 +257,31 @@ def _collect_agent_metrics(self, base_url, headers, extra_tags, max_pages): platform = (agent.get("platform") or "unknown").lower() by_platform[platform] = by_platform.get(platform, 0) + 1 - def_status = (agent.get("defender_status") or "unknown").lower().replace(" ", "_") - by_defender_status[def_status] = by_defender_status.get(def_status, 0) + 1 + def_status = (agent.get("defender_status") + or "unknown").lower().replace(" ", "_") + by_defender_status[def_status] = by_defender_status.get( + def_status, 0) + 1 - fw_status = (agent.get("firewall_status") or "unknown").lower().replace(" ", "_") - by_firewall_status[fw_status] = by_firewall_status.get(fw_status, 0) + 1 + fw_status = (agent.get("firewall_status") + or "unknown").lower().replace(" ", "_") + by_firewall_status[fw_status] = by_firewall_status.get( + fw_status, 0) + 1 self.gauge("huntress.agents.total", len(agents), tags=extra_tags) - self.gauge("huntress.agents.pages_fetched", pages_fetched, tags=extra_tags) + self.gauge("huntress.agents.pages_fetched", + pages_fetched, tags=extra_tags) for platform, count in by_platform.items(): - self.gauge("huntress.agents.count", count, tags=extra_tags + [f"platform:{platform}"]) + self.gauge("huntress.agents.count", count, + tags=extra_tags + [f"platform:{platform}"]) for status, count in by_defender_status.items(): - self.gauge("huntress.agents.defender_status", count, tags=extra_tags + [f"defender_status:{status}"]) + self.gauge("huntress.agents.defender_status", count, + tags=extra_tags + [f"defender_status:{status}"]) for status, count in by_firewall_status.items(): - self.gauge("huntress.agents.firewall_status", count, tags=extra_tags + [f"firewall_status:{status}"]) + self.gauge("huntress.agents.firewall_status", count, + tags=extra_tags + [f"firewall_status:{status}"]) # ------------------------------------------------------------------ # # Auth # @@ -296,7 +326,8 @@ def _query_page(self, base_url, headers, esql, range_start, range_end, page_toke if page_token: body["page_token"] = page_token - response = self._request_with_retry(method="POST", url=url, headers=headers, json_body=body) + response = self._request_with_retry( + method="POST", url=url, headers=headers, json_body=body) data = response.json() logs = data.get("logs", []) pagination = data.get("pagination", {}) @@ -309,7 +340,8 @@ def _query_page(self, base_url, headers, esql, range_start, range_end, page_toke def _request_with_retry(self, method, url, headers, json_body=None, params=None): """Execute an HTTP request with retry logic per PRD §7.""" - timeout = self.init_config.get("request_timeout", self.DEFAULT_REQUEST_TIMEOUT) + timeout = self.init_config.get( + "request_timeout", self.DEFAULT_REQUEST_TIMEOUT) max_retries_5xx = 3 max_retries_408 = 2 backoff_5xx = [5, 10, 20] @@ -333,15 +365,20 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) return resp if resp.status_code == 400: - self.count("huntress.siem.errors", 1, tags=["error_type:bad_request"]) - raise Exception(f"Huntress API 400 Bad Request: {resp.text}") + self.count("huntress.siem.errors", 1, + tags=["error_type:bad_request"]) + raise Exception( + f"Huntress API 400 Bad Request: {resp.text}") if resp.status_code == 401: - self.count("huntress.siem.errors", 1, tags=["error_type:auth_failure"]) - raise Exception("Huntress API 401 Unauthorized — check huntress_api_key and huntress_secret_key") + self.count("huntress.siem.errors", 1, tags=[ + "error_type:auth_failure"]) + raise Exception( + "Huntress API 401 Unauthorized — check huntress_api_key and huntress_secret_key") if resp.status_code == 404: - raise Exception("Huntress API 404 — SIEM feature may not be enabled on this account") + raise Exception( + "Huntress API 404 — SIEM feature may not be enabled on this account") if resp.status_code == 408: if attempt < max_retries_408: @@ -355,18 +392,24 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) time.sleep(wait) attempt += 1 continue - self.count("huntress.siem.errors", 1, tags=["error_type:timeout"]) - raise Exception("Huntress API 408 Query Timeout — query may be too broad") + self.count("huntress.siem.errors", 1, + tags=["error_type:timeout"]) + raise Exception( + "Huntress API 408 Query Timeout — query may be too broad") if resp.status_code == 413: - raise Exception("Huntress API 413 Memory Limit — narrow the query with KEEP or WHERE clauses") + raise Exception( + "Huntress API 413 Memory Limit — narrow the query with KEEP or WHERE clauses") if resp.status_code == 422: - self.count("huntress.siem.errors", 1, tags=["error_type:invalid_query"]) - raise Exception(f"Huntress API 422 Invalid ES|QL query: {resp.text}") + self.count("huntress.siem.errors", 1, tags=[ + "error_type:invalid_query"]) + raise Exception( + f"Huntress API 422 Invalid ES|QL query: {resp.text}") if resp.status_code == 429: - self.log.warning("Huntress API 429 Rate Limited — sleeping 60s then retrying") + self.log.warning( + "Huntress API 429 Rate Limited — sleeping 60s then retrying") time.sleep(60) continue @@ -383,10 +426,13 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) time.sleep(wait) attempt += 1 continue - self.count("huntress.siem.errors", 1, tags=["error_type:server_error"]) - raise Exception(f"Huntress API {resp.status_code} Server Error after {max_retries_5xx} retries") + self.count("huntress.siem.errors", 1, tags=[ + "error_type:server_error"]) + raise Exception( + f"Huntress API {resp.status_code} Server Error after {max_retries_5xx} retries") - raise Exception(f"Huntress API unexpected status {resp.status_code}: {resp.text}") + raise Exception( + f"Huntress API unexpected status {resp.status_code}: {resp.text}") except requests.exceptions.RequestException as exc: if attempt < max_retries_5xx: @@ -401,8 +447,10 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) time.sleep(wait) attempt += 1 continue - self.count("huntress.siem.errors", 1, tags=["error_type:connection_error"]) - raise Exception(f"Huntress API connection error after {max_retries_5xx} retries: {exc}") from exc + self.count("huntress.siem.errors", 1, tags=[ + "error_type:connection_error"]) + raise Exception( + f"Huntress API connection error after {max_retries_5xx} retries: {exc}") from exc # ------------------------------------------------------------------ # # Log transformation # @@ -415,7 +463,8 @@ def _extract_service(self, tags): return "huntress-siem" def _transform_log(self, raw_log, tags, service): - message = raw_log.get("log.original") or raw_log.get("message") or json.dumps(raw_log) + message = raw_log.get("log.original") or raw_log.get( + "message") or json.dumps(raw_log) timestamp = raw_log.get("@timestamp") date_ms = None @@ -424,7 +473,8 @@ def _transform_log(self, raw_log, tags, service): if isinstance(timestamp, (int, float)): date_ms = int(timestamp) else: - dt = datetime.fromisoformat(str(timestamp).replace("Z", "+00:00")) + dt = datetime.fromisoformat( + str(timestamp).replace("Z", "+00:00")) date_ms = int(dt.timestamp() * 1000) except Exception: pass @@ -470,7 +520,8 @@ def _load_checkpoint(self, checkpoint_key): def _save_checkpoint(self, checkpoint_key, timestamp_iso): key = self.CHECKPOINT_CACHE_KEY_PREFIX + checkpoint_key - payload = json.dumps({"last_collected_at": timestamp_iso, "schema_version": 1}) + payload = json.dumps( + {"last_collected_at": timestamp_iso, "schema_version": 1}) self.write_persistent_cache(key, payload) # ------------------------------------------------------------------ # @@ -483,7 +534,8 @@ def _get_or_refresh_org_cache(self, base_url, headers, instance_hash, ttl_second if cached: try: fetched_at_str = cached.get("fetched_at", "") - fetched_at = datetime.fromisoformat(fetched_at_str.replace("Z", "+00:00")) + fetched_at = datetime.fromisoformat( + fetched_at_str.replace("Z", "+00:00")) age = (datetime.now(timezone.utc) - fetched_at).total_seconds() if ttl_seconds == 0 or age < ttl_seconds: return cached @@ -519,8 +571,10 @@ def _fetch_org_cache(self, base_url, headers, account_id): orgs = {} page = 1 while True: - url = base_url + self.HUNTRESS_ORGS_ENDPOINT.format(account_id=account_id) - resp = self._request_with_retry("GET", url, headers, params={"limit": 500, "page": page}) + url = base_url + \ + self.HUNTRESS_ORGS_ENDPOINT.format(account_id=account_id) + resp = self._request_with_retry("GET", url, headers, params={ + "limit": 500, "page": page}) data = resp.json() org_list = data.get("organizations", []) for org in org_list: @@ -531,7 +585,8 @@ def _fetch_org_cache(self, base_url, headers, account_id): } pagination = data.get("pagination", {}) if pagination.get("next_page_token") or ( - pagination.get("current_page", 1) < pagination.get("total_pages", 1) + pagination.get("current_page", 1) < pagination.get( + "total_pages", 1) ): page += 1 else: @@ -566,7 +621,8 @@ def _get_org_tags(self, raw_log, org_cache): # Strategy 1: match by organization.id org_id = raw_log.get("organization.id") or ( - raw_log.get("organization", {}).get("id") if isinstance(raw_log.get("organization"), dict) else None + raw_log.get("organization", {}).get("id") if isinstance( + raw_log.get("organization"), dict) else None ) if org_id is not None: org = orgs.get(str(org_id)) @@ -579,7 +635,8 @@ def _get_org_tags(self, raw_log, org_cache): # Strategy 2: match by organization.name (reverse lookup) org_name = raw_log.get("organization.name") or ( - raw_log.get("organization", {}).get("name") if isinstance(raw_log.get("organization"), dict) else None + raw_log.get("organization", {}).get("name") if isinstance( + raw_log.get("organization"), dict) else None ) if org_name: for oid, org in orgs.items(): @@ -598,11 +655,10 @@ def _get_org_tags(self, raw_log, org_cache): # ------------------------------------------------------------------ # def _instance_hash(self, instance): - """Stable hash for a Huntress account (api key + secret + base url).""" + """Stable hash for a Huntress account (api key + base url).""" key = "|".join( [ instance.get("huntress_api_key", ""), - instance.get("huntress_secret_key", ""), instance.get("huntress_base_url", self.DEFAULT_BASE_URL), ] ) From accc6893c6146da2256b51cf71a7cc006ac85cef Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Fri, 29 May 2026 17:49:06 -0500 Subject: [PATCH 15/40] Use non-secret stable discriminator not derived from keys --- huntress/checks.d/huntress.py | 18 ++- huntress/datadog_checks/huntress/huntress.py | 18 ++- huntress/tests/test_huntress.py | 125 +++++++++++++------ 3 files changed, 98 insertions(+), 63 deletions(-) diff --git a/huntress/checks.d/huntress.py b/huntress/checks.d/huntress.py index 391652fc90..76da61bc64 100644 --- a/huntress/checks.d/huntress.py +++ b/huntress/checks.d/huntress.py @@ -1,5 +1,5 @@ import base64 -import hashlib +import zlib import json import time from datetime import datetime, timedelta, timezone @@ -655,15 +655,11 @@ def _get_org_tags(self, raw_log, org_cache): # ------------------------------------------------------------------ # def _instance_hash(self, instance): - """Stable hash for a Huntress account (api key + base url).""" - key = "|".join( - [ - instance.get("huntress_api_key", ""), - instance.get("huntress_base_url", self.DEFAULT_BASE_URL), - ] - ) - return hashlib.sha256(key.encode()).hexdigest()[:12] + """Stable non-secret identifier for this Huntress instance.""" + key = instance.get("instance_id") or instance.get( + "huntress_base_url", self.DEFAULT_BASE_URL) + return format(zlib.crc32(key.encode("utf-8")) & 0xFFFFFFFF, "08x") def _query_hash(self, esql_query): - """Short hash to build a per-query checkpoint key.""" - return hashlib.sha256(esql_query.encode()).hexdigest()[:8] + """Short non-cryptographic checksum for a per-query checkpoint key.""" + return format(zlib.crc32(esql_query.encode("utf-8")) & 0xFFFFFFFF, "08x") diff --git a/huntress/datadog_checks/huntress/huntress.py b/huntress/datadog_checks/huntress/huntress.py index 391652fc90..76da61bc64 100644 --- a/huntress/datadog_checks/huntress/huntress.py +++ b/huntress/datadog_checks/huntress/huntress.py @@ -1,5 +1,5 @@ import base64 -import hashlib +import zlib import json import time from datetime import datetime, timedelta, timezone @@ -655,15 +655,11 @@ def _get_org_tags(self, raw_log, org_cache): # ------------------------------------------------------------------ # def _instance_hash(self, instance): - """Stable hash for a Huntress account (api key + base url).""" - key = "|".join( - [ - instance.get("huntress_api_key", ""), - instance.get("huntress_base_url", self.DEFAULT_BASE_URL), - ] - ) - return hashlib.sha256(key.encode()).hexdigest()[:12] + """Stable non-secret identifier for this Huntress instance.""" + key = instance.get("instance_id") or instance.get( + "huntress_base_url", self.DEFAULT_BASE_URL) + return format(zlib.crc32(key.encode("utf-8")) & 0xFFFFFFFF, "08x") def _query_hash(self, esql_query): - """Short hash to build a per-query checkpoint key.""" - return hashlib.sha256(esql_query.encode()).hexdigest()[:8] + """Short non-cryptographic checksum for a per-query checkpoint key.""" + return format(zlib.crc32(esql_query.encode("utf-8")) & 0xFFFFFFFF, "08x") diff --git a/huntress/tests/test_huntress.py b/huntress/tests/test_huntress.py index f7be6c5de2..2106866382 100644 --- a/huntress/tests/test_huntress.py +++ b/huntress/tests/test_huntress.py @@ -40,7 +40,8 @@ def _make_check(**kwargs): # Use an in-memory dict to prevent persistent cache from bleeding between tests _cache = {} check.read_persistent_cache = lambda key: _cache.get(key) - check.write_persistent_cache = lambda key, value: _cache.__setitem__(key, value) + check.write_persistent_cache = lambda key, value: _cache.__setitem__( + key, value) return check, inst @@ -72,7 +73,8 @@ def test_happy_path_single_page(): fixture["logs"] = _load_fixture("siem_query_page1.json")["logs"][:1] with ( - patch.object(check, "_request_with_retry", return_value=_mock_response(200, fixture)), + patch.object(check, "_request_with_retry", + return_value=_mock_response(200, fixture)), patch.object(check, "_send_logs_batch") as mock_send, patch.object(check, "_save_checkpoint") as mock_save, patch("time.time", side_effect=[1000.0, 1001.0]), @@ -111,7 +113,8 @@ def test_happy_path_multi_page(): check.check(instance) assert mock_send.call_count == 2 - all_logs = mock_send.call_args_list[0][0][0] + mock_send.call_args_list[1][0][0] + all_logs = mock_send.call_args_list[0][0][0] + \ + mock_send.call_args_list[1][0][0] assert len(all_logs) == 3 mock_save.assert_called_once() @@ -125,7 +128,8 @@ def test_auth_failure_401(): check, instance = _make_check() with ( - patch.object(check, "_request_with_retry", side_effect=Exception("Huntress API 401 Unauthorized")), + patch.object(check, "_request_with_retry", + side_effect=Exception("Huntress API 401 Unauthorized")), patch.object(check, "_save_checkpoint") as mock_save, patch.object(check, "count"), patch("time.time", side_effect=[1000.0, 1001.0]), @@ -143,9 +147,11 @@ def test_auth_failure_401_raises_via_retry(): with patch("requests.request", return_value=resp), patch.object(check, "count") as mock_count: with pytest.raises(Exception, match="401"): - check._request_with_retry("POST", "https://api.huntress.io/v1/siem/query", {}, json_body={}) + check._request_with_retry( + "POST", "https://api.huntress.io/v1/siem/query", {}, json_body={}) - mock_count.assert_called_with("huntress.siem.errors", 1, tags=["error_type:auth_failure"]) + mock_count.assert_called_with("huntress.siem.errors", 1, tags=[ + "error_type:auth_failure"]) # =========================================================================== @@ -169,7 +175,8 @@ def test_query_timeout_408(): assert mock_sleep.call_count == 2 sleep_args = [c[0][0] for c in mock_sleep.call_args_list] assert sleep_args == [2, 4] - mock_count.assert_called_with("huntress.siem.errors", 1, tags=["error_type:timeout"]) + mock_count.assert_called_with( + "huntress.siem.errors", 1, tags=["error_type:timeout"]) # =========================================================================== @@ -183,7 +190,8 @@ def test_rate_limit_429_then_success(): resp_200 = _mock_response(200, _load_fixture("siem_query_empty.json")) with patch("requests.request", side_effect=[resp_429, resp_200]), patch("time.sleep") as mock_sleep: - result = check._request_with_retry("POST", "https://x/q", {}, json_body={}) + result = check._request_with_retry( + "POST", "https://x/q", {}, json_body={}) mock_sleep.assert_called_once_with(60) assert result.status_code == 200 @@ -202,7 +210,8 @@ def test_invalid_esql_422(): with pytest.raises(Exception, match="422"): check._request_with_retry("POST", "https://x/q", {}, json_body={}) - mock_count.assert_called_with("huntress.siem.errors", 1, tags=["error_type:invalid_query"]) + mock_count.assert_called_with("huntress.siem.errors", 1, tags=[ + "error_type:invalid_query"]) # =========================================================================== @@ -229,7 +238,8 @@ def capture_request(method, url, headers, json_body=None, params=None): return _mock_response(200, _load_fixture("siem_query_empty.json")) with ( - patch.object(check, "_request_with_retry", side_effect=capture_request), + patch.object(check, "_request_with_retry", + side_effect=capture_request), patch("time.time", side_effect=[1000.0, 1001.0]), ): check.check(instance) @@ -255,7 +265,8 @@ def capture_request(method, url, headers, json_body=None, params=None): fixed_now = datetime(2026, 5, 27, 14, 0, 0, tzinfo=timezone.utc) with ( - patch.object(check, "_request_with_retry", side_effect=capture_request), + patch.object(check, "_request_with_retry", + side_effect=capture_request), patch("datadog_checks.huntress.huntress.datetime") as mock_dt, patch("time.time", side_effect=[1000.0, 1001.0]), ): @@ -265,8 +276,10 @@ def capture_request(method, url, headers, json_body=None, params=None): assert captured_bodies body = captured_bodies[0] - range_start = datetime.fromisoformat(body["range_start"].replace("Z", "+00:00")) - range_end = datetime.fromisoformat(body["range_end"].replace("Z", "+00:00")) + range_start = datetime.fromisoformat( + body["range_start"].replace("Z", "+00:00")) + range_end = datetime.fromisoformat( + body["range_end"].replace("Z", "+00:00")) diff_seconds = (range_end - range_start).total_seconds() assert abs(diff_seconds - 900) < 2 @@ -283,7 +296,8 @@ def test_esql_validation_rejects_bad_query(): def test_esql_validation_accepts_from_logs(): - check, instance = _make_check(esql_query="FROM logs | KEEP @timestamp, message") + check, instance = _make_check( + esql_query="FROM logs | KEEP @timestamp, message") with ( patch.object( check, "_request_with_retry", return_value=_mock_response(200, _load_fixture("siem_query_empty.json")) @@ -335,14 +349,16 @@ def test_log_transformation(): def test_log_transformation_fallback_message(): check, instance = _make_check() - raw = {"message": "fallback used", "@timestamp": "2026-05-27T14:00:00.000Z"} + raw = {"message": "fallback used", + "@timestamp": "2026-05-27T14:00:00.000Z"} payload = check._transform_log(raw, [], "huntress-siem") assert payload["message"] == "fallback used" def test_log_transformation_json_fallback(): check, instance = _make_check() - raw = {"event.category": "network", "@timestamp": "2026-05-27T14:00:00.000Z"} + raw = {"event.category": "network", + "@timestamp": "2026-05-27T14:00:00.000Z"} payload = check._transform_log(raw, [], "huntress-siem") assert "event.category" in payload["message"] @@ -365,7 +381,8 @@ def test_batching_over_1000_logs(): fixture = {"logs": large_log_list, "pagination": {}} with ( - patch.object(check, "_request_with_retry", return_value=_mock_response(200, fixture)), + patch.object(check, "_request_with_retry", + return_value=_mock_response(200, fixture)), patch.object(check, "_send_logs_batch") as mock_send, patch("time.time", side_effect=[1000.0, 1001.0]), ): @@ -388,7 +405,8 @@ def test_max_pages_per_run_cap(): page1 = _load_fixture("siem_query_page1.json") with ( - patch.object(check, "_request_with_retry", return_value=_mock_response(200, page1)), + patch.object(check, "_request_with_retry", + return_value=_mock_response(200, page1)), patch.object(check, "_save_checkpoint") as mock_save, patch("time.time", side_effect=[1000.0, 1001.0]), ): @@ -416,7 +434,8 @@ def test_org_enrichment_cache_hit(): "101": {"name": "Acme Inc.", "key": "acme", "account_id": 42}, }, } - check.write_persistent_cache(check.ORG_CACHE_KEY_PREFIX + instance_hash, json.dumps(cache)) + check.write_persistent_cache( + check.ORG_CACHE_KEY_PREFIX + instance_hash, json.dumps(cache)) page1 = _load_fixture("siem_query_page1.json") page1["pagination"] = {} @@ -543,7 +562,8 @@ def test_org_enrichment_no_org_match(): "account_id": 42, "orgs": {"101": {"name": "Acme Inc.", "key": "acme", "account_id": 42}}, } - raw_log = {"@timestamp": "2026-05-27T14:00:00.000Z", "message": "no org field"} + raw_log = {"@timestamp": "2026-05-27T14:00:00.000Z", + "message": "no org field"} tags = check._get_org_tags(raw_log, org_cache) assert "huntress_account_id:42" in tags @@ -557,10 +577,12 @@ def test_org_enrichment_no_org_match(): def test_multi_instance_isolation(): instance_a = _make_instance( + instance_id="account_a", huntress_api_key="key_a", huntress_secret_key="secret_a", ) instance_b = _make_instance( + instance_id="account_b", huntress_api_key="key_b", huntress_secret_key="secret_b", ) @@ -569,13 +591,15 @@ def test_multi_instance_isolation(): check_a.log = MagicMock() _cache_a = {} check_a.read_persistent_cache = lambda key: _cache_a.get(key) - check_a.write_persistent_cache = lambda key, value: _cache_a.__setitem__(key, value) + check_a.write_persistent_cache = lambda key, value: _cache_a.__setitem__( + key, value) check_b = HuntressCheck("huntress", {}, [instance_b]) check_b.log = MagicMock() _cache_b = {} check_b.read_persistent_cache = lambda key: _cache_b.get(key) - check_b.write_persistent_cache = lambda key, value: _cache_b.__setitem__(key, value) + check_b.write_persistent_cache = lambda key, value: _cache_b.__setitem__( + key, value) hash_a = check_a._instance_hash(instance_a) hash_b = check_b._instance_hash(instance_b) @@ -690,7 +714,8 @@ def test_rate_limit_headers_missing_gracefully(): check._last_api_call_limit = None check._last_api_call_remaining = None - resp = _mock_response(200, {}, headers={"Content-Type": "application/json"}) + resp = _mock_response( + 200, {}, headers={"Content-Type": "application/json"}) check._parse_rate_limit_headers(resp) assert check._last_api_call_limit is None @@ -745,7 +770,8 @@ def test_validation_neither_configured_raises(): check.log = MagicMock() _cache = {} check.read_persistent_cache = lambda key: _cache.get(key) - check.write_persistent_cache = lambda key, value: _cache.__setitem__(key, value) + check.write_persistent_cache = lambda key, value: _cache.__setitem__( + key, value) with pytest.raises(Exception, match="at least one"): check.check(instance) @@ -761,7 +787,8 @@ def test_validation_metrics_only_instance(): check.log = MagicMock() _cache = {} check.read_persistent_cache = lambda key: _cache.get(key) - check.write_persistent_cache = lambda key, value: _cache.__setitem__(key, value) + check.write_persistent_cache = lambda key, value: _cache.__setitem__( + key, value) agents_resp = _mock_response(200, {"agents": [], "pagination": {}}) with ( @@ -794,11 +821,14 @@ def test_agent_metrics_emitted(): check.log = MagicMock() _cache = {} check.read_persistent_cache = lambda key: _cache.get(key) - check.write_persistent_cache = lambda key, value: _cache.__setitem__(key, value) + check.write_persistent_cache = lambda key, value: _cache.__setitem__( + key, value) agents = [ - {"platform": "windows", "defender_status": "Healthy", "firewall_status": "Enabled"}, - {"platform": "windows", "defender_status": "Healthy", "firewall_status": "Disabled"}, + {"platform": "windows", "defender_status": "Healthy", + "firewall_status": "Enabled"}, + {"platform": "windows", "defender_status": "Healthy", + "firewall_status": "Disabled"}, {"platform": "darwin", "defender_status": None, "firewall_status": "Enabled"}, ] agents_resp = _mock_response(200, {"agents": agents, "pagination": {}}) @@ -816,22 +846,30 @@ def capture_gauge(name, value, tags=None): check.check(instance) assert emitted[("huntress.agents.total", ("source:huntress",))] == 3 - assert emitted[("huntress.agents.count", ("platform:windows", "source:huntress"))] == 2 - assert emitted[("huntress.agents.count", ("platform:darwin", "source:huntress"))] == 1 - assert emitted[("huntress.agents.defender_status", ("defender_status:healthy", "source:huntress"))] == 2 - assert emitted[("huntress.agents.defender_status", ("defender_status:unknown", "source:huntress"))] == 1 - assert emitted[("huntress.agents.firewall_status", ("firewall_status:enabled", "source:huntress"))] == 2 - assert emitted[("huntress.agents.firewall_status", ("firewall_status:disabled", "source:huntress"))] == 1 + assert emitted[("huntress.agents.count", + ("platform:windows", "source:huntress"))] == 2 + assert emitted[("huntress.agents.count", + ("platform:darwin", "source:huntress"))] == 1 + assert emitted[("huntress.agents.defender_status", + ("defender_status:healthy", "source:huntress"))] == 2 + assert emitted[("huntress.agents.defender_status", + ("defender_status:unknown", "source:huntress"))] == 1 + assert emitted[("huntress.agents.firewall_status", + ("firewall_status:enabled", "source:huntress"))] == 2 + assert emitted[("huntress.agents.firewall_status", + ("firewall_status:disabled", "source:huntress"))] == 1 def test_agent_metrics_pagination(): """Agent collection paginates until no next_page_token.""" - instance = _make_agents_instance(metrics={"agents": {"enabled": True, "max_pages": 5}}) + instance = _make_agents_instance( + metrics={"agents": {"enabled": True, "max_pages": 5}}) check = HuntressCheck("huntress", {}, [instance]) check.log = MagicMock() _cache = {} check.read_persistent_cache = lambda key: _cache.get(key) - check.write_persistent_cache = lambda key, value: _cache.__setitem__(key, value) + check.write_persistent_cache = lambda key, value: _cache.__setitem__( + key, value) page1 = { "agents": [{"platform": "windows", "defender_status": "Healthy", "firewall_status": "Enabled"}], @@ -862,19 +900,23 @@ def capture_gauge(name, value, tags=None): check.check(instance) assert emitted[("huntress.agents.total", ("source:huntress",))] == 2 - assert emitted[("huntress.agents.count", ("platform:windows", "source:huntress"))] == 1 - assert emitted[("huntress.agents.count", ("platform:linux", "source:huntress"))] == 1 + assert emitted[("huntress.agents.count", + ("platform:windows", "source:huntress"))] == 1 + assert emitted[("huntress.agents.count", + ("platform:linux", "source:huntress"))] == 1 assert len(call_urls) == 2 def test_agent_metrics_max_pages_cap(): """Agent collection stops at max_pages even when more pages exist.""" - instance = _make_agents_instance(metrics={"agents": {"enabled": True, "max_pages": 1}}) + instance = _make_agents_instance( + metrics={"agents": {"enabled": True, "max_pages": 1}}) check = HuntressCheck("huntress", {}, [instance]) check.log = MagicMock() _cache = {} check.read_persistent_cache = lambda key: _cache.get(key) - check.write_persistent_cache = lambda key, value: _cache.__setitem__(key, value) + check.write_persistent_cache = lambda key, value: _cache.__setitem__( + key, value) page_with_more = { "agents": [{"platform": "windows", "defender_status": "Healthy", "firewall_status": "Enabled"}], @@ -882,7 +924,8 @@ def test_agent_metrics_max_pages_cap(): } with ( - patch.object(check, "_request_with_retry", return_value=_mock_response(200, page_with_more)), + patch.object(check, "_request_with_retry", + return_value=_mock_response(200, page_with_more)), patch("time.time", side_effect=[1000.0, 1001.0]), ): check.check(instance) From 27671cf0a920aa9bca5db90081fe0ef6195548f2 Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Fri, 29 May 2026 17:54:42 -0500 Subject: [PATCH 16/40] Apply linting --- huntress/checks.d/huntress.py | 176 +++++++------------ huntress/datadog_checks/huntress/huntress.py | 176 +++++++------------ huntress/tests/test_huntress.py | 123 +++++-------- 3 files changed, 159 insertions(+), 316 deletions(-) diff --git a/huntress/checks.d/huntress.py b/huntress/checks.d/huntress.py index 76da61bc64..00006654fe 100644 --- a/huntress/checks.d/huntress.py +++ b/huntress/checks.d/huntress.py @@ -1,7 +1,7 @@ import base64 -import zlib import json import time +import zlib from datetime import datetime, timedelta, timezone import requests @@ -43,23 +43,18 @@ def check(self, instance): api_key = instance.get("huntress_api_key", "").strip() secret_key = instance.get("huntress_secret_key", "").strip() - base_url = instance.get("huntress_base_url", - self.DEFAULT_BASE_URL).rstrip("/") - max_pages = int(instance.get("max_pages_per_run", - self.DEFAULT_MAX_PAGES_PER_RUN)) - min_interval = int(instance.get( - "min_collection_interval", self.DEFAULT_MIN_COLLECTION_INTERVAL)) + base_url = instance.get("huntress_base_url", self.DEFAULT_BASE_URL).rstrip("/") + max_pages = int(instance.get("max_pages_per_run", self.DEFAULT_MAX_PAGES_PER_RUN)) + min_interval = int(instance.get("min_collection_interval", self.DEFAULT_MIN_COLLECTION_INTERVAL)) enrich_orgs = instance.get("enrich_with_org_tags", True) - org_ttl = int(instance.get("org_cache_ttl_seconds", - self.DEFAULT_ORG_CACHE_TTL)) + org_ttl = int(instance.get("org_cache_ttl_seconds", self.DEFAULT_ORG_CACHE_TTL)) extra_tags = list(instance.get("tags", [])) log_queries = instance.get("log_queries") or [] metrics_config = instance.get("metrics") or {} agents_config = metrics_config.get("agents") or {} collect_agents = bool(agents_config.get("enabled", False)) - agents_max_pages = int(agents_config.get( - "max_pages", self.DEFAULT_AGENT_MAX_PAGES)) + agents_max_pages = int(agents_config.get("max_pages", self.DEFAULT_AGENT_MAX_PAGES)) if not api_key: raise ConfigurationError("huntress_api_key is required") @@ -77,8 +72,7 @@ def check(self, instance): # Org enrichment cache is shared across all queries for this instance org_cache = None if enrich_orgs and log_queries: - org_cache = self._get_or_refresh_org_cache( - base_url, headers, instance_hash, org_ttl) + org_cache = self._get_or_refresh_org_cache(base_url, headers, instance_hash, org_ttl) success = False @@ -89,14 +83,11 @@ def check(self, instance): query_tags = list(query_def.get("tags", [])) if not query_name: - raise ConfigurationError( - "Each entry in log_queries must have a non-empty 'name'") + raise ConfigurationError("Each entry in log_queries must have a non-empty 'name'") if not esql: - raise ConfigurationError( - "Each entry in log_queries must have a non-empty esql_query") + raise ConfigurationError("Each entry in log_queries must have a non-empty esql_query") if not esql.lower().lstrip().startswith("from logs"): - raise ConfigurationError( - f"esql_query must begin with 'FROM logs' (case-insensitive): {esql!r}") + raise ConfigurationError(f"esql_query must begin with 'FROM logs' (case-insensitive): {esql!r}") # Each query tracks its own checkpoint so queries are independently resumable checkpoint_key = instance_hash + "_" + self._query_hash(esql) @@ -107,39 +98,30 @@ def check(self, instance): base_url, headers, esql, checkpoint_key, max_pages, min_interval, org_cache, all_tags ) - self.gauge("huntress.siem.logs_collected", - logs, tags=query_metric_tags) - self.gauge("huntress.siem.pages_fetched", - pages, tags=query_metric_tags) + self.gauge("huntress.siem.logs_collected", logs, tags=query_metric_tags) + self.gauge("huntress.siem.pages_fetched", pages, tags=query_metric_tags) if collect_agents: - self._collect_agent_metrics( - base_url, headers, extra_tags, agents_max_pages) + self._collect_agent_metrics(base_url, headers, extra_tags, agents_max_pages) success = True except Exception as exc: self.log.error("Huntress check run failed: %s", exc) - self.count("huntress.siem.errors", 1, - tags=extra_tags + ["error_type:run_failure"]) - self.service_check(self.SERVICE_CHECK_NAME, - self.CRITICAL, tags=extra_tags) + self.count("huntress.siem.errors", 1, tags=extra_tags + ["error_type:run_failure"]) + self.service_check(self.SERVICE_CHECK_NAME, self.CRITICAL, tags=extra_tags) raise finally: duration = time.time() - start_time - self.gauge("huntress.siem.run_duration_seconds", - duration, tags=extra_tags) + self.gauge("huntress.siem.run_duration_seconds", duration, tags=extra_tags) if self._last_api_call_limit is not None: - self.gauge("huntress.siem.api_call_limit", - self._last_api_call_limit, tags=extra_tags) + self.gauge("huntress.siem.api_call_limit", self._last_api_call_limit, tags=extra_tags) if self._last_api_call_remaining is not None: - self.gauge("huntress.siem.api_call_remaining", - self._last_api_call_remaining, tags=extra_tags) + self.gauge("huntress.siem.api_call_remaining", self._last_api_call_remaining, tags=extra_tags) if success: - self.service_check(self.SERVICE_CHECK_NAME, - self.OK, tags=extra_tags) + self.service_check(self.SERVICE_CHECK_NAME, self.OK, tags=extra_tags) # ------------------------------------------------------------------ # # Per-query SIEM execution # @@ -153,8 +135,7 @@ def _run_query(self, base_url, headers, esql, checkpoint_key, max_pages, min_int if range_start is None: default_start = now - timedelta(seconds=min_interval) - range_start = default_start.strftime( - "%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + range_start = default_start.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" else: # Add 1ms to avoid re-fetching the boundary event from the previous run try: @@ -178,16 +159,13 @@ def _run_query(self, base_url, headers, esql, checkpoint_key, max_pages, min_int hit_page_cap = False while True: - logs, next_token = self._query_page( - base_url, headers, esql, range_start, range_end, page_token) + logs, next_token = self._query_page(base_url, headers, esql, range_start, range_end, page_token) if logs: batch = [] for raw in logs: - org_tags = self._get_org_tags( - raw, org_cache) if org_cache else [] - payload = self._transform_log( - raw, all_tags + org_tags, service_tag) + org_tags = self._get_org_tags(raw, org_cache) if org_cache else [] + payload = self._transform_log(raw, all_tags + org_tags, service_tag) batch.append(payload) if len(batch) >= self.MAX_LOGS_PER_BATCH: self._send_logs_batch(batch) @@ -257,31 +235,23 @@ def _collect_agent_metrics(self, base_url, headers, extra_tags, max_pages): platform = (agent.get("platform") or "unknown").lower() by_platform[platform] = by_platform.get(platform, 0) + 1 - def_status = (agent.get("defender_status") - or "unknown").lower().replace(" ", "_") - by_defender_status[def_status] = by_defender_status.get( - def_status, 0) + 1 + def_status = (agent.get("defender_status") or "unknown").lower().replace(" ", "_") + by_defender_status[def_status] = by_defender_status.get(def_status, 0) + 1 - fw_status = (agent.get("firewall_status") - or "unknown").lower().replace(" ", "_") - by_firewall_status[fw_status] = by_firewall_status.get( - fw_status, 0) + 1 + fw_status = (agent.get("firewall_status") or "unknown").lower().replace(" ", "_") + by_firewall_status[fw_status] = by_firewall_status.get(fw_status, 0) + 1 self.gauge("huntress.agents.total", len(agents), tags=extra_tags) - self.gauge("huntress.agents.pages_fetched", - pages_fetched, tags=extra_tags) + self.gauge("huntress.agents.pages_fetched", pages_fetched, tags=extra_tags) for platform, count in by_platform.items(): - self.gauge("huntress.agents.count", count, - tags=extra_tags + [f"platform:{platform}"]) + self.gauge("huntress.agents.count", count, tags=extra_tags + [f"platform:{platform}"]) for status, count in by_defender_status.items(): - self.gauge("huntress.agents.defender_status", count, - tags=extra_tags + [f"defender_status:{status}"]) + self.gauge("huntress.agents.defender_status", count, tags=extra_tags + [f"defender_status:{status}"]) for status, count in by_firewall_status.items(): - self.gauge("huntress.agents.firewall_status", count, - tags=extra_tags + [f"firewall_status:{status}"]) + self.gauge("huntress.agents.firewall_status", count, tags=extra_tags + [f"firewall_status:{status}"]) # ------------------------------------------------------------------ # # Auth # @@ -326,8 +296,7 @@ def _query_page(self, base_url, headers, esql, range_start, range_end, page_toke if page_token: body["page_token"] = page_token - response = self._request_with_retry( - method="POST", url=url, headers=headers, json_body=body) + response = self._request_with_retry(method="POST", url=url, headers=headers, json_body=body) data = response.json() logs = data.get("logs", []) pagination = data.get("pagination", {}) @@ -340,8 +309,7 @@ def _query_page(self, base_url, headers, esql, range_start, range_end, page_toke def _request_with_retry(self, method, url, headers, json_body=None, params=None): """Execute an HTTP request with retry logic per PRD §7.""" - timeout = self.init_config.get( - "request_timeout", self.DEFAULT_REQUEST_TIMEOUT) + timeout = self.init_config.get("request_timeout", self.DEFAULT_REQUEST_TIMEOUT) max_retries_5xx = 3 max_retries_408 = 2 backoff_5xx = [5, 10, 20] @@ -365,20 +333,15 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) return resp if resp.status_code == 400: - self.count("huntress.siem.errors", 1, - tags=["error_type:bad_request"]) - raise Exception( - f"Huntress API 400 Bad Request: {resp.text}") + self.count("huntress.siem.errors", 1, tags=["error_type:bad_request"]) + raise Exception(f"Huntress API 400 Bad Request: {resp.text}") if resp.status_code == 401: - self.count("huntress.siem.errors", 1, tags=[ - "error_type:auth_failure"]) - raise Exception( - "Huntress API 401 Unauthorized — check huntress_api_key and huntress_secret_key") + self.count("huntress.siem.errors", 1, tags=["error_type:auth_failure"]) + raise Exception("Huntress API 401 Unauthorized — check huntress_api_key and huntress_secret_key") if resp.status_code == 404: - raise Exception( - "Huntress API 404 — SIEM feature may not be enabled on this account") + raise Exception("Huntress API 404 — SIEM feature may not be enabled on this account") if resp.status_code == 408: if attempt < max_retries_408: @@ -392,24 +355,18 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) time.sleep(wait) attempt += 1 continue - self.count("huntress.siem.errors", 1, - tags=["error_type:timeout"]) - raise Exception( - "Huntress API 408 Query Timeout — query may be too broad") + self.count("huntress.siem.errors", 1, tags=["error_type:timeout"]) + raise Exception("Huntress API 408 Query Timeout — query may be too broad") if resp.status_code == 413: - raise Exception( - "Huntress API 413 Memory Limit — narrow the query with KEEP or WHERE clauses") + raise Exception("Huntress API 413 Memory Limit — narrow the query with KEEP or WHERE clauses") if resp.status_code == 422: - self.count("huntress.siem.errors", 1, tags=[ - "error_type:invalid_query"]) - raise Exception( - f"Huntress API 422 Invalid ES|QL query: {resp.text}") + self.count("huntress.siem.errors", 1, tags=["error_type:invalid_query"]) + raise Exception(f"Huntress API 422 Invalid ES|QL query: {resp.text}") if resp.status_code == 429: - self.log.warning( - "Huntress API 429 Rate Limited — sleeping 60s then retrying") + self.log.warning("Huntress API 429 Rate Limited — sleeping 60s then retrying") time.sleep(60) continue @@ -426,13 +383,10 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) time.sleep(wait) attempt += 1 continue - self.count("huntress.siem.errors", 1, tags=[ - "error_type:server_error"]) - raise Exception( - f"Huntress API {resp.status_code} Server Error after {max_retries_5xx} retries") + self.count("huntress.siem.errors", 1, tags=["error_type:server_error"]) + raise Exception(f"Huntress API {resp.status_code} Server Error after {max_retries_5xx} retries") - raise Exception( - f"Huntress API unexpected status {resp.status_code}: {resp.text}") + raise Exception(f"Huntress API unexpected status {resp.status_code}: {resp.text}") except requests.exceptions.RequestException as exc: if attempt < max_retries_5xx: @@ -447,10 +401,8 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) time.sleep(wait) attempt += 1 continue - self.count("huntress.siem.errors", 1, tags=[ - "error_type:connection_error"]) - raise Exception( - f"Huntress API connection error after {max_retries_5xx} retries: {exc}") from exc + self.count("huntress.siem.errors", 1, tags=["error_type:connection_error"]) + raise Exception(f"Huntress API connection error after {max_retries_5xx} retries: {exc}") from exc # ------------------------------------------------------------------ # # Log transformation # @@ -463,8 +415,7 @@ def _extract_service(self, tags): return "huntress-siem" def _transform_log(self, raw_log, tags, service): - message = raw_log.get("log.original") or raw_log.get( - "message") or json.dumps(raw_log) + message = raw_log.get("log.original") or raw_log.get("message") or json.dumps(raw_log) timestamp = raw_log.get("@timestamp") date_ms = None @@ -473,8 +424,7 @@ def _transform_log(self, raw_log, tags, service): if isinstance(timestamp, (int, float)): date_ms = int(timestamp) else: - dt = datetime.fromisoformat( - str(timestamp).replace("Z", "+00:00")) + dt = datetime.fromisoformat(str(timestamp).replace("Z", "+00:00")) date_ms = int(dt.timestamp() * 1000) except Exception: pass @@ -520,8 +470,7 @@ def _load_checkpoint(self, checkpoint_key): def _save_checkpoint(self, checkpoint_key, timestamp_iso): key = self.CHECKPOINT_CACHE_KEY_PREFIX + checkpoint_key - payload = json.dumps( - {"last_collected_at": timestamp_iso, "schema_version": 1}) + payload = json.dumps({"last_collected_at": timestamp_iso, "schema_version": 1}) self.write_persistent_cache(key, payload) # ------------------------------------------------------------------ # @@ -534,8 +483,7 @@ def _get_or_refresh_org_cache(self, base_url, headers, instance_hash, ttl_second if cached: try: fetched_at_str = cached.get("fetched_at", "") - fetched_at = datetime.fromisoformat( - fetched_at_str.replace("Z", "+00:00")) + fetched_at = datetime.fromisoformat(fetched_at_str.replace("Z", "+00:00")) age = (datetime.now(timezone.utc) - fetched_at).total_seconds() if ttl_seconds == 0 or age < ttl_seconds: return cached @@ -571,10 +519,8 @@ def _fetch_org_cache(self, base_url, headers, account_id): orgs = {} page = 1 while True: - url = base_url + \ - self.HUNTRESS_ORGS_ENDPOINT.format(account_id=account_id) - resp = self._request_with_retry("GET", url, headers, params={ - "limit": 500, "page": page}) + url = base_url + self.HUNTRESS_ORGS_ENDPOINT.format(account_id=account_id) + resp = self._request_with_retry("GET", url, headers, params={"limit": 500, "page": page}) data = resp.json() org_list = data.get("organizations", []) for org in org_list: @@ -585,8 +531,7 @@ def _fetch_org_cache(self, base_url, headers, account_id): } pagination = data.get("pagination", {}) if pagination.get("next_page_token") or ( - pagination.get("current_page", 1) < pagination.get( - "total_pages", 1) + pagination.get("current_page", 1) < pagination.get("total_pages", 1) ): page += 1 else: @@ -621,8 +566,7 @@ def _get_org_tags(self, raw_log, org_cache): # Strategy 1: match by organization.id org_id = raw_log.get("organization.id") or ( - raw_log.get("organization", {}).get("id") if isinstance( - raw_log.get("organization"), dict) else None + raw_log.get("organization", {}).get("id") if isinstance(raw_log.get("organization"), dict) else None ) if org_id is not None: org = orgs.get(str(org_id)) @@ -635,8 +579,7 @@ def _get_org_tags(self, raw_log, org_cache): # Strategy 2: match by organization.name (reverse lookup) org_name = raw_log.get("organization.name") or ( - raw_log.get("organization", {}).get("name") if isinstance( - raw_log.get("organization"), dict) else None + raw_log.get("organization", {}).get("name") if isinstance(raw_log.get("organization"), dict) else None ) if org_name: for oid, org in orgs.items(): @@ -656,8 +599,7 @@ def _get_org_tags(self, raw_log, org_cache): def _instance_hash(self, instance): """Stable non-secret identifier for this Huntress instance.""" - key = instance.get("instance_id") or instance.get( - "huntress_base_url", self.DEFAULT_BASE_URL) + key = instance.get("instance_id") or instance.get("huntress_base_url", self.DEFAULT_BASE_URL) return format(zlib.crc32(key.encode("utf-8")) & 0xFFFFFFFF, "08x") def _query_hash(self, esql_query): diff --git a/huntress/datadog_checks/huntress/huntress.py b/huntress/datadog_checks/huntress/huntress.py index 76da61bc64..00006654fe 100644 --- a/huntress/datadog_checks/huntress/huntress.py +++ b/huntress/datadog_checks/huntress/huntress.py @@ -1,7 +1,7 @@ import base64 -import zlib import json import time +import zlib from datetime import datetime, timedelta, timezone import requests @@ -43,23 +43,18 @@ def check(self, instance): api_key = instance.get("huntress_api_key", "").strip() secret_key = instance.get("huntress_secret_key", "").strip() - base_url = instance.get("huntress_base_url", - self.DEFAULT_BASE_URL).rstrip("/") - max_pages = int(instance.get("max_pages_per_run", - self.DEFAULT_MAX_PAGES_PER_RUN)) - min_interval = int(instance.get( - "min_collection_interval", self.DEFAULT_MIN_COLLECTION_INTERVAL)) + base_url = instance.get("huntress_base_url", self.DEFAULT_BASE_URL).rstrip("/") + max_pages = int(instance.get("max_pages_per_run", self.DEFAULT_MAX_PAGES_PER_RUN)) + min_interval = int(instance.get("min_collection_interval", self.DEFAULT_MIN_COLLECTION_INTERVAL)) enrich_orgs = instance.get("enrich_with_org_tags", True) - org_ttl = int(instance.get("org_cache_ttl_seconds", - self.DEFAULT_ORG_CACHE_TTL)) + org_ttl = int(instance.get("org_cache_ttl_seconds", self.DEFAULT_ORG_CACHE_TTL)) extra_tags = list(instance.get("tags", [])) log_queries = instance.get("log_queries") or [] metrics_config = instance.get("metrics") or {} agents_config = metrics_config.get("agents") or {} collect_agents = bool(agents_config.get("enabled", False)) - agents_max_pages = int(agents_config.get( - "max_pages", self.DEFAULT_AGENT_MAX_PAGES)) + agents_max_pages = int(agents_config.get("max_pages", self.DEFAULT_AGENT_MAX_PAGES)) if not api_key: raise ConfigurationError("huntress_api_key is required") @@ -77,8 +72,7 @@ def check(self, instance): # Org enrichment cache is shared across all queries for this instance org_cache = None if enrich_orgs and log_queries: - org_cache = self._get_or_refresh_org_cache( - base_url, headers, instance_hash, org_ttl) + org_cache = self._get_or_refresh_org_cache(base_url, headers, instance_hash, org_ttl) success = False @@ -89,14 +83,11 @@ def check(self, instance): query_tags = list(query_def.get("tags", [])) if not query_name: - raise ConfigurationError( - "Each entry in log_queries must have a non-empty 'name'") + raise ConfigurationError("Each entry in log_queries must have a non-empty 'name'") if not esql: - raise ConfigurationError( - "Each entry in log_queries must have a non-empty esql_query") + raise ConfigurationError("Each entry in log_queries must have a non-empty esql_query") if not esql.lower().lstrip().startswith("from logs"): - raise ConfigurationError( - f"esql_query must begin with 'FROM logs' (case-insensitive): {esql!r}") + raise ConfigurationError(f"esql_query must begin with 'FROM logs' (case-insensitive): {esql!r}") # Each query tracks its own checkpoint so queries are independently resumable checkpoint_key = instance_hash + "_" + self._query_hash(esql) @@ -107,39 +98,30 @@ def check(self, instance): base_url, headers, esql, checkpoint_key, max_pages, min_interval, org_cache, all_tags ) - self.gauge("huntress.siem.logs_collected", - logs, tags=query_metric_tags) - self.gauge("huntress.siem.pages_fetched", - pages, tags=query_metric_tags) + self.gauge("huntress.siem.logs_collected", logs, tags=query_metric_tags) + self.gauge("huntress.siem.pages_fetched", pages, tags=query_metric_tags) if collect_agents: - self._collect_agent_metrics( - base_url, headers, extra_tags, agents_max_pages) + self._collect_agent_metrics(base_url, headers, extra_tags, agents_max_pages) success = True except Exception as exc: self.log.error("Huntress check run failed: %s", exc) - self.count("huntress.siem.errors", 1, - tags=extra_tags + ["error_type:run_failure"]) - self.service_check(self.SERVICE_CHECK_NAME, - self.CRITICAL, tags=extra_tags) + self.count("huntress.siem.errors", 1, tags=extra_tags + ["error_type:run_failure"]) + self.service_check(self.SERVICE_CHECK_NAME, self.CRITICAL, tags=extra_tags) raise finally: duration = time.time() - start_time - self.gauge("huntress.siem.run_duration_seconds", - duration, tags=extra_tags) + self.gauge("huntress.siem.run_duration_seconds", duration, tags=extra_tags) if self._last_api_call_limit is not None: - self.gauge("huntress.siem.api_call_limit", - self._last_api_call_limit, tags=extra_tags) + self.gauge("huntress.siem.api_call_limit", self._last_api_call_limit, tags=extra_tags) if self._last_api_call_remaining is not None: - self.gauge("huntress.siem.api_call_remaining", - self._last_api_call_remaining, tags=extra_tags) + self.gauge("huntress.siem.api_call_remaining", self._last_api_call_remaining, tags=extra_tags) if success: - self.service_check(self.SERVICE_CHECK_NAME, - self.OK, tags=extra_tags) + self.service_check(self.SERVICE_CHECK_NAME, self.OK, tags=extra_tags) # ------------------------------------------------------------------ # # Per-query SIEM execution # @@ -153,8 +135,7 @@ def _run_query(self, base_url, headers, esql, checkpoint_key, max_pages, min_int if range_start is None: default_start = now - timedelta(seconds=min_interval) - range_start = default_start.strftime( - "%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + range_start = default_start.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" else: # Add 1ms to avoid re-fetching the boundary event from the previous run try: @@ -178,16 +159,13 @@ def _run_query(self, base_url, headers, esql, checkpoint_key, max_pages, min_int hit_page_cap = False while True: - logs, next_token = self._query_page( - base_url, headers, esql, range_start, range_end, page_token) + logs, next_token = self._query_page(base_url, headers, esql, range_start, range_end, page_token) if logs: batch = [] for raw in logs: - org_tags = self._get_org_tags( - raw, org_cache) if org_cache else [] - payload = self._transform_log( - raw, all_tags + org_tags, service_tag) + org_tags = self._get_org_tags(raw, org_cache) if org_cache else [] + payload = self._transform_log(raw, all_tags + org_tags, service_tag) batch.append(payload) if len(batch) >= self.MAX_LOGS_PER_BATCH: self._send_logs_batch(batch) @@ -257,31 +235,23 @@ def _collect_agent_metrics(self, base_url, headers, extra_tags, max_pages): platform = (agent.get("platform") or "unknown").lower() by_platform[platform] = by_platform.get(platform, 0) + 1 - def_status = (agent.get("defender_status") - or "unknown").lower().replace(" ", "_") - by_defender_status[def_status] = by_defender_status.get( - def_status, 0) + 1 + def_status = (agent.get("defender_status") or "unknown").lower().replace(" ", "_") + by_defender_status[def_status] = by_defender_status.get(def_status, 0) + 1 - fw_status = (agent.get("firewall_status") - or "unknown").lower().replace(" ", "_") - by_firewall_status[fw_status] = by_firewall_status.get( - fw_status, 0) + 1 + fw_status = (agent.get("firewall_status") or "unknown").lower().replace(" ", "_") + by_firewall_status[fw_status] = by_firewall_status.get(fw_status, 0) + 1 self.gauge("huntress.agents.total", len(agents), tags=extra_tags) - self.gauge("huntress.agents.pages_fetched", - pages_fetched, tags=extra_tags) + self.gauge("huntress.agents.pages_fetched", pages_fetched, tags=extra_tags) for platform, count in by_platform.items(): - self.gauge("huntress.agents.count", count, - tags=extra_tags + [f"platform:{platform}"]) + self.gauge("huntress.agents.count", count, tags=extra_tags + [f"platform:{platform}"]) for status, count in by_defender_status.items(): - self.gauge("huntress.agents.defender_status", count, - tags=extra_tags + [f"defender_status:{status}"]) + self.gauge("huntress.agents.defender_status", count, tags=extra_tags + [f"defender_status:{status}"]) for status, count in by_firewall_status.items(): - self.gauge("huntress.agents.firewall_status", count, - tags=extra_tags + [f"firewall_status:{status}"]) + self.gauge("huntress.agents.firewall_status", count, tags=extra_tags + [f"firewall_status:{status}"]) # ------------------------------------------------------------------ # # Auth # @@ -326,8 +296,7 @@ def _query_page(self, base_url, headers, esql, range_start, range_end, page_toke if page_token: body["page_token"] = page_token - response = self._request_with_retry( - method="POST", url=url, headers=headers, json_body=body) + response = self._request_with_retry(method="POST", url=url, headers=headers, json_body=body) data = response.json() logs = data.get("logs", []) pagination = data.get("pagination", {}) @@ -340,8 +309,7 @@ def _query_page(self, base_url, headers, esql, range_start, range_end, page_toke def _request_with_retry(self, method, url, headers, json_body=None, params=None): """Execute an HTTP request with retry logic per PRD §7.""" - timeout = self.init_config.get( - "request_timeout", self.DEFAULT_REQUEST_TIMEOUT) + timeout = self.init_config.get("request_timeout", self.DEFAULT_REQUEST_TIMEOUT) max_retries_5xx = 3 max_retries_408 = 2 backoff_5xx = [5, 10, 20] @@ -365,20 +333,15 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) return resp if resp.status_code == 400: - self.count("huntress.siem.errors", 1, - tags=["error_type:bad_request"]) - raise Exception( - f"Huntress API 400 Bad Request: {resp.text}") + self.count("huntress.siem.errors", 1, tags=["error_type:bad_request"]) + raise Exception(f"Huntress API 400 Bad Request: {resp.text}") if resp.status_code == 401: - self.count("huntress.siem.errors", 1, tags=[ - "error_type:auth_failure"]) - raise Exception( - "Huntress API 401 Unauthorized — check huntress_api_key and huntress_secret_key") + self.count("huntress.siem.errors", 1, tags=["error_type:auth_failure"]) + raise Exception("Huntress API 401 Unauthorized — check huntress_api_key and huntress_secret_key") if resp.status_code == 404: - raise Exception( - "Huntress API 404 — SIEM feature may not be enabled on this account") + raise Exception("Huntress API 404 — SIEM feature may not be enabled on this account") if resp.status_code == 408: if attempt < max_retries_408: @@ -392,24 +355,18 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) time.sleep(wait) attempt += 1 continue - self.count("huntress.siem.errors", 1, - tags=["error_type:timeout"]) - raise Exception( - "Huntress API 408 Query Timeout — query may be too broad") + self.count("huntress.siem.errors", 1, tags=["error_type:timeout"]) + raise Exception("Huntress API 408 Query Timeout — query may be too broad") if resp.status_code == 413: - raise Exception( - "Huntress API 413 Memory Limit — narrow the query with KEEP or WHERE clauses") + raise Exception("Huntress API 413 Memory Limit — narrow the query with KEEP or WHERE clauses") if resp.status_code == 422: - self.count("huntress.siem.errors", 1, tags=[ - "error_type:invalid_query"]) - raise Exception( - f"Huntress API 422 Invalid ES|QL query: {resp.text}") + self.count("huntress.siem.errors", 1, tags=["error_type:invalid_query"]) + raise Exception(f"Huntress API 422 Invalid ES|QL query: {resp.text}") if resp.status_code == 429: - self.log.warning( - "Huntress API 429 Rate Limited — sleeping 60s then retrying") + self.log.warning("Huntress API 429 Rate Limited — sleeping 60s then retrying") time.sleep(60) continue @@ -426,13 +383,10 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) time.sleep(wait) attempt += 1 continue - self.count("huntress.siem.errors", 1, tags=[ - "error_type:server_error"]) - raise Exception( - f"Huntress API {resp.status_code} Server Error after {max_retries_5xx} retries") + self.count("huntress.siem.errors", 1, tags=["error_type:server_error"]) + raise Exception(f"Huntress API {resp.status_code} Server Error after {max_retries_5xx} retries") - raise Exception( - f"Huntress API unexpected status {resp.status_code}: {resp.text}") + raise Exception(f"Huntress API unexpected status {resp.status_code}: {resp.text}") except requests.exceptions.RequestException as exc: if attempt < max_retries_5xx: @@ -447,10 +401,8 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) time.sleep(wait) attempt += 1 continue - self.count("huntress.siem.errors", 1, tags=[ - "error_type:connection_error"]) - raise Exception( - f"Huntress API connection error after {max_retries_5xx} retries: {exc}") from exc + self.count("huntress.siem.errors", 1, tags=["error_type:connection_error"]) + raise Exception(f"Huntress API connection error after {max_retries_5xx} retries: {exc}") from exc # ------------------------------------------------------------------ # # Log transformation # @@ -463,8 +415,7 @@ def _extract_service(self, tags): return "huntress-siem" def _transform_log(self, raw_log, tags, service): - message = raw_log.get("log.original") or raw_log.get( - "message") or json.dumps(raw_log) + message = raw_log.get("log.original") or raw_log.get("message") or json.dumps(raw_log) timestamp = raw_log.get("@timestamp") date_ms = None @@ -473,8 +424,7 @@ def _transform_log(self, raw_log, tags, service): if isinstance(timestamp, (int, float)): date_ms = int(timestamp) else: - dt = datetime.fromisoformat( - str(timestamp).replace("Z", "+00:00")) + dt = datetime.fromisoformat(str(timestamp).replace("Z", "+00:00")) date_ms = int(dt.timestamp() * 1000) except Exception: pass @@ -520,8 +470,7 @@ def _load_checkpoint(self, checkpoint_key): def _save_checkpoint(self, checkpoint_key, timestamp_iso): key = self.CHECKPOINT_CACHE_KEY_PREFIX + checkpoint_key - payload = json.dumps( - {"last_collected_at": timestamp_iso, "schema_version": 1}) + payload = json.dumps({"last_collected_at": timestamp_iso, "schema_version": 1}) self.write_persistent_cache(key, payload) # ------------------------------------------------------------------ # @@ -534,8 +483,7 @@ def _get_or_refresh_org_cache(self, base_url, headers, instance_hash, ttl_second if cached: try: fetched_at_str = cached.get("fetched_at", "") - fetched_at = datetime.fromisoformat( - fetched_at_str.replace("Z", "+00:00")) + fetched_at = datetime.fromisoformat(fetched_at_str.replace("Z", "+00:00")) age = (datetime.now(timezone.utc) - fetched_at).total_seconds() if ttl_seconds == 0 or age < ttl_seconds: return cached @@ -571,10 +519,8 @@ def _fetch_org_cache(self, base_url, headers, account_id): orgs = {} page = 1 while True: - url = base_url + \ - self.HUNTRESS_ORGS_ENDPOINT.format(account_id=account_id) - resp = self._request_with_retry("GET", url, headers, params={ - "limit": 500, "page": page}) + url = base_url + self.HUNTRESS_ORGS_ENDPOINT.format(account_id=account_id) + resp = self._request_with_retry("GET", url, headers, params={"limit": 500, "page": page}) data = resp.json() org_list = data.get("organizations", []) for org in org_list: @@ -585,8 +531,7 @@ def _fetch_org_cache(self, base_url, headers, account_id): } pagination = data.get("pagination", {}) if pagination.get("next_page_token") or ( - pagination.get("current_page", 1) < pagination.get( - "total_pages", 1) + pagination.get("current_page", 1) < pagination.get("total_pages", 1) ): page += 1 else: @@ -621,8 +566,7 @@ def _get_org_tags(self, raw_log, org_cache): # Strategy 1: match by organization.id org_id = raw_log.get("organization.id") or ( - raw_log.get("organization", {}).get("id") if isinstance( - raw_log.get("organization"), dict) else None + raw_log.get("organization", {}).get("id") if isinstance(raw_log.get("organization"), dict) else None ) if org_id is not None: org = orgs.get(str(org_id)) @@ -635,8 +579,7 @@ def _get_org_tags(self, raw_log, org_cache): # Strategy 2: match by organization.name (reverse lookup) org_name = raw_log.get("organization.name") or ( - raw_log.get("organization", {}).get("name") if isinstance( - raw_log.get("organization"), dict) else None + raw_log.get("organization", {}).get("name") if isinstance(raw_log.get("organization"), dict) else None ) if org_name: for oid, org in orgs.items(): @@ -656,8 +599,7 @@ def _get_org_tags(self, raw_log, org_cache): def _instance_hash(self, instance): """Stable non-secret identifier for this Huntress instance.""" - key = instance.get("instance_id") or instance.get( - "huntress_base_url", self.DEFAULT_BASE_URL) + key = instance.get("instance_id") or instance.get("huntress_base_url", self.DEFAULT_BASE_URL) return format(zlib.crc32(key.encode("utf-8")) & 0xFFFFFFFF, "08x") def _query_hash(self, esql_query): diff --git a/huntress/tests/test_huntress.py b/huntress/tests/test_huntress.py index 2106866382..06d9f8c4bf 100644 --- a/huntress/tests/test_huntress.py +++ b/huntress/tests/test_huntress.py @@ -40,8 +40,7 @@ def _make_check(**kwargs): # Use an in-memory dict to prevent persistent cache from bleeding between tests _cache = {} check.read_persistent_cache = lambda key: _cache.get(key) - check.write_persistent_cache = lambda key, value: _cache.__setitem__( - key, value) + check.write_persistent_cache = lambda key, value: _cache.__setitem__(key, value) return check, inst @@ -73,8 +72,7 @@ def test_happy_path_single_page(): fixture["logs"] = _load_fixture("siem_query_page1.json")["logs"][:1] with ( - patch.object(check, "_request_with_retry", - return_value=_mock_response(200, fixture)), + patch.object(check, "_request_with_retry", return_value=_mock_response(200, fixture)), patch.object(check, "_send_logs_batch") as mock_send, patch.object(check, "_save_checkpoint") as mock_save, patch("time.time", side_effect=[1000.0, 1001.0]), @@ -113,8 +111,7 @@ def test_happy_path_multi_page(): check.check(instance) assert mock_send.call_count == 2 - all_logs = mock_send.call_args_list[0][0][0] + \ - mock_send.call_args_list[1][0][0] + all_logs = mock_send.call_args_list[0][0][0] + mock_send.call_args_list[1][0][0] assert len(all_logs) == 3 mock_save.assert_called_once() @@ -128,8 +125,7 @@ def test_auth_failure_401(): check, instance = _make_check() with ( - patch.object(check, "_request_with_retry", - side_effect=Exception("Huntress API 401 Unauthorized")), + patch.object(check, "_request_with_retry", side_effect=Exception("Huntress API 401 Unauthorized")), patch.object(check, "_save_checkpoint") as mock_save, patch.object(check, "count"), patch("time.time", side_effect=[1000.0, 1001.0]), @@ -147,11 +143,9 @@ def test_auth_failure_401_raises_via_retry(): with patch("requests.request", return_value=resp), patch.object(check, "count") as mock_count: with pytest.raises(Exception, match="401"): - check._request_with_retry( - "POST", "https://api.huntress.io/v1/siem/query", {}, json_body={}) + check._request_with_retry("POST", "https://api.huntress.io/v1/siem/query", {}, json_body={}) - mock_count.assert_called_with("huntress.siem.errors", 1, tags=[ - "error_type:auth_failure"]) + mock_count.assert_called_with("huntress.siem.errors", 1, tags=["error_type:auth_failure"]) # =========================================================================== @@ -175,8 +169,7 @@ def test_query_timeout_408(): assert mock_sleep.call_count == 2 sleep_args = [c[0][0] for c in mock_sleep.call_args_list] assert sleep_args == [2, 4] - mock_count.assert_called_with( - "huntress.siem.errors", 1, tags=["error_type:timeout"]) + mock_count.assert_called_with("huntress.siem.errors", 1, tags=["error_type:timeout"]) # =========================================================================== @@ -190,8 +183,7 @@ def test_rate_limit_429_then_success(): resp_200 = _mock_response(200, _load_fixture("siem_query_empty.json")) with patch("requests.request", side_effect=[resp_429, resp_200]), patch("time.sleep") as mock_sleep: - result = check._request_with_retry( - "POST", "https://x/q", {}, json_body={}) + result = check._request_with_retry("POST", "https://x/q", {}, json_body={}) mock_sleep.assert_called_once_with(60) assert result.status_code == 200 @@ -210,8 +202,7 @@ def test_invalid_esql_422(): with pytest.raises(Exception, match="422"): check._request_with_retry("POST", "https://x/q", {}, json_body={}) - mock_count.assert_called_with("huntress.siem.errors", 1, tags=[ - "error_type:invalid_query"]) + mock_count.assert_called_with("huntress.siem.errors", 1, tags=["error_type:invalid_query"]) # =========================================================================== @@ -238,8 +229,7 @@ def capture_request(method, url, headers, json_body=None, params=None): return _mock_response(200, _load_fixture("siem_query_empty.json")) with ( - patch.object(check, "_request_with_retry", - side_effect=capture_request), + patch.object(check, "_request_with_retry", side_effect=capture_request), patch("time.time", side_effect=[1000.0, 1001.0]), ): check.check(instance) @@ -265,8 +255,7 @@ def capture_request(method, url, headers, json_body=None, params=None): fixed_now = datetime(2026, 5, 27, 14, 0, 0, tzinfo=timezone.utc) with ( - patch.object(check, "_request_with_retry", - side_effect=capture_request), + patch.object(check, "_request_with_retry", side_effect=capture_request), patch("datadog_checks.huntress.huntress.datetime") as mock_dt, patch("time.time", side_effect=[1000.0, 1001.0]), ): @@ -276,10 +265,8 @@ def capture_request(method, url, headers, json_body=None, params=None): assert captured_bodies body = captured_bodies[0] - range_start = datetime.fromisoformat( - body["range_start"].replace("Z", "+00:00")) - range_end = datetime.fromisoformat( - body["range_end"].replace("Z", "+00:00")) + range_start = datetime.fromisoformat(body["range_start"].replace("Z", "+00:00")) + range_end = datetime.fromisoformat(body["range_end"].replace("Z", "+00:00")) diff_seconds = (range_end - range_start).total_seconds() assert abs(diff_seconds - 900) < 2 @@ -296,8 +283,7 @@ def test_esql_validation_rejects_bad_query(): def test_esql_validation_accepts_from_logs(): - check, instance = _make_check( - esql_query="FROM logs | KEEP @timestamp, message") + check, instance = _make_check(esql_query="FROM logs | KEEP @timestamp, message") with ( patch.object( check, "_request_with_retry", return_value=_mock_response(200, _load_fixture("siem_query_empty.json")) @@ -349,16 +335,14 @@ def test_log_transformation(): def test_log_transformation_fallback_message(): check, instance = _make_check() - raw = {"message": "fallback used", - "@timestamp": "2026-05-27T14:00:00.000Z"} + raw = {"message": "fallback used", "@timestamp": "2026-05-27T14:00:00.000Z"} payload = check._transform_log(raw, [], "huntress-siem") assert payload["message"] == "fallback used" def test_log_transformation_json_fallback(): check, instance = _make_check() - raw = {"event.category": "network", - "@timestamp": "2026-05-27T14:00:00.000Z"} + raw = {"event.category": "network", "@timestamp": "2026-05-27T14:00:00.000Z"} payload = check._transform_log(raw, [], "huntress-siem") assert "event.category" in payload["message"] @@ -381,8 +365,7 @@ def test_batching_over_1000_logs(): fixture = {"logs": large_log_list, "pagination": {}} with ( - patch.object(check, "_request_with_retry", - return_value=_mock_response(200, fixture)), + patch.object(check, "_request_with_retry", return_value=_mock_response(200, fixture)), patch.object(check, "_send_logs_batch") as mock_send, patch("time.time", side_effect=[1000.0, 1001.0]), ): @@ -405,8 +388,7 @@ def test_max_pages_per_run_cap(): page1 = _load_fixture("siem_query_page1.json") with ( - patch.object(check, "_request_with_retry", - return_value=_mock_response(200, page1)), + patch.object(check, "_request_with_retry", return_value=_mock_response(200, page1)), patch.object(check, "_save_checkpoint") as mock_save, patch("time.time", side_effect=[1000.0, 1001.0]), ): @@ -434,8 +416,7 @@ def test_org_enrichment_cache_hit(): "101": {"name": "Acme Inc.", "key": "acme", "account_id": 42}, }, } - check.write_persistent_cache( - check.ORG_CACHE_KEY_PREFIX + instance_hash, json.dumps(cache)) + check.write_persistent_cache(check.ORG_CACHE_KEY_PREFIX + instance_hash, json.dumps(cache)) page1 = _load_fixture("siem_query_page1.json") page1["pagination"] = {} @@ -562,8 +543,7 @@ def test_org_enrichment_no_org_match(): "account_id": 42, "orgs": {"101": {"name": "Acme Inc.", "key": "acme", "account_id": 42}}, } - raw_log = {"@timestamp": "2026-05-27T14:00:00.000Z", - "message": "no org field"} + raw_log = {"@timestamp": "2026-05-27T14:00:00.000Z", "message": "no org field"} tags = check._get_org_tags(raw_log, org_cache) assert "huntress_account_id:42" in tags @@ -591,15 +571,13 @@ def test_multi_instance_isolation(): check_a.log = MagicMock() _cache_a = {} check_a.read_persistent_cache = lambda key: _cache_a.get(key) - check_a.write_persistent_cache = lambda key, value: _cache_a.__setitem__( - key, value) + check_a.write_persistent_cache = lambda key, value: _cache_a.__setitem__(key, value) check_b = HuntressCheck("huntress", {}, [instance_b]) check_b.log = MagicMock() _cache_b = {} check_b.read_persistent_cache = lambda key: _cache_b.get(key) - check_b.write_persistent_cache = lambda key, value: _cache_b.__setitem__( - key, value) + check_b.write_persistent_cache = lambda key, value: _cache_b.__setitem__(key, value) hash_a = check_a._instance_hash(instance_a) hash_b = check_b._instance_hash(instance_b) @@ -714,8 +692,7 @@ def test_rate_limit_headers_missing_gracefully(): check._last_api_call_limit = None check._last_api_call_remaining = None - resp = _mock_response( - 200, {}, headers={"Content-Type": "application/json"}) + resp = _mock_response(200, {}, headers={"Content-Type": "application/json"}) check._parse_rate_limit_headers(resp) assert check._last_api_call_limit is None @@ -770,8 +747,7 @@ def test_validation_neither_configured_raises(): check.log = MagicMock() _cache = {} check.read_persistent_cache = lambda key: _cache.get(key) - check.write_persistent_cache = lambda key, value: _cache.__setitem__( - key, value) + check.write_persistent_cache = lambda key, value: _cache.__setitem__(key, value) with pytest.raises(Exception, match="at least one"): check.check(instance) @@ -787,8 +763,7 @@ def test_validation_metrics_only_instance(): check.log = MagicMock() _cache = {} check.read_persistent_cache = lambda key: _cache.get(key) - check.write_persistent_cache = lambda key, value: _cache.__setitem__( - key, value) + check.write_persistent_cache = lambda key, value: _cache.__setitem__(key, value) agents_resp = _mock_response(200, {"agents": [], "pagination": {}}) with ( @@ -821,14 +796,11 @@ def test_agent_metrics_emitted(): check.log = MagicMock() _cache = {} check.read_persistent_cache = lambda key: _cache.get(key) - check.write_persistent_cache = lambda key, value: _cache.__setitem__( - key, value) + check.write_persistent_cache = lambda key, value: _cache.__setitem__(key, value) agents = [ - {"platform": "windows", "defender_status": "Healthy", - "firewall_status": "Enabled"}, - {"platform": "windows", "defender_status": "Healthy", - "firewall_status": "Disabled"}, + {"platform": "windows", "defender_status": "Healthy", "firewall_status": "Enabled"}, + {"platform": "windows", "defender_status": "Healthy", "firewall_status": "Disabled"}, {"platform": "darwin", "defender_status": None, "firewall_status": "Enabled"}, ] agents_resp = _mock_response(200, {"agents": agents, "pagination": {}}) @@ -846,30 +818,22 @@ def capture_gauge(name, value, tags=None): check.check(instance) assert emitted[("huntress.agents.total", ("source:huntress",))] == 3 - assert emitted[("huntress.agents.count", - ("platform:windows", "source:huntress"))] == 2 - assert emitted[("huntress.agents.count", - ("platform:darwin", "source:huntress"))] == 1 - assert emitted[("huntress.agents.defender_status", - ("defender_status:healthy", "source:huntress"))] == 2 - assert emitted[("huntress.agents.defender_status", - ("defender_status:unknown", "source:huntress"))] == 1 - assert emitted[("huntress.agents.firewall_status", - ("firewall_status:enabled", "source:huntress"))] == 2 - assert emitted[("huntress.agents.firewall_status", - ("firewall_status:disabled", "source:huntress"))] == 1 + assert emitted[("huntress.agents.count", ("platform:windows", "source:huntress"))] == 2 + assert emitted[("huntress.agents.count", ("platform:darwin", "source:huntress"))] == 1 + assert emitted[("huntress.agents.defender_status", ("defender_status:healthy", "source:huntress"))] == 2 + assert emitted[("huntress.agents.defender_status", ("defender_status:unknown", "source:huntress"))] == 1 + assert emitted[("huntress.agents.firewall_status", ("firewall_status:enabled", "source:huntress"))] == 2 + assert emitted[("huntress.agents.firewall_status", ("firewall_status:disabled", "source:huntress"))] == 1 def test_agent_metrics_pagination(): """Agent collection paginates until no next_page_token.""" - instance = _make_agents_instance( - metrics={"agents": {"enabled": True, "max_pages": 5}}) + instance = _make_agents_instance(metrics={"agents": {"enabled": True, "max_pages": 5}}) check = HuntressCheck("huntress", {}, [instance]) check.log = MagicMock() _cache = {} check.read_persistent_cache = lambda key: _cache.get(key) - check.write_persistent_cache = lambda key, value: _cache.__setitem__( - key, value) + check.write_persistent_cache = lambda key, value: _cache.__setitem__(key, value) page1 = { "agents": [{"platform": "windows", "defender_status": "Healthy", "firewall_status": "Enabled"}], @@ -900,23 +864,19 @@ def capture_gauge(name, value, tags=None): check.check(instance) assert emitted[("huntress.agents.total", ("source:huntress",))] == 2 - assert emitted[("huntress.agents.count", - ("platform:windows", "source:huntress"))] == 1 - assert emitted[("huntress.agents.count", - ("platform:linux", "source:huntress"))] == 1 + assert emitted[("huntress.agents.count", ("platform:windows", "source:huntress"))] == 1 + assert emitted[("huntress.agents.count", ("platform:linux", "source:huntress"))] == 1 assert len(call_urls) == 2 def test_agent_metrics_max_pages_cap(): """Agent collection stops at max_pages even when more pages exist.""" - instance = _make_agents_instance( - metrics={"agents": {"enabled": True, "max_pages": 1}}) + instance = _make_agents_instance(metrics={"agents": {"enabled": True, "max_pages": 1}}) check = HuntressCheck("huntress", {}, [instance]) check.log = MagicMock() _cache = {} check.read_persistent_cache = lambda key: _cache.get(key) - check.write_persistent_cache = lambda key, value: _cache.__setitem__( - key, value) + check.write_persistent_cache = lambda key, value: _cache.__setitem__(key, value) page_with_more = { "agents": [{"platform": "windows", "defender_status": "Healthy", "firewall_status": "Enabled"}], @@ -924,8 +884,7 @@ def test_agent_metrics_max_pages_cap(): } with ( - patch.object(check, "_request_with_retry", - return_value=_mock_response(200, page_with_more)), + patch.object(check, "_request_with_retry", return_value=_mock_response(200, page_with_more)), patch("time.time", side_effect=[1000.0, 1001.0]), ): check.check(instance) From 57b2113d3c7dc531e1c666dac975acb1186a6fe5 Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Fri, 29 May 2026 18:06:10 -0500 Subject: [PATCH 17/40] Update codecov (add huntress) --- .codecov.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.codecov.yml b/.codecov.yml index d11d93b364..e05b33f017 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -67,6 +67,10 @@ coverage: target: 75 flags: - grafana + Huntress: + target: 75 + flags: + - huntress JFrog_Platform: target: 75 flags: @@ -469,6 +473,11 @@ flags: paths: - hikaricp/datadog_checks/hikaricp - hikaricp/tests + huntress: + carryforward: true + paths: + - huntress/datadog_checks/huntress + - huntress/tests jfrog_platform_self_hosted: carryforward: true paths: From a92e10d2b252b41b14b268fae9aaa3484730ea01 Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Fri, 29 May 2026 18:16:27 -0500 Subject: [PATCH 18/40] Add Huntress to test-all.yml --- .github/workflows/test-all.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index 7a5be903a9..94b0871654 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -538,6 +538,25 @@ jobs: test-py3: ${{ inputs.test-py3 }} setup-env-vars: "${{ inputs.setup-env-vars }}" secrets: inherit + j8ecbaad: + uses: DataDog/integrations-core/.github/workflows/test-target.yml@574d63ba88365ffbab915280ceddbaa333c63d6a + with: + job-name: Huntress + target: huntress + platform: linux + runner: '["ubuntu-22.04"]' + repo: "${{ inputs.repo }}" + context: ${{ inputs.context }} + python-version: "${{ inputs.python-version }}" + latest: ${{ inputs.latest }} + agent-image: "${{ inputs.agent-image }}" + agent-image-py2: "${{ inputs.agent-image-py2 }}" + agent-image-windows: "${{ inputs.agent-image-windows }}" + agent-image-windows-py2: "${{ inputs.agent-image-windows-py2 }}" + test-py2: ${{ inputs.test-py2 }} + test-py3: ${{ inputs.test-py3 }} + setup-env-vars: "${{ inputs.setup-env-vars }}" + secrets: inherit j43aee0d: uses: DataDog/integrations-core/.github/workflows/test-target.yml@574d63ba88365ffbab915280ceddbaa333c63d6a with: From c31bf35492f7a7e6175d3abccb80905711e730b2 Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Fri, 29 May 2026 18:29:47 -0500 Subject: [PATCH 19/40] Add missing spec item --- huntress/assets/configuration/spec.yaml | 2 ++ huntress/datadog_checks/huntress/data/conf.yaml.example | 2 ++ 2 files changed, 4 insertions(+) diff --git a/huntress/assets/configuration/spec.yaml b/huntress/assets/configuration/spec.yaml index ad3f55cb90..78dcb2116f 100644 --- a/huntress/assets/configuration/spec.yaml +++ b/huntress/assets/configuration/spec.yaml @@ -61,6 +61,8 @@ files: example: - name: "all-logs" esql_query: "FROM logs" + tags: + - "log_type:all" - name: enrich_with_org_tags description: | When true, the check fetches Huntress organization metadata (org name, diff --git a/huntress/datadog_checks/huntress/data/conf.yaml.example b/huntress/datadog_checks/huntress/data/conf.yaml.example index d360300ffa..f3f59dd2a7 100644 --- a/huntress/datadog_checks/huntress/data/conf.yaml.example +++ b/huntress/datadog_checks/huntress/data/conf.yaml.example @@ -84,6 +84,8 @@ instances: # log_queries: # - name: all-logs # esql_query: FROM logs + # tags: + # - log_type:all ## @param enrich_with_org_tags - boolean - optional - default: true ## When true, the check fetches Huntress organization metadata (org name, From 9c1c205388dc62a1fad8f0486fa121a615d06f6e Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Mon, 1 Jun 2026 11:15:51 -0500 Subject: [PATCH 20/40] Remove old conf artifact --- huntress/conf.d/huntress.yaml.example | 45 --------------------------- 1 file changed, 45 deletions(-) delete mode 100644 huntress/conf.d/huntress.yaml.example diff --git a/huntress/conf.d/huntress.yaml.example b/huntress/conf.d/huntress.yaml.example deleted file mode 100644 index 95086765cf..0000000000 --- a/huntress/conf.d/huntress.yaml.example +++ /dev/null @@ -1,45 +0,0 @@ -init_config: {} - -instances: - ## --- Account 1 --- - - ## Required: Huntress API credentials (Basic Auth). - ## Each instance has its own key pair, allowing multiple Huntress accounts - ## to feed into the same Datadog org simultaneously. - huntress_api_key: "" - huntress_secret_key: "" - - ## Required: ES|QL query to execute. Must begin with "FROM logs" (case-insensitive). - ## The query determines which log fields are returned. Use KEEP to select specific - ## columns, or omit for all ECS fields. Do NOT add time range filters — range_start - ## and range_end are managed automatically by the check. - esql_query: "FROM logs" - - ## Optional: Whether to enrich logs with Huntress organization metadata - ## (org name, org key, account ID). Fetches /v1/account and - ## /v1/accounts/{id}/organizations on first run and caches for org_cache_ttl_seconds. - ## Default: true - enrich_with_org_tags: true - - ## Optional: How long (seconds) to cache the org metadata before re-fetching. - ## Default: 3600 (1 hour). Set to 0 to refresh on every run. - org_cache_ttl_seconds: 3600 - - ## Optional: Collection interval in seconds. Must be >= 60 to stay within Huntress - ## rate limits (60 req/min). Default: 900 (15 minutes). - min_collection_interval: 900 - - ## Optional: Maximum number of pages to fetch per run (200 logs/page). - ## Protects against runaway pagination on large backlogs. - ## Default: 100 (up to 20,000 logs per run). - max_pages_per_run: 100 - - ## Optional: Huntress API base URL. Override for testing or sandbox environments. - huntress_base_url: "https://api.huntress.io" - - ## Optional: Additional tags attached to every log event forwarded to Datadog. - ## huntress_account_id, huntress_org_name, and huntress_org_key are added - ## automatically when enrich_with_org_tags is true. - tags: - - "source:huntress" - - "service:huntress-siem" - - "env:production" From 752339caf1363edcca36344541b2a5322ac40647 Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Tue, 2 Jun 2026 10:13:17 -0700 Subject: [PATCH 21/40] Apply suggestions from code review Co-authored-by: domalessi <111786334+domalessi@users.noreply.github.com> --- huntress/README.md | 11 +++++------ .../assets/monitors/huntress_siem_check_failed.json | 2 +- .../monitors/huntress_siem_no_logs_collected.json | 4 ++-- huntress/assets/service_checks.json | 2 +- huntress/metadata.csv | 6 +++--- 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/huntress/README.md b/huntress/README.md index e717faa456..1b8286a36c 100644 --- a/huntress/README.md +++ b/huntress/README.md @@ -6,7 +6,7 @@ This integration polls the Huntress Managed SIEM API using ES|QL queries and forwards all security events to Datadog as logs. Each collection run: -1. Loads a checkpoint - the timestamp of the last successful collection +1. Loads a checkpoint (the timestamp of the last successful collection) 2. Executes a configurable ES|QL query for the elapsed time window 3. Paginates through all result pages 4. Optionally enriches each log with Huntress organization metadata (org name, key, account ID) @@ -72,7 +72,7 @@ datadog-agent integration install -t datadog-huntress==1.0.0 ### Multiple Huntress accounts -Add additional blocks under `instances:` - each runs independently with its own checkpoint, org metadata cache, and metrics: +Add additional blocks under `instances:`. Each block runs independently with its own checkpoint, org metadata cache, and metrics: ```yaml instances: @@ -118,7 +118,7 @@ instances: ### Rate limit considerations -The Huntress API allows 60 requests per minute per API key pair. A typical run with 3 SIEM queries and agent metrics uses roughly 5 - 25 requests, well within this budget. For large accounts with high log volume or thousands of agents, consider splitting concerns across two instances using separate API key pairs: +The Huntress API allows 60 requests per minute per API key pair. A typical run with 3 SIEM queries and agent metrics uses roughly 5–25 requests, well within this budget. For large accounts with high log volume or thousands of agents, consider splitting concerns across two instances using separate API key pairs: ```yaml instances: @@ -143,7 +143,7 @@ instances: All logs collected from the Huntress Managed SIEM API are forwarded to Datadog with: -- `ddsource: huntress` - enables automatic log pipeline processing +- `ddsource: huntress`: enables automatic log pipeline processing - ECS field names preserved as top-level log attributes (for example, `event.category`, `host.hostname`, `user.name`) - Organization metadata tags when `enrich_with_org_tags: true` (for example, `huntress_org_name`, `huntress_org_key`, `huntress_account_id`) @@ -209,9 +209,8 @@ The Huntress API allows 60 requests per minute. The integration logs a warning w **Duplicate logs after Agent restart** -This is expected on the first restart after a failed run - the checkpoint is only advanced when all pages are successfully sent. Subsequent runs resume from the last successful checkpoint. +This is expected on the first restart after a failed run. The checkpoint is only advanced after all pages are successfully sent. -Need help? Contact [Datadog support][2]. ## Support diff --git a/huntress/assets/monitors/huntress_siem_check_failed.json b/huntress/assets/monitors/huntress_siem_check_failed.json index 4e255f6694..4575309a6b 100644 --- a/huntress/assets/monitors/huntress_siem_check_failed.json +++ b/huntress/assets/monitors/huntress_siem_check_failed.json @@ -8,7 +8,7 @@ ], "description": "The Huntress SIEM Agent check has entered a CRITICAL state, meaning the last collection run failed. Logs may no longer be flowing from Huntress to Datadog.", "definition": { - "name": "Huntress SIEM collection run failed on {{host.name}}", + "name": "Huntress SIEM collection run failed", "type": "service check", "query": "\"huntress.siem.check_status\".over(\"*\").by(\"error_type\").last(2).count_by_status()", "message": "The Huntress SIEM Agent check on {{host.name}} has entered a CRITICAL state, meaning the last collection run failed. Logs may no longer be flowing from Huntress to Datadog.\n\nCommon causes:\n- Invalid or rotated API credentials (`huntress_api_key` / `huntress_secret_key`)\n- Network connectivity issue between the Agent host and `api.huntress.io`\n- Invalid `esql_query` in the instance configuration\n- Huntress SIEM feature not enabled on this account\n\nCheck the Agent logs for details:\n```\nsudo datadog-agent check huntress\n```", diff --git a/huntress/assets/monitors/huntress_siem_no_logs_collected.json b/huntress/assets/monitors/huntress_siem_no_logs_collected.json index 35cab53dab..6239e602e0 100644 --- a/huntress/assets/monitors/huntress_siem_no_logs_collected.json +++ b/huntress/assets/monitors/huntress_siem_no_logs_collected.json @@ -2,13 +2,13 @@ "version": 2, "created_at": "2026-05-29", "last_updated_at": "2026-05-29", - "title": "Huntress SIEM no logs collected in 2 hours", + "title": "No Huntress SIEM logs collected in 2 hours", "tags": [ "integration:huntress" ], "description": "The Huntress SIEM integration collected 0 logs in the last 2 hours.", "definition": { - "name": "Huntress SIEM no logs collected in 2 hours", + "name": "No Huntress SIEM logs collected in 2 hours", "type": "query alert", "query": "sum(last_2h):sum:huntress.siem.logs_collected{*} by {query_name}.as_count() < 1", "message": "The Huntress SIEM integration collected 0 logs in the last 2 hours.\n\nThis may be expected in a brand-new deployment or a quiet environment, but if you expect ongoing log flow, investigate:\n\n1. Verify the Agent check is running: `sudo datadog-agent check huntress`\n2. Confirm `esql_query` returns results in the Huntress Partner Portal\n3. Check that the Huntress account has active endpoints and SIEM log sources\n4. Review `huntress.siem.check_status` — if CRITICAL, the check itself is failing", diff --git a/huntress/assets/service_checks.json b/huntress/assets/service_checks.json index f5b604d8c2..e67893ec2b 100644 --- a/huntress/assets/service_checks.json +++ b/huntress/assets/service_checks.json @@ -9,6 +9,6 @@ ], "groups": [], "name": "Huntress SIEM Check Status", - "description": "Returns `CRITICAL` if the Huntress SIEM collection run fails for any reason (auth error, network error, invalid query, etc.), returns `OK` otherwise." + "description": "Returns `CRITICAL` if the Huntress SIEM collection run fails for any reason (such as an auth error, network error, or invalid query); returns `OK` otherwise." } ] \ No newline at end of file diff --git a/huntress/metadata.csv b/huntress/metadata.csv index d289e4ddc8..76482a3c11 100644 --- a/huntress/metadata.csv +++ b/huntress/metadata.csv @@ -1,5 +1,5 @@ metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric -huntress.siem.logs_collected,gauge,,,, Number of SIEM log events collected from Huntress in the last run.,0,huntress,siem logs collected, +huntress.siem.logs_collected,gauge,,,,Number of SIEM log events collected from Huntress in the last run.,0,huntress,siem logs collected, huntress.siem.pages_fetched,gauge,,,,Number of API result pages fetched from Huntress in the last run.,0,huntress,siem pages fetched, huntress.siem.run_duration_seconds,gauge,,second,,Wall time of the Huntress SIEM collection run.,0,huntress,siem run duration, huntress.siem.errors,count,,,,Number of errors encountered during the Huntress SIEM collection run.,0,huntress,siem errors, @@ -7,6 +7,6 @@ huntress.siem.api_call_limit,gauge,,,,Huntress API rate limit (requests per minu huntress.siem.api_call_remaining,gauge,,,,Remaining Huntress API requests in the current rate limit window as reported by the x-huntress-api-call-remaining response header.,0,huntress,api call remaining, huntress.agents.total,gauge,,,,Total number of Huntress agents registered in the account.,0,huntress,agents total, huntress.agents.count,gauge,,,,Number of Huntress agents broken down by platform tag (windows/darwin/linux).,0,huntress,agents count, -huntress.agents.defender_status,gauge,,,,Number of Huntress agents broken down by defender_status tag (healthy/unhealthy/etc).,0,huntress,agents defender status, -huntress.agents.firewall_status,gauge,,,,Number of Huntress agents broken down by firewall_status tag (enabled/disabled/isolated/etc).,0,huntress,agents firewall status, +huntress.agents.defender_status,gauge,,,,Number of Huntress agents by Defender AV status, tagged as defender_status.,0,huntress,agents defender status, +huntress.agents.firewall_status,gauge,,,,Number of Huntress agents by firewall status, tagged as firewall_status.,0,huntress,agents firewall status, huntress.agents.pages_fetched,gauge,,,,Number of API pages fetched to retrieve the full agent list in the last run.,0,huntress,agents pages fetched, From 26f4835e7906777410ddd513164d0ba6259fc5bd Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Tue, 2 Jun 2026 10:52:40 -0700 Subject: [PATCH 22/40] Move validation to dedicated section --- huntress/README.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/huntress/README.md b/huntress/README.md index 1b8286a36c..6398ee7cb0 100644 --- a/huntress/README.md +++ b/huntress/README.md @@ -62,13 +62,15 @@ datadog-agent integration install -t datadog-huntress==1.0.0 sudo launchctl stop com.datadoghq.agent && sudo launchctl start com.datadoghq.agent ``` -4. Verify the check: +### Validation - ```bash - sudo datadog-agent check huntress - ``` +Run the Agent status command and look for `huntress` under the **Checks** section: + +```bash +sudo datadog-agent check huntress +``` - Logs appear in Datadog Log Explorer filtered by `source:huntress` within one collection interval (default: 15 minutes). +Logs appear in Datadog Log Explorer filtered by `source:huntress` within one collection interval (default: 15 minutes). ### Multiple Huntress accounts From e41ae650e1a1de8cc8bece9c6706d7d28d0db26a Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Tue, 2 Jun 2026 10:53:04 -0700 Subject: [PATCH 23/40] Replace "Data Collected" section with recommendation --- huntress/README.md | 29 +++-------------------------- 1 file changed, 3 insertions(+), 26 deletions(-) diff --git a/huntress/README.md b/huntress/README.md index 6398ee7cb0..bc08de810f 100644 --- a/huntress/README.md +++ b/huntress/README.md @@ -151,37 +151,15 @@ All logs collected from the Huntress Managed SIEM API are forwarded to Datadog w ### Metrics -**SIEM metrics** (always emitted per collection run): - -| Metric | Type | Tags | Description | -| ------------------------------------ | ----- | ------------- | ----------------------------------------------------- | -| `huntress.siem.logs_collected` | Gauge | `query_name:` | Log events collected per query per run | -| `huntress.siem.pages_fetched` | Gauge | `query_name:` | API pages fetched per query per run | -| `huntress.siem.run_duration_seconds` | Gauge | | Wall time of the full collection run | -| `huntress.siem.errors` | Count | `error_type:` | Errors by type | -| `huntress.siem.api_call_limit` | Gauge | | Total API requests allowed per minute (from Huntress) | -| `huntress.siem.api_call_remaining` | Gauge | | API requests remaining in the current minute | - -**Agent metrics** (emitted when `metrics.agents.enabled: true`): - -| Metric | Type | Tags | Description | -| --------------------------------- | ----- | ------------------ | ---------------------------------------------- | -| `huntress.agents.total` | Gauge | | Total agent count | -| `huntress.agents.count` | Gauge | `platform:` | Agent count by platform (windows/darwin/linux) | -| `huntress.agents.defender_status` | Gauge | `defender_status:` | Agent count by Defender AV status | -| `huntress.agents.firewall_status` | Gauge | `firewall_status:` | Agent count by firewall status | -| `huntress.agents.pages_fetched` | Gauge | | API pages used to fetch the agent list | - -See [metadata.csv][1] for a full list of metrics. +See [metadata.csv][1] for a list of metrics provided by this integration. ### Events -The Huntress integration does not emit Datadog events. +The Huntress integration does not include any events. ### Service Checks -**`huntress.siem.check_status`** -Returns `CRITICAL` if the collection run fails for any reason; `OK` otherwise. +See [service_checks.json][2] for a list of service checks provided by this integration. ## Troubleshooting @@ -213,7 +191,6 @@ The Huntress API allows 60 requests per minute. The integration logs a warning w This is expected on the first restart after a failed run. The checkpoint is only advanced after all pages are successfully sent. - ## Support For questions and support, [contact Datadog support][2]. From 13b35eec391e8ae42d48453aaa6f999ae571c107 Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Tue, 2 Jun 2026 11:24:44 -0700 Subject: [PATCH 24/40] Fix metadata column issue (suggestion added a comma that broke validation) --- huntress/metadata.csv | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/huntress/metadata.csv b/huntress/metadata.csv index 76482a3c11..cdd5644bcb 100644 --- a/huntress/metadata.csv +++ b/huntress/metadata.csv @@ -7,6 +7,6 @@ huntress.siem.api_call_limit,gauge,,,,Huntress API rate limit (requests per minu huntress.siem.api_call_remaining,gauge,,,,Remaining Huntress API requests in the current rate limit window as reported by the x-huntress-api-call-remaining response header.,0,huntress,api call remaining, huntress.agents.total,gauge,,,,Total number of Huntress agents registered in the account.,0,huntress,agents total, huntress.agents.count,gauge,,,,Number of Huntress agents broken down by platform tag (windows/darwin/linux).,0,huntress,agents count, -huntress.agents.defender_status,gauge,,,,Number of Huntress agents by Defender AV status, tagged as defender_status.,0,huntress,agents defender status, -huntress.agents.firewall_status,gauge,,,,Number of Huntress agents by firewall status, tagged as firewall_status.,0,huntress,agents firewall status, +huntress.agents.defender_status,gauge,,,,Number of Huntress agents by Defender AV status tagged as defender_status.,0,huntress,agents defender status, +huntress.agents.firewall_status,gauge,,,,Number of Huntress agents by firewall status tagged as firewall_status.,0,huntress,agents firewall status, huntress.agents.pages_fetched,gauge,,,,Number of API pages fetched to retrieve the full agent list in the last run.,0,huntress,agents pages fetched, From a4d682034c2cc15cc1ee2efd6687ea8232a36865 Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Tue, 2 Jun 2026 11:26:58 -0700 Subject: [PATCH 25/40] Remove en dash as validation rejects it as a non-ascii character --- huntress/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/huntress/README.md b/huntress/README.md index bc08de810f..15a5897160 100644 --- a/huntress/README.md +++ b/huntress/README.md @@ -120,7 +120,7 @@ instances: ### Rate limit considerations -The Huntress API allows 60 requests per minute per API key pair. A typical run with 3 SIEM queries and agent metrics uses roughly 5–25 requests, well within this budget. For large accounts with high log volume or thousands of agents, consider splitting concerns across two instances using separate API key pairs: +The Huntress API allows 60 requests per minute per API key pair. A typical run with 3 SIEM queries and agent metrics uses roughly 5 to 25 requests, well within this budget. For large accounts with high log volume or thousands of agents, consider splitting concerns across two instances using separate API key pairs: ```yaml instances: From 008500803ba5ec3967e809ea77acd0504cd7bbd5 Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Tue, 2 Jun 2026 11:37:42 -0700 Subject: [PATCH 26/40] Fix config default value --- huntress/datadog_checks/huntress/data/conf.yaml.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/huntress/datadog_checks/huntress/data/conf.yaml.example b/huntress/datadog_checks/huntress/data/conf.yaml.example index f3f59dd2a7..c81fce0431 100644 --- a/huntress/datadog_checks/huntress/data/conf.yaml.example +++ b/huntress/datadog_checks/huntress/data/conf.yaml.example @@ -36,7 +36,7 @@ instances: # # service: - ## @param min_collection_interval - number - optional - default: 15 + ## @param min_collection_interval - number - optional - default: 900 ## This changes the collection interval of the check. For more information, see: ## https://docs.datadoghq.com/developers/write_agent_check/#collection-interval # From 3ac80f3dadeaaa3eb792a766988ed55854eb6061 Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Tue, 2 Jun 2026 11:48:36 -0700 Subject: [PATCH 27/40] Fix config validation issue - add missing `example` property --- huntress/assets/configuration/spec.yaml | 1 + huntress/datadog_checks/huntress/data/conf.yaml.example | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/huntress/assets/configuration/spec.yaml b/huntress/assets/configuration/spec.yaml index 78dcb2116f..ea3bc3dba9 100644 --- a/huntress/assets/configuration/spec.yaml +++ b/huntress/assets/configuration/spec.yaml @@ -18,6 +18,7 @@ files: - template: instances/default overrides: min_collection_interval.value.default: 900 + min_collection_interval.value.example: 900 - name: huntress_api_key required: true secret: true diff --git a/huntress/datadog_checks/huntress/data/conf.yaml.example b/huntress/datadog_checks/huntress/data/conf.yaml.example index c81fce0431..4c822cdb53 100644 --- a/huntress/datadog_checks/huntress/data/conf.yaml.example +++ b/huntress/datadog_checks/huntress/data/conf.yaml.example @@ -40,7 +40,7 @@ instances: ## This changes the collection interval of the check. For more information, see: ## https://docs.datadoghq.com/developers/write_agent_check/#collection-interval # - # min_collection_interval: 15 + # min_collection_interval: 900 ## @param empty_default_hostname - boolean - optional - default: false ## This forces the check to send metrics with no hostname. From 31bf819a7279e16078a3aa583b7842ad0eb46974 Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Wed, 3 Jun 2026 14:34:55 -0700 Subject: [PATCH 28/40] Apply suggestions from code review Co-authored-by: domalessi <111786334+domalessi@users.noreply.github.com> --- huntress/README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/huntress/README.md b/huntress/README.md index 15a5897160..3473ec51e7 100644 --- a/huntress/README.md +++ b/huntress/README.md @@ -4,7 +4,7 @@ [Huntress](https://www.huntress.com/) is a managed security platform providing endpoint detection and response (EDR), antivirus, security awareness training, and a Managed SIEM product that continuously collects and analyzes endpoint telemetry. -This integration polls the Huntress Managed SIEM API using ES|QL queries and forwards all security events to Datadog as logs. Each collection run: +This integration polls the Huntress Managed SIEM API using ES|QL queries and forwards all security events to Datadog as logs. During each collection run, the integration: 1. Loads a checkpoint (the timestamp of the last successful collection) 2. Executes a configurable ES|QL query for the elapsed time window @@ -21,7 +21,7 @@ This integration is designed for managed security providers (MSPs) and enterpris - Datadog Agent 7.x or later - A Huntress account with the Managed SIEM feature enabled -- Huntress API credentials (public API key + secret key) from the Huntress Partner Portal under **Settings > API Credentials** +- Huntress API credentials (public API key and secret key) from the Huntress Partner Portal under **Settings > API Credentials** ### Installation @@ -64,13 +64,13 @@ datadog-agent integration install -t datadog-huntress==1.0.0 ### Validation -Run the Agent status command and look for `huntress` under the **Checks** section: +Run the following command to validate the integration is collecting data: ```bash sudo datadog-agent check huntress ``` -Logs appear in Datadog Log Explorer filtered by `source:huntress` within one collection interval (default: 15 minutes). +Logs appear in Datadog Log Explorer filtered by `source:huntress`. Allow up to 15 minutes for the first logs to appear. ### Multiple Huntress accounts @@ -145,7 +145,7 @@ instances: All logs collected from the Huntress Managed SIEM API are forwarded to Datadog with: -- `ddsource: huntress`: enables automatic log pipeline processing +- `ddsource: huntress`: a reserved log attribute that identifies the log source and triggers automatic pipeline processing - ECS field names preserved as top-level log attributes (for example, `event.category`, `host.hostname`, `user.name`) - Organization metadata tags when `enrich_with_org_tags: true` (for example, `huntress_org_name`, `huntress_org_key`, `huntress_account_id`) @@ -159,7 +159,7 @@ The Huntress integration does not include any events. ### Service Checks -See [service_checks.json][2] for a list of service checks provided by this integration. +See [service_checks.json][3] for a list of service checks provided by this integration. ## Troubleshooting @@ -176,20 +176,20 @@ Inspect the `error_type` tag to identify the root cause: | `error_type` | Cause | Resolution | | ------------------ | ---------------------------------- | ------------------------------------------------------------- | -| `auth_failure` | Invalid or rotated API credentials | Update `huntress_api_key` / `huntress_secret_key` | +| `auth_failure` | Invalid or rotated API credentials | Update `huntress_api_key` or `huntress_secret_key` | | `timeout` | ES\|QL query too broad | Add a `KEEP` or `WHERE` clause to the query | | `invalid_query` | Malformed ES\|QL | Fix the `esql_query` value in the failing `log_queries` entry | | `server_error` | Transient Huntress API error | Check [Huntress status page](https://status.huntress.com) | | `connection_error` | Network issue | Verify connectivity from the Agent host to `api.huntress.io` | | `run_failure` | Unexpected error during collection | Check Agent logs for the full stack trace | -**`huntress.siem.api_call_remaining` is very low or zero** +**`huntress.siem.api_call_remaining` is low or zero** The Huntress API allows 60 requests per minute. The integration logs a warning when fewer than 10 requests remain in a given minute. If this happens regularly, reduce `max_pages_per_run` or increase `min_collection_interval` to spread out collection runs. **Duplicate logs after Agent restart** -This is expected on the first restart after a failed run. The checkpoint is only advanced after all pages are successfully sent. +This is expected on the first restart after a failed run. The checkpoint advances only after all pages are successfully sent. ## Support From 1d44e327e6f35401996f290db9f8e633cada8148 Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Wed, 3 Jun 2026 23:08:18 -0700 Subject: [PATCH 29/40] Update README with recommendations: MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Configuration step 1: "Create" → "Copy the [example configuration file][4]…and edit it", steps collapsed from 3 → 2, "fully-annotated" hyphen removed - Multiple Huntress accounts: Moved before ### Validation, demoted to ####, conditional lead-in sentence added - Config reference table: No\* → Conditional for both conditional fields, column widths realigned, footnote \* replaced with **Note:** paragraph - Rate limits: Renamed from "Rate limit considerations" - Link references: Added [3] for service_checks.json (was broken before) and [4] for the example config file on GitHub --- huntress/README.md | 62 +++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/huntress/README.md b/huntress/README.md index 3473ec51e7..aca2e26fdd 100644 --- a/huntress/README.md +++ b/huntress/README.md @@ -33,9 +33,7 @@ datadog-agent integration install -t datadog-huntress==1.0.0 ### Configuration -1. Create the configuration file at `/etc/datadog-agent/conf.d/huntress.yaml` (Linux/macOS) or `C:\ProgramData\Datadog\conf.d\huntress.yaml` (Windows). A fully-annotated example is at `datadog_checks/huntress/data/conf.yaml.example`. - -2. Edit `huntress.yaml` with your credentials: +1. Copy the [example configuration file][4] to `/etc/datadog-agent/conf.d/huntress/conf.yaml` (Linux/macOS) or `C:\ProgramData\Datadog\conf.d\huntress\conf.yaml` (Windows) and edit it with your credentials: ```yaml init_config: {} @@ -52,7 +50,7 @@ datadog-agent integration install -t datadog-huntress==1.0.0 - "env:production" ``` -3. Restart the Agent: +2. Restart the Agent: ```bash # Linux (systemd) @@ -62,19 +60,9 @@ datadog-agent integration install -t datadog-huntress==1.0.0 sudo launchctl stop com.datadoghq.agent && sudo launchctl start com.datadoghq.agent ``` -### Validation - -Run the following command to validate the integration is collecting data: - -```bash -sudo datadog-agent check huntress -``` - -Logs appear in Datadog Log Explorer filtered by `source:huntress`. Allow up to 15 minutes for the first logs to appear. - -### Multiple Huntress accounts +#### Multiple Huntress accounts -Add additional blocks under `instances:`. Each block runs independently with its own checkpoint, org metadata cache, and metrics: +If you manage multiple Huntress accounts, add an additional block under `instances:` for each one. Each block runs independently with its own checkpoint, org metadata cache, and metrics: ```yaml instances: @@ -93,6 +81,16 @@ instances: tags: ["source:huntress", "env:staging"] ``` +### Validation + +Run the following command to validate the integration is collecting data: + +```bash +sudo datadog-agent check huntress +``` + +Logs appear in Datadog Log Explorer filtered by `source:huntress`. Allow up to 15 minutes for the first logs to appear. + ### Configuration reference **`init_config` options** (apply to all instances): @@ -103,22 +101,22 @@ instances: **`instances` options** (per Huntress account): -| Field | Required | Default | Description | -| -------------------------- | -------- | ------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `huntress_api_key` | Yes | - | Huntress public API key | -| `huntress_secret_key` | Yes | - | Huntress secret API key | -| `log_queries` | No\* | - | List of query objects; each has `name` (required), `esql_query` (required, must begin with `FROM logs`), and `tags` (optional). `name` is used as the `query_name` tag on metrics | -| `metrics.agents.enabled` | No\* | `false` | Collect agent fleet metrics (total, by platform, by Defender/firewall status) | -| `metrics.agents.max_pages` | No | `20` | Max pages of agents to fetch per run (500 agents/page) | -| `enrich_with_org_tags` | No | `true` | Fetch and attach org metadata as log tags | -| `org_cache_ttl_seconds` | No | `3600` | How long to cache org metadata (seconds) | -| `max_pages_per_run` | No | `100` | Page cap per query per run (~20,000 logs maximum) | -| `huntress_base_url` | No | `https://api.huntress.io` | Override for sandbox environments | -| `tags` | No | `[]` | Extra tags on every forwarded metric and log | +| Field | Required | Default | Description | +| -------------------------- | ----------- | ------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `huntress_api_key` | Yes | - | Huntress public API key | +| `huntress_secret_key` | Yes | - | Huntress secret API key | +| `log_queries` | Conditional | - | List of query objects; each has `name` (required), `esql_query` (required, must begin with `FROM logs`), and `tags` (optional). `name` is used as the `query_name` tag on metrics | +| `metrics.agents.enabled` | Conditional | `false` | Collect agent fleet metrics (total, by platform, by Defender/firewall status) | +| `metrics.agents.max_pages` | No | `20` | Max pages of agents to fetch per run (500 agents/page) | +| `enrich_with_org_tags` | No | `true` | Fetch and attach org metadata as log tags | +| `org_cache_ttl_seconds` | No | `3600` | How long to cache org metadata (seconds) | +| `max_pages_per_run` | No | `100` | Page cap per query per run (~20,000 logs maximum) | +| `huntress_base_url` | No | `https://api.huntress.io` | Override for sandbox environments | +| `tags` | No | `[]` | Extra tags on every forwarded metric and log | -\* At least one of `log_queries` or `metrics.agents.enabled: true` must be configured per instance. +**Note:** At least one of `log_queries` or `metrics.agents.enabled: true` must be configured per instance. -### Rate limit considerations +### Rate limits The Huntress API allows 60 requests per minute per API key pair. A typical run with 3 SIEM queries and agent metrics uses roughly 5 to 25 requests, well within this budget. For large accounts with high log volume or thousands of agents, consider splitting concerns across two instances using separate API key pairs: @@ -176,7 +174,7 @@ Inspect the `error_type` tag to identify the root cause: | `error_type` | Cause | Resolution | | ------------------ | ---------------------------------- | ------------------------------------------------------------- | -| `auth_failure` | Invalid or rotated API credentials | Update `huntress_api_key` or `huntress_secret_key` | +| `auth_failure` | Invalid or rotated API credentials | Update `huntress_api_key` or `huntress_secret_key` | | `timeout` | ES\|QL query too broad | Add a `KEEP` or `WHERE` clause to the query | | `invalid_query` | Malformed ES\|QL | Fix the `esql_query` value in the failing `log_queries` entry | | `server_error` | Transient Huntress API error | Check [Huntress status page](https://status.huntress.com) | @@ -197,3 +195,5 @@ For questions and support, [contact Datadog support][2]. [1]: https://github.com/DataDog/integrations-extras/blob/master/huntress/metadata.csv [2]: https://docs.datadoghq.com/help/ +[3]: https://github.com/DataDog/integrations-extras/blob/master/huntress/service_checks.json +[4]: https://github.com/DataDog/integrations-extras/blob/master/huntress/datadog_checks/huntress/data/conf.yaml.example From 29045e1b0b3fa7977a39e2198db325f862f77f87 Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Tue, 9 Jun 2026 14:04:49 -0500 Subject: [PATCH 30/40] Fix timestamp parsing, 429 retry loop, and org tag lookup; add run summary and KEEP guidance Bug fixes: - Truncate nanosecond-precision @timestamp values to microseconds before parsing; Python's fromisoformat() only handles up to 6 fractional digits, causing silent date_ms=None on all Huntress log events - Match organization_id flat field (actual API response shape) in addition to organization.id when resolving org enrichment tags - Cap 429 retry loop at one retry with error_type:rate_limited counting; the previous implementation retried indefinitely, blocking the check thread Observability: - Emit a self.log.info() summary after each successful run reporting logs collected and pages fetched per query, and agent count/pages when metrics.agents is enabled - Surface the same summary as the service check message so it appears in `datadog-agent status` output - Warn on the first page when log entries contain only uuid and organization_id, indicating a missing KEEP clause in the ES|QL query Docs and config: - Document the Huntress API KEEP requirement in spec.yaml, conf.yaml.example, and README; despite what the Huntress API docs state, queries without an explicit KEEP return only uuid and organization_id, not all fields - Update all example queries in README and conf.yaml.example to include a representative KEEP clause --- huntress/README.md | 33 +++++------ huntress/assets/configuration/spec.yaml | 13 ++++- .../huntress/data/conf.yaml.example | 14 ++++- huntress/datadog_checks/huntress/huntress.py | 56 ++++++++++++++++--- 4 files changed, 86 insertions(+), 30 deletions(-) diff --git a/huntress/README.md b/huntress/README.md index aca2e26fdd..6e5aa9f5f9 100644 --- a/huntress/README.md +++ b/huntress/README.md @@ -43,7 +43,7 @@ datadog-agent integration install -t datadog-huntress==1.0.0 huntress_secret_key: "" log_queries: - name: "all-logs" - esql_query: "FROM logs" + esql_query: "FROM logs | KEEP @timestamp, message, host.hostname, event.category, event.code, event.provider, event.id, user.target.name, user.target.domain, winlog.system.EventID" tags: - "source:huntress" - "service:huntress-siem" @@ -70,14 +70,14 @@ instances: huntress_secret_key: "" log_queries: - name: "all-logs" - esql_query: "FROM logs" + esql_query: "FROM logs | KEEP @timestamp, message, host.hostname, event.category, event.code, event.provider, event.id, user.target.name, user.target.domain, winlog.system.EventID" tags: ["source:huntress", "env:production"] - huntress_api_key: "" huntress_secret_key: "" log_queries: - name: "all-logs" - esql_query: "FROM logs" + esql_query: "FROM logs | KEEP @timestamp, message, host.hostname, event.category, event.code, event.provider, event.id, user.target.name, user.target.domain, winlog.system.EventID" tags: ["source:huntress", "env:staging"] ``` @@ -101,18 +101,18 @@ Logs appear in Datadog Log Explorer filtered by `source:huntress`. Allow up to 1 **`instances` options** (per Huntress account): -| Field | Required | Default | Description | -| -------------------------- | ----------- | ------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `huntress_api_key` | Yes | - | Huntress public API key | -| `huntress_secret_key` | Yes | - | Huntress secret API key | -| `log_queries` | Conditional | - | List of query objects; each has `name` (required), `esql_query` (required, must begin with `FROM logs`), and `tags` (optional). `name` is used as the `query_name` tag on metrics | -| `metrics.agents.enabled` | Conditional | `false` | Collect agent fleet metrics (total, by platform, by Defender/firewall status) | -| `metrics.agents.max_pages` | No | `20` | Max pages of agents to fetch per run (500 agents/page) | -| `enrich_with_org_tags` | No | `true` | Fetch and attach org metadata as log tags | -| `org_cache_ttl_seconds` | No | `3600` | How long to cache org metadata (seconds) | -| `max_pages_per_run` | No | `100` | Page cap per query per run (~20,000 logs maximum) | -| `huntress_base_url` | No | `https://api.huntress.io` | Override for sandbox environments | -| `tags` | No | `[]` | Extra tags on every forwarded metric and log | +| Field | Required | Default | Description | +| -------------------------- | ----------- | ------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `huntress_api_key` | Yes | - | Huntress public API key | +| `huntress_secret_key` | Yes | - | Huntress secret API key | +| `log_queries` | Conditional | - | List of query objects; each has `name` (required), `esql_query` (required, must begin with `FROM logs`), and `tags` (optional). `name` is used as the `query_name` tag on metrics. **Always include a `KEEP` clause** - despite what the Huntress API docs state, queries without `KEEP` return only `uuid` and `organization_id`, not all fields. | +| `metrics.agents.enabled` | Conditional | `false` | Collect agent fleet metrics (total, by platform, by Defender/firewall status) | +| `metrics.agents.max_pages` | No | `20` | Max pages of agents to fetch per run (500 agents/page) | +| `enrich_with_org_tags` | No | `true` | Fetch and attach org metadata as log tags | +| `org_cache_ttl_seconds` | No | `3600` | How long to cache org metadata (seconds) | +| `max_pages_per_run` | No | `100` | Page cap per query per run (~20,000 logs maximum) | +| `huntress_base_url` | No | `https://api.huntress.io` | Override for sandbox environments | +| `tags` | No | `[]` | Extra tags on every forwarded metric and log | **Note:** At least one of `log_queries` or `metrics.agents.enabled: true` must be configured per instance. @@ -127,7 +127,7 @@ instances: huntress_secret_key: "" log_queries: - name: "all-logs" - esql_query: "FROM logs" + esql_query: "FROM logs | KEEP @timestamp, message, host.hostname, event.category, event.code, event.provider, event.id, user.target.name, user.target.domain, winlog.system.EventID" # Instance 2: agent metrics only (isolated rate limit budget) - huntress_api_key: "" @@ -167,6 +167,7 @@ See [service_checks.json][3] for a list of service checks provided by this integ - Verify the API key pair is valid by checking the Huntress Partner Portal - Confirm the Managed SIEM feature is enabled on the account - Check that each `log_queries[].esql_query` begins with `FROM logs` +- The Huntress SIEM API requires an explicit `KEEP` clause to return log fields - without one, responses contain only `uuid` and `organization_id`. Add `| KEEP @timestamp, message, host.hostname, event.category, event.code, ...` to your query. The Agent logs will show a warning if this is detected. **`huntress.siem.errors` count is increasing** diff --git a/huntress/assets/configuration/spec.yaml b/huntress/assets/configuration/spec.yaml index ea3bc3dba9..7e359b0ade 100644 --- a/huntress/assets/configuration/spec.yaml +++ b/huntress/assets/configuration/spec.yaml @@ -46,6 +46,13 @@ files: targeted collection for different log types within the same account. Do not add time range filters — range_start and range_end are managed automatically by the check. + + IMPORTANT: Always include a KEEP clause to select the fields you want. + Despite what the Huntress API documentation states, queries without an + explicit KEEP clause return only uuid and organization_id — not all fields. + Example: FROM logs | KEEP @timestamp, message, host.hostname, event.category, + event.code, event.provider, event.id, user.target.name, user.target.domain, + winlog.system.EventID value: type: array items: @@ -60,10 +67,10 @@ files: items: type: string example: - - name: "all-logs" - esql_query: "FROM logs" + - name: "windows-security-events" + esql_query: "FROM logs | KEEP @timestamp, message, host.hostname, event.category, event.code, event.provider, event.id, user.target.name, user.target.domain, winlog.system.EventID" tags: - - "log_type:all" + - "log_type:windows" - name: enrich_with_org_tags description: | When true, the check fetches Huntress organization metadata (org name, diff --git a/huntress/datadog_checks/huntress/data/conf.yaml.example b/huntress/datadog_checks/huntress/data/conf.yaml.example index 4c822cdb53..0f6d28fe49 100644 --- a/huntress/datadog_checks/huntress/data/conf.yaml.example +++ b/huntress/datadog_checks/huntress/data/conf.yaml.example @@ -80,12 +80,20 @@ instances: ## targeted collection for different log types within the same account. ## Do not add time range filters — range_start and range_end are managed ## automatically by the check. + ## + ## IMPORTANT: Always include a KEEP clause to select the fields you want. + ## Despite what the Huntress API documentation states, queries without an + ## explicit KEEP clause return only uuid and organization_id — not all fields. + ## Example: FROM logs | KEEP @timestamp, message, host.hostname, event.category, + ## event.code, event.provider, event.id, user.target.name, user.target.domain, + ## winlog.system.EventID # # log_queries: - # - name: all-logs - # esql_query: FROM logs + # - name: windows-security-events + # esql_query: FROM logs | KEEP @timestamp, message, host.hostname, event.category, + # event.code, event.provider, event.id, user.target.name, user.target.domain, winlog.system.EventID # tags: - # - log_type:all + # - log_type:windows ## @param enrich_with_org_tags - boolean - optional - default: true ## When true, the check fetches Huntress organization metadata (org name, diff --git a/huntress/datadog_checks/huntress/huntress.py b/huntress/datadog_checks/huntress/huntress.py index 00006654fe..0602d60590 100644 --- a/huntress/datadog_checks/huntress/huntress.py +++ b/huntress/datadog_checks/huntress/huntress.py @@ -1,5 +1,6 @@ import base64 import json +import re import time import zlib from datetime import datetime, timedelta, timezone @@ -34,6 +35,10 @@ class HuntressCheck(AgentCheck): SERVICE_CHECK_NAME = "huntress.siem.check_status" + # Fields present in every response regardless of KEEP clause; logs containing + # only these keys have no content because the query is missing a KEEP verb. + _BARE_LOG_KEYS = frozenset({"uuid", "organization_id"}) + def check(self, instance): start_time = time.time() @@ -75,6 +80,7 @@ def check(self, instance): org_cache = self._get_or_refresh_org_cache(base_url, headers, instance_hash, org_ttl) success = False + run_summary = [] try: for query_def in log_queries: @@ -100,11 +106,18 @@ def check(self, instance): self.gauge("huntress.siem.logs_collected", logs, tags=query_metric_tags) self.gauge("huntress.siem.pages_fetched", pages, tags=query_metric_tags) + run_summary.append(f"query '{query_name}': {logs} log(s) collected, {pages} page(s) fetched") if collect_agents: - self._collect_agent_metrics(base_url, headers, extra_tags, agents_max_pages) + agents_total, agents_pages = self._collect_agent_metrics( + base_url, headers, extra_tags, agents_max_pages + ) + run_summary.append(f"agent metrics: {agents_total} agent(s), {agents_pages} page(s) fetched") success = True + self.log.info( + "Huntress check complete — %s", "; ".join(run_summary) if run_summary else "no data collected" + ) except Exception as exc: self.log.error("Huntress check run failed: %s", exc) @@ -162,6 +175,15 @@ def _run_query(self, base_url, headers, esql, checkpoint_key, max_pages, min_int logs, next_token = self._query_page(base_url, headers, esql, range_start, range_end, page_token) if logs: + if pages_fetched == 0 and all(set(raw.keys()) <= self._BARE_LOG_KEYS for raw in logs[:3]): + self.log.warning( + "Huntress SIEM query %r returned logs with no content fields " + "(only uuid and organization_id). The Huntress API requires an explicit " + "KEEP clause to return log fields. Add one to your esql_query, e.g.: " + "FROM logs | KEEP @timestamp, message, host.hostname, event.category, event.code, ...", + esql[:80], + ) + batch = [] for raw in logs: org_tags = self._get_org_tags(raw, org_cache) if org_cache else [] @@ -253,6 +275,8 @@ def _collect_agent_metrics(self, base_url, headers, extra_tags, max_pages): for status, count in by_firewall_status.items(): self.gauge("huntress.agents.firewall_status", count, tags=extra_tags + [f"firewall_status:{status}"]) + return len(agents), pages_fetched + # ------------------------------------------------------------------ # # Auth # # ------------------------------------------------------------------ # @@ -312,10 +336,12 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) timeout = self.init_config.get("request_timeout", self.DEFAULT_REQUEST_TIMEOUT) max_retries_5xx = 3 max_retries_408 = 2 + max_retries_429 = 1 backoff_5xx = [5, 10, 20] backoff_408 = [2, 4] attempt = 0 + attempt_429 = 0 while True: try: @@ -366,9 +392,19 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) raise Exception(f"Huntress API 422 Invalid ES|QL query: {resp.text}") if resp.status_code == 429: - self.log.warning("Huntress API 429 Rate Limited — sleeping 60s then retrying") - time.sleep(60) - continue + if attempt_429 < max_retries_429: + self.log.warning( + "Huntress API 429 Rate Limited — sleeping 60s then retrying (attempt %d/%d)", + attempt_429 + 1, + max_retries_429, + ) + time.sleep(60) + attempt_429 += 1 + continue + self.count("huntress.siem.errors", 1, tags=["error_type:rate_limited"]) + raise Exception( + "Huntress API 429 Rate Limited after retry — reduce max_pages_per_run or increase min_collection_interval" + ) if 500 <= resp.status_code < 600: if attempt < max_retries_5xx: @@ -424,7 +460,9 @@ def _transform_log(self, raw_log, tags, service): if isinstance(timestamp, (int, float)): date_ms = int(timestamp) else: - dt = datetime.fromisoformat(str(timestamp).replace("Z", "+00:00")) + # Truncate sub-microsecond precision (e.g. nanoseconds) that fromisoformat rejects + ts_str = re.sub(r'(\.\d{6})\d+', r'\1', str(timestamp)).replace("Z", "+00:00") + dt = datetime.fromisoformat(ts_str) date_ms = int(dt.timestamp() * 1000) except Exception: pass @@ -564,9 +602,11 @@ def _get_org_tags(self, raw_log, org_cache): if account_id is not None: base_tags.append(f"huntress_account_id:{account_id}") - # Strategy 1: match by organization.id - org_id = raw_log.get("organization.id") or ( - raw_log.get("organization", {}).get("id") if isinstance(raw_log.get("organization"), dict) else None + # Strategy 1: match by organization id (flat field from API, dot-notation, or nested) + org_id = ( + raw_log.get("organization_id") + or raw_log.get("organization.id") + or (raw_log.get("organization", {}).get("id") if isinstance(raw_log.get("organization"), dict) else None) ) if org_id is not None: org = orgs.get(str(org_id)) From 29771cd224a3a68d1c7341a5294538716bd6f207 Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Tue, 9 Jun 2026 14:59:26 -0500 Subject: [PATCH 31/40] Update timestamp parsing --- huntress/datadog_checks/huntress/huntress.py | 10 +++++----- huntress/tests/test_huntress.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/huntress/datadog_checks/huntress/huntress.py b/huntress/datadog_checks/huntress/huntress.py index 0602d60590..396bd65970 100644 --- a/huntress/datadog_checks/huntress/huntress.py +++ b/huntress/datadog_checks/huntress/huntress.py @@ -454,16 +454,16 @@ def _transform_log(self, raw_log, tags, service): message = raw_log.get("log.original") or raw_log.get("message") or json.dumps(raw_log) timestamp = raw_log.get("@timestamp") - date_ms = None + ts_seconds = None if timestamp: try: if isinstance(timestamp, (int, float)): - date_ms = int(timestamp) + ts_seconds = float(timestamp) else: # Truncate sub-microsecond precision (e.g. nanoseconds) that fromisoformat rejects ts_str = re.sub(r'(\.\d{6})\d+', r'\1', str(timestamp)).replace("Z", "+00:00") dt = datetime.fromisoformat(ts_str) - date_ms = int(dt.timestamp() * 1000) + ts_seconds = dt.timestamp() except Exception: pass @@ -473,8 +473,8 @@ def _transform_log(self, raw_log, tags, service): "ddtags": ",".join(tags), "service": service, } - if date_ms is not None: - payload["date"] = date_ms + if ts_seconds is not None: + payload["timestamp"] = ts_seconds # Preserve all ECS fields as top-level log attributes for key, value in raw_log.items(): diff --git a/huntress/tests/test_huntress.py b/huntress/tests/test_huntress.py index 06d9f8c4bf..13c037ce3c 100644 --- a/huntress/tests/test_huntress.py +++ b/huntress/tests/test_huntress.py @@ -326,7 +326,7 @@ def test_log_transformation(): assert payload["ddsource"] == "huntress" assert payload["service"] == "huntress-siem" assert payload["ddtags"] == "source:huntress,env:test,huntress_account_id:42" - assert payload["date"] == 1779890400000 + assert payload["timestamp"] == 1779890400.0 assert payload["event.provider"] == "Microsoft-Windows-Security-Auditing" assert payload["host.hostname"] == "DESKTOP-ABC123" assert "@timestamp" not in payload From e98e03c049a61ffe8ac9bce8e01b67394a08a55f Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Tue, 9 Jun 2026 15:23:44 -0500 Subject: [PATCH 32/40] Update log format --- huntress/datadog_checks/huntress/__about__.py | 2 +- huntress/datadog_checks/huntress/huntress.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/huntress/datadog_checks/huntress/__about__.py b/huntress/datadog_checks/huntress/__about__.py index 5becc17c04..f102a9cadf 100644 --- a/huntress/datadog_checks/huntress/__about__.py +++ b/huntress/datadog_checks/huntress/__about__.py @@ -1 +1 @@ -__version__ = "1.0.0" +__version__ = "0.0.1" diff --git a/huntress/datadog_checks/huntress/huntress.py b/huntress/datadog_checks/huntress/huntress.py index 396bd65970..b1261141a2 100644 --- a/huntress/datadog_checks/huntress/huntress.py +++ b/huntress/datadog_checks/huntress/huntress.py @@ -100,13 +100,15 @@ def check(self, instance): all_tags = extra_tags + query_tags query_metric_tags = extra_tags + [f"query_name:{query_name}"] - logs, pages = self._run_query( + logs, pages, q_start, q_end = self._run_query( base_url, headers, esql, checkpoint_key, max_pages, min_interval, org_cache, all_tags ) self.gauge("huntress.siem.logs_collected", logs, tags=query_metric_tags) self.gauge("huntress.siem.pages_fetched", pages, tags=query_metric_tags) - run_summary.append(f"query '{query_name}': {logs} log(s) collected, {pages} page(s) fetched") + run_summary.append( + f"query '{query_name}': {logs} log(s) collected, {pages} page(s) fetched [{q_start} → {q_end}]" + ) if collect_agents: agents_total, agents_pages = self._collect_agent_metrics( @@ -141,7 +143,7 @@ def check(self, instance): # ------------------------------------------------------------------ # def _run_query(self, base_url, headers, esql, checkpoint_key, max_pages, min_interval, org_cache, all_tags): - """Paginate a single ES|QL query. Returns (logs_collected, pages_fetched).""" + """Paginate a single ES|QL query. Returns (logs_collected, pages_fetched, range_start, range_end).""" range_start = self._load_checkpoint(checkpoint_key) now = datetime.now(timezone.utc) range_end = now.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" @@ -213,7 +215,7 @@ def _run_query(self, base_url, headers, esql, checkpoint_key, max_pages, min_int if not hit_page_cap: self._save_checkpoint(checkpoint_key, range_end) - return total_logs, pages_fetched + return total_logs, pages_fetched, range_start, range_end # ------------------------------------------------------------------ # # Agent metrics # From 739534ea7d729174d0788c4e548379717b600d02 Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Tue, 9 Jun 2026 16:11:10 -0500 Subject: [PATCH 33/40] Add log batching fallback and log declaration --- huntress/README.md | 7 ++- huntress/assets/configuration/spec.yaml | 5 ++ .../huntress/data/conf.yaml.example | 20 ++++++ huntress/datadog_checks/huntress/huntress.py | 63 ++++++++++++++++--- huntress/tests/test_huntress.py | 4 +- 5 files changed, 89 insertions(+), 10 deletions(-) diff --git a/huntress/README.md b/huntress/README.md index 6e5aa9f5f9..37125ca37a 100644 --- a/huntress/README.md +++ b/huntress/README.md @@ -163,12 +163,17 @@ See [service_checks.json][3] for a list of service checks provided by this integ **No logs in Datadog after first run** -- Run `sudo datadog-agent check huntress` and inspect the output +- Run `sudo datadog-agent check huntress` and inspect the output - the summary line reports how many logs were collected and the exact time range queried - Verify the API key pair is valid by checking the Huntress Partner Portal - Confirm the Managed SIEM feature is enabled on the account +- Confirm `min_collection_interval: 900` is set in your instance config - without it, the Agent uses its default 15-second interval, and each run queries only a 15-second window with no new events - Check that each `log_queries[].esql_query` begins with `FROM logs` - The Huntress SIEM API requires an explicit `KEEP` clause to return log fields - without one, responses contain only `uuid` and `organization_id`. Add `| KEEP @timestamp, message, host.hostname, event.category, event.code, ...` to your query. The Agent logs will show a warning if this is detected. +**`LogsSent: 0` in `datadog-agent status` even though logs are being collected** + +This is expected. The integration sends logs directly to the Datadog Logs Intake API rather than through the Agent's internal log pipeline, so the Logs Agent counters (`LogsProcessed`, `LogsSent`) will always read 0. Use the Agent check output or Datadog Log Explorer filtered by `source:huntress` to confirm logs are arriving. + **`huntress.siem.errors` count is increasing** Inspect the `error_type` tag to identify the root cause: diff --git a/huntress/assets/configuration/spec.yaml b/huntress/assets/configuration/spec.yaml index 7e359b0ade..06b8ebcb5e 100644 --- a/huntress/assets/configuration/spec.yaml +++ b/huntress/assets/configuration/spec.yaml @@ -126,3 +126,8 @@ files: agents: enabled: true max_pages: 20 + - template: logs + example: + - type: integration + source: huntress + service: huntress diff --git a/huntress/datadog_checks/huntress/data/conf.yaml.example b/huntress/datadog_checks/huntress/data/conf.yaml.example index 0f6d28fe49..b720af3cd5 100644 --- a/huntress/datadog_checks/huntress/data/conf.yaml.example +++ b/huntress/datadog_checks/huntress/data/conf.yaml.example @@ -132,3 +132,23 @@ instances: # agents: # enabled: true # max_pages: 20 + +## Log Section +## +## type - required - Type of log input source (tcp / udp / file / windows_event). +## port / path / channel_path - required - Set port if type is tcp or udp. +## Set path if type is file. +## Set channel_path if type is windows_event. +## source - required - Attribute that defines which integration sent the logs. +## encoding - optional - For file specifies the file encoding. Default is utf-8. Other +## possible values are utf-16-le and utf-16-be. +## service - optional - The name of the service that generates the log. +## Overrides any `service` defined in the `init_config` section. +## tags - optional - Add tags to the collected logs. +## +## Discover Datadog log collection: https://docs.datadoghq.com/logs/log_collection/ +# +# logs: +# - type: integration +# source: huntress +# service: huntress diff --git a/huntress/datadog_checks/huntress/huntress.py b/huntress/datadog_checks/huntress/huntress.py index b1261141a2..665e48eff1 100644 --- a/huntress/datadog_checks/huntress/huntress.py +++ b/huntress/datadog_checks/huntress/huntress.py @@ -1,4 +1,5 @@ import base64 +import gzip import json import re import time @@ -146,17 +147,17 @@ def _run_query(self, base_url, headers, esql, checkpoint_key, max_pages, min_int """Paginate a single ES|QL query. Returns (logs_collected, pages_fetched, range_start, range_end).""" range_start = self._load_checkpoint(checkpoint_key) now = datetime.now(timezone.utc) - range_end = now.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + range_end = now.strftime("%Y-%m-%dT%H:%M:%S") if range_start is None: default_start = now - timedelta(seconds=min_interval) - range_start = default_start.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + range_start = default_start.strftime("%Y-%m-%dT%H:%M:%S") else: - # Add 1ms to avoid re-fetching the boundary event from the previous run + # Advance by 1 second so the boundary event isn't re-fetched next run try: dt = datetime.fromisoformat(range_start.replace("Z", "+00:00")) - dt = dt + timedelta(milliseconds=1) - range_start = dt.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + dt = dt + timedelta(seconds=1) + range_start = dt.strftime("%Y-%m-%dT%H:%M:%S") except Exception: pass @@ -490,8 +491,56 @@ def _transform_log(self, raw_log, tags, service): # ------------------------------------------------------------------ # def _send_logs_batch(self, logs_batch): - for log_payload in logs_batch: - self.send_log(log_payload) + api_key = self.agentConfig.get('api_key') or self.agentConfig.get('dd_api_key') + if not api_key: + for log_payload in logs_batch: + self.send_log(log_payload) + return + + # Split into sub-batches that respect the 5MB uncompressed limit and 1000-entry cap + sub_batch = [] + sub_batch_bytes = 0 + for log in logs_batch: + log_bytes = len(json.dumps(log).encode('utf-8')) + if sub_batch and sub_batch_bytes + log_bytes > self.MAX_BATCH_SIZE_BYTES: + self._post_logs_to_intake(sub_batch, api_key) + sub_batch = [] + sub_batch_bytes = 0 + sub_batch.append(log) + sub_batch_bytes += log_bytes + if sub_batch: + self._post_logs_to_intake(sub_batch, api_key) + + def _post_logs_to_intake(self, logs_batch, api_key): + site = self.agentConfig.get('site') or 'datadoghq.com' + url = f'https://http-intake.logs.{site}/api/v2/logs' + timeout = self.init_config.get('request_timeout', self.DEFAULT_REQUEST_TIMEOUT) + body = gzip.compress(json.dumps(logs_batch).encode('utf-8')) + + for attempt in range(3): + resp = requests.post( + url, + headers={ + 'DD-API-KEY': api_key, + 'Content-Type': 'application/json', + 'Content-Encoding': 'gzip', + }, + data=body, + timeout=timeout, + ) + if resp.status_code in (200, 202): + return + if resp.status_code in (408, 429, 500, 503) and attempt < 2: + wait = (attempt + 1) * 5 + self.log.warning( + "Datadog Logs intake HTTP %d — retrying in %ds (attempt %d/2)", + resp.status_code, + wait, + attempt + 1, + ) + time.sleep(wait) + continue + raise Exception(f"Datadog Logs intake returned HTTP {resp.status_code}: {resp.text[:200]}") # ------------------------------------------------------------------ # # Checkpoint # diff --git a/huntress/tests/test_huntress.py b/huntress/tests/test_huntress.py index 13c037ce3c..cf0ebf31da 100644 --- a/huntress/tests/test_huntress.py +++ b/huntress/tests/test_huntress.py @@ -235,8 +235,8 @@ def capture_request(method, url, headers, json_body=None, params=None): check.check(instance) assert captured_bodies, "No SIEM query was made" - # range_start should be 1ms after saved_ts - assert captured_bodies[0]["range_start"] == "2026-05-27T13:00:00.001Z" + # range_start should be 1 second after saved_ts (API only accepts second-precision timestamps) + assert captured_bodies[0]["range_start"] == "2026-05-27T13:00:01" # =========================================================================== From 982c08aee72658350f2ddcfce516293e216a33b7 Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Wed, 10 Jun 2026 13:20:35 -0500 Subject: [PATCH 34/40] Apply suggestions from code review Co-authored-by: domalessi <111786334+domalessi@users.noreply.github.com> --- huntress/README.md | 8 ++++---- huntress/assets/monitors/huntress_siem_errors_high.json | 2 +- .../assets/monitors/huntress_siem_no_logs_collected.json | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/huntress/README.md b/huntress/README.md index 37125ca37a..b6268cf5f7 100644 --- a/huntress/README.md +++ b/huntress/README.md @@ -2,7 +2,7 @@ ## Overview -[Huntress](https://www.huntress.com/) is a managed security platform providing endpoint detection and response (EDR), antivirus, security awareness training, and a Managed SIEM product that continuously collects and analyzes endpoint telemetry. +[Huntress][5] is a managed security platform providing endpoint detection and response (EDR), antivirus, security awareness training, and a Managed SIEM product that continuously collects and analyzes endpoint telemetry. This integration polls the Huntress Managed SIEM API using ES|QL queries and forwards all security events to Datadog as logs. During each collection run, the integration: @@ -105,7 +105,7 @@ Logs appear in Datadog Log Explorer filtered by `source:huntress`. Allow up to 1 | -------------------------- | ----------- | ------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `huntress_api_key` | Yes | - | Huntress public API key | | `huntress_secret_key` | Yes | - | Huntress secret API key | -| `log_queries` | Conditional | - | List of query objects; each has `name` (required), `esql_query` (required, must begin with `FROM logs`), and `tags` (optional). `name` is used as the `query_name` tag on metrics. **Always include a `KEEP` clause** - despite what the Huntress API docs state, queries without `KEEP` return only `uuid` and `organization_id`, not all fields. | +| `log_queries` | Conditional | - | List of query objects; each has `name` (required), `esql_query` (required, must begin with `FROM logs`), and `tags` (optional). `name` is used as the `query_name` tag on metrics. **Always include a `KEEP` clause.** Despite what the Huntress API docs state, queries without `KEEP` return only `uuid` and `organization_id`, not all fields. | | `metrics.agents.enabled` | Conditional | `false` | Collect agent fleet metrics (total, by platform, by Defender/firewall status) | | `metrics.agents.max_pages` | No | `20` | Max pages of agents to fetch per run (500 agents/page) | | `enrich_with_org_tags` | No | `true` | Fetch and attach org metadata as log tags | @@ -168,7 +168,7 @@ See [service_checks.json][3] for a list of service checks provided by this integ - Confirm the Managed SIEM feature is enabled on the account - Confirm `min_collection_interval: 900` is set in your instance config - without it, the Agent uses its default 15-second interval, and each run queries only a 15-second window with no new events - Check that each `log_queries[].esql_query` begins with `FROM logs` -- The Huntress SIEM API requires an explicit `KEEP` clause to return log fields - without one, responses contain only `uuid` and `organization_id`. Add `| KEEP @timestamp, message, host.hostname, event.category, event.code, ...` to your query. The Agent logs will show a warning if this is detected. +- The Huntress SIEM API requires an explicit `KEEP` clause to return log fields. Without one, responses contain only `uuid` and `organization_id`. Add `| KEEP @timestamp, message, host.hostname, event.category, event.code, ...` to your query. The Agent logs will show a warning if this is detected. **`LogsSent: 0` in `datadog-agent status` even though logs are being collected** @@ -183,7 +183,7 @@ Inspect the `error_type` tag to identify the root cause: | `auth_failure` | Invalid or rotated API credentials | Update `huntress_api_key` or `huntress_secret_key` | | `timeout` | ES\|QL query too broad | Add a `KEEP` or `WHERE` clause to the query | | `invalid_query` | Malformed ES\|QL | Fix the `esql_query` value in the failing `log_queries` entry | -| `server_error` | Transient Huntress API error | Check [Huntress status page](https://status.huntress.com) | +| `server_error` | Transient Huntress API error | Check [Huntress status page][6] | | `connection_error` | Network issue | Verify connectivity from the Agent host to `api.huntress.io` | | `run_failure` | Unexpected error during collection | Check Agent logs for the full stack trace | diff --git a/huntress/assets/monitors/huntress_siem_errors_high.json b/huntress/assets/monitors/huntress_siem_errors_high.json index 3c9250330c..ce769df1e5 100644 --- a/huntress/assets/monitors/huntress_siem_errors_high.json +++ b/huntress/assets/monitors/huntress_siem_errors_high.json @@ -6,7 +6,7 @@ "tags": [ "integration:huntress" ], - "description": "The Huntress SIEM integration has recorded errors in the last 30 minutes.", + "description": "The Huntress SIEM integration has recorded errors in the last 30 minutes. Repeated errors may result in log collection gaps, reducing security event visibility in Datadog.", "definition": { "name": "Huntress SIEM collection errors detected", "type": "query alert", diff --git a/huntress/assets/monitors/huntress_siem_no_logs_collected.json b/huntress/assets/monitors/huntress_siem_no_logs_collected.json index 6239e602e0..cd0c51bc0f 100644 --- a/huntress/assets/monitors/huntress_siem_no_logs_collected.json +++ b/huntress/assets/monitors/huntress_siem_no_logs_collected.json @@ -6,7 +6,7 @@ "tags": [ "integration:huntress" ], - "description": "The Huntress SIEM integration collected 0 logs in the last 2 hours.", + "description": "The Huntress SIEM integration collected 0 logs in the last 2 hours. This may indicate a collection failure or a disruption in security event visibility from Huntress endpoints.", "definition": { "name": "No Huntress SIEM logs collected in 2 hours", "type": "query alert", From 75117132ba77d42669045b54f3cc8bcabd2ec6cb Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Wed, 10 Jun 2026 13:36:54 -0500 Subject: [PATCH 35/40] Reorder readme sections --- huntress/README.md | 48 +++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/huntress/README.md b/huntress/README.md index b6268cf5f7..ea13741f2e 100644 --- a/huntress/README.md +++ b/huntress/README.md @@ -81,17 +81,28 @@ instances: tags: ["source:huntress", "env:staging"] ``` -### Validation +#### Rate limits -Run the following command to validate the integration is collecting data: +The Huntress API allows 60 requests per minute per API key pair. A typical run with 3 SIEM queries and agent metrics uses roughly 5 to 25 requests, well within this budget. For large accounts with high log volume or thousands of agents, consider splitting concerns across two instances using separate API key pairs: -```bash -sudo datadog-agent check huntress -``` +```yaml +instances: + # Instance 1: SIEM log collection only + - huntress_api_key: "" + huntress_secret_key: "" + log_queries: + - name: "all-logs" + esql_query: "FROM logs | KEEP @timestamp, message, host.hostname, event.category, event.code, event.provider, event.id, user.target.name, user.target.domain, winlog.system.EventID" -Logs appear in Datadog Log Explorer filtered by `source:huntress`. Allow up to 15 minutes for the first logs to appear. + # Instance 2: agent metrics only (isolated rate limit budget) + - huntress_api_key: "" + huntress_secret_key: "" + metrics: + agents: + enabled: true +``` -### Configuration reference +#### Configuration reference **`init_config` options** (apply to all instances): @@ -116,27 +127,16 @@ Logs appear in Datadog Log Explorer filtered by `source:huntress`. Allow up to 1 **Note:** At least one of `log_queries` or `metrics.agents.enabled: true` must be configured per instance. -### Rate limits - -The Huntress API allows 60 requests per minute per API key pair. A typical run with 3 SIEM queries and agent metrics uses roughly 5 to 25 requests, well within this budget. For large accounts with high log volume or thousands of agents, consider splitting concerns across two instances using separate API key pairs: +### Validation -```yaml -instances: - # Instance 1: SIEM log collection only - - huntress_api_key: "" - huntress_secret_key: "" - log_queries: - - name: "all-logs" - esql_query: "FROM logs | KEEP @timestamp, message, host.hostname, event.category, event.code, event.provider, event.id, user.target.name, user.target.domain, winlog.system.EventID" +Run the following command to validate the integration is collecting data: - # Instance 2: agent metrics only (isolated rate limit budget) - - huntress_api_key: "" - huntress_secret_key: "" - metrics: - agents: - enabled: true +```bash +sudo datadog-agent check huntress ``` +Logs appear in Datadog Log Explorer filtered by `source:huntress`. Allow up to 15 minutes for the first logs to appear. + ## Data Collected ### Logs From 49381e0ed55d5bf052f2e163d0027f1ec094987d Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Wed, 10 Jun 2026 13:55:25 -0500 Subject: [PATCH 36/40] Add esql name as log tag --- huntress/datadog_checks/huntress/huntress.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/huntress/datadog_checks/huntress/huntress.py b/huntress/datadog_checks/huntress/huntress.py index 665e48eff1..f80a571191 100644 --- a/huntress/datadog_checks/huntress/huntress.py +++ b/huntress/datadog_checks/huntress/huntress.py @@ -98,7 +98,7 @@ def check(self, instance): # Each query tracks its own checkpoint so queries are independently resumable checkpoint_key = instance_hash + "_" + self._query_hash(esql) - all_tags = extra_tags + query_tags + all_tags = extra_tags + query_tags + [f"huntress_log_name:{query_name}"] query_metric_tags = extra_tags + [f"query_name:{query_name}"] logs, pages, q_start, q_end = self._run_query( From 933d83d9c58d33e1df963be7d203bfdd4ebd88b0 Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Mon, 22 Jun 2026 14:12:34 -0600 Subject: [PATCH 37/40] Fix missing reference --- huntress/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/huntress/README.md b/huntress/README.md index ea13741f2e..9f11315da9 100644 --- a/huntress/README.md +++ b/huntress/README.md @@ -203,3 +203,4 @@ For questions and support, [contact Datadog support][2]. [2]: https://docs.datadoghq.com/help/ [3]: https://github.com/DataDog/integrations-extras/blob/master/huntress/service_checks.json [4]: https://github.com/DataDog/integrations-extras/blob/master/huntress/datadog_checks/huntress/data/conf.yaml.example +[5]: https://www.huntress.com/ From ad8d214bc7ca529b14daf872356fbc4abc201d7b Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Mon, 22 Jun 2026 14:19:22 -0600 Subject: [PATCH 38/40] Fix linting error --- huntress/datadog_checks/huntress/huntress.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/huntress/datadog_checks/huntress/huntress.py b/huntress/datadog_checks/huntress/huntress.py index f80a571191..abdd5333a2 100644 --- a/huntress/datadog_checks/huntress/huntress.py +++ b/huntress/datadog_checks/huntress/huntress.py @@ -405,9 +405,11 @@ def _request_with_retry(self, method, url, headers, json_body=None, params=None) attempt_429 += 1 continue self.count("huntress.siem.errors", 1, tags=["error_type:rate_limited"]) - raise Exception( - "Huntress API 429 Rate Limited after retry — reduce max_pages_per_run or increase min_collection_interval" + exceptionText = ( + "Huntress API 429 Rate Limited after retry — " + "reduce max_pages_per_run or increase min_collection_interval" ) + raise Exception(exceptionText) if 500 <= resp.status_code < 600: if attempt < max_retries_5xx: From eb133bbfe52c1dbec3701355985f128a6b464c29 Mon Sep 17 00:00:00 2001 From: Kyle Taylor Date: Mon, 22 Jun 2026 14:40:13 -0600 Subject: [PATCH 39/40] Add last missing reference --- huntress/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/huntress/README.md b/huntress/README.md index 9f11315da9..712a1a7a1e 100644 --- a/huntress/README.md +++ b/huntress/README.md @@ -204,3 +204,4 @@ For questions and support, [contact Datadog support][2]. [3]: https://github.com/DataDog/integrations-extras/blob/master/huntress/service_checks.json [4]: https://github.com/DataDog/integrations-extras/blob/master/huntress/datadog_checks/huntress/data/conf.yaml.example [5]: https://www.huntress.com/ +[6]: https://status.huntress.com/ \ No newline at end of file From 6a29a0881ac543934c456acf55e3875eb57b1c5a Mon Sep 17 00:00:00 2001 From: domalessi <111786334+domalessi@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:18:50 -0400 Subject: [PATCH 40/40] Apply suggestions from code review Co-authored-by: domalessi <111786334+domalessi@users.noreply.github.com> --- huntress/README.md | 28 ++++++++++++++-------------- huntress/manifest.json | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/huntress/README.md b/huntress/README.md index 712a1a7a1e..41cfcdd586 100644 --- a/huntress/README.md +++ b/huntress/README.md @@ -2,7 +2,7 @@ ## Overview -[Huntress][5] is a managed security platform providing endpoint detection and response (EDR), antivirus, security awareness training, and a Managed SIEM product that continuously collects and analyzes endpoint telemetry. +[Huntress][5] is a managed security platform that provides endpoint detection and response (EDR), antivirus, security awareness training, and a Managed SIEM product that continuously collects and analyzes endpoint telemetry. This integration polls the Huntress Managed SIEM API using ES|QL queries and forwards all security events to Datadog as logs. During each collection run, the integration: @@ -62,7 +62,7 @@ datadog-agent integration install -t datadog-huntress==1.0.0 #### Multiple Huntress accounts -If you manage multiple Huntress accounts, add an additional block under `instances:` for each one. Each block runs independently with its own checkpoint, org metadata cache, and metrics: +If you manage multiple Huntress accounts, add an additional block under `instances:` for each account. Each block runs independently with its own checkpoint, org metadata cache, and metrics: ```yaml instances: @@ -83,7 +83,7 @@ instances: #### Rate limits -The Huntress API allows 60 requests per minute per API key pair. A typical run with 3 SIEM queries and agent metrics uses roughly 5 to 25 requests, well within this budget. For large accounts with high log volume or thousands of agents, consider splitting concerns across two instances using separate API key pairs: +The Huntress API allows 60 requests per minute per API key pair. A typical run with 3 SIEM queries and agent metrics uses roughly 5 to 25 requests, well within the limit. For large accounts with high log volume or thousands of agents, consider distributing the load across two instances using separate API key pairs: ```yaml instances: @@ -116,7 +116,7 @@ instances: | -------------------------- | ----------- | ------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `huntress_api_key` | Yes | - | Huntress public API key | | `huntress_secret_key` | Yes | - | Huntress secret API key | -| `log_queries` | Conditional | - | List of query objects; each has `name` (required), `esql_query` (required, must begin with `FROM logs`), and `tags` (optional). `name` is used as the `query_name` tag on metrics. **Always include a `KEEP` clause.** Despite what the Huntress API docs state, queries without `KEEP` return only `uuid` and `organization_id`, not all fields. | +| `log_queries` | Conditional | - | List of query objects; each has `name` (required), `esql_query` (required, must begin with `FROM logs`), and `tags` (optional). `name` is used as the `query_name` tag on metrics. Always include a `KEEP` clause. Queries without `KEEP` return only `uuid` and `organization_id`. | | `metrics.agents.enabled` | Conditional | `false` | Collect agent fleet metrics (total, by platform, by Defender/firewall status) | | `metrics.agents.max_pages` | No | `20` | Max pages of agents to fetch per run (500 agents/page) | | `enrich_with_org_tags` | No | `true` | Fetch and attach org metadata as log tags | @@ -163,16 +163,16 @@ See [service_checks.json][3] for a list of service checks provided by this integ **No logs in Datadog after first run** -- Run `sudo datadog-agent check huntress` and inspect the output - the summary line reports how many logs were collected and the exact time range queried -- Verify the API key pair is valid by checking the Huntress Partner Portal -- Confirm the Managed SIEM feature is enabled on the account -- Confirm `min_collection_interval: 900` is set in your instance config - without it, the Agent uses its default 15-second interval, and each run queries only a 15-second window with no new events -- Check that each `log_queries[].esql_query` begins with `FROM logs` -- The Huntress SIEM API requires an explicit `KEEP` clause to return log fields. Without one, responses contain only `uuid` and `organization_id`. Add `| KEEP @timestamp, message, host.hostname, event.category, event.code, ...` to your query. The Agent logs will show a warning if this is detected. +- Run `sudo datadog-agent check huntress` and inspect the output. The summary line reports how many logs were collected and the exact time range queried. +- Verify the API key pair is valid by checking the Huntress Partner Portal. +- Confirm the Managed SIEM feature is enabled on the account. +- Confirm `min_collection_interval: 900` is set in your instance config. Otherwise, the Agent uses its default 15-second interval, and each run queries only a 15-second window with no new events. +- Check that each `log_queries[].esql_query` begins with `FROM logs`. +- The Huntress SIEM API requires an explicit `KEEP` clause to return log fields. Without one, responses contain only `uuid` and `organization_id`. Add `| KEEP @timestamp, message, host.hostname, event.category, event.code, ...` to your query. The Agent logs show a warning if this is detected. **`LogsSent: 0` in `datadog-agent status` even though logs are being collected** -This is expected. The integration sends logs directly to the Datadog Logs Intake API rather than through the Agent's internal log pipeline, so the Logs Agent counters (`LogsProcessed`, `LogsSent`) will always read 0. Use the Agent check output or Datadog Log Explorer filtered by `source:huntress` to confirm logs are arriving. +This is expected. The integration sends logs directly to the Datadog Logs Intake API rather than through the Agent's internal log pipeline, so the Logs Agent counters (`LogsProcessed`, `LogsSent`) always read 0. Use the Agent check output or Datadog Log Explorer filtered by `source:huntress` to confirm logs are arriving. **`huntress.siem.errors` count is increasing** @@ -181,8 +181,8 @@ Inspect the `error_type` tag to identify the root cause: | `error_type` | Cause | Resolution | | ------------------ | ---------------------------------- | ------------------------------------------------------------- | | `auth_failure` | Invalid or rotated API credentials | Update `huntress_api_key` or `huntress_secret_key` | -| `timeout` | ES\|QL query too broad | Add a `KEEP` or `WHERE` clause to the query | -| `invalid_query` | Malformed ES\|QL | Fix the `esql_query` value in the failing `log_queries` entry | +| `timeout` | ES|QL query too broad | Add a `KEEP` or `WHERE` clause to the query | +| `invalid_query` | Malformed ES|QL | Fix the `esql_query` value in the failing `log_queries` entry | | `server_error` | Transient Huntress API error | Check [Huntress status page][6] | | `connection_error` | Network issue | Verify connectivity from the Agent host to `api.huntress.io` | | `run_failure` | Unexpected error during collection | Check Agent logs for the full stack trace | @@ -201,7 +201,7 @@ For questions and support, [contact Datadog support][2]. [1]: https://github.com/DataDog/integrations-extras/blob/master/huntress/metadata.csv [2]: https://docs.datadoghq.com/help/ -[3]: https://github.com/DataDog/integrations-extras/blob/master/huntress/service_checks.json +[3]: https://github.com/DataDog/integrations-extras/blob/master/huntress/assets/service_checks.json [4]: https://github.com/DataDog/integrations-extras/blob/master/huntress/datadog_checks/huntress/data/conf.yaml.example [5]: https://www.huntress.com/ [6]: https://status.huntress.com/ \ No newline at end of file diff --git a/huntress/manifest.json b/huntress/manifest.json index 8c0d207a22..3cbb3a6951 100644 --- a/huntress/manifest.json +++ b/huntress/manifest.json @@ -9,7 +9,7 @@ "configuration": "README.md#Setup", "support": "README.md#Support", "changelog": "CHANGELOG.md", - "description": "Collect and forward Huntress Managed SIEM logs to Datadog for security monitoring.", + "description": "Collect and forward Huntress SIEM logs to Datadog for security monitoring.", "title": "Huntress", "media": [], "resources": [