diff --git a/content/docs/ingest-data/databases/meta.json b/content/docs/ingest-data/databases/meta.json index 64f76d1..5eb66ba 100644 --- a/content/docs/ingest-data/databases/meta.json +++ b/content/docs/ingest-data/databases/meta.json @@ -5,6 +5,7 @@ "mysql", "mongodb", "redis", + "sql-server", "elasticsearch" ] } diff --git a/content/docs/ingest-data/databases/mongodb.mdx b/content/docs/ingest-data/databases/mongodb.mdx index 9c7759f..89b3ef2 100644 --- a/content/docs/ingest-data/databases/mongodb.mdx +++ b/content/docs/ingest-data/databases/mongodb.mdx @@ -1,37 +1,39 @@ --- title: MongoDB -description: Monitor MongoDB metrics with Parseable using OpenTelemetry +description: Send MongoDB logs, metrics, and traces to Parseable with OpenTelemetry --- -Monitor your MongoDB databases by collecting metrics using the OpenTelemetry Collector and sending them to Parseable. +MongoDB is usually only one part of a larger service path. Parseable helps you bring the database logs, database metrics, and application traces for MongoDB-backed workloads into one place. The reference setup below uses a three-member replica set, one Percona MongoDB exporter per member, an OpenTelemetry Collector, and a continuously running client workload. -## Overview +## What the integration collects -The OpenTelemetry Collector's MongoDB receiver collects metrics from standalone MongoDB clusters including: +| Signal | Source | Parseable dataset | +|---|---|---| +| Logs | `mongod` logs from every replica-set member | `mongodb-logs` | +| Metrics | OpenTelemetry MongoDB receiver and Percona MongoDB exporter | `mongodb-metrics` | +| Traces | Instrumented MongoDB client spans | `mongodb-traces` | -- **Server Statistics** - Connections, operations, network traffic -- **Database Metrics** - Document counts, storage sizes -- **Collection Metrics** - Index usage, document operations -- **Replication Metrics** - Replica set status and lag + +MongoDB server telemetry provides logs and metrics. Distributed traces come from the applications or workloads that execute MongoDB operations. + ## Prerequisites -- MongoDB 4.0+ (supports 4.0, 5.0, 6.0, 7.0) -- OpenTelemetry Collector with `mongodb` receiver -- Parseable instance running and accessible +- MongoDB 6.0 or later +- OpenTelemetry Collector Contrib +- [Percona MongoDB Exporter](https://github.com/percona/mongodb_exporter) for extended Prometheus metrics +- A Parseable ingestor endpoint and API key -### Database User Setup +## Configure MongoDB -Create a monitoring user with the `clusterMonitor` role: +Create a monitoring user in the `admin` database: ```javascript -// Connect to admin database use admin -// Create monitoring user db.createUser({ user: "otel", - pwd: "your-secure-password", + pwd: "replace-me", roles: [ { role: "clusterMonitor", db: "admin" }, { role: "read", db: "local" } @@ -39,47 +41,81 @@ db.createUser({ }) ``` -## OpenTelemetry Collector Configuration +For a replica set, start every member with the same replica-set name and a persistent log path: -### Basic Configuration +```bash +mongod \ + --replSet rs0 \ + --bind_ip_all \ + --logpath /data/db/mongod.log \ + --logappend \ + --profile 1 \ + --slowms 50 +``` -Create an `otel-collector-config.yaml` file: +Initiate the replica set once: -```yaml -receivers: - mongodb: - hosts: - - endpoint: localhost:27017 - username: otel - password: ${env:MONGODB_PASSWORD} - collection_interval: 60s - initial_delay: 1s - tls: - insecure: true +```javascript +rs.initiate({ + _id: "rs0", + members: [ + { _id: 0, host: "mongo1:27017", priority: 2 }, + { _id: 1, host: "mongo2:27017", priority: 1 }, + { _id: 2, host: "mongo3:27017", priority: 1 } + ] +}) +``` -exporters: - otlphttp/parseable: - endpoint: "http://parseable:8000" - headers: - Authorization: "Basic YWRtaW46YWRtaW4=" - X-P-Stream: "mongodb-metrics" - X-P-Log-Source: "otel-metrics" - tls: - insecure: true +Run an exporter for each member. Direct connections preserve member-level replication and storage metrics: -service: - pipelines: - metrics: - receivers: [mongodb] - exporters: [otlphttp/parseable] +```yaml +mongo-exporter1: + image: percona/mongodb_exporter:0.51.0 + environment: + MONGODB_URI: >- + mongodb://otel:${MONGODB_PASSWORD}@mongo1:27017/admin + ?authSource=admin&directConnection=true + command: + - --mongodb.direct-connect + - --compatible-mode + - --discovering-mode + - --collector.diagnosticdata + - --collector.replicasetstatus + - --collector.dbstats + - --collector.topmetrics + - --collector.indexstats + - --collector.collstats ``` -### Replica Set Configuration +## Collector configuration -For monitoring MongoDB replica sets: +Mount each MongoDB data volume read-only into the Collector and create `otel-collector.yaml`. The exporter examples expect `PARSEABLE_ENDPOINT` and `PARSEABLE_API_KEY` to be set in the Collector environment: ```yaml receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + + filelog/mongodb: + include: + - /mongo/mongo1/mongod.log + - /mongo/mongo2/mongod.log + - /mongo/mongo3/mongod.log + start_at: end + include_file_name: true + include_file_path: true + operators: + - type: move + from: body + to: attributes["mongodb.raw"] + - type: add + field: body + value: MongoDB structured log + mongodb: hosts: - endpoint: mongo1:27017 @@ -88,132 +124,168 @@ receivers: username: otel password: ${env:MONGODB_PASSWORD} replica_set: rs0 - collection_interval: 60s - timeout: 1m + collection_interval: 15s + initial_delay: 1s + timeout: 10s tls: - insecure: false - insecure_skip_verify: false - ca_file: /path/to/ca.crt + insecure: true + + prometheus/mongodb_exporter: + config: + scrape_configs: + - job_name: mongodb + scrape_interval: 15s + static_configs: + - targets: + - mongo-exporter1:9216 + - mongo-exporter2:9216 + - mongo-exporter3:9216 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 512 + spike_limit_mib: 128 + resource/mongodb: + attributes: + - key: service.name + value: mongodb + action: upsert + - key: deployment.environment.name + value: production + action: upsert + batch: + timeout: 1s exporters: - otlphttp/parseable: - endpoint: "http://parseable:8000" + otlphttp/parseable_logs: + endpoint: ${env:PARSEABLE_ENDPOINT} + encoding: json headers: - Authorization: "Basic YWRtaW46YWRtaW4=" - X-P-Stream: "mongodb-metrics" - X-P-Log-Source: "otel-metrics" - tls: - insecure: true + X-API-Key: "${env:PARSEABLE_API_KEY}" + X-P-Stream: mongodb-logs + X-P-Log-Source: otel-logs + Content-Type: application/json + retry_on_failure: + enabled: true + max_elapsed_time: 0s + + otlphttp/parseable_metrics: + endpoint: ${env:PARSEABLE_ENDPOINT} + encoding: json + headers: + X-API-Key: "${env:PARSEABLE_API_KEY}" + X-P-Stream: mongodb-metrics + X-P-Log-Source: otel-metrics + Content-Type: application/json + retry_on_failure: + enabled: true + max_elapsed_time: 0s + + otlphttp/parseable_traces: + endpoint: ${env:PARSEABLE_ENDPOINT} + encoding: json + headers: + X-API-Key: "${env:PARSEABLE_API_KEY}" + X-P-Stream: mongodb-traces + X-P-Log-Source: otel-traces + Content-Type: application/json + retry_on_failure: + enabled: true + max_elapsed_time: 0s service: pipelines: + logs: + receivers: [filelog/mongodb] + processors: [memory_limiter, resource/mongodb, batch] + exporters: [otlphttp/parseable_logs] metrics: - receivers: [mongodb] - exporters: [otlphttp/parseable] + receivers: [mongodb, prometheus/mongodb_exporter] + processors: [memory_limiter, resource/mongodb, batch] + exporters: [otlphttp/parseable_metrics] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/parseable_traces] ``` -## Configuration Options - -| Parameter | Default | Description | -|-----------|---------|-------------| -| `hosts` | `localhost:27017` | List of MongoDB endpoints | -| `username` | - | Database username | -| `password` | - | Database password | -| `replica_set` | - | Replica set name for autodiscovery | -| `collection_interval` | `1m` | Metrics collection interval | -| `timeout` | `1m` | Command timeout | -| `direct_connection` | `false` | Disable autodiscovery | -| `tls.insecure` | - | Disable TLS | - -## Collected Metrics - -The MongoDB receiver collects the following metrics: - -| Metric | Description | -|--------|-------------| -| `mongodb.cache.operations` | Cache operations count | -| `mongodb.collection.count` | Number of collections | -| `mongodb.connection.count` | Active connections | -| `mongodb.cursor.count` | Open cursors | -| `mongodb.cursor.timeout.count` | Timed out cursors | -| `mongodb.database.count` | Number of databases | -| `mongodb.document.operation.count` | Document operations | -| `mongodb.global_lock.time` | Global lock time | -| `mongodb.index.count` | Number of indexes | -| `mongodb.index.size` | Index size in bytes | -| `mongodb.memory.usage` | Memory usage | -| `mongodb.network.io.receive` | Network bytes received | -| `mongodb.network.io.transmit` | Network bytes transmitted | -| `mongodb.operation.count` | Operation counts by type | -| `mongodb.storage.size` | Storage size in bytes | - -## Running the Collector - -### Docker +The `filelog` operators retain the original structured log under `mongodb.raw` while keeping the OTLP body type stable. This avoids schema conflicts when different MongoDB log records have different nested shapes. -```bash -docker run -d \ - --name otel-collector \ - -v $(pwd)/otel-collector-config.yaml:/etc/otelcol/config.yaml \ - -e MONGODB_PASSWORD=your-password \ - otel/opentelemetry-collector-contrib:latest -``` +## Generate representative activity -### Docker Compose +The reference workload creates customers, products, inventory, orders, events, a time-series measurements collection, and a churn collection. It then exercises transactions, inserts, updates, aggregation pipelines, lookups, index access, collection scans, TTL deletion, and intentional duplicate-key failures. -```yaml -version: '3.8' -services: - otel-collector: - image: otel/opentelemetry-collector-contrib:latest - volumes: - - ./otel-collector-config.yaml:/etc/otelcol/config.yaml - environment: - - MONGODB_PASSWORD=${MONGODB_PASSWORD} - depends_on: - - mongodb - - parseable +```javascript +const d = db.getSiblingDB("telemetry") + +d.orders.insertOne({ + customerId: 42, + status: "paid", + total: 99.95, + items: [{ productId: 7, quantity: 1 }], + createdAt: new Date() +}) + +d.orders.aggregate([ + { $match: { createdAt: { $gte: new Date(Date.now() - 3600000) } } }, + { $unwind: "$items" }, + { $lookup: { + from: "products", + localField: "items.productId", + foreignField: "_id", + as: "product" + } }, + { $group: { _id: "$status", orders: { $sum: 1 }, revenue: { $sum: "$total" } } } +]).toArray() ``` -## Querying MongoDB Metrics in Parseable +Instrument the MongoDB client and send spans to the Collector's OTLP HTTP endpoint. Recommended attributes include `db.system.name=mongodb`, `db.namespace`, `db.collection.name`, `db.operation.name`, `server.address`, and `server.port`. + +## Verify ingestion -Once data is flowing, query your MongoDB metrics: +```sql +SELECT count(*) FROM "mongodb-logs" +WHERE p_timestamp > NOW() - INTERVAL '10 minutes'; +``` ```sql --- Get connection counts over time -SELECT p_timestamp, connection_count -FROM "mongodb-metrics" -ORDER BY p_timestamp DESC -LIMIT 100 - --- Find databases with high storage usage -SELECT database_name, storage_size, index_size +SELECT metric_name, count(*) FROM "mongodb-metrics" -WHERE p_timestamp > NOW() - INTERVAL '1 hour' -ORDER BY storage_size DESC +WHERE p_timestamp > NOW() - INTERVAL '10 minutes' +GROUP BY metric_name +ORDER BY metric_name; ``` -## Troubleshooting +```sql +SELECT span_name, count(*) +FROM "mongodb-traces" +WHERE p_timestamp > NOW() - INTERVAL '10 minutes' +GROUP BY span_name; +``` + +Use the resulting metrics for connections, operation rates, memory, WiredTiger cache, locks, cursors, collection/index sizes, replication state, and replica lag. + +## View in Parseable -### Connection Issues +Once the Collector starts exporting, the three datasets appear separately in Parseable. The logs dataset helps you inspect MongoDB server events, the metrics dataset shows replica-set and exporter metrics, and the traces dataset shows the client-side database spans produced by your application workload. -If the collector can't connect to MongoDB: +![MongoDB logs in Parseable](./static/mongodb-logs.png) -1. Verify MongoDB is accepting connections on the configured port -2. Check the user has `clusterMonitor` role -3. Verify authentication database is correct (usually `admin`) -4. Check firewall rules allow the connection +![MongoDB metrics in Parseable](./static/mongodb-metrics.png) -### Missing Metrics +![MongoDB traces in Parseable](./static/mongodb-traces.png) -If some metrics are not appearing: +## Dashboards -1. Ensure the monitoring user has proper roles -2. Verify the replica set name is correct (if using replica sets) -3. Check timeout settings if MongoDB is slow to respond +Public Parseable dashboard templates are available in [parseablehq/dashboards](https://github.com/parseablehq/dashboards). Choose a MongoDB template and update its dataset variables if necessary. -## Next Steps +Exporter versions can change metric names. Use the template version that matches your Percona exporter, or translate the queries using the `metric_name` values present in `mongodb-metrics`. + +## Troubleshooting -- Set up [alerts](/docs/user-guide/alerting) for database performance thresholds -- Create [dashboards](/docs/user-guide/dashboards) for MongoDB monitoring -- Explore [SQL queries](/docs/user-guide/sql-editor) for custom analysis +- Verify the monitoring account has `clusterMonitor` and access to the `local` database. +- Verify `rs.status()` is healthy before starting the exporters. +- Verify every exporter endpoint returns metrics on port `9216`. +- Verify the Collector can read every `mongod.log` volume. +- An HTTP `405` from Parseable normally means the exporter is pointed at a query/UI endpoint rather than the ingestor. diff --git a/content/docs/ingest-data/databases/postgresql.mdx b/content/docs/ingest-data/databases/postgresql.mdx index 53e46ef..fb5012b 100644 --- a/content/docs/ingest-data/databases/postgresql.mdx +++ b/content/docs/ingest-data/databases/postgresql.mdx @@ -1,222 +1,263 @@ --- title: PostgreSQL -description: Monitor PostgreSQL metrics and logs with Parseable using OpenTelemetry +description: Send PostgreSQL logs, metrics, and traces to Parseable with OpenTelemetry --- -Monitor your PostgreSQL databases by collecting metrics and logs using the OpenTelemetry Collector and sending them to Parseable. +PostgreSQL issues are easier to understand when query activity, database health, and application spans sit next to each other. This guide shows how to send PostgreSQL logs, metrics, and traces to Parseable using a primary and replica, `pg_stat_statements`, `pg_exporter`, an OpenTelemetry Collector, and a continuously running SQL workload. -## Overview +## What the integration collects -The OpenTelemetry Collector's PostgreSQL receiver collects metrics from PostgreSQL databases including: +| Signal | Source | Parseable dataset | +|---|---|---| +| Logs | PostgreSQL JSON logs from the primary and replicas | `postgres-logs` | +| Metrics | OpenTelemetry PostgreSQL receiver and `pg_exporter` | `postgres-metrics` | +| Traces | Instrumented database client spans | `postgres-traces` | -- **Database Statistics** - Connections, transactions, queries -- **Table Metrics** - Row counts, dead tuples, table sizes -- **Index Metrics** - Index usage and efficiency -- **Query Performance** - Query samples and execution times -- **Replication Metrics** - Lag and replication status +The native receiver covers database and table health. `pg_exporter` adds the statement, WAL, replication, vacuum, lock, index, and I/O families commonly used by PostgreSQL dashboards. -## Prerequisites + +PostgreSQL does not create distributed traces for SQL clients. Instrument the application or workload that issues SQL and send its spans to the Collector's OTLP receiver. + -- PostgreSQL 12+ (see [supported versions](https://www.postgresql.org/support/versioning)) -- OpenTelemetry Collector with `postgresql` receiver -- Parseable instance running and accessible +## Prerequisites -### Database User Setup +- A supported PostgreSQL release +- OpenTelemetry Collector Contrib +- [`pg_exporter`](https://github.com/nbari/pg_exporter) for dashboard-compatible metrics +- A Parseable ingestor endpoint and API key + +## Configure PostgreSQL + +Enable statement statistics, I/O timing, replication data, and JSON logs: + +```ini +shared_preload_libraries = 'pg_stat_statements' +pg_stat_statements.track = all +track_io_timing = on +wal_level = replica +max_wal_senders = 10 +logging_collector = on +log_destination = 'jsonlog' +log_directory = 'log' +log_filename = 'postgresql-%Y-%m-%d_%H%M%S.json' +log_connections = on +log_disconnections = on +log_checkpoints = on +log_lock_waits = on +``` -Create a monitoring user with the required permissions: +Create the extension and a least-privilege exporter account: ```sql --- Create monitoring user -CREATE USER otel WITH PASSWORD 'your-secure-password'; +CREATE EXTENSION IF NOT EXISTS pg_stat_statements; --- Grant required permissions -GRANT SELECT ON pg_stat_database TO otel; -GRANT pg_monitor TO otel; +CREATE ROLE postgres_exporter + LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION + PASSWORD 'replace-me'; --- For query sample collection (optional) -CREATE EXTENSION IF NOT EXISTS pg_stat_statements; +GRANT pg_monitor TO postgres_exporter; +GRANT CONNECT ON DATABASE telemetry TO postgres_exporter; ``` -## OpenTelemetry Collector Configuration +Start `pg_exporter` with the collectors required by your dashboards: -### Basic Configuration +```yaml +pg-exporter: + image: ghcr.io/nbari/pg_exporter:0.17.2 + environment: + PG_EXPORTER_DSN: >- + postgresql://postgres_exporter:${POSTGRES_EXPORTER_PASSWORD} + @postgres:5432/telemetry?sslmode=disable + command: + - --collector.locks + - --collector.database + - --collector.stat + - --collector.stat_io + - --collector.replication + - --collector.index + - --collector.statements + - --statements.top-n=50 +``` + +## Collector configuration -Create an `otel-collector-config.yaml` file: +Mount the PostgreSQL data directories read-only into the Collector and create `otel-collector.yaml`. The exporter examples expect `PARSEABLE_ENDPOINT` and `PARSEABLE_API_KEY` to be set in the Collector environment: ```yaml receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + + filelog/postgresql: + include: + - /postgres/primary/log/*.json + - /postgres/replica/log/*.json + start_at: end + include_file_name: true + include_file_path: true + postgresql: - endpoint: localhost:5432 + endpoint: postgres:5432 transport: tcp - username: otel - password: ${env:POSTGRESQL_PASSWORD} - databases: - - mydb + username: postgres_exporter + password: ${env:POSTGRES_EXPORTER_PASSWORD} + databases: [telemetry] collection_interval: 10s tls: insecure: true + prometheus/pg_exporter: + config: + scrape_configs: + - job_name: postgresql + scrape_interval: 15s + static_configs: + - targets: [pg-exporter:9432] + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 512 + spike_limit_mib: 128 + resource/postgresql: + attributes: + - key: service.name + value: postgresql + action: upsert + - key: deployment.environment.name + value: production + action: upsert + batch: + timeout: 1s + exporters: - otlphttp/parseable: - endpoint: "http://parseable:8000" + otlphttp/parseable_logs: + endpoint: ${env:PARSEABLE_ENDPOINT} + encoding: json headers: - Authorization: "Basic YWRtaW46YWRtaW4=" - X-P-Stream: "postgresql-metrics" - X-P-Log-Source: "otel-metrics" - tls: - insecure: true + X-API-Key: "${env:PARSEABLE_API_KEY}" + X-P-Stream: postgres-logs + X-P-Log-Source: otel-logs + Content-Type: application/json + retry_on_failure: + enabled: true + max_elapsed_time: 0s + + otlphttp/parseable_metrics: + endpoint: ${env:PARSEABLE_ENDPOINT} + encoding: json + headers: + X-API-Key: "${env:PARSEABLE_API_KEY}" + X-P-Stream: postgres-metrics + X-P-Log-Source: otel-metrics + Content-Type: application/json + retry_on_failure: + enabled: true + max_elapsed_time: 0s + + otlphttp/parseable_traces: + endpoint: ${env:PARSEABLE_ENDPOINT} + encoding: json + headers: + X-API-Key: "${env:PARSEABLE_API_KEY}" + X-P-Stream: postgres-traces + X-P-Log-Source: otel-traces + Content-Type: application/json + retry_on_failure: + enabled: true + max_elapsed_time: 0s service: pipelines: + logs: + receivers: [filelog/postgresql] + processors: [memory_limiter, resource/postgresql, batch] + exporters: [otlphttp/parseable_logs] metrics: - receivers: [postgresql] - exporters: [otlphttp/parseable] + receivers: [postgresql, prometheus/pg_exporter] + processors: [memory_limiter, resource/postgresql, batch] + exporters: [otlphttp/parseable_metrics] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/parseable_traces] ``` -### Advanced Configuration with Query Samples +For TLS-enabled PostgreSQL, replace `tls.insecure: true` with the appropriate CA and client certificate settings. -For detailed query performance monitoring: +## Generate representative activity -```yaml -receivers: - postgresql: - endpoint: localhost:5432 - transport: tcp - username: otel - password: ${env:POSTGRESQL_PASSWORD} - databases: - - mydb - collection_interval: 10s - tls: - insecure: false - insecure_skip_verify: false - ca_file: /path/to/ca.crt - cert_file: /path/to/client.crt - key_file: /path/to/client.key - events: - db.server.query_sample: - enabled: true - db.server.top_query: - enabled: true - query_sample_collection: - max_rows_per_query: 100 - top_query_collection: - max_rows_per_query: 100 - top_n_query: 100 +The reference workload creates customers, products, inventory, orders, order items, events, measurements, and churn tables. It continuously performs transactions, joins, aggregates, updates, deletes, lock contention, temporary-file sorts, and intentional constraint errors. -exporters: - otlphttp/parseable: - endpoint: "http://parseable:8000" - headers: - Authorization: "Basic YWRtaW46YWRtaW4=" - X-P-Stream: "postgresql-metrics" - X-P-Log-Source: "otel-metrics" - tls: - insecure: true +```sql +BEGIN; -service: - pipelines: - metrics: - receivers: [postgresql] - exporters: [otlphttp/parseable] -``` +INSERT INTO orders(customer_id, status, total, created_at) +VALUES (42, 'paid', 99.95, NOW()); -## Configuration Options - -| Parameter | Default | Description | -|-----------|---------|-------------| -| `endpoint` | `localhost:5432` | PostgreSQL server endpoint | -| `transport` | `tcp` | Transport protocol (`tcp` or `unix`) | -| `username` | - | Database username | -| `password` | - | Database password | -| `databases` | `[]` | List of databases to monitor (empty = all) | -| `collection_interval` | `10s` | Metrics collection interval | -| `tls.insecure` | `false` | Disable TLS | -| `tls.insecure_skip_verify` | `true` | Skip certificate verification | - -## Collected Metrics - -The PostgreSQL receiver collects the following metrics: - -| Metric | Description | -|--------|-------------| -| `postgresql.backends` | Number of active connections | -| `postgresql.commits` | Number of committed transactions | -| `postgresql.rollbacks` | Number of rolled back transactions | -| `postgresql.database.size` | Database size in bytes | -| `postgresql.rows` | Number of rows by operation type | -| `postgresql.blocks_read` | Number of disk blocks read | -| `postgresql.blocks_hit` | Number of buffer hits | -| `postgresql.deadlocks` | Number of deadlocks detected | -| `postgresql.temp_files` | Number of temporary files created | - -## Running the Collector - -### Docker - -```bash -docker run -d \ - --name otel-collector \ - -v $(pwd)/otel-collector-config.yaml:/etc/otelcol/config.yaml \ - -e POSTGRESQL_PASSWORD=your-password \ - otel/opentelemetry-collector-contrib:latest -``` +UPDATE inventory +SET available = available - 1, + reserved = reserved + 1 +WHERE product_id = 7; -### Docker Compose +COMMIT; -```yaml -version: '3.8' -services: - otel-collector: - image: otel/opentelemetry-collector-contrib:latest - volumes: - - ./otel-collector-config.yaml:/etc/otelcol/config.yaml - environment: - - POSTGRESQL_PASSWORD=${POSTGRESQL_PASSWORD} - depends_on: - - postgres - - parseable +SELECT status, count(*), sum(total) +FROM orders +WHERE created_at > NOW() - INTERVAL '1 hour' +GROUP BY status; ``` -## Querying PostgreSQL Metrics in Parseable +Instrument the client with OpenTelemetry. Database spans should include `db.system.name=postgresql`, `db.namespace`, `db.operation.name`, `db.query.summary`, `server.address`, and `server.port`. Avoid recording secrets or unredacted SQL parameters. -Once data is flowing, query your PostgreSQL metrics: +## Verify ingestion ```sql --- Get connection count over time -SELECT p_timestamp, backends -FROM "postgresql-metrics" -ORDER BY p_timestamp DESC -LIMIT 100 - --- Find databases with high rollback rates -SELECT database_name, commits, rollbacks, - (rollbacks::float / NULLIF(commits + rollbacks, 0)) * 100 as rollback_pct -FROM "postgresql-metrics" -WHERE p_timestamp > NOW() - INTERVAL '1 hour' +SELECT count(*) FROM "postgres-logs" +WHERE p_timestamp > NOW() - INTERVAL '10 minutes'; ``` -## Troubleshooting +```sql +SELECT metric_name, count(*) +FROM "postgres-metrics" +WHERE p_timestamp > NOW() - INTERVAL '10 minutes' +GROUP BY metric_name +ORDER BY metric_name; +``` + +```sql +SELECT span_name, count(*) +FROM "postgres-traces" +WHERE p_timestamp > NOW() - INTERVAL '10 minutes' +GROUP BY span_name; +``` + +Use these signals to build panels for connections, query latency, cache hit ratio, locks, vacuum, WAL, replication lag, table activity, database size, and top statements. + +## View in Parseable -### Connection Issues +After the Collector exports data, Parseable keeps PostgreSQL logs, metrics, and traces in separate datasets. This makes it easier to move from a database log line to the related health metrics or client spans without mixing the signal types. -If the collector can't connect to PostgreSQL: +![PostgreSQL logs in Parseable](./static/postgres-logs.png) -1. Verify PostgreSQL is accepting connections on the configured port -2. Check firewall rules allow the connection -3. Verify the username and password are correct -4. Check TLS settings match your PostgreSQL configuration +![PostgreSQL metrics in Parseable](./static/postgres-metrics.png) -### Missing Metrics +![PostgreSQL traces in Parseable](./static/postgres-traces.png) -If some metrics are not appearing: +## Dashboards -1. Ensure the monitoring user has `pg_monitor` role -2. Verify `pg_stat_statements` extension is installed for query metrics -3. Check the `databases` list includes your target databases +Public Parseable dashboard templates are available in [parseablehq/dashboards](https://github.com/parseablehq/dashboards). Choose a PostgreSQL template and update its dataset variables if you use different names. -## Next Steps +Dashboards written for another PostgreSQL exporter may require metric-name changes. Confirm the expected names against the `metric_name` field in `postgres-metrics`. + +## Troubleshooting -- Set up [alerts](/docs/user-guide/alerting) for database performance thresholds -- Create [dashboards](/docs/user-guide/dashboards) for PostgreSQL monitoring -- Explore [SQL queries](/docs/user-guide/sql-editor) for custom analysis +- Confirm the database account has `pg_monitor` and can connect to every configured database. +- Confirm `shared_preload_libraries` contains `pg_stat_statements` and PostgreSQL was restarted after changing it. +- Confirm `pg_exporter` exposes `/metrics` and the Collector can scrape port `9432`. +- Confirm the Collector can read the PostgreSQL log volume. +- An HTTP `405` from Parseable normally indicates that the exporter is using the query/UI endpoint instead of the ingestor. diff --git a/content/docs/ingest-data/databases/redis.mdx b/content/docs/ingest-data/databases/redis.mdx index 6173e20..429cc6e 100644 --- a/content/docs/ingest-data/databases/redis.mdx +++ b/content/docs/ingest-data/databases/redis.mdx @@ -1,251 +1,256 @@ --- title: Redis -description: Monitor Redis metrics with Parseable using OpenTelemetry +description: Send Redis logs, metrics, and traces to Parseable with OpenTelemetry --- -Monitor your Redis instances by collecting metrics using the OpenTelemetry Collector and sending them to Parseable. +Redis often sits on the fast path for application traffic, so small changes in latency, memory, or command behavior can matter quickly. This guide shows how to send Redis logs, metrics, and traces to Parseable using a Redis primary with two replicas, one `redis_exporter` per node, an OpenTelemetry Collector, and a continuously running client workload. -## Overview +## What the integration collects -The OpenTelemetry Collector's Redis receiver collects metrics from Redis instances using the `INFO` command including: +| Signal | Source | Parseable dataset | +|---|---|---| +| Logs | Redis server logs and optional `MONITOR` command audit output | `redis-logs` | +| Metrics | OpenTelemetry Redis receiver and Prometheus `redis_exporter` metrics | `redis-metrics` | +| Traces | Instrumented Redis client or workload spans | `redis-traces` | -- **Server Statistics** - Uptime, connected clients, memory usage -- **Memory Metrics** - Used memory, peak memory, fragmentation -- **Persistence Metrics** - RDB and AOF status -- **Replication Metrics** - Master/replica status and lag -- **Command Statistics** - Commands processed, keyspace hits/misses + +Redis does not emit distributed traces by itself. Trace the application or workload that sends Redis commands and export those spans through OTLP. + ## Prerequisites -- Redis 4.0+ -- OpenTelemetry Collector with `redis` receiver -- Parseable instance running and accessible +- Redis 6.0 or later +- OpenTelemetry Collector Contrib +- A Parseable ingestor endpoint and API key +- `redis_exporter` when you need Prometheus-compatible dashboard metrics -### Redis Authentication (Optional) +The Collector must be able to reach Redis, the exporter, the Redis log files, and the Parseable **ingestor**. Do not point the OTLP exporter at a query-only endpoint. -If your Redis instance requires authentication: +## Configure Redis -```bash -# Redis 6.0+ with ACL -redis-cli ACL SETUSER otel on >your-password +info +client +slowlog +latency allkeys +The following options enable persistence, latency data, slow-command history, and a file that the Collector can tail: -# Redis < 6.0 with requirepass -# Set password in redis.conf: requirepass your-password +```ini +appendonly yes +appendfsync everysec +slowlog-log-slower-than 1000 +slowlog-max-len 2048 +latency-monitor-threshold 10 +loglevel verbose +logfile /data/redis-server.log +requirepass ${REDIS_PASSWORD} ``` -## OpenTelemetry Collector Configuration +For a replica, add: -### Basic Configuration +```ini +replicaof redis-primary 6379 +masterauth ${REDIS_PASSWORD} +``` -Create an `otel-collector-config.yaml` file: +Run one exporter for each Redis node so role and replication metrics retain their node identity: ```yaml -receivers: - redis: - endpoint: "localhost:6379" - collection_interval: 10s - password: ${env:REDIS_PASSWORD} - -exporters: - otlphttp/parseable: - endpoint: "http://parseable:8000" - headers: - Authorization: "Basic YWRtaW46YWRtaW4=" - X-P-Stream: "redis-metrics" - X-P-Log-Source: "otel-metrics" - tls: - insecure: true - -service: - pipelines: - metrics: - receivers: [redis] - exporters: [otlphttp/parseable] +redis-exporter-primary: + image: oliver006/redis_exporter:v1.84.0 + environment: + REDIS_ADDR: redis://redis-primary:6379 + REDIS_PASSWORD: ${REDIS_PASSWORD} ``` -### TLS Configuration +## Collector configuration -For Redis instances with TLS enabled: +Create `otel-collector.yaml`. The exporter examples expect `PARSEABLE_ENDPOINT` and `PARSEABLE_API_KEY` to be set in the Collector environment: ```yaml receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + + filelog/redis: + include: + - /redis/primary/redis-server.log + - /redis/replica1/redis-server.log + - /redis/replica2/redis-server.log + start_at: end + include_file_name: true + include_file_path: true + redis: - endpoint: "localhost:6379" - collection_interval: 10s - username: otel + endpoint: redis-primary:6379 password: ${env:REDIS_PASSWORD} - tls: - insecure: false - ca_file: /path/to/ca.crt - cert_file: /path/to/client.crt - key_file: /path/to/client.key + collection_interval: 10s + + prometheus/redis_exporter: + config: + scrape_configs: + - job_name: redis + scrape_interval: 15s + static_configs: + - targets: + - redis-exporter-primary:9121 + - redis-exporter-replica1:9121 + - redis-exporter-replica2:9121 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 512 + spike_limit_mib: 128 + resource/redis: + attributes: + - key: service.name + value: redis + action: upsert + - key: deployment.environment.name + value: production + action: upsert + batch: + timeout: 1s exporters: - otlphttp/parseable: - endpoint: "http://parseable:8000" + otlphttp/parseable_logs: + endpoint: ${env:PARSEABLE_ENDPOINT} + encoding: json + headers: + X-API-Key: "${env:PARSEABLE_API_KEY}" + X-P-Stream: redis-logs + X-P-Log-Source: otel-logs + Content-Type: application/json + retry_on_failure: + enabled: true + max_elapsed_time: 0s + + otlphttp/parseable_metrics: + endpoint: ${env:PARSEABLE_ENDPOINT} + encoding: json headers: - Authorization: "Basic YWRtaW46YWRtaW4=" - X-P-Stream: "redis-metrics" - X-P-Log-Source: "otel-metrics" - tls: - insecure: true + X-API-Key: "${env:PARSEABLE_API_KEY}" + X-P-Stream: redis-metrics + X-P-Log-Source: otel-metrics + Content-Type: application/json + retry_on_failure: + enabled: true + max_elapsed_time: 0s + + otlphttp/parseable_traces: + endpoint: ${env:PARSEABLE_ENDPOINT} + encoding: json + headers: + X-API-Key: "${env:PARSEABLE_API_KEY}" + X-P-Stream: redis-traces + X-P-Log-Source: otel-traces + Content-Type: application/json + retry_on_failure: + enabled: true + max_elapsed_time: 0s service: pipelines: + logs: + receivers: [filelog/redis] + processors: [memory_limiter, resource/redis, batch] + exporters: [otlphttp/parseable_logs] metrics: - receivers: [redis] - exporters: [otlphttp/parseable] + receivers: [redis, prometheus/redis_exporter] + processors: [memory_limiter, resource/redis, batch] + exporters: [otlphttp/parseable_metrics] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/parseable_traces] ``` -### Unix Socket Configuration - -For Redis instances using Unix sockets: +Set the environment variables before starting the Collector: -```yaml -receivers: - redis: - endpoint: "/var/run/redis/redis.sock" - transport: unix - collection_interval: 10s - password: ${env:REDIS_PASSWORD} +```bash +export REDIS_PASSWORD='replace-me' +export PARSEABLE_ENDPOINT='https://ingestor.example.com' +export PARSEABLE_API_KEY='' +``` -exporters: - otlphttp/parseable: - endpoint: "http://parseable:8000" - headers: - Authorization: "Basic YWRtaW46YWRtaW4=" - X-P-Stream: "redis-metrics" - X-P-Log-Source: "otel-metrics" - tls: - insecure: true +Mount each Redis data volume read-only at the paths referenced by `filelog/redis`. -service: - pipelines: - metrics: - receivers: [redis] - exporters: [otlphttp/parseable] -``` +## Generate representative activity -## Configuration Options - -| Parameter | Default | Description | -|-----------|---------|-------------| -| `endpoint` | - | Redis server endpoint (required) | -| `transport` | `tcp` | Transport protocol (`tcp` or `unix`) | -| `username` | - | Username for Redis 6.0+ ACL | -| `password` | - | Redis password | -| `collection_interval` | `10s` | Metrics collection interval | -| `tls.insecure` | `true` | Disable TLS | -| `tls.ca_file` | - | CA certificate path | -| `tls.cert_file` | - | Client certificate path | -| `tls.key_file` | - | Client key path | - -## Collected Metrics - -The Redis receiver collects the following metrics: - -| Metric | Description | -|--------|-------------| -| `redis.clients.connected` | Number of connected clients | -| `redis.clients.blocked` | Number of blocked clients | -| `redis.clients.max_input_buffer` | Biggest input buffer | -| `redis.clients.max_output_buffer` | Biggest output buffer | -| `redis.commands` | Total commands processed | -| `redis.commands.processed` | Commands processed per second | -| `redis.connections.received` | Total connections received | -| `redis.connections.rejected` | Rejected connections | -| `redis.cpu.time` | CPU time consumed | -| `redis.db.avg_ttl` | Average TTL of keys | -| `redis.db.expires` | Keys with expiration | -| `redis.db.keys` | Total keys in database | -| `redis.keys.evicted` | Evicted keys | -| `redis.keys.expired` | Expired keys | -| `redis.keyspace.hits` | Keyspace hits | -| `redis.keyspace.misses` | Keyspace misses | -| `redis.memory.fragmentation_ratio` | Memory fragmentation ratio | -| `redis.memory.lua` | Lua memory usage | -| `redis.memory.peak` | Peak memory usage | -| `redis.memory.rss` | Resident set size | -| `redis.memory.used` | Used memory | -| `redis.net.input` | Network input bytes | -| `redis.net.output` | Network output bytes | -| `redis.rdb.changes_since_last_save` | Changes since last RDB save | -| `redis.replication.backlog_first_byte_offset` | Replication backlog offset | -| `redis.replication.offset` | Replication offset | -| `redis.slaves.connected` | Connected replicas | -| `redis.uptime` | Server uptime in seconds | - -## Running the Collector - -### Docker +The workload used to validate this integration continuously exercises strings, hashes, lists, sets, sorted sets, HyperLogLog, bitmaps, pub/sub, TTL churn, Lua scripts, cache hits and misses, and Redis Streams consumer groups. ```bash -docker run -d \ - --name otel-collector \ - -v $(pwd)/otel-collector-config.yaml:/etc/otelcol/config.yaml \ - -e REDIS_PASSWORD=your-password \ - otel/opentelemetry-collector-contrib:latest +redis-cli -h redis-primary -a "$REDIS_PASSWORD" XGROUP \ + CREATE orders:stream processors 0 MKSTREAM + +while true; do + redis-cli -h redis-primary -a "$REDIS_PASSWORD" \ + XADD orders:stream MAXLEN '~' 10000 '*' status created + redis-cli -h redis-primary -a "$REDIS_PASSWORD" INCR workload:cycles + redis-cli -h redis-primary -a "$REDIS_PASSWORD" GET missing:key + sleep 2 +done ``` -### Docker Compose +Instrument these client operations with an OpenTelemetry SDK and send spans to `http://otel-collector:4318/v1/traces`. Recommended span attributes include `db.system.name=redis`, `db.namespace`, `db.operation.name`, `server.address`, and `server.port`. -```yaml -version: '3.8' -services: - otel-collector: - image: otel/opentelemetry-collector-contrib:latest - volumes: - - ./otel-collector-config.yaml:/etc/otelcol/config.yaml - environment: - - REDIS_PASSWORD=${REDIS_PASSWORD} - depends_on: - - redis - - parseable +### Optional command audit logs + +`MONITOR` produces a live stream of every command and can be redirected to a file consumed by `filelog/redis`: + +```bash +redis-cli -h redis-primary -a "$REDIS_PASSWORD" MONITOR \ + >> /logs/redis-monitor.log ``` -## Querying Redis Metrics in Parseable + +`MONITOR` can expose command arguments and adds overhead. Use it only for controlled demonstrations or short diagnostic windows. Do not use it as a default production audit mechanism. + + +## Verify ingestion -Once data is flowing, query your Redis metrics: +```sql +SELECT count(*) FROM "redis-logs" +WHERE p_timestamp > NOW() - INTERVAL '10 minutes'; +``` ```sql --- Get memory usage over time -SELECT p_timestamp, memory_used, memory_peak -FROM "redis-metrics" -ORDER BY p_timestamp DESC -LIMIT 100 - --- Calculate cache hit ratio -SELECT - p_timestamp, - keyspace_hits, - keyspace_misses, - (keyspace_hits::float / NULLIF(keyspace_hits + keyspace_misses, 0)) * 100 as hit_ratio +SELECT metric_name, count(*) FROM "redis-metrics" -WHERE p_timestamp > NOW() - INTERVAL '1 hour' -ORDER BY p_timestamp DESC +WHERE p_timestamp > NOW() - INTERVAL '10 minutes' +GROUP BY metric_name +ORDER BY metric_name; ``` -## Troubleshooting +```sql +SELECT span_name, count(*) +FROM "redis-traces" +WHERE p_timestamp > NOW() - INTERVAL '10 minutes' +GROUP BY span_name; +``` + +Expected metric families include native names such as `redis.clients.connected` and exporter names such as `redis_commands_total`, `redis_memory_used_bytes`, and `redis_db_keys`. + +## View in Parseable -### Connection Issues +With the Redis datasets in place, you can inspect server logs, watch Redis and exporter metrics, and open traces from the client workload that is issuing Redis commands. This gives you enough context to see whether an application slowdown is coming from Redis itself or from the code path around it. -If the collector can't connect to Redis: +![Redis logs in Parseable](./static/redis-logs.png) -1. Verify Redis is accepting connections on the configured port -2. Check the password is correct -3. Verify `protected-mode` settings in Redis -4. Check firewall rules allow the connection +![Redis metrics in Parseable](./static/redis-metrics.png) -### Missing Metrics +![Redis traces in Parseable](./static/redis-traces.png) -If some metrics are not appearing: +## Dashboards -1. Ensure the Redis user has permission to run `INFO` command -2. Check if specific features (like replication) are enabled -3. Verify the Redis version supports the metrics +Parseable maintains reusable public dashboard templates in the [parseablehq/dashboards](https://github.com/parseablehq/dashboards) repository. Start with a Redis template and adjust dataset names if you changed the defaults in this guide. -## Next Steps +Grafana dashboards built for `redis_exporter` are useful references. Dashboards built for the direct Redis datasource or Telegraf require metric-name translation. + +## Troubleshooting -- Set up [alerts](/docs/user-guide/alerting) for cache performance thresholds -- Create [dashboards](/docs/user-guide/dashboards) for Redis monitoring -- Explore [SQL queries](/docs/user-guide/sql-editor) for custom analysis +- Check the Collector log for Redis authentication, scrape, and OTLP export errors. +- Verify `redis-cli INFO` works with the monitoring credentials. +- Verify every exporter returns metrics from `/metrics`. +- An HTTP `405` from Parseable usually means the Collector is pointed at the query/UI endpoint instead of the ingestor. +- Keep Redis, exporter, and OTLP receiver ports private; only the Collector needs outbound access to Parseable. diff --git a/content/docs/ingest-data/databases/sql-server.mdx b/content/docs/ingest-data/databases/sql-server.mdx new file mode 100644 index 0000000..9d3fec0 --- /dev/null +++ b/content/docs/ingest-data/databases/sql-server.mdx @@ -0,0 +1,298 @@ +--- +title: Microsoft SQL Server +description: Send SQL Server logs, metrics, and traces to Parseable with OpenTelemetry +--- + +SQL Server emits useful signals from a few different places: ERRORLOG files, dynamic management views, query telemetry, and the applications that execute T-SQL. This guide brings those signals into Parseable with the OpenTelemetry SQL Server and SQL Query receivers, ERRORLOG collection, and a continuously running SQL client workload. + +## What the integration collects + +| Signal | Source | Parseable dataset | +|---|---|---| +| Logs | SQL Server ERRORLOG, active query samples, and top-query events | `sqlserver-logs` | +| Metrics | OpenTelemetry SQL Server receiver and optional SQL Query metrics | `sqlserver-metrics` | +| Traces | Instrumented SQL client spans | `sqlserver-traces` | + + +SQL Server does not emit end-to-end application traces. Instrument the application or workload executing T-SQL and send its spans to the Collector over OTLP. + + +## Prerequisites + +- SQL Server 2019 or later +- OpenTelemetry Collector Contrib +- A Parseable ingestor endpoint and API key +- A monitoring login with access to SQL Server dynamic management views + +SQL Server Developer Edition is licensed only for development and testing. Use an appropriately licensed edition in production. + +## Create a monitoring login + +Run as a SQL Server administrator: + +```sql +CREATE LOGIN otel WITH PASSWORD = 'replace-with-a-strong-password'; +GRANT VIEW ANY DATABASE TO otel; + +-- SQL Server 2022 and later +GRANT VIEW SERVER PERFORMANCE STATE TO otel; + +-- Use VIEW SERVER STATE instead on older SQL Server releases. +``` + +Enable Query Store in the application database: + +```sql +ALTER DATABASE telemetry SET QUERY_STORE = ON ( + OPERATION_MODE = READ_WRITE, + QUERY_CAPTURE_MODE = AUTO, + MAX_STORAGE_SIZE_MB = 256, + INTERVAL_LENGTH_MINUTES = 1 +); +``` + +For SQL Server on Linux, ERRORLOG is normally available at `/var/opt/mssql/log/errorlog`. Mount the log directory read-only into the Collector. + +## Collector configuration + +Create `otel-collector.yaml`. The exporter examples expect `PARSEABLE_ENDPOINT` and `PARSEABLE_API_KEY` to be set in the Collector environment: + +```yaml +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + + filelog/sqlserver: + include: + - /mssql/log/errorlog + - /mssql/log/errorlog.* + exclude: + - /mssql/log/*.xel + start_at: end + include_file_name: true + include_file_path: true + + sqlserver: + collection_interval: 10s + username: otel + password: ${env:MSSQL_PASSWORD} + server: sqlserver + port: 1433 + events: + db.server.query_sample: + enabled: true + db.server.top_query: + enabled: true + top_query_collection: + lookback_time: 60s + max_query_sample_count: 1000 + top_query_count: 100 + collection_interval: 30s + query_sample_collection: + max_rows_per_query: 100 + metrics: + sqlserver.database.io: + enabled: true + sqlserver.database.latency: + enabled: true + sqlserver.database.operations: + enabled: true + sqlserver.database.tempdb.space: + enabled: true + sqlserver.deadlock.rate: + enabled: true + sqlserver.memory.usage: + enabled: true + sqlserver.os.wait.duration: + enabled: true + sqlserver.processes.blocked: + enabled: true + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 700 + spike_limit_mib: 150 + transform/sqlserver_query_logs: + error_mode: ignore + log_statements: + - context: log + statements: + - set(body, "SQL Server query telemetry event") + resource/sqlserver: + attributes: + - key: service.name + value: sqlserver + action: upsert + - key: deployment.environment.name + value: production + action: upsert + batch: + timeout: 1s + +exporters: + otlphttp/parseable_logs: + endpoint: ${env:PARSEABLE_ENDPOINT} + encoding: json + headers: + X-API-Key: "${env:PARSEABLE_API_KEY}" + X-P-Stream: sqlserver-logs + X-P-Log-Source: otel-logs + Content-Type: application/json + retry_on_failure: + enabled: true + max_elapsed_time: 0s + + otlphttp/parseable_metrics: + endpoint: ${env:PARSEABLE_ENDPOINT} + encoding: json + headers: + X-API-Key: "${env:PARSEABLE_API_KEY}" + X-P-Stream: sqlserver-metrics + X-P-Log-Source: otel-metrics + Content-Type: application/json + retry_on_failure: + enabled: true + max_elapsed_time: 0s + + otlphttp/parseable_traces: + endpoint: ${env:PARSEABLE_ENDPOINT} + encoding: json + headers: + X-API-Key: "${env:PARSEABLE_API_KEY}" + X-P-Stream: sqlserver-traces + X-P-Log-Source: otel-traces + Content-Type: application/json + retry_on_failure: + enabled: true + max_elapsed_time: 0s + +service: + pipelines: + logs/errorlog: + receivers: [filelog/sqlserver] + processors: [memory_limiter, resource/sqlserver, batch] + exporters: [otlphttp/parseable_logs] + logs/query_events: + receivers: [sqlserver] + processors: + - memory_limiter + - transform/sqlserver_query_logs + - resource/sqlserver + - batch + exporters: [otlphttp/parseable_logs] + metrics: + receivers: [sqlserver] + processors: [memory_limiter, resource/sqlserver, batch] + exporters: [otlphttp/parseable_metrics] + traces: + receivers: [otlp] + processors: [memory_limiter, batch] + exporters: [otlphttp/parseable_traces] +``` + +The query-event transform keeps the log body type stable while retaining query attributes such as `db.query.text`, elapsed time, reads, writes, query hashes, wait type, and blocking session ID. + +### Add dashboard-compatible custom metrics + +Some dashboards expect `mssql_*` metric names rather than native `sqlserver.*` names. Add the SQL Query receiver when necessary: + +```yaml +receivers: + sqlquery/dashboard_metrics: + driver: sqlserver + datasource: ${env:MSSQL_DATASOURCE} + collection_interval: 15s + queries: + - sql: | + SELECT COALESCE(DB_NAME(database_id), 'master') AS database_name, + CAST(COUNT_BIG(*) AS bigint) AS connection_count + FROM sys.dm_exec_sessions + WHERE is_user_process = 1 + GROUP BY database_id + metrics: + - metric_name: mssql_connections + value_column: connection_count + attribute_columns: [database_name] + + - sql: | + SELECT CAST(SUM(io_stall_read_ms + io_stall_write_ms) AS bigint) + AS io_stall_total + FROM sys.dm_io_virtual_file_stats(NULL, NULL) + metrics: + - metric_name: mssql_io_stall_total + value_column: io_stall_total +``` + +Add `sqlquery/dashboard_metrics` to the metrics pipeline. Limit custom performance-counter queries to the counters used by your dashboards; exporting every SQL Server counter creates unnecessary volume. + +## Generate representative activity + +The reference workload creates customer, product, inventory, order, event, measurement, and churn tables. It exercises transactions, joins, aggregates, full scans, tempdb sorts, lock waits, deliberate deadlocks, duplicate-key errors, and ERRORLOG entries. + +```sql +SET XACT_ABORT ON; +BEGIN TRAN; + +INSERT dbo.Orders(ExternalId, CustomerId, Status, Total) +VALUES(NEWID(), 42, 'paid', 99.95); + +UPDATE dbo.Inventory +SET Available = Available - 1, + Reserved = Reserved + 1, + UpdatedAt = SYSUTCDATETIME() +WHERE ProductId = 7; + +COMMIT; + +RAISERROR('SQL Server telemetry workload completed', 10, 1) WITH LOG; +``` + +Instrument `SqlClient`, JDBC, ODBC, or the client library used by your application. Useful span attributes include `db.system.name=microsoft.sql_server`, `db.namespace`, `db.operation.name`, `db.query.summary`, `server.address`, and `server.port`. + +## Verify ingestion + +```sql +SELECT count(*) FROM "sqlserver-logs" +WHERE p_timestamp > NOW() - INTERVAL '10 minutes'; +``` + +```sql +SELECT metric_name, count(*) +FROM "sqlserver-metrics" +WHERE p_timestamp > NOW() - INTERVAL '10 minutes' +GROUP BY metric_name +ORDER BY metric_name; +``` + +```sql +SELECT span_name, count(*) +FROM "sqlserver-traces" +WHERE p_timestamp > NOW() - INTERVAL '10 minutes' +GROUP BY span_name; +``` + +## View in Parseable + +SQL Server data lands in Parseable as logs, metrics, and traces. Use the logs dataset for ERRORLOG and query events, the metrics dataset for SQL Server receiver and custom SQL Query metrics, and the traces dataset for the application spans around T-SQL calls. + +![SQL Server logs in Parseable](./static/sqlserver-logs.png) + +![SQL Server metrics in Parseable](./static/sqlserver-metrics.png) + +![SQL Server traces in Parseable](./static/sqlserver-traces.png) + +## Dashboards + +Public Parseable dashboard templates are available in [parseablehq/dashboards](https://github.com/parseablehq/dashboards). SQL Server templates may use native `sqlserver.*`, custom `mssql_*`, or generic performance-counter families; match the Collector receivers to the template requirements. + +## Troubleshooting + +- SQL Server 2022 and later require `VIEW SERVER PERFORMANCE STATE`; earlier releases use `VIEW SERVER STATE`. +- If the Collector reports certificate parsing errors against a development container, configure a valid SQL Server certificate. Do not disable certificate validation in production. +- An HTTP `405` from Parseable normally means the exporter is pointed at the query/UI endpoint rather than the ingestor. +- If query events produce log schema errors, keep ERRORLOG and query events in separate pipelines and normalize the query-event body as shown above. diff --git a/content/docs/ingest-data/databases/static/mongodb-logs.png b/content/docs/ingest-data/databases/static/mongodb-logs.png new file mode 100644 index 0000000..6815f2f Binary files /dev/null and b/content/docs/ingest-data/databases/static/mongodb-logs.png differ diff --git a/content/docs/ingest-data/databases/static/mongodb-metrics.png b/content/docs/ingest-data/databases/static/mongodb-metrics.png new file mode 100644 index 0000000..01bdd2b Binary files /dev/null and b/content/docs/ingest-data/databases/static/mongodb-metrics.png differ diff --git a/content/docs/ingest-data/databases/static/mongodb-traces.png b/content/docs/ingest-data/databases/static/mongodb-traces.png new file mode 100644 index 0000000..70a2d5d Binary files /dev/null and b/content/docs/ingest-data/databases/static/mongodb-traces.png differ diff --git a/content/docs/ingest-data/databases/static/postgres-logs.png b/content/docs/ingest-data/databases/static/postgres-logs.png new file mode 100644 index 0000000..3fa0cbe Binary files /dev/null and b/content/docs/ingest-data/databases/static/postgres-logs.png differ diff --git a/content/docs/ingest-data/databases/static/postgres-metrics.png b/content/docs/ingest-data/databases/static/postgres-metrics.png new file mode 100644 index 0000000..ef3d359 Binary files /dev/null and b/content/docs/ingest-data/databases/static/postgres-metrics.png differ diff --git a/content/docs/ingest-data/databases/static/postgres-traces.png b/content/docs/ingest-data/databases/static/postgres-traces.png new file mode 100644 index 0000000..b7553d4 Binary files /dev/null and b/content/docs/ingest-data/databases/static/postgres-traces.png differ diff --git a/content/docs/ingest-data/databases/static/redis-logs.png b/content/docs/ingest-data/databases/static/redis-logs.png new file mode 100644 index 0000000..a158ed5 Binary files /dev/null and b/content/docs/ingest-data/databases/static/redis-logs.png differ diff --git a/content/docs/ingest-data/databases/static/redis-metrics.png b/content/docs/ingest-data/databases/static/redis-metrics.png new file mode 100644 index 0000000..ff2247a Binary files /dev/null and b/content/docs/ingest-data/databases/static/redis-metrics.png differ diff --git a/content/docs/ingest-data/databases/static/redis-traces.png b/content/docs/ingest-data/databases/static/redis-traces.png new file mode 100644 index 0000000..5cceaca Binary files /dev/null and b/content/docs/ingest-data/databases/static/redis-traces.png differ diff --git a/content/docs/ingest-data/databases/static/sqlserver-logs.png b/content/docs/ingest-data/databases/static/sqlserver-logs.png new file mode 100644 index 0000000..cfb5825 Binary files /dev/null and b/content/docs/ingest-data/databases/static/sqlserver-logs.png differ diff --git a/content/docs/ingest-data/databases/static/sqlserver-metrics.png b/content/docs/ingest-data/databases/static/sqlserver-metrics.png new file mode 100644 index 0000000..544f61e Binary files /dev/null and b/content/docs/ingest-data/databases/static/sqlserver-metrics.png differ diff --git a/content/docs/ingest-data/databases/static/sqlserver-traces.png b/content/docs/ingest-data/databases/static/sqlserver-traces.png new file mode 100644 index 0000000..a3f1cea Binary files /dev/null and b/content/docs/ingest-data/databases/static/sqlserver-traces.png differ diff --git a/content/docs/ingest-data/infrastructure/meta.json b/content/docs/ingest-data/infrastructure/meta.json new file mode 100644 index 0000000..d549be2 --- /dev/null +++ b/content/docs/ingest-data/infrastructure/meta.json @@ -0,0 +1,6 @@ +{ + "title": "Infrastructure", + "pages": [ + "windows-server" + ] +} diff --git a/content/docs/ingest-data/infrastructure/static/windowserver-logs.png b/content/docs/ingest-data/infrastructure/static/windowserver-logs.png new file mode 100644 index 0000000..1c4a5d2 Binary files /dev/null and b/content/docs/ingest-data/infrastructure/static/windowserver-logs.png differ diff --git a/content/docs/ingest-data/infrastructure/static/windowserver-metrics.png b/content/docs/ingest-data/infrastructure/static/windowserver-metrics.png new file mode 100644 index 0000000..039f035 Binary files /dev/null and b/content/docs/ingest-data/infrastructure/static/windowserver-metrics.png differ diff --git a/content/docs/ingest-data/infrastructure/windows-server.mdx b/content/docs/ingest-data/infrastructure/windows-server.mdx new file mode 100644 index 0000000..5c88e2c --- /dev/null +++ b/content/docs/ingest-data/infrastructure/windows-server.mdx @@ -0,0 +1,370 @@ +--- +title: Windows Server +description: Send Windows Event Logs, windows_exporter metrics, and application traces to Parseable +--- + +Windows Server gives you Event Logs, host metrics, and application traces, but those signals are useful only when they are easy to inspect together. This guide sends them to Parseable using a Windows Server VM on Google Compute Engine, `windows_exporter`, an OpenTelemetry Collector running as `LocalSystem`, and a scheduled PowerShell workload. + +## What the integration collects + +| Signal | Source | Parseable dataset | +|---|---|---| +| Logs | Windows Application, System, and Security Event Logs | `windows-logs` | +| Metrics | Prometheus metrics from `windows_exporter` | `windows-metrics` | +| Traces | Instrumented IIS/.NET applications or a synthetic workload | `windows-traces` | + + +Windows Server produces Event Logs and performance metrics, but not distributed application traces. Instrument IIS/.NET applications or use the test workload below to validate the trace pipeline. + + +## Prerequisites + +- Windows Server 2016 or later +- Administrator access +- Outbound access to the Parseable ingestor +- OpenTelemetry Collector Contrib for Windows +- [`windows_exporter`](https://github.com/prometheus-community/windows_exporter) + +Run the Collector on the Windows server. The Windows Event Log receiver uses Windows APIs and is not supported on Linux. + +## Optional: create a GCP Windows VM + +The following example creates Windows Server 2025 on an existing VPC and subnet: + +```bash +gcloud compute instances create "$VM_NAME" \ + --project="$PROJECT_ID" \ + --zone="$ZONE" \ + --machine-type=e2-standard-4 \ + --network="$NETWORK" \ + --subnet="$SUBNET" \ + --image-project=windows-cloud \ + --image-family=windows-2025 \ + --boot-disk-size=100GB \ + --boot-disk-type=pd-balanced \ + --tags=windows-iap-rdp,windows-telemetry \ + --shielded-secure-boot \ + --shielded-vtpm \ + --shielded-integrity-monitoring \ + --no-service-account \ + --no-scopes +``` + +Create credentials with `gcloud compute reset-windows-password`. Prefer IAP TCP forwarding or restrict direct RDP on port `3389` to your public IP. Do not expose exporter or OTLP ports publicly. + +## Install windows_exporter + +Run PowerShell as Administrator: + +```powershell +New-Item C:\Telemetry -ItemType Directory -Force + +$url = "https://github.com/prometheus-community/windows_exporter/" + + "releases/download/v0.31.7/windows_exporter-0.31.7-amd64.msi" +$msi = "C:\Telemetry\windows_exporter.msi" + +Invoke-WebRequest $url -OutFile $msi + +$args = @( + "/i", $msi, "/qn", "/norestart", + "ENABLED_COLLECTORS=cpu,cpu_info,logical_disk,memory,net,os," + + "physical_disk,process,scheduled_task,service,system,tcp", + "LISTEN_ADDR=127.0.0.1" +) + +$process = Start-Process msiexec.exe -ArgumentList $args -Wait -PassThru +$process.ExitCode +``` + +An exit code of `0` means success; `3010` means success with a required reboot. + +```powershell +Get-Service windows_exporter +curl.exe -s http://127.0.0.1:9182/health +``` + +## Install the OpenTelemetry Collector + +Download a current Windows `otelcol-contrib` release from the [OpenTelemetry Collector releases](https://github.com/open-telemetry/opentelemetry-collector-releases/releases). This example uses a validated release: + +```powershell +$dir = "C:\otelcol-contrib" +New-Item $dir -ItemType Directory -Force + +$url = "https://github.com/open-telemetry/" + + "opentelemetry-collector-releases/releases/download/v0.153.0/" + + "otelcol-contrib_0.153.0_windows_amd64.tar.gz" + +Invoke-WebRequest $url -OutFile "$dir\otelcol.tar.gz" +tar -xzf "$dir\otelcol.tar.gz" -C $dir + +New-Item "C:\ProgramData\otelcol-contrib\storage" \ + -ItemType Directory -Force +``` + +## Collector configuration + +Create `C:\otelcol-contrib\config.yaml`: + +```yaml +extensions: + health_check: + endpoint: 127.0.0.1:13133 + file_storage: + directory: C:\ProgramData\otelcol-contrib\storage + create_directory: true + +receivers: + prometheus/windows: + config: + scrape_configs: + - job_name: windows + scrape_interval: 15s + scrape_timeout: 10s + static_configs: + - targets: ["127.0.0.1:9182"] + + windows_event_log/application: + channel: Application + start_at: end + raw: true + event_driven_scraping: true + storage: file_storage + + windows_event_log/system: + channel: System + start_at: end + raw: true + event_driven_scraping: true + storage: file_storage + + windows_event_log/security: + channel: Security + start_at: end + raw: true + event_driven_scraping: true + storage: file_storage + + otlp: + protocols: + grpc: + endpoint: 127.0.0.1:4317 + http: + endpoint: 127.0.0.1:4318 + +processors: + memory_limiter: + check_interval: 1s + limit_mib: 512 + spike_limit_mib: 128 + resourcedetection/gcp: + detectors: [env, system, gcp] + timeout: 5s + override: false + resource/windows: + attributes: + - key: service.name + value: windows-server + action: upsert + - key: service.instance.id + value: ${env:COMPUTERNAME} + action: upsert + - key: deployment.environment.name + value: production + action: upsert + batch: + timeout: 1s + +exporters: + otlphttp/parseable_logs: + endpoint: ${env:PARSEABLE_ENDPOINT} + encoding: json + headers: + X-API-Key: "${env:PARSEABLE_API_KEY}" + X-P-Stream: windows-logs + X-P-Log-Source: otel-logs + Content-Type: application/json + retry_on_failure: + enabled: true + max_elapsed_time: 0s + + otlphttp/parseable_metrics: + endpoint: ${env:PARSEABLE_ENDPOINT} + encoding: json + headers: + X-API-Key: "${env:PARSEABLE_API_KEY}" + X-P-Stream: windows-metrics + X-P-Log-Source: otel-metrics + Content-Type: application/json + retry_on_failure: + enabled: true + max_elapsed_time: 0s + + otlphttp/parseable_traces: + endpoint: ${env:PARSEABLE_ENDPOINT} + encoding: json + headers: + X-API-Key: "${env:PARSEABLE_API_KEY}" + X-P-Stream: windows-traces + X-P-Log-Source: otel-traces + Content-Type: application/json + retry_on_failure: + enabled: true + max_elapsed_time: 0s + +service: + extensions: [health_check, file_storage] + pipelines: + logs: + receivers: + - windows_event_log/application + - windows_event_log/system + - windows_event_log/security + processors: + - memory_limiter + - resourcedetection/gcp + - resource/windows + - batch + exporters: [otlphttp/parseable_logs] + metrics: + receivers: [prometheus/windows] + processors: + - memory_limiter + - resourcedetection/gcp + - resource/windows + - batch + exporters: [otlphttp/parseable_metrics] + traces: + receivers: [otlp] + processors: [memory_limiter, resourcedetection/gcp, batch] + exporters: [otlphttp/parseable_traces] +``` + +`PARSEABLE_ENDPOINT` must be the ingestor endpoint, not the query or UI endpoint. Use HTTPS in production so the API key is protected in transit. + +Validate and register the Collector as `LocalSystem` so it can access the Security Event Log: + +```powershell +cd C:\otelcol-contrib +.\otelcol-contrib.exe validate --config=.\config.yaml + +$binary = '"C:\otelcol-contrib\otelcol-contrib.exe" ' + + '--config="C:\otelcol-contrib\config.yaml"' + +New-Service -Name "otelcol-contrib" ` + -DisplayName "OpenTelemetry Collector Contrib" ` + -BinaryPathName $binary ` + -StartupType Automatic + +Start-Service otelcol-contrib +``` + +## Generate test logs and traces + +For real traces, use OpenTelemetry auto-instrumentation for IIS or .NET. To validate a new installation, create `C:\otelcol-contrib\run.ps1` that writes an Event Log entry and posts one OTLP span: + +```powershell +if (-not [Diagnostics.EventLog]::SourceExists("WindowsTelemetryDemo")) { + New-EventLog -LogName Application -Source WindowsTelemetryDemo +} + +Write-EventLog -LogName Application ` + -Source WindowsTelemetryDemo ` + -EventId 1001 ` + -EntryType Information ` + -Message "Windows telemetry workload completed" + +$now = ([long][DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() * 1000000L) +$trace = ([guid]::NewGuid().ToString("N") + [guid]::NewGuid().ToString("N")).Substring(0,32) +$span = [guid]::NewGuid().ToString("N").Substring(0,16) + +$payload = @{ + resourceSpans = @(@{ + resource = @{ attributes = @( + @{ key="service.name"; value=@{ stringValue="windows-workload" } } + ) } + scopeSpans = @(@{ + scope = @{ name="windows-telemetry-workload" } + spans = @(@{ + traceId=$trace + spanId=$span + name="windows.synthetic.workload" + kind=1 + startTimeUnixNano=$now.ToString() + endTimeUnixNano=($now + 1000000L).ToString() + status=@{ code=1 } + }) + }) + }) +} | ConvertTo-Json -Depth 20 -Compress + +Invoke-RestMethod -Method Post ` + -Uri http://127.0.0.1:4318/v1/traces ` + -ContentType application/json ` + -Body $payload | Out-Null +``` + +Run it every minute as `SYSTEM`: + +```powershell +$action = New-ScheduledTaskAction ` + -Execute "powershell.exe" ` + -Argument '-ExecutionPolicy Bypass -File .\run.ps1' ` + -WorkingDirectory "C:\otelcol-contrib" + +$trigger = New-ScheduledTaskTrigger ` + -Once -At (Get-Date).AddMinutes(1) ` + -RepetitionInterval (New-TimeSpan -Minutes 1) ` + -RepetitionDuration (New-TimeSpan -Days 3650) + +$principal = New-ScheduledTaskPrincipal ` + -UserId SYSTEM -LogonType ServiceAccount -RunLevel Highest + +Register-ScheduledTask -TaskName WindowsTelemetryWorkload ` + -Action $action -Trigger $trigger -Principal $principal -Force +``` + +## Verify ingestion + +```sql +SELECT count(*) FROM "windows-logs" +WHERE p_timestamp > NOW() - INTERVAL '10 minutes'; +``` + +```sql +SELECT metric_name, count(*) +FROM "windows-metrics" +WHERE p_timestamp > NOW() - INTERVAL '10 minutes' +GROUP BY metric_name +ORDER BY metric_name; +``` + +```sql +SELECT span_name, count(*) +FROM "windows-traces" +WHERE p_timestamp > NOW() - INTERVAL '10 minutes' +GROUP BY span_name; +``` + +Expected metrics include `windows_cpu_time_total`, `windows_logical_disk_free_bytes`, `windows_memory_physical_free_bytes`, `windows_net_bytes_received_total`, `windows_os_info`, and `windows_service_state`. + +## View in Parseable + +Once the Windows Server Collector is running, Event Logs and host metrics appear as separate datasets in Parseable. Use the logs view to inspect Application, System, and Security events, and use the metrics view to follow CPU, memory, disk, network, service, and process-level signals. + +![Windows Server logs in Parseable](./static/windowserver-logs.png) + +![Windows Server metrics in Parseable](./static/windowserver-metrics.png) + +## Dashboards + +Public Windows Server dashboard templates are available in [parseablehq/dashboards](https://github.com/parseablehq/dashboards). Current templates use `windows_*` metrics from `windows_exporter`. Older dashboards that use `wmi_*` names require query translation. + +## Troubleshooting + +- `Get-Service windows_exporter,otelcol-contrib` should show both services as running. +- Check exporter health at `http://127.0.0.1:9182/health` and Collector health at `http://127.0.0.1:13133`. +- Validate the Collector with `otelcol-contrib.exe validate --config=config.yaml` before restarting the service. +- An HTTP `405` from Parseable indicates that the exporter is pointed at a query/UI endpoint instead of the ingestor. +- Run the Collector interactively to expose authentication, TLS, network, and schema errors. +- A scheduled task result of `0` means success. Use a short script path such as `run.ps1` to avoid accidental line breaks in task arguments. diff --git a/content/docs/ingest-data/streaming/aws-msk.mdx b/content/docs/ingest-data/streaming/aws-msk.mdx new file mode 100644 index 0000000..422ea51 --- /dev/null +++ b/content/docs/ingest-data/streaming/aws-msk.mdx @@ -0,0 +1,480 @@ +--- +title: Amazon MSK on EKS +description: Connect Parseable ingestors running in Amazon EKS to Amazon MSK with IAM authentication +--- + +If your applications are already publishing JSON records to Amazon MSK, this setup lets Parseable read them directly from Kafka without adding another moving part in the middle. Parseable ingestors run on EKS, use EKS Pod Identity for AWS IAM authentication, and connect to MSK with short-lived credentials instead of long-lived AWS keys or Kafka secrets. + + +Use a Parseable image that includes the Kafka connector and the `aws-msk` OAuth provider. The connector runs inside each Parseable ingestor; it is not a separate Kafka Connect deployment. + + +## Architecture + +```text +Application producers + | + | JSON records + v ++---------------------------+ +| Amazon MSK | +| topic -> partitions | +| IAM listener: TCP 9098 | ++-------------+-------------+ + | + | SASL_SSL + OAUTHBEARER + | AWS-signed, short-lived token + v ++-----------------------------------------------------------+ +| Amazon EKS | +| | +| EKS Pod Identity Agent <---- EKS Auth / AWS STS | +| | | +| | temporary role credentials | +| v | +| Parseable ingestor StatefulSet | +| - ingestor-0 \ | +| - ingestor-1 +-- one Kafka consumer group | +| - ingestor-N / partitions distributed across pods | ++-----------------------------+-----------------------------+ + | + | validate, batch, infer schema + v + Parseable stream named after topic + | + v + S3-compatible object store + | + v + Parseable querier and API +``` + +There are two parts to keep in mind here. The first is the data path: Parseable connects to the IAM-enabled brokers on port `9098`, joins a consumer group, and starts reading topic partitions. The second is the identity path: EKS Pod Identity gives the ingestor temporary AWS credentials, Parseable signs a short-lived OAuth bearer token from those credentials, and MSK validates that token against the IAM permissions on the role. + +At runtime, the flow is straightforward: + +1. EKS injects Pod Identity credential variables and a projected token into pods using the associated Kubernetes ServiceAccount. +2. Parseable resolves credentials through the AWS SDK default credential chain. +3. Parseable generates and refreshes an AWS MSK IAM token for the configured region. +4. `librdkafka` connects with `SASL_SSL` and `OAUTHBEARER`. +5. MSK distributes topic partitions across Parseable ingestors in the same consumer group. +6. Each ingestor batches valid JSON records by partition. The Kafka topic name becomes the Parseable stream name. +7. Parseable writes processed data to its configured object store and commits Kafka offsets. + +## Prerequisites + +- An active Amazon MSK Provisioned cluster running Apache Kafka `2.7.1` or later. +- An Amazon EKS cluster with Linux EC2 worker nodes. +- Network routing between the EKS nodes or pod subnets and the MSK broker subnets. +- An MSK security group rule allowing TCP `9098` from the EKS node security group or the pod security group. +- A Kafka topic containing JSON records. +- Parseable running in distributed mode with an ingestor StatefulSet. +- AWS CLI and `kubectl` access with permission to update MSK, IAM, and EKS resources. + +The examples below use these placeholders: + +| Placeholder | Example | +| --- | --- | +| `` | `ap-south-1` | +| `` | `123456789012` | +| `` | `parseable-eks` | +| `` | `parseable` | +| `` | `parseable-ingestor` | +| `` | `parseable-msk` | +| `` | The identifier after the cluster name in the MSK ARN | +| `` | `parseable-events` | +| `` | `parseable-ingestor` | + +## 1. Enable IAM authentication on MSK + +Check the current authentication and encryption settings: + +```bash +aws kafka describe-cluster-v2 \ + --cluster-arn "" \ + --region "" \ + --query 'ClusterInfo.Provisioned.{Authentication:ClientAuthentication,Encryption:EncryptionInfo.EncryptionInTransit}' +``` + +If IAM authentication is disabled, retrieve the current MSK version token: + +```bash +CURRENT_VERSION=$(aws kafka describe-cluster-v2 \ + --cluster-arn "" \ + --region "" \ + --query 'ClusterInfo.CurrentVersion' \ + --output text) +``` + +Enable IAM authentication, require TLS between clients and brokers, and disable unauthenticated clients: + +```bash +OPERATION_ARN=$(aws kafka update-security \ + --cluster-arn "" \ + --current-version "$CURRENT_VERSION" \ + --client-authentication '{"Sasl":{"Iam":{"Enabled":true}},"Unauthenticated":{"Enabled":false}}' \ + --encryption-info '{"EncryptionInTransit":{"ClientBroker":"TLS"}}' \ + --region "" \ + --query 'ClusterOperationArn' \ + --output text) +``` + + +Changing `ClientBroker` from `TLS_PLAINTEXT` to `TLS` disables plaintext endpoints. Disabling unauthenticated access also requires every client to use a configured authentication mechanism. Migrate existing clients before applying this change. + + +After the update starts, wait until the operation returns `UPDATE_COMPLETE`: + +```bash +aws kafka describe-cluster-operation \ + --cluster-operation-arn "$OPERATION_ARN" \ + --region "" \ + --query 'ClusterOperationInfo.OperationState' \ + --output text +``` + +Retrieve the IAM bootstrap address: + +```bash +aws kafka get-bootstrap-brokers \ + --cluster-arn "" \ + --region "" \ + --query 'BootstrapBrokerStringSaslIam' \ + --output text +``` + +This returns a comma-separated broker list on port `9098`. Use that exact value for `P_KAFKA_BOOTSTRAP_SERVERS`. Do not switch it to the plaintext `9092` or the TLS-only `9094` bootstrap address. + +See [Update Amazon MSK security settings](https://docs.aws.amazon.com/msk/latest/developerguide/msk-update-security.html) and [IAM access control](https://docs.aws.amazon.com/msk/latest/developerguide/iam-access-control.html). + +## 2. Configure network access + +The Parseable pods must be able to resolve the private MSK broker hostnames and connect to every broker on TCP `9098`. + +If EKS and MSK are in the same VPC, this usually comes down to three checks. The MSK security group must allow inbound `9098` from the EKS node or pod security group. The subnets must be routable to each other. The network ACLs must also allow request and return traffic. + +If EKS and MSK live in different VPCs or different accounts, set up private connectivity first. VPC peering, Transit Gateway, or [MSK multi-VPC private connectivity](https://docs.aws.amazon.com/msk/latest/developerguide/aws-access-mult-vpc.html) are the usual options. In cross-account setups, make sure the cluster policy and IAM trust settings line up with the role used by the ingestors. + +Test DNS and TCP from an ingestor or a temporary pod: + +```bash +kubectl -n exec -- \ + getent hosts + +kubectl -n exec -- \ + sh -c 'nc -vz 9098' +``` + +## 3. Install EKS Pod Identity Agent + +EKS Pod Identity needs the agent on every worker node. If you are using EKS Auto Mode, this is already handled for you. For other EKS clusters, install the add-on first. + +```bash +aws eks create-addon \ + --cluster-name "" \ + --addon-name eks-pod-identity-agent \ + --region "" + +aws eks wait addon-active \ + --cluster-name "" \ + --addon-name eks-pod-identity-agent \ + --region "" +``` + +The worker node role must include `eks-auth:AssumeRoleForPodIdentity`. The AWS-managed `AmazonEKSWorkerNodePolicy` already includes that permission. If your nodes are in private subnets, they also need a path to the EKS Auth API, either through normal outbound access or through an EKS Auth VPC interface endpoint. + +See [Set up the EKS Pod Identity Agent](https://docs.aws.amazon.com/eks/latest/userguide/pod-id-agent-setup.html). + +## 4. Create the least-privilege MSK consumer policy + +The ingestor only needs data-plane permissions for the cluster, the topics it reads from, and the consumer group it joins. It does not need MSK admin permissions or producer permissions. + +Create `parseable-msk-consumer-policy.json`: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "ConnectToCluster", + "Effect": "Allow", + "Action": [ + "kafka-cluster:Connect" + ], + "Resource": [ + "arn:aws:kafka:::cluster//" + ] + }, + { + "Sid": "ReadTopic", + "Effect": "Allow", + "Action": [ + "kafka-cluster:DescribeTopic", + "kafka-cluster:ReadData" + ], + "Resource": [ + "arn:aws:kafka:::topic///" + ] + }, + { + "Sid": "UseConsumerGroup", + "Effect": "Allow", + "Action": [ + "kafka-cluster:DescribeGroup", + "kafka-cluster:AlterGroup" + ], + "Resource": [ + "arn:aws:kafka:::group///" + ] + } + ] +} +``` + +The topic and group resource names must match `P_KAFKA_CONSUMER_TOPICS` and `P_KAFKA_CONSUMER_GROUP_ID`. For multiple topics, add each topic ARN or use a narrowly scoped wildcard. + +Create the customer-managed policy: + +```bash +aws iam create-policy \ + --policy-name ParseableMskConsumer \ + --policy-document file://parseable-msk-consumer-policy.json +``` + +These actions are the permissions AWS documents for consuming data. See [Common MSK IAM authorization use cases](https://docs.aws.amazon.com/msk/latest/developerguide/iam-access-control-use-cases.html). + +## 5. Create or update the ingestor IAM role + +Create `parseable-pod-identity-trust.json`: + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowEksPodIdentity", + "Effect": "Allow", + "Principal": { + "Service": "pods.eks.amazonaws.com" + }, + "Action": [ + "sts:AssumeRole", + "sts:TagSession" + ], + "Condition": { + "StringEquals": { + "aws:RequestTag/kubernetes-namespace": "", + "aws:RequestTag/kubernetes-service-account": "" + } + } + } + ] +} +``` + +Create the role and attach the consumer policy: + +```bash +aws iam create-role \ + --role-name ParseableIngestorRole \ + --assume-role-policy-document file://parseable-pod-identity-trust.json + +aws iam attach-role-policy \ + --role-name ParseableIngestorRole \ + --policy-arn arn:aws:iam:::policy/ParseableMskConsumer +``` + + +One Kubernetes ServiceAccount can have only one EKS Pod Identity role association. If the Parseable ServiceAccount already has a role for S3, attach the MSK consumer policy to that existing role. Do not create a second association. The role must contain every permission required by that workload, including its object-store permissions. + + +For least privilege in distributed mode, use a dedicated ingestor ServiceAccount. A shared ServiceAccount gives both ingestor and querier pods the same IAM permissions even if only the ingestor has Kafka environment variables. + +## 6. Associate the role with the ServiceAccount + +Create the ServiceAccount if your deployment does not already provide it: + +```yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: + namespace: +``` + +Associate the role: + +```bash +aws eks create-pod-identity-association \ + --cluster-name "" \ + --namespace "" \ + --service-account "" \ + --role-arn "arn:aws:iam:::role/ParseableIngestorRole" \ + --region "" +``` + +Unlike IAM Roles for Service Accounts (IRSA), EKS Pod Identity does not use a ServiceAccount annotation. New pods using this ServiceAccount receive the credential environment and token mount automatically. + +See [Assign an IAM role to a Kubernetes ServiceAccount](https://docs.aws.amazon.com/eks/latest/userguide/pod-id-association.html). + +## 7. Configure the Parseable ingestor + +Add the Kafka variables only to the ingestor container. The following is a StatefulSet pod-template fragment: + +```yaml +spec: + template: + spec: + serviceAccountName: + containers: + - name: parseable + env: + - name: P_KAFKA_BOOTSTRAP_SERVERS + value: ":9098,:9098" + - name: P_KAFKA_CONSUMER_TOPICS + value: "" + - name: P_KAFKA_CONSUMER_GROUP_ID + value: "" + - name: P_KAFKA_CONSUMER_GROUP_INSTANCE_ID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: P_KAFKA_CONSUMER_AUTO_OFFSET_RESET + value: "earliest" + - name: P_KAFKA_SECURITY_PROTOCOL + value: "SASL_SSL" + - name: P_KAFKA_SASL_MECHANISM + value: "OAUTHBEARER" + - name: P_KAFKA_OAUTH_PROVIDER + value: "aws-msk" + - name: P_KAFKA_AWS_REGION + value: "" +``` + +### Required and recommended variables + +| Environment variable | Required | Value and purpose | +| --- | --- | --- | +| `P_KAFKA_BOOTSTRAP_SERVERS` | Yes | Comma-separated `BootstrapBrokerStringSaslIam` result. Empty values are rejected. | +| `P_KAFKA_CONSUMER_TOPICS` | Yes | Comma-separated topic names. Each topic becomes a Parseable stream. | +| `P_KAFKA_CONSUMER_GROUP_ID` | Recommended | Stable group shared by all ingestor replicas. It must match the IAM group resource. | +| `P_KAFKA_CONSUMER_GROUP_INSTANCE_ID` | Recommended | Unique, stable member ID. A StatefulSet pod name is a good value. | +| `P_KAFKA_CONSUMER_AUTO_OFFSET_RESET` | Recommended | `earliest` reads history when no committed offset exists; `latest` starts with new records. `group` keeps the client fallback when no committed offset exists. | +| `P_KAFKA_SECURITY_PROTOCOL` | Yes | Must be `SASL_SSL` for MSK IAM. | +| `P_KAFKA_SASL_MECHANISM` | Yes | Must be `OAUTHBEARER`. | +| `P_KAFKA_OAUTH_PROVIDER` | Yes | Set to `aws-msk` to use application-managed AWS signing. | +| `P_KAFKA_AWS_REGION` | Recommended | MSK region. If absent, Parseable checks `AWS_REGION`, `AWS_DEFAULT_REGION`, then the AWS SDK region chain. | + +Do not set `P_KAFKA_OAUTH_TOKEN_ENDPOINT_URL`, `P_KAFKA_OAUTH_CLIENT_ID`, or `P_KAFKA_OAUTH_CLIENT_SECRET` for AWS MSK. Those variables configure standard OIDC providers, not AWS IAM signing. Do not inject `AWS_ACCESS_KEY_ID` or `AWS_SECRET_ACCESS_KEY`; Pod Identity supplies rotating credentials. + +If Helm manages the StatefulSet, add this configuration to the chart values or template and run `helm upgrade`. A direct `kubectl patch` is useful for testing but is overwritten by the next Helm reconciliation. + +Apply the deployment and watch the rollout: + +```bash +kubectl -n rollout status \ + statefulset/ \ + --timeout=10m +``` + +## Scaling behavior + +- All ingestor replicas should use the same consumer group ID. +- Each replica should use a unique group instance ID. StatefulSet pod names remain stable across restarts and reduce unnecessary group churn. +- Kafka assigns each partition to only one consumer in the group. To use all ingestor replicas, create at least as many partitions as ingestors. +- `P_KAFKA_PARTITION_LISTENER_CONCURRENCY` controls processing concurrency within each ingestor. The default is `2`. +- `P_KAFKA_CONSUMER_BUFFER_SIZE` and `P_KAFKA_CONSUMER_BUFFER_TIMEOUT` control per-partition micro-batches. Defaults are `10000` records and `10000ms`. +- A committed group offset takes precedence over `P_KAFKA_CONSUMER_AUTO_OFFSET_RESET`. The reset setting matters when no valid committed offset exists. + +## Validate the integration + +Start by confirming that the ServiceAccount is actually linked to the IAM role you expect. + +### Confirm the identity association + +```bash +aws eks list-pod-identity-associations \ + --cluster-name "" \ + --region "" \ + --query "associations[?namespace=='' && serviceAccount=='']" +``` + +After the rollout, the ingestor pod should receive the Pod Identity environment variables and projected token automatically. + +### Confirm Pod Identity injection + +After the rollout, the pod should contain `AWS_CONTAINER_CREDENTIALS_FULL_URI` and `AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE`: + +```bash +kubectl -n exec -- \ + sh -c 'env | sed -n "/^AWS_/s/=.*$/=/p" | sort' +``` + +Once the pod has credentials, check the Parseable container logs. At this point you mainly want to confirm that it can reach the brokers, authenticate, and stay in the consumer group without repeated reconnects. + +### Check connector logs + +```bash +kubectl -n logs \ + --container parseable \ + --since=10m +``` + +The pod should stay ready without authentication or authorization errors. Then publish at least one valid JSON record with a producer that is authorized separately from Parseable: + +```json +{"message":"hello from Amazon MSK","level":"info"} +``` + +The producer identity needs `kafka-cluster:Connect`, `kafka-cluster:DescribeTopic`, and `kafka-cluster:WriteData`. Do not permanently grant producer permissions to the Parseable consumer role. + +Parseable creates the stream lazily after the first record is consumed and the batch is flushed. You can verify that the stream appeared with: + +```bash +curl --header "X-API-Key: $PARSEABLE_API_KEY" \ + "https:///api/v1/logstream" +``` + +Then run a small query to confirm the test record is present: + +```bash +curl --header "X-API-Key: $PARSEABLE_API_KEY" \ + --header 'Content-Type: application/json' \ + --data '{"query":"SELECT * FROM \"\" LIMIT 10","startTime":"10m","endTime":"now"}' \ + "https:///api/v1/query" +``` + +Kafka-ingested records include `p_user_agent: kafka`. + +## Security recommendations + +- Keep MSK and EKS on private networks; public broker access is not required. +- Restrict the MSK security group to the EKS node or ingestor pod security group. +- Scope IAM resources to the exact cluster, topics, and consumer-group names. +- Use a dedicated producer identity for applications writing to MSK. +- Use a dedicated ingestor ServiceAccount when the deployment supports separate ingestor and querier identities. +- Avoid static AWS credentials in Secrets, environment variables, and container images. +- Keep unauthenticated and plaintext MSK listeners disabled after all clients migrate. +- Audit token use and denied actions with AWS CloudTrail and MSK broker logs. + +## Troubleshooting + +| Symptom | Likely cause | Resolution | +| --- | --- | --- | +| `BootstrapBrokerStringSaslIam` is empty or `None` | IAM authentication is disabled or the security update is incomplete. | Wait for `UPDATE_COMPLETE`, then retrieve the brokers again. | +| Connection timeout | DNS, routing, network ACL, or security-group rule blocks port `9098`. | Test DNS and TCP from the pod; verify both directions between EKS and MSK subnets. | +| Failed to generate an MSK IAM token | Pod Identity Agent, association, region, or AWS credential chain is missing. | Check the add-on, ServiceAccount name, role association, injected `AWS_*` variables, and `P_KAFKA_AWS_REGION`. | +| `SASL authentication failed` | Wrong bootstrap type, OAuth provider, region, or credentials. | Use the IAM brokers on `9098`, `SASL_SSL`, `OAUTHBEARER`, and `P_KAFKA_OAUTH_PROVIDER=aws-msk`. | +| `TopicAuthorizationFailed` | Topic ARN or topic actions do not match. | Grant `DescribeTopic` and `ReadData` on the exact topic resource. | +| `GroupAuthorizationFailed` | Group ARN differs from `P_KAFKA_CONSUMER_GROUP_ID`. | Update the group IAM resource or the Parseable group ID. | +| `UnknownTopicOrPartition` | Topic does not exist or the configured name is wrong. | Create the topic and verify `P_KAFKA_CONSUMER_TOPICS`. | +| Ingestors are ready but no stream appears | No record has arrived, the batch timeout has not elapsed, or payloads are not valid JSON. | Publish a JSON test record and wait at least one buffer interval. | +| Only some ingestors consume | The topic has fewer partitions than consumer replicas. | Increase the topic partition count or reduce ingestor replicas. | +| New pods have no credentials | Role association was created after the pods started. | Restart or roll out the ingestor StatefulSet. | + +## IRSA alternative + +IAM Roles for Service Accounts (IRSA) also works because Parseable uses the AWS SDK default credential chain. With IRSA, configure the cluster OIDC provider, use a web-identity trust policy, and annotate the ServiceAccount with `eks.amazonaws.com/role-arn`. + +Choose either EKS Pod Identity or IRSA for a ServiceAccount. Avoid configuring both credential mechanisms on the same workload. See [IAM roles for service accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html). diff --git a/content/docs/ingest-data/streaming/gcp-managed-kafka.mdx b/content/docs/ingest-data/streaming/gcp-managed-kafka.mdx new file mode 100644 index 0000000..c639381 --- /dev/null +++ b/content/docs/ingest-data/streaming/gcp-managed-kafka.mdx @@ -0,0 +1,451 @@ +--- +title: Google Cloud Managed Kafka +description: Connect Parseable to Google Cloud Managed Service for Apache Kafka with OAuth and Application Default Credentials +--- + +Use this guide when your application already writes JSON records to Google Cloud Managed Service for Apache Kafka and you want Parseable to consume those records directly. The setup works with Parseable running in Google Kubernetes Engine (GKE) or on a Compute Engine VM. + +Authentication uses `SASL_SSL` with `OAUTHBEARER`. Google's local authentication server obtains short-lived credentials through Application Default Credentials (ADC) and exposes them to Parseable on `127.0.0.1:14293`. No service account key is required. + + +Use a Parseable image that includes the Kafka connector and the `oidc` OAuth provider. For non-Java clients such as Parseable, Google requires the local authentication server supplied in the `googleapis/managedkafka` repository. + + +## Architecture + +```text +Application producers + | + | JSON records + v ++--------------------------------------+ +| Google Cloud Managed Kafka | +| topic -> partitions | +| private bootstrap address: TCP 9092 | ++-------------------+------------------+ + | + | SASL_SSL + OAUTHBEARER + v ++----------------------------------------------------------------+ +| GKE pod or Compute Engine VM | +| | +| Google metadata server / Workload Identity | +| | | +| | ADC | +| v | +| kafka_gcp_credentials_server.py | +| listens only on 127.0.0.1:14293 | +| | | +| | short-lived OAuth token | +| v | +| Parseable ingestor(s) -> one Kafka consumer group | ++---------------------------+------------------------------------+ + | + | validate, batch, infer schema + v + Parseable stream named after topic + | + v + configured object store + | + v + Parseable querier and API +``` + +The local authentication server and Parseable must share the same network namespace. On GKE, run the server as a sidecar in each ingestor pod. On Compute Engine, run the server on the same VM before starting Parseable. + +Do not expose port `14293` through a Kubernetes Service, load balancer, or firewall. The endpoint returns live credentials and is intended to remain local to the workload. + +## Prerequisites + +- An active Google Cloud Managed Service for Apache Kafka cluster. +- A topic containing JSON records. +- A VPC subnet connected to the Managed Kafka cluster. +- Parseable running in distributed mode on GKE, or Parseable running on a Compute Engine VM. +- A Google Cloud identity authorized to connect and consume. +- Google Cloud CLI and `kubectl` access where applicable. + +The examples use these placeholders: + +| Placeholder | Example | +| --- | --- | +| `` | `observability-prod` | +| `` | `123456789012` | +| `` | `us-central1` | +| `` | `parseable-kafka` | +| `` | `parseable-events` | +| `` | `parseable-ingestor` | +| `` | `parseable` | +| `` | `parseable-ingestor` | + +## 1. Configure network access + +Managed Kafka runs in a Google-managed tenant VPC. Connecting a subnet creates Private Service Connect endpoints and Cloud DNS records in the consumer VPC. The connected subnet must be in the same region as the Kafka cluster. + +View the cluster's connected subnets and bootstrap address: + +```bash +gcloud managed-kafka clusters describe \ + --project= \ + --location= +``` + +Retrieve only the bootstrap address: + +```bash +BOOTSTRAP=$(gcloud managed-kafka clusters describe \ + --project= \ + --location= \ + --format='value(bootstrapAddress)') + +echo "$BOOTSTRAP" +``` + +The address normally uses port `9092`. Unlike self-managed Kafka, this endpoint requires TLS and authentication. + +The GKE node or pod network, or the Compute Engine VM, must: + +1. Use a VPC connected to the Managed Kafka cluster. +2. Resolve the bootstrap and broker DNS names created in that VPC. +3. Permit egress to the cluster on TCP `9092`. + +See [Configure Managed Kafka networking](https://cloud.google.com/managed-service-for-apache-kafka/docs/networking-kafka). + +## 2. Create the workload identity + +Create a Google IAM service account for the Parseable Kafka workload: + +```bash +gcloud iam service-accounts create parseable-kafka \ + --project= \ + --display-name='Parseable Managed Kafka consumer' +``` + +The service account email is: + +```text +parseable-kafka@.iam.gserviceaccount.com +``` + +Grant the Managed Kafka Client role in the project containing the Kafka cluster: + +```bash +gcloud projects add-iam-policy-binding \ + --member='serviceAccount:parseable-kafka@.iam.gserviceaccount.com' \ + --role='roles/managedkafka.client' +``` + +This role provides `managedkafka.clusters.connect`. Kafka ACLs separately control topic and consumer-group operations. + + +If the same identity also supplies Parseable's object-store credentials, grant it the required storage permissions as well. Prefer a dedicated ingestor identity so query-only workloads do not receive Kafka access. + + +See [Configure SASL authentication](https://cloud.google.com/managed-service-for-apache-kafka/docs/authentication-kafka). + +## 3. Configure Kafka ACLs + +IAM authorizes the principal to connect. Kafka ACLs authorize reads from the topic and use of the consumer group. + +Set reusable variables: + +```bash +PROJECT_ID= +REGION= +KAFKA_CLUSTER= +TOPIC= +GROUP_ID= +PRINCIPAL='User:parseable-kafka@.iam.gserviceaccount.com' +``` + +Allow topic reads: + +```bash +gcloud managed-kafka acls create "topic/$TOPIC" \ + --project="$PROJECT_ID" \ + --cluster="$KAFKA_CLUSTER" \ + --location="$REGION" \ + --acl-entry="principal=$PRINCIPAL,operation=READ,permission-type=ALLOW,host=*" \ + --acl-entry="principal=$PRINCIPAL,operation=DESCRIBE,permission-type=ALLOW,host=*" +``` + +Allow consumer-group membership and offset commits: + +```bash +gcloud managed-kafka acls create "consumerGroup/$GROUP_ID" \ + --project="$PROJECT_ID" \ + --cluster="$KAFKA_CLUSTER" \ + --location="$REGION" \ + --acl-entry="principal=$PRINCIPAL,operation=READ,permission-type=ALLOW,host=*" \ + --acl-entry="principal=$PRINCIPAL,operation=DESCRIBE,permission-type=ALLOW,host=*" +``` + +If an ACL resource already exists, add entries to it instead of creating it again. Keep the ACL topic and consumer-group names synchronized with the Parseable environment variables. + +See [Create a Managed Kafka ACL](https://cloud.google.com/managed-service-for-apache-kafka/docs/kafka-acls/create-kafka-acls). + +## 4. Package the Google local authentication server + +Google maintains `kafka_gcp_credentials_server.py` in the [googleapis/managedkafka repository](https://github.com/googleapis/managedkafka/tree/main/kafka-auth-local-server). It binds to `localhost:14293`, uses ADC, refreshes access tokens, and returns the token response expected by Kafka OIDC clients. + +For production, pin the repository to a reviewed commit and build an internal image. Copy the official script into a build directory and add this `Dockerfile`: + +```dockerfile +FROM python:3.12-slim + +WORKDIR /app +COPY kafka_gcp_credentials_server.py /app/ + +RUN pip install --no-cache-dir 'google-auth[urllib3]>=2.40.3' + +CMD ["python", "/app/kafka_gcp_credentials_server.py"] +``` + +Build and push it to Artifact Registry: + +```bash +docker build -t -docker.pkg.dev///managed-kafka-auth: . +docker push -docker.pkg.dev///managed-kafka-auth: +``` + +Google requires `google-auth` version `2.40.3` or later for direct GKE Workload Identity Federation with this server. + +## 5A. Run Parseable on GKE + +### Enable Workload Identity Federation + +Enable Workload Identity Federation for the GKE cluster if it is not already enabled: + +```bash +gcloud container clusters update \ + --project= \ + --location= \ + --workload-pool=.svc.id.goog +``` + +Create the Kubernetes ServiceAccount: + +```bash +kubectl create serviceaccount \ + --namespace +``` + +Allow it to impersonate the Google IAM service account: + +```bash +gcloud iam service-accounts add-iam-policy-binding \ + parseable-kafka@.iam.gserviceaccount.com \ + --project= \ + --role='roles/iam.workloadIdentityUser' \ + --member='serviceAccount:.svc.id.goog[/]' + +kubectl annotate serviceaccount \ + --namespace \ + iam.gke.io/gcp-service-account=parseable-kafka@.iam.gserviceaccount.com \ + --overwrite +``` + +This keyless linked-service-account method gives the authentication server a stable service account email that also matches the Kafka ACL principal. + +### Add the authentication sidecar and Kafka environment + +Add the local authentication server to the same pod as each Parseable ingestor. Containers in one pod share `127.0.0.1` and the Kubernetes ServiceAccount identity. + +```yaml +spec: + template: + spec: + serviceAccountName: + containers: + - name: parseable + env: + - name: P_KAFKA_BOOTSTRAP_SERVERS + value: "" + - name: P_KAFKA_CONSUMER_TOPICS + value: "" + - name: P_KAFKA_CONSUMER_GROUP_ID + value: "" + - name: P_KAFKA_CONSUMER_GROUP_INSTANCE_ID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: P_KAFKA_CONSUMER_AUTO_OFFSET_RESET + value: "earliest" + - name: P_KAFKA_SECURITY_PROTOCOL + value: "SASL_SSL" + - name: P_KAFKA_SASL_MECHANISM + value: "OAUTHBEARER" + - name: P_KAFKA_OAUTH_PROVIDER + value: "oidc" + - name: P_KAFKA_OAUTH_TOKEN_ENDPOINT_URL + value: "http://127.0.0.1:14293" + - name: P_KAFKA_OAUTH_CLIENT_ID + value: "unused" + - name: P_KAFKA_OAUTH_CLIENT_SECRET + value: "unused" + + - name: managed-kafka-auth + image: -docker.pkg.dev///managed-kafka-auth: + resources: + requests: + cpu: 20m + memory: 64Mi + limits: + memory: 128Mi + readinessProbe: + exec: + command: + - python + - -c + - import socket; socket.create_connection(('127.0.0.1', 14293), 2).close() + periodSeconds: 5 +``` + +Use a startup gate or native sidecar ordering when your Kubernetes version supports it. If Parseable starts before the authentication server, the connector can initially fail; restarting the Parseable container after the sidecar is ready resolves the race. + +If Helm manages the StatefulSet, add the sidecar and environment through chart values or templates. Direct StatefulSet edits are overwritten by later Helm reconciliation. + +### Direct GKE principal alternative + +Managed Kafka also supports direct Workload Identity Federation principals on GKE `1.31.1-gke.1241000` or later. This mode requires: + +- `iam.gke.io/return-principal-id-as-email: "true"` on the Kubernetes ServiceAccount. +- `google-auth>=2.40.3` in the local server image. +- `roles/managedkafka.client` and Kafka ACLs granted to the resulting GKE principal. + +Use the linked Google IAM service account method above when you want one stable email principal across GKE and Compute Engine. + +## 5B. Run Parseable on Compute Engine + +Attach `parseable-kafka@.iam.gserviceaccount.com` to the VM with the `cloud-platform` access scope. The IAM roles on the service account remain the authorization boundary. + +Install and start the official local server before Parseable: + +```bash +git clone https://github.com/googleapis/managedkafka.git +cd managedkafka +git checkout + +python3 -m venv /opt/managed-kafka-auth +/opt/managed-kafka-auth/bin/pip install \ + -r kafka-auth-local-server/requirements.txt + +/opt/managed-kafka-auth/bin/python \ + kafka-auth-local-server/kafka_gcp_credentials_server.py +``` + +For production, run it as a `systemd` service with `Restart=always`, and start Parseable only after port `14293` is listening. + +Set the same Parseable variables used in GKE: + +```bash +export P_KAFKA_BOOTSTRAP_SERVERS='' +export P_KAFKA_CONSUMER_TOPICS='' +export P_KAFKA_CONSUMER_GROUP_ID='' +export P_KAFKA_CONSUMER_GROUP_INSTANCE_ID="$(hostname)" +export P_KAFKA_CONSUMER_AUTO_OFFSET_RESET='earliest' +export P_KAFKA_SECURITY_PROTOCOL='SASL_SSL' +export P_KAFKA_SASL_MECHANISM='OAUTHBEARER' +export P_KAFKA_OAUTH_PROVIDER='oidc' +export P_KAFKA_OAUTH_TOKEN_ENDPOINT_URL='http://127.0.0.1:14293' +export P_KAFKA_OAUTH_CLIENT_ID='unused' +export P_KAFKA_OAUTH_CLIENT_SECRET='unused' +``` + +The VM metadata server supplies ADC. Do not create or download a service account key. + +## Parseable environment reference + +| Environment variable | Required | Value and purpose | +| --- | --- | --- | +| `P_KAFKA_BOOTSTRAP_SERVERS` | Yes | Bootstrap address returned by `gcloud managed-kafka clusters describe`. Empty values are rejected. | +| `P_KAFKA_CONSUMER_TOPICS` | Yes | Comma-separated topics. Each topic becomes a Parseable stream. | +| `P_KAFKA_CONSUMER_GROUP_ID` | Recommended | Shared by all ingestors and matched by the Kafka consumer-group ACL. | +| `P_KAFKA_CONSUMER_GROUP_INSTANCE_ID` | Recommended | Unique stable member ID, such as a StatefulSet pod name or VM hostname. | +| `P_KAFKA_CONSUMER_AUTO_OFFSET_RESET` | Recommended | `earliest` reads history when no committed offset exists; `latest` starts with new records. | +| `P_KAFKA_SECURITY_PROTOCOL` | Yes | `SASL_SSL`. Managed Kafka does not support plaintext connections. | +| `P_KAFKA_SASL_MECHANISM` | Yes | `OAUTHBEARER`. | +| `P_KAFKA_OAUTH_PROVIDER` | Yes | `oidc`, because `librdkafka` retrieves tokens from the local HTTP endpoint. | +| `P_KAFKA_OAUTH_TOKEN_ENDPOINT_URL` | Yes | `http://127.0.0.1:14293`. | +| `P_KAFKA_OAUTH_CLIENT_ID` | Yes | `unused`; required by the OIDC client configuration. | +| `P_KAFKA_OAUTH_CLIENT_SECRET` | Yes | `unused`; required by the OIDC client configuration. | + +Do not set `P_KAFKA_AWS_REGION` for Google Cloud. Do not set `GOOGLE_APPLICATION_CREDENTIALS` to a service account key when Workload Identity or an attached VM service account is available. + +## Scaling behavior + +- All ingestors use one consumer group, so Managed Kafka distributes topic partitions across replicas. +- Each group instance ID must be unique. StatefulSet pod names are stable and work well for static membership. +- A topic needs at least as many partitions as active ingestors to use every replica. +- `P_KAFKA_PARTITION_LISTENER_CONCURRENCY` defaults to `2` per ingestor. +- `P_KAFKA_CONSUMER_BUFFER_SIZE` defaults to `10000` records per partition. +- `P_KAFKA_CONSUMER_BUFFER_TIMEOUT` defaults to `10000ms`; a new stream may not appear until the first batch flushes. +- Committed offsets take precedence over `P_KAFKA_CONSUMER_AUTO_OFFSET_RESET`. + +## Validate the integration + +Check the local server without printing the returned token: + +```bash +curl --fail --silent --output /dev/null \ + http://127.0.0.1:14293 +``` + +On GKE, run the check in the authentication sidecar: + +```bash +kubectl -n exec \ + --container managed-kafka-auth -- \ + python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:14293', timeout=5).read()" +``` + +Check ingestor logs: + +```bash +kubectl -n logs \ + --container parseable \ + --since=10m +``` + +Publish one valid JSON record with a separately authorized producer: + +```json +{"message":"hello from Google Cloud Managed Kafka","level":"info"} +``` + +The Parseable stream is created lazily after the record is consumed and its batch is flushed. Query it: + +```bash +curl --header "X-API-Key: $PARSEABLE_API_KEY" \ + --header 'Content-Type: application/json' \ + --data '{"query":"SELECT * FROM \"\" LIMIT 10","startTime":"10m","endTime":"now"}' \ + "https:///api/v1/query" +``` + +Kafka-ingested records include `p_user_agent: kafka`. + +## Security recommendations + +- Use Workload Identity Federation or an attached VM service account; avoid service account keys. +- Keep the local token endpoint bound to `127.0.0.1` and never expose it outside the pod or VM. +- Pin and review the Google authentication-server source used in your image. +- Use separate producer and consumer principals and ACLs. +- Scope topic and group ACLs to exact names or narrow prefixes. +- Use a dedicated identity for Parseable ingestors when possible. +- Keep GKE, VM, and Managed Kafka traffic on connected private VPC networks. + +## Troubleshooting + +| Symptom | Likely cause | Resolution | +| --- | --- | --- | +| Bootstrap hostname does not resolve | Workload VPC or subnet is not connected to the cluster. | Connect the subnet and verify Cloud DNS visibility from the pod or VM. | +| Connection timeout to port `9092` | Routing or egress firewall issue. | Test DNS and TCP from the same pod or VM running Parseable. | +| Connection refused on `127.0.0.1:14293` | Local authentication server is not running or is in a different pod/host. | Run it as an ingestor sidecar or on the same VM and start it before Parseable. | +| Local server cannot obtain credentials | Workload Identity, VM service account, or ADC configuration is missing. | Verify the KSA/GSA link or attached VM service account and metadata access. | +| Local server cannot determine principal | Credential type does not expose an email principal. | Use the linked GSA method or set `GOOGLE_MANAGED_KAFKA_AUTH_PRINCIPAL` to the authorized principal. | +| `SASL authentication failed` | Wrong endpoint, OAuth settings, or principal. | Use `SASL_SSL`, `OAUTHBEARER`, `oidc`, localhost token endpoint, and `unused` client credentials. | +| Authorization error for topic | Topic ACL is missing or uses a different principal/topic. | Add `READ` and `DESCRIBE` entries to `topic/`. | +| Authorization error for group | Consumer-group ACL differs from the configured group ID. | Add the principal to `consumerGroup/`. | +| `UnknownTopicOrPartition` | Topic does not exist or its name is wrong. | Create the topic and verify `P_KAFKA_CONSUMER_TOPICS`. | +| No Parseable stream appears | No record has arrived, buffer has not flushed, or payload is not valid JSON. | Publish a JSON record and wait at least one buffer interval. | diff --git a/content/docs/ingest-data/streaming/kafka.mdx b/content/docs/ingest-data/streaming/kafka.mdx index b9db935..6a84d8c 100644 --- a/content/docs/ingest-data/streaming/kafka.mdx +++ b/content/docs/ingest-data/streaming/kafka.mdx @@ -4,13 +4,15 @@ redirect_from: - /streaming/kafka --- -The Parseable Kafka Connector enables log ingestion from Apache Kafka into Parseable, providing a high-performance, scalable, and efficient logging pipeline. +The Parseable Kafka connector lets ingestors read JSON records from Kafka topics and store them as Parseable datasets. Use it when Kafka is already part of your data path and you want Parseable to consume from the topic directly instead of adding another shipper in between. + + +For provider-specific managed Kafka setup, follow the [Amazon MSK on EKS guide](/docs/ingest-data/streaming/aws-msk) or the [Google Cloud Managed Kafka guide](/docs/ingest-data/streaming/gcp-managed-kafka). + ## Features -- Consumer & Producer Support: Supports both consuming and producing messages (ready to use for DLT). -- Configurable Buffering & Performance Settings: Optimized for high-throughput data processing. -- Security Integration: Supports SSL/TLS and SASL authentication. -- Fault Tolerance & Partitioning: Handles partition balancing, offsets, and error handling. + +The connector supports Kafka consumer configuration, configurable per-partition buffering, TLS and SASL authentication, consumer-group based scaling, and partition-aware processing. Producer settings are documented for future dead-letter-topic support, but normal ingestion only requires the consumer path. ## Configuration Options @@ -22,9 +24,7 @@ The Parseable Kafka Connector enables log ingestion from Apache Kafka into Parse | `--partition-listener-concurrency` | `P_KAFKA_PARTITION_LISTENER_CONCURRENCY` | `2` | Number of parallel threads for Kafka partition listeners. | Determines the number of threads used to process Kafka partitions. | | `--bad-data-policy` | `P_CONNECTOR_BAD_DATA_POLICY` | `fail` | Policy for handling bad data. | Determines how the client should handle corrupt or invalid messages. Options: fail, drop (not yet supported), dlt (not yet supported). | -- All parameters can be set using command-line arguments or environment variables. -- Environment variables take precedence over default values. -- When configuring both producer and consumer, make sure to specify relevant options in their respective sections. +All parameters can be set using command-line arguments or environment variables. Environment variables take precedence over default values. For ingestion, start with the general and consumer configuration tables, then add security settings based on your Kafka cluster. For more details, refer to [Kafka's official documentation](https://kafka.apache.org/documentation). @@ -107,10 +107,11 @@ For more details, refer to [Kafka's official documentation on producer configura | `--kerberos-service-name` | `P_KAFKA_KERBEROS_SERVICE_NAME` | `None` | Kerberos service name. | Required when using GSSAPI SASL mechanism. | | `--kerberos-principal` | `P_KAFKA_KERBEROS_PRINCIPAL` | `None` | Kerberos principal. | Used for Kerberos authentication. | | `--kerberos-keytab` | `P_KAFKA_KERBEROS_KEYTAB` | `None` | Path to Kerberos keytab file. | Required when using Kerberos authentication. | -| `--oauth-token-endpoint` | `P_KAFKA_OAUTH_TOKEN_ENDPOINT` | `None` | OAuth Bearer token endpoint. | Required when using OAUTHBEARER SASL mechanism. | +| `--oauth-provider` | `P_KAFKA_OAUTH_PROVIDER` | `None` | OAuth provider. | Use `aws-msk` for Amazon MSK IAM or `oidc` for a standard OIDC token endpoint. | +| `--oauth-token-endpoint-url` | `P_KAFKA_OAUTH_TOKEN_ENDPOINT_URL` | `None` | OAuth Bearer token endpoint. | Required for the `oidc` provider. Not used for Amazon MSK IAM. | | `--oauth-client-id` | `P_KAFKA_OAUTH_CLIENT_ID` | `None` | OAuth client ID. | Used for authentication with an OAuth provider. | | `--oauth-client-secret` | `P_KAFKA_OAUTH_CLIENT_SECRET` | `None` | OAuth client secret. | Used to authenticate the OAuth client. | -| `--oauth-scope` | `P_KAFKA_OAUTH_SCOPE` | `None` | OAuth scope. | Defines the permissions requested from the OAuth provider. | +| `--aws-region` | `P_KAFKA_AWS_REGION` | `None` | AWS region for MSK IAM token signing. | Falls back to `AWS_REGION`, `AWS_DEFAULT_REGION`, then the AWS SDK region chain. | ### Security Configuration Combinations @@ -147,15 +148,16 @@ Required parameters: - `--kerberos-service-name` and `--kerberos-principal` (for GSSAPI mechanism) - `--kerberos-keytab` (if using Kerberos authentication) -#### OAuth Bearer Token Authentication (Not supported yet) -- `--security-protocol=SASL_SSL or SASL_PLAINTEXT` +#### OAuth Bearer Token Authentication +- `--security-protocol=SASL_SSL` - `--sasl-mechanism=OAUTHBEARER` -Required parameters: -- `--oauth-token-endpoint` -- `--oauth-client-id` -- `--oauth-client-secret` -- `--oauth-scope` (if required by the OAuth provider) +Provider-specific parameters: + +- Amazon MSK IAM: `--oauth-provider=aws-msk` and `--aws-region=`. AWS credentials are resolved through the AWS SDK default credential chain. +- Standard OIDC: `--oauth-provider=oidc`, `--oauth-token-endpoint-url`, `--oauth-client-id`, and `--oauth-client-secret`. + +`SASL_PLAINTEXT` is rejected for OAuth bearer tokens because it would expose the token without TLS. #### Examples diff --git a/content/docs/ingest-data/streaming/meta.json b/content/docs/ingest-data/streaming/meta.json index cc0ffb0..f57e4e7 100644 --- a/content/docs/ingest-data/streaming/meta.json +++ b/content/docs/ingest-data/streaming/meta.json @@ -3,6 +3,8 @@ "pages": [ "cribl", "kafka", + "aws-msk", + "gcp-managed-kafka", "redpanda", "rabbitmq", "nats" diff --git a/content/docs/ingestion.mdx b/content/docs/ingestion.mdx index e49678e..3c3edfc 100644 --- a/content/docs/ingestion.mdx +++ b/content/docs/ingestion.mdx @@ -3,7 +3,7 @@ title: Ingestion redirect_from: - /ingestion --- -import { IconPlug, IconCode, IconWand, IconSettings, IconDatabase, IconChartDots, IconTimeline,IconBrandAws,IconBrandAzure,IconBrandGoogle } from '@tabler/icons-react'; +import { IconPlug, IconCode, IconWand, IconSettings, IconDatabase, IconServer, IconChartDots, IconTimeline,IconBrandAws,IconBrandAzure,IconBrandGoogle } from '@tabler/icons-react'; Ingestion is the process of sending Telemetry signals (Metrics, Events, Logs, Traces) into Parseable. @@ -30,7 +30,7 @@ You can use HTTP headers to control how data is ingested and processed. | Header | Description | Example | Possible Values | |--------|-------------|---------|-----------------| | `X-P-Stream` | Target dataset name. Creates the dataset if it doesn't exist. | `nginx-logs` | Valid dataset name | -| `Authorization` | Basic auth credentials (base64 encoded `username:password`) | `Basic YWRtaW46YWRtaW4=` | Valid credentials | +| `X-API-Key` | API key used to authorize the request | `px_api_key` | Valid Parseable API key | | `X-P-Tenant-ID` | Target tenant ID. | `tenant-123` | Valid tenant ID - only applicable for multi-tenant setups (for example Parseable Cloud) | | `Content-Type` | Content type of the request body | `application/json` | `application/json`,`application/protobuf` | @@ -154,10 +154,18 @@ Ingest logs and metrics from MongoDB databases. Collect logs and metrics from Redis databases. +} title='Microsoft SQL Server'> +Collect SQL Server logs, metrics, query events, and client traces. + + } title='Elasticsearch'> Migrate or sync data from Elasticsearch to Parseable. +} title='Windows Server'> +Collect Windows Event Logs, windows_exporter metrics, and application traces. + + } title='Docker'> Collect logs from Docker containers using logging drivers. diff --git a/content/docs/meta.json b/content/docs/meta.json index 4014a6c..463f73e 100644 --- a/content/docs/meta.json +++ b/content/docs/meta.json @@ -22,6 +22,7 @@ "ingest-data/ai-agents", "ingest-data/logging-agents", "ingest-data/databases", + "ingest-data/infrastructure", "ingest-data/containers", "ingest-data/streaming", "ingest-data/prometheus",