|
| 1 | +receivers: |
| 2 | + # Troubleshooting |
| 3 | + prometheus: |
| 4 | + config: |
| 5 | + scrape_configs: |
| 6 | + - job_name: 'otelcol' |
| 7 | + scrape_interval: 30s |
| 8 | + static_configs: |
| 9 | + - targets: |
| 10 | + - '0.0.0.0:8888' |
| 11 | + - ${env:CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT} |
| 12 | + # Data sources: logs |
| 13 | + fluentforward: |
| 14 | + endpoint: '0.0.0.0:24225' |
| 15 | + # Configured via OpAMP w/ authentication |
| 16 | + # Data sources: traces, metrics, logs |
| 17 | + # otlp/hyperdx: |
| 18 | + # protocols: |
| 19 | + # grpc: |
| 20 | + # include_metadata: true |
| 21 | + # endpoint: '0.0.0.0:4317' |
| 22 | + # http: |
| 23 | + # cors: |
| 24 | + # allowed_origins: ['*'] |
| 25 | + # allowed_headers: ['*'] |
| 26 | + # include_metadata: true |
| 27 | + # endpoint: '0.0.0.0:4318' |
| 28 | +processors: |
| 29 | + transform: |
| 30 | + log_statements: |
| 31 | + - context: log |
| 32 | + error_mode: ignore |
| 33 | + statements: |
| 34 | + # JSON parsing: Extends log attributes with the fields from structured log body content, either as an OTEL map or |
| 35 | + # as a string containing JSON content. |
| 36 | + - set(log.cache, ExtractPatterns(log.body, "(?P<0>(\\{.*\\}))")) where |
| 37 | + IsString(log.body) |
| 38 | + - merge_maps(log.attributes, ParseJSON(log.cache["0"]), "upsert") |
| 39 | + where IsMap(log.cache) |
| 40 | + - flatten(log.attributes) where IsMap(log.cache) |
| 41 | + - merge_maps(log.attributes, log.body, "upsert") where IsMap(log.body) |
| 42 | + - context: log |
| 43 | + error_mode: ignore |
| 44 | + conditions: |
| 45 | + - severity_number == 0 and severity_text == "" |
| 46 | + statements: |
| 47 | + # Infer: extract the first log level keyword from the first 256 characters of the body |
| 48 | + - set(log.cache["substr"], log.body.string) where Len(log.body.string) |
| 49 | + < 256 |
| 50 | + - set(log.cache["substr"], Substring(log.body.string, 0, 256)) where |
| 51 | + Len(log.body.string) >= 256 |
| 52 | + - set(log.cache, ExtractPatterns(log.cache["substr"], |
| 53 | + "(?i)(?P<0>(alert|crit|emerg|fatal|error|err|warn|notice|debug|dbug|trace))")) |
| 54 | + # Infer: detect FATAL |
| 55 | + - set(log.severity_number, SEVERITY_NUMBER_FATAL) where |
| 56 | + IsMatch(log.cache["0"], "(?i)(alert|crit|emerg|fatal)") |
| 57 | + - set(log.severity_text, "fatal") where log.severity_number == |
| 58 | + SEVERITY_NUMBER_FATAL |
| 59 | + # Infer: detect ERROR |
| 60 | + - set(log.severity_number, SEVERITY_NUMBER_ERROR) where |
| 61 | + IsMatch(log.cache["0"], "(?i)(error|err)") |
| 62 | + - set(log.severity_text, "error") where log.severity_number == |
| 63 | + SEVERITY_NUMBER_ERROR |
| 64 | + # Infer: detect WARN |
| 65 | + - set(log.severity_number, SEVERITY_NUMBER_WARN) where |
| 66 | + IsMatch(log.cache["0"], "(?i)(warn|notice)") |
| 67 | + - set(log.severity_text, "warn") where log.severity_number == |
| 68 | + SEVERITY_NUMBER_WARN |
| 69 | + # Infer: detect DEBUG |
| 70 | + - set(log.severity_number, SEVERITY_NUMBER_DEBUG) where |
| 71 | + IsMatch(log.cache["0"], "(?i)(debug|dbug)") |
| 72 | + - set(log.severity_text, "debug") where log.severity_number == |
| 73 | + SEVERITY_NUMBER_DEBUG |
| 74 | + # Infer: detect TRACE |
| 75 | + - set(log.severity_number, SEVERITY_NUMBER_TRACE) where |
| 76 | + IsMatch(log.cache["0"], "(?i)(trace)") |
| 77 | + - set(log.severity_text, "trace") where log.severity_number == |
| 78 | + SEVERITY_NUMBER_TRACE |
| 79 | + # Infer: else |
| 80 | + - set(log.severity_text, "info") where log.severity_number == 0 |
| 81 | + - set(log.severity_number, SEVERITY_NUMBER_INFO) where |
| 82 | + log.severity_number == 0 |
| 83 | + - context: log |
| 84 | + error_mode: ignore |
| 85 | + statements: |
| 86 | + # Normalize the severity_text case |
| 87 | + - set(log.severity_text, ConvertCase(log.severity_text, "lower")) |
| 88 | + resourcedetection: |
| 89 | + detectors: |
| 90 | + - env |
| 91 | + - system |
| 92 | + - docker |
| 93 | + timeout: 5s |
| 94 | + override: false |
| 95 | + batch: |
| 96 | + memory_limiter: |
| 97 | + # 80% of maximum memory up to 2G |
| 98 | + limit_mib: 1500 |
| 99 | + # 25% of limit up to 2G |
| 100 | + spike_limit_mib: 512 |
| 101 | + check_interval: 5s |
| 102 | +connectors: |
| 103 | + routing/logs: |
| 104 | + default_pipelines: [logs/out-default] |
| 105 | + error_mode: ignore |
| 106 | + table: |
| 107 | + - context: log |
| 108 | + statement: route() where IsMatch(attributes["rr-web.event"], ".*") |
| 109 | + pipelines: [logs/out-rrweb] |
| 110 | +exporters: |
| 111 | + debug: |
| 112 | + verbosity: detailed |
| 113 | + sampling_initial: 5 |
| 114 | + sampling_thereafter: 200 |
| 115 | + clickhouse/rrweb: |
| 116 | + database: ${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE} |
| 117 | + endpoint: ${env:CLICKHOUSE_ENDPOINT} |
| 118 | + password: ${env:CLICKHOUSE_PASSWORD} |
| 119 | + username: ${env:CLICKHOUSE_USER} |
| 120 | + ttl: 720h |
| 121 | + logs_table_name: hyperdx_sessions |
| 122 | + timeout: 5s |
| 123 | + retry_on_failure: |
| 124 | + enabled: true |
| 125 | + initial_interval: 5s |
| 126 | + max_interval: 30s |
| 127 | + max_elapsed_time: 300s |
| 128 | + clickhouse: |
| 129 | + database: ${env:HYPERDX_OTEL_EXPORTER_CLICKHOUSE_DATABASE} |
| 130 | + endpoint: ${env:CLICKHOUSE_ENDPOINT} |
| 131 | + password: ${env:CLICKHOUSE_PASSWORD} |
| 132 | + username: ${env:CLICKHOUSE_USER} |
| 133 | + ttl: 720h |
| 134 | + timeout: 5s |
| 135 | + retry_on_failure: |
| 136 | + enabled: true |
| 137 | + initial_interval: 5s |
| 138 | + max_interval: 30s |
| 139 | + max_elapsed_time: 300s |
| 140 | +extensions: |
| 141 | + health_check: |
| 142 | + endpoint: :13133 |
| 143 | +service: |
| 144 | + telemetry: |
| 145 | + metrics: |
| 146 | + readers: |
| 147 | + - pull: |
| 148 | + exporter: |
| 149 | + prometheus: |
| 150 | + host: '0.0.0.0' |
| 151 | + port: 8888 |
| 152 | + logs: |
| 153 | + level: ${HYPERDX_LOG_LEVEL} |
| 154 | + extensions: [health_check] |
| 155 | + pipelines: |
| 156 | + traces: |
| 157 | + # receivers: [otlp/hyperdx] |
| 158 | + processors: [memory_limiter, batch] |
| 159 | + exporters: [clickhouse] |
| 160 | + metrics: |
| 161 | + # receivers: [otlp/hyperdx, prometheus] |
| 162 | + processors: [memory_limiter, batch] |
| 163 | + exporters: [clickhouse] |
| 164 | + logs/in: |
| 165 | + # receivers: [otlp/hyperdx, fluentforward] |
| 166 | + exporters: [routing/logs] |
| 167 | + logs/out-default: |
| 168 | + receivers: [routing/logs] |
| 169 | + processors: [memory_limiter, transform, batch] |
| 170 | + exporters: [clickhouse] |
| 171 | + logs/out-rrweb: |
| 172 | + receivers: [routing/logs] |
| 173 | + processors: [memory_limiter, batch] |
| 174 | + exporters: [clickhouse/rrweb] |
0 commit comments