[examples] Add example config for Couchbase (open-telemetry#10894)

* Add example config for couchbase

* changelog

CHANGELOG.md

@@ -17,7 +17,7 @@
 - `transformprocessor`: Add byte slice literal to the grammar. Add new SpanID and TraceID functions that take a byte slice and return a Span/Trace ID. (#10487)
 - `elasticsearchreceiver`: Add integration test for elasticsearch receiver (#10165)
 - `datadogexporter`: Some config validation and unmarshaling steps are now done on `Validate` and `Unmarshal` instead of `Sanitize` (#8829)
-
+- `examples`: Add an example for scraping Couchbase metrics (#10894)
 ### 🧰 Bug fixes 🧰
 
 - `kubletetstatsreceiver`: Bring back `k8s.container.name` attribute (#10848)

examples/couchbase/README.md

@@ -0,0 +1,29 @@
+# Couchbase Example
+
+This example shows how you can configure the collector to scrape a [Couchbase server](https://www.couchbase.com/products/server) using its built-in Prometheus metrics exporter.
+
+This [example config](./otel-collector-config.yaml) retrieves a subset of metrics important for monitoring a Couchbase server, transforms them to follow OpenTelemetry conventions, then exports again to prometheus so that they may be easily viewed.
+
+## Running
+
+First build the collector docker image by running the following command in the root of this repository:
+```sh
+make docker-otelcontribcol
+```
+
+Then, start Couchbase, the collector, and a Prometheus server by running the following command in this directory:
+```sh
+docker-compose up -d --remove-orphans --build --force-recreate
+```
+
+After Couchbase is running, you'll need to initialize Couchbase and create some buckets:
+```sh
+./scripts/setup.sh
+```
+
+You should now be able to view metrics being scraped from the Couchbase instance every 5 seconds in Prometheus at http:https://localhost:9090 (all metrics are prefixed with `couchbase_`)
+
+Once you are done with the example, you may shut it down:
+```sh
+docker-compose down
+```

examples/couchbase/docker-compose.yaml

@@ -0,0 +1,24 @@
+version: "3"
+services:
+ couchbase:
+ image: couchbase:enterprise-7.1.0
+ ports:
+ - 8091:8091
+ deploy:
+ resources:
+ limits:
+ cpus: 0.50
+ memory: 1512M
+ opentelemetry-collector-contrib:
+ image: otel/opentelemetry-collector-contrib:0.53.0
+ command: ["--config=/etc/otel-collector-config.yml"]
+ volumes:
+ - ./otel-collector-config.yaml:/etc/otel-collector-config.yml
+ depends_on:
+ - couchbase
+ prometheus:
+ image: prom/prometheus:v2.36.1
+ volumes:
+ - ./prometheus-config.yaml:/etc/prometheus/prometheus.yml
+ ports:
+ - 9090:9090

examples/couchbase/otel-collector-config.yaml

@@ -0,0 +1,146 @@
+receivers:
+ prometheus/couchbase:
+ config:
+ scrape_configs:
+ - job_name: 'couchbase'
+ scrape_interval: 5s
+ static_configs:
+ - targets: ['couchbase:8091']
+ basic_auth:
+ username: 'otelu'
+ password: 'otelpassword'
+ metric_relabel_configs:
+ # Include only a few key metrics
+ - source_labels: [ __name__ ]
+ regex: "(kv_ops)|\
+ (kv_vb_curr_items)|\
+ (kv_num_vbuckets)|\
+ (kv_ep_cursor_memory_freed_bytes)|\
+ (kv_total_memory_used_bytes)|\
+ (kv_ep_num_value_ejects)|\
+ (kv_ep_mem_high_wat)|\
+ (kv_ep_mem_low_wat)|\
+ (kv_ep_tmp_oom_errors)|\
+ (kv_ep_oom_errors)"
+ action: keep
+
+processors:
+ filter/couchbase:
+ # Filter out prometheus scraping meta-metrics.
+ metrics:
+ exclude:
+ match_type: strict
+ metric_names:
+ - scrape_samples_post_metric_relabeling
+ - scrape_series_added
+ - scrape_duration_seconds
+ - scrape_samples_scraped
+ - up
+
+ metricstransform/couchbase:
+ transforms:
+ # Rename from prometheus metric name to OTel metric name.
+ # We cannot do this with metric_relabel_configs, as the prometheus receiver does not
+ # allow metric renames at this time.
+ - include: kv_ops
+ match_type: strict
+ action: update
+ new_name: "couchbase.bucket.operation.count"
+ - include: kv_vb_curr_items
+ match_type: strict
+ action: update
+ new_name: "couchbase.bucket.item.count"
+ - include: kv_num_vbuckets
+ match_type: strict
+ action: update
+ new_name: "couchbase.bucket.vbucket.count"
+ - include: kv_ep_cursor_memory_freed_bytes
+ match_type: strict
+ action: update
+ new_name: "couchbase.bucket.memory.usage.free"
+ - include: kv_total_memory_used_bytes
+ match_type: strict
+ action: update
+ new_name: "couchbase.bucket.memory.usage.used"
+ - include: kv_ep_num_value_ejects
+ match_type: strict
+ action: update
+ new_name: "couchbase.bucket.item.ejection.count"
+ - include: kv_ep_mem_high_wat
+ match_type: strict
+ action: update
+ new_name: "couchbase.bucket.memory.high_water_mark.limit"
+ - include: kv_ep_mem_low_wat
+ match_type: strict
+ action: update
+ new_name: "couchbase.bucket.memory.low_water_mark.limit"
+ - include: kv_ep_tmp_oom_errors
+ match_type: strict
+ action: update
+ new_name: "couchbase.bucket.error.oom.count.recoverable"
+ - include: kv_ep_oom_errors
+ match_type: strict
+ action: update
+ new_name: "couchbase.bucket.error.oom.count.unrecoverable"
+ # Combine couchbase.bucket.error.oom.count.x and couchbase.bucket.memory.usage.x
+ # metrics.
+ - include: '^couchbase\.bucket\.error\.oom\.count\.(?P<error_type>unrecoverable|recoverable)$$'
+ match_type: regexp
+ action: combine
+ new_name: "couchbase.bucket.error.oom.count"
+ - include: '^couchbase\.bucket\.memory\.usage\.(?P<state>free|used)$$'
+ match_type: regexp
+ action: combine
+ new_name: "couchbase.bucket.memory.usage"
+ # Aggregate "result" label on operation count to keep label sets consistent across the metric datapoints
+ - include: 'couchbase.bucket.operation.count'
+ match_type: strict
+ action: update
+ operations:
+ - action: aggregate_labels
+ label_set: ["bucket", "op"]
+ aggregation_type: sum
+
+ transform/couchbase:
+ metrics:
+ queries:
+ - convert_gauge_to_sum("cumulative", true) where metric.name == "couchbase.bucket.operation.count"
+ - set(metric.description, "Number of operations on the bucket.") where metric.name == "couchbase.bucket.operation.count"
+ - set(metric.unit, "{operations}") where metric.name == "couchbase.bucket.operation.count"
+
+ - convert_gauge_to_sum("cumulative", false) where metric.name == "couchbase.bucket.item.count"
+ - set(metric.description, "Number of items that belong to the bucket.") where metric.name == "couchbase.bucket.item.count"
+ - set(metric.unit, "{items}") where metric.name == "couchbase.bucket.item.count"
+
+ - convert_gauge_to_sum("cumulative", false) where metric.name == "couchbase.bucket.vbucket.count"
+ - set(metric.description, "Number of non-resident vBuckets.") where metric.name == "couchbase.bucket.vbucket.count"
+ - set(metric.unit, "{vbuckets}") where metric.name == "couchbase.bucket.vbucket.count"
+
+ - convert_gauge_to_sum("cumulative", false) where metric.name == "couchbase.bucket.memory.usage"
+ - set(metric.description, "Usage of total memory available to the bucket.") where metric.name == "couchbase.bucket.memory.usage"
+ - set(metric.unit, "By") where metric.name == "couchbase.bucket.memory.usage"
+
+ - convert_gauge_to_sum("cumulative", true) where metric.name == "couchbase.bucket.item.ejection.count"
+ - set(metric.description, "Number of item value ejections from memory to disk.") where metric.name == "couchbase.bucket.item.ejection.count"
+ - set(metric.unit, "{ejections}") where metric.name == "couchbase.bucket.item.ejection.count"
+
+ - convert_gauge_to_sum("cumulative", true) where metric.name == "couchbase.bucket.error.oom.count"
+ - set(metric.description, "Number of out of memory errors.") where metric.name == "couchbase.bucket.error.oom.count"
+ - set(metric.unit, "{errors}") where metric.name == "couchbase.bucket.error.oom.count"
+
+ - set(metric.description, "The memory usage at which items will be ejected.") where metric.name == "couchbase.bucket.memory.high_water_mark.limit"
+ - set(metric.unit, "By") where metric.name == "couchbase.bucket.memory.high_water_mark.limit"
+
+ - set(metric.description, "The memory usage at which ejections will stop that were previously triggered by a high water mark breach.") where metric.name == "couchbase.bucket.memory.low_water_mark.limit"
+ - set(metric.unit, "By") where metric.name == "couchbase.bucket.memory.low_water_mark.limit"
+
+exporters:
+ prometheus:
+ endpoint: "0.0.0.0:9123"
+
+service:
+ pipelines:
+ metrics/couchbase:
+ receivers: [prometheus/couchbase]
+ processors: [filter/couchbase, metricstransform/couchbase, transform/couchbase]
+ exporters: [prometheus]

examples/couchbase/prometheus-config.yaml

@@ -0,0 +1,10 @@
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+ # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+ - job_name: "otel-collector"
+ # metrics_path defaults to '/metrics'
+ # scheme defaults to 'http'.
+ scrape_interval: 5s
+ static_configs:
+ - targets: ["opentelemetry-collector-contrib:9123"]

examples/couchbase/scripts/setup.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+set -e
+
+CONTAINER_NAME=${CONTAINER_NAME:-"couchbase_couchbase_1"}
+
+# Wait until couchbase is responsive
+until curl --silent http:https://127.0.0.1:8091/pools > /dev/null; do
+ >&2 echo "Waiting for Couchbase Server to be available ..."
+ sleep 1
+done
+
+
+echo "Initializing cluster..."
+docker exec "$CONTAINER_NAME" couchbase-cli cluster-init -c 127.0.0.1 --cluster-username otelu --cluster-password otelpassword \
+ --cluster-name otelc --cluster-ramsize 512 --cluster-index-ramsize 256 --services data,index,query,fts \
+ --index-storage-setting default
+
+echo "Creating buckets..."
+docker exec "$CONTAINER_NAME" couchbase-cli bucket-create -c 127.0.0.1 --username otelu --password otelpassword --bucket-type couchbase --bucket-ramsize 256 --bucket test_bucket_1
+
+docker exec "$CONTAINER_NAME" couchbase-cli bucket-create -c 127.0.0.1 --username otelu --password otelpassword --bucket-type couchbase --bucket-ramsize 256 --bucket test_bucket_2
+
+echo "Creating users..."
+docker exec "$CONTAINER_NAME" couchbase-cli user-manage -c 127.0.0.1:8091 -u otelu -p otelpassword --set --rbac-username sysadmin --rbac-password otelpassword \
+ --rbac-name "sysadmin" --roles admin --auth-domain local
+
+docker exec "$CONTAINER_NAME" couchbase-cli user-manage -c 127.0.0.1:8091 -u otelu -p otelpassword --set --rbac-username admin --rbac-password otelpassword \
+ --rbac-name "admin" --roles bucket_full_access[*] --auth-domain local
+
+# Need to wait until query service is ready to process N1QL queries
+echo "Waiting for query service to be ready..."
+sleep 20
+
+# Create otelb indexes
+echo "Create test_bucket_1 index"
+docker exec "$CONTAINER_NAME" cbq -u otelu -p otelpassword -s "CREATE PRIMARY INDEX idx_primary ON \`test_bucket_1\`;"
+
+# Create test_bucket indexes
+echo "Create test_bucket_2 index"
+docker exec "$CONTAINER_NAME" cbq -u otelu -p otelpassword -s "CREATE PRIMARY INDEX idx_primary ON \`test_bucket_2\`;"