diff --git a/docs/guides/proxysql/monitoring/_index.md b/docs/guides/proxysql/monitoring/_index.md new file mode 100644 index 0000000000..759e5afbff --- /dev/null +++ b/docs/guides/proxysql/monitoring/_index.md @@ -0,0 +1,10 @@ +--- +title: Monitoring ProxySQL +menu: + docs_{{ .version }}: + identifier: guides-proxysql-monitoring + name: Monitoring + parent: guides-proxysql +weight: 120 +menu_name: docs_{{ .version }} +--- diff --git a/docs/guides/proxysql/monitoring/builtin-prometheus/examples/mysql.yaml b/docs/guides/proxysql/monitoring/builtin-prometheus/examples/mysql.yaml new file mode 100644 index 0000000000..6a7aaf96d1 --- /dev/null +++ b/docs/guides/proxysql/monitoring/builtin-prometheus/examples/mysql.yaml @@ -0,0 +1,19 @@ +apiVersion: kubedb.com/v1alpha2 +kind: MySQL +metadata: + name: mysql-grp + namespace: demo +spec: + version: "5.7.36" + replicas: 3 + topology: + mode: GroupReplication + storageType: Durable + storage: + storageClassName: "standard" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + terminationPolicy: WipeOut diff --git a/docs/guides/proxysql/monitoring/builtin-prometheus/examples/prom-config.yaml b/docs/guides/proxysql/monitoring/builtin-prometheus/examples/prom-config.yaml new file mode 100644 index 0000000000..fc3c0399bc --- /dev/null +++ b/docs/guides/proxysql/monitoring/builtin-prometheus/examples/prom-config.yaml @@ -0,0 +1,68 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-config + labels: + app: prometheus-demo + namespace: monitoring +data: + prometheus.yml: |- + global: + scrape_interval: 5s + evaluation_interval: 5s + scrape_configs: + - job_name: 'kubedb-databases' + honor_labels: true + scheme: http + kubernetes_sd_configs: + - role: endpoints + # by default Prometheus server select all Kubernetes services as possible target. + # relabel_config is used to filter only desired endpoints + relabel_configs: + # keep only those services that has "prometheus.io/scrape","prometheus.io/path" and "prometheus.io/port" anootations + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape, __meta_kubernetes_service_annotation_prometheus_io_port] + separator: ; + regex: true;(.*) + action: keep + # currently KubeDB supported databases uses only "http" scheme to export metrics. so, drop any service that uses "https" scheme. + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] + action: drop + regex: https + # only keep the stats services created by KubeDB for monitoring purpose which has "-stats" suffix + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*-stats) + action: keep + # service created by KubeDB will have "app.kubernetes.io/name" and "app.kubernetes.io/instance" annotations. keep only those services that have these annotations. + - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name] + separator: ; + regex: (.*) + action: keep + # read the metric path from "prometheus.io/path: " annotation + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + # read the port from "prometheus.io/port: " annotation and update scraping address accordingly + - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] + action: replace + target_label: __address__ + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + # add service namespace as label to the scraped metrics + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $1 + action: replace + # add service name as a label to the scraped metrics + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $1 + action: replace + # add stats service's labels to the scraped metrics + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) \ No newline at end of file diff --git a/docs/guides/proxysql/monitoring/builtin-prometheus/examples/proxysql.yaml b/docs/guides/proxysql/monitoring/builtin-prometheus/examples/proxysql.yaml new file mode 100644 index 0000000000..71650953cc --- /dev/null +++ b/docs/guides/proxysql/monitoring/builtin-prometheus/examples/proxysql.yaml @@ -0,0 +1,17 @@ +apiVersion: kubedb.com/v1alpha2 +kind: ProxySQL +metadata: + name: proxy-server + namespace: demo +spec: + version: "2.4.4-debian" + replicas: 3 + mode: GroupReplication + backend: + name: mysql-grp + syncUsers: true + monitor: + agent: prometheus.io/operator + terminationPolicy: WipeOut + healthChecker: + failureThreshold: 3 diff --git a/docs/guides/proxysql/monitoring/builtin-prometheus/images/built-prom.png b/docs/guides/proxysql/monitoring/builtin-prometheus/images/built-prom.png new file mode 100644 index 0000000000..5c583b861b Binary files /dev/null and b/docs/guides/proxysql/monitoring/builtin-prometheus/images/built-prom.png differ diff --git a/docs/guides/proxysql/monitoring/builtin-prometheus/index.md b/docs/guides/proxysql/monitoring/builtin-prometheus/index.md new file mode 100644 index 0000000000..dfd4aaa682 --- /dev/null +++ b/docs/guides/proxysql/monitoring/builtin-prometheus/index.md @@ -0,0 +1,387 @@ +--- +title: Monitor ProxySQL using Builtin Prometheus Discovery +menu: + docs_{{ .version }}: + identifier: guides-proxysql-monitoring-builtinprometheus + name: Builtin Prometheus + parent: guides-proxysql-monitoring + weight: 30 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Monitoring ProxySQL with builtin Prometheus + +This tutorial will show you how to monitor ProxySQL database using builtin [Prometheus](https://github.com/prometheus/prometheus) scraper. + +## Before You Begin + +- At first, you need to have a Kubernetes cluster, and the kubectl command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +- Install KubeDB operator in your cluster following the steps [here](/docs/setup/README.md). + +- If you are not familiar with how to configure Prometheus to scrape metrics from various Kubernetes resources, please read the tutorial from [here](https://github.com/appscode/third-party-tools/tree/master/monitoring/prometheus/builtin). + +- To learn how Prometheus monitoring works with KubeDB in general, please visit [here](/docs/guides/proxysql/monitoring/overview). + +- To keep Prometheus resources isolated, we are going to use a separate namespace called `monitoring` to deploy respective monitoring resources. We are going to deploy database in `demo` namespace. + + ```bash + $ kubectl create ns monitoring + namespace/monitoring created + + $ kubectl create ns demo + namespace/demo created + ``` + +> Note: YAML files used in this tutorial are stored in [docs/guides/proxysql/monitoring/builtin-prometheus/examples](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/guides/proxysql/monitoring/builtin-prometheus/examples) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Deploy MySQL as ProxySQL Backend + +We need a mysql backend for the proxysql server. So we are creating one with the below yaml. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: MySQL +metadata: + name: mysql-grp + namespace: demo +spec: + version: "5.7.36" + replicas: 3 + topology: + mode: GroupReplication + storageType: Durable + storage: + storageClassName: "standard" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + terminationPolicy: WipeOut +``` + +```bash +$ kubectl create -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/proxysql/monitoring/builtin-prometheus/example/mysql.yaml +mysql.kubedb.com/mysql-grp created +``` + +After applying the above yaml wait for the MySQL to be Ready. + +## Deploy ProxySQL with Monitoring Enabled + +At first, let's deploy an ProxySQL server with monitoring enabled. Below is the ProxySQL object that we are going to create. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: ProxySQL +metadata: + name: proxy-server + namespace: demo +spec: + version: "2.4.4-debian" + replicas: 3 + mode: GroupReplication + backend: + name: mysql-grp + syncUsers: true + monitor: + agent: prometheus.io/operator + terminationPolicy: WipeOut + healthChecker: + failureThreshold: 3 + +``` + +Here, + +- `spec.monitor.agent: prometheus.io/builtin` specifies that we are going to monitor this server using builtin Prometheus scraper. + +Let's create the ProxySQL crd we have shown above. + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/proxysql/monitoring/builtin-prometheus/examples/proxysql.yaml +proxysql.kubedb.com/proxysql-server created +``` + +Now, wait for the server to go into `Running` state. + +```bash +$ kubectl get proxysql -n demo proxy-server +NAME VERSION STATUS AGE +proxy-server 2.4.4-debian Ready 76s +``` + +KubeDB will create a separate stats service with name `{ProxySQL crd name}-stats` for monitoring purpose. + +```bash +$ kubectl get svc -n demo --selector="app.kubernetes.io/instance=proxy-server" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +proxy-server ClusterIP 10.106.32.194 6033/TCP 2m3s +proxy-server-pods ClusterIP None 6032/TCP,6033/TCP 2m3s +proxy-server-stats ClusterIP 10.109.106.92 6070/TCP 2m2s +``` + +Here, `proxy-server-stats ` service has been created for monitoring purpose. Let's describe the service. + +```bash +$ kubectl describe svc -n demo proxy-server-stats +Name: proxy-server-stats +Namespace: demo +Labels: app.kubernetes.io/instance=proxy-server + app.kubernetes.io/managed-by=kubedb.com + app.kubernetes.io/name=proxysqls.kubedb.com + kubedb.com/role=stats +Annotations: monitoring.appscode.com/agent: prometheus.io/builtin + prometheus.io/path: /metrics + prometheus.io/port: 6070 + prometheus.io/scrape: true +Selector: app.kubernetes.io/instance=proxy-server,app.kubernetes.io/managed-by=kubedb.com,app.kubernetes.io/name=proxysqls.kubedb.com +Type: ClusterIP +IP: 10.109.106.92 +Port: metrics 6070/TCP +TargetPort: metrics/TCP +Endpoints: 10.244.0.34:6070 +Session Affinity: None +Events: +``` + +You can see that the service contains following annotations. + +```bash +prometheus.io/path: /metrics +prometheus.io/port: 6070 +prometheus.io/scrape: true +``` + +The Prometheus server will discover the service endpoint using these specifications and will scrape metrics from the exporter. + +## Configure Prometheus Server + +Now, we have to configure a Prometheus scraping job to scrape the metrics using this service. We are going to configure scraping job similar to this [kubernetes-service-endpoints](https://github.com/appscode/third-party-tools/tree/master/monitoring/prometheus/builtin#kubernetes-service-endpoints) job that scrapes metrics from endpoints of a service. + +Let's configure a Prometheus scraping job to collect metrics from this service. + +```yaml +- job_name: 'kubedb-databases' + honor_labels: true + scheme: http + kubernetes_sd_configs: + - role: endpoints + # by default Prometheus server select all Kubernetes services as possible target. + # relabel_config is used to filter only desired endpoints + relabel_configs: + # keep only those services that has "prometheus.io/scrape","prometheus.io/path" and "prometheus.io/port" annotations + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape, __meta_kubernetes_service_annotation_prometheus_io_port] + separator: ; + regex: true;(.*) + action: keep + # currently KubeDB supported databases uses only "http" scheme to export metrics. so, drop any service that uses "https" scheme. + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] + action: drop + regex: https + # only keep the stats services created by KubeDB for monitoring purpose which has "-stats" suffix + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*-stats) + action: keep + # service created by KubeDB will have "app.kubernetes.io/name" and "app.kubernetes.io/instance" annotations. keep only those services that have these annotations. + - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name] + separator: ; + regex: (.*) + action: keep + # read the metric path from "prometheus.io/path: " annotation + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + # read the port from "prometheus.io/port: " annotation and update scraping address accordingly + - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] + action: replace + target_label: __address__ + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + # add service namespace as label to the scraped metrics + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $1 + action: replace + # add service name as a label to the scraped metrics + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $1 + action: replace + # add stats service's labels to the scraped metrics + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) +``` + +### Configure Existing Prometheus Server + +If you already have a Prometheus server running, you have to add above scraping job in the `ConfigMap` used to configure the Prometheus server. Then, you have to restart it for the updated configuration to take effect. + +>If you don't use a persistent volume for Prometheus storage, you will lose your previously scraped data on restart. + +### Deploy New Prometheus Server + +If you don't have any existing Prometheus server running, you have to deploy one. In this section, we are going to deploy a Prometheus server in `monitoring` namespace to collect metrics using this stats service. + +**Create ConfigMap:** + +At first, create a ConfigMap with the scraping configuration. Bellow, the YAML of ConfigMap that we are going to create in this tutorial. + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-config + labels: + app: prometheus-demo + namespace: monitoring +data: + prometheus.yml: |- + global: + scrape_interval: 5s + evaluation_interval: 5s + scrape_configs: + - job_name: 'kubedb-databases' + honor_labels: true + scheme: http + kubernetes_sd_configs: + - role: endpoints + # by default Prometheus server select all Kubernetes services as possible target. + # relabel_config is used to filter only desired endpoints + relabel_configs: + # keep only those services that has "prometheus.io/scrape","prometheus.io/path" and "prometheus.io/port" anootations + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape, __meta_kubernetes_service_annotation_prometheus_io_port] + separator: ; + regex: true;(.*) + action: keep + # currently KubeDB supported databases uses only "http" scheme to export metrics. so, drop any service that uses "https" scheme. + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme] + action: drop + regex: https + # only keep the stats services created by KubeDB for monitoring purpose which has "-stats" suffix + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*-stats) + action: keep + # service created by KubeDB will have "app.kubernetes.io/name" and "app.kubernetes.io/instance" annotations. keep only those services that have these annotations. + - source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name] + separator: ; + regex: (.*) + action: keep + # read the metric path from "prometheus.io/path: " annotation + - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + # read the port from "prometheus.io/port: " annotation and update scraping address accordingly + - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port] + action: replace + target_label: __address__ + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + # add service namespace as label to the scraped metrics + - source_labels: [__meta_kubernetes_namespace] + separator: ; + regex: (.*) + target_label: namespace + replacement: $1 + action: replace + # add service name as a label to the scraped metrics + - source_labels: [__meta_kubernetes_service_name] + separator: ; + regex: (.*) + target_label: service + replacement: $1 + action: replace + # add stats service's labels to the scraped metrics + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) +``` + +Let's create above `ConfigMap`, + +```bash +$ kubectl apply -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/proxysql/monitoring/builtin-prometheus/examples/prom-config.yaml +configmap/prometheus-config created +``` + +**Create RBAC:** + +If you are using an RBAC enabled cluster, you have to give necessary RBAC permissions for Prometheus. Let's create necessary RBAC stuffs for Prometheus, + +```bash +$ kubectl apply -f https://github.com/appscode/third-party-tools/raw/master/monitoring/prometheus/builtin/artifacts/rbac.yaml +clusterrole.rbac.authorization.k8s.io/prometheus created +serviceaccount/prometheus created +clusterrolebinding.rbac.authorization.k8s.io/prometheus created +``` + +>YAML for the RBAC resources created above can be found [here](https://github.com/appscode/third-party-tools/blob/master/monitoring/prometheus/builtin/artifacts/rbac.yaml). + +**Deploy Prometheus:** + +Now, we are ready to deploy Prometheus server. We are going to use following [deployment](https://github.com/appscode/third-party-tools/blob/master/monitoring/prometheus/builtin/artifacts/deployment.yaml) to deploy Prometheus server. + +Let's deploy the Prometheus server. + +```bash +$ kubectl apply -f https://github.com/appscode/third-party-tools/raw/master/monitoring/prometheus/builtin/artifacts/deployment.yaml +deployment.apps/prometheus created +``` + +### Verify Monitoring Metrics + +Prometheus server is listening to port `9090`. We are going to use [port forwarding](https://kubernetes.io/docs/tasks/access-application-cluster/port-forward-access-application-cluster/) to access Prometheus dashboard. + +At first, let's check if the Prometheus pod is in `Running` state. + +```bash +$ kubectl get pod -n monitoring -l=app=prometheus +NAME READY STATUS RESTARTS AGE +prometheus-5dff66b455-cz9td 1/1 Running 0 42s +``` + +Now, run following command on a separate terminal to forward 9090 port of `prometheus-8568c86d86-95zhn` pod, + +```bash +$ kubectl port-forward -n monitoring prometheus-8568c86d86-95zhn 9090 +Forwarding from 127.0.0.1:9090 -> 9090 +Forwarding from [::1]:9090 -> 9090 +``` + +Now, we can access the dashboard at `localhost:9090`. Open [http://localhost:9090](http://localhost:9090) in your browser. You should see the endpoint of `proxy-server-stats` service as one of the targets. + +

+  Prometheus Target +

+ +Check the labels marked with red rectangle. These labels confirm that the metrics are coming from `ProxySQL` server `proxy-server` through stats service `proxy-server-stats`. + +Now, you can view the collected metrics and create a graph from homepage of this Prometheus dashboard. You can also use this Prometheus server as data source for [Grafana](https://grafana.com/) and create beautiful dashboard with collected metrics. + +## Cleaning up + +To cleanup the Kubernetes resources created by this tutorial, run following commands + +```bash +kubectl delete proxysql -n demo proxy-server + +kubectl delete -n monitoring deployment.apps/prometheus + +kubectl delete -n monitoring clusterrole.rbac.authorization.k8s.io/prometheus +kubectl delete -n monitoring serviceaccount/prometheus +kubectl delete -n monitoring clusterrolebinding.rbac.authorization.k8s.io/prometheus + +kubectl delete ns demo +kubectl delete ns monitoring +``` \ No newline at end of file diff --git a/docs/guides/proxysql/monitoring/overview/images/database-monitoring-overview.svg b/docs/guides/proxysql/monitoring/overview/images/database-monitoring-overview.svg new file mode 100644 index 0000000000..395eefb334 --- /dev/null +++ b/docs/guides/proxysql/monitoring/overview/images/database-monitoring-overview.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/guides/proxysql/monitoring/overview/index.md b/docs/guides/proxysql/monitoring/overview/index.md new file mode 100644 index 0000000000..4ae93a1723 --- /dev/null +++ b/docs/guides/proxysql/monitoring/overview/index.md @@ -0,0 +1,109 @@ +--- +title: ProxySQL Monitoring Overview +description: ProxySQL Monitoring Overview +menu: + docs_{{ .version }}: + identifier: guides-proxysql-monitoring-overview + name: Overview + parent: guides-proxysql-monitoring + weight: 10 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Monitoring ProxySQL with KubeDB + +KubeDB has native support for monitoring via [Prometheus](https://prometheus.io/). You can use builtin [Prometheus](https://github.com/prometheus/prometheus) scraper or [Prometheus operator](https://github.com/prometheus-operator/prometheus-operator) to monitor KubeDB managed databases. This tutorial will show you how database monitoring works with KubeDB and how to configure Database crd to enable monitoring. + +## Overview + +KubeDB uses Prometheus [exporter](https://prometheus.io/docs/instrumenting/exporters/#databases) images to export Prometheus metrics for respective databases. Following diagram shows the logical flow of database monitoring with KubeDB. + +

+  Database Monitoring Flow +

+ +When a user creates a database crd with `spec.monitor` section configured, KubeDB operator provisions the respective database and injects an exporter image as sidecar to the database pod. It also creates a dedicated stats service with name `{database-crd-name}-stats` for monitoring. Prometheus server can scrape metrics using this stats service. + +## Configure Monitoring + +In order to enable monitoring for a database, you have to configure `spec.monitor` section. KubeDB provides following options to configure `spec.monitor` section: + +| Field | Type | Uses | +| -------------------------------------------------- | ---------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| `spec.monitor.agent` | `Required` | Type of the monitoring agent that will be used to monitor this database. It can be `prometheus.io/builtin` or `prometheus.io/operator`. | +| `spec.monitor.prometheus.exporter.port` | `Optional` | Port number where the exporter side car will serve metrics. | +| `spec.monitor.prometheus.exporter.args` | `Optional` | Arguments to pass to the exporter sidecar. | +| `spec.monitor.prometheus.exporter.env` | `Optional` | List of environment variables to set in the exporter sidecar container. | +| `spec.monitor.prometheus.exporter.resources` | `Optional` | Resources required by exporter sidecar container. | +| `spec.monitor.prometheus.exporter.securityContext` | `Optional` | Security options the exporter should run with. | +| `spec.monitor.prometheus.serviceMonitor.labels` | `Optional` | Labels for `ServiceMonitor` crd. | +| `spec.monitor.prometheus.serviceMonitor.interval` | `Optional` | Interval at which metrics should be scraped. | + +## Sample Configuration + +A sample YAML for Redis crd with `spec.monitor` section configured to enable monitoring with [Prometheus operator](https://github.com/prometheus-operator/prometheus-operator) is shown below. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: Redis +metadata: + name: sample-redis + namespace: databases +spec: + version: 6.0.6 + terminationPolicy: WipeOut + configSecret: # configure Redis to use password for authentication + name: redis-config + storageType: Durable + storage: + storageClassName: default + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + k8s-app: prometheus + exporter: + args: + - --redis.password=$(REDIS_PASSWORD) + env: + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: _name_of_secret_with_redis_password + key: password # key with the password + resources: + requests: + memory: 512Mi + cpu: 200m + limits: + memory: 512Mi + cpu: 250m + securityContext: + runAsUser: 2000 + allowPrivilegeEscalation: false +``` + +Assume that above Redis is configured to use basic authentication. So, exporter image also need to provide password to collect metrics. We have provided it through `spec.monitor.args` field. + +Here, we have specified that we are going to monitor this server using Prometheus operator through `spec.monitor.agent: prometheus.io/operator`. KubeDB will create a `ServiceMonitor` crd in `monitoring` namespace and this `ServiceMonitor` will have `k8s-app: prometheus` label. + +> As native ProxySQL provides builtin monitoring utilities , KubeDB ProxySQL do not depend on external exporter container. Which means one don't need to configure the `spec.monitor.agent.exporter` field. + +## Next Steps + +- Learn how to monitor `ProxySQL` server with KubeDB using [builtin-Prometheus](/docs/guides/proxysql/monitoring/builtin-prometheus) and using [Prometheus operator](/docs/guides/proxysql/monitoring/prometheus-operator). +- Learn how to monitor `ElasticSearch` database with KubeDB using [builtin-Prometheus](/docs/guides/elasticsearch/monitoring/using-builtin-prometheus.md) and using [Prometheus operator](/docs/guides/elasticsearch/monitoring/using-prometheus-operator.md). +- Learn how to monitor `PostgreSQL` database with KubeDB using [builtin-Prometheus](/docs/guides/postgres/monitoring/using-builtin-prometheus.md) and using [Prometheus operator](/docs/guides/postgres/monitoring/using-prometheus-operator.md). +- Learn how to monitor `MySQL` database with KubeDB using [builtin-Prometheus](/docs/guides/mysql/monitoring/builtin-prometheus/index.md) and using [Prometheus operator](/docs/guides/mysql/monitoring/prometheus-operator/index.md). +- Learn how to monitor `MongoDB` database with KubeDB using [builtin-Prometheus](/docs/guides/mongodb/monitoring/using-builtin-prometheus.md) and using [Prometheus operator](/docs/guides/mongodb/monitoring/using-prometheus-operator.md). +- Learn how to monitor `Redis` server with KubeDB using [builtin-Prometheus](/docs/guides/redis/monitoring/using-builtin-prometheus.md) and using [Prometheus operator](/docs/guides/redis/monitoring/using-prometheus-operator.md). +- Learn how to monitor `Memcached` server with KubeDB using [builtin-Prometheus](/docs/guides/memcached/monitoring/using-builtin-prometheus.md) and using [Prometheus operator](/docs/guides/memcached/monitoring/using-prometheus-operator.md). \ No newline at end of file diff --git a/docs/guides/proxysql/monitoring/prometheus-operator/examples/mysql.yaml b/docs/guides/proxysql/monitoring/prometheus-operator/examples/mysql.yaml new file mode 100644 index 0000000000..6a7aaf96d1 --- /dev/null +++ b/docs/guides/proxysql/monitoring/prometheus-operator/examples/mysql.yaml @@ -0,0 +1,19 @@ +apiVersion: kubedb.com/v1alpha2 +kind: MySQL +metadata: + name: mysql-grp + namespace: demo +spec: + version: "5.7.36" + replicas: 3 + topology: + mode: GroupReplication + storageType: Durable + storage: + storageClassName: "standard" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + terminationPolicy: WipeOut diff --git a/docs/guides/proxysql/monitoring/prometheus-operator/examples/proxysql.yaml b/docs/guides/proxysql/monitoring/prometheus-operator/examples/proxysql.yaml new file mode 100644 index 0000000000..f2bf5dfb2c --- /dev/null +++ b/docs/guides/proxysql/monitoring/prometheus-operator/examples/proxysql.yaml @@ -0,0 +1,23 @@ +apiVersion: kubedb.com/v1alpha2 +kind: ProxySQL +metadata: + name: proxy-server + namespace: demo +spec: + version: "2.4.4-debian" + replicas: 3 + mode: GroupReplication + backend: + name: mysql-grp + syncUsers: true + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + k8s-app: prometheus + interval: 10s + terminationPolicy: WipeOut + healthChecker: + failureThreshold: 3 diff --git a/docs/guides/proxysql/monitoring/prometheus-operator/images/prom-end.png b/docs/guides/proxysql/monitoring/prometheus-operator/images/prom-end.png new file mode 100644 index 0000000000..9987dd9b54 Binary files /dev/null and b/docs/guides/proxysql/monitoring/prometheus-operator/images/prom-end.png differ diff --git a/docs/guides/proxysql/monitoring/prometheus-operator/index.md b/docs/guides/proxysql/monitoring/prometheus-operator/index.md new file mode 100644 index 0000000000..5ec1030354 --- /dev/null +++ b/docs/guides/proxysql/monitoring/prometheus-operator/index.md @@ -0,0 +1,358 @@ +--- +title: Monitor ProxySQL using Prometheus Operator +menu: + docs_{{ .version }}: + identifier: guides-proxysql-monitoring-prometheusoperator + name: Prometheus Operator + parent: guides-proxysql-monitoring + weight: 20 +menu_name: docs_{{ .version }} +section_menu_id: guides +--- + +> New to KubeDB? Please start [here](/docs/README.md). + +# Monitoring ProxySQL Using Prometheus operator + +[Prometheus operator](https://github.com/prometheus-operator/prometheus-operator) provides simple and Kubernetes native way to deploy and configure Prometheus server. This tutorial will show you how to use Prometheus operator to monitor ProxySQL server deployed with KubeDB. + +## Before You Begin + +- At first, you need to have a Kubernetes cluster, and the kubectl command-line tool must be configured to communicate with your cluster. If you do not already have a cluster, you can create one by using [kind](https://kind.sigs.k8s.io/docs/user/quick-start/). + +- To learn how Prometheus monitoring works with KubeDB in general, please visit [here](/docs/guides/proxysql/monitoring/overview). + +- To keep database resources isolated, this tutorial uses a separate namespace called `demo` throughout this tutorial. Run the following command to prepare your cluster: + + ```bash + $ kubectl create ns demo + namespace/demo created + ``` + +- We need a [Prometheus operator](https://github.com/prometheus-operator/prometheus-operator) instance running. If you don't already have a running instance, deploy one following the docs from [here](https://github.com/appscode/third-party-tools/blob/master/monitoring/prometheus/operator/README.md). + +- If you already don't have a Prometheus server running, deploy one following tutorial from [here](https://github.com/appscode/third-party-tools/blob/master/monitoring/prometheus/operator/README.md#deploy-prometheus-server). + +> Note: YAML files used in this tutorial are stored in [/docs/guides/proxysql/monitoring/prometheus-operator/examples](https://github.com/kubedb/docs/tree/{{< param "info.version" >}}/docs/guides/proxysql/monitoring/prometheus-operator/examples) folder in GitHub repository [kubedb/docs](https://github.com/kubedb/docs). + +## Find out required labels for ServiceMonitor + +We need to know the labels used to select `ServiceMonitor` by a `Prometheus` crd. We are going to provide these labels in `spec.monitor.prometheus.labels` field of ProxySQL crd so that KubeDB creates `ServiceMonitor` object accordingly. + +At first, let's find out the available Prometheus server in our cluster. + +```bash +$ kubectl get prometheus --all-namespaces +NAMESPACE NAME VERSION REPLICAS AGE +default prometheus 1 2m19s +``` + +> If you don't have any Prometheus server running in your cluster, deploy one following the guide specified in **Before You Begin** section. + +Now, let's view the YAML of the available Prometheus server `prometheus` in `default` namespace. + +```yaml +$ kubectl get prometheus -n default prometheus -o yaml +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"monitoring.coreos.com/v1","kind":"Prometheus","metadata":{"annotations":{},"labels":{"prometheus":"prometheus"},"name":"prometheus","namespace":"default"},"spec":{"replicas":1,"resources":{"requests":{"memory":"400Mi"}},"serviceAccountName":"prometheus","serviceMonitorNamespaceSelector":{"matchLabels":{"prometheus":"prometheus"}},"serviceMonitorSelector":{"matchLabels":{"k8s-app":"prometheus"}}}} + creationTimestamp: "2020-08-25T04:02:07Z" + generation: 1 + labels: + prometheus: prometheus + ... + manager: kubectl + operation: Update + time: "2020-08-25T04:02:07Z" + name: prometheus + namespace: default + resourceVersion: "2087" + selfLink: /apis/monitoring.coreos.com/v1/namespaces/default/prometheuses/prometheus + uid: 972a50cb-b751-418b-b2bc-e0ecc9232730 +spec: + replicas: 1 + resources: + requests: + memory: 400Mi + serviceAccountName: prometheus + serviceMonitorNamespaceSelector: + matchLabels: + prometheus: prometheus + serviceMonitorSelector: + matchLabels: + k8s-app: prometheus +``` + +- `spec.serviceMonitorSelector` field specifies which ServiceMonitors should be included. The Above label `k8s-app: prometheus` is used to select `ServiceMonitors` by its selector. So, we are going to use this label in `spec.monitor.prometheus.labels` field of ProxySQL crd. +- `spec.serviceMonitorNamespaceSelector` field specifies that the `ServiceMonitors` can be selected outside the Prometheus namespace by Prometheus using namespace selector. The Above label `prometheus: prometheus` is used to select the namespace where the `ServiceMonitor` is created. + +### Add Label to database namespace + +KubeDB creates a `ServiceMonitor` in database namespace `demo`. We need to add label to `demo` namespace. Prometheus will select this namespace by using its `spec.serviceMonitorNamespaceSelector` field. + +Let's add label `prometheus: prometheus` to `demo` namespace, + +```bash +$ kubectl patch namespace demo -p '{"metadata":{"labels": {"prometheus":"prometheus"}}}' +namespace/demo patched +``` + +## Deploy MySQL as ProxySQL Backend + +We need a mysql backend for the proxysql server. So we are creating one with the below yaml. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: MySQL +metadata: + name: mysql-grp + namespace: demo +spec: + version: "5.7.36" + replicas: 3 + topology: + mode: GroupReplication + storageType: Durable + storage: + storageClassName: "standard" + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + terminationPolicy: WipeOut +``` + +```bash +$ kubectl create -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/proxysql/monitoring/builtin-prometheus/example/mysql.yaml +mysql.kubedb.com/mysql-grp created +``` + +After applying the above yaml wait for the MySQL to be Ready. + +## Deploy ProxySQL with Monitoring Enabled + +At first, let's deploy an ProxySQL database with monitoring enabled. Below is the ProxySQL object that we are going to create. + +```yaml +apiVersion: kubedb.com/v1alpha2 +kind: ProxySQL +metadata: + name: proxy-server + namespace: demo +spec: + version: "2.4.4-debian" + replicas: 3 + mode: GroupReplication + backend: + name: mysql-grp + syncUsers: true + monitor: + agent: prometheus.io/operator + prometheus: + serviceMonitor: + labels: + release: prometheus + k8s-app: prometheus + interval: 10s + terminationPolicy: WipeOut + healthChecker: + failureThreshold: 3 +``` + +Here, + +- `monitor.agent: prometheus.io/operator` indicates that we are going to monitor this server using Prometheus operator. + +- `monitor.prometheus.labels` specifies that KubeDB should create `ServiceMonitor` with these labels. + +- `monitor.prometheus.interval` indicates that the Prometheus server should scrape metrics from this server with 10 seconds interval. + +Let's create the ProxySQL object that we have shown above, + +```bash +$ kubectl create -f https://github.com/kubedb/docs/raw/{{< param "info.version" >}}/docs/guides/proxysql/monitoring/prometheus-operator/examples/proxysql.yaml +proxysql.kubedb.com/proxy-server created +``` + +Now, wait for the server to go into `Ready` state. + +```bash +$ kubectl get proxysql -n demo proxy-server +NAME VERSION STATUS AGE +proxy-server 2.4.4-debian Ready 59s +``` + +KubeDB will create a separate stats service with name `{ProxySQL crd name}-stats` for monitoring purpose. + +```bash +$ $ kubectl get svc -n demo --selector="app.kubernetes.io/instance=proxy-server" +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +proxy-server ClusterIP 10.99.96.226 6033/TCP 107s +proxy-server-pods ClusterIP None 6032/TCP,6033/TCP 107s +proxy-server-stats ClusterIP 10.101.190.67 6070/TCP 107s +``` + +Here, `proxy-server-stats` service has been created for monitoring purpose. + +Let's describe this stats service. + +```yaml +$ kubectl describe svc -n demo proxy-server-stats +Name: proxy-server-stats +Namespace: demo +Labels: app.kubernetes.io/instance=proxy-server + app.kubernetes.io/managed-by=kubedb.com + app.kubernetes.io/name=proxysqls.kubedb.com + kubedb.com/role=stats +Annotations: monitoring.appscode.com/agent: prometheus.io/operator +Selector: app.kubernetes.io/instance=proxy-server,app.kubernetes.io/managed-by=kubedb.com,app.kubernetes.io/name=proxysqls.kubedb.com +Type: ClusterIP +IP: 10.101.190.67 +Port: metrics 6070/TCP +TargetPort: metrics/TCP +Endpoints: 10.244.0.31:6070 +Session Affinity: None +Events: +``` + +Notice the `Labels` and `Port` fields. `ServiceMonitor` will use these information to target its endpoints. + +KubeDB will also create a `ServiceMonitor` crd in `demo` namespace that select the endpoints of `proxy-server-stats` service. Verify that the `ServiceMonitor` crd has been created. + +```bash +$ kubectl get servicemonitor -n demo +NAME AGE +proxy-server-stats 4m8s +``` + +Let's verify that the `ServiceMonitor` has the label that we had specified in `spec.monitor` section of ProxySQL crd. + +```yaml +$ kubectl get servicemonitor -n demo proxy-server-stats -o yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + creationTimestamp: "2021-03-19T10:09:03Z" + generation: 1 + labels: + app.kubernetes.io/component: database + app.kubernetes.io/instance: proxy-server + app.kubernetes.io/managed-by: kubedb.com + app.kubernetes.io/name: proxysqls.kubedb.com + k8s-app: prometheus + managedFields: + - apiVersion: monitoring.coreos.com/v1 + fieldsType: FieldsV1 + fieldsV1: + f:metadata: + f:labels: + .: {} + f:app.kubernetes.io/component: {} + f:app.kubernetes.io/instance: {} + f:app.kubernetes.io/managed-by: {} + f:app.kubernetes.io/name: {} + f:k8s-app: {} + f:ownerReferences: {} + f:spec: + .: {} + f:endpoints: {} + f:namespaceSelector: + .: {} + f:matchNames: {} + f:selector: + .: {} + f:matchLabels: + .: {} + f:app.kubernetes.io/instance: {} + f:app.kubernetes.io/managed-by: {} + f:app.kubernetes.io/name: {} + f:kubedb.com/role: {} + manager: proxysql-operator + operation: Update + time: "2021-03-19T10:09:03Z" + name: proxy-server-stats + namespace: demo + ownerReferences: + - apiVersion: v1 + blockOwnerDeletion: true + controller: true + kind: Service + name: proxy-server-stats + uid: 08260a99-0984-4d90-bf68-34080ad0ee5b + resourceVersion: "241637" + selfLink: /apis/monitoring.coreos.com/v1/namespaces/demo/servicemonitors/proxy-server-stats + uid: 4f022d98-d2d8-490f-9548-f6367d03ae1f +spec: + endpoints: + - bearerTokenSecret: + key: "" + honorLabels: true + interval: 10s + path: /metrics + port: metrics + namespaceSelector: + matchNames: + - demo + selector: + matchLabels: + app.kubernetes.io/instance: proxy-server + app.kubernetes.io/managed-by: kubedb.com + app.kubernetes.io/name: proxysqls.kubedb.com + kubedb.com/role: stats +``` + +Notice that the `ServiceMonitor` has label `k8s-app: prometheus` that we had specified in ProxySQL crd. + +Also notice that the `ServiceMonitor` has selector which match the labels we have seen in the `proxy-server-stats` service. It also, target the `prom-http` port that we have seen in the stats service. + +## Verify Monitoring Metrics + +At first, let's find out the respective Prometheus pod for `prometheus` Prometheus server. + +```bash +$ kubectl get pod -n default -l=app=prometheus +NAME READY STATUS RESTARTS AGE +prometheus-prometheus-0 3/3 Running 1 16m +``` + +Prometheus server is listening to port `9090` of `prometheus-prometheus-0` pod. We are going to use [port forwarding](https://kubernetes.io/docs/tasks/access-application-cluster/port-forward-access-application-cluster/) to access Prometheus dashboard. + +Run following command on a separate terminal to forward the port 9090 of `prometheus-prometheus-0` pod, + +```bash +$ kubectl port-forward -n default prometheus-prometheus-0 9090 +Forwarding from 127.0.0.1:9090 -> 9090 +Forwarding from [::1]:9090 -> 9090 +``` + +Now, we can access the dashboard at `localhost:9090`. Open [http://localhost:9090](http://localhost:9090) in your browser. You should see `prom-http` endpoint of `proxy-server-stats` service as one of the targets. + +

+  Prometheus Target +

+ +Check the `endpoint` and `service` labels. It verifies that the target is our expected database. Now, you can view the collected metrics and create a graph from homepage of this Prometheus dashboard. You can also use this Prometheus server as data source for [Grafana](https://grafana.com/) and create beautiful dashboard with collected metrics. + +## Cleaning up + +To cleanup the Kubernetes resources created by this tutorial, run following commands + +```bash +# cleanup database +kubectl delete proxysql -n demo proxy-server + +# cleanup Prometheus resources +kubectl delete -f https://raw.githubusercontent.com/appscode/third-party-tools/master/monitoring/prometheus/operator/artifacts/prometheus.yaml + +kubectl delete -f https://raw.githubusercontent.com/appscode/third-party-tools/master/monitoring/prometheus/operator/artifacts/prometheus-rbac.yaml + +# cleanup Prometheus operator resources +kubectl delete -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/release-0.41/bundle.yaml + +# delete namespace +kubectl delete ns demo +``` \ No newline at end of file