[nodeutilization]: prometheus usage client through prometheus metrics

README.md

@@ -126,6 +126,14 @@ These are top level keys in the Descheduler Policy that you can use to configure
 | `maxNoOfPodsToEvictTotal` |`int`| `nil` | maximum number of pods evicted per rescheduling cycle (summed through all strategies) |
 | `metricsCollector` |`object`| `nil` | configures collection of metrics for actual resource utilization |
 | `metricsCollector.enabled` |`bool`| `false` | enables kubernetes [metrics server](https://kubernetes-sigs.github.io/metrics-server/) collection |
+| `prometheus` |`object`| `nil` | configures collection of Prometheus metrics for actual resource utilization |
+| `prometheus.url` |`string`| `nil` | points to a Prometheus server url |
+| `prometheus.insecureSkipVerify` |`bool`| `nil` | disables server certificate chain and host name verification |
+| `prometheus.authToken` |`object`| `nil` | sets Prometheus server authentication token |
+| `prometheus.authToken.raw` |`string`| `nil` | set the authentication token as a raw string (takes precedence over secretReference) |
+| `prometheus.authToken.secretReference` |`object`| `nil` | read the authentication token from a kubernetes secret (the secret is expected to contain the token under `prometheusAuthToken` data key) |
+| `prometheus.authToken.secretReference.namespace` |`string`| `nil` | authentication token kubernetes secret namespace (the curent rbac allows to get secrets from kube-system namespace) |
+| `prometheus.authToken.secretReference.name` |`string`| `nil` | authentication token kubernetes secret name |
 
 ### Evictor Plugin configuration (Default Evictor)
 
@@ -162,6 +170,13 @@ maxNoOfPodsToEvictPerNamespace: 5000 # you don't need to set this, unlimited if
 maxNoOfPodsToEvictTotal: 5000 # you don't need to set this, unlimited if not set
 metricsCollector:
   enabled: true # you don't need to set this, metrics are not collected if not set
+prometheus: # you don't need to set this, prometheus client will not get created if not set
+  url: http://prometheus-kube-prometheus-prometheus.prom.svc.cluster.local
+  insecureSkipVerify: true
+  authToken:
+    secretReference:
+      namespace: "kube-system"
+      name: "authtoken"
 profiles:
   - name: ProfileName
     pluginConfig:
@@ -287,6 +302,11 @@ design for scheduling pods onto nodes. This means that resource usage as reporte
 like `kubectl top`) may differ from the calculated consumption, due to these components reporting
 actual usage metrics. Metrics-based descheduling can be enabled by setting `metricsUtilization.metricsServer` field.
 In order to have the plugin consume the metrics the metric collector needs to be configured as well.
+Alternatively, it is possible to create a prometheus client and configure a prometheus query to consume
+metrics outside of the kubernetes metrics server. The query is expected to return a vector of values for
+each node. The values are expected to be any real number within <0; 1> interval. During eviction only
+a single pod is evicted at most from each overutilized node. There's currently no support for evicting
+more. Kubernetes metric server takes precedence over Prometheus.
 See `metricsCollector` field at [Top Level configuration](#top-level-configuration) for available options.
 
 **Parameters:**
@@ -300,6 +320,7 @@ See `metricsCollector` field at [Top Level configuration](#top-level-configurati
 |`evictableNamespaces`|(see [namespace filtering](#namespace-filtering))|
 |`metricsUtilization`|object|
 |`metricsUtilization.metricsServer`|bool|
+|`metricsUtilization.prometheus.query`|string|
 
 
 **Example:**
@@ -322,6 +343,8 @@ profiles:
           "pods": 50
         metricsUtilization:
           metricsServer: true
+          # prometheus:
+          #   query: instance:node_cpu:rate:sum
     plugins:
       balance:
         enabled:

cmd/descheduler/app/options/options.go

@@ -20,6 +20,7 @@ package options
 import (
 	"time"
 
+	promapi "github.com/prometheus/client_golang/api"
 	"github.com/spf13/pflag"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -49,6 +50,7 @@ type DeschedulerServer struct {
 	Client            clientset.Interface
 	EventClient       clientset.Interface
 	MetricsClient     metricsclient.Interface
+	PrometheusClient  promapi.Client
 	SecureServing     *apiserveroptions.SecureServingOptionsWithLoopback
 	SecureServingInfo *apiserver.SecureServingInfo
 	DisableMetrics    bool

kubernetes/base/rbac.yaml

@@ -36,6 +36,15 @@ rules:
   resources: ["nodes", "pods"]
   verbs: ["get", "list"]
 ---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: descheduler-role
+rules:
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get", "list", "watch"]
+---
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -54,3 +63,16 @@ subjects:
   - name: descheduler-sa
     kind: ServiceAccount
     namespace: kube-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: descheduler-role-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: descheduler-role
+subjects:
+  - name: descheduler-sa
+    kind: ServiceAccount
+    namespace: kube-system

pkg/api/types.go

@@ -44,6 +44,9 @@ type DeschedulerPolicy struct {
 
 	// MetricsCollector configures collection of metrics about actual resource utilization
 	MetricsCollector MetricsCollector
+
+	// Prometheus enables metrics collection through Prometheus
+	Prometheus Prometheus
 }
 
 // Namespaces carries a list of included/excluded namespaces
@@ -94,3 +97,25 @@ type MetricsCollector struct {
 	// Later, the collection can be extended to other providers.
 	Enabled bool
 }
+
+type Prometheus struct {
+	URL                string
+	AuthToken          AuthToken
+	InsecureSkipVerify bool
+}
+
+type AuthToken struct {
+	// raw for a raw authentication token
+	Raw string
+	// secretReference references an authentication token.
+	// secrets are expected to be created under the descheduler's namespace.
+	SecretReference SecretReference
+}
+
+// SecretReference holds a reference to a Secret
+type SecretReference struct {
+	// namespace is the namespace of the secret.
+	Namespace string
+	// name is the name of the secret.
+	Name string
+}

pkg/api/v1alpha2/types.go

@@ -43,6 +43,9 @@ type DeschedulerPolicy struct {
 
 	// MetricsCollector configures collection of metrics for actual resource utilization
 	MetricsCollector MetricsCollector `json:"metricsCollector,omitempty"`
+
+	// Prometheus enables metrics collection through Prometheus
+	Prometheus Prometheus `json:"prometheus,omitempty"`
 }
 
 type DeschedulerProfile struct {
@@ -76,3 +79,25 @@ type MetricsCollector struct {
 	// Later, the collection can be extended to other providers.
 	Enabled bool `json:"enabled"`
 }
+
+type Prometheus struct {
+	URL                string    `json:"url,omitempty"`
+	AuthToken          AuthToken `json:"authToken,omitempty"`
+	InsecureSkipVerify bool      `json:"insecureSkipVerify,omitempty"`
+}
+
+type AuthToken struct {
+	// raw for a raw authentication token
+	Raw string `json:"raw,omitempty"`
+	// secretReference references an authentication token.
+	// secrets are expected to be created under the descheduler's namespace.
+	SecretReference SecretReference `json:"secretReference,omitempty"`
+}
+
+// SecretReference holds a reference to a Secret
+type SecretReference struct {
+	// namespace is the namespace of the secret.
+	Namespace string `json:"namespace,omitempty"`
+	// name is the name of the secret.
+	Name string `json:"name,omitempty"`
+}