Matthias Andreas Benkard | 832a54e | 2019-01-29 09:27:38 +0100 | [diff] [blame] | 1 | /* |
| 2 | Copyright 2017 The Kubernetes Authors. |
| 3 | |
| 4 | Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | you may not use this file except in compliance with the License. |
| 6 | You may obtain a copy of the License at |
| 7 | |
| 8 | http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | |
| 10 | Unless required by applicable law or agreed to in writing, software |
| 11 | distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | See the License for the specific language governing permissions and |
| 14 | limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | package metrics |
| 18 | |
| 19 | import ( |
| 20 | "fmt" |
| 21 | "strconv" |
| 22 | "time" |
| 23 | |
| 24 | "github.com/prometheus/client_golang/prometheus" |
| 25 | |
| 26 | "k8s.io/apiserver/pkg/admission" |
| 27 | ) |
| 28 | |
| 29 | const ( |
| 30 | namespace = "apiserver" |
| 31 | subsystem = "admission" |
| 32 | ) |
| 33 | |
| 34 | var ( |
| 35 | // Use buckets ranging from 25 ms to ~2.5 seconds. |
| 36 | latencyBuckets = prometheus.ExponentialBuckets(25000, 2.5, 5) |
| 37 | latencySummaryMaxAge = 5 * time.Hour |
| 38 | |
| 39 | // Metrics provides access to all admission metrics. |
| 40 | Metrics = newAdmissionMetrics() |
| 41 | ) |
| 42 | |
| 43 | // ObserverFunc is a func that emits metrics. |
| 44 | type ObserverFunc func(elapsed time.Duration, rejected bool, attr admission.Attributes, stepType string, extraLabels ...string) |
| 45 | |
| 46 | const ( |
| 47 | stepValidate = "validate" |
| 48 | stepAdmit = "admit" |
| 49 | ) |
| 50 | |
| 51 | // WithControllerMetrics is a decorator for named admission handlers. |
| 52 | func WithControllerMetrics(i admission.Interface, name string) admission.Interface { |
| 53 | return WithMetrics(i, Metrics.ObserveAdmissionController, name) |
| 54 | } |
| 55 | |
| 56 | // WithStepMetrics is a decorator for a whole admission phase, i.e. admit or validation.admission step. |
| 57 | func WithStepMetrics(i admission.Interface) admission.Interface { |
| 58 | return WithMetrics(i, Metrics.ObserveAdmissionStep) |
| 59 | } |
| 60 | |
| 61 | // WithMetrics is a decorator for admission handlers with a generic observer func. |
| 62 | func WithMetrics(i admission.Interface, observer ObserverFunc, extraLabels ...string) admission.Interface { |
| 63 | return &pluginHandlerWithMetrics{ |
| 64 | Interface: i, |
| 65 | observer: observer, |
| 66 | extraLabels: extraLabels, |
| 67 | } |
| 68 | } |
| 69 | |
| 70 | // pluginHandlerWithMetrics decorates a admission handler with metrics. |
| 71 | type pluginHandlerWithMetrics struct { |
| 72 | admission.Interface |
| 73 | observer ObserverFunc |
| 74 | extraLabels []string |
| 75 | } |
| 76 | |
| 77 | // Admit performs a mutating admission control check and emit metrics. |
| 78 | func (p pluginHandlerWithMetrics) Admit(a admission.Attributes) error { |
| 79 | mutatingHandler, ok := p.Interface.(admission.MutationInterface) |
| 80 | if !ok { |
| 81 | return nil |
| 82 | } |
| 83 | |
| 84 | start := time.Now() |
| 85 | err := mutatingHandler.Admit(a) |
| 86 | p.observer(time.Since(start), err != nil, a, stepAdmit, p.extraLabels...) |
| 87 | return err |
| 88 | } |
| 89 | |
| 90 | // Validate performs a non-mutating admission control check and emits metrics. |
| 91 | func (p pluginHandlerWithMetrics) Validate(a admission.Attributes) error { |
| 92 | validatingHandler, ok := p.Interface.(admission.ValidationInterface) |
| 93 | if !ok { |
| 94 | return nil |
| 95 | } |
| 96 | |
| 97 | start := time.Now() |
| 98 | err := validatingHandler.Validate(a) |
| 99 | p.observer(time.Since(start), err != nil, a, stepValidate, p.extraLabels...) |
| 100 | return err |
| 101 | } |
| 102 | |
| 103 | // AdmissionMetrics instruments admission with prometheus metrics. |
| 104 | type AdmissionMetrics struct { |
| 105 | step *metricSet |
| 106 | controller *metricSet |
| 107 | webhook *metricSet |
| 108 | } |
| 109 | |
| 110 | // newAdmissionMetrics create a new AdmissionMetrics, configured with default metric names. |
| 111 | func newAdmissionMetrics() *AdmissionMetrics { |
| 112 | // Admission metrics for a step of the admission flow. The entire admission flow is broken down into a series of steps |
| 113 | // Each step is identified by a distinct type label value. |
| 114 | step := newMetricSet("step", |
| 115 | []string{"type", "operation", "group", "version", "resource", "subresource", "rejected"}, |
| 116 | "Admission sub-step %s, broken out for each operation and API resource and step type (validate or admit).", true) |
| 117 | |
| 118 | // Built-in admission controller metrics. Each admission controller is identified by name. |
| 119 | controller := newMetricSet("controller", |
| 120 | []string{"name", "type", "operation", "group", "version", "resource", "subresource", "rejected"}, |
| 121 | "Admission controller %s, identified by name and broken out for each operation and API resource and type (validate or admit).", false) |
| 122 | |
| 123 | // Admission webhook metrics. Each webhook is identified by name. |
| 124 | webhook := newMetricSet("webhook", |
| 125 | []string{"name", "type", "operation", "group", "version", "resource", "subresource", "rejected"}, |
| 126 | "Admission webhook %s, identified by name and broken out for each operation and API resource and type (validate or admit).", false) |
| 127 | |
| 128 | step.mustRegister() |
| 129 | controller.mustRegister() |
| 130 | webhook.mustRegister() |
| 131 | return &AdmissionMetrics{step: step, controller: controller, webhook: webhook} |
| 132 | } |
| 133 | |
| 134 | func (m *AdmissionMetrics) reset() { |
| 135 | m.step.reset() |
| 136 | m.controller.reset() |
| 137 | m.webhook.reset() |
| 138 | } |
| 139 | |
| 140 | // ObserveAdmissionStep records admission related metrics for a admission step, identified by step type. |
| 141 | func (m *AdmissionMetrics) ObserveAdmissionStep(elapsed time.Duration, rejected bool, attr admission.Attributes, stepType string, extraLabels ...string) { |
| 142 | gvr := attr.GetResource() |
| 143 | m.step.observe(elapsed, append(extraLabels, stepType, string(attr.GetOperation()), gvr.Group, gvr.Version, gvr.Resource, attr.GetSubresource(), strconv.FormatBool(rejected))...) |
| 144 | } |
| 145 | |
| 146 | // ObserveAdmissionController records admission related metrics for a built-in admission controller, identified by it's plugin handler name. |
| 147 | func (m *AdmissionMetrics) ObserveAdmissionController(elapsed time.Duration, rejected bool, attr admission.Attributes, stepType string, extraLabels ...string) { |
| 148 | gvr := attr.GetResource() |
| 149 | m.controller.observe(elapsed, append(extraLabels, stepType, string(attr.GetOperation()), gvr.Group, gvr.Version, gvr.Resource, attr.GetSubresource(), strconv.FormatBool(rejected))...) |
| 150 | } |
| 151 | |
| 152 | // ObserveWebhook records admission related metrics for a admission webhook. |
| 153 | func (m *AdmissionMetrics) ObserveWebhook(elapsed time.Duration, rejected bool, attr admission.Attributes, stepType string, extraLabels ...string) { |
| 154 | gvr := attr.GetResource() |
| 155 | m.webhook.observe(elapsed, append(extraLabels, stepType, string(attr.GetOperation()), gvr.Group, gvr.Version, gvr.Resource, attr.GetSubresource(), strconv.FormatBool(rejected))...) |
| 156 | } |
| 157 | |
| 158 | type metricSet struct { |
| 159 | latencies *prometheus.HistogramVec |
| 160 | latenciesSummary *prometheus.SummaryVec |
| 161 | } |
| 162 | |
| 163 | func newMetricSet(name string, labels []string, helpTemplate string, hasSummary bool) *metricSet { |
| 164 | var summary *prometheus.SummaryVec |
| 165 | if hasSummary { |
| 166 | summary = prometheus.NewSummaryVec( |
| 167 | prometheus.SummaryOpts{ |
| 168 | Namespace: namespace, |
| 169 | Subsystem: subsystem, |
| 170 | Name: fmt.Sprintf("%s_admission_latencies_seconds_summary", name), |
| 171 | Help: fmt.Sprintf(helpTemplate, "latency summary"), |
| 172 | MaxAge: latencySummaryMaxAge, |
| 173 | }, |
| 174 | labels, |
| 175 | ) |
| 176 | } |
| 177 | |
| 178 | return &metricSet{ |
| 179 | latencies: prometheus.NewHistogramVec( |
| 180 | prometheus.HistogramOpts{ |
| 181 | Namespace: namespace, |
| 182 | Subsystem: subsystem, |
| 183 | Name: fmt.Sprintf("%s_admission_latencies_seconds", name), |
| 184 | Help: fmt.Sprintf(helpTemplate, "latency histogram"), |
| 185 | Buckets: latencyBuckets, |
| 186 | }, |
| 187 | labels, |
| 188 | ), |
| 189 | |
| 190 | latenciesSummary: summary, |
| 191 | } |
| 192 | } |
| 193 | |
| 194 | // MustRegister registers all the prometheus metrics in the metricSet. |
| 195 | func (m *metricSet) mustRegister() { |
| 196 | prometheus.MustRegister(m.latencies) |
| 197 | if m.latenciesSummary != nil { |
| 198 | prometheus.MustRegister(m.latenciesSummary) |
| 199 | } |
| 200 | } |
| 201 | |
| 202 | // Reset resets all the prometheus metrics in the metricSet. |
| 203 | func (m *metricSet) reset() { |
| 204 | m.latencies.Reset() |
| 205 | if m.latenciesSummary != nil { |
| 206 | m.latenciesSummary.Reset() |
| 207 | } |
| 208 | } |
| 209 | |
| 210 | // Observe records an observed admission event to all metrics in the metricSet. |
| 211 | func (m *metricSet) observe(elapsed time.Duration, labels ...string) { |
| 212 | elapsedMicroseconds := float64(elapsed / time.Microsecond) |
| 213 | m.latencies.WithLabelValues(labels...).Observe(elapsedMicroseconds) |
| 214 | if m.latenciesSummary != nil { |
| 215 | m.latenciesSummary.WithLabelValues(labels...).Observe(elapsedMicroseconds) |
| 216 | } |
| 217 | } |