Matthias Andreas Benkard | 832a54e | 2019-01-29 09:27:38 +0100 | [diff] [blame] | 1 | /* |
| 2 | Copyright 2016 The Kubernetes Authors. |
| 3 | |
| 4 | Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | you may not use this file except in compliance with the License. |
| 6 | You may obtain a copy of the License at |
| 7 | |
| 8 | http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | |
| 10 | Unless required by applicable law or agreed to in writing, software |
| 11 | distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | See the License for the specific language governing permissions and |
| 14 | limitations under the License. |
| 15 | */ |
| 16 | |
| 17 | package storage |
| 18 | |
| 19 | import ( |
| 20 | "crypto/tls" |
| 21 | "crypto/x509" |
| 22 | "io/ioutil" |
| 23 | "strings" |
| 24 | |
| 25 | "github.com/golang/glog" |
| 26 | |
| 27 | "k8s.io/apimachinery/pkg/runtime" |
| 28 | "k8s.io/apimachinery/pkg/runtime/schema" |
| 29 | "k8s.io/apimachinery/pkg/util/sets" |
| 30 | "k8s.io/apiserver/pkg/features" |
| 31 | "k8s.io/apiserver/pkg/storage/storagebackend" |
| 32 | "k8s.io/apiserver/pkg/storage/value" |
| 33 | utilfeature "k8s.io/apiserver/pkg/util/feature" |
| 34 | ) |
| 35 | |
| 36 | // Backend describes the storage servers, the information here should be enough |
| 37 | // for health validations. |
| 38 | type Backend struct { |
| 39 | // the url of storage backend like: https://etcd.domain:2379 |
| 40 | Server string |
| 41 | // the required tls config |
| 42 | TLSConfig *tls.Config |
| 43 | } |
| 44 | |
| 45 | // StorageFactory is the interface to locate the storage for a given GroupResource |
| 46 | type StorageFactory interface { |
| 47 | // New finds the storage destination for the given group and resource. It will |
| 48 | // return an error if the group has no storage destination configured. |
| 49 | NewConfig(groupResource schema.GroupResource) (*storagebackend.Config, error) |
| 50 | |
| 51 | // ResourcePrefix returns the overridden resource prefix for the GroupResource |
| 52 | // This allows for cohabitation of resources with different native types and provides |
| 53 | // centralized control over the shape of etcd directories |
| 54 | ResourcePrefix(groupResource schema.GroupResource) string |
| 55 | |
| 56 | // Backends gets all backends for all registered storage destinations. |
| 57 | // Used for getting all instances for health validations. |
| 58 | Backends() []Backend |
| 59 | } |
| 60 | |
| 61 | // DefaultStorageFactory takes a GroupResource and returns back its storage interface. This result includes: |
| 62 | // 1. Merged etcd config, including: auth, server locations, prefixes |
| 63 | // 2. Resource encodings for storage: group,version,kind to store as |
| 64 | // 3. Cohabitating default: some resources like hpa are exposed through multiple APIs. They must agree on 1 and 2 |
| 65 | type DefaultStorageFactory struct { |
| 66 | // StorageConfig describes how to create a storage backend in general. |
| 67 | // Its authentication information will be used for every storage.Interface returned. |
| 68 | StorageConfig storagebackend.Config |
| 69 | |
| 70 | Overrides map[schema.GroupResource]groupResourceOverrides |
| 71 | |
| 72 | DefaultResourcePrefixes map[schema.GroupResource]string |
| 73 | |
| 74 | // DefaultMediaType is the media type used to store resources. If it is not set, "application/json" is used. |
| 75 | DefaultMediaType string |
| 76 | |
| 77 | // DefaultSerializer is used to create encoders and decoders for the storage.Interface. |
| 78 | DefaultSerializer runtime.StorageSerializer |
| 79 | |
| 80 | // ResourceEncodingConfig describes how to encode a particular GroupVersionResource |
| 81 | ResourceEncodingConfig ResourceEncodingConfig |
| 82 | |
| 83 | // APIResourceConfigSource indicates whether the *storage* is enabled, NOT the API |
| 84 | // This is discrete from resource enablement because those are separate concerns. How this source is configured |
| 85 | // is left to the caller. |
| 86 | APIResourceConfigSource APIResourceConfigSource |
| 87 | |
| 88 | // newStorageCodecFn exists to be overwritten for unit testing. |
| 89 | newStorageCodecFn func(opts StorageCodecConfig) (codec runtime.Codec, err error) |
| 90 | } |
| 91 | |
| 92 | type groupResourceOverrides struct { |
| 93 | // etcdLocation contains the list of "special" locations that are used for particular GroupResources |
| 94 | // These are merged on top of the StorageConfig when requesting the storage.Interface for a given GroupResource |
| 95 | etcdLocation []string |
| 96 | // etcdPrefix is the base location for a GroupResource. |
| 97 | etcdPrefix string |
| 98 | // etcdResourcePrefix is the location to use to store a particular type under the `etcdPrefix` location |
| 99 | // If empty, the default mapping is used. If the default mapping doesn't contain an entry, it will use |
| 100 | // the ToLowered name of the resource, not including the group. |
| 101 | etcdResourcePrefix string |
| 102 | // mediaType is the desired serializer to choose. If empty, the default is chosen. |
| 103 | mediaType string |
| 104 | // serializer contains the list of "special" serializers for a GroupResource. Resource=* means for the entire group |
| 105 | serializer runtime.StorageSerializer |
| 106 | // cohabitatingResources keeps track of which resources must be stored together. This happens when we have multiple ways |
| 107 | // of exposing one set of concepts. autoscaling.HPA and extensions.HPA as a for instance |
| 108 | // The order of the slice matters! It is the priority order of lookup for finding a storage location |
| 109 | cohabitatingResources []schema.GroupResource |
| 110 | // encoderDecoratorFn is optional and may wrap the provided encoder prior to being serialized. |
| 111 | encoderDecoratorFn func(runtime.Encoder) runtime.Encoder |
| 112 | // decoderDecoratorFn is optional and may wrap the provided decoders (can add new decoders). The order of |
| 113 | // returned decoders will be priority for attempt to decode. |
| 114 | decoderDecoratorFn func([]runtime.Decoder) []runtime.Decoder |
| 115 | // transformer is optional and shall encrypt that resource at rest. |
| 116 | transformer value.Transformer |
| 117 | // disablePaging will prevent paging on the provided resource. |
| 118 | disablePaging bool |
| 119 | } |
| 120 | |
| 121 | // Apply overrides the provided config and options if the override has a value in that position |
| 122 | func (o groupResourceOverrides) Apply(config *storagebackend.Config, options *StorageCodecConfig) { |
| 123 | if len(o.etcdLocation) > 0 { |
| 124 | config.ServerList = o.etcdLocation |
| 125 | } |
| 126 | if len(o.etcdPrefix) > 0 { |
| 127 | config.Prefix = o.etcdPrefix |
| 128 | } |
| 129 | |
| 130 | if len(o.mediaType) > 0 { |
| 131 | options.StorageMediaType = o.mediaType |
| 132 | } |
| 133 | if o.serializer != nil { |
| 134 | options.StorageSerializer = o.serializer |
| 135 | } |
| 136 | if o.encoderDecoratorFn != nil { |
| 137 | options.EncoderDecoratorFn = o.encoderDecoratorFn |
| 138 | } |
| 139 | if o.decoderDecoratorFn != nil { |
| 140 | options.DecoderDecoratorFn = o.decoderDecoratorFn |
| 141 | } |
| 142 | if o.transformer != nil { |
| 143 | config.Transformer = o.transformer |
| 144 | } |
| 145 | if o.disablePaging { |
| 146 | config.Paging = false |
| 147 | } |
| 148 | } |
| 149 | |
| 150 | var _ StorageFactory = &DefaultStorageFactory{} |
| 151 | |
| 152 | const AllResources = "*" |
| 153 | |
| 154 | func NewDefaultStorageFactory( |
| 155 | config storagebackend.Config, |
| 156 | defaultMediaType string, |
| 157 | defaultSerializer runtime.StorageSerializer, |
| 158 | resourceEncodingConfig ResourceEncodingConfig, |
| 159 | resourceConfig APIResourceConfigSource, |
| 160 | specialDefaultResourcePrefixes map[schema.GroupResource]string, |
| 161 | ) *DefaultStorageFactory { |
| 162 | config.Paging = utilfeature.DefaultFeatureGate.Enabled(features.APIListChunking) |
| 163 | if len(defaultMediaType) == 0 { |
| 164 | defaultMediaType = runtime.ContentTypeJSON |
| 165 | } |
| 166 | return &DefaultStorageFactory{ |
| 167 | StorageConfig: config, |
| 168 | Overrides: map[schema.GroupResource]groupResourceOverrides{}, |
| 169 | DefaultMediaType: defaultMediaType, |
| 170 | DefaultSerializer: defaultSerializer, |
| 171 | ResourceEncodingConfig: resourceEncodingConfig, |
| 172 | APIResourceConfigSource: resourceConfig, |
| 173 | DefaultResourcePrefixes: specialDefaultResourcePrefixes, |
| 174 | |
| 175 | newStorageCodecFn: NewStorageCodec, |
| 176 | } |
| 177 | } |
| 178 | |
| 179 | func (s *DefaultStorageFactory) SetEtcdLocation(groupResource schema.GroupResource, location []string) { |
| 180 | overrides := s.Overrides[groupResource] |
| 181 | overrides.etcdLocation = location |
| 182 | s.Overrides[groupResource] = overrides |
| 183 | } |
| 184 | |
| 185 | func (s *DefaultStorageFactory) SetEtcdPrefix(groupResource schema.GroupResource, prefix string) { |
| 186 | overrides := s.Overrides[groupResource] |
| 187 | overrides.etcdPrefix = prefix |
| 188 | s.Overrides[groupResource] = overrides |
| 189 | } |
| 190 | |
| 191 | // SetDisableAPIListChunking allows a specific resource to disable paging at the storage layer, to prevent |
| 192 | // exposure of key names in continuations. This may be overridden by feature gates. |
| 193 | func (s *DefaultStorageFactory) SetDisableAPIListChunking(groupResource schema.GroupResource) { |
| 194 | overrides := s.Overrides[groupResource] |
| 195 | overrides.disablePaging = true |
| 196 | s.Overrides[groupResource] = overrides |
| 197 | } |
| 198 | |
| 199 | // SetResourceEtcdPrefix sets the prefix for a resource, but not the base-dir. You'll end up in `etcdPrefix/resourceEtcdPrefix`. |
| 200 | func (s *DefaultStorageFactory) SetResourceEtcdPrefix(groupResource schema.GroupResource, prefix string) { |
| 201 | overrides := s.Overrides[groupResource] |
| 202 | overrides.etcdResourcePrefix = prefix |
| 203 | s.Overrides[groupResource] = overrides |
| 204 | } |
| 205 | |
| 206 | func (s *DefaultStorageFactory) SetSerializer(groupResource schema.GroupResource, mediaType string, serializer runtime.StorageSerializer) { |
| 207 | overrides := s.Overrides[groupResource] |
| 208 | overrides.mediaType = mediaType |
| 209 | overrides.serializer = serializer |
| 210 | s.Overrides[groupResource] = overrides |
| 211 | } |
| 212 | |
| 213 | func (s *DefaultStorageFactory) SetTransformer(groupResource schema.GroupResource, transformer value.Transformer) { |
| 214 | overrides := s.Overrides[groupResource] |
| 215 | overrides.transformer = transformer |
| 216 | s.Overrides[groupResource] = overrides |
| 217 | } |
| 218 | |
| 219 | // AddCohabitatingResources links resources together the order of the slice matters! its the priority order of lookup for finding a storage location |
| 220 | func (s *DefaultStorageFactory) AddCohabitatingResources(groupResources ...schema.GroupResource) { |
| 221 | for _, groupResource := range groupResources { |
| 222 | overrides := s.Overrides[groupResource] |
| 223 | overrides.cohabitatingResources = groupResources |
| 224 | s.Overrides[groupResource] = overrides |
| 225 | } |
| 226 | } |
| 227 | |
| 228 | func (s *DefaultStorageFactory) AddSerializationChains(encoderDecoratorFn func(runtime.Encoder) runtime.Encoder, decoderDecoratorFn func([]runtime.Decoder) []runtime.Decoder, groupResources ...schema.GroupResource) { |
| 229 | for _, groupResource := range groupResources { |
| 230 | overrides := s.Overrides[groupResource] |
| 231 | overrides.encoderDecoratorFn = encoderDecoratorFn |
| 232 | overrides.decoderDecoratorFn = decoderDecoratorFn |
| 233 | s.Overrides[groupResource] = overrides |
| 234 | } |
| 235 | } |
| 236 | |
| 237 | func getAllResourcesAlias(resource schema.GroupResource) schema.GroupResource { |
| 238 | return schema.GroupResource{Group: resource.Group, Resource: AllResources} |
| 239 | } |
| 240 | |
| 241 | func (s *DefaultStorageFactory) getStorageGroupResource(groupResource schema.GroupResource) schema.GroupResource { |
| 242 | for _, potentialStorageResource := range s.Overrides[groupResource].cohabitatingResources { |
| 243 | if s.APIResourceConfigSource.AnyVersionForGroupEnabled(potentialStorageResource.Group) { |
| 244 | return potentialStorageResource |
| 245 | } |
| 246 | } |
| 247 | |
| 248 | return groupResource |
| 249 | } |
| 250 | |
| 251 | // New finds the storage destination for the given group and resource. It will |
| 252 | // return an error if the group has no storage destination configured. |
| 253 | func (s *DefaultStorageFactory) NewConfig(groupResource schema.GroupResource) (*storagebackend.Config, error) { |
| 254 | chosenStorageResource := s.getStorageGroupResource(groupResource) |
| 255 | |
| 256 | // operate on copy |
| 257 | storageConfig := s.StorageConfig |
| 258 | codecConfig := StorageCodecConfig{ |
| 259 | StorageMediaType: s.DefaultMediaType, |
| 260 | StorageSerializer: s.DefaultSerializer, |
| 261 | } |
| 262 | |
| 263 | if override, ok := s.Overrides[getAllResourcesAlias(chosenStorageResource)]; ok { |
| 264 | override.Apply(&storageConfig, &codecConfig) |
| 265 | } |
| 266 | if override, ok := s.Overrides[chosenStorageResource]; ok { |
| 267 | override.Apply(&storageConfig, &codecConfig) |
| 268 | } |
| 269 | |
| 270 | var err error |
| 271 | codecConfig.StorageVersion, err = s.ResourceEncodingConfig.StorageEncodingFor(chosenStorageResource) |
| 272 | if err != nil { |
| 273 | return nil, err |
| 274 | } |
| 275 | codecConfig.MemoryVersion, err = s.ResourceEncodingConfig.InMemoryEncodingFor(groupResource) |
| 276 | if err != nil { |
| 277 | return nil, err |
| 278 | } |
| 279 | codecConfig.Config = storageConfig |
| 280 | |
| 281 | storageConfig.Codec, err = s.newStorageCodecFn(codecConfig) |
| 282 | if err != nil { |
| 283 | return nil, err |
| 284 | } |
| 285 | glog.V(3).Infof("storing %v in %v, reading as %v from %#v", groupResource, codecConfig.StorageVersion, codecConfig.MemoryVersion, codecConfig.Config) |
| 286 | |
| 287 | return &storageConfig, nil |
| 288 | } |
| 289 | |
| 290 | // Backends returns all backends for all registered storage destinations. |
| 291 | // Used for getting all instances for health validations. |
| 292 | func (s *DefaultStorageFactory) Backends() []Backend { |
| 293 | servers := sets.NewString(s.StorageConfig.ServerList...) |
| 294 | |
| 295 | for _, overrides := range s.Overrides { |
| 296 | servers.Insert(overrides.etcdLocation...) |
| 297 | } |
| 298 | |
| 299 | tlsConfig := &tls.Config{ |
| 300 | InsecureSkipVerify: true, |
| 301 | } |
| 302 | if len(s.StorageConfig.CertFile) > 0 && len(s.StorageConfig.KeyFile) > 0 { |
| 303 | cert, err := tls.LoadX509KeyPair(s.StorageConfig.CertFile, s.StorageConfig.KeyFile) |
| 304 | if err != nil { |
| 305 | glog.Errorf("failed to load key pair while getting backends: %s", err) |
| 306 | } else { |
| 307 | tlsConfig.Certificates = []tls.Certificate{cert} |
| 308 | } |
| 309 | } |
| 310 | if len(s.StorageConfig.CAFile) > 0 { |
| 311 | if caCert, err := ioutil.ReadFile(s.StorageConfig.CAFile); err != nil { |
| 312 | glog.Errorf("failed to read ca file while getting backends: %s", err) |
| 313 | } else { |
| 314 | caPool := x509.NewCertPool() |
| 315 | caPool.AppendCertsFromPEM(caCert) |
| 316 | tlsConfig.RootCAs = caPool |
| 317 | tlsConfig.InsecureSkipVerify = false |
| 318 | } |
| 319 | } |
| 320 | |
| 321 | backends := []Backend{} |
| 322 | for server := range servers { |
| 323 | backends = append(backends, Backend{ |
| 324 | Server: server, |
| 325 | // We can't share TLSConfig across different backends to avoid races. |
| 326 | // For more details see: http://pr.k8s.io/59338 |
| 327 | TLSConfig: tlsConfig.Clone(), |
| 328 | }) |
| 329 | } |
| 330 | return backends |
| 331 | } |
| 332 | |
| 333 | func (s *DefaultStorageFactory) ResourcePrefix(groupResource schema.GroupResource) string { |
| 334 | chosenStorageResource := s.getStorageGroupResource(groupResource) |
| 335 | groupOverride := s.Overrides[getAllResourcesAlias(chosenStorageResource)] |
| 336 | exactResourceOverride := s.Overrides[chosenStorageResource] |
| 337 | |
| 338 | etcdResourcePrefix := s.DefaultResourcePrefixes[chosenStorageResource] |
| 339 | if len(groupOverride.etcdResourcePrefix) > 0 { |
| 340 | etcdResourcePrefix = groupOverride.etcdResourcePrefix |
| 341 | } |
| 342 | if len(exactResourceOverride.etcdResourcePrefix) > 0 { |
| 343 | etcdResourcePrefix = exactResourceOverride.etcdResourcePrefix |
| 344 | } |
| 345 | if len(etcdResourcePrefix) == 0 { |
| 346 | etcdResourcePrefix = strings.ToLower(chosenStorageResource.Resource) |
| 347 | } |
| 348 | |
| 349 | return etcdResourcePrefix |
| 350 | } |