Skip to content

Commit

Permalink
refactor(cluster): Add clusterCredentialRef in cluster for support va…
Browse files Browse the repository at this point in the history
…lidate cluster version and import repeatly

1. Add clusterCredentialRef in cluster for support validate cluster version and import repeatly
2. Refactor cluster and machine provider for reuse code and extension
3. Refactor cluster and machine controller for reuse code and extension
4. Support delete cluster for import later which means don't remove node when cluster is in terminating
5. Register provider change to non-critical path
6. Optimize cluster and machine health check in large-scale scene
  • Loading branch information
QianChenglong committed May 7, 2020
1 parent fba9e30 commit 3f8071a
Show file tree
Hide file tree
Showing 73 changed files with 3,273 additions and 2,138 deletions.
270 changes: 270 additions & 0 deletions api/meta/v1/types.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,270 @@
/*
* Tencent is pleased to support the open source community by making TKEStack
* available.
*
* Copyright (C) 2012-2019 Tencent. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the “License”); you may not use
* this file except in compliance with the License. You may obtain a copy of the
* License at
*
* https://opensource.org/licenses/Apache-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an “AS IS” BASIS, WITHOUT
* WARRANTIES OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/

package v1

import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
)

// TypeMeta describes an individual object in an API response or request
// with strings representing the type of the object and its API schema version.
// Structures that are versioned or persisted should inline TypeMeta.
//
// +k8s:deepcopy-gen=false
type TypeMeta struct {
// Kind is a string value representing the REST resource this object represents.
// Servers may infer this from the endpoint the client submits requests to.
// Cannot be updated.
// In CamelCase.
// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
// +optional
Kind string `json:"kind,omitempty" protobuf:"bytes,1,opt,name=kind"`

// APIVersion defines the versioned schema of this representation of an object.
// Servers should convert recognized schemas to the latest internal value, and
// may reject unrecognized values.
// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
// +optional
APIVersion string `json:"apiVersion,omitempty" protobuf:"bytes,2,opt,name=apiVersion"`
}

// ListMeta describes metadata that synthetic resources must have, including lists and
// various status objects. A resource may have only one of {ObjectMeta, ListMeta}.
type ListMeta struct {
// selfLink is a URL representing this object.
// Populated by the system.
// Read-only.
//
// DEPRECATED
// Kubernetes will stop propagating this field in 1.20 release and the field is planned
// to be removed in 1.21 release.
// +optional
SelfLink string `json:"selfLink,omitempty" protobuf:"bytes,1,opt,name=selfLink"`

// String that identifies the server's internal version of this object that
// can be used by clients to determine when objects have changed.
// Value must be treated as opaque by clients and passed unmodified back to the server.
// Populated by the system.
// Read-only.
// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency
// +optional
ResourceVersion string `json:"resourceVersion,omitempty" protobuf:"bytes,2,opt,name=resourceVersion"`

// continue may be set if the user set a limit on the number of items returned, and indicates that
// the server has more data available. The value is opaque and may be used to issue another request
// to the endpoint that served this list to retrieve the next set of available objects. Continuing a
// consistent list may not be possible if the server configuration has changed or more than a few
// minutes have passed. The resourceVersion field returned when using this continue value will be
// identical to the value in the first response, unless you have received this token from an error
// message.
Continue string `json:"continue,omitempty" protobuf:"bytes,3,opt,name=continue"`

// remainingItemCount is the number of subsequent items in the list which are not included in this
// list response. If the list request contained label or field selectors, then the number of
// remaining items is unknown and the field will be left unset and omitted during serialization.
// If the list is complete (either because it is not chunking or because this is the last chunk),
// then there are no more remaining items and this field will be left unset and omitted during
// serialization.
// Servers older than v1.15 do not set this field.
// The intended use of the remainingItemCount is *estimating* the size of a collection. Clients
// should not rely on the remainingItemCount to be set or to be exact.
// +optional
RemainingItemCount *int64 `json:"remainingItemCount,omitempty" protobuf:"bytes,4,opt,name=remainingItemCount"`
}

// ObjectMeta is metadata that all persisted resources must have, which includes all objects
// users must create.
type ObjectMeta struct {
// Name must be unique within a namespace. Is required when creating resources, although
// some resources may allow a client to request the generation of an appropriate name
// automatically. Name is primarily intended for creation idempotence and configuration
// definition.
// Cannot be updated.
// More info: http:https://kubernetes.io/docs/user-guide/identifiers#names
// +optional
Name string `json:"name,omitempty" protobuf:"bytes,1,opt,name=name"`

// GenerateName is an optional prefix, used by the server, to generate a unique
// name ONLY IF the Name field has not been provided.
// If this field is used, the name returned to the client will be different
// than the name passed. This value will also be combined with a unique suffix.
// The provided value has the same validation rules as the Name field,
// and may be truncated by the length of the suffix required to make the value
// unique on the server.
//
// If this field is specified and the generated name exists, the server will
// NOT return a 409 - instead, it will either return 201 Created or 500 with Reason
// ServerTimeout indicating a unique name could not be found in the time allotted, and the client
// should retry (optionally after the time indicated in the Retry-After header).
//
// Applied only if Name is not specified.
// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#idempotency
// +optional
GenerateName string `json:"generateName,omitempty" protobuf:"bytes,2,opt,name=generateName"`

// Namespace defines the space within each name must be unique. An empty namespace is
// equivalent to the "default" namespace, but "default" is the canonical representation.
// Not all objects are required to be scoped to a namespace - the value of this field for
// those objects will be empty.
//
// Must be a DNS_LABEL.
// Cannot be updated.
// More info: http:https://kubernetes.io/docs/user-guide/namespaces
// +optional
Namespace string `json:"namespace,omitempty" protobuf:"bytes,3,opt,name=namespace"`

// SelfLink is a URL representing this object.
// Populated by the system.
// Read-only.
//
// DEPRECATED
// Kubernetes will stop propagating this field in 1.20 release and the field is planned
// to be removed in 1.21 release.
// +optional
SelfLink string `json:"selfLink,omitempty" protobuf:"bytes,4,opt,name=selfLink"`

// UID is the unique in time and space value for this object. It is typically generated by
// the server on successful creation of a resource and is not allowed to change on PUT
// operations.
//
// Populated by the system.
// Read-only.
// More info: http:https://kubernetes.io/docs/user-guide/identifiers#uids
// +optional
UID types.UID `json:"uid,omitempty" protobuf:"bytes,5,opt,name=uid,casttype=k8s.io/kubernetes/pkg/types.UID"`

// An opaque value that represents the internal version of this object that can
// be used by clients to determine when objects have changed. May be used for optimistic
// concurrency, change detection, and the watch operation on a resource or set of resources.
// Clients must treat these values as opaque and passed unmodified back to the server.
// They may only be valid for a particular resource or set of resources.
//
// Populated by the system.
// Read-only.
// Value must be treated as opaque by clients and .
// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency
// +optional
ResourceVersion string `json:"resourceVersion,omitempty" protobuf:"bytes,6,opt,name=resourceVersion"`

// A sequence number representing a specific generation of the desired state.
// Populated by the system. Read-only.
// +optional
Generation int64 `json:"generation,omitempty" protobuf:"varint,7,opt,name=generation"`

// CreationTimestamp is a timestamp representing the server time when this object was
// created. It is not guaranteed to be set in happens-before order across separate operations.
// Clients may not set this value. It is represented in RFC3339 form and is in UTC.
//
// Populated by the system.
// Read-only.
// Null for lists.
// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
// +optional
CreationTimestamp metav1.Time `json:"creationTimestamp,omitempty" protobuf:"bytes,8,opt,name=creationTimestamp"`

// DeletionTimestamp is RFC 3339 date and time at which this resource will be deleted. This
// field is set by the server when a graceful deletion is requested by the user, and is not
// directly settable by a client. The resource is expected to be deleted (no longer visible
// from resource lists, and not reachable by name) after the time in this field, once the
// finalizers list is empty. As long as the finalizers list contains items, deletion is blocked.
// Once the deletionTimestamp is set, this value may not be unset or be set further into the
// future, although it may be shortened or the resource may be deleted prior to this time.
// For example, a user may request that a pod is deleted in 30 seconds. The Kubelet will react
// by sending a graceful termination signal to the containers in the pod. After that 30 seconds,
// the Kubelet will send a hard termination signal (SIGKILL) to the container and after cleanup,
// remove the pod from the API. In the presence of network partitions, this object may still
// exist after this timestamp, until an administrator or automated process can determine the
// resource is fully terminated.
// If not set, graceful deletion of the object has not been requested.
//
// Populated by the system when a graceful deletion is requested.
// Read-only.
// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
// +optional
DeletionTimestamp *metav1.Time `json:"deletionTimestamp,omitempty" protobuf:"bytes,9,opt,name=deletionTimestamp"`

// Number of seconds allowed for this object to gracefully terminate before
// it will be removed from the system. Only set when deletionTimestamp is also set.
// May only be shortened.
// Read-only.
// +optional
DeletionGracePeriodSeconds *int64 `json:"deletionGracePeriodSeconds,omitempty" protobuf:"varint,10,opt,name=deletionGracePeriodSeconds"`

// Map of string keys and values that can be used to organize and categorize
// (scope and select) objects. May match selectors of replication controllers
// and services.
// More info: http:https://kubernetes.io/docs/user-guide/labels
// +optional
Labels map[string]string `json:"labels,omitempty" protobuf:"bytes,11,rep,name=labels"`

// Annotations is an unstructured key value map stored with a resource that may be
// set by external tools to store and retrieve arbitrary metadata. They are not
// queryable and should be preserved when modifying objects.
// More info: http:https://kubernetes.io/docs/user-guide/annotations
// +optional
Annotations map[string]string `json:"annotations,omitempty" protobuf:"bytes,12,rep,name=annotations"`

// List of objects depended by this object. If ALL objects in the list have
// been deleted, this object will be garbage collected. If this object is managed by a controller,
// then an entry in this list will point to this controller, with the controller field set to true.
// There cannot be more than one managing controller.
// +optional
// +patchMergeKey=uid
// +patchStrategy=merge
OwnerReferences []metav1.OwnerReference `json:"ownerReferences,omitempty" patchStrategy:"merge" patchMergeKey:"uid" protobuf:"bytes,13,rep,name=ownerReferences"`

// Must be empty before the object is deleted from the registry. Each entry
// is an identifier for the responsible component that will remove the entry
// from the list. If the deletionTimestamp of the object is non-nil, entries
// in this list can only be removed.
// Finalizers may be processed and removed in any order. Order is NOT enforced
// because it introduces significant risk of stuck finalizers.
// finalizers is a shared field, any actor with permission can reorder it.
// If the finalizer list is processed in order, then this can lead to a situation
// in which the component responsible for the first finalizer in the list is
// waiting for a signal (field value, external system, or other) produced by a
// component responsible for a finalizer later in the list, resulting in a deadlock.
// Without enforced ordering finalizers are free to order amongst themselves and
// are not vulnerable to ordering changes in the list.
// +optional
// +patchStrategy=merge
Finalizers []string `json:"finalizers,omitempty" patchStrategy:"merge" protobuf:"bytes,14,rep,name=finalizers"`

// The name of the cluster which the object belongs to.
// This is used to distinguish resources with same name and namespace in different clusters.
// This field is not set anywhere right now and apiserver is going to ignore it if set in create or update request.
// +optional
ClusterName string `json:"clusterName,omitempty" protobuf:"bytes,15,opt,name=clusterName"`

// ManagedFields maps workflow-id and version to the set of fields
// that are managed by that workflow. This is mostly for internal
// housekeeping, and users typically shouldn't need to set or
// understand this field. A workflow can be the user's name, a
// controller's name, or the name of a specific apply path like
// "ci-cd". The set of fields is always in the version that the
// workflow used when modifying the object.
//
// +optional
ManagedFields []metav1.ManagedFieldsEntry `json:"managedFields,omitempty" protobuf:"bytes,17,rep,name=managedFields"`

// Extand
// TenantID is used to distinct different user.
TenantID string `json:"tenantID" protobuf:"bytes,80,opt,name=tenantID"`
}
16 changes: 11 additions & 5 deletions api/openapi/zz_generated.openapi.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

27 changes: 27 additions & 0 deletions api/platform/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@
package platform

import (
"errors"
"fmt"
"math/rand"
"time"

"tkestack.io/tke/pkg/util/ssh"
Expand All @@ -37,3 +40,27 @@ func (in *ClusterMachine) SSH() (*ssh.SSH, error) {
}
return ssh.New(sshConfig)
}

func (in *Cluster) Host() (string, error) {
addrs := make(map[AddressType][]ClusterAddress)
for _, one := range in.Status.Addresses {
addrs[one.Type] = append(addrs[one.Type], one)
}

var address *ClusterAddress
if len(addrs[AddressInternal]) != 0 {
address = &addrs[AddressInternal][rand.Intn(len(addrs[AddressInternal]))]
} else if len(addrs[AddressAdvertise]) != 0 {
address = &addrs[AddressAdvertise][rand.Intn(len(addrs[AddressAdvertise]))]
} else {
if len(addrs[AddressReal]) != 0 {
address = &addrs[AddressReal][rand.Intn(len(addrs[AddressReal]))]
}
}

if address == nil {
return "", errors.New("can't find valid address")
}

return fmt.Sprintf("%s:%d", address.Host, address.Port), nil
}
24 changes: 8 additions & 16 deletions api/platform/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,9 @@ type ClusterSpec struct {
NetworkDevice string
// +optional
ClusterCIDR string
// ServiceCIDR is used to set a separated CIDR for k8s service, it's exclusive with MaxClusterServiceNum.
// +optional
ServiceCIDR *string
// +optional
// DNSDomain is the dns domain used by k8s services. Defaults to "cluster.local".
DNSDomain string
Expand All @@ -106,9 +109,11 @@ type ClusterSpec struct {
// +optional
SchedulerExtraArgs map[string]string

// ServiceCIDR is used to set a separated CIDR for k8s service, it's exclusive with MaxClusterServiceNum.
// +optionals
ServiceCIDR *string
// ClusterCredentialRef for isolate sensitive information.
// If not specified, cluster controller will create one;
// If specified, provider must make sure is valid.
// +optional
ClusterCredentialRef *corev1.LocalObjectReference
}

// ClusterStatus represents information about the status of a cluster.
Expand Down Expand Up @@ -164,19 +169,6 @@ const (
// NetworkType defines the network type of cluster.
type NetworkType string

const (
// NetworkPhysics indicates the communication network using the physics network to establish the pod between nodes.
NetworkPhysics NetworkType = "Physics"
// NetworkVPC indicates the communication network using the VPC to establish the pod between nodes.
NetworkVPC NetworkType = "VPC"
// NetworkFlannel indicates the communication network using the flannel to establish the pod between nodes.
NetworkFlannel NetworkType = "Flannel"
// NetworkCalico indicates the communication network using the calico to establish the pod between nodes.
NetworkCalico NetworkType = "Calico"
// NetworkIPIP indicates the communication network using the IPIP to establish the pod between nodes.
NetworkIPIP NetworkType = "IPIP"
)

// GPUType defines the gpu type of cluster.
type GPUType string

Expand Down
Loading

0 comments on commit 3f8071a

Please sign in to comment.