feat(platform): support ccr mirrors (#2055)

tkestack · Aug 11, 2022 · 4252618 · 4252618
1 parent 374967e
commit 4252618
Show file tree

Hide file tree

Showing 13 changed files with 48 additions and 16 deletions.
diff --git a/build/docker/tools/provider-res/Makefile b/build/docker/tools/provider-res/Makefile
@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations under the License.
 
 IMAGE = provider-res
-VERSION = v1.21.4-3
+VERSION = v1.21.4-4
 REGISTRY_PREFIX ?= tkestack
 
 COMMON_SELF_DIR := $(dir $(lastword $(MAKEFILE_LIST)))

diff --git a/build/docker/tools/tke-installer/release.sh b/build/docker/tools/tke-installer/release.sh
@@ -23,7 +23,7 @@ set -o pipefail
 REGISTRY_PREFIX=${REGISTRY_PREFIX:-tkestack}
 BUILDER=${BUILDER:-default}
 VERSION=${VERSION:-$(git describe --dirty --always --tags | sed 's/-/./g')}
-PROVIDER_RES_VERSION=v1.21.4-3
+PROVIDER_RES_VERSION=v1.21.4-4
 K8S_VERSION=${PROVIDER_RES_VERSION%-*}
 DOCKER_VERSION=19.03.14
 CONTAINERD_VERSION=1.5.4

diff --git a/cmd/tke-installer/app/installer/images/images.go b/cmd/tke-installer/app/installer/images/images.go
@@ -117,7 +117,7 @@ var baseComponents = BaseComponents{
  TKEPlatformController: containerregistry.Image{Name: "tke-platform-controller", Tag: Version},
  TKERegistryAPI: containerregistry.Image{Name: "tke-registry-api", Tag: Version},
  TKERegistryController: containerregistry.Image{Name: "tke-registry-controller", Tag: Version},
- ProviderRes: containerregistry.Image{Name: "provider-res", Tag: "v1.21.4-3"},
+ ProviderRes: containerregistry.Image{Name: "provider-res", Tag: "v1.21.4-4"},
  TKEGateway: containerregistry.Image{Name: "tke-gateway", Tag: Version},
 
  NginxIngress: containerregistry.Image{Name: "ingress-nginx-controller", Tag: "v1.1.3"},

diff --git a/pkg/platform/provider/baremetal/cluster/create.go b/pkg/platform/provider/baremetal/cluster/create.go
@@ -178,6 +178,7 @@ func (p *Provider) EnsureRegistryHosts(ctx context.Context, c *v1.Cluster) error
  if c.Spec.TenantID != "" {
  domains = append(domains, c.Spec.TenantID+"."+p.Config.Registry.Domain)
  }
+ domains = append(domains, constants.MirrorsRegistryHostName)
  for _, machine := range machines {
  machineSSH, err := machine.SSH()
  if err != nil {
@@ -186,7 +187,11 @@ func (p *Provider) EnsureRegistryHosts(ctx context.Context, c *v1.Cluster) error
 
  for _, one := range domains {
  remoteHosts := hosts.RemoteHosts{Host: one, SSH: machineSSH}
- err := remoteHosts.Set(p.Config.Registry.IP)
+ ip := p.Config.Registry.IP
+ if len(p.Config.Registry.IP) == 0 {
+ ip = c.GetMainIP()
+ }
+ err := remoteHosts.Set(ip)
  if err != nil {
  return errors.Wrap(err, machine.IP)
  }

diff --git a/pkg/platform/provider/baremetal/cluster/kubeadm.go b/pkg/platform/provider/baremetal/cluster/kubeadm.go
@@ -34,6 +34,7 @@ import (
  "tkestack.io/tke/pkg/platform/provider/baremetal/phases/kubeadm"
  v1 "tkestack.io/tke/pkg/platform/types/v1"
  "tkestack.io/tke/pkg/util/apiclient"
+ "tkestack.io/tke/pkg/util/containerregistry"
  "tkestack.io/tke/pkg/util/json"
  "tkestack.io/tke/pkg/util/version"
 )
@@ -183,7 +184,7 @@ func (p *Provider) getClusterConfiguration(c *v1.Cluster) *kubeadmv1beta2.Cluste
  DNS: kubeadmv1beta2.DNS{
  Type: kubeadmv1beta2.CoreDNS,
  },
- ImageRepository: p.Config.Registry.Prefix,
+ ImageRepository: containerregistry.GetPrefix(),
  ClusterName: c.Name,
  FeatureGates: map[string]bool{
  "IPv6DualStack": c.Cluster.Spec.Features.IPv6DualStack},

diff --git a/pkg/platform/provider/baremetal/cluster/provider.go b/pkg/platform/provider/baremetal/cluster/provider.go
@@ -36,7 +36,6 @@ import (
  clusterprovider "tkestack.io/tke/pkg/platform/provider/cluster"
  "tkestack.io/tke/pkg/platform/types"
  "tkestack.io/tke/pkg/spec"
- "tkestack.io/tke/pkg/util/containerregistry"
  "tkestack.io/tke/pkg/util/log"
 )
 
@@ -166,8 +165,6 @@ func NewProvider() (*Provider, error) {
  }
  p.Config = cfg
 
- containerregistry.Init(cfg.Registry.Domain, cfg.Registry.Namespace)
-
  // Run for compatibility with installer.
  // TODO: Installer reuse platform components
  if cfg.PlatformAPIClientConfig != "" {

diff --git a/pkg/platform/provider/baremetal/conf/containerd/config.toml b/pkg/platform/provider/baremetal/conf/containerd/config.toml
@@ -41,3 +41,6 @@ state = "/run/containerd"
  [plugins."io.containerd.grpc.v1.cri".registry.configs."{{.}}".tls]
  insecure_skip_verify=true
  {{end}}
+ [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+ [plugins."io.containerd.grpc.v1.cri".registry.mirrors."ccr.ccs.tencentyun.com"]
+ endpoint = ["http:https://mirrors.tke.com"]
diff --git a/pkg/platform/provider/baremetal/constants/constants.go b/pkg/platform/provider/baremetal/constants/constants.go
@@ -133,7 +133,8 @@ const (
  // MinNumCPU mininum cpu number.
  MinNumCPU = 2
 
- APIServerHostName = "api.tke.com"
+ APIServerHostName = "api.tke.com"
+ MirrorsRegistryHostName = "mirrors.tke.com"
 
  // include itself
  NeedUpgradeCoreDNSLowerK8sVersion = "1.19.0"

diff --git a/pkg/platform/provider/baremetal/machine/create.go b/pkg/platform/provider/baremetal/machine/create.go
@@ -155,9 +155,14 @@ func (p *Provider) EnsureRegistryHosts(ctx context.Context, machine *platformv1.
  if machine.Spec.TenantID != "" {
  domains = append(domains, machine.Spec.TenantID+"."+p.config.Registry.Domain)
  }
+ domains = append(domains, constants.MirrorsRegistryHostName)
  for _, one := range domains {
  remoteHosts := hosts.RemoteHosts{Host: one, SSH: machineSSH}
- err := remoteHosts.Set(p.config.Registry.IP)
+ ip := p.config.Registry.IP
+ if len(p.config.Registry.IP) == 0 {
+ ip = cluster.GetMainIP()
+ }
+ err := remoteHosts.Set(ip)
  if err != nil {
  return err
  }

diff --git a/pkg/platform/provider/baremetal/machine/provider.go b/pkg/platform/provider/baremetal/machine/provider.go
@@ -32,7 +32,6 @@ import (
  "tkestack.io/tke/pkg/platform/provider/baremetal/constants"
  "tkestack.io/tke/pkg/platform/provider/baremetal/validation"
  machineprovider "tkestack.io/tke/pkg/platform/provider/machine"
- "tkestack.io/tke/pkg/util/containerregistry"
  "tkestack.io/tke/pkg/util/log"
 )
 
@@ -117,8 +116,6 @@ func NewProvider() (*Provider, error) {
  }
  p.config = cfg
 
- containerregistry.Init(cfg.Registry.Domain, cfg.Registry.Namespace)
-
  // Run for compatibility with installer.
  // TODO: Installer reuse platform components
  if cfg.PlatformAPIClientConfig != "" {

diff --git a/pkg/platform/provider/edge/cluster/provider.go b/pkg/platform/provider/edge/cluster/provider.go
@@ -39,7 +39,6 @@ import (
  clusterprovider "tkestack.io/tke/pkg/platform/provider/cluster"
  "tkestack.io/tke/pkg/platform/types"
  "tkestack.io/tke/pkg/spec"
- "tkestack.io/tke/pkg/util/containerregistry"
  "tkestack.io/tke/pkg/util/log"
 )
 
@@ -176,8 +175,6 @@ func NewProvider() (*Provider, error) {
  }
  p.bconfig = cfg
 
- containerregistry.Init(cfg.Registry.Domain, cfg.Registry.Namespace)
-
  // Run for compatibility with installer.
  // TODO: Installer reuse platform components
  if cfg.PlatformAPIClientConfig != "" {

diff --git a/pkg/platform/types/types.go b/pkg/platform/types/types.go
@@ -73,6 +73,19 @@ func (c *Cluster) RESTConfig() (*rest.Config, error) {
  return c.restConfig, nil
 }
 
+func (c *Cluster) GetMainIP() string {
+ mainIP := c.Spec.Machines[0].IP
+ if c.Spec.Features.HA != nil {
+ if c.Spec.Features.HA.TKEHA != nil {
+ mainIP = c.Spec.Features.HA.TKEHA.VIP
+ }
+ if c.Spec.Features.HA.ThirdPartyHA != nil {
+ mainIP = c.Spec.Features.HA.ThirdPartyHA.VIP
+ }
+ }
+ return mainIP
+}
+
 func (c *Cluster) Clientset() (kubernetes.Interface, error) {
  config, err := c.RESTConfig()
  if err != nil {

diff --git a/pkg/platform/types/v1/types.go b/pkg/platform/types/v1/types.go
@@ -58,6 +58,19 @@ func (c *Cluster) Clientset() (kubernetes.Interface, error) {
  return kubernetes.NewForConfig(config)
 }
 
+func (c *Cluster) GetMainIP() string {
+ mainIP := c.Spec.Machines[0].IP
+ if c.Spec.Features.HA != nil {
+ if c.Spec.Features.HA.TKEHA != nil {
+ mainIP = c.Spec.Features.HA.TKEHA.VIP
+ }
+ if c.Spec.Features.HA.ThirdPartyHA != nil {
+ mainIP = c.Spec.Features.HA.ThirdPartyHA.VIP
+ }
+ }
+ return mainIP
+}
+
 func (c *Cluster) ClientsetForBootstrap() (kubernetes.Interface, error) {
  config, err := c.RESTConfigForBootstrap()
  if err != nil {