Skip to content

Commit

Permalink
feat(platform): support deploy clusternet app
Browse files Browse the repository at this point in the history
  • Loading branch information
leonarliu committed Jul 5, 2022
1 parent fb637f7 commit 465e9e4
Show file tree
Hide file tree
Showing 7 changed files with 275 additions and 1 deletion.
10 changes: 10 additions & 0 deletions api/platform/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,16 @@ const (
const (
// RegistrationCommandAnno contains base64 registration command of cluster net
RegistrationCommandAnno = "tkestack.io/registration-command"
// AnywhereEdtionLabel describe which anywhere edition will be deployed
AnywhereEdtionLabel = "tkestack.io/anywhere-edtion"
// AnywhereSubscriptionNameAnno describe sub name
AnywhereSubscriptionNameAnno = "tkestack.io/anywhere-subscription-name"
// AnywhereSubscriptionNameAnno describe sub namespace
AnywhereSubscriptionNamespaceAnno = "tkestack.io/anywhere-subscription-namespace"
// AnywhereLocalizationsAnno contains base64 localizations json data
AnywhereLocalizationsAnno = "tkestack.io/anywhere-localizations"
// AnywhereMachinesAnno contains base64 machines json data
AnywhereMachinesAnno = "tkestack.io/anywhere-machines"
)

// KubeVendorType describe the kubernetes provider of the cluster
Expand Down
10 changes: 10 additions & 0 deletions api/platform/v1/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,16 @@ const (
const (
// RegistrationCommandAnno contains base64 registration command of cluster net
RegistrationCommandAnno = "tkestack.io/registration-command"
// AnywhereEdtionLabel describe which anywhere edition will be deployed
AnywhereEdtionLabel = "tkestack.io/anywhere-edtion"
// AnywhereSubscriptionNameAnno describe sub name
AnywhereSubscriptionNameAnno = "tkestack.io/anywhere-subscription-name"
// AnywhereSubscriptionNameAnno describe sub namespace
AnywhereSubscriptionNamespaceAnno = "tkestack.io/anywhere-subscription-namespace"
// AnywhereLocalizationsAnno contains base64 localizations json data
AnywhereLocalizationsAnno = "tkestack.io/anywhere-localizations"
// AnywhereMachinesAnno contains base64 machines json data
AnywhereMachinesAnno = "tkestack.io/anywhere-machines"
)

// KubeVendorType describe the kubernetes provider of the cluster
Expand Down
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ require (
github.com/casbin/casbin/v2 v2.2.1
github.com/chartmuseum/helm-push v0.9.0
github.com/chartmuseum/storage v0.11.0
github.com/clusternet/apis v0.6.0
github.com/clusternet/clusternet v0.6.0 // indirect
github.com/coreos/go-oidc v2.2.1+incompatible
github.com/coreos/prometheus-operator v0.38.1-0.20200506070354-4231c1d4b313
github.com/cyphar/filepath-securejoin v0.2.2
Expand Down
45 changes: 45 additions & 0 deletions go.sum

Large diffs are not rendered by default.

94 changes: 94 additions & 0 deletions pkg/platform/provider/baremetal/cluster/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"context"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"io/ioutil"
"math/rand"
Expand All @@ -32,16 +33,20 @@ import (
"strings"
"time"

appsv1alpha1 "github.com/clusternet/apis/apps/v1alpha1"
"github.com/imdario/mergo"
"github.com/pkg/errors"
"github.com/segmentio/ksuid"
"github.com/thoas/go-funk"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/rest"
bootstraputil "k8s.io/cluster-bootstrap/token/util"
kubeaggregatorclientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
utilsnet "k8s.io/utils/net"
runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
platformv1 "tkestack.io/tke/api/platform/v1"
"tkestack.io/tke/pkg/platform/provider/baremetal/constants"
"tkestack.io/tke/pkg/platform/provider/baremetal/images"
Expand All @@ -65,6 +70,7 @@ import (
"tkestack.io/tke/pkg/platform/provider/util/mark"
v1 "tkestack.io/tke/pkg/platform/types/v1"
"tkestack.io/tke/pkg/util/apiclient"
"tkestack.io/tke/pkg/util/clusternet"
"tkestack.io/tke/pkg/util/cmdstring"
containerregistryutil "tkestack.io/tke/pkg/util/containerregistry"
"tkestack.io/tke/pkg/util/hosts"
Expand Down Expand Up @@ -1555,3 +1561,91 @@ func (p *Provider) EnsureClusternetRegistration(ctx context.Context, c *v1.Clust
}
return nil
}

func (p *Provider) EnsureAnywhereEdtion(ctx context.Context, c *v1.Cluster) error {
if c.Labels[platformv1.AnywhereEdtionLabel] == "" {
log.FromContext(ctx).Info("anywhere edtion is empty, skip EnsureAnywhereEdtion")
return nil
}
config, err := rest.InClusterConfig()
if err != nil {
return err
}
hubClient, err := clusternet.GetHubClient(config)
if err != nil {
return err
}
current, err := clusternet.GetManagedCluster(hubClient, c.Name)
if err != nil {
return err
}

if c.Annotations[platformv1.AnywhereLocalizationsAnno] != "" {
localizationsJSON, err := base64.StdEncoding.DecodeString(c.Annotations[platformv1.AnywhereLocalizationsAnno])
if err != nil {
return fmt.Errorf("decode localizations failed: %v", err)

}
localizations := new(appsv1alpha1.LocalizationList)
err = json.Unmarshal(localizationsJSON, localizations)
if err != nil {
return fmt.Errorf("unmarshal localization failed %v", err)
}

for _, l := range localizations.Items {
l.Namespace = current.Namespace
err := hubClient.Create(ctx, &l)
if err != nil && !apierrors.IsAlreadyExists(err) {
return fmt.Errorf("create localization %+v failed: %v", l, err)
}
}
}

desired := current.DeepCopy()
desired.Labels[platformv1.AnywhereEdtionLabel] = c.Labels[platformv1.AnywhereEdtionLabel]
err = hubClient.Patch(ctx, desired, runtimeclient.MergeFrom(current))
if err != nil {
return fmt.Errorf("patch managed cls failed %v", err)
}
return nil
}

func (p *Provider) EnsureCheckAnywhereSubscription(ctx context.Context, c *v1.Cluster) error {
if c.Annotations[platformv1.AnywhereSubscriptionNameAnno] == "" {
log.FromContext(ctx).Info("anywhere subscription name is empty, skip subscription")
return nil
}
config, err := rest.InClusterConfig()
if err != nil {
return err
}
hubClient, err := clusternet.GetHubClient(config)
if err != nil {
return err
}
mcls, err := clusternet.GetManagedCluster(hubClient, c.Name)
if err != nil {
return err
}
sub, err := clusternet.GetSubscription(hubClient, c.Annotations[platformv1.AnywhereSubscriptionNameAnno], c.Annotations[platformv1.AnywhereSubscriptionNamespaceAnno])
if err != nil {
return err
}
for _, feed := range sub.Spec.Feeds {
err = wait.PollImmediate(5*time.Second, 5*time.Minute, func() (bool, error) {
hr, err := clusternet.GetHelmRelease(hubClient, clusternet.GenerateHelmReleaseName(sub.Name, feed), mcls.Namespace)
if err != nil {
return false, fmt.Errorf("get helmrelease %s failed: %v", feed.Name, err)
}
if hr.Status.Phase != "deployed" {
return false, fmt.Errorf("helm release phase: %s", hr.Status.Phase)
}
return true, nil
})
if err != nil {
return err
}
}
return nil

}
5 changes: 4 additions & 1 deletion pkg/platform/provider/baremetal/cluster/provider.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ func NewProvider() (*Provider, error) {
p.EnsureNvidiaDriver,
p.EnsureNvidiaContainerRuntime,
p.EnsureContainerRuntime,
p.EnsureKubernetesImages,
// p.EnsureKubernetesImages,
p.EnsureKubelet,
p.EnsureCNIPlugins,
p.EnsureConntrackTools,
Expand Down Expand Up @@ -120,6 +120,9 @@ func NewProvider() (*Provider, error) {
p.EnsureGalaxy,
p.EnsureCilium,

p.EnsureAnywhereEdtion,
p.EnsureCheckAnywhereSubscription,

p.EnsurePatchAnnotation, // wait rest master ready
p.EnsureMarkControlPlane,
p.EnsureKeepalivedWithLBOption,
Expand Down
110 changes: 110 additions & 0 deletions pkg/util/clusternet/clusternet.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
package clusternet

import (
"context"
"errors"
"fmt"

appsv1alpha1 "github.com/clusternet/apis/apps/v1alpha1"
clustersv1beta1 "github.com/clusternet/apis/clusters/v1beta1"
"k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
"sigs.k8s.io/controller-runtime/pkg/client"

"k8s.io/client-go/rest"
)

var (
scheme = runtime.NewScheme()
)

func init() {
utilruntime.Must(clientgoscheme.AddToScheme(scheme))

utilruntime.Must(clustersv1beta1.AddToScheme(scheme))
utilruntime.Must(appsv1alpha1.AddToScheme(scheme))
}

func GetHubClient(config *rest.Config) (client.Client, error) {
var err error

if config == nil {
return nil, errors.New("empty hub restconfig file")
}

config.ContentConfig.ContentType = "application/json"

if err != nil {
return nil, fmt.Errorf("fail to get hub cluster rest config ,err is %s\n", err)
}

clusternetClient, err := client.New(config, client.Options{
Scheme: scheme,
})
if err != nil {
return nil, fmt.Errorf("fail to build a clusternet clien, error is %s", err)
}

return clusternetClient, nil
}

func GetManagedCluster(clientSet client.Client, name string) (*clustersv1beta1.ManagedCluster, error) {

mcSet := clustersv1beta1.ManagedClusterList{}

err := clientSet.List(context.TODO(), &mcSet, client.MatchingFields{"metadata.name": name})

if err != nil {
return nil, fmt.Errorf("fail to get managed cluster object which name is %s,err is %s", name, err)
}

if len(mcSet.Items) == 0 {
return nil, fmt.Errorf("not find a managed cluster named %s", name)
}

// mcs should have only one item
mc := mcSet.Items[0]

return &mc, nil
}

func GetSubscription(clientSet client.Client, name, namespace string) (*appsv1alpha1.Subscription, error) {
if namespace == "" {
namespace = "default"
}
sub := new(appsv1alpha1.Subscription)
sub.Name = name
sub.Namespace = namespace
key, err := client.ObjectKeyFromObject(sub)
if err != nil {
return nil, fmt.Errorf("get subscription key failed: %v", err)
}
err = clientSet.Get(context.TODO(), key, sub)
if err != nil {
return nil, fmt.Errorf("get subscription failed: %v", err)
}
return sub, nil
}

func GenerateHelmReleaseName(subName string, feed appsv1alpha1.Feed) string {
return fmt.Sprintf("%s-helm-%s-%s", subName, feed.Namespace, feed.Name)
}

func GetHelmRelease(clientSet client.Client, name, namespace string) (*appsv1alpha1.HelmRelease, error) {
if namespace == "" {
namespace = "default"
}
hr := new(appsv1alpha1.HelmRelease)
hr.Name = name
hr.Namespace = namespace
key, err := client.ObjectKeyFromObject(hr)
if err != nil {
return nil, fmt.Errorf("get hemrelease key failed: %v", err)
}
err = clientSet.Get(context.TODO(), key, hr)
if err != nil {
return nil, fmt.Errorf("get helmrelease %s in %s failed: %v", name, namespace, err)
}
return hr, nil
}

0 comments on commit 465e9e4

Please sign in to comment.