diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 52948881e..13987a7d9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,4 +26,6 @@ jobs: password: ${{ secrets.REGISTRY_PASSWORD }} - run: | + VERSION=$(git describe --dirty --always --tags | sed 's/-/./g') make release + make release-test diff --git a/.gitignore b/.gitignore index ebc607d6e..a5e72390b 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ # Misc .DS_Store +.env .env.local .env.development.local .env.test.local diff --git a/Makefile b/Makefile index 13f4025ff..79ef5efad 100644 --- a/Makefile +++ b/Makefile @@ -111,12 +111,17 @@ test: ## quick-release: Quick release tke .PHONY: quick-release quick-release: - build/docker/tools/tke-installer/build.sh -q + build/docker/tools/tke-installer/release.sh -q ## release: Release tke .PHONY: release release: - build/docker/tools/tke-installer/build.sh + build/docker/tools/tke-installer/release.sh + +## release-test: test release +.PHONY: release +release-test: + go test tkestack.io/tke/test/e2e_installer ## help: Show this help info. .PHONY: help diff --git a/build/docker/tools/tke-installer/build.sh b/build/docker/tools/tke-installer/build.sh index b92647119..2b8406e91 100755 --- a/build/docker/tools/tke-installer/build.sh +++ b/build/docker/tools/tke-installer/build.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#! /usr/bin/env bash # Tencent is pleased to support the open source community by making TKEStack # available. @@ -16,93 +16,36 @@ # WARRANTIES OF ANY KIND, either express or implied. See the License for the # specific language governing permissions and limitations under the License. + set -o errexit set -o nounset set -o pipefail -REGISTRY_PREFIX=tkestack -VERSION=$(git describe --dirty --always --tags | sed 's/-/./g') -PROVIDER_RES_VERSION=v1.14.6-1 -K8S_VERION=${PROVIDER_RES_VERSION%-*} -OUTPUT_DIR=_output/ -DST_DIR=$(mktemp -d) -#DST_DIR="/var/folders/20/n4jpmhjs0hd9hjxg80yr2nww0000gn/T/tmp.uN71o6ID" -echo "$DST_DIR" || exit -SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") - -function usage() { - cat <"$DST_DIR"/images.tar.gz -} +cd $(dirname $0) -pwd +echo "===> Begin to create $target in $(pwd)" -quick=false -while getopts "hq" o; do - case "${o}" in - h) - usage - ;; - q) - quick=true - ;; - *) - usage - ;; - esac -done -shift $((OPTIND-1)) +echo "Step.1 cleanup" +rm -f $target -make build BINS="tke-installer" OS=linux ARCH=amd64 VERSION="$VERSION" +echo "Step.2 prepare package" +chmod +x *.sh +cp -f init_installer.sh $target +tar -czf package.tgz install.sh res -prepare_baremetal_provider -prepare_tke_installer -if [[ "${quick}" == "false" ]]; then - prepare_images -fi +echo "Step.3 generate installer" +cat package.tgz >>$target +chmod +x $target +rm -f package.tgz -build_installer_image +echo "===> Success to create $target" diff --git a/build/docker/tools/tke-installer/init_installer.sh b/build/docker/tools/tke-installer/init_installer.sh new file mode 100755 index 000000000..d720eb040 --- /dev/null +++ b/build/docker/tools/tke-installer/init_installer.sh @@ -0,0 +1,56 @@ +#! /usr/bin/env bash + +# Tencent is pleased to support the open source community by making TKEStack +# available. +# +# Copyright (C) 2012-2019 Tencent. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at +# +# https://opensource.org/licenses/Apache-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +umask 0022 +unset IFS +unset OFS +unset LD_PRELOAD +unset LD_LIBRARY_PATH + +export PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + +die() +{ + echo '[FAIL] Operation failed.' >&2 + exit 1 +} + +cd `dirname "${0}"` || exit 1 +cwd=`pwd` || exit 1 +file=`basename "${0}"` || exit 1 + +me="${cwd}/${file}" +tmp="${me}.tmp" + +rm -rf "${tmp}" || die +mkdir "${tmp}" && cd "${tmp}" || die + +tailNum=`sed -n '/^#real installing packages append below/{=;q;}' ${me}` +tailNum=$((tailNum +1)) +tail -n +${tailNum} "${me}" >package.tgz || die +tar -zxf package.tgz || die + +./install.sh || die +cd "${cwd}" || die +exit 0 + +#real installing packages append below diff --git a/build/docker/tools/tke-installer/install.sh b/build/docker/tools/tke-installer/install.sh new file mode 100755 index 000000000..482995400 --- /dev/null +++ b/build/docker/tools/tke-installer/install.sh @@ -0,0 +1,129 @@ +#! /usr/bin/env bash + +# Tencent is pleased to support the open source community by making TKEStack +# available. +# +# Copyright (C) 2012-2019 Tencent. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at +# +# https://opensource.org/licenses/Apache-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +umask 0022 +unset IFS +unset OFS +unset LD_PRELOAD +unset LD_LIBRARY_PATH + +export PATH='/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' + +VERSION=latest + +INSTALL_DIR=/opt/tke-installer +DATA_DIR=$INSTALL_DIR/data +OPTIONS="--name tke-installer -d --privileged --net=host -v/etc/hosts:/app/hosts -v/var/run/docker.sock:/var/run/docker.sock -v$DATA_DIR:/app/data -v$INSTALL_DIR/conf:/app/conf" + +function prefight() { + echo "Step.1 prefight" + + if [ "root" != "$(whoami)" ]; then + echo "only root can execute this script" + exit 1 + fi +} + +function ensure_docker() { + echo "Step.2 ensure docker is ok" + + if ! [ -x "$(command -v docker)" ]; then + echo "command docker not find" + install_docker + fi + if ! systemctl is-active --quiet docker; then + echo "docker status is not running" + install_docker + fi +} + +function install_docker() { + echo "install docker [doing]" + + tar xvaf "res/docker.tgz" -C /usr/bin --strip-components=1 + cp -v res/docker.service /etc/systemd/system + + systemctl daemon-reload + + # becuase first start docker may be restart some times + systemctl start docker || : + for i in {1..60}; do + if systemctl is-active --quiet docker; then + break + fi + sleep 1 + done + if (( i == 10)); then + echo "start docker failed, please check docker service." + exit 1 + fi + + echo "install docker [ok]" +} + +function load_image() { + echo "Step.3 load tke-installer image [doing]" + + docker load -i res/tke-installer.tgz + + echo "Step.3 load tke-installer image [ok]" +} + +function clean_old_data() { + echo "Step.4 clean old data [doing]" + + rm -rf $DATA_DIR || : + docker rm -f tke-installer || : + + echo "Step.4 clean old data [ok]" +} + +function start_installer() { + echo "Step.5 start tke-installer [doing]" + + docker run $OPTIONS tkestack/tke-installer:$VERSION + + echo "Step.5 start tke-installer [ok]" +} + +function check_installer() { + echo "Step.6 check tke-installer status [doing]" + + ip=$(ip route get 1 | awk '{print $NF;exit}') + url="http://$ip:8080/index.html" + if ! curl -sSf $url >/dev/null; then + echo "check installer status error" + docker logs tke-installer + exit 1 + fi + + echo "Step.6 check tke-installer status [ok]" + + echo "Please use your browser which can connect this machine to open $url for install TKE!" +} + +prefight +ensure_docker +load_image +clean_old_data +start_installer +check_installer diff --git a/build/docker/tools/tke-installer/release.sh b/build/docker/tools/tke-installer/release.sh new file mode 100755 index 000000000..5984ff7b5 --- /dev/null +++ b/build/docker/tools/tke-installer/release.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash + +# Tencent is pleased to support the open source community by making TKEStack +# available. +# +# Copyright (C) 2012-2019 Tencent. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use +# this file except in compliance with the License. You may obtain a copy of the +# License at +# +# https://opensource.org/licenses/Apache-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +REGISTRY_PREFIX=${REGISTRY_PREFIX:-tkestack} +VERSION=${VERSION:-$(git describe --dirty --always --tags | sed 's/-/./g')} +INSTALLER=tke-installer-x86_64-$VERSION.run +PROVIDER_RES_VERSION=v1.14.6-1 +K8S_VERION=${PROVIDER_RES_VERSION%-*} +OUTPUT_DIR=_output +DST_DIR=$(mktemp -d) +#DST_DIR="/var/folders/20/n4jpmhjs0hd9hjxg80yr2nww0000gn/T/tmp.uN71o6ID" +echo "$DST_DIR" || exit +SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") + +function usage() { + cat < $installer_dir/res/tke-installer.tgz + + sed -i "s;VERSION=.*;VERSION=$VERSION;g" $installer_dir/install.sh + + $installer_dir/build.sh $INSTALLER + cp -v $installer_dir/$INSTALLER $OUTPUT_DIR + + echo "build tke-installer success! OUTPUT => $OUTPUT_DIR/$INSTALLER" + sha256sum $OUTPUT_DIR/$INSTALLER > $OUTPUT_DIR/$INSTALLER.sha256 + + if [[ "${BUILDER}" == "tke" ]]; then + coscmd upload $installer_dir/$INSTALLER $INSTALLER + coscmd upload $installer_dir/$INSTALLER.sha256 $INSTALLER.sha256 + fi + + rm -rf $installer_dir +} + +function prepare_images() { + if [[ "${BUILDER}" == "tke" ]]; then + make push VERSION="$VERSION" + else + make image VERSION="$VERSION" + fi + + GENERATE_IMAGES_BIN="$OUTPUT_DIR"/$(go env GOOS)/$(go env GOARCH)/tke-generate-images + make build BINS=tke-generate-images VERSION="$VERSION" + + $GENERATE_IMAGES_BIN + $GENERATE_IMAGES_BIN | sed "s;^;$REGISTRY_PREFIX/;" | xargs -n1 -I{} sh -c "docker pull {} || exit 1" + $GENERATE_IMAGES_BIN | sed "s;^;$REGISTRY_PREFIX/;" | xargs docker save | gzip -c >"$DST_DIR"/images.tar.gz +} + +pwd + +quick=false +while getopts "hq" o; do + case "${o}" in + h) + usage + ;; + q) + quick=true + ;; + *) + usage + ;; + esac +done +shift $((OPTIND-1)) + +make build BINS="tke-installer" OS=linux ARCH=amd64 VERSION="$VERSION" + +prepare_baremetal_provider +prepare_tke_installer +if [[ "${quick}" == "false" ]]; then + prepare_images +fi + +build_installer_image +build_installer + +rm -rf $DST_DIR diff --git a/cmd/tke-installer/app/installer/images/images.go b/cmd/tke-installer/app/installer/images/images.go index da3d3d210..6dbd96745 100644 --- a/cmd/tke-installer/app/installer/images/images.go +++ b/cmd/tke-installer/app/installer/images/images.go @@ -28,6 +28,7 @@ import ( ) type Components struct { + LocalTCR containerregistry.Image Busybox containerregistry.Image Keepalived containerregistry.Image InfluxDB containerregistry.Image @@ -61,6 +62,7 @@ func (c Components) Get(name string) *containerregistry.Image { var Version = version.Get().GitVersion var components = Components{ + LocalTCR: containerregistry.Image{Name: "local-tcr", Tag: "v1.0.0"}, Busybox: containerregistry.Image{Name: "busybox", Tag: "1.31.0"}, Keepalived: containerregistry.Image{Name: "keepalived", Tag: "2.0.16-r0"}, InfluxDB: containerregistry.Image{Name: "influxdb", Tag: "1.7.6-alpine"}, diff --git a/cmd/tke-installer/app/installer/installer.go b/cmd/tke-installer/app/installer/installer.go index 041fe542c..635fec94b 100644 --- a/cmd/tke-installer/app/installer/installer.go +++ b/cmd/tke-installer/app/installer/installer.go @@ -128,7 +128,7 @@ type CreateClusterPara struct { // Config is the installer config type Config struct { - Basic Basic `json:"basic"` + Basic *Basic `json:"basic"` Auth Auth `json:"auth"` Registry Registry `json:"registry"` Business *Business `json:"business,omitempty"` @@ -185,6 +185,10 @@ func (r *Registry) Namespace() string { return r.TKERegistry.Namespace } +func (r *Registry) Prefix() string { + return path.Join(r.Domain(), r.Namespace()) +} + type TKERegistry struct { Domain string `json:"domain" validate:"hostname_rfc1123"` Namespace string `json:"namespace"` @@ -249,7 +253,7 @@ type ThirdPartyHA struct { type Gateway struct { Domain string `json:"domain"` - Cert Cert `json:"cert"` + Cert *Cert `json:"cert"` } type Cert struct { @@ -291,10 +295,10 @@ type handler struct { type ClusterProgressStatus string const ( - statusUnknown = "Unknown" - statusDoing = "Doing" - statusSuccess = "Success" - statusFailed = "Failed" + StatusUnknown = "Unknown" + StatusDoing = "Doing" + StatusSuccess = "Success" + StatusFailed = "Failed" ) const ( @@ -330,7 +334,7 @@ func NewTKE(config *config.Config) *TKE { c.clusterProviders = new(sync.Map) c.process = new(ClusterProgress) - c.process.Status = statusUnknown + c.process.Status = StatusUnknown return c } @@ -689,6 +693,13 @@ func (t *TKE) prepare() errors.APIStatus { } func (t *TKE) SetConfigDefault(config *Config) { + if config.Basic == nil { + config.Basic = &Basic{ + Username: "admin", + Password: []byte("admin"), + } + } + if config.Registry.TKERegistry != nil { config.Registry.TKERegistry.Namespace = "library" config.Registry.TKERegistry.Username = config.Basic.Username @@ -700,16 +711,41 @@ func (t *TKE) SetConfigDefault(config *Config) { config.Auth.TKEAuth.Password = config.Basic.Password } + if config.Gateway != nil { + if config.Gateway.Domain == "" { + config.Gateway.Domain = "console.tke.com" + } + if config.Gateway.Cert == nil { + config.Gateway.Cert = &Cert{ + SelfSignedCert: &SelfSignedCert{}, + } + } + } + } func (t *TKE) SetClusterDefault(cluster *platformv1.Cluster, config *Config) { + if cluster.APIVersion == "" { + cluster.APIVersion = platformv1.SchemeGroupVersion.String() + } + if cluster.Kind == "" { + cluster.Kind = "Cluster" + } cluster.Name = "global" cluster.Spec.DisplayName = "TKE" - cluster.Spec.TenantID = defaultTeantID if t.Para.Config.Auth.TKEAuth != nil { cluster.Spec.TenantID = t.Para.Config.Auth.TKEAuth.TenantID } cluster.Spec.Version = k8sVersion + if cluster.Spec.ClusterCIDR == "" { + cluster.Spec.ClusterCIDR = "10.244.0.0/16" + } + if cluster.Spec.Type == "" { + cluster.Spec.Type = platformv1.ClusterBaremetal + } + if cluster.Spec.NetworkDevice == "" { + cluster.Spec.NetworkDevice = "eth0" + } if config.HA != nil && config.HA.ThirdPartyHA != nil { cluster.Status.Addresses = append(cluster.Status.Addresses, platformv1.ClusterAddress{ @@ -810,15 +846,12 @@ func (t *TKE) initProviderConfig() error { if err != nil { return err } - c.Registry.Domain = t.Para.Config.Registry.Domain() - if t.Para.Config.Registry.ThirdPartyRegistry == nil && - t.Para.Config.Registry.TKERegistry != nil { - ip := t.Cluster.Spec.Machines[0].IP - if t.Para.Config.HA != nil { - ip = t.Para.Config.HA.VIP() - } - c.Registry.IP = ip + c.Registry.Prefix = t.Para.Config.Registry.Prefix() + ip, err := util.GetExternalIP() + if err != nil { + return pkgerrors.Wrap(err, "get external ip error") } + c.Registry.IP = ip return c.Save(pluginConfigFile) } @@ -840,7 +873,7 @@ func (t *TKE) createCluster(req *restful.Request, rsp *restful.Response) { if apiStatus != nil { _ = rsp.WriteHeaderAndJson(int(apiStatus.Status().Code), apiStatus.Status(), restful.MIME_JSON) } else { - _ = rsp.WriteEntity(t.Para) + _ = rsp.WriteHeaderAndEntity(http.StatusCreated, t.Para) } } @@ -895,42 +928,6 @@ func (t *TKE) findClusterProgress(request *restful.Request, response *restful.Re if apiStatus != nil { response.WriteHeaderAndJson(int(apiStatus.Status().Code), apiStatus.Status(), restful.MIME_JSON) } else { - if t.process.Status == statusSuccess { - if t.Para.Config.Gateway != nil { - var host string - if t.Para.Config.Gateway.Domain != "" { - host = t.Para.Config.Gateway.Domain - } else if t.Para.Config.HA != nil { - host = t.Para.Config.HA.VIP() - } else { - host = t.Para.Cluster.Spec.Machines[0].IP - } - t.process.URL = fmt.Sprintf("http://%s", host) - - t.process.Username = t.Para.Config.Basic.Username - t.process.Password = t.Para.Config.Basic.Password - - if t.Para.Config.Gateway.Cert.SelfSignedCert != nil { - t.process.CACert, _ = ioutil.ReadFile(constants.CACrtFile) - } - - if t.Para.Config.Gateway.Domain != "" { - t.process.Hosts = append(t.process.Hosts, t.Para.Config.Gateway.Domain) - } - - cfg, _ := t.getKubeconfig() - t.process.Kubeconfig, _ = runtime.Encode(clientcmdlatest.Codec, cfg) - } - - if t.Para.Config.Registry.TKERegistry != nil { - t.process.Hosts = append(t.process.Hosts, t.Para.Config.Registry.TKERegistry.Domain) - } - - t.process.Servers = t.servers - if t.Para.Config.HA != nil { - t.process.Servers = append(t.process.Servers, t.Para.Config.HA.VIP()) - } - } response.WriteEntity(t.process) } } @@ -943,7 +940,7 @@ func (t *TKE) do() { if t.Step == 0 { t.log.Print("===>starting install task") - t.process.Status = statusDoing + t.process.Status = StatusDoing } if t.runAfterClusterReady() { @@ -955,7 +952,7 @@ func (t *TKE) do() { start := time.Now() err := t.steps[t.Step].Func() if err != nil { - t.process.Status = statusFailed + t.process.Status = StatusFailed t.log.Printf("%d.%s [Failed] [%fs] error %s", t.Step, t.steps[t.Step].Name, time.Since(start).Seconds(), err) return } @@ -965,7 +962,41 @@ func (t *TKE) do() { t.backup() } - t.process.Status = statusSuccess + t.process.Status = StatusSuccess + if t.Para.Config.Gateway != nil { + var host string + if t.Para.Config.Gateway.Domain != "" { + host = t.Para.Config.Gateway.Domain + } else if t.Para.Config.HA != nil { + host = t.Para.Config.HA.VIP() + } else { + host = t.Para.Cluster.Spec.Machines[0].IP + } + t.process.URL = fmt.Sprintf("http://%s", host) + + t.process.Username = t.Para.Config.Basic.Username + t.process.Password = t.Para.Config.Basic.Password + + if t.Para.Config.Gateway.Cert.SelfSignedCert != nil { + t.process.CACert, _ = ioutil.ReadFile(constants.CACrtFile) + } + + if t.Para.Config.Gateway.Domain != "" { + t.process.Hosts = append(t.process.Hosts, t.Para.Config.Gateway.Domain) + } + + cfg, _ := t.getKubeconfig() + t.process.Kubeconfig, _ = runtime.Encode(clientcmdlatest.Codec, cfg) + } + + if t.Para.Config.Registry.TKERegistry != nil { + t.process.Hosts = append(t.process.Hosts, t.Para.Config.Registry.TKERegistry.Domain) + } + + t.process.Servers = t.servers + if t.Para.Config.HA != nil { + t.process.Servers = append(t.process.Servers, t.Para.Config.HA.VIP()) + } t.log.Printf("===>install task [Sucesss] [%fs]", time.Since(start).Seconds()) } @@ -1118,6 +1149,9 @@ func (t *TKE) loadImages() error { continue } nameAndTag := strings.Split(imageNames[1], ":") + if nameAndTag[0] == "tke-installer" { // no need to push installer image for speed up + continue + } if nameAndTag[1] == "" { t.log.Printf("skip invalid tag:name=%s", image) continue @@ -1151,41 +1185,11 @@ func (t *TKE) setupLocalRegistry() error { return err } - // for pull image from local registry on node - ip, err := util.GetExternalIP() + data, err := ioutil.ReadFile("hosts") if err != nil { - return pkgerrors.Wrap(err, "get external ip error") - } - err = t.injectRemoteHosts([]string{t.Para.Config.Registry.Domain()}, ip) - if err != nil { - return pkgerrors.Wrap(err, "inject remote hosts error") - } - - return nil -} - -func (t *TKE) injectRemoteHosts(host []string, ip string) error { - for _, machine := range t.Cluster.Spec.Machines { - sshConfig := &ssh.Config{ - User: machine.Username, - Host: machine.IP, - Port: int(machine.Port), - Password: string(machine.Password), - PrivateKey: machine.PrivateKey, - PassPhrase: machine.PassPhrase, - } - s, err := ssh.New(sshConfig) - if err != nil { - return err - } - for _, one := range host { - remoteHosts := hosts.RemoteHosts{Host: one, SSH: s} - err = remoteHosts.Set(ip) - if err != nil { - return err - } - } + return err } + t.log.Print(string(data)) return nil } @@ -1322,6 +1326,24 @@ func (t *TKE) prepareCertificates() error { } func (t *TKE) prepareBaremetalProviderConfig() error { + c, err := baremetalconfig.New(pluginConfigFile) + if err != nil { + return err + } + if t.Para.Config.Registry.ThirdPartyRegistry == nil && + t.Para.Config.Registry.TKERegistry != nil { + ip := t.Cluster.Spec.Machines[0].IP + if t.Para.Config.HA != nil { + ip = t.Para.Config.HA.VIP() + } + c.Registry.IP = ip + } + + err = c.Save(pluginConfigFile) + if err != nil { + return err + } + configMaps := []struct { Name string File string @@ -1911,6 +1933,7 @@ func (t *TKE) pushImages() error { cmd := exec.Command("sh", "-c", fmt.Sprintf("docker images --format='{{.Repository}}:{{.Tag}}' --filter='reference=%s'", imagesFilter), ) + t.log.Print(cmd) out, err := cmd.Output() if err != nil { return pkgerrors.Wrap(err, "docker images error") @@ -1918,6 +1941,9 @@ func (t *TKE) pushImages() error { tkeImages := strings.Split(strings.TrimSpace(string(out)), "\n") for i, image := range tkeImages { nameAndTag := strings.Split(image, ":") + if len(nameAndTag) != 2 { + continue + } if nameAndTag[1] == "" { t.log.Printf("skip invalid tag:name=%s", image) continue diff --git a/go.mod b/go.mod index f1fd52380..a42876910 100644 --- a/go.mod +++ b/go.mod @@ -48,6 +48,7 @@ require ( github.com/go-playground/locales v0.12.1 // indirect github.com/go-playground/universal-translator v0.16.0 // indirect github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48 + github.com/golang/glog v0.0.0 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect github.com/golang/snappy v0.0.1 github.com/google/gofuzz v1.0.0 @@ -59,14 +60,18 @@ require ( github.com/hashicorp/go-hclog v0.8.0 github.com/hashicorp/go-plugin v0.0.0-20181212150838-f444068e8f5a github.com/hashicorp/go-uuid v1.0.1 + github.com/howeyc/fsnotify v0.9.0 github.com/influxdata/influxdb1-client v0.0.0-20190402204710-8ff2fc3824fc github.com/jinzhu/configor v1.1.1 github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3 + github.com/joho/godotenv v1.3.0 github.com/json-iterator/go v1.1.7 github.com/kr/fs v0.1.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/leodido/go-urn v1.1.0 // indirect github.com/moul/http2curl v1.0.0 // indirect + github.com/onsi/ginkgo v1.8.0 + github.com/onsi/gomega v1.5.0 github.com/opencontainers/go-digest v1.0.0-rc1 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect github.com/parnurzeal/gorequest v0.2.15 @@ -79,11 +84,13 @@ require ( github.com/rs/cors v1.6.0 github.com/russellhaering/goxmldsig v0.0.0-20180430223755-7acd5e4a6ef7 // indirect github.com/segmentio/ksuid v1.0.2 + github.com/sirupsen/logrus v1.4.2 github.com/spf13/afero v1.2.2 github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.4.0 github.com/stretchr/testify v1.4.0 + github.com/tencentcloud/tencentcloud-sdk-go v3.0.107+incompatible github.com/thoas/go-funk v0.4.0 go.etcd.io/bbolt v1.3.3 // indirect go.uber.org/zap v1.10.0 @@ -101,14 +108,6 @@ require ( gopkg.in/square/go-jose.v2 v2.3.1 gopkg.in/yaml.v2 v2.2.4 gotest.tools v2.2.0+incompatible - k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf - k8s.io/utils v0.0.0-20190801114015-581e00157fb1 -) - -require ( - github.com/golang/glog v0.0.0 - github.com/howeyc/fsnotify v0.9.0 - github.com/sirupsen/logrus v1.4.2 k8s.io/api v0.0.0 k8s.io/apiextensions-apiserver v0.0.0 k8s.io/apimachinery v0.0.0 @@ -118,4 +117,6 @@ require ( k8s.io/component-base v0.0.0 k8s.io/klog v0.4.0 k8s.io/kube-aggregator v0.0.0 + k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf + k8s.io/utils v0.0.0-20190801114015-581e00157fb1 ) diff --git a/go.sum b/go.sum index 63a044a2e..344c7dbea 100644 --- a/go.sum +++ b/go.sum @@ -390,6 +390,8 @@ github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3 h1:sHsPfNMAG70QAvKbd github.com/jinzhu/copier v0.0.0-20180308034124-7e38e58719c3/go.mod h1:yL958EeXv8Ylng6IfnvG4oflryUi3vgA3xPs9hmII1s= github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7 h1:SMvOWPJCES2GdFracYbBQh93GXac8fq7HeN6JnpduB8= github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.0.0-20160907122059-bcac9884e750/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= @@ -635,6 +637,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/tencentcloud/tencentcloud-sdk-go v3.0.107+incompatible h1:uUGvqiepdES1dmkFrJ9zI1ZKTue7VG5ACLlAmizIdqw= +github.com/tencentcloud/tencentcloud-sdk-go v3.0.107+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4= github.com/thoas/go-funk v0.4.0 h1:KBaa5NL7NMtsFlQaD8nQMbDt1wuM+OOaNQyYNYQFhVo= github.com/thoas/go-funk v0.4.0/go.mod h1:mlR+dHGb+4YgXkf13rkQTuzrneeHANxOm6+ZnEV9HsA= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8 h1:ndzgwNDnKIqyCvHTXaCqh9KlOWKvBry6nuXMJmonVsE= diff --git a/pkg/platform/provider/baremetal/images/images.go b/pkg/platform/provider/baremetal/images/images.go index 18dc5adbc..8612fd4f6 100644 --- a/pkg/platform/provider/baremetal/images/images.go +++ b/pkg/platform/provider/baremetal/images/images.go @@ -23,7 +23,16 @@ import ( "tkestack.io/tke/pkg/util/containerregistry" ) +const ( + k8sVersion = "v1.14.6" +) + type Components struct { + KubeAPIServer containerregistry.Image + KubeControllerManager containerregistry.Image + KubeScheduler containerregistry.Image + KubeProxy containerregistry.Image + ETCD containerregistry.Image CoreDNS containerregistry.Image Pause containerregistry.Image @@ -42,6 +51,11 @@ func (c Components) Get(name string) *containerregistry.Image { } var components = Components{ + KubeAPIServer: containerregistry.Image{Name: "kube-apiserver", Tag: k8sVersion}, + KubeControllerManager: containerregistry.Image{Name: "kube-controller-manager", Tag: k8sVersion}, + KubeScheduler: containerregistry.Image{Name: "kube-scheduler", Tag: k8sVersion}, + KubeProxy: containerregistry.Image{Name: "kube-proxy", Tag: k8sVersion}, + ETCD: containerregistry.Image{Name: "etcd", Tag: "v3.3.12"}, CoreDNS: containerregistry.Image{Name: "coredns", Tag: "1.2.6"}, Pause: containerregistry.Image{Name: "pause", Tag: "3.1"}, diff --git a/test/e2e_installer/bootstrap_test.go b/test/e2e_installer/bootstrap_test.go new file mode 100644 index 000000000..d258d1836 --- /dev/null +++ b/test/e2e_installer/bootstrap_test.go @@ -0,0 +1,151 @@ +/* + * Tencent is pleased to support the open source community by making TKEStack + * available. + * + * Copyright (C) 2012-2019 Tencent. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the “License”); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://opensource.org/licenses/Apache-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an “AS IS” BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package e2e_installer_test + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "time" + + "k8s.io/apimachinery/pkg/util/wait" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + platformv1 "tkestack.io/tke/api/platform/v1" + "tkestack.io/tke/cmd/tke-installer/app/installer" + "tkestack.io/tke/pkg/util/ssh" + "tkestack.io/tke/test/util/cloudprovider" + "tkestack.io/tke/test/util/cloudprovider/tencent" +) + +var _ = Describe("bootstrap", func() { + var nodes []cloudprovider.Instance + var nodesSSH []ssh.Interface + var provider cloudprovider.Provider + needDelete := false + + BeforeEach(func() { + var err error + provider = tencent.NewTencentProvider() + nodes, err = provider.CreateInstances(3) + Expect(err).To(BeNil()) + + nodesSSH = make([]ssh.Interface, len(nodes)) + for i, one := range nodes { + fmt.Printf("ensure ssh %d %s is ready\n", i, one.InternalIP) + s, err := ssh.New(&ssh.Config{ + User: one.Username, + Password: one.Password, + Host: one.InternalIP, + Port: int(one.Port), + }) + Expect(err).To(BeNil()) + for j := 1; j <= 10; j++ { + err = s.Ping() + if err == nil { + break + } + time.Sleep(5 * time.Second) + } + nodesSSH[i] = s + } + }) + + AfterEach(func() { + if !needDelete { + return + } + var instanceIDs []*string + for i, one := range nodes { + fmt.Printf("delete instance %d %s\n", i, one.InternalIP) + instanceIDs = append(instanceIDs, &nodes[i].InstanceID) + } + err := provider.DeleteInstances(instanceIDs) + Expect(err).To(BeNil()) + }) + + It("should bootstrap successfuly", func() { + By("install installer") + version := os.Getenv("VERSION") + cmd := fmt.Sprintf("wget https://tke-release-1251707795.cos.ap-guangzhou.myqcloud.com/tke-installer-x86_64-%s.run{,.sha256} && sha256sum --check --status tke-installer-x86_64-%s.run.sha256 && chmod +x tke-installer-x86_64-%s.run && ./tke-installer-x86_64-%s.run", + version, version, version, version) + _, err := nodesSSH[0].CombinedOutput(cmd) + Expect(err).To(BeNil()) + + By("prepare parametes") + url := fmt.Sprintf("http://%s:8080/api/cluster", nodes[0].InternalIP) + para := new(installer.CreateClusterPara) + for _, one := range nodes[1:] { + para.Cluster.Spec.Machines = append(para.Cluster.Spec.Machines, platformv1.ClusterMachine{ + IP: one.InternalIP, + Port: one.Port, + Username: one.Username, + Password: []byte(one.Password), + }) + } + para.Config.Auth.TKEAuth = &installer.TKEAuth{} + para.Config.Registry.TKERegistry = &installer.TKERegistry{Domain: "registry.tke.com"} + para.Config.Business = &installer.Business{} + para.Config.Monitor = &installer.Monitor{ + InfluxDBMonitor: &installer.InfluxDBMonitor{ + LocalInfluxDBMonitor: &installer.LocalInfluxDBMonitor{}, + }, + } + para.Config.Gateway = &installer.Gateway{} + body, err := json.Marshal(para) + Expect(err).To(BeNil()) + + By("post data to installer for install") + resp, err := http.Post(url, "application/json", bytes.NewReader(body)) + Expect(err).To(BeNil()) + defer resp.Body.Close() + Expect(resp.StatusCode).To(Equal(http.StatusCreated)) + + By("wait install finish") + err = wait.PollImmediate(5*time.Second, 30*time.Minute, func() (bool, error) { + url := fmt.Sprintf("http://%s:8080/api/cluster/global/progress", nodes[0].InternalIP) + resp, err := http.Get(url) + if err != nil { + return false, nil + } + defer resp.Body.Close() + data, err := ioutil.ReadAll(resp.Body) + Expect(err).To(BeNil()) + progress := new(installer.ClusterProgress) + err = json.Unmarshal(data, progress) + Expect(err).To(BeNil()) + switch progress.Status { + case installer.StatusUnknown, installer.StatusDoing: + return false, nil + case installer.StatusFailed: + return false, fmt.Errorf("install failed:\n%s", progress.Data) + case installer.StatusSuccess: + return true, nil + default: + return false, fmt.Errorf("unknown install progress status: %s", progress.Status) + } + }) + Expect(err).To(BeNil()) + needDelete = true + }) +}) diff --git a/test/e2e_installer/e2e_installer_suite_test.go b/test/e2e_installer/e2e_installer_suite_test.go new file mode 100644 index 000000000..5b9b08237 --- /dev/null +++ b/test/e2e_installer/e2e_installer_suite_test.go @@ -0,0 +1,13 @@ +package e2e_installer_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestE2EInstaller(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "E2EInstaller Suite") +} diff --git a/test/util/cloudprovider/interface.go b/test/util/cloudprovider/interface.go new file mode 100644 index 000000000..c4118e764 --- /dev/null +++ b/test/util/cloudprovider/interface.go @@ -0,0 +1,34 @@ +/* + * Tencent is pleased to support the open source community by making TKEStack + * available. + * + * Copyright (C) 2012-2019 Tencent. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the “License”); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://opensource.org/licenses/Apache-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an “AS IS” BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package cloudprovider + +type Provider interface { + // CreateInstances create count instances which wait they are running and return info for ssh + CreateInstances(count int64) ([]Instance, error) + // DeleteInstances delete instances + DeleteInstances(instanceIDs []*string) error +} + +type Instance struct { + InstanceID string + InternalIP string + Port int32 + Username string + Password string +} diff --git a/test/util/cloudprovider/tencent/tencent.go b/test/util/cloudprovider/tencent/tencent.go new file mode 100644 index 000000000..5816324c3 --- /dev/null +++ b/test/util/cloudprovider/tencent/tencent.go @@ -0,0 +1,108 @@ +/* + * Tencent is pleased to support the open source community by making TKEStack + * available. + * + * Copyright (C) 2012-2019 Tencent. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the “License”); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://opensource.org/licenses/Apache-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an “AS IS” BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tencent + +import ( + "os" + "time" + + "github.com/joho/godotenv" + "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common" + "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/profile" + cvm "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cvm/v20170312" + "k8s.io/apimachinery/pkg/util/wait" + "tkestack.io/tke/test/util/cloudprovider" +) + +func init() { + godotenv.Load() // for local dev +} + +func NewTencentProvider() cloudprovider.Provider { + p := &provider{} + + credential := common.NewCredential( + os.Getenv("SECRET_ID"), + os.Getenv("SECRET_KEY"), + ) + cpf := profile.NewClientProfile() + p.cvmClient, _ = cvm.NewClient(credential, os.Getenv("REGION"), cpf) + + return p +} + +type provider struct { + cvmClient *cvm.Client +} + +func (p *provider) CreateInstances(count int64) ([]cloudprovider.Instance, error) { + result := make([]cloudprovider.Instance, count) + + request := cvm.NewRunInstancesRequest() + err := request.FromJsonString(os.Getenv("CREATE_INSTANCES_PARAM")) + if err != nil { + return nil, err + } + request.InstanceCount = &count + response, err := p.cvmClient.RunInstances(request) + if err != nil { + return nil, err + } + + err = wait.PollImmediate(5*time.Second, 10*time.Minute, func() (bool, error) { + describeInstancesRequest := cvm.NewDescribeInstancesRequest() + describeInstancesRequest.InstanceIds = response.Response.InstanceIdSet + describeInstancesResponse, err := p.cvmClient.DescribeInstances(describeInstancesRequest) + if err != nil { + return false, nil + } + for _, one := range describeInstancesResponse.Response.InstanceSet { + if *one.InstanceState != "RUNNING" { + return false, nil + } + } + for i, one := range describeInstancesResponse.Response.InstanceSet { + result[i] = cloudprovider.Instance{ + InstanceID: *one.InstanceId, + InternalIP: *one.PrivateIpAddresses[0], + Port: 22, + Username: "root", + Password: os.Getenv("PASSWORD"), + } + } + return true, nil + }) + if err != nil { + _ = p.DeleteInstances(response.Response.InstanceIdSet) + return nil, err + } + + return result, nil +} + +func (p *provider) DeleteInstances(instanceIDs []*string) error { + request := cvm.NewTerminateInstancesRequest() + request.InstanceIds = instanceIDs + _, err := p.cvmClient.TerminateInstances(request) + if err != nil { + return err + } + + return nil +} diff --git a/test/util/cloudprovider/tencent/tencent_test.go b/test/util/cloudprovider/tencent/tencent_test.go new file mode 100644 index 000000000..884defc7d --- /dev/null +++ b/test/util/cloudprovider/tencent/tencent_test.go @@ -0,0 +1,32 @@ +/* + * Tencent is pleased to support the open source community by making TKEStack + * available. + * + * Copyright (C) 2012-2019 Tencent. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the “License”); you may not use + * this file except in compliance with the License. You may obtain a copy of the + * License at + * + * https://opensource.org/licenses/Apache-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an “AS IS” BASIS, WITHOUT + * WARRANTIES OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tencent + +import ( + "fmt" + "testing" +) + +func TestCreateInstance(t *testing.T) { + fmt.Println(NewTencentProvider().CreateInstances(2)) +} + +func TestDeleteInstance(t *testing.T) { + //NewTencentProvider().DeleteInstances("") +}