Merge pull request istio#10388 from rkpagadala/master

fix release scripts for prow

release/gcb/docker_tag_push_lib.sh

@@ -82,33 +82,12 @@ function add_docker_creds() {
  local PUSH_HUB
  PUSH_HUB="$1"
 
- local ADD_DOCKER_KEY
- ADD_DOCKER_KEY="true"
- if [[ "${ADD_DOCKER_KEY}" != "true" ]]; then
- return
- fi
-
- if [[ "${PUSH_HUB}" == "docker.io/istio" ]]; then
- echo "using istio cred for docker"
- KEYRING="Secrets"
- KEY="DockerHub"
-
- gsutil -q cp gs:https://istio-secrets/dockerhub_config.json.enc "$HOME/.docker/config.json.enc"
- gcloud kms decrypt \
- --ciphertext-file="$HOME/.docker/config.json.enc" \
- --plaintext-file="$HOME/.docker/config.json" \
- --location=global \
- --keyring="${KEYRING}" \
- --key="${KEY}"
- return
- fi
-
- if [[ "${PUSH_HUB}" == "docker.io/testistio" ]]; then
- gsutil -q cp "gs:https://istio-secrets/docker.test.json" "$HOME/.docker/config.json"
- fi
-
+ cp -r "${DOCKER_CONFIG}" "$HOME/.docker"
+ export DOCKER_CONFIG="$HOME/.docker"
  if [[ "${PUSH_HUB}" == gcr.io* ]]; then
  gcloud auth configure-docker -q
+ elif [[ "${PUSH_HUB}" == "docker.io/testistio" ]]; then
+ gsutil -q cp "gs:https://istio-secrets/docker.test.json" "$HOME/.docker/config.json"
  fi
 }
 

release/gcb/gcb_lib.sh

@@ -17,52 +17,17 @@
 
 # This script is meant to be sourced, has a set of functions used by scripts on gcb
 
-# this function sets variable TEST_INFRA_DIR and githubctl
-function githubctl_setup() {
- git clone https://github.com/istio/test-infra.git -b master --depth 1
- TEST_INFRA_DIR="${PWD}/test-infra"
- pushd "${TEST_INFRA_DIR}" || exit 1
- if [[ -f "/workspace/githubctl" ]]; then
- githubctl="/workspace/githubctl"
- ls -l "$githubctl"
- chmod +x "$githubctl"
- ls -l "$githubctl"
- else
- bazel build //toolbox/githubctl
- githubctl="${TEST_INFRA_DIR}/bazel-bin/toolbox/githubctl/linux_amd64_stripped/githubctl" 
- fi
- popd || exit 1
-
- export TEST_INFRA_DIR
- export githubctl
-}
-
 #sets GITHUB_KEYFILE to github auth file
 function github_keys() {
- local LOCAL_DIR
- LOCAL_DIR="$(mktemp -d /tmp/github.XXXX)"
- local KEYFILE_ENC
- KEYFILE_ENC="$LOCAL_DIR/keyfile.enc"
- local KEYFILE_TEMP
- KEYFILE_TEMP="$LOCAL_DIR/keyfile.txt"
- local KEYRING
- KEYRING="Secrets"
- local KEY
- KEY="DockerHub"
-
- GITHUB_KEYFILE="${KEYFILE_TEMP}"
+ GITHUB_KEYFILE="${GITHUB_TOKEN_FILE}"
  export GITHUB_KEYFILE
-
-
- gsutil -q cp "gs:https://${CB_GITHUB_TOKEN_FILE_PATH}" "${KEYFILE_ENC}"
- gcloud kms decrypt \
- --ciphertext-file="$KEYFILE_ENC" \
- --plaintext-file="$KEYFILE_TEMP" \
- --location=global \
- --keyring="${KEYRING}" \
- --key="${KEY}"
-
+
  if [[ -n "$CB_TEST_GITHUB_TOKEN_FILE_PATH" ]]; then
- gsutil -q cp "gs:https://${CB_TEST_GITHUB_TOKEN_FILE_PATH}" "${KEYFILE_TEMP}"
+ local LOCAL_DIR
+ LOCAL_DIR="$(mktemp -d /tmp/github.XXXX)"
+ local KEYFILE_TEMP
+ KEYFILE_TEMP="$LOCAL_DIR/keyfile.txt"
+ GITHUB_KEYFILE="${KEYFILE_TEMP}"
+ gsutil -q cp "gs:https://${CB_TEST_GITHUB_TOKEN_FILE_PATH}" "${KEYFILE_TEMP}"
  fi
 }

release/gcb/generate_manifest.sh

@@ -28,10 +28,23 @@ set -x
 # shellcheck disable=SC1091
 source "/workspace/gcb_env.sh"
 
-SCRIPTPATH=$( cd "$(dirname "$0")" ; pwd -P )
-# shellcheck source=release/gcb/gcb_lib.sh
+# this function sets variable TEST_INFRA_DIR and githubctl
+function githubctl_setup() {
+ GOPATH=$PWD/go
+ mkdir -p go/bin
+ GOBIN=$GOPATH/bin
+
+ go get -u istio.io/test-infra/toolbox/githubctl
+ githubctl="$GOBIN/githubctl"
+ export githubctl
+ TEST_INFRA_DIR="$GOPATH/src/istio.io/test-infra/"
+ export TEST_INFRA_DIR
+}
 
-source "${SCRIPTPATH}/gcb_lib.sh"
+function github_keys() {
+ GITHUB_KEYFILE="${GITHUB_TOKEN_FILE}"
+ export GITHUB_KEYFILE
+}
 
 function checkout_code() {
  local REPO=$1
@@ -75,23 +88,14 @@ EOF
 
  if [[ "${CB_VERIFY_CONSISTENCY}" == "true" ]]; then
  # Consistency check not needed for CNI
- checkout_code "proxy" "HEAD" .
+ checkout_code "proxy" "PROXY_REPO_SHA" .
  pushd proxy 
- PROXY_HEAD_SHA=$(git rev-parse HEAD)
- PROXY_HEAD_API_SHA=$(grep ISTIO_API istio.deps -A 4 | grep lastStableSHA | cut -f 4 -d '"')
+ PROXY_API_SHA=$(grep ISTIO_API istio.deps -A 4 | grep lastStableSHA | cut -f 4 -d '"')
  popd
- if [ "$PROXY_HEAD_SHA" != "$PROXY_REPO_SHA" ]; then
- echo "inconsistent shas PROXY_HEAD_SHA $PROXY_HEAD_SHA != $PROXY_REPO_SHA PROXY_REPO_SHA" 1>&2
- exit 16
- fi
- if [ "$PROXY_HEAD_API_SHA" != "$API_REPO_SHA" ]; then
- echo "inconsistent shas PROXY_HEAD_API_SHA $PROXY_HEAD_API_SHA != $API_REPO_SHA API_REPO_SHA" 1>&2
- exit 17
- fi
- if [ "$ISTIO_HEAD_SHA" != "$ISTIO_REPO_SHA" ]; then
- echo "inconsistent shas ISTIO_HEAD_SHA $ISTIO_HEAD_SHA != $ISTIO_REPO_SHA ISTIO_REPO_SHA" 1>&2
- exit 18
- fi
+ if [ "$PROXY_API_SHA" != "$API_REPO_SHA" ]; then
+ echo "inconsistent shas PROXY_API_SHA $PROXY_API_SHA != $API_REPO_SHA API_REPO_SHA" 1>&2
+ exit 17
+ fi
  fi
 }
 
@@ -209,20 +213,19 @@ popd
 CLONE_DIR=$(mktemp -d)
 pushd "${CLONE_DIR}"
  githubctl_setup
- gsutil -q cp -P "$githubctl" "gs:https://${CB_GCS_RELEASE_TOOLS_PATH}/"
 
+ MANIFEST_FILE="/workspace/manifest.txt"
  BASE_MANIFEST_URL="gs:https://${CB_GCS_BUILD_BUCKET}/release-tools/${CB_BRANCH}-manifest.txt"
  BASE_MASTER_MANIFEST_URL="gs:https://${CB_GCS_BUILD_BUCKET}/release-tools/master-manifest.txt"
 
  NEW_BRANCH="false"
- gsutil -q cp "${BASE_MANIFEST_URL}" "manifest.txt" || NEW_BRANCH="true"
+ gsutil -q cp "${BASE_MANIFEST_URL}" "$MANIFEST_FILE" || NEW_BRANCH="true"
  if [[ "${NEW_BRANCH}" == "true" ]]; then
  if [[ "${CB_COMMIT}" == "" ]]; then
  CB_COMMIT="HEAD" # just use head of branch as green sha
  fi
- gsutil -q cp "${BASE_MASTER_MANIFEST_URL}" "manifest.txt"
+ gsutil -q cp "${BASE_MASTER_MANIFEST_URL}" "$MANIFEST_FILE"
  fi
- MANIFEST_FILE="$PWD/manifest.txt"
 
  git clone "https://github.com/${CB_GITHUB_ORG}/istio" -b "${CB_BRANCH}"
 
@@ -238,8 +241,3 @@ pushd "${CLONE_DIR}"
 
 popd # "${CLONE_DIR}"
 rm -rf "${CLONE_DIR}"
-
- gsutil -mq cp -P istio/release/gcb/*sh "gs:https://${CB_GCS_RELEASE_TOOLS_PATH}/gcb/"
- gsutil -mq cp -P istio/release/gcb/*json "gs:https://${CB_GCS_RELEASE_TOOLS_PATH}/gcb/"
- gsutil -mq cp -P istio/release/pipeline/*sh "gs:https://${CB_GCS_RELEASE_TOOLS_PATH}/pipeline/"
-

release/gcb/istio_checkout_code.sh

@@ -44,15 +44,17 @@ function checkout_code() {
 function istio_code_init_manifest() {
  local MANIFEST_FILE=$1
 
- ISTIO_SHA=$(grep "istio" "$MANIFEST_FILE" | cut -f 2 -d " ")
  API_SHA=$( grep "api" "$MANIFEST_FILE" | cut -f 2 -d " ")
  PROXY_SHA=$(grep "proxy" "$MANIFEST_FILE" | cut -f 2 -d " ")
  CNI_SHA=$( grep "cni" "$MANIFEST_FILE" | cut -f 2 -d " ")
 
  checkout_code "cni" "${CNI_SHA}" "/workspace/go/src/istio.io"
  checkout_code "proxy" "${PROXY_SHA}" "/workspace/src"
  checkout_code "api" "${API_SHA}" "/workspace/go/src/istio.io"
- checkout_code "istio" "${ISTIO_SHA}" "/workspace/go/src/istio.io"
+
+# istio is checkout by the initialization script in pipeline repo
+# ISTIO_SHA=$(grep "istio" "$MANIFEST_FILE" | cut -f 2 -d " ")
+# checkout_code "istio" "${ISTIO_SHA}" "/workspace/go/src/istio.io"
 }
 
 istio_code_init_manifest "/workspace/manifest.txt"

release/gcb/run_build.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# Copyright 2018 Istio Authors
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http:https://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Exit immediately for non zero status
+set -e
+# Check unset variables
+set -u
+# Print commands
+set -x
+
+cd /workspace || exit 1
+# /output is used to store release artifacts
+mkdir /output
+
+# start actual build steps
+/workspace/generate_manifest.sh
+/workspace/istio_checkout_code.sh
+
+cd /workspace/go/src/istio.io/istio || exit 2
+/workspace/cloud_builder.sh
+
+cd /workspace || exit 3
+/workspace/store_artifacts.sh
+/workspace/rel_push_docker_build_version.sh
+/workspace/helm_charts.sh
+/workspace/helm_values.sh

release/gcb/run_daily_release.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+# Copyright 2018 Istio Authors
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http:https://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+
+#copy files over to final destination
+gsutil -m cp -r "gs:https://$CB_GCS_BUILD_PATH" "gs:https://$CB_GCS_FULL_STAGING_PATH"
+
+if [[ "$CB_GITHUB_ORG" != "istio" ]]; then
+ if [[ "$CB_BRANCH" == *"release"* ]] || [[ "$CB_BRANCH" == "master" ]]; then
+ echo "not messing up daily builds with testing"
+ exit 0
+ fi
+fi
+
+cd /workspace || exit 1
+# run the release steps
+./rel_push_docker_daily.sh
+./rel_daily_complete.sh

release/gcb/run_monthly_release.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Copyright 2018 Istio Authors
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http:https://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -x
+
+#copy files over to final destination
+gsutil -m cp -r "gs:https://$CB_GCS_BUILD_PATH" "gs:https://$CB_GCS_FULL_STAGING_PATH"
+
+
+cd /workspace || exit 1
+# for testing use your own GITHUB_ORG (your own private org)
+# also set CB_TEST_GITHUB_TOKEN_FILE_PATH so that your github creds are used
+
+# run the release steps
+./github_publish_release.sh
+./github_tag_release.sh