migrating(eks): add example of migrating node groups with zero downtime

.travis.yml

@@ -14,10 +14,6 @@ before_install:
 - source ${GOPATH}/src/github.com/pulumi/scripts/ci/prepare-environment.sh
 - source ${PULUMI_SCRIPTS}/ci/keep-failed-tests.sh
 - sudo apt-get update && sudo apt-get install -y apt-transport-https
-- curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
-- echo "deb https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee -a /etc/apt/sources.list.d/kubernetes.list
-- sudo apt-get update
-- sudo apt-get install -y kubectl
 install:
 - source ${PULUMI_SCRIPTS}/ci/install-common-toolchain.sh
 - curl -L https://get.pulumi.com/ | bash
@@ -32,10 +28,15 @@ install:
  | bash
 - helm init -c
 - helm repo add bitnami https://charts.bitnami.com/bitnami
+# Install aws-iam-authenticator
+# See: https://docs.aws.amazon.com/eks/latest/userguide/install-aws-iam-authenticator.html)
 - curl -o aws-iam-authenticator https://amazon-eks.s3-us-west-2.amazonaws.com/1.13.7/2019-06-11/bin/linux/amd64/aws-iam-authenticator
 - chmod +x ./aws-iam-authenticator
-- mkdir -p $HOME/bin && cp ./aws-iam-authenticator $HOME/bin/aws-iam-authenticator
--
+- sudo mv aws-iam-authenticator /usr/local/bin
+# Install kubectl
+- curl -LO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl
+- chmod +x ./kubectl
+- sudo mv kubectl /usr/local/bin
 before_script:
 - "${PULUMI_SCRIPTS}/ci/ensure-dependencies"
 after_failure:

aws-ts-eks-migrate-nodegroups/Pulumi.yaml

@@ -0,0 +1,3 @@
+name: aws-ts-eks-migrate-nodegroups
+description: Creates an EKS cluster with node groups and a workload. Then covers how to add an additional node group, and use it to migrate the workload with zero downtime.
+runtime: nodejs

aws-ts-eks-migrate-nodegroups/README.md

@@ -0,0 +1,4 @@
+# examples/aws-ts-eks-migrate-nodegroups
+
+Creates an EKS cluster with node groups and a workload, and showcases how add an
+additional node group to use for workload migration with zero downtime.

aws-ts-eks-migrate-nodegroups/echoserver.ts

@@ -0,0 +1,161 @@
+import * as eks from "@pulumi/eks";
+import * as k8s from "@pulumi/kubernetes";
+import * as pulumi from "@pulumi/pulumi";
+
+// Create the echoserver workload's Service, Deployment and Ingress.
+interface EchoserverArgs {
+ replicas: pulumi.Input<number>;
+ namespace: pulumi.Input<string>;
+ ingressClass: pulumi.Input<string>;
+ provider: k8s.Provider;
+}
+export function create(
+ name: string,
+ args: EchoserverArgs,
+): k8s.core.v1.Service {
+
+ const labels = {app: name};
+
+ // Create the Service.
+ const service = createService(name, {
+ labels: labels,
+ namespace: args.namespace,
+ provider: args.provider,
+ });
+ const serviceName = service.metadata.name;
+
+ // Deploy the echoserver in the general, standard nodegroup.
+ const deployment = createDeployment(name, {
+ replicas: args.replicas,
+ labels: labels,
+ namespace: args.namespace,
+ provider: args.provider,
+ });
+
+ // Create the Ingress.
+ const ingress = createIngress(name, {
+ labels: labels,
+ namespace: args.namespace,
+ ingressClass: args.ingressClass,
+ serviceName: serviceName,
+ provider: args.provider,
+ });
+
+ return service;
+}
+
+interface EchoserverServiceArgs {
+ labels: pulumi.Input<any>;
+ namespace: pulumi.Input<string>;
+ provider: k8s.Provider;
+}
+export function createService(
+ name: string,
+ args: EchoserverServiceArgs,
+): k8s.core.v1.Service {
+ return new k8s.core.v1.Service(
+ name,
+ {
+ metadata: {
+ labels: args.labels,
+ namespace: args.namespace,
+ },
+ spec: {
+ type: "ClusterIP",
+ ports: [{port: 80, protocol: "TCP", targetPort: "http"}],
+ selector: args.labels,
+ },
+ },
+ {
+ provider: args.provider,
+ },
+ );
+}
+
+interface EchoserverDeploymentArgs {
+ replicas: pulumi.Input<number>;
+ labels: pulumi.Input<any>;
+ namespace: pulumi.Input<string>;
+ provider: k8s.Provider;
+}
+export function createDeployment(
+ name: string,
+ args: EchoserverDeploymentArgs,
+): k8s.apps.v1.Deployment {
+ return new k8s.apps.v1.Deployment(name,
+ {
+ metadata: {
+ labels: args.labels,
+ namespace: args.namespace,
+ },
+ spec: {
+ replicas: args.replicas,
+ selector: { matchLabels: args.labels },
+ template: {
+ metadata: { labels: args.labels, namespace: args.namespace },
+ spec: {
+ restartPolicy: "Always",
+ containers: [
+ {
+ name: name,
+ image: "gcr.io/google-containers/echoserver:1.5",
+ ports: [{ name: "http", containerPort: 8080 }],
+ },
+ ],
+ },
+ },
+ },
+ },
+ {
+ provider: args.provider,
+ },
+ );
+}
+
+interface EchoserverIngressArgs {
+ labels: pulumi.Input<any>;
+ namespace: pulumi.Input<string>;
+ ingressClass: pulumi.Input<string>;
+ serviceName: pulumi.Input<string>;
+ provider: k8s.Provider;
+}
+export function createIngress(
+ name: string,
+ args: EchoserverIngressArgs,
+): k8s.extensions.v1beta1.Ingress {
+ // TODO(metral): change to k8s.networking.v1beta.Ingress
+ // when EKS supports >= 1.14.
+ return new k8s.extensions.v1beta1.Ingress(
+ name,
+ {
+ metadata: {
+ labels: args.labels,
+ namespace: args.namespace,
+ annotations: {
+ "kubernetes.io/ingress.class": args.ingressClass,
+ },
+ },
+ spec: {
+ rules: [
+ {
+ host: "apps.example.com",
+ http: {
+ paths: [
+ {
+ path: "/echoserver",
+ backend: {
+ serviceName: args.serviceName,
+ servicePort: "http",
+ },
+ },
+ ],
+ },
+ },
+ ],
+ },
+ },
+ {
+ provider: args.provider,
+ },
+ );
+}

aws-ts-eks-migrate-nodegroups/iam.ts

@@ -0,0 +1,51 @@
+import * as aws from "@pulumi/aws";
+import * as pulumi from "@pulumi/pulumi";
+import * as iam from "./iam";
+
+const managedPolicyArns: string[] = [
+ "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy",
+ "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy",
+ "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly",
+];
+
+// Creates a role and attaches the EKS worker node IAM managed policies
+export function createRole(name: string): aws.iam.Role {
+ const role = new aws.iam.Role(name, {
+ assumeRolePolicy: aws.iam.assumeRolePolicyForPrincipal({
+ Service: "ec2.amazonaws.com",
+ }),
+ });
+
+ let counter = 0;
+ for (const policy of managedPolicyArns) {
+ // Create RolePolicyAttachment without returning it.
+ const rpa = new aws.iam.RolePolicyAttachment(`${name}-policy-${counter++}`,
+ { policyArn: policy, role: role },
+ );
+ }
+
+ return role;
+}
+
+// Creates a collection of IAM roles.
+export function createRoles(name: string, quantity: number): aws.iam.Role[] {
+ const roles: aws.iam.Role[] = [];
+
+ for (let i = 0; i < quantity; i++) {
+ roles.push(iam.createRole(`${name}-role-${i}`));
+ }
+
+ return roles;
+}
+
+// Creates a collection of IAM instance profiles from the given roles.
+export function createInstanceProfiles(name: string, roles: aws.iam.Role[]): aws.iam.InstanceProfile[] {
+ const profiles: aws.iam.InstanceProfile[] = [];
+
+ for (let i = 0; i < roles.length; i++) {
+ const role = roles[i];
+ profiles.push(new aws.iam.InstanceProfile(`${name}-instanceProfile-${i}`, {role: role}));
+ }
+
+ return profiles;
+}

aws-ts-eks-migrate-nodegroups/index.ts

@@ -0,0 +1,83 @@
+import * as awsx from "@pulumi/awsx"; import * as eks from "@pulumi/eks";
+import * as k8s from "@pulumi/kubernetes";
+import * as pulumi from "@pulumi/pulumi";
+import * as echoserver from "./echoserver";
+import * as iam from "./iam";
+import * as nginx from "./nginx";
+import * as utils from "./utils";
+
+const projectName = pulumi.getProject();
+
+// Allocate a new VPC with custom settings, and a public & private subnet per AZ.
+const vpc = new awsx.ec2.Vpc(`${projectName}`, {
+ cidrBlock: "172.16.0.0/16",
+ subnets: [{ type: "public" }, { type: "private" }],
+});
+
+// Export VPC ID and Subnets.
+export const vpcId = vpc.id;
+export const allVpcSubnets = vpc.privateSubnetIds.concat(vpc.publicSubnetIds);
+
+// Create 3 IAM Roles and matching InstanceProfiles to use with the nodegroups.
+const roles = iam.createRoles(projectName, 3);
+const instanceProfiles = iam.createInstanceProfiles(projectName, roles);
+
+// Create an EKS cluster.
+const myCluster = new eks.Cluster(`${projectName}`, {
+ version: "1.13",
+ vpcId: vpcId,
+ subnetIds: allVpcSubnets,
+ nodeAssociatePublicIpAddress: false,
+ skipDefaultNodeGroup: true,
+ deployDashboard: false,
+ instanceRoles: roles,
+ enabledClusterLogTypes: ["api", "audit", "authenticator",
+ "controllerManager", "scheduler"],
+});
+export const kubeconfig = myCluster.kubeconfig;
+export const clusterName = myCluster.core.cluster.name;
+
+// Create a Standard node group of t2.medium workers.
+const ngStandard = utils.createNodeGroup(`${projectName}-ng-standard`, {
+ ami: "ami-03a55127c613349a7", // k8s v1.13.7 in us-west-2
+ instanceType: "t2.medium",
+ desiredCapacity: 3,
+ cluster: myCluster,
+ instanceProfile: instanceProfiles[0],
+});
+
+// Create a 2xlarge node group of t3.2xlarge workers with taints on the nodes
+// dedicated for the NGINX Ingress Controller.
+const ng2xlarge = utils.createNodeGroup(`${projectName}-ng-2xlarge`, {
+ ami: "ami-0355c210cb3f58aa2", // k8s v1.12.7 in us-west-2
+ instanceType: "t3.2xlarge",
+ desiredCapacity: 3,
+ cluster: myCluster,
+ instanceProfile: instanceProfiles[1],
+ taints: {"nginx": { value: "true", effect: "NoSchedule"}},
+});
+
+// Create a Namespace for NGINX Ingress Controller and the echoserver workload.
+const namespace = new k8s.core.v1.Namespace("apps", undefined, { provider: myCluster.provider });
+export const namespaceName = namespace.metadata.name;
+
+// Deploy the NGINX Ingress Controller on the specified node group.
+const image: string = "quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.25.0";
+const ingressClass: string = "my-nginx-class";
+const nginxService = nginx.create("nginx-ing-cntlr", {
+ image: image,
+ replicas: 3,
+ namespace: namespaceName,
+ ingressClass: ingressClass,
+ provider: myCluster.provider,
+ nodeSelectorTermValues: ["t3.2xlarge"],
+});
+export const nginxServiceUrl = nginxService.status.loadBalancer.ingress[0].hostname;
+
+// Deploy the echoserver Workload on the Standard node group.
+const echoserverDeployment = echoserver.create("echoserver", {
+ replicas: 3,
+ namespace: namespaceName,
+ ingressClass: ingressClass,
+ provider: myCluster.provider,
+});