Add dedicated fail-restart fixture, fix krane_test fetch_restarted_at…

… method

test/fixtures/downward_api/configmap-data.yml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: hello-cloud-configmap-data
+  labels:
+    name: hello-cloud-configmap-data
+    app: hello-cloud
+data:
+  datapoint1: value1
+  datapoint2: value2

test/fixtures/downward_api/web.yml.erb

@@ -0,0 +1,85 @@
+apiVersion: extensions/v1beta1
+kind: Ingress
+metadata:
+  name: web
+  labels:
+    name: web
+    app: hello-cloud
+spec:
+  rules:
+  - host: hello-cloud.example.com
+    http:
+      paths:
+      - backend:
+          serviceName: web
+          servicePort: 80
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: web
+  labels:
+    name: web
+    app: hello-cloud
+spec:
+  ports:
+  - port: 80
+    name: http
+    targetPort: http
+  selector:
+    name: web
+    app: hello-cloud
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: web
+  annotations:
+    shipit.shopify.io/restart: "true"
+  labels:
+    name: web
+    app: hello-cloud
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: web
+      app: hello-cloud
+  progressDeadlineSeconds: 60
+  template:
+    metadata:
+      labels:
+        name: web
+        app: hello-cloud
+    spec:
+      containers:
+      - name: app
+        image: busybox
+        imagePullPolicy: IfNotPresent
+        command: ["tail", "-f", "/dev/null"]
+        ports:
+        - containerPort: 80
+          name: http
+        env:
+        - name: GITHUB_REV
+          value: "<%= current_sha %>"
+        - name: CONFIG
+          valueFrom:
+            configMapKeyRef:
+              name: hello-cloud-configmap-data
+              key: datapoint1
+        volumeMounts:
+        - name: podinfo
+          mountPath: /etc/podinfo
+      - name: sidecar
+        image: busybox
+        imagePullPolicy: IfNotPresent
+        command: ["tail", "-f", "/dev/null"]
+      volumes:
+      - name: podinfo
+        downwardAPI:
+          items:
+          - path: "annotations"
+            fieldRef:
+              fieldPath: metadata.annotations
+

test/integration/krane_test.rb

@@ -6,16 +6,16 @@ def test_restart_black_box
     assert_deploy_success(
       deploy_fixtures("hello-cloud", subset: ["configmap-data.yml", "web.yml.erb", "redis.yml"], render_erb: true)
     )
-    refute(fetch_restarted_at("web"), "RESTARTED_AT env on fresh deployment")
-    refute(fetch_restarted_at("redis"), "RESTARTED_AT env on fresh deployment")
+    refute(fetch_restarted_at("web"), "restart annotation on fresh deployment")
+    refute(fetch_restarted_at("redis"), "restart annotation on fresh deployment")
 
     out, err, status = krane_black_box("restart", "#{@namespace} #{KubeclientHelper::TEST_CONTEXT} --deployments web")
     assert_empty(out)
     assert_match("Success", err)
     assert_predicate(status, :success?)
 
-    assert(fetch_restarted_at("web"), "no RESTARTED_AT env is present after the restart")
-    refute(fetch_restarted_at("redis"), "RESTARTED_AT env is present")
+    assert(fetch_restarted_at("web"), "no restart annotation is present after the restart")
+    refute(fetch_restarted_at("redis"), "restart annotation is present")
   end
 
   def test_run_success_black_box
@@ -142,8 +142,6 @@ def task_runner_pods
 
   def fetch_restarted_at(deployment_name)
     deployment = apps_v1_kubeclient.get_deployment(deployment_name, @namespace)
-    containers = deployment.spec.template.spec.containers
-    app_container = containers.find { |c| c["name"] == "app" }
-    app_container&.env&.find { |n| n.name == "RESTARTED_AT" }
+    deployment.spec.template.metadata.annotations&.dig(Krane::RestartTask::RESTART_TRIGGER_ANNOTATION)
   end
 end

test/integration/restart_task_test.rb

@@ -188,6 +188,44 @@ def test_restart_not_existing_namespace
       in_order: true)
   end
 
+  def test_restart_failure
+    success = deploy_fixtures("downward_api", subset: ["configmap-data.yml", "web.yml.erb"],
+      render_erb: true) do |fixtures|
+      deployment = fixtures["web.yml.erb"]["Deployment"].first
+      deployment["spec"]["progressDeadlineSeconds"] = 30
+      container = deployment["spec"]["template"]["spec"]["containers"].first
+      container["readinessProbe"] = {
+        "failureThreshold" => 1,
+        "periodSeconds" => 1,
+        "initialDelaySeconds" => 0,
+        "exec" => {
+          "command" => [
+            "/bin/sh",
+            "-c",
+            "test $(cat /etc/podinfo/annotations | grep -s kubectl.kubernetes.io/restartedAt -c) -eq 0",
+          ],
+        },
+      }
+    end
+    assert_deploy_success(success)
+
+    restart = build_restart_task
+    assert_raises(Krane::DeploymentTimeoutError) { restart.perform!(deployments: %w(web)) }
+
+    assert_logs_match_all([
+      "Triggered `Deployment/web` restart",
+      "Deployment/web rollout timed out",
+      "Result: TIMED OUT",
+      "Timed out waiting for 1 resource to restart",
+      "Deployment/web: TIMED OUT",
+      "The following containers have not passed their readiness probes",
+      "app must exit 0 from the following command",
+      "Final status: 2 replicas, 1 updatedReplica, 1 availableReplica, 1 unavailableReplica",
+      "Unhealthy: Readiness probe failed",
+    ],
+      in_order: true)
+  end
+
   def test_restart_successful_with_partial_availability
     result = deploy_fixtures("slow-cloud", subset: %w(web-deploy-1.yml)) do |fixtures|
       web = fixtures["web-deploy-1.yml"]["Deployment"].first