Add 'expect' assertions, support 'step' in expressions

Allow users to add assertions after steps are run using the 'expect'
key, which can be either a string expression or an array of expressions.
We return the error from the first that fails.

Add the results of the step into the context, and allow users to
reference step results using the 'step' context name.

.github/workflows/push.yaml

@@ -50,7 +50,7 @@ jobs:
  -v $(pwd)/examples:/app/examples \
  dctest --results-file /app/examples/results.json examples /app/examples/00-intro.yaml \
  || true
- jq --exit-status '[.pass == 3, .fail == 1] | all' examples/results.json
+ jq --exit-status '[.pass == 4, .fail == 1] | all' examples/results.json
 
  - name: Run Examples with --continue-on-error
  run: |
@@ -61,7 +61,7 @@ jobs:
  -v $(pwd)/examples:/app/examples \
  dctest --continue-on-error --results-file /app/examples/results.json examples /app/examples/00-intro.yaml \
  || true
- jq --exit-status '[.pass == 4, .fail == 3] | all' examples/results.json
+ jq --exit-status '[.pass == 5, .fail == 3] | all' examples/results.json
 
 
  - name: Setup Fixtures

examples/00-intro.yaml

@@ -58,6 +58,18 @@ tests:
  run: |
  [ "" == "${FOO}" ]
 
+ expect-assertions:
+ name: Expect test
+ steps:
+ - exec: node1
+ run: echo -n "2"
+ expect: step.stdout == "2"
+ - exec: node1
+ run: echo -n "3"
+ expect:
+ - step.stdout != "2"
+ - step.stdout == "3"
+
  fail-exit-code:
  name: Failing due to non-zero exit code
  steps:

schema.yaml

@@ -34,6 +34,10 @@ properties:
  properties:
  env: { "$ref": "#/$defs/env" }
  exec: { type: string, expression: InterpolatedText }
+ expect:
+ oneOf:
+ - { type: string, expression: Expression }
+ - { type: array, items: { type: string, expression: Expression } }
  name: { type: string }
  if: { type: string, expression: Expression, default: "success()" }
  index: { type: integer }

src/dctest/core.cljs

@@ -192,6 +192,7 @@ Options:
  interpolate #(expr/interpolate-text step-context %)
  step (-> step
  (update :exec interpolate)
+ (update :expect #(if (string? %) [%] %))
  (update :index #(or % 1))
  (update :run (fn [command]
  (if (string? command)
@@ -204,6 +205,14 @@ Options:
  {:delay-ms interval
  :check-fn #(not (:error %))})
 
+ step-context (assoc step-context :step results)
+ run-expect (fn [expr]
+ (when-not (expr/read-eval step-context expr)
+ {:error {:message (str "Expectation failure: " expr)}}))
+ expect-results (when-not (:error results)
+ (first (keep run-expect (:expect step))))
+ results (merge expect-results results)
+
  outcome (if (:error results) :fail :pass)
  results (merge results
  {:outcome outcome}

src/dctest/expressions.cljs

@@ -13,7 +13,7 @@
 (declare read-ast)
 
 (def supported-contexts
- #{"env" "process"})
+ #{"env" "process" "step"})
 
 (def stdlib
  {

test/dctest/expressions_test.cljs

@@ -174,10 +174,12 @@
 
  ;; Supported contexts
  (let [contexts {:env {"FOO" 1}
- :process {"pid" 123}}]
+ :process {"pid" 123}
+ :step {:code 0 :stdout "hi" :stderr ""}}]
  (are [expected text] (= expected (expr/read-eval contexts text))
- 1 "env.FOO"
- 123 "process.pid"))
+ 1 "env.FOO"
+ 123 "process.pid"
+ "hi" "step.stdout"))
 
  ;; Coerce keywords to strings inside context
  (is (= 123