specsuite: Start of the harness for the specification test suite

cmd/hclspecsuite/README.md

@@ -0,0 +1,4 @@
+# `hclspecsuite`
+
+`hclspecsuite` is the test harness for
+[the HCL specification test suite](../../specsuite/README.md).

cmd/hclspecsuite/log.go

@@ -0,0 +1,3 @@
+package main
+
+type LogCallback func(testName string, testFile *TestFile)

cmd/hclspecsuite/main.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+ "fmt"
+ "os"
+ "os/exec"
+
+ "github.com/hashicorp/hcl2/hclparse"
+
+ "github.com/hashicorp/hcl2/hcl"
+ "golang.org/x/crypto/ssh/terminal"
+)
+
+func main() {
+ os.Exit(realMain(os.Args[1:]))
+}
+
+func realMain(args []string) int {
+ if len(args) != 2 {
+ fmt.Fprintf(os.Stderr, "Usage: hclspecsuite <tests-dir> <hcldec-file>\n")
+ return 2
+ }
+
+ testsDir := args[0]
+ hcldecPath := args[1]
+
+ hcldecPath, err := exec.LookPath(hcldecPath)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "%s\n", err)
+ return 2
+ }
+
+ parser := hclparse.NewParser()
+
+ runner := &Runner{
+ parser: parser,
+ hcldecPath: hcldecPath,
+ baseDir: testsDir,
+ log: func(name string, file *TestFile) {
+ fmt.Printf("- %s\n", name)
+ },
+ }
+ diags := runner.Run()
+
+ if len(diags) != 0 {
+ os.Stderr.WriteString("\n")
+ color := terminal.IsTerminal(int(os.Stderr.Fd()))
+ w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
+ if err != nil {
+ w = 80
+ }
+ diagWr := hcl.NewDiagnosticTextWriter(os.Stderr, parser.Files(), uint(w), color)
+ diagWr.WriteDiagnostics(diags)
+ return 2
+ }
+
+ return 0
+}

cmd/hclspecsuite/runner.go

@@ -0,0 +1,164 @@
+package main
+
+import (
+ "bytes"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "sort"
+ "strings"
+
+ "github.com/hashicorp/hcl2/hcl"
+ "github.com/hashicorp/hcl2/hclparse"
+ "github.com/zclconf/go-cty/cty"
+)
+
+type Runner struct {
+ parser *hclparse.Parser
+ hcldecPath string
+ baseDir string
+ log LogCallback
+}
+
+func (r *Runner) Run() hcl.Diagnostics {
+ return r.runDir(r.baseDir)
+}
+
+func (r *Runner) runDir(dir string) hcl.Diagnostics {
+ var diags hcl.Diagnostics
+
+ infos, err := ioutil.ReadDir(dir)
+ if err != nil {
+ diags = append(diags, &hcl.Diagnostic{
+ Severity: hcl.DiagError,
+ Summary: "Failed to read test directory",
+ Detail: fmt.Sprintf("The directory %q could not be opened: %s.", dir, err),
+ })
+ return diags
+ }
+
+ var tests []string
+ var subDirs []string
+ for _, info := range infos {
+ name := info.Name()
+ if strings.HasPrefix(name, ".") {
+ continue
+ }
+
+ if info.IsDir() {
+ subDirs = append(subDirs, name)
+ }
+ if strings.HasSuffix(name, ".t") {
+ tests = append(tests, name)
+ }
+ }
+ sort.Strings(tests)
+ sort.Strings(subDirs)
+
+ for _, filename := range tests {
+ filename = filepath.Join(r.baseDir, filename)
+ testDiags := r.runTest(filename)
+ diags = append(diags, testDiags...)
+ }
+
+ for _, dirName := range subDirs {
+ dir := filepath.Join(r.baseDir, dirName)
+ dirDiags := r.runDir(dir)
+ diags = append(diags, dirDiags...)
+ }
+
+ return diags
+}
+
+func (r *Runner) runTest(filename string) hcl.Diagnostics {
+ prettyName := r.prettyTestName(filename)
+ tf, diags := r.LoadTestFile(filename)
+ if diags.HasErrors() {
+ // We'll still log, so it's clearer which test the diagnostics belong to.
+ if r.log != nil {
+ r.log(prettyName, nil)
+ }
+ return diags
+ }
+
+ if r.log != nil {
+ r.log(prettyName, tf)
+ }
+
+ basePath := filename[:len(filename)-2]
+ specFilename := basePath + ".hcldec"
+ nativeFilename := basePath + ".hcl"
+ //jsonFilename := basePath + ".hcl.json"
+
+ if _, err := os.Stat(specFilename); err != nil {
+ diags = append(diags, &hcl.Diagnostic{
+ Severity: hcl.DiagError,
+ Summary: "Missing .hcldec file",
+ Detail: fmt.Sprintf("No specification file for test %s: %s.", prettyName, err),
+ })
+ return diags
+ }
+
+ if _, err := os.Stat(nativeFilename); err == nil {
+
+ }
+
+ return diags
+}
+
+func (r *Runner) hcldecTransform(specFile, inputFile string) (cty.Value, hcl.Diagnostics) {
+ var diags hcl.Diagnostics
+ var outBuffer bytes.Buffer
+ var errBuffer bytes.Buffer
+
+ cmd := &exec.Cmd{
+ Path: r.hcldecPath,
+ Args: []string{
+ "--spec=" + specFile,
+ "--diags=json",
+ inputFile,
+ },
+ Stdout: &outBuffer,
+ Stderr: &errBuffer,
+ }
+ err := cmd.Run()
+ if _, isExit := err.(*exec.ExitError); !isExit {
+ diags = append(diags, &hcl.Diagnostic{
+ Severity: hcl.DiagError,
+ Summary: "Failed to run hcldec",
+ Detail: fmt.Sprintf("Sub-program hcldec failed to start: %s.", err),
+ })
+ return cty.DynamicVal, diags
+ }
+
+ if err != nil {
+ // If we exited unsuccessfully then we'll expect diagnostics on stderr
+ // TODO: implement that
+ } else {
+ // Otherwise, we expect a JSON result value on stdout
+ // TODO: implement that
+ }
+
+ return cty.DynamicVal, diags
+}
+
+func (r *Runner) prettyDirName(dir string) string {
+ rel, err := filepath.Rel(r.baseDir, dir)
+ if err != nil {
+ return filepath.ToSlash(dir)
+ }
+ return filepath.ToSlash(rel)
+}
+
+func (r *Runner) prettyTestName(filename string) string {
+ dir := filepath.Dir(filename)
+ dirName := r.prettyDirName(dir)
+ filename = filepath.Base(filename)
+ testName := filename[:len(filename)-2]
+ if dirName == "." {
+ return testName
+ }
+ return fmt.Sprintf("%s/%s", dirName, testName)
+}

cmd/hclspecsuite/test_file.go

@@ -0,0 +1,78 @@
+package main
+
+import (
+ "fmt"
+
+ "github.com/hashicorp/hcl2/ext/typeexpr"
+ "github.com/hashicorp/hcl2/hcl"
+ "github.com/zclconf/go-cty/cty"
+ "github.com/zclconf/go-cty/cty/convert"
+)
+
+type TestFile struct {
+ Result cty.Value
+ ResultType cty.Type
+
+ Traversals []hcl.Traversal
+}
+
+func (r *Runner) LoadTestFile(filename string) (*TestFile, hcl.Diagnostics) {
+ f, diags := r.parser.ParseHCLFile(filename)
+ if diags.HasErrors() {
+ return nil, diags
+ }
+
+ content, moreDiags := f.Body.Content(testFileSchema)
+ diags = append(diags, moreDiags...)
+ if moreDiags.HasErrors() {
+ return nil, diags
+ }
+
+ ret := &TestFile{
+ ResultType: cty.DynamicPseudoType,
+ }
+
+ if typeAttr, exists := content.Attributes["result_type"]; exists {
+ ty, moreDiags := typeexpr.TypeConstraint(typeAttr.Expr)
+ diags = append(diags, moreDiags...)
+ if !moreDiags.HasErrors() {
+ ret.ResultType = ty
+ }
+ }
+
+ if resultAttr, exists := content.Attributes["result"]; exists {
+ resultVal, moreDiags := resultAttr.Expr.Value(nil)
+ diags = append(diags, moreDiags...)
+ if !moreDiags.HasErrors() {
+ resultVal, err := convert.Convert(resultVal, ret.ResultType)
+ if err != nil {
+ diags = diags.Append(&hcl.Diagnostic{
+ Severity: hcl.DiagError,
+ Summary: "Invalid result value",
+ Detail: fmt.Sprintf("The result value does not conform to the given result type: %s.", err),
+ Subject: resultAttr.Expr.Range().Ptr(),
+ })
+ } else {
+ ret.Result = resultVal
+ }
+ }
+ }
+
+ return ret, diags
+}
+
+var testFileSchema = &hcl.BodySchema{
+ Attributes: []hcl.AttributeSchema{
+ {
+ Name: "result",
+ },
+ {
+ Name: "result_type",
+ },
+ },
+ Blocks: []hcl.BlockHeaderSchema{
+ {
+ Type: "traversals",
+ },
+ },
+}

specsuite/README.md

@@ -0,0 +1,37 @@
+# HCL Language Test Suite
+
+This directory contains an implementation-agnostic test suite that can be used
+to verify the correct behavior not only of the HCL implementation in _this_
+repository but also of possible other implementations.
+
+The harness for running this suite -- a Go program in this directory -- uses
+the `hcldec` program as a level of abstraction to avoid depending directly on
+the Go implementation. As a result, other HCL implementations must also
+include a version of `hcldec` in order to run this spec.
+
+The tests defined in this suite each correspond to a detail of
+[the HCL spec](../hcl/spec.md). This suite is separate from and not a
+substitute for direct unit tests of a given implementation that would presumably
+also exercise that implementation's own programmatic API.
+
+To run the suite, first build the harness using Go:
+
+```
+go install github.com/hashicorp/hcl2/cmd/hclspecsuite
+```
+
+Then run it, passing it the directory containing the test definitions (the
+"tests" subdirectory of this directory) and the path to the `hcldec` executable
+to use.
+
+For example, if working in the root of this repository and using the `hcldec`
+implementation from here:
+
+```
+go install ./cmd/hcldec
+hclspecsuite ./specsuite/tests $GOPATH/bin/hcldec
+```
+
+For developers working on the Go implementation of HCL from this repository,
+please note that this spec suite is run as part of a normal `go test ./...`
+execution for this whole repository and so does not need to be run separately.

specsuite/tests/empty.hcl.json

@@ -0,0 +1 @@
+{}

specsuite/tests/empty.hcldec

@@ -0,0 +1,3 @@
+literal {
+ value = "ok"
+}

specsuite/tests/empty.t

@@ -0,0 +1,9 @@
+# This test ensures that we can successfully parse an empty file.
+# Since an empty file has no content, the hcldec spec for this test is
+# just a literal value, which we test below.
+
+result = "ok"
+
+traversals {
+ # Explicitly no traversals
+}