Merge pull request hashicorp#466 from hashicorp/unused-backport

Backport unused key tracking from main to the Vault fork branch

.circleci/config.yml

@@ -0,0 +1,180 @@
+version: 2.1
+
+orbs:
+ win: circleci/[email protected]
+
+references:
+ environment: &ENVIRONMENT
+ GOMAXPROCS: 4
+ GO111MODULE: "on"
+ GOPROXY: https://proxy.golang.org/
+ TEST_RESULTS_DIR: &TEST_RESULTS_DIR /tmp/test-results
+ WIN_TEST_RESULTS: &WIN_TEST_RESULTS c:\Users\circleci\AppData\Local\Temp\test-results
+
+commands:
+ git-verify:
+ steps:
+ - run: 
+ name: "Verify no code was generated"
+ command: |
+ if [[ -z $(git status --porcelain) ]]; then
+ echo "Git directory is clean."
+ else
+ echo "Git is dirty. Run `make fmt` and `make generate` locally and commit any formatting fixes or generated code."
+ git status --porcelain
+ exit 1
+ fi
+
+ run-gotests:
+ parameters:
+ cmd:
+ type: string
+ platform:
+ type: string
+ steps:
+ - run:
+ name: "Run go tests"
+ command: |
+ PACKAGE_NAMES=$(go list ./... | circleci tests split --split-by=timings --timings-type=classname)
+ echo "Running $(echo $PACKAGE_NAMES | wc -w) packages"
+ echo $PACKAGE_NAMES
+ << parameters.cmd >> --format=short-verbose --junitfile $TEST_RESULTS_DIR/hcl2/gotestsum-report.xml -- -p 2 -cover -coverprofile=<< parameters.platform >>_cov_$CIRCLE_NODE_INDEX.part $PACKAGE_NAMES
+
+jobs:
+ go-checks:
+ docker:
+ - image: circleci/golang:<< parameters.go-version >>
+ environment: 
+ <<: *ENVIRONMENT
+ parameters:
+ go-version:
+ type: string
+ steps:
+ - checkout
+ - run: go mod verify
+ - run: make fmt
+ - git-verify
+
+ linux-tests:
+ docker:
+ - image: circleci/golang:<< parameters.go-version >>
+ environment: 
+ <<: *ENVIRONMENT
+ parameters:
+ go-version:
+ type: string
+ parallelism: 4
+ steps:
+ - checkout
+ - attach_workspace:
+ at: .
+ - run: mkdir -p $TEST_RESULTS_DIR/hcl2
+ - run-gotests:
+ cmd: "gotestsum"
+ platform: "linux"
+
+ # save coverage report parts
+ - persist_to_workspace:
+ root: .
+ paths:
+ - linux_cov_*.part
+
+ - store_test_results:
+ path: *TEST_RESULTS_DIR
+ - store_artifacts:
+ path: *TEST_RESULTS_DIR
+
+ win-tests:
+ executor: 
+ name: win/default
+ shell: bash --login -eo pipefail
+ environment:
+ <<: *ENVIRONMENT
+ working_directory: c:\gopath\src\github.com\hashicorp\hcl
+ parameters:
+ go-version:
+ type: string
+ gotestsum-version:
+ type: string
+ steps: 
+ - add_ssh_keys:
+ fingerprints:
+ - "3f:fc:7c:7b:7f:45:55:70:d0:7a:6b:26:7f:0d:50:e9"
+ - run: git config --global core.autocrlf false
+ - checkout
+ - attach_workspace:
+ at: .
+ # - git-verify
+ - run: 
+ name: Setup (remove pre-installed golang version)
+ command: |
+ rm -rf "c:\Go"
+ mkdir -p $TEST_RESULTS_DIR/hcl2
+ - restore_cache:
+ keys:
+ - win-golang-<< parameters.go-version >>-cache-v1
+ - win-gomod-cache-{{ checksum "go.mod" }}-v1
+
+ - run: 
+ name: Install go version << parameters.go-version >>
+ command: | 
+ if [ ! -d "c:\go" ]; then
+ echo "Cache not found, installing new version of go"
+ curl --fail --location https://dl.google.com/go/go<< parameters.go-version >>.windows-amd64.zip --output go.zip
+ unzip go.zip -d "/c"
+ fi
+ - run: 
+ name: Go mod download
+ command: |
+ go mod verify
+ go mod download
+ 
+ - save_cache:
+ key: win-golang-<< parameters.go-version >>-cache-v1
+ paths:
+ - /go
+
+ - save_cache:
+ key: win-gomod-cache-{{ checksum "go.mod" }}-v1
+ paths:
+ - c:\Windows\system32\config\systemprofile\go\pkg\mod
+
+ - run:
+ name: Install gotestsum
+ command: |
+ curl --fail --location https://github.com/gotestyourself/gotestsum/releases/download/v<< parameters.gotestsum-version >>/gotestsum_<< parameters.gotestsum-version >>_windows_amd64.tar.gz --output gotestsum.tar.gz
+ tar -xvzf gotestsum.tar.gz
+ - run-gotests:
+ cmd: "./gotestsum.exe"
+ platform: "win"
+
+ # save coverage report parts
+ - persist_to_workspace:
+ root: .
+ paths:
+ - win_cov_*.part
+
+ - store_test_results:
+ path: *WIN_TEST_RESULTS
+ - store_artifacts:
+ path: *WIN_TEST_RESULTS
+
+workflows:
+ hcl2:
+ jobs:
+ # - go-checks:
+ # matrix:
+ # parameters:
+ # go-version: ["1.14"]
+ # name: go-checks-<< matrix.go-version >>
+ - linux-tests:
+ matrix:
+ parameters:
+ go-version: ["1.14"]
+ name: linux-test-go-<< matrix.go-version >>
+ - win-tests:
+ matrix:
+ parameters:
+ go-version: ["1.12"]
+ gotestsum-version: ["0.4.1"]
+ name: win-test-go-<< matrix.go-version >>

decoder.go

@@ -505,7 +505,7 @@ func expandObject(node ast.Node, result reflect.Value) ast.Node {
  // we need to un-flatten the ast enough to decode
  newNode := &ast.ObjectItem{
  Keys: []*ast.ObjectKey{
- &ast.ObjectKey{
+ {
  Token: keyToken,
  },
  },
@@ -628,6 +628,18 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value)
  decodedFields := make([]string, 0, len(fields))
  decodedFieldsVal := make([]reflect.Value, 0)
  unusedKeysVal := make([]reflect.Value, 0)
+
+ // fill unusedNodeKeys with keys from the AST
+ // a slice because we have to do equals case fold to match Filter
+ unusedNodeKeys := make(map[string][]token.Pos, 0)
+ for _, item := range list.Items {
+ for _, k := range item.Keys {
+ fn := k.Token.Value().(string)
+ sl := unusedNodeKeys[fn]
+ unusedNodeKeys[fn] = append(sl, k.Token.Pos)
+ }
+ }
+
  for _, f := range fields {
  field, fieldValue := f.field, f.val
  if !fieldValue.IsValid() {
@@ -661,7 +673,7 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value)
 
  fieldValue.SetString(item.Keys[0].Token.Value().(string))
  continue
- case "unusedKeys":
+ case "unusedKeyPositions":
  unusedKeysVal = append(unusedKeysVal, fieldValue)
  continue
  }
@@ -682,8 +694,9 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value)
  continue
  }
 
- // Track the used key
+ // Track the used keys
  usedKeys[fieldName] = struct{}{}
+ unusedNodeKeys = removeCaseFold(unusedNodeKeys, fieldName)
 
  // Create the field name and decode. We range over the elements
  // because we actually want the value.
@@ -716,6 +729,13 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value)
  }
  }
 
+ if len(unusedNodeKeys) > 0 {
+ // like decodedFields, populated the unusedKeys field(s)
+ for _, v := range unusedKeysVal {
+ v.Set(reflect.ValueOf(unusedNodeKeys))
+ }
+ }
+
  return nil
 }
 
@@ -727,3 +747,17 @@ func findNodeType() reflect.Type {
  value := reflect.ValueOf(nodeContainer).FieldByName("Node")
  return value.Type()
 }
+
+func removeCaseFold(xs map[string][]token.Pos, y string) map[string][]token.Pos {
+ var toDel []string
+
+ for i := range xs {
+ if strings.EqualFold(i, y) {
+ toDel = append(toDel, i)
+ }
+ }
+ for _, i := range toDel {
+ delete(xs, i)
+ }
+ return xs
+}

decoder_test.go

@@ -1,6 +1,7 @@
 package hcl
 
 import (
+ "github.com/hashicorp/hcl/hcl/token"
  "io/ioutil"
  "path/filepath"
  "reflect"
@@ -786,6 +787,52 @@ func TestDecode_structureMapInvalid(t *testing.T) {
  }
 }
 
+func TestDecode_structureMapExtraKeys(t *testing.T) {
+ type hclVariable struct {
+ A int
+ B int
+ Found []string `hcl:",decodedFields"`
+ Extra map[string][]token.Pos `hcl:",unusedKeyPositions"`
+ }
+
+ q := hclVariable{
+ A: 1,
+ B: 2,
+ Found: []string{"A", "B"},
+ Extra: map[string][]token.Pos {
+ "extra1": {{
+ Line: 3,
+ Column: 1,
+ Offset: 12,
+ }},
+ "extra2": {{
+ Line: 4,
+ Column: 1,
+ Offset: 23,
+ }},
+ },
+ }
+
+ var p hclVariable
+ ast, _ := Parse(testReadFile(t, "structure_map_extra_keys.hcl"))
+ DecodeObject(&p, ast)
+ if !reflect.DeepEqual(p, q) {
+ t.Fatal("not equal")
+ }
+
+ p.Extra = map[string][]token.Pos{
+ "extra1": {{}},
+ "extra2": {{}},
+ }
+
+ var j hclVariable
+ ast, _ = Parse(testReadFile(t, "structure_map_extra_keys.json"))
+ DecodeObject(&j, ast)
+ if !reflect.DeepEqual(p, j) {
+ t.Fatal("not equal")
+ }
+}
+
 func TestDecode_interfaceNonPointer(t *testing.T) {
  var value interface{}
  err := Decode(value, testReadFile(t, "basic_int_string.hcl"))

go.mod

@@ -1,3 +1,5 @@
 module github.com/hashicorp/hcl
 
+go 1.15
+
 require github.com/davecgh/go-spew v1.1.1

hcl/parser/parser_test.go

@@ -240,7 +240,7 @@ func TestListType_lineComment(t *testing.T) {
  comment := l.comment[i]
 
  if (lt.LineComment == nil) != (comment == "") {
- t.Fatalf("bad: %s", lt)
+ t.Fatalf("bad: %s", lt.Token.Value())
  }
 
  if comment == "" {

test-fixtures/structure_map_extra_keys.hcl

@@ -0,0 +1,4 @@
+a = 1
+b = 2
+extra1 = 3
+extra2 = 4

test-fixtures/structure_map_extra_keys.json

@@ -0,0 +1,6 @@
+{
+ "a": 1,
+ "b": 2,
+ "extra1": 3,
+ "extra2": 4
+}