Skip to content

Commit

Permalink
chore: unify toml packages being used
Browse files Browse the repository at this point in the history
Drop BurntSushi one, and use /v2 of pelletier package.
There is indirect use of v1 which should hopefully go away once we move
away from sonobouy.

Signed-off-by: Andrey Smirnov <[email protected]>
  • Loading branch information
smira committed May 29, 2024
1 parent 4feb94c commit 0359c85
Show file tree
Hide file tree
Showing 8 changed files with 68 additions and 51 deletions.
5 changes: 2 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ require (
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.2
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azcertificates v1.1.0
github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.1.0
github.com/BurntSushi/toml v1.3.2
github.com/alexflint/go-filemutex v1.3.0
github.com/aws/aws-sdk-go-v2/config v1.27.11
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1
Expand Down Expand Up @@ -113,7 +112,7 @@ require (
github.com/opencontainers/image-spec v1.1.0
github.com/opencontainers/runtime-spec v1.2.0
github.com/packethost/packngo v0.31.0
github.com/pelletier/go-toml v1.9.5
github.com/pelletier/go-toml/v2 v2.2.2
github.com/pin/tftp/v3 v3.1.0
github.com/pmorjan/kmod v1.1.1
github.com/prometheus/procfs v0.14.0
Expand Down Expand Up @@ -302,7 +301,7 @@ require (
github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect
github.com/opencontainers/selinux v1.11.0 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
github.com/pierrec/lz4/v4 v4.1.15 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
Expand Down
2 changes: 0 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,6 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mx
github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8=
github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=
github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
Expand Down
4 changes: 2 additions & 2 deletions internal/pkg/containers/cri/containerd/containerd.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
"bytes"
"path/filepath"

"github.com/BurntSushi/toml"
"github.com/pelletier/go-toml/v2"

"github.com/siderolabs/talos/pkg/machinery/config/config"
"github.com/siderolabs/talos/pkg/machinery/constants"
Expand Down Expand Up @@ -39,7 +39,7 @@ func GenerateCRIConfig(r config.Registries) ([]byte, error) {

var buf bytes.Buffer

if err := toml.NewEncoder(&buf).Encode(&ctrdCfg); err != nil {
if err := toml.NewEncoder(&buf).SetIndentTables(true).Encode(&ctrdCfg); err != nil {
return nil, err
}

Expand Down
18 changes: 11 additions & 7 deletions internal/pkg/containers/cri/containerd/hosts.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import (
"strings"

"github.com/containerd/containerd/v2/core/remotes/docker"
"github.com/pelletier/go-toml"
"github.com/pelletier/go-toml/v2"

"github.com/siderolabs/talos/pkg/machinery/config/config"
)
Expand Down Expand Up @@ -128,11 +128,14 @@ func GenerateHosts(cfg config.Registries, basePath string) (*HostsConfig, error)

configureEndpoint(u.Host, directoryName, hostsToml.HostConfigs[endpoint], directory)

tomlBytes, err := toml.Marshal(hostsToml)
if err != nil {
var tomlBuf bytes.Buffer

if err := toml.NewEncoder(&tomlBuf).SetIndentTables(true).Encode(hostsToml); err != nil {
return nil, err
}

tomlBytes := tomlBuf.Bytes()

// this is an ugly hack, and neither TOML format nor go-toml library make it easier
//
// we need to marshal each endpoint in the order they are specified in the config, but go-toml defines
Expand All @@ -146,7 +149,7 @@ func GenerateHosts(cfg config.Registries, basePath string) (*HostsConfig, error)
// [host."bar.foo"]
//
// but this is invalid TOML, as `[host]' is repeated, so we do an ugly hack and remove it below
const hostPrefix = "\n[host]\n"
const hostPrefix = "[host]\n"

if i > 0 {
if bytes.HasPrefix(tomlBytes, []byte(hostPrefix)) {
Expand Down Expand Up @@ -206,16 +209,17 @@ func GenerateHosts(cfg config.Registries, basePath string) (*HostsConfig, error)

configureEndpoint(hostname, directoryName, hostsToml.HostConfigs[defaultHost], directory)

marshaled, err := toml.Marshal(hostsToml)
if err != nil {
var tomlBuf bytes.Buffer

if err = toml.NewEncoder(&tomlBuf).SetIndentTables(true).Encode(hostsToml); err != nil {
return nil, err
}

directory.Files = append(directory.Files,
&HostsFile{
Name: "hosts.toml",
Mode: 0o600,
Contents: marshaled,
Contents: tomlBuf.Bytes(),
},
)

Expand Down
22 changes: 11 additions & 11 deletions internal/pkg/containers/cri/containerd/hosts_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ func TestGenerateHostsWithTLS(t *testing.T) {
{
Name: "hosts.toml",
Mode: 0o600,
Contents: []byte("\n[host]\n\n [host.\"https://registry-1.docker.io\"]\n capabilities = [\"pull\", \"resolve\"]\n\n [host.\"https://registry-2.docker.io\"]\n capabilities = [\"pull\", \"resolve\"]\n skip_verify = true\n"), //nolint:lll
Contents: []byte("[host]\n [host.'https://registry-1.docker.io']\n capabilities = ['pull', 'resolve']\n [host.'https://registry-2.docker.io']\n capabilities = ['pull', 'resolve']\n skip_verify = true\n"), //nolint:lll
},
},
},
Expand All @@ -83,7 +83,7 @@ func TestGenerateHostsWithTLS(t *testing.T) {
{
Name: "hosts.toml",
Mode: 0o600,
Contents: []byte("\n[host]\n\n [host.\"https://some.host:123\"]\n ca = \"/etc/cri/conf.d/hosts/some.host_123_/some.host:123-ca.crt\"\n client = [[\"/etc/cri/conf.d/hosts/some.host_123_/some.host:123-client.crt\", \"/etc/cri/conf.d/hosts/some.host_123_/some.host:123-client.key\"]]\n skip_verify = true\n"), //nolint:lll
Contents: []byte("[host]\n [host.'https://some.host:123']\n ca = '/etc/cri/conf.d/hosts/some.host_123_/some.host:123-ca.crt'\n client = [['/etc/cri/conf.d/hosts/some.host_123_/some.host:123-client.crt', '/etc/cri/conf.d/hosts/some.host_123_/some.host:123-client.key']]\n skip_verify = true\n"), //nolint:lll
},
},
},
Expand All @@ -92,7 +92,7 @@ func TestGenerateHostsWithTLS(t *testing.T) {
{
Name: "hosts.toml",
Mode: 0o600,
Contents: []byte("\n[host]\n\n [host.\"https://registry-2.docker.io\"]\n skip_verify = true\n"),
Contents: []byte("[host]\n [host.'https://registry-2.docker.io']\n skip_verify = true\n"),
},
},
},
Expand Down Expand Up @@ -132,7 +132,7 @@ func TestGenerateHostsWithoutTLS(t *testing.T) {
{
Name: "hosts.toml",
Mode: 0o600,
Contents: []byte("\n[host]\n\n [host.\"https://registry-1.docker.io\"]\n capabilities = [\"pull\", \"resolve\"]\n\n [host.\"https://registry-2.docker.io\"]\n capabilities = [\"pull\", \"resolve\"]\n"), //nolint:lll
Contents: []byte("[host]\n [host.'https://registry-1.docker.io']\n capabilities = ['pull', 'resolve']\n [host.'https://registry-2.docker.io']\n capabilities = ['pull', 'resolve']\n"), //nolint:lll
},
},
},
Expand All @@ -141,7 +141,7 @@ func TestGenerateHostsWithoutTLS(t *testing.T) {
{
Name: "hosts.toml",
Mode: 0o600,
Contents: []byte("\n[host]\n\n [host.\"https://some.host:123\"]\n"),
Contents: []byte("[host]\n [host.'https://some.host:123']\n"),
},
},
},
Expand All @@ -150,7 +150,7 @@ func TestGenerateHostsWithoutTLS(t *testing.T) {
{
Name: "hosts.toml",
Mode: 0o600,
Contents: []byte("\n[host]\n\n [host.\"https://my-registry\"]\n capabilities = [\"pull\", \"resolve\"]\n"),
Contents: []byte("[host]\n [host.'https://my-registry']\n capabilities = ['pull', 'resolve']\n"),
},
},
},
Expand Down Expand Up @@ -205,7 +205,7 @@ func TestGenerateHostsTLSWildcard(t *testing.T) {
{
Name: "hosts.toml",
Mode: 0o600,
Contents: []byte("\n[host]\n\n [host.\"https://my-registry1\"]\n ca = \"/etc/cri/conf.d/hosts/_default/my-registry1-ca.crt\"\n capabilities = [\"pull\", \"resolve\"]\n\n [host.\"https://my-registry2\"]\n capabilities = [\"pull\", \"resolve\"]\n"), //nolint:lll
Contents: []byte("[host]\n [host.'https://my-registry1']\n capabilities = ['pull', 'resolve']\n ca = '/etc/cri/conf.d/hosts/_default/my-registry1-ca.crt'\n [host.'https://my-registry2']\n capabilities = ['pull', 'resolve']\n"), //nolint:lll
},
},
},
Expand All @@ -219,7 +219,7 @@ func TestGenerateHostsTLSWildcard(t *testing.T) {
{
Name: "hosts.toml",
Mode: 0o600,
Contents: []byte("\n[host]\n\n [host.\"https://my-registry1\"]\n ca = \"/etc/cri/conf.d/hosts/my-registry1/my-registry1-ca.crt\"\n"),
Contents: []byte("[host]\n [host.'https://my-registry1']\n ca = '/etc/cri/conf.d/hosts/my-registry1/my-registry1-ca.crt'\n"),
},
},
},
Expand Down Expand Up @@ -269,7 +269,7 @@ func TestGenerateHostsWithHarbor(t *testing.T) {
{
Name: "hosts.toml",
Mode: 0o600,
Contents: []byte("\n[host]\n\n [host.\"https://harbor/v2/mirrors/proxy.docker.io\"]\n capabilities = [\"pull\", \"resolve\"]\n override_path = true\n skip_verify = true\n"),
Contents: []byte("[host]\n [host.'https://harbor/v2/mirrors/proxy.docker.io']\n capabilities = ['pull', 'resolve']\n override_path = true\n skip_verify = true\n"),
},
},
},
Expand All @@ -278,7 +278,7 @@ func TestGenerateHostsWithHarbor(t *testing.T) {
{
Name: "hosts.toml",
Mode: 0o600,
Contents: []byte("\n[host]\n\n [host.\"https://harbor/v2/mirrors/proxy.ghcr.io\"]\n capabilities = [\"pull\", \"resolve\"]\n override_path = true\n skip_verify = true\n"),
Contents: []byte("[host]\n [host.'https://harbor/v2/mirrors/proxy.ghcr.io']\n capabilities = ['pull', 'resolve']\n override_path = true\n skip_verify = true\n"),
},
},
},
Expand All @@ -287,7 +287,7 @@ func TestGenerateHostsWithHarbor(t *testing.T) {
{
Name: "hosts.toml",
Mode: 0o600,
Contents: []byte("\n[host]\n\n [host.\"https://harbor\"]\n skip_verify = true\n"),
Contents: []byte("[host]\n [host.'https://harbor']\n skip_verify = true\n"),
},
},
},
Expand Down
21 changes: 11 additions & 10 deletions internal/pkg/containers/cri/containerd/testdata/cri.toml
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
[plugins]
[plugins."io.containerd.grpc.v1.cri"]
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/etc/cri/conf.d/hosts"
[plugins."io.containerd.grpc.v1.cri".registry.configs]
[plugins."io.containerd.grpc.v1.cri".registry.configs."some.host:123"]
[plugins."io.containerd.grpc.v1.cri".registry.configs."some.host:123".auth]
username = "root"
password = "secret"
auth = "auth"
identitytoken = "token"
[plugins.'io.containerd.grpc.v1.cri']
[plugins.'io.containerd.grpc.v1.cri'.registry]
config_path = '/etc/cri/conf.d/hosts'

[plugins.'io.containerd.grpc.v1.cri'.registry.configs]
[plugins.'io.containerd.grpc.v1.cri'.registry.configs.'some.host:123']
[plugins.'io.containerd.grpc.v1.cri'.registry.configs.'some.host:123'.auth]
username = 'root'
password = 'secret'
auth = 'auth'
identitytoken = 'token'
24 changes: 18 additions & 6 deletions internal/pkg/toml/merge.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,24 +7,36 @@ package toml
import (
"bytes"
"fmt"
"os"

"github.com/BurntSushi/toml"
"github.com/pelletier/go-toml/v2"

"github.com/siderolabs/talos/pkg/machinery/config/merge"
)

func tomlDecodeFile(path string, dest any) error {
f, err := os.Open(path)
if err != nil {
return err
}

defer f.Close() //nolint:errcheck

return toml.NewDecoder(f).Decode(dest)
}

// Merge several TOML documents in files into one.
//
// Merge process relies on generic map[string]interface{} merge which might not always be correct.
// Merge process relies on generic map[string]any merge which might not always be correct.
func Merge(parts []string) ([]byte, error) {
merged := map[string]interface{}{}
merged := map[string]any{}

var header []byte

for _, part := range parts {
partial := map[string]interface{}{}
partial := map[string]any{}

if _, err := toml.DecodeFile(part, &partial); err != nil {
if err := tomlDecodeFile(part, &partial); err != nil {
return nil, fmt.Errorf("error decoding %q: %w", part, err)
}

Expand All @@ -40,7 +52,7 @@ func Merge(parts []string) ([]byte, error) {
_, _ = out.Write(header)
_ = out.WriteByte('\n')

if err := toml.NewEncoder(&out).Encode(merged); err != nil {
if err := toml.NewEncoder(&out).SetIndentTables(true).Encode(merged); err != nil {
return nil, fmt.Errorf("error encoding merged config: %w", err)
}

Expand Down
23 changes: 13 additions & 10 deletions internal/pkg/toml/testdata/expected.toml
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,19 @@
version = 2

[metrics]
address = "0.0.0.0:11234"
address = '0.0.0.0:11234'

[plugins]
[plugins."io.containerd.grpc.v1.cri"]
sandbox_image = "registry.k8s.io/pause:3.8"
[plugins."io.containerd.grpc.v1.cri".containerd]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
[plugins.'io.containerd.grpc.v1.cri']
sandbox_image = 'registry.k8s.io/pause:3.8'

[plugins.'io.containerd.grpc.v1.cri'.containerd]
[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes]
[plugins.'io.containerd.grpc.v1.cri'.containerd.runtimes.runc]
discard_unpacked_layers = true
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/etc/cri/conf.d/hosts"
[plugins."io.containerd.grpc.v1.cri".registry.configs]
runtime_type = 'io.containerd.runc.v2'

[plugins.'io.containerd.grpc.v1.cri'.registry]
config_path = '/etc/cri/conf.d/hosts'

[plugins.'io.containerd.grpc.v1.cri'.registry.configs]

0 comments on commit 0359c85

Please sign in to comment.