ingest/ledgerbackend: Add version check for protocol 21 (#5346)

* Remove protocol 20 from integration tests

* Implement a version check for protocol 21 when creating a new captive-core instance.

* Add common functions to get the stellar-core build version and protocol version and update ledgerexporter and captivecorebackend to use these functions.

.github/workflows/horizon.yml

@@ -13,7 +13,7 @@ jobs:
  os: [ubuntu-20.04, ubuntu-22.04]
  go: ["1.21", "1.22"]
  pg: [12, 16]
- protocol-version: [20, 21]
+ protocol-version: [21]
  runs-on: ${{ matrix.os }}
  services:
  postgres:
@@ -36,9 +36,6 @@ jobs:
  PROTOCOL_21_CORE_DEBIAN_PKG_VERSION: 21.0.0-1872.c6f474133.focal
  PROTOCOL_21_CORE_DOCKER_IMG: stellar/stellar-core:21
  PROTOCOL_21_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:21.0.0-rc2-73
- PROTOCOL_20_CORE_DEBIAN_PKG_VERSION: 21.0.0-1872.c6f474133.focal
- PROTOCOL_20_CORE_DOCKER_IMG: stellar/stellar-core:21
- PROTOCOL_20_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:21.0.0-rc2-73
  PGHOST: localhost
  PGPORT: 5432
  PGUSER: postgres
@@ -101,7 +98,7 @@ jobs:
  - name: Calculate the source hash
  id: calculate_source_hash
  run: |
- combined_hash=$(echo "horizon-hash-${{ hashFiles('./horizon') }}-${{ hashFiles('./clients/horizonclient/**') }}-${{ hashFiles('./protocols/horizon/**') }}-${{ hashFiles('./txnbuild/**') }}-${{ hashFiles('./ingest/**') }}-${{ hashFiles('./xdr/**') }}-${{ hashFiles('./services/**') }}-${{ env.PROTOCOL_20_CORE_DOCKER_IMG }}-${{ env.PROTOCOL_19_CORE_DOCKER_IMG }}-${{ env.PREFIX }}" | sha256sum | cut -d ' ' -f 1)
+ combined_hash=$(echo "horizon-hash-${{ hashFiles('./horizon') }}-${{ hashFiles('./clients/horizonclient/**') }}-${{ hashFiles('./protocols/horizon/**') }}-${{ hashFiles('./txnbuild/**') }}-${{ hashFiles('./ingest/**') }}-${{ hashFiles('./xdr/**') }}-${{ hashFiles('./services/**') }}-${{ env.PROTOCOL_21_CORE_DOCKER_IMG }}-${{ env.PREFIX }}" | sha256sum | cut -d ' ' -f 1)
  echo "COMBINED_SOURCE_HASH=$combined_hash" >> "$GITHUB_ENV"
 
  - name: Restore Horizon binary and integration tests source hash to cache

exp/services/ledgerexporter/internal/app.go

@@ -104,7 +104,7 @@ func (a *App) init(ctx context.Context, runtimeSettings RuntimeSettings) error {
  collectors.NewGoCollector(),
  )
 
- if a.config, err = NewConfig(runtimeSettings); err != nil {
+ if a.config, err = NewConfig(runtimeSettings, nil); err != nil {
  return errors.Wrap(err, "Could not load configuration")
  }
  if archive, err = a.config.GenerateHistoryArchive(ctx, logger); err != nil {

exp/services/ledgerexporter/internal/config.go

@@ -5,8 +5,6 @@ import (
  _ "embed"
  "fmt"
  "os"
- "os/exec"
- "strings"
 
  "github.com/stellar/go/historyarchive"
  "github.com/stellar/go/ingest/ledgerbackend"
@@ -73,19 +71,24 @@ type Config struct {
 
  CoreVersion string
  SerializedCaptiveCoreToml []byte
+ CoreBuildVersionFn ledgerbackend.CoreBuildVersionFunc
 }
 
 // This will generate the config based on settings
 //
 // settings - requested settings
 //
 // return - *Config or an error if any range validation failed.
-func NewConfig(settings RuntimeSettings) (*Config, error) {
+func NewConfig(settings RuntimeSettings, getCoreVersionFn ledgerbackend.CoreBuildVersionFunc) (*Config, error) {
  config := &Config{}
 
  config.StartLedger = uint32(settings.StartLedger)
  config.EndLedger = uint32(settings.EndLedger)
  config.Mode = settings.Mode
+ config.CoreBuildVersionFn = ledgerbackend.CoreBuildVersion
+ if getCoreVersionFn != nil {
+ config.CoreBuildVersionFn = getCoreVersionFn
+ }
 
  logger.Infof("Requested export mode of %v with start=%d, end=%d", settings.Mode.Name(), config.StartLedger, config.EndLedger)
 
@@ -194,26 +197,11 @@ func (config *Config) GenerateCaptiveCoreConfig(coreBinFromPath string) (ledgerb
  }, nil
 }
 
-// By default, it points to exec.Command, overridden for testing purpose
-var execCommand = exec.Command
-
-// Executes the "stellar-core version" command and parses its output to extract
-// the core version
-// The output of the "version" command is expected to be a multi-line string where the
-// first line is the core version in format "vX.Y.Z-*".
 func (c *Config) setCoreVersionInfo() (err error) {
- versionCmd := execCommand(c.StellarCoreConfig.StellarCoreBinaryPath, "version")
- versionOutput, err := versionCmd.Output()
+ c.CoreVersion, err = c.CoreBuildVersionFn(c.StellarCoreConfig.StellarCoreBinaryPath)
  if err != nil {
- return fmt.Errorf("failed to execute stellar-core version command: %w", err)
- }
-
- // Split the output into lines
- rows := strings.Split(string(versionOutput), "\n")
- if len(rows) == 0 || len(rows[0]) == 0 {
- return fmt.Errorf("stellar-core version not found")
+ return fmt.Errorf("failed to set stellar-core version: %w", err)
  }
- c.CoreVersion = rows[0]
  logger.Infof("stellar-core version: %s", c.CoreVersion)
  return nil
 }

exp/services/ledgerexporter/internal/config_test.go

@@ -3,8 +3,6 @@ package ledgerexporter
 import (
  "context"
  "fmt"
- "os"
- "os/exec"
  "testing"
 
  "github.com/stellar/go/network"
@@ -13,7 +11,6 @@ import (
  "github.com/stretchr/testify/require"
 
  "github.com/stellar/go/historyarchive"
- "github.com/stellar/go/support/errors"
 )
 
 func TestNewConfig(t *testing.T) {
@@ -23,7 +20,7 @@ func TestNewConfig(t *testing.T) {
  mockArchive.On("GetRootHAS").Return(historyarchive.HistoryArchiveState{CurrentLedger: 5}, nil).Once()
 
  config, err := NewConfig(
- RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/test.toml", Mode: Append})
+ RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/test.toml", Mode: Append}, nil)
 
  require.NoError(t, err)
  err = config.ValidateAndSetLedgerRange(ctx, mockArchive)
@@ -43,7 +40,7 @@ func TestNewConfig(t *testing.T) {
 func TestGenerateHistoryArchiveFromPreconfiguredNetwork(t *testing.T) {
  ctx := context.Background()
  config, err := NewConfig(
- RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/valid_captive_core_preconfigured.toml", Mode: Append})
+ RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/valid_captive_core_preconfigured.toml", Mode: Append}, nil)
  require.NoError(t, err)
 
  _, err = config.GenerateHistoryArchive(ctx, nil)
@@ -53,7 +50,7 @@ func TestGenerateHistoryArchiveFromPreconfiguredNetwork(t *testing.T) {
 func TestGenerateHistoryArchiveFromManulConfiguredNetwork(t *testing.T) {
  ctx := context.Background()
  config, err := NewConfig(
- RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/valid_captive_core_manual.toml", Mode: Append})
+ RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/valid_captive_core_manual.toml", Mode: Append}, nil)
  require.NoError(t, err)
 
  _, err = config.GenerateHistoryArchive(ctx, nil)
@@ -62,28 +59,28 @@ func TestGenerateHistoryArchiveFromManulConfiguredNetwork(t *testing.T) {
 
 func TestNewConfigUserAgent(t *testing.T) {
  config, err := NewConfig(
- RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/useragent.toml"})
+ RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/useragent.toml"}, nil)
  require.NoError(t, err)
  require.Equal(t, config.UserAgent, "useragent_x")
 }
 
 func TestResumeDisabled(t *testing.T) {
  // resumable is only enabled when mode is Append
  config, err := NewConfig(
- RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/test.toml", Mode: ScanFill})
+ RuntimeSettings{StartLedger: 2, EndLedger: 3, ConfigFilePath: "test/test.toml", Mode: ScanFill}, nil)
  require.NoError(t, err)
  require.False(t, config.Resumable())
 }
 
 func TestInvalidConfigFilePath(t *testing.T) {
  _, err := NewConfig(
- RuntimeSettings{ConfigFilePath: "test/notfound.toml"})
+ RuntimeSettings{ConfigFilePath: "test/notfound.toml"}, nil)
  require.ErrorContains(t, err, "config file test/notfound.toml was not found")
 }
 
 func TestNoCaptiveCoreBin(t *testing.T) {
  cfg, err := NewConfig(
- RuntimeSettings{ConfigFilePath: "test/no_core_bin.toml"})
+ RuntimeSettings{ConfigFilePath: "test/no_core_bin.toml"}, nil)
  require.NoError(t, err)
 
  _, err = cfg.GenerateCaptiveCoreConfig("")
@@ -92,31 +89,31 @@ func TestNoCaptiveCoreBin(t *testing.T) {
 
 func TestDefaultCaptiveCoreBin(t *testing.T) {
  cfg, err := NewConfig(
- RuntimeSettings{ConfigFilePath: "test/no_core_bin.toml"})
+ RuntimeSettings{ConfigFilePath: "test/no_core_bin.toml"},
+ func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil })
  require.NoError(t, err)
 
- cmdOut = "v20.2.0-2-g6e73c0a88\n"
  ccConfig, err := cfg.GenerateCaptiveCoreConfig("/test/default/stellar-core")
  require.NoError(t, err)
  require.Equal(t, ccConfig.BinaryPath, "/test/default/stellar-core")
 }
 
 func TestInvalidCaptiveCorePreconfiguredNetwork(t *testing.T) {
  _, err := NewConfig(
- RuntimeSettings{ConfigFilePath: "test/invalid_preconfigured_network.toml"})
+ RuntimeSettings{ConfigFilePath: "test/invalid_preconfigured_network.toml"}, nil)
 
  require.ErrorContains(t, err, "invalid captive core config")
 }
 
 func TestValidCaptiveCorePreconfiguredNetwork(t *testing.T) {
  cfg, err := NewConfig(
- RuntimeSettings{ConfigFilePath: "test/valid_captive_core_preconfigured.toml"})
+ RuntimeSettings{ConfigFilePath: "test/valid_captive_core_preconfigured.toml"},
+ func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil })
  require.NoError(t, err)
 
  require.Equal(t, cfg.StellarCoreConfig.NetworkPassphrase, network.PublicNetworkPassphrase)
  require.Equal(t, cfg.StellarCoreConfig.HistoryArchiveUrls, network.PublicNetworkhistoryArchiveURLs)
 
- cmdOut = "v20.2.0-2-g6e73c0a88\n"
  ccConfig, err := cfg.GenerateCaptiveCoreConfig("")
  require.NoError(t, err)
 
@@ -131,13 +128,13 @@ func TestValidCaptiveCorePreconfiguredNetwork(t *testing.T) {
 
 func TestValidCaptiveCoreManualNetwork(t *testing.T) {
  cfg, err := NewConfig(
- RuntimeSettings{ConfigFilePath: "test/valid_captive_core_manual.toml"})
+ RuntimeSettings{ConfigFilePath: "test/valid_captive_core_manual.toml"},
+ func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil })
  require.NoError(t, err)
  require.Equal(t, cfg.CoreVersion, "")
  require.Equal(t, cfg.StellarCoreConfig.NetworkPassphrase, "test")
  require.Equal(t, cfg.StellarCoreConfig.HistoryArchiveUrls, []string{"http:https://testarchive"})
 
- cmdOut = "v20.2.0-2-g6e73c0a88\n"
  ccConfig, err := cfg.GenerateCaptiveCoreConfig("")
  require.NoError(t, err)
 
@@ -152,12 +149,12 @@ func TestValidCaptiveCoreManualNetwork(t *testing.T) {
 
 func TestValidCaptiveCoreOverridenToml(t *testing.T) {
  cfg, err := NewConfig(
- RuntimeSettings{ConfigFilePath: "test/valid_captive_core_override.toml"})
+ RuntimeSettings{ConfigFilePath: "test/valid_captive_core_override.toml"},
+ func(string) (string, error) { return "v20.2.0-2-g6e73c0a88", nil })
  require.NoError(t, err)
  require.Equal(t, cfg.StellarCoreConfig.NetworkPassphrase, network.PublicNetworkPassphrase)
  require.Equal(t, cfg.StellarCoreConfig.HistoryArchiveUrls, network.PublicNetworkhistoryArchiveURLs)
 
- cmdOut = "v20.2.0-2-g6e73c0a88\n"
  ccConfig, err := cfg.GenerateCaptiveCoreConfig("")
  require.NoError(t, err)
 
@@ -173,13 +170,13 @@ func TestValidCaptiveCoreOverridenToml(t *testing.T) {
 
 func TestValidCaptiveCoreOverridenArchiveUrls(t *testing.T) {
  cfg, err := NewConfig(
- RuntimeSettings{ConfigFilePath: "test/valid_captive_core_override_archives.toml"})
+ RuntimeSettings{ConfigFilePath: "test/valid_captive_core_override_archives.toml"},
+ func(string) (string, error) { return "v20.2.0-2-g6e73c0a88\n", nil })
  require.NoError(t, err)
 
  require.Equal(t, cfg.StellarCoreConfig.NetworkPassphrase, network.PublicNetworkPassphrase)
  require.Equal(t, cfg.StellarCoreConfig.HistoryArchiveUrls, []string{"http:https://testarchive"})
 
- cmdOut = "v20.2.0-2-g6e73c0a88\n"
  ccConfig, err := cfg.GenerateCaptiveCoreConfig("")
  require.NoError(t, err)
 
@@ -194,7 +191,8 @@ func TestValidCaptiveCoreOverridenArchiveUrls(t *testing.T) {
 
 func TestInvalidCaptiveCoreTomlPath(t *testing.T) {
  _, err := NewConfig(
- RuntimeSettings{ConfigFilePath: "test/invalid_captive_core_toml_path.toml"})
+ RuntimeSettings{ConfigFilePath: "test/invalid_captive_core_toml_path.toml"},
+ nil)
  require.ErrorContains(t, err, "Failed to load captive-core-toml-path file")
 }
 
@@ -293,7 +291,7 @@ func TestValidateStartAndEndLedger(t *testing.T) {
  mockedHasCtr++
  }
  config, err := NewConfig(
- RuntimeSettings{StartLedger: tt.startLedger, EndLedger: tt.endLedger, ConfigFilePath: "test/validate_start_end.toml", Mode: tt.mode})
+ RuntimeSettings{StartLedger: tt.startLedger, EndLedger: tt.endLedger, ConfigFilePath: "test/validate_start_end.toml", Mode: tt.mode}, nil)
  require.NoError(t, err)
  err = config.ValidateAndSetLedgerRange(ctx, mockArchive)
  if tt.errMsg != "" {
@@ -365,7 +363,7 @@ func TestAdjustedLedgerRangeBoundedMode(t *testing.T) {
  for _, tt := range tests {
  t.Run(tt.name, func(t *testing.T) {
  config, err := NewConfig(
- RuntimeSettings{StartLedger: tt.start, EndLedger: tt.end, ConfigFilePath: tt.configFile, Mode: ScanFill})
+ RuntimeSettings{StartLedger: tt.start, EndLedger: tt.end, ConfigFilePath: tt.configFile, Mode: ScanFill}, nil)
 
  require.NoError(t, err)
  err = config.ValidateAndSetLedgerRange(ctx, mockArchive)
@@ -428,7 +426,7 @@ func TestAdjustedLedgerRangeUnBoundedMode(t *testing.T) {
  for _, tt := range tests {
  t.Run(tt.name, func(t *testing.T) {
  config, err := NewConfig(
- RuntimeSettings{StartLedger: tt.start, EndLedger: tt.end, ConfigFilePath: tt.configFile, Mode: Append})
+ RuntimeSettings{StartLedger: tt.start, EndLedger: tt.end, ConfigFilePath: tt.configFile, Mode: Append}, nil)
  require.NoError(t, err)
  err = config.ValidateAndSetLedgerRange(ctx, mockArchive)
  require.NoError(t, err)
@@ -438,72 +436,3 @@ func TestAdjustedLedgerRangeUnBoundedMode(t *testing.T) {
  }
  mockArchive.AssertExpectations(t)
 }
-
-var cmdOut = ""
-
-func fakeExecCommand(command string, args ...string) *exec.Cmd {
- cs := append([]string{"-test.run=TestExecCmdHelperProcess", "--", command}, args...)
- cmd := exec.Command(os.Args[0], cs...)
- cmd.Env = append(os.Environ(), "GO_EXEC_CMD_HELPER_PROCESS=1", "CMD_OUT="+cmdOut)
- return cmd
-}
-
-func init() {
- execCommand = fakeExecCommand
-}
-
-func TestExecCmdHelperProcess(t *testing.T) {
- if os.Getenv("GO_EXEC_CMD_HELPER_PROCESS") != "1" {
- return
- }
- fmt.Fprint(os.Stdout, os.Getenv("CMD_OUT"))
- os.Exit(0)
-}
-
-func TestSetCoreVersionInfo(t *testing.T) {
- execCommand = fakeExecCommand
- tests := []struct {
- name string
- commandOutput string
- expectedError error
- expectedCoreVer string
- }{
- {
- name: "version found",
- commandOutput: "v20.2.0-2-g6e73c0a88\n" +
- "rust version: rustc 1.74.1 (a28077b28 2023-12-04)\n" +
- "soroban-env-host: \n" +
- " curr:\n" +
- " package version: 20.2.0\n" +
- " git version: 1bfc0f2a2ee134efc1e1b0d5270281d0cba61c2e\n" +
- " ledger protocol version: 20\n" +
- " pre-release version: 0\n" +
- " rs-stellar-xdr:\n" +
- " package version: 20.1.0\n" +
- " git version: 8b9d623ef40423a8462442b86997155f2c04d3a1\n" +
- " base XDR git version: b96148cd4acc372cc9af17b909ffe4b12c43ecb6\n",
- expectedError: nil,
- expectedCoreVer: "v20.2.0-2-g6e73c0a88",
- },
- {
- name: "core version not found",
- commandOutput: "",
- expectedError: errors.New("stellar-core version not found"),
- expectedCoreVer: "",
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- config := Config{}
- cmdOut = tt.commandOutput
- err := config.setCoreVersionInfo()
-
- if tt.expectedError != nil {
- require.EqualError(t, err, tt.expectedError.Error())
- } else {
- require.NoError(t, err)
- require.Equal(t, tt.expectedCoreVer, config.CoreVersion)
- }
- })
- }
-}