From e2059097f8f20ce0c4c3d6b2a1f6e7f1baa8e19c Mon Sep 17 00:00:00 2001 From: Dmytro Kozhevin Date: Tue, 21 May 2024 16:25:14 -0400 Subject: [PATCH 1/9] Bump core to the stable v21 build. --- .github/workflows/horizon.yml | 12 ++++++------ exp/tools/dump-ledger-state/Dockerfile | 2 +- exp/tools/dump-ledger-state/stellar-core-testnet.cfg | 2 ++ exp/tools/dump-ledger-state/stellar-core.cfg | 2 ++ 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/horizon.yml b/.github/workflows/horizon.yml index ddf0da7f11..0568edd1ce 100644 --- a/.github/workflows/horizon.yml +++ b/.github/workflows/horizon.yml @@ -32,11 +32,11 @@ jobs: env: HORIZON_INTEGRATION_TESTS_ENABLED: true HORIZON_INTEGRATION_TESTS_CORE_MAX_SUPPORTED_PROTOCOL: ${{ matrix.protocol-version }} - PROTOCOL_21_CORE_DEBIAN_PKG_VERSION: 21.0.0-1812.rc1.a10329cca.focal - PROTOCOL_21_CORE_DOCKER_IMG: stellar/unsafe-stellar-core:21.0.0-1812.rc1.a10329cca.focal + PROTOCOL_21_CORE_DEBIAN_PKG_VERSION: 21.0.0-1872.c6f474133.focal + PROTOCOL_21_CORE_DOCKER_IMG: stellar/stellar-core:21 PROTOCOL_21_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:21.0.0-rc2-73 - PROTOCOL_20_CORE_DEBIAN_PKG_VERSION: 21.0.0-1812.rc1.a10329cca.focal - PROTOCOL_20_CORE_DOCKER_IMG: stellar/unsafe-stellar-core:21.0.0-1812.rc1.a10329cca.focal + PROTOCOL_20_CORE_DEBIAN_PKG_VERSION: 21.0.0-1872.c6f474133.focal + PROTOCOL_20_CORE_DOCKER_IMG: stellar/stellar-core:21 PROTOCOL_20_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:21.0.0-rc2-73 PGHOST: localhost PGPORT: 5432 @@ -125,7 +125,7 @@ jobs: name: Test (and push) verify-range image runs-on: ubuntu-22.04 env: - STELLAR_CORE_VERSION: 19.14.0-1500.5664eff4e.focal + STELLAR_CORE_VERSION: 21.0.0-1872.c6f474133.focal CAPTIVE_CORE_STORAGE_PATH: /tmp steps: - uses: actions/checkout@v3 @@ -155,7 +155,7 @@ jobs: name: Test and push the Ledger Exporter images runs-on: ubuntu-latest env: - STELLAR_CORE_VERSION: 21.0.0-1812.rc1.a10329cca.focal + STELLAR_CORE_VERSION: 21.0.0-1872.c6f474133.focal steps: - uses: actions/checkout@v3 with: diff --git a/exp/tools/dump-ledger-state/Dockerfile b/exp/tools/dump-ledger-state/Dockerfile index 91b258f0a8..5ffcb9c0a2 100644 --- a/exp/tools/dump-ledger-state/Dockerfile +++ b/exp/tools/dump-ledger-state/Dockerfile @@ -1,6 +1,6 @@ FROM ubuntu:22.04 -ENV STELLAR_CORE_VERSION=19.14.0-1500.5664eff4e.focal +ENV STELLAR_CORE_VERSION=21.0.0-1872.c6f474133.focal ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl wget gnupg apt-utils diff --git a/exp/tools/dump-ledger-state/stellar-core-testnet.cfg b/exp/tools/dump-ledger-state/stellar-core-testnet.cfg index 5162295622..ebdc3ae4cc 100644 --- a/exp/tools/dump-ledger-state/stellar-core-testnet.cfg +++ b/exp/tools/dump-ledger-state/stellar-core-testnet.cfg @@ -8,6 +8,8 @@ UNSAFE_QUORUM=true FAILURE_SAFETY=1 CATCHUP_RECENT=8640 +DEPRECATED_SQL_LEDGER_STATE=false + [HISTORY.cache] get="cp /opt/stellar/history-cache/{0} {1}" diff --git a/exp/tools/dump-ledger-state/stellar-core.cfg b/exp/tools/dump-ledger-state/stellar-core.cfg index 0b4e454681..b25c86d17a 100644 --- a/exp/tools/dump-ledger-state/stellar-core.cfg +++ b/exp/tools/dump-ledger-state/stellar-core.cfg @@ -6,6 +6,8 @@ DATABASE="postgresql://dbname=core host=localhost user=circleci" NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015" CATCHUP_RECENT=1 +DEPRECATED_SQL_LEDGER_STATE=false + [HISTORY.cache] get="cp /opt/stellar/history-cache/{0} {1}" From 19ac62ffeb2fad774ab3212dd3fac79e854d5989 Mon Sep 17 00:00:00 2001 From: Dmytro Kozhevin Date: Tue, 21 May 2024 17:57:13 -0400 Subject: [PATCH 2/9] Try using 'unsafe-stellar-core' docker image --- .github/workflows/horizon.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/horizon.yml b/.github/workflows/horizon.yml index 0568edd1ce..f1ebd4cdd5 100644 --- a/.github/workflows/horizon.yml +++ b/.github/workflows/horizon.yml @@ -33,10 +33,10 @@ jobs: HORIZON_INTEGRATION_TESTS_ENABLED: true HORIZON_INTEGRATION_TESTS_CORE_MAX_SUPPORTED_PROTOCOL: ${{ matrix.protocol-version }} PROTOCOL_21_CORE_DEBIAN_PKG_VERSION: 21.0.0-1872.c6f474133.focal - PROTOCOL_21_CORE_DOCKER_IMG: stellar/stellar-core:21 + PROTOCOL_21_CORE_DOCKER_IMG: stellar/unsafe-stellar-core:21.0.0-1872.c6f474133.focal PROTOCOL_21_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:21.0.0-rc2-73 PROTOCOL_20_CORE_DEBIAN_PKG_VERSION: 21.0.0-1872.c6f474133.focal - PROTOCOL_20_CORE_DOCKER_IMG: stellar/stellar-core:21 + PROTOCOL_20_CORE_DOCKER_IMG: stellar/unsafe-stellar-core:21.0.0-1872.c6f474133.focal PROTOCOL_20_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:21.0.0-rc2-73 PGHOST: localhost PGPORT: 5432 From 889d2339df49d5de396a8f5e447c69501f092be9 Mon Sep 17 00:00:00 2001 From: Dmytro Kozhevin Date: Tue, 21 May 2024 19:10:36 -0400 Subject: [PATCH 3/9] Revert "Try using 'unsafe-stellar-core' docker image" This reverts commit 19ac62ffeb2fad774ab3212dd3fac79e854d5989. --- .github/workflows/horizon.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/horizon.yml b/.github/workflows/horizon.yml index f1ebd4cdd5..0568edd1ce 100644 --- a/.github/workflows/horizon.yml +++ b/.github/workflows/horizon.yml @@ -33,10 +33,10 @@ jobs: HORIZON_INTEGRATION_TESTS_ENABLED: true HORIZON_INTEGRATION_TESTS_CORE_MAX_SUPPORTED_PROTOCOL: ${{ matrix.protocol-version }} PROTOCOL_21_CORE_DEBIAN_PKG_VERSION: 21.0.0-1872.c6f474133.focal - PROTOCOL_21_CORE_DOCKER_IMG: stellar/unsafe-stellar-core:21.0.0-1872.c6f474133.focal + PROTOCOL_21_CORE_DOCKER_IMG: stellar/stellar-core:21 PROTOCOL_21_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:21.0.0-rc2-73 PROTOCOL_20_CORE_DEBIAN_PKG_VERSION: 21.0.0-1872.c6f474133.focal - PROTOCOL_20_CORE_DOCKER_IMG: stellar/unsafe-stellar-core:21.0.0-1872.c6f474133.focal + PROTOCOL_20_CORE_DOCKER_IMG: stellar/stellar-core:21 PROTOCOL_20_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:21.0.0-rc2-73 PGHOST: localhost PGPORT: 5432 From 4048f0f4abb451e9b2fe4bbb3084f4dd87f47330 Mon Sep 17 00:00:00 2001 From: Dmytro Kozhevin Date: Tue, 21 May 2024 20:30:17 -0400 Subject: [PATCH 4/9] Add DEPRECATED_SQL_LEDGER_STATE=false to (most) of the Core configs --- exp/lighthorizon/build/ledgerexporter/captive-core-pubnet.cfg | 2 ++ exp/lighthorizon/build/ledgerexporter/captive-core-testnet.cfg | 2 ++ ingest/ledgerbackend/configs/captive-core-pubnet.cfg | 2 ++ ingest/ledgerbackend/configs/captive-core-testnet.cfg | 2 ++ .../horizon/docker/captive-core-classic-integration-tests.cfg | 2 ++ services/horizon/docker/captive-core-integration-tests.cfg | 2 ++ .../docker/captive-core-integration-tests.soroban-rpc.cfg | 2 ++ .../captive-core-reingest-range-classic-integration-tests.cfg | 2 ++ .../docker/captive-core-reingest-range-integration-tests.cfg | 1 + services/horizon/docker/captive-core-standalone.cfg | 2 ++ .../horizon/docker/stellar-core-classic-integration-tests.cfg | 1 + services/horizon/docker/stellar-core-integration-tests.cfg | 1 + services/horizon/docker/stellar-core-pubnet.cfg | 2 ++ services/horizon/docker/stellar-core-standalone.cfg | 1 + services/horizon/docker/stellar-core-testnet.cfg | 2 ++ services/horizon/docker/verify-range/captive-core-pubnet.cfg | 2 ++ services/horizon/docker/verify-range/stellar-core.cfg | 2 ++ 17 files changed, 30 insertions(+) diff --git a/exp/lighthorizon/build/ledgerexporter/captive-core-pubnet.cfg b/exp/lighthorizon/build/ledgerexporter/captive-core-pubnet.cfg index d891626756..911f5f5a3a 100644 --- a/exp/lighthorizon/build/ledgerexporter/captive-core-pubnet.cfg +++ b/exp/lighthorizon/build/ledgerexporter/captive-core-pubnet.cfg @@ -3,6 +3,8 @@ DATABASE = "sqlite3:///cc/stellar.db" FAILURE_SAFETY=1 +DEPRECATED_SQL_LEDGER_STATE=false + # WARNING! Do not use this config in production. Quorum sets should # be carefully selected manually. NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015" diff --git a/exp/lighthorizon/build/ledgerexporter/captive-core-testnet.cfg b/exp/lighthorizon/build/ledgerexporter/captive-core-testnet.cfg index 0cd9b2f496..f9182afc64 100644 --- a/exp/lighthorizon/build/ledgerexporter/captive-core-testnet.cfg +++ b/exp/lighthorizon/build/ledgerexporter/captive-core-testnet.cfg @@ -4,6 +4,8 @@ DATABASE = "sqlite3:///cc/stellar.db" UNSAFE_QUORUM=true FAILURE_SAFETY=1 +DEPRECATED_SQL_LEDGER_STATE=false + [[HOME_DOMAINS]] HOME_DOMAIN="testnet.stellar.org" QUALITY="HIGH" diff --git a/ingest/ledgerbackend/configs/captive-core-pubnet.cfg b/ingest/ledgerbackend/configs/captive-core-pubnet.cfg index 5af59efaf9..d2dd309521 100644 --- a/ingest/ledgerbackend/configs/captive-core-pubnet.cfg +++ b/ingest/ledgerbackend/configs/captive-core-pubnet.cfg @@ -5,6 +5,8 @@ FAILURE_SAFETY=1 HTTP_PORT=11626 PEER_PORT=11725 +DEPRECATED_SQL_LEDGER_STATE=false + [[HOME_DOMAINS]] HOME_DOMAIN="publicnode.org" QUALITY="HIGH" diff --git a/ingest/ledgerbackend/configs/captive-core-testnet.cfg b/ingest/ledgerbackend/configs/captive-core-testnet.cfg index 9abeecc8f5..ac56c41308 100644 --- a/ingest/ledgerbackend/configs/captive-core-testnet.cfg +++ b/ingest/ledgerbackend/configs/captive-core-testnet.cfg @@ -2,6 +2,8 @@ NETWORK_PASSPHRASE="Test SDF Network ; September 2015" UNSAFE_QUORUM=true FAILURE_SAFETY=1 +DEPRECATED_SQL_LEDGER_STATE=false + [[HOME_DOMAINS]] HOME_DOMAIN="testnet.stellar.org" QUALITY="HIGH" diff --git a/services/horizon/docker/captive-core-classic-integration-tests.cfg b/services/horizon/docker/captive-core-classic-integration-tests.cfg index ed8ac3ed73..552a2a6975 100644 --- a/services/horizon/docker/captive-core-classic-integration-tests.cfg +++ b/services/horizon/docker/captive-core-classic-integration-tests.cfg @@ -4,6 +4,8 @@ ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true UNSAFE_QUORUM=true FAILURE_SAFETY=0 +DEPRECATED_SQL_LEDGER_STATE=false + [[VALIDATORS]] NAME="local_core" HOME_DOMAIN="core.local" diff --git a/services/horizon/docker/captive-core-integration-tests.cfg b/services/horizon/docker/captive-core-integration-tests.cfg index 275599bacd..c7a57f5557 100644 --- a/services/horizon/docker/captive-core-integration-tests.cfg +++ b/services/horizon/docker/captive-core-integration-tests.cfg @@ -4,6 +4,8 @@ ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true UNSAFE_QUORUM=true FAILURE_SAFETY=0 +DEPRECATED_SQL_LEDGER_STATE=false + ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true # Lower the TTL of persistent ledger entries # so that ledger entry extension/restoring becomes testeable diff --git a/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg b/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg index 9a7ad9d769..8551fe52a5 100644 --- a/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg +++ b/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg @@ -4,6 +4,8 @@ ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true UNSAFE_QUORUM=true FAILURE_SAFETY=0 +DEPRECATED_SQL_LEDGER_STATE=false + ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true # Lower the TTL of persistent ledger entries # so that ledger entry extension/restoring becomes testeable diff --git a/services/horizon/docker/captive-core-reingest-range-classic-integration-tests.cfg b/services/horizon/docker/captive-core-reingest-range-classic-integration-tests.cfg index 4902cf8d15..2f0672af78 100644 --- a/services/horizon/docker/captive-core-reingest-range-classic-integration-tests.cfg +++ b/services/horizon/docker/captive-core-reingest-range-classic-integration-tests.cfg @@ -1,5 +1,7 @@ ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true +DEPRECATED_SQL_LEDGER_STATE=false + [[VALIDATORS]] NAME="local_core" HOME_DOMAIN="core.local" diff --git a/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg b/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg index 44820f5933..32025bed27 100644 --- a/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg +++ b/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg @@ -2,6 +2,7 @@ ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true TESTING_SOROBAN_HIGH_LIMIT_OVERRIDE=true TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME=10 ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true +DEPRECATED_SQL_LEDGER_STATE=false [[VALIDATORS]] NAME="local_core" diff --git a/services/horizon/docker/captive-core-standalone.cfg b/services/horizon/docker/captive-core-standalone.cfg index d54b0ecae1..b19c48a471 100644 --- a/services/horizon/docker/captive-core-standalone.cfg +++ b/services/horizon/docker/captive-core-standalone.cfg @@ -3,6 +3,8 @@ PEER_PORT=11725 UNSAFE_QUORUM=true FAILURE_SAFETY=0 +DEPRECATED_SQL_LEDGER_STATE=false + [[VALIDATORS]] NAME="local_core" HOME_DOMAIN="core.local" diff --git a/services/horizon/docker/stellar-core-classic-integration-tests.cfg b/services/horizon/docker/stellar-core-classic-integration-tests.cfg index e27cfe14ed..a264cbcbaf 100644 --- a/services/horizon/docker/stellar-core-classic-integration-tests.cfg +++ b/services/horizon/docker/stellar-core-classic-integration-tests.cfg @@ -13,6 +13,7 @@ UNSAFE_QUORUM=true FAILURE_SAFETY=0 DATABASE="postgresql://user=postgres password=mysecretpassword host=core-postgres port=5641 dbname=stellar" +DEPRECATED_SQL_LEDGER_STATE=false [QUORUM_SET] THRESHOLD_PERCENT=100 diff --git a/services/horizon/docker/stellar-core-integration-tests.cfg b/services/horizon/docker/stellar-core-integration-tests.cfg index 594a35b244..99c8c19b6f 100644 --- a/services/horizon/docker/stellar-core-integration-tests.cfg +++ b/services/horizon/docker/stellar-core-integration-tests.cfg @@ -13,6 +13,7 @@ UNSAFE_QUORUM=true FAILURE_SAFETY=0 DATABASE="postgresql://user=postgres password=mysecretpassword host=core-postgres port=5641 dbname=stellar" +DEPRECATED_SQL_LEDGER_STATE=false # Lower the TTL of persistent ledger entries # so that ledger entry extension/restoring becomes testeable diff --git a/services/horizon/docker/stellar-core-pubnet.cfg b/services/horizon/docker/stellar-core-pubnet.cfg index 94b29c4b4e..2ec0800647 100644 --- a/services/horizon/docker/stellar-core-pubnet.cfg +++ b/services/horizon/docker/stellar-core-pubnet.cfg @@ -9,6 +9,8 @@ DATABASE="postgresql://user=postgres password=mysecretpassword host=host.docker. NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015" CATCHUP_RECENT=100 +DEPRECATED_SQL_LEDGER_STATE=false + [HISTORY.cache] get="cp /opt/stellar/history-cache/{0} {1}" diff --git a/services/horizon/docker/stellar-core-standalone.cfg b/services/horizon/docker/stellar-core-standalone.cfg index a2b7e806c9..20e74dd5cf 100644 --- a/services/horizon/docker/stellar-core-standalone.cfg +++ b/services/horizon/docker/stellar-core-standalone.cfg @@ -14,6 +14,7 @@ UNSAFE_QUORUM=true FAILURE_SAFETY=0 DATABASE="postgresql://user=postgres password=mysecretpassword host=host.docker.internal port=5641 dbname=stellar" +DEPRECATED_SQL_LEDGER_STATE=false [QUORUM_SET] THRESHOLD_PERCENT=100 diff --git a/services/horizon/docker/stellar-core-testnet.cfg b/services/horizon/docker/stellar-core-testnet.cfg index cf8546a3e9..08df39eaf7 100644 --- a/services/horizon/docker/stellar-core-testnet.cfg +++ b/services/horizon/docker/stellar-core-testnet.cfg @@ -12,6 +12,8 @@ UNSAFE_QUORUM=true FAILURE_SAFETY=1 CATCHUP_RECENT=100 +DEPRECATED_SQL_LEDGER_STATE=false + [HISTORY.cache] get="cp /opt/stellar/history-cache/{0} {1}" diff --git a/services/horizon/docker/verify-range/captive-core-pubnet.cfg b/services/horizon/docker/verify-range/captive-core-pubnet.cfg index 5a702711fe..48a20b590e 100644 --- a/services/horizon/docker/verify-range/captive-core-pubnet.cfg +++ b/services/horizon/docker/verify-range/captive-core-pubnet.cfg @@ -2,6 +2,8 @@ PEER_PORT=11725 FAILURE_SAFETY=1 +DEPRECATED_SQL_LEDGER_STATE=false + [[HOME_DOMAINS]] HOME_DOMAIN="stellar.org" QUALITY="HIGH" diff --git a/services/horizon/docker/verify-range/stellar-core.cfg b/services/horizon/docker/verify-range/stellar-core.cfg index 6139f9f528..e3e48f10f6 100644 --- a/services/horizon/docker/verify-range/stellar-core.cfg +++ b/services/horizon/docker/verify-range/stellar-core.cfg @@ -4,6 +4,8 @@ LOG_FILE_PATH="" NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015" CATCHUP_RECENT=100 +DEPRECATED_SQL_LEDGER_STATE=false + AUTOMATIC_MAINTENANCE_COUNT=0 NODE_NAMES=[ From a4b08f46e1e17b6fe6a3fd34ae4462f7179425d2 Mon Sep 17 00:00:00 2001 From: Dmytro Kozhevin Date: Wed, 22 May 2024 11:11:36 -0400 Subject: [PATCH 5/9] Try using `EXPERIMENTAL_BUCKETLIST_DB` for compatibility reasons --- exp/lighthorizon/build/ledgerexporter/captive-core-pubnet.cfg | 2 +- exp/lighthorizon/build/ledgerexporter/captive-core-testnet.cfg | 2 +- exp/tools/dump-ledger-state/stellar-core-testnet.cfg | 2 +- exp/tools/dump-ledger-state/stellar-core.cfg | 2 +- ingest/ledgerbackend/configs/captive-core-pubnet.cfg | 2 +- ingest/ledgerbackend/configs/captive-core-testnet.cfg | 2 +- .../horizon/docker/captive-core-classic-integration-tests.cfg | 2 +- services/horizon/docker/captive-core-integration-tests.cfg | 2 +- .../docker/captive-core-integration-tests.soroban-rpc.cfg | 2 +- .../captive-core-reingest-range-classic-integration-tests.cfg | 2 +- .../docker/captive-core-reingest-range-integration-tests.cfg | 2 +- services/horizon/docker/captive-core-standalone.cfg | 2 +- .../horizon/docker/stellar-core-classic-integration-tests.cfg | 2 +- services/horizon/docker/stellar-core-integration-tests.cfg | 2 +- services/horizon/docker/stellar-core-pubnet.cfg | 2 +- services/horizon/docker/stellar-core-standalone.cfg | 2 +- services/horizon/docker/stellar-core-testnet.cfg | 2 +- services/horizon/docker/verify-range/captive-core-pubnet.cfg | 2 +- services/horizon/docker/verify-range/stellar-core.cfg | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/exp/lighthorizon/build/ledgerexporter/captive-core-pubnet.cfg b/exp/lighthorizon/build/ledgerexporter/captive-core-pubnet.cfg index 911f5f5a3a..6379725b8d 100644 --- a/exp/lighthorizon/build/ledgerexporter/captive-core-pubnet.cfg +++ b/exp/lighthorizon/build/ledgerexporter/captive-core-pubnet.cfg @@ -3,7 +3,7 @@ DATABASE = "sqlite3:///cc/stellar.db" FAILURE_SAFETY=1 -DEPRECATED_SQL_LEDGER_STATE=false +EXPERIMENTAL_BUCKETLIST_DB=true # WARNING! Do not use this config in production. Quorum sets should # be carefully selected manually. diff --git a/exp/lighthorizon/build/ledgerexporter/captive-core-testnet.cfg b/exp/lighthorizon/build/ledgerexporter/captive-core-testnet.cfg index f9182afc64..9c7dadc527 100644 --- a/exp/lighthorizon/build/ledgerexporter/captive-core-testnet.cfg +++ b/exp/lighthorizon/build/ledgerexporter/captive-core-testnet.cfg @@ -4,7 +4,7 @@ DATABASE = "sqlite3:///cc/stellar.db" UNSAFE_QUORUM=true FAILURE_SAFETY=1 -DEPRECATED_SQL_LEDGER_STATE=false +EXPERIMENTAL_BUCKETLIST_DB=true [[HOME_DOMAINS]] HOME_DOMAIN="testnet.stellar.org" diff --git a/exp/tools/dump-ledger-state/stellar-core-testnet.cfg b/exp/tools/dump-ledger-state/stellar-core-testnet.cfg index ebdc3ae4cc..a02221e795 100644 --- a/exp/tools/dump-ledger-state/stellar-core-testnet.cfg +++ b/exp/tools/dump-ledger-state/stellar-core-testnet.cfg @@ -8,7 +8,7 @@ UNSAFE_QUORUM=true FAILURE_SAFETY=1 CATCHUP_RECENT=8640 -DEPRECATED_SQL_LEDGER_STATE=false +EXPERIMENTAL_BUCKETLIST_DB=true [HISTORY.cache] get="cp /opt/stellar/history-cache/{0} {1}" diff --git a/exp/tools/dump-ledger-state/stellar-core.cfg b/exp/tools/dump-ledger-state/stellar-core.cfg index b25c86d17a..0d97346ce6 100644 --- a/exp/tools/dump-ledger-state/stellar-core.cfg +++ b/exp/tools/dump-ledger-state/stellar-core.cfg @@ -6,7 +6,7 @@ DATABASE="postgresql://dbname=core host=localhost user=circleci" NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015" CATCHUP_RECENT=1 -DEPRECATED_SQL_LEDGER_STATE=false +EXPERIMENTAL_BUCKETLIST_DB=true [HISTORY.cache] get="cp /opt/stellar/history-cache/{0} {1}" diff --git a/ingest/ledgerbackend/configs/captive-core-pubnet.cfg b/ingest/ledgerbackend/configs/captive-core-pubnet.cfg index d2dd309521..6d23046f56 100644 --- a/ingest/ledgerbackend/configs/captive-core-pubnet.cfg +++ b/ingest/ledgerbackend/configs/captive-core-pubnet.cfg @@ -5,7 +5,7 @@ FAILURE_SAFETY=1 HTTP_PORT=11626 PEER_PORT=11725 -DEPRECATED_SQL_LEDGER_STATE=false +EXPERIMENTAL_BUCKETLIST_DB=true [[HOME_DOMAINS]] HOME_DOMAIN="publicnode.org" diff --git a/ingest/ledgerbackend/configs/captive-core-testnet.cfg b/ingest/ledgerbackend/configs/captive-core-testnet.cfg index ac56c41308..3f0dec5095 100644 --- a/ingest/ledgerbackend/configs/captive-core-testnet.cfg +++ b/ingest/ledgerbackend/configs/captive-core-testnet.cfg @@ -2,7 +2,7 @@ NETWORK_PASSPHRASE="Test SDF Network ; September 2015" UNSAFE_QUORUM=true FAILURE_SAFETY=1 -DEPRECATED_SQL_LEDGER_STATE=false +EXPERIMENTAL_BUCKETLIST_DB=true [[HOME_DOMAINS]] HOME_DOMAIN="testnet.stellar.org" diff --git a/services/horizon/docker/captive-core-classic-integration-tests.cfg b/services/horizon/docker/captive-core-classic-integration-tests.cfg index 552a2a6975..2f95a0ee54 100644 --- a/services/horizon/docker/captive-core-classic-integration-tests.cfg +++ b/services/horizon/docker/captive-core-classic-integration-tests.cfg @@ -4,7 +4,7 @@ ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true UNSAFE_QUORUM=true FAILURE_SAFETY=0 -DEPRECATED_SQL_LEDGER_STATE=false +EXPERIMENTAL_BUCKETLIST_DB=true [[VALIDATORS]] NAME="local_core" diff --git a/services/horizon/docker/captive-core-integration-tests.cfg b/services/horizon/docker/captive-core-integration-tests.cfg index c7a57f5557..02c59d7057 100644 --- a/services/horizon/docker/captive-core-integration-tests.cfg +++ b/services/horizon/docker/captive-core-integration-tests.cfg @@ -4,7 +4,7 @@ ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true UNSAFE_QUORUM=true FAILURE_SAFETY=0 -DEPRECATED_SQL_LEDGER_STATE=false +EXPERIMENTAL_BUCKETLIST_DB=true ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true # Lower the TTL of persistent ledger entries diff --git a/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg b/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg index 8551fe52a5..cf4d514975 100644 --- a/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg +++ b/services/horizon/docker/captive-core-integration-tests.soroban-rpc.cfg @@ -4,7 +4,7 @@ ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true UNSAFE_QUORUM=true FAILURE_SAFETY=0 -DEPRECATED_SQL_LEDGER_STATE=false +EXPERIMENTAL_BUCKETLIST_DB=true ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true # Lower the TTL of persistent ledger entries diff --git a/services/horizon/docker/captive-core-reingest-range-classic-integration-tests.cfg b/services/horizon/docker/captive-core-reingest-range-classic-integration-tests.cfg index 2f0672af78..735f58b739 100644 --- a/services/horizon/docker/captive-core-reingest-range-classic-integration-tests.cfg +++ b/services/horizon/docker/captive-core-reingest-range-classic-integration-tests.cfg @@ -1,6 +1,6 @@ ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true -DEPRECATED_SQL_LEDGER_STATE=false +EXPERIMENTAL_BUCKETLIST_DB=true [[VALIDATORS]] NAME="local_core" diff --git a/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg b/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg index 32025bed27..4744fd390e 100644 --- a/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg +++ b/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg @@ -2,7 +2,7 @@ ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true TESTING_SOROBAN_HIGH_LIMIT_OVERRIDE=true TESTING_MINIMUM_PERSISTENT_ENTRY_LIFETIME=10 ENABLE_SOROBAN_DIAGNOSTIC_EVENTS=true -DEPRECATED_SQL_LEDGER_STATE=false +EXPERIMENTAL_BUCKETLIST_DB=true [[VALIDATORS]] NAME="local_core" diff --git a/services/horizon/docker/captive-core-standalone.cfg b/services/horizon/docker/captive-core-standalone.cfg index b19c48a471..f5042a14df 100644 --- a/services/horizon/docker/captive-core-standalone.cfg +++ b/services/horizon/docker/captive-core-standalone.cfg @@ -3,7 +3,7 @@ PEER_PORT=11725 UNSAFE_QUORUM=true FAILURE_SAFETY=0 -DEPRECATED_SQL_LEDGER_STATE=false +EXPERIMENTAL_BUCKETLIST_DB=true [[VALIDATORS]] NAME="local_core" diff --git a/services/horizon/docker/stellar-core-classic-integration-tests.cfg b/services/horizon/docker/stellar-core-classic-integration-tests.cfg index a264cbcbaf..f2b20b4927 100644 --- a/services/horizon/docker/stellar-core-classic-integration-tests.cfg +++ b/services/horizon/docker/stellar-core-classic-integration-tests.cfg @@ -13,7 +13,7 @@ UNSAFE_QUORUM=true FAILURE_SAFETY=0 DATABASE="postgresql://user=postgres password=mysecretpassword host=core-postgres port=5641 dbname=stellar" -DEPRECATED_SQL_LEDGER_STATE=false +EXPERIMENTAL_BUCKETLIST_DB=true [QUORUM_SET] THRESHOLD_PERCENT=100 diff --git a/services/horizon/docker/stellar-core-integration-tests.cfg b/services/horizon/docker/stellar-core-integration-tests.cfg index 99c8c19b6f..0d5ec5cc43 100644 --- a/services/horizon/docker/stellar-core-integration-tests.cfg +++ b/services/horizon/docker/stellar-core-integration-tests.cfg @@ -13,7 +13,7 @@ UNSAFE_QUORUM=true FAILURE_SAFETY=0 DATABASE="postgresql://user=postgres password=mysecretpassword host=core-postgres port=5641 dbname=stellar" -DEPRECATED_SQL_LEDGER_STATE=false +EXPERIMENTAL_BUCKETLIST_DB=true # Lower the TTL of persistent ledger entries # so that ledger entry extension/restoring becomes testeable diff --git a/services/horizon/docker/stellar-core-pubnet.cfg b/services/horizon/docker/stellar-core-pubnet.cfg index 2ec0800647..7f250b10c7 100644 --- a/services/horizon/docker/stellar-core-pubnet.cfg +++ b/services/horizon/docker/stellar-core-pubnet.cfg @@ -9,7 +9,7 @@ DATABASE="postgresql://user=postgres password=mysecretpassword host=host.docker. NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015" CATCHUP_RECENT=100 -DEPRECATED_SQL_LEDGER_STATE=false +EXPERIMENTAL_BUCKETLIST_DB=true [HISTORY.cache] get="cp /opt/stellar/history-cache/{0} {1}" diff --git a/services/horizon/docker/stellar-core-standalone.cfg b/services/horizon/docker/stellar-core-standalone.cfg index 20e74dd5cf..41c8a377aa 100644 --- a/services/horizon/docker/stellar-core-standalone.cfg +++ b/services/horizon/docker/stellar-core-standalone.cfg @@ -14,7 +14,7 @@ UNSAFE_QUORUM=true FAILURE_SAFETY=0 DATABASE="postgresql://user=postgres password=mysecretpassword host=host.docker.internal port=5641 dbname=stellar" -DEPRECATED_SQL_LEDGER_STATE=false +EXPERIMENTAL_BUCKETLIST_DB=true [QUORUM_SET] THRESHOLD_PERCENT=100 diff --git a/services/horizon/docker/stellar-core-testnet.cfg b/services/horizon/docker/stellar-core-testnet.cfg index 08df39eaf7..55c5cea462 100644 --- a/services/horizon/docker/stellar-core-testnet.cfg +++ b/services/horizon/docker/stellar-core-testnet.cfg @@ -12,7 +12,7 @@ UNSAFE_QUORUM=true FAILURE_SAFETY=1 CATCHUP_RECENT=100 -DEPRECATED_SQL_LEDGER_STATE=false +EXPERIMENTAL_BUCKETLIST_DB=true [HISTORY.cache] get="cp /opt/stellar/history-cache/{0} {1}" diff --git a/services/horizon/docker/verify-range/captive-core-pubnet.cfg b/services/horizon/docker/verify-range/captive-core-pubnet.cfg index 48a20b590e..cedbbc65ab 100644 --- a/services/horizon/docker/verify-range/captive-core-pubnet.cfg +++ b/services/horizon/docker/verify-range/captive-core-pubnet.cfg @@ -2,7 +2,7 @@ PEER_PORT=11725 FAILURE_SAFETY=1 -DEPRECATED_SQL_LEDGER_STATE=false +EXPERIMENTAL_BUCKETLIST_DB=true [[HOME_DOMAINS]] HOME_DOMAIN="stellar.org" diff --git a/services/horizon/docker/verify-range/stellar-core.cfg b/services/horizon/docker/verify-range/stellar-core.cfg index e3e48f10f6..91bb190131 100644 --- a/services/horizon/docker/verify-range/stellar-core.cfg +++ b/services/horizon/docker/verify-range/stellar-core.cfg @@ -4,7 +4,7 @@ LOG_FILE_PATH="" NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015" CATCHUP_RECENT=100 -DEPRECATED_SQL_LEDGER_STATE=false +EXPERIMENTAL_BUCKETLIST_DB=true AUTOMATIC_MAINTENANCE_COUNT=0 From f8370744a53d5dcc513d1a66a79b888bf0fa921a Mon Sep 17 00:00:00 2001 From: shawn Date: Wed, 22 May 2024 09:35:38 -0700 Subject: [PATCH 6/9] Update horizon.yml force all integration tests to have HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_USE_DB=true by default --- .github/workflows/horizon.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/horizon.yml b/.github/workflows/horizon.yml index 0568edd1ce..fe7b2df7a8 100644 --- a/.github/workflows/horizon.yml +++ b/.github/workflows/horizon.yml @@ -32,6 +32,7 @@ jobs: env: HORIZON_INTEGRATION_TESTS_ENABLED: true HORIZON_INTEGRATION_TESTS_CORE_MAX_SUPPORTED_PROTOCOL: ${{ matrix.protocol-version }} + HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_USE_DB: true PROTOCOL_21_CORE_DEBIAN_PKG_VERSION: 21.0.0-1872.c6f474133.focal PROTOCOL_21_CORE_DOCKER_IMG: stellar/stellar-core:21 PROTOCOL_21_SOROBAN_RPC_DOCKER_IMG: stellar/soroban-rpc:21.0.0-rc2-73 From eba82759ce228fadf6f601729f2dd78ece3667b8 Mon Sep 17 00:00:00 2001 From: shawn Date: Fri, 24 May 2024 12:35:56 -0700 Subject: [PATCH 7/9] captive core use_db flag default to true for integration tests should always be set to true, that flag is deprecated and defaulted to true, there is no test paths validating it being false. --- services/horizon/internal/test/integration/integration.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/horizon/internal/test/integration/integration.go b/services/horizon/internal/test/integration/integration.go index d755d00252..e12f77f693 100644 --- a/services/horizon/internal/test/integration/integration.go +++ b/services/horizon/internal/test/integration/integration.go @@ -441,7 +441,7 @@ func (i *Test) getDefaultIngestArgs(postgres *dbtest.DB) map[string]string { "stellar-core-binary-path": i.coreConfig.binaryPath, "captive-core-config-path": i.coreConfig.configPath, "captive-core-http-port": "21626", - "captive-core-use-db": strconv.FormatBool(i.coreConfig.useDB), + "captive-core-use-db": "true", "captive-core-storage-path": i.coreConfig.storagePath, "ingest": "true"}) } From 772ca144d8540fba2ac138a8e63adfe04fd85943 Mon Sep 17 00:00:00 2001 From: Shawn Reuland Date: Fri, 24 May 2024 13:50:51 -0700 Subject: [PATCH 8/9] #5295: dealing with cc use db assumptions in tests --- services/horizon/internal/flags_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/horizon/internal/flags_test.go b/services/horizon/internal/flags_test.go index a30ea3a404..65d1da524c 100644 --- a/services/horizon/internal/flags_test.go +++ b/services/horizon/internal/flags_test.go @@ -86,6 +86,7 @@ func Test_createCaptiveCoreDefaultConfig(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + tt.config.CaptiveCoreTomlParams.UseDB = true e := setCaptiveCoreConfiguration(&tt.config, ApplyOptions{RequireCaptiveCoreFullConfig: true}) if tt.errStr == "" { @@ -192,6 +193,7 @@ func Test_createCaptiveCoreConfig(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + tt.config.CaptiveCoreTomlParams.UseDB = true e := setCaptiveCoreConfiguration(&tt.config, ApplyOptions{RequireCaptiveCoreFullConfig: tt.requireCaptiveCoreConfig}) if tt.errStr == "" { From 2d097092c715285320ee57bd0245d774dcff8398 Mon Sep 17 00:00:00 2001 From: Shawn Reuland Date: Fri, 24 May 2024 17:15:12 -0700 Subject: [PATCH 9/9] #5295: fixing captive core use_db flag assumptions in tests --- services/horizon/internal/integration/parameters_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/horizon/internal/integration/parameters_test.go b/services/horizon/internal/integration/parameters_test.go index eb22e9a068..133950d6f3 100644 --- a/services/horizon/internal/integration/parameters_test.go +++ b/services/horizon/internal/integration/parameters_test.go @@ -505,7 +505,7 @@ func TestDeprecatedOutputs(t *testing.T) { stdLog.SetOutput(os.Stderr) testConfig := integration.GetTestConfig() - testConfig.HorizonIngestParameters = map[string]string{"captive-core-use-db": "false"} + testConfig.HorizonIngestParameters = map[string]string{"captive-core-use-db": "true"} test := integration.NewTest(t, *testConfig) err := test.StartHorizon() assert.NoError(t, err)