Merge release-horizon-v2.23.0 into master

.github/workflows/go.yml

@@ -18,7 +18,7 @@ jobs:
  strategy:
  matrix:
  os: [ubuntu-latest]
- go: [1.19.1]
+ go: [1.19]
  runs-on: ${{ matrix.os }}
  steps:
  - uses: actions/checkout@v3
@@ -38,7 +38,7 @@ jobs:
  strategy:
  matrix:
  os: [ubuntu-latest]
- go: [1.18.6, 1.19.1]
+ go: [1.18.6, 1.19]
  runs-on: ${{ matrix.os }}
  steps:
  - uses: actions/checkout@v3
@@ -56,7 +56,7 @@ jobs:
  strategy:
  matrix:
  os: [ubuntu-latest]
- go: [1.18.6, 1.19.1]
+ go: [1.18.6, 1.19]
  pg: [9.6.5, 10]
  runs-on: ${{ matrix.os }}
  services:

.github/workflows/horizon-release.yml

@@ -22,7 +22,7 @@ jobs:
 
  - uses: ./.github/actions/setup-go
  with:
- go-version: 1.19.1
+ go-version: 1.19
 
  - name: Check dependencies
  run: ./gomod.sh

.github/workflows/horizon.yml

@@ -12,7 +12,7 @@ jobs:
  strategy:
  matrix:
  os: [ubuntu-latest]
- go: [1.18.6, 1.19.1]
+ go: [1.18.6, 1.19]
  pg: [9.6.5]
  ingestion-backend: [db, captive-core, captive-core-remote-storage]
  protocol-version: [18, 19]

exp/orderbook/batch.go

@@ -87,15 +87,19 @@ func (tx *orderBookBatchedUpdates) apply(ledger uint32) error {
  return errUnexpectedLedger
  }
 
+ reallocatePairs := map[tradingPair]struct{}{}
+
  for _, operation := range tx.operations {
  switch operation.operationType {
  case addOfferOperationType:
  if err := tx.orderbook.addOffer(*operation.offer); err != nil {
  panic(errors.Wrap(err, "could not apply update in batch"))
  }
  case removeOfferOperationType:
- if _, ok := tx.orderbook.tradingPairForOffer[operation.offerID]; !ok {
+ if pair, ok := tx.orderbook.tradingPairForOffer[operation.offerID]; !ok {
  continue
+ } else {
+ reallocatePairs[pair] = struct{}{}
  }
  if err := tx.orderbook.removeOffer(operation.offerID); err != nil {
  panic(errors.Wrap(err, "could not apply update in batch"))
@@ -114,5 +118,9 @@ func (tx *orderBookBatchedUpdates) apply(ledger uint32) error {
 
  tx.orderbook.lastLedger = ledger
 
+ for pair := range reallocatePairs {
+ tx.orderbook.venuesForSellingAsset[pair.sellingAsset].reallocate()
+ tx.orderbook.venuesForBuyingAsset[pair.buyingAsset].reallocate()
+ }
  return nil
 }

exp/orderbook/edges.go

@@ -15,6 +15,10 @@ type edgeSet []edge
 type edge struct {
  key int32
  value Venues
+
+ // reallocate is set to true when some offers were removed from the edge.
+ // See edgeSet.reallocate() godoc for more information.
+ reallocate bool
 }
 
 func (e edgeSet) find(key int32) int {
@@ -85,6 +89,7 @@ func (e edgeSet) removeOffer(key int32, offerID xdr.Int64) (edgeSet, bool) {
  return slices.Delete(e, i, i+1), true
  }
  e[i].value.offers = updatedOffers
+ e[i].reallocate = true
  return e, true
 }
 
@@ -101,3 +106,18 @@ func (e edgeSet) removePool(key int32) edgeSet {
  e[i].value = Venues{offers: e[i].value.offers}
  return e
 }
+
+// reallocate recreates offers slice when edge.reallocate is set to true and
+// this is true after an offer is removed.
+// Without periodic reallocations an arbitrary account could create 1000s of
+// offers in an orderbook, then remove them but the space occupied by these
+// offers would not be released by GC because an array used internally is
+// the same. This can lead to DoS attack by OOM.
+func (e edgeSet) reallocate() {
+ for i := 0; i < len(e); i++ {
+ if e[i].reallocate {
+ e[i].value.offers = append([]xdr.OfferEntry(nil), e[i].value.offers[:]...)
+ e[i].reallocate = false
+ }
+ }
+}

exp/orderbook/edges_test.go

@@ -0,0 +1,46 @@
+package orderbook
+
+import (
+ "runtime"
+ "testing"
+
+ "github.com/stellar/go/xdr"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestRemoveOffersMemoryUsage(t *testing.T) {
+ edges := edgeSet{}
+ for i := 0; i < 2000; i++ {
+ edges = edges.addOffer(1, xdr.OfferEntry{
+ SellerId: xdr.MustAddress("GCZFUQEPMLGUE2NB5RR7C3I2LTTLOEBM7GYD7PDKI4SU5HHWTDB553WD"),
+ OfferId: xdr.Int64(i),
+ })
+ }
+
+ var afterAdded, afterRemoved, afterReallocate runtime.MemStats
+ runtime.ReadMemStats(&afterAdded)
+
+ t.Logf("after added: %d\n", afterAdded.HeapInuse)
+
+ // Remove all offers except one
+ for i := 0; i < 2000-1; i++ {
+ var removed bool
+ edges, removed = edges.removeOffer(1, xdr.Int64(i))
+ require.True(t, removed)
+
+ }
+
+ runtime.ReadMemStats(&afterRemoved)
+ t.Logf("after removed: %d\n", afterRemoved.HeapInuse)
+
+ require.True(t, edges[0].reallocate)
+ edges.reallocate()
+ runtime.GC()
+
+ runtime.ReadMemStats(&afterReallocate)
+ t.Logf("after reallocate: %d\n", afterReallocate.HeapInuse)
+
+ assert.Less(t, afterReallocate.HeapInuse, afterAdded.HeapInuse)
+ assert.Less(t, afterReallocate.HeapInuse, afterRemoved.HeapInuse)
+}

exp/orderbook/graph_test.go

@@ -463,22 +463,26 @@ func TestAddOffersOrderBook(t *testing.T) {
  {
  assetStringToID[usdAsset.String()],
  makeVenues(quarterOffer, fiftyCentsOffer, dollarOffer),
+ false,
  },
  {
  assetStringToID[eurAsset.String()],
  makeVenues(eurOffer, twoEurOffer, threeEurOffer),
+ false,
  },
  },
  {
  {
  assetStringToID[eurAsset.String()],
  makeVenues(eurUsdOffer, otherEurUsdOffer),
+ false,
  },
  },
  {
  {
  assetStringToID[usdAsset.String()],
  makeVenues(usdEurOffer),
+ false,
  },
  },
  },
@@ -488,20 +492,24 @@ func TestAddOffersOrderBook(t *testing.T) {
  {
  assetStringToID[eurAsset.String()],
  makeVenues(usdEurOffer),
+ false,
  },
  {
  assetStringToID[nativeAsset.String()],
  makeVenues(quarterOffer, fiftyCentsOffer, dollarOffer),
+ false,
  },
  },
  {
  {
  assetStringToID[usdAsset.String()],
  makeVenues(eurUsdOffer, otherEurUsdOffer),
+ false,
  },
  {
  assetStringToID[nativeAsset.String()],
  makeVenues(eurOffer, twoEurOffer, threeEurOffer),
+ false,
  },
  },
  },
@@ -705,22 +713,26 @@ func TestUpdateOfferOrderBook(t *testing.T) {
  {
  assetStringToID[usdAsset.String()],
  makeVenues(quarterOffer, fiftyCentsOffer, dollarOffer),
+ false,
  },
  {
  assetStringToID[eurAsset.String()],
  makeVenues(eurOffer, twoEurOffer, threeEurOffer),
+ false,
  },
  },
  {
  {
  assetStringToID[eurAsset.String()],
  makeVenues(otherEurUsdOffer, eurUsdOffer),
+ false,
  },
  },
  {
  {
  assetStringToID[usdAsset.String()],
  makeVenues(usdEurOffer),
+ false,
  },
  },
  },
@@ -730,20 +742,24 @@ func TestUpdateOfferOrderBook(t *testing.T) {
  {
  assetStringToID[nativeAsset.String()],
  makeVenues(quarterOffer, fiftyCentsOffer, dollarOffer),
+ false,
  },
  {
  assetStringToID[eurAsset.String()],
  makeVenues(usdEurOffer),
+ false,
  },
  },
  {
  {
  assetStringToID[nativeAsset.String()],
  makeVenues(eurOffer, twoEurOffer, threeEurOffer),
+ false,
  },
  {
  assetStringToID[usdAsset.String()],
  makeVenues(otherEurUsdOffer, eurUsdOffer),
+ false,
  },
  },
  },
@@ -876,16 +892,19 @@ func TestRemoveOfferOrderBook(t *testing.T) {
  {
  assetStringToID[usdAsset.String()],
  makeVenues(quarterOffer, fiftyCentsOffer),
+ false,
  },
  {
  assetStringToID[eurAsset.String()],
  makeVenues(eurOffer, twoEurOffer, threeEurOffer),
+ false,
  },
  },
  {
  {
  assetStringToID[eurAsset.String()],
  makeVenues(eurUsdOffer),
+ false,
  },
  },
  {},
@@ -896,16 +915,19 @@ func TestRemoveOfferOrderBook(t *testing.T) {
  {
  assetStringToID[nativeAsset.String()],
  makeVenues(quarterOffer, fiftyCentsOffer),
+ false,
  },
  },
  {
  {
  assetStringToID[nativeAsset.String()],
  makeVenues(eurOffer, twoEurOffer, threeEurOffer),
+ false,
  },
  {
  assetStringToID[usdAsset.String()],
  makeVenues(eurUsdOffer),
+ false,
  },
  },
  },

services/horizon/CHANGELOG.md

@@ -6,10 +6,16 @@ file. This project adheres to [Semantic Versioning](https://semver.org/).
 
 ## Unreleased
 
-### Fixes
+## 2.23.0
+
+**Upgrading to this version will trigger a state rebuild. During this process, Horizon will not ingest new ledgers.**
 
-* The ingestion subsystem will now properly use a pool of history archives if more than one is provided ([#4687](https://github.com/stellar/go/pull/4687)).
+### Fixes
 
+* Improve performance of `/claimable_balances` filters. This change should significantly improve `?asset=` and `?claimant=` filters. ([#4690](https://github.com/stellar/go/pull/4690)).
+* Reallocate slices after offer removals in order book graph. This is done to prevent keeping a large chunks of allocated but unused memory that can lead to OOM crash.
+* The ingestion subsystem will now properly use a pool of history archives if more than one is provided. ([#4687](https://github.com/stellar/go/pull/4687))
+* Add `horizon ingest build-state` command which builds state at a specific ledger. Useful for debugging. ([#4636](https://github.com/stellar/go/pull/4636))
 
 ## 2.22.1