calculate multiple times the creation block of a contract and select …

…the lower one, use provider manager on API layer

api/api.go

@@ -19,6 +19,7 @@ import (
  "github.com/vocdoni/census3/helpers/queue"
  "github.com/vocdoni/census3/helpers/web3"
  "github.com/vocdoni/census3/scanner/providers"
+ "github.com/vocdoni/census3/scanner/providers/manager"
  web3provider "github.com/vocdoni/census3/scanner/providers/web3"
  "go.vocdoni.io/dvote/api/censusdb"
  storagelayer "go.vocdoni.io/dvote/data"
@@ -40,7 +41,7 @@ type Census3APIConf struct {
  DataDir string
  GroupKey string
  Web3Providers *web3.Web3Pool
- HolderProviders map[uint64]providers.HolderProvider
+ HolderProviders *manager.ProviderManager
  AdminToken string
 }
 
@@ -53,7 +54,7 @@ type census3API struct {
  w3p *web3.Web3Pool
  storage storagelayer.Storage
  downloader *downloader.Downloader
- holderProviders map[uint64]providers.HolderProvider
+ holderProviders *manager.ProviderManager
  cache *lru.Cache[CacheKey, any]
  router *httprouter.HTTProuter
 }
@@ -238,13 +239,14 @@ func (capi *census3API) CreateInitialTokens(tokensPath string) error {
 
  // get the correct holder provider for the token type
  tokenType := providers.TokenTypeID(token.Type)
- provider, exists := capi.holderProviders[tokenType]
- if !exists {
+ provider, err := capi.holderProviders.GetProvider(ctx, tokenType)
+ if err != nil {
  log.Warnw("token type provided in initial list not supported, check provider is set. SKIPPING...",
  "tokenID", token.ID,
  "chainID", token.ChainID,
  "externalID", token.ExternalID,
- "type", token.Type)
+ "type", token.Type,
+ "error", err)
  continue
  }
  if !provider.IsExternal() {

api/helpers.go

@@ -272,9 +272,9 @@ func (capi *census3API) CalculateStrategyHolders(ctx context.Context,
  Decimals: token.Decimals,
  ExternalID: t.ExternalID,
  }
- provider, exists := capi.holderProviders[token.TypeID]
- if !exists {
- return nil, nil, 0, fmt.Errorf("provider not found for token type id %d", token.TypeID)
+ provider, err := capi.holderProviders.GetProvider(ctx, token.TypeID)
+ if err != nil {
+ return nil, nil, 0, fmt.Errorf("provider not found for token type id %d: %w", token.TypeID, err)
  }
  if !provider.IsExternal() {
  if err := provider.SetRef(web3.Web3ProviderRef{

api/holders.go

@@ -106,9 +106,9 @@ func (capi *census3API) listHoldersAtLastBlock(address common.Address,
  return nil, 0, ErrCantGetTokenHolders.WithErr(err)
  }
  // if the token is external, return an error
- provider, exists := capi.holderProviders[tokenData.TypeID]
- if !exists {
- return nil, 0, ErrCantCreateCensus.With("token type not supported")
+ provider, err := capi.holderProviders.GetProvider(internalCtx, tokenData.TypeID)
+ if err != nil {
+ return nil, 0, ErrCantCreateCensus.WithErr(fmt.Errorf("token type not supported: %w", err))
  }
  if provider.IsExternal() {
  return nil, 0, ErrCantCreateCensus.With("not implemented for external providers")

api/tokens.go

@@ -9,8 +9,10 @@ import (
  "errors"
  "fmt"
  "math/big"
+ "net/http"
  "strconv"
  "strings"
+ "time"
 
  "github.com/ethereum/go-ethereum/common"
  "github.com/vocdoni/census3/db/annotations"
@@ -37,6 +39,10 @@ func (capi *census3API) initTokenHandlers() error {
  api.MethodAccessTypePublic, capi.getToken); err != nil {
  return err
  }
+ if err := capi.endpoint.RegisterMethod("/tokens/startblock", "POST",
+ api.MethodAccessTypePublic, capi.tokenStartBlock); err != nil {
+ return err
+ }
  if err := capi.endpoint.RegisterMethod("/tokens/{tokenID}", "DELETE",
  api.MethodAccessTypeAdmin, capi.launchDeleteToken); err != nil {
  return err
@@ -242,9 +248,9 @@ func (capi *census3API) createToken(msg *api.APIdata, ctx *httprouter.HTTPContex
  defer cancel()
  // get the correct holder provider for the token type
  tokenType := providers.TokenTypeID(req.Type)
- provider, exists := capi.holderProviders[tokenType]
- if !exists {
- return ErrCantCreateCensus.With("token type not supported")
+ provider, err := capi.holderProviders.GetProvider(internalCtx, tokenType)
+ if err != nil {
+ return ErrCantCreateCensus.WithErr(fmt.Errorf("token type not supported: %w", err))
  }
  if !provider.IsExternal() {
  if err := provider.SetRef(web3.Web3ProviderRef{
@@ -541,9 +547,9 @@ func (capi *census3API) getToken(msg *api.APIdata, ctx *httprouter.HTTPContext)
  atBlock := uint64(tokenData.LastBlock)
  tokenProgress := 100
  if !tokenData.Synced {
- provider, exists := capi.holderProviders[tokenData.TypeID]
- if !exists {
- return ErrCantCreateCensus.With("token type not supported")
+ provider, err := capi.holderProviders.GetProvider(internalCtx, tokenData.TypeID)
+ if err != nil {
+ return ErrCantCreateCensus.WithErr(fmt.Errorf("token type not supported: %w", err))
  }
  if !provider.IsExternal() {
  if err := provider.SetRef(web3.Web3ProviderRef{
@@ -601,6 +607,42 @@ func (capi *census3API) getToken(msg *api.APIdata, ctx *httprouter.HTTPContext)
  return ctx.Send(res, api.HTTPstatusOK)
 }
 
+func (capi *census3API) tokenStartBlock(msg *api.APIdata, ctx *httprouter.HTTPContext) error {
+ req := Token{}
+ if err := json.Unmarshal(msg.Data, &req); err != nil {
+ log.Errorf("error unmarshalling token information: %s", err)
+ return ErrMalformedToken.WithErr(err)
+ }
+ tokenType := providers.TokenTypeID(req.Type)
+ // get token information from the database
+ internalCtx, cancel := context.WithTimeout(ctx.Request.Context(), getTokenTimeout)
+ defer cancel()
+ provider, err := capi.holderProviders.GetProvider(internalCtx, tokenType)
+ if err != nil {
+ return ErrCantCreateCensus.WithErr(fmt.Errorf("token type not supported: %w", err))
+ }
+ if provider.IsExternal() {
+ return ctx.Send([]byte("type not supported"), http.StatusBadRequest)
+ }
+ if err := provider.SetRef(web3.Web3ProviderRef{
+ HexAddress: common.HexToAddress(req.ID).Hex(),
+ ChainID: req.ChainID,
+ }); err != nil {
+ return ErrInitializingWeb3.WithErr(err)
+ }
+ go func() {
+ bgCtx, cancel := context.WithTimeout(context.Background(), time.Minute*10)
+ defer cancel()
+ startBlock, err := provider.CreationBlock(bgCtx, nil)
+ if err != nil {
+ log.Error(err)
+ return
+ }
+ log.Infow("start block calculated", "startBlock", startBlock, "tokenID", req.ID, "chainID", req.ChainID)
+ }()
+ return ctx.Send([]byte("ok"), api.HTTPstatusOK)
+}
+
 func (capi *census3API) getTokenHolder(msg *api.APIdata, ctx *httprouter.HTTPContext) error {
  // get contract address from the tokenID query param and decode check if
  // it is provided, if not return an error
@@ -771,7 +813,7 @@ func (capi *census3API) enqueueTokenHoldersCSV(msg *api.APIdata, ctx *httprouter
 // supported types of token contracts.
 func (capi *census3API) getTokenTypes(msg *api.APIdata, ctx *httprouter.HTTPContext) error {
  supportedTypes := []string{}
- for _, provider := range capi.holderProviders {
+ for _, provider := range capi.holderProviders.Providers(ctx.Request.Context()) {
  supportedTypes = append(supportedTypes, provider.TypeName())
  }
  res, err := json.Marshal(TokenTypes{supportedTypes})

cmd/census3/main.go

@@ -214,7 +214,7 @@ func main() {
  DataDir: config.dataDir,
  Web3Providers: w3p,
  GroupKey: config.connectKey,
- HolderProviders: pm.Providers(ctx),
+ HolderProviders: pm,
  AdminToken: config.adminToken,
  })
  if err != nil {

scanner/providers/web3/web3_provider.go

@@ -63,15 +63,18 @@ func creationBlock(client *web3.Client, ctx context.Context, addr common.Address
  if err != nil {
  return 0, err
  }
- var creationBlock uint64
+ var minCreationBlock uint64
  for i := 0; i < web3.DefaultMaxWeb3ClientRetries; i++ {
- creationBlock, err = creationBlockInRange(client, ctx, addr, 0, lastBlock)
- if err == nil {
- break
+ creationBlock, err := creationBlockInRange(client, ctx, addr, 0, lastBlock)
+ if err != nil {
+ time.Sleep(RetryWeb3Cooldown)
+ continue
+ }
+ if minCreationBlock == 0 || creationBlock < minCreationBlock {
+ minCreationBlock = creationBlock
  }
- time.Sleep(RetryWeb3Cooldown)
  }
- return creationBlock, err
+ return minCreationBlock, err
 }
 
 // creationBlockInRange function finds the block number of a contract between