Cleans up chifra chunks index --pin and some test cases

CHANGES.md

@@ -3,13 +3,32 @@
 
 This file details changes made to TrueBlocks per version. See the [migration notes](./MIGRATIONS.md) for changes from previous versions.
 
-## v0.65.0 (2023/05/10)
+## v0.65.0 (2023/05/17)
 
 Added optional gRPC server for `chifra names` route.
+Correction to the Unchained Index after block 15,300,000 - requires a re-run of `chifra init -all` or a re-scrape of the entire index.
+Adds --dry_run (unimplemented) to `chifra init`
+Cleaned up a few data model markdown files
+Bumped version to 0.64.1.
+Removed abiSource from Function data model
+Removed C++ testing for acctlib which is no longer needed
+Adds `--unripe` option to `chifra list`
+Improves auto-gen code with a small cleanup
+Cleans up reporting on "bogus" token transfers (i.e., token transfer events without a corresponding change in balance)
+Removes a number of "fields" from transaction export including datesh, time, day, week, etc.
+Removes dups from command line for chifra export
+Enabled better reporting in verbose mode for chifra list
+Allow for using dates in command line for chifra state
+Fix to scraper getting stuck when there are no new blocks
+Full support for `--pin` option in `chifra chunks index`. Tested and ready to go.
+A whole bunch of cleanup across multiple packages. Lots of smoke, no real fire.
+Creates an optional gRPC server for names. This was put in place for future enhancements.
+Updates the Manifest to correct a previous error -- requires a migration
+A lot of improvements including `--first_block / --last_block` and `--max_addrs` options to aid debugging.
 
 ## v0.64.0 (2023/04/20)
 
-This is not a minor release that closes a few random issues and adds a small feature to better manage the caches.
+This is a minor release that closes a few random issues and adds a small feature to better manage the caches.
 
 ## Issues closed
 

src/apps/chifra/internal/chunks/handle_addresses_belongs.go

@@ -94,6 +94,10 @@ func (opts *ChunksOptions) handleResolvedRecords(modelChan chan types.Modeler[ty
  }
  }
  s.Appearances = good
+ s.AddressRecord.Count = uint32(len(good))
+ }
+ if len(s.Appearances) == 0 {
+ continue
  }
  modelChan <- &s
  cnt++

src/apps/chifra/internal/chunks/handle_pin_manifest.go

@@ -3,10 +3,13 @@ package chunksPkg
 import (
  "context"
  "fmt"
+ "os"
  "strings"
  "time"
 
+ "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/base"
  "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/cache"
+ "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/colors"
  "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/index"
  "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger"
  "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/output"
@@ -17,6 +20,8 @@ import (
 )
 
 func (opts *ChunksOptions) HandlePinManifest(blockNums []uint64) error {
+ firstBlock := mustParseUint(os.Getenv("TB_CHUNK_PIN_FIRST_BLOCK"))
+
  ctx, cancel := context.WithCancel(context.Background())
  fetchData := func(modelChan chan types.Modeler[types.RawModeler], errorChan chan error) {
  man := simpleManifest{
@@ -32,6 +37,15 @@ func (opts *ChunksOptions) HandlePinManifest(blockNums []uint64) error {
  }
 
  pinChunk := func(walker *index.CacheWalker, path string, first bool) (bool, error) {
+ rng, err := base.RangeFromFilenameE(path)
+ if err != nil {
+ return false, err
+ }
+ if rng.First < firstBlock {
+ logger.Info("Skipping", path)
+ return true, nil
+ }
+
  if path != cache.ToBloomPath(path) {
  return false, fmt.Errorf("should not happen in pinChunk")
  }
@@ -43,14 +57,21 @@ func (opts *ChunksOptions) HandlePinManifest(blockNums []uint64) error {
 
  if pinning.LocalDaemonRunning() {
  man.Chunks = append(man.Chunks, result.Local)
- logger.Progress(true, "Pinning:", result.Local, spaces)
- } else {
+ // logger.Progress(true, colors.Green+"Pinned local: ", result.Local, spaces, colors.Off)
+ }
+
+ if opts.Remote {
  man.Chunks = append(man.Chunks, result.Remote)
- logger.Progress(true, "Pinning:", result.Remote, spaces)
+ // logger.Progress(true, colors.BrightBlue+"Pinned remote:", result.Remote, spaces, colors.Off)
  }
 
- if opts.Globals.Verbose {
- logger.Progress(true, "Pinning:", path)
+ if !result.Matches {
+ logger.Warn("Local and remote pins do not match")
+ logger.Warn(colors.Yellow+result.Local.BloomHash.String(), "-", result.Local.IndexHash, colors.Off)
+ logger.Warn(colors.Yellow+result.Remote.BloomHash.String(), "-", result.Remote.IndexHash, colors.Off)
+ logger.Fatal("Failed")
+ } else if opts.Remote && pinning.LocalDaemonRunning() {
+ logger.Info(colors.BrightGreen+"Matches: "+result.Remote.BloomHash.String(), "-", result.Remote.IndexHash, colors.Off)
  }
 
  sleep := opts.Sleep

src/apps/chifra/internal/chunks/validate.go

@@ -119,7 +119,7 @@ func (opts *ChunksOptions) validateChunks() error {
  return err
  }
 
- if opts.FirstBlock != 0 || opts.LastBlock != utils.NOPOS || opts.MaxAddrs > 0 {
+ if opts.FirstBlock != 0 || opts.LastBlock != utils.NOPOS || opts.MaxAddrs != utils.NOPOS {
  if opts.FirstBlock >= opts.LastBlock {
  msg := fmt.Sprintf("first_block (%d) must be strictly earlier than last_block (%d).", opts.FirstBlock, opts.LastBlock)
  return validate.Usage(msg)

src/apps/chifra/internal/names/handle_terms.go

@@ -18,6 +18,7 @@ func (opts *NamesOptions) HandleTerms() error {
  var fetchData func(modelChan chan types.Modeler[types.RawName], errorChan chan error)
 
  apiMode := opts.Globals.IsApiMode()
+ testMode := opts.Globals.TestMode
 
  var conn *grpc.ClientConn
  var client proto.NamesClient
@@ -38,8 +39,12 @@ func (opts *NamesOptions) HandleTerms() error {
  } else {
  // Report the error only if we know that the server is running or the user wants us
  // to be verbose
- if grpcErr != nil && (!errors.Is(grpcErr, proto.ErrServerNotRunning) || opts.Globals.Verbose) {
- logger.Error("gRPC connection error:", grpcErr)
+ if grpcErr != nil {
+ if errors.Is(grpcErr, proto.ErrServerNotRunning) && opts.Globals.Verbose {
+ logger.Warn(grpcErr, "falling back to file-based search")
+ } else if !testMode {
+ logger.Error("gRPC connection error:", grpcErr)
+ }
  }
 
  namesArray, err := names.LoadNamesArray(opts.Globals.Chain, opts.getType(), names.SortByAddress, opts.Terms)

src/apps/chifra/pkg/pinning/pin_file.go

@@ -3,6 +3,7 @@ package pinning
 import (
  "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/base"
  "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/cache"
+ "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/colors"
  "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/config"
  "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/file"
  "github.com/TrueBlocks/trueblocks-core/src/apps/chifra/pkg/logger"
@@ -22,27 +23,36 @@ type PinResult struct {
 // TODO: BOGUS - WE HAVE TO HAVE A SOLUTION FOR THE TIMESTAMP FILE --PIN --REMOTE ON THE CHIFRA WHEN ROUTINES?
 func PinTimestamps(chain string, isRemote bool) error {
  path := config.GetPathToIndex(chain) + "ts.bin"
- var hash base.IpfsHash
+ var localHash base.IpfsHash
+ var remoteHash base.IpfsHash
  var err error
  if LocalDaemonRunning() {
+ logger.Info("Pinning timestamps to local daemon")
  localService, _ := NewPinningService(chain, Local)
- if hash, err = localService.PinFile(path, true); err != nil {
+ if localHash, err = localService.PinFile(path, true); err != nil {
  logger.Fatal("Error in PinTimestamps", err)
  return err
  }
  }
 
  if isRemote {
+ logger.Info("Pinning timestamps to remote daemon")
  remoteService, _ := NewPinningService(chain, Pinata)
- if hash, err = remoteService.PinFile(path, true); err != nil {
+ if remoteHash, err = remoteService.PinFile(path, true); err != nil {
  logger.Fatal("Error in PinTimestamps", err)
  return err
  }
  }
 
  fn := config.GetPathToCache(chain) + "/tmp/ts.ipfs_hash.fil"
- logger.Info("Pinning timestamp file", fn, "to", hash)
- file.StringToAsciiFile(fn, hash.String())
+ logger.Info("Pinning timestamp file", fn, "to", localHash)
+ file.StringToAsciiFile(fn, localHash.String())
+ if LocalDaemonRunning() && isRemote {
+ if localHash != remoteHash {
+ logger.Info("Local and remote hashes differ in PinTimestamps", localHash, remoteHash)
+ }
+ }
+
  return nil
 }
 
@@ -62,6 +72,7 @@ func PinChunk(chain, path string, isRemote bool) (PinResult, error) {
 
  isLocal := LocalDaemonRunning()
  if isLocal {
+ // logger.Info(colors.Magenta+"Pinning locally...", colors.Off)
  if result.Local.BloomHash, result.err = localService.PinFile(bloomFile, true); result.err != nil {
  return PinResult{}, result.err
  }
@@ -70,9 +81,11 @@ func PinChunk(chain, path string, isRemote bool) (PinResult, error) {
  return PinResult{}, result.err
  }
  result.Local.IndexSize = file.FileSize(indexFile)
+ logger.Info(colors.Magenta+"Pinned", rng, "local to ", result.Local.BloomHash, result.Local.IndexHash, colors.Off)
  }
 
  if isRemote {
+ logger.Info(colors.Magenta+"Pinning remotely...", colors.Off)
  if result.Remote.BloomHash, result.err = remoteService.PinFile(bloomFile, false); result.err != nil {
  return PinResult{}, result.err
  }
@@ -81,6 +94,7 @@ func PinChunk(chain, path string, isRemote bool) (PinResult, error) {
  return PinResult{}, result.err
  }
  result.Remote.IndexSize = file.FileSize(indexFile)
+ logger.Info(colors.Magenta+"Pinned", rng, "remote to", result.Remote.BloomHash, result.Remote.IndexHash, colors.Off)
  }
 
  // TODO: We used to use this to report an error between local and remote pinning, but it got turned off. Turn it back on

src/apps/chifra/pkg/types/types_function.go

@@ -104,7 +104,7 @@ func (s *SimpleFunction) Model(verbose bool, format string, extraOptions map[str
  }
  return result
  }
- if extraOptions["verbose"] == true {
+ if verbose {
  inputs := getParameterModels(s.Inputs)
  if inputs != nil {
  model["inputs"] = inputs

test/gold/tools/ethNames/ethNames_fromFile.txt

@@ -2,29 +2,35 @@ chifra names --file names.txt
 TEST[DATE|TIME] Terms: [LockTrip]
 TEST[DATE|TIME] File: names.txt
 TEST[DATE|TIME] Format: txt
+EROR[DATE|TIME] gRPC connection error: gRPC server not running
 WARN[DATE|TIME] No results for [chifra names --file names.txt]
 TEST[DATE|TIME] Terms: [UCASH]
 TEST[DATE|TIME] NoHeader: true
 TEST[DATE|TIME] File: names.txt
 TEST[DATE|TIME] Format: txt
+EROR[DATE|TIME] gRPC connection error: gRPC server not running
 50-Tokens:ERC20 0x92e52a1a235d9a103d970901066ce910aacefd37 UCASH UCASH 8 physically-fun-perch
 TEST[DATE|TIME] Terms: [XES]
 TEST[DATE|TIME] NoHeader: true
 TEST[DATE|TIME] File: names.txt
 TEST[DATE|TIME] Format: txt
+EROR[DATE|TIME] gRPC connection error: gRPC server not running
 50-Tokens:ERC20 0xa017ac5fac5941f95010b12570b812c974469c2c Proxeus XES 18 factually-prepared-weevil
 TEST[DATE|TIME] Terms: [b052f8a33d8bb0]
 TEST[DATE|TIME] NoHeader: true
 TEST[DATE|TIME] File: names.txt
 TEST[DATE|TIME] Format: txt
+EROR[DATE|TIME] gRPC connection error: gRPC server not running
 50-Tokens:ERC20 0xb052f8a33d8bb068414eade06af6955199f9f010 EcoRealEstate ECOREAL 18 only-growing-spider
 TEST[DATE|TIME] Terms: [68414eade06af6955199f9f010]
 TEST[DATE|TIME] NoHeader: true
 TEST[DATE|TIME] File: names.txt
 TEST[DATE|TIME] Format: txt
+EROR[DATE|TIME] gRPC connection error: gRPC server not running
 50-Tokens:ERC20 0xb052f8a33d8bb068414eade06af6955199f9f010 EcoRealEstate ECOREAL 18 only-growing-spider
 TEST[DATE|TIME] Terms: [0xb7cb1c96db6b22b0d3d9536e0108d062bd488f74 WTC Walton]
 TEST[DATE|TIME] NoHeader: true
 TEST[DATE|TIME] File: names.txt
 TEST[DATE|TIME] Format: txt
+EROR[DATE|TIME] gRPC connection error: gRPC server not running
 50-Tokens:ERC20 0xb7cb1c96db6b22b0d3d9536e0108d062bd488f74 Walton Token WTC 18 briefly-delicate-fawn

test/gold/tools/ethNames/ethNames_prefund_custom_search.txt

@@ -4,7 +4,7 @@ TEST[DATE|TIME] Custom: true
 TEST[DATE|TIME] Prefund: true
 TEST[DATE|TIME] Verbose: true
 TEST[DATE|TIME] Format: txt
-EROR[DATE|TIME] gRPC connection error: gRPC server not running
+WARN[DATE|TIME] gRPC server not running falling back to file-based search
 tags address name symbol source decimals petname
 81-Custom 0x0000000000000000000000000000000000000001 Account_1 SYM_1 Testing 1 abnormally-able-ant
 80-Prefund 0x001762430ea9c3a26e5749afdb70da5f78ddbb8c Prefund_0001 Genesis blatantly-distinct-crow

test/gold/tools/ethNames/ethNames_simple_verbose.txt

@@ -2,7 +2,7 @@ chifra names etwork --verbose
 TEST[DATE|TIME] Terms: [etwork]
 TEST[DATE|TIME] Verbose: true
 TEST[DATE|TIME] Format: txt
-EROR[DATE|TIME] gRPC connection error: gRPC server not running
+WARN[DATE|TIME] gRPC server not running falling back to file-based search
 tags address name symbol decimals petname
 31-Gitcoin:Grants 0x011c2c1433b36a524b548444ae2f20c015427d04 Grant 0491 - Tokamak Network: Decentralized, Scalable, Turing Complete Layer 2 Plasma brightly-ultimate-bengal
 31-Giveth:Project 0x035495b687850f79cb1652f416ea7ec19907557c Bees network plainly-climbing-coral