Enable state dumping and loading

.circleci/config.yml

@@ -41,6 +41,9 @@ jobs:
  - run:
  name: Test postman
  command: poetry run pytest test/test_postman.py
+ - run:
+ name: Test dumping/loading
+ command: poetry run pytest test/test_dump.py
  - run:
  name: Test plugin - dockerized
  command: ./test/test_plugin.sh

README.md

@@ -23,7 +23,6 @@ brew install gmp
 
 ## Disclaimer
 - Devnet should not be used as a replacement for Alpha testnet. After testing on Devnet, be sure to test on testnet!
-- Hash calculation of transactions and blocks differs from the one used in Alpha testnet.
 - Specifying a block by its hash/number is not supported. All interaction is done with the latest block.
 - Read more in [interaction](#interaction-api).
 
@@ -39,14 +38,21 @@ optional arguments:
  -v, --version Print the version
  --host HOST Specify the address to listen at; defaults to localhost (use the address the program outputs on start)
  --port PORT, -p PORT Specify the port to listen at; defaults to 5000
+ --load-path LOAD_PATH
+ Specify the path from which the state is loaded on
+ startup
+ --dump-path DUMP_PATH
+ Specify the path to dump to
+ --dump-on DUMP_ON Specify when to dump; can dump on: exit, transaction
 ```
 
-## Run - Docker
+## Run with Docker
 Devnet is available as a Docker container ([shardlabs/starknet-devnet](https://hub.docker.com/repository/docker/shardlabs/starknet-devnet)):
 ```text
 docker pull shardlabs/starknet-devnet
 ```
 
+### Host and port with Docker
 The server inside the container listens to the port 5000, which you need to publish to a desired `<PORT>` on your host machine:
 ```text
 docker run -it -p [HOST:]<PORT>:5000 shardlabs/starknet-devnet
@@ -104,6 +110,60 @@ constructor(MockStarknetMessaging mockStarknetMessaging_) public {
 }
 ```
 
+## Dumping
+To preserve your Devnet instance for future use, there are several options:
+
+- Dumping on exit (handles Ctrl+C, i.e. SIGINT, doesn't handle SIGKILL):
+```
+starknet-devnet --dump-on exit --dump-path <PATH>
+```
+
+- Dumping after each transaction (done in background, doesn't block):
+```
+starknet-devnet --dump-on transaction --dump-path <PATH>
+```
+
+- Dumping on request (replace `<HOST>`, `<PORT>` and `<PATH>` with your own):
+```
+curl -X POST http:https://<HOST>:<PORT>/dump -d '{ "path": <PATH> }' -H "Content-Type: application/json"
+```
+
+## Loading
+To load a preserved Devnet instance, run:
+```
+starknet-devnet --load-path <PATH>
+```
+
+## Enabling dumping and loading with Docker
+To enable dumping and loading if running Devnet in a Docker container, you must bind the container path with the path on your host machine.
+
+This example:
+- Relies on [Docker bind mount](https://docs.docker.com/storage/bind-mounts/); try [Docker volume](https://docs.docker.com/storage/volumes/) instead.
+- Assumes that `/actual/dumpdir` exists. If unsure, use absolute paths.
+- Assumes you are listening on `127.0.0.1:5000`. However, leave the `--host 0.0.0.0` part as it is.
+
+If there is `dump.pkl` inside `/actual/dumpdir`, you can load it with:
+```
+docker run -it \
+ -p 127.0.0.1:5000:5000 \
+ --mount type=bind,source=/actual/dumpdir,target=/dumpdir \
+ shardlabs/starknet-devnet \
+ poetry run starknet-devnet \
+ --host 0.0.0.0 --port 5000 \
+ --load-path /dumpdir/dump.pkl
+```
+
+To dump to `/actual/dumpdir/dump.pkl` on Devnet shutdown, run:
+```
+docker run -it \
+ -p 127.0.0.1:5000:5000 \
+ --mount type=bind,source=/actual/dumpdir,target=/dumpdir \
+ shardlabs/starknet-devnet \
+ poetry run starknet-devnet \
+ --host 0.0.0.0 --port 5000 \
+ --dump-on exit --dump-path /dumpdir/dump.pkl
+```
+
 ## Development - Prerequisite
 If you're a developer willing to contribute, be sure to have installed [Poetry](https://pypi.org/project/poetry/).
 

pyproject.toml

@@ -15,6 +15,7 @@ python = "^3.7"
 Flask = {extras = ["async"], version = "^2.0.2"}
 flask-cors = "^3.0.10"
 cairo-lang = "0.7.1"
+dill = "^0.3.4"
 
 [tool.poetry.dev-dependencies]
 pylint = "^2.12.2"

starknet_devnet/dump.py

@@ -0,0 +1,43 @@
+"""Dumping utilities."""
+
+import multiprocessing
+
+import dill as pickle
+
+from .util import DumpOn
+
+def perform_dump(dumpable, path):
+ """Performs the very act of dumping."""
+ with open(path, "wb") as file:
+ pickle.dump(dumpable, file)
+
+class Dumper:
+ """Class for dumping objects."""
+
+ def __init__(self, dumpable):
+ """Specify the `dumpable` object to be dumped."""
+
+ self.dumpable = dumpable
+
+ self.dump_path: str = None
+ """Where to dump."""
+
+ self.dump_on: DumpOn = None
+ """When to dump."""
+
+ def dump(self, path: str):
+ """Dump to `path`."""
+ path = path or self.dump_path
+ assert path, "No dump_path defined"
+ print("Dumping Devnet to:", path)
+
+ multiprocessing.Process(
+ target=perform_dump,
+ args=[self.dumpable, path]
+ ).start()
+ # don't .join(), let it run in background
+
+ def dump_if_required(self):
+ """Dump only if specified in self.dump_on."""
+ if self.dump_on == DumpOn.TRANSACTION:
+ self.dump(self.dump_path)

starknet_devnet/server.py

@@ -4,6 +4,9 @@
 
 import os
 import json
+import signal
+import sys
+import dill as pickle
 
 from flask import Flask, request, jsonify, abort
 from flask.wrappers import Response
@@ -15,8 +18,9 @@
 from werkzeug.datastructures import MultiDict
 
 from .constants import CAIRO_LANG_VERSION
+from .dump import Dumper
 from .starknet_wrapper import StarknetWrapper
-from .util import custom_int, fixed_length_hex, parse_args
+from .util import DumpOn, custom_int, fixed_length_hex, parse_args
 
 app = Flask(__name__)
 CORS(app)
@@ -43,7 +47,9 @@ async def add_transaction():
  else:
  abort(Response(f"Invalid tx_type: {tx_type}.", 400))
 
+ # after tx
  await starknet_wrapper.postman_flush()
+ dumper.dump_if_required()
 
  return jsonify({
  "code": StarkErrorCode.TRANSACTION_RECEIVED.name,
@@ -204,19 +210,54 @@ def validate_load_messaging_contract(request_dict: dict):
  abort(Response(error_message, 400))
  return network_url
 
+@app.route("/dump", methods=["POST"])
+def dump():
+ """Dumps the starknet_wrapper"""
+
+ request_dict = request.json or {}
+ dump_path = request_dict.get("path") or dumper.dump_path
+ if not dump_path:
+ abort(Response("No path provided", 400))
+
+ dumper.dump(dump_path)
+ return Response(status=200)
+
+def dump_on_exit(_signum, _frame):
+ """Dumps on exit."""
+ dumper.dump(dumper.dump_path)
+ sys.exit(0)
+
 starknet_wrapper = StarknetWrapper()
+dumper = Dumper(starknet_wrapper)
 
 def main():
  """Runs the server."""
 
+ # pylint: disable=global-statement, invalid-name
+ global starknet_wrapper
+
  # reduce startup logging
  os.environ["WERKZEUG_RUN_MAIN"] = "true"
 
  args = parse_args()
+
  # Uncomment this once fork support is added
  # origin = Origin(args.fork) if args.fork else NullOrigin()
  # starknet_wrapper.set_origin(origin)
 
+ if args.load_path:
+ try:
+ starknet_wrapper = StarknetWrapper.load(args.load_path)
+ except (FileNotFoundError, pickle.UnpicklingError):
+ sys.exit(f"Error: Cannot load from {args.load_path}. Make sure the file exists and contains a Devnet dump.")
+
+ if args.dump_on == DumpOn.EXIT:
+ signal.signal(signal.SIGTERM, dump_on_exit)
+ signal.signal(signal.SIGINT, dump_on_exit)
+
+ dumper.dump_path = args.dump_path
+ dumper.dump_on = args.dump_on
+
  app.run(host=args.host, port=args.port)
 
 if __name__ == "__main__":

starknet_devnet/starknet_wrapper.py

@@ -7,6 +7,7 @@
 from copy import deepcopy
 from typing import Dict
 
+import dill as pickle
 from starkware.starknet.business_logic.internal_transaction import InternalInvokeFunction
 from starkware.starknet.business_logic.state import CarriedState
 from starkware.starknet.definitions.transaction_type import TransactionType
@@ -18,13 +19,16 @@
 from starkware.starknet.services.api.feeder_gateway.block_hash import calculate_block_hash
 
 from .origin import NullOrigin, Origin
-from .util import Choice, StarknetDevnetException, TxStatus, fixed_length_hex, DummyExecutionInfo
+from .util import Choice, StarknetDevnetException, TxStatus, fixed_length_hex, DummyExecutionInfo, enable_pickling
 from .contract_wrapper import ContractWrapper
 from .transaction_wrapper import TransactionWrapper, DeployTransactionWrapper, InvokeTransactionWrapper
 from .postman_wrapper import GanachePostmanWrapper
 from .constants import FAILURE_REASON_KEY
 
-class StarknetWrapper: # pylint: disable=too-many-instance-attributes
+enable_pickling()
+
+#pylint: disable=too-many-instance-attributes
+class StarknetWrapper:
  """
  Wraps a Starknet instance and stores data to be returned by the server:
  contract states, transactions, blocks, storages.
@@ -55,6 +59,12 @@ def __init__(self):
  self.__l1_provider = None
  """Saves the L1 URL being used for L1 <> L2 communication."""
 
+ @classmethod
+ def load(cls, path: str) -> "StarknetWrapper":
+ """Load a serialized instance of this class from `path`."""
+ with open(path, "rb") as file:
+ return pickle.load(file)
+
  async def __preserve_current_state(self, state: CarriedState):
  self.__current_carried_state = deepcopy(state)
  self.__current_carried_state.shared_state = state.shared_state