diff --git a/.circleci/config.yml b/.circleci/config.yml index cee07e451..5ea1dc2b0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -41,6 +41,9 @@ jobs: - run: name: Test postman command: poetry run pytest test/test_postman.py + - run: + name: Test dumping/loading + command: poetry run pytest test/test_dump.py - run: name: Test plugin - dockerized command: ./test/test_plugin.sh diff --git a/.pylintrc b/.pylintrc index 5d310c5ec..02dd8c7cb 100644 --- a/.pylintrc +++ b/.pylintrc @@ -1,2 +1,5 @@ [FORMAT] max-line-length=150 + +[BASIC] +min-public-methods=1 diff --git a/README.md b/README.md index e18acf93a..ff446b680 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,6 @@ brew install gmp ## Disclaimer - Devnet should not be used as a replacement for Alpha testnet. After testing on Devnet, be sure to test on testnet! -- Hash calculation of transactions and blocks differs from the one used in Alpha testnet. - Specifying a block by its hash/number is not supported. All interaction is done with the latest block. - Read more in [interaction](#interaction-api). @@ -39,14 +38,21 @@ optional arguments: -v, --version Print the version --host HOST Specify the address to listen at; defaults to localhost (use the address the program outputs on start) --port PORT, -p PORT Specify the port to listen at; defaults to 5000 + --load-path LOAD_PATH + Specify the path from which the state is loaded on + startup + --dump-path DUMP_PATH + Specify the path to dump to + --dump-on DUMP_ON Specify when to dump; can dump on: exit, transaction ``` -## Run - Docker +## Run with Docker Devnet is available as a Docker container ([shardlabs/starknet-devnet](https://hub.docker.com/repository/docker/shardlabs/starknet-devnet)): ```text docker pull shardlabs/starknet-devnet ``` +### Host and port with Docker The server inside the container listens to the port 5000, which you need to publish to a desired `` on your host machine: ```text docker run -it -p [HOST:]:5000 shardlabs/starknet-devnet @@ -104,6 +110,60 @@ constructor(MockStarknetMessaging mockStarknetMessaging_) public { } ``` +## Dumping +To preserve your Devnet instance for future use, there are several options: + +- Dumping on exit (handles Ctrl+C, i.e. SIGINT, doesn't handle SIGKILL): +``` +starknet-devnet --dump-on exit --dump-path +``` + +- Dumping after each transaction (done in background, doesn't block): +``` +starknet-devnet --dump-on transaction --dump-path +``` + +- Dumping on request (replace ``, `` and `` with your own): +``` +curl -X POST http://:/dump -d '{ "path": }' -H "Content-Type: application/json" +``` + +## Loading +To load a preserved Devnet instance, run: +``` +starknet-devnet --load-path +``` + +## Enabling dumping and loading with Docker +To enable dumping and loading if running Devnet in a Docker container, you must bind the container path with the path on your host machine. + +This example: +- Relies on [Docker bind mount](https://docs.docker.com/storage/bind-mounts/); try [Docker volume](https://docs.docker.com/storage/volumes/) instead. +- Assumes that `/actual/dumpdir` exists. If unsure, use absolute paths. +- Assumes you are listening on `127.0.0.1:5000`. However, leave the `--host 0.0.0.0` part as it is. + +If there is `dump.pkl` inside `/actual/dumpdir`, you can load it with: +``` +docker run -it \ + -p 127.0.0.1:5000:5000 \ + --mount type=bind,source=/actual/dumpdir,target=/dumpdir \ + shardlabs/starknet-devnet \ + poetry run starknet-devnet \ + --host 0.0.0.0 --port 5000 \ + --load-path /dumpdir/dump.pkl +``` + +To dump to `/actual/dumpdir/dump.pkl` on Devnet shutdown, run: +``` +docker run -it \ + -p 127.0.0.1:5000:5000 \ + --mount type=bind,source=/actual/dumpdir,target=/dumpdir \ + shardlabs/starknet-devnet \ + poetry run starknet-devnet \ + --host 0.0.0.0 --port 5000 \ + --dump-on exit --dump-path /dumpdir/dump.pkl +``` + ## Development - Prerequisite If you're a developer willing to contribute, be sure to have installed [Poetry](https://pypi.org/project/poetry/). diff --git a/poetry.lock b/poetry.lock index 1f3354c4f..ca5edb62d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -211,6 +211,17 @@ toolz = ">=0.8.0" [package.extras] cython = ["cython"] +[[package]] +name = "dill" +version = "0.3.4" +description = "serialize all of python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*" + +[package.extras] +graph = ["objgraph (>=1.7.2)"] + [[package]] name = "ecdsa" version = "0.17.0" @@ -1116,7 +1127,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "cde3f4e66424a8dda07da89f6398b07eac66b02f4f554549fbc3be13ae64c2e6" +content-hash = "1f3974c7a7d4363f8e897b06297be2399f911a5d73462b1f8103d8ef5f1488da" [metadata.files] aiohttp = [ @@ -1254,6 +1265,10 @@ colorama = [ cytoolz = [ {file = "cytoolz-0.11.2.tar.gz", hash = "sha256:ea23663153806edddce7e4153d1d407d62357c05120a4e8485bddf1bd5ab22b4"}, ] +dill = [ + {file = "dill-0.3.4-py2.py3-none-any.whl", hash = "sha256:7e40e4a70304fd9ceab3535d36e58791d9c4a776b38ec7f7ec9afc8d3dca4d4f"}, + {file = "dill-0.3.4.zip", hash = "sha256:9f9734205146b2b353ab3fec9af0070237b6ddae78452af83d2fca84d739e675"}, +] ecdsa = [ {file = "ecdsa-0.17.0-py2.py3-none-any.whl", hash = "sha256:5cf31d5b33743abe0dfc28999036c849a69d548f994b535e527ee3cb7f3ef676"}, {file = "ecdsa-0.17.0.tar.gz", hash = "sha256:b9f500bb439e4153d0330610f5d26baaf18d17b8ced1bc54410d189385ea68aa"}, diff --git a/pyproject.toml b/pyproject.toml index dc068a8b1..c49a576fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ python = "^3.7" Flask = {extras = ["async"], version = "^2.0.2"} flask-cors = "^3.0.10" cairo-lang = "0.7.1" +dill = "^0.3.4" [tool.poetry.dev-dependencies] pylint = "^2.12.2" diff --git a/starknet_devnet/dump.py b/starknet_devnet/dump.py new file mode 100644 index 000000000..3cfe9efcb --- /dev/null +++ b/starknet_devnet/dump.py @@ -0,0 +1,38 @@ +"""Dumping utilities.""" + +import multiprocessing + +import dill as pickle + +from .util import DumpOn + +class Dumper: + """Class for dumping objects.""" + + def __init__(self, dumpable): + """Specify the `dumpable` object to be dumped.""" + + self.dumpable = dumpable + + self.dump_path: str = None + """Where to dump.""" + + self.dump_on: DumpOn = None + """When to dump.""" + + def __write_file(self, path): + """Writes the dump to disk.""" + with open(path, "wb") as file: + pickle.dump(self.dumpable, file) + + def dump(self, path: str=None): + """Dump to `path`.""" + path = path or self.dump_path + assert path, "No dump_path defined" + print("Dumping Devnet to:", path) + + multiprocessing.Process( + target=self.__write_file, + args=[path] + ).start() + # don't .join(), let it run in background diff --git a/starknet_devnet/server.py b/starknet_devnet/server.py index 619559aa0..5c8975869 100644 --- a/starknet_devnet/server.py +++ b/starknet_devnet/server.py @@ -4,6 +4,9 @@ import os import json +import signal +import sys +import dill as pickle from flask import Flask, request, jsonify, abort from flask.wrappers import Response @@ -15,8 +18,9 @@ from werkzeug.datastructures import MultiDict from .constants import CAIRO_LANG_VERSION +from .dump import Dumper from .starknet_wrapper import StarknetWrapper -from .util import custom_int, fixed_length_hex, parse_args +from .util import DumpOn, custom_int, fixed_length_hex, parse_args app = Flask(__name__) CORS(app) @@ -43,7 +47,10 @@ async def add_transaction(): else: abort(Response(f"Invalid tx_type: {tx_type}.", 400)) + # after tx await starknet_wrapper.postman_flush() + if dumper.dump_on == DumpOn.TRANSACTION: + dumper.dump() return jsonify({ "code": StarkErrorCode.TRANSACTION_RECEIVED.name, @@ -204,19 +211,54 @@ def validate_load_messaging_contract(request_dict: dict): abort(Response(error_message, 400)) return network_url +@app.route("/dump", methods=["POST"]) +def dump(): + """Dumps the starknet_wrapper""" + + request_dict = request.json or {} + dump_path = request_dict.get("path") or dumper.dump_path + if not dump_path: + abort(Response("No path provided", 400)) + + dumper.dump(dump_path) + return Response(status=200) + +def dump_on_exit(_signum, _frame): + """Dumps on exit.""" + dumper.dump(dumper.dump_path) + sys.exit(0) + starknet_wrapper = StarknetWrapper() +dumper = Dumper(starknet_wrapper) def main(): """Runs the server.""" + # pylint: disable=global-statement, invalid-name + global starknet_wrapper + # reduce startup logging os.environ["WERKZEUG_RUN_MAIN"] = "true" args = parse_args() + # Uncomment this once fork support is added # origin = Origin(args.fork) if args.fork else NullOrigin() # starknet_wrapper.set_origin(origin) + if args.load_path: + try: + starknet_wrapper = StarknetWrapper.load(args.load_path) + except (FileNotFoundError, pickle.UnpicklingError): + sys.exit(f"Error: Cannot load from {args.load_path}. Make sure the file exists and contains a Devnet dump.") + + if args.dump_on == DumpOn.EXIT: + for sig in [signal.SIGTERM, signal.SIGINT]: + signal.signal(sig, dump_on_exit) + + dumper.dump_path = args.dump_path + dumper.dump_on = args.dump_on + app.run(host=args.host, port=args.port) if __name__ == "__main__": diff --git a/starknet_devnet/starknet_wrapper.py b/starknet_devnet/starknet_wrapper.py index 1263e61f5..ccacd2d08 100644 --- a/starknet_devnet/starknet_wrapper.py +++ b/starknet_devnet/starknet_wrapper.py @@ -7,6 +7,7 @@ from copy import deepcopy from typing import Dict +import dill as pickle from starkware.starknet.business_logic.internal_transaction import InternalInvokeFunction from starkware.starknet.business_logic.state import CarriedState from starkware.starknet.definitions.transaction_type import TransactionType @@ -18,13 +19,16 @@ from starkware.starknet.services.api.feeder_gateway.block_hash import calculate_block_hash from .origin import NullOrigin, Origin -from .util import Choice, StarknetDevnetException, TxStatus, fixed_length_hex, DummyExecutionInfo +from .util import Choice, StarknetDevnetException, TxStatus, fixed_length_hex, DummyExecutionInfo, enable_pickling from .contract_wrapper import ContractWrapper from .transaction_wrapper import TransactionWrapper, DeployTransactionWrapper, InvokeTransactionWrapper from .postman_wrapper import GanachePostmanWrapper from .constants import FAILURE_REASON_KEY -class StarknetWrapper: # pylint: disable=too-many-instance-attributes +enable_pickling() + +#pylint: disable=too-many-instance-attributes +class StarknetWrapper: """ Wraps a Starknet instance and stores data to be returned by the server: contract states, transactions, blocks, storages. @@ -55,6 +59,12 @@ def __init__(self): self.__l1_provider = None """Saves the L1 URL being used for L1 <> L2 communication.""" + @staticmethod + def load(path: str) -> "StarknetWrapper": + """Load a serialized instance of this class from `path`.""" + with open(path, "rb") as file: + return pickle.load(file) + async def __preserve_current_state(self, state: CarriedState): self.__current_carried_state = deepcopy(state) self.__current_carried_state.shared_state = state.shared_state diff --git a/starknet_devnet/util.py b/starknet_devnet/util.py index 0e1e64787..2d7552bc4 100644 --- a/starknet_devnet/util.py +++ b/starknet_devnet/util.py @@ -5,8 +5,11 @@ from dataclasses import dataclass from enum import Enum, auto import argparse +import sys from starkware.starkware_utils.error_handling import StarkException +from starkware.starknet.testing.contract import StarknetContract + from . import __version__ class TxStatus(Enum): @@ -66,6 +69,20 @@ def fixed_length_hex(arg: int) -> str: # # otherwise a URL; perhaps check validity # return name +class DumpOn(Enum): + """Enumerate possible dumping frequencies.""" + EXIT = auto() + TRANSACTION = auto() + +DUMP_ON_OPTIONS = [e.name.lower() for e in DumpOn] +DUMP_ON_OPTIONS_STRINGIFIED = ", ".join(DUMP_ON_OPTIONS) + +def parse_dump_on(option: str): + """Parse dumping frequency option.""" + if option in DUMP_ON_OPTIONS: + return DumpOn[option.upper()] + sys.exit(f"Error: Invalid --dump-on option: {option}. Valid options: {DUMP_ON_OPTIONS_STRINGIFIED}") + DEFAULT_HOST = "localhost" DEFAULT_PORT = 5000 def parse_args(): @@ -91,6 +108,19 @@ def parse_args(): help=f"Specify the port to listen at; defaults to {DEFAULT_PORT}", default=DEFAULT_PORT ) + parser.add_argument( + "--load-path", + help="Specify the path from which the state is loaded on startup" + ) + parser.add_argument( + "--dump-path", + help="Specify the path to dump to" + ) + parser.add_argument( + "--dump-on", + help=f"Specify when to dump; can dump on: {DUMP_ON_OPTIONS_STRINGIFIED}", + type=parse_dump_on + ) # Uncomment this once fork support is added # parser.add_argument( # "--fork", "-f", @@ -99,7 +129,11 @@ def parse_args(): # "or network name (alpha or alpha-mainnet)", # ) - return parser.parse_args() + args = parser.parse_args() + if args.dump_on and not args.dump_path: + sys.exit("Error: --dump-path required if --dump-on present") + + return args class StarknetDevnetException(StarkException): """ @@ -123,3 +157,16 @@ def __init__(self): self.retdata = [] self.internal_calls = [] self.l2_to_l1_messages = [] + +def enable_pickling(): + """ + Extends the `StarknetContract` class to enable pickling. + """ + def contract_getstate(self): + return self.__dict__ + + def contract_setstate(self, state): + self.__dict__ = state + + StarknetContract.__getstate__ = contract_getstate + StarknetContract.__setstate__ = contract_setstate diff --git a/test/test_dump.py b/test/test_dump.py new file mode 100644 index 000000000..c591a9dca --- /dev/null +++ b/test/test_dump.py @@ -0,0 +1,186 @@ +""" +Test server state serialization (dumping/loading). +""" + +import os +import signal +import time +import requests + +import pytest + +from .util import call, deploy, invoke, run_devnet_in_background +from .settings import GATEWAY_URL + +ARTIFACTS_PATH = "starknet-hardhat-example/starknet-artifacts/contracts" +CONTRACT_PATH = f"{ARTIFACTS_PATH}/contract.cairo/contract.json" +ABI_PATH = f"{ARTIFACTS_PATH}/contract.cairo/contract_abi.json" +DUMP_PATH = "dump.pkl" + +@pytest.fixture(autouse=True) +def run_before_and_after_test(): + """Cleanup after tests finish.""" + + # before test + # nothing + + yield + + # after test + if os.path.isfile(DUMP_PATH): + os.remove(DUMP_PATH) + +def send_dump_request(dump_path: str=None): + """Send HTTP request to trigger dumping.""" + json_load = { "path": dump_path } if dump_path else None + return requests.post(f"{GATEWAY_URL}/dump", json=json_load) + +def assert_dump_present(dump_path: str, sleep_seconds=2): + """Assert there is a non-empty dump file.""" + time.sleep(sleep_seconds) + assert os.path.isfile(dump_path) + assert os.path.getsize(dump_path) > 0 + +def assert_no_dump_present(dump_path: str, sleep_seconds=2): + """Assert there is no dump file.""" + time.sleep(sleep_seconds) + assert not os.path.isfile(dump_path) + +def dump_and_assert(dump_path: str=None): + """Assert no dump file before dump and assert some dump file after dump.""" + assert_no_dump_present(dump_path) + resp = send_dump_request(dump_path) + assert resp.status_code == 200 + assert_dump_present(dump_path) + +def assert_not_alive(): + """Assert devnet is not alive.""" + try: + requests.get(f"{GATEWAY_URL}/is_alive") + raise RuntimeError("Should have failed before this line.") + except requests.exceptions.ConnectionError: + pass + +def deploy_empty_contract(): + """ + Deploy sample contract with balance = 0. + Returns contract address. + """ + deploy_dict = deploy(CONTRACT_PATH, inputs=["0"]) + contract_address = deploy_dict["address"] + initial_balance = call("get_balance", contract_address, ABI_PATH) + assert initial_balance == "0" + return contract_address + +def test_load_if_no_file(): + """Test loading if dump file not present.""" + assert_no_dump_present(DUMP_PATH) + devnet_proc = run_devnet_in_background("--load-path", DUMP_PATH) + devnet_proc.wait() + + assert devnet_proc.returncode != 0 + expected_msg = f"Error: Cannot load from {DUMP_PATH}. Make sure the file exists and contains a Devnet dump.\n" + assert devnet_proc.stderr.read().decode("utf-8") == expected_msg + +def test_dumping_if_path_not_provided(): + """Assert failure if dumping attempted without a known path.""" + devnet_proc = run_devnet_in_background() + resp = send_dump_request() + assert resp.status_code == 400 + devnet_proc.kill() + +def test_dumping_if_path_provided_as_cli_option(): + """Test dumping if path provided as CLI option""" + devnet_proc = run_devnet_in_background("--dump-path", DUMP_PATH) + resp = send_dump_request() + assert resp.status_code == 200 + assert_dump_present(DUMP_PATH) + devnet_proc.kill() + +def test_dumping_via_endpoint(): + """Test dumping via endpoint.""" + # init devnet + contract + devnet_proc = run_devnet_in_background() + contract_address = deploy_empty_contract() + + invoke("increase_balance", ["10", "20"], contract_address, ABI_PATH) + balance_after_invoke = call("get_balance", contract_address, ABI_PATH) + assert balance_after_invoke == "30" + + dump_and_assert(DUMP_PATH) + + devnet_proc.kill() + assert_not_alive() + + # spawn new devnet, load from dump path + loaded_devnet_proc = run_devnet_in_background("--load-path", DUMP_PATH) + loaded_balance = call("get_balance", contract_address, ABI_PATH) + assert loaded_balance == balance_after_invoke + + # assure that new invokes can be made + invoke("increase_balance", ["15", "25"], contract_address, ABI_PATH) + balance_after_invoke_on_loaded = call("get_balance", contract_address, abi_path=ABI_PATH) + assert balance_after_invoke_on_loaded == "70" + + os.remove(DUMP_PATH) + loaded_devnet_proc.kill() + assert_no_dump_present(DUMP_PATH) + +def test_dumping_on_exit(): + """Test dumping on exit.""" + devnet_proc = run_devnet_in_background("--dump-on", "exit", "--dump-path", DUMP_PATH) + contract_address = deploy_empty_contract() + + invoke("increase_balance", ["10", "20"], contract_address, ABI_PATH) + balance_after_invoke = call("get_balance", contract_address, ABI_PATH) + assert balance_after_invoke == "30" + + assert_no_dump_present(DUMP_PATH) + devnet_proc.send_signal(signal.SIGINT) # simulate Ctrl+C because devnet can't handle kill + assert_dump_present(DUMP_PATH, sleep_seconds=3) + +def test_invalid_dump_on_option(): + """Test behavior when invalid dump-on is provided.""" + devnet_proc = run_devnet_in_background("--dump-on", "obviously-invalid", "--dump-path", DUMP_PATH) + devnet_proc.wait() + + assert devnet_proc.returncode != 0 + expected_msg = b"Error: Invalid --dump-on option: obviously-invalid. Valid options: exit, transaction\n" + assert devnet_proc.stderr.read() == expected_msg + +def test_dump_path_not_present_with_dump_on_present(): + """Test behavior when dump-path is not present and dump-on is.""" + devnet_proc = run_devnet_in_background("--dump-on", "exit") + devnet_proc.wait() + + assert devnet_proc.returncode != 0 + expected_msg = b"Error: --dump-path required if --dump-on present\n" + assert devnet_proc.stderr.read() == expected_msg + +def assert_load(dump_path: str, contract_address: str, expected_value: str): + """Load from `dump_path` and assert get_balance at `contract_address` returns `expected_value`.""" + devnet_loaded_proc = run_devnet_in_background("--load-path", dump_path) + assert call("get_balance", contract_address, ABI_PATH) == expected_value + devnet_loaded_proc.kill() + os.remove(dump_path) + +def test_dumping_on_each_tx(): + """Test dumping on each transaction.""" + devnet_proc = run_devnet_in_background("--dump-on", "transaction", "--dump-path", DUMP_PATH) + + # deploy + contract_address = deploy_empty_contract() + assert_dump_present(DUMP_PATH) + dump_after_deploy_path = "dump_after_deploy.pkl" + os.rename(DUMP_PATH, dump_after_deploy_path) + + # invoke + invoke("increase_balance", ["5", "5"], contract_address, ABI_PATH) + assert_dump_present(DUMP_PATH) + dump_after_invoke_path = "dump_after_invoke.pkl" + os.rename(DUMP_PATH, dump_after_invoke_path) + + devnet_proc.kill() + + assert_load(dump_after_deploy_path, contract_address, "0") + assert_load(dump_after_invoke_path, contract_address, "10") diff --git a/test/test_plugin.sh b/test/test_plugin.sh index 99aff59d1..98cead04a 100755 --- a/test/test_plugin.sh +++ b/test/test_plugin.sh @@ -48,4 +48,4 @@ if [ ! -f "$TEST_FILE" ]; then exit 1 fi -npx hardhat test "$TEST_FILE" +npx hardhat test --no-compile "$TEST_FILE" diff --git a/test/util.py b/test/util.py index 96b14dc7f..b2e4b4f15 100644 --- a/test/util.py +++ b/test/util.py @@ -14,13 +14,19 @@ class ReturnCodeAssertionError(AssertionError): """Error to be raised when the return code of an executed process is not as expected.""" -def run_devnet_in_background(sleep_seconds=0): - """Run starknet-devnet in background. Return the process handle. Optionally sleep.""" - command = ["poetry", "run", "starknet-devnet", "--host", HOST, "--port", PORT] +def run_devnet_in_background(*args, sleep_seconds=3): + """ + Runs starknet-devnet in background. + By default sleeps 3 second after spawning devnet. + Accepts extra args to pass to `starknet-devnet` command. + Returns the process handle. + """ + command = ["poetry", "run", "starknet-devnet", "--host", HOST, "--port", PORT, *args] # pylint: disable=consider-using-with - proc = subprocess.Popen(command, close_fds=True) + proc = subprocess.Popen(command, close_fds=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) time.sleep(sleep_seconds) atexit.register(proc.kill) + return proc def assert_equal(actual, expected, explanation=None): """Assert that the two values are equal. Optionally provide explanation."""