Organize scripts and python code

hacking_testing/minetest_env.py → minetests/minetest_env.py

@@ -3,159 +3,16 @@
 import os
 import random
 import shutil
-import subprocess
 import uuid
 from typing import Any, Dict, List, Optional, Tuple
 
 import gym
 import matplotlib.pyplot as plt
 import numpy as np
-import proto_python.objects_pb2 as pb_objects
 import psutil
 import zmq
-from proto_python.objects_pb2 import KeyType
-
-# Define default keys / buttons
-KEY_MAP = {
- "FORWARD": KeyType.FORWARD,
- "BACKWARD": KeyType.BACKWARD,
- "LEFT": KeyType.LEFT,
- "RIGHT": KeyType.RIGHT,
- "JUMP": KeyType.JUMP,
- "SNEAK": KeyType.SNEAK, # shift key in menus
- "DIG": KeyType.DIG, # left mouse button
- "MIDDLE": KeyType.MIDDLE, # middle mouse button
- "PLACE": KeyType.PLACE, # right mouse button
- "DROP": KeyType.DROP,
- "HOTBAR_NEXT": KeyType.HOTBAR_NEXT, # mouse wheel up
- "HOTBAR_PREV": KeyType.HOTBAR_PREV, # mouse wheel down
- "SLOT_1": KeyType.SLOT_1,
- "SLOT_2": KeyType.SLOT_2,
- "SLOT_3": KeyType.SLOT_3,
- "SLOT_4": KeyType.SLOT_4,
- "SLOT_5": KeyType.SLOT_5,
- "SLOT_6": KeyType.SLOT_6,
- "SLOT_7": KeyType.SLOT_7,
- "SLOT_8": KeyType.SLOT_8,
- "INVENTORY": KeyType.INVENTORY,
-}
-INV_KEY_MAP = {value: key for key, value in KEY_MAP.items()}
-
-# Define noop action
-NOOP_ACTION = {key: 0 for key in KEY_MAP.keys()}
-NOOP_ACTION.update({"MOUSE": np.zeros(2, dtype=int)})
-
-
-def unpack_pb_obs(received_obs: str):
- pb_obs = pb_objects.Observation()
- pb_obs.ParseFromString(received_obs)
- obs = np.frombuffer(pb_obs.image.data, dtype=np.uint8).reshape(
- pb_obs.image.height,
- pb_obs.image.width,
- 3,
- )
- last_action = unpack_pb_action(pb_obs.action) if pb_obs.action else None
- rew = pb_obs.reward
- done = pb_obs.terminal
- # TODO receive extra infos
- info = {}
- return obs, rew, done, info, last_action
-
-
-def unpack_pb_action(pb_action: pb_objects.Action):
- action = dict(NOOP_ACTION)
- action["MOUSE"] = [pb_action.mouseDx, pb_action.mouseDy]
- for key_event in pb_action.keyEvents:
- if key_event.key in INV_KEY_MAP and key_event.eventType == pb_objects.PRESS:
- key_name = INV_KEY_MAP[key_event.key]
- action[key_name] = 1
- return action
-
-
-def pack_pb_action(action: Dict[str, Any]):
- pb_action = pb_objects.Action()
- pb_action.mouseDx, pb_action.mouseDy = action["MOUSE"]
- for key, v in action.items():
- if key == "MOUSE":
- continue
- pb_action.keyEvents.append(
- pb_objects.KeyboardEvent(
- key=KEY_MAP[key],
- eventType=pb_objects.PRESS if v else pb_objects.RELEASE,
- ),
- )
- return pb_action
-
-
-def start_minetest_server(
- minetest_path: str = "bin/minetest",
- config_path: str = "minetest.conf",
- log_path: str = "log/{}.log",
- server_port: int = 30000,
- world_dir: str = "newworld",
-):
- cmd = [
- minetest_path,
- "--server",
- "--world",
- world_dir,
- "--gameid",
- "minetest", # TODO does this have to be unique?
- "--config",
- config_path,
- "--port",
- str(server_port),
- ]
- stdout_file = log_path.format("server_stdout")
- stderr_file = log_path.format("server_stderr")
- with open(stdout_file, "w") as out, open(stderr_file, "w") as err:
- server_process = subprocess.Popen(cmd, stdout=out, stderr=err)
- return server_process
-
-
-def start_minetest_client(
- minetest_path: str = "bin/minetest",
- config_path: str = "minetest.conf",
- log_path: str = "log/{}.log",
- client_port: int = 5555,
- server_port: int = 30000,
- cursor_img: str = "cursors/mouse_cursor_white_16x16.png",
- client_name: str = "MinetestAgent",
- xvfb_headless: bool = False,
-):
- cmd = [
- minetest_path,
- "--name",
- client_name,
- "--password",
- "1234",
- "--address",
- "0.0.0.0", # listen to all interfaces
- "--port",
- str(server_port),
- "--go",
- "--dumb",
- "--client-address",
- "tcp:https://localhost:" + str(client_port),
- "--record",
- "--noresizing",
- "--config",
- config_path,
- ]
- if xvfb_headless:
- # hide window
- cmd.insert(0, "-a") # allow restarts
- cmd.insert(0, "xvfb-run")
- # don't render to screen
- cmd.append("--headless")
- if cursor_img:
- cmd.extend(["--cursor-image", cursor_img])
-
- stdout_file = log_path.format("client_stdout")
- stderr_file = log_path.format("client_stderr")
- with open(stdout_file, "w") as out, open(stderr_file, "w") as err:
- client_process = subprocess.Popen(cmd, stdout=out, stderr=err)
- return client_process
+from minetests.utils import (KEY_MAP, pack_pb_action, start_minetest_client,
+ start_minetest_server, unpack_pb_obs)
 
 
 class Minetest(gym.Env):

hacking_testing/test_loop.py → minetests/scripts/test_loop.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-from minetest_env import Minetest
+from minetests.minetest_env import Minetest
 
 env = Minetest(
  seed=42,
@@ -8,7 +8,7 @@
  clientmods=["random_v0"],
 )
 
-render = False
+render = True
 obs = env.reset()
 done = False
 while not done:

hacking_testing/test_loop_parallel.py → minetests/scripts/test_loop_parallel.py

@@ -1,7 +1,7 @@
 from typing import Any, Dict, Optional
 
 from gym.wrappers import TimeLimit
-from minetest_env import Minetest
+from minetests.minetest_env import Minetest
 from stable_baselines3.common.vec_env import DummyVecEnv, SubprocVecEnv
 
 if __name__ == "__main__":

minetests/utils.py

@@ -0,0 +1,148 @@
+import subprocess
+from typing import Any, Dict
+
+import numpy as np
+from minetests.proto import objects_pb2 as pb_objects
+from minetests.proto.objects_pb2 import KeyType
+
+# Define default keys / buttons
+KEY_MAP = {
+ "FORWARD": KeyType.FORWARD,
+ "BACKWARD": KeyType.BACKWARD,
+ "LEFT": KeyType.LEFT,
+ "RIGHT": KeyType.RIGHT,
+ "JUMP": KeyType.JUMP,
+ "SNEAK": KeyType.SNEAK, # shift key in menus
+ "DIG": KeyType.DIG, # left mouse button
+ "MIDDLE": KeyType.MIDDLE, # middle mouse button
+ "PLACE": KeyType.PLACE, # right mouse button
+ "DROP": KeyType.DROP,
+ "HOTBAR_NEXT": KeyType.HOTBAR_NEXT, # mouse wheel up
+ "HOTBAR_PREV": KeyType.HOTBAR_PREV, # mouse wheel down
+ "SLOT_1": KeyType.SLOT_1,
+ "SLOT_2": KeyType.SLOT_2,
+ "SLOT_3": KeyType.SLOT_3,
+ "SLOT_4": KeyType.SLOT_4,
+ "SLOT_5": KeyType.SLOT_5,
+ "SLOT_6": KeyType.SLOT_6,
+ "SLOT_7": KeyType.SLOT_7,
+ "SLOT_8": KeyType.SLOT_8,
+ "INVENTORY": KeyType.INVENTORY,
+}
+INV_KEY_MAP = {value: key for key, value in KEY_MAP.items()}
+
+# Define noop action
+NOOP_ACTION = {key: 0 for key in KEY_MAP.keys()}
+NOOP_ACTION.update({"MOUSE": np.zeros(2, dtype=int)})
+
+
+def unpack_pb_obs(received_obs: str):
+ pb_obs = pb_objects.Observation()
+ pb_obs.ParseFromString(received_obs)
+ obs = np.frombuffer(pb_obs.image.data, dtype=np.uint8).reshape(
+ pb_obs.image.height,
+ pb_obs.image.width,
+ 3,
+ )
+ last_action = unpack_pb_action(pb_obs.action) if pb_obs.action else None
+ rew = pb_obs.reward
+ done = pb_obs.terminal
+ # TODO receive extra infos
+ info = {}
+ return obs, rew, done, info, last_action
+
+
+def unpack_pb_action(pb_action: pb_objects.Action):
+ action = dict(NOOP_ACTION)
+ action["MOUSE"] = [pb_action.mouseDx, pb_action.mouseDy]
+ for key_event in pb_action.keyEvents:
+ if key_event.key in INV_KEY_MAP and key_event.eventType == pb_objects.PRESS:
+ key_name = INV_KEY_MAP[key_event.key]
+ action[key_name] = 1
+ return action
+
+
+def pack_pb_action(action: Dict[str, Any]):
+ pb_action = pb_objects.Action()
+ pb_action.mouseDx, pb_action.mouseDy = action["MOUSE"]
+ for key, v in action.items():
+ if key == "MOUSE":
+ continue
+ pb_action.keyEvents.append(
+ pb_objects.KeyboardEvent(
+ key=KEY_MAP[key],
+ eventType=pb_objects.PRESS if v else pb_objects.RELEASE,
+ ),
+ )
+ return pb_action
+
+
+def start_minetest_server(
+ minetest_path: str = "bin/minetest",
+ config_path: str = "minetest.conf",
+ log_path: str = "log/{}.log",
+ server_port: int = 30000,
+ world_dir: str = "newworld",
+):
+ cmd = [
+ minetest_path,
+ "--server",
+ "--world",
+ world_dir,
+ "--gameid",
+ "minetest", # TODO does this have to be unique?
+ "--config",
+ config_path,
+ "--port",
+ str(server_port),
+ ]
+ stdout_file = log_path.format("server_stdout")
+ stderr_file = log_path.format("server_stderr")
+ with open(stdout_file, "w") as out, open(stderr_file, "w") as err:
+ server_process = subprocess.Popen(cmd, stdout=out, stderr=err)
+ return server_process
+
+
+def start_minetest_client(
+ minetest_path: str = "bin/minetest",
+ config_path: str = "minetest.conf",
+ log_path: str = "log/{}.log",
+ client_port: int = 5555,
+ server_port: int = 30000,
+ cursor_img: str = "cursors/mouse_cursor_white_16x16.png",
+ client_name: str = "MinetestAgent",
+ xvfb_headless: bool = False,
+):
+ cmd = [
+ minetest_path,
+ "--name",
+ client_name,
+ "--password",
+ "1234",
+ "--address",
+ "0.0.0.0", # listen to all interfaces
+ "--port",
+ str(server_port),
+ "--go",
+ "--dumb",
+ "--client-address",
+ "tcp:https://localhost:" + str(client_port),
+ "--record",
+ "--noresizing",
+ "--config",
+ config_path,
+ ]
+ if xvfb_headless:
+ # hide window
+ cmd.insert(0, "-a") # allow restarts
+ cmd.insert(0, "xvfb-run")
+ # don't render to screen
+ cmd.append("--headless")
+ if cursor_img:
+ cmd.extend(["--cursor-image", cursor_img])
+
+ stdout_file = log_path.format("client_stdout")
+ stderr_file = log_path.format("client_stderr")
+ with open(stdout_file, "w") as out, open(stderr_file, "w") as err:
+ client_process = subprocess.Popen(cmd, stdout=out, stderr=err)
+ return client_process

scripts/compile_proto.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+protoc -I=../proto/ --python_out=../minetests/proto --cpp_out=../src ../proto/*.proto

hacking_testing/data_recorder.py → scripts/data_recorder.py

@@ -1,7 +1,7 @@
 import os
 
 import zmq
-from minetest_env import unpack_pb_obs
+from minetests.utils import unpack_pb_obs
 
 
 class DataRecorder:

scripts/record_client.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+exec bin/minetest \
+ --name MinetestAgent \
+ --password whyisthisnecessary \
+ --address 0.0.0.0 \
+ --port 30000 \
+ --go \
+ --client-address "tcp:https://*:5555" \
+ --record \
+ --noresizing \
+ --cursor-image "cursors/mouse_cursor_white_16x16.png" \
+ --config scripts/minetest.conf

scripts/server.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+exec bin/minetest \
+ --server \
+ --world newworld \
+ --gameid minetest \
+ --config scripts/minetest.conf