merge commit

.gitignore

@@ -37,6 +37,17 @@ build/.cmake/
 # GNU Patch reject file
 *.rej
 
+# Logs
+log/*
+
+# Protobuf
+**/*.pb.cc
+**/*.pb.h
+**/*_pb2.py
+
+# Python
+**/__pycache__/
+
 newworld/
 ## Non-static Minetest directories or symlinks to these
 /bin/

hacking_testing/client.sh

@@ -1,2 +1,2 @@
 #!/bin/bash
-exec bin/minetest --name me --password whyisthisnecessary --address 0.0.0.0 --port 30000 --go --dumb --dumb-port "5555"  # --record --record-port "tcp:https://*:9001"
+exec bin/minetest --name me --password whyisthisnecessary --address 0.0.0.0 --port 30000 --go --dumb --client-address "tcp:https://localhost:5555" --record --noresizing --cursor-image "cursors/mouse_cursor_white_16x16.png"

hacking_testing/compile_proto.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+protoc -I=../proto/ --python_out=proto_python/ --cpp_out=../src ../proto/**/*.proto

hacking_testing/minetest.conf

@@ -1,2 +1,7 @@
-default_privs = interact, shout
-developer_mode = false
+default_privs = interact, shout, debug
+developer_mode = false
+# TODO why are these options not working?
+# don't show debug info
+#show_debug = false
+# don't show chat
+#console_alpha = 0

hacking_testing/minetest_env.py

@@ -1,6 +1,20 @@
+import os
+import subprocess
+import uuid
+
 import gym
+import matplotlib.pyplot as plt
+import numpy as np
 import zmq
 from gym.spaces import Box, Dict, Discrete
+from proto_python.client import dumb_inputs_pb2 as dumb_inputs
+
+# TODO read from the minetest.conf file
+DISPLAY_SIZE = (1024, 600)
+FOV_Y = 72 # degrees
+FOV_X = FOV_Y * DISPLAY_SIZE[0] / DISPLAY_SIZE[1]
+MAX_MOUSE_MOVE_X = 180 / FOV_X * DISPLAY_SIZE[0]
+MAX_MOUSE_MOVE_Y = 180 / FOV_Y * DISPLAY_SIZE[1]
 
 KEYS = [
  "FORWARD",
@@ -10,7 +24,8 @@
  "JUMP",
  "SNEAK",
  "DIG", # left mouse
- "PLACE", # right mouse
+ "MIDDLE", # middle mouse
+ "PLACE", # right mouse
  "DROP",
  "HOTBAR_NEXT",
  "HOTBAR_PREVIOUS",
@@ -23,9 +38,9 @@
  "SLOT7",
  "SLOT8",
  # these keys open the inventory/menu
- # "ESC",
- # "INVENTORY",
- "AUX1",
+ "ESC",
+ "INVENTORY",
+ # "AUX1",
  # these keys lead to errors:
  # "CHAT", "CMD",
  # these keys are probably uninteresting
@@ -39,37 +54,201 @@
 ]
 
 
+def start_minetest_server(
+ minetest_path: str = "bin/minetest",
+ config_path: str = "minetest.conf",
+ log_dir: str = "log",
+ world: str = None,
+):
+ if world:
+ world_path = world
+ else:
+ world_path = str(uuid.uuid4())
+ cmd = [
+ minetest_path,
+ "--server",
+ "--world",
+ world_path,
+ "--gameid",
+ "minetest",
+ "--config",
+ config_path,
+ ]
+ os.makedirs(log_dir, exist_ok=True)
+ stdout_file = os.path.join(log_dir, "server_stdout.log")
+ stderr_file = os.path.join(log_dir, "server_stderr.log")
+ with open(stdout_file, "w") as out, open(stderr_file, "w") as err:
+ server_process = subprocess.Popen(cmd, stdout=out, stderr=err)
+ return server_process
+
+
+def start_minetest_client(
+ minetest_path: str = "bin/minetest",
+ log_dir: str = "log",
+ client_port: int = 5555,
+ cursor_img: str = "cursors/mouse_cursor_white_16x16.png",
+ client_name: str = "MinetestAgent",
+):
+ cmd = [
+ minetest_path,
+ "--name",
+ client_name,
+ "--password",
+ "1234",
+ "--address",
+ "0.0.0.0", # listen to all interfaces
+ "--port",
+ "30000", # TODO this should be the same as in minetest.conf
+ "--go",
+ "--dumb",
+ "--client-address",
+ "tcp:https://localhost:" + str(client_port),
+ "--record",
+ "--noresizing",
+ ]
+ if cursor_img:
+ cmd.extend(["--cursor-image", cursor_img])
+
+ os.makedirs(log_dir, exist_ok=True)
+ stdout_file = os.path.join(log_dir, "client_stdout.log")
+ stderr_file = os.path.join(log_dir, "client_stderr.log")
+ with open(stdout_file, "w") as out, open(stderr_file, "w") as err:
+ client_process = subprocess.Popen(cmd, stdout=out, stderr=err)
+ return client_process
+
+
 class Minetest(gym.Env):
- def __init__(self, port: int = 5555):
- # Define action space
- self.action_space = Dict({
- **{key.lower(): Discrete(2) for key in KEYS},
- **{"mouse": Box(-11, 11, shape=(2,), dtype=int)},
- })
- # TODO define obs and reward space
+ metadata = {"render.modes": ["rgb_array", "human"]}
+
+ def __init__(
+ self,
+ socket_port: int = 5555,
+ minetest_executable: os.PathLike = None, 
+ log_dir: os.PathLike = None,
+ config_path: os.PathLike = None
+ ):
+ # Define action and observation space
+ self.action_space = Dict(
+ {
+ **{key.lower(): Discrete(2) for key in KEYS},
+ **{
+ "mouse": Box(
+ np.array([-MAX_MOUSE_MOVE_X, -MAX_MOUSE_MOVE_Y]),
+ np.array([MAX_MOUSE_MOVE_X, MAX_MOUSE_MOVE_Y]),
+ shape=(2,),
+ dtype=int,
+ ),
+ },
+ },
+ )
+ self.observation_space = Box(0, 255, shape=(*DISPLAY_SIZE, 3), dtype=np.uint8)
+
+ root_dir = os.path.dirname(os.path.dirname(__file__))
+ if minetest_executable is None:
+ minetest_executable = os.path.join(root_dir, "bin", "minetest")
+ if log_dir is None:
+ log_dir = os.path.join(root_dir, "log")
+ if config_path is None:
+ config_path = os.path.join(root_dir, "minetest.conf")
+
+ # Start Minetest server and client
+ self.server_process = start_minetest_server(
+ minetest_executable, config_path, log_dir, "newworld"
+ )
+ self.client_process = start_minetest_client(minetest_executable, log_dir, socket_port)
 
  # Setup ZMQ
- self.port = port
+ self.socket_port = socket_port
  self.context = zmq.Context()
  self.socket = self.context.socket(zmq.REP)
- self.socket.bind(f"tcp:https://*:{self.port}")
+ self.socket.bind(f"tcp:https://*:{self.socket_port}")
+
+ self.last_obs = None
+ self.render_fig = None
+ self.render_img = None
 
  def reset(self):
  print("Waiting for obs...")
- obs = self.socket.recv()
- print("Received obs: {}".format(obs))
+ byte_obs = self.socket.recv()
+ obs = np.frombuffer(byte_obs, dtype=np.uint8).reshape(
+ DISPLAY_SIZE[1],
+ DISPLAY_SIZE[0],
+ 3,
+ )
+ self.last_obs = obs
+ print("Received obs: {}".format(obs.shape))
  return obs
 
  def step(self, action):
- print("Sending action: {}".format(action))
  # make mouse action serializable
- action["mouse"] = action["mouse"].tolist()
- self.socket.send_json(action)
+ pb_action = dumb_inputs.InputAction()
+ pb_action.mouseDx, pb_action.mouseDy = action["mouse"]
+ for key, v in action.items():
+ if key == "mouse":
+ continue
+ pb_action.keyEvents.append(
+ dumb_inputs.KeyboardEvent(
+ key=key,
+ eventType=dumb_inputs.PRESS if v else dumb_inputs.RELEASE,
+ ),
+ )
+
+ print("Sending action: {}".format(action))
+ self.socket.send(pb_action.SerializeToString())
+
+ # TODO more robust check for whether a server/client is alive while receiving observations
+ for process in [self.server_process, self.client_process]:
+ if process.poll() is not None:
+ return self.last_obs, 0.0, True, {}
+
  print("Waiting for obs...")
- next_obs = self.socket.recv()
- print("Received obs: {}".format(next_obs))
+ byte_next_obs = self.socket.recv()
+ next_obs = np.frombuffer(byte_next_obs, dtype=np.uint8).reshape(
+ DISPLAY_SIZE[1],
+ DISPLAY_SIZE[0],
+ 3,
+ )
+ self.last_obs = next_obs
+ print("Received obs: {}".format(next_obs.shape))
  # TODO receive rewards etc.
- rew = 0.
+ rew = 0.0
  done = False
  info = {}
  return next_obs, rew, done, info
+
+ def render(self, render_mode: str = "human"):
+ if render_mode is None:
+ gym.logger.warn(
+ "You are calling render method without specifying any render mode. "
+ "You can specify the render_mode at initialization, "
+ f'e.g. gym("{self.spec.id}", render_mode="rgb_array")',
+ )
+ return
+ if render_mode == "human":
+ if self.render_img is None:
+ # Setup figure
+ plt.rcParams["toolbar"] = "None"
+ plt.rcParams["figure.autolayout"] = True
+
+ self.render_fig = plt.figure(
+ num="Minetest",
+ figsize=(3 * DISPLAY_SIZE[0] / DISPLAY_SIZE[1], 3),
+ )
+ self.render_img = self.render_fig.gca().imshow(
+ self.last_obs,
+ )
+ self.render_fig.gca().axis("off")
+ self.render_fig.gca().margins(0, 0)
+ self.render_fig.gca().autoscale_view()
+ else:
+ self.render_img.set_data(self.last_obs)
+ plt.draw(), plt.pause(1e-3)
+ elif render_mode == "rgb_array":
+ return self.last_obs
+
+ def close(self):
+ if self.render_fig is not None:
+ plt.close()
+ self.socket.close()
+ self.client_process.kill()
+ self.server_process.kill()

hacking_testing/test_loop.py

@@ -2,7 +2,14 @@
 
 env = Minetest()
 obs = env.reset()
+render = False
 done = False
 while not done:
- action = env.action_space.sample()
- obs, rew, done, info = env.step(action)
+ try:
+ action = env.action_space.sample()
+ obs, rew, done, info = env.step(action)
+ if render:
+ env.render()
+ except KeyboardInterrupt:
+ break
+env.close()

minetest.conf

@@ -1,10 +1,8 @@
-name = me
-update_last_checked = 1670077920
-menu_last_game = minetest
-selected_world_path = /home/edwin/minetest/bin/../worlds/new
+name = MinetestAgent
+selected_world_path = /home/ax37/code/cpp/eai-minetest/bin/../worlds/world
 server_announce = false
 enable_damage = true
 creative_mode = false
+update_last_checked = 1670094822
+menu_last_game = minetest
 mainmenu_last_selected_world = 1
-screen_h = 813
-screen_w = 1024

proto/client/dumb_inputs.proto

@@ -1,6 +1,6 @@
 syntax = "proto3";
 
-message InputEvent {
+message InputAction {
  repeated KeyboardEvent keyEvents = 1;
  sint32 mouseDx = 2;
  sint32 mouseDy = 3;