Proximal Policy Optimization for cartpole balance.

cartpole-gazebo-ppo/cartpole_controller/CMakeLists.txt

@@ -0,0 +1,197 @@
+cmake_minimum_required(VERSION 2.8.3)
+project(cartpole_controller)
+
+## Compile as C++11, supported in ROS Kinetic and newer
+# add_compile_options(-std=c++11)
+
+## Find catkin macros and libraries
+## if COMPONENTS list like find_package(catkin REQUIRED COMPONENTS xyz)
+## is used, also find other catkin packages
+find_package(catkin REQUIRED COMPONENTS
+
+)
+
+## System dependencies are found with CMake's conventions
+# find_package(Boost REQUIRED COMPONENTS system)
+
+
+## Uncomment this if the package has a setup.py. This macro ensures
+## modules and global scripts declared therein get installed
+## See http:https://ros.org/doc/api/catkin/html/user_guide/setup_dot_py.html
+# catkin_python_setup()
+
+################################################
+## Declare ROS messages, services and actions ##
+################################################
+
+## To declare and build messages, services or actions from within this
+## package, follow these steps:
+## * Let MSG_DEP_SET be the set of packages whose message types you use in
+## your messages/services/actions (e.g. std_msgs, actionlib_msgs, ...).
+## * In the file package.xml:
+## * add a build_depend tag for "message_generation"
+## * add a build_depend and a run_depend tag for each package in MSG_DEP_SET
+## * If MSG_DEP_SET isn't empty the following dependency has been pulled in
+## but can be declared for certainty nonetheless:
+## * add a run_depend tag for "message_runtime"
+## * In this file (CMakeLists.txt):
+## * add "message_generation" and every package in MSG_DEP_SET to
+## find_package(catkin REQUIRED COMPONENTS ...)
+## * add "message_runtime" and every package in MSG_DEP_SET to
+## catkin_package(CATKIN_DEPENDS ...)
+## * uncomment the add_*_files sections below as needed
+## and list every .msg/.srv/.action file to be processed
+## * uncomment the generate_messages entry below
+## * add every package in MSG_DEP_SET to generate_messages(DEPENDENCIES ...)
+
+## Generate messages in the 'msg' folder
+# add_message_files(
+# FILES
+# Message1.msg
+# Message2.msg
+# )
+
+## Generate services in the 'srv' folder
+# add_service_files(
+# FILES
+# Service1.srv
+# Service2.srv
+# )
+
+## Generate actions in the 'action' folder
+# add_action_files(
+# FILES
+# Action1.action
+# Action2.action
+# )
+
+## Generate added messages and services with any dependencies listed here
+# generate_messages(
+# DEPENDENCIES
+# std_msgs
+# )
+
+################################################
+## Declare ROS dynamic reconfigure parameters ##
+################################################
+
+## To declare and build dynamic reconfigure parameters within this
+## package, follow these steps:
+## * In the file package.xml:
+## * add a build_depend and a run_depend tag for "dynamic_reconfigure"
+## * In this file (CMakeLists.txt):
+## * add "dynamic_reconfigure" to
+## find_package(catkin REQUIRED COMPONENTS ...)
+## * uncomment the "generate_dynamic_reconfigure_options" section below
+## and list every .cfg file to be processed
+
+## Generate dynamic reconfigure parameters in the 'cfg' folder
+# generate_dynamic_reconfigure_options(
+# cfg/DynReconf1.cfg
+# cfg/DynReconf2.cfg
+# )
+
+###################################
+## catkin specific configuration ##
+###################################
+## The catkin_package macro generates cmake config files for your package
+## Declare things to be passed to dependent projects
+## INCLUDE_DIRS: uncomment this if your package contains header files
+## LIBRARIES: libraries you create in this project that dependent projects also need
+## CATKIN_DEPENDS: catkin_packages dependent projects also need
+## DEPENDS: system dependencies of this project that dependent projects also need
+catkin_package(
+# INCLUDE_DIRS include
+# LIBRARIES cartpole_controller
+# CATKIN_DEPENDS ros_control ros_controllers rospy std_msgs
+# DEPENDS system_lib
+)
+
+###########
+## Build ##
+###########
+
+## Specify additional locations of header files
+## Your package locations should be listed before other locations
+include_directories(
+# include
+ ${catkin_INCLUDE_DIRS}
+)
+
+## Declare a C++ library
+# add_library(${PROJECT_NAME}
+# src/${PROJECT_NAME}/cartpole_controller.cpp
+# )
+
+## Add cmake target dependencies of the library
+## as an example, code may need to be generated before libraries
+## either from message generation or dynamic reconfigure
+# add_dependencies(${PROJECT_NAME} ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS})
+
+## Declare a C++ executable
+## With catkin_make all packages are built within a single CMake context
+## The recommended prefix ensures that target names across packages don't collide
+# add_executable(${PROJECT_NAME}_node src/cartpole_controller_node.cpp)
+
+## Rename C++ executable without prefix
+## The above recommended prefix causes long target names, the following renames the
+## target back to the shorter version for ease of user use
+## e.g. "rosrun someones_pkg node" instead of "rosrun someones_pkg someones_pkg_node"
+# set_target_properties(${PROJECT_NAME}_node PROPERTIES OUTPUT_NAME node PREFIX "")
+
+## Add cmake target dependencies of the executable
+## same as for the library above
+# add_dependencies(${PROJECT_NAME}_node ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS})
+
+## Specify libraries to link a library or executable target against
+# target_link_libraries(${PROJECT_NAME}_node
+# ${catkin_LIBRARIES}
+# )
+
+#############
+## Install ##
+#############
+
+# all install targets should use catkin DESTINATION variables
+# See http:https://ros.org/doc/api/catkin/html/adv_user_guide/variables.html
+
+## Mark executable scripts (Python etc.) for installation
+## in contrast to setup.py, you can choose the destination
+# install(PROGRAMS
+# scripts/my_python_script
+# DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION}
+# )
+
+## Mark executables and/or libraries for installation
+# install(TARGETS ${PROJECT_NAME} ${PROJECT_NAME}_node
+# ARCHIVE DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION}
+# LIBRARY DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION}
+# RUNTIME DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION}
+# )
+
+## Mark cpp header files for installation
+# install(DIRECTORY include/${PROJECT_NAME}/
+# DESTINATION ${CATKIN_PACKAGE_INCLUDE_DESTINATION}
+# FILES_MATCHING PATTERN "*.h"
+# PATTERN ".svn" EXCLUDE
+# )
+
+## Mark other files for installation (e.g. launch and bag files, etc.)
+# install(FILES
+# # myfile1
+# # myfile2
+# DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION}
+# )
+
+#############
+## Testing ##
+#############
+
+## Add gtest based cpp test target and link libraries
+# catkin_add_gtest(${PROJECT_NAME}-test test/test_cartpole_controller.cpp)
+# if(TARGET ${PROJECT_NAME}-test)
+# target_link_libraries(${PROJECT_NAME}-test ${PROJECT_NAME})
+# endif()
+
+## Add folders to be run by python nosetests
+# catkin_add_nosetests(test)

cartpole-gazebo-ppo/cartpole_controller/config/joint_position_control.yaml

@@ -0,0 +1,41 @@
+joint_state_controller:
+ type: joint_state_controller/JointStateController
+ publish_rate: 120
+
+# Position Controllers ---------------------------------------
+stand_cart_position_controller:
+ type: effort_controllers/JointPositionController
+ joint: stand_cart
+ pid: {p: 100, i: 0.01, d: 10}
+
+cart_pole_position_controller:
+ type: effort_controllers/JointPositionController
+ joint: cart_pole
+ pid: {p: 100, i: 0.01, d: 10}
+
+
+# waist_pendulum_position_controller:
+# type: effort_controllers/JointPositionController
+# joint: waist_pendulum
+# pid: {p: 100.0, i: 0.01, d: 10.0}
+
+# top_pendulum_position_controller:
+# type: effort_controllers/JointPositionController
+# joint: top_pendulum
+# pid: {p: 100.0, i: 0.01, d: 10.0}
+
+# shankL_footL_position_controller:
+# type: effort_controllers/JointPositionController
+# joint: shankL_footL
+# pid: {p: 100.0, i: 0.01, d: 10.0}
+
+# waist_boom_position_controller:
+# type: effort_controllers/JointPositionController
+# joint: waist_boom
+# pid: {p: 100.0, i: 0.01, d: 10.0}
+
+
+# pivot_boom_position_controller:
+# type: effort_controllers/JointPositionController
+# joint: boom_pivot
+# pid: {p: 100.0, i: 0.01, d: 10.0}

cartpole-gazebo-ppo/cartpole_controller/package.xml

@@ -0,0 +1,71 @@
+<?xml version="1.0"?>
+<package format="2">
+ <name>cartpole_controller</name>
+ <version>0.0.0</version>
+ <description>The cartpole_controller package</description>
+
+ <!-- One maintainer tag required, multiple allowed, one person per tag -->
+ <!-- Example: -->
+ <!-- <maintainer email="[email protected]">Jane Doe</maintainer> -->
+ <maintainer email="[email protected]">imitaion</maintainer>
+
+
+ <!-- One license tag required, multiple allowed, one license per tag -->
+ <!-- Commonly used license strings: -->
+ <!-- BSD, MIT, Boost Software License, GPLv2, GPLv3, LGPLv2.1, LGPLv3 -->
+ <license>TODO</license>
+
+
+ <!-- Url tags are optional, but multiple are allowed, one per tag -->
+ <!-- Optional attribute type can be: website, bugtracker, or repository -->
+ <!-- Example: -->
+ <!-- <url type="website">http:https://wiki.ros.org/cartpole_controller</url> -->
+
+
+ <!-- Author tags are optional, multiple are allowed, one per tag -->
+ <!-- Authors do not have to be maintainers, but could be -->
+ <!-- Example: -->
+ <!-- <author email="[email protected]">Jane Doe</author> -->
+
+
+ <!-- The *depend tags are used to specify dependencies -->
+ <!-- Dependencies can be catkin packages or system dependencies -->
+ <!-- Examples: -->
+ <!-- Use depend as a shortcut for packages that are both build and exec dependencies -->
+ <!-- <depend>roscpp</depend> -->
+ <!-- Note that this is equivalent to the following: -->
+ <!-- <build_depend>roscpp</build_depend> -->
+ <!-- <exec_depend>roscpp</exec_depend> -->
+ <!-- Use build_depend for packages you need at compile time: -->
+ <!-- <build_depend>message_generation</build_depend> -->
+ <!-- Use build_export_depend for packages you need in order to build against this package: -->
+ <!-- <build_export_depend>message_generation</build_export_depend> -->
+ <!-- Use buildtool_depend for build tool packages: -->
+ <!-- <buildtool_depend>catkin</buildtool_depend> -->
+ <!-- Use exec_depend for packages you need at runtime: -->
+ <!-- <exec_depend>message_runtime</exec_depend> -->
+ <!-- Use test_depend for packages you need only for testing: -->
+ <!-- <test_depend>gtest</test_depend> -->
+ <!-- Use doc_depend for packages you need only for building documentation: -->
+ <!-- <doc_depend>doxygen</doc_depend> -->
+ <buildtool_depend>catkin</buildtool_depend>
+ <build_depend>ros_control</build_depend>
+ <build_depend>ros_controllers</build_depend>
+ <build_depend>rospy</build_depend>
+ <build_depend>std_msgs</build_depend>
+ <build_export_depend>ros_control</build_export_depend>
+ <build_export_depend>ros_controllers</build_export_depend>
+ <build_export_depend>rospy</build_export_depend>
+ <build_export_depend>std_msgs</build_export_depend>
+ <exec_depend>ros_control</exec_depend>
+ <exec_depend>ros_controllers</exec_depend>
+ <exec_depend>rospy</exec_depend>
+ <exec_depend>std_msgs</exec_depend>
+
+
+ <!-- The export tag contains other, unspecified, tags -->
+ <export>
+ <!-- Other tools can request additional information be placed here -->
+
+ </export>
+</package>

cartpole-gazebo-ppo/cartpole_controller/src/model/checkpoint

@@ -0,0 +1,2 @@
+model_checkpoint_path: "model.ckpt"
+all_model_checkpoint_paths: "model.ckpt"

cartpole-gazebo-ppo/cartpole_controller/src/policy_net.py

@@ -0,0 +1,52 @@
+import gym
+import numpy as np
+import tensorflow as tf
+
+
+class Policy_net:
+ def __init__(self, name: str, temp=0.1):
+ """
+ :param name: string
+ :param env: gym env
+ :param temp: temperature of boltzmann distribution
+ """
+
+ ob_space = 4
+ act_space = 2
+
+ with tf.variable_scope(name):
+ self.obs = tf.placeholder(dtype=tf.float32, shape=[None, 4], name='obs')
+
+ with tf.variable_scope('policy_net'):
+ layer_1 = tf.layers.dense(inputs=self.obs, units=20, activation=tf.tanh)
+ layer_2 = tf.layers.dense(inputs=layer_1, units=20, activation=tf.tanh)
+ layer_3 = tf.layers.dense(inputs=layer_2, units=2, activation=tf.tanh)
+ self.act_probs = tf.layers.dense(inputs=tf.divide(layer_3, temp), units=2, activation=tf.nn.softmax)
+
+ with tf.variable_scope('value_net'):
+ layer_1 = tf.layers.dense(inputs=self.obs, units=20, activation=tf.tanh)
+ layer_2 = tf.layers.dense(inputs=layer_1, units=20, activation=tf.tanh)
+ self.v_preds = tf.layers.dense(inputs=layer_2, units=1, activation=None)
+
+ self.act_stochastic = tf.multinomial(tf.log(self.act_probs), num_samples=1)
+ self.act_stochastic = tf.reshape(self.act_stochastic, shape=[-1])
+
+ self.act_deterministic = tf.argmax(self.act_probs, axis=1)
+
+ self.scope = tf.get_variable_scope().name
+
+ def act(self, obs, stochastic=True):
+ if stochastic:
+ return tf.get_default_session().run([self.act_stochastic, self.v_preds], feed_dict={self.obs: obs})
+ else:
+ return tf.get_default_session().run([self.act_deterministic, self.v_preds], feed_dict={self.obs: obs})
+
+ def get_action_prob(self, obs):
+ return tf.get_default_session().run(self.act_probs, feed_dict={self.obs: obs})
+
+ def get_variables(self):
+ return tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, self.scope)
+
+ def get_trainable_variables(self):
+ return tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, self.scope)
+