Added Room Layout results

LayoutStruct/LayoutStruct.py

@@ -0,0 +1,17 @@
+import open3d as o3d
+
+class LayoutComp:
+ def __init__(self, comp_dict):
+ """
+ :param comp_dict: Component dictionary {"type": type, "plane": plane, "poly": poly}
+ """
+
+ self.type = comp_dict["type"]
+ self.plane = comp_dict["plane"]
+ self.poly = comp_dict["poly"]
+
+
+class LayoutStruct:
+ def __init__(self, comp_list, axis_align_matrix):
+ self.comp_list = comp_list
+ self.axis_align_matrix = axis_align_matrix

LayoutStruct/LayoutStructVis.py

@@ -0,0 +1,74 @@
+import open3d as o3d
+import numpy as np
+
+from LayoutStruct.LayoutStruct import LayoutStruct
+from LayoutStruct.line_mesh import LineMesh
+
+class LayoutStructVis:
+ def __init__(self):
+ pass
+
+ def get_layout_wireframe(self, comp_list, clr=np.array([1., 0, 0])):
+ # Add edges
+ # ------------------------------------
+
+ wireframe = o3d.geometry.TriangleMesh()
+ for comp_ind, comp in enumerate(comp_list):
+
+ comp_poly = comp.poly
+
+ clr = clr
+ comp_corners = []
+ comp_edges = []
+
+ comp_poly = np.round(comp_poly, decimals=3)
+ for corner_ind, comp_corner in enumerate(comp_poly):
+ if corner_ind < len(comp_poly) - 2:
+ comp_corners.append(comp_corner)
+ comp_edges.append([corner_ind, corner_ind + 1])
+ # elif corner_ind == len(comp_poly) - 2:
+ # comp_corners.append(comp_corner)
+ # comp_edges.append([corner_ind, 0])
+ elif corner_ind == len(comp_poly) - 2:
+ comp_corners.append(comp_corner)
+ comp_edges.append([0, corner_ind])
+ # comp_edges.append([0,2])
+
+ comp_corners = np.array(comp_corners)
+ comp_edges = np.array(comp_edges)
+
+ compidate_edges_o3d = o3d.geometry.LineSet()
+ compidate_edges_o3d.points = o3d.utility.Vector3dVector(comp_corners)
+ compidate_edges_o3d.lines = o3d.utility.Vector2iVector(comp_edges)
+ compidate_edges_o3d.colors = \
+ o3d.utility.Vector3dVector(np.ones_like(comp_corners) * clr[None, :])
+
+ # print("comp_corners", comp_corners)
+ # print("comp_edges", comp_edges)
+ # print("clr", list(np.ones_like(comp_corners) * clr[None, :]))
+ # line_mesh1 = LineMesh(comp_corners, list(comp_edges), list(np.ones_like(comp_corners) * clr[None, :]),
+ # radius=0.02)
+ line_mesh1 = LineMesh(comp_corners, list(comp_edges), clr[:],
+ radius=0.02)
+ line_mesh1_geoms = line_mesh1.cylinder_segments
+
+ # line_sets.append(line_mesh1_geoms)
+
+ for g in line_mesh1_geoms:
+ wireframe += g
+
+ # break
+
+ return wireframe
+
+ def visualize_layout(self, layout):
+ """
+
+ :param layout:
+ :type layout: LayoutStruct
+ :return:
+ """
+
+ wireframe = self.get_layout_wireframe(layout.comp_list)
+
+ o3d.visualization.draw_geometries([wireframe])

LayoutStruct/line_mesh.py

@@ -0,0 +1,92 @@
+import numpy as np
+import open3d as o3d
+
+# Reference Implementation:
+# https://github.com/intel-isl/Open3D/pull/738#issuecomment-564785941
+# https://github.com/intel-isl/Open3D/issues/1480
+
+def align_vector_to_another(a=np.array([0, 0, 1]), b=np.array([1, 0, 0])):
+ """
+ Aligns vector a to vector b with axis angle rotation
+ """
+ if np.array_equal(a, b):
+ return None, None
+ axis_ = np.cross(a, b)
+ axis_ = axis_ / np.linalg.norm(axis_)
+ angle = np.arccos(np.dot(a, b))
+
+ return axis_, angle
+
+
+def normalized(a, axis=-1, order=2):
+ """Normalizes a numpy array of points"""
+ l2 = np.atleast_1d(np.linalg.norm(a, order, axis))
+ l2[l2 == 0] = 1
+ return a / np.expand_dims(l2, axis), l2
+
+
+class LineMesh(object):
+ def __init__(self, points, lines=None, colors=[0, 1, 0], radius=0.15):
+ """Creates a line represented as sequence of cylinder triangular meshes
+
+ Arguments:
+ points {ndarray} -- Numpy array of ponts Nx3.
+
+ Keyword Arguments:
+ lines {list[list] or None} -- List of point index pairs denoting line segments. If None, implicit lines from ordered pairwise points. (default: {None})
+ colors {list} -- list of colors, or single color of the line (default: {[0, 1, 0]})
+ radius {float} -- radius of cylinder (default: {0.15})
+ """
+ self.points = np.array(points)
+ self.lines = np.array(
+ lines) if lines is not None else self.lines_from_ordered_points(self.points)
+ self.colors = np.array(colors)
+ self.radius = radius
+ self.cylinder_segments = []
+
+ self.create_line_mesh()
+
+ @staticmethod
+ def lines_from_ordered_points(points):
+ lines = [[i, i + 1] for i in range(0, points.shape[0] - 1, 1)]
+ return np.array(lines)
+
+ def create_line_mesh(self):
+ first_points = self.points[self.lines[:, 0], :]
+ second_points = self.points[self.lines[:, 1], :]
+ line_segments = second_points - first_points
+ line_segments_unit, line_lengths = normalized(line_segments)
+
+ z_axis = np.array([0, 0, 1])
+ # Create triangular mesh cylinder segments of line
+ for i in range(line_segments_unit.shape[0]):
+ line_segment = line_segments_unit[i, :]
+ line_length = line_lengths[i]
+ # get axis angle rotation to allign cylinder with line segment
+ axis, angle = align_vector_to_another(z_axis, line_segment)
+ # Get translation vector
+ translation = first_points[i, :] + line_segment * line_length * 0.5
+ # create cylinder and apply transformations
+ cylinder_segment = o3d.geometry.TriangleMesh.create_cylinder(self.radius, line_length)
+ cylinder_segment = cylinder_segment.translate(
+ translation, relative=False)
+ if axis is not None:
+ axis_a = axis * angle
+ cylinder_segment = cylinder_segment.rotate(
+ R=o3d.geometry.get_rotation_matrix_from_axis_angle(axis_a))#, center=True)
+ # cylinder_segment = cylinder_segment.rotate(
+ # axis_a, center=True, type=o3d.geometry.RotationType.AxisAngle)
+ # color cylinder
+ color = self.colors if self.colors.ndim == 1 else self.colors[i, :]
+ cylinder_segment.paint_uniform_color(color)
+ self.cylinder_segments.append(cylinder_segment)
+
+ def add_line(self, vis):
+ """Adds this line to the visualizer"""
+ for cylinder in self.cylinder_segments:
+ vis.add_geometry(cylinder)
+
+ def remove_line(self, vis):
+ """Removes this line from the visualizer"""
+ for cylinder in self.cylinder_segments:
+ vis.remove_geometry(cylinder)

LayoutStruct/utils.py

@@ -0,0 +1,72 @@
+import numpy as np
+from shapely.geometry import Polygon
+
+def svd(A):
+ u, s, vh = np.linalg.svd(A)
+ S = np.zeros(A.shape)
+ S[:s.shape[0], :s.shape[0]] = np.diag(s)
+ return u, S, vh
+
+
+def fit_plane_LSE(points):
+ # points: Nx4 homogeneous 3d points
+ # return: 1d array of four elements [a, b, c, d] of
+ # ax+by+cz+d = 0
+ assert points.shape[0] >= 3 # at least 3 points needed
+ U, S, Vt = svd(points)
+ null_space = Vt[-1, :]
+ return null_space
+
+def poly_2d_np2shapely(poly_np):
+ # x, y, z = poly_np[:, 0], poly_np[:, 1], poly_np[:, 2]
+ # poly_shapely = Polygon(x, y, z)
+ poly_shapely = Polygon([tuple(poly_vertex) for poly_vertex in poly_np])
+ return poly_shapely
+
+def project_poly2plane_2d(poly, plane, norm=False):
+ polygon_xyz = np.array(poly)
+
+ dists_to_plane = polygon_xyz.dot(plane[:3][:, None]) + plane[3]
+ poly_proj = polygon_xyz - dists_to_plane * plane[:3][None, :]
+
+ # polygon_xy = polygon_xyz[:, :2] / polygon_xyz[:, 2][:, None]
+ # polygon_xy = poly_proj[:, ::2] / poly_proj[:, 1][:, None]
+
+ plane_normal_dir = np.argmax(np.abs(plane[:3]))
+ if plane_normal_dir == 0:
+ polygon_xy = poly_proj[:, 1:]
+ if norm:
+ polygon_xy /= polygon_xy[:, 0][:, None] + 1e-4
+ elif plane_normal_dir == 1:
+ polygon_xy = poly_proj[:, ::2]
+ if norm:
+ polygon_xy /= polygon_xy[:, 1][:, None] + 1e-4
+ elif plane_normal_dir == 2:
+ polygon_xy = poly_proj[:, :2]
+ if norm:
+ polygon_xy /= polygon_xy[:, 2][:, None] + 1e-4
+
+ return polygon_xy
+
+def project_points_to_plane_xy(points, plane, norm=False):
+ dists_to_plane = points.dot(plane[:3][:, None]) + plane[3]
+ pcd_proj = points - dists_to_plane * plane[:3][None, :]
+
+ # pcd_proj_xy = pcd_proj[:, ::2] / pcd_proj[:, 1][:, None]
+ # pcd_proj_xy = pcd_proj[:, ::2]
+
+ plane_normal_dir = np.argmax(np.abs(plane[:3]))
+ if plane_normal_dir == 0:
+ pcd_proj_xy = pcd_proj[:, 1:]
+ if norm:
+ pcd_proj_xy /= pcd_proj[:, 0][:, None] + 1e-4
+ elif plane_normal_dir == 1:
+ pcd_proj_xy = pcd_proj[:, ::2]
+ if norm:
+ pcd_proj_xy /= pcd_proj[:, 1][:, None] + 1e-4
+ elif plane_normal_dir == 2:
+ pcd_proj_xy = pcd_proj[:, :2]
+ if norm:
+ pcd_proj_xy /= pcd_proj[:, 2][:, None] + 1e-4
+
+ return pcd_proj_xy, dists_to_plane

demo.py

@@ -5,50 +5,73 @@
 
 from absl import flags
 from absl import app
+
+from LayoutStruct.LayoutStruct import LayoutStruct
+from LayoutStruct.LayoutStructVis import LayoutStructVis
+
 FLAGS = flags.FLAGS
 
 flags.DEFINE_string('scene', 'scene0000_00', 'scene ID')
 flags.DEFINE_string('shapenet_dir', '/media/shreyas/4aa82be1-14a8-47f7-93a7-171e3ebac2b0/Datasets/ShapeNetCore.v2', 'shapenet dir')
+flags.DEFINE_bool('add_objects', False, 'add objects to the scene')
+flags.DEFINE_bool('add_layout', False, 'add room layout to the scene')
 
-def visualize(sceneID):
-
- mctsObjs = MCTSObjs(sceneID, FLAGS.shapenet_dir)
-
- # Create mesh with Bounding Boxes
- lineGeometryList = []
- geometryList = []
- for obj in mctsObjs.mctsObjList[0:]:
- geometryList.append(obj.transMesh)
- col = [0, 0, 1]
- lineSets = drawOpen3dCylLines([obj.orientedBB], col)
- lineGeometryList.append(lineSets)
-
- finalMesh = deepcopy(mctsObjs.sceneMesh)
- for i in range(0, len(geometryList)):
- finalMesh += lineGeometryList[i]
- finalMesh += mctsObjs.mctsObjList[i].transMesh
-
- objMesh = deepcopy(geometryList[0])
- for i in range(1, len(geometryList)):
- objMesh += geometryList[i]
-
+def visualize(sceneID, add_objects=True, add_layout=True):
  # Visualize
  vis = o3d.visualization.Visualizer()
  vis.create_window(window_name='Open3D', width=640, height=480, left=0, top=0,
  visible=True)
  vis.get_render_option().mesh_show_back_face = True
 
- sceneMeshVert = np.array(mctsObjs.sceneMesh.vertices)
- trans = np.zeros((3,)) * 1.0
- trans[0] = np.max(sceneMeshVert[:, 0]) - np.min(sceneMeshVert[:, 0])
- vis.add_geometry(mctsObjs.sceneMesh)
- vis.add_geometry(objMesh.translate(trans * 1.1))
- vis.add_geometry(finalMesh.translate(trans * 1.1 * 2))
+ if add_objects:
+ mctsObjs = MCTSObjs(sceneID, FLAGS.shapenet_dir)
+
+ # Create mesh with Bounding Boxes
+ lineGeometryList = []
+ geometryList = []
+ for obj in mctsObjs.mctsObjList[0:]:
+ geometryList.append(obj.transMesh)
+ col = [0, 0, 1]
+ lineSets = drawOpen3dCylLines([obj.orientedBB], col)
+ lineGeometryList.append(lineSets)
+
+ finalMesh = deepcopy(mctsObjs.sceneMesh)
+ for i in range(0, len(geometryList)):
+ finalMesh += lineGeometryList[i]
+ finalMesh += mctsObjs.mctsObjList[i].transMesh
+
+ objMesh = deepcopy(geometryList[0])
+ for i in range(1, len(geometryList)):
+ objMesh += geometryList[i]
+
+
+
+ sceneMeshVert = np.array(mctsObjs.sceneMesh.vertices)
+ trans = np.zeros((3,)) * 1.0
+ trans[0] = np.max(sceneMeshVert[:, 0]) - np.min(sceneMeshVert[:, 0])
+ vis.add_geometry(mctsObjs.sceneMesh)
+ vis.add_geometry(objMesh.translate(trans * 1.1))
+ vis.add_geometry(finalMesh.translate(trans * 1.1 * 2))
+
+ if add_layout:
+ sceneLayoutPath = os.path.join("outputs/scans/", sceneID, 'monte_carlo/FinalLayout.pickle')
+ if not os.path.isfile(sceneLayoutPath):
+ print("WARNING: Layout missing for scene %s" % sceneID)
+ else:
+ with open(sceneLayoutPath, 'rb') as f:
+ layout_struct = pickle.load(f) # type: LayoutStruct
+
+ wireframe = LayoutStructVis().get_layout_wireframe(layout_struct.comp_list)
+
+ vis.add_geometry(wireframe)
 
  vis.run()
 
 
 def main(argv):
+ add_objects = FLAGS.add_objects
+ add_layout = FLAGS.add_layout
+
  if FLAGS.scene == 'scene0000_00':
 
  with open('scannetv2_val.txt', 'r') as f:
@@ -61,7 +84,7 @@ def main(argv):
  while (True):
  sceneID = random.choice(sceneIDs)
  downloadScanNetScene(sceneID)
- visualize(sceneID)
+ visualize(sceneID, add_objects=add_objects, add_layout=add_layout)