diff --git a/_Example_Workflows/Multi-View-Images_to_3DGS_to_3DMesh(DMTet and DiffRast).json b/_Example_Workflows/Multi-View-Images_to_3DGS_to_3DMesh(DMTet and DiffRast).json index 345e7f7..38c4ba1 100644 --- a/_Example_Workflows/Multi-View-Images_to_3DGS_to_3DMesh(DMTet and DiffRast).json +++ b/_Example_Workflows/Multi-View-Images_to_3DGS_to_3DMesh(DMTet and DiffRast).json @@ -1,44 +1,7 @@ { - "last_node_id": 46, - "last_link_id": 69, + "last_node_id": 47, + "last_link_id": 75, "nodes": [ - { - "id": 4, - "type": "[Comfy3D] Generate Orbit Camera Poses", - "pos": [ - 435, - 148 - ], - "size": { - "0": 908.027099609375, - "1": 153.19027709960938 - }, - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "reference_images", - "type": "IMAGE", - "link": 1 - } - ], - "outputs": [ - { - "name": "orbit_camposes", - "type": "ORBIT_CAMPOSES", - "links": [], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Generate Orbit Camera Poses" - }, - "widgets_values": [ - "#([start_reference_image_index : end_reference_image_index], orbit_radius, elevation_angle [-90, 90], start_azimuth_angle [0, 360], end_azimuth_angle [0, 360])\n([0:120], 1.75, 0, 0, 360)" - ] - }, { "id": 9, "type": "MaskToImage", @@ -51,7 +14,7 @@ "1": 26 }, "flags": {}, - "order": 11, + "order": 13, "mode": 0, "inputs": [ { @@ -75,92 +38,6 @@ "Node name for S&R": "MaskToImage" } }, - { - "id": 27, - "type": "[Comfy3D] Stack Orbit Camera Poses", - "pos": [ - 1034, - 367 - ], - "size": { - "0": 315, - "1": 586 - }, - "flags": {}, - "order": 0, - "mode": 0, - "outputs": [ - { - "name": "orbit_camposes", - "type": "ORBIT_CAMPOSES", - "links": [ - 36, - 40 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "orbit_radius_list", - "type": "FLOAT", - "links": null, - "shape": 6 - }, - { - "name": "elevation_list", - "type": "FLOAT", - "links": null, - "shape": 6 - }, - { - "name": "azimuth_list", - "type": "FLOAT", - "links": null, - "shape": 6 - }, - { - "name": "orbit_center_X_list", - "type": "FLOAT", - "links": null, - "shape": 6 - }, - { - "name": "orbit_center_Y_list", - "type": "FLOAT", - "links": null, - "shape": 6 - }, - { - "name": "orbit_center_Z_list", - "type": "FLOAT", - "links": null, - "shape": 6 - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Stack Orbit Camera Poses" - }, - "widgets_values": [ - 1.75, - 1.75, - 0.1, - 0, - 0, - 0, - 0, - -0.01, - 3, - 0, - 0, - 0.1, - 0, - 0, - 0.1, - 0, - 0, - 0.1 - ] - }, { "id": 10, "type": "PreviewImage", @@ -175,7 +52,7 @@ "flags": { "collapsed": true }, - "order": 14, + "order": 17, "mode": 0, "inputs": [ { @@ -202,7 +79,7 @@ "flags": { "collapsed": true }, - "order": 6, + "order": 7, "mode": 0, "inputs": [ { @@ -224,10 +101,10 @@ ], "size": [ 315, - 559 + 270 ], "flags": {}, - "order": 20, + "order": 22, "mode": 0, "inputs": [ { @@ -286,7 +163,7 @@ "1": 26 }, "flags": {}, - "order": 22, + "order": 24, "mode": 0, "inputs": [ { @@ -354,13 +231,13 @@ "1": 106 }, "flags": {}, - "order": 15, + "order": 0, "mode": 0, "inputs": [ { "name": "gs_ply", "type": "GS_PLY", - "link": 30 + "link": null } ], "outputs": [ @@ -383,72 +260,6 @@ "-z" ] }, - { - "id": 7, - "type": "VHS_LoadImages", - "pos": [ - 113, - 54 - ], - "size": [ - 243.60000610351562, - 194 - ], - "flags": {}, - "order": 1, - "mode": 0, - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 1, - 22, - 31, - 58 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": [ - 2 - ], - "shape": 3, - "slot_index": 1 - }, - { - "name": "INT", - "type": "INT", - "links": null, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "VHS_LoadImages" - }, - "widgets_values": { - "directory": "Images_enhanced", - "image_load_cap": 0, - "skip_first_images": 0, - "select_every_nth": 1, - "choose folder to upload": "image", - "videopreview": { - "hidden": false, - "paused": false, - "params": { - "frame_load_cap": 0, - "skip_first_images": 0, - "filename": "Images_enhanced", - "type": "input", - "format": "folder", - "select_every_nth": 1 - } - } - } - }, { "id": 39, "type": "Reroute", @@ -461,7 +272,7 @@ 26 ], "flags": {}, - "order": 7, + "order": 8, "mode": 0, "inputs": [ { @@ -485,44 +296,6 @@ "horizontal": false } }, - { - "id": 8, - "type": "InvertMask", - "pos": [ - 545, - 6 - ], - "size": { - "0": 210, - "1": 26 - }, - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "mask", - "type": "MASK", - "link": 2 - } - ], - "outputs": [ - { - "name": "MASK", - "type": "MASK", - "links": [ - 8, - 33, - 61 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "InvertMask" - } - }, { "id": 41, "type": "Reroute", @@ -535,7 +308,7 @@ 26 ], "flags": {}, - "order": 13, + "order": 14, "mode": 0, "inputs": [ { @@ -571,7 +344,7 @@ 26 ], "flags": {}, - "order": 4, + "order": 10, "mode": 0, "inputs": [ { @@ -597,42 +370,18 @@ } }, { - "id": 22, - "type": "Note", + "id": 43, + "type": "Reroute", "pos": [ - 1967, - 649 + 2195, + -25 + ], + "size": [ + 75, + 26 ], - "size": { - "0": 347.9384460449219, - "1": 141.6276092529297 - }, "flags": {}, - "order": 2, - "mode": 0, - "title": "Switch 3DGS Axis Note", - "properties": { - "text": "" - }, - "widgets_values": [ - "Triplane Gaussian Transformers to Preview:\n axis x to: -y\n axis y to: +z\n axis z to: -x\n\nGaussian Splatting to Preview:\n axis x to: +x\n axis y to: -y\n axis z to: -z" - ], - "color": "#432", - "bgcolor": "#653" - }, - { - "id": 43, - "type": "Reroute", - "pos": [ - 2195, - -25 - ], - "size": [ - 75, - 26 - ], - "flags": {}, - "order": 9, + "order": 16, "mode": 0, "inputs": [ { @@ -656,97 +405,6 @@ "horizontal": false } }, - { - "id": 25, - "type": "[Comfy3D] Gaussian Splatting", - "pos": [ - 1397, - 54 - ], - "size": { - "0": 531.5999755859375, - "1": 758 - }, - "flags": {}, - "order": 12, - "mode": 0, - "inputs": [ - { - "name": "reference_images", - "type": "IMAGE", - "link": 31 - }, - { - "name": "reference_masks", - "type": "MASK", - "link": 33 - }, - { - "name": "reference_orbit_camera_poses", - "type": "ORBIT_CAMPOSES", - "link": 36 - }, - { - "name": "points_cloud_to_initialize_gaussian", - "type": "POINTCLOUD", - "link": null - }, - { - "name": "ply_to_initialize_gaussian", - "type": "GS_PLY", - "link": null - }, - { - "name": "mesh_to_initialize_gaussian", - "type": "MESH", - "link": null - } - ], - "outputs": [ - { - "name": "gs_ply", - "type": "GS_PLY", - "links": [ - 30, - 49, - 57 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Gaussian Splatting" - }, - "widgets_values": [ - 49.1, - 1000, - 3, - 10000, - 0.2, - 3, - 0, - 0, - 0.5, - 0.01, - 0.05, - 0.005, - 0.005, - 0.001, - 0.00002, - 0.02, - 500, - 5000, - 3, - 0.01, - 100, - 100000, - 100, - 700, - 0.01, - 0 - ] - }, { "id": 35, "type": "[Comfy3D] Convert 3DGS To Pointcloud", @@ -759,13 +417,13 @@ "1": 26 }, "flags": {}, - "order": 17, + "order": 19, "mode": 0, "inputs": [ { "name": "gs_ply", "type": "GS_PLY", - "link": 57 + "link": 74 } ], "outputs": [ @@ -797,7 +455,7 @@ "flags": { "collapsed": false }, - "order": 25, + "order": 27, "mode": 0, "inputs": [ { @@ -824,7 +482,7 @@ "flags": { "collapsed": false }, - "order": 21, + "order": 23, "mode": 0, "inputs": [ { @@ -849,7 +507,7 @@ 500 ], "flags": {}, - "order": 24, + "order": 11, "mode": 0, "inputs": [ { @@ -881,7 +539,7 @@ "1": 58 }, "flags": {}, - "order": 19, + "order": 5, "mode": 0, "inputs": [ { @@ -920,7 +578,7 @@ "1": 154 }, "flags": {}, - "order": 3, + "order": 1, "mode": 0, "outputs": [ { @@ -955,7 +613,7 @@ 26 ], "flags": {}, - "order": 10, + "order": 12, "mode": 0, "inputs": [ { @@ -991,13 +649,13 @@ "1": 198 }, "flags": {}, - "order": 16, + "order": 20, "mode": 0, "inputs": [ { "name": "gs_ply", "type": "GS_PLY", - "link": 49 + "link": 75 }, { "name": "render_orbit_camera_poses", @@ -1125,7 +783,7 @@ "1": 314 }, "flags": {}, - "order": 23, + "order": 21, "mode": 0, "inputs": [ { @@ -1184,7 +842,7 @@ "1": 58 }, "flags": {}, - "order": 27, + "order": 26, "mode": 0, "inputs": [ { @@ -1221,7 +879,7 @@ "1": 334 }, "flags": {}, - "order": 26, + "order": 25, "mode": 0, "inputs": [ { @@ -1277,6 +935,346 @@ 0.5, false ] + }, + { + "id": 7, + "type": "VHS_LoadImages", + "pos": [ + 113, + 54 + ], + "size": [ + 243.60000610351562, + 194 + ], + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 1, + 22, + 58, + 70 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": [ + 2 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "INT", + "type": "INT", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadImages" + }, + "widgets_values": { + "directory": "Images_enhanced", + "image_load_cap": 0, + "skip_first_images": 0, + "select_every_nth": 1, + "choose folder to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_images": 0, + "filename": "Images_enhanced", + "type": "input", + "format": "folder", + "select_every_nth": 1 + } + } + } + }, + { + "id": 4, + "type": "[Comfy3D] Generate Orbit Camera Poses", + "pos": [ + 435, + 148 + ], + "size": { + "0": 908.027099609375, + "1": 153.19027709960938 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "reference_images", + "type": "IMAGE", + "link": 1 + } + ], + "outputs": [ + { + "name": "orbit_camposes", + "type": "ORBIT_CAMPOSES", + "links": [], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Generate Orbit Camera Poses" + }, + "widgets_values": [ + "#([start_reference_image_index : end_reference_image_index], orbit_radius, elevation_angle [-90, 90], start_azimuth_angle [0, 360], end_azimuth_angle [0, 360])\n([0:120], 1.75, 0, 0, 360)" + ] + }, + { + "id": 27, + "type": "[Comfy3D] Stack Orbit Camera Poses", + "pos": [ + 1034, + 367 + ], + "size": { + "0": 315, + "1": 586 + }, + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "orbit_camposes", + "type": "ORBIT_CAMPOSES", + "links": [ + 40, + 72 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "orbit_radius_list", + "type": "FLOAT", + "links": null, + "shape": 6 + }, + { + "name": "elevation_list", + "type": "FLOAT", + "links": null, + "shape": 6 + }, + { + "name": "azimuth_list", + "type": "FLOAT", + "links": null, + "shape": 6 + }, + { + "name": "orbit_center_X_list", + "type": "FLOAT", + "links": null, + "shape": 6 + }, + { + "name": "orbit_center_Y_list", + "type": "FLOAT", + "links": null, + "shape": 6 + }, + { + "name": "orbit_center_Z_list", + "type": "FLOAT", + "links": null, + "shape": 6 + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Stack Orbit Camera Poses" + }, + "widgets_values": [ + 1.75, + 1.75, + 0.1, + 0, + 0, + 0, + 0, + -0.01, + 3, + 0, + 0, + 0.1, + 0, + 0, + 0.1, + 0, + 0, + 0.1 + ] + }, + { + "id": 8, + "type": "InvertMask", + "pos": [ + 545, + 6 + ], + "size": { + "0": 210, + "1": 26 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "mask", + "type": "MASK", + "link": 2 + } + ], + "outputs": [ + { + "name": "MASK", + "type": "MASK", + "links": [ + 8, + 61, + 73 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "InvertMask" + } + }, + { + "id": 47, + "type": "[Comfy3D] Gaussian Splatting", + "pos": [ + 1415, + 52 + ], + "size": { + "0": 531.5999755859375, + "1": 734 + }, + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "reference_images", + "type": "IMAGE", + "link": 70 + }, + { + "name": "reference_masks", + "type": "MASK", + "link": 73 + }, + { + "name": "reference_orbit_camera_poses", + "type": "ORBIT_CAMPOSES", + "link": 72 + }, + { + "name": "points_cloud_to_initialize_gaussian", + "type": "POINTCLOUD", + "link": null + }, + { + "name": "ply_to_initialize_gaussian", + "type": "GS_PLY", + "link": null + }, + { + "name": "mesh_to_initialize_gaussian", + "type": "MESH", + "link": null + } + ], + "outputs": [ + { + "name": "gs_ply", + "type": "GS_PLY", + "links": [ + 74, + 75 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "[Comfy3D] Gaussian Splatting" + }, + "widgets_values": [ + 49.1, + 30000, + 1, + 0.2, + 3, + 0, + 0, + 0.5, + 0.0025, + 0.05, + 0.005, + 0.001, + 0.00016, + 0.0000016, + 0.01, + 30000, + 10000, + 3, + 0.01, + 500, + 15000, + 100, + 3000, + 0.0002, + 3 + ] + }, + { + "id": 22, + "type": "Note", + "pos": [ + 1964, + 668 + ], + "size": { + "0": 347.9384460449219, + "1": 141.6276092529297 + }, + "flags": {}, + "order": 4, + "mode": 0, + "title": "Switch 3DGS Axis Note", + "properties": { + "text": "" + }, + "widgets_values": [ + "Triplane Gaussian Transformers to Preview:\n axis x to: -y\n axis y to: +z\n axis z to: -x\n\nGaussian Splatting to Preview:\n axis x to: +x\n axis y to: -y\n axis z to: -z" + ], + "color": "#432", + "bgcolor": "#653" } ], "links": [ @@ -1336,38 +1334,6 @@ 0, "GS_PLY" ], - [ - 30, - 25, - 0, - 24, - 0, - "GS_PLY" - ], - [ - 31, - 7, - 0, - 25, - 0, - "IMAGE" - ], - [ - 33, - 8, - 0, - 25, - 1, - "MASK" - ], - [ - 36, - 27, - 0, - 25, - 2, - "ORBIT_CAMPOSES" - ], [ 38, 29, @@ -1400,14 +1366,6 @@ 0, "MASK" ], - [ - 49, - 25, - 0, - 32, - 0, - "GS_PLY" - ], [ 50, 32, @@ -1456,14 +1414,6 @@ 3, "MESH" ], - [ - 57, - 25, - 0, - 35, - 0, - "GS_PLY" - ], [ 58, 7, @@ -1535,9 +1485,62 @@ 38, 0, "IMAGE" + ], + [ + 70, + 7, + 0, + 47, + 0, + "IMAGE" + ], + [ + 72, + 27, + 0, + 47, + 2, + "ORBIT_CAMPOSES" + ], + [ + 73, + 8, + 0, + 47, + 1, + "MASK" + ], + [ + 74, + 47, + 0, + 35, + 0, + "GS_PLY" + ], + [ + 75, + 47, + 0, + 32, + 0, + "GS_PLY" ] ], - "groups": [], + "groups": [ + { + "title": "3DGS to Mesh using DMTet and DiffRast", + "bounding": [ + 1970, + -674, + 2380, + 1294 + ], + "color": "#3f789e", + "font_size": 24, + "locked": false + } + ], "config": {}, "extra": {}, "version": 0.4 diff --git a/_Example_Workflows/Triplane_Gaussian_Transformers_to_3DGS(DMTet and DiffRast).json b/_Example_Workflows/Triplane_Gaussian_Transformers_to_3DGS(DMTet and DiffRast).json deleted file mode 100644 index 36ca888..0000000 --- a/_Example_Workflows/Triplane_Gaussian_Transformers_to_3DGS(DMTet and DiffRast).json +++ /dev/null @@ -1,1229 +0,0 @@ -{ - "last_node_id": 31, - "last_link_id": 46, - "nodes": [ - { - "id": 4, - "type": "LoadImage", - "pos": [ - 192, - 214 - ], - "size": { - "0": 354.32330322265625, - "1": 315.4895935058594 - }, - "flags": {}, - "order": 0, - "mode": 0, - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 14 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "MASK", - "type": "MASK", - "links": [ - 10 - ], - "shape": 3, - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "LoadImage" - }, - "widgets_values": [ - "a_pikachu_with_smily_face-removebg.png", - "image" - ] - }, - { - "id": 11, - "type": "Note", - "pos": [ - 859, - 428 - ], - "size": { - "0": 347.9384460449219, - "1": 141.6276092529297 - }, - "flags": {}, - "order": 1, - "mode": 0, - "title": "Switch 3DGS Axis Note", - "properties": { - "text": "" - }, - "widgets_values": [ - "Triplane Gaussian Transformers to Preview:\n axis x to: -y\n axis y to: +z\n axis z to: -x\n\nGaussian Splatting to Preview:\n axis x to: +x\n axis y to: -y\n axis z to: -z" - ], - "color": "#432", - "bgcolor": "#653" - }, - { - "id": 14, - "type": "[Comfy3D] Triplane Gaussian Transformers", - "pos": [ - 767, - 631 - ], - "size": { - "0": 252, - "1": 98 - }, - "flags": {}, - "order": 8, - "mode": 0, - "inputs": [ - { - "name": "reference_image", - "type": "IMAGE", - "link": 14 - }, - { - "name": "reference_mask", - "type": "MASK", - "link": 15 - }, - { - "name": "tgs_model", - "type": "TGS_MODEL", - "link": 16 - }, - { - "name": "cam_dist", - "type": "FLOAT", - "link": 28, - "widget": { - "name": "cam_dist" - } - } - ], - "outputs": [ - { - "name": "gs_ply", - "type": "GS_PLY", - "links": [ - 17 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Triplane Gaussian Transformers" - }, - "widgets_values": [ - 1.9000000000000001 - ] - }, - { - "id": 9, - "type": "InvertMask", - "pos": [ - 576, - 690 - ], - "size": { - "0": 140, - "1": 26 - }, - "flags": {}, - "order": 5, - "mode": 0, - "inputs": [ - { - "name": "mask", - "type": "MASK", - "link": 10 - } - ], - "outputs": [ - { - "name": "MASK", - "type": "MASK", - "links": [ - 15 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "InvertMask" - } - }, - { - "id": 2, - "type": "[Comfy3D] Load Triplane Gaussian Transformers", - "pos": [ - 441, - 765 - ], - "size": { - "0": 294, - "1": 41.596431732177734 - }, - "flags": {}, - "order": 2, - "mode": 0, - "outputs": [ - { - "name": "tgs_model", - "type": "TGS_MODEL", - "links": [ - 16 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Load Triplane Gaussian Transformers" - } - }, - { - "id": 10, - "type": "[Comfy3D] Save 3DGS", - "pos": [ - 1285, - 800 - ], - "size": { - "0": 275.57391357421875, - "1": 58 - }, - "flags": {}, - "order": 11, - "mode": 0, - "inputs": [ - { - "name": "gs_ply", - "type": "GS_PLY", - "link": 12 - } - ], - "outputs": [ - { - "name": "save_path", - "type": "STRING", - "links": [ - 13 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Save 3DGS" - }, - "widgets_values": [ - "MeshTest\\3DGS_Picacho.ply" - ] - }, - { - "id": 24, - "type": "ImpactFloat", - "pos": [ - 434, - 41 - ], - "size": { - "0": 315, - "1": 58 - }, - "flags": {}, - "order": 3, - "mode": 0, - "outputs": [ - { - "name": "FLOAT", - "type": "FLOAT", - "links": [ - 28, - 29, - 30 - ], - "shape": 3, - "slot_index": 0 - } - ], - "title": "Camera Distance", - "properties": { - "Node name for S&R": "ImpactFloat" - }, - "widgets_values": [ - 1.9000000000000001 - ] - }, - { - "id": 23, - "type": "[Comfy3D] Stack Orbit Camera Poses", - "pos": [ - 884, - -218 - ], - "size": { - "0": 320.79998779296875, - "1": 586 - }, - "flags": {}, - "order": 6, - "mode": 0, - "inputs": [ - { - "name": "orbit_radius_start", - "type": "FLOAT", - "link": 29, - "widget": { - "name": "orbit_radius_start" - } - }, - { - "name": "orbit_radius_stop", - "type": "FLOAT", - "link": 30, - "widget": { - "name": "orbit_radius_stop" - } - } - ], - "outputs": [ - { - "name": "orbit_camposes", - "type": "ORBIT_CAMPOSES", - "links": [ - 27, - 33 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "orbit_radius_list", - "type": "FLOAT", - "links": null, - "shape": 6 - }, - { - "name": "elevation_list", - "type": "FLOAT", - "links": null, - "shape": 6 - }, - { - "name": "azimuth_list", - "type": "FLOAT", - "links": null, - "shape": 6 - }, - { - "name": "orbit_center_X_list", - "type": "FLOAT", - "links": null, - "shape": 6 - }, - { - "name": "orbit_center_Y_list", - "type": "FLOAT", - "links": null, - "shape": 6 - }, - { - "name": "orbit_center_Z_list", - "type": "FLOAT", - "links": null, - "shape": 6 - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Stack Orbit Camera Poses" - }, - "widgets_values": [ - 1.9000000000000001, - 1.9000000000000001, - 0.1, - -45, - 45, - 45, - 0, - -0.01, - 12, - 0, - 0, - 0.1, - 0, - 0, - 0.1, - 0, - 0, - 0.1 - ] - }, - { - "id": 27, - "type": "ImpactFloat", - "pos": [ - 915, - -357 - ], - "size": { - "0": 315, - "1": 58 - }, - "flags": {}, - "order": 4, - "mode": 0, - "outputs": [ - { - "name": "FLOAT", - "type": "FLOAT", - "links": [ - 35, - 36 - ], - "shape": 3, - "slot_index": 0 - } - ], - "title": "Camera Fovy", - "properties": { - "Node name for S&R": "ImpactFloat" - }, - "widgets_values": [ - 49.1 - ] - }, - { - "id": 13, - "type": "[Comfy3D] Switch 3DGS Axis", - "pos": [ - 1052, - 630 - ], - "size": { - "0": 210, - "1": 106 - }, - "flags": {}, - "order": 10, - "mode": 0, - "inputs": [ - { - "name": "gs_ply", - "type": "GS_PLY", - "link": 17 - } - ], - "outputs": [ - { - "name": "switched_gs_ply", - "type": "GS_PLY", - "links": [ - 12, - 38 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Switch 3DGS Axis" - }, - "widgets_values": [ - "-y", - "+z", - "-x" - ] - }, - { - "id": 16, - "type": "MaskToImage", - "pos": [ - 2390, - 150 - ], - "size": { - "0": 210, - "1": 26 - }, - "flags": {}, - "order": 17, - "mode": 0, - "inputs": [ - { - "name": "mask", - "type": "MASK", - "link": 19 - } - ], - "outputs": [ - { - "name": "IMAGE", - "type": "IMAGE", - "links": [ - 20 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "MaskToImage" - } - }, - { - "id": 18, - "type": "[Comfy3D] Gaussian Splatting Orbit Renderer", - "pos": [ - 1695.2593051633276, - 22.362099354583734 - ], - "size": { - "0": 556.800048828125, - "1": 198 - }, - "flags": {}, - "order": 14, - "mode": 0, - "inputs": [ - { - "name": "gs_ply", - "type": "GS_PLY", - "link": 39 - }, - { - "name": "render_orbit_camera_poses", - "type": "ORBIT_CAMPOSES", - "link": 27 - }, - { - "name": "render_orbit_camera_fovy", - "type": "FLOAT", - "link": 35, - "widget": { - "name": "render_orbit_camera_fovy" - } - } - ], - "outputs": [ - { - "name": "rendered_gs_images", - "type": "IMAGE", - "links": [ - 18, - 41 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "rendered_gs_masks", - "type": "MASK", - "links": [ - 19, - 42 - ], - "shape": 3, - "slot_index": 1 - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Gaussian Splatting Orbit Renderer" - }, - "widgets_values": [ - 1024, - 1024, - 49.1, - 0, - 0, - 0 - ] - }, - { - "id": 29, - "type": "Reroute", - "pos": [ - 2121.2593051633326, - -47.637900645416266 - ], - "size": [ - 75, - 26 - ], - "flags": {}, - "order": 7, - "mode": 0, - "inputs": [ - { - "name": "", - "type": "*", - "link": 36, - "widget": { - "name": "value" - } - } - ], - "outputs": [ - { - "name": "", - "type": "FLOAT", - "links": [ - 44 - ], - "slot_index": 0 - } - ], - "properties": { - "showOutputText": false, - "horizontal": false - } - }, - { - "id": 28, - "type": "Reroute", - "pos": [ - 1292, - 325 - ], - "size": [ - 75, - 26 - ], - "flags": {}, - "order": 9, - "mode": 0, - "inputs": [ - { - "name": "", - "type": "*", - "link": 33 - } - ], - "outputs": [ - { - "name": "", - "type": "ORBIT_CAMPOSES", - "links": [ - 45 - ], - "slot_index": 0 - } - ], - "properties": { - "showOutputText": false, - "horizontal": false - } - }, - { - "id": 12, - "type": "[Comfy3D] Preview 3DGS", - "pos": [ - 1590, - 800 - ], - "size": [ - 600, - 500 - ], - "flags": { - "collapsed": false - }, - "order": 13, - "mode": 0, - "inputs": [ - { - "name": "gs_file_path", - "type": "STRING", - "link": 13, - "widget": { - "name": "gs_file_path" - } - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Preview 3DGS" - }, - "widgets_values": [ - "", - null - ] - }, - { - "id": 17, - "type": "PreviewImage", - "pos": [ - 2650, - -160 - ], - "size": { - "0": 215.39028930664062, - "1": 246 - }, - "flags": { - "collapsed": false - }, - "order": 19, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 20 - } - ], - "properties": { - "Node name for S&R": "PreviewImage" - } - }, - { - "id": 30, - "type": "[Comfy3D] Switch 3DGS Axis", - "pos": [ - 1301, - 420 - ], - "size": { - "0": 210, - "1": 106 - }, - "flags": {}, - "order": 12, - "mode": 0, - "inputs": [ - { - "name": "gs_ply", - "type": "GS_PLY", - "link": 38 - } - ], - "outputs": [ - { - "name": "switched_gs_ply", - "type": "GS_PLY", - "links": [ - 39, - 46 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Switch 3DGS Axis" - }, - "widgets_values": [ - "+x", - "-y", - "-z" - ] - }, - { - "id": 20, - "type": "[Comfy3D] Convert 3DGS To Pointcloud", - "pos": [ - 1580, - 494 - ], - "size": { - "0": 218.39999389648438, - "1": 26 - }, - "flags": {}, - "order": 15, - "mode": 0, - "inputs": [ - { - "name": "gs_ply", - "type": "GS_PLY", - "link": 46 - } - ], - "outputs": [ - { - "name": "points_cloud", - "type": "POINTCLOUD", - "links": [ - 23 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Convert 3DGS To Pointcloud" - } - }, - { - "id": 31, - "type": "[Comfy3D] Bake Texture To Mesh", - "pos": [ - 2343, - 286 - ], - "size": { - "0": 531.5999755859375, - "1": 334 - }, - "flags": {}, - "order": 20, - "mode": 0, - "inputs": [ - { - "name": "reference_images", - "type": "IMAGE", - "link": 41 - }, - { - "name": "reference_masks", - "type": "MASK", - "link": 42 - }, - { - "name": "reference_orbit_camera_poses", - "type": "ORBIT_CAMPOSES", - "link": 45 - }, - { - "name": "mesh", - "type": "MESH", - "link": 43 - }, - { - "name": "reference_orbit_camera_fovy", - "type": "FLOAT", - "link": 44, - "widget": { - "name": "reference_orbit_camera_fovy" - } - } - ], - "outputs": [ - { - "name": "trained_mesh", - "type": "MESH", - "links": [ - 40 - ], - "shape": 3, - "slot_index": 0 - }, - { - "name": "baked_texture", - "type": "IMAGE", - "links": null, - "shape": 3 - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Bake Texture To Mesh" - }, - "widgets_values": [ - 49.1, - 1024, - 1024, - 1000, - 3, - 0.1, - false, - 0.01, - 0.5, - false - ] - }, - { - "id": 25, - "type": "[Comfy3D] Save 3D Mesh", - "pos": [ - 2904, - 286 - ], - "size": { - "0": 315, - "1": 58 - }, - "flags": {}, - "order": 21, - "mode": 0, - "inputs": [ - { - "name": "mesh", - "type": "MESH", - "link": 40 - } - ], - "outputs": [ - { - "name": "save_path", - "type": "STRING", - "links": [ - 32 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Save 3D Mesh" - }, - "widgets_values": [ - "MeshTest\\3DMesh_Picacho.obj" - ] - }, - { - "id": 26, - "type": "[Comfy3D] Preview 3DMesh", - "pos": [ - 3250, - 286 - ], - "size": [ - 600, - 500 - ], - "flags": {}, - "order": 22, - "mode": 0, - "inputs": [ - { - "name": "mesh_file_path", - "type": "STRING", - "link": 32, - "widget": { - "name": "mesh_file_path" - } - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Preview 3DMesh" - }, - "widgets_values": [ - "", - null - ] - }, - { - "id": 15, - "type": "PreviewImage", - "pos": [ - 2390, - -160 - ], - "size": { - "0": 210, - "1": 246 - }, - "flags": { - "collapsed": false - }, - "order": 16, - "mode": 0, - "inputs": [ - { - "name": "images", - "type": "IMAGE", - "link": 18 - } - ], - "properties": { - "Node name for S&R": "PreviewImage" - } - }, - { - "id": 19, - "type": "[Comfy3D] Deep Marching Tetrahedrons", - "pos": [ - 1833, - 425 - ], - "size": { - "0": 443.4000244140625, - "1": 314 - }, - "flags": {}, - "order": 18, - "mode": 0, - "inputs": [ - { - "name": "reference_points_cloud", - "type": "POINTCLOUD", - "link": 23 - }, - { - "name": "reference_images", - "type": "IMAGE", - "link": null - }, - { - "name": "reference_masks", - "type": "MASK", - "link": null - } - ], - "outputs": [ - { - "name": "trained_mesh", - "type": "MESH", - "links": [ - 43 - ], - "shape": 3, - "slot_index": 0 - } - ], - "properties": { - "Node name for S&R": "[Comfy3D] Deep Marching Tetrahedrons" - }, - "widgets_values": [ - 5000, - 1, - 0.1, - 1, - 1, - 128, - 0.0001, - 2, - 128, - 5 - ] - } - ], - "links": [ - [ - 10, - 4, - 1, - 9, - 0, - "MASK" - ], - [ - 12, - 13, - 0, - 10, - 0, - "GS_PLY" - ], - [ - 13, - 10, - 0, - 12, - 0, - "STRING" - ], - [ - 14, - 4, - 0, - 14, - 0, - "IMAGE" - ], - [ - 15, - 9, - 0, - 14, - 1, - "MASK" - ], - [ - 16, - 2, - 0, - 14, - 2, - "TGS_MODEL" - ], - [ - 17, - 14, - 0, - 13, - 0, - "GS_PLY" - ], - [ - 18, - 18, - 0, - 15, - 0, - "IMAGE" - ], - [ - 19, - 18, - 1, - 16, - 0, - "MASK" - ], - [ - 20, - 16, - 0, - 17, - 0, - "IMAGE" - ], - [ - 23, - 20, - 0, - 19, - 0, - "POINTCLOUD" - ], - [ - 27, - 23, - 0, - 18, - 1, - "ORBIT_CAMPOSES" - ], - [ - 28, - 24, - 0, - 14, - 3, - "FLOAT" - ], - [ - 29, - 24, - 0, - 23, - 0, - "FLOAT" - ], - [ - 30, - 24, - 0, - 23, - 1, - "FLOAT" - ], - [ - 32, - 25, - 0, - 26, - 0, - "STRING" - ], - [ - 33, - 23, - 0, - 28, - 0, - "*" - ], - [ - 35, - 27, - 0, - 18, - 2, - "FLOAT" - ], - [ - 36, - 27, - 0, - 29, - 0, - "*" - ], - [ - 38, - 13, - 0, - 30, - 0, - "GS_PLY" - ], - [ - 39, - 30, - 0, - 18, - 0, - "GS_PLY" - ], - [ - 40, - 31, - 0, - 25, - 0, - "MESH" - ], - [ - 41, - 18, - 0, - 31, - 0, - "IMAGE" - ], - [ - 42, - 18, - 1, - 31, - 1, - "MASK" - ], - [ - 43, - 19, - 0, - 31, - 3, - "MESH" - ], - [ - 44, - 29, - 0, - 31, - 4, - "FLOAT" - ], - [ - 45, - 28, - 0, - 31, - 2, - "ORBIT_CAMPOSES" - ], - [ - 46, - 30, - 0, - 20, - 0, - "GS_PLY" - ] - ], - "groups": [], - "config": {}, - "extra": {}, - "version": 0.4 -} \ No newline at end of file diff --git a/algorithms/main_3DGS.py b/algorithms/main_3DGS.py index a0604d5..a4dc0cb 100644 --- a/algorithms/main_3DGS.py +++ b/algorithms/main_3DGS.py @@ -15,36 +15,34 @@ class GSParams: def __init__(self, - training_iterations=3000, + training_iterations=30_000, batch_size=1, - loss_scale=10000, lambda_ssim=0.2, lambda_alpha=3, lambda_offset=0, lambda_offset_opacity=0, invert_bg_prob = 0.5, - feature_lr=0.01, + feature_lr=0.0025, opacity_lr=0.05, scaling_lr=0.005, - rotation_lr=0.005, - position_lr_init=0.001, - position_lr_final=0.00002, - position_lr_delay_mult=0.02, - position_lr_max_steps=500, - num_pts=5000, + rotation_lr=0.001, + position_lr_init=0.00016, + position_lr_final=0.0000016, + position_lr_delay_mult=0.01, + position_lr_max_steps=30_000, + num_pts=10_000, K=3, percent_dense=0.01, - density_start_iter=100, - density_end_iter=100000, + density_start_iter=500, + density_end_iter=15_000, densification_interval=100, - opacity_reset_interval=700, - densify_grad_threshold=0.01, - sh_degree=0): + opacity_reset_interval=3000, + densify_grad_threshold=0.0002, + sh_degree=3): # training params self.training_iterations = training_iterations self.batch_size = batch_size - self.loss_scale = loss_scale self.lambda_ssim = lambda_ssim self.lambda_alpha = lambda_alpha self.lambda_offset = lambda_offset @@ -212,25 +210,27 @@ def training(self): masks_batch_torch = torch.cat(masks_batch, dim=0) ref_masks_batch_torch = torch.cat(ref_masks_batch, dim=0) + #loss_scaler = self.gs_params.loss_scale * step_ratio + # rgb loss - loss += (1 - self.gs_params.lambda_ssim) * self.gs_params.loss_scale * step_ratio * F.mse_loss(masked_rendered_img_batch_torch, masked_ref_img_batch_torch) + loss += (1 - self.gs_params.lambda_ssim) * F.l1_loss(masked_rendered_img_batch_torch, masked_ref_img_batch_torch) # alpha loss - loss += self.gs_params.lambda_alpha * self.gs_params.loss_scale * step_ratio * F.mse_loss(masks_batch_torch, ref_masks_batch_torch) + loss += self.gs_params.lambda_alpha * F.mse_loss(masks_batch_torch, ref_masks_batch_torch) # D-SSIM loss # [1, 3, H, W] in [0, 1] #loss += self.lambda_ssim * (1 - self.ssim_loss(X, Y)) - loss += self.gs_params.lambda_ssim * self.gs_params.loss_scale * step_ratio * (1 - self.ms_ssim_loss(masked_ref_img_batch_torch, masked_rendered_img_batch_torch)) + loss += self.gs_params.lambda_ssim * (1 - self.ms_ssim_loss(masked_ref_img_batch_torch, masked_rendered_img_batch_torch)) if self.gs_params.lambda_offset > 0: # Reference offset loss offset_norm = self.renderer.gaussians.get_xyz_offset.norm(dim=-1, keepdim=True) - loss += self.gs_params.lambda_offset * self.gs_params.loss_scale * step_ratio * torch.mean(offset_norm) + loss += self.gs_params.lambda_offset * torch.mean(offset_norm) if self.gs_params.lambda_offset_opacity > 0: # Alpha penalty loss - loss += self.gs_params.lambda_offset_opacity * self.gs_params.loss_scale * step_ratio * torch.mean(offset_norm.detach() * self.renderer.gaussians.get_opacity) + loss += self.gs_params.lambda_offset_opacity * torch.mean(offset_norm.detach() * self.renderer.gaussians.get_opacity) # optimize step @@ -245,12 +245,12 @@ def training(self): self.renderer.gaussians.add_densification_stats(viewspace_point_tensor, visibility_filter) if step % self.gs_params.densification_interval == 0: - #self.renderer.gaussians.densify_and_prune(self.gs_params.densify_grad_threshold, min_opacity=0.01, extent=4, max_screen_size=1) + self.renderer.gaussians.densify_and_prune(self.gs_params.densify_grad_threshold, min_opacity=0.005, extent=4, max_screen_size=1) #self.renderer.gaussians.densify_and_prune_by_compatness(self.gs_params.K, min_opacity=0.01, extent=4, max_screen_size=1) #self.renderer.gaussians.densify_by_clone_and_split(self.gs_params.densify_grad_threshold, extent=4) - self.renderer.gaussians.densify_by_compatness(self.gs_params.K) - self.renderer.gaussians.prune(min_opacity=0.01, extent=4, max_screen_size=1, max_offset=0.1) + #self.renderer.gaussians.densify_by_compatness(self.gs_params.K) + #self.renderer.gaussians.prune(min_opacity=0.01, extent=4, max_screen_size=1, max_offset=0.1) if step % self.gs_params.opacity_reset_interval == 0: self.renderer.gaussians.reset_opacity() diff --git a/nodes.py b/nodes.py index ca9762f..718c60f 100644 --- a/nodes.py +++ b/nodes.py @@ -721,31 +721,30 @@ def INPUT_TYPES(cls): "reference_masks": ("MASK",), "reference_orbit_camera_poses": ("ORBIT_CAMPOSES",), # (orbit radius, elevation, azimuth, orbit center X, orbit center Y, orbit center Z) "reference_orbit_camera_fovy": ("FLOAT", {"default": 49.1, "min": 0.0, "max": 180.0, "step": 0.1}), - "training_iterations": ("INT", {"default": 1000, "min": 1, "max": 100000}), - "batch_size": ("INT", {"default": 3, "min": 1, "max": 0xffffffffffffffff}), - "loss_value_scale": ("FLOAT", {"default": 10000.0, "min": 1.0}), + "training_iterations": ("INT", {"default": 30_000, "min": 1, "max": 0xffffffffffffffff}), + "batch_size": ("INT", {"default": 1, "min": 1, "max": 0xffffffffffffffff}), "ms_ssim_loss_weight": ("FLOAT", {"default": 0.2, "min": 0.0, "max": 1.0, }), "alpha_loss_weight": ("FLOAT", {"default": 3, "min": 0.0, }), "offset_loss_weight": ("FLOAT", {"default": 0.0, "min": 0.0, }), "offset_opacity_loss_weight": ("FLOAT", {"default": 0.0, "min": 0.0, }), "invert_background_probability": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.1}), - "feature_learning_rate": ("FLOAT", {"default": 0.01, "min": 0.00001, "step": 0.00001}), - "opacity_learning_rate": ("FLOAT", {"default": 0.05, "min": 0.00001, "step": 0.00001}), - "scaling_learning_rate": ("FLOAT", {"default": 0.005, "min": 0.00001, "step": 0.00001}), - "rotation_learning_rate": ("FLOAT", {"default": 0.005, "min": 0.00001, "step": 0.00001}), - "position_learning_rate_init": ("FLOAT", {"default": 0.001, "min": 0.00001, "step": 0.00001}), - "position_learning_rate_final": ("FLOAT", {"default": 0.00002, "min": 0.00001, "step": 0.00001}), - "position_learning_rate_delay_mult": ("FLOAT", {"default": 0.02, "min": 0.00001, "step": 0.00001}), - "position_learning_rate_max_steps": ("INT", {"default": 500, "min": 1}), - "initial_gaussians_num": ("INT", {"default": 5000, "min": 1}), - "K_nearest_neighbors": ("INT", {"default": 3, "min": 1}), + "feature_learning_rate": ("FLOAT", {"default": 0.0025, "min": 0.000001, "step": 0.000001}), + "opacity_learning_rate": ("FLOAT", {"default": 0.05, "min": 0.000001, "step": 0.000001}), + "scaling_learning_rate": ("FLOAT", {"default": 0.005, "min": 0.000001, "step": 0.000001}), + "rotation_learning_rate": ("FLOAT", {"default": 0.001, "min": 0.000001, "step": 0.000001}), + "position_learning_rate_init": ("FLOAT", {"default": 0.00016, "min": 0.000001, "step": 0.000001}), + "position_learning_rate_final": ("FLOAT", {"default": 0.0000016, "min": 0.0000001, "step": 0.0000001}), + "position_learning_rate_delay_mult": ("FLOAT", {"default": 0.01, "min": 0.000001, "step": 0.000001}), + "position_learning_rate_max_steps": ("INT", {"default": 30_000, "min": 1, "max": 0xffffffffffffffff}), + "initial_gaussians_num": ("INT", {"default": 10_000, "min": 1, "max": 0xffffffffffffffff}), + "K_nearest_neighbors": ("INT", {"default": 3, "min": 1, "max": 0xffffffffffffffff}), "percent_dense": ("FLOAT", {"default": 0.01, "min": 0.00001, "step": 0.00001}), - "density_start_iterations": ("INT", {"default": 100, "min": 0}), - "density_end_iterations": ("INT", {"default": 100000, "min": 0}), - "densification_interval": ("INT", {"default": 100, "min": 1}), - "opacity_reset_interval": ("INT", {"default": 700, "min": 1}), - "densify_grad_threshold": ("FLOAT", {"default": 0.01, "min": 0.00001, "step": 0.00001}), - "gaussian_sh_degree": ("INT", {"default": 0, "min": 0}), + "density_start_iterations": ("INT", {"default": 500, "min": 0, "max": 0xffffffffffffffff}), + "density_end_iterations": ("INT", {"default": 15_000, "min": 0, "max": 0xffffffffffffffff}), + "densification_interval": ("INT", {"default": 100, "min": 1, "max": 0xffffffffffffffff}), + "opacity_reset_interval": ("INT", {"default": 3000, "min": 1, "max": 0xffffffffffffffff}), + "densify_grad_threshold": ("FLOAT", {"default": 0.0002, "min": 0.00001, "step": 0.00001}), + "gaussian_sh_degree": ("INT", {"default": 3, "min": 0}), }, "optional": { diff --git a/web/js/threeVisualizer.js b/web/js/threeVisualizer.js index aca3e4a..863ef0a 100644 --- a/web/js/threeVisualizer.js +++ b/web/js/threeVisualizer.js @@ -119,8 +119,9 @@ async function main(filepath="") { loader.load( url, function ( gltf ) { const model = gltf.scene; - model.position.set( 1, 1, 0 ); - model.scale.set( 0.01, 0.01, 0.01 ); + //model.position.set( 1, 1, 0 ); + //model.scale.set( 0.01, 0.01, 0.01 ); + model.scale.set( 3, 3, 3 ); scene.add( model );