added evaluation code

README.md

@@ -111,6 +111,19 @@ ${GLAMR_ROOT}
 | |-- H36M
 ```
 
+
+# Evaluation
+First, run GLAMR on the test set of the dataset you want to evaluate. For example, to run GLAMR on the 3DPW test set:
+```
+python global_recon/run_dataset.py --dataset 3dpw --cfg glamr_3dpw --out_dir out/3dpw
+```
+Next, evaluate the results generated by GLAMR:
+```
+python eval/eval.py --dataset 3dpw --results_dir out/3dpw
+```
+Similarly, to evaluate on Dynamic Human3.6M, you can replace the `3dpw` to `h36m` for the dataset and config.
+
+
 ### AMASS
 The following command processes the original [AMASS](https://amass.is.tue.mpg.de/) dataset into a processed version used in the code:
 ```

global_recon/cfg/glamr_3dpw.yml

@@ -0,0 +1,111 @@
+dataset: 3dpw
+
+# recon model
+grecon_model_name: global_recon_model
+grecon_model_specs:
+ motion_traj_cfg: joint_motion_traj_demo
+ # flags
+ est_type: hybrik
+ flag_infer_motion_traj: true
+ flag_pred_traj: true
+ flag_opt_traj: true
+ flag_opt_cam: true
+ flag_fixed_cam: false
+ flag_init_cam_all_frames: false
+ flag_opt_cam_from_person_pose: true
+
+
+opt_stage_specs:
+
+ init_opt:
+
+ opt_lr: 1.e-2
+ opt_niters: 200
+ opt_variables: ['local_xy', 'local_heading']
+
+ loss_cfg:
+ rel_transform:
+ trans_weight: 0.0
+ weight: 200
+
+ kp_2d:
+ weight: 1.0
+ min_conf: 0.3
+ kp_2d_dist:
+ weight: 1.0
+ min_conf: 0.3
+ monitor_only: true
+
+ cam_traj_rot:
+ rot_type: 6d
+ weight: 1.e+5
+
+ traj_rot_smoothness:
+ weight: 1.e+3
+
+ # reg loss
+ local_traj_dxy_reg:
+ weight: 3.e+2
+ local_traj_dheading_reg_new:
+ weight: 3.e+3
+ local_traj_rot_reg:
+ weight: 5.e+2
+ local_traj_z_reg:
+ weight: 1.e+2
+ cam_inv_trans_residual_reg:
+ weight: 1.e+2
+
+ # smoothness loss
+ cam_inv_rot_smoothness:
+ weight: 1.e+1
+ cam_origin_smoothness:
+ weight: 1.e+2
+ cam_up_reg:
+ weight: 1.e+5
+
+
+ main_opt:
+
+ opt_lr: 1.e-4
+ opt_niters: 500
+ opt_variables: ['local_xy', 'local_heading', 'local_dheading', 'local_dxy', 'local_rot']
+
+ loss_cfg:
+ rel_transform:
+ trans_weight: 0.0
+ weight: 200
+
+ kp_2d:
+ weight: 1.0
+ min_conf: 0.3
+ kp_2d_dist:
+ weight: 1.0
+ min_conf: 0.3
+ monitor_only: true
+
+ cam_traj_rot:
+ rot_type: 6d
+ weight: 1.e+5
+
+ traj_rot_smoothness:
+ weight: 1.e+3
+
+ # reg loss
+ local_traj_dxy_reg:
+ weight: 3.e+2
+ local_traj_dheading_reg_new:
+ weight: 3.e+3
+ local_traj_rot_reg:
+ weight: 5.e+2
+ local_traj_z_reg:
+ weight: 1.e+2
+ cam_inv_trans_residual_reg:
+ weight: 1.e+2
+
+ # smoothness loss
+ cam_inv_rot_smoothness:
+ weight: 1.e+1
+ cam_origin_smoothness:
+ weight: 1.e+2
+ cam_up_reg:
+ weight: 1.e+5

global_recon/cfg/glamr_h36m.yml

@@ -0,0 +1,110 @@
+dataset: h36m
+
+# recon model
+grecon_model_name: global_recon_model
+grecon_model_specs:
+ motion_traj_cfg: joint_motion_traj_demo
+ # flags
+ est_type: hybrik
+ flag_infer_motion_traj: true
+ flag_pred_traj: true
+ flag_opt_traj: true
+ flag_opt_cam: true
+ flag_fixed_cam: false
+ flag_init_cam_all_frames: false
+
+
+opt_stage_specs:
+
+ init_opt:
+
+ opt_lr: 1.e-2
+ opt_niters: 200
+ opt_variables: ['cam', 'local_xy', 'local_heading']
+
+ loss_cfg:
+ rel_transform:
+ trans_weight: 0.0
+ weight: 200
+
+ kp_2d:
+ weight: 1.0
+ min_conf: 0.3
+ kp_2d_dist:
+ weight: 1.0
+ min_conf: 0.3
+ monitor_only: true
+
+ cam_traj_rot:
+ rot_type: 6d
+ weight: 1.e+5
+
+ traj_rot_smoothness:
+ weight: 1.e+3
+
+ # reg loss
+ local_traj_dxy_reg:
+ weight: 3.e+2
+ local_traj_dheading_reg_new:
+ weight: 3.e+3
+ local_traj_rot_reg:
+ weight: 5.e+2
+ local_traj_z_reg:
+ weight: 1.e+2
+ cam_inv_trans_residual_reg:
+ weight: 1.e+2
+
+ # smoothness loss
+ cam_inv_rot_smoothness:
+ weight: 1.e+4
+ cam_origin_smoothness:
+ weight: 1.e+4
+ cam_up_reg:
+ weight: 1.e+5
+
+
+ main_opt:
+
+ opt_lr: 1.e-4
+ opt_niters: 500
+ opt_variables: ['cam', 'local_xy', 'local_heading', 'world_dheading', 'local_dxy', 'local_rot']
+
+ loss_cfg:
+ rel_transform:
+ trans_weight: 0.0
+ weight: 200
+
+ kp_2d:
+ weight: 1.0
+ min_conf: 0.3
+ kp_2d_dist:
+ weight: 1.0
+ min_conf: 0.3
+ monitor_only: true
+
+ cam_traj_rot:
+ rot_type: 6d
+ weight: 1.e+5
+
+ traj_rot_smoothness:
+ weight: 1.e+3
+
+ # reg loss
+ local_traj_dxy_reg:
+ weight: 3.e+2
+ local_traj_dheading_reg_new:
+ weight: 3.e+3
+ local_traj_rot_reg:
+ weight: 5.e+2
+ local_traj_z_reg:
+ weight: 1.e+2
+ cam_inv_trans_residual_reg:
+ weight: 1.e+2
+
+ # smoothness loss
+ cam_inv_rot_smoothness:
+ weight: 1.e+4
+ cam_origin_smoothness:
+ weight: 1.e+4
+ cam_up_reg:
+ weight: 1.e+5

global_recon/eval_dataset.py

@@ -0,0 +1,56 @@
+import os, sys
+sys.path.append(os.path.join(os.getcwd()))
+sys.path.append(os.path.join(os.getcwd(), 'kama'))
+import os.path as osp
+import pickle
+import argparse
+import numpy as np
+import torch
+from global_recon.utils.evaluator import Evaluator
+
+
+test_sequences = {
+ '3dpw': ['downtown_arguing_00', 'downtown_bar_00', 'downtown_bus_00', 'downtown_cafe_00', 'downtown_car_00', 'downtown_crossStreets_00', 'downtown_downstairs_00', 
+ 'downtown_enterShop_00', 'downtown_rampAndStairs_00', 'downtown_runForBus_00', 'downtown_runForBus_01', 'downtown_sitOnStairs_00', 'downtown_stairs_00',
+ 'downtown_upstairs_00', 'downtown_walkBridge_01', 'downtown_walkUphill_00', 'downtown_walking_00', 'downtown_warmWelcome_00', 'downtown_weeklyMarket_00',
+ 'downtown_windowShopping_00', 'flat_guitar_01', 'flat_packBags_00', 'office_phoneCall_00', 'outdoors_fencing_01']
+}
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--dataset', default='3dpw')
+parser.add_argument('--results_dir', default='out/3dpw')
+parser.add_argument('--gpu', type=int, default=0)
+parser.add_argument('--seeds', default="1")
+args = parser.parse_args()
+
+results_dir = args.results_dir
+sequences = test_sequences[args.dataset]
+seeds = [int(x) for x in args.seeds.split(',')]
+multi_seeds = len(seeds) > 1
+
+if torch.cuda.is_available() and args.gpu >= 0:
+ device = torch.device('cuda', index=args.gpu)
+ torch.cuda.set_device(args.gpu)
+else:
+ device = torch.device('cpu')
+torch.torch.set_grad_enabled(False)
+
+evaluator = Evaluator(results_dir, args.dataset, device=device, log_file=f'{results_dir}/log_eval.txt', compute_sample=multi_seeds)
+seed_evaluator = Evaluator(results_dir, args.dataset, device=device, log_file=f'{results_dir}/log_eval_seed.txt', compute_sample=multi_seeds)
+
+for sind, seq_name in enumerate(sequences[:2]):
+ metrics_dict_arr = []
+ evaluator.log.info(f'{sind}/{len(sequences)} evaluating global reconstruction for {seq_name}')
+
+ for seed in seeds:
+ data_file = f'{results_dir}/{seq_name}/grecon/{seq_name}_seed{seed}.pkl'
+ data = pickle.load(open(data_file, 'rb'))
+
+ metrics_dict = seed_evaluator.compute_sequence_metrics(data, seq_name, accumulate=False)
+ metrics_dict_arr.append(metrics_dict)
+
+ metrics_dict_allseeds = evaluator.metrics_from_multiple_seeds(metrics_dict_arr)
+ evaluator.update_accumulated_metrics(metrics_dict_allseeds, seq_name)
+ evaluator.print_metrics(metrics_dict_allseeds, prefix=f'{sind}/{len(sequences)} --- All seeds {seq_name} --- ', print_accum=False)
+
+evaluator.print_metrics(prefix=f'Total ------- ', print_accum=True)