fixed evaluation code, final results

collect_results.py

@@ -28,19 +28,19 @@ for expname in experiments:
         if not file_path.is_file():
             print(file_path)
             continue
-    with open(file_path, 'r') as f:
-        lines = f.readlines()
-        lines = lines[:8]
-        lines = [l.strip() for l in lines]
-        metric_name, metric_val = zip(*(s.split(":") for s in lines))
-        try:
-            metric_val = [round(float(x.strip().split("'")[1]), 6) for x in metric_val] 
-            metric_val = [float(str(x).ljust(5, '0')) for x in metric_val]
-        except:
-            print('buggy file at:', file_path)
-            continue        
-        name = f'{expname}_{sample_variant}'
-        metr_d[name] = dict(zip(metric_name, metric_val))
+        with open(file_path, 'r') as f:
+            lines = f.readlines()
+            lines = lines[:8]
+            lines = [l.strip() for l in lines]
+            metric_name, metric_val = zip(*(s.split(":") for s in lines))
+            try:
+                metric_val = [round(float(x.strip().split("'")[1]), 6) for x in metric_val] 
+                metric_val = [float(str(x).ljust(5, '0')) for x in metric_val]
+            except:
+                print('buggy file at:', file_path)
+                continue        
+            name = f'{expname}_{sample_variant}'
+            metr_d[name] = dict(zip(metric_name, metric_val))
 
 write_json(metr_d, f'{dir}/final_mets.json')
 list_of_tup_expname_metrs = [(k,v) for k,v in metr_d.items()]
@@ -57,11 +57,11 @@ df.sort_index()
 html = df.to_html(classes='table table-striped text-center', justify='center')
   
 # write html to file
-# text_file = open(f'{dir}/metrics.html', "w")
-# text_file.write(html)
-# text_file.close()
-# text_file = open(f'./metrics.html', "w")
-# text_file.write(html)
-# text_file.close()
+text_file = open(f'{dir}/metrics.html', "w")
+text_file.write(html)
+text_file.close()
+text_file = open(f'./metrics.html', "w")
+text_file.write(html)
+text_file.close()
 
 latex = df.to_latex(index=True)
\ No newline at end of file

spawn_jobs.py

@@ -34,7 +34,7 @@ base_dir = Path("/is/cluster/work/nathanasiou/experiments/teach/babel-amass/teac
 #             cmd.extend(['--extras', f'align={a} slerp_ws={s}'])
 #             run(cmd)
 
-# # submission only samples
+# submission only samples
 # for exp in ['teach-motion_branch_false-teacher_force_false', 'teach-motion_branch_false-teacher_force_true',
 #             'teach-motion_branch_true-teacher_force_false', 'teach-motion_branch_true-teacher_force_true']:
 #     exp_path = Path(f'{base_dir}/{exp}/')
@@ -57,33 +57,33 @@ base_dir = Path("/is/cluster/work/nathanasiou/experiments/teach/babel-amass/teac
 
 # teach/independent evaluation
 
-base_dir = Path("/is/cluster/work/nathanasiou/experiments/teach/babel-amass/teach-post-submission")
-exps_to_sample = ['teach-motion_branch_false-teacher_force_false',
-                  'teach-motion_branch_false-teacher_force_true',
-                  'teach-motion_branch_true-teacher_force_false',
-                  'teach-motion_branch_true-teacher_force_true',
-                  'temos-motion_branch_false-teacher_force_false',
-                  'temos-motion_branch_true-teacher_force_false',
-                  'teach-hist10-teacher_force',
-                  'teach-hist10-teacher_force_false',
-                  'teach-hist5-teacher_force',
-                  'teach-hist5-teacher_force_false']
+# base_dir = Path("/is/cluster/work/nathanasiou/experiments/teach/babel-amass/teach-post-submission")
+# exps_to_sample = ['teach-motion_branch_false-teacher_force_false',
+#                   'teach-motion_branch_false-teacher_force_true',
+#                   'teach-motion_branch_true-teacher_force_false',
+#                   'teach-motion_branch_true-teacher_force_true',
+#                   'temos-motion_branch_false-teacher_force_false',
+#                   'temos-motion_branch_true-teacher_force_false',
+#                   'teach-hist10-teacher_force',
+#                   'teach-hist10-teacher_force_false',
+#                   'teach-hist5-teacher_force',
+#                   'teach-hist5-teacher_force_false']
 
 # pairs_type = ['samples_no-slerp_unaligned_pairs',
 #               'samples_slerp_aligned_pairs',
 #               'samples_slerp_unaligned_pairs',
 #               'samples_no-slerp_aligned_pairs']
 
-set_folder = 'checkpoint-last/val'
-slerp = ['true', 'false']
-align = ['true', 'false']
-for exp in exps_to_sample:
-    for slerp in ['true', 'false']:
-        for align in ['true', 'false']:
-            samples_path = Path(f'{base_dir}/{exp}/')
-            cmd = ['python', 'cluster/single_run.py', '--folder', str(samples_path), '--mode', 'eval', '--bid', '40']
-            cmd.extend(['--extras', f'align={align} slerp={slerp}'])
-            run(cmd)
+# set_folder = 'checkpoint-last/val'
+# slerp = ['true', 'false']
+# align = ['true', 'false']
+# for exp in exps_to_sample:
+#     for slerp in ['true', 'false']:
+#         for align in ['true', 'false']:
+#             samples_path = Path(f'{base_dir}/{exp}/')
+#             cmd = ['python', 'cluster/single_run.py', '--folder', str(samples_path), '--mode', 'eval', '--bid', '40']
+#             cmd.extend(['--extras', f'align={align} slerp={slerp}'])
+#             run(cmd)
 
 # submission only samples
 for exp in ['teach-motion_branch_false-teacher_force_false', 'teach-motion_branch_false-teacher_force_true',
@@ -95,32 +95,32 @@ for exp in ['teach-motion_branch_false-teacher_force_false', 'teach-motion_branc
                     f"mode=sequence quality=false num=5 res=med separate_actions={fig_var} fake_trans=false"])
         run(cmd)
 
-# single example rendering
-for exp in ['teach-motion_branch_false-teacher_force_false', 'teach-motion_branch_false-teacher_force_true',
-            'teach-motion_branch_true-teacher_force_false', 'teach-motion_branch_true-teacher_force_true',
-            'temos-motion_branch_false-teacher_force_false', 'temos-motion_branch_true-teacher_force_false']:
-    samples_path = Path(f'{base_dir}/{exp}/samples_slerp_aligned_pairs/checkpoint-last/submission')
-    cmd = ['python', 'cluster/single_run.py', '--folder', str(samples_path), '--mode', 'render', '--bid', '20']
-    cmd.extend([ '--extras',
-                f"mode=video quality=false num=5 res=med fake_trans=trans"])
-    run(cmd)
-
+# # single example rendering
+# for exp in ['teach-motion_branch_false-teacher_force_false', 'teach-motion_branch_false-teacher_force_true',
+#             'teach-motion_branch_true-teacher_force_false', 'teach-motion_branch_true-teacher_force_true',
+#             'temos-motion_branch_false-teacher_force_false', 'temos-motion_branch_true-teacher_force_false']:
+#     samples_path = Path(f'{base_dir}/{exp}/samples_slerp_aligned_pairs/checkpoint-last/submission')
+#     cmd = ['python', 'cluster/single_run.py', '--folder', str(samples_path), '--mode', 'render', '--bid', '20']
+#     cmd.extend([ '--extras',
+#                 f"mode=video quality=false num=5 res=med fake_trans=trans"])
+#     run(cmd)
 
-# single example rendering
-for exp in ['teach-motion_branch_false-teacher_force_false', 'teach-motion_branch_false-teacher_force_true',
-            'teach-motion_branch_true-teacher_force_false', 'teach-motion_branch_true-teacher_force_true',
-            'temos-motion_branch_false-teacher_force_false', 'temos-motion_branch_true-teacher_force_false']:
-    samples_path = Path(f'{base_dir}/{exp}/samples_slerp_aligned_pairs/checkpoint-last/submission/1183-3.npy')
-    for fig_var in ['true', 'false']:
 
-        cmd = ['python', 'cluster/single_run.py', '--folder', str(samples_path), '--mode', 'render', '--bid', '20']
-        cmd.extend([ '--extras',
-                    f"mode=sequence quality=false num=5 res=med separate_actions={fig_var} fake_trans=true"])
-        run(cmd)
-        cmd = ['python', 'cluster/single_run.py', '--folder', str(samples_path), '--mode', 'render', '--bid', '20']
-        cmd.extend([ '--extras',
-                    f"mode=sequence quality=false num=5 res=med separate_actions={fig_var} fake_trans=false"])
-        run(cmd)
+# # single example rendering
+# for exp in ['teach-motion_branch_false-teacher_force_false', 'teach-motion_branch_false-teacher_force_true',
+#             'teach-motion_branch_true-teacher_force_false', 'teach-motion_branch_true-teacher_force_true',
+#             'temos-motion_branch_false-teacher_force_false', 'temos-motion_branch_true-teacher_force_false']:
+#     samples_path = Path(f'{base_dir}/{exp}/samples_slerp_aligned_pairs/checkpoint-last/submission/1183-3.npy')
+#     for fig_var in ['true', 'false']:
+
+#         cmd = ['python', 'cluster/single_run.py', '--folder', str(samples_path), '--mode', 'render', '--bid', '20']
+#         cmd.extend([ '--extras',
+#                     f"mode=sequence quality=false num=5 res=med separate_actions={fig_var} fake_trans=true"])
+#         run(cmd)
+#         cmd = ['python', 'cluster/single_run.py', '--folder', str(samples_path), '--mode', 'render', '--bid', '20']
+#         cmd.extend([ '--extras',
+#                     f"mode=sequence quality=false num=5 res=med separate_actions={fig_var} fake_trans=false"])
+#         run(cmd)
 
 # temos/teach final exps
 # models = ['teach', 'temos']