merge request

cluster/base.py

@@ -17,8 +17,8 @@ CONDOR_FD = 'condor_logs'
 GPUS = {'v100-p16': ('\"Tesla V100-PCIE-16GB\"', 'tesla', 16000),
         'v100-p32': ('\"Tesla V100-PCIE-32GB\"', 'tesla', 32000),
         'v100-s32': ('\"Tesla V100-SXM2-32GB\"', 'tesla', 32000),
-        # 'a100-sm80': ('\"NVIDIA A100-SXM-80GB\"', 'nvidia', 80000),
-        #'a100-sxm40': ('\"NVIDIA A100-SXM4-40GB\"', 'nvidia', 40000),
+        'a100-sm80': ('\"NVIDIA A100-SXM-80GB\"', 'nvidia', 80000),
+        'a100-sxm40': ('\"NVIDIA A100-SXM4-40GB\"', 'nvidia', 40000),
         'quadro6000': ('\"Quadro RTX 6000\"', 'quadro', 24000),
         'rtx2080ti': ('\"NVIDIA GeForce RTX 2080 Ti\"', 'rtx', 11000)
         }
@@ -57,7 +57,7 @@ def get_gpus(min_mem=32000, arch=('tesla', 'quadro', 'rtx', 'nvidia')):
 def launch_task_on_cluster(configs: List[Dict[str, str]],
                            num_exp: int = 1, mode: str = 'train',
                            bid_amount: int = 10, num_workers: int = 8,
-                           memory: int = 64000, gpu_min_mem:int = 32000,
+                           memory: int = 64000, gpu_min_mem:int = 20000,
                            gpu_arch: Optional[List[Tuple[str, ...]]] = 
                            ('tesla', 'quadro', 'rtx', 'nvidia')) -> None:
 

spawn_jobs.py

@@ -18,34 +18,49 @@ def run(cmd):
 # teach/independent sampling
 base_dir = Path("/is/cluster/work/nathanasiou/experiments/teach/babel-amass/teach-post-submission")
 # temos/teach final exps
-motion_branch = ['true']
-teacher_forcing = ['true', 'false']
-hist_frames= [1,2,5,10]
-for hf in hist_frames:
-    for tf in teacher_forcing:
-        cmd = ['python', 'cluster/single_run.py', '--expname', 'teach-post-submission', '--mode', 'train', '--bid', '20']
-        cmd.extend(['--run-id', f'teach-hf-{hf}-motion_branch_true-teacher_force_{tf}', '--extras',
-                    f"model.teacher_forcing={tf} model.hist_frames={hf} model.motion_branch=true data.dtype=separate_pairs callback.render.every_n_epochs=50 model=teach"])
-        run(cmd)
-
-end_script()
 
 # exps_to_sample = ['teach-motion_branch_false-teacher_force_false',
 #                   'teach-motion_branch_false-teacher_force_true',
 #                   'teach-motion_branch_true-teacher_force_false',
 #                   'teach-motion_branch_true-teacher_force_true',
 #                   'temos-motion_branch_false-teacher_force_false',
-#                   'temos-motion_branch_true-teacher_force_false']
+#                   'temos-motion_branch_true-teacher_force_false'
+# models = ['teach', 'temos']
+# motion_branch = ['true', 'false']
+# teacher_forcing = ['true', 'false']
+# hist_frames= [1, 2, 5, 10]
+# for hf in hist_frames:
+#     for is_motion_branch in motion_branch:
+#         for tf in teacher_forcing:
+#             if hf == 5 or hf == 10 and is_motion_branch == 'false':
+#                 continue
+#             else:
+#                 cmd = ['python', 'cluster/single_run.py', '--expname', 'teach-post-submission', '--mode', 'train', '--bid', '20']
+#                 cmd.extend(['--run-id', f'teach-hf-{hf}-motion_branch_{is_motion_branch}-teacher_force_{tf}', '--extras',
+#                         f"model.teacher_forcing={tf} model.hist_frames={hf} model.motion_branch={is_motion_branch} data.dtype=separate_pairs callback.render.every_n_epochs=50 model=teach"])
+#                 run(cmd)
+
+base_dir = Path("/ps/scratch/ps_shared/mpetrovich/3dv_cam_ready_exps/teach/babel-amass/teach-post-submission-29Aug")
 
+exps_to_sample = ['teach-hf-1-motion_branch_true-teacher_force_false',
+                  'teach-hf-1-motion_branch_true-teacher_force_true',
+                  'teach-hf-2-motion_branch_true-teacher_force_false',
+                  'teach-hf-2-motion_branch_true-teacher_force_true',
+                  'teach-hf-5-motion_branch_true-teacher_force_false',
+                  'teach-hf-5-motion_branch_true-teacher_force_true',
+                  'teach-hf-10-motion_branch_true-teacher_force_false',
+                  'teach-hf-10-motion_branch_true-teacher_force_true']
 # slerp = ['null', 8]
 # align = ['full', 'trans']
-# for exp in exps_to_sample:
-#     exp_path = Path(f'{base_dir}/{exp}/')
-#     for s in slerp:
-#         for a in align:
-#             cmd = ['python', 'cluster/single_run.py', '--folder', str(exp_path), '--mode', 'sample', '--bid', '20']
-#             cmd.extend(['--extras', f'align={a} slerp_ws={s}'])
-#             run(cmd)
+combinations = [['null', 'trans'], [8, 'full']]
+for exp in exps_to_sample:
+    exp_path = Path(f'{base_dir}/{exp}/')
+    for c in combinations:
+        slerp = c[0]
+        align = c[1]
+        cmd = ['python', 'cluster/single_run.py', '--folder', str(exp_path), '--mode', 'sample', '--bid', '20']
+        cmd.extend(['--extras', f'align={align} slerp_ws={slerp}'])
+        run(cmd)
 
 # submission only samples
 
@@ -111,15 +126,15 @@ exps_to_sample = ['teach-motion_branch_false-teacher_force_false',
 #            run(cmd)
 
 # submission only samples
-for exp in ['teach-motion_branch_true-teacher_force_false',
-            'temos-motion_branch_true-teacher_force_false']:
-    samples_path = Path(f'{base_dir}/{exp}/samples_vertices_slerp_aligned_pairs/checkpoint-last/submission')
-    for fig_var in ['true', 'false']:
-        cmd = ['python', 'cluster/single_run.py', '--folder', str(samples_path), '--mode', 'render', '--bid', '20']
-        cmd.extend([ '--extras',
-                    f"mode=sequence quality=false num=5 res=med separate_actions={fig_var} fake_trans=false"])
-        run(cmd)
-
+# for exp in ['teach-motion_branch_false-teacher_force_false', 'teach-motion_branch_false-teacher_force_true',
+#             'teach-motion_branch_true-teacher_force_false', 'teach-motion_branch_true-teacher_force_true']:
+#     samples_path = Path(f'{base_dir}/{exp}/samples_slerp_aligned_pairs/checkpoint-last/submission')
+#     for fig_var in ['true', 'false']:
+#         cmd = ['python', 'cluster/single_run.py', '--folder', str(samples_path), '--mode', 'render', '--bid', '20']
+#         cmd.extend([ '--extras',
+#                     f"mode=sequence quality=false num=5 res=med separate_actions={fig_var} fake_trans=false"])
+#         run(cmd)
+end_script()
 # # single example rendering
 for exp in ['teach-motion_branch_true-teacher_force_false',
             'temos-motion_branch_true-teacher_force_false']:

temos/render/blender/camera.py

@@ -30,7 +30,7 @@ class Camera:
                 camera.data.lens = 140
         elif mode == "video":
             if is_mesh:
-                camera.data.lens = 110
+                camera.data.lens = 90
             else:
                 camera.data.lens = 140
 
@@ -60,4 +60,4 @@ class Camera:
         self.camera.location.x += delta_root[0]
         self.camera.location.y += delta_root[1]
 
-        self._root = newroot
\ No newline at end of file
+        self._root = newroot