diff --git a/LICENSE b/LICENSE index 0dc1f32d..1bfecbc4 100644 --- a/LICENSE +++ b/LICENSE @@ -2,12 +2,12 @@ License Software Copyright License for non-commercial scientific research purposes Please read carefully the following terms and conditions and any accompanying documentation before you download -and/or use the EMOCA model, data and software, (the "Model & Software"), including 3D meshes, software, and scripts. -By downloading and/or using the Model & Software (including downloading, cloning, installing, and any other use -of this github repository), you acknowledge that you have read these terms and conditions, understand them, and -agree to be bound by them. If you do not agree with these terms and conditions, you must not download and/or use -the Model & Software. Any infringement of the terms of this agreement will automatically terminate your rights -under this License +and/or use any of the models present in the INFERNO library, data and software, (the "Model & Software"), +including 3D meshes, software, and scripts. By downloading and/or using the Model & Software +(including downloading, cloning, installing, and any other use of this github repository), you acknowledge that +you have read these terms and conditions, understand them, and agree to be bound by them. If you do not agree +with these terms and conditions, you must not download and/or use the Model & Software. Any infringement +of the terms of this agreement will automatically terminate your rights under this License Ownership / Licensees The Model & Software and the associated materials has been developed at the diff --git a/external/Deep3DFaceRecon_pytorch b/external/Deep3DFaceRecon_pytorch deleted file mode 160000 index da7efde8..00000000 --- a/external/Deep3DFaceRecon_pytorch +++ /dev/null @@ -1 +0,0 @@ -Subproject commit da7efde87b4cd5f4346394d5e7f0ac217906a0ab diff --git a/external/Deep3DFaceRecon_pytorch b/external/Deep3DFaceRecon_pytorch new file mode 120000 index 00000000..3ddf3a6d --- /dev/null +++ b/external/Deep3DFaceRecon_pytorch @@ -0,0 +1 @@ +../../gdl_inferno/external/Deep3DFaceRecon_pytorch/ \ No newline at end of file diff --git a/inferno_apps/MotionPrior/LICENSE b/inferno_apps/MotionPrior/LICENSE new file mode 100644 index 00000000..ea7a6d59 --- /dev/null +++ b/inferno_apps/MotionPrior/LICENSE @@ -0,0 +1,88 @@ +License + +Software Copyright License for non-commercial scientific research purposes +Please read carefully the following terms and conditions and any accompanying documentation before you download +and/or use any of the MotionPrior models, data and software, (the "Model & Software"), +including 3D meshes, software, and scripts. By downloading and/or using the Model & Software +(including downloading, cloning, installing, and any other use of this github repository), you acknowledge that +you have read these terms and conditions, understand them, and agree to be bound by them. If you do not agree +with these terms and conditions, you must not download and/or use the Model & Software. Any infringement +of the terms of this agreement will automatically terminate your rights under this License + +Ownership / Licensees +The Model & Software and the associated materials has been developed at the +Max Planck Institute for Intelligent Systems (hereinafter "MPI"). + +Any copyright or patent right is owned by and proprietary material of the +Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (hereinafter “MPG”; MPI and MPG hereinafter +collectively “Max-Planck”) hereinafter the “Licensor”. + +License Grant +Licensor grants you (Licensee) personally a single-user, non-exclusive, non-transferable, free of charge right: + + • To install the Model & Software on computers owned, leased or otherwise controlled by you and/or your organization. + • To use the Model & Software for the sole purpose of performing peaceful non-commercial scientific research, + non-commercial education, or non-commercial artistic projects. + +Any other use, in particular any use for commercial, pornographic, military, or surveillance purposes is prohibited. +This includes, without limitation, incorporation in a commercial product, use in a commercial service, +or production of other artefacts for commercial purposes. + +The Model & Software may not be used to create fake, libelous, misleading, or defamatory content of any kind, excluding +analyses in peer-reviewed scientific research. + +The Model & Software may not be reproduced, modified and/or made available in any form to any third party +without Max-Planck’s prior written permission. + +The Model & Software may not be used for pornographic purposes or to generate pornographic material whether +commercial or not. This license also prohibits the use of the Model & Software to train methods/algorithms/neural +networks/etc. for commercial use of any kind. By downloading the Model & Software, you agree not to reverse engineer it. + +No Distribution +The Model & Software and the license herein granted shall not be copied, shared, distributed, re-sold, offered +for re-sale, transferred or sub-licensed in whole or in part except that you may make one copy for archive +purposes only. + +Disclaimer of Representations and Warranties +You expressly acknowledge and agree that the Model & Software results from basic research, is provided “AS IS”, +may contain errors, and that any use of the Model & Software is at your sole risk. +LICENSOR MAKES NO REPRESENTATIONS +OR WARRANTIES OF ANY KIND CONCERNING THE MODEL & SOFTWARE, NEITHER EXPRESS NOR IMPLIED, AND THE ABSENCE OF ANY +LEGAL OR ACTUAL DEFECTS, WHETHER DISCOVERABLE OR NOT. Specifically, and not to limit the foregoing, licensor +makes no representations or warranties (i) regarding the merchantability or fitness for a particular purpose of +the Model & Software, (ii) that the use of the Model & Software will not infringe any patents, copyrights or other +intellectual property rights of a third party, and (iii) that the use of the Model & Software will not cause any +damage of any kind to you or a third party. + +Limitation of Liability +Because this Model & Software License Agreement qualifies as a donation, according to Section 521 of the German +Civil Code (Bürgerliches Gesetzbuch – BGB) Licensor as a donor is liable for intent and gross negligence only. +If the Licensor fraudulently conceals a legal or material defect, they are obliged to compensate the Licensee +for the resulting damage. + +Licensor shall be liable for loss of data only up to the amount of typical recovery costs which would have +arisen had proper and regular data backup measures been taken. For the avoidance of doubt Licensor shall be +liable in accordance with the German Product Liability Act in the event of product liability. The foregoing +applies also to Licensor’s legal representatives or assistants in performance. Any further liability shall +be excluded. Patent claims generated through the usage of the Model & Software cannot be directed towards the copyright holders. +The Model & Software is provided in the state of development the licensor defines. If modified or extended by +Licensee, the Licensor makes no claims about the fitness of the Model & Software and is not responsible +for any problems such modifications cause. + +No Maintenance Services +You understand and agree that Licensor is under no obligation to provide either maintenance services, +update services, notices of latent defects, or corrections of defects with regard to the Model & Software. +Licensor nevertheless reserves the right to update, modify, or discontinue the Model & Software at any time. + +Defects of the Model & Software must be notified in writing to the Licensor with a comprehensible description +of the error symptoms. The notification of the defect should enable the reproduction of the error. +The Licensee is encouraged to communicate any use, results, modification or publication. + +Publications using the Model & Software +You acknowledge that the Model & Software is a valuable scientific resource and agree to appropriately reference +the following paper in any publication making use of the Model & Software. + +Commercial licensing opportunities +For commercial uses of the Model & Software, please send email to ps-license@tue.mpg.de + +This Agreement shall be governed by the laws of the Federal Republic of Germany except for the UN Sales Convention. diff --git a/inferno_apps/TalkingHead/LICENSE b/inferno_apps/TalkingHead/LICENSE new file mode 100644 index 00000000..7aebc7d6 --- /dev/null +++ b/inferno_apps/TalkingHead/LICENSE @@ -0,0 +1,88 @@ +License + +Software Copyright License for non-commercial scientific research purposes +Please read carefully the following terms and conditions and any accompanying documentation before you download +and/or use any of the TalkingHead models, data and software, (the "Model & Software"), +including 3D meshes, software, and scripts. By downloading and/or using the Model & Software +(including downloading, cloning, installing, and any other use of this github repository), you acknowledge that +you have read these terms and conditions, understand them, and agree to be bound by them. If you do not agree +with these terms and conditions, you must not download and/or use the Model & Software. Any infringement +of the terms of this agreement will automatically terminate your rights under this License + +Ownership / Licensees +The Model & Software and the associated materials has been developed at the +Max Planck Institute for Intelligent Systems (hereinafter "MPI"). + +Any copyright or patent right is owned by and proprietary material of the +Max-Planck-Gesellschaft zur Förderung der Wissenschaften e.V. (hereinafter “MPG”; MPI and MPG hereinafter +collectively “Max-Planck”) hereinafter the “Licensor”. + +License Grant +Licensor grants you (Licensee) personally a single-user, non-exclusive, non-transferable, free of charge right: + + • To install the Model & Software on computers owned, leased or otherwise controlled by you and/or your organization. + • To use the Model & Software for the sole purpose of performing peaceful non-commercial scientific research, + non-commercial education, or non-commercial artistic projects. + +Any other use, in particular any use for commercial, pornographic, military, or surveillance purposes is prohibited. +This includes, without limitation, incorporation in a commercial product, use in a commercial service, +or production of other artefacts for commercial purposes. + +The Model & Software may not be used to create fake, libelous, misleading, or defamatory content of any kind, excluding +analyses in peer-reviewed scientific research. + +The Model & Software may not be reproduced, modified and/or made available in any form to any third party +without Max-Planck’s prior written permission. + +The Model & Software may not be used for pornographic purposes or to generate pornographic material whether +commercial or not. This license also prohibits the use of the Model & Software to train methods/algorithms/neural +networks/etc. for commercial use of any kind. By downloading the Model & Software, you agree not to reverse engineer it. + +No Distribution +The Model & Software and the license herein granted shall not be copied, shared, distributed, re-sold, offered +for re-sale, transferred or sub-licensed in whole or in part except that you may make one copy for archive +purposes only. + +Disclaimer of Representations and Warranties +You expressly acknowledge and agree that the Model & Software results from basic research, is provided “AS IS”, +may contain errors, and that any use of the Model & Software is at your sole risk. +LICENSOR MAKES NO REPRESENTATIONS +OR WARRANTIES OF ANY KIND CONCERNING THE MODEL & SOFTWARE, NEITHER EXPRESS NOR IMPLIED, AND THE ABSENCE OF ANY +LEGAL OR ACTUAL DEFECTS, WHETHER DISCOVERABLE OR NOT. Specifically, and not to limit the foregoing, licensor +makes no representations or warranties (i) regarding the merchantability or fitness for a particular purpose of +the Model & Software, (ii) that the use of the Model & Software will not infringe any patents, copyrights or other +intellectual property rights of a third party, and (iii) that the use of the Model & Software will not cause any +damage of any kind to you or a third party. + +Limitation of Liability +Because this Model & Software License Agreement qualifies as a donation, according to Section 521 of the German +Civil Code (Bürgerliches Gesetzbuch – BGB) Licensor as a donor is liable for intent and gross negligence only. +If the Licensor fraudulently conceals a legal or material defect, they are obliged to compensate the Licensee +for the resulting damage. + +Licensor shall be liable for loss of data only up to the amount of typical recovery costs which would have +arisen had proper and regular data backup measures been taken. For the avoidance of doubt Licensor shall be +liable in accordance with the German Product Liability Act in the event of product liability. The foregoing +applies also to Licensor’s legal representatives or assistants in performance. Any further liability shall +be excluded. Patent claims generated through the usage of the Model & Software cannot be directed towards the copyright holders. +The Model & Software is provided in the state of development the licensor defines. If modified or extended by +Licensee, the Licensor makes no claims about the fitness of the Model & Software and is not responsible +for any problems such modifications cause. + +No Maintenance Services +You understand and agree that Licensor is under no obligation to provide either maintenance services, +update services, notices of latent defects, or corrections of defects with regard to the Model & Software. +Licensor nevertheless reserves the right to update, modify, or discontinue the Model & Software at any time. + +Defects of the Model & Software must be notified in writing to the Licensor with a comprehensible description +of the error symptoms. The notification of the defect should enable the reproduction of the error. +The Licensee is encouraged to communicate any use, results, modification or publication. + +Publications using the Model & Software +You acknowledge that the Model & Software is a valuable scientific resource and agree to appropriately reference +the following paper in any publication making use of the Model & Software. + +Commercial licensing opportunities +For commercial uses of the Model & Software, please send email to ps-license@tue.mpg.de + +This Agreement shall be governed by the laws of the Federal Republic of Germany except for the UN Sales Convention. diff --git a/inferno_apps/TalkingHead/demos/demo_eval_talking_head_on_audio.py b/inferno_apps/TalkingHead/demos/demo_eval_talking_head_on_audio.py index c862ebea..0779ca83 100644 --- a/inferno_apps/TalkingHead/demos/demo_eval_talking_head_on_audio.py +++ b/inferno_apps/TalkingHead/demos/demo_eval_talking_head_on_audio.py @@ -97,14 +97,14 @@ def eval_talking_head_on_audio( # val_ids = ['M032', 'M033', 'M034', 'M035', 'W033', 'W035', 'W036'] # test_ids = ['M037', 'M039', 'M040', 'M041', 'M042', 'W037', 'W038', 'W040'] - def main(): parser = argparse.ArgumentParser() # add the input folder arg parser.add_argument('--path_to_audio', type=str, default= str(get_path_to_assets() / "data/EMOTE_test_example_data/01_gday.wav")) parser.add_argument('--output_folder', type=str, default="results", help="Output folder to save the results to.") - parser.add_argument('--model_name', type=str, default='EMOTE', help='Name of the model to use.') + # parser.add_argument('--model_name', type=str, default='EMOTE', help='Name of the model to use.') + parser.add_argument('--model_name', type=str, default='EMOTEv2', help='Name of the model to use.') parser.add_argument('--path_to_models', type=str, default=str(get_path_to_assets() / "TalkingHead/models")) parser.add_argument('--save_video', type=bool, default=True, help="If true, output images will be saved") parser.add_argument('--save_flame', type=bool, default=False, help="If true, output FLAME values for shape, expression, jaw pose will be saved") @@ -116,11 +116,8 @@ def main(): parser.add_argument('--subject_style', type=str, default='M003', help=f"Which subject style to use. Styles available: \n{training_ids}") parser.add_argument('--neutral_mesh_path', type=str, default='', help="Path to the neutral mesh. If blank, the default FLAME mean face will be used") parser.add_argument('--emotion', type=str, default='all', help="The emotion to generate. One of: neutral, Happy, Sad, Surprise, Fear, Disgust, Anger, Contempt. If 'all', all emotions will be generated.") - parser.add_argument('--intensity', type=str, default='2', help="The emotion intesntiy. One of: 0, 1, 2. If 'all', all emotions will be generated.") - # parser.add_argument('--neutral_mesh_path', type=str, default='/is/cluster/work/rdanecek/faceformer/templates/FaceTalk_170809_00138_TA.ply', - # parser.add_argument('--neutral_mesh_path', type=str, default='/is/cluster/work/rdanecek/faceformer/templates/FaceTalk_170731_00024_TA.ply', - # help="Path to the neutral mesh. If blank, the default FLAME mean face will be used") - + parser.add_argument('--intensity', type=str, default='2', help="The emotion intentsity. One of: 0, 1, 2. If 'all', all emotions will be generated.") + args = parser.parse_args() root = args.path_to_models @@ -155,7 +152,7 @@ def main(): silent_frames_end=args.silent_frames_end, silent_emotion_start=args.silent_emotion_start, silent_emotion_end=args.silent_emotion_end, - outfolder=args.output_folder, + outfolder=str(Path(args.output_folder) / args.model_name), identity_idx = subject_id, save_flame=args.save_flame, save_meshes=args.save_mesh, diff --git a/inferno_apps/TalkingHead/training/train_emote_stage_1.py b/inferno_apps/TalkingHead/training/train_emote_stage_1.py index 1c82b788..8998fb1d 100644 --- a/inferno_apps/TalkingHead/training/train_emote_stage_1.py +++ b/inferno_apps/TalkingHead/training/train_emote_stage_1.py @@ -161,18 +161,19 @@ def submit_trainings(): dataset = "mead_pseudo_gt" # reconstruction_type = "EMICA_mead_mp_lr_mse_15" ## old version of data used in EMOTE paper reconstruction_type = "EMICA-MEAD_flame2020" ## new version of data with much better reconstructions - batching = "fixed_length_bs32_seq32" + # batching = "fixed_length_bs32_seq32" + batching = "fixed_length_bs4_45gb" preprocessor = "flame_tex" ### MEAD splits ## split = "random_70_15_15" ## split = "random_by_sequence_random_70_15_15" - # split = "random_by_sequence_sorted_70_15_15" + # split = "random_by_sequence_sorted_70_15_15" ## identity overlap between train/val/test (not used in paper EMOTE) ## split = "random_by_identityV2_random_70_15_15" - split = "random_by_identityV2_sorted_70_15_15" + split = "random_by_identityV2_sorted_70_15_15" ## split used to train EMOTE (no identity overlap between train/val/test) ## split = "specific_identity_random_80_20_M003" - # split = "specific_identity_sorted_80_20_M003" + # split = "specific_identity_sorted_80_20_M003" ## specific identity (M003) for quick testing ## split = "specific_identity_random_80_20_M005" # split = "specific_identity_sorted_80_20_M005" @@ -186,9 +187,10 @@ def submit_trainings(): ## 4) Motion prior - Choose your motion prior (aka FLINT) - motion_prior_path = get_path_to_assets() / "MotionPrior" / "models" - motion_prior_name = "FLINT" - + # motion_prior_path = get_path_to_assets() / "MotionPrior" / "models" + motion_prior_path = Path("MotionPrior") / "models" + # motion_prior_name = "FLINT" ## FLINT in EMOTE paper + motion_prior_name = "FLINTv2" ## FLINT of EMOTE v2 fixed_overrides = [] ## specify the motion prior for EMOTE diff --git a/inferno_apps/TalkingHead/training/train_emote_stage_2.py b/inferno_apps/TalkingHead/training/train_emote_stage_2.py index 71d9d5f9..fbb8200b 100644 --- a/inferno_apps/TalkingHead/training/train_emote_stage_2.py +++ b/inferno_apps/TalkingHead/training/train_emote_stage_2.py @@ -24,8 +24,8 @@ from inferno.utils.other import get_path_to_assets -submit_ = False -# submit_ = True +# submit_ = False +submit_ = True def submit_reconfigured_trainings(): @@ -49,14 +49,20 @@ def submit_reconfigured_trainings(): w_lip_reading_dis = w_emotion_dis * wl_multiplier lr_emo_weights = [ - (w_lip_reading, w_lip_reading_dis, w_emotion, w_emotion_dis) + # (w_lip_reading, w_lip_reading_dis, w_emotion, w_emotion_dis), ## good weight for EMOTE v1 + # (w_lip_reading, w_lip_reading_dis, w_emotion*1.2, w_emotion_dis*1.2), + # (w_lip_reading, w_lip_reading_dis, w_emotion*0.8, w_emotion_dis*0.8), ## to strong for EMOTE v2 + # (w_lip_reading, w_lip_reading_dis, w_emotion*0.7, w_emotion_dis*0.7), + (w_lip_reading, w_lip_reading_dis, w_emotion*0.6, w_emotion_dis*0.6), + # (w_lip_reading, w_lip_reading_dis, w_emotion*0.5, w_emotion_dis*0.5), ## ok for EMOTE v2 + # (w_lip_reading, w_lip_reading_dis, w_emotion*0.25, w_emotion_dis*0.25), + # (w_lip_reading, w_lip_reading_dis, w_emotion*0.1, w_emotion_dis*0.1), ] ## 4) Set the paths to the models path_to_talkinghead_models = "" - ##5) EMOTE-stage-1 model names to be finetuned - resume_folders = [] + ##5) EMOTE-stage-1 model names to be finetuned # resume_folders += [""] @@ -80,9 +86,10 @@ def submit_reconfigured_trainings(): ## 7) set the video emotion network(s) to be used - path_to_video_emotion = get_path_to_assets() / "VideoEmotionRecognition" / "models" + # path_to_video_emotion = get_path_to_assets() / "VideoEmotionRecognition" / "models" + path_to_video_emotion = Path("/is/cluster/work/rdanecek/video_emotion_recognition/trainings/") video_emotion_networks = [] - video_emotion_networks += ["VideoEmotionAndIntensityClassifier"] + video_emotion_networks += ["2023_04_12_15-14-44_6653765603916292046_VideoEmotionClassifier_MEADP__TSC_NPE_L_early"] # video_emotion_networks += [""] @@ -107,6 +114,9 @@ def submit_reconfigured_trainings(): # reconstruction_type = "EMICA_mead_mp_lr_mse_15" ## old version of data used in EMOTE paper reconstruction_type = "EMICA-MEAD_flame2020" ## new version of data with much better reconstructions batching = "fixed_length_bs4_45gb" + + # if not submit_: + # batching = "fixed_length_bs2" ## small for debugging on workstations if "rendering" in conf: preprocessor = None @@ -119,7 +129,7 @@ def submit_reconfigured_trainings(): ## split = "random_by_sequence_random_70_15_15" # split = "random_by_sequence_sorted_70_15_15" ## split = "random_by_identityV2_random_70_15_15" - split = "random_by_identityV2_sorted_70_15_15" + split = "random_by_identityV2_sorted_70_15_15" ## EMOTE split ## split = "specific_identity_random_80_20_M003" # split = "specific_identity_sorted_80_20_M003" ## split = "specific_identity_random_80_20_M005" @@ -186,12 +196,14 @@ def submit_reconfigured_trainings(): if "bertprior_wild_rendering_ex_vid" in conf: fixed_overrides += [f'learning.losses.emotion_video_loss_disentangled.network_path={str(path_to_video_emotion / video_emo_net)}'] fixed_overrides += [f'learning.losses.emotion_video_loss_disentangled.use_real_video_for_reference={str(use_real_video_for_emotion_reference)}'] - fixed_overrides += ['model.max_epochs=2'] + fixed_overrides += [f'learning.losses.emotion_video_loss_disentangled.target_method_image={str(reconstruction_type)}'] + # fixed_overrides += ['model.max_epochs=2'] if "bertprior_wild_rendering_ex" == conf: fixed_overrides += [f'learning.losses.emotion_loss.use_real_video_for_reference={str(use_real_video_for_emotion_reference)}'] fixed_overrides += [f'learning.losses.emotion_loss_disentangled.use_real_video_for_reference={str(use_real_video_for_emotion_reference)}'] - fixed_overrides += ['model.max_epochs=2'] + fixed_overrides += [f'learning.losses.emotion_loss_disentangled.target_method_image={str(reconstruction_type)}'] + # fixed_overrides += ['model.max_epochs=2'] if "bertprior_wild_rendering_ex" in conf: fixed_overrides += [f'+learning.losses.lip_reading_loss.use_real_video_for_reference={str(use_real_video_for_lip_reference)}']