Affect bug fix

pliang279 · Sep 21, 2021 · 3e5085f · 3e5085f
1 parent 1a12949
commit 3e5085f
Show file tree

Hide file tree

Showing 22 changed files with 93 additions and 123 deletions.
diff --git a/.gitignore b/.gitignore
@@ -150,3 +150,7 @@ dmypy.json
 
 # End of https://www.gitignore.io/api/macos,python,visualstudiocode.DS_Store
 *.pt
+*.pkl
+*.hdf5
+*.png
+.vector_cache/*
diff --git a/README.md b/README.md
@@ -80,10 +80,10 @@ We welcome new contributions to MultiBench through new research areas, datasets,
 
 We release the processed datasets [here](https://drive.google.com/drive/folders/1IXZAjOEWFOGLxAK9JKvwlG2D9LThK6c5?usp=sharing). The raw datasets are also publicly available at [MultimodalSDK](https://github.com/A2Zadeh/CMU-MultimodalSDK) for MOSI and MOSEI, [MUsTARD](https://github.com/soujanyaporia/MUStARD) and [UR-Funny](https://github.com/ROC-HCI/UR-FUNNY). You can obtain processed data with `datasets/affect/get_data.py`, note that `sarcasm` means [MUsTARD](https://github.com/soujanyaporia/MUStARD) and `humor` means [UR-FUNNY](https://github.com/ROC-HCI/UR-FUNNY), please remember to use `regression` for MOSI and MOSEI `task` and `classcification` for MUsTARD and UR-FUNNY.
 
-There are several example scripts for running affect datasets under examples/affect/. For example, to run UR-Funny with simple late fusion, do
+There are several example scripts for running affect datasets under examples/affect/. For example, to run affect datasets with simple late fusion, do
 
 ```
-python3 examples/affect/humor_late_fusion.py
+python3 examples/affect/affect_late_fusion.py
 ```
 
 ### Healthcare

diff --git a/datasets/affect/get_data.py b/datasets/affect/get_data.py
@@ -196,7 +196,7 @@ def __len__(self):
 def get_dataloader(
  filepath: str, batch_size: int = 32, max_seq_len=50, train_shuffle: bool = True,
  num_workers: int = 4, flatten_time_series: bool = False, task=None,
- raw_path='/home/pliang/multibench/affect/mosi/mosi.hdf5') -> DataLoader:
+ raw_path='/home/paul/MultiBench/mosi.hdf5') -> DataLoader:
  with open(filepath, "rb") as f:
  alldata = pickle.load(f)
 

diff --git a/examples/affect/affect_ef_transformer.py → ..._examples/affect/affect_ef_transformer.py b/examples/affect/affect_ef_transformer.py → ..._examples/affect/affect_ef_transformer.py
@@ -1,4 +1,4 @@
-import sys
+impor:t sys
 import os
 sys.path.append(os.getcwd())
 sys.path.append(os.path.dirname(os.path.dirname(os.getcwd())))
@@ -16,6 +16,7 @@
 traindata, validdata, testdata = get_dataloader('/home/pliang/multibench/affect/processed/mosi_raw.pkl')
 
 # mosi/mosei
+
 encoders = [Transformer(409, 512).cuda()]
 head = MLP(512, 256, 1).cuda()
 

diff --git a/examples/affect/affect_gradblend.py → ...cated_examples/affect/affect_gradblend.py b/examples/affect/affect_gradblend.py → ...cated_examples/affect/affect_gradblend.py
@@ -14,7 +14,7 @@
 from private_test_scripts.all_in_one import all_in_one_train
 
 # mosi_raw.pkl, mosei_raw.pkl, sarcasm.pkl, humor.pkl
-traindata, validdata, testdata = get_dataloader('/home/pliang/multibench/affect/processed/mosi_raw.pkl')
+traindata, validdata, testdata = get_dataloader('/home/paul/MultiBench/mosi_raw.pkl')
 
 # mosi/mosei
 encoders=[GRU(35,70,dropout=True,has_padding=True).cuda(), \
@@ -31,7 +31,7 @@
 
 all_modules = [*encoders, head, *unimodal_heads]
 
-fusion = Concat.cuda()
+fusion = Concat().cuda()
 
 def trainprocess():
  train(encoders, head, unimodal_heads, fusion, traindata, validdata, 300, lr=0.005, AUPRC=False, savedir='gb.pt')

diff --git a/examples/affect/affect_lf_transformer.py → ..._examples/affect/affect_lf_transformer.py b/examples/affect/affect_lf_transformer.py → ..._examples/affect/affect_lf_transformer.py
@@ -13,12 +13,12 @@
 from private_test_scripts.all_in_one import all_in_one_train
 
 # mosi_raw.pkl, mosei_raw.pkl, sarcasm.pkl, humor.pkl
-traindata, validdata, testdata = get_dataloader('/home/pliang/multibench/affect/processed/mosi_raw.pkl')
+traindata, validdata, testdata = get_dataloader('/home/paul/MultiBench/mosi_raw.pkl')
 
 # mosi/mosei
-encoders = [Transformer(74, 150).cuda(), Transformer(35, 70).cuda(), Transformer(300, 600).cuda()]
+encoders = [Transformer(74, 150).cuda(), Transformer(35, 75).cuda(), Transformer(300, 600).cuda()]
 
-head = MLP(820, 512, 1).cuda()
+head = MLP(825, 512, 1).cuda()
 
 # humor/sarcasm
 # encoders = [Transformer().cuda()] * 3
@@ -30,11 +30,11 @@
 
 
 def trainprocess():
- train(encoders, fusion, head, traindata, validdata, 1000, task="regression", optimtype=torch.optim.AdamW,
+ train(encoders, fusion, head, traindata, validdata, 10, task="regression", optimtype=torch.optim.AdamW, is_packed=True,
  lr=1e-4, save='mosi_lf_best.pt', weight_decay=0.01, objective=torch.nn.L1Loss())
 
 all_in_one_train(trainprocess, all_modules)
 
 print("Testing:")
 model = torch.load('mosi_lf_best.pt').cuda()
-test(model, testdata, True, torch.nn.L1Loss(), "regression")
+test(model, testdata, 'affect',True, torch.nn.L1Loss(), "posneg-classification")
diff --git a/examples/affect/affect_mctn.py → deprecated_examples/affect/affect_mctn.py b/examples/affect/affect_mctn.py → deprecated_examples/affect/affect_mctn.py
diff --git a/examples/affect/affect_mfm.py → deprecated_examples/affect/affect_mfm.py b/examples/affect/affect_mfm.py → deprecated_examples/affect/affect_mfm.py
diff --git a/deprecated_examples/affect/affect_mult.py b/deprecated_examples/affect/affect_mult.py
@@ -1,31 +1,40 @@
 import sys
 import os
 sys.path.append(os.getcwd())
-
+sys.path.append(os.path.dirname(os.path.dirname(os.getcwd())))
 import torch
-from torch import nn
-from fusions.mult import MULTModel
-from training_structures.unimodal import train, test
-from unimodals.common_models import MLP
 
+from fusions.common_fusions import ConcatEarly
 from datasets.affect.get_data import get_dataloader
+from unimodals.common_models import MLP
+from fusions.mult import MULTModel
 
+from training_structures.Supervised_Learning import train, test
 
-traindata, validdata, testdata = get_dataloader('../affect/processed/mosei_senti_data.pkl')
+from private_test_scripts.all_in_one import all_in_one_train
 
-#mosi
-# encoders=GRU(325,512,dropout=True,has_padding=True).cuda()
-# head=MLP(512,256, 1).cuda()
+# mosi_raw.pkl, mosei_raw.pkl, sarcasm.pkl, humor.pkl
+traindata, validdata, testdata = get_dataloader('/home/pliang/multibench/affect/processed/mosi_raw.pkl')
 
-#mosei
-encoders = MULTModel(3).cuda()
-head = nn.Identity()
+# mosi/mosei
+encoders = [MULTModel(409, 512).cuda()]
+head = MLP(512, 256, 1).cuda()
 
-train(encoders, head, traindata, validdata, 1000, True, True, task="regression", optimtype=torch.optim.AdamW, lr=1e-5,
- save='mosei_mult_best.pt', weight_decay=0.01,criterion=torch.nn.L1Loss(), regularization=False)
+# humor/sarcasm
+# encoders = [Transformer(early=True).cuda()]
+# head = MLP(1128, 512, 1).cuda()
 
+all_modules = [*encoders, head]
+
+fusion = ConcatEarly().cuda()
+
+
+def trainprocess():
+ train(encoders, fusion, head, traindata, validdata, 1000, task="regression", optimtype=torch.optim.AdamW,
+ lr=1e-4, save='mosi_ef_best.pt', weight_decay=0.01, objective=torch.nn.L1Loss())
+
+all_in_one_train(trainprocess, all_modules)
 
 print("Testing:")
-model = torch.load('mosei_mult_best.pt').cuda()
+model = torch.load('mosi_ef_best.pt').cuda()
 test(model, testdata, True, torch.nn.L1Loss(), "regression")
-
diff --git a/examples/affect/affect_unimodal.py → ...ecated_examples/affect/affect_unimodal.py b/examples/affect/affect_unimodal.py → ...ecated_examples/affect/affect_unimodal.py
@@ -14,7 +14,7 @@
 from private_test_scripts.all_in_one import all_in_one_train
 
 # mosi_raw.pkl, mosei_raw.pkl, sarcasm.pkl, humor.pkl
-traindata, validdata, testdata = get_dataloader('/home/pliang/multibench/affect/processed/mosi_raw.pkl')
+traindata, validdata, testdata = get_dataloader('/home/paul/MultiBench/mosi_raw.pkl')
 
 
 modal_num = 2

diff --git a/deprecated_examples/affect/humor_late_fusion.py b/deprecated_examples/affect/humor_late_fusion.py
@@ -4,13 +4,13 @@
 
 import torch
 
-from training_structures.Simple_Late_Fusion import train, test
+from training_structures.Supervised_Learning import train, test
 from fusions.common_fusions import Concat
-from get_data import get_dataloader
+from datasets.affect.get_data import get_dataloader
 from unimodals.common_models import GRU, MLP
 
 # Support mosi/mosi_unaligned/mosei/mosei_unaligned
-traindata, validdata, testdata = get_dataloader('../affect/processed/humor_data.pkl')
+traindata, validdata, testdata = get_dataloader('/home/pliang/multibench/affect/processed/humor_data.pkl')
 
 # humor 371 81 300
 encoders = GRU(752, 1128, dropout=True, has_padding=True).cuda()
@@ -23,9 +23,9 @@
 fusion = Concat().cuda()
 
 # Support simple late_fusion and late_fusion with removing bias
-train(encoders, fusion, head, traindata, validdata, 1000, True, True, \
+train(encoders, fusion, head, traindata, validdata, 1000, is_packed=True, early_stop=True, \
  task="classification", optimtype=torch.optim.AdamW, lr=1e-5, save='humor_lf_best.pt', \
- weight_decay=0.01, criterion=torch.nn.MSELoss(), regularization=False)
+ weight_decay=0.01, objective=torch.nn.MSELoss())
 
 print("Testing:")
 model=torch.load('humor_lf_best.pt').cuda()

diff --git a/examples/affect/affect_early_fusion.py b/examples/affect/affect_early_fusion.py
@@ -3,21 +3,21 @@
 sys.path.append(os.getcwd())
 sys.path.append(os.path.dirname(os.path.dirname(os.getcwd())))
 import torch
-
+os.environ['CUDA_VISIBLE_DEVICES']='1'
 from fusions.common_fusions import ConcatEarly
 from datasets.affect.get_data import get_dataloader
-from unimodals.common_models import GRU, MLP
+from unimodals.common_models import GRU, MLP,Sequential,Identity
 
 from training_structures.Supervised_Learning import train, test
 
 from private_test_scripts.all_in_one import all_in_one_train
 
 # mosi_raw.pkl, mosei_raw.pkl, sarcasm.pkl, humor.pkl
-traindata, validdata, testdata = get_dataloader('/home/pliang/multibench/affect/processed/mosi_raw.pkl')
+traindata, validdata, testdata = get_dataloader('/home/paul/MultiBench/mosi_raw.pkl')
 
 # mosi/mosei
-encoders = [GRU(409, 512, dropout=True, has_padding=True).cuda()]
-head = MLP(512, 256, 1).cuda()
+encoders = [Identity().cuda(),Identity().cuda(),Identity().cuda()]
+head = Sequential(GRU(409,512,dropout=True,has_padding=True),MLP(512, 256, 1)).cuda()
 
 # humor/sarcasm
 # encoders = GRU(752, 1128, dropout=True, has_padding=True).cuda()
@@ -29,11 +29,11 @@
 
 
 def trainprocess():
- train(encoders, fusion, head, traindata, validdata, 1000, task="regression", optimtype=torch.optim.AdamW,
+ train(encoders, fusion, head, traindata, validdata, 1000, task="regression", optimtype=torch.optim.AdamW,is_packed=True,
  lr=1e-4, save='mosi_ef_best.pt', weight_decay=0.01, objective=torch.nn.L1Loss())
 
 all_in_one_train(trainprocess, all_modules)
 
 print("Testing:")
 model = torch.load('mosi_ef_best.pt').cuda()
-test(model, testdata, True, torch.nn.L1Loss(), "regression")
+test(model, testdata,'affect', is_packed=True, criterion=torch.nn.L1Loss(), task="posneg-classification")
diff --git a/examples/affect/affect_late_fusion.py b/examples/affect/affect_late_fusion.py
@@ -15,7 +15,7 @@
 
 # mosi_raw.pkl, mosei_raw.pkl, sarcasm.pkl, humor.pkl
 traindata, validdata, test_robust = \
- get_dataloader('/home/pliang/leslie/test/MultiBench/datasets/affect/mosi_raw.pkl')
+ get_dataloader('/home/paul/MultiBench/mosi_raw.pkl')
 
 # mosi/mosei
 encoders=[GRU(35,70,dropout=True,has_padding=True).cuda(), \
@@ -34,7 +34,7 @@
 fusion = Concat().cuda()
 
 def trainprocess():
- train(encoders, fusion, head, traindata, validdata, 1000, task="regression", optimtype=torch.optim.AdamW, is_packed=True,
+ train(encoders, fusion, head, traindata, validdata, 200, task="regression", optimtype=torch.optim.AdamW, is_packed=True,
  early_stop=True, lr=1e-4, save='mosi_lf_best.pt', weight_decay=0.01, objective=torch.nn.L1Loss())
 
 
@@ -43,7 +43,7 @@ def trainprocess():
 print("Testing:")
 model = torch.load('mosi_lf_best.pt').cuda()
 
-test(model=model, test_dataloaders_all=test_robust, dataset='mosi', is_packed=True, criterion=torch.nn.L1Loss(), task='regression')
+test(model=model, test_dataloaders_all=test_robust, dataset='mosi', is_packed=True, criterion=torch.nn.L1Loss(), task='posneg-classification')
 
 
 
diff --git a/examples/affect/affect_mult.py b/examples/affect/affect_mult.py
diff --git a/examples/affect/humor_late_fusion.py b/examples/affect/humor_late_fusion.py
diff --git a/fusions/__pycache__/__init__.cpython-38.pyc b/fusions/__pycache__/__init__.cpython-38.pyc
diff --git a/fusions/__pycache__/common_fusions.cpython-38.pyc b/fusions/__pycache__/common_fusions.cpython-38.pyc
diff --git a/objective_functions/objectives_for_supervised_learning.py b/objective_functions/objectives_for_supervised_learning.py
@@ -9,6 +9,8 @@ def criterioning(pred,truth,criterion):
  return criterion(pred,truth.long().cuda())
  elif type(criterion)==torch.nn.modules.loss.BCEWithLogitsLoss or type(criterion)==torch.nn.MSELoss:
  return criterion(pred,truth.float().cuda())
+ elif type(criterion)==torch.nn.L1Loss:
+ return criterion(pred,truth.float().cuda())
 
 # objective for MFM
 # ce_weight: weight of simple supervised loss