modularized the code

sMamooler · Jan 2, 2022 · ad105cf · ad105cf
1 parent 0a833c1
commit ad105cf
Show file tree

Hide file tree

Showing 282 changed files with 5,184 additions and 3,259 deletions.
diff --git a/Ringier_Project/Images/DuasImagens.jpg → Images/DuasImagens.jpg b/Ringier_Project/Images/DuasImagens.jpg → Images/DuasImagens.jpg
diff --git a/Ringier_Project/Images/car.jpeg → Images/car.jpeg b/Ringier_Project/Images/car.jpeg → Images/car.jpeg
diff --git a/Ringier_Project/Images/car.jpg → Images/car.jpg b/Ringier_Project/Images/car.jpg → Images/car.jpg
diff --git a/Ringier_Project/Images/car.png → Images/car.png b/Ringier_Project/Images/car.png → Images/car.png
diff --git a/Ringier_Project/Images/cow&sheep.jpg → Images/cow&sheep.jpg b/Ringier_Project/Images/cow&sheep.jpg → Images/cow&sheep.jpg
diff --git a/Ringier_Project/Images/cow.jpg → Images/cow.jpg b/Ringier_Project/Images/cow.jpg → Images/cow.jpg
diff --git a/Ringier_Project/Images/cow_comic.jpg → Images/cow_comic.jpg b/Ringier_Project/Images/cow_comic.jpg → Images/cow_comic.jpg
diff --git a/Ringier_Project/Images/dog.jpg → Images/dog.jpg b/Ringier_Project/Images/dog.jpg → Images/dog.jpg
diff --git a/Ringier_Project/Images/dog_sketch.jpg → Images/dog_sketch.jpg b/Ringier_Project/Images/dog_sketch.jpg → Images/dog_sketch.jpg
diff --git a/Ringier_Project/Images/duck-rabbit.jpeg → Images/duck-rabbit.jpeg b/Ringier_Project/Images/duck-rabbit.jpeg → Images/duck-rabbit.jpeg
diff --git a/Ringier_Project/Images/duck-rabbit.jpg → Images/duck-rabbit.jpg b/Ringier_Project/Images/duck-rabbit.jpg → Images/duck-rabbit.jpg
diff --git a/Ringier_Project/Images/duck.jpg → Images/duck.jpg b/Ringier_Project/Images/duck.jpg → Images/duck.jpg
diff --git a/Ringier_Project/Images/face.jpeg → Images/face.jpeg b/Ringier_Project/Images/face.jpeg → Images/face.jpeg
diff --git a/Ringier_Project/Images/face.jpg → Images/face.jpg b/Ringier_Project/Images/face.jpg → Images/face.jpg
diff --git a/Ringier_Project/Images/face.png → Images/face.png b/Ringier_Project/Images/face.png → Images/face.png
diff --git a/Ringier_Project/Images/fall_painting.jpg → Images/fall_painting.jpg b/Ringier_Project/Images/fall_painting.jpg → Images/fall_painting.jpg
diff --git a/Ringier_Project/Images/happy_colors.jpg → Images/happy_colors.jpg b/Ringier_Project/Images/happy_colors.jpg → Images/happy_colors.jpg
diff --git a/Ringier_Project/Images/happy_woman.jpg → Images/happy_woman.jpg b/Ringier_Project/Images/happy_woman.jpg → Images/happy_woman.jpg
diff --git a/Ringier_Project/Images/happy_woman_b&w.jpg → Images/happy_woman_b&w.jpg b/Ringier_Project/Images/happy_woman_b&w.jpg → Images/happy_woman_b&w.jpg
diff --git a/Ringier_Project/Images/horse&dog.jpg → Images/horse&dog.jpg b/Ringier_Project/Images/horse&dog.jpg → Images/horse&dog.jpg
diff --git a/Ringier_Project/Images/horse&dog2.jpg → Images/horse&dog2.jpg b/Ringier_Project/Images/horse&dog2.jpg → Images/horse&dog2.jpg
diff --git a/Ringier_Project/Images/horse&dog3.jpg → Images/horse&dog3.jpg b/Ringier_Project/Images/horse&dog3.jpg → Images/horse&dog3.jpg
diff --git a/Ringier_Project/Images/horse&sheep.jpg → Images/horse&sheep.jpg b/Ringier_Project/Images/horse&sheep.jpg → Images/horse&sheep.jpg
diff --git a/Ringier_Project/Images/horse.jpg → Images/horse.jpg b/Ringier_Project/Images/horse.jpg → Images/horse.jpg
diff --git a/Ringier_Project/Images/horse_comic.jpg → Images/horse_comic.jpg b/Ringier_Project/Images/horse_comic.jpg → Images/horse_comic.jpg
diff --git a/Ringier_Project/Images/horse_sketch.jpg → Images/horse_sketch.jpg b/Ringier_Project/Images/horse_sketch.jpg → Images/horse_sketch.jpg
diff --git a/Ringier_Project/Images/rabbit.jpeg → Images/rabbit.jpeg b/Ringier_Project/Images/rabbit.jpeg → Images/rabbit.jpeg
diff --git a/Ringier_Project/Images/rabbit.jpg → Images/rabbit.jpg b/Ringier_Project/Images/rabbit.jpg → Images/rabbit.jpg
diff --git a/Ringier_Project/Images/rainy_night.jpg → Images/rainy_night.jpg b/Ringier_Project/Images/rainy_night.jpg → Images/rainy_night.jpg
diff --git a/Ringier_Project/Images/sad_painting.jpg → Images/sad_painting.jpg b/Ringier_Project/Images/sad_painting.jpg → Images/sad_painting.jpg
diff --git a/Ringier_Project/Images/sad_woman.jpg → Images/sad_woman.jpg b/Ringier_Project/Images/sad_woman.jpg → Images/sad_woman.jpg
diff --git a/...ier_Project/Images/sad_woman_colorful.jpg → Images/sad_woman_colorful.jpg b/...ier_Project/Images/sad_woman_colorful.jpg → Images/sad_woman_colorful.jpg
diff --git a/Ringier_Project/Images/scary_painting.jpg → Images/scary_painting.jpg b/Ringier_Project/Images/scary_painting.jpg → Images/scary_painting.jpg
diff --git a/...roject/Images/scary_painting_colorful.jpg → Images/scary_painting_colorful.jpg b/...roject/Images/scary_painting_colorful.jpg → Images/scary_painting_colorful.jpg
diff --git a/Ringier_Project/Images/sheep.jpg → Images/sheep.jpg b/Ringier_Project/Images/sheep.jpg → Images/sheep.jpg
diff --git a/Ringier_Project/Images/sheep_comic.jpg → Images/sheep_comic.jpg b/Ringier_Project/Images/sheep_comic.jpg → Images/sheep_comic.jpg
diff --git a/Ringier_Project/Images/two_faces.jpg → Images/two_faces.jpg b/Ringier_Project/Images/two_faces.jpg → Images/two_faces.jpg
diff --git a/Ringier_Project/Images/vase.jpg → Images/vase.jpg b/Ringier_Project/Images/vase.jpg → Images/vase.jpg
diff --git a/Ringier_Project/Images/vase1.jpg → Images/vase1.jpg b/Ringier_Project/Images/vase1.jpg → Images/vase1.jpg
diff --git a/Ringier_Project/Images/vase_or_face.png → Images/vase_or_face.png b/Ringier_Project/Images/vase_or_face.png → Images/vase_or_face.png
diff --git a/Ringier_Project/CLIP-VIL/README.md b/Ringier_Project/CLIP-VIL/README.md
diff --git a/Ringier_Project/CLIP-VIL/emotion_clip.ipynb b/Ringier_Project/CLIP-VIL/emotion_clip.ipynb
diff --git a/Ringier_Project/CLIP-VIL/gradcam_clip.ipynb b/Ringier_Project/CLIP-VIL/gradcam_clip.ipynb
diff --git a/Ringier_Project/CLIP-VIL/image_text_grad_vis.pdf b/Ringier_Project/CLIP-VIL/image_text_grad_vis.pdf
diff --git a/Ringier_Project/CLIP-VIL/mixed_saliency.pdf b/Ringier_Project/CLIP-VIL/mixed_saliency.pdf
diff --git a/Ringier_Project/CLIP-VIL/per-word_gradcam_clip.ipynb b/Ringier_Project/CLIP-VIL/per-word_gradcam_clip.ipynb
diff --git a/Ringier_Project/CLIP-VIL/pos&neg_saliency.pdf b/Ringier_Project/CLIP-VIL/pos&neg_saliency.pdf
diff --git a/Ringier_Project/Images/.ipynb_checkpoints/duck-rabbit-checkpoint.jpg b/Ringier_Project/Images/.ipynb_checkpoints/duck-rabbit-checkpoint.jpg
diff --git a/Ringier_Project/Images/.ipynb_checkpoints/fall_painting-checkpoint.jpg b/Ringier_Project/Images/.ipynb_checkpoints/fall_painting-checkpoint.jpg
diff --git a/Ringier_Project/Images/.ipynb_checkpoints/vase-checkpoint.jpg b/Ringier_Project/Images/.ipynb_checkpoints/vase-checkpoint.jpg
diff --git a/code/README.md b/code/README.md
@@ -0,0 +1,6 @@
+# Ringier_Project
+
+
+# TODO:
+[-] complete requirement.txt
+
diff --git a/Ringier_Project/CLIP-VIL/auxilary.py → code/auxilary.py b/Ringier_Project/CLIP-VIL/auxilary.py → code/auxilary.py
diff --git a/...ect/CLIP-VIL/bpe_simple_vocab_16e6.txt.gz → code/bpe_simple_vocab_16e6.txt.gz b/...ect/CLIP-VIL/bpe_simple_vocab_16e6.txt.gz → code/bpe_simple_vocab_16e6.txt.gz
diff --git a/Ringier_Project/CLIP-VIL/clip_.py → code/clip_.py b/Ringier_Project/CLIP-VIL/clip_.py → code/clip_.py
@@ -1,3 +1,8 @@
+"""
+taken from https://github.com/hila-chefer/Transformer-MM-Explainability
+added similarity_score 
+"""
+
 import hashlib
 import os
 import urllib
@@ -191,3 +196,15 @@ def tokenize(texts: Union[str, List[str]], context_length: int = 77) -> torch.Lo
         result[i, :len(tokens)] = torch.tensor(tokens)
 
     return result
+
+
+def similarity_score(clip_model, image, target_features):
+
+    image_features = clip_model.encode_image(image)
+
+    image_features_norm = image_features.norm(dim=-1, keepdim=True)
+    image_features_new = image_features / image_features_norm
+    target_features_norm = target_features.norm(dim=-1, keepdim=True)
+    target_features_new = target_features / target_features_norm
+
+    return image_features_new[0].dot(target_features_new[0])*100
diff --git a/code/image_utils.py b/code/image_utils.py
@@ -0,0 +1,24 @@
+import numpy as np
+import cv2
+
+def show_cam_on_image(img, mask, neg_saliency=False):
+    if neg_saliency:
+        heatmap = cv2.applyColorMap(np.uint8(255 * mask), cv2.COLORMAP_RAINBOW)
+    else:
+        heatmap = cv2.applyColorMap(np.uint8(255 * mask), cv2.COLORMAP_JET)
+
+    heatmap = np.float32(heatmap) / 255
+    cam = heatmap + np.float32(img)
+    cam = cam / np.max(cam)
+    return cam
+
+def show_overlapped_cam(img, neg_mask, pos_mask):
+    neg_heatmap = cv2.applyColorMap(np.uint8(255 * neg_mask), cv2.COLORMAP_RAINBOW)
+    pos_heatmap = cv2.applyColorMap(np.uint8(255 * pos_mask), cv2.COLORMAP_JET)
+    neg_heatmap = np.float32(neg_heatmap) / 255
+    pos_heatmap = np.float32(pos_heatmap) / 255
+    # try different options: sum, average, ...
+    heatmap = neg_heatmap + pos_heatmap
+    cam = heatmap + np.float32(img)
+    cam = cam / np.max(cam)
+    return cam
diff --git a/Ringier_Project/CLIP-VIL/model.py → code/model.py b/Ringier_Project/CLIP-VIL/model.py → code/model.py
@@ -1,3 +1,7 @@
+"""
+taken from https://github.com/hila-chefer/Transformer-MM-Explainability
+"""
+
 from collections import OrderedDict
 from typing import Tuple, Union
 

diff --git a/code/notebooks/ARTEMIS_EMOTIONS.json b/code/notebooks/ARTEMIS_EMOTIONS.json
diff --git a/code/notebooks/affectnet_emotions.ipynb b/code/notebooks/affectnet_emotions.ipynb
diff --git a/code/notebooks/artemis_emotions.ipynb b/code/notebooks/artemis_emotions.ipynb
diff --git a/...oject/CLIP-VIL/cross_image_saliency.ipynb → code/notebooks/cross_image_saliency.ipynb b/...oject/CLIP-VIL/cross_image_saliency.ipynb → code/notebooks/cross_image_saliency.ipynb
diff --git a/code/notebooks/global_vis.ipynb b/code/notebooks/global_vis.ipynb
diff --git a/code/notebooks/perword_vis.ipynb b/code/notebooks/perword_vis.ipynb