Merge branch 'master' of github.com:pieterwolfert/engagement-l2tor

.gitignore

@@ -2,3 +2,4 @@ data/
 *.pyc
 *.ipynb_checkpoints/
 architecture.png
+models/

script/emotionnetwork.py

@@ -2,10 +2,12 @@
 import csv
 from keras.utils import plot_model
 from preprocessing import Preprocessing
-from keras.layers import Activation, Input, Conv2D, Dense, Dropout, MaxPooling2D, SeparableConv2D, GlobalAveragePooling2D
+from keras.layers import Activation, Input, Conv2D, Dense, Dropout, \
+ MaxPooling2D, SeparableConv2D, GlobalAveragePooling2D
 from keras import layers
 from keras.models import Model
-from keras.optimizers import Adam
+from keras.callbacks import CSVLogger, ReduceLROnPlateau, ModelCheckpoint
+from keras.optimizers import Adam, SGD
 from keras.layers.normalization import BatchNormalization
 import matplotlib.pyplot as plt
 
@@ -75,17 +77,24 @@ def getmodel():
  x = GlobalAveragePooling2D()(x)
  output = Dense(8, activation='softmax', name='predictions')(x)
  model = Model(inputs=img_input, outputs=output)
+ sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
+ model.compile(loss='binary_crossentropy', optimizer='sgd', metrics=['accuracy'])
  return model
 
+def train(x_train, y_train, x_test, y_test):
+ model = getmodel()
+ lr_plateau = ReduceLROnPlateau(monitor='val_loss', patience=3, verbose=1, factor=0.5)
+ checkpoint = ModelCheckpoint(filepath='./models/'+"emotions+'.hdf5',
+ verbose=1, save_best_only=True)
+ #plot_model(model, to_file="architecture.png")
+ model.fit(x_train, y_train, epochs=100, batch_size=64,\
+ callbacks=[lr_plateau, checkpoint],\
+ validation_data=(x_test, y_test))
+
 def main():
  prep = Preprocessing(data_dir)
- x_train, y_train, x_test, y_test = prep.loadData("train.txt", "test.txt")
- print(np.shape(x_train))
- model = getmodel()
- plot_model(model, to_file="architecture.png")
- #model.fit(x_train, y_train, epochs=100, batch_size=64)
+ train(prep.loadData("train.txt", "test.txt"))
  #score = model.evaluate(x_test, y_test, batch_size=32)
 
-
 if __name__=="__main__":
  main()

script/gaze_predict.py

@@ -1,4 +1,5 @@
 import numpy as np
+import time
 from skimage.io import imread
 import matplotlib.pyplot as plt
 import scipy.io as sio
@@ -186,6 +187,7 @@ def getGaze(e, image):
  return [x,y]
 
 if __name__=="__main__":
+ start = time.time()
  #this main method is for testing purposes
  #predictions = getGaze([0.60, 0.2679], 'script/test.jpg')
  predictions = getGaze([0.54, 0.28], 'script/5.jpg')

script/predict_gaze.py

@@ -1,4 +1,6 @@
 import numpy as np
+from typing import List, NewType, Union, Tuple
+import time
 from skimage.io import imread
 import matplotlib.pyplot as plt
 import scipy.io as sio
@@ -9,21 +11,22 @@
 import caffe
 
 class Gaze():
- def __init__(self, model_def, model_weights):
+ def __init__(self, model_def: str, model_weights: str):
  self.network = caffe.Net(model_def, model_weights, caffe.TEST)
 
- def getImage(self):
+ def getImage(self) -> np.array:
  """Helper function to return image."""
  return self.image
 
- def getEyeImage(self):
+ def getEyeImage(self) -> np.array:
  return self.eye_image_resize.transpose(0,2,3,1)[0]
 
- def getHeatmap(self):
+ def getHeatmap(self) -> np.array:
  """Get heatmap to see where the gaze is predicted."""
  return self.heatmap
 
- def getGaze(self, image, headloc, alpha=0.3):
+ def getGaze(self, image: np.array, headloc:List[float], alpha=0.3)\
+ -> List[float]:
  """Returns x,y coordinates of the gaze location
 
  Keyword Arguments:
@@ -42,7 +45,7 @@ def getGaze(self, image, headloc, alpha=0.3):
  self.y = int(self.predictions[1] * np.shape(self.image)[1])
  return [self.x, self.y]
 
- def predictGaze(self):
+ def predictGaze(self) -> List[float]:
  """Loads data in network and does a forward pass."""
  self.network.blobs['data'].data[...] = self.img_resize
  self.network.blobs['face'].data[...] = self.eye_image_resize
@@ -60,7 +63,7 @@ def visualizeGaze(self):
  ax.add_patch(Circle((self.head_x, self.head_y), 10, color = 'r'))
  plt.show()
 
- def prepImages(self, img, e, alpha):
+ def prepImages(self, img: np.array, e: List[float], alpha) -> np.array:
  """
  Output images of prepImages are exactly the same as the matlab ones
 
@@ -102,15 +105,16 @@ def prepImages(self, img, e, alpha):
  eye_image = imresize(im_face, input_shape, interp='bicubic')
  eye_image = eye_image.astype('float32')
  eye_image_resize = eye_image[:,:,[2,1,0]] - imagenet_mean
- eye_image_resize = np.rot90(np.fliplr(eye_image_resize)).astype('float32')
+ eye_image_resize = np.rot90(np.fliplr(eye_image_resize))\
+ .astype('float32')
  #get everything in the right input format for the network
  img_resize, eye_image_resize = self.fit_shape_of_inputs(img_resize,\
  eye_image_resize)
  z = self.eyeGrid(img, [x1, x2, y1, y2])
  z = z.astype('float32')
  return img_resize, eye_image_resize, z
 
- def eyeGrid(self, img, headlocs):
+ def eyeGrid(self, img: np.array, headlocs: List[float]) -> np.array:
  """Calculates the relative location of the eye.
 
  Keyword Arguments:
@@ -133,8 +137,10 @@ def eyeGrid(self, img, headlocs):
  eyes_grid_flat = eyes_grid_flat.reshape(1, len(eyes_grid_flat), 1, 1)
  return eyes_grid_flat
 
- def postProcessing(self, f_val):
- """Combines the 5 outputs into one heatmap and calculates the gaze location
+ def postProcessing(self, f_val: List[float])\
+ -> Union[np.array, List[float]]:
+ """Combines the 5 outputs into one heatmap
+ and calculates the gaze location.
 
  Keyword arguments:
  f_val -- output of the Caffe model
@@ -176,16 +182,18 @@ def postProcessing(self, f_val):
  x_predict = cols/227
  return final_map, [x_predict, y_predict]
 
- def alpha_exponentiate(self, x, alpha=0.3):
+ def alpha_exponentiate(self, x, alpha=0.3) -> float:
  return np.exp(alpha * x) / np.sum(np.exp(alpha*x.flatten()))
 
- def ind2sub2(self, array_shape, ind):
+ def ind2sub2(self, array_shape: Tuple[float, float], ind: float)\
+ -> List[float]:
  """Python implementation of the equivalent matlab method"""
  rows = (ind / array_shape[1])
- cols = (ind % array_shape[1]) # or numpy.mod(ind.astype('int'), array_shape[1])
+ cols = (ind % array_shape[1])
  return [rows, cols]
 
- def shifted_mapping(self, x, delta_x, is_topleft_corner):
+ def shifted_mapping(self, x: int, delta_x: int, is_topleft_corner: bool)\
+ -> int:
  if is_topleft_corner:
  if x == 0:
  return 0
@@ -197,33 +205,38 @@ def shifted_mapping(self, x, delta_x, is_topleft_corner):
  ix = 3 * (x + 1) - 1 - delta_x
  return min(14, ix)
 
- def fit_shape_of_inputs(self, img_resize, eye_image_resize):
+ def fit_shape_of_inputs(self, img_resize: np.array,\
+ eye_image_resize: np.array):
  """Fits the input for the forward pass."""
  input_image_resize = img_resize.reshape([img_resize.shape[0], \
  img_resize.shape[1], \
  img_resize.shape[2], 1])
  input_image_resize = input_image_resize.transpose(3, 2, 0, 1)
 
  eye_image_resize = eye_image_resize.reshape([eye_image_resize.shape[0], \
-  eye_image_resize.shape[1], \
-  eye_image_resize.shape[2], 1])
+ eye_image_resize.shape[1], \
+ eye_image_resize.shape[2], 1])
  eye_image_resize = eye_image_resize.transpose(3, 2, 0, 1)
  return input_image_resize, eye_image_resize
 
 if __name__ == '__main__':
- model_def = '/home/pieter/projects/engagement-l2tor/data/model/deploy_demo.prototxt'
- model_weights = '/home/pieter/projects/engagement-l2tor/data/model/binary_w.caffemodel'
+ model_def =\
+ '/home/pieter/projects/engagement-l2tor/data/model/deploy_demo.prototxt'
+ model_weights =\
+ '/home/pieter/projects/engagement-l2tor/data/model/binary_w.caffemodel'
  gazemachine = Gaze(model_def, model_weights)
+ start = time.time()
  image = imread('script/images/5.jpg')
  e = [0.54, 0.28]
- #e = [y, x]
+ #e = [x,y]
  predictions = gazemachine.getGaze(image, e)
- print(predictions)
- plt.imshow(gazemachine.getHeatmap())
- plt.show()
- gazemachine.visualizeGaze()
- plt.imshow(gazemachine.getEyeImage())
- plt.show()
+ end = time.time()
+ #print(predictions)
+ #plt.imshow(gazemachine.getHeatmap())
+ #plt.show()
+ #gazemachine.visualizeGaze()
+ #plt.imshow(gazemachine.getEyeImage())
+ #plt.show()
  #image = imread('script/images/test.jpg')
  #image = np.fliplr(image)
  #e = [0.60, .2679]

script/preprocessing.py

@@ -2,6 +2,8 @@
 import csv
 from scipy.misc import imread
 from scipy.misc import imresize
+from os import listdir
+from os.path import isfile, join
 
 class Preprocessing:
  def __init__(self, datadir):
@@ -26,6 +28,20 @@ def getFileNames(self, filename):
  temp.append(row[0][:-4])
  return temp
 
+ def getFrames(self, filename, folder):
+ """gets filenames of frames"""
+ pth = self.datadir + folder + '/' + filename + '/'
+ filelist = []
+ for f in listdir(pth):
+ if isfile(join(pth, f)):
+ filelist.append(f)
+ return filelist
+
+
+ onlyfiles = [f for f in listdir(pth) if isfile(join(pth, f))]
+ print(onlyfiles)
+ return len(onlyfiles)
+
  def getLabels(self, filename, length):
  labels = np.zeros(shape=(length, 8))
  with open(self.datadir + filename) as f:
@@ -49,3 +65,18 @@ def loadImages(self, filenames, labels, folder, size):
  except FileNotFoundError as f:
  pass
  return np.asarray(x_train), np.asarray(y_train)
+
+def main():
+ data_dir = "/home/pieter/data/emoreact/"
+ prep = Preprocessing(data_dir)
+ file_count = 0
+ frames_count = 0
+ for group in ['train', 'test', 'validation']:
+ file_dict = {}
+ temp = prep.getFileNames(group + ".txt")
+ file_count += len(temp)
+ for i in temp:
+ file_dict[i] = prep.getFrames(i, group)
+
+if __name__=="__main__":
+ main()