Source code for bob.rppg.cvpr14.extract_utils

#!/usr/bin/env python
# encoding: utf-8

import os, sys
import numpy

import bob.ip.draw
import bob.ip.color


[docs]def kp66_to_mask(image, keypoints, indent=10, plot=False): """builds a mask on the lower part of the face The mask is built using selected keypoints retrieved by a Discriminative Response Map Fitting (DRMF) algorithm. Note that the DRMF is not implemented here, and that the keypoints are loaded from file (and are not provided in the package). Note also that this function is explicitly made for the keypoints set generated by the Matlab software downloaded from http://ibug.doc.ic.ac.uk/resources/drmf-matlab-code-cvpr-2013/ Update: this function also works when using :py:class:`bob.ip.dlib.DlibLandmarkExtraction` Parameters ---------- image: numpy.ndarray The current frame. keypoints: numpy.ndarray the set of 66 keypoints retrieved by DRMF. indent: int The percentage of the facewidth [in pixels] by which selected keypoints are shifted inside the face to build the mask. THe facewidth is defined by the distance between the two keypoints located on the right and left edge of the face, at the eyes' height. plot: bool If set to True, plots the current face with the selected keypoints and the built mask. Returns ------- mask: numpy.ndarray A boolean array of the size of the original image, where the region corresponding to the mask is True. mask_points: :obj:`list` of :obj:`tuple` The points corresponding to vertices of the mask. """ assert keypoints.shape[0] == 66 or keypoints.shape[0] == 68, "You should provide a set of 66 (DRMF) or 68 (dlib ) keypoints" if plot: imkey = numpy.copy(image) # there are 9 points to define the mask mask_points = [] for k in range(keypoints.shape[0]): if k == 1 or k == 3 or k == 5 or k == 8 or k == 11 or k == 13 or k == 15 or k == 41 or k == 47: mask_points.append([int(keypoints[k, 0]), int(keypoints[k, 1])]) if plot: bob.ip.draw.cross(imkey, (int(keypoints[k, 0]), int(keypoints[k, 1])), 4, (255,0,0)) # indent in pixel will be the percentage provided of the face width # face width is defined as the distance between point 0 and point 6 face_width = numpy.sqrt(numpy.sum((numpy.array(mask_points[0]) - numpy.array(mask_points[6]))**2)) indent = int(float(indent)/100 * face_width) # left contour mask_points[0][1] += indent mask_points[1][1] += indent mask_points[2][1] += indent # chin tip mask_points[3][0] -= indent # right contour mask_points[4][1] -= indent mask_points[5][1] -= indent mask_points[6][1] -= indent # below left eye mask_points[7][0] += indent # below right eye mask_points[8][0] += indent # swap left and right eye such that vertices are following each other swp = mask_points[7] mask_points[7] = mask_points[8] mask_points[8] = swp if plot: for k in range(len(mask_points)-1): bob.ip.draw.line(imkey, (mask_points[k][0], mask_points[k][1]), (mask_points[k+1][0], mask_points[k+1][1]), (0,255,0)) bob.ip.draw.line(imkey, (mask_points[0][0], mask_points[0][1]), (mask_points[8][0], mask_points[8][1]), (0,255,0)) from matplotlib import pyplot pyplot.imshow(numpy.rollaxis(numpy.rollaxis(imkey, 2),2)) pyplot.title('Built mask') pyplot.show() mask = get_mask(image, mask_points) return mask_points, mask
[docs]def get_mask(image, mask_points): """returns a boolean array where the mask is True. It turns mask points into a region of interest and returns the corresponding boolean array, of the same size as the image. Taken from https://github.com/jdoepfert/roipoly.py/blob/master/roipoly.py Parameters ---------- image: numpy.ndarray The current frame. mask_points: :obj:`list` of :obj:`tuple` The points corresponding to vertices of the mask. Returns ------- mask: numpy.ndarray A boolean array of the size of the original image, where the region corresponding to the mask is True. """ import matplotlib.path as mplPath ny = image.shape[1] nx = image.shape[2] poly_verts = [(mask_points[0][1], mask_points[0][0])] for i in range(len(mask_points)-1, -1, -1): poly_verts.append((mask_points[i][1], mask_points[i][0])) x, y = numpy.meshgrid(numpy.arange(nx), numpy.arange(ny)) x, y = x.flatten(), y.flatten() points = numpy.vstack((x,y)).T ROIpath = mplPath.Path(poly_verts, closed=True) grid = ROIpath.contains_points(points).reshape((ny,nx)) grid = grid.astype('bool') return grid
[docs]def get_good_features_to_track(face, npoints, quality=0.01, min_distance=10, plot=False): """applies the openCV function "good features to track" Parameters ---------- face: numpy.ndarray The cropped face image npoints: int The maximum number of strong corners you want to detect quality: :obj:`float` The minimum relative quality of the detected corners. Note that increasing this value decreases the number of detected corners. Defaluts to 0.01. min_distance: int minimum euclidean distance between detected corners. plot: bool if we should plot the currently selected features to track. Returns ------- corners: numpy.ndarray the detected strong corners. """ from cv2 import goodFeaturesToTrack gray = bob.ip.color.rgb_to_gray(face) corners = goodFeaturesToTrack(gray, npoints, quality, min_distance) if plot: display = numpy.copy(face) int_corners = numpy.int0(corners) for i in int_corners: x,y = i.ravel() bob.ip.draw.cross(display, (y, x), 3, (255,0,0)) from matplotlib import pyplot pyplot.imshow(numpy.rollaxis(numpy.rollaxis(display, 2),2)) pyplot.title('Good features to track') pyplot.show() return corners
[docs]def track_features(previous, current, previous_points, plot=False): """projects the features from the previous frame in the current frame. Parameters ---------- previous: numpy.ndarray the previous frame. current: numpy.ndarray the current frame. previous_points: numpy.ndarray the set of keypoints to track (in the previous frame). plot: bool Plots the keypoints projected on the current frame. Returns ------- current_points: numpy.ndarray the set of keypoints in the current frame. """ prev_gray = bob.ip.color.rgb_to_gray(previous) curr_gray = bob.ip.color.rgb_to_gray(current) from cv2 import calcOpticalFlowPyrLK current_points = calcOpticalFlowPyrLK(prev_gray, curr_gray, prevPts=previous_points, nextPts=None) if plot: display = numpy.copy(current) int_corners = numpy.int0(current_points[0]) for i in int_corners: x,y = i.ravel() bob.ip.draw.cross(display, (y, x), 3, (255,0,0)) from matplotlib import pyplot pyplot.imshow(numpy.rollaxis(numpy.rollaxis(display, 2),2)) pyplot.title('Result of the tracked features') pyplot.show() return current_points[0]
[docs]def find_transformation(previous_points, current_points): """finds the transformation matrix from previous points to current points. The transformation matrix is found using estimateRigidTransform (fancier alternatives have been tried, but are not that stable). Parameters ---------- previous_points: numpy.ndarray Set of 'starting' 2d points current_points: numpy.ndarray Set of 'destination' 2d points Returns ------- transformation_matrix: numpy.ndarray the affine transformation matrix between the two sets of points. """ from cv2 import estimateRigidTransform return estimateRigidTransform(previous_points, current_points, False)
[docs]def get_current_mask_points(previous_mask_points, transfo_matrix): """projects the previous mask points to get the current mask. Parameters ---------- previous_mask_points: numpy.ndarray The points forming the mask in the previous frame transformation_matrix: numpy.ndarray the affine transformation matrix between the two sets of points. Returns ------- current_mask_points: numpy.ndarray The points forming the mask in the current frame """ previous_mask_points = numpy.array([previous_mask_points], dtype='float64') from cv2 import transform new_mask_points = transform(previous_mask_points, transfo_matrix) return new_mask_points[0].tolist()
[docs]def compute_average_colors_mask(image, mask, plot=False): """computes the average green color within a given mask. Parameters ---------- image: numpy.ndarray The image containing the face. mask: numpy.ndarray A boolean array of the size of the original image, where the region corresponding to the mask is True. plot: bool Plot the mask as an overlay on the original image. Returns ------- color: numpy.ndarray The average RGB colors inside the mask ROI. """ if plot: from matplotlib import pyplot display = numpy.copy(image) display[:, mask] = 255 pyplot.imshow(numpy.rollaxis(numpy.rollaxis(display, 2),2)) pyplot.title('Mask overlaid on the original frame') pyplot.show() [red, green, blue] = image[:, mask] return numpy.array([numpy.mean(red), numpy.mean(green), numpy.mean(blue)])
[docs]def compute_average_colors_wholeface(image, plot=False): """computes the average green color within the provided face image Parameters ---------- image: numpy.ndarray The cropped face image plot: bool Plot the mask as an overlay on the original image. Returns ------- color: :obj:`float` The average green color inside the face """ if plot: from matplotlib import pyplot display = numpy.copy(image) pyplot.imshow(numpy.rollaxis(numpy.rollaxis(display, 2),2)) pyplot.title('Face area used to compute the mean green value') pyplot.show() green = image[1, :] return numpy.mean(green)