Coverage for src/bob/pad/face/utils/load_utils.py: 51%
152 statements
« prev ^ index » next coverage.py v7.6.5, created at 2024-11-14 23:14 +0100
1import random
3from collections import OrderedDict
4from functools import partial
6import numpy
8from imageio import get_reader
9from PIL import Image
11import bob.io.image
13from bob.bio.face.annotator import (
14 bounding_box_from_annotation,
15 min_face_size_validator,
16)
17from bob.bio.face.color import rgb_to_hsv, rgb_to_yuv
18from bob.bio.video.annotator import normalize_annotations
21def block(X, block_size, block_overlap, flat=False):
22 """
23 Parameters
24 ----------
25 X : numpy.ndarray
26 The image to be split into blocks.
27 block_size : tuple
28 The size of the block.
29 block_overlap : tuple
30 The overlap of the block.
32 Returns
33 -------
34 numpy.ndarray
35 The image split into blocks.
36 """
38 assert len(block_size) == 2
39 assert len(block_overlap) == 2
41 size_ov_h = int(block_size[0] - block_overlap[0])
42 size_ov_w = int(block_size[1] - block_overlap[1])
43 n_blocks_h = int((X.shape[0] - block_overlap[0]) / size_ov_h)
44 n_blocks_w = int((X.shape[1] - block_overlap[1]) / size_ov_w)
46 blocks = numpy.zeros(shape=(n_blocks_h, n_blocks_w, size_ov_h, size_ov_w))
47 for h in range(n_blocks_h):
48 for w in range(n_blocks_w):
49 blocks[h, w, :, :] = X[
50 h * size_ov_h : h * size_ov_h + block_size[0],
51 w * size_ov_w : w * size_ov_w + block_size[1],
52 ]
54 if flat:
55 return blocks.reshape(
56 n_blocks_h * n_blocks_w, blocks.shape[2], blocks.shape[3]
57 )
59 return blocks
62def scale(image, scaling_factor):
63 """
64 Scales and image using PIL
66 Parameters
67 ----------
69 image:
70 Input image to be scaled
72 new_shape: tuple
73 Shape of the rescaled image
77 """
79 if isinstance(scaling_factor, float):
80 new_size = tuple(
81 (numpy.array(image.shape) * scaling_factor).astype(numpy.int)
82 )
83 return bob.io.image.to_bob(
84 numpy.array(
85 Image.fromarray(bob.io.image.to_matplotlib(image)).resize(
86 size=new_size
87 ),
88 dtype="float",
89 )
90 )
92 elif isinstance(scaling_factor, tuple):
93 if len(scaling_factor) > 2:
94 scaling_factor = scaling_factor[1:]
96 return bob.io.image.to_bob(
97 numpy.array(
98 Image.fromarray(bob.io.image.to_matplotlib(image)).resize(
99 size=scaling_factor
100 ),
101 dtype="float",
102 )
103 )
104 else:
105 raise ValueError(f"Scaling factor not supported: {scaling_factor}")
108def frames(path):
109 """Yields the frames of a video file.
111 Parameters
112 ----------
113 path : str
114 Path to the video file.
116 Yields
117 ------
118 numpy.ndarray
119 A frame of the video. The size is (3, 240, 320).
120 """
121 video = get_reader(path)
122 for frame in video:
123 yield bob.io.image.to_bob(frame)
126def number_of_frames(path):
127 """returns the number of frames of a video file.
129 Parameters
130 ----------
131 path : str
132 Path to the video file.
134 Returns
135 -------
136 int
137 The number of frames. Then, it yields the frames.
138 """
139 video = get_reader(path)
140 return video.count_frames()
143def bbx_cropper(frame, annotations):
144 for source in ("direct", "eyes", None):
145 try:
146 bbx = bounding_box_from_annotation(source=source, **annotations)
147 break
148 except Exception:
149 if source is None:
150 raise
151 return frame[..., bbx.top : bbx.bottom, bbx.left : bbx.right]
154def min_face_size_normalizer(annotations, max_age=15, **kwargs):
155 return normalize_annotations(
156 annotations, partial(min_face_size_validator, **kwargs), max_age=max_age
157 )
160def yield_faces(pad_sample, cropper, normalizer=None):
161 """Yields face images of a padfile. It uses the annotations from the
162 database. The annotations are further normalized.
164 Parameters
165 ----------
166 pad_sample
167 The pad sample to return the faces.
168 cropper : collections.abc.Callable
169 A face image cropper that works with database's annotations.
170 normalizer : collections.abc.Callable
171 If not None, it should be a function that takes all the annotations of
172 the whole video and yields normalized annotations frame by frame. It
173 should yield same as ``annotations.items()``.
175 Yields
176 ------
177 numpy.ndarray
178 Face images
180 Raises
181 ------
182 ValueError
183 If the database returns None for annotations.
184 """
186 # read annotation
187 annotations = pad_sample.annotations
188 if annotations is None:
189 raise ValueError("No annotations were returned.")
191 if normalizer is not None:
192 annotations = OrderedDict(normalizer(annotations))
194 # normalize annotations and crop faces
195 for frame_id, frame in enumerate(pad_sample.data):
196 annot = annotations.get(str(frame_id), None)
197 if annot is None:
198 continue
199 face = cropper(frame, annotations=annot)
200 if face is not None:
201 yield face
204def scale_face(face, face_height, face_width=None):
205 """Scales a face image to the given size.
207 Parameters
208 ----------
209 face : numpy.ndarray
210 The face image. It can be 2D or 3D in bob image format.
211 face_height : int
212 The height of the scaled face.
213 face_width : :obj:`None`, optional
214 The width of the scaled face. If None, face_height is used.
216 Returns
217 -------
218 numpy.ndarray
219 The scaled face.
220 """
221 face_width = face_height if face_width is None else face_width
222 shape = list(face.shape)
223 shape[-2:] = (face_height, face_width)
224 # scaled_face = numpy.empty(shape, dtype="float64")
225 scaled_face = scale(face, tuple(shape))
226 return scaled_face
229def blocks(data, block_size, block_overlap=(0, 0)):
230 """Extracts patches of an image
232 Parameters
233 ----------
234 data : numpy.ndarray
235 The image in gray-scale, color, or color video format.
236 block_size : (int, int)
237 The size of patches
238 block_overlap : (:obj:`int`, :obj:`int`), optional
239 The size of overlap of patches
241 Returns
242 -------
243 numpy.ndarray
244 The patches.
246 Raises
247 ------
248 ValueError
249 If data dimension is not between 2 and 4 (inclusive).
250 """
252 data = numpy.asarray(data)
253 # if a gray scale image:
254 if data.ndim == 2:
255 output = block(data, block_size, block_overlap, flat=True)
256 # if a color image:
257 elif data.ndim == 3:
258 # out_shape = list(data.shape[0:1]) + list(
259 # block_output_shape(data[0], block_size, block_overlap, flat=True)
260 # )
262 # output = numpy.empty(out_shape, dtype=data.dtype)
263 output = []
264 for i, img2d in enumerate(data):
265 output.append(block(img2d, block_size, block_overlap, flat=True))
266 output = numpy.moveaxis(output, 0, 1)
267 # if a color video:
268 elif data.ndim == 4:
269 output = [blocks(img3d, block_size, block_overlap) for img3d in data]
270 output = numpy.concatenate(output, axis=0)
271 else:
272 raise ValueError("Unknown data dimension {}".format(data.ndim))
273 return output
276def block_generator(input, block_size, block_overlap=(0, 0)):
277 """Performs a block decomposition of a 2D or 3D array/image
279 It works exactly as :any:`bob.ip.base.block` except that it yields the blocks
280 one by one instead of concatenating them. It also works with color images.
282 Parameters
283 ----------
284 input : :any:`numpy.ndarray`
285 A 2D array (Height, Width) or a color image (Bob format: Channels,
286 Height, Width).
287 block_size : (:obj:`int`, :obj:`int`)
288 The size of the blocks in which the image is decomposed.
289 block_overlap : (:obj:`int`, :obj:`int`), optional
290 The overlap of the blocks.
292 Yields
293 ------
294 array_like
295 A block view of the image. Modifying the blocks will change the original
296 image as well. This is different from :any:`bob.ip.base.block`.
298 Raises
299 ------
300 ValueError
301 If the block_overlap is not smaller than block_size.
302 If the block_size is bigger than the image size.
303 """
304 block_h, block_w = block_size
305 overlap_h, overlap_w = block_overlap
306 img_h, img_w = input.shape[-2:]
308 if overlap_h >= block_h or overlap_w >= block_w:
309 raise ValueError(
310 "block_overlap: {} must be smaller than block_size: {}.".format(
311 block_overlap, block_size
312 )
313 )
314 if img_h < block_h or img_w < block_w:
315 raise ValueError(
316 "block_size: {} must be smaller than the image size: {}.".format(
317 block_size, input.shape[-2:]
318 )
319 )
321 # Determine the number of block per row and column
322 size_ov_h = block_h - overlap_h
323 size_ov_w = block_w - overlap_w
325 # Perform the block decomposition
326 for h in range(0, img_h - block_h + 1, size_ov_h):
327 for w in range(0, img_w - block_w + 1, size_ov_w):
328 yield input[..., h : h + block_h, w : w + block_w]
331def blocks_generator(data, block_size, block_overlap=(0, 0)):
332 """Yields patches of an image
334 Parameters
335 ----------
336 data : numpy.ndarray
337 The image in gray-scale, color, or color video format.
338 block_size : (int, int)
339 The size of patches
340 block_overlap : (:obj:`int`, :obj:`int`), optional
341 The size of overlap of patches
343 Yields
344 ------
345 numpy.ndarray
346 The patches.
348 Raises
349 ------
350 ValueError
351 If data dimension is not between 2 and 4 (inclusive).
352 """
353 data = numpy.asarray(data)
354 if 1 < data.ndim < 4:
355 for patch in block_generator(data, block_size, block_overlap):
356 yield patch
357 # if a color video:
358 elif data.ndim == 4:
359 for frame in data:
360 for patch in block_generator(frame, block_size, block_overlap):
361 yield patch
362 else:
363 raise ValueError("Unknown data dimension {}".format(data.ndim))
366def color_augmentation(image, channels=("rgb",)):
367 """Converts an RGB image to different color channels.
369 Parameters
370 ----------
371 image : numpy.ndarray
372 The image in RGB Bob format.
373 channels : :obj:`tuple`, optional
374 List of channels to convert the image to. It can be any of ``rgb``,
375 ``yuv``, ``hsv``.
377 Returns
378 -------
379 numpy.ndarray
380 The image that contains several channels:
381 ``(3*len(channels), height, width)``.
382 """
383 final_image = []
385 if "rgb" in channels:
386 final_image.append(image)
388 if "yuv" in channels:
389 final_image.append(rgb_to_yuv(image))
391 if "hsv" in channels:
392 final_image.append(rgb_to_hsv(image))
394 return numpy.concatenate(final_image, axis=0)
397def random_sample(A, size):
398 """Randomly selects ``size`` samples from the array ``A``"""
399 return A[numpy.random.choice(A.shape[0], size, replace=False), ...]
402def random_patches(image, block_size, n_random_patches=1):
403 """Extracts N random patches of block_size from an image"""
404 h, w = image.shape[-2:]
405 bh, bw = block_size
406 if h < block_size[0] or w < block_size[1]:
407 raise ValueError("block_size must be smaller than image shape")
408 hl = numpy.random.randint(0, h - bh, size=n_random_patches)
409 wl = numpy.random.randint(0, w - bw, size=n_random_patches)
410 for ch, cw in zip(hl, wl):
411 yield image[..., ch : ch + bh, cw : cw + bw]
414def extract_patches(
415 image, block_size, block_overlap=(0, 0), n_random_patches=None
416):
417 """Yields either all patches from an image or N random patches."""
418 if n_random_patches is None:
419 return blocks_generator(image, block_size, block_overlap)
420 else:
421 return random_patches(
422 image, block_size, n_random_patches=n_random_patches
423 )
426def the_giant_video_loader(
427 pad_sample,
428 region="whole",
429 scaling_factor=None,
430 cropper=None,
431 normalizer=None,
432 patches=False,
433 block_size=(96, 96),
434 block_overlap=(0, 0),
435 random_patches_per_frame=None,
436 augment=None,
437 multiple_bonafide_patches=1,
438 keep_pa_samples=None,
439 keep_bf_samples=None,
440):
441 """Loads a video pad file frame by frame and optionally applies
442 transformations.
444 Parameters
445 ----------
446 pad_sample
447 The pad sample
448 region : str
449 Either `whole` or `crop`. If whole, it will return the whole frame.
450 Otherwise, you need to provide a cropper and a normalizer.
451 scaling_factor : float
452 If given, will scale images to this factor.
453 cropper
454 The cropper to use
455 normalizer
456 The normalizer to use
457 patches : bool
458 If true, will extract patches from images.
459 block_size : tuple
460 Size of the patches
461 block_overlap : tuple
462 Size of overlap of the patches
463 random_patches_per_frame : int
464 If not None, will only take this much patches per frame
465 augment
466 If given, frames will be transformed using this function.
467 multiple_bonafide_patches : int
468 Will use more random patches for bonafide samples
469 keep_pa_samples : float
470 If given, will drop some PA samples.
471 keep_bf_samples : float
472 If given, will drop some BF samples.
474 Returns
475 -------
476 object
477 A generator that yields the samples.
479 Raises
480 ------
481 ValueError
482 If region is not whole or crop.
483 """
484 if region == "whole":
485 generator = iter(pad_sample.data)
486 elif region == "crop":
487 generator = yield_faces(
488 pad_sample, cropper=cropper, normalizer=normalizer
489 )
490 else:
491 raise ValueError("Invalid region value: `{}'".format(region))
493 if scaling_factor is not None:
494 generator = (scale(frame, scaling_factor) for frame in generator)
495 if patches:
496 if random_patches_per_frame is None:
497 generator = (
498 patch
499 for frame in generator
500 for patch in blocks_generator(frame, block_size, block_overlap)
501 )
502 else:
503 if pad_sample.is_bonafide:
504 random_patches_per_frame *= multiple_bonafide_patches
505 generator = (
506 patch
507 for frame in generator
508 for patch in random_sample(
509 blocks(frame, block_size, block_overlap),
510 random_patches_per_frame,
511 )
512 )
514 if augment is not None:
515 generator = (augment(frame) for frame in generator)
517 if keep_pa_samples is not None and not pad_sample.is_bonafide:
518 generator = (
519 frame for frame in generator if random.random() < keep_pa_samples
520 )
522 if keep_bf_samples is not None and pad_sample.is_bonafide:
523 generator = (
524 frame for frame in generator if random.random() < keep_bf_samples
525 )
527 return generator