import os
import numpy
from scipy.interpolate import RectBivariateSpline
def activation_visualization(image, data, level, alpha=0.5, source_shape=None,
crop=False, zoom=None, border=2, negate=False, return_mask=False,
**kwargs):
"""
Makes a visualiztion image of activation data overlaid on the image.
Params:
image The original image.
data The single channel feature map.
alpha The darkening to apply in inactive regions of the image.
level The threshold of activation levels to highlight.
"""
if len(image.shape) == 2:
# Puff up grayscale image to RGB.
image = image[:,:,None] * numpy.array([[[1, 1, 1]]])
surface = activation_surface(data, target_shape=image.shape[:2],
source_shape=source_shape, **kwargs)
if negate:
surface = -surface
level = -level
if crop:
# crop to source_shape
if source_shape is not None:
ch, cw = ((t - s) // 2 for s, t in zip(
source_shape, image.shape[:2]))
image = image[ch:ch+source_shape[0], cw:cw+source_shape[1]]
surface = surface[ch:ch+source_shape[0], cw:cw+source_shape[1]]
if crop is True:
crop = surface.shape
elif not hasattr(crop, '__len__'):
crop = (crop, crop)
if zoom is not None:
source_rect = best_sub_rect(surface >= level, crop, zoom,
pad=border)
else:
source_rect = (0, surface.shape[0], 0, surface.shape[1])
image = zoom_image(image, source_rect, crop)
surface = zoom_image(surface, source_rect, crop)
mask = (surface >= level)
# Add a yellow border at the edge of the mask for contrast
result = (mask[:, :, None] * (1 - alpha) + alpha) * image
if border:
edge = mask_border(mask)[:,:,None]
result = numpy.maximum(edge * numpy.array([[[200, 200, 0]]]), result)
if not return_mask:
return result
mask_image = (1 - mask[:, :, None]) * numpy.array(
[[[0, 0, 0, 255 * (1 - alpha)]]], dtype=numpy.uint8)
if border:
mask_image = numpy.maximum(edge * numpy.array([[[200, 200, 0, 255]]]),
mask_image)
return result, mask_image
def activation_surface(data, target_shape=None, source_shape=None,
scale_offset=None, deg=1, pad=True):
"""
Generates an upsampled activation sample.
Params:
target_shape Shape of the output array.
source_shape The centered shape of the output to match with data
when upscaling. Defaults to the whole target_shape.
scale_offset The amount by which to scale, then offset data
dimensions to end up with target dimensions. A pair of pairs.
deg Degree of interpolation to apply (1 = linear, etc).
pad True to zero-pad the edge instead of doing a funny edge interp.
"""
# Default is that nothing is resized.
if target_shape is None:
target_shape = data.shape
# Make a default scale_offset to fill the image if there isn't one
if scale_offset is None:
scale = tuple(float(ts) / ds
for ts, ds in zip(target_shape, data.shape))
offset = tuple(0.5 * s - 0.5 for s in scale)
else:
scale, offset = (v for v in zip(*scale_offset))
# Now we adjust offsets to take into account cropping and so on
if source_shape is not None:
offset = tuple(o + (ts - ss) / 2.0
for o, ss, ts in zip(offset, source_shape, target_shape))
# Pad the edge with zeros for sensible edge behavior
if pad:
zeropad = numpy.zeros(
(data.shape[0] + 2, data.shape[1] + 2), dtype=data.dtype)
zeropad[1:-1, 1:-1] = data
data = zeropad
offset = tuple((o - s) for o, s in zip(offset, scale))
# Upsample linearly
ty, tx = (numpy.arange(ts) for ts in target_shape)
sy, sx = (numpy.arange(ss) * s + o
for ss, s, o in zip(data.shape, scale, offset))
levels = RectBivariateSpline(
sy, sx, data, kx=deg, ky=deg)(ty, tx, grid=True)
# Return the mask.
return levels
def mask_border(mask, border=2):
"""Given a mask computes a border mask"""
from scipy import ndimage
struct = ndimage.generate_binary_structure(2, 2)
erosion = numpy.ones((mask.shape[0] + 10, mask.shape[1] + 10), dtype='int')
erosion[5:5+mask.shape[0], 5:5+mask.shape[1]] = ~mask
for _ in range(border):
erosion = ndimage.binary_erosion(erosion, struct)
return ~mask ^ erosion[5:5+mask.shape[0], 5:5+mask.shape[1]]
def bounding_rect(mask, pad=0):
"""Returns (r, b, l, r) boundaries so that all nonzero pixels in mask
have locations (i, j) with t <= i < b, and l <= j < r."""
nz = mask.nonzero()
if len(nz[0]) == 0:
# print('no pixels')
return (0, mask.shape[0], 0, mask.shape[1])
(t, b), (l, r) = [(max(0, p.min() - pad), min(s, p.max() + 1 + pad))
for p, s in zip(nz, mask.shape)]
return (t, b, l, r)
def best_sub_rect(mask, shape, max_zoom=None, pad=2):
"""Finds the smallest subrectangle containing all the nonzeros of mask,
matching the aspect ratio of shape, and where the zoom-up ratio is no
more than max_zoom"""
t, b, l, r = bounding_rect(mask, pad=pad)
height = max(b - t, int(round(float(shape[0]) * (r - l) / shape[1])))
if max_zoom is not None:
height = int(max(round(float(shape[0]) / max_zoom), height))
width = int(round(float(shape[1]) * height / shape[0]))
nt = min(mask.shape[0] - height, max(0, (b + t - height) // 2))
nb = nt + height
nl = min(mask.shape[1] - width, max(0, (r + l - width) // 2))
nr = nl + width
return (nt, nb, nl, nr)
def zoom_image(img, source_rect, target_shape=None):
"""Zooms pixels from the source_rect of img to target_shape."""
import warnings
from scipy.ndimage import zoom
if target_shape is None:
target_shape = img.shape
st, sb, sl, sr = source_rect
source = img[st:sb, sl:sr]
if source.shape == target_shape:
return source
zoom_tuple = tuple(float(t) / s
for t, s in zip(target_shape, source.shape[:2])
) + (1,) * (img.ndim - 2)
with warnings.catch_warnings():
warnings.simplefilter('ignore', UserWarning) # "output shape of zoom"
target = zoom(source, zoom_tuple)
assert target.shape[:2] == target_shape, (target.shape, target_shape)
return target
def scale_offset(dilations):
if len(dilations) == 0:
return (1, 0)
scale, offset = scale_offset(dilations[1:])
kernel, stride, padding = dilations[0]
scale *= stride
offset *= stride
offset += (kernel - 1) / 2.0 - padding
return scale, offset
def choose_level(feature_map, percentile=0.8):
'''
Chooses the top 80% level (or whatever the level chosen).
'''
data_range = numpy.sort(feature_map.flatten())
return numpy.interp(
percentile, numpy.linspace(0, 1, len(data_range)), data_range)
def dilations(modulelist):
result = []
for module in modulelist:
settings = tuple(getattr(module, n, d)
for n, d in (('kernel_size', 1), ('stride', 1), ('padding', 0)))
settings = (((s, s) if not isinstance(s, tuple) else s)
for s in settings)
if settings != ((1, 1), (1, 1), (0, 0)):
result.append(zip(*settings))
return zip(*result)
def grid_scale_offset(modulelist):
'''Returns (yscale, yoffset), (xscale, xoffset) given a list of modules'''
return tuple(scale_offset(d) for d in dilations(modulelist))