-
Notifications
You must be signed in to change notification settings - Fork 0
/
visualization.py
279 lines (257 loc) · 11.7 KB
/
visualization.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
import torch
import numpy as np
import matplotlib.pyplot as plt
import open3d as o3d
from collections import namedtuple
from itertools import accumulate
from typing import Optional, Tuple
from matplotlib.ticker import AutoMinorLocator
DEFAULT_TRANSITIONS = (15, 6, 4, 11, 13, 6)
BLUE = (94/255, 129/255, 160/255)
GREEN = (163/255, 190/255, 128/255)
RED = (191/255, 97/255, 106/255)
PURPLE = (180/255, 142/255, 160/255)
OPACITY = 1.0
def make_colorwheel(transitions: tuple=DEFAULT_TRANSITIONS) -> np.ndarray:
"""Creates a colorwheel (borrowed/modified from flowpy).
A colorwheel defines the transitions between the six primary hues:
Red(255, 0, 0), Yellow(255, 255, 0), Green(0, 255, 0), Cyan(0, 255, 255), Blue(0, 0, 255) and Magenta(255, 0, 255).
Args:
transitions: Contains the length of the six transitions, based on human color perception.
Returns:
colorwheel: The RGB values of the transitions in the color space.
Notes:
For more information, see:
https://web.archive.org/web/20051107102013/http://members.shaw.ca/quadibloc/other/colint.htm
http://vision.middlebury.edu/flow/flowEval-iccv07.pdf
"""
colorwheel_length = sum(transitions)
# The red hue is repeated to make the colorwheel cyclic
base_hues = map(
np.array, ([255, 0, 0], [255, 255, 0], [0, 255, 0], [0, 255, 255], [0, 0, 255], [255, 0, 255], [255, 0, 0])
)
colorwheel = np.zeros((colorwheel_length, 3), dtype="uint8")
hue_from = next(base_hues)
start_index = 0
for hue_to, end_index in zip(base_hues, accumulate(transitions)):
transition_length = end_index - start_index
colorwheel[start_index:end_index] = np.linspace(hue_from, hue_to, transition_length, endpoint=False)
hue_from = hue_to
start_index = end_index
return colorwheel
def flow_to_rgb(
flow: np.ndarray,
flow_max_radius: Optional[float]=None,
background: Optional[str]="bright",
) -> np.ndarray:
"""Creates a RGB representation of an optical flow (borrowed/modified from flowpy).
Args:
flow: scene flow.
flow[..., 0] should be the x-displacement
flow[..., 1] should be the y-displacement
flow[..., 2] should be the z-displacement
flow_max_radius: Set the radius that gives the maximum color intensity, useful for comparing different flows.
Default: The normalization is based on the input flow maximum radius.
background: States if zero-valued flow should look 'bright' or 'dark'.
Returns: An array of RGB colors.
"""
valid_backgrounds = ("bright", "dark")
if background not in valid_backgrounds:
raise ValueError(f"background should be one the following: {valid_backgrounds}, not {background}.")
wheel = make_colorwheel()
# For scene flow, it's reasonable to assume displacements in x and y directions only for visualization pursposes.
complex_flow = flow[..., 0] + 1j * flow[..., 1]
radius, angle = np.abs(complex_flow), np.angle(complex_flow)
if flow_max_radius is None:
flow_max_radius = np.max(radius)
if flow_max_radius > 0:
radius /= flow_max_radius
ncols = len(wheel)
# Map the angles from (-pi, pi] to [0, 2pi) to [0, ncols - 1)
angle[angle < 0] += 2 * np.pi
angle = angle * ((ncols - 1) / (2 * np.pi))
# Make the wheel cyclic for interpolation
wheel = np.vstack((wheel, wheel[0]))
# Interpolate the hues
(angle_fractional, angle_floor), angle_ceil = np.modf(angle), np.ceil(angle)
angle_fractional = angle_fractional.reshape((angle_fractional.shape) + (1,))
float_hue = (
wheel[angle_floor.astype(np.int32)] * (1 - angle_fractional) + wheel[angle_ceil.astype(np.int32)\
] * angle_fractional
)
ColorizationArgs = namedtuple(
'ColorizationArgs', ['move_hue_valid_radius', 'move_hue_oversized_radius', 'invalid_color']
)
def move_hue_on_V_axis(hues, factors):
return hues * np.expand_dims(factors, -1)
def move_hue_on_S_axis(hues, factors):
return 255. - np.expand_dims(factors, -1) * (255. - hues)
if background == "dark":
parameters = ColorizationArgs(
move_hue_on_V_axis, move_hue_on_S_axis, np.array([255, 255, 255], dtype=np.float32)
)
else:
parameters = ColorizationArgs(move_hue_on_S_axis, move_hue_on_V_axis, np.array([0, 0, 0\
], dtype=np.float32))
colors = parameters.move_hue_valid_radius(float_hue, radius)
oversized_radius_mask = radius > 1
colors[oversized_radius_mask] = parameters.move_hue_oversized_radius(
float_hue[oversized_radius_mask],
1 / radius[oversized_radius_mask]
)
return colors.astype(np.uint8)
def calibration_pattern(
pixel_size: int=151,
flow_max_radius: float=1,
**flow_to_rgb_args
) -> Tuple[np.ndarray, np.ndarray]:
"""Generates a calibration pattern to add as a legend to the scene flow plots.
Args:
pixel_size: Radius of the square test pattern.
flow_max_radius: The maximum radius value represented by the calibration pattern.
flow_to_rgb_args: kwargs passed to the flow_to_rgb function.
Returns:
calibration_img: The RGB image representation of the calibration pattern.
calibration_flow: The flow represented in the calibration_pattern.
"""
half_width = pixel_size // 2
y_grid, x_grid = np.mgrid[:pixel_size, :pixel_size]
u = flow_max_radius * (x_grid / half_width - 1)
v = flow_max_radius * (y_grid / half_width - 1)
flow = np.zeros((pixel_size, pixel_size, 2))
flow[..., 0] = u
flow[..., 1] = v
flow_to_rgb_args["flow_max_radius"] = flow_max_radius
img = flow_to_rgb(flow, **flow_to_rgb_args)
return img, flow
def attach_calibration_pattern(ax, **calibration_pattern_kwargs):
"""Attach a calibration pattern to axes.
This function uses calibration_pattern to generate a figure.
Args:
calibration_pattern_kwargs: kwargs, optional
Parameters to be given to the calibration_pattern function.
Returns:
image_axes: matplotlib.AxesImage
See matplotlib.imshow documentation
Useful for changing the image dynamically
circle_artist: matplotlib.artist
See matplotlib.circle documentation
Useful for removing the circle from the figure
"""
pattern, flow = calibration_pattern(**calibration_pattern_kwargs)
flow_max_radius = calibration_pattern_kwargs.get("flow_max_radius", 1)
extent = (-flow_max_radius, flow_max_radius) * 2
image = ax.imshow(pattern, extent=extent)
ax.spines["top"].set_visible(False)
ax.spines["right"].set_visible(False)
for spine in ("bottom", "left"):
ax.spines[spine].set_position("zero")
ax.spines[spine].set_linewidth(1)
ax.xaxis.set_minor_locator(AutoMinorLocator())
ax.yaxis.set_minor_locator(AutoMinorLocator())
attach_coord(ax, flow, extent=extent)
circle = plt.Circle((0, 0), flow_max_radius, fill=False, lw=1)
ax.add_artist(circle)
return image, circle
def attach_coord(ax, flow, extent=None):
"""Attach the flow value to the coordinate tooltip.
It allows you to see on the same figure, the RGB value of the pixel and the underlying value of the flow.
Shows cartesian and polar coordinates.
Args:
ax: matplotlib.axes
The axes the arrows should be plotted on.
flow: numpy.ndarray
scene flow.
flow[..., 0] should be the x-displacement
flow[..., 1] should be the y-displacement
extent: sequence_like, optional
Use this parameters in combination with matplotlib.imshow to resize the RGB plot.
See matplotlib.imshow extent parameter.
See attach_calibration_pattern
"""
height, width, _ = flow.shape
base_format = ax.format_coord
if extent != None:
left, right, bottom, top = extent
x_ratio = width / (right - left)
y_ratio = height / (top - bottom)
def new_format_coord(x, y):
if extent is None:
int_x = int(x + 0.5)
int_y = int(y + 0.5)
else:
int_x = int((x - left) * x_ratio)
int_y = int((y - bottom) * y_ratio)
if 0 <= int_x < width and 0 <= int_y < height:
format_string = "Coord: x={}, y={} / Flow: ".format(int_x, int_y)
u, v = flow[int_y, int_x, :]
if np.isnan(u) or np.isnan(v):
format_string += "invalid"
else:
complex_flow = u - 1j * v
r, h = np.abs(complex_flow), np.angle(complex_flow, deg=True)
format_string += ("u={:.2f}, v={:.2f} (cartesian) ρ={:.2f}, θ={:.2f}° (polar)"
.format(u, v, r, h))
return format_string
else:
return base_format(x, y)
ax.format_coord = new_format_coord
def custom_draw_geometry_with_key_callback(pcds):
def change_background_to_real_black(vis):
opt = vis.get_render_option()
# opt.background_color = np.asarray([7/255, 54/255, 66/255])
opt.background_color = np.asarray([0., 0., 0.])
return False
def change_background_to_black(vis):
opt = vis.get_render_option()
opt.background_color = np.asarray([76/255, 86/255, 106/255])
return False
def load_render_option(vis):
vis.get_render_option().load_from_json(
"../../TestData/renderoption.json")
return False
def capture_depth(vis):
depth = vis.capture_depth_float_buffer()
plt.imshow(np.asarray(depth))
plt.show()
return False
def capture_image(vis):
image = vis.capture_screen_float_buffer()
plt.imshow(np.asarray(image))
plt.show()
return False
key_to_callback = {}
key_to_callback[ord("B")] = change_background_to_real_black
key_to_callback[ord("K")] = change_background_to_black
key_to_callback[ord("R")] = load_render_option
key_to_callback[ord(",")] = capture_depth
key_to_callback[ord(".")] = capture_image
o3d.visualization.draw_geometries_with_key_callbacks(pcds, key_to_callback)
def show_flows(pc1, pc2, flow):
if type(pc1) != np.ndarray:
pc1 = pc1.cpu().numpy()
pc2 = pc2.cpu().numpy()
flow = flow.detach().cpu().numpy()
pc1_deform = pc1 + flow
pc1_o3d = o3d.geometry.PointCloud()
pc1_o3d.points = o3d.utility.Vector3dVector(pc1)
pc1_o3d.paint_uniform_color(BLUE)
pc1_o3d.estimate_normals(search_param=o3d.geometry.KDTreeSearchParamHybrid(radius=0.1, max_nn=30))
pc1_o3d.orient_normals_to_align_with_direction()
pc2_o3d = o3d.geometry.PointCloud()
pc2_o3d.points = o3d.utility.Vector3dVector(pc2)
pc2_o3d.paint_uniform_color(GREEN)
pc2_o3d.estimate_normals(search_param=o3d.geometry.KDTreeSearchParamHybrid(radius=0.1, max_nn=30))
pc2_o3d.orient_normals_to_align_with_direction()
pc1_def_o3d = o3d.geometry.PointCloud()
pc1_def_o3d.points = o3d.utility.Vector3dVector(pc1_deform)
pc1_def_o3d.paint_uniform_color(RED)
pc1_def_o3d.estimate_normals(search_param=o3d.geometry.KDTreeSearchParamHybrid(radius=0.1, max_nn=30))
pc1_def_o3d.orient_normals_to_align_with_direction()
o3d.visualization.draw_geometries([pc1_o3d, pc2_o3d, pc1_def_o3d])
# ANCHOR: new plot style
pc1_o3d_gt = o3d.geometry.PointCloud()
colors_flow = flow_to_rgb(flow)
pc1_o3d_gt.points = o3d.utility.Vector3dVector(pc1)
pc1_o3d_gt.colors = o3d.utility.Vector3dVector(colors_flow / 255.0)
custom_draw_geometry_with_key_callback([pc1_o3d_gt]) # Press 'k' to see with dark background.