Add Advanced Optical Flow scripts

Someone asked me about doing a field of receptors before. These scripts
show how to do that. Also, added example scripts for calling the linear
polar and log polar methods added previously which power
find_rotscale().
This commit is contained in:
Kwabena W. Agyeman 2018-01-08 00:10:55 -05:00
parent 8530b26867
commit 7516d7699f
8 changed files with 360 additions and 10 deletions

View File

@ -141,26 +141,35 @@ void imlib_logpolar(image_t *img, bool linear, bool reverse)
void imlib_phasecorrelate(image_t *img0, image_t *img1, rectangle_t *roi0, rectangle_t *roi1, bool logpolar, float *x_offset, float *y_offset, float *response)
{
image_t img0alt, img1alt;
rectangle_t roi0alt, roi1alt;
if (logpolar) {
img0alt.w = roi0->w;
img0alt.h = roi0->h;
img0alt.bpp = img0->bpp;
img0alt.data = fb_alloc0(image_size(img0));
img0alt.data = fb_alloc0(image_size(&img0alt));
imlib_logpolar_int(&img0alt, img0, roi0, false, false);
roi0alt.x = 0;
roi0alt.y = 0;
roi0alt.w = roi0->w;
roi0alt.h = roi0->h;
img1alt.w = roi1->w;
img1alt.h = roi1->h;
img1alt.bpp = img1->bpp;
img1alt.data = fb_alloc0(image_size(img1));
img1alt.data = fb_alloc0(image_size(&img1alt));
imlib_logpolar_int(&img1alt, img1, roi1, false, false);
roi1alt.x = 0;
roi1alt.y = 0;
roi1alt.w = roi1->w;
roi1alt.h = roi1->h;
}
fft2d_controller_t fft0, fft1;
fft2d_alloc(&fft0, logpolar ? &img0alt : img0, roi0);
fft2d_alloc(&fft1, logpolar ? &img1alt : img1, roi1);
fft2d_alloc(&fft0, logpolar ? &img0alt : img0, logpolar ? &roi0alt : roi0);
fft2d_alloc(&fft1, logpolar ? &img1alt : img1, logpolar ? &roi1alt : roi1);
fft2d_run(&fft0);
fft2d_run(&fft1);
@ -174,10 +183,10 @@ void imlib_phasecorrelate(image_t *img0, image_t *img1, rectangle_t *roi0, recta
float gb_i = -fft1.data[i+1]; // complex conjugate...
float hp_r = (ga_r * gb_r) - (ga_i * gb_i); // hadamard product
float hp_i = (ga_r * gb_i) + (ga_i * gb_r); // hadamard product
float mag = fast_sqrtf((hp_r*hp_r)+(hp_i*hp_i)); // magnitude
float mag_inv = mag ? (1 / mag) : 0;
fft0.data[i+0] = hp_r * mag_inv;
fft0.data[i+1] = hp_i * mag_inv;
float mag = 1 / fast_sqrtf((hp_r*hp_r)+(hp_i*hp_i)); // magnitude
// Replace first fft with phase correlation...
fft0.data[i+0] = hp_r * mag;
fft0.data[i+1] = hp_i * mag;
}
ifft2d_run(&fft0);
@ -243,6 +252,21 @@ void imlib_phasecorrelate(image_t *img0, image_t *img1, rectangle_t *roi0, recta
*y_offset = -f_off_y;
}
if ((*x_offset < (-roi0->w/2))
|| ((roi0->w/2) <= *x_offset)
|| (*y_offset < (-roi0->h/2))
|| ((roi0->h/2) <= *y_offset)
|| isnanf(*x_offset)
|| isinff(*x_offset)
|| isnanf(*y_offset)
|| isinff(*y_offset)
|| isnanf(*response)
|| isinff(*response)) { // Noise Filter
*x_offset = 0;
*y_offset = 0;
*response = 0;
}
fft2d_dealloc(); // fft1
fft2d_dealloc(); // fft0

View File

@ -516,8 +516,8 @@ STATIC const mp_map_elem_t globals_dict_table[] = {
// FFT Resolutions
{ MP_OBJ_NEW_QSTR(MP_QSTR_B64X32), MP_OBJ_NEW_SMALL_INT(FRAMESIZE_64X32)}, /* 64x32 */
{ MP_OBJ_NEW_QSTR(MP_QSTR_B64X64), MP_OBJ_NEW_SMALL_INT(FRAMESIZE_64X64)}, /* 64x64 */
{ MP_OBJ_NEW_QSTR(MP_QSTR_B128X64), MP_OBJ_NEW_SMALL_INT(FRAMESIZE_64X64)}, /* 128x64 */
{ MP_OBJ_NEW_QSTR(MP_QSTR_B128X128), MP_OBJ_NEW_SMALL_INT(FRAMESIZE_64X64)}, /* 128x128 */
{ MP_OBJ_NEW_QSTR(MP_QSTR_B128X64), MP_OBJ_NEW_SMALL_INT(FRAMESIZE_128X64)}, /* 128x64 */
{ MP_OBJ_NEW_QSTR(MP_QSTR_B128X128), MP_OBJ_NEW_SMALL_INT(FRAMESIZE_128X128)}, /* 128x128 */
// Other
{ MP_OBJ_NEW_QSTR(MP_QSTR_LCD), MP_OBJ_NEW_SMALL_INT(FRAMESIZE_LCD)}, /* 128x160 */
{ MP_OBJ_NEW_QSTR(MP_QSTR_QQVGA2), MP_OBJ_NEW_SMALL_INT(FRAMESIZE_QQVGA2)}, /* 128x160 */

View File

@ -0,0 +1,21 @@
# Linear Polar Mapping Example
#
# This example shows off re-projecting the image using a linear polar
# transformation. Linear polar images are useful in that rotations
# become translations in the X direction and linear changes
# in scale become linear translations in the Y direction.
import sensor, image, time
sensor.reset() # Initialize the camera sensor.
sensor.set_pixformat(sensor.RGB565) # or sensor.GRAYSCALE
sensor.set_framesize(sensor.QQVGA) # or sensor.QVGA (or others)
sensor.skip_frames(time = 2000) # Let new settings take affect.
clock = time.clock() # Tracks FPS.
while(True):
clock.tick() # Track elapsed milliseconds between snapshots().
img = sensor.snapshot().linpolar(reverse=False)
print(clock.fps()) # Note: Your OpenMV Cam runs about half as fast while
# connected to your computer. The FPS should increase once disconnected.

View File

@ -0,0 +1,21 @@
# Log Polar Mapping Example
#
# This example shows off re-projecting the image using a log polar
# transformation. Log polar images are useful in that rotations
# become translations in the X direction and exponential changes
# in scale (x2, x4, etc.) become linear translations in the Y direction.
import sensor, image, time
sensor.reset() # Initialize the camera sensor.
sensor.set_pixformat(sensor.RGB565) # or sensor.GRAYSCALE
sensor.set_framesize(sensor.QQVGA) # or sensor.QVGA (or others)
sensor.skip_frames(time = 2000) # Let new settings take affect.
clock = time.clock() # Tracks FPS.
while(True):
clock.tick() # Track elapsed milliseconds between snapshots().
img = sensor.snapshot().logpolar(reverse=False)
print(clock.fps()) # Note: Your OpenMV Cam runs about half as fast while
# connected to your computer. The FPS should increase once disconnected.

View File

@ -0,0 +1,73 @@
# Image Patches Absolute Optical Flow Rotation/Scale
#
# This example shows off using your OpenMV Cam to measure
# rotation/scale by comparing the current and a previous
# image against each other. Note that only rotation/scale is
# handled - not X and Y translation in this mode.
#
# However, this examples goes beyond doing optical flow on the whole
# image at once. Instead it breaks up the process by working on groups
# of pixels in the image. This gives you a "new" image of results.
#
# Note that surfaces need to have some type of "edge" on them for the
# algorithm to work. A featureless surface produces crazy results.
# NOTE: Unless you have a very nice test rig this example is hard to see usefulness of...
BLOCK_W = 16 # pow2
BLOCK_H = 16 # pow2
# To run this demo effectively please mount your OpenMV Cam on a steady
# base and SLOWLY rotate the camera around the lens and move the camera
# forward/backwards to see the numbers change.
# I.e. Z direction changes only.
import sensor, image, time, math
# NOTE!!! You have to use a small power of 2 resolution when using
# find_rotscale(). This is because the algorithm is powered by
# something called phase correlation which does the image comparison
# using FFTs. A non-power of 2 resolution requires padding to a power
# of 2 which reduces the usefulness of the algorithm results. Please
# use a resolution like B128X128 or B128X64 (2x faster).
# Your OpenMV Cam supports power of 2 resolutions of 64x32, 64x64,
# 128x64, and 128x128. If you want a resolution of 32x32 you can create
# it by doing "img.pool(2, 2)" on a 64x64 image.
sensor.reset() # Reset and initialize the sensor.
sensor.set_pixformat(sensor.GRAYSCALE) # Set pixel format to GRAYSCALE (or RGB565)
sensor.set_framesize(sensor.B128X128) # Set frame size to 128x128... (or 128x64)...
sensor.skip_frames(time = 2000) # Wait for settings take effect.
clock = time.clock() # Create a clock object to track the FPS.
# Take from the main frame buffer's RAM to allocate a second frame buffer.
# There's a lot more RAM in the frame buffer than in the MicroPython heap.
# However, after doing this you have a lot less RAM for some algorithms...
# So, be aware that it's a lot easier to get out of RAM issues now.
extra_fb = sensor.alloc_extra_fb(sensor.width(), sensor.height(), sensor.GRAYSCALE)
extra_fb.replace(sensor.snapshot())
while(True):
clock.tick() # Track elapsed milliseconds between snapshots().
img = sensor.snapshot() # Take a picture and return the image.
for y in range(0, sensor.height(), BLOCK_H):
for x in range(0, sensor.width(), BLOCK_W):
# For this example we never update the old image to measure absolute change.
rotscale_obj = extra_fb.find_rotscale(img, \
roi = (x, y, BLOCK_W, BLOCK_H), template_roi = (x, y, BLOCK_W, BLOCK_H))
# Below 0.1 or so (YMMV) and the results are just noise.
if(rotscale_obj.response() > 0.1):
rotation_change = rotscale_obj.rot_offset()
zoom_amount = 1.0 + rotscale_obj.scale_offset()
pixel_x = x + (BLOCK_W//2) + int(math.sin(rotation_change) * zoom_amount * (BLOCK_W//4))
pixel_y = y + (BLOCK_H//2) + int(math.cos(rotation_change) * zoom_amount * (BLOCK_H//4))
img.draw_line((x + BLOCK_W//2, y + BLOCK_H//2, pixel_x, pixel_y), \
color = 255)
else:
img.draw_line((x + BLOCK_W//2, y + BLOCK_H//2, x + BLOCK_W//2, y + BLOCK_H//2), \
color = 0)
print(clock.fps())

View File

@ -0,0 +1,69 @@
# Image Patches Absolute Optical Flow Translation
#
# This example shows off using your OpenMV Cam to measure translation
# in the X and Y direction by comparing the current and a previous
# image against each other. Note that only X and Y translation is
# handled - not rotation/scale in this mode.
#
# However, this examples goes beyond doing optical flow on the whole
# image at once. Instead it breaks up the process by working on groups
# of pixels in the image. This gives you a "new" image of results.
#
# Note that surfaces need to have some type of "edge" on them for the
# algorithm to work. A featureless surface produces crazy results.
BLOCK_W = 16 # pow2
BLOCK_H = 16 # pow2
# To run this demo effectively please mount your OpenMV Cam on a steady
# base and SLOWLY translate it to the left, right, up, and down and
# watch the numbers change. Note that you can see displacement numbers
# up +- half of the hoizontal and vertical resolution.
import sensor, image, time
# NOTE!!! You have to use a small power of 2 resolution when using
# find_displacement(). This is because the algorithm is powered by
# something called phase correlation which does the image comparison
# using FFTs. A non-power of 2 resolution requires padding to a power
# of 2 which reduces the usefulness of the algorithm results. Please
# use a resolution like B128X128 or B128X64 (2x faster).
# Your OpenMV Cam supports power of 2 resolutions of 64x32, 64x64,
# 128x64, and 128x128. If you want a resolution of 32x32 you can create
# it by doing "img.pool(2, 2)" on a 64x64 image.
sensor.reset() # Reset and initialize the sensor.
sensor.set_pixformat(sensor.GRAYSCALE) # Set pixel format to GRAYSCALE (or RGB565)
sensor.set_framesize(sensor.B128X128) # Set frame size to 128x128... (or 128x64)...
sensor.skip_frames(time = 2000) # Wait for settings take effect.
clock = time.clock() # Create a clock object to track the FPS.
# Take from the main frame buffer's RAM to allocate a second frame buffer.
# There's a lot more RAM in the frame buffer than in the MicroPython heap.
# However, after doing this you have a lot less RAM for some algorithms...
# So, be aware that it's a lot easier to get out of RAM issues now.
extra_fb = sensor.alloc_extra_fb(sensor.width(), sensor.height(), sensor.GRAYSCALE)
extra_fb.replace(sensor.snapshot())
while(True):
clock.tick() # Track elapsed milliseconds between snapshots().
img = sensor.snapshot() # Take a picture and return the image.
for y in range(0, sensor.height(), BLOCK_H):
for x in range(0, sensor.width(), BLOCK_W):
# For this example we never update the old image to measure absolute change.
displacement_obj = extra_fb.find_displacement(img, \
roi = (x, y, BLOCK_W, BLOCK_H), template_roi = (x, y, BLOCK_W, BLOCK_H))
# Below 0.1 or so (YMMV) and the results are just noise.
if(displacement_obj.response() > 0.1):
pixel_x = x + (BLOCK_W//2) + int(displacement_obj.x_offset())
pixel_y = y + (BLOCK_H//2) + int(displacement_obj.y_offset())
img.draw_line((x + BLOCK_W//2, y + BLOCK_H//2, pixel_x, pixel_y), \
color = 255)
else:
img.draw_line((x + BLOCK_W//2, y + BLOCK_H//2, x + BLOCK_W//2, y + BLOCK_H//2), \
color = 0)
print(clock.fps())

View File

@ -0,0 +1,73 @@
# Image Patches Differential Optical Flow Rotation/Scale
#
# This example shows off using your OpenMV Cam to measure
# rotation/scale by comparing the current and the previous
# image against each other. Note that only rotation/scale is
# handled - not X and Y translation in this mode.
#
# However, this examples goes beyond doing optical flow on the whole
# image at once. Instead it breaks up the process by working on groups
# of pixels in the image. This gives you a "new" image of results.
#
# Note that surfaces need to have some type of "edge" on them for the
# algorithm to work. A featureless surface produces crazy results.
# NOTE: Unless you have a very nice test rig this example is hard to see usefulness of...
BLOCK_W = 16 # pow2
BLOCK_H = 16 # pow2
# To run this demo effectively please mount your OpenMV Cam on a steady
# base and SLOWLY rotate the camera around the lens and move the camera
# forward/backwards to see the numbers change.
# I.e. Z direction changes only.
import sensor, image, time, math
# NOTE!!! You have to use a small power of 2 resolution when using
# find_rotscale(). This is because the algorithm is powered by
# something called phase correlation which does the image comparison
# using FFTs. A non-power of 2 resolution requires padding to a power
# of 2 which reduces the usefulness of the algorithm results. Please
# use a resolution like B128X128 or B128X64 (2x faster).
# Your OpenMV Cam supports power of 2 resolutions of 64x32, 64x64,
# 128x64, and 128x128. If you want a resolution of 32x32 you can create
# it by doing "img.pool(2, 2)" on a 64x64 image.
sensor.reset() # Reset and initialize the sensor.
sensor.set_pixformat(sensor.GRAYSCALE) # Set pixel format to GRAYSCALE (or RGB565)
sensor.set_framesize(sensor.B128X128) # Set frame size to 128x128... (or 128x64)...
sensor.skip_frames(time = 2000) # Wait for settings take effect.
clock = time.clock() # Create a clock object to track the FPS.
# Take from the main frame buffer's RAM to allocate a second frame buffer.
# There's a lot more RAM in the frame buffer than in the MicroPython heap.
# However, after doing this you have a lot less RAM for some algorithms...
# So, be aware that it's a lot easier to get out of RAM issues now.
extra_fb = sensor.alloc_extra_fb(sensor.width(), sensor.height(), sensor.GRAYSCALE)
extra_fb.replace(sensor.snapshot())
while(True):
clock.tick() # Track elapsed milliseconds between snapshots().
img = sensor.snapshot() # Take a picture and return the image.
for y in range(0, sensor.height(), BLOCK_H):
for x in range(0, sensor.width(), BLOCK_W):
rotscale_obj = extra_fb.find_rotscale(img, \
roi = (x, y, BLOCK_W, BLOCK_H), template_roi = (x, y, BLOCK_W, BLOCK_H))
# Below 0.1 or so (YMMV) and the results are just noise.
if(rotscale_obj.response() > 0.1):
rotation_change = rotscale_obj.rot_offset()
zoom_amount = 1.0 + rotscale_obj.scale_offset()
pixel_x = x + (BLOCK_W//2) + int(math.sin(rotation_change) * zoom_amount * (BLOCK_W//4))
pixel_y = y + (BLOCK_H//2) + int(math.cos(rotation_change) * zoom_amount * (BLOCK_H//4))
img.draw_line((x + BLOCK_W//2, y + BLOCK_H//2, pixel_x, pixel_y), \
color = 255)
else:
img.draw_line((x + BLOCK_W//2, y + BLOCK_H//2, x + BLOCK_W//2, y + BLOCK_H//2), \
color = 0)
extra_fb.replace(img)
print(clock.fps())

View File

@ -0,0 +1,69 @@
# Image Patches Differential Optical Flow Translation
#
# This example shows off using your OpenMV Cam to measure translation
# in the X and Y direction by comparing the current and the previous
# image against each other. Note that only X and Y translation is
# handled - not rotation/scale in this mode.
#
# However, this examples goes beyond doing optical flow on the whole
# image at once. Instead it breaks up the process by working on groups
# of pixels in the image. This gives you a "new" image of results.
#
# Note that surfaces need to have some type of "edge" on them for the
# algorithm to work. A featureless surface produces crazy results.
BLOCK_W = 16 # pow2
BLOCK_H = 16 # pow2
# To run this demo effectively please mount your OpenMV Cam on a steady
# base and SLOWLY translate it to the left, right, up, and down and
# watch the numbers change. Note that you can see displacement numbers
# up +- half of the hoizontal and vertical resolution.
import sensor, image, time
# NOTE!!! You have to use a small power of 2 resolution when using
# find_displacement(). This is because the algorithm is powered by
# something called phase correlation which does the image comparison
# using FFTs. A non-power of 2 resolution requires padding to a power
# of 2 which reduces the usefulness of the algorithm results. Please
# use a resolution like B128X128 or B128X64 (2x faster).
# Your OpenMV Cam supports power of 2 resolutions of 64x32, 64x64,
# 128x64, and 128x128. If you want a resolution of 32x32 you can create
# it by doing "img.pool(2, 2)" on a 64x64 image.
sensor.reset() # Reset and initialize the sensor.
sensor.set_pixformat(sensor.GRAYSCALE) # Set pixel format to GRAYSCALE (or RGB565)
sensor.set_framesize(sensor.B128X128) # Set frame size to 128x128... (or 128x64)...
sensor.skip_frames(time = 2000) # Wait for settings take effect.
clock = time.clock() # Create a clock object to track the FPS.
# Take from the main frame buffer's RAM to allocate a second frame buffer.
# There's a lot more RAM in the frame buffer than in the MicroPython heap.
# However, after doing this you have a lot less RAM for some algorithms...
# So, be aware that it's a lot easier to get out of RAM issues now.
extra_fb = sensor.alloc_extra_fb(sensor.width(), sensor.height(), sensor.GRAYSCALE)
extra_fb.replace(sensor.snapshot())
while(True):
clock.tick() # Track elapsed milliseconds between snapshots().
img = sensor.snapshot() # Take a picture and return the image.
for y in range(0, sensor.height(), BLOCK_H):
for x in range(0, sensor.width(), BLOCK_W):
displacement_obj = extra_fb.find_displacement(img, \
roi = (x, y, BLOCK_W, BLOCK_H), template_roi = (x, y, BLOCK_W, BLOCK_H))
# Below 0.1 or so (YMMV) and the results are just noise.
if(displacement_obj.response() > 0.1):
pixel_x = x + (BLOCK_W//2) + int(displacement_obj.x_offset())
pixel_y = y + (BLOCK_H//2) + int(displacement_obj.y_offset())
img.draw_line((x + BLOCK_W//2, y + BLOCK_H//2, pixel_x, pixel_y), \
color = 255)
else:
img.draw_line((x + BLOCK_W//2, y + BLOCK_H//2, x + BLOCK_W//2, y + BLOCK_H//2), \
color = 0)
extra_fb.replace(img)
print(clock.fps())