diff --git a/src/libtf/cortex-m4/libtf.a b/src/libtf/cortex-m4/libtf.a index d521f35b8..7222974dc 100644 Binary files a/src/libtf/cortex-m4/libtf.a and b/src/libtf/cortex-m4/libtf.a differ diff --git a/src/libtf/cortex-m4/libtf.h b/src/libtf/cortex-m4/libtf.h index 142e2ac9c..2a59b7c55 100644 --- a/src/libtf/cortex-m4/libtf.h +++ b/src/libtf/cortex-m4/libtf.h @@ -15,40 +15,48 @@ extern "C" { // Errors are printed to stdout. int libtf_get_input_data_hwc(const unsigned char *model_data, // TensorFlow Lite binary model (8-bit quant). unsigned char *tensor_arena, // As big as you can make it scratch buffer. - const unsigned int tensor_arena_size, // Size of the above scratch buffer. + unsigned int tensor_arena_size, // Size of the above scratch buffer. unsigned int *input_height, // Height for the model. unsigned int *input_width, // Width for the model. - unsigned int *input_channels); // Channels for the model (1 for grayscale8 and 3 for rgb888). + unsigned int *input_channels, // Channels for the model (1 for grayscale8 and 3 for rgb888). + bool *signed_or_unsigned, // True if input is int8_t ([0:255]->[-128:127]), False if input is uint8_t ([0:255]->[0:255]). + bool *is_float); // Actual is float32 (not optimal - network should be fixed). Input should be ([0:255]->[-1.0f:+1.0f]). // Call this second to get the shape of the model output. // Returns 0 on success and 1 on failure. // Errors are printed to stdout. int libtf_get_output_data_hwc(const unsigned char *model_data, // TensorFlow Lite binary model (8-bit quant). unsigned char *tensor_arena, // As big as you can make it scratch buffer. - const unsigned int tensor_arena_size, // Size of the above scratch buffer. + unsigned int tensor_arena_size, // Size of the above scratch buffer. unsigned int *output_height, // Height for the model. unsigned int *output_width, // Width for the model. - unsigned int *output_channels); // Channels for the model (1 for grayscale8 and 3 for rgb888). + unsigned int *output_channels, // Channels for the model (1 for grayscale8 and 3 for rgb888). + bool *signed_or_unsigned, // True if output is int8_t ([-128:127]->[0:255]->[0.0f:1.0f]), False if output is uint8_t ([0:255]->[0:255]->[0.0f:1.0f]). + bool *is_float); // Actual is float32 (not optimal - network should be fixed). Output should be ([-1.0f:+1.0f]->[0:255]->[0.0f:1.0f]). // Callback to populate the model input data byte array (laid out in [height][width][channel] order). typedef void (*libtf_input_data_callback_t)(void *callback_data, - unsigned char *model_input, + void *model_input, const unsigned int input_height, const unsigned int input_width, - const unsigned int input_channels); + const unsigned int input_channels, + const bool signed_or_unsigned, // True if input is int8_t ([0:255]->[-128:127]), False if input is uint8_t ([0:255]->[0:255]). + const bool is_float); // Actual is float32 (not optimal - network should be fixed). Input should be ([0:255]->[-1.0f:+1.0f]). // Callback to use the model output data byte array (laid out in [height][width][channel] order). typedef void (*libtf_output_data_callback_t)(void *callback_data, - unsigned char *model_output, + void *model_output, const unsigned int output_height, const unsigned int output_width, - const unsigned int output_channels); + const unsigned int output_channels, + const bool signed_or_unsigned, // True if output is int8_t ([-128:127]->[0:255]->[0.0f:1.0f]), False if output is uint8_t ([0:255]->[0:255]->[0.0f:1.0f]). + const bool is_float); // Actual is float32 (not optimal - network should be fixed). Output should be ([-1.0f:+1.0f]->[0:255]->[0.0f:1.0f]). // Returns 0 on success and 1 on failure. // Errors are printed to stdout. int libtf_invoke(const unsigned char *model_data, // TensorFlow Lite binary model (8-bit quant). unsigned char *tensor_arena, // As big as you can make it scratch buffer. - const unsigned int tensor_arena_size, // Size of the above scratch buffer. + unsigned int tensor_arena_size, // Size of the above scratch buffer. libtf_input_data_callback_t input_callback, // Callback to populate the model input data byte array. void *input_callback_data, // User data structure passed to input callback. libtf_output_data_callback_t output_callback, // Callback to use the model output data byte array. diff --git a/src/libtf/cortex-m4/libtf_person_detect_model_data.a b/src/libtf/cortex-m4/libtf_person_detect_model_data.a index 511a9ef0c..df1a1ebfc 100644 Binary files a/src/libtf/cortex-m4/libtf_person_detect_model_data.a and b/src/libtf/cortex-m4/libtf_person_detect_model_data.a differ diff --git a/src/libtf/cortex-m7/libtf.a b/src/libtf/cortex-m7/libtf.a index 3a4247524..63b82eb5f 100644 Binary files a/src/libtf/cortex-m7/libtf.a and b/src/libtf/cortex-m7/libtf.a differ diff --git a/src/libtf/cortex-m7/libtf.h b/src/libtf/cortex-m7/libtf.h index 142e2ac9c..2a59b7c55 100644 --- a/src/libtf/cortex-m7/libtf.h +++ b/src/libtf/cortex-m7/libtf.h @@ -15,40 +15,48 @@ extern "C" { // Errors are printed to stdout. int libtf_get_input_data_hwc(const unsigned char *model_data, // TensorFlow Lite binary model (8-bit quant). unsigned char *tensor_arena, // As big as you can make it scratch buffer. - const unsigned int tensor_arena_size, // Size of the above scratch buffer. + unsigned int tensor_arena_size, // Size of the above scratch buffer. unsigned int *input_height, // Height for the model. unsigned int *input_width, // Width for the model. - unsigned int *input_channels); // Channels for the model (1 for grayscale8 and 3 for rgb888). + unsigned int *input_channels, // Channels for the model (1 for grayscale8 and 3 for rgb888). + bool *signed_or_unsigned, // True if input is int8_t ([0:255]->[-128:127]), False if input is uint8_t ([0:255]->[0:255]). + bool *is_float); // Actual is float32 (not optimal - network should be fixed). Input should be ([0:255]->[-1.0f:+1.0f]). // Call this second to get the shape of the model output. // Returns 0 on success and 1 on failure. // Errors are printed to stdout. int libtf_get_output_data_hwc(const unsigned char *model_data, // TensorFlow Lite binary model (8-bit quant). unsigned char *tensor_arena, // As big as you can make it scratch buffer. - const unsigned int tensor_arena_size, // Size of the above scratch buffer. + unsigned int tensor_arena_size, // Size of the above scratch buffer. unsigned int *output_height, // Height for the model. unsigned int *output_width, // Width for the model. - unsigned int *output_channels); // Channels for the model (1 for grayscale8 and 3 for rgb888). + unsigned int *output_channels, // Channels for the model (1 for grayscale8 and 3 for rgb888). + bool *signed_or_unsigned, // True if output is int8_t ([-128:127]->[0:255]->[0.0f:1.0f]), False if output is uint8_t ([0:255]->[0:255]->[0.0f:1.0f]). + bool *is_float); // Actual is float32 (not optimal - network should be fixed). Output should be ([-1.0f:+1.0f]->[0:255]->[0.0f:1.0f]). // Callback to populate the model input data byte array (laid out in [height][width][channel] order). typedef void (*libtf_input_data_callback_t)(void *callback_data, - unsigned char *model_input, + void *model_input, const unsigned int input_height, const unsigned int input_width, - const unsigned int input_channels); + const unsigned int input_channels, + const bool signed_or_unsigned, // True if input is int8_t ([0:255]->[-128:127]), False if input is uint8_t ([0:255]->[0:255]). + const bool is_float); // Actual is float32 (not optimal - network should be fixed). Input should be ([0:255]->[-1.0f:+1.0f]). // Callback to use the model output data byte array (laid out in [height][width][channel] order). typedef void (*libtf_output_data_callback_t)(void *callback_data, - unsigned char *model_output, + void *model_output, const unsigned int output_height, const unsigned int output_width, - const unsigned int output_channels); + const unsigned int output_channels, + const bool signed_or_unsigned, // True if output is int8_t ([-128:127]->[0:255]->[0.0f:1.0f]), False if output is uint8_t ([0:255]->[0:255]->[0.0f:1.0f]). + const bool is_float); // Actual is float32 (not optimal - network should be fixed). Output should be ([-1.0f:+1.0f]->[0:255]->[0.0f:1.0f]). // Returns 0 on success and 1 on failure. // Errors are printed to stdout. int libtf_invoke(const unsigned char *model_data, // TensorFlow Lite binary model (8-bit quant). unsigned char *tensor_arena, // As big as you can make it scratch buffer. - const unsigned int tensor_arena_size, // Size of the above scratch buffer. + unsigned int tensor_arena_size, // Size of the above scratch buffer. libtf_input_data_callback_t input_callback, // Callback to populate the model input data byte array. void *input_callback_data, // User data structure passed to input callback. libtf_output_data_callback_t output_callback, // Callback to use the model output data byte array. diff --git a/src/libtf/cortex-m7/libtf_person_detect_model_data.a b/src/libtf/cortex-m7/libtf_person_detect_model_data.a index fa3ce65d5..f94e85e2a 100644 Binary files a/src/libtf/cortex-m7/libtf_person_detect_model_data.a and b/src/libtf/cortex-m7/libtf_person_detect_model_data.a differ diff --git a/src/omv/boards/OPENMV4/omv_boardconfig.h b/src/omv/boards/OPENMV4/omv_boardconfig.h index 756046998..f8404038c 100644 --- a/src/omv/boards/OPENMV4/omv_boardconfig.h +++ b/src/omv/boards/OPENMV4/omv_boardconfig.h @@ -120,7 +120,7 @@ #define OMV_FB_SIZE (400K) // FB memory: header + VGA/GS image #define OMV_FB_ALLOC_SIZE (96K) // minimum fb alloc size #define OMV_STACK_SIZE (15K) -#define OMV_HEAP_SIZE (230K) +#define OMV_HEAP_SIZE (229K) #define OMV_LINE_BUF_SIZE (3K) // Image line buffer round(640 * 2BPP * 2 buffers). #define OMV_MSC_BUF_SIZE (12K) // USB MSC bot data diff --git a/src/omv/boards/OPENMV4P/omv_boardconfig.h b/src/omv/boards/OPENMV4P/omv_boardconfig.h index b3fbcacfd..62a76daa5 100644 --- a/src/omv/boards/OPENMV4P/omv_boardconfig.h +++ b/src/omv/boards/OPENMV4P/omv_boardconfig.h @@ -124,7 +124,7 @@ #define OMV_FB_SIZE (30M) // FB memory: header + VGA/GS image #define OMV_FB_ALLOC_SIZE (1M) // minimum fb alloc size #define OMV_STACK_SIZE (15K) -#define OMV_HEAP_SIZE (230K) +#define OMV_HEAP_SIZE (229K) #define OMV_SDRAM_SIZE (32 * 1024 * 1024) // This needs to be here for UVC firmware. #define OMV_SDRAM_TEST (0) diff --git a/src/omv/py/py_tf.c b/src/omv/py/py_tf.c index 451b6062e..aedc1fa24 100644 --- a/src/omv/py/py_tf.c +++ b/src/omv/py/py_tf.c @@ -28,17 +28,21 @@ typedef struct py_tf_model_obj { mp_obj_base_t base; unsigned char *model_data; unsigned int model_data_len, height, width, channels; + bool signed_or_unsigned; + bool is_float; } py_tf_model_obj_t; STATIC void py_tf_model_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) { py_tf_model_obj_t *self = self_in; mp_printf(print, - "{\"len\":%d, \"height\":%d, \"width\":%d, \"channels\":%d}", + "{\"len\":%d, \"height\":%d, \"width\":%d, \"channels\":%d, \"signed\":%d, \"is_float\":%d}", self->model_data_len, self->height, self->width, - self->channels); + self->channels, + self->signed_or_unsigned, + self->is_float); } // TF Classification Object @@ -163,7 +167,9 @@ STATIC mp_obj_t int_py_tf_load(mp_obj_t path_obj, bool alloc_mode, bool helper_m tensor_arena_size, &tf_model->height, &tf_model->width, - &tf_model->channels), + &tf_model->channels, + &tf_model->signed_or_unsigned, + &tf_model->is_float), py_tf_putchar_buffer - (PY_TF_PUTCHAR_BUFFER_LEN - py_tf_putchar_buffer_len)); fb_free(); // free fb_alloc_all() @@ -211,12 +217,16 @@ typedef struct py_tf_input_data_callback_data { } py_tf_input_data_callback_data_t; STATIC void py_tf_input_data_callback(void *callback_data, - unsigned char *model_input, + void *model_input, const unsigned int input_height, const unsigned int input_width, - const unsigned int input_channels) + const unsigned int input_channels, + const bool signed_or_unsigned, + const bool is_float) { py_tf_input_data_callback_data_t *arg = (py_tf_input_data_callback_data_t *) callback_data; + int shift = signed_or_unsigned ? 128 : 0; + float fscale = 1.0f / (signed_or_unsigned ? 128.0f: 255.0f); float xscale = input_width / ((float) arg->roi->w); float yscale = input_height / ((float) arg->roi->h); @@ -235,15 +245,25 @@ STATIC void py_tf_input_data_callback(void *callback_data, int index = row + x; switch (input_channels) { case 1: { - model_input[index] = COLOR_BINARY_TO_GRAYSCALE(pixel); + if (!is_float) { + ((uint8_t *) model_input)[index] = COLOR_BINARY_TO_GRAYSCALE(pixel) ^ shift; + } else { + ((float *) model_input)[index] = (COLOR_BINARY_TO_GRAYSCALE(pixel) - shift) * fscale; + } break; } case 3: { int index_3 = index * 3; pixel = COLOR_BINARY_TO_RGB565(pixel); - model_input[index_3 + 0] = COLOR_RGB565_TO_R8(pixel); - model_input[index_3 + 1] = COLOR_RGB565_TO_G8(pixel); - model_input[index_3 + 2] = COLOR_RGB565_TO_B8(pixel); + if (!is_float) { + ((uint8_t *) model_input)[index_3 + 0] = COLOR_RGB565_TO_R8(pixel) ^ shift; + ((uint8_t *) model_input)[index_3 + 1] = COLOR_RGB565_TO_G8(pixel) ^ shift; + ((uint8_t *) model_input)[index_3 + 2] = COLOR_RGB565_TO_B8(pixel) ^ shift; + } else { + ((float *) model_input)[index_3 + 0] = (COLOR_RGB565_TO_R8(pixel) - shift) * fscale; + ((float *) model_input)[index_3 + 1] = (COLOR_RGB565_TO_G8(pixel) - shift) * fscale; + ((float *) model_input)[index_3 + 2] = (COLOR_RGB565_TO_B8(pixel) - shift) * fscale; + } break; } default: { @@ -263,15 +283,25 @@ STATIC void py_tf_input_data_callback(void *callback_data, int index = row + x; switch (input_channels) { case 1: { - model_input[index] = pixel; + if (!is_float) { + ((uint8_t *) model_input)[index] = pixel ^ shift; + } else { + ((float *) model_input)[index] = (pixel - shift) * fscale; + } break; } case 3: { int index_3 = index * 3; pixel = COLOR_GRAYSCALE_TO_RGB565(pixel); - model_input[index_3 + 0] = COLOR_RGB565_TO_R8(pixel); - model_input[index_3 + 1] = COLOR_RGB565_TO_G8(pixel); - model_input[index_3 + 2] = COLOR_RGB565_TO_B8(pixel); + if (!is_float) { + ((uint8_t *) model_input)[index_3 + 0] = COLOR_RGB565_TO_R8(pixel) ^ shift; + ((uint8_t *) model_input)[index_3 + 1] = COLOR_RGB565_TO_G8(pixel) ^ shift; + ((uint8_t *) model_input)[index_3 + 2] = COLOR_RGB565_TO_B8(pixel) ^ shift; + } else { + ((float *) model_input)[index_3 + 0] = (COLOR_RGB565_TO_R8(pixel) - shift) * fscale; + ((float *) model_input)[index_3 + 1] = (COLOR_RGB565_TO_G8(pixel) - shift) * fscale; + ((float *) model_input)[index_3 + 2] = (COLOR_RGB565_TO_B8(pixel) - shift) * fscale; + } break; } default: { @@ -291,14 +321,24 @@ STATIC void py_tf_input_data_callback(void *callback_data, int index = row + x; switch (input_channels) { case 1: { - model_input[index] = COLOR_RGB565_TO_GRAYSCALE(pixel); + if (!is_float) { + ((uint8_t *) model_input)[index] = COLOR_RGB565_TO_GRAYSCALE(pixel) ^ shift; + } else { + ((float *) model_input)[index] = (COLOR_RGB565_TO_GRAYSCALE(pixel) - shift) * fscale; + } break; } case 3: { int index_3 = index * 3; - model_input[index_3 + 0] = COLOR_RGB565_TO_R8(pixel); - model_input[index_3 + 1] = COLOR_RGB565_TO_G8(pixel); - model_input[index_3 + 2] = COLOR_RGB565_TO_B8(pixel); + if (!is_float) { + ((uint8_t *) model_input)[index_3 + 0] = COLOR_RGB565_TO_R8(pixel) ^ shift; + ((uint8_t *) model_input)[index_3 + 1] = COLOR_RGB565_TO_G8(pixel) ^ shift; + ((uint8_t *) model_input)[index_3 + 2] = COLOR_RGB565_TO_B8(pixel) ^ shift; + } else { + ((float *) model_input)[index_3 + 0] = (COLOR_RGB565_TO_R8(pixel) - shift) * fscale; + ((float *) model_input)[index_3 + 1] = (COLOR_RGB565_TO_G8(pixel) - shift) * fscale; + ((float *) model_input)[index_3 + 2] = (COLOR_RGB565_TO_B8(pixel) - shift) * fscale; + } break; } default: { @@ -320,19 +360,27 @@ typedef struct py_tf_classify_output_data_callback_data { } py_tf_classify_output_data_callback_data_t; STATIC void py_tf_classify_output_data_callback(void *callback_data, - unsigned char *model_output, + void *model_output, const unsigned int output_height, const unsigned int output_width, - const unsigned int output_channels) + const unsigned int output_channels, + const bool signed_or_unsigned, + const bool is_float) { py_tf_classify_output_data_callback_data_t *arg = (py_tf_classify_output_data_callback_data_t *) callback_data; + int shift = signed_or_unsigned ? 128 : 0; + float fscale = signed_or_unsigned ? 127.0f: 255.0f; PY_ASSERT_TRUE_MSG(output_height == 1, "Expected model output height to be 1!"); PY_ASSERT_TRUE_MSG(output_width == 1, "Expected model output width to be 1!"); arg->out = mp_obj_new_list(output_channels, NULL); for (unsigned int i = 0; i < output_channels; i++) { - ((mp_obj_list_t *) arg->out)->items[i] = mp_obj_new_float(model_output[i] / 255.0f); + if (!is_float) { + ((mp_obj_list_t *) arg->out)->items[i] = mp_obj_new_float((((uint8_t *) model_output)[i] ^ shift) / 255.0f); + } else { + ((mp_obj_list_t *) arg->out)->items[i] = mp_obj_new_float(((((float *) model_output)[i] * fscale) + shift) / 255.0f); + } } } @@ -422,12 +470,16 @@ typedef struct py_tf_segment_output_data_callback_data { } py_tf_segment_output_data_callback_data_t; STATIC void py_tf_segment_output_data_callback(void *callback_data, - unsigned char *model_output, + void *model_output, const unsigned int output_height, const unsigned int output_width, - const unsigned int output_channels) + const unsigned int output_channels, + const bool signed_or_unsigned, + const bool is_float) { py_tf_segment_output_data_callback_data_t *arg = (py_tf_segment_output_data_callback_data_t *) callback_data; + int shift = signed_or_unsigned ? 128 : 0; + float fscale = signed_or_unsigned ? 127.0f: 255.0f; arg->out = mp_obj_new_list(output_channels, NULL); for (unsigned int i = 0; i < output_channels; i++) { @@ -443,7 +495,11 @@ STATIC void py_tf_segment_output_data_callback(void *callback_data, uint8_t *row_ptr = IMAGE_COMPUTE_GRAYSCALE_PIXEL_ROW_PTR(&img, y); for (unsigned int x = 0; i < output_width; x++) { unsigned int col = x * output_channels; - IMAGE_PUT_GRAYSCALE_PIXEL_FAST(row_ptr, x, model_output[row + col + i]); + if (!is_float) { + IMAGE_PUT_GRAYSCALE_PIXEL_FAST(row_ptr, x, ((uint8_t *) model_output)[row + col + i] ^ shift); + } else { + IMAGE_PUT_GRAYSCALE_PIXEL_FAST(row_ptr, x, ((((float *) model_output)[i] * fscale) + shift)); + } } } } @@ -488,17 +544,23 @@ mp_obj_t py_tf_len(mp_obj_t self_in) { return mp_obj_new_int(((py_tf_model_obj_t mp_obj_t py_tf_height(mp_obj_t self_in) { return mp_obj_new_int(((py_tf_model_obj_t *) self_in)->height); } mp_obj_t py_tf_width(mp_obj_t self_in) { return mp_obj_new_int(((py_tf_model_obj_t *) self_in)->width); } mp_obj_t py_tf_channels(mp_obj_t self_in) { return mp_obj_new_int(((py_tf_model_obj_t *) self_in)->channels); } +mp_obj_t py_tf_signed(mp_obj_t self_in) { return mp_obj_new_int(((py_tf_model_obj_t *) self_in)->signed_or_unsigned); } +mp_obj_t py_tf_is_float(mp_obj_t self_in) { return mp_obj_new_int(((py_tf_model_obj_t *) self_in)->is_float); } STATIC MP_DEFINE_CONST_FUN_OBJ_1(py_tf_len_obj, py_tf_len); STATIC MP_DEFINE_CONST_FUN_OBJ_1(py_tf_height_obj, py_tf_height); STATIC MP_DEFINE_CONST_FUN_OBJ_1(py_tf_width_obj, py_tf_width); STATIC MP_DEFINE_CONST_FUN_OBJ_1(py_tf_channels_obj, py_tf_channels); +STATIC MP_DEFINE_CONST_FUN_OBJ_1(py_tf_signed_obj, py_tf_signed); +STATIC MP_DEFINE_CONST_FUN_OBJ_1(py_tf_is_float_obj, py_tf_is_float); STATIC const mp_rom_map_elem_t locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_len), MP_ROM_PTR(&py_tf_len_obj) }, { MP_ROM_QSTR(MP_QSTR_height), MP_ROM_PTR(&py_tf_height_obj) }, { MP_ROM_QSTR(MP_QSTR_width), MP_ROM_PTR(&py_tf_width_obj) }, { MP_ROM_QSTR(MP_QSTR_channels), MP_ROM_PTR(&py_tf_channels_obj) }, + { MP_ROM_QSTR(MP_QSTR_signed), MP_ROM_PTR(&py_tf_signed_obj) }, + { MP_ROM_QSTR(MP_QSTR_is_float), MP_ROM_PTR(&py_tf_is_float_obj) }, { MP_ROM_QSTR(MP_QSTR_classify), MP_ROM_PTR(&py_tf_classify_obj) }, { MP_ROM_QSTR(MP_QSTR_segment), MP_ROM_PTR(&py_tf_segment_obj) } }; diff --git a/src/omv/py/qstrdefsomv.h b/src/omv/py/qstrdefsomv.h index 501a2d30c..50b5adf11 100644 --- a/src/omv/py/qstrdefsomv.h +++ b/src/omv/py/qstrdefsomv.h @@ -1217,6 +1217,8 @@ Q(tf_model) // duplicate Q(height) // duplicate Q(width) Q(channels) +Q(signed) +Q(is_float) // Classify // duplicate Q(classify)