Libtf: Add support for replacing/adding built-in models.

This commit is contained in:
iabdalkader 2022-02-08 01:44:23 +02:00
parent 2ee2a163b7
commit 9de1f50a65
5 changed files with 181 additions and 36 deletions

View File

@ -34,6 +34,7 @@ MKDIR = $(Q)mkdir
ECHO = $(Q)@echo
MAKE = $(Q)make
CAT = $(Q)cat
TFLITE2C = tflite2c.py
# Targets
TARGET ?= OPENMV4
@ -43,6 +44,7 @@ FIRMWARE = firmware
# Directories
TOP_DIR=$(shell pwd)
BUILD=$(TOP_DIR)/build
TOOLS=$(TOP_DIR)/../tools
FW_DIR=$(BUILD)/bin
OMV_DIR=omv
UVC_DIR=uvc

32
src/lib/libtf/Makefile Normal file
View File

@ -0,0 +1,32 @@
# This file is part of the OpenMV project.
#
# Copyright (c) 2013-2021 Ibrahim Abdelkader <iabdalkader@openmv.io>
# Copyright (c) 2013-2021 Kwabena W. Agyeman <kwagyeman@openmv.io>
#
# This work is licensed under the MIT license, see the file LICENSE for details.
#
# LIBTF Makefile
GENERATED := $(BUILD)/libtf_builtin_models.h $(BUILD)/libtf_builtin_models.c
OBJS = $(BUILD)/libtf_builtin_models.o
OBJ_DIRS = $(sort $(dir $(OBJS)))
all: | $(OBJ_DIRS) $(OBJS)
$(OBJ_DIRS):
$(MKDIR) -p $@
$(GENERATED): $(wildcard models/*)
$(ECHO) "GEN $@"
$(PYTHON) $(TOOLS)/$(TFLITE2C) --input models > $(BUILD)/libtf_builtin_models.c
$(PYTHON) $(TOOLS)/$(TFLITE2C) --input models --header > $(BUILD)/libtf_builtin_models.h
$(OBJS): $(GENERATED)
$(BUILD)/%.o : %.c
$(ECHO) "CC $<"
$(CC) $(CFLAGS) -c -o $@ $<
$(BUILD)/%.o : %.s
$(ECHO) "AS $<"
$(AS) $(AFLAGS) $< -o $@
-include $(OBJS:%.o=%.d)

View File

@ -20,6 +20,7 @@
#include "py_image.h"
#include "ff_wrapper.h"
#include "py_tf.h"
#include "libtf_builtin_models.h"
#define GRAYSCALE_RANGE ((COLOR_GRAYSCALE_MAX) - (COLOR_GRAYSCALE_MIN))
#define GRAYSCALE_MID (((GRAYSCALE_RANGE) + 1) / 2)
@ -155,11 +156,17 @@ STATIC mp_obj_t int_py_tf_load(mp_obj_t path_obj, bool alloc_mode, bool helper_m
const char *path = mp_obj_str_get_str(path_obj);
py_tf_model_obj_t *tf_model = m_new_obj(py_tf_model_obj_t);
tf_model->base.type = &py_tf_model_type;
tf_model->model_data = NULL;
if (!strcmp(path, "person_detection")) {
tf_model->model_data = (unsigned char *) g_person_detect_model_data;
tf_model->model_data_len = g_person_detect_model_data_len;
} else {
for (int i=0; i<MP_ARRAY_SIZE(libtf_builtin_models); i++) {
const libtf_builtin_model_t *model = &libtf_builtin_models[i];
if (!strcmp(path, model->name)) {
tf_model->model_data = (unsigned char *) model->data;
tf_model->model_data_len = model->size;
}
}
if (tf_model->model_data == NULL) {
#if defined(IMLIB_ENABLE_IMAGE_FILE_IO)
FIL fp;
file_read_open(&fp, path);
@ -207,12 +214,31 @@ STATIC mp_obj_t int_py_tf_load(mp_obj_t path_obj, bool alloc_mode, bool helper_m
STATIC mp_obj_t py_tf_load(uint n_args, const mp_obj_t *args, mp_map_t *kw_args)
{
return int_py_tf_load(args[0],
py_helper_keyword_int(n_args, args, 1, kw_args, MP_OBJ_NEW_QSTR(MP_QSTR_load_to_fb), false),
false);
bool alloc_mode = py_helper_keyword_int(n_args, args, 1, kw_args, MP_OBJ_NEW_QSTR(MP_QSTR_load_to_fb), false);
return int_py_tf_load(args[0], alloc_mode, false);
}
STATIC MP_DEFINE_CONST_FUN_OBJ_KW(py_tf_load_obj, 1, py_tf_load);
STATIC mp_obj_t py_tf_load_builtin_model(mp_obj_t path_obj)
{
mp_obj_t net = int_py_tf_load(path_obj, false, false);
const char *path = mp_obj_str_get_str(path_obj);
mp_obj_t labels = mp_obj_new_list(0, NULL);
for (int i=0; i<MP_ARRAY_SIZE(libtf_builtin_models); i++) {
const libtf_builtin_model_t *model = &libtf_builtin_models[i];
if (!strcmp(path, model->name)) {
for (int l=0; l < model->n_labels; l++) {
const char *label = model->labels[l];
mp_obj_list_append(labels, mp_obj_new_str(label, strlen(label)));
}
break;
}
}
return mp_obj_new_tuple(2, (mp_obj_t []) {labels, net});
}
STATIC MP_DEFINE_CONST_FUN_OBJ_1(py_tf_load_builtin_model_obj, py_tf_load_builtin_model);
STATIC mp_obj_t py_tf_free_from_fb()
{
fb_alloc_free_till_mark_past_mark_permanent();
@ -775,17 +801,19 @@ STATIC const mp_obj_type_t py_tf_model_type = {
STATIC const mp_rom_map_elem_t globals_dict_table[] = {
{ MP_ROM_QSTR(MP_QSTR___name__), MP_OBJ_NEW_QSTR(MP_QSTR_tf) },
#ifdef IMLIB_ENABLE_TF
{ MP_ROM_QSTR(MP_QSTR_load), MP_ROM_PTR(&py_tf_load_obj) },
{ MP_ROM_QSTR(MP_QSTR_free_from_fb), MP_ROM_PTR(&py_tf_free_from_fb_obj) },
{ MP_ROM_QSTR(MP_QSTR_classify), MP_ROM_PTR(&py_tf_classify_obj) },
{ MP_ROM_QSTR(MP_QSTR_segment), MP_ROM_PTR(&py_tf_segment_obj) },
{ MP_ROM_QSTR(MP_QSTR_detect), MP_ROM_PTR(&py_tf_detect_obj) },
{ MP_ROM_QSTR(MP_QSTR_load), MP_ROM_PTR(&py_tf_load_obj) },
{ MP_ROM_QSTR(MP_QSTR_load_builtin_model), MP_ROM_PTR(&py_tf_load_builtin_model_obj) },
{ MP_ROM_QSTR(MP_QSTR_free_from_fb), MP_ROM_PTR(&py_tf_free_from_fb_obj) },
{ MP_ROM_QSTR(MP_QSTR_classify), MP_ROM_PTR(&py_tf_classify_obj) },
{ MP_ROM_QSTR(MP_QSTR_segment), MP_ROM_PTR(&py_tf_segment_obj) },
{ MP_ROM_QSTR(MP_QSTR_detect), MP_ROM_PTR(&py_tf_detect_obj) },
#else
{ MP_ROM_QSTR(MP_QSTR_load), MP_ROM_PTR(&py_func_unavailable_obj) },
{ MP_ROM_QSTR(MP_QSTR_free_from_fb), MP_ROM_PTR(&py_func_unavailable_obj) },
{ MP_ROM_QSTR(MP_QSTR_classify), MP_ROM_PTR(&py_func_unavailable_obj) },
{ MP_ROM_QSTR(MP_QSTR_segment), MP_ROM_PTR(&py_func_unavailable_obj) },
{ MP_ROM_QSTR(MP_QSTR_detect), MP_ROM_PTR(&py_func_unavailable_obj) }
{ MP_ROM_QSTR(MP_QSTR_load), MP_ROM_PTR(&py_func_unavailable_obj) },
{ MP_ROM_QSTR(MP_QSTR_load_builtin_model), MP_ROM_PTR(&py_func_unavailable_obj) },
{ MP_ROM_QSTR(MP_QSTR_free_from_fb), MP_ROM_PTR(&py_func_unavailable_obj) },
{ MP_ROM_QSTR(MP_QSTR_classify), MP_ROM_PTR(&py_func_unavailable_obj) },
{ MP_ROM_QSTR(MP_QSTR_segment), MP_ROM_PTR(&py_func_unavailable_obj) },
{ MP_ROM_QSTR(MP_QSTR_detect), MP_ROM_PTR(&py_func_unavailable_obj) }
#endif // IMLIB_ENABLE_TF
};

View File

@ -46,7 +46,8 @@ OMV_CFLAGS += -I$(TOP_DIR)/$(MLX90621_DIR)/include/
OMV_CFLAGS += -I$(TOP_DIR)/$(MLX90640_DIR)/include/
OMV_CFLAGS += -I$(TOP_DIR)/$(MLX90641_DIR)/include/
OMV_CFLAGS += -I$(TOP_DIR)/$(PIXART_DIR)/include/
OMV_CFLAGS += -I$(TOP_DIR)/$(TENSORFLOW_DIR)/$(CPU)/
OMV_CFLAGS += -I$(TOP_DIR)/$(TENSORFLOW_DIR)/
OMV_CFLAGS += -I$(BUILD)/$(TENSORFLOW_DIR)/
OMV_CFLAGS += -I$(TOP_DIR)/$(LIBPDM_DIR)/
ifeq ($(OMV_ENABLE_BL), 1)
@ -216,11 +217,12 @@ FIRM_OBJ += $(addprefix $(BUILD)/$(OMV_DIR)/imlib/, \
zbar.o \
)
FIRM_OBJ += $(wildcard $(BUILD)/$(TENSORFLOW_DIR)/*.o)
FIRM_OBJ += $(wildcard $(BUILD)/$(OMV_DIR)/ports/$(PORT)/*.o)
FIRM_OBJ += $(wildcard $(BUILD)/$(MICROPY_DIR)/modules/*.o)
FIRM_OBJ += $(wildcard $(BUILD)/$(MICROPY_DIR)/ports/$(PORT)/modules/*.o)
#------------- MicroPy Objects -------------------#
FIRM_OBJ += $(wildcard $(BUILD)/$(MICROPY_DIR)/modules/*.o)
FIRM_OBJ += $(wildcard $(BUILD)/$(MICROPY_DIR)/ports/$(PORT)/modules/*.o)
FIRM_OBJ += $(wildcard $(BUILD)/$(MICROPY_DIR)/py/*.o)
FIRM_OBJ += $(wildcard $(BUILD)/$(MICROPY_DIR)/boards/$(TARGET)/*.o)
@ -582,6 +584,7 @@ export CFLAGS
export LDFLAGS
export TOP_DIR
export BUILD
export TOOLS
export TARGET
export STARTUP
export SYSTEM
@ -589,6 +592,8 @@ export FROZEN_MANIFEST
export PORT
export HAL_DIR
export CMSIS_DIR
export PYTHON
export TFLITE2C
###################################################
all: $(OPENMV)
@ -599,36 +604,37 @@ $(FW_DIR):
$(MKDIR) -p $@
FIRMWARE_OBJS: | $(BUILD) $(FW_DIR)
$(MAKE) -C $(CMSIS_DIR) BUILD=$(BUILD)/$(CMSIS_DIR) CFLAGS="$(CFLAGS) -fno-strict-aliasing -MMD"
$(MAKE) -C $(MICROPY_DIR)/ports/$(PORT) BUILD=$(BUILD)/$(MICROPY_DIR) $(MICROPY_ARGS)
$(MAKE) -C $(HAL_DIR) BUILD=$(BUILD)/$(HAL_DIR) CFLAGS="$(CFLAGS) -MMD"
$(MAKE) -C $(LEPTON_DIR) BUILD=$(BUILD)/$(LEPTON_DIR) CFLAGS="$(CFLAGS) -MMD"
$(MAKE) -C $(CMSIS_DIR) BUILD=$(BUILD)/$(CMSIS_DIR) CFLAGS="$(CFLAGS) -fno-strict-aliasing -MMD"
$(MAKE) -C $(TENSORFLOW_DIR) BUILD=$(BUILD)/$(TENSORFLOW_DIR) CFLAGS="$(CFLAGS) -MMD"
$(MAKE) -C $(MICROPY_DIR)/ports/$(PORT) BUILD=$(BUILD)/$(MICROPY_DIR) $(MICROPY_ARGS)
$(MAKE) -C $(HAL_DIR) BUILD=$(BUILD)/$(HAL_DIR) CFLAGS="$(CFLAGS) -MMD"
$(MAKE) -C $(LEPTON_DIR) BUILD=$(BUILD)/$(LEPTON_DIR) CFLAGS="$(CFLAGS) -MMD"
ifeq ($(MICROPY_PY_IMU), 1)
$(MAKE) -C $(LSM6DS3_DIR) BUILD=$(BUILD)/$(LSM6DS3_DIR) CFLAGS="$(CFLAGS) -MMD"
$(MAKE) -C $(LSM6DSOX_DIR) BUILD=$(BUILD)/$(LSM6DSOX_DIR) CFLAGS="$(CFLAGS) -MMD"
$(MAKE) -C $(LSM6DS3_DIR) BUILD=$(BUILD)/$(LSM6DS3_DIR) CFLAGS="$(CFLAGS) -MMD"
$(MAKE) -C $(LSM6DSOX_DIR) BUILD=$(BUILD)/$(LSM6DSOX_DIR) CFLAGS="$(CFLAGS) -MMD"
endif
ifeq ($(MICROPY_PY_WINC1500), 1)
$(MAKE) -C $(WINC1500_DIR) BUILD=$(BUILD)/$(WINC1500_DIR) CFLAGS="$(CFLAGS) -MMD"
$(MAKE) -C $(WINC1500_DIR) BUILD=$(BUILD)/$(WINC1500_DIR) CFLAGS="$(CFLAGS) -MMD"
endif
$(MAKE) -C $(MLX90621_DIR) BUILD=$(BUILD)/$(MLX90621_DIR) CFLAGS="$(CFLAGS) -MMD"
$(MAKE) -C $(MLX90640_DIR) BUILD=$(BUILD)/$(MLX90640_DIR) CFLAGS="$(CFLAGS) -MMD"
$(MAKE) -C $(MLX90641_DIR) BUILD=$(BUILD)/$(MLX90641_DIR) CFLAGS="$(CFLAGS) -MMD"
$(MAKE) -C $(PIXART_DIR) BUILD=$(BUILD)/$(PIXART_DIR) CFLAGS="$(CFLAGS) -MMD"
$(MAKE) -C $(OMV_DIR) BUILD=$(BUILD)/$(OMV_DIR) CFLAGS="$(CFLAGS) -MMD"
$(MAKE) -C $(MLX90621_DIR) BUILD=$(BUILD)/$(MLX90621_DIR) CFLAGS="$(CFLAGS) -MMD"
$(MAKE) -C $(MLX90640_DIR) BUILD=$(BUILD)/$(MLX90640_DIR) CFLAGS="$(CFLAGS) -MMD"
$(MAKE) -C $(MLX90641_DIR) BUILD=$(BUILD)/$(MLX90641_DIR) CFLAGS="$(CFLAGS) -MMD"
$(MAKE) -C $(PIXART_DIR) BUILD=$(BUILD)/$(PIXART_DIR) CFLAGS="$(CFLAGS) -MMD"
$(MAKE) -C $(OMV_DIR) BUILD=$(BUILD)/$(OMV_DIR) CFLAGS="$(CFLAGS) -MMD"
ifeq ($(CUBEAI), 1)
$(MAKE) -C $(CUBEAI_DIR) BUILD=$(BUILD)/$(CUBEAI_DIR) CFLAGS="$(CFLAGS) -fno-strict-aliasing -MMD"
$(MAKE) -C $(CUBEAI_DIR) BUILD=$(BUILD)/$(CUBEAI_DIR) CFLAGS="$(CFLAGS) -fno-strict-aliasing -MMD"
endif
ifeq ($(OMV_ENABLE_UVC), 1)
UVC_OBJS: FIRMWARE_OBJS
$(MAKE) -C $(UVC_DIR) BUILD=$(BUILD)/$(UVC_DIR) CFLAGS="$(UVC_CFLAGS) -MMD"
$(MAKE) -C $(UVC_DIR) BUILD=$(BUILD)/$(UVC_DIR) CFLAGS="$(UVC_CFLAGS) -MMD"
endif
ifeq ($(OMV_ENABLE_CM4), 1)
CM4_OBJS: FIRMWARE_OBJS
$(MAKE) -C $(CM4_DIR) BUILD=$(BUILD)/$(CM4_DIR) CFLAGS="$(CM4_CFLAGS) -MMD"
$(MAKE) -C $(CM4_DIR) BUILD=$(BUILD)/$(CM4_DIR) CFLAGS="$(CM4_CFLAGS) -MMD"
endif
ifeq ($(OMV_ENABLE_BL), 1)
BOOTLOADER_OBJS: FIRMWARE_OBJS
$(MAKE) -C $(BOOTLDR_DIR) BUILD=$(BUILD)/$(BOOTLDR_DIR) CFLAGS="$(BL_CFLAGS) -MMD"
$(MAKE) -C $(BOOTLDR_DIR) BUILD=$(BUILD)/$(BOOTLDR_DIR) CFLAGS="$(BL_CFLAGS) -MMD"
endif
# This target generates the main/app firmware image located at 0x08010000

77
tools/tflite2c.py Executable file
View File

@ -0,0 +1,77 @@
#!/usr/bin/env python3
# This file is part of the OpenMV project.
#
# Copyright (c) 2013-2022 Ibrahim Abdelkader <iabdalkader@openmv.io>
# Copyright (c) 2013-2022 Kwabena W. Agyeman <kwagyeman@openmv.io>
#
# This work is licensed under the MIT license, see the file LICENSE for details.
#
# This script converts tflite models and labels to a C structs.
import sys, os
import glob
import argparse
import binascii
def main():
parser = argparse.ArgumentParser(description='Converts TFLite models to C file.')
parser.add_argument('--input', action = 'store', help = 'Input tflite models directory.', required=True)
parser.add_argument('--header', action = 'store_true', help = 'Generate header file.', required=False, default=False)
args = parser.parse_args()
libtf_builtin_models = []
print('/* NOTE: This file is auto-generated. */\n')
models_list = glob.glob(os.path.join(args.input, "*tflite"))
if (args.header):
# Generate the header file
print('// Built-in TFLite Models.')
print('typedef struct {')
print(' const char *name;')
print(' const unsigned int n_labels;')
print(' const char **labels;')
print(' const unsigned int size;')
print(' const unsigned char *data;')
print('}libtf_builtin_model_t;\n')
print('extern const libtf_builtin_model_t libtf_builtin_models[{:d}];'.format(len(models_list)))
else:
# Generate the C file
print('#include "libtf_builtin_models.h"')
for model_file in models_list:
model_size = os.path.getsize(model_file)
model_name = os.path.basename(os.path.splitext(model_file)[0])
labels_file = os.path.splitext(model_file)[0]+'.txt'
# Generate model labels.
labels = []
n_labels = 0
with open(labels_file, 'r') as f:
labels = ['"{:s}"'.format(l.strip()) for l in f.readlines()]
n_labels = len(labels)
print('const char *libtf_{:s}_labels[] = {{{:s}}};'.format(model_name, ', '.join(labels)))
# Generate model data.
print('const unsigned char libtf_{:s}_data[] = {{'.format(model_name))
with open(model_file, 'rb') as f:
for chunk in iter(lambda: f.read(12), b''):
print(' ', end='')
print(' '.join(['0x{:02x},'.format(x) for x in chunk]))
print('};')
# Store model info in builtin models table.
libtf_builtin_models.append([
model_name,
n_labels,
'libtf_{:s}_labels'.format(model_name),
model_size,
'libtf_{:s}_data'.format(model_name)]
)
# Generate built-in models table.
print('const libtf_builtin_model_t libtf_builtin_models[{:d}] = {{'.format(len(models_list)))
for model in libtf_builtin_models:
print(' {{"{:s}", {:d}, {:s}, {:d}, {:s}}},'.format(*model))
print('};')
if __name__ == '__main__':
main()