From efca0603a7a3678837b4bcae074bd92ca5c05991 Mon Sep 17 00:00:00 2001 From: iabdalkader Date: Tue, 10 Nov 2020 22:44:53 +0200 Subject: [PATCH] Add micro speech module. --- src/Makefile | 5 + src/micropython | 2 +- src/omv/Makefile | 1 + src/omv/boards/PORTENTA/omv_boardconfig.mk | 1 + src/omv/py/py_micro_speech.c | 130 +++++++++++++++++++++ src/omv/py/qstrdefsomv.h | 3 + 6 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 src/omv/py/py_micro_speech.c diff --git a/src/Makefile b/src/Makefile index f0ff54bcb..5d6d66955 100755 --- a/src/Makefile +++ b/src/Makefile @@ -149,6 +149,10 @@ ifeq ($(MICROPY_PY_AUDIO), 1) MP_CFLAGS += -DMICROPY_PY_AUDIO=1 MICROPY_ARGS += MICROPY_PY_AUDIO=1 endif +ifeq ($(MICROPY_PY_MICRO_SPEECH), 1) +MP_CFLAGS += -DMICROPY_PY_MICRO_SPEECH=1 +MICROPY_ARGS += MICROPY_PY_MICRO_SPEECH=1 +endif OMV_CFLAGS += -I$(TOP_DIR)/$(OMV_DIR)/ OMV_CFLAGS += -I$(TOP_DIR)/$(OMV_DIR)/py/ @@ -316,6 +320,7 @@ FIRM_OBJ += $(addprefix $(BUILD)/$(OMV_DIR)/py/, \ py_tf.o \ py_imu.o \ py_audio.o \ + py_micro_speech.o \ ) diff --git a/src/micropython b/src/micropython index a0d393a06..bea4a37fc 160000 --- a/src/micropython +++ b/src/micropython @@ -1 +1 @@ -Subproject commit a0d393a0627a7fd4825a3ae8f14c7eb5e2a70de5 +Subproject commit bea4a37fcc82eb2be64ed2cde61866faba50e8c4 diff --git a/src/omv/Makefile b/src/omv/Makefile index 93367a096..9f1ccbede 100644 --- a/src/omv/Makefile +++ b/src/omv/Makefile @@ -107,6 +107,7 @@ SRCS += $(addprefix py/, \ py_tf.c \ py_imu.c \ py_audio.c \ + py_micro_speech.c \ ) OBJS = $(addprefix $(BUILD)/, $(SRCS:.c=.o)) diff --git a/src/omv/boards/PORTENTA/omv_boardconfig.mk b/src/omv/boards/PORTENTA/omv_boardconfig.mk index f10b25a6d..ff2b6a3af 100755 --- a/src/omv/boards/PORTENTA/omv_boardconfig.mk +++ b/src/omv/boards/PORTENTA/omv_boardconfig.mk @@ -17,3 +17,4 @@ MICROPY_PY_WINC1500 = 0 MICROPY_PY_LWIP = 1 MICROPY_PY_NETWORK_CYW43 = 1 MICROPY_PY_AUDIO = 1 +MICROPY_PY_MICRO_SPEECH = 1 diff --git a/src/omv/py/py_micro_speech.c b/src/omv/py/py_micro_speech.c new file mode 100644 index 000000000..bcf0dc00d --- /dev/null +++ b/src/omv/py/py_micro_speech.c @@ -0,0 +1,130 @@ +/* + * This file is part of the OpenMV project. + * + * Copyright (c) 2013-2019 Ibrahim Abdelkader + * Copyright (c) 2013-2019 Kwabena W. Agyeman + * + * This work is licensed under the MIT license, see the file LICENSE for details. + * + * Micro Speech Python module. + */ +#include +#include "systick.h" +#include "py_assert.h" +#include "py_helper.h" +#include "py/binary.h" +#include "fb_alloc.h" +#include "omv_boardconfig.h" +#include "py/obj.h" +#include "py/objarray.h" +#include "libtf.h" + +#if MICROPY_PY_MICRO_SPEECH +#define kMaxAudioSampleSize (512) +#define kAudioSampleFrequency (16000) +// The following values are derived from values used during model training. +// If you change the way you preprocess the input, update all these constants. +#define kFeatureSliceSize (40) +#define kFeatureSliceCount (49) +#define kFeatureElementCount (kFeatureSliceSize * kFeatureSliceCount) +#define kFeatureSliceStrideMs (20) +#define kFeatureSliceDurationMs (30) + +#define RAISE_OS_EXCEPTION(msg) nlr_raise(mp_obj_new_exception_msg(&mp_type_OSError, msg)) +typedef struct _py_micro_speech_obj { + mp_obj_base_t base; + uint32_t n_slices; + int8_t spectrogram[kFeatureElementCount]; +} py_micro_speech_obj_t; + +static const mp_obj_type_t py_micro_speech_type; + +static void py_micro_speech_print(const mp_print_t *print, mp_obj_t self_in, mp_print_kind_t kind) +{ + //py_micro_speech_obj_t *microspeech = MP_OBJ_TO_PTR(self_in); + printf("micro speech object\n"); +} + +mp_obj_t py_micro_speech_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *args) +{ + // check arguments + //mp_arg_check_num(n_args, n_kw, 1, MP_OBJ_FUN_ARGS_MAX, true); + py_micro_speech_obj_t *o = m_new_obj(py_micro_speech_obj_t); + o->base.type = &py_micro_speech_type; + o->n_slices = 0; + memset(o->spectrogram, 0, kFeatureElementCount); + if (libtf_initialize_micro_features() != 0) { + RAISE_OS_EXCEPTION("Failed to initialize micro features!"); + } + return MP_OBJ_FROM_PTR(o); +} + +mp_obj_t py_micro_speech_audio_callback(mp_obj_t self_in, mp_obj_t buf_in) +{ + py_micro_speech_obj_t *microspeech = MP_OBJ_TO_PTR(self_in); + mp_buffer_info_t pcmbuf; + mp_get_buffer_raise(buf_in, &pcmbuf, MP_BUFFER_READ); + //size_t typesize = mp_binary_get_size('@', pcmbuf.typecode, NULL); + + if ((pcmbuf.len / 2) != kMaxAudioSampleSize) { + RAISE_OS_EXCEPTION("Audio data size too small!"); + } + + uint32_t slice_index = 0; + if (microspeech->n_slices < kFeatureSliceCount) { + slice_index = microspeech->n_slices++; + } else { + // Spectrogram is full, move old data up in the spectrogram + // and put the new slice at the end of the spectrogram. + // +-----------+ +-----------+ + // | data@20ms | --> | data@40ms | + // +-----------+ -- +-----------+ + // | data@40ms | -- --> | data@60ms | + // +-----------+ -- -- +-----------+ + // | data@60ms | -- -- | data@80ms | + // +-----------+ -- +-----------+ + // | data@80ms | -- || + // +-----------+ +-----------+ + slice_index = (kFeatureSliceCount - 1); + memmove(microspeech->spectrogram, + microspeech->spectrogram + kFeatureSliceSize, + kFeatureElementCount - kFeatureSliceSize); + } + printf("slice index %ld\n", slice_index); + size_t num_samples_read; + int8_t *new_slice = microspeech->spectrogram + (slice_index * kFeatureSliceSize); + if (libtf_generate_micro_features((int16_t*) pcmbuf.buf, + kMaxAudioSampleSize, kFeatureSliceSize, new_slice, &num_samples_read)) { + RAISE_OS_EXCEPTION("Feature generation failed!"); + } + return mp_const_none; +} +STATIC MP_DEFINE_CONST_FUN_OBJ_2(py_micro_speech_audio_callback_obj, py_micro_speech_audio_callback); + +STATIC const mp_rom_map_elem_t py_micro_speech_locals_dict_table[] = { + // instance methods + { MP_ROM_QSTR(MP_QSTR_audio_callback), MP_ROM_PTR(&py_micro_speech_audio_callback_obj) }, + // class constants +}; +STATIC MP_DEFINE_CONST_DICT(py_micro_speech_locals_dict, py_micro_speech_locals_dict_table); + +static const mp_obj_type_t py_micro_speech_type = { + { &mp_type_type }, + .name = MP_QSTR_MicroSpeech, + .print = py_micro_speech_print, + .make_new = py_micro_speech_make_new, + .locals_dict = (mp_obj_dict_t*)&py_micro_speech_locals_dict, +}; + +STATIC const mp_rom_map_elem_t module_globals_table[] = { + { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_micro_speech) }, + { MP_ROM_QSTR(MP_QSTR_MicroSpeech), MP_ROM_PTR(&py_micro_speech_type) }, +}; + +STATIC MP_DEFINE_CONST_DICT(module_globals, module_globals_table); + +const mp_obj_module_t micro_speech_module = { + .base = { &mp_type_module }, + .globals = (mp_obj_dict_t*)&module_globals, +}; +#endif //MICROPY_PY_MICRO_SPEECH diff --git a/src/omv/py/qstrdefsomv.h b/src/omv/py/qstrdefsomv.h index ba798b9c7..33427c039 100644 --- a/src/omv/py/qstrdefsomv.h +++ b/src/omv/py/qstrdefsomv.h @@ -1280,3 +1280,6 @@ Q(frequency) Q(read_pdm) Q(start_streaming) Q(stop_streaming) +Q(micro_speech) +Q(MicroSpeech) +Q(audio_callback)