From 8e34e31a2ae9ceed31c2e0c222124b4029b0c130 Mon Sep 17 00:00:00 2001 From: Zdzislaw Goik Date: Mon, 10 Apr 2023 20:36:12 +0200 Subject: [PATCH] feat: usb support for esp32s3 (#47) * fix: completed esp32s3 boards config * feat: enable usb cdc on boot * fix: memory_type has to be octal qio_opi for psram to init * fix: cannot flash from platformio without --no-stub flag * fix: fixed pinouts copy paste issues * feat: add system header to unify future traits * feat: add rudimentary usb cdc video streaming impl * feat: add rudimentary impl for usb tracker in main * fix: revert wrovers3 board names * feat: disable radio to reduce power usage * feat: add v2 serial comms with packet headers --------- authored-by: Sebastian Fitt --- ESP/ini/boards.ini | 20 +++--- ESP/ini/pinouts.ini | 36 +++++------ ESP/lib/src/etvr_system.hpp | 10 +++ ESP/lib/src/usb/etvr_eye_tracker_usb.cpp | 61 ++++++++++++++++++ ESP/lib/src/usb/etvr_eye_tracker_usb.hpp | 7 +++ ESP/src/main.cpp | 78 ++++++++++++++++-------- 6 files changed, 158 insertions(+), 54 deletions(-) create mode 100644 ESP/lib/src/etvr_system.hpp create mode 100644 ESP/lib/src/usb/etvr_eye_tracker_usb.cpp create mode 100644 ESP/lib/src/usb/etvr_eye_tracker_usb.hpp diff --git a/ESP/ini/boards.ini b/ESP/ini/boards.ini index f2ff379..345c02c 100644 --- a/ESP/ini/boards.ini +++ b/ESP/ini/boards.ini @@ -148,29 +148,29 @@ upload_flags = [env:wrovers3] board = esp32-s3-devkitc-1 -#board_build.f_flash = 80000000L -#board_build.flash_mode = qio +board_build.arduino.memory_type = qio_opi build_type = debug build_flags = ${env.build_flags} - -DCORE_DEBUG_LEVEL=4 ; set the debug level - -DDEBUG_MODE=1 ; Set the debug mode - ; CAMERA PINOUT DEFINITIONS + -DCORE_DEBUG_LEVEL=4 + -DDEBUG_MODE=1 + -DARDUINO_USB_CDC_ON_BOOT=1 ${pinoutsESPS3WROVER.build_flags} +upload_flags = --no-stub [env:wrovers3_release] board = esp32-s3-devkitc-1 -#board_build.f_flash = 80000000L -#board_build.flash_mode = qio +board_build.arduino.memory_type = qio_opi build_type = release build_flags = ${env.build_flags} - -DDEBUG_MODE=0 ; Set the debug mode + -DDEBUG_MODE=0 -DCORE_DEBUG_LEVEL=1 + -DARDUINO_USB_CDC_ON_BOOT=1 -DDEBUG_ESP_OTA - ; CAMERA PINOUT DEFINITIONS ${pinoutsESPS3WROVER.build_flags} +upload_flags = --no-stub [env:wrovers3_OTA] -extends = env:wrover_release +extends = env:wrovers3_release upload_port = ${ota.otaserverip} upload_protocol = espota upload_flags = diff --git a/ESP/ini/pinouts.ini b/ESP/ini/pinouts.ini index be62523..a054ca9 100644 --- a/ESP/ini/pinouts.ini +++ b/ESP/ini/pinouts.ini @@ -203,21 +203,21 @@ vsync_gpio_num = 6 href_gpio_num = 7 pclk_gpio_num = 13 build_flags = - '-DCAMERA_MODULE_NAME="ESP32_CAM_WROVER_KIT"' - -DCONFIG_CAMERA_MODULE_WROVER_KIT=1 - -DPWDN_GPIO_NUM=${pinoutsESPWROVER.PWDN_GPIO_NUM} - -DRESET_GPIO_NUM=${pinoutsESPWROVER.RESET_GPIO_NUM} - -DXCLK_GPIO_NUM=${pinoutsESPWROVER.XCLK_GPIO_NUM} - -DSIOD_GPIO_NUM=${pinoutsESPWROVER.SIOD_GPIO_NUM} - -DSIOC_GPIO_NUM=${pinoutsESPWROVER.SIOC_GPIO_NUM} - -DY9_GPIO_NUM=${pinoutsESPWROVER.Y9_GPIO_NUM} - -DY8_GPIO_NUM=${pinoutsESPWROVER.Y8_GPIO_NUM} - -DY7_GPIO_NUM=${pinoutsESPWROVER.Y7_GPIO_NUM} - -DY6_GPIO_NUM=${pinoutsESPWROVER.Y6_GPIO_NUM} - -DY5_GPIO_NUM=${pinoutsESPWROVER.Y5_GPIO_NUM} - -DY4_GPIO_NUM=${pinoutsESPWROVER.Y4_GPIO_NUM} - -DY3_GPIO_NUM=${pinoutsESPWROVER.Y3_GPIO_NUM} - -DY2_GPIO_NUM=${pinoutsESPWROVER.Y2_GPIO_NUM} - -DVSYNC_GPIO_NUM=${pinoutsESPWROVER.VSYNC_GPIO_NUM} - -DHREF_GPIO_NUM=${pinoutsESPWROVER.HREF_GPIO_NUM} - -DPCLK_GPIO_NUM=${pinoutsESPWROVER.PCLK_GPIO_NUM} + '-DCAMERA_MODULE_NAME="ESPS3WROVER"' + -DCONFIG_CAMERA_MODULE_ESPS3WROVER=1 + -DPWDN_GPIO_NUM=${pinoutsESPS3WROVER.PWDN_GPIO_NUM} + -DRESET_GPIO_NUM=${pinoutsESPS3WROVER.RESET_GPIO_NUM} + -DXCLK_GPIO_NUM=${pinoutsESPS3WROVER.XCLK_GPIO_NUM} + -DSIOD_GPIO_NUM=${pinoutsESPS3WROVER.SIOD_GPIO_NUM} + -DSIOC_GPIO_NUM=${pinoutsESPS3WROVER.SIOC_GPIO_NUM} + -DY9_GPIO_NUM=${pinoutsESPS3WROVER.Y9_GPIO_NUM} + -DY8_GPIO_NUM=${pinoutsESPS3WROVER.Y8_GPIO_NUM} + -DY7_GPIO_NUM=${pinoutsESPS3WROVER.Y7_GPIO_NUM} + -DY6_GPIO_NUM=${pinoutsESPS3WROVER.Y6_GPIO_NUM} + -DY5_GPIO_NUM=${pinoutsESPS3WROVER.Y5_GPIO_NUM} + -DY4_GPIO_NUM=${pinoutsESPS3WROVER.Y4_GPIO_NUM} + -DY3_GPIO_NUM=${pinoutsESPS3WROVER.Y3_GPIO_NUM} + -DY2_GPIO_NUM=${pinoutsESPS3WROVER.Y2_GPIO_NUM} + -DVSYNC_GPIO_NUM=${pinoutsESPS3WROVER.VSYNC_GPIO_NUM} + -DHREF_GPIO_NUM=${pinoutsESPS3WROVER.HREF_GPIO_NUM} + -DPCLK_GPIO_NUM=${pinoutsESPS3WROVER.PCLK_GPIO_NUM} diff --git a/ESP/lib/src/etvr_system.hpp b/ESP/lib/src/etvr_system.hpp new file mode 100644 index 0000000..e232fda --- /dev/null +++ b/ESP/lib/src/etvr_system.hpp @@ -0,0 +1,10 @@ +#ifndef INCLUDED_ETVR_SYSTEM_HPP +#define INCLUDED_ETVR_SYSTEM_HPP + +#if CONFIG_CAMERA_MODULE_ESPS3WROVER +#define ETVR_EYE_TRACKER_USB_API +#else +#define ETVR_EYE_TRACKER_WEB_API +#endif + +#endif // INCLUDED_ETVR_SYSTEM_HPP diff --git a/ESP/lib/src/usb/etvr_eye_tracker_usb.cpp b/ESP/lib/src/usb/etvr_eye_tracker_usb.cpp new file mode 100644 index 0000000..331c695 --- /dev/null +++ b/ESP/lib/src/usb/etvr_eye_tracker_usb.cpp @@ -0,0 +1,61 @@ +#include "etvr_eye_tracker_usb.hpp" + +#include +#include +#include + +const char* const ETVR_HEADER = "\xff\xa0"; +const char* const ETVR_HEADER_FRAME = "\xff\xa1"; + +void etvr_eye_tracker_usb_init() { + Serial.begin(3000000); + Serial.flush(); +} + +void etvr_eye_tracker_usb_loop() { + int64_t last_frame = 0; + if (!last_frame) + last_frame = esp_timer_get_time(); + + long last_request_time = 0; + camera_fb_t* fb = NULL; + esp_err_t err = ESP_OK; + + size_t len = 0; + uint8_t* buf = NULL; + + uint8_t len_bytes[2]; + + while (true) { + fb = esp_camera_fb_get(); + if (fb) { + len = fb->len; + buf = fb->buf; + } else { + log_e("Camera capture failed with response: %s", esp_err_to_name(err)); + err = ESP_FAIL; + } + if (err == ESP_OK) + Serial.write(ETVR_HEADER, 2); + Serial.write(ETVR_HEADER_FRAME, 2); + len_bytes[0] = len & 0xFF; + len_bytes[1] = (len >> CHAR_BIT) & 0xFF; + Serial.write(len_bytes, 2); + Serial.write((const char*)buf, len); + if (fb) { + esp_camera_fb_return(fb); + fb = NULL; + buf = NULL; + } else if (buf) { + free(buf); + buf = NULL; + } + if (err != ESP_OK) + break; + long request_end = millis(); + long latency = request_end - last_request_time; + last_request_time = request_end; + log_d("Size: %uKB, Time: %ums (%ifps)\n", len / 1024, latency, + 1000 / latency); + } +} diff --git a/ESP/lib/src/usb/etvr_eye_tracker_usb.hpp b/ESP/lib/src/usb/etvr_eye_tracker_usb.hpp new file mode 100644 index 0000000..cfd6de1 --- /dev/null +++ b/ESP/lib/src/usb/etvr_eye_tracker_usb.hpp @@ -0,0 +1,7 @@ +#ifndef INCLUDED_ETVR_EYE_TRACKER_USB_HPP +#define INCLUDED_ETVR_EYE_TRACKER_USB_HPP + +void etvr_eye_tracker_usb_init(); +void etvr_eye_tracker_usb_loop(); + +#endif // INCLUDED_ETVR_EYE_TRACKER_USB_HPP diff --git a/ESP/src/main.cpp b/ESP/src/main.cpp index a94e1ce..33cc7f1 100644 --- a/ESP/src/main.cpp +++ b/ESP/src/main.cpp @@ -1,13 +1,22 @@ +#include + #include + +#include #include #include +#include + +#ifdef ETVR_EYE_TRACKER_WEB_API #include #include #include #include +#endif // ETVR_EYE_TRACKER_WEB_API -#include -#include +#ifdef ETVR_EYE_TRACKER_USB_API +#include +#endif // ETVR_EYE_TRACKER_USB_API /** * @brief ProjectConfig object @@ -22,35 +31,18 @@ LEDManager ledManager(33); #ifndef SIM_ENABLED CameraHandler cameraHandler(deviceConfig); #endif // SIM_ENABLED + +#ifdef ETVR_EYE_TRACKER_WEB_API WiFiHandler wifiHandler(deviceConfig, WIFI_SSID, WIFI_PASSWORD, WIFI_CHANNEL); - -// ImprovHandler improvHandler(deviceConfig); - -#ifndef SIM_ENABLED -APIServer apiServer(deviceConfig, cameraHandler, "/control"); -#else -APIServer apiServer(deviceConfig, wifiStateManager, "/control"); -#endif // SIM_ENABLED - MDNSHandler mdnsHandler(deviceConfig); - -#ifndef SIM_ENABLED +#ifdef SIM_ENABLED +APIServer apiServer(deviceConfig, wifiStateManager, "/control"); +#else +APIServer apiServer(deviceConfig, cameraHandler, "/control"); StreamServer streamServer; #endif // SIM_ENABLED -void setup() { - setCpuFrequencyMhz(240); // set to 240mhz for performance boost - Serial.begin(115200); - Logo::printASCII(); - Serial.flush(); - ledManager.begin(); -#ifndef SIM_ENABLED - deviceConfig.attach(cameraHandler); -#endif // SIM_ENABLED - deviceConfig.attach(mdnsHandler); - deviceConfig.attach(wifiHandler); - deviceConfig.initConfig(); - deviceConfig.load(); +void etvr_eye_tracker_web_init() { wifiHandler._enable_adhoc = ENABLE_ADHOC; wifiHandler.begin(); mdnsHandler.startMDNS(); @@ -88,7 +80,41 @@ void setup() { } } } +#endif // ETVR_EYE_TRACKER_WEB_API + +void setup() { + setCpuFrequencyMhz(240); + Serial.begin(115200); + Logo::printASCII(); + Serial.flush(); + ledManager.begin(); + +#ifndef SIM_ENABLED + deviceConfig.attach(cameraHandler); +#endif // SIM_ENABLED + +#ifdef ETVR_EYE_TRACKER_WEB_API + deviceConfig.attach(mdnsHandler); + deviceConfig.attach(wifiHandler); +#endif // ETVR_EYE_TRACKER_WEB_API + + deviceConfig.initConfig(); + deviceConfig.load(); + +#ifdef ETVR_EYE_TRACKER_WEB_API + etvr_eye_tracker_web_init(); +#else // ETVR_EYE_TRACKER_WEB_API + WiFi.disconnect(true); +#endif // ETVR_EYE_TRACKER_WEB_API + +#ifdef ETVR_EYE_TRACKER_USB_API + etvr_eye_tracker_usb_init(); +#endif // ETVR_EYE_TRACKER_USB_API +} void loop() { ledManager.handleLED(); +#ifdef ETVR_EYE_TRACKER_USB_API + etvr_eye_tracker_usb_loop(); +#endif // ETVR_EYE_TRACKER_USB_API }