Initial Commit

.clang-format

@@ -0,0 +1,9 @@
+BasedOnStyle: Google
+IndentWidth: 2
+ColumnLimit: 80
+AllowShortFunctionsOnASingleLine: Empty
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+BreakBeforeBraces: Attach
+DerivePointerAlignment: false
+PointerAlignment: Right

.github/workflows/build.yaml

@@ -0,0 +1,28 @@
+name: Build
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        target: [esp32s3, linux]
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repo
+      uses: actions/checkout@v2
+      with:
+        submodules: 'recursive'
+
+    - name: Build
+      run: |
+        docker run -v $PWD:/project -w /project -u 0 \
+        -e HOME=/tmp -e WIFI_SSID=A -e WIFI_PASSWORD=B -e OPENAI_API_KEY=X \
+        espressif/idf:latest \
+        /bin/bash -c 'idf.py --preview set-target ${{ matrix.target }} && idf.py build'
+      shell: bash

.github/workflows/clang-format-check.yml

@@ -0,0 +1,13 @@
+name: clang-format Check
+on: [push, pull_request]
+jobs:
+  formatting-check:
+    name: Formatting Check
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Run clang-format style check for C/C++/Protobuf programs.
+      uses: jidicula/[email protected]
+      with:
+        clang-format-version: '17'
+        check-path: 'src'

.gitignore

@@ -0,0 +1,4 @@
+build
+sdkconfig
+sdkconfig.old
+managed_components

.gitmodules

@@ -0,0 +1,12 @@
+[submodule "components/srtp"]
+	path = components/srtp
+	url = https://[email protected]/sepfy/esp_ports
+[submodule "deps/libpeer"]
+	path = deps/libpeer
+	url = https://github.com/sean-der/libpeer
+[submodule "components/esp-libopus"]
+	path = components/esp-libopus
+	url = https://github.com/XasWorks/esp-libopus.git
+[submodule "components/esp-protocols"]
+	path = components/esp-protocols
+	url = https://github.com/espressif/esp-protocols.git

CMakeLists.txt

@@ -0,0 +1,35 @@
+cmake_minimum_required(VERSION 3.19)
+
+# Audio Sending is implemented, but not performant enough yet
+add_compile_definitions(SEND_AUDIO=0)
+
+if(NOT IDF_TARGET STREQUAL linux)
+  if(NOT DEFINED ENV{WIFI_SSID} OR NOT DEFINED ENV{WIFI_PASSWORD})
+    message(FATAL_ERROR "Env variables WIFI_SSID and WIFI_PASSWORD must be set")
+  endif()
+
+  add_compile_definitions(WIFI_SSID="$ENV{WIFI_SSID}")
+  add_compile_definitions(WIFI_PASSWORD="$ENV{WIFI_PASSWORD}")
+endif()
+
+if(NOT DEFINED ENV{OPENAI_API_KEY})
+  message(FATAL_ERROR "Env variable OPENAI_API_KEY must be set")
+endif()
+
+add_compile_definitions(OPENAI_API_KEY="$ENV{OPENAI_API_KEY}")
+add_compile_definitions(OPENAI_REALTIMEAPI="https://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview-2024-10-01")
+
+set(COMPONENTS src)
+set(EXTRA_COMPONENT_DIRS "src" "components/srtp" "components/peer" "components/esp-libopus")
+
+if(IDF_TARGET STREQUAL linux)
+	add_compile_definitions(LINUX_BUILD=1)
+  list(APPEND EXTRA_COMPONENT_DIRS
+    $ENV{IDF_PATH}/examples/protocols/linux_stubs/esp_stubs
+    "components/esp-protocols/common_components/linux_compat/esp_timer"
+    "components/esp-protocols/common_components/linux_compat/freertos"
+    )
+endif()
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(src)

README.md

@@ -0,0 +1,48 @@
+# Open RealtimeAPI Embedded SDK
+
+# Table of Contents
+
+- [Docs](#docs)
+- [Installation](#installation)
+- [Usage](#usage)
+
+## Platform/Device Support
+
+This SDK has been developed tested on a `esp32s3` and `linux`. You don't need any physical hardware
+to run this SDK. You can use it from Linux directly.
+
+To use it on hardware purchase either of these microcontrollers. Others may work, but this is what
+has been developed against.
+
+* [Freenove ESP32-S3-WROOM](https://www.amazon.com/gp/product/B0BMQ8F7FN)
+* [Sonatino - ESP32-S3 Audio Development Board](https://www.amazon.com/gp/product/B0BVY8RJNP)
+
+You can get a ESP32S3 for much less money on eBay/AliExpress.
+
+## Installation
+
+`protoc` must be in your path with `protobufc` installed.
+
+Call `set-target` with the platform you are targetting. Today only `linux` and `esp32s3` are supported.
+* `idf.py set-target esp32s3`
+
+Configure device specific settings. None needed at this time
+* `idf.py menuconfig`
+
+Set your Wifi SSID + Password as env variables
+* `export WIFI_SSID=foo`
+* `export WIFI_PASSWORD=bar`
+* `export OPENAI_API_KEY=bing`
+
+Build
+* `idf.py build`
+
+If you built for `esp32s3` run the following to flash to the device
+* `sudo -E idf.py flash`
+
+If you built for `linux` you can run the binary directly
+* `./build/src.elf`
+
+See [build.yaml](.github/workflows/build.yaml) for a Docker command to do this all in one step.
+
+## Usage

components/peer/CMakeLists.txt

@@ -0,0 +1,24 @@
+set(PEER_PROJECT_PATH "../../deps/libpeer")
+file(GLOB CODES "${PEER_PROJECT_PATH}/src/*.c")
+
+idf_component_register(
+  SRCS ${CODES}
+  INCLUDE_DIRS "${PEER_PROJECT_PATH}/src"
+  REQUIRES mbedtls srtp json esp_netif
+)
+
+# Disable building of usrsctp
+file(READ ${CMAKE_CURRENT_SOURCE_DIR}/../../deps/libpeer/src/config.h INPUT_CONTENT)
+string(REPLACE "#define HAVE_USRSCTP" "" MODIFIED_CONTENT ${INPUT_CONTENT})
+file(WRITE ${CMAKE_CURRENT_SOURCE_DIR}/../../deps/libpeer/src/config.h ${MODIFIED_CONTENT})
+
+# Disable KeepAlives
+file(READ ${CMAKE_CURRENT_SOURCE_DIR}/../../deps/libpeer/src/config.h INPUT_CONTENT)
+string(REPLACE "#define KEEPALIVE_CONNCHECK 10000" "#define KEEPALIVE_CONNCHECK 0" MODIFIED_CONTENT ${INPUT_CONTENT})
+file(WRITE ${CMAKE_CURRENT_SOURCE_DIR}/../../deps/libpeer/src/config.h ${MODIFIED_CONTENT})
+
+
+if(NOT IDF_TARGET STREQUAL linux)
+  add_definitions("-DESP32")
+endif()
+add_definitions("-DHTTP_DO_NOT_USE_CUSTOM_CONFIG -DMQTT_DO_NOT_USE_CUSTOM_CONFIG -DDISABLE_PEER_SIGNALING=true")

dependencies.lock

@@ -0,0 +1,10 @@
+dependencies:
+  idf:
+    source:
+      type: idf
+    version: 5.5.0
+direct_dependencies:
+- idf
+manifest_hash: 655e4ae2c4a00dc0e9b6d66aa2a909e40e81c57604a11f1553343408aeddfb41
+target: esp32s3
+version: 2.0.0

partitions.csv

@@ -0,0 +1,6 @@
+# ESP-IDF Partition Table
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,     0x9000,  0x6000,
+phy_init, data, phy,     0xf000,  0x1000,
+factory,  app,  factory, 0x10000, 0x180000,
+

sdkconfig.defaults

@@ -0,0 +1,31 @@
+# ESP Event Loop on Linux
+CONFIG_ESP_EVENT_POST_FROM_ISR=n
+CONFIG_ESP_EVENT_POST_FROM_IRAM_ISR=n
+
+# Disable TLS verification
+# Production needs to include specific cert chain you care about
+CONFIG_ESP_TLS_INSECURE=y
+CONFIG_ESP_TLS_SKIP_SERVER_CERT_VERIFY=y
+
+# Enable DTLS-SRTP
+CONFIG_MBEDTLS_SSL_PROTO_DTLS=y
+
+# libpeer requires large stack allocations
+CONFIG_ESP_MAIN_TASK_STACK_SIZE=8192
+
+# Defaults to partitions.csv
+CONFIG_PARTITION_TABLE_CUSTOM=y
+
+# Set highest CPU Freq
+CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240=y
+
+CONFIG_SPIRAM=y
+CONFIG_SPIRAM_MODE_OCT=y
+
+# Disable Watchdog
+# CONFIG_ESP_INT_WDT is not set
+# CONFIG_ESP_TASK_WDT_EN is not set
+
+# Enable Compiler Optimization
+CONFIG_COMPILER_OPTIMIZATION_PERF=y
+CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_DISABLE=y

src/CMakeLists.txt

@@ -0,0 +1,22 @@
+set(COMMON_SRC "webrtc.cpp" "main.cpp" "http.cpp")
+
+if(IDF_TARGET STREQUAL linux)
+	idf_component_register(
+		SRCS ${COMMON_SRC}
+		REQUIRES peer esp-libopus esp_http_client)
+else()
+	idf_component_register(
+		SRCS ${COMMON_SRC} "wifi.cpp" "media.cpp"
+		REQUIRES driver esp_wifi nvs_flash peer esp_psram esp-libopus esp_http_client)
+endif()
+
+idf_component_get_property(lib peer COMPONENT_LIB)
+target_compile_options(${lib} PRIVATE -Wno-error=restrict)
+target_compile_options(${lib} PRIVATE -Wno-error=stringop-truncation)
+
+idf_component_get_property(lib srtp COMPONENT_LIB)
+target_compile_options(${lib} PRIVATE -Wno-error=incompatible-pointer-types)
+
+idf_component_get_property(lib esp-libopus COMPONENT_LIB)
+target_compile_options(${lib} PRIVATE -Wno-error=maybe-uninitialized)
+target_compile_options(${lib} PRIVATE -Wno-error=stringop-overread)

src/http.cpp

@@ -0,0 +1,97 @@
+#include <esp_http_client.h>
+#include <esp_log.h>
+#include <string.h>
+
+#include "main.h"
+
+#ifndef MIN
+#define MIN(a, b) (((a) < (b)) ? (a) : (b))
+#endif
+
+esp_err_t oai_http_event_handler(esp_http_client_event_t *evt) {
+  static int output_len;
+  switch (evt->event_id) {
+    case HTTP_EVENT_REDIRECT:
+      ESP_LOGD(LOG_TAG, "HTTP_EVENT_REDIRECT");
+      esp_http_client_set_header(evt->client, "From", "[email protected]");
+      esp_http_client_set_header(evt->client, "Accept", "text/html");
+      esp_http_client_set_redirection(evt->client);
+      break;
+    case HTTP_EVENT_ERROR:
+      ESP_LOGD(LOG_TAG, "HTTP_EVENT_ERROR");
+      break;
+    case HTTP_EVENT_ON_CONNECTED:
+      ESP_LOGD(LOG_TAG, "HTTP_EVENT_ON_CONNECTED");
+      break;
+    case HTTP_EVENT_HEADER_SENT:
+      ESP_LOGD(LOG_TAG, "HTTP_EVENT_HEADER_SENT");
+      break;
+    case HTTP_EVENT_ON_HEADER:
+      ESP_LOGD(LOG_TAG, "HTTP_EVENT_ON_HEADER, key=%s, value=%s",
+               evt->header_key, evt->header_value);
+      break;
+    case HTTP_EVENT_ON_DATA: {
+      ESP_LOGD(LOG_TAG, "HTTP_EVENT_ON_DATA, len=%d", evt->data_len);
+      if (esp_http_client_is_chunked_response(evt->client)) {
+        ESP_LOGE(LOG_TAG, "Chunked HTTP response not supported");
+#ifndef LINUX_BUILD
+        esp_restart();
+#endif
+      }
+
+      if (output_len == 0 && evt->user_data) {
+        memset(evt->user_data, 0, MAX_HTTP_OUTPUT_BUFFER);
+      }
+
+      // If user_data buffer is configured, copy the response into the buffer
+      int copy_len = 0;
+      if (evt->user_data) {
+        // The last byte in evt->user_data is kept for the NULL character in
+        // case of out-of-bound access.
+        copy_len = MIN(evt->data_len, (MAX_HTTP_OUTPUT_BUFFER - output_len));
+        if (copy_len) {
+          memcpy(((char *)evt->user_data) + output_len, evt->data, copy_len);
+        }
+      }
+      output_len += copy_len;
+
+      break;
+    }
+    case HTTP_EVENT_ON_FINISH:
+      ESP_LOGD(LOG_TAG, "HTTP_EVENT_ON_FINISH");
+      output_len = 0;
+      break;
+    case HTTP_EVENT_DISCONNECTED:
+      ESP_LOGI(LOG_TAG, "HTTP_EVENT_DISCONNECTED");
+      output_len = 0;
+      break;
+  }
+  return ESP_OK;
+}
+
+void oai_http_request(char *offer, char *answer) {
+  esp_http_client_config_t config;
+  memset(&config, 0, sizeof(esp_http_client_config_t));
+
+  config.url = OPENAI_REALTIMEAPI;
+  config.event_handler = oai_http_event_handler;
+  config.user_data = answer;
+
+  snprintf(answer, MAX_HTTP_OUTPUT_BUFFER, "Bearer %s", OPENAI_API_KEY);
+
+  esp_http_client_handle_t client = esp_http_client_init(&config);
+  esp_http_client_set_method(client, HTTP_METHOD_POST);
+  esp_http_client_set_header(client, "Content-Type", "application/sdp");
+  esp_http_client_set_header(client, "Authorization", answer);
+  esp_http_client_set_post_field(client, offer, strlen(offer));
+
+  esp_err_t err = esp_http_client_perform(client);
+  if (err != ESP_OK || esp_http_client_get_status_code(client) != 201) {
+    ESP_LOGE(LOG_TAG, "Error perform http request %s", esp_err_to_name(err));
+#ifndef LINUX_BUILD
+    esp_restart();
+#endif
+  }
+
+  esp_http_client_cleanup(client);
+}

src/idf_component.yml

@@ -0,0 +1,3 @@
+dependencies:
+  idf:
+    version: ">=4.1.0"

src/main.cpp

@@ -0,0 +1,32 @@
+#include "main.h"
+
+#include <esp_event.h>
+#include <esp_log.h>
+#include <peer.h>
+
+#ifndef LINUX_BUILD
+#include "nvs_flash.h"
+
+extern "C" void app_main(void) {
+  esp_err_t ret = nvs_flash_init();
+  if (ret == ESP_ERR_NVS_NO_FREE_PAGES ||
+      ret == ESP_ERR_NVS_NEW_VERSION_FOUND) {
+    ESP_ERROR_CHECK(nvs_flash_erase());
+    ret = nvs_flash_init();
+  }
+  ESP_ERROR_CHECK(ret);
+
+  ESP_ERROR_CHECK(esp_event_loop_create_default());
+  peer_init();
+  oai_init_audio_capture();
+  oai_init_audio_decoder();
+  oai_wifi();
+  oai_webrtc();
+}
+#else
+int main(void) {
+  ESP_ERROR_CHECK(esp_event_loop_create_default());
+  peer_init();
+  oai_webrtc();
+}
+#endif