diff --git a/components/esp_driver_i2s/i2s_common.c b/components/esp_driver_i2s/i2s_common.c index bcf15a342afa..c89cd245cb6e 100644 --- a/components/esp_driver_i2s/i2s_common.c +++ b/components/esp_driver_i2s/i2s_common.c @@ -568,6 +568,11 @@ uint32_t i2s_get_source_clk_freq(i2s_clock_src_t clk_src, uint32_t mclk_freq_hz) if (clk_src == I2S_CLK_SRC_APLL) { return i2s_set_get_apll_freq(mclk_freq_hz); } +#endif +#ifdef I2S_LL_DEFAULT_CLK_SRC + if (clk_src == I2S_CLK_SRC_DEFAULT) { + clk_src = I2S_LL_DEFAULT_CLK_SRC; + } #endif esp_clk_tree_src_get_freq_hz(clk_src, ESP_CLK_TREE_SRC_FREQ_PRECISION_CACHED, &clk_freq); return clk_freq; diff --git a/components/esp_driver_i2s/test_apps/i2s/main/test_i2s.c b/components/esp_driver_i2s/test_apps/i2s/main/test_i2s.c index 04cbdb4fb75f..323db87653a6 100644 --- a/components/esp_driver_i2s/test_apps/i2s/main/test_i2s.c +++ b/components/esp_driver_i2s/test_apps/i2s/main/test_i2s.c @@ -632,7 +632,7 @@ TEST_CASE("I2S_loopback_test", "[i2s]") TEST_ESP_OK(i2s_del_channel(rx_handle)); } -#if SOC_I2S_NUM > 1 +#if SOC_I2S_NUM > 1 && !CONFIG_IDF_TARGET_ESP32P4 TEST_CASE("I2S_master_write_slave_read_test", "[i2s]") { i2s_chan_handle_t tx_handle; @@ -798,10 +798,10 @@ TEST_CASE("I2S_default_PLL_clock_test", "[i2s]") TEST_ESP_OK(i2s_new_channel(&chan_cfg, NULL, &rx_handle)); TEST_ESP_OK(i2s_channel_init_std_mode(rx_handle, &std_cfg)); -// ESP32-P4 has no PLL except XTAL -#if !CONFIG_IDF_TARGET_ESP32P4 +#ifdef I2S_LL_DEFAULT_CLK_SRC + std_cfg.clk_cfg.clk_src = I2S_LL_DEFAULT_CLK_SRC; +#endif i2s_test_common_sample_rate(rx_handle, &std_cfg.clk_cfg); -#endif // CONFIG_IDF_TARGET_ESP32P4 #if SOC_I2S_SUPPORTS_XTAL std_cfg.clk_cfg.clk_src = I2S_CLK_SRC_XTAL; i2s_test_common_sample_rate(rx_handle, &std_cfg.clk_cfg); diff --git a/components/hal/esp32p4/include/hal/i2s_ll.h b/components/hal/esp32p4/include/hal/i2s_ll.h index 4573faf372d2..65aeb6e9f888 100644 --- a/components/hal/esp32p4/include/hal/i2s_ll.h +++ b/components/hal/esp32p4/include/hal/i2s_ll.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: 2023-2024 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2023-2025 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ @@ -23,6 +23,7 @@ #include "soc/soc_etm_source.h" #include "hal/i2s_types.h" #include "hal/hal_utils.h" +#include "hal/config.h" #ifdef __cplusplus @@ -41,7 +42,14 @@ extern "C" { #define I2S_LL_SLOT_FRAME_BIT_MAX 512 // Up-to 512 bits in one frame, determined by MAX(half_sample_bits) * 2 #define I2S_LL_XTAL_CLK_FREQ (40 * 1000000) // XTAL_CLK: 40MHz -#define I2S_LL_DEFAULT_CLK_FREQ I2S_LL_XTAL_CLK_FREQ // No PLL clock source on P4, use XTAL as default +#if HAL_CONFIG(CHIP_SUPPORT_MIN_REV) >= 300 +#define I2S_LL_DEFAULT_CLK_FREQ (160 * 1000000) // PLL_F160M_CLK: 160MHz +#define I2S_LL_DEFAULT_CLK_SRC I2S_CLK_SRC_PLL_160M +#else +#define I2S_LL_DEFAULT_CLK_FREQ I2S_LL_XTAL_CLK_FREQ // No PLL clock source before version 3, use XTAL as default +#define I2S_LL_DEFAULT_CLK_SRC I2S_CLK_SRC_XTAL +#endif + #define I2S_LL_ETM_EVENT_TABLE(i2s_port, chan_dir, event) \ (uint32_t[SOC_I2S_NUM][2][I2S_ETM_EVENT_MAX]){ \ @@ -414,6 +422,15 @@ static inline uint32_t i2s_ll_get_clk_src(i2s_clock_src_t src) return 1; case I2S_CLK_SRC_EXTERNAL: return 2; + case I2S_CLK_SRC_DEFAULT: +#if HAL_CONFIG(CHIP_SUPPORT_MIN_REV) >= 300 + return 3; + // Only support PLL_160M on P4 ver3 and later + case I2S_CLK_SRC_PLL_160M: + return 3; +#else + return 0; +#endif default: HAL_ASSERT(false && "unsupported clock source"); return -1; diff --git a/components/nvs_flash/host_test/nvs_host_test/main/test_nvs.cpp b/components/nvs_flash/host_test/nvs_host_test/main/test_nvs.cpp index 431e8892037c..b0762d9e54ab 100644 --- a/components/nvs_flash/host_test/nvs_host_test/main/test_nvs.cpp +++ b/components/nvs_flash/host_test/nvs_host_test/main/test_nvs.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ @@ -3806,6 +3806,82 @@ TEST_CASE("nvs multiple write with same key but different types", "[nvs]") TEST_ESP_OK(nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME)); } +TEST_CASE("nvs multiple write with same key blob and string involved", "[nvs]") +{ + PartitionEmulationFixture f(0, 10); + + nvs_handle_t handle_1; + const uint32_t NVS_FLASH_SECTOR = 6; + const uint32_t NVS_FLASH_SECTOR_COUNT_MIN = 3; + TEMPORARILY_DISABLED(f.emu.setBounds(NVS_FLASH_SECTOR, NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN);) + + for (uint16_t j = NVS_FLASH_SECTOR; j < NVS_FLASH_SECTOR + NVS_FLASH_SECTOR_COUNT_MIN; ++j) { + f.erase(j); + } + TEST_ESP_OK(nvs::NVSPartitionManager::get_instance()->init_custom(f.part(), + NVS_FLASH_SECTOR, + NVS_FLASH_SECTOR_COUNT_MIN)); + + TEST_ESP_OK(nvs_open("namespace1", NVS_READWRITE, &handle_1)); + + nvs_erase_all(handle_1); + + const char key_name[] = "foo"; + + // integer variables + int32_t v32; + int8_t v8; + + // string + #define str_data_len 64 + const char str_data[] = "string data"; + char str_buf[str_data_len] = {0}; + size_t str_len = str_data_len; + + // blob + #define blob_data_len 64 + uint8_t blob_data[blob_data_len] = {0}; + uint8_t blob_buf[blob_data_len] = {0}; + size_t blob_read_size; + + // first write is i32 + TEST_ESP_OK(nvs_set_i32(handle_1, key_name, (int32_t)12345678)); + + TEST_ESP_ERR(nvs_get_i8(handle_1, key_name, &v8), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_get_i32(handle_1, key_name, &v32)); + TEST_ESP_ERR(nvs_get_str(handle_1, key_name, str_buf, &str_len), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_ERR(nvs_get_blob(handle_1, key_name, blob_buf, &blob_read_size), ESP_ERR_NVS_NOT_FOUND); + + + // second write is string + TEST_ESP_OK(nvs_set_str(handle_1, key_name, str_data)); + + TEST_ESP_ERR(nvs_get_i8(handle_1, key_name, &v8), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_ERR(nvs_get_i32(handle_1, key_name, &v32), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_get_str(handle_1, key_name, str_buf, &str_len)); + TEST_ESP_ERR(nvs_get_blob(handle_1, key_name, blob_buf, &blob_read_size), ESP_ERR_NVS_NOT_FOUND); + + // third write is blob + TEST_ESP_OK(nvs_set_blob(handle_1, key_name, blob_data, blob_data_len)); + + TEST_ESP_ERR(nvs_get_i8(handle_1, key_name, &v8), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_ERR(nvs_get_i32(handle_1, key_name, &v32), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_ERR(nvs_get_str(handle_1, key_name, str_buf, &str_len), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_OK(nvs_get_blob(handle_1, key_name, blob_buf, &blob_read_size)); + + // fourth write is i8 + TEST_ESP_OK(nvs_set_i8(handle_1, key_name, (int8_t)12)); + + TEST_ESP_OK(nvs_get_i8(handle_1, key_name, &v8)); + TEST_ESP_ERR(nvs_get_i32(handle_1, key_name, &v32), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_ERR(nvs_get_str(handle_1, key_name, str_buf, &str_len), ESP_ERR_NVS_NOT_FOUND); + TEST_ESP_ERR(nvs_get_blob(handle_1, key_name, blob_buf, &blob_read_size), ESP_ERR_NVS_NOT_FOUND); + + nvs_close(handle_1); + + TEST_ESP_OK(nvs_flash_deinit_partition(NVS_DEFAULT_PART_NAME)); +} + TEST_CASE("nvs find key tests", "[nvs]") { const size_t buff_len = 4096; diff --git a/components/nvs_flash/src/nvs_page.cpp b/components/nvs_flash/src/nvs_page.cpp index f818ddb56ffd..ddec64ef226b 100644 --- a/components/nvs_flash/src/nvs_page.cpp +++ b/components/nvs_flash/src/nvs_page.cpp @@ -250,33 +250,19 @@ esp_err_t Page::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, c return ESP_OK; } -esp_err_t Page::readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize, uint8_t chunkIdx, VerOffset chunkStart) +// Reads the data entries of the variable length item. +// The metadata entry is already read in the item object. +// index is the index of the metadata entry on the page. +// data is pointer to the buffer where the data will be copied to. It has to be at least +// item.varLength.dataSize bytes long. +// The function returns ESP_OK if the data was read successfully, or an error code if there was an error. +esp_err_t Page::readVariableLengthItemData(const Item& item, const size_t index, void* data) { - size_t index = 0; - Item item; - if (mState == PageState::INVALID) { return ESP_ERR_NVS_INVALID_STATE; } - esp_err_t rc = findItem(nsIndex, datatype, key, index, item, chunkIdx, chunkStart); - if (rc != ESP_OK) { - return rc; - } - - if (!isVariableLengthType(datatype)) { - if (dataSize != getAlignmentForType(datatype)) { - return ESP_ERR_NVS_TYPE_MISMATCH; - } - - memcpy(data, item.data, dataSize); - return ESP_OK; - } - - if (dataSize < static_cast(item.varLength.dataSize)) { - return ESP_ERR_NVS_INVALID_LENGTH; - } - + esp_err_t rc; uint8_t* dst = reinterpret_cast(data); size_t left = item.varLength.dataSize; for (size_t i = index + 1; i < index + item.span; ++i) { @@ -301,6 +287,36 @@ esp_err_t Page::readItem(uint8_t nsIndex, ItemType datatype, const char* key, vo return ESP_OK; } +esp_err_t Page::readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize, uint8_t chunkIdx, VerOffset chunkStart) +{ + size_t index = 0; + Item item; + + if (mState == PageState::INVALID) { + return ESP_ERR_NVS_INVALID_STATE; + } + + esp_err_t rc = findItem(nsIndex, datatype, key, index, item, chunkIdx, chunkStart); + if (rc != ESP_OK) { + return rc; + } + + if (!isVariableLengthType(datatype)) { + if (dataSize != getAlignmentForType(datatype)) { + return ESP_ERR_NVS_TYPE_MISMATCH; + } + + memcpy(data, item.data, dataSize); + return ESP_OK; + } + + if (dataSize < static_cast(item.varLength.dataSize)) { + return ESP_ERR_NVS_INVALID_LENGTH; + } + + return readVariableLengthItemData(item, index, data); +} + esp_err_t Page::cmpItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize, uint8_t chunkIdx, VerOffset chunkStart) { size_t index = 0; @@ -330,9 +346,23 @@ esp_err_t Page::cmpItem(uint8_t nsIndex, ItemType datatype, const char* key, con return ESP_ERR_NVS_INVALID_LENGTH; } + // We have metadata of the variable length data chunk. It contains the length of the data and the crc32. + // As a first step we can calculate the crc32 of the data buffer to be compared with the crc32 of the item in the flash. + // If they are not equal, immediately return ESP_ERR_NVS_CONTENT_DIFFERS. + // If they are equal, to avoid crc32 collision false positive, we will read the data from the flash entry by entry and compare + // it with the respective chunk of input data buffer. The crc32 of the data read from the flash will be calculated on the fly. + // At the end, we will compare the crc32 of the data read from the flash with the crc32 of the metadata item in the flash to make sure + // that the data in the flash is not corrupted. + if (Item::calculateCrc32(reinterpret_cast(data), item.varLength.dataSize) != item.varLength.dataCrc32) { + return ESP_ERR_NVS_CONTENT_DIFFERS; + } + const uint8_t* dst = reinterpret_cast(data); size_t left = item.varLength.dataSize; - for (size_t i = index + 1; i < index + item.span; ++i) { + uint32_t accumulatedCRC32; + size_t initial_index = index + 1; + + for (size_t i = initial_index; i < index + item.span; ++i) { Item ditem; rc = readEntry(i, ditem); if (rc != ESP_OK) { @@ -343,11 +373,18 @@ esp_err_t Page::cmpItem(uint8_t nsIndex, ItemType datatype, const char* key, con if (memcmp(dst, ditem.rawData, willCopy)) { return ESP_ERR_NVS_CONTENT_DIFFERS; } + + // Calculate the crc32 of the actual ditem.rawData buffer. Do not pass accumulatedCRC32 in the first call. + // In the first call, calculateCrc32 will use its default. In the subsequent calls, accumulatedCRC32 is the crc32 of the previous buffer. + accumulatedCRC32 = Item::calculateCrc32(ditem.rawData, willCopy, (i == initial_index) ? nullptr : &accumulatedCRC32); + left -= willCopy; dst += willCopy; } - if (Item::calculateCrc32(reinterpret_cast(data), item.varLength.dataSize) != item.varLength.dataCrc32) { - return ESP_ERR_NVS_NOT_FOUND; + // Check if the CRC32 calculated on the fly matches the variable length data CRC32 indicated in the metadata entry. + // If they are not equal, it means the data in the flash is corrupt, we will return ESP_ERR_NVS_CONTENT_DIFFERS. + if (accumulatedCRC32 != item.varLength.dataCrc32) { + return ESP_ERR_NVS_CONTENT_DIFFERS; } return ESP_OK; diff --git a/components/nvs_flash/src/nvs_page.hpp b/components/nvs_flash/src/nvs_page.hpp index 7adb75bef79a..953682b157fa 100644 --- a/components/nvs_flash/src/nvs_page.hpp +++ b/components/nvs_flash/src/nvs_page.hpp @@ -88,6 +88,8 @@ class Page : public intrusive_list_node, public ExceptionlessAllocatable esp_err_t writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize, uint8_t chunkIdx = CHUNK_ANY); + esp_err_t readVariableLengthItemData(const Item& item, const size_t index, void* data); + esp_err_t readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize, uint8_t chunkIdx = CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY); esp_err_t cmpItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize, uint8_t chunkIdx = CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY); diff --git a/components/nvs_flash/src/nvs_storage.cpp b/components/nvs_flash/src/nvs_storage.cpp index e5db0caefed6..7d75a3a0467d 100644 --- a/components/nvs_flash/src/nvs_storage.cpp +++ b/components/nvs_flash/src/nvs_storage.cpp @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: 2015-2024 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2015-2025 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ @@ -51,7 +51,7 @@ void Storage::clearNamespaces() esp_err_t Storage::populateBlobIndices(TBlobIndexList& blobIdxList) { - for (auto it = mPageManager.begin(); it != mPageManager.end(); ++it) { + for(auto it = mPageManager.begin(); it != mPageManager.end(); ++it) { Page& p = *it; size_t itemIndex = 0; Item item; @@ -60,10 +60,10 @@ esp_err_t Storage::populateBlobIndices(TBlobIndexList& blobIdxList) * logic in pagemanager will remove the earlier index. So we should never find a * duplicate index at this point */ - while (p.findItem(Page::NS_ANY, ItemType::BLOB_IDX, nullptr, itemIndex, item) == ESP_OK) { + while(p.findItem(Page::NS_ANY, ItemType::BLOB_IDX, nullptr, itemIndex, item) == ESP_OK) { BlobIndexNode* entry = new (std::nothrow) BlobIndexNode; - if (!entry) return ESP_ERR_NO_MEM; + if(!entry) return ESP_ERR_NO_MEM; item.getKey(entry->key, sizeof(entry->key)); entry->nsIndex = item.nsIndex; @@ -89,7 +89,7 @@ esp_err_t Storage::populateBlobIndices(TBlobIndexList& blobIdxList) // later by the call to eraseOrphanDataBlobs(). void Storage::eraseMismatchedBlobIndexes(TBlobIndexList& blobIdxList) { - for (auto it = mPageManager.begin(); it != mPageManager.end(); ++it) { + for(auto it = mPageManager.begin(); it != mPageManager.end(); ++it) { Page& p = *it; size_t itemIndex = 0; Item item; @@ -98,7 +98,7 @@ void Storage::eraseMismatchedBlobIndexes(TBlobIndexList& blobIdxList) * 1) VER_0_OFFSET <= chunkIndex < VER_1_OFFSET-1 => Version0 chunks * 2) VER_1_OFFSET <= chunkIndex < VER_ANY => Version1 chunks */ - while (p.findItem(Page::NS_ANY, ItemType::BLOB_DATA, nullptr, itemIndex, item) == ESP_OK) { + while(p.findItem(Page::NS_ANY, ItemType::BLOB_DATA, nullptr, itemIndex, item) == ESP_OK) { auto iter = std::find_if(blobIdxList.begin(), blobIdxList.end(), @@ -107,7 +107,7 @@ void Storage::eraseMismatchedBlobIndexes(TBlobIndexList& blobIdxList) && (item.nsIndex == e.nsIndex) && (item.chunkIndex >= static_cast (e.chunkStart)) && (item.chunkIndex < static_cast ((e.chunkStart == nvs::VerOffset::VER_0_OFFSET) ? nvs::VerOffset::VER_1_OFFSET : nvs::VerOffset::VER_ANY));}); - if (iter != std::end(blobIdxList)) { + if(iter != std::end(blobIdxList)) { // accumulate the size iter->observedDataSize += item.varLength.dataSize; iter->observedChunkCount++; @@ -117,15 +117,15 @@ void Storage::eraseMismatchedBlobIndexes(TBlobIndexList& blobIdxList) } auto iter = blobIdxList.begin(); - while (iter != blobIdxList.end()) + while(iter != blobIdxList.end()) { - if ( (iter->observedDataSize != iter->dataSize) || (iter->observedChunkCount != iter->chunkCount) ) + if( (iter->observedDataSize != iter->dataSize) || (iter->observedChunkCount != iter->chunkCount) ) { // Delete blob_index from flash // This is very rare case, so we can loop over all pages - for (auto it = mPageManager.begin(); it != mPageManager.end(); ++it) { + for(auto it = mPageManager.begin(); it != mPageManager.end(); ++it) { // skip pages in non eligible states - if (it->state() == nvs::Page::PageState::CORRUPT + if(it->state() == nvs::Page::PageState::CORRUPT || it->state() == nvs::Page::PageState::INVALID || it->state() == nvs::Page::PageState::UNINITIALIZED){ continue; @@ -153,7 +153,7 @@ void Storage::eraseMismatchedBlobIndexes(TBlobIndexList& blobIdxList) void Storage::eraseOrphanDataBlobs(TBlobIndexList& blobIdxList) { - for (auto it = mPageManager.begin(); it != mPageManager.end(); ++it) { + for(auto it = mPageManager.begin(); it != mPageManager.end(); ++it) { Page& p = *it; size_t itemIndex = 0; Item item; @@ -162,7 +162,7 @@ void Storage::eraseOrphanDataBlobs(TBlobIndexList& blobIdxList) * 1) VER_0_OFFSET <= chunkIndex < VER_1_OFFSET-1 => Version0 chunks * 2) VER_1_OFFSET <= chunkIndex < VER_ANY => Version1 chunks */ - while (p.findItem(Page::NS_ANY, ItemType::BLOB_DATA, nullptr, itemIndex, item) == ESP_OK) { + while(p.findItem(Page::NS_ANY, ItemType::BLOB_DATA, nullptr, itemIndex, item) == ESP_OK) { auto iter = std::find_if(blobIdxList.begin(), blobIdxList.end(), @@ -171,7 +171,7 @@ void Storage::eraseOrphanDataBlobs(TBlobIndexList& blobIdxList) && (item.nsIndex == e.nsIndex) && (item.chunkIndex >= static_cast (e.chunkStart)) && (item.chunkIndex < static_cast (e.chunkStart) + e.chunkCount);}); - if (iter == std::end(blobIdxList)) { + if(iter == std::end(blobIdxList)) { p.eraseItem(item.nsIndex, item.datatype, item.key, item.chunkIndex); } @@ -183,7 +183,7 @@ void Storage::eraseOrphanDataBlobs(TBlobIndexList& blobIdxList) esp_err_t Storage::init(uint32_t baseSector, uint32_t sectorCount) { auto err = mPageManager.load(mPartition, baseSector, sectorCount); - if (err != ESP_OK) { + if(err != ESP_OK) { mState = StorageState::INVALID; return err; } @@ -191,25 +191,25 @@ esp_err_t Storage::init(uint32_t baseSector, uint32_t sectorCount) // load namespaces list clearNamespaces(); std::fill_n(mNamespaceUsage.data(), mNamespaceUsage.byteSize() / 4, 0); - for (auto it = mPageManager.begin(); it != mPageManager.end(); ++it) { + for(auto it = mPageManager.begin(); it != mPageManager.end(); ++it) { Page& p = *it; size_t itemIndex = 0; Item item; - while (p.findItem(Page::NS_INDEX, ItemType::U8, nullptr, itemIndex, item) == ESP_OK) { + while(p.findItem(Page::NS_INDEX, ItemType::U8, nullptr, itemIndex, item) == ESP_OK) { NamespaceEntry* entry = new (std::nothrow) NamespaceEntry; - if (!entry) { + if(!entry) { mState = StorageState::INVALID; return ESP_ERR_NO_MEM; } item.getKey(entry->mName, sizeof(entry->mName)); err = item.getValue(entry->mIndex); - if (err != ESP_OK) { + if(err != ESP_OK) { delete entry; return err; } - if (mNamespaceUsage.set(entry->mIndex, true) != ESP_OK) { + if(mNamespaceUsage.set(entry->mIndex, true) != ESP_OK) { delete entry; return ESP_FAIL; } @@ -217,17 +217,17 @@ esp_err_t Storage::init(uint32_t baseSector, uint32_t sectorCount) itemIndex += item.span; } } - if (mNamespaceUsage.set(0, true) != ESP_OK) { + if(mNamespaceUsage.set(0, true) != ESP_OK) { return ESP_FAIL; } - if (mNamespaceUsage.set(255, true) != ESP_OK) { + if(mNamespaceUsage.set(255, true) != ESP_OK) { return ESP_FAIL; } // Populate list of multi-page index entries. TBlobIndexList blobIdxList; err = populateBlobIndices(blobIdxList); - if (err != ESP_OK) { + if(err != ESP_OK) { mState = StorageState::INVALID; return ESP_ERR_NO_MEM; } @@ -254,13 +254,16 @@ bool Storage::isValid() const return mState == StorageState::ACTIVE; } -esp_err_t Storage::findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item, uint8_t chunkIdx, VerOffset chunkStart) +esp_err_t Storage::findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item, uint8_t chunkIdx, VerOffset chunkStart, size_t* itemIndex) { - for (auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) { - size_t itemIndex = 0; - auto err = it->findItem(nsIndex, datatype, key, itemIndex, item, chunkIdx, chunkStart); - if (err == ESP_OK) { + for(auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) { + size_t tmpItemIndex = 0; + auto err = it->findItem(nsIndex, datatype, key, tmpItemIndex, item, chunkIdx, chunkStart); + if(err == ESP_OK) { page = it; + if(itemIndex) { + *itemIndex = tmpItemIndex; + } return ESP_OK; } } @@ -282,7 +285,7 @@ esp_err_t Storage::writeMultiPageBlob(uint8_t nsIndex, const char* key, const vo max_pages = (Page::CHUNK_ANY-1)/2; } - if (dataSize > max_pages * Page::CHUNK_MAX_SIZE) { + if(dataSize > max_pages * Page::CHUNK_MAX_SIZE) { return ESP_ERR_NVS_VALUE_TOO_LONG; } @@ -290,16 +293,16 @@ esp_err_t Storage::writeMultiPageBlob(uint8_t nsIndex, const char* key, const vo Page& page = getCurrentPage(); size_t tailroom = page.getVarDataTailroom(); size_t chunkSize = 0; - if (chunkCount == 0U && ((tailroom < dataSize) || (tailroom == 0 && dataSize == 0)) && tailroom < Page::CHUNK_MAX_SIZE/10) { + if(chunkCount == 0U && ((tailroom < dataSize) || (tailroom == 0 && dataSize == 0)) && tailroom < Page::CHUNK_MAX_SIZE/10) { /** This is the first chunk and tailroom is too small ***/ - if (page.state() != Page::PageState::FULL) { + if(page.state() != Page::PageState::FULL) { err = page.markFull(); - if (err != ESP_OK) { + if(err != ESP_OK) { return err; } } err = mPageManager.requestNewPage(); - if (err != ESP_OK) { + if(err != ESP_OK) { return err; } else if(getCurrentPage().getVarDataTailroom() == tailroom) { /* We got the same page or we are not improving.*/ @@ -307,7 +310,7 @@ esp_err_t Storage::writeMultiPageBlob(uint8_t nsIndex, const char* key, const vo } else { continue; } - } else if (!tailroom) { + } else if(!tailroom) { err = ESP_ERR_NVS_NOT_ENOUGH_SPACE; break; } @@ -322,32 +325,32 @@ esp_err_t Storage::writeMultiPageBlob(uint8_t nsIndex, const char* key, const vo static_cast (data) + offset, chunkSize, static_cast (chunkStart) + chunkCount); chunkCount++; - if (err != ESP_OK) { + if(err != ESP_OK) { NVS_ASSERT_OR_RETURN(err != ESP_ERR_NVS_PAGE_FULL, err); break; } else { UsedPageNode* node = new (std::nothrow) UsedPageNode(); - if (!node) { + if(!node) { err = ESP_ERR_NO_MEM; break; } node->mPage = &page; usedPages.push_back(node); - if (remainingSize || (tailroom - chunkSize) < Page::ENTRY_SIZE) { - if (page.state() != Page::PageState::FULL) { + if(remainingSize || (tailroom - chunkSize) < Page::ENTRY_SIZE) { + if(page.state() != Page::PageState::FULL) { err = page.markFull(); - if (err != ESP_OK) { + if(err != ESP_OK) { break; } } err = mPageManager.requestNewPage(); - if (err != ESP_OK) { + if(err != ESP_OK) { break; } } } offset += chunkSize; - if (!remainingSize) { + if(!remainingSize) { /* All pages are stored. Now store the index.*/ Item item; std::fill_n(item.data, sizeof(item.data), 0xff); @@ -359,12 +362,12 @@ esp_err_t Storage::writeMultiPageBlob(uint8_t nsIndex, const char* key, const vo NVS_ASSERT_OR_RETURN(err != ESP_ERR_NVS_PAGE_FULL, err); break; } - } while (1); + } while(1); - if (err != ESP_OK) { + if(err != ESP_OK) { /* Anything failed, then we should erase all the written chunks*/ int ii=0; - for (auto it = std::begin(usedPages); it != std::end(usedPages); it++) { + for(auto it = std::begin(usedPages); it != std::end(usedPages); it++) { it->mPage->eraseItem(nsIndex, ItemType::BLOB_DATA, key, ii++); } } @@ -372,200 +375,265 @@ esp_err_t Storage::writeMultiPageBlob(uint8_t nsIndex, const char* key, const vo return err; } +// datatype BLOB is written as BLOB_INDEX and BLOB_DATA and is searched for previous value as BLOB_INDEX and/or BLOB +// datatype BLOB_INDEX and BLOB_DATA are not supported as input parameters, the layer above should always use BLOB esp_err_t Storage::writeItem(uint8_t nsIndex, ItemType datatype, const char* key, const void* data, size_t dataSize) { - if (mState != StorageState::ACTIVE) { + if(mState != StorageState::ACTIVE) { return ESP_ERR_NVS_NOT_INITIALIZED; } + // pointer to the page where the existing item was found Page* findPage = nullptr; + // index of the item in the page where the existing item was found + size_t itemIndex = 0; + // page sequence number helping to detect whether the page with old value was relocated during the new write + uint32_t findPageSeqNumber = UINT32_MAX; + + // indicates the datatype representation match between the old value and the new one bool matchedTypePageFound = false; + + // contains the item with the old value, if found Item item; - esp_err_t err; - if (datatype == ItemType::BLOB) { - err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item); - if(err == ESP_OK) { + esp_err_t err = ESP_OK; + + // Try to find existing item with the same key and namespace index + // We are performing the findItem with datatype specified (it is not ANY) to ensure the hash list lookup is done. + if(datatype == ItemType::BLOB) { + // Specific lookup if performed for input datatype BLOB. The searched datatype for exact match is BLOB_INDEX. + // The BLOB_INDEX is used to store the metadata of the (originally typed) BLOB data in current V2 implementation. + err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item, Page::CHUNK_ANY, VerOffset::VER_ANY, &itemIndex); + if(err == ESP_OK && findPage != nullptr) { matchedTypePageFound = true; } - } else { #ifdef CONFIG_NVS_LEGACY_DUP_KEYS_COMPATIBILITY - err = findItem(nsIndex, datatype, key, findPage, item); + // In legacy mode, we also try to find the item as BLOB if not found as BLOB_INDEX. + // In this mode, it is possible to have multiple active values under the same (logical) key. + // For BLOBs (which may have different physical representations in V1 it is BLOB, in V2 it is BLOB_INDEX) it in turn means + // that we have to check both datatypes to find the old value. + // The general case for compatibility flag disabled is below and handles all datatypes including BLOB. + // To save some cycles, we do not compile both findItem calls in this case. + if(err == ESP_ERR_NVS_NOT_FOUND) { + // If not found as BLOB_INDEX, try to find it as BLOB (legacy support). + err = findItem(nsIndex, ItemType::BLOB, key, findPage, item, Page::CHUNK_ANY, VerOffset::VER_ANY, &itemIndex); + if(err == ESP_OK && findPage != nullptr) { + matchedTypePageFound = false; // datatype does not match, we cannot extract chunkStart from the item + + // keep the sequence number of the page where the item was found for later check of relocation + err = findPage->getSeqNumber(findPageSeqNumber); + if(err != ESP_OK) { + return err; + } + } + } +#endif + } else { + // Handle all other data types than BLOB + err = findItem(nsIndex, datatype, key, findPage, item, Page::CHUNK_ANY, VerOffset::VER_ANY, &itemIndex); if(err == ESP_OK && findPage != nullptr) { matchedTypePageFound = true; + + // keep the sequence number of the page where the item was found for later check of relocation + err = findPage->getSeqNumber(findPageSeqNumber); + if(err != ESP_OK) { + return err; + } } -#else - err = findItem(nsIndex, ItemType::ANY, key, findPage, item); - if(err == ESP_OK && datatype == item.datatype) { - matchedTypePageFound = true; + } + +#ifndef CONFIG_NVS_LEGACY_DUP_KEYS_COMPATIBILITY + // If the item was not found under assumed datatype, try to find it as ANY. + if(findPage == nullptr) { + + // We should not find BLOB_DATA chunks as CHUNK_ANY is never used by the BLOB_DATA. + err = findItem(nsIndex, nvs::ItemType::ANY, key, findPage, item, Page::CHUNK_ANY, VerOffset::VER_ANY, &itemIndex); + if(err == ESP_OK && findPage != nullptr) { + // keep the sequence number of the page where the item was found for later check of relocation + err = findPage->getSeqNumber(findPageSeqNumber); + if(err != ESP_OK) { + return err; + } + // item was found with the same key and namespace index but data type is different + matchedTypePageFound = false; } -#endif } +#endif + // Here the findPage is either nullptr or points to the page where the item was found. + // The matchedTypePageFound is true if the old value item was found and its datatype representation matches the new one. + // This flag is used to determine if the item should be checked for same value. if (err != ESP_OK && err != ESP_ERR_NVS_NOT_FOUND) { return err; } + // Handle value update if (datatype == ItemType::BLOB) { VerOffset prevStart, nextStart; prevStart = nextStart = VerOffset::VER_0_OFFSET; if (matchedTypePageFound) { - // Do a sanity check that the item in question is actually being modified. + // Do a check that the item in question is actually being modified. // If it isn't, it is cheaper to purposefully not write out new data. // since it may invoke an erasure of flash. - if (cmpMultiPageBlob(nsIndex, key, data, dataSize) == ESP_OK) { + if(cmpMultiPageBlob(nsIndex, key, data, dataSize) == ESP_OK) { return ESP_OK; } - if (findPage->state() == Page::PageState::UNINITIALIZED || - findPage->state() == Page::PageState::INVALID) { - err = findItem(nsIndex, datatype, key, findPage, item); - if (err != ESP_OK) { - return err; - } - } - /* Get the version of the previous index with same */ + // Get the version of the previous index with same prevStart = item.blobIndex.chunkStart; NVS_ASSERT_OR_RETURN(prevStart == VerOffset::VER_0_OFFSET || prevStart == VerOffset::VER_1_OFFSET, ESP_FAIL); - - /* Toggle the version by changing the offset */ + // Toggle the version by changing the offset nextStart = (prevStart == VerOffset::VER_1_OFFSET) ? VerOffset::VER_0_OFFSET : VerOffset::VER_1_OFFSET; } - /* Write the blob with new version*/ + // Write the blob with new version err = writeMultiPageBlob(nsIndex, key, data, dataSize, nextStart); - if (err == ESP_ERR_NVS_PAGE_FULL) { + if(err == ESP_ERR_NVS_PAGE_FULL) { return ESP_ERR_NVS_NOT_ENOUGH_SPACE; } + if (err != ESP_OK) { return err; } - - if (matchedTypePageFound) { - /* Erase the blob with earlier version*/ - err = eraseMultiPageBlob(nsIndex, key, prevStart); - - if (err == ESP_ERR_FLASH_OP_FAIL) { - return ESP_ERR_NVS_REMOVE_FAILED; - } - if (err != ESP_OK) { - return err; - } - - findPage = nullptr; - } else { - /* Support for earlier versions where BLOBS were stored without index */ - err = findItem(nsIndex, datatype, key, findPage, item); - if (err != ESP_OK && err != ESP_ERR_NVS_NOT_FOUND) { - return err; - } - } } else { - // Do a sanity check that the item in question is actually being modified. + // Do a check that the item in question is actually being modified. // If it isn't, it is cheaper to purposefully not write out new data. // since it may invoke an erasure of flash. - if (matchedTypePageFound && + if(matchedTypePageFound && findPage->cmpItem(nsIndex, datatype, key, data, dataSize) == ESP_OK) { return ESP_OK; } Page& page = getCurrentPage(); err = page.writeItem(nsIndex, datatype, key, data, dataSize); - if (err == ESP_ERR_NVS_PAGE_FULL) { - if (page.state() != Page::PageState::FULL) { + if(err == ESP_ERR_NVS_PAGE_FULL) { + if(page.state() != Page::PageState::FULL) { err = page.markFull(); - if (err != ESP_OK) { + if(err != ESP_OK) { return err; } } err = mPageManager.requestNewPage(); - if (err != ESP_OK) { + if(err != ESP_OK) { return err; } err = getCurrentPage().writeItem(nsIndex, datatype, key, data, dataSize); - if (err == ESP_ERR_NVS_PAGE_FULL) { + if(err == ESP_ERR_NVS_PAGE_FULL) { return ESP_ERR_NVS_NOT_ENOUGH_SPACE; } - if (err != ESP_OK) { - return err; - } - } else if (err != ESP_OK) { + } + + if (err != ESP_OK) { return err; } } - if (findPage) { - if (findPage->state() == Page::PageState::UNINITIALIZED || - findPage->state() == Page::PageState::INVALID) { -#ifdef CONFIG_NVS_LEGACY_DUP_KEYS_COMPATIBILITY - err = findItem(nsIndex, datatype, key, findPage, item); -#else - err = findItem(nsIndex, ItemType::ANY, key, findPage, item); -#endif - if (err != ESP_OK) { + // Delete previous value + // Note: The old entry won't be deleted if the new value is the same as the old value - code won't reach here in that case. + + // If findPage is null then previous value was not present in NVS and nothig is to be deleted. + if(findPage == nullptr) { + return err; + } + + if(item.datatype == ItemType::BLOB_IDX) { + // If the item found was BLOB_INDEX, the eraseMultiPageBlob is used to erase the whole multi-page blob. + // It is not necessary to check the potential page relocation as the function will find the blob again anyway. + VerOffset prevStart = item.blobIndex.chunkStart; + err = eraseMultiPageBlob(nsIndex, key, prevStart); + + if(err == ESP_ERR_FLASH_OP_FAIL) { + return ESP_ERR_NVS_REMOVE_FAILED; + } + } else { + // For all other data types, we have to check the potential page relocation. + + // The findPage might have been relocated as a part of space reclaiming. + // First indication is the page state. It might become the "spare" page thus changing the state from FULL or ACTIVE. + bool wasRelocated = false; + + if( findPage->state() != Page::PageState::ACTIVE && + findPage->state() != Page::PageState::FULL) { + wasRelocated = true; + } + + // Other indication of the multi step relocation is the page sequence number. If the sequence number is different than page + // sequence number at the moment initial item was found, the page was relocated. + if(!wasRelocated) { + uint32_t newPageSeqNumber; + err = findPage->getSeqNumber(newPageSeqNumber); + if(err != ESP_OK) { return err; } + if(newPageSeqNumber != findPageSeqNumber) { + wasRelocated = true; + } } -#ifdef CONFIG_NVS_LEGACY_DUP_KEYS_COMPATIBILITY - err = findPage->eraseItem(nsIndex, datatype, key); -#else - err = findPage->eraseItem(nsIndex, ItemType::ANY, key); -#endif - if (err == ESP_ERR_FLASH_OP_FAIL) { - return ESP_ERR_NVS_REMOVE_FAILED; + + if(wasRelocated) { + // The page was relocated. We have to find the old value again from the beginning. + // As the item was already found before relocation, we can use the exact datatype from item. + err = findItem(nsIndex, item.datatype, key, findPage, item, Page::CHUNK_ANY, VerOffset::VER_ANY, &itemIndex); + if(err != ESP_OK) { + return err; + } } - if (err != ESP_OK) { - return err; + + // Page containing the old value is now refreshed. We can erase the old value. + err = findPage->eraseEntryAndSpan(itemIndex); + if(err == ESP_ERR_FLASH_OP_FAIL) { + return ESP_ERR_NVS_REMOVE_FAILED; } } + #ifdef DEBUG_STORAGE debugCheck(); #endif - return ESP_OK; + return err; } esp_err_t Storage::createOrOpenNamespace(const char* nsName, bool canCreate, uint8_t& nsIndex) { - if (mState != StorageState::ACTIVE) { + if(mState != StorageState::ACTIVE) { return ESP_ERR_NVS_NOT_INITIALIZED; } auto it = std::find_if(mNamespaces.begin(), mNamespaces.end(), [=] (const NamespaceEntry& e) -> bool { return strncmp(nsName, e.mName, sizeof(e.mName) - 1) == 0; }); - if (it == std::end(mNamespaces)) { - if (!canCreate) { + if(it == std::end(mNamespaces)) { + if(!canCreate) { return ESP_ERR_NVS_NOT_FOUND; } uint8_t ns; bool ns_state; - for (ns = 1; ns < 255; ++ns) { - if (mNamespaceUsage.get(ns, &ns_state) != ESP_OK) { + for(ns = 1; ns < 255; ++ns) { + if(mNamespaceUsage.get(ns, &ns_state) != ESP_OK) { return ESP_FAIL; } - if (!ns_state) { + if(!ns_state) { break; } } - if (ns == 255) { + if(ns == 255) { return ESP_ERR_NVS_NOT_ENOUGH_SPACE; } auto err = writeItem(Page::NS_INDEX, ItemType::U8, nsName, &ns, sizeof(ns)); - if (err != ESP_OK) { + if(err != ESP_OK) { return err; } - if (mNamespaceUsage.set(ns, true) != ESP_OK) { + if(mNamespaceUsage.set(ns, true) != ESP_OK) { return ESP_FAIL; } nsIndex = ns; NamespaceEntry* entry = new (std::nothrow) NamespaceEntry; - if (!entry) { + if(!entry) { return ESP_ERR_NO_MEM; } @@ -584,10 +652,12 @@ esp_err_t Storage::readMultiPageBlob(uint8_t nsIndex, const char* key, void* dat { Item item; Page* findPage = nullptr; + size_t itemIndex = 0; + - /* First read the blob index */ + // First read the blob index auto err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item); - if (err != ESP_OK) { + if(err != ESP_OK) { return err; } @@ -597,30 +667,34 @@ esp_err_t Storage::readMultiPageBlob(uint8_t nsIndex, const char* key, void* dat NVS_ASSERT_OR_RETURN(dataSize == item.blobIndex.dataSize, ESP_FAIL); - /* Now read corresponding chunks */ - for (uint8_t chunkNum = 0; chunkNum < chunkCount; chunkNum++) { - err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, static_cast (chunkStart) + chunkNum); - if (err != ESP_OK) { - if (err == ESP_ERR_NVS_NOT_FOUND) { + // Now read related blob data chunks + // Remember the itemIndex as it is used to fast locate the entry in the page + for(uint8_t chunkNum = 0; chunkNum < chunkCount; chunkNum++) { + err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, static_cast (chunkStart) + chunkNum, nvs::VerOffset::VER_ANY, &itemIndex); + if(err != ESP_OK) { + if(err == ESP_ERR_NVS_NOT_FOUND) { break; } return err; } - if (item.varLength.dataSize > dataSize - offset) { - /* The size of the entry in the index is inconsistent with the sum of the sizes of chunks */ + + // Check if the blob data chunk length indicated for actual item still fits into the total length of the buffer + if(item.varLength.dataSize > dataSize - offset) { err = ESP_ERR_NVS_INVALID_LENGTH; break; } - err = findPage->readItem(nsIndex, ItemType::BLOB_DATA, key, static_cast(data) + offset, item.varLength.dataSize, static_cast (chunkStart) + chunkNum); - if (err != ESP_OK) { + + err = findPage->readVariableLengthItemData(item, itemIndex, static_cast(data) + offset); + if(err != ESP_OK) { return err; } + NVS_ASSERT_OR_RETURN(static_cast (chunkStart) + chunkNum == item.chunkIndex, ESP_FAIL); offset += item.varLength.dataSize; } - if (err == ESP_ERR_NVS_NOT_FOUND || err == ESP_ERR_NVS_INVALID_LENGTH) { + if(err == ESP_ERR_NVS_NOT_FOUND || err == ESP_ERR_NVS_INVALID_LENGTH) { // cleanup if a chunk is not found or the size is inconsistent eraseMultiPageBlob(nsIndex, key); } @@ -634,10 +708,11 @@ esp_err_t Storage::cmpMultiPageBlob(uint8_t nsIndex, const char* key, const void { Item item; Page* findPage = nullptr; + size_t itemIndex = 0; - /* First read the blob index */ + // First read the blob index auto err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item); - if (err != ESP_OK) { + if(err != ESP_OK) { return err; } @@ -646,21 +721,34 @@ esp_err_t Storage::cmpMultiPageBlob(uint8_t nsIndex, const char* key, const void size_t readSize = item.blobIndex.dataSize; size_t offset = 0; - if (dataSize != readSize) { + if(dataSize != readSize) { return ESP_ERR_NVS_CONTENT_DIFFERS; } - /* Now read corresponding chunks */ - for (uint8_t chunkNum = 0; chunkNum < chunkCount; chunkNum++) { - err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, static_cast (chunkStart) + chunkNum); - if (err != ESP_OK) { - if (err == ESP_ERR_NVS_NOT_FOUND) { + // Now read corresponding chunks + for(uint8_t chunkNum = 0; chunkNum < chunkCount; chunkNum++) { + err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, static_cast (chunkStart) + chunkNum, nvs::VerOffset::VER_ANY, &itemIndex); + if(err != ESP_OK) { + if(err == ESP_ERR_NVS_NOT_FOUND) { break; } return err; } + + if(item.varLength.dataSize > dataSize - offset) { + // The size of the entry in the index is bigger than the size of the remaining data to be compared + return ESP_ERR_NVS_CONTENT_DIFFERS; + } + + // calculate crc32 of the incoming data window related to the BLOB_DATA chunk and compare it with the crc32 from the BLOB_DATA metadata entry + // Different crc32 indicates data mismatch. + // If crc32 matches, we have to compare the data in the chunk with the buffer data to exclude crc32 collision. + if (Item::calculateCrc32(reinterpret_cast(data), item.varLength.dataSize) != item.varLength.dataCrc32) { + return ESP_ERR_NVS_CONTENT_DIFFERS; + } + err = findPage->cmpItem(nsIndex, ItemType::BLOB_DATA, key, static_cast(data) + offset, item.varLength.dataSize, static_cast (chunkStart) + chunkNum); - if (err != ESP_OK) { + if(err != ESP_OK) { return err; } NVS_ASSERT_OR_RETURN(static_cast (chunkStart) + chunkNum == item.chunkIndex, ESP_FAIL); @@ -674,21 +762,21 @@ esp_err_t Storage::cmpMultiPageBlob(uint8_t nsIndex, const char* key, const void esp_err_t Storage::readItem(uint8_t nsIndex, ItemType datatype, const char* key, void* data, size_t dataSize) { - if (mState != StorageState::ACTIVE) { + if(mState != StorageState::ACTIVE) { return ESP_ERR_NVS_NOT_INITIALIZED; } Item item; Page* findPage = nullptr; - if (datatype == ItemType::BLOB) { + if(datatype == ItemType::BLOB) { auto err = readMultiPageBlob(nsIndex, key, data, dataSize); - if (err != ESP_ERR_NVS_NOT_FOUND) { + if(err != ESP_ERR_NVS_NOT_FOUND) { return err; } // else check if the blob is stored with earlier version format without index } auto err = findItem(nsIndex, datatype, key, findPage, item); - if (err != ESP_OK) { + if(err != ESP_OK) { return err; } return findPage->readItem(nsIndex, datatype, key, data, dataSize); @@ -697,67 +785,100 @@ esp_err_t Storage::readItem(uint8_t nsIndex, ItemType datatype, const char* key, esp_err_t Storage::eraseMultiPageBlob(uint8_t nsIndex, const char* key, VerOffset chunkStart) { - if (mState != StorageState::ACTIVE) { + if(mState != StorageState::ACTIVE) { return ESP_ERR_NVS_NOT_INITIALIZED; } Item item; Page* findPage = nullptr; + size_t itemIndex = 0; + uint8_t chunkCount = 0; - auto err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item, Page::CHUNK_ANY, chunkStart); - if (err != ESP_OK) { + auto err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item, Page::CHUNK_ANY, chunkStart, &itemIndex); + if(err != ESP_OK) { return err; } + + chunkCount = item.blobIndex.chunkCount; + // Erase the index first and make children blobs orphan - err = findPage->eraseItem(nsIndex, ItemType::BLOB_IDX, key, Page::CHUNK_ANY, chunkStart); - if (err != ESP_OK) { + err = findPage->eraseEntryAndSpan(itemIndex); + if(err != ESP_OK) { return err; } // If caller requires delete of VER_ANY // We may face dirty NVS partition and version duplicates can be there // Make second attempt to delete index and ignore eventual not found - if(chunkStart == VerOffset::VER_ANY) - { - err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item, Page::CHUNK_ANY, chunkStart); - if (err == ESP_OK) { - err = findPage->eraseItem(nsIndex, ItemType::BLOB_IDX, key, Page::CHUNK_ANY, chunkStart); - if (err != ESP_OK) { + if(chunkStart == VerOffset::VER_ANY) { + // Specific case called during initialisation of the storage + // We need to delete all chunks with the same key and namespace index + + // If there exists another BLOB_IDX with the same key and namespace index, delete it + // Ignore potential error if the item is not found + err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item, Page::CHUNK_ANY, chunkStart, &itemIndex); + if(err == ESP_OK) { + err = findPage->eraseEntryAndSpan(itemIndex); + if(err != ESP_OK) { return err; } - } else if (err != ESP_ERR_NVS_NOT_FOUND) { + } else if(err != ESP_ERR_NVS_NOT_FOUND) { return err; } - } - - // setup limits for chunkIndex-es to be deleted - uint8_t minChunkIndex = (uint8_t) VerOffset::VER_0_OFFSET; - uint8_t maxChunkIndex = (uint8_t) VerOffset::VER_ANY; - if(chunkStart == VerOffset::VER_0_OFFSET) { - maxChunkIndex = (uint8_t) VerOffset::VER_1_OFFSET; - } else if (chunkStart == VerOffset::VER_1_OFFSET) { - minChunkIndex = (uint8_t) VerOffset::VER_1_OFFSET; - } - - for (auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) { - size_t itemIndex = 0; - do { - err = it->findItem(nsIndex, ItemType::BLOB_DATA, key, itemIndex, item); - if (err == ESP_ERR_NVS_NOT_FOUND) { - break; - } else if (err == ESP_OK) { - // check if item.chunkIndex is within the version range indicated by chunkStart, if so, delete it - if((item.chunkIndex >= minChunkIndex) && (item.chunkIndex < maxChunkIndex)) { + // To delete all chunks, we will visit every page and delete all chunks regardless of chunkIndex + // This approach cannot use the hash list as the chunkIndex is not known. + for(auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) { + // reset itemIndex to zero for each page to search from the beginning + itemIndex = 0; + do { + // (Re)Try to find the item at the position starting at the itemIndex + err = it->findItem(nsIndex, ItemType::BLOB_DATA, key, itemIndex, item); + + // If the item is not found, we can break the actual loop and continue with the next page + if(err == ESP_ERR_NVS_NOT_FOUND) { + break; + } else if(err == ESP_OK) { err = it->eraseEntryAndSpan(itemIndex); + + // advance itemIndex to the next potential entry on the page + // findItem checks the consistency of the entry metadata so we can safely assume the span is non-zero + itemIndex += item.span; } + if(err != ESP_OK) { + return err; + } + // Continue the loop until all items on the current page are found and erased + } while(err == ESP_OK && itemIndex < Page::ENTRY_COUNT); + } - // continue findItem until end of page - itemIndex += item.span; + } else { + // Most common condition + // The caller has specified the chunk version, delete all chunks within the chunk index range indicated by the BLOB_IDX entry + // The loop will iterate the chunk index, page will be found and chunk index will be erased + // This approach uses the hash list to find the item on the page, so it is efficient. + uint8_t minChunkIndex = (uint8_t) VerOffset::VER_ANY; + uint8_t maxChunkIndex = (uint8_t) VerOffset::VER_ANY; + + if(chunkStart == VerOffset::VER_0_OFFSET) { + minChunkIndex = (uint8_t) VerOffset::VER_0_OFFSET; + maxChunkIndex = minChunkIndex + chunkCount; + } else if(chunkStart == VerOffset::VER_1_OFFSET) { + minChunkIndex = (uint8_t) VerOffset::VER_1_OFFSET; + maxChunkIndex = minChunkIndex + chunkCount; + } + + for(uint8_t chunkIndex = minChunkIndex; chunkIndex < maxChunkIndex; chunkIndex++) { + err = findItem(nsIndex, ItemType::BLOB_DATA, key, findPage, item, chunkIndex, nvs::VerOffset::VER_ANY, &itemIndex); + if(err != ESP_OK) { + return err; } + + // Erase the entry + err = findPage->eraseEntryAndSpan(itemIndex); if(err != ESP_OK) { return err; } - } while (err == ESP_OK && itemIndex < Page::ENTRY_COUNT); + } } return ESP_OK; @@ -765,41 +886,40 @@ esp_err_t Storage::eraseMultiPageBlob(uint8_t nsIndex, const char* key, VerOffse esp_err_t Storage::eraseItem(uint8_t nsIndex, ItemType datatype, const char* key) { - if (mState != StorageState::ACTIVE) { + if(mState != StorageState::ACTIVE) { return ESP_ERR_NVS_NOT_INITIALIZED; } - if (datatype == ItemType::BLOB) { - return eraseMultiPageBlob(nsIndex, key); - } - Item item; Page* findPage = nullptr; - auto err = findItem(nsIndex, datatype, key, findPage, item); - if (err != ESP_OK) { + esp_err_t err = ESP_OK; + size_t itemIndex = 0; + + err = findItem(nsIndex, datatype, key, findPage, item, Page::CHUNK_ANY, VerOffset::VER_ANY, &itemIndex); + if(err != ESP_OK) { return err; } - - if (item.datatype == ItemType::BLOB_DATA || item.datatype == ItemType::BLOB_IDX) { - return eraseMultiPageBlob(nsIndex, key); + // If the item found is BLOB_IDX, the eraseMultiPageBlob is used to erase the whole multi-page blob. + if (item.datatype == ItemType::BLOB_IDX) { + return eraseMultiPageBlob(nsIndex, key, item.blobIndex.chunkStart); } - return findPage->eraseItem(nsIndex, datatype, key); + return findPage->eraseEntryAndSpan(itemIndex); } esp_err_t Storage::eraseNamespace(uint8_t nsIndex) { - if (mState != StorageState::ACTIVE) { + if(mState != StorageState::ACTIVE) { return ESP_ERR_NVS_NOT_INITIALIZED; } - for (auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) { - while (true) { + for(auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) { + while(true) { auto err = it->eraseItem(nsIndex, ItemType::ANY, nullptr); - if (err == ESP_ERR_NVS_NOT_FOUND) { + if(err == ESP_ERR_NVS_NOT_FOUND) { break; } - else if (err != ESP_OK) { + else if(err != ESP_OK) { return err; } } @@ -810,14 +930,14 @@ esp_err_t Storage::eraseNamespace(uint8_t nsIndex) esp_err_t Storage::findKey(const uint8_t nsIndex, const char* key, ItemType* datatype) { - if (mState != StorageState::ACTIVE) { + if(mState != StorageState::ACTIVE) { return ESP_ERR_NVS_NOT_INITIALIZED; } Item item; Page* findPage = nullptr; auto err = findItem(nsIndex, ItemType::ANY, key, findPage, item); - if (err != ESP_OK) { + if(err != ESP_OK) { return err; } @@ -830,32 +950,37 @@ esp_err_t Storage::findKey(const uint8_t nsIndex, const char* key, ItemType* dat esp_err_t Storage::getItemDataSize(uint8_t nsIndex, ItemType datatype, const char* key, size_t& dataSize) { - if (mState != StorageState::ACTIVE) { + if(mState != StorageState::ACTIVE) { return ESP_ERR_NVS_NOT_INITIALIZED; } Item item; Page* findPage = nullptr; - auto err = findItem(nsIndex, datatype, key, findPage, item); - if (err != ESP_OK) { - if (datatype != ItemType::BLOB) { - return err; - } + esp_err_t err = ESP_OK; + + // If requested datatype is BLOB, first try to find the item with datatype BLOB_IDX - new format + // If not found, try to find the item with datatype BLOB - old format. + if(datatype == ItemType::BLOB) { err = findItem(nsIndex, ItemType::BLOB_IDX, key, findPage, item); - if (err != ESP_OK) { + if(err == ESP_OK) { + dataSize = item.blobIndex.dataSize; + return err; + } else if(err != ESP_ERR_NVS_NOT_FOUND) { return err; } - dataSize = item.blobIndex.dataSize; - return ESP_OK; } + err = findItem(nsIndex, datatype, key, findPage, item); + if(err != ESP_OK) { + return err; + } dataSize = item.varLength.dataSize; return ESP_OK; } void Storage::debugDump() { - for (auto p = mPageManager.begin(); p != mPageManager.end(); ++p) { + for(auto p = mPageManager.begin(); p != mPageManager.end(); ++p) { p->debugDump(); } } @@ -865,15 +990,15 @@ void Storage::debugCheck() { std::map keys; - for (auto p = mPageManager.begin(); p != mPageManager.end(); ++p) { + for(auto p = mPageManager.begin(); p != mPageManager.end(); ++p) { size_t itemIndex = 0; size_t usedCount = 0; Item item; - while (p->findItem(Page::NS_ANY, ItemType::ANY, nullptr, itemIndex, item) == ESP_OK) { + while(p->findItem(Page::NS_ANY, ItemType::ANY, nullptr, itemIndex, item) == ESP_OK) { std::stringstream keyrepr; keyrepr << static_cast(item.nsIndex) << "_" << static_cast(item.datatype) << "_" << item.key <<"_"<(item.chunkIndex); std::string keystr = keyrepr.str(); - if (keys.find(keystr) != std::end(keys)) { + if(keys.find(keystr) != std::end(keys)) { printf("Duplicate key: %s\n", keystr.c_str()); debugDump(); assert(0); @@ -897,24 +1022,24 @@ esp_err_t Storage::calcEntriesInNamespace(uint8_t nsIndex, size_t& usedEntries) { usedEntries = 0; - if (mState != StorageState::ACTIVE) { + if(mState != StorageState::ACTIVE) { return ESP_ERR_NVS_NOT_INITIALIZED; } - for (auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) { + for(auto it = std::begin(mPageManager); it != std::end(mPageManager); ++it) { size_t itemIndex = 0; Item item; - while (true) { + while(true) { auto err = it->findItem(nsIndex, ItemType::ANY, nullptr, itemIndex, item); - if (err == ESP_ERR_NVS_NOT_FOUND) { + if(err == ESP_ERR_NVS_NOT_FOUND) { break; } - else if (err != ESP_OK) { + else if(err != ESP_OK) { return err; } usedEntries += item.span; itemIndex += item.span; - if (itemIndex >= it->ENTRY_COUNT) break; + if(itemIndex >= it->ENTRY_COUNT) break; } } return ESP_OK; @@ -926,7 +1051,7 @@ void Storage::fillEntryInfo(Item &item, nvs_entry_info_t &info) strncpy(info.key, item.key, sizeof(info.key) - 1); info.key[sizeof(info.key) - 1] = '\0'; - for (auto &name : mNamespaces) { + for(auto &name : mNamespaces) { if(item.nsIndex == name.mIndex) { strlcpy(info.namespace_name, name.mName, sizeof(info.namespace_name)); break; @@ -940,7 +1065,7 @@ bool Storage::findEntry(nvs_opaque_iterator_t* it, const char* namespace_name) it->nsIndex = Page::NS_ANY; it->page = mPageManager.begin(); - if (namespace_name != nullptr) { + if(namespace_name != nullptr) { if(createOrOpenNamespace(namespace_name, false, it->nsIndex) != ESP_OK) { return false; } @@ -977,7 +1102,7 @@ bool Storage::nextEntry(nvs_opaque_iterator_t* it) Item item; esp_err_t err; - for (auto page = it->page; page != mPageManager.end(); ++page) { + for(auto page = it->page; page != mPageManager.end(); ++page) { do { err = page->findItem(it->nsIndex, (ItemType)it->type, nullptr, it->entryIndex, item); it->entryIndex += item.span; @@ -986,7 +1111,7 @@ bool Storage::nextEntry(nvs_opaque_iterator_t* it) it->page = page; return true; } - } while (err != ESP_ERR_NVS_NOT_FOUND); + } while(err != ESP_ERR_NVS_NOT_FOUND); it->entryIndex = 0; } diff --git a/components/nvs_flash/src/nvs_storage.hpp b/components/nvs_flash/src/nvs_storage.hpp index 3ea642e773b5..1c63d1c8f087 100644 --- a/components/nvs_flash/src/nvs_storage.hpp +++ b/components/nvs_flash/src/nvs_storage.hpp @@ -153,7 +153,7 @@ class Storage : public intrusive_list_node, public ExceptionlessAllocat void fillEntryInfo(Item &item, nvs_entry_info_t &info); - esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item, uint8_t chunkIdx = Page::CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY); + esp_err_t findItem(uint8_t nsIndex, ItemType datatype, const char* key, Page* &page, Item& item, uint8_t chunkIdx = Page::CHUNK_ANY, VerOffset chunkStart = VerOffset::VER_ANY, size_t* itemIndex = NULL); protected: Partition *mPartition; diff --git a/components/nvs_flash/src/nvs_types.cpp b/components/nvs_flash/src/nvs_types.cpp index 24073fc30697..81379c8591ed 100644 --- a/components/nvs_flash/src/nvs_types.cpp +++ b/components/nvs_flash/src/nvs_types.cpp @@ -33,9 +33,12 @@ uint32_t Item::calculateCrc32WithoutValue() const return result; } -uint32_t Item::calculateCrc32(const uint8_t* data, size_t size) +uint32_t Item::calculateCrc32(const uint8_t* data, size_t size, uint32_t* initial_crc32) { uint32_t result = 0xffffffff; + if(initial_crc32) { + result = *initial_crc32; + } result = esp_rom_crc32_le(result, data, size); return result; } diff --git a/components/nvs_flash/src/nvs_types.hpp b/components/nvs_flash/src/nvs_types.hpp index 5eda9abcd990..5cb3056a45e0 100644 --- a/components/nvs_flash/src/nvs_types.hpp +++ b/components/nvs_flash/src/nvs_types.hpp @@ -93,7 +93,7 @@ class Item uint32_t calculateCrc32() const; uint32_t calculateCrc32WithoutValue() const; - static uint32_t calculateCrc32(const uint8_t* data, size_t size); + static uint32_t calculateCrc32(const uint8_t* data, size_t size, uint32_t* initial_crc32 = nullptr); void getKey(char* dst, size_t dstSize) { diff --git a/components/soc/esp32p4/include/soc/clk_tree_defs.h b/components/soc/esp32p4/include/soc/clk_tree_defs.h index bd4c7a51b2a6..f9df80221c5f 100644 --- a/components/soc/esp32p4/include/soc/clk_tree_defs.h +++ b/components/soc/esp32p4/include/soc/clk_tree_defs.h @@ -338,7 +338,8 @@ typedef enum { * @brief I2S clock source enum */ typedef enum { - I2S_CLK_SRC_DEFAULT = SOC_MOD_CLK_XTAL, /*!< Select XTAL as the default source clock */ + I2S_CLK_SRC_DEFAULT = 0, /*!< Auto select maximum clock source asdefault source clock */ + I2S_CLK_SRC_PLL_160M = SOC_MOD_CLK_PLL_F160M, /*!< Select PLL_F160M as the source clock (only supported on P4 hw_ver3) */ I2S_CLK_SRC_XTAL = SOC_MOD_CLK_XTAL, /*!< Select XTAL as the source clock */ I2S_CLK_SRC_APLL = SOC_MOD_CLK_APLL, /*!< Select APLL as the source clock */ I2S_CLK_SRC_EXTERNAL = -1, /*!< Select external clock as source clock */ diff --git a/examples/openthread/ot_ci_function.py b/examples/openthread/ot_ci_function.py index 0c438e9f601c..bc32af980cb7 100644 --- a/examples/openthread/ot_ci_function.py +++ b/examples/openthread/ot_ci_function.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Unlicense OR CC0-1.0 # !/usr/bin/env python3 # this file defines some functions for testing cli and br under pytest framework +import logging import os import re import socket @@ -10,6 +11,7 @@ import time from functools import wraps from typing import Callable +from typing import Optional from typing import Tuple import netifaces @@ -19,19 +21,33 @@ def extract_address( - command: str, pattern: str, default_return: str = '' + command: str, + pattern: str, + default_return: str = '', + retries: int = 3, + delay: int = 2, ) -> Callable[[Callable[[str], str]], Callable[[IdfDut], str]]: def decorator(func: Callable[[str], str]) -> Callable[[IdfDut], str]: @wraps(func) def wrapper(dut: IdfDut) -> str: - clean_buffer(dut) - execute_command(dut, command) - try: - result = dut.expect(pattern, timeout=5)[1].decode() - except Exception as e: - print(f'Error: {e}') - return default_return - return func(result) + # requires Python3.10 + # last_exception: Exception | None = None + last_exception: Optional[Exception] = None + for attempt in range(1, retries + 1): + try: + clean_buffer(dut) + execute_command(dut, command) + result = dut.expect(pattern, timeout=5)[1].decode() + return func(result) + except Exception as e: + logging.exception(f'[{command}] Attempt {attempt}/{retries} failed: {e}') + last_exception = e + if attempt < retries: + time.sleep(delay) + + if last_exception: + logging.exception(f'[{command}] Giving up after {retries} retries.') + return default_return return wrapper @@ -152,7 +168,7 @@ def getDeviceRole(dut: IdfDut) -> str: wait(dut, 1) execute_command(dut, 'state') role = dut.expect(r'\W+(\w+)\W+Done', timeout=5)[1].decode() - print(role) + logging.info(role) return str(role) @@ -173,6 +189,12 @@ def init_thread(dut: IdfDut) -> None: reset_thread(dut) +def stop_thread(dut: IdfDut) -> None: + execute_command(dut, 'thread stop') + dut.expect('disabled', timeout=20) + reset_thread(dut) + + def reset_thread(dut: IdfDut) -> None: execute_command(dut, 'factoryreset') dut.expect('OpenThread attached to netif', timeout=20) @@ -180,28 +202,28 @@ def reset_thread(dut: IdfDut) -> None: clean_buffer(dut) +def hardreset_dut(dut: IdfDut) -> None: + dut.serial.hard_reset() + time.sleep(5) + execute_command(dut, 'factoryreset') + + # get the mleid address of the thread -def get_mleid_addr(dut: IdfDut) -> str: - dut_adress = '' - execute_command(dut, 'ipaddr mleid') - dut_adress = dut.expect(r'\n((?:\w+:){7}\w+)\r', timeout=5)[1].decode() - return dut_adress +@extract_address('ipaddr mleid', r'\n((?:\w+:){7}\w+)\r') +def get_mleid_addr(addr: str) -> str: + return addr # get the rloc address of the thread -def get_rloc_addr(dut: IdfDut) -> str: - dut_adress = '' - execute_command(dut, 'ipaddr rloc') - dut_adress = dut.expect(r'\n((?:\w+:){7}\w+)\r', timeout=5)[1].decode() - return dut_adress +@extract_address('ipaddr rloc', r'\n((?:\w+:){7}\w+)\r') +def get_rloc_addr(addr: str) -> str: + return addr # get the linklocal address of the thread -def get_linklocal_addr(dut: IdfDut) -> str: - dut_adress = '' - execute_command(dut, 'ipaddr linklocal') - dut_adress = dut.expect(r'\n((?:\w+:){7}\w+)\r', timeout=5)[1].decode() - return dut_adress +@extract_address('ipaddr linklocal', r'\n((?:\w+:){7}\w+)\r') +def get_linklocal_addr(addr: str) -> str: + return addr # get the global unicast address of the thread: @@ -311,7 +333,7 @@ def get_host_interface_name() -> str: interface_name = config.get('interface_name') if interface_name: if interface_name == 'eth0': - print( + logging.warning( f"Warning: 'eth0' is not recommended as a valid network interface. " f"Please check and update the 'interface_name' in the configuration file: " f'{config_path}' @@ -319,9 +341,9 @@ def get_host_interface_name() -> str: else: return str(interface_name) else: - print("Warning: Configuration file found but 'interface_name' is not defined.") + logging.warning("Warning: Configuration file found but 'interface_name' is not defined.") except Exception as e: - print(f'Error: Failed to read or parse {config_path}. Details: {e}') + logging.error(f'Error: Failed to read or parse {config_path}. Details: {e}') if 'eth1' in netifaces.interfaces(): return 'eth1' @@ -339,8 +361,8 @@ def check_if_host_receive_ra(br: IdfDut) -> bool: omrprefix = get_omrprefix(br) command = 'ip -6 route | grep ' + str(interface_name) out_str = subprocess.getoutput(command) - print('br omrprefix: ', str(omrprefix)) - print('host route table:\n', str(out_str)) + logging.info(f'br omrprefix: {omrprefix}') + logging.info(f'host route table:\n {out_str}') return str(omrprefix) in str(out_str) @@ -405,7 +427,7 @@ def create_host_udp_server(myudp: udp_parameter) -> None: AF_INET = socket.AF_INET6 else: AF_INET = socket.AF_INET - print('The host start to create udp server!') + logging.info('The host start to create udp server!') if_index = socket.if_nametoindex(interface_name) sock = socket.socket(AF_INET, socket.SOCK_DGRAM) sock.bind((myudp.addr, myudp.port)) @@ -418,13 +440,14 @@ def create_host_udp_server(myudp: udp_parameter) -> None: ) sock.settimeout(myudp.timeout) myudp.init_flag = True - print('The host start to receive message!') + logging.info('The host start to receive message!') myudp.udp_bytes = (sock.recvfrom(1024))[0] - print('The host has received message: ', myudp.udp_bytes) - except socket.error: - print('The host did not receive message!') + udp_str = str(myudp.udp_bytes) + logging.info(f'The host has received message: {udp_str}') + except OSError: + logging.error('The host did not receive message!') finally: - print('Close the socket.') + logging.info('Close the socket.') sock.close() @@ -439,10 +462,10 @@ def host_udp_send_message(udp_target: udp_parameter) -> None: sock.bind(('::', 12350)) sock.setsockopt(socket.SOL_SOCKET, socket.SO_BINDTODEVICE, interface_name.encode()) sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_HOPS, 32) - print('Host is sending message') + logging.info('Host is sending message') sock.sendto(udp_target.udp_bytes, (udp_target.addr, udp_target.port)) - except socket.error: - print('Host cannot send message') + except OSError: + logging.error('Host cannot send message') finally: sock.close() @@ -482,13 +505,13 @@ def host_close_service() -> None: command = 'ps auxww | grep avahi-publish-s' out_bytes = subprocess.check_output(command, shell=True, timeout=5) out_str = out_bytes.decode('utf-8') - print('host close service avahi status:\n', out_str) + logging.info(f'host close service avahi status:\n {out_str}') service_info = [line for line in out_str.splitlines() if 'testxxx _testxxx._udp' in line] for line in service_info: - print('Process:', line) + logging.info(f'Process:{line}') pid = line.split()[1] command = 'kill -9 ' + pid - print('kill ', pid) + logging.info(f'kill {pid}') subprocess.call(command, shell=True, timeout=5) time.sleep(1) @@ -521,24 +544,24 @@ def open_host_interface() -> None: def get_domain() -> str: hostname = socket.gethostname() - print('hostname is: ', hostname) + logging.info(f'hostname is: {hostname}') command = 'ps -auxww | grep avahi-daemon | grep running' out_str = subprocess.getoutput(command) - print('avahi status:\n', out_str) + logging.info(f'avahi status:\n {out_str}') role = re.findall(r'\[([\w\W]+)\.local\]', str(out_str))[0] - print('active host is: ', role) + logging.info(f'active host is: {role}') return str(role) def flush_ipv6_addr_by_interface() -> None: interface_name = get_host_interface_name() - print(f'flush ipv6 addr : {interface_name}') + logging.info(f'flush ipv6 addr : {interface_name}') command_show_addr = f'ip -6 addr show dev {interface_name}' command_show_route = f'ip -6 route show dev {interface_name}' addr_before = subprocess.getoutput(command_show_addr) route_before = subprocess.getoutput(command_show_route) - print(f'Before flush, IPv6 addresses: \n{addr_before}') - print(f'Before flush, IPv6 routes: \n{route_before}') + logging.info(f'Before flush, IPv6 addresses: \n{addr_before}') + logging.info(f'Before flush, IPv6 routes: \n{route_before}') subprocess.run(['ip', 'link', 'set', interface_name, 'down']) subprocess.run(['ip', '-6', 'addr', 'flush', 'dev', interface_name]) subprocess.run(['ip', '-6', 'route', 'flush', 'dev', interface_name]) @@ -546,8 +569,8 @@ def flush_ipv6_addr_by_interface() -> None: time.sleep(5) addr_after = subprocess.getoutput(command_show_addr) route_after = subprocess.getoutput(command_show_route) - print(f'After flush, IPv6 addresses: \n{addr_after}') - print(f'After flush, IPv6 routes: \n{route_after}') + logging.info(f'After flush, IPv6 addresses: \n{addr_after}') + logging.info(f'After flush, IPv6 routes: \n{route_after}') class tcp_parameter: @@ -576,28 +599,29 @@ def create_host_tcp_server(mytcp: tcp_parameter) -> None: AF_INET = socket.AF_INET6 else: AF_INET = socket.AF_INET - print('The host start to create a tcp server!') + logging.info('The host start to create a tcp server!') sock = socket.socket(AF_INET, socket.SOCK_STREAM) sock.bind((mytcp.addr, mytcp.port)) sock.listen(5) mytcp.listen_flag = True - print('The tcp server is waiting for connection!') + logging.info('The tcp server is waiting for connection!') sock.settimeout(mytcp.timeout) connfd, addr = sock.accept() - print('The tcp server connected with ', addr) + logging.info(f'The tcp server connected with {addr}') mytcp.recv_flag = True mytcp.tcp_bytes = connfd.recv(1024) - print('The tcp server has received message: ', mytcp.tcp_bytes) + tcp_str = str(mytcp.tcp_bytes) + logging.info(f'The tcp server has received message: {tcp_str}') except socket.error: if mytcp.recv_flag: - print('The tcp server did not receive message!') + logging.error('The tcp server did not receive message!') else: - print('The tcp server fail to connect!') + logging.error('The tcp server fail to connect!') finally: - print('Close the socket.') + logging.info('Close the socket.') sock.close() @@ -617,22 +641,19 @@ def decimal_to_hex(decimal_str: str) -> str: return hex_str -def get_omrprefix(br: IdfDut) -> str: - execute_command(br, 'br omrprefix') - omrprefix = br.expect(r'Local: ((?:\w+:){4}):/\d+\r', timeout=5)[1].decode() - return str(omrprefix) +@extract_address('br omrprefix', r'Local: ((?:\w+:){4}):/\d+\r') +def get_omrprefix(addr: str) -> str: + return addr -def get_onlinkprefix(br: IdfDut) -> str: - execute_command(br, 'br onlinkprefix') - onlinkprefix = br.expect(r'Local: ((?:\w+:){4}):/\d+\r', timeout=5)[1].decode() - return str(onlinkprefix) +@extract_address('br onlinkprefix', r'Local: ((?:\w+:){4}):/\d+\r') +def get_onlinkprefix(addr: str) -> str: + return addr -def get_nat64prefix(br: IdfDut) -> str: - execute_command(br, 'br nat64prefix') - nat64prefix = br.expect(r'Local: ((?:\w+:){6}):/\d+', timeout=5)[1].decode() - return str(nat64prefix) +@extract_address('br nat64prefix', r'Local: ((?:\w+:){6}):/\d+') +def get_nat64prefix(addr: str) -> str: + return addr def execute_command(dut: IdfDut, command: str, prefix: str = 'ot ') -> None: @@ -645,3 +666,17 @@ def get_ouput_string(dut: IdfDut, command: str, wait_time: int) -> str: tmp = dut.expect(pexpect.TIMEOUT, timeout=wait_time) clean_buffer(dut) return str(tmp) + + +def wait_for_host_network(host: str = '8.8.8.8', retries: int = 6, interval: int = 10) -> None: + for attempt in range(1, retries + 1): + try: + subprocess.run(['ping', '-c', '1', '-W', '2', host], check=True) + logging.info(f'Host network reachable on attempt {attempt}') + return + except subprocess.CalledProcessError: + logging.info(f'Ping attempt {attempt} failed, retrying in {interval} seconds...') + if attempt < retries: + time.sleep(interval) + else: + raise RuntimeError(f'Host network is not reachable after {retries} attempts.') diff --git a/examples/openthread/pytest_otbr.py b/examples/openthread/pytest_otbr.py index 36b880179387..670f6bd8eed1 100644 --- a/examples/openthread/pytest_otbr.py +++ b/examples/openthread/pytest_otbr.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Unlicense OR CC0-1.0 # !/usr/bin/env python3 import copy +import logging import os.path import random import re @@ -76,7 +77,7 @@ @pytest.fixture(scope='module', name='Init_avahi') def fixture_Init_avahi() -> bool: - print('Init Avahi') + logging.info('Init Avahi') ocf.start_avahi() time.sleep(10) return True @@ -84,7 +85,7 @@ def fixture_Init_avahi() -> bool: @pytest.fixture(name='Init_interface') def fixture_Init_interface() -> bool: - print('Init interface') + logging.info('Init interface') ocf.flush_ipv6_addr_by_interface() # The sleep time is set based on experience; reducing it might cause the host to be unready. time.sleep(30) @@ -109,7 +110,7 @@ def fixture_Init_interface() -> bool: # Case 1: Thread network formation and attaching @pytest.mark.supported_targets @pytest.mark.openthread_br -@pytest.mark.flaky(reruns=1, reruns_delay=1) +@pytest.mark.flaky(reruns=1, reruns_delay=5) @pytest.mark.parametrize( 'config, count, app_path, target, port', [ @@ -136,7 +137,7 @@ def fixture_Init_interface() -> bool: ], indirect=True, ) -def test_thread_connect(dut:Tuple[IdfDut, IdfDut, IdfDut]) -> None: +def test_thread_connect(dut: Tuple[IdfDut, IdfDut, IdfDut]) -> None: br = dut[2] cli_h2 = dut[1] dut[0].serial.stop_redirect_thread() @@ -164,9 +165,9 @@ def test_thread_connect(dut:Tuple[IdfDut, IdfDut, IdfDut]) -> None: rx_nums = ocf.ot_ping(br, cli_mleid_addr, count=5)[1] assert rx_nums == 5 finally: - ocf.execute_command(br, 'factoryreset') for cli in cli_list: - ocf.execute_command(cli, 'factoryreset') + ocf.stop_thread(cli) + ocf.stop_thread(br) time.sleep(3) @@ -193,7 +194,7 @@ def formBasicWiFiThreadNetwork(br:IdfDut, cli:IdfDut) -> None: # Case 2: Bidirectional IPv6 connectivity @pytest.mark.supported_targets @pytest.mark.openthread_br -@pytest.mark.flaky(reruns=1, reruns_delay=1) +@pytest.mark.flaky(reruns=1, reruns_delay=5) @pytest.mark.parametrize( 'config, count, app_path, target, port', [ @@ -210,7 +211,7 @@ def formBasicWiFiThreadNetwork(br:IdfDut, cli:IdfDut) -> None: ], indirect=True, ) -def test_Bidirectional_IPv6_connectivity(Init_interface:bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> None: +def test_Bidirectional_IPv6_connectivity(Init_interface: bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> None: br = dut[2] cli = dut[1] assert Init_interface @@ -220,10 +221,10 @@ def test_Bidirectional_IPv6_connectivity(Init_interface:bool, dut: Tuple[IdfDut, try: assert ocf.is_joined_wifi_network(br) cli_global_unicast_addr = ocf.get_global_unicast_addr(cli, br) - print('cli_global_unicast_addr', cli_global_unicast_addr) + logging.info(f'cli_global_unicast_addr {cli_global_unicast_addr}') command = 'ping ' + str(cli_global_unicast_addr) + ' -c 10' out_str = subprocess.getoutput(command) - print('ping result:\n', str(out_str)) + logging.info(f'ping result:\n{out_str}') role = re.findall(r' (\d+)%', str(out_str))[0] assert role != '100' interface_name = ocf.get_host_interface_name() @@ -231,22 +232,24 @@ def test_Bidirectional_IPv6_connectivity(Init_interface:bool, dut: Tuple[IdfDut, out_bytes = subprocess.check_output(command, shell=True, timeout=5) out_str = out_bytes.decode('utf-8') onlinkprefix = ocf.get_onlinkprefix(br) - host_global_unicast_addr = re.findall(r'\W+(%s(?:\w+:){3}\w+)\W+' % onlinkprefix, str(out_str)) + pattern = rf'\W+({onlinkprefix}(?:\w+:){{3}}\w+)\W+' + host_global_unicast_addr = re.findall(pattern, out_str) rx_nums = 0 for ip_addr in host_global_unicast_addr: - txrx_nums = ocf.ot_ping(cli, str(ip_addr), count=5) + txrx_nums = ocf.ot_ping(cli, str(ip_addr), count=10) rx_nums = rx_nums + int(txrx_nums[1]) + logging.debug(f'rx_nums: {rx_nums}') assert rx_nums != 0 finally: - ocf.execute_command(br, 'factoryreset') - ocf.execute_command(cli, 'factoryreset') + ocf.stop_thread(cli) + ocf.stop_thread(br) time.sleep(3) # Case 3: Multicast forwarding from Wi-Fi to Thread network @pytest.mark.supported_targets @pytest.mark.openthread_br -@pytest.mark.flaky(reruns=1, reruns_delay=1) +@pytest.mark.flaky(reruns=1, reruns_delay=5) @pytest.mark.parametrize( 'config, count, app_path, target, port', [ @@ -263,7 +266,7 @@ def test_Bidirectional_IPv6_connectivity(Init_interface:bool, dut: Tuple[IdfDut, ], indirect=True, ) -def test_multicast_forwarding_A(Init_interface:bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> None: +def test_multicast_forwarding_A(Init_interface: bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> None: br = dut[2] cli = dut[1] assert Init_interface @@ -278,7 +281,7 @@ def test_multicast_forwarding_A(Init_interface:bool, dut: Tuple[IdfDut, IdfDut, interface_name = ocf.get_host_interface_name() command = 'ping -I ' + str(interface_name) + ' -t 64 ff04::125 -c 10' out_str = subprocess.getoutput(command) - print('ping result:\n', str(out_str)) + logging.info(f'ping result:\n{out_str}') role = re.findall(r' (\d+)%', str(out_str))[0] assert role != '100' ocf.execute_command(cli, 'udp open') @@ -292,15 +295,15 @@ def test_multicast_forwarding_A(Init_interface:bool, dut: Tuple[IdfDut, IdfDut, ocf.execute_command(cli, 'udp close') cli.expect('Done', timeout=5) finally: - ocf.execute_command(br, 'factoryreset') - ocf.execute_command(cli, 'factoryreset') + ocf.stop_thread(cli) + ocf.stop_thread(br) time.sleep(3) # Case 4: Multicast forwarding from Thread to Wi-Fi network @pytest.mark.supported_targets @pytest.mark.openthread_br -@pytest.mark.flaky(reruns=1, reruns_delay=1) +@pytest.mark.flaky(reruns=1, reruns_delay=5) @pytest.mark.parametrize( 'config, count, app_path, target, port', [ @@ -317,7 +320,7 @@ def test_multicast_forwarding_A(Init_interface:bool, dut: Tuple[IdfDut, IdfDut, ], indirect=True, ) -def test_multicast_forwarding_B(Init_interface:bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> None: +def test_multicast_forwarding_B(Init_interface: bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> None: br = dut[2] cli = dut[1] assert Init_interface @@ -346,8 +349,8 @@ def test_multicast_forwarding_B(Init_interface:bool, dut: Tuple[IdfDut, IdfDut, while udp_mission.is_alive(): time.sleep(1) finally: - ocf.execute_command(br, 'factoryreset') - ocf.execute_command(cli, 'factoryreset') + ocf.stop_thread(cli) + ocf.stop_thread(br) time.sleep(3) assert b'hello' in myudp.udp_bytes @@ -355,7 +358,7 @@ def test_multicast_forwarding_B(Init_interface:bool, dut: Tuple[IdfDut, IdfDut, # Case 5: discover dervice published by Thread device @pytest.mark.supported_targets @pytest.mark.openthread_br -@pytest.mark.flaky(reruns=1, reruns_delay=1) +@pytest.mark.flaky(reruns=1, reruns_delay=5) @pytest.mark.parametrize( 'config, count, app_path, target, port', [ @@ -372,7 +375,9 @@ def test_multicast_forwarding_B(Init_interface:bool, dut: Tuple[IdfDut, IdfDut, ], indirect=True, ) -def test_service_discovery_of_Thread_device(Init_interface:bool, Init_avahi:bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> None: +def test_service_discovery_of_Thread_device( + Init_interface: bool, Init_avahi: bool, dut: Tuple[IdfDut, IdfDut, IdfDut] +) -> None: br = dut[2] cli = dut[1] assert Init_interface @@ -384,14 +389,14 @@ def test_service_discovery_of_Thread_device(Init_interface:bool, Init_avahi:bool assert ocf.is_joined_wifi_network(br) command = 'avahi-browse -rt _testyyy._udp' out_str = subprocess.getoutput(command) - print('avahi-browse:\n', str(out_str)) + logging.info(f'avahi-browse:\n{out_str}') assert 'myTest' not in str(out_str) hostname = 'myTest' command = 'srp client host name ' + hostname ocf.execute_command(cli, command) cli.expect('Done', timeout=5) cli_global_unicast_addr = ocf.get_global_unicast_addr(cli, br) - print('cli_global_unicast_addr', cli_global_unicast_addr) + logging.info(f'cli_global_unicast_addr {cli_global_unicast_addr}') command = 'srp client host address ' + str(cli_global_unicast_addr) ocf.execute_command(cli, command) cli.expect('Done', timeout=5) @@ -404,18 +409,18 @@ def test_service_discovery_of_Thread_device(Init_interface:bool, Init_avahi:bool ocf.wait(cli, 3) command = 'avahi-browse -rt _testyyy._udp' out_str = subprocess.getoutput(command) - print('avahi-browse:\n', str(out_str)) + logging.info(f'avahi-browse:\n {out_str}') assert 'myTest' in str(out_str) finally: - ocf.execute_command(br, 'factoryreset') - ocf.execute_command(cli, 'factoryreset') + ocf.stop_thread(cli) + ocf.stop_thread(br) time.sleep(3) # Case 6: discover dervice published by Wi-Fi device @pytest.mark.supported_targets @pytest.mark.openthread_br -@pytest.mark.flaky(reruns=1, reruns_delay=1) +@pytest.mark.flaky(reruns=1, reruns_delay=5) @pytest.mark.parametrize( 'config, count, app_path, target, port', [ @@ -432,7 +437,9 @@ def test_service_discovery_of_Thread_device(Init_interface:bool, Init_avahi:bool ], indirect=True, ) -def test_service_discovery_of_WiFi_device(Init_interface:bool, Init_avahi:bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> None: +def test_service_discovery_of_WiFi_device( + Init_interface: bool, Init_avahi: bool, dut: Tuple[IdfDut, IdfDut, IdfDut] +) -> None: br = dut[2] cli = dut[1] assert Init_interface @@ -448,7 +455,7 @@ def test_service_discovery_of_WiFi_device(Init_interface:bool, Init_avahi:bool, cli.expect('Done', timeout=5) ocf.wait(cli, 1) domain_name = ocf.get_domain() - print('domain name is: ', domain_name) + logging.info(f'domain name is: {domain_name}') command = 'dns resolve ' + domain_name + '.default.service.arpa.' ocf.execute_command(cli, command) @@ -476,15 +483,15 @@ def test_service_discovery_of_WiFi_device(Init_interface:bool, Init_avahi:bool, finally: ocf.host_close_service() sp.terminate() - ocf.execute_command(br, 'factoryreset') - ocf.execute_command(cli, 'factoryreset') + ocf.stop_thread(cli) + ocf.stop_thread(br) time.sleep(3) # Case 7: ICMP communication via NAT64 @pytest.mark.supported_targets @pytest.mark.openthread_br -@pytest.mark.flaky(reruns=1, reruns_delay=1) +@pytest.mark.flaky(reruns=1, reruns_delay=5) @pytest.mark.parametrize( 'config, count, app_path, target, port', [ @@ -501,7 +508,7 @@ def test_service_discovery_of_WiFi_device(Init_interface:bool, Init_avahi:bool, ], indirect=True, ) -def test_ICMP_NAT64(Init_interface:bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> None: +def test_ICMP_NAT64(Init_interface: bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> None: br = dut[2] cli = dut[1] assert Init_interface @@ -511,19 +518,19 @@ def test_ICMP_NAT64(Init_interface:bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> try: assert ocf.is_joined_wifi_network(br) host_ipv4_address = ocf.get_host_ipv4_address() - print('host_ipv4_address: ', host_ipv4_address) + logging.info(f'host_ipv4_address: {host_ipv4_address}') rx_nums = ocf.ot_ping(cli, str(host_ipv4_address), count=5)[1] assert rx_nums != 0 finally: - ocf.execute_command(br, 'factoryreset') - ocf.execute_command(cli, 'factoryreset') + ocf.stop_thread(cli) + ocf.stop_thread(br) time.sleep(3) # Case 8: UDP communication via NAT64 @pytest.mark.supported_targets @pytest.mark.openthread_br -@pytest.mark.flaky(reruns=1, reruns_delay=1) +@pytest.mark.flaky(reruns=1, reruns_delay=5) @pytest.mark.parametrize( 'config, count, app_path, target, port', [ @@ -540,7 +547,7 @@ def test_ICMP_NAT64(Init_interface:bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> ], indirect=True, ) -def test_UDP_NAT64(Init_interface:bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> None: +def test_UDP_NAT64(Init_interface: bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> None: br = dut[2] cli = dut[1] assert Init_interface @@ -555,7 +562,7 @@ def test_UDP_NAT64(Init_interface:bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> N cli.expect('Done', timeout=5) ocf.wait(cli, 3) host_ipv4_address = ocf.get_host_ipv4_address() - print('host_ipv4_address: ', host_ipv4_address) + logging.info(f'host_ipv4_address: {host_ipv4_address}') myudp = ocf.udp_parameter('INET4', host_ipv4_address, 5090, '', False, 15.0, b'') udp_mission = threading.Thread(target=ocf.create_host_udp_server, args=(myudp, )) udp_mission.start() @@ -571,8 +578,8 @@ def test_UDP_NAT64(Init_interface:bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> N while udp_mission.is_alive(): time.sleep(1) finally: - ocf.execute_command(br, 'factoryreset') - ocf.execute_command(cli, 'factoryreset') + ocf.stop_thread(cli) + ocf.stop_thread(br) time.sleep(3) assert b'hello' in myudp.udp_bytes @@ -580,7 +587,7 @@ def test_UDP_NAT64(Init_interface:bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> N # Case 9: TCP communication via NAT64 @pytest.mark.supported_targets @pytest.mark.openthread_br -@pytest.mark.flaky(reruns=1, reruns_delay=1) +@pytest.mark.flaky(reruns=1, reruns_delay=5) @pytest.mark.parametrize( 'config, count, app_path, target, port', [ @@ -597,7 +604,7 @@ def test_UDP_NAT64(Init_interface:bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> N ], indirect=True, ) -def test_TCP_NAT64(Init_interface:bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> None: +def test_TCP_NAT64(Init_interface: bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> None: br = dut[2] cli = dut[1] assert Init_interface @@ -613,7 +620,7 @@ def test_TCP_NAT64(Init_interface:bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> N ocf.wait(cli, 3) host_ipv4_address = ocf.get_host_ipv4_address() connect_address = ocf.get_ipv6_from_ipv4(host_ipv4_address, br) - print('connect_address is: ', connect_address) + logging.info(f'connect_address is: {connect_address}') mytcp = ocf.tcp_parameter('INET4', host_ipv4_address, 12345, False, False, 15.0, b'') tcp_mission = threading.Thread(target=ocf.create_host_tcp_server, args=(mytcp, )) tcp_mission.start() @@ -634,8 +641,8 @@ def test_TCP_NAT64(Init_interface:bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> N while tcp_mission.is_alive(): time.sleep(1) finally: - ocf.execute_command(br, 'factoryreset') - ocf.execute_command(cli, 'factoryreset') + ocf.stop_thread(cli) + ocf.stop_thread(br) time.sleep(3) assert b'hello' in mytcp.tcp_bytes @@ -644,6 +651,7 @@ def test_TCP_NAT64(Init_interface:bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> N @pytest.mark.esp32h2 @pytest.mark.esp32c6 @pytest.mark.openthread_sleep +@pytest.mark.flaky(reruns=1, reruns_delay=5) @pytest.mark.parametrize( 'config, count, app_path, target, port', [ @@ -671,6 +679,7 @@ def test_TCP_NAT64(Init_interface:bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> N def test_ot_sleepy_device(dut: Tuple[IdfDut, IdfDut]) -> None: leader = dut[0] sleepy_device = dut[1] + ocf.hardreset_dut(sleepy_device) fail_info = re.compile(r'Core\W*?\d\W*?register dump') try: ocf.init_thread(leader) @@ -697,13 +706,14 @@ def test_ot_sleepy_device(dut: Tuple[IdfDut, IdfDut]) -> None: assert not bool(fail_info.search(str(output))) finally: ocf.execute_command(leader, 'factoryreset') + ocf.hardreset_dut(sleepy_device) time.sleep(3) # Case 11: Basic startup Test of BR @pytest.mark.supported_targets @pytest.mark.openthread_br -@pytest.mark.flaky(reruns=1, reruns_delay=1) +@pytest.mark.flaky(reruns=1, reruns_delay=5) @pytest.mark.parametrize( 'config, count, app_path, target, port', [ @@ -738,14 +748,14 @@ def test_basic_startup(dut: Tuple[IdfDut, IdfDut]) -> None: br.expect('Done', timeout=5) assert ocf.wait_for_join(br, 'leader') finally: - ocf.execute_command(br, 'factoryreset') + ocf.stop_thread(br) time.sleep(3) # Case 12: Curl a website via DNS and NAT64 @pytest.mark.supported_targets @pytest.mark.openthread_bbr -@pytest.mark.flaky(reruns=1, reruns_delay=1) +@pytest.mark.flaky(reruns=1, reruns_delay=5) @pytest.mark.parametrize( 'config, count, app_path, target, port', [ @@ -762,7 +772,7 @@ def test_basic_startup(dut: Tuple[IdfDut, IdfDut]) -> None: ], indirect=True, ) -def test_NAT64_DNS(Init_interface:bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> None: +def test_NAT64_DNS(Init_interface: bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> None: br = dut[2] cli = dut[1] assert Init_interface @@ -770,24 +780,25 @@ def test_NAT64_DNS(Init_interface:bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> N formBasicWiFiThreadNetwork(br, cli) try: + ocf.wait_for_host_network() ocf.execute_command(br, 'bbr') br.expect('server16', timeout=5) ocf.execute_command(cli, 'dns64server 8.8.8.8') cli.expect('Done', timeout=5) command = 'curl http://www.espressif.com' message = ocf.get_ouput_string(cli, command, 10) - assert '' in str(message) + assert 'html' in str(message) assert '301 Moved Permanently' in str(message) finally: - ocf.execute_command(br, 'factoryreset') - ocf.execute_command(cli, 'factoryreset') + ocf.stop_thread(cli) + ocf.stop_thread(br) time.sleep(3) # Case 13: Meshcop discovery of Border Router @pytest.mark.supported_targets @pytest.mark.openthread_br -@pytest.mark.flaky(reruns=1, reruns_delay=1) +@pytest.mark.flaky(reruns=1, reruns_delay=5) @pytest.mark.parametrize( 'config, count, app_path, target, port', [ @@ -803,7 +814,7 @@ def test_NAT64_DNS(Init_interface:bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> N ], indirect=True, ) -def test_br_meshcop(Init_interface:bool, Init_avahi:bool, dut: Tuple[IdfDut, IdfDut]) -> None: +def test_br_meshcop(Init_interface: bool, Init_avahi: bool, dut: Tuple[IdfDut, IdfDut]) -> None: br = dut[1] assert Init_interface assert Init_avahi @@ -829,9 +840,9 @@ def test_br_meshcop(Init_interface:bool, Init_avahi:bool, dut: Tuple[IdfDut, Idf except subprocess.CalledProcessError as e: output_bytes = e.stdout finally: - print('out_bytes: ', output_bytes) + logging.info(f'out_bytes: {output_bytes!r}') output_str = str(output_bytes) - print('out_str: ', output_str) + logging.info(f'out_str: {output_str}') assert 'hostname = [esp-ot-br.local]' in str(output_str) assert ('address = [' + ipv4_address + ']') in str(output_str) @@ -842,14 +853,14 @@ def test_br_meshcop(Init_interface:bool, Init_avahi:bool, dut: Tuple[IdfDut, Idf assert 'vn=OpenThread' in str(output_str) assert 'rv=1' in str(output_str) finally: - ocf.execute_command(br, 'factoryreset') + ocf.stop_thread(br) time.sleep(3) # Case 14: Curl a website over HTTPS via DNS and NAT64 @pytest.mark.supported_targets @pytest.mark.openthread_bbr -@pytest.mark.flaky(reruns=1, reruns_delay=1) +@pytest.mark.flaky(reruns=1, reruns_delay=5) @pytest.mark.parametrize( 'config, count, app_path, target, port', [ @@ -866,7 +877,7 @@ def test_br_meshcop(Init_interface:bool, Init_avahi:bool, dut: Tuple[IdfDut, Idf ], indirect=True, ) -def test_https_NAT64_DNS(Init_interface:bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> None: +def test_https_NAT64_DNS(Init_interface: bool, dut: Tuple[IdfDut, IdfDut, IdfDut]) -> None: br = dut[2] cli = dut[1] assert Init_interface @@ -874,22 +885,23 @@ def test_https_NAT64_DNS(Init_interface:bool, dut: Tuple[IdfDut, IdfDut, IdfDut] formBasicWiFiThreadNetwork(br, cli) try: + ocf.wait_for_host_network() ocf.execute_command(cli, 'dns64server 8.8.8.8') cli.expect('Done', timeout=5) command = 'curl https://www.example.com/' message = ocf.get_ouput_string(cli, command, 20) - assert '' in str(message) - assert 'This domain is for use in illustrative examples in documents' in str(message) + assert 'html' in str(message) + assert 'This domain is for use in' in str(message) finally: - ocf.execute_command(br, 'factoryreset') - ocf.execute_command(cli, 'factoryreset') + ocf.stop_thread(cli) + ocf.stop_thread(br) time.sleep(3) # Case 15: Thread network formation and attaching with TREL @pytest.mark.supported_targets @pytest.mark.openthread_br -@pytest.mark.flaky(reruns=1, reruns_delay=1) +@pytest.mark.flaky(reruns=1, reruns_delay=5) @pytest.mark.parametrize( 'config, count, app_path, target, port', [ @@ -935,16 +947,16 @@ def test_trel_connect(dut: Tuple[IdfDut, IdfDut]) -> None: rx_nums = ocf.ot_ping(trel_s3, trel_mleid_addr, count=10)[1] assert rx_nums > 5 finally: - ocf.execute_command(trel_s3, 'factoryreset') for trel in trel_list: - ocf.execute_command(trel, 'factoryreset') + ocf.stop_thread(trel) + ocf.stop_thread(trel_s3) time.sleep(3) # Case 16: Thread network BR lib check @pytest.mark.supported_targets @pytest.mark.openthread_br -@pytest.mark.flaky(reruns=1, reruns_delay=1) +@pytest.mark.flaky(reruns=1, reruns_delay=5) @pytest.mark.parametrize( 'config, count, app_path, target, port', [ @@ -974,6 +986,7 @@ def test_br_lib_check(dut: Tuple[IdfDut, IdfDut]) -> None: # Case 17: SSED test @pytest.mark.openthread_sleep +@pytest.mark.flaky(reruns=1, reruns_delay=5) @pytest.mark.parametrize( 'config, count, app_path, target, port', [ @@ -1002,6 +1015,7 @@ def test_ot_ssed_device(dut: Tuple[IdfDut, IdfDut]) -> None: leader = dut[0] ssed_device = dut[1] try: + ocf.hardreset_dut(ssed_device) # CI device must have external XTAL to run SSED case, we will check this here first ssed_device.expect('32k XTAL in use', timeout=10) ocf.init_thread(leader) @@ -1041,4 +1055,5 @@ def test_ot_ssed_device(dut: Tuple[IdfDut, IdfDut]) -> None: ocf.ping_and_check(dut=leader, target=ssed_address, tx_total=10, timeout=6) finally: ocf.execute_command(leader, 'factoryreset') + ocf.hardreset_dut(ssed_device) time.sleep(3)