Add rBoot partition support (#2258)

This PR adds partition table support to the ESP8266 bootloader (rBoot). See #2254 #2251.

The ESP8266 flash layout has been reverted to the original Sming 4.2 layout, with the addition of the partition table in the sector before the RF calibration data at end of flash.

Update rBoot

Update bootloader to read ROM addresses from partition table on boot. These overwrite whatever values have been set in its own configuration, thus avoiding inconsistencies. rBoot has been forked to simplify management.

An additional partition subtype has been added for RF calibration data, so existing applications running off the develop branch will need re-building and re-flashing.

The partition table may now be freely relocated for the Esp8266 (+ Host) to allow compatibility with existing devices which must be upgrade OTA.

Methods have been added to OTA classes to allow update regions to be set using partitions; this is safer than working with raw flash addresses but the underlying mechanism hasn't changed.

.gitmodules

@@ -63,7 +63,7 @@
 	ignore = dirty
 [submodule "rboot"]
 	path = Sming/Components/rboot/rboot
-	url = https://github.com/raburton/rboot.git
+	url = https://github.com/mikee47/rboot
 	ignore = dirty
 [submodule "spiffs"]
 	path = Sming/Components/spiffs/spiffs

Sming/Arch/Esp8266/Components/esp8266/startup.cpp

@@ -55,9 +55,8 @@ extern "C" void WEAK_ATTR user_rf_pre_init(void)
 
 extern "C" uint32 ICACHE_FLASH_ATTR  WEAK_ATTR user_rf_cal_sector_set(void)
 {
-	// RF calibration stored in last sector of sysParam
-	auto sysParam = *Storage::findPartition(Storage::Partition::SubType::Data::sysParam);
-	return ((sysParam.address() + sysParam.size()) / SPI_FLASH_SEC_SIZE) - 1;
+	auto rfCal = *Storage::findPartition(Storage::Partition::SubType::Data::rfCal);
+	return rfCal.address();
 }
 
 #ifdef SDK_INTERNAL
@@ -71,16 +70,14 @@ extern "C" void ICACHE_FLASH_ATTR WEAK_ATTR user_pre_init(void)
 	Storage::initialize();
 
 	auto sysParam = *Storage::findPartition(Storage::Partition::SubType::Data::sysParam);
+	auto rfCal = *Storage::findPartition(Storage::Partition::SubType::Data::rfCal);
 	auto phy = *Storage::findPartition(Storage::Partition::SubType::Data::phy);
 
-	// RF calibration stored in last sector of sysParam
-	auto sysParamSize = sysParam.size() - SPI_FLASH_SEC_SIZE;
-
 	static const partition_item_t partitions[] = {
 			{SYSTEM_PARTITION_BOOTLOADER,		0,						SPI_FLASH_SEC_SIZE},
 			{SYSTEM_PARTITION_PHY_DATA,			phy.address(),			phy.size()},
-			{SYSTEM_PARTITION_SYSTEM_PARAMETER,	sysParam.address(),		sysParamSize},
-			{SYSTEM_PARTITION_RF_CAL,			sysParam.address() + sysParamSize,	SPI_FLASH_SEC_SIZE},
+			{SYSTEM_PARTITION_SYSTEM_PARAMETER,	sysParam.address(),		sysParam.size()},
+			{SYSTEM_PARTITION_RF_CAL,			rfCal.address(),		rfCal.size()},
 	};
 
 	enum flash_size_map sizeMap = system_get_flash_size_map();

Sming/Arch/Esp8266/options.json

@@ -6,5 +6,23 @@
                 "filename": "$(FLASH_INIT_DATA_VCC)"
             }
         }
+    },
+    "alternate": {
+        "description": "ESP8266 layout with critical partitions at start of flash",
+        "partition_table_offset": "0x2000",
+        "partitions": {
+            "rf_cal": {
+                "address": "0x3000"
+            },
+            "phy_init": {
+                "address": "0x4000"
+            },
+            "sys_param": {
+                "address": "0x5000"
+            },
+            "rom0": {
+                "address": "0x8000"
+            }
+        }
     }
 }

Sming/Arch/Esp8266/spiffs-two-roms.hw

@@ -2,11 +2,14 @@
 	"name": "Two ROM slots with single SPIFFS",
 	"base_config": "spiffs",
 	"partitions": {
+		"rom0": {
+			"subtype": "ota_0"
+		},
 		"rom1": {
 			"address": "0x108000",
 			"size": "992K",
 			"type": "app",
-			"subtype": "ota_0",
+			"subtype": "ota_1",
 			"filename": "$(RBOOT_ROM_1_BIN)"
 		}
 	}

Sming/Arch/Esp8266/standard.hw

@@ -2,7 +2,7 @@
 	"name": "Standard config with single ROM",
 	"comment": "Should work with any Esp8266 variant",
 	"arch": "Esp8266",
-	"partition_table_offset": "0x2000",
+	"partition_table_offset": "self.devices[0].size - 0x6000",
 	"devices": {
 		"spiFlash": {
 			"type": "flash",
@@ -12,25 +12,31 @@
 		}
 	},
 	"partitions": {
+		"rom0": {
+			"address": "0x002000",
+			"size": "992K",
+			"type": "app",
+			"subtype": "factory",
+			"filename": "$(RBOOT_ROM_0_BIN)"
+		},
+		"rf_cal": {
+			"address": "self.device.size - 0x5000",
+			"size": "4K",
+			"type": "data",
+			"subtype": "rfcal"
+		},
 		"phy_init": {
-			"address": "0x003000",
+			"address": "self.device.size - 0x4000",
 			"size": "4K",
 			"type": "data",
 			"subtype": "phy",
 			"filename": "$(FLASH_INIT_DATA)"
 		},
 		"sys_param": {
-			"address": "0x004000",
-			"size": "16K",
+			"address": "self.device.size - 0x3000",
+			"size": "12K",
 			"type": "data",
 			"subtype": "sysparam"
-		},
-		"rom0": {
-			"address": "0x008000",
-			"size": "992K",
-			"type": "app",
-			"subtype": "factory",
-			"filename": "$(RBOOT_ROM_0_BIN)"
 		}
 	}
-}
+}

Sming/Arch/Esp8266/two-rom-mode.hw

@@ -3,13 +3,14 @@
 	"base_config": "standard",
 	"partitions": {
 		"rom0": {
+			"subtype": "ota_0",
 			"size": "480K"
 		},
 		"rom1": {
 			"address": "0x80000",
-			"size": "512K",
+			"size": "488K",
 			"type": "app",
-			"subtype": "ota_0",
+			"subtype": "ota_1",
 			"filename": "$(RBOOT_ROM_1_BIN)"
 		}
 	}

Sming/Components/Storage/README.rst

@@ -84,6 +84,16 @@ If using this approach, remember to updated your project's ``component.mk`` with
 and verify the layout is correct using ``make map``.
 
 
+OTA updates
+-----------
+
+When planning OTA updates please check that the displayed partition map corresponds to your project.
+For example, the partition table requires a free sector so must not overlap other partitions.
+
+Your OTA update process must include a step to write the partition table to the correct location.
+
+It is not necessary to update the bootloader. See :component:`rboot` for further information.
+
 
 Custom configurations
 ---------------------
@@ -173,6 +183,19 @@ To customise the hardware configuration for a project, for example 'my_project':
    This will flash everything: bootloader, partition table and all defined partitions (those with a ``filename`` entry).
 
 
+.. note::
+
+   The build system isn't smart enough to track dependencies for partition build targets.
+
+   To rebuild these manually type::
+
+      make partbuild
+
+   These will be removed when ``make clean`` is run, but you can also clean them separately thus::
+
+      make part-clean
+
+
 Partition maps
 --------------
 
@@ -356,7 +379,7 @@ you can take advantage of the partition API to manage them as follows:
 -  Create an instance of your custom device and make a call to :cpp:func:`Storage::registerDevice`
    in your ``init()`` function (or elsewhere if more appropriate).
 
- 
+
 API
 ---
 

Sming/Components/Storage/Tools/hwconfig/config.py

@@ -46,6 +46,7 @@ def from_name(cls, name):
         config.load(name)
         if options != '':
             config.parse_options(options.split(','))
+        config.resolve_expressions()
         config.partitions.sort()
         return config
 
@@ -73,6 +74,9 @@ def parse_options(self, options):
             temp.pop('description', None)
             self.parse_dict(temp)
 
+    def resolve_expressions(self):
+        self.partitions.offset = eval(str(self.partitions.offset))
+
     def parse_dict(self, data):
         base_config = data.pop('base_config', None)
         if base_config is not None:
@@ -89,7 +93,7 @@ def parse_dict(self, data):
             elif k == 'arch':
                 self.arch = v
             elif k == 'partition_table_offset':
-                self.partitions.offset = parse_int(v)
+                self.partitions.offset = v
             elif k == 'devices':
                 self.devices.parse_dict(v)
             elif k == 'comment':
@@ -132,7 +136,7 @@ def buildVars(self):
         return res
 
     def verify(self, secure):
-        self.partitions.verify(self.arch, secure)
+        self.partitions.verify(self.arch, self.devices[0], secure)
 
     def map(self):
         return partition.Map(self.partitions, self.devices)

Sming/Components/Storage/Tools/hwconfig/hwconfig.py

@@ -44,7 +44,7 @@ def handle_flashcheck(args, config, part):
     for e in list:
         addr, filename = e.split('=')
         addr = int(addr, 0)
-        part = config.partitions.find_by_address(addr)
+        part = config.partitions.find_by_address(config.devices[0], addr)
         if part is None:
             raise InputError("No partition contains address 0x%08x" % addr)
         if part.address != addr:
@@ -113,5 +113,5 @@ def main():
     try:
         main()
     except InputError as e:
-        print(e, file=sys.stderr)
+        print("** ERROR! %s" % e, file=sys.stderr)
         sys.exit(2)

Sming/Components/Storage/Tools/hwconfig/partition.py

@@ -16,11 +16,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import struct, hashlib, storage, binascii
+import struct, hashlib, storage, binascii, copy
 from common import *
 
 MAX_PARTITION_LENGTH = 0xC00  # 3K for partition data (96 entries) leaves 1K in a 4K sector for signature
 MD5_PARTITION_BEGIN = b"\xEB\xEB" + b"\xFF" * 14  # The first 2 bytes are like magic numbers for MD5 sum
+FLASH_SECTOR_SIZE = 0x1000
 PARTITION_TABLE_SIZE = 0x1000  # Size of partition table
 PARTITION_ENTRY_SIZE = 32
 
@@ -84,6 +85,7 @@
         "nvs_keys": 0x04,
         "efuse": 0x05,
         "sysparam": 0x40,
+        "rfcal": 0x41,
         "esphttpd": 0x80,
         "fat": 0x81,
         "spiffs": 0x82,
@@ -154,6 +156,10 @@ def offset_str(self):
     def buildVars(self):
         dict = {}
         dict['PARTITION_NAMES'] = " ".join(p.name for p in self)
+        buildparts = [p for p in self if p.build is not None]
+        dict['PARTITIONS_WITH_TARGETS'] = " ".join(p.name for p in buildparts)
+        dict['PARTITION_BUILD_TARGETS'] = " ".join(p.filename for p in buildparts)
+
         for p in self:
             dict.update(p.buildVars())
         return dict
@@ -198,19 +204,28 @@ def find_by_name(self, name):
                 return p
         return None
 
-    def find_by_address(self, addr):
+    def find_by_address(self, device, addr):
         for p in self:
-            if p.contains(addr):
+            if p.device == device and p.contains(addr):
                 return p
         return None
 
-    def verify(self, arch, secure):
+    def verify(self, arch, spiFlash, secure):
         """Verify partition layout
         """
         # verify each partition individually
         for p in self:
             p.verify(arch, secure)
 
+        if self.offset % FLASH_SECTOR_SIZE != 0:
+            raise InputError("Partition table offset not aligned to flash sector")
+
+        p = self.find_by_address(spiFlash, self.offset)
+        if p is None:
+            p = self.find_by_address(spiFlash, self.offset + PARTITION_TABLE_SIZE - 1)
+        if not p is None:
+            raise InputError("Partition table conflict with '%s'" % p.name)
+
         # check on duplicate name
         names = [p.name for p in self]
         duplicates = set(n for n in names if names.count(n) > 1)
@@ -224,7 +239,10 @@ def verify(self, arch, secure):
             raise InputError("Partition names must be unique")
 
         # check for overlaps
-        minPartitionAddress = self.offset + PARTITION_TABLE_SIZE
+        if arch == 'Esp32':
+            minPartitionAddress = self.offset + PARTITION_TABLE_SIZE
+        else:
+            minPartitionAddress = 0x00002000
         dev = ''
         last = None
         for p in self:
@@ -345,7 +363,7 @@ def parse_dict(self, data, devices):
                 if k == 'device':
                     self.device = devices.find_by_name(v)
                 elif k == 'address':
-                    self.address = parse_int(v)
+                    self.address = eval(str(v))
                 elif k == 'size':
                     self.size = parse_int(v)
                 elif k == 'filename':
@@ -523,13 +541,17 @@ def add_unused(address, last_end):
             if address > last_end + 1:
                 add('(unused)', last_end + 1, address - last_end - 1)
 
+        partitions = copy.copy(table)
+
         if table.offset == 0:
             last = None
         else:
-            add("Boot Sector", 0, table.offset)
-            last = add("Partition Table", table.offset, PARTITION_TABLE_SIZE)
+            last = add('Boot Sector', 0, min(table.offset, partitions[0].address))
+            p = Entry(device, 'Partition Table', table.offset, PARTITION_TABLE_SIZE, 0xff, 0xff)
+            partitions.append(p)
+            partitions.sort()
 
-        for p in table:
+        for p in partitions:
             if last is not None:
                 if p.device != last.device:
                     add_unused(last.device.size, last.end())