From 0fb1ebc3c0fddfddcd35b771df2248bc329e765c Mon Sep 17 00:00:00 2001 From: sticilface Date: Fri, 6 Nov 2020 20:31:43 +0000 Subject: [PATCH 01/12] Add FSTools with examples of how to convert between SPIFFS and LITTLEFS. --- libraries/FSTools/FSTools.cpp | 249 ++++++++++++++++++ libraries/FSTools/FSTools.h | 95 +++++++ .../examples/Basic_example/Basic_example.ino | 132 ++++++++++ .../custom_FS_example/custom_FS_example.ino | 139 ++++++++++ libraries/FSTools/package.json | 11 + 5 files changed, 626 insertions(+) create mode 100644 libraries/FSTools/FSTools.cpp create mode 100644 libraries/FSTools/FSTools.h create mode 100644 libraries/FSTools/examples/Basic_example/Basic_example.ino create mode 100644 libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino create mode 100644 libraries/FSTools/package.json diff --git a/libraries/FSTools/FSTools.cpp b/libraries/FSTools/FSTools.cpp new file mode 100644 index 0000000000..5bb1940018 --- /dev/null +++ b/libraries/FSTools/FSTools.cpp @@ -0,0 +1,249 @@ +#include "FSTools.h" +#include "LittleFS.h" +#include +#include + + +namespace FST { + + const layout layout_512k32 (0x40273000, 0x4027B000, 0x100, 0x1000 ); + const layout layout_512k64 (0x4026B000, 0x4027B000, 0x100, 0x1000 ); + const layout layout_512k128 (0x4025B000, 0x4027B000, 0x100, 0x1000 ); + + const layout layout_1m64 (0x402EB000, 0x402FB000, 0x100, 0x1000 ); + const layout layout_1m128 (0x402DB000, 0x402FB000, 0x100, 0x1000 ); + const layout layout_1m144 (0x402D7000, 0x402FB000, 0x100, 0x1000 ); + const layout layout_1m160 (0x402D3000, 0x402FB000, 0x100, 0x1000 ); + const layout layout_1m192 (0x402CB000, 0x402FB000, 0x100, 0x1000 ); + const layout layout_1m256 (0x402BB000, 0x402FB000, 0x100, 0x1000 ); + const layout layout_1m512 (0x4027B000, 0x402FB000, 0x100, 0x2000 ); + + const layout layout_2m64 (0x403F0000, 0x403FB000, 0x100, 0x1000 ); + const layout layout_2m128 (0x403E0000, 0x403FB000, 0x100, 0x1000 ); + const layout layout_2m256 (0x403C0000, 0x403FB000, 0x100, 0x1000 ); + const layout layout_2m512 (0x40380000, 0x403FA000, 0x100, 0x2000 ); + const layout layout_2m1m (0x40300000, 0x403FA000, 0x100, 0x2000 ); + + const layout layout_4m1m (0x40500000, 0x405FA000, 0x100, 0x2000 ); + const layout layout_4m2m (0x40400000, 0x405FA000, 0x100, 0x2000 ); + const layout layout_4m3m (0x40300000, 0x405FA000, 0x100, 0x2000 ); + + const layout layout_8m6m (0x40400000, 0x409FA000, 0x100, 0x2000 ); + const layout layout_8m7m (0x40300000, 0x409FA000, 0x100, 0x2000 ); + + const layout layout_16m14m (0x40400000, 0x411FA000, 0x100, 0x2000 ); + const layout layout_16m15m (0x40300000, 0x411FA000, 0x100, 0x2000 ); + +}; + +FSTools::FSTools() +{ + +} + +FSTools::~FSTools() +{ + reset(); +} + + +bool FSTools::attemptToMountFS(fs::FS & fs) +{ + LittleFSConfig littleFSCfg(false); + SPIFFSConfig SPIFFSCfg(false); + // try to apply the "safe" no format config to the FS... doesn't matter which.. just need one to apply.. + if (!fs.setConfig(littleFSCfg) && ! fs.setConfig(SPIFFSCfg) ) + { + return false; + } + return fs.begin(); +} + + +bool FSTools::mountAlternativeFS( FST::FS_t type, const FST::layout layout, bool keepMounted ) +{ + FSConfig * pCfg{nullptr}; + LittleFSConfig littleFSCfg(false); + SPIFFSConfig SPIFFSCfg(false); + reset(); + + switch (type) { + case FST::SPIFFS : { + _pFS.reset( new FS(FSImplPtr(new spiffs_impl::SPIFFSImpl (_getStartAddr(layout) , _getSize(layout) , layout.page, layout.block, 5))) ); + pCfg = &SPIFFSCfg; + break; + } + case FST::LITTLEFS : { + _pFS.reset( new FS(FSImplPtr(new littlefs_impl::LittleFSImpl(_getStartAddr(layout) , _getSize(layout) , layout.page, layout.block, 5))) ); + pCfg = &littleFSCfg; + break; + } + }; + + if (_pFS && pCfg && _pFS->setConfig(*pCfg) && _pFS->begin()) { + if (!keepMounted) { + _pFS->end(); + } + _mounted = true; + _layout = &layout; + return true; + } + + if (_pFS) { + _pFS.reset(); + } + _mounted = false; + return false; + +}; + + +bool FSTools::mounted() +{ + return _mounted; +}; + + + + +bool FSTools::moveFS(fs::FS & destinationFS) +{ + uint32_t sourceFileCount = 0; + uint32_t sourceByteTotal = 0; + bool result = false; + + // do not init new fs... until old one is copied to test... + if (!_mounted || !_pFS) { + //Serial.println("Source FS not mounted"); + return false; + } + + + uint32_t startSector = (ESP.getSketchSize() + FLASH_SECTOR_SIZE - 1) & (~(FLASH_SECTOR_SIZE - 1)); + //uint32_t endSector = (uint32_t)&_FS_start - FST::lowestSPIFFSstartAddr; + uint32_t lowestFSStart = 0x40300000; + + if (_layout) { + lowestFSStart = _layout->startAddr; + } + + uint32_t endSector = lowestFSStart - 0x40200000; + + uint32_t tempFSsize = endSector - startSector; + + //Serial.printf("TempFS: start: %u, end: %u, size: %u, sketchSize = %u, _FS_start = %u\n", startSector, endSector, tempFSsize, ESP.getSketchSize(), (uint32_t)&_FS_start ); + + fileListIterator(*_pFS, "/", [&sourceFileCount, &sourceByteTotal, this](File & f) { + if (f) { + sourceFileCount++; + sourceByteTotal += f.size(); + _dumpFileInfo(f); + } + }); + + //Serial.printf("%u Files Found Total Size = %u\n", sourceFileCount, sourceByteTotal); + //Serial.printf("Size of dummy FS = %u\n", tempFSsize); + + FS tempFS = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(startSector, tempFSsize, FS_PHYS_PAGE, FS_PHYS_BLOCK, 5))); + + if (tempFS.format() && tempFS.begin()) { + if (_copyFS(*_pFS,tempFS)) { + //Serial.println("Files copied to temp File System"); + reset(); + if (destinationFS.format() && destinationFS.begin()) // must format then mount the new FS + { + if (_copyFS(tempFS, destinationFS)) { + //Serial.println("Files copied back to new FS"); + result = true; + } + } else { + //Serial.println("Error Mounting "); + } + } else { + //Serial.println("Copy Failed"); + } + tempFS.end(); + } else { + //Serial.println("Failed to begin() TempFS"); + } + return result; +}; + +void FSTools::reset() +{ + _mounted = false; + _layout = nullptr; + if (_pFS) { + _pFS->end(); + _pFS.reset(); + } +} + +void FSTools::fileListIterator(FS & fs, const char * dirName, FST::FileCb Cb ) +{ + Dir dir = fs.openDir(dirName); + while (dir.next()) { + //const char * fileName = dir.fileName(); + if (dir.isFile()) { + File f = dir.openFile("r"); + if (Cb) { + Cb(f); + } + } else { + fileListIterator(fs, dir.fileName().c_str() , Cb); + } + } +} + + +uint32_t FSTools::_getStartAddr(const FST::layout & layout) +{ + return ( layout.startAddr - 0x40200000 ); +} + +uint32_t FSTools::_getSize(const FST::layout & layout) +{ + return (layout.endAddr - layout.startAddr ); +} + + +void FSTools::_dumpFileInfo(File & f) +{ + if (f) { + //Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size() ); + } +} + +bool FSTools::_copyFS(FS & sourceFS, FS & destFS) +{ + uint32_t sourceFileCount = 0; + uint32_t sourceByteTotal = 0; + + fileListIterator(sourceFS, "/", [&sourceFileCount, &sourceByteTotal](File & f) { + if (f) { + sourceFileCount++; + sourceByteTotal += f.size(); + } + }); + + size_t count = 0; + fileListIterator(sourceFS, "/", [&count, &destFS](File & sourceFile){ + if (sourceFile) { + File destFile = destFS.open(sourceFile.fullName(), "w"); + bool copied = false; + if (destFile) { + destFile.setTimeout(5000); + size_t written = destFile.write(sourceFile); + if (written == sourceFile.size()) { + copied = true; + count++; + } + } + destFile.close(); + sourceFile.close(); + } + }); + + return (count == sourceFileCount); + +} diff --git a/libraries/FSTools/FSTools.h b/libraries/FSTools/FSTools.h new file mode 100644 index 0000000000..ce5eea6534 --- /dev/null +++ b/libraries/FSTools/FSTools.h @@ -0,0 +1,95 @@ +#pragma once + +#include +#include +#include +/* + + A temporary FS is made between the END of the sketch... and the start of the partition you try to mount, to maximise the available space for copying the FS. + The WORST case this is at 0x40300000 which is for a 3m FS on 4m flash.. leaving 460Kb for copying. + + + I have not worked out a way to prevent linking of ALL the layouts except to use #ifdef guards. Ideas welcome as it uses 400B ROM. + */ + +namespace FST { + +struct layout { + layout(uint32_t s, uint32_t e, uint32_t p, uint32_t b) : startAddr(s), endAddr(e), page(p), block(b) {}; + uint32_t startAddr{0}; + uint32_t endAddr{0}; + uint32_t page{0}; + uint32_t block{0}; +}; + +enum FS_t : uint8_t { + SPIFFS, + LITTLEFS +}; + +extern const layout layout_512k32; +extern const layout layout_512k64; +extern const layout layout_512k128; + +extern const layout layout_1m64; +extern const layout layout_1m128; +extern const layout layout_1m144; +extern const layout layout_1m160; +extern const layout layout_1m192; +extern const layout layout_1m256; +extern const layout layout_1m512; + +extern const layout layout_2m64; +extern const layout layout_2m128; +extern const layout layout_2m256; +extern const layout layout_2m512; +extern const layout layout_2m1m; + +extern const layout layout_4m1m; +extern const layout layout_4m2m; +extern const layout layout_4m3m; + +extern const layout layout_8m6m; +extern const layout layout_8m7m; + +extern const layout layout_16m14m; +extern const layout layout_16m15m; + + +typedef std::function FileCb; + + +}; + + + +//376884 +//376452 + + +class FSTools { +public: + + + + FSTools(); + ~FSTools(); + bool attemptToMountFS(fs::FS & fs); + bool mountAlternativeFS( FST::FS_t type, const FST::layout layout, bool keepMounted = false ); + bool mounted(); + bool moveFS(fs::FS & destinationFS); + void reset(); + void fileListIterator(FS & fs, const char * dirName, FST::FileCb Cb ); + +private: + uint32_t _getStartAddr(const FST::layout & layout); + uint32_t _getSize(const FST::layout & layout); + void _dumpFileInfo(File & f); + bool _copyFS(FS & sourceFS, FS & destFS); + + std::unique_ptr _pFS; + bool _mounted{false}; + const FST::layout * _layout{nullptr}; + + +}; \ No newline at end of file diff --git a/libraries/FSTools/examples/Basic_example/Basic_example.ino b/libraries/FSTools/examples/Basic_example/Basic_example.ino new file mode 100644 index 0000000000..b5a7d3279b --- /dev/null +++ b/libraries/FSTools/examples/Basic_example/Basic_example.ino @@ -0,0 +1,132 @@ +/* + + This sketch will convert SPIFFS partitions to LittleFS on ESP8266 + + Change the `TARGET_FS_LAYOUT` to the partition layout that you want target + ie what you are trying to copy from. + + Include in the sketch whatever you want the destination to be, in this case LittleFS, + but it could be SPIFFS to convert back if need be. + + How it works: It creates a LittleFS partition between the end of the sketch and the + start of whatever filesystem you set as target. This has IMPORTANT implications for the + amount of data you can move!!! eg a 4Mb flash module with a 3Mb SPIFFS partition only leaves + about 450k for the temp file system, so if you have more data than that on your 3Mb SPIFFS it + will fail. + + */ + + + +#include +#include +#include +#include +#include +#include + + +#define TARGET_FS_LAYOUT FST::layout_4m3m + +FSTools fstools; + +#ifndef STASSID +#define STASSID "xxxx" +#define STAPSK "xxxx" +#endif + +const char* ssid = STASSID; +const char* password = STAPSK; + + +bool migrateFS() +{ + if (!fstools.attemptToMountFS(LittleFS)) { // Attempts to mount LittleFS without autoformat... + Serial.println(F("Default FS not found")); + if (fstools.mountAlternativeFS( FST::SPIFFS /* FST::LITTLEFS */ , TARGET_FS_LAYOUT , true )) { + Serial.println(F("Alternative found")); + if (fstools.moveFS(LittleFS)) { + Serial.println(F("FileSystem Moved New FS contents:")); + fstools.fileListIterator(LittleFS, "/", [](File & f) { + Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size() ); + }); + return true; + } + } + } + return false; +} + + +void initWiFiOTA() +{ + WiFi.mode(WIFI_STA); + WiFi.begin(ssid, password); + while (WiFi.waitForConnectResult() != WL_CONNECTED) { + Serial.println("Connection Failed! Rebooting..."); + delay(5000); + ESP.restart(); + } + + ArduinoOTA.onStart([]() { + String type; + if (ArduinoOTA.getCommand() == U_FLASH) { + type = "sketch"; + } else { // U_FS + type = "filesystem"; + } + + // NOTE: if updating FS this would be the place to unmount FS using FS.end() + Serial.println("Start updating " + type); + }); + ArduinoOTA.onEnd([]() { + Serial.println("\nEnd"); + }); + ArduinoOTA.onProgress([](unsigned int progress, unsigned int total) { + Serial.printf("Progress: %u%%\r", (progress / (total / 100))); + }); + ArduinoOTA.onError([](ota_error_t error) { + Serial.printf("Error[%u]: ", error); + if (error == OTA_AUTH_ERROR) { + Serial.println("Auth Failed"); + } else if (error == OTA_BEGIN_ERROR) { + Serial.println("Begin Failed"); + } else if (error == OTA_CONNECT_ERROR) { + Serial.println("Connect Failed"); + } else if (error == OTA_RECEIVE_ERROR) { + Serial.println("Receive Failed"); + } else if (error == OTA_END_ERROR) { + Serial.println("End Failed"); + } + }); + ArduinoOTA.begin(); +} + +void setup() { + + WiFi.persistent(false); + WiFi.disconnect(true); + Serial.begin(115200); + + Serial.println(); + Serial.printf("SDK Version: %s\n", ESP.getSdkVersion() ); + Serial.println("Core Version: " + ESP.getCoreVersion() ); + Serial.println("Full Version: " + ESP.getFullVersion() ); + + Serial.printf("Sketch size: %u\n", ESP.getSketchSize()); + Serial.printf("Free size: %u\n", ESP.getFreeSketchSpace()); + + Serial.println("Booting"); + + migrateFS(); // MUST call this before calling your own begin(); + + initWiFiOTA(); + + Serial.println("Ready"); + Serial.print("IP address: "); + Serial.println(WiFi.localIP()); +} + +void loop() { + ArduinoOTA.handle(); +} diff --git a/libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino b/libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino new file mode 100644 index 0000000000..41c5ecb366 --- /dev/null +++ b/libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino @@ -0,0 +1,139 @@ +/* + + This sketch will convert SPIFFS partitions to a custom FS on ESP8266 + + Change the `TARGET_FS_LAYOUT` to the partition layout that you want target + ie what you are trying to copy from. + + This ksetch shows how to create a FS different to the one provided for by the sketch defaults. + This is useful if you need to use an intermediate sketch to move the FS but need to maintain the + sketch size limit of say 512kb. + + How it works: It creates a LittleFS partition between the end of the sketch and the + start of whatever filesystem you set as target. This has IMPORTANT implications for the + amount of data you can move!!! eg a 4Mb flash module with a 3Mb SPIFFS partition only leaves + about 450k for the temp file system, so if you have more data than that on your 3Mb SPIFFS it + will fail. + + */ + + + +#include +#include +#include +#include +#include +#include + + +#define TARGET_FS_LAYOUT FST::layout_4m3m + +const uint32_t startSector = FST::layout_4m1m.startAddr - 0x40200000; +const uint32_t tempFSsize = FST::layout_4m1m.endAddr - FST::layout_4m1m.startAddr; + +fs::FS LittleFS_Different = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(startSector, tempFSsize, FS_PHYS_PAGE, FS_PHYS_BLOCK, 5))); + + +FSTools fstools; + +#ifndef STASSID +#define STASSID "xxxx" +#define STAPSK "xxxx" +#endif + +const char* ssid = STASSID; +const char* password = STAPSK; + + +bool migrateFS() +{ + if (!fstools.attemptToMountFS(LittleFS_Different)) { // Attempts to mount LittleFS without autoformat... + Serial.println(F("Default FS not found")); + if (fstools.mountAlternativeFS( FST::SPIFFS /* FST::LITTLEFS */ , TARGET_FS_LAYOUT , true )) { + Serial.println(F("Alternative found")); + if (fstools.moveFS(LittleFS_Different)) { + Serial.println(F("FileSystem Moved New FS contents:")); + fstools.fileListIterator(LittleFS_Different, "/", [](File & f) { + Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size() ); + }); + return true; + } + } + } + return false; +} + + +void initWiFiOTA() +{ + WiFi.mode(WIFI_STA); + WiFi.begin(ssid, password); + while (WiFi.waitForConnectResult() != WL_CONNECTED) { + Serial.println("Connection Failed! Rebooting..."); + delay(5000); + ESP.restart(); + } + + ArduinoOTA.onStart([]() { + String type; + if (ArduinoOTA.getCommand() == U_FLASH) { + type = "sketch"; + } else { // U_FS + type = "filesystem"; + } + + // NOTE: if updating FS this would be the place to unmount FS using FS.end() + Serial.println("Start updating " + type); + }); + ArduinoOTA.onEnd([]() { + Serial.println("\nEnd"); + }); + ArduinoOTA.onProgress([](unsigned int progress, unsigned int total) { + Serial.printf("Progress: %u%%\r", (progress / (total / 100))); + }); + ArduinoOTA.onError([](ota_error_t error) { + Serial.printf("Error[%u]: ", error); + if (error == OTA_AUTH_ERROR) { + Serial.println("Auth Failed"); + } else if (error == OTA_BEGIN_ERROR) { + Serial.println("Begin Failed"); + } else if (error == OTA_CONNECT_ERROR) { + Serial.println("Connect Failed"); + } else if (error == OTA_RECEIVE_ERROR) { + Serial.println("Receive Failed"); + } else if (error == OTA_END_ERROR) { + Serial.println("End Failed"); + } + }); + ArduinoOTA.begin(); +} + +void setup() { + + WiFi.persistent(false); + WiFi.disconnect(true); + Serial.begin(115200); + + Serial.println(); + Serial.printf("SDK Version: %s\n", ESP.getSdkVersion() ); + Serial.println("Core Version: " + ESP.getCoreVersion() ); + Serial.println("Full Version: " + ESP.getFullVersion() ); + + Serial.printf("Sketch size: %u\n", ESP.getSketchSize()); + Serial.printf("Free size: %u\n", ESP.getFreeSketchSpace()); + + Serial.println("Booting"); + + migrateFS(); // MUST call this before calling your own begin(); + + initWiFiOTA(); + + Serial.println("Ready"); + Serial.print("IP address: "); + Serial.println(WiFi.localIP()); +} + +void loop() { + ArduinoOTA.handle(); +} diff --git a/libraries/FSTools/package.json b/libraries/FSTools/package.json new file mode 100644 index 0000000000..2602e180a7 --- /dev/null +++ b/libraries/FSTools/package.json @@ -0,0 +1,11 @@ +{ + "name": "FSTools", + "keywords": "SPIFFS LittleFS", + "description": "A library that manages convertion between SPIFFS and LITTLEFS as well as mounting partitions outside of sketch default.", + "homepage": "", + "author": "sticilface", + "version": "1.0.0", + "frameworks": "arduino", + "platforms": "esp8266" +} + From 745db9ab55199f280ccf0da1392148a4cfbd63ba Mon Sep 17 00:00:00 2001 From: sticilface Date: Fri, 6 Nov 2020 22:33:01 +0000 Subject: [PATCH 02/12] Oops. Need to pass layout by reference in order to capture the correct address. Took a while to find that. There maybe a better way to store all these configs --- libraries/FSTools/FSTools.cpp | 18 +++++++++--------- libraries/FSTools/FSTools.h | 2 +- libraries/FSTools/package.json | 19 +++++++++---------- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/libraries/FSTools/FSTools.cpp b/libraries/FSTools/FSTools.cpp index 5bb1940018..c281b374fd 100644 --- a/libraries/FSTools/FSTools.cpp +++ b/libraries/FSTools/FSTools.cpp @@ -34,6 +34,7 @@ namespace FST { const layout layout_16m14m (0x40400000, 0x411FA000, 0x100, 0x2000 ); const layout layout_16m15m (0x40300000, 0x411FA000, 0x100, 0x2000 ); + }; FSTools::FSTools() @@ -60,7 +61,7 @@ bool FSTools::attemptToMountFS(fs::FS & fs) } -bool FSTools::mountAlternativeFS( FST::FS_t type, const FST::layout layout, bool keepMounted ) +bool FSTools::mountAlternativeFS( FST::FS_t type, const FST::layout & layout, bool keepMounted ) { FSConfig * pCfg{nullptr}; LittleFSConfig littleFSCfg(false); @@ -119,16 +120,16 @@ bool FSTools::moveFS(fs::FS & destinationFS) } - uint32_t startSector = (ESP.getSketchSize() + FLASH_SECTOR_SIZE - 1) & (~(FLASH_SECTOR_SIZE - 1)); + uint32_t startSector = (ESP.getSketchSize() + FLASH_SECTOR_SIZE - 1) & (~(FLASH_SECTOR_SIZE - 1)); //uint32_t endSector = (uint32_t)&_FS_start - FST::lowestSPIFFSstartAddr; - uint32_t lowestFSStart = 0x40300000; + uint32_t lowestFSStart = 0x40300000; - if (_layout) { - lowestFSStart = _layout->startAddr; - } + if (_layout) { + lowestFSStart = _layout->startAddr; + //Serial.printf("** _layout->startADDR = 0x%08x\n", _layout->startAddr ); + } - uint32_t endSector = lowestFSStart - 0x40200000; - + uint32_t endSector = lowestFSStart - 0x40200000; uint32_t tempFSsize = endSector - startSector; //Serial.printf("TempFS: start: %u, end: %u, size: %u, sketchSize = %u, _FS_start = %u\n", startSector, endSector, tempFSsize, ESP.getSketchSize(), (uint32_t)&_FS_start ); @@ -183,7 +184,6 @@ void FSTools::fileListIterator(FS & fs, const char * dirName, FST::FileCb Cb ) { Dir dir = fs.openDir(dirName); while (dir.next()) { - //const char * fileName = dir.fileName(); if (dir.isFile()) { File f = dir.openFile("r"); if (Cb) { diff --git a/libraries/FSTools/FSTools.h b/libraries/FSTools/FSTools.h index ce5eea6534..2cca0b0e82 100644 --- a/libraries/FSTools/FSTools.h +++ b/libraries/FSTools/FSTools.h @@ -75,7 +75,7 @@ class FSTools { FSTools(); ~FSTools(); bool attemptToMountFS(fs::FS & fs); - bool mountAlternativeFS( FST::FS_t type, const FST::layout layout, bool keepMounted = false ); + bool mountAlternativeFS( FST::FS_t type, const FST::layout & layout, bool keepMounted = false ); bool mounted(); bool moveFS(fs::FS & destinationFS); void reset(); diff --git a/libraries/FSTools/package.json b/libraries/FSTools/package.json index 2602e180a7..d10f43fc87 100644 --- a/libraries/FSTools/package.json +++ b/libraries/FSTools/package.json @@ -1,11 +1,10 @@ { - "name": "FSTools", - "keywords": "SPIFFS LittleFS", - "description": "A library that manages convertion between SPIFFS and LITTLEFS as well as mounting partitions outside of sketch default.", - "homepage": "", - "author": "sticilface", - "version": "1.0.0", - "frameworks": "arduino", - "platforms": "esp8266" -} - + "name": "FSTools", + "keywords": "SPIFFS LittleFS", + "description": "A library that manages convertion between SPIFFS and LITTLEFS as well as mounting partitions outside of sketch default.", + "homepage": "", + "author": "sticilface", + "version": "1.0.0", + "frameworks": "arduino", + "platforms": "esp8266" + } \ No newline at end of file From cbf4073695d9b6b6314461566e24538321042e47 Mon Sep 17 00:00:00 2001 From: sticilface Date: Sat, 7 Nov 2020 08:36:54 +0000 Subject: [PATCH 03/12] Update FSTools.cpp fix ESP.h to Esp.h --- libraries/FSTools/FSTools.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/FSTools/FSTools.cpp b/libraries/FSTools/FSTools.cpp index c281b374fd..fd34514c03 100644 --- a/libraries/FSTools/FSTools.cpp +++ b/libraries/FSTools/FSTools.cpp @@ -1,7 +1,7 @@ #include "FSTools.h" #include "LittleFS.h" #include -#include +#include namespace FST { From d382a33c7d3eaf791f4bbf8e787adbb5d0fb3740 Mon Sep 17 00:00:00 2001 From: sticilface Date: Sat, 7 Nov 2020 19:23:09 +0000 Subject: [PATCH 04/12] Fix unused variable i --- cores/esp8266/FS.h | 1 - 1 file changed, 1 deletion(-) diff --git a/cores/esp8266/FS.h b/cores/esp8266/FS.h index b1f1c528aa..c4715936d4 100644 --- a/cores/esp8266/FS.h +++ b/cores/esp8266/FS.h @@ -88,7 +88,6 @@ class File : public Stream uint8_t obuf[256]; size_t doneLen = 0; size_t sentLen; - int i; while (src.available() > sizeof(obuf)){ src.read(obuf, sizeof(obuf)); From ee1481aa7c827434cfd680703384dce3e7309ea1 Mon Sep 17 00:00:00 2001 From: sticilface Date: Sat, 7 Nov 2020 19:24:31 +0000 Subject: [PATCH 05/12] Parsed with restyle.sh. Compile with all errors. --- libraries/FSTools/FSTools.cpp | 14 ++-- libraries/FSTools/FSTools.h | 9 +-- .../examples/Basic_example/Basic_example.ino | 58 ++++++++--------- .../custom_FS_example/custom_FS_example.ino | 64 +++++++++---------- 4 files changed, 67 insertions(+), 78 deletions(-) diff --git a/libraries/FSTools/FSTools.cpp b/libraries/FSTools/FSTools.cpp index c281b374fd..d71fe1cb26 100644 --- a/libraries/FSTools/FSTools.cpp +++ b/libraries/FSTools/FSTools.cpp @@ -138,7 +138,7 @@ bool FSTools::moveFS(fs::FS & destinationFS) if (f) { sourceFileCount++; sourceByteTotal += f.size(); - _dumpFileInfo(f); + //_dumpFileInfo(f); } }); @@ -207,12 +207,12 @@ uint32_t FSTools::_getSize(const FST::layout & layout) } -void FSTools::_dumpFileInfo(File & f) -{ - if (f) { - //Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size() ); - } -} +// void FSTools::_dumpFileInfo(File & f) +// { +// if (f) { +// Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size() ); +// } +// } bool FSTools::_copyFS(FS & sourceFS, FS & destFS) { diff --git a/libraries/FSTools/FSTools.h b/libraries/FSTools/FSTools.h index 2cca0b0e82..8d48e5cd77 100644 --- a/libraries/FSTools/FSTools.h +++ b/libraries/FSTools/FSTools.h @@ -55,7 +55,6 @@ extern const layout layout_8m7m; extern const layout layout_16m14m; extern const layout layout_16m15m; - typedef std::function FileCb; @@ -63,15 +62,9 @@ typedef std::function FileCb; -//376884 -//376452 - - class FSTools { public: - - FSTools(); ~FSTools(); bool attemptToMountFS(fs::FS & fs); @@ -84,7 +77,7 @@ class FSTools { private: uint32_t _getStartAddr(const FST::layout & layout); uint32_t _getSize(const FST::layout & layout); - void _dumpFileInfo(File & f); +// void _dumpFileInfo(File & f); bool _copyFS(FS & sourceFS, FS & destFS); std::unique_ptr _pFS; diff --git a/libraries/FSTools/examples/Basic_example/Basic_example.ino b/libraries/FSTools/examples/Basic_example/Basic_example.ino index b5a7d3279b..6feca9ea66 100644 --- a/libraries/FSTools/examples/Basic_example/Basic_example.ino +++ b/libraries/FSTools/examples/Basic_example/Basic_example.ino @@ -2,19 +2,19 @@ This sketch will convert SPIFFS partitions to LittleFS on ESP8266 - Change the `TARGET_FS_LAYOUT` to the partition layout that you want target + Change the `TARGET_FS_LAYOUT` to the partition layout that you want target ie what you are trying to copy from. - Include in the sketch whatever you want the destination to be, in this case LittleFS, - but it could be SPIFFS to convert back if need be. + Include in the sketch whatever you want the destination to be, in this case LittleFS, + but it could be SPIFFS to convert back if need be. - How it works: It creates a LittleFS partition between the end of the sketch and the - start of whatever filesystem you set as target. This has IMPORTANT implications for the - amount of data you can move!!! eg a 4Mb flash module with a 3Mb SPIFFS partition only leaves - about 450k for the temp file system, so if you have more data than that on your 3Mb SPIFFS it - will fail. + How it works: It creates a LittleFS partition between the end of the sketch and the + start of whatever filesystem you set as target. This has IMPORTANT implications for the + amount of data you can move!!! eg a 4Mb flash module with a 3Mb SPIFFS partition only leaves + about 450k for the temp file system, so if you have more data than that on your 3Mb SPIFFS it + will fail. - */ +*/ @@ -28,7 +28,7 @@ #define TARGET_FS_LAYOUT FST::layout_4m3m -FSTools fstools; +FSTools fstools; #ifndef STASSID #define STASSID "xxxx" @@ -39,27 +39,25 @@ const char* ssid = STASSID; const char* password = STAPSK; -bool migrateFS() -{ - if (!fstools.attemptToMountFS(LittleFS)) { // Attempts to mount LittleFS without autoformat... - Serial.println(F("Default FS not found")); - if (fstools.mountAlternativeFS( FST::SPIFFS /* FST::LITTLEFS */ , TARGET_FS_LAYOUT , true )) { - Serial.println(F("Alternative found")); - if (fstools.moveFS(LittleFS)) { +bool migrateFS() { + if (!fstools.attemptToMountFS(LittleFS)) { // Attempts to mount LittleFS without autoformat... + Serial.println(F("Default FS not found")); + if (fstools.mountAlternativeFS(FST::SPIFFS /* FST::LITTLEFS */, TARGET_FS_LAYOUT, true)) { + Serial.println(F("Alternative found")); + if (fstools.moveFS(LittleFS)) { Serial.println(F("FileSystem Moved New FS contents:")); fstools.fileListIterator(LittleFS, "/", [](File & f) { - Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size() ); + Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size()); }); - return true; + return true; } } } - return false; + return false; } -void initWiFiOTA() -{ +void initWiFiOTA() { WiFi.mode(WIFI_STA); WiFi.begin(ssid, password); while (WiFi.waitForConnectResult() != WL_CONNECTED) { @@ -107,20 +105,20 @@ void setup() { WiFi.persistent(false); WiFi.disconnect(true); Serial.begin(115200); - - Serial.println(); - Serial.printf("SDK Version: %s\n", ESP.getSdkVersion() ); - Serial.println("Core Version: " + ESP.getCoreVersion() ); - Serial.println("Full Version: " + ESP.getFullVersion() ); + + Serial.println(); + Serial.printf("SDK Version: %s\n", ESP.getSdkVersion()); + Serial.println("Core Version: " + ESP.getCoreVersion()); + Serial.println("Full Version: " + ESP.getFullVersion()); Serial.printf("Sketch size: %u\n", ESP.getSketchSize()); Serial.printf("Free size: %u\n", ESP.getFreeSketchSpace()); - + Serial.println("Booting"); - migrateFS(); // MUST call this before calling your own begin(); + migrateFS(); // MUST call this before calling your own begin(); - initWiFiOTA(); + initWiFiOTA(); Serial.println("Ready"); Serial.print("IP address: "); diff --git a/libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino b/libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino index 41c5ecb366..6ca8d8c9b6 100644 --- a/libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino +++ b/libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino @@ -2,20 +2,20 @@ This sketch will convert SPIFFS partitions to a custom FS on ESP8266 - Change the `TARGET_FS_LAYOUT` to the partition layout that you want target + Change the `TARGET_FS_LAYOUT` to the partition layout that you want target ie what you are trying to copy from. - This ksetch shows how to create a FS different to the one provided for by the sketch defaults. - This is useful if you need to use an intermediate sketch to move the FS but need to maintain the - sketch size limit of say 512kb. + This ksetch shows how to create a FS different to the one provided for by the sketch defaults. + This is useful if you need to use an intermediate sketch to move the FS but need to maintain the + sketch size limit of say 512kb. - How it works: It creates a LittleFS partition between the end of the sketch and the - start of whatever filesystem you set as target. This has IMPORTANT implications for the - amount of data you can move!!! eg a 4Mb flash module with a 3Mb SPIFFS partition only leaves - about 450k for the temp file system, so if you have more data than that on your 3Mb SPIFFS it - will fail. + How it works: It creates a LittleFS partition between the end of the sketch and the + start of whatever filesystem you set as target. This has IMPORTANT implications for the + amount of data you can move!!! eg a 4Mb flash module with a 3Mb SPIFFS partition only leaves + about 450k for the temp file system, so if you have more data than that on your 3Mb SPIFFS it + will fail. - */ +*/ @@ -29,13 +29,13 @@ #define TARGET_FS_LAYOUT FST::layout_4m3m -const uint32_t startSector = FST::layout_4m1m.startAddr - 0x40200000; -const uint32_t tempFSsize = FST::layout_4m1m.endAddr - FST::layout_4m1m.startAddr; +const uint32_t startSector = FST::layout_4m1m.startAddr - 0x40200000; +const uint32_t tempFSsize = FST::layout_4m1m.endAddr - FST::layout_4m1m.startAddr; fs::FS LittleFS_Different = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(startSector, tempFSsize, FS_PHYS_PAGE, FS_PHYS_BLOCK, 5))); -FSTools fstools; +FSTools fstools; #ifndef STASSID #define STASSID "xxxx" @@ -46,27 +46,25 @@ const char* ssid = STASSID; const char* password = STAPSK; -bool migrateFS() -{ - if (!fstools.attemptToMountFS(LittleFS_Different)) { // Attempts to mount LittleFS without autoformat... - Serial.println(F("Default FS not found")); - if (fstools.mountAlternativeFS( FST::SPIFFS /* FST::LITTLEFS */ , TARGET_FS_LAYOUT , true )) { - Serial.println(F("Alternative found")); - if (fstools.moveFS(LittleFS_Different)) { +bool migrateFS() { + if (!fstools.attemptToMountFS(LittleFS_Different)) { // Attempts to mount LittleFS without autoformat... + Serial.println(F("Default FS not found")); + if (fstools.mountAlternativeFS(FST::SPIFFS /* FST::LITTLEFS */, TARGET_FS_LAYOUT, true)) { + Serial.println(F("Alternative found")); + if (fstools.moveFS(LittleFS_Different)) { Serial.println(F("FileSystem Moved New FS contents:")); fstools.fileListIterator(LittleFS_Different, "/", [](File & f) { - Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size() ); + Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size()); }); - return true; + return true; } } } - return false; + return false; } -void initWiFiOTA() -{ +void initWiFiOTA() { WiFi.mode(WIFI_STA); WiFi.begin(ssid, password); while (WiFi.waitForConnectResult() != WL_CONNECTED) { @@ -114,20 +112,20 @@ void setup() { WiFi.persistent(false); WiFi.disconnect(true); Serial.begin(115200); - - Serial.println(); - Serial.printf("SDK Version: %s\n", ESP.getSdkVersion() ); - Serial.println("Core Version: " + ESP.getCoreVersion() ); - Serial.println("Full Version: " + ESP.getFullVersion() ); + + Serial.println(); + Serial.printf("SDK Version: %s\n", ESP.getSdkVersion()); + Serial.println("Core Version: " + ESP.getCoreVersion()); + Serial.println("Full Version: " + ESP.getFullVersion()); Serial.printf("Sketch size: %u\n", ESP.getSketchSize()); Serial.printf("Free size: %u\n", ESP.getFreeSketchSpace()); - + Serial.println("Booting"); - migrateFS(); // MUST call this before calling your own begin(); + migrateFS(); // MUST call this before calling your own begin(); - initWiFiOTA(); + initWiFiOTA(); Serial.println("Ready"); Serial.print("IP address: "); From 4afa2d450250d1f51351ec3922e490d928c21729 Mon Sep 17 00:00:00 2001 From: sticilface Date: Sat, 7 Nov 2020 19:40:22 +0000 Subject: [PATCH 06/12] remove unused variable --- libraries/FSTools/FSTools.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/libraries/FSTools/FSTools.cpp b/libraries/FSTools/FSTools.cpp index 33f6b40b7a..e1fb8a214f 100644 --- a/libraries/FSTools/FSTools.cpp +++ b/libraries/FSTools/FSTools.cpp @@ -230,12 +230,10 @@ bool FSTools::_copyFS(FS & sourceFS, FS & destFS) fileListIterator(sourceFS, "/", [&count, &destFS](File & sourceFile){ if (sourceFile) { File destFile = destFS.open(sourceFile.fullName(), "w"); - bool copied = false; if (destFile) { destFile.setTimeout(5000); size_t written = destFile.write(sourceFile); if (written == sourceFile.size()) { - copied = true; count++; } } From 5022cf0ff752e37a58488a7ea74958201dad8c6e Mon Sep 17 00:00:00 2001 From: sticilface Date: Sat, 7 Nov 2020 19:40:45 +0000 Subject: [PATCH 07/12] fix different sign complication error --- cores/esp8266/FS.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cores/esp8266/FS.h b/cores/esp8266/FS.h index c4715936d4..d3883838ad 100644 --- a/cores/esp8266/FS.h +++ b/cores/esp8266/FS.h @@ -89,7 +89,7 @@ class File : public Stream size_t doneLen = 0; size_t sentLen; - while (src.available() > sizeof(obuf)){ + while (src.available() > (int)sizeof(obuf)){ src.read(obuf, sizeof(obuf)); sentLen = write(obuf, sizeof(obuf)); doneLen = doneLen + sentLen; From 9feb47ab06c2c6a8cf4abe7262d93decf9299433 Mon Sep 17 00:00:00 2001 From: sticilface Date: Sun, 15 Nov 2020 11:01:42 +0000 Subject: [PATCH 08/12] Fix indentation to spaces Run test/restyle.sh Remove commented code Use #ifdef blocks for debugging. `DEBUG_ESP_CORE` and `DEBUG_ESP_PORT` use `static constexpr layout` which saves ROM ~500B for unused vars --- libraries/FSTools/FSTools.cpp | 360 +++++++++++++++++----------------- libraries/FSTools/FSTools.h | 123 ++++++------ 2 files changed, 246 insertions(+), 237 deletions(-) diff --git a/libraries/FSTools/FSTools.cpp b/libraries/FSTools/FSTools.cpp index e1fb8a214f..9ef602c88c 100644 --- a/libraries/FSTools/FSTools.cpp +++ b/libraries/FSTools/FSTools.cpp @@ -4,38 +4,13 @@ #include -namespace FST { - const layout layout_512k32 (0x40273000, 0x4027B000, 0x100, 0x1000 ); - const layout layout_512k64 (0x4026B000, 0x4027B000, 0x100, 0x1000 ); - const layout layout_512k128 (0x4025B000, 0x4027B000, 0x100, 0x1000 ); +#if defined(DEBUG_ESP_CORE) +#define FSTOOLSDEBUG(_1, ...) { DEBUG_ESP_PORT.printf_P( PSTR(_1),##__VA_ARGS__); } +#else +#define FSTOOLSDEBUG(...) {} +#endif - const layout layout_1m64 (0x402EB000, 0x402FB000, 0x100, 0x1000 ); - const layout layout_1m128 (0x402DB000, 0x402FB000, 0x100, 0x1000 ); - const layout layout_1m144 (0x402D7000, 0x402FB000, 0x100, 0x1000 ); - const layout layout_1m160 (0x402D3000, 0x402FB000, 0x100, 0x1000 ); - const layout layout_1m192 (0x402CB000, 0x402FB000, 0x100, 0x1000 ); - const layout layout_1m256 (0x402BB000, 0x402FB000, 0x100, 0x1000 ); - const layout layout_1m512 (0x4027B000, 0x402FB000, 0x100, 0x2000 ); - - const layout layout_2m64 (0x403F0000, 0x403FB000, 0x100, 0x1000 ); - const layout layout_2m128 (0x403E0000, 0x403FB000, 0x100, 0x1000 ); - const layout layout_2m256 (0x403C0000, 0x403FB000, 0x100, 0x1000 ); - const layout layout_2m512 (0x40380000, 0x403FA000, 0x100, 0x2000 ); - const layout layout_2m1m (0x40300000, 0x403FA000, 0x100, 0x2000 ); - - const layout layout_4m1m (0x40500000, 0x405FA000, 0x100, 0x2000 ); - const layout layout_4m2m (0x40400000, 0x405FA000, 0x100, 0x2000 ); - const layout layout_4m3m (0x40300000, 0x405FA000, 0x100, 0x2000 ); - - const layout layout_8m6m (0x40400000, 0x409FA000, 0x100, 0x2000 ); - const layout layout_8m7m (0x40300000, 0x409FA000, 0x100, 0x2000 ); - - const layout layout_16m14m (0x40400000, 0x411FA000, 0x100, 0x2000 ); - const layout layout_16m15m (0x40300000, 0x411FA000, 0x100, 0x2000 ); - - -}; FSTools::FSTools() { @@ -44,204 +19,235 @@ FSTools::FSTools() FSTools::~FSTools() { - reset(); + reset(); } bool FSTools::attemptToMountFS(fs::FS & fs) { - LittleFSConfig littleFSCfg(false); - SPIFFSConfig SPIFFSCfg(false); - // try to apply the "safe" no format config to the FS... doesn't matter which.. just need one to apply.. - if (!fs.setConfig(littleFSCfg) && ! fs.setConfig(SPIFFSCfg) ) - { - return false; - } - return fs.begin(); + LittleFSConfig littleFSCfg(false); + SPIFFSConfig SPIFFSCfg(false); + // try to apply the "safe" no format config to the FS... doesn't matter which.. just need one to apply correctly.. + if (!fs.setConfig(littleFSCfg) && ! fs.setConfig(SPIFFSCfg)) + { + return false; + } + return fs.begin(); } -bool FSTools::mountAlternativeFS( FST::FS_t type, const FST::layout & layout, bool keepMounted ) +bool FSTools::mountAlternativeFS(FST::FS_t type, const FST::layout & layout, bool keepMounted) { - FSConfig * pCfg{nullptr}; + FSConfig * pCfg{nullptr}; LittleFSConfig littleFSCfg(false); - SPIFFSConfig SPIFFSCfg(false); + SPIFFSConfig SPIFFSCfg(false); reset(); - switch (type) { - case FST::SPIFFS : { - _pFS.reset( new FS(FSImplPtr(new spiffs_impl::SPIFFSImpl (_getStartAddr(layout) , _getSize(layout) , layout.page, layout.block, 5))) ); - pCfg = &SPIFFSCfg; - break; - } - case FST::LITTLEFS : { - _pFS.reset( new FS(FSImplPtr(new littlefs_impl::LittleFSImpl(_getStartAddr(layout) , _getSize(layout) , layout.page, layout.block, 5))) ); - pCfg = &littleFSCfg; - break; - } - }; - - if (_pFS && pCfg && _pFS->setConfig(*pCfg) && _pFS->begin()) { - if (!keepMounted) { - _pFS->end(); - } - _mounted = true; - _layout = &layout; - return true; - } - - if (_pFS) { - _pFS.reset(); - } - _mounted = false; - return false; - -}; + switch (type) + { + case FST::SPIFFS : + { + _pFS.reset(new FS(FSImplPtr(new spiffs_impl::SPIFFSImpl(_getStartAddr(layout), _getSize(layout), layout.page, layout.block, 5)))); + pCfg = &SPIFFSCfg; + break; + } + case FST::LITTLEFS : + { + _pFS.reset(new FS(FSImplPtr(new littlefs_impl::LittleFSImpl(_getStartAddr(layout), _getSize(layout), layout.page, layout.block, 5)))); + pCfg = &littleFSCfg; + break; + } + }; + if (_pFS && pCfg && _pFS->setConfig(*pCfg) && _pFS->begin()) + { + if (!keepMounted) + { + _pFS->end(); + } + _mounted = true; + _layout = &layout; + return true; + } -bool FSTools::mounted() -{ - return _mounted; + if (_pFS) + { + _pFS.reset(); + } + _mounted = false; + return false; }; +bool FSTools::mounted() +{ + return _mounted; +}; bool FSTools::moveFS(fs::FS & destinationFS) { - uint32_t sourceFileCount = 0; - uint32_t sourceByteTotal = 0; - bool result = false; - - // do not init new fs... until old one is copied to test... - if (!_mounted || !_pFS) { - //Serial.println("Source FS not mounted"); - return false; - } - - - uint32_t startSector = (ESP.getSketchSize() + FLASH_SECTOR_SIZE - 1) & (~(FLASH_SECTOR_SIZE - 1)); - //uint32_t endSector = (uint32_t)&_FS_start - FST::lowestSPIFFSstartAddr; - uint32_t lowestFSStart = 0x40300000; - - if (_layout) { - lowestFSStart = _layout->startAddr; - //Serial.printf("** _layout->startADDR = 0x%08x\n", _layout->startAddr ); - } - - uint32_t endSector = lowestFSStart - 0x40200000; - uint32_t tempFSsize = endSector - startSector; - - //Serial.printf("TempFS: start: %u, end: %u, size: %u, sketchSize = %u, _FS_start = %u\n", startSector, endSector, tempFSsize, ESP.getSketchSize(), (uint32_t)&_FS_start ); - - fileListIterator(*_pFS, "/", [&sourceFileCount, &sourceByteTotal, this](File & f) { - if (f) { - sourceFileCount++; - sourceByteTotal += f.size(); - //_dumpFileInfo(f); + uint32_t sourceFileCount = 0; + uint32_t sourceByteTotal = 0; + bool result = false; + + if (!_mounted || !_pFS) + { + FSTOOLSDEBUG("Source FS not mounted\n"); + return false; + } + + uint32_t startSector = (ESP.getSketchSize() + FLASH_SECTOR_SIZE - 1) & (~(FLASH_SECTOR_SIZE - 1)); + uint32_t lowestFSStart = 0x40300000; + + if (_layout) + { + lowestFSStart = _layout->startAddr; + FSTOOLSDEBUG("_layout->startADDR = 0x%08x\n", _layout->startAddr); } + + uint32_t endSector = lowestFSStart - 0x40200000; + uint32_t tempFSsize = endSector - startSector; + + FSTOOLSDEBUG("TempFS: start: %u, end: %u, size: %u, sketchSize = %u, _FS_start = %u\n", startSector, endSector, tempFSsize, ESP.getSketchSize(), (uint32_t)&_FS_start); + + fileListIterator(*_pFS, "/", [&sourceFileCount, &sourceByteTotal, this](File & f) + { + if (f) + { + sourceFileCount++; + sourceByteTotal += f.size(); + +#ifdef DEBUG_ESP_CORE + _dumpFileInfo(f); +#endif + + } }); - //Serial.printf("%u Files Found Total Size = %u\n", sourceFileCount, sourceByteTotal); - //Serial.printf("Size of dummy FS = %u\n", tempFSsize); + FSTOOLSDEBUG("%u Files Found Total Size = %u\n", sourceFileCount, sourceByteTotal); + FSTOOLSDEBUG("Size of dummy FS = %u\n", tempFSsize); FS tempFS = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(startSector, tempFSsize, FS_PHYS_PAGE, FS_PHYS_BLOCK, 5))); - if (tempFS.format() && tempFS.begin()) { - if (_copyFS(*_pFS,tempFS)) { - //Serial.println("Files copied to temp File System"); - reset(); - if (destinationFS.format() && destinationFS.begin()) // must format then mount the new FS - { - if (_copyFS(tempFS, destinationFS)) { - //Serial.println("Files copied back to new FS"); - result = true; - } - } else { - //Serial.println("Error Mounting "); - } - } else { - //Serial.println("Copy Failed"); - } - tempFS.end(); - } else { - //Serial.println("Failed to begin() TempFS"); - } - return result; -}; + if (tempFS.format() && tempFS.begin()) + { + if (_copyFS(*_pFS, tempFS)) + { + FSTOOLSDEBUG("Files copied to temp File System\n"); + reset(); + if (destinationFS.format() && destinationFS.begin()) // must format then mount the new FS + { + if (_copyFS(tempFS, destinationFS)) + { + FSTOOLSDEBUG("Files copied back to new FS\n"); + result = true; + } + } + else + { + FSTOOLSDEBUG("Error Mounting\n"); + } + } + else + { + FSTOOLSDEBUG("Copy Failed\n"); + } + tempFS.end(); + } + else + { + FSTOOLSDEBUG("Failed to begin() TempFS\n"); + } + return result; +}; void FSTools::reset() { - _mounted = false; - _layout = nullptr; - if (_pFS) { - _pFS->end(); - _pFS.reset(); - } + _mounted = false; + _layout = nullptr; + if (_pFS) + { + _pFS->end(); + _pFS.reset(); + } } -void FSTools::fileListIterator(FS & fs, const char * dirName, FST::FileCb Cb ) +void FSTools::fileListIterator(FS & fs, const char * dirName, FST::FileCb Cb) { Dir dir = fs.openDir(dirName); - while (dir.next()) { - if (dir.isFile()) { - File f = dir.openFile("r"); - if (Cb) { - Cb(f); + while (dir.next()) + { + if (dir.isFile()) + { + File f = dir.openFile("r"); + if (Cb) + { + Cb(f); + } + } + else + { + fileListIterator(fs, dir.fileName().c_str(), Cb); } - } else { - fileListIterator(fs, dir.fileName().c_str() , Cb); - } } } uint32_t FSTools::_getStartAddr(const FST::layout & layout) { - return ( layout.startAddr - 0x40200000 ); + return (layout.startAddr - 0x40200000); } uint32_t FSTools::_getSize(const FST::layout & layout) { - return (layout.endAddr - layout.startAddr ); -} - + return (layout.endAddr - layout.startAddr); +} -// void FSTools::_dumpFileInfo(File & f) -// { -// if (f) { -// Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size() ); -// } -// } +#ifdef DEBUG_ESP_CORE +void FSTools::_dumpFileInfo(File & f) +{ + if (f) + { + DEBUG_ESP_PORT.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size()); + } +} +#endif bool FSTools::_copyFS(FS & sourceFS, FS & destFS) { - uint32_t sourceFileCount = 0; - uint32_t sourceByteTotal = 0; - - fileListIterator(sourceFS, "/", [&sourceFileCount, &sourceByteTotal](File & f) { - if (f) { - sourceFileCount++; - sourceByteTotal += f.size(); - } + uint32_t sourceFileCount = 0; + uint32_t sourceByteTotal = 0; + + fileListIterator(sourceFS, "/", [&sourceFileCount, &sourceByteTotal](File & f) + { + if (f) + { + sourceFileCount++; + sourceByteTotal += f.size(); + } }); - size_t count = 0; - fileListIterator(sourceFS, "/", [&count, &destFS](File & sourceFile){ - if (sourceFile) { - File destFile = destFS.open(sourceFile.fullName(), "w"); - if (destFile) { - destFile.setTimeout(5000); - size_t written = destFile.write(sourceFile); - if (written == sourceFile.size()) { - count++; - } - } - destFile.close(); - sourceFile.close(); + size_t count = 0; + fileListIterator(sourceFS, "/", [&count, &destFS](File & sourceFile) + { + if (sourceFile) + { + File destFile = destFS.open(sourceFile.fullName(), "w"); + if (destFile) + { + destFile.setTimeout(5000); // this value was chosen empirically as it failed with default timeout. + size_t written = destFile.write(sourceFile); + if (written == sourceFile.size()) + { + count++; + } + } + destFile.close(); + sourceFile.close(); } - }); + }); + + return (count == sourceFileCount); - return (count == sourceFileCount); - } diff --git a/libraries/FSTools/FSTools.h b/libraries/FSTools/FSTools.h index 8d48e5cd77..4722a1742b 100644 --- a/libraries/FSTools/FSTools.h +++ b/libraries/FSTools/FSTools.h @@ -1,88 +1,91 @@ #pragma once -#include +#include #include #include /* - - A temporary FS is made between the END of the sketch... and the start of the partition you try to mount, to maximise the available space for copying the FS. - The WORST case this is at 0x40300000 which is for a 3m FS on 4m flash.. leaving 460Kb for copying. + A temporary FS is made between the END of the sketch... and the start of the partition you try to mount, to maximise the available space for copying the FS. + The WORST case this is at 0x40300000 which is for a 3m FS on 4m flash.. leaving 460Kb for copying. - I have not worked out a way to prevent linking of ALL the layouts except to use #ifdef guards. Ideas welcome as it uses 400B ROM. - */ +*/ -namespace FST { -struct layout { - layout(uint32_t s, uint32_t e, uint32_t p, uint32_t b) : startAddr(s), endAddr(e), page(p), block(b) {}; - uint32_t startAddr{0}; - uint32_t endAddr{0}; - uint32_t page{0}; - uint32_t block{0}; -}; -enum FS_t : uint8_t { - SPIFFS, - LITTLEFS -}; +namespace FST +{ -extern const layout layout_512k32; -extern const layout layout_512k64; -extern const layout layout_512k128; + struct layout + { + constexpr layout(uint32_t s, uint32_t e, uint32_t p, uint32_t b) : startAddr(s), endAddr(e), page(p), block(b) {}; + const uint32_t startAddr{0}; + const uint32_t endAddr{0}; + const uint32_t page{0}; + const uint32_t block{0}; + }; -extern const layout layout_1m64; -extern const layout layout_1m128; -extern const layout layout_1m144; -extern const layout layout_1m160; -extern const layout layout_1m192; -extern const layout layout_1m256; -extern const layout layout_1m512; + enum FS_t : uint8_t + { + SPIFFS, + LITTLEFS + }; -extern const layout layout_2m64; -extern const layout layout_2m128; -extern const layout layout_2m256; -extern const layout layout_2m512; -extern const layout layout_2m1m; + static constexpr layout layout_512k32 = { 0x40273000, 0x4027B000, 0x100, 0x1000 }; + static constexpr layout layout_512k64 = { 0x4026B000, 0x4027B000, 0x100, 0x1000 }; + static constexpr layout layout_512k128 = { 0x4025B000, 0x4027B000, 0x100, 0x1000 }; -extern const layout layout_4m1m; -extern const layout layout_4m2m; -extern const layout layout_4m3m; + static constexpr layout layout_1m64 = { 0x402EB000, 0x402FB000, 0x100, 0x1000 }; + static constexpr layout layout_1m128 = { 0x402DB000, 0x402FB000, 0x100, 0x1000 }; + static constexpr layout layout_1m144 = { 0x402D7000, 0x402FB000, 0x100, 0x1000 }; + static constexpr layout layout_1m160 = { 0x402D3000, 0x402FB000, 0x100, 0x1000 }; + static constexpr layout layout_1m192 = { 0x402CB000, 0x402FB000, 0x100, 0x1000 }; + static constexpr layout layout_1m256 = { 0x402BB000, 0x402FB000, 0x100, 0x1000 }; + static constexpr layout layout_1m512 = { 0x4027B000, 0x402FB000, 0x100, 0x2000 }; -extern const layout layout_8m6m; -extern const layout layout_8m7m; + static constexpr layout layout_2m64 = { 0x403F0000, 0x403FB000, 0x100, 0x1000 }; + static constexpr layout layout_2m128 = { 0x403E0000, 0x403FB000, 0x100, 0x1000 }; + static constexpr layout layout_2m256 = { 0x403C0000, 0x403FB000, 0x100, 0x1000 }; + static constexpr layout layout_2m512 = { 0x40380000, 0x403FA000, 0x100, 0x2000 }; + static constexpr layout layout_2m1m = { 0x40300000, 0x403FA000, 0x100, 0x2000 }; -extern const layout layout_16m14m; -extern const layout layout_16m15m; + static constexpr layout layout_4m1m = { 0x40500000, 0x405FA000, 0x100, 0x2000 }; + static constexpr layout layout_4m2m = { 0x40400000, 0x405FA000, 0x100, 0x2000 }; + static constexpr layout layout_4m3m = { 0x40300000, 0x405FA000, 0x100, 0x2000 }; -typedef std::function FileCb; + static constexpr layout layout_8m6m = { 0x40400000, 0x409FA000, 0x100, 0x2000 }; + static constexpr layout layout_8m7m = { 0x40300000, 0x409FA000, 0x100, 0x2000 }; + static constexpr layout layout_16m14m = { 0x40400000, 0x411FA000, 0x100, 0x2000 }; + static constexpr layout layout_16m15m = { 0x40300000, 0x411FA000, 0x100, 0x2000 }; -}; + typedef std::function FileCb; +}; -class FSTools { +class FSTools +{ public: - FSTools(); - ~FSTools(); - bool attemptToMountFS(fs::FS & fs); - bool mountAlternativeFS( FST::FS_t type, const FST::layout & layout, bool keepMounted = false ); - bool mounted(); - bool moveFS(fs::FS & destinationFS); - void reset(); - void fileListIterator(FS & fs, const char * dirName, FST::FileCb Cb ); + FSTools(); + ~FSTools(); + bool attemptToMountFS(fs::FS & fs); + bool mountAlternativeFS(FST::FS_t type, const FST::layout & layout, bool keepMounted = false); + bool mounted(); + bool moveFS(fs::FS & destinationFS); + void reset(); + void fileListIterator(FS & fs, const char * dirName, FST::FileCb Cb); private: - uint32_t _getStartAddr(const FST::layout & layout); - uint32_t _getSize(const FST::layout & layout); -// void _dumpFileInfo(File & f); - bool _copyFS(FS & sourceFS, FS & destFS); - - std::unique_ptr _pFS; - bool _mounted{false}; - const FST::layout * _layout{nullptr}; - + uint32_t _getStartAddr(const FST::layout & layout); + uint32_t _getSize(const FST::layout & layout); +#ifdef DEBUG_ESP_CORE + void _dumpFileInfo(File & f); +#endif + bool _copyFS(FS & sourceFS, FS & destFS); + + std::unique_ptr _pFS; + bool _mounted{false}; + const FST::layout * _layout{nullptr}; }; \ No newline at end of file From 9f80841319e854ed96091d94488c64e0a941ebe8 Mon Sep 17 00:00:00 2001 From: sticilface Date: Fri, 20 Nov 2020 20:52:13 +0000 Subject: [PATCH 09/12] Update FSTools.cpp Add yield in between file copy --- libraries/FSTools/FSTools.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/FSTools/FSTools.cpp b/libraries/FSTools/FSTools.cpp index 9ef602c88c..c18e968b76 100644 --- a/libraries/FSTools/FSTools.cpp +++ b/libraries/FSTools/FSTools.cpp @@ -245,6 +245,7 @@ bool FSTools::_copyFS(FS & sourceFS, FS & destFS) } destFile.close(); sourceFile.close(); + yield(); } }); From 319f1d066dcbaa70edf64c5a5c084e642e5b9393 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 15 Jun 2022 23:54:10 +0200 Subject: [PATCH 10/12] style --- .../FSTools/examples/Basic_example/Basic_example.ino | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libraries/FSTools/examples/Basic_example/Basic_example.ino b/libraries/FSTools/examples/Basic_example/Basic_example.ino index 6feca9ea66..acacb362a5 100644 --- a/libraries/FSTools/examples/Basic_example/Basic_example.ino +++ b/libraries/FSTools/examples/Basic_example/Basic_example.ino @@ -40,13 +40,13 @@ const char* password = STAPSK; bool migrateFS() { - if (!fstools.attemptToMountFS(LittleFS)) { // Attempts to mount LittleFS without autoformat... + if (!fstools.attemptToMountFS(LittleFS)) { // Attempts to mount LittleFS without autoformat... Serial.println(F("Default FS not found")); - if (fstools.mountAlternativeFS(FST::SPIFFS /* FST::LITTLEFS */, TARGET_FS_LAYOUT, true)) { + if (fstools.mountAlternativeFS(FST::SPIFFS /* FST::LITTLEFS */, TARGET_FS_LAYOUT, true)) { Serial.println(F("Alternative found")); if (fstools.moveFS(LittleFS)) { Serial.println(F("FileSystem Moved New FS contents:")); - fstools.fileListIterator(LittleFS, "/", [](File & f) { + fstools.fileListIterator(LittleFS, "/", [](File& f) { Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size()); }); return true; @@ -70,7 +70,7 @@ void initWiFiOTA() { String type; if (ArduinoOTA.getCommand() == U_FLASH) { type = "sketch"; - } else { // U_FS + } else { // U_FS type = "filesystem"; } @@ -116,7 +116,7 @@ void setup() { Serial.println("Booting"); - migrateFS(); // MUST call this before calling your own begin(); + migrateFS(); // MUST call this before calling your own begin(); initWiFiOTA(); From 7e9968ca88121c348d8804b57985d6cd4d208549 Mon Sep 17 00:00:00 2001 From: david gauchard Date: Wed, 15 Jun 2022 23:55:49 +0200 Subject: [PATCH 11/12] style --- .../custom_FS_example/custom_FS_example.ino | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino b/libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino index 6ca8d8c9b6..3dc2e804fc 100644 --- a/libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino +++ b/libraries/FSTools/examples/custom_FS_example/custom_FS_example.ino @@ -30,7 +30,7 @@ #define TARGET_FS_LAYOUT FST::layout_4m3m const uint32_t startSector = FST::layout_4m1m.startAddr - 0x40200000; -const uint32_t tempFSsize = FST::layout_4m1m.endAddr - FST::layout_4m1m.startAddr; +const uint32_t tempFSsize = FST::layout_4m1m.endAddr - FST::layout_4m1m.startAddr; fs::FS LittleFS_Different = FS(FSImplPtr(new littlefs_impl::LittleFSImpl(startSector, tempFSsize, FS_PHYS_PAGE, FS_PHYS_BLOCK, 5))); @@ -39,7 +39,7 @@ FSTools fstools; #ifndef STASSID #define STASSID "xxxx" -#define STAPSK "xxxx" +#define STAPSK "xxxx" #endif const char* ssid = STASSID; @@ -47,13 +47,13 @@ const char* password = STAPSK; bool migrateFS() { - if (!fstools.attemptToMountFS(LittleFS_Different)) { // Attempts to mount LittleFS without autoformat... + if (!fstools.attemptToMountFS(LittleFS_Different)) { // Attempts to mount LittleFS without autoformat... Serial.println(F("Default FS not found")); - if (fstools.mountAlternativeFS(FST::SPIFFS /* FST::LITTLEFS */, TARGET_FS_LAYOUT, true)) { + if (fstools.mountAlternativeFS(FST::SPIFFS /* FST::LITTLEFS */, TARGET_FS_LAYOUT, true)) { Serial.println(F("Alternative found")); if (fstools.moveFS(LittleFS_Different)) { Serial.println(F("FileSystem Moved New FS contents:")); - fstools.fileListIterator(LittleFS_Different, "/", [](File & f) { + fstools.fileListIterator(LittleFS_Different, "/", [](File& f) { Serial.printf_P(PSTR(" File: %-30s [%8uB]\n"), f.fullName(), f.size()); }); return true; @@ -77,7 +77,7 @@ void initWiFiOTA() { String type; if (ArduinoOTA.getCommand() == U_FLASH) { type = "sketch"; - } else { // U_FS + } else { // U_FS type = "filesystem"; } @@ -123,7 +123,7 @@ void setup() { Serial.println("Booting"); - migrateFS(); // MUST call this before calling your own begin(); + migrateFS(); // MUST call this before calling your own begin(); initWiFiOTA(); From 3c43f18510e39e919d8f3dbcd935d3ee6698729d Mon Sep 17 00:00:00 2001 From: david gauchard Date: Thu, 16 Jun 2022 00:11:18 +0200 Subject: [PATCH 12/12] style --- libraries/FSTools/examples/Basic_example/Basic_example.ino | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/FSTools/examples/Basic_example/Basic_example.ino b/libraries/FSTools/examples/Basic_example/Basic_example.ino index acacb362a5..efa8b5740d 100644 --- a/libraries/FSTools/examples/Basic_example/Basic_example.ino +++ b/libraries/FSTools/examples/Basic_example/Basic_example.ino @@ -32,7 +32,7 @@ FSTools fstools; #ifndef STASSID #define STASSID "xxxx" -#define STAPSK "xxxx" +#define STAPSK "xxxx" #endif const char* ssid = STASSID; pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy