From 0591eeff8e18f93516277830687e2a88a351a41f Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Wed, 8 Jan 2020 19:25:14 +0100 Subject: [PATCH 01/31] auto sort timers --- src/times_utils.py | 82 +++++++++++++++++++++++++++++------- tests/test_http_set_timer.py | 20 +++++++++ tests/test_power_timer.py | 1 - tests/test_time_utils.py | 17 ++++---- 4 files changed, 95 insertions(+), 25 deletions(-) diff --git a/src/times_utils.py b/src/times_utils.py index 27d51bc..a11023f 100644 --- a/src/times_utils.py +++ b/src/times_utils.py @@ -4,11 +4,9 @@ import constants import ure import utime +from micropython import const - -def parse_time(clock_time): - hours, minutes = clock_time.split(':') - return int(hours), int(minutes) +MAX_DAY_MINUTES = const(24 * 60 + 59) def iter_times(times): @@ -38,13 +36,25 @@ def validate_times(times): return True -def parse_timers(data): +def clock_time2minutes(clock_time): + hours, minutes = clock_time.split(':') + return int(hours) * 60 + int(minutes) + + +def minutes2clock_time(minutes): + hours = int(minutes / 60) + minutes = minutes - (hours * 60) + return hours, minutes + + +def timers2list(data): print('parse_timers:', repr(data)) regex = ure.compile(r'^\D*(\d+:\d+)\D+(\d+:\d+)\D*$') - data = data.strip().split('\n') + data = data.strip().replace('\r\n', '\n').split('\n') - last_time = None - for no, line in enumerate(data, 1): + times = [] + for line_no, line in enumerate(data, 1): + print(line_no, repr(line)) line = line.strip() if not line: continue @@ -52,20 +62,60 @@ def parse_timers(data): match = regex.match(line) if not match: print('Error in: %r' % line) - raise ValueError('Wrong time in line %i' % no) + raise ValueError('Wrong time in line %i' % line_no) - start_time = parse_time(match.group(1)) - end_time = parse_time(match.group(2)) + start_minutes = clock_time2minutes(match.group(1)) + end_minutes = clock_time2minutes(match.group(2)) del match # collect match object gc.collect() - if start_time >= end_time or (last_time is not None and start_time <= last_time): - print('Error in: %r' % line) - raise ValueError('Times in line %i are not sequential!' % no) + if not 0 <= start_minutes <= MAX_DAY_MINUTES: + print('Error in start time: %r' % line) + raise ValueError('No valid start time in %i!' % line_no) - last_time = end_time - yield (start_time, end_time) + if not 0 <= end_minutes <= MAX_DAY_MINUTES: + print('Error in end time: %r' % line) + raise ValueError('No valid end time in %i!' % line_no) + + if start_minutes >= end_minutes: + print('Error in: %r (%r >= %r)' % (line, start_minutes, end_minutes)) + raise ValueError('Times in line %i are not sequential!' % line_no) + + times.append(start_minutes) + times.append(end_minutes) + + times.sort() + return times + + +def parse_timers(data): + times = timers2list(data) + gc.collect() + + last_time = None + pos = 0 + line_no = 1 + while pos < len(times): + start_minutes = times[pos] + pos += 1 + end_minutes = times[pos] + + print( + line_no, + start_minutes, + end_minutes, + minutes2clock_time(start_minutes), + minutes2clock_time(end_minutes)) + + if last_time is not None and start_minutes <= last_time: + raise ValueError('Times in line %i are not sequential!' % line_no) + + yield (minutes2clock_time(start_minutes), minutes2clock_time(end_minutes)) + + last_time = end_minutes + pos += 1 + line_no += 1 def pformat_timers(times): diff --git a/tests/test_http_set_timer.py b/tests/test_http_set_timer.py index 2fdc963..cdbe341 100644 --- a/tests/test_http_set_timer.py +++ b/tests/test_http_set_timer.py @@ -58,3 +58,23 @@ async def test_set_timer(self): ) ) assert self.web_server.message == 'Timers saved and activated.' + + async def test_sort_timers(self): + + # send not ordered timer list: + # 21:30 - 22:45 + # 07:00 - 08:00 + + response = await self.get_request( + request_line=( + b"GET" + b" /set_timer/submit/" + b"?timers=21%3A30+-+22%3A45%0D%0A07%3A00+-+08%3A00" + b" HTTP/1.1" + ) + ) + assert response == b'HTTP/1.0 303 Moved\r\nLocation: /set_timer/form/\r\n\r\n' + assert self.web_server.message == 'Timers saved and deactivated.' + + timers = pformat_timers(restore_timers()) + assert timers == '07:00 - 08:00\n21:30 - 22:45' diff --git a/tests/test_power_timer.py b/tests/test_power_timer.py index b24378b..2bf16ca 100644 --- a/tests/test_power_timer.py +++ b/tests/test_power_timer.py @@ -62,7 +62,6 @@ def test_relay_switch_timers_and_overwrite(self): assert pformat_timers(timers) == '10:00 - 20:00' assert list(iter_times(timers)) == [(True, (10, 0, 0)), (False, (20, 0, 0))] - print('***********') save_active_days((0, 1, 2, 3, 4, 5, 6)) assert get_active_days() == (0, 1, 2, 3, 4, 5, 6) diff --git a/tests/test_time_utils.py b/tests/test_time_utils.py index b218d5e..349fbf3 100644 --- a/tests/test_time_utils.py +++ b/tests/test_time_utils.py @@ -39,6 +39,15 @@ def test_parse_timers2(self): ((19, 0), (20, 0)) ) + def test_reorder_timers1(self): + assert tuple(parse_timers(''' + 10:00 - 12:00 + 10:30 - 13:00 + ''')) == ( + ((10, 0), (10, 30)), + ((12, 0), (13, 0)) + ) + def test_parse_timers_not_sequential1(self): with self.assertRaises(ValueError) as cm: tuple(parse_timers(''' @@ -46,14 +55,6 @@ def test_parse_timers_not_sequential1(self): ''')) self.assertEqual(cm.exception.args[0], 'Times in line 1 are not sequential!') - def test_parse_timers_not_sequential2(self): - with self.assertRaises(ValueError) as cm: - tuple(parse_timers(''' - 10:00 - 12:00 - 10:30 - 13:00 - ''')) - self.assertEqual(cm.exception.args[0], 'Times in line 2 are not sequential!') - def test_parse_timers_error1(self): with self.assertRaises(ValueError) as cm: tuple(parse_timers(''' From a867246c3c366e0d1ed60e0ab84c14b98e071ce7 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Thu, 9 Jan 2020 06:33:12 +0100 Subject: [PATCH 02/31] update docker-yaota8266 and update github action --- .github/workflows/compile_firmware.yml | 12 +++++++++--- .github/workflows/compile_yaota8266.yml | 2 +- docker-yaota8266 | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/compile_firmware.yml b/.github/workflows/compile_firmware.yml index 340efc6..55ff841 100644 --- a/.github/workflows/compile_firmware.yml +++ b/.github/workflows/compile_firmware.yml @@ -19,10 +19,16 @@ jobs: run: | make update make yaota8266-rsa-keys - cp docker-yaota8266/yaota8266/config.h.example docker-yaota8266/yaota8266/config.h - - name: 'Compile MicroPython Firmware via docker' + - name: 'Compile MicroPython normal Firmware via docker' run: | - make build-firmware + make build-firmware-combined + ls -la build/ + file build/firmware-combined.bin + + - name: 'Compile MicroPython OTA Firmware via docker' + run: | + make build-ota-firmware ls -la build/ file build/firmware-ota.bin + file build/firmware-ota.bin.ota diff --git a/.github/workflows/compile_yaota8266.yml b/.github/workflows/compile_yaota8266.yml index c7113e3..71f1209 100644 --- a/.github/workflows/compile_yaota8266.yml +++ b/.github/workflows/compile_yaota8266.yml @@ -19,7 +19,7 @@ jobs: run: | make update make yaota8266-rsa-keys - cp docker-yaota8266/yaota8266/config.h.example docker-yaota8266/yaota8266/config.h + make assert-yaota8266-setup - name: 'Compile yaota8266.bin via docker' run: | diff --git a/docker-yaota8266 b/docker-yaota8266 index 1eccfdf..6a79743 160000 --- a/docker-yaota8266 +++ b/docker-yaota8266 @@ -1 +1 @@ -Subproject commit 1eccfdf4da996fc68b737929d8422f216002c038 +Subproject commit 6a797438b3b1a7c22adba1c72b2d5923a814c0ff From 813eecd3eb4b20b68d9580dc0201d3a47d5700a5 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Thu, 9 Jan 2020 08:02:12 +0100 Subject: [PATCH 03/31] Change Context to a instance, so it's possible to make it atomic for tests --- helpers/minimal_webserver.py | 2 +- helpers/ntp_sync_test.py | 2 +- helpers/ram_hex_viewer.py | 2 +- mpy_tests/test_times_utils.py | 2 +- src/main.py | 2 +- tests/base.py | 4 ++-- tests/conftest.py | 8 ++++++-- tests/mocks/_patches.py | 26 ++++++++++++++++++++++++++ tests/test_power_timer.py | 2 +- 9 files changed, 40 insertions(+), 10 deletions(-) diff --git a/helpers/minimal_webserver.py b/helpers/minimal_webserver.py index 7749234..ae12d7f 100644 --- a/helpers/minimal_webserver.py +++ b/helpers/minimal_webserver.py @@ -216,7 +216,7 @@ def run(self): def main(): from context import Context - context = Context + context = Context() import wifi wifi.init(context) diff --git a/helpers/ntp_sync_test.py b/helpers/ntp_sync_test.py index 3cc77de..07ef812 100644 --- a/helpers/ntp_sync_test.py +++ b/helpers/ntp_sync_test.py @@ -15,7 +15,7 @@ def print_times(): if __name__ == '__main__': - context = Context + context = Context() ntp_sync(context) print_times() diff --git a/helpers/ram_hex_viewer.py b/helpers/ram_hex_viewer.py index cbd0978..4ce98dc 100644 --- a/helpers/ram_hex_viewer.py +++ b/helpers/ram_hex_viewer.py @@ -244,7 +244,7 @@ def run(self): def main(): from context import Context - context = Context + context = Context() import wifi wifi.init(context) diff --git a/mpy_tests/test_times_utils.py b/mpy_tests/test_times_utils.py index d9db4e5..f3413f8 100644 --- a/mpy_tests/test_times_utils.py +++ b/mpy_tests/test_times_utils.py @@ -94,7 +94,7 @@ def run_all_times_utils_tests(): assert results == (0, 1, 2, 3, 4), results print('OK\n') - context = Context + context = Context() save_timers([]) results = tuple(restore_timers()) diff --git a/src/main.py b/src/main.py index 91e96fe..d8c4b08 100644 --- a/src/main.py +++ b/src/main.py @@ -16,7 +16,7 @@ def main(): from context import Context - context = Context + context = Context() import wifi wifi.init(context) diff --git a/tests/base.py b/tests/base.py index 924e1af..312b623 100644 --- a/tests/base.py +++ b/tests/base.py @@ -24,7 +24,7 @@ def setUp(self): machine.RTC().datetime((2019, 5, 1, 4, 13, 12, 11, 0)) config_files = get_all_config_files() assert not config_files, f'Config files exists before test start: %s' % config_files - self.context = Context + self.context = Context() self.context.power_timer_timers = None print('No config files, ok.') @@ -45,7 +45,7 @@ class WebServerTestCase(MicropythonMixin, asynctest.TestCase): def setUp(self): super().setUp() - context = Context # no instance! + context = Context() context.watchdog = Watchdog(context) diff --git a/tests/conftest.py b/tests/conftest.py index 0dccb1c..1e58c7f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,10 +1,14 @@ import os import sys +import pytest import tests.mocks._patches # noqa - extend existing python modules from utils.constants import BASE_PATH, SRC_PATH sys.path.append(str(BASE_PATH / 'tests' / 'mocks')) - -os.chdir(SRC_PATH) sys.path.insert(0, '.') + + +@pytest.fixture(autouse=True) +def chdir_src(): + os.chdir(SRC_PATH) diff --git a/tests/mocks/_patches.py b/tests/mocks/_patches.py index 1cbb6ad..f88bcc9 100644 --- a/tests/mocks/_patches.py +++ b/tests/mocks/_patches.py @@ -6,6 +6,8 @@ import sys import traceback +import context + def _print_exception(e): traceback.print_exception(None, e, sys.exc_info()[2]) @@ -35,3 +37,27 @@ def no_exit(no=None): sys.exit = no_exit + +# Save context values and defauls for NonSharedContext.__init__() +_default_context = dict( + (attr, getattr(context.Context, attr)) + for attr in dir(context.Context) + if not attr.startswith('_') +) +# print(_default_context) + + +class NonSharedContext: + """ + Same as context.Context, but all attributes are init in __init__() + So they are reset on every test method + """ + + def __init__(self): + # print('Init NonSharedContext') + for attr, value in _default_context.items(): + # print(attr, repr(value)) + setattr(self, attr, value) + + +context.Context = NonSharedContext diff --git a/tests/test_power_timer.py b/tests/test_power_timer.py index 2bf16ca..0c60a32 100644 --- a/tests/test_power_timer.py +++ b/tests/test_power_timer.py @@ -19,7 +19,7 @@ class PowerTimerTestCase(MicropythonBaseTestCase): def setUp(self): super().setUp() machine.RTC().datetime((2019, 5, 1, 4, 13, 12, 11, 0)) - self.context = Context + self.context = Context() self.context.power_timer_timers = None def test_update_relay_switch_without_timers(self): From 580a0985038dbad3c59700d0a647c9216177a9ed Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Thu, 9 Jan 2020 08:17:26 +0100 Subject: [PATCH 04/31] +pytest-randomly --- Pipfile | 1 + Pipfile.lock | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Pipfile b/Pipfile index 99a7c45..4dd7219 100644 --- a/Pipfile +++ b/Pipfile @@ -12,6 +12,7 @@ pttydev = "*" flake8 = "*" pytest = "*" asynctest = "*" +pytest-randomly = "*" pytest-cov = "*" [requires] diff --git a/Pipfile.lock b/Pipfile.lock index 3efcedd..9521179 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "a4c73c862764144aee87a6f4598aab61fdac774f9402b1a71d2b9e97beb9ce34" + "sha256": "0e958d6e6d56c0df00fd1496a1e49e4b85b98f6ef2a593cdff77be5596cc1cb0" }, "pipfile-spec": 6, "requires": { @@ -212,6 +212,14 @@ "index": "pypi", "version": "==2.8.1" }, + "pytest-randomly": { + "hashes": [ + "sha256:1ccaba7214858c681ff73da232f585a6fcccf0fe504e78ac38d077b3cba00b4e", + "sha256:2e89768ea851723b733cc57eb9e33b425490d1ebc858e7a122ade602d9cb58b6" + ], + "index": "pypi", + "version": "==3.2.0" + }, "six": { "hashes": [ "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", From 5643e3d7afbbafaeb85afb3bf7681d3fc8ec153e Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Thu, 9 Jan 2020 08:17:47 +0100 Subject: [PATCH 05/31] WIP: Fix non atomic tests --- .github/workflows/pythonapp.yml | 2 -- tests/base.py | 11 ++++++----- tests/test_power_timer.py | 6 +----- 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/pythonapp.yml index ebd4cd3..e705d16 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/pythonapp.yml @@ -35,7 +35,5 @@ jobs: make docker-build - name: 'Run pytest' - env: - PYTHONPATH: src run: | pipenv run pytest diff --git a/tests/base.py b/tests/base.py index 312b623..30ff595 100644 --- a/tests/base.py +++ b/tests/base.py @@ -5,6 +5,7 @@ import asynctest import machine from context import Context +from power_timer import update_power_timer from uasyncio import StreamReader, StreamWriter from utils.constants import SRC_PATH from watchdog import Watchdog @@ -19,13 +20,15 @@ def get_all_config_files(): class MicropythonMixin: - def setUp(self): + def setUp(self, rtc_time=None): super().setUp() - machine.RTC().datetime((2019, 5, 1, 4, 13, 12, 11, 0)) + if rtc_time is None: + rtc_time = (2019, 5, 1, 4, 13, 12, 11, 0) + machine.RTC().datetime(rtc_time) config_files = get_all_config_files() assert not config_files, f'Config files exists before test start: %s' % config_files self.context = Context() - self.context.power_timer_timers = None + update_power_timer(self.context) print('No config files, ok.') def tearDown(self): @@ -46,9 +49,7 @@ class WebServerTestCase(MicropythonMixin, asynctest.TestCase): def setUp(self): super().setUp() context = Context() - context.watchdog = Watchdog(context) - self.web_server = WebServer(context=context, version='v0.1') async def get_request(self, request_line): diff --git a/tests/test_power_timer.py b/tests/test_power_timer.py index 0c60a32..7530938 100644 --- a/tests/test_power_timer.py +++ b/tests/test_power_timer.py @@ -3,7 +3,6 @@ import constants import machine import utime -from context import Context from mpy_tests.test_times_utils import assert_current_timer, save_active_days, save_timers from pins import Pins from power_timer import active_today, get_info_text, update_power_timer @@ -17,10 +16,7 @@ class PowerTimerTestCase(MicropythonBaseTestCase): def setUp(self): - super().setUp() - machine.RTC().datetime((2019, 5, 1, 4, 13, 12, 11, 0)) - self.context = Context() - self.context.power_timer_timers = None + super().setUp(rtc_time=(2019, 5, 1, 4, 13, 12, 11, 0)) def test_update_relay_switch_without_timers(self): with mock_py_config_context(): From d4a82c682d19756f8d554f2242866503f84e6634 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Thu, 9 Jan 2020 10:22:55 +0100 Subject: [PATCH 06/31] update "inisetup.py": Format only if needed and update boot.py code and add tests --- helpers/convert_filesystem.py | 2 ++ src/inisetup.py | 68 +++++++++++++++++++---------------- tests/test_inisetup.py | 14 ++++++++ 3 files changed, 53 insertions(+), 31 deletions(-) create mode 100644 tests/test_inisetup.py diff --git a/helpers/convert_filesystem.py b/helpers/convert_filesystem.py index a680bb0..8b4c07b 100644 --- a/helpers/convert_filesystem.py +++ b/helpers/convert_filesystem.py @@ -58,6 +58,8 @@ def convert_filesystem2littlefs(force=False): filesystem_hex_dump(line_count=5, chunk_size=16) print('Detected filesystem:', detect_filesystem()) + else: + print('No need to format the flash filesystem, ok.') if __name__ == '__main__': diff --git a/src/inisetup.py b/src/inisetup.py index 0c35780..c04acb9 100644 --- a/src/inisetup.py +++ b/src/inisetup.py @@ -1,47 +1,46 @@ """ based on origin: - micropython/ports/esp8266/modules/inisetup.py + https://github.com/micropython/micropython/blob/master/ports/esp8266/modules/inisetup.py """ +import esp +import flashbdev import uos -from flashbdev import bdev - - -def check_bootsec(): - buf = bytearray(bdev.SEC_SIZE) - bdev.readblocks(0, buf) - empty = True - for b in buf: - if b != 0xff: - empty = False - break - if empty: - return True - fs_corrupted() - - -def fs_corrupted(): - import time - while True: - print("""\ -The FAT filesystem starting at sector %d with size %d sectors appears to -be corrupted. If you had important data there, you may want to make a flash -snapshot to try to recover it. Otherwise, perform factory reprogramming -of MicroPython firmware (completely erase flash, followed by firmware -programming). -""" % (bdev.START_SEC, bdev.blocks)) - time.sleep(3) + +FS_FAT = 'FAT' +FS_LITTLEFS = 'LittleFS' + + +def detect_filesystem(): + buf = bytearray(16) + flashbdev.bdev.readblocks(0, buf) + if buf[3:8] == b'MSDOS': + return FS_FAT + elif buf[8:16] == b'littlefs': + return FS_LITTLEFS + return 'unknown' def setup(): - check_bootsec() print("Performing initial setup") - uos.VfsLfs2.mkfs(bdev) - vfs = uos.VfsLfs2(bdev) + + filesystem = detect_filesystem() + print('Detected filesystem: %r' % filesystem) + if filesystem != FS_LITTLEFS: + print('Erase flash start sector 0x%x' % flashbdev.bdev.START_SEC) + esp.flash_erase(flashbdev.bdev.START_SEC) + + print('convert to littlefs2') + uos.VfsLfs2.mkfs(flashbdev.bdev) + + print('mount filesystem') + vfs = uos.VfsLfs2(flashbdev.bdev) uos.mount(vfs, '/') + with open("boot.py", "w") as f: f.write("""\ print('boot.py') # noqa isort:skip import gc +import sys import esp import micropython @@ -61,6 +60,13 @@ def setup(): # https://forum.micropython.org/viewtopic.php?f=2&t=7345&p=42390#p42390 gc.threshold(8192) +# default is: +# sys.path=['', '/lib', '/'] +# But we would like to be possible to overwrite frozen modues with .mpy on flash drive ;) +sys.path.insert(0, '.') +print('sys.path=%r' % sys.path) + + print('boot.py END') """) return vfs diff --git a/tests/test_inisetup.py b/tests/test_inisetup.py new file mode 100644 index 0000000..99979c8 --- /dev/null +++ b/tests/test_inisetup.py @@ -0,0 +1,14 @@ +from unittest import TestCase + + +class TestIniSetupTestCase(TestCase): + def test_boot_py_content(self): + with open('boot.py', 'r') as f: + boot_py_content = f.read() + + with open('inisetup.py', 'r') as f: + inisetup_py_content = f.read() + + assert boot_py_content in inisetup_py_content, ( + 'boot.py code in inisetup.py is not up-to-date' + ) From 713cf807fd111d4458f2747bc1af242de50eeff5 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Thu, 9 Jan 2020 11:42:39 +0100 Subject: [PATCH 07/31] Bugfix tests and refactor NonCachesImporter --- src/power_timer.py | 33 +++++++++++++++----------- src/times_utils.py | 4 ++++ tests/base.py | 31 +++++++++++++++++------- tests/mocks/_patches.py | 7 ++++-- tests/test_config_files.py | 14 +++++++++++ tests/test_power_timer.py | 11 ++++----- tests/utils/mock_py_config.py | 44 ++++++++++++++++++++++++++++------- 7 files changed, 104 insertions(+), 40 deletions(-) diff --git a/src/power_timer.py b/src/power_timer.py index 0f4a61d..620c969 100644 --- a/src/power_timer.py +++ b/src/power_timer.py @@ -51,14 +51,11 @@ def update_power_timer(context): if __debug__: print('Update power timer:', utime.time()) - if context.power_timer_timers is None: - if __debug__: - print('restore timers') - from times_utils import restore_timers - context.power_timer_timers = restore_timers() - del restore_timers - del sys.modules['times_utils'] - gc.collect() + from times_utils import restore_timers + context.power_timer_timers = restore_timers() + del restore_timers + del sys.modules['times_utils'] + gc.collect() from rtc import get_dict_from_rtc rtc_memory_dict = get_dict_from_rtc() @@ -67,6 +64,8 @@ def update_power_timer(context): manual_overwrite = rtc_memory_dict.get(constants.RTC_KEY_MANUAL_OVERWRITE, 0) current_state = rtc_memory_dict.get(constants.RTC_KEY_MANUAL_OVERWRITE_TYPE) + if __debug__ and current_state: + print('manual overwrite:', repr(current_state)) del rtc_memory_dict del sys.modules['rtc'] @@ -75,7 +74,8 @@ def update_power_timer(context): context.power_timer_today_active = active_today() if context.power_timer_active and context.power_timer_today_active: - # Update power timer state + if __debug__: + print('Update power timer state') from times_utils import get_current_timer last_timer_epoch, turn_on, context.power_timer_next_timer_epoch = get_current_timer(context) @@ -86,10 +86,12 @@ def update_power_timer(context): context.power_timer_turn_on = turn_on if last_timer_epoch is None: - print('No timer scheduled') + if __debug__: + print('No timer scheduled') else: if current_state is None or manual_overwrite < last_timer_epoch: - print('Set state from timer') + if __debug__: + print('Set state from timer') # Note: # The current power state is the **opposite** of the next one. # In other words: If the **next** timer will "turn on" (==True) then we are @@ -97,14 +99,17 @@ def update_power_timer(context): current_state = not turn_on if current_state is None: - # No timer to scheduled and no manual overwrite + if __debug__: + print('No timer to scheduled and no manual overwrite') return True if current_state: - print('Switch on') + if __debug__: + print('Switch on') Pins.relay.on() else: - print('Switch off') + if __debug__: + print('Switch off') Pins.relay.off() gc.collect() diff --git a/src/times_utils.py b/src/times_utils.py index a11023f..16dad50 100644 --- a/src/times_utils.py +++ b/src/times_utils.py @@ -179,8 +179,12 @@ def get_current_timer(self, context): return last and next switching point in "(hours, minutes)" and if the next point turns ON or OFF """ + if __DEBUG__: + print('context.power_timer_timers:', context.power_timer_timers) assert context.power_timer_timers is not None, 'Timers not loaded, yet?!?' turn_on_times = tuple(iter_times(context.power_timer_timers)) + if __DEBUG__: + print('turn on times:', turn_on_times) now_hour_minute_sec = (self.hour, self.minute, self.second) # print('%02i:%02i:%02i' % now_hour_minute_sec) diff --git a/tests/base.py b/tests/base.py index 30ff595..13ea317 100644 --- a/tests/base.py +++ b/tests/base.py @@ -5,7 +5,9 @@ import asynctest import machine from context import Context +from pins import Pins from power_timer import update_power_timer +from rtc import clear_rtc_dict from uasyncio import StreamReader, StreamWriter from utils.constants import SRC_PATH from watchdog import Watchdog @@ -22,14 +24,25 @@ def get_all_config_files(): class MicropythonMixin: def setUp(self, rtc_time=None): super().setUp() + + # Always start with OFF state: + Pins.relay.off() + Pins.power_led.off() + + # Start with empty RTC memory: + clear_rtc_dict() + if rtc_time is None: rtc_time = (2019, 5, 1, 4, 13, 12, 11, 0) machine.RTC().datetime(rtc_time) + config_files = get_all_config_files() - assert not config_files, f'Config files exists before test start: %s' % config_files - self.context = Context() - update_power_timer(self.context) - print('No config files, ok.') + if config_files: + raise AssertionError(f'Config files exists before test start: %s' % config_files) + else: + print('No config files, ok.') + + self.context = Context() # mocks._patches.NonSharedContext def tearDown(self): config_files = get_all_config_files() @@ -46,13 +59,13 @@ class MicropythonBaseTestCase(MicropythonMixin, TestCase): class WebServerTestCase(MicropythonMixin, asynctest.TestCase): - def setUp(self): - super().setUp() - context = Context() - context.watchdog = Watchdog(context) - self.web_server = WebServer(context=context, version='v0.1') + def setUp(self, rtc_time=None): + super().setUp(rtc_time=rtc_time) + self.context.watchdog = Watchdog(self.context) + self.web_server = WebServer(context=self.context, version='v0.1') async def get_request(self, request_line): + update_power_timer(self.context) reader = asynctest.mock.Mock(StreamReader) writer = StreamWriter() diff --git a/tests/mocks/_patches.py b/tests/mocks/_patches.py index f88bcc9..013d7cc 100644 --- a/tests/mocks/_patches.py +++ b/tests/mocks/_patches.py @@ -54,9 +54,12 @@ class NonSharedContext: """ def __init__(self): - # print('Init NonSharedContext') + self.reset() + + def reset(self): + print('NonSharedContext.reset():') for attr, value in _default_context.items(): - # print(attr, repr(value)) + print(f'\t{attr} = {value!r}') setattr(self, attr, value) diff --git a/tests/test_config_files.py b/tests/test_config_files.py index f8a7c88..c86566d 100644 --- a/tests/test_config_files.py +++ b/tests/test_config_files.py @@ -2,8 +2,10 @@ from config_files import restore_py_config, save_py_config from mpy_tests.test_config_files import test_json_config +from mpy_tests.test_times_utils import save_timers from tests.base import MicropythonBaseTestCase from tests.utils.mock_py_config import mock_py_config_context +from times_utils import restore_timers class JsonConfigTestCase(MicropythonBaseTestCase): @@ -42,3 +44,15 @@ def test_py_config(self): save_py_config(module_name='test_dict', value={'foo': 1.2}) assert restore_py_config(module_name='test_dict') == {'foo': 1.2} + + def test_mock_py_config_context_with_timers(self): + with mock_py_config_context(): + save_timers(( + ((1, 2), (3, 4)), + )) + assert restore_timers() == (((1, 2), (3, 4)),) + + save_timers(( + ((5, 6), (7, 8)), + )) + assert restore_timers() == (((5, 6), (7, 8)),) diff --git a/tests/test_power_timer.py b/tests/test_power_timer.py index 7530938..4500a99 100644 --- a/tests/test_power_timer.py +++ b/tests/test_power_timer.py @@ -38,14 +38,11 @@ def test_update_relay_switch_in_1min(self): assert machine.RTC().datetime((2000, 1, 1, 6, 0, 0, 0, 0)) assert localtime_isoformat(sep=' ') == '2000-01-01 00:00:00' - update_power_timer(self.context) - print(get_info_text(self.context)) - - self.assertEqual( - get_info_text(self.context), - 'Switch on in 1 min at 00:01 h.' - ) + timers = restore_timers() + assert pformat_timers(timers) == '00:01 - 01:00' + assert list(iter_times(timers)) == [(True, (0, 1, 0)), (False, (1, 0, 0))] + update_power_timer(self.context) self.assertEqual(get_info_text(self.context), 'Switch on in 1 min at 00:01 h.') def test_relay_switch_timers_and_overwrite(self): diff --git a/tests/utils/mock_py_config.py b/tests/utils/mock_py_config.py index 487cfa1..84191a4 100644 --- a/tests/utils/mock_py_config.py +++ b/tests/utils/mock_py_config.py @@ -1,4 +1,4 @@ -import importlib +import ast import sys import traceback from contextlib import contextmanager @@ -6,6 +6,10 @@ from unittest import mock +class MockedModule: + pass + + class NonCachesImporter: """ replace __import__() with a non-cached import @@ -33,17 +37,41 @@ def mocked_import(self, module_name, *args, **kwargs): else: print('Existing file:', file_path) + # It's very difficult to disable the CPython import caching. + # What doesn't work is, e.g.: + # * `del sys.modules[module_name]` before import + # * module = importlib.reload(module) + # * Using low-level stuff from `importlib.util` + # + # So we parse the content via ast.literal_eval() and create + # a mocked sys.modules entry + with file_path.open('r') as f: - print('Content:', repr(f.read())) + content = f.read() + print('Content:', repr(content)) - spec = importlib.util.spec_from_file_location(module_name, file_path) - module = importlib.util.module_from_spec(spec) - sys.modules[module_name] = module - spec.loader.exec_module(module) + module = MockedModule() + for line in content.splitlines(): + if line == 'from micropython import const': + continue + + var_name, data = line.split('=') - # module = importlib.reload(module) # will fail! + var_name = var_name.strip() + data = data.strip() + + if 'const(' in data: + # 'const(123)' -> '123' + data = data.split('(', 1)[1].rstrip(')') + + data = ast.literal_eval(data) + setattr(module, var_name, data) + break # there is currently one one value + + # Add to sys.modules, so that "del sys.modules[module_name]" will work + # used in config_files.restore_py_config() + sys.modules[module_name] = module - print('imported:', module, id(module)) print('%s.value: %r' % (module_name, getattr(module, 'value', '-'))) return module From e018ec20091b8ac9f5d8158db67f2d28fd356f63 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Thu, 9 Jan 2020 15:58:15 +0100 Subject: [PATCH 08/31] clarify difference between `firmware-ota.bin` and `firmware-ota.bin.ota` --- README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3bb55da..c808fe4 100644 --- a/README.md +++ b/README.md @@ -166,7 +166,9 @@ After you have created your own RSA keys and `config.h`, you can compile `yaota8 The compiled files are stored here: * `~/micropython-sonoff-webswitch/build/yaota8266.bin` -* `~/micropython-sonoff-webswitch/build/firmware-ota.bin` +* `~/micropython-sonoff-webswitch/build/firmware-ota.bin` <- for flashing +* `~/micropython-sonoff-webswitch/build/firmware-ota.bin.ota` <- used in hard-OTA process + ### flash yaota8266 and firmware @@ -185,6 +187,9 @@ For other devices just use `esptool` directly, e.g.: ~/micropython-sonoff-webswitch$ pipenv run esptool.py --port /dev/ttyUSB0 --baud 460800 write_flash 0x3c000 build/firmware-ota.bin ``` +**Note:** + +The file `firmware-ota.bin` must be flash with `esptool.py` **not** the `firmware-ota.bin.ota` ! This file is ues in hard-OTA update process. ## flash micropython From 3526d481cde4ac2efcfa7ddaa71fed465f3949e4 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Thu, 9 Jan 2020 16:45:58 +0100 Subject: [PATCH 09/31] cleanup inisetup.py --- src/inisetup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/inisetup.py b/src/inisetup.py index c04acb9..8c81b4c 100644 --- a/src/inisetup.py +++ b/src/inisetup.py @@ -1,6 +1,8 @@ """ based on origin: https://github.com/micropython/micropython/blob/master/ports/esp8266/modules/inisetup.py + Will be only called from "_boot.py" if uos.mount(flashbdev.bdev, '/') failed, see: + https://github.com/micropython/micropython/blob/master/ports/esp8266/modules/_boot.py """ import esp import flashbdev @@ -33,8 +35,7 @@ def setup(): uos.VfsLfs2.mkfs(flashbdev.bdev) print('mount filesystem') - vfs = uos.VfsLfs2(flashbdev.bdev) - uos.mount(vfs, '/') + uos.mount(flashbdev.bdev, '/') with open("boot.py", "w") as f: f.write("""\ @@ -69,4 +70,3 @@ def setup(): print('boot.py END') """) - return vfs From 39c5187655200b2624e0448c7eebc30b30ad4bb1 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Thu, 9 Jan 2020 16:46:38 +0100 Subject: [PATCH 10/31] Use own "_boot.py" case of: https://github.com/micropython/micropython/pull/5509 --- micropython_config/manifest.py | 7 ++++++- src/_boot.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 src/_boot.py diff --git a/micropython_config/manifest.py b/micropython_config/manifest.py index 05b8b4a..c05eac7 100644 --- a/micropython_config/manifest.py +++ b/micropython_config/manifest.py @@ -3,10 +3,15 @@ micropython/ports/esp8266/boards/manifest_release.py """ -freeze('$(PORT_DIR)/modules') +# Include only needed files from ".../micropython/ports/esp8266/modules/": +#freeze('$(PORT_DIR)/modules', '_boot.py') # use own case of: https://github.com/micropython/micropython/pull/5509 +freeze('$(PORT_DIR)/modules', 'flashbdev.py') +freeze('$(PORT_DIR)/modules', 'ntptime.py') + freeze('$(PORT_DIR)/sdist') # project sources, mounted via docker + #freeze('$(MPY_DIR)/tools', ('upip.py', 'upip_utarfile.py')) #freeze('$(MPY_DIR)/drivers/dht', 'dht.py') #freeze('$(MPY_DIR)/drivers/onewire') diff --git a/src/_boot.py b/src/_boot.py new file mode 100644 index 0000000..f1c355b --- /dev/null +++ b/src/_boot.py @@ -0,0 +1,13 @@ +import gc +gc.threshold((gc.mem_free() + gc.mem_alloc()) // 4) +import uos +from flashbdev import bdev + +if bdev: + try: + uos.mount(bdev, '/') + except AttributeError: + import inisetup + inisetup.setup() + +gc.collect() From 1953b3dc278147e168c0cec066043cc27a46bdb2 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Thu, 9 Jan 2020 17:01:58 +0100 Subject: [PATCH 11/31] Add/rename OTA helper scripts --- helpers/do_hard_ota_updates.py | 10 ++++++++++ helpers/{do_ota_update.py => do_soft_ota_updates.py} | 0 2 files changed, 10 insertions(+) create mode 100644 helpers/do_hard_ota_updates.py rename helpers/{do_ota_update.py => do_soft_ota_updates.py} (100%) diff --git a/helpers/do_hard_ota_updates.py b/helpers/do_hard_ota_updates.py new file mode 100644 index 0000000..f5eb56f --- /dev/null +++ b/helpers/do_hard_ota_updates.py @@ -0,0 +1,10 @@ +""" + Start "hard" OTA updates +""" + +import machine + +if __name__ == '__main__': + print('Force hard-OTA via yaota8266') + machine.RTC().memory('yaotaota') + machine.reset() diff --git a/helpers/do_ota_update.py b/helpers/do_soft_ota_updates.py similarity index 100% rename from helpers/do_ota_update.py rename to helpers/do_soft_ota_updates.py From 66210e6cb1a38d47a2eea60249d3f140674b3849 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Thu, 9 Jan 2020 17:02:16 +0100 Subject: [PATCH 12/31] Update "make hard-ota" to starte the server --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c3d718d..e52d87b 100644 --- a/Makefile +++ b/Makefile @@ -172,7 +172,7 @@ flash-ota-firmware: verify ## Flash build/firmware-ota.bin to location 0x3c000 v pipenv run esptool.py --port /dev/ttyUSB0 --baud 460800 write_flash -fs 1MB -fm dout 0x3c000 build/firmware-ota.bin hard-ota: ## Start yaota8266 live-ota to hard-OTA Update the firmware file build/firmware-ota.bin.ota - python3 docker-yaota8266/yaota8266/ota-client/ota_client.py live-ota build/firmware-ota.bin.ota + python3 docker-yaota8266/yaota8266/cli.py ota build/firmware-ota.bin.ota soft-ota: ## Start soft-OTA updates: Compile .py to .mpy and push missing/updated files (*.mpy, *.css, *.html etc.) to the device python3 start_soft_ota_server.py From b9160233ad76464c6140b6358e128f6dea87318f Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Thu, 9 Jan 2020 17:03:34 +0100 Subject: [PATCH 13/31] Bugfix: "__DEBUG__" -> "__debug__" --- src/times_utils.py | 4 ++-- tests/mocks/_patches.py | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/times_utils.py b/src/times_utils.py index 16dad50..16f0c48 100644 --- a/src/times_utils.py +++ b/src/times_utils.py @@ -179,11 +179,11 @@ def get_current_timer(self, context): return last and next switching point in "(hours, minutes)" and if the next point turns ON or OFF """ - if __DEBUG__: + if __debug__: print('context.power_timer_timers:', context.power_timer_timers) assert context.power_timer_timers is not None, 'Timers not loaded, yet?!?' turn_on_times = tuple(iter_times(context.power_timer_timers)) - if __DEBUG__: + if __debug__: print('turn on times:', turn_on_times) now_hour_minute_sec = (self.hour, self.minute, self.second) diff --git a/tests/mocks/_patches.py b/tests/mocks/_patches.py index 013d7cc..dcb4315 100644 --- a/tests/mocks/_patches.py +++ b/tests/mocks/_patches.py @@ -1,7 +1,7 @@ """ Expand existing python modules """ -import builtins + import gc import sys import traceback @@ -29,8 +29,6 @@ def _mem_alloc(): gc.mem_alloc = _mem_alloc -builtins.__DEBUG__ = True - def no_exit(no=None): print(f'sys.exit({no!r}) called!') From 38e887794d3e3cc888489c40de8147792ff96230 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Thu, 9 Jan 2020 17:11:00 +0100 Subject: [PATCH 14/31] move tests/conftest.py into tests/__init__.py --- tests/__init__.py | 11 +++++++++++ tests/conftest.py | 14 -------------- 2 files changed, 11 insertions(+), 14 deletions(-) delete mode 100644 tests/conftest.py diff --git a/tests/__init__.py b/tests/__init__.py index e69de29..4c56f0b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,11 @@ +import os +import sys + +from utils.constants import BASE_PATH, SRC_PATH + +os.chdir(SRC_PATH) + +sys.path.append(str(BASE_PATH / 'tests' / 'mocks')) +sys.path.insert(0, '.') + +import tests.mocks._patches # noqa - extend existing python modules diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index 1e58c7f..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,14 +0,0 @@ -import os -import sys - -import pytest -import tests.mocks._patches # noqa - extend existing python modules -from utils.constants import BASE_PATH, SRC_PATH - -sys.path.append(str(BASE_PATH / 'tests' / 'mocks')) -sys.path.insert(0, '.') - - -@pytest.fixture(autouse=True) -def chdir_src(): - os.chdir(SRC_PATH) From 53dacc17f27f1857373cbeb79c62b751363a94b5 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Thu, 16 Jan 2020 08:48:37 +0100 Subject: [PATCH 15/31] update docker-yaota8266 submodule --- docker-yaota8266 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-yaota8266 b/docker-yaota8266 index 6a79743..d37b355 160000 --- a/docker-yaota8266 +++ b/docker-yaota8266 @@ -1 +1 @@ -Subproject commit 6a797438b3b1a7c22adba1c72b2d5923a814c0ff +Subproject commit d37b355b5fc6f49414559e4fd24523405ccdb9d5 From 1d0216748cda89f0203354c26d379b26c0590a70 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Thu, 16 Jan 2020 08:48:46 +0100 Subject: [PATCH 16/31] update requirements --- Pipfile.lock | 96 ++++++++++++++++++++++++++-------------------------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 9521179..f4e794c 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -33,39 +33,39 @@ }, "coverage": { "hashes": [ - "sha256:189aac76d6e0d7af15572c51892e7326ee451c076c5a50a9d266406cd6c49708", - "sha256:1bf7ba2af1d373a1750888724f84cffdfc697738f29a353c98195f98fc011509", - "sha256:1f4ee8e2e4243971618bc16fcc4478317405205f135e95226c2496e2a3b8dbbf", - "sha256:225e79a5d485bc1642cb7ba02281419c633c216cdc6b26c26494ba959f09e69f", - "sha256:23688ff75adfa8bfa2a67254d889f9bdf9302c27241d746e17547c42c732d3f4", - "sha256:28f7f73b34a05e23758e860a89a7f649b85c6749e252eff60ebb05532d180e86", - "sha256:2d0cb9b1fe6ad0d915d45ad3d87f03a38e979093a98597e755930db1f897afae", - "sha256:47874b4711c5aeb295c31b228a758ce3d096be83dc37bd56da48ed99efb8813b", - "sha256:511ec0c00840e12fb4e852e4db58fa6a01ca4da72f36a9766fae344c3d502033", - "sha256:53e7438fef0c97bc248f88ba1edd10268cd94d5609970aaf87abbe493691af87", - "sha256:569f9ee3025682afda6e9b0f5bb14897c0db03f1a1dc088b083dd36e743f92bb", - "sha256:593853aa1ac6dcc6405324d877544c596c9d948ef20d2e9512a0f5d2d3202356", - "sha256:5b0a07158360d22492f9abd02a0f2ee7981b33f0646bf796598b7673f6bbab14", - "sha256:7ca3db38a61f3655a2613ee2c190d63639215a7a736d3c64cc7bbdb002ce6310", - "sha256:7d1cc7acc9ce55179616cf72154f9e648136ea55987edf84addbcd9886ffeba2", - "sha256:88b51153657612aea68fa684a5b88037597925260392b7bb4509d4f9b0bdd889", - "sha256:955ec084f549128fa2702f0b2dc696392001d986b71acd8fd47424f28289a9c3", - "sha256:b251c7092cbb6d789d62dc9c9e7c4fb448c9138b51285c36aeb72462cad3600e", - "sha256:bd82b684bb498c60ef47bb1541a50e6d006dde8579934dcbdbc61d67d1ea70d9", - "sha256:bfe102659e2ec13b86c7f3b1db6c9a4e7beea4255058d006351339e6b342d5d2", - "sha256:c1e4e39e43057396a5e9d069bfbb6ffeee892e40c5d2effbd8cd71f34ee66c4d", - "sha256:cb2b74c123f65e8166f7e1265829a6c8ed755c3cd16d7f50e75a83456a5f3fd7", - "sha256:cca38ded59105f7705ef6ffe1e960b8db6c7d8279c1e71654a4775ab4454ca15", - "sha256:cf908840896f7aa62d0ec693beb53264b154f972eb8226fb864ac38975590c4f", - "sha256:d095a7b473f8a95f7efe821f92058c8a2ecfb18f8db6677ae3819e15dc11aaae", - "sha256:d22b4297e7e4225ccf01f1aa55e7a96412ea0796b532dd614c3fcbafa341128e", - "sha256:d4a2b578a7a70e0c71f662705262f87a456f1e6c1e40ada7ea699abaf070a76d", - "sha256:ddeb42a3d5419434742bf4cc71c9eaa22df3b76808e23a82bd0b0bd360f1a9f1", - "sha256:e65a5aa1670db6263f19fdc03daee1d7dbbadb5cb67fd0a1f16033659db13c1d", - "sha256:eaad65bd20955131bcdb3967a4dea66b4e4d4ca488efed7c00d91ee0173387e8", - "sha256:f45fba420b94165c17896861bb0e8b27fb7abdcedfeb154895d8553df90b7b00" - ], - "version": "==5.0.2" + "sha256:15cf13a6896048d6d947bf7d222f36e4809ab926894beb748fc9caa14605d9c3", + "sha256:1daa3eceed220f9fdb80d5ff950dd95112cd27f70d004c7918ca6dfc6c47054c", + "sha256:1e44a022500d944d42f94df76727ba3fc0a5c0b672c358b61067abb88caee7a0", + "sha256:25dbf1110d70bab68a74b4b9d74f30e99b177cde3388e07cc7272f2168bd1477", + "sha256:3230d1003eec018ad4a472d254991e34241e0bbd513e97a29727c7c2f637bd2a", + "sha256:3dbb72eaeea5763676a1a1efd9b427a048c97c39ed92e13336e726117d0b72bf", + "sha256:5012d3b8d5a500834783689a5d2292fe06ec75dc86ee1ccdad04b6f5bf231691", + "sha256:51bc7710b13a2ae0c726f69756cf7ffd4362f4ac36546e243136187cfcc8aa73", + "sha256:527b4f316e6bf7755082a783726da20671a0cc388b786a64417780b90565b987", + "sha256:722e4557c8039aad9592c6a4213db75da08c2cd9945320220634f637251c3894", + "sha256:76e2057e8ffba5472fd28a3a010431fd9e928885ff480cb278877c6e9943cc2e", + "sha256:77afca04240c40450c331fa796b3eab6f1e15c5ecf8bf2b8bee9706cd5452fef", + "sha256:7afad9835e7a651d3551eab18cbc0fdb888f0a6136169fbef0662d9cdc9987cf", + "sha256:9bea19ac2f08672636350f203db89382121c9c2ade85d945953ef3c8cf9d2a68", + "sha256:a8b8ac7876bc3598e43e2603f772d2353d9931709345ad6c1149009fd1bc81b8", + "sha256:b0840b45187699affd4c6588286d429cd79a99d509fe3de0f209594669bb0954", + "sha256:b26aaf69713e5674efbde4d728fb7124e429c9466aeaf5f4a7e9e699b12c9fe2", + "sha256:b63dd43f455ba878e5e9f80ba4f748c0a2156dde6e0e6e690310e24d6e8caf40", + "sha256:be18f4ae5a9e46edae3f329de2191747966a34a3d93046dbdf897319923923bc", + "sha256:c312e57847db2526bc92b9bfa78266bfbaabac3fdcd751df4d062cd4c23e46dc", + "sha256:c60097190fe9dc2b329a0eb03393e2e0829156a589bd732e70794c0dd804258e", + "sha256:c62a2143e1313944bf4a5ab34fd3b4be15367a02e9478b0ce800cb510e3bbb9d", + "sha256:cc1109f54a14d940b8512ee9f1c3975c181bbb200306c6d8b87d93376538782f", + "sha256:cd60f507c125ac0ad83f05803063bed27e50fa903b9c2cfee3f8a6867ca600fc", + "sha256:d513cc3db248e566e07a0da99c230aca3556d9b09ed02f420664e2da97eac301", + "sha256:d649dc0bcace6fcdb446ae02b98798a856593b19b637c1b9af8edadf2b150bea", + "sha256:d7008a6796095a79544f4da1ee49418901961c97ca9e9d44904205ff7d6aa8cb", + "sha256:da93027835164b8223e8e5af2cf902a4c80ed93cb0909417234f4a9df3bcd9af", + "sha256:e69215621707119c6baf99bda014a45b999d37602cb7043d943c76a59b05bf52", + "sha256:ea9525e0fef2de9208250d6c5aeeee0138921057cd67fcef90fbed49c4d62d37", + "sha256:fca1669d464f0c9831fd10be2eef6b86f5ebd76c724d1e0706ebdff86bb4adf0" + ], + "version": "==5.0.3" }, "ecdsa": { "hashes": [ @@ -98,11 +98,11 @@ }, "importlib-metadata": { "hashes": [ - "sha256:073a852570f92da5f744a3472af1b61e28e9f78ccf0c9117658dc32b15de7b45", - "sha256:d95141fbfa7ef2ec65cfd945e2af7e5a6ddbd7c8d9a25e66ff3be8e3daf9f60f" + "sha256:bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359", + "sha256:f17c015735e1a88296994c0697ecea7e11db24290941983b08c9feb30921e6d8" ], "markers": "python_version < '3.8'", - "version": "==1.3.0" + "version": "==1.4.0" }, "mccabe": { "hashes": [ @@ -113,10 +113,10 @@ }, "more-itertools": { "hashes": [ - "sha256:b84b238cce0d9adad5ed87e745778d20a3f8487d0f0cb8b8a586816c7496458d", - "sha256:c833ef592a0324bcc6a60e48440da07645063c453880c9477ceb22490aec1564" + "sha256:1a2a32c72400d365000412fe08eb4a24ebee89997c18d3d147544f70f5403b39", + "sha256:c468adec578380b6281a114cb8a5db34eb1116277da92d7c46f904f0b52d3288" ], - "version": "==8.0.2" + "version": "==8.1.0" }, "mpycntrl": { "hashes": [ @@ -214,18 +214,18 @@ }, "pytest-randomly": { "hashes": [ - "sha256:1ccaba7214858c681ff73da232f585a6fcccf0fe504e78ac38d077b3cba00b4e", - "sha256:2e89768ea851723b733cc57eb9e33b425490d1ebc858e7a122ade602d9cb58b6" + "sha256:8282a5a2066ed93d8d3eca6290538f34d09be05b7256ba94dba6fe3d2b1c514f", + "sha256:d9273db715bc7f8945cdb3efb994a9d15596a6087fb71a2e87030c5f2677317e" ], "index": "pypi", - "version": "==3.2.0" + "version": "==3.2.1" }, "six": { "hashes": [ - "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", - "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" + "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", + "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" ], - "version": "==1.13.0" + "version": "==1.14.0" }, "wcwidth": { "hashes": [ @@ -243,10 +243,10 @@ }, "zipp": { "hashes": [ - "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e", - "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335" + "sha256:8dda78f06bd1674bd8720df8a50bb47b6e1233c503a4eed8e7810686bde37656", + "sha256:d38fbe01bbf7a3593a32bc35a9c4453c32bc42b98c377f9bff7e9f8da157786c" ], - "version": "==0.6.0" + "version": "==1.0.0" } }, "develop": {} From 68dbe03fa6bd96c84a450c9d5d07bde049c23291 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Wed, 22 Jan 2020 10:28:29 +0100 Subject: [PATCH 17/31] Add `make thonny` --- Makefile | 4 +- Pipfile | 1 + Pipfile.lock | 171 ++++++++++++++++++++++++++++++++++++++++++++++-- README.md | 1 + upload_files.py | 23 ------- 5 files changed, 169 insertions(+), 31 deletions(-) delete mode 100644 upload_files.py diff --git a/Makefile b/Makefile index e52d87b..effa562 100644 --- a/Makefile +++ b/Makefile @@ -43,8 +43,10 @@ update: docker-build ## update git repositories/submodules, virtualenv, docker git submodule update --init --recursive ; \ fi python3 -m pip install --upgrade pipenv - pipenv sync + pipenv update +thonny: ## run Thonny IDE to access the Micropython REPL (Python prompt) + pipenv run thonny & test: update ## Run pytest PYTHONPATH=src pipenv run pytest diff --git a/Pipfile b/Pipfile index 4dd7219..5b23c52 100644 --- a/Pipfile +++ b/Pipfile @@ -14,6 +14,7 @@ pytest = "*" asynctest = "*" pytest-randomly = "*" pytest-cov = "*" +thonny = "*" [requires] python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock index f4e794c..2f6e635 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "0e958d6e6d56c0df00fd1496a1e49e4b85b98f6ef2a593cdff77be5596cc1cb0" + "sha256": "8f738b64e637a415d77dacb249d982d1ee9bec6198208f569ee1b5b70a188399" }, "pipfile-spec": 6, "requires": { @@ -16,6 +16,20 @@ ] }, "default": { + "astroid": { + "hashes": [ + "sha256:71ea07f44df9568a75d0f354c49143a4575d90645e9fead6dfb52c26a85ed13a", + "sha256:840947ebfa8b58f318d42301cf8c0a20fd794a33b61cc4638e28e9e61ba32f42" + ], + "version": "==2.3.3" + }, + "asttokens": { + "hashes": [ + "sha256:284831ac3e33be743ca6ac018316b66abfd8b1a49d6366ce6c7b1fd07504a21b", + "sha256:f58af645756597143629a4ac1fe78bc670b4429018ad741ac1f4cfd4504fc436" + ], + "version": "==2.0.3" + }, "asynctest": { "hashes": [ "sha256:5da6118a7e6d6b54d83a8f7197769d046922a44d2a99c21382f0a6e4fadae676", @@ -67,6 +81,13 @@ ], "version": "==5.0.3" }, + "docutils": { + "hashes": [ + "sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af", + "sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc" + ], + "version": "==0.16" + }, "ecdsa": { "hashes": [ "sha256:867ec9cf6df0b03addc8ef66b56359643cb5d0c1dc329df76ba7ecfe256c8061", @@ -104,6 +125,46 @@ "markers": "python_version < '3.8'", "version": "==1.4.0" }, + "isort": { + "hashes": [ + "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1", + "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd" + ], + "version": "==4.3.21" + }, + "jedi": { + "hashes": [ + "sha256:1349c1e8c107095a55386628bb3b2a79422f3a2cab8381e34ce19909e0cf5064", + "sha256:e909527104a903606dd63bea6e8e888833f0ef087057829b89a18364a856f807" + ], + "version": "==0.15.2" + }, + "lazy-object-proxy": { + "hashes": [ + "sha256:0c4b206227a8097f05c4dbdd323c50edf81f15db3b8dc064d08c62d37e1a504d", + "sha256:194d092e6f246b906e8f70884e620e459fc54db3259e60cf69a4d66c3fda3449", + "sha256:1be7e4c9f96948003609aa6c974ae59830a6baecc5376c25c92d7d697e684c08", + "sha256:4677f594e474c91da97f489fea5b7daa17b5517190899cf213697e48d3902f5a", + "sha256:48dab84ebd4831077b150572aec802f303117c8cc5c871e182447281ebf3ac50", + "sha256:5541cada25cd173702dbd99f8e22434105456314462326f06dba3e180f203dfd", + "sha256:59f79fef100b09564bc2df42ea2d8d21a64fdcda64979c0fa3db7bdaabaf6239", + "sha256:8d859b89baf8ef7f8bc6b00aa20316483d67f0b1cbf422f5b4dc56701c8f2ffb", + "sha256:9254f4358b9b541e3441b007a0ea0764b9d056afdeafc1a5569eee1cc6c1b9ea", + "sha256:9651375199045a358eb6741df3e02a651e0330be090b3bc79f6d0de31a80ec3e", + "sha256:97bb5884f6f1cdce0099f86b907aa41c970c3c672ac8b9c8352789e103cf3156", + "sha256:9b15f3f4c0f35727d3a0fba4b770b3c4ebbb1fa907dbcc046a1d2799f3edd142", + "sha256:a2238e9d1bb71a56cd710611a1614d1194dc10a175c1e08d75e1a7bcc250d442", + "sha256:a6ae12d08c0bf9909ce12385803a543bfe99b95fe01e752536a60af2b7797c62", + "sha256:ca0a928a3ddbc5725be2dd1cf895ec0a254798915fb3a36af0964a0a4149e3db", + "sha256:cb2c7c57005a6804ab66f106ceb8482da55f5314b7fcb06551db1edae4ad1531", + "sha256:d74bb8693bf9cf75ac3b47a54d716bbb1a92648d5f781fc799347cfc95952383", + "sha256:d945239a5639b3ff35b70a88c5f2f491913eb94871780ebfabb2568bd58afc5a", + "sha256:eba7011090323c1dadf18b3b689845fd96a61ba0a1dfbd7f24b921398affc357", + "sha256:efa1909120ce98bbb3777e8b6f92237f5d5c8ea6758efea36a473e1d38f7d3e4", + "sha256:f3900e8a5de27447acbf900b4750b0ddfd7ec1ea7fbaf11dfa911141bc522af0" + ], + "version": "==1.4.3" + }, "mccabe": { "hashes": [ "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", @@ -126,6 +187,32 @@ "index": "pypi", "version": "==0.0.6" }, + "mypy": { + "hashes": [ + "sha256:0a9a45157e532da06fe56adcfef8a74629566b607fa2c1ac0122d1ff995c748a", + "sha256:2c35cae79ceb20d47facfad51f952df16c2ae9f45db6cb38405a3da1cf8fc0a7", + "sha256:4b9365ade157794cef9685791032521233729cb00ce76b0ddc78749abea463d2", + "sha256:53ea810ae3f83f9c9b452582261ea859828a9ed666f2e1ca840300b69322c474", + "sha256:634aef60b4ff0f650d3e59d4374626ca6153fcaff96ec075b215b568e6ee3cb0", + "sha256:7e396ce53cacd5596ff6d191b47ab0ea18f8e0ec04e15d69728d530e86d4c217", + "sha256:7eadc91af8270455e0d73565b8964da1642fe226665dd5c9560067cd64d56749", + "sha256:7f672d02fffcbace4db2b05369142e0506cdcde20cea0e07c7c2171c4fd11dd6", + "sha256:85baab8d74ec601e86134afe2bcccd87820f79d2f8d5798c889507d1088287bf", + "sha256:87c556fb85d709dacd4b4cb6167eecc5bbb4f0a9864b69136a0d4640fdc76a36", + "sha256:a6bd44efee4dc8c3324c13785a9dc3519b3ee3a92cada42d2b57762b7053b49b", + "sha256:c6d27bd20c3ba60d5b02f20bd28e20091d6286a699174dfad515636cb09b5a72", + "sha256:e2bb577d10d09a2d8822a042a23b8d62bc3b269667c9eb8e60a6edfa000211b1", + "sha256:f97a605d7c8bc2c6d1172c2f0d5a65b24142e11a58de689046e62c2d632ca8c1" + ], + "version": "==0.761" + }, + "mypy-extensions": { + "hashes": [ + "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d", + "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8" + ], + "version": "==0.4.3" + }, "packaging": { "hashes": [ "sha256:aec3fdbb8bc9e4bb65f0634b9f551ced63983a529d6a8931817d52fdd0816ddb", @@ -133,6 +220,13 @@ ], "version": "==20.0" }, + "parso": { + "hashes": [ + "sha256:55cf25df1a35fd88b878715874d2c4dc1ad3f0eebd1e0266a67e1f55efccfbe1", + "sha256:5c1f7791de6bd5dbbeac8db0ef5594b36799de198b3f7f7014643b0c5536b9d3" + ], + "version": "==0.5.2" + }, "pluggy": { "hashes": [ "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", @@ -182,6 +276,13 @@ ], "version": "==2.1.1" }, + "pylint": { + "hashes": [ + "sha256:3db5468ad013380e987410a8d6956226963aed94ecb5f9d3a28acca6d9ac36cd", + "sha256:886e6afc935ea2590b462664b161ca9a5e40168ea99e5300935f6591ad467df4" + ], + "version": "==2.4.4" + }, "pyparsing": { "hashes": [ "sha256:4c830582a84fb022400b85429791bc551f1f4871c33f23e44f353119e92f969f", @@ -198,11 +299,11 @@ }, "pytest": { "hashes": [ - "sha256:6b571215b5a790f9b41f19f3531c53a45cf6bb8ef2988bc1ff9afb38270b25fa", - "sha256:e41d489ff43948babd0fad7ad5e49b8735d5d55e26628a58673c39ff61d95de4" + "sha256:1d122e8be54d1a709e56f82e2d85dcba3018313d64647f38a91aec88c239b600", + "sha256:c13d1943c63e599b98cf118fcb9703e4d7bde7caa9a432567bcdcae4bf512d20" ], "index": "pypi", - "version": "==5.3.2" + "version": "==5.3.4" }, "pytest-cov": { "hashes": [ @@ -220,6 +321,13 @@ "index": "pypi", "version": "==3.2.1" }, + "send2trash": { + "hashes": [ + "sha256:60001cc07d707fe247c94f74ca6ac0d3255aabcb930529690897ca2a39db28b2", + "sha256:f1691922577b6fa12821234aeb57599d887c4900b9ca537948d2dac34aea888b" + ], + "version": "==1.5.0" + }, "six": { "hashes": [ "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", @@ -227,6 +335,49 @@ ], "version": "==1.14.0" }, + "thonny": { + "hashes": [ + "sha256:8ddee49f51ba1e1c2125e2119db6c52bc6806853e8cae42b373da1a52bce98e9", + "sha256:c221c520a34fd0d2903da9fd737146438eb29d1a8a539cfb6ecf90625cf0dd1f" + ], + "index": "pypi", + "version": "==3.2.7" + }, + "typed-ast": { + "hashes": [ + "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355", + "sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919", + "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa", + "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652", + "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75", + "sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01", + "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d", + "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1", + "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907", + "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c", + "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3", + "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b", + "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614", + "sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb", + "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b", + "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41", + "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6", + "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34", + "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe", + "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4", + "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7" + ], + "markers": "implementation_name == 'cpython' and python_version < '3.8'", + "version": "==1.4.1" + }, + "typing-extensions": { + "hashes": [ + "sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2", + "sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d", + "sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575" + ], + "version": "==3.7.4.1" + }, "wcwidth": { "hashes": [ "sha256:8fd29383f539be45b20bd4df0dc29c20ba48654a41e661925e612311e9f3c603", @@ -241,12 +392,18 @@ ], "version": "==0.57.0" }, + "wrapt": { + "hashes": [ + "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" + ], + "version": "==1.11.2" + }, "zipp": { "hashes": [ - "sha256:8dda78f06bd1674bd8720df8a50bb47b6e1233c503a4eed8e7810686bde37656", - "sha256:d38fbe01bbf7a3593a32bc35a9c4453c32bc42b98c377f9bff7e9f8da157786c" + "sha256:57147f6b0403b59f33fd357f169f860e031303415aeb7d04ede4839d23905ab8", + "sha256:7ae5ccaca427bafa9760ac3cd8f8c244bfc259794b5b6bb9db4dda2241575d09" ], - "version": "==1.0.0" + "version": "==2.0.0" } }, "develop": {} diff --git a/README.md b/README.md index c808fe4..1db96e8 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,7 @@ make targets: docker-pull pull docker images docker-build pull and build docker images update update git repositories/submodules, virtualenv, docker images and build local docker image + thonny run Thonny IDE to access the Micropython REPL (Python prompt) test Run pytest micropython_shell start a bash shell in docker container "local/micropython:latest" unix-port-shell start micropython unix port interpreter diff --git a/upload_files.py b/upload_files.py deleted file mode 100644 index e3801dc..0000000 --- a/upload_files.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python3 - -from pathlib import Path - -from utils.main import lint_and_compile -from utils.sync import sync - -PORT = 8266 - - -if __name__ == '__main__': - src_path = Path('src') - bdist_path = Path('bdist') - - lint_and_compile(src_path, bdist_path) - - print('_' * 100) - print('Sync bdist files\n') - - sync( - src_path=bdist_path, - verbose=True, - ) From d389c1c1feaf206e45d788660a4cbed7a3cb8bd7 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Wed, 22 Jan 2020 10:58:28 +0100 Subject: [PATCH 18/31] update README --- README.md | 171 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 106 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index 1db96e8..c39c7c4 100644 --- a/README.md +++ b/README.md @@ -80,24 +80,22 @@ All existing screenshots can be found here: * [jedie.github.io/blob/master/screenshots/WebSwitch/](https://github.com/jedie/jedie.github.io/blob/master/screenshots/WebSwitch/README.creole) -## prepare +# setup a device + + +## prepare the hardware: * Open the device * make a connection with a UART-USB converter with 3.3V option and to the `3.3V`, `GND`, `TX` and `RX` pins +* **WARNING:** DO NOT CONNECT DEVICES TO MAINS POWER WHILE THE COVER IS OPEN AND CIRCUIT BOARD IS EXPOSED!!! -Very good information to get started can you found here: https://github.com/tsaarni/mqtt-micropython-smartsocket +Very good information to get started can you found here: +- https://templates.blakadder.com/sonoff_S20.html +- https://github.com/tsaarni/mqtt-micropython-smartsocket -## quickstart - -Clone the sources, and setup virtualenv via `pipenv`: -```bash -~$ git clone https://github.com/jedie/micropython-sonoff-webswitch.git -~$ cd micropython-sonoff-webswitch -~/micropython-sonoff-webswitch$ make update -``` -overview: +## Overview of installing the software on the device: * Generate yaota8266 RSA keys, create `config.h` and compile yaota8266 and firmware * Flash yaota8266 and firmware @@ -106,7 +104,15 @@ overview: * Connect device to WiFi * start soft-OTA to put all missing files to the device -## compile own firmware + +## setup project + +Clone the sources, and setup virtualenv via `pipenv`: +```bash +~$ git clone https://github.com/jedie/micropython-sonoff-webswitch.git +~$ cd micropython-sonoff-webswitch +~/micropython-sonoff-webswitch$ make update +``` To see all make targets, just call make, e.g.: ```bash @@ -134,6 +140,8 @@ make targets: miniterm Low level debug device via miniterm.py (from pyserial) to /dev/ttyUSB0 ``` +## compile own firmware + ### docker-yaota8266/yaota8266/config.h You must create `docker-yaota8266/yaota8266/config.h` and insert your RSA modulus line. @@ -161,7 +169,7 @@ You should backup theses files; After you have created your own RSA keys and `config.h`, you can compile `yaota8266.bin` and `firmware-ota.bin`, e.g.: ```bash ~/micropython-sonoff-webswitch$ make yaota8266-build -~/micropython-sonoff-webswitch$ make build-firmware +~/micropython-sonoff-webswitch$ make build-ota-firmware ``` The compiled files are stored here: @@ -171,86 +179,94 @@ The compiled files are stored here: * `~/micropython-sonoff-webswitch/build/firmware-ota.bin.ota` <- used in hard-OTA process -### flash yaota8266 and firmware +### Deploying the firmware: flash yaota8266 and firmware + +ESP8266 needs to be put into Programming Mode before the firmware can be uploaded. To put the ESP8266 into Programming Mode: + +* Press and hold the power button before connecting +* Connect device via UART-USB converter +* After about 2 seconds: release the button + +Now `esptool` can be used. But only for *one* operation! After each esptool call, you must disconnect the device from the USB and repeat this procedure! + +The first time, the flash memory must be erased, call: + +```bash +~/micropython-sonoff-webswitch$ make erase-flash +``` -After you have called `make yaota8266-build` and `make build-firmware` you can flash your device: +After `erase-flash` and after you have called `make yaota8266-build` and `make build-ota-firmware` you can flash your device: ```bash +# put into Programming Mode and call: ~/micropython-sonoff-webswitch$ make flash-yaota8266 -~/micropython-sonoff-webswitch$ make flash-firmware + +# Again, put into Programming Mode and call: +~/micropython-sonoff-webswitch$ make flash-ota-firmware ``` -**Importand**: the flash make targets are for the Sonoff ESP8266 and may **not work** on other ESP8266 devices! +**Importand**: These flash commands are for the Sonoff device and may **not work** on other ESP8266 devices! For other devices just use `esptool` directly, e.g.: ```bash -~/micropython-sonoff-webswitch$ pipenv run esptool.py --port /dev/ttyUSB0 --baud 460800 write_flash 0 build/yaota8266.bin -~/micropython-sonoff-webswitch$ pipenv run esptool.py --port /dev/ttyUSB0 --baud 460800 write_flash 0x3c000 build/firmware-ota.bin +~/micropython-sonoff-webswitch$ pipenv run esptool.py --port /dev/ttyUSB0 write_flash 0 build/yaota8266.bin +~/micropython-sonoff-webswitch$ pipenv run esptool.py --port /dev/ttyUSB0 write_flash 0x3c000 build/firmware-ota.bin ``` **Note:** The file `firmware-ota.bin` must be flash with `esptool.py` **not** the `firmware-ota.bin.ota` ! This file is ues in hard-OTA update process. -## flash micropython - -* Flash last MicroPython firmware to your device, see: http://docs.micropython.org/en/latest/esp8266/tutorial/intro.html - -overview: - -* Press and hold the power button before connecting -* Connect device via UART-USB converter -* Now you can release the button. -* To erase the Flash with esptool, call: `./erase_flash.sh` -* disconnect the device from PC and reconnect with pressed button again -* To flash micropython call: `./flash_firmware.sh` (This script will download micropython, if not already done and compares the sha256 hash) -* disconnect device and reconnect **without** pressing the power button -Notes: In my experience flashing the S20 needs `-fs 1MB -fm dout 0x0`. +### access the Micropython REPL -**Importand**: +Once you have the firmware on the device you can access the Micropython REPL (Python prompt). +There are different ways to do this. I used [thonny](https://github.com/thonny/thonny/) and his [MicroPython support](https://github.com/thonny/thonny/wiki/MicroPython) -The micropython version and the `mpy_cross` version must be **the same**! -Otherwise the compiled `.mpy` can't run on the device! +`thonny` is installed via pipenv, to start the IDE, just call: +```bash +~/micropython-sonoff-webswitch$ make thonny +``` -## WiFi config +First steps in thonny: +* Activate ESP8266 mode: `Tools / Options / Interpreter` select: `MicroPython (ESP8266)` +* Activate `files` Tab: `View / Files` -To connect the device with your WIFI it needs the SSID/password. -Several WLAN network access data can be specified. +Now you should be able to access the Micropython REPL via `Ctrl-F2`. +You can also start scripts on the device: -Copy and edit [_config_wifi-example.json](https://github.com/jedie/micropython-sonoff-webswitch/blob/master/_config_wifi-example.json) to `src/_config_wifi.json` +* In `This computer` TAB go to: `.../micropython-sonoff-webswitch/helpers` +* Open a file by double-click e.g.: `mpy_information.py` in the editor +* Run the script on the device by `F5` -## bootstrap +### format flash filesystem -If micropython is on the device, only the sources and your WiFi credentials file needs to be uploaded. +Now you have flashed the `yaota8266` bootloader and MicroPython Firmware and you must have access to the Micropython REPL. -This 'bootstrapping' can be done in different ways. +To format the flash filesystem to `littlefs2`, just run `.../micropython-sonoff-webswitch/helpers/convert_filesystem.py` on the device. -### bootstrap via USB +### copy missing files to the device -Connect device via TTL-USB-converter to your PC and run [upload_files.py](https://github.com/jedie/micropython-sonoff-webswitch/blob/master/upload_files.py) +After format the flash filesystem as `littlefs2`: copy missing files to the device, using soft-OTA: -This script will do this: +* Connect Device to your WiFi network +* start soft-OTA client and server -* compile `src` files with [mpy-cross](https://pypi.org/project/mpy-cross/) to `bdist` -* upload all files from `bdist` to the device via [mpycntrl](https://github.com/kr-g/mpycntrl) -The upload is a little bit slow. Bootstrap via WiFi is faster, see below: +#### create src/_config_wifi.json +The device needs SSID/passwords to be able to log in to different WLANs. It reads the credentials from the file `_config_wifi.json`. This file must be created. The template is [_config_wifi-example.json](https://github.com/jedie/micropython-sonoff-webswitch/blob/master/_config_wifi-example.json). -### bootstrap via WiFi +Copy and edit [_config_wifi-example.json](https://github.com/jedie/micropython-sonoff-webswitch/blob/master/_config_wifi-example.json) to `src/_config_wifi.json` -overview: +All missing files in `src` will be copied to device via soft-OTA. -* Connect the device to your WiFi network -* put your `_config_wifi.json` to the root of the device -* Run `ota_client.py` to upload all `*.mpy` compiled files -I used [thonny](https://github.com/thonny/thonny) for this. With thonny it's easy to upload the config file and execute scripts on the device. +#### connect device to WiFi -To connect to your WiFi network, edit and run this: +To connect the device on the fist start to your WiFi network, edit and run this: ```python import time, network @@ -262,6 +278,30 @@ while not sta_if.isconnected(): print('connected:', sta_if.ifconfig()) ``` +Just copy&paste this code snippet into Thonny IDE, insert your credentials and run it via `F5` + + +### run soft-OTA manually + +After the device is connected to your WiFi, it can run soft-OTA and copy all missing `src` files. + +Start soft-OTA server with: +```bash +~/micropython-sonoff-webswitch$ make soft-ota +``` + +At the same time, open [.../micropython-sonoff-webswitch/src/ota_client.py](https://github.com/jedie/micropython-sonoff-webswitch/blob/master/src/ota_client.py) in Thonny and start it via `F5`. + + +Now the soft-OTA should be run: + +* The device will be connected +* All missing/new files from `src` will be transfered to the device + + +Now the device setup is done ;) + + ### OTA updates **Note**: There are two kinds of OTA updates: @@ -269,7 +309,7 @@ print('connected:', sta_if.ifconfig()) * 'hard' OTA update via **yaota8266** bootloader that will replace the complete firmware. * 'soft' OTA updates via pure micropython script that will only upload new files to the flash filesystem. -The 'hard' OTA via **yaota8266** is work-in-progress and will currenlty not work, see: https://github.com/jedie/micropython-sonoff-webswitch/issues/33 +The 'hard' OTA via **yaota8266** is work-in-progress, see: https://github.com/jedie/micropython-sonoff-webswitch/issues/33 #### 'soft' OTA updates @@ -279,11 +319,10 @@ The device will run the [/src/ota_client.py](https://github.com/jedie/micropytho The script waits some time for the OTA server and after the timeout the normal web server will be started. -To start the `OTA Server`, do this: +To start the `soft-OTA Server`, do this: ```bash -~$ cd micropython-sonoff-webswitch -~/micropython-sonoff-webswitch$ pipenv run start_ota_server.py +~/micropython-sonoff-webswitch$ make soft-ota ``` If server runs: reboot device and look to the output of the OTA server. @@ -298,15 +337,17 @@ The OTA update implementation does: ## project structure -* `./bdist/` - Contains the compiled files that will be uploaded to the device. -* `./helpers/` - Some device tests/helper scripts for developing (normaly not needed) +* `./bdist/` - compiled `.mpy` files (and `.html`, `.css` files) that will be uploaded to the device in `soft-OTA` +* `./build/` - compiled firmware files (`firmware-*.bin` and `yaota8266.bin`) for flashing and `hard-OTA` +* `./docker-yaota8266/` - git submodule https://github.com/jedie/docker-yaota8266 to compile yaota8266.bin via docker +* `./helpers/` - Some device tests/helper scripts for bootstrap and developing +* `./micropython_config/` - Config files used to compile MicroPython Firmware * `./mpy_tests/` - tests that can be run on micropython device (will be also run by pytest with mocks) -* `./ota/` - source code of the OTA server +* `./sdist/` - Contains all modules that will be freezes into firmware, created via `utils/make_sdist.py` +* `./soft_ota/` - source code of the OTA server * `./src/` - device source files * `./tests/` - some pytest files (run on host with CPython) * `./utils/` - utils for local run (compile, code lint, sync with mpycntrl) -* `./start_ota_server.py` - Starts the local OTA server (will compile, lint the `src` files and create `bdist`) -* `./upload_files.py` - Upload files via USB (will compile, lint the `src` files and create `bdist`) ## Links From 5bdbc7cccf75dc2af4b228e520373c4ed9cad703 Mon Sep 17 00:00:00 2001 From: Jens Diemer Date: Wed, 22 Jan 2020 11:45:56 +0100 Subject: [PATCH 19/31] diagram_import.drawio --- Untitled Diagram.drawio | 1 + 1 file changed, 1 insertion(+) create mode 100644 Untitled Diagram.drawio diff --git a/Untitled Diagram.drawio b/Untitled Diagram.drawio new file mode 100644 index 0000000..6e26935 --- /dev/null +++ b/Untitled Diagram.drawio @@ -0,0 +1 @@ +5VjbbuIwEP2aqLsrFeXG7RFo2XYvUiUe9vJmsEmsOnHkmIb063dMnMS5sFCprJAWIeE5Ho+dc2zPBMtbRPvPAiXhd44Js1wb7y3vznJdx3ddS31tnBfIZKiBQFCsnWpgRV+JBm2N7igmacNRcs4kTZrghscx2cgGhoTgWdNty1lz1gQFpAOsNoh10R8Uy1A/hTuu8QdCg7Cc2RlNi54Ilc76SdIQYZ4ZkHdveQvBuSxa0X5BmCKv5KUYtzzSWy1MkFieM+DLKBMTXzzv02+/H/2M5vLh662O8oLYTj+wXqzMSwZIjGeKSLA2DKUp3VjePJQRA8CB5pbHUuvmjMFOpeDPFVlTQGCBIv8Jhj3wh34J/ALg1h44pX2n9oxdWblpPRFBIyKJ0GCxSII72tVklJsFiYDIvzAwraSAPUw4zCJyGJfVYg+1gKGhc4kJwpCkL81lIL3ngipcNcMTp7BA19bnwyvj6NPhju1miJTvxIboUaa4JwL5XitQwUMnEDSMx66hw955wz5yT++jQPBdot2IkGRv9ZxftC7d7SOSdqUqKZge4dKQsgpraulMWlyZujWIeisrXg8rIwbTzjF9gWagmktByCvB6srgeMfgstM+MKXh1jPy0yBKFAFbaoxai/aIdpyWLiBLjAnWJysLqSSrBG1UbwZ3evOwH9XP1On4DumKp8Vy3B6xnOlgan7GXe1Gl5LO70jXJPsaSRxdGYfDLocmhfYHMggGljeD5hrS4KFzcUidNAbj45WxfC20jo7eKokg5TFP83SQ6BJkOL+5KZiFnyWja8O6sYZ3xkVhROiwDzzKJsVFql9wxlVijnlMVD1AGWtBiNEgVvUD8K5y+FypQqHImumOiGKspunVtKl6u9w4S+YjmSNvJU9DYc/vyRTupSQdn5UooPoKwWsluVBV67lpglEpGdn+owzxnvroASPndGavkngjs19MsElHsCUVUYbU8bly9qo66RR7F7vAph3yHqOEC3kPrxnijGvnAiS5reLR78mljtdDkn8pkkrVDJbogaX/4F4uK3qvde69rijvdE2DWb+JFy9F9f8Z3v0f \ No newline at end of file From eb354b22e03fde3e3044082f7b2848d9701fb202 Mon Sep 17 00:00:00 2001 From: Jens Diemer Date: Wed, 22 Jan 2020 12:44:28 +0100 Subject: [PATCH 20/31] Update README.md --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index c39c7c4..15d242c 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,12 @@ All existing screenshots can be found here: * [jedie.github.io/blob/master/screenshots/WebSwitch/](https://github.com/jedie/jedie.github.io/blob/master/screenshots/WebSwitch/README.creole) +### OTA overview + +![ESP8266_OTA.png](https://raw.githubusercontent.com/jedie/jedie.github.io/master/diagrams/micropython/ESP8266_OTA.png) + + + # setup a device From ffda6cc764e8ba244383a38b5efa69426a618967 Mon Sep 17 00:00:00 2001 From: Jens Diemer Date: Wed, 22 Jan 2020 17:11:20 +0100 Subject: [PATCH 21/31] Update README.md Manual format not needed, will be done by https://github.com/jedie/micropython-sonoff-webswitch/blob/master/src/inisetup.py --- README.md | 9 --------- 1 file changed, 9 deletions(-) diff --git a/README.md b/README.md index 15d242c..5ab7346 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,6 @@ Very good information to get started can you found here: * Generate yaota8266 RSA keys, create `config.h` and compile yaota8266 and firmware * Flash yaota8266 and firmware -* format device flash to `littlefs2` with: `helpers/convert_filesystem.py` * create `_config_wifi.json` * Connect device to WiFi * start soft-OTA to put all missing files to the device @@ -245,14 +244,6 @@ You can also start scripts on the device: * Open a file by double-click e.g.: `mpy_information.py` in the editor * Run the script on the device by `F5` - -### format flash filesystem - -Now you have flashed the `yaota8266` bootloader and MicroPython Firmware and you must have access to the Micropython REPL. - -To format the flash filesystem to `littlefs2`, just run `.../micropython-sonoff-webswitch/helpers/convert_filesystem.py` on the device. - - ### copy missing files to the device After format the flash filesystem as `littlefs2`: copy missing files to the device, using soft-OTA: From 07877900ad73516056fa2e78a16ce32204ebf7f1 Mon Sep 17 00:00:00 2001 From: Jens Diemer Date: Fri, 24 Jan 2020 10:33:59 +0100 Subject: [PATCH 22/31] Update Makefile use pipenv run to fix #44 --- Makefile | 8 ++++---- Pipfile | 3 +++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index effa562..be97b22 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ test: update ## Run pytest .PHONY: sdist sdist: - python3 utils/make_sdist.py + pipenv run make_sdist micropython_shell: sdist docker-build ## start a bash shell in docker container "local/micropython:latest" @@ -174,10 +174,10 @@ flash-ota-firmware: verify ## Flash build/firmware-ota.bin to location 0x3c000 v pipenv run esptool.py --port /dev/ttyUSB0 --baud 460800 write_flash -fs 1MB -fm dout 0x3c000 build/firmware-ota.bin hard-ota: ## Start yaota8266 live-ota to hard-OTA Update the firmware file build/firmware-ota.bin.ota - python3 docker-yaota8266/yaota8266/cli.py ota build/firmware-ota.bin.ota + pipenv run docker-yaota8266/yaota8266/cli.py ota build/firmware-ota.bin.ota soft-ota: ## Start soft-OTA updates: Compile .py to .mpy and push missing/updated files (*.mpy, *.css, *.html etc.) to the device - python3 start_soft_ota_server.py + pipenv run start_soft_ota_server.py miniterm: ## Low level debug device via miniterm.py (from pyserial) to /dev/ttyUSB0 - pipenv run miniterm.py /dev/ttyUSB0 115200 \ No newline at end of file + pipenv run miniterm.py /dev/ttyUSB0 115200 diff --git a/Pipfile b/Pipfile index 5b23c52..66ca8b6 100644 --- a/Pipfile +++ b/Pipfile @@ -18,3 +18,6 @@ thonny = "*" [requires] python_version = "3.6" + +[scripts] +make_sdist = "python utils/make_sdist.py" From a170fc69d468f0cf04381ff65c933c0ed9945a9e Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Wed, 29 Jan 2020 10:36:53 +0100 Subject: [PATCH 23/31] Fix: Error the command start_soft_ota_server.py could not be found within PATH or Pipfile's [scripts] See: https://github.com/jedie/micropython-sonoff-webswitch/issues/47 --- Makefile | 2 +- Pipfile | 3 ++- Pipfile.lock | 30 +++++++++++++++--------------- soft_ota/ota_server.py | 5 +++++ tests/test_soft_ota_server.py | 33 +++++++++++++++++++++++++++++++++ 5 files changed, 56 insertions(+), 17 deletions(-) create mode 100644 tests/test_soft_ota_server.py diff --git a/Makefile b/Makefile index be97b22..2aabf81 100644 --- a/Makefile +++ b/Makefile @@ -177,7 +177,7 @@ hard-ota: ## Start yaota8266 live-ota to hard-OTA Update the firmware file buil pipenv run docker-yaota8266/yaota8266/cli.py ota build/firmware-ota.bin.ota soft-ota: ## Start soft-OTA updates: Compile .py to .mpy and push missing/updated files (*.mpy, *.css, *.html etc.) to the device - pipenv run start_soft_ota_server.py + pipenv run start_soft_ota_server miniterm: ## Low level debug device via miniterm.py (from pyserial) to /dev/ttyUSB0 pipenv run miniterm.py /dev/ttyUSB0 115200 diff --git a/Pipfile b/Pipfile index 66ca8b6..bf775a0 100644 --- a/Pipfile +++ b/Pipfile @@ -20,4 +20,5 @@ thonny = "*" python_version = "3.6" [scripts] -make_sdist = "python utils/make_sdist.py" +make_sdist = "python3 utils/make_sdist.py" +start_soft_ota_server = "python3 start_soft_ota_server.py" diff --git a/Pipfile.lock b/Pipfile.lock index 2f6e635..bdb2d8f 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -119,11 +119,11 @@ }, "importlib-metadata": { "hashes": [ - "sha256:bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359", - "sha256:f17c015735e1a88296994c0697ecea7e11db24290941983b08c9feb30921e6d8" + "sha256:06f5b3a99029c7134207dd882428a66992a9de2bef7c2b699b5641f9886c3302", + "sha256:b97607a1a18a5100839aec1dc26a1ea17ee0d93b20b0f008d80a5a050afb200b" ], "markers": "python_version < '3.8'", - "version": "==1.4.0" + "version": "==1.5.0" }, "isort": { "hashes": [ @@ -134,10 +134,10 @@ }, "jedi": { "hashes": [ - "sha256:1349c1e8c107095a55386628bb3b2a79422f3a2cab8381e34ce19909e0cf5064", - "sha256:e909527104a903606dd63bea6e8e888833f0ef087057829b89a18364a856f807" + "sha256:b4f4052551025c6b0b0b193b29a6ff7bdb74c52450631206c262aef9f7159ad2", + "sha256:d5c871cb9360b414f981e7072c52c33258d598305280fef91c6cae34739d65d5" ], - "version": "==0.15.2" + "version": "==0.16.0" }, "lazy-object-proxy": { "hashes": [ @@ -215,17 +215,17 @@ }, "packaging": { "hashes": [ - "sha256:aec3fdbb8bc9e4bb65f0634b9f551ced63983a529d6a8931817d52fdd0816ddb", - "sha256:fe1d8331dfa7cc0a883b49d75fc76380b2ab2734b220fbb87d774e4fd4b851f8" + "sha256:170748228214b70b672c581a3dd610ee51f733018650740e98c7df862a583f73", + "sha256:e665345f9eef0c621aa0bf2f8d78cf6d21904eef16a93f020240b704a57f1334" ], - "version": "==20.0" + "version": "==20.1" }, "parso": { "hashes": [ - "sha256:55cf25df1a35fd88b878715874d2c4dc1ad3f0eebd1e0266a67e1f55efccfbe1", - "sha256:5c1f7791de6bd5dbbeac8db0ef5594b36799de198b3f7f7014643b0c5536b9d3" + "sha256:1376bdc8cb81377ca481976933773295218a2df47d3e1182ba76d372b1acb128", + "sha256:597f36de5102a8db05ffdf7ecdc761838b86565a4a111604c6e78beaedf1b045" ], - "version": "==0.5.2" + "version": "==0.6.0" }, "pluggy": { "hashes": [ @@ -400,10 +400,10 @@ }, "zipp": { "hashes": [ - "sha256:57147f6b0403b59f33fd357f169f860e031303415aeb7d04ede4839d23905ab8", - "sha256:7ae5ccaca427bafa9760ac3cd8f8c244bfc259794b5b6bb9db4dda2241575d09" + "sha256:ccc94ed0909b58ffe34430ea5451f07bc0c76467d7081619a454bf5c98b89e28", + "sha256:feae2f18633c32fc71f2de629bfb3bd3c9325cd4419642b1f1da42ee488d9b98" ], - "version": "==2.0.0" + "version": "==2.1.0" } }, "develop": {} diff --git a/soft_ota/ota_server.py b/soft_ota/ota_server.py index 044af9e..09abda8 100644 --- a/soft_ota/ota_server.py +++ b/soft_ota/ota_server.py @@ -9,6 +9,7 @@ """ import asyncio import hashlib +import os import socket import time from pathlib import Path @@ -354,6 +355,10 @@ async def port_scan_and_serve(self, port): return clients print('.', end='', flush=True) + if os.environ.get('OTA_NO_LOOP') is not None: + # Used in unittests, see: tests/test_soft_ota_server.py + print('Exit after one try, because OTA_NO_LOOP is set, ok.') + return clients time.sleep(2) def run(self, port): diff --git a/tests/test_soft_ota_server.py b/tests/test_soft_ota_server.py new file mode 100644 index 0000000..985923c --- /dev/null +++ b/tests/test_soft_ota_server.py @@ -0,0 +1,33 @@ +import os +import subprocess +from unittest import TestCase + +from utils.constants import BASE_PATH + + +class SoftOtaServerTestCase(TestCase): + def test_make_soft_ota(self): + env = os.environ.copy() + env['OTA_NO_LOOP'] = 'yes' + output = subprocess.check_output( + ['make soft-ota'], + env=env, + cwd=str(BASE_PATH), + shell=True, + universal_newlines=True + ) + print(output) + + assert 'Lint code with flake8' in output + assert 'flake8, ok.' in output + + assert 'Compile via mpy_cross' in output + assert '+ src/webswitch.py -> bdist/webswitch.mpy' in output + + assert 'Copy files...' in output + assert '+ src/boot.py -> bdist/boot.py' in output + + assert 'Start OTA Server' in output + assert 'Wait vor devices on port: 8267' in output + assert 'Exit after one try, because OTA_NO_LOOP is set, ok.' in output + assert 'Update 0 device(s), ok.' in output From 7be99594edccb87c3846546700f046667a52fbe0 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Wed, 5 Feb 2020 15:32:12 +0100 Subject: [PATCH 24/31] +helpers/pico_uasyncio_webserver.py --- helpers/pico_uasyncio_webserver.py | 39 ++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 helpers/pico_uasyncio_webserver.py diff --git a/helpers/pico_uasyncio_webserver.py b/helpers/pico_uasyncio_webserver.py new file mode 100644 index 0000000..e7f16b2 --- /dev/null +++ b/helpers/pico_uasyncio_webserver.py @@ -0,0 +1,39 @@ +""" + A very, very minimalistic uasyncio based micropython web server +""" + +import network +import uasyncio +import utime + + +async def request_handler(reader, writer): + print('-' * 100) + peername = writer.get_extra_info('peername') + method, url, http_version = (await reader.readline()).decode().strip().split() + print(peername, method, url, http_version) + + while True: + line = await reader.readline() + if line == b'\r\n': # header ends + break + print(line.decode().strip()) + + await writer.awrite(b'HTTP/1.0 200 OK\r\n') + await writer.awrite(b'Content-type: text/plain; charset=utf-8\r\n\r\n') + + await writer.awrite(b'Your IP: %s port:%s\n' % peername) + await writer.awrite(b'Device time:%s\n' % utime.time()) + + await writer.aclose() + + +if __name__ == '__main__': + sta_if = network.WLAN(network.STA_IF) + print('WiFi information:', sta_if.ifconfig()) + + loop = uasyncio.get_event_loop() + loop.create_task(uasyncio.start_server(request_handler, '0.0.0.0', 80)) + + print('Web server started...') + loop.run_forever() From 72b361f1ac0e7fed522c0b5f0d0f7d321e6b479c Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Wed, 5 Feb 2020 17:20:27 +0100 Subject: [PATCH 25/31] update requirements --- Pipfile.lock | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index bdb2d8f..4f701a4 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -174,10 +174,10 @@ }, "more-itertools": { "hashes": [ - "sha256:1a2a32c72400d365000412fe08eb4a24ebee89997c18d3d147544f70f5403b39", - "sha256:c468adec578380b6281a114cb8a5db34eb1116277da92d7c46f904f0b52d3288" + "sha256:5dd8bcf33e5f9513ffa06d5ad33d78f31e1931ac9a18f33d37e77a180d393a7c", + "sha256:b1ddb932186d8a6ac451e1d95844b382f55e12686d51ca0c68b6f61f2ab7a507" ], - "version": "==8.1.0" + "version": "==8.2.0" }, "mpycntrl": { "hashes": [ @@ -222,10 +222,10 @@ }, "parso": { "hashes": [ - "sha256:1376bdc8cb81377ca481976933773295218a2df47d3e1182ba76d372b1acb128", - "sha256:597f36de5102a8db05ffdf7ecdc761838b86565a4a111604c6e78beaedf1b045" + "sha256:56b2105a80e9c4df49de85e125feb6be69f49920e121406f15e7acde6c9dfc57", + "sha256:951af01f61e6dccd04159042a0706a31ad437864ec6e25d0d7a96a9fbb9b0095" ], - "version": "==0.6.0" + "version": "==0.6.1" }, "pluggy": { "hashes": [ @@ -299,11 +299,11 @@ }, "pytest": { "hashes": [ - "sha256:1d122e8be54d1a709e56f82e2d85dcba3018313d64647f38a91aec88c239b600", - "sha256:c13d1943c63e599b98cf118fcb9703e4d7bde7caa9a432567bcdcae4bf512d20" + "sha256:0d5fe9189a148acc3c3eb2ac8e1ac0742cb7618c084f3d228baaec0c254b318d", + "sha256:ff615c761e25eb25df19edddc0b970302d2a9091fbce0e7213298d85fb61fef6" ], "index": "pypi", - "version": "==5.3.4" + "version": "==5.3.5" }, "pytest-cov": { "hashes": [ From c339dd81acb513ae4ebfbf681c37cbba820b2e48 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Wed, 5 Feb 2020 17:21:02 +0100 Subject: [PATCH 26/31] update submodule docker-yaota8266 --- docker-yaota8266 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-yaota8266 b/docker-yaota8266 index d37b355..3dab2b5 160000 --- a/docker-yaota8266 +++ b/docker-yaota8266 @@ -1 +1 @@ -Subproject commit d37b355b5fc6f49414559e4fd24523405ccdb9d5 +Subproject commit 3dab2b5326d1d03404c5e757150c1ef8d30bdada From 387068b7a2ab93d5d59d1567beac8a652f839062 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Wed, 5 Feb 2020 17:21:37 +0100 Subject: [PATCH 27/31] Bugfix helpers/do_soft_ota_updates.py --- helpers/do_soft_ota_updates.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helpers/do_soft_ota_updates.py b/helpers/do_soft_ota_updates.py index 0c0e4d4..4a9b798 100644 --- a/helpers/do_soft_ota_updates.py +++ b/helpers/do_soft_ota_updates.py @@ -21,5 +21,5 @@ import gc gc.collect() - from ota_client import OtaUpdate - OtaUpdate().run() + from ota_client import SoftOtaUpdate + SoftOtaUpdate().run() From ca972eede39c59364e0eca603eb76a9834586d4d Mon Sep 17 00:00:00 2001 From: Jens Diemer Date: Thu, 6 Feb 2020 18:54:01 +0100 Subject: [PATCH 28/31] Update README.md --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 5ab7346..b7b31d5 100644 --- a/README.md +++ b/README.md @@ -192,6 +192,8 @@ ESP8266 needs to be put into Programming Mode before the firmware can be uploade * Connect device via UART-USB converter * After about 2 seconds: release the button +Maybe you must give the user the permissions to access the USB Port, e.g.: `sudo usermod -a -G dialout $USER` Otherwise you will get a error message like: `Permission denied: '/dev/ttyUSB0'` + Now `esptool` can be used. But only for *one* operation! After each esptool call, you must disconnect the device from the USB and repeat this procedure! The first time, the flash memory must be erased, call: @@ -222,6 +224,7 @@ For other devices just use `esptool` directly, e.g.: The file `firmware-ota.bin` must be flash with `esptool.py` **not** the `firmware-ota.bin.ota` ! This file is ues in hard-OTA update process. +More information about flashing can be found in the official documentation here: http://docs.micropython.org/en/latest/esp8266/tutorial/intro.html ### access the Micropython REPL From cfe37ae0ec4911b0c697917eab6ddf095d46c2e9 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Wed, 5 Feb 2020 17:22:00 +0100 Subject: [PATCH 29/31] update helpers/wifi_connect.py --- helpers/wifi_connect.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/helpers/wifi_connect.py b/helpers/wifi_connect.py index da8615e..e40e677 100644 --- a/helpers/wifi_connect.py +++ b/helpers/wifi_connect.py @@ -1,9 +1,10 @@ -import network -from wifi_connect import connect +""" + Connect to WiFi via saved settings in: + /_config_wifi.json +""" +import wifi +from context import Context if __name__ == '__main__': - # Connect or reconnect - sta_if = network.WLAN(network.STA_IF) - sta_if.disconnect() - connect(station=sta_if) - print('connected:', sta_if.isconnected()) + context = Context() + wifi.init(context) From abe2b93f39797ad40152add23d2ef9b9c4690d11 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Thu, 6 Feb 2020 19:43:21 +0100 Subject: [PATCH 30/31] WIP: refactor device reset --- src/constants.py | 12 ++++++++++++ src/context.py | 9 +++++---- src/http_internals.html | 10 ++++++---- src/http_internals.py | 9 ++++++++- src/ntp.py | 13 +++++++------ src/power_timer.py | 9 ++------- src/tasks.py | 42 +++++++++++++++++++++++++++++++++++++++++ src/watchdog.py | 5 ++++- src/watchdog_checks.py | 15 +++++---------- src/webswitch.py | 27 +++++++------------------- src/wifi.py | 12 +++++++----- 11 files changed, 105 insertions(+), 58 deletions(-) create mode 100644 src/tasks.py diff --git a/src/constants.py b/src/constants.py index f51edd9..9f94266 100644 --- a/src/constants.py +++ b/src/constants.py @@ -39,9 +39,21 @@ H2SEC = const(60 * 60) # multiplier for calc hours into seconds ONE_DAY_SEC = const(1 * 24 * 60 * 60) + + NTP_MIN_TIME_EPOCH = const(599616000) # epoch 1.1.2019 + +# The ESP8266 has a very inacurate RTC! So a sync is often needed. NTP_SYNC_WAIT_TIME_SEC = const(1 * 60 * 60) # sync NTP every 1 h +# The NTP sync must not exceed this time span, otherwise the device will be reset. +# This value should be chosen carefully. With each reset the relay can switch! +# e.g.: Possibly the wireless overnight is switched off for several hours but during +# the night a light should be switched on. Then the light would go out from time to time, +# while reset the device ;) +NTP_LOST_SYNC_DEFAULT_SEC = const(7 * 60 * 60) # Reset device if NTP sync lost in this time + + TIMEZONE_PY_CFG_NAME = 'timezone' DEFAULT_OFFSET_H = const(0) diff --git a/src/context.py b/src/context.py index 39fae10..32c73de 100644 --- a/src/context.py +++ b/src/context.py @@ -4,16 +4,17 @@ class Context: power_timer_today_active = True power_timer_timers = None power_timer_turn_on = None - power_timer_next_timer = None # epoch power_timer_next_timer_epoch = None - wifi_first_connect_time = None # epoch + wifi_first_connect_epoch = None # epoch + wifi_last_connect_epoch = 0 # epoch wifi_connected = 0 wifi_not_connected = 0 - ntp_next_sync = 0 # epoch + ntp_last_sync_epoch = 0 # epoch + ntp_next_sync_epoch = 0 # epoch - watchdog_last_feed = 0 # epoch + watchdog_last_feed_epoch = 0 # epoch watchdog_check_count = 0 watchdog_reset_count = 0 watchdog_last_reset_reason = 0 diff --git a/src/http_internals.html b/src/http_internals.html index a72ac82..1f58a4f 100644 --- a/src/http_internals.html +++ b/src/http_internals.html @@ -9,24 +9,26 @@ Today active:{power_timer_today_active} Timers:{power_timer_timers} Turn on:{power_timer_turn_on} - Next timer:{power_timer_next_timer} + Next timer:{power_timer_next_timer_epoch}
WiFi: - + +
First connection time:{wifi_first_connect_time}
First connection time:{wifi_first_connect_epoch}
Last connection time:{wifi_last_connect_epoch}
Connected count:{wifi_connected}
Not connected count:{wifi_not_connected}
NTP sync: - + +
next sync:{ntp_next_sync} sec.
last sync:{ntp_last_sync_epoch}
next sync:{ntp_next_sync_epoch}
Watchdog: - + diff --git a/src/http_internals.py b/src/http_internals.py index 7f432a4..5ac44e1 100644 --- a/src/http_internals.py +++ b/src/http_internals.py @@ -24,12 +24,19 @@ async def get_show(server, reader, writer, querystring, body): 'minimal_modules': ', '.join(server.context.minimal_modules) } + from timezone import localtime_isoformat + for attr_name in dir(server.context): if attr_name.startswith('_'): continue - value = str(getattr(server.context, attr_name)) + value = getattr(server.context, attr_name) print(attr_name, value) + if value is not None and attr_name.endswith('_epoch'): + value = localtime_isoformat(sep=' ', epoch=value) + else: + value = str(value) + template_context[attr_name] = value await server.send_html_page( diff --git a/src/ntp.py b/src/ntp.py index 5b1117f..aca817c 100644 --- a/src/ntp.py +++ b/src/ntp.py @@ -59,15 +59,16 @@ def _ntp_sync(): def ntp_sync(context): """ - will be called from watchdog_checks.check() - Must return True if everything is ok. + Sync the RTC time via NTP. + + Called from tasks.periodical_tasks() + updates context.wifi_last_connect_epoch """ - if context.ntp_next_sync > utime.time(): + if context.ntp_next_sync_epoch > utime.time(): print('NTP sync not needed, yet.') return True sync_done = _ntp_sync() # update RTC via NTP if sync_done is True: - context.ntp_next_sync = utime.time() + constants.NTP_SYNC_WAIT_TIME_SEC - - return sync_done + context.ntp_last_sync_epoch = utime.time() + context.ntp_next_sync_epoch = context.ntp_last_sync_epoch + constants.NTP_SYNC_WAIT_TIME_SEC diff --git a/src/power_timer.py b/src/power_timer.py index 620c969..37c7da2 100644 --- a/src/power_timer.py +++ b/src/power_timer.py @@ -43,9 +43,7 @@ def active_today(): def update_power_timer(context): """ Sets relay switch on/off on schedule and manual override. - - Will be called from watchdog_checks.check() - Must return True if everything is ok. + Called from tasks.periodical_tasks() """ gc.collect() if __debug__: @@ -101,9 +99,7 @@ def update_power_timer(context): if current_state is None: if __debug__: print('No timer to scheduled and no manual overwrite') - return True - - if current_state: + elif current_state: if __debug__: print('Switch on') Pins.relay.on() @@ -113,4 +109,3 @@ def update_power_timer(context): Pins.relay.off() gc.collect() - return True diff --git a/src/tasks.py b/src/tasks.py new file mode 100644 index 0000000..8657265 --- /dev/null +++ b/src/tasks.py @@ -0,0 +1,42 @@ +import gc +import sys + + +def reset(reason): + from reset import ResetDevice + ResetDevice(reason=reason).reset() + + +async def periodical_tasks(context): + """ + Periodical tasks, started from webswitch.WebServer.feed_watchdog + """ + if context.ntp_last_sync_epoch > 0: # A NTP sync was successful in the past + + # Sets relay switch on/off on schedule and manual override: + + from power_timer import update_power_timer + update_power_timer(context) + + del update_power_timer + del sys.modules['power_timer'] + gc.collect() + + # Ensure that the device is connected to the WiFi: + + from wifi import ensure_connection + is_connected = ensure_connection(context) + + del ensure_connection + del sys.modules['wifi'] + gc.collect() + + if is_connected: + # Sync the RTC time via NTP: + + from ntp import ntp_sync + ntp_sync(context) + + del ntp_sync + del sys.modules['ntp'] + gc.collect() diff --git a/src/watchdog.py b/src/watchdog.py index b59f690..e0523bf 100644 --- a/src/watchdog.py +++ b/src/watchdog.py @@ -32,7 +32,10 @@ def _timer_callback(self, timer): self.garbage_collection() def feed(self): - self.context.watchdog_last_feed = utime.time() + """ + Will be called from webswitch.WebServer.periodical_tasks() + """ + self.context.watchdog_last_feed_epoch = utime.time() def collect_import_cache(self): if self.context.minimal_modules is not None: diff --git a/src/watchdog_checks.py b/src/watchdog_checks.py index 4e59a28..a7bfedf 100644 --- a/src/watchdog_checks.py +++ b/src/watchdog_checks.py @@ -1,5 +1,3 @@ - - import gc import constants @@ -38,18 +36,15 @@ def can_bind_web_server_port(): def check(context): gc.collect() try: - if utime.time() - context.watchdog_last_feed > constants.WATCHDOG_TIMEOUT: - reset(reason='Feed timeout') - - from wifi import ensure_connection - if ensure_connection(context) is not True: - reset(reason='No Wifi connection') - - gc.collect() + if utime.time() - context.watchdog_last_feed_epoch > constants.WATCHDOG_TIMEOUT: + reset(reason='Watchdog feed timeout') if can_bind_web_server_port(): reset(reason='Web Server down') + if utime.time() - context.ntp_last_sync_epoch > constants.NTP_LOST_SYNC_DEFAULT_SEC: + reset(reason='NTP sync lost') + except MemoryError as e: context.watchdog.garbage_collection() reset(reason='Memory error: %s' % e) diff --git a/src/webswitch.py b/src/webswitch.py index 53b94b8..ab47acf 100644 --- a/src/webswitch.py +++ b/src/webswitch.py @@ -132,30 +132,17 @@ async def request_handler(self, reader, writer): Pins.power_led.on() print('----------------------------------------------------------------------------------') - async def feed_watchdog(self): + async def periodical_tasks(self): """ - Start some periodical tasks and feed the watchdog + Start periodical tasks and feed the watchdog """ while True: - gc.collect() - - from power_timer import update_power_timer - if update_power_timer(self.context) is not True: - from reset import ResetDevice - ResetDevice(reason='Update power timer error').reset() - - del update_power_timer - del sys.modules['power_timer'] - gc.collect() - - from ntp import ntp_sync - if ntp_sync(self.context) is not True: - from reset import ResetDevice - ResetDevice(reason='NTP sync error').reset() + from tasks import periodical_tasks + await periodical_tasks(self.context) - del ntp_sync - del sys.modules['ntp'] + del periodical_tasks + del sys.modules['tasks'] gc.collect() self.context.watchdog.feed() @@ -166,7 +153,7 @@ async def feed_watchdog(self): def run(self): loop = uasyncio.get_event_loop() loop.create_task(uasyncio.start_server(self.request_handler, '0.0.0.0', 80)) - loop.create_task(self.feed_watchdog()) + loop.create_task(self.periodical_tasks()) from led_dim_level_cfg import restore_power_led_level Pins.power_led.on() diff --git a/src/wifi.py b/src/wifi.py index 796352f..baac7ce 100644 --- a/src/wifi.py +++ b/src/wifi.py @@ -2,6 +2,7 @@ import sys import network +import utime from pins import Pins @@ -9,7 +10,7 @@ def ensure_connection(context): """ will be called from: - main.py - - watchdog_checks.check() + - tasks.periodical_tasks() Must return True if connected to WiFi. """ @@ -20,6 +21,7 @@ def ensure_connection(context): if station.isconnected(): print('Still connected:', station.ifconfig()) context.wifi_connected += 1 + context.wifi_last_connect_epoch = utime.time() return True context.wifi_not_connected += 1 @@ -31,10 +33,10 @@ def ensure_connection(context): del connect del sys.modules['wifi_connect'] - if context.wifi_first_connect_time is None: - context.wifi_first_connect_time = connected_time - - return True + if connected_time: + if context.wifi_first_connect_epoch is None: + context.wifi_first_connect_epoch = connected_time + return True def init(context): From 6b1f8dc0ebe0fd8ed90123d13c10d7db1bebfaf7 Mon Sep 17 00:00:00 2001 From: JensDiemer Date: Thu, 6 Feb 2020 20:08:18 +0100 Subject: [PATCH 31/31] set version to v0.12.0 --- src/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.py b/src/main.py index d8c4b08..d1c33af 100644 --- a/src/main.py +++ b/src/main.py @@ -1,6 +1,6 @@ -__version__ = 'v0.11.1' +__version__ = 'v0.12.0' def main(): pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy

Last feed:{watchdog_last_feed}
Last feed:{watchdog_last_feed_epoch}
Check count:{watchdog_check_count}
Reset count:{watchdog_reset_count}
Last reset reason:{watchdog_last_reset_reason}