Merge upstream tag 'v24.9.26' into development

schlimmchen · Sep 27, 2024 · 5d8bb8f · 5d8bb8f
2 parents 20159f3 + 3559007
commit 5d8bb8f
Show file tree

Hide file tree

Showing 20 changed files with 520 additions and 379 deletions.
diff --git a/include/MqttHandleHass.h b/include/MqttHandleHass.h
@@ -6,7 +6,7 @@
 #include <TaskSchedulerDeclarations.h>
 
 // mqtt discovery device classes
-enum {
+enum DeviceClassType {
     DEVICE_CLS_NONE = 0,
     DEVICE_CLS_CURRENT,
     DEVICE_CLS_ENERGY,
@@ -15,20 +15,34 @@ enum {
     DEVICE_CLS_FREQ,
     DEVICE_CLS_TEMP,
     DEVICE_CLS_POWER_FACTOR,
-    DEVICE_CLS_REACTIVE_POWER
+    DEVICE_CLS_REACTIVE_POWER,
+    DEVICE_CLS_CONNECTIVITY,
+    DEVICE_CLS_DURATION,
+    DEVICE_CLS_SIGNAL_STRENGTH,
+    DEVICE_CLS_TEMPERATURE,
+    DEVICE_CLS_RESTART
 };
-const char* const deviceClasses[] = { 0, "current", "energy", "power", "voltage", "frequency", "temperature", "power_factor", "reactive_power" };
-enum {
+const char* const deviceClass_name[] = { 0, "current", "energy", "power", "voltage", "frequency", "temperature", "power_factor", "reactive_power", "connectivity", "duration", "signal_strength", "temperature", "restart" };
+
+enum StateClassType {
     STATE_CLS_NONE = 0,
     STATE_CLS_MEASUREMENT,
     STATE_CLS_TOTAL_INCREASING
 };
-const char* const stateClasses[] = { 0, "measurement", "total_increasing" };
+const char* const stateClass_name[] = { 0, "measurement", "total_increasing" };
+
+enum CategoryType {
+    CATEGORY_NONE = 0,
+    CATEGORY_CONFIG,
+    CATEGORY_DIAGNOSTIC
+};
+const char* const category_name[] = { 0, "config", "diagnostic" };
+
 
 typedef struct {
     FieldId_t fieldId; // field id
-    uint8_t deviceClsId; // device class
-    uint8_t stateClsId; // state class
+    DeviceClassType deviceClsId; // device class
+    StateClassType stateClsId; // state class
 } byteAssign_fieldDeviceClass_t;
 
 const byteAssign_fieldDeviceClass_t deviceFieldAssignment[] = {
@@ -61,13 +75,24 @@ class MqttHandleHassClass {
 
 private:
     void loop();
-    void publish(const String& subtopic, const String& payload);
-    void publishDtuSensor(const char* name, const char* device_class, const char* category, const char* icon, const char* unit_of_measure, const char* subTopic);
-    void publishDtuBinarySensor(const char* name, const char* device_class, const char* category, const char* payload_on, const char* payload_off, const char* subTopic = "");
-    void publishInverterField(std::shared_ptr<InverterAbstract> inv, const ChannelType_t type, const ChannelNum_t channel, const byteAssign_fieldDeviceClass_t fieldType, const bool clear = false);
-    void publishInverterButton(std::shared_ptr<InverterAbstract> inv, const char* caption, const char* icon, const char* category, const char* deviceClass, const char* subTopic, const char* payload);
-    void publishInverterNumber(std::shared_ptr<InverterAbstract> inv, const char* caption, const char* icon, const char* category, const char* commandTopic, const char* stateTopic, const char* unitOfMeasure, const int16_t min = 1, const int16_t max = 100, float step = 1.0);
-    void publishInverterBinarySensor(std::shared_ptr<InverterAbstract> inv, const char* caption, const char* subTopic, const char* payload_on, const char* payload_off);
+    static void publish(const String& subtopic, const String& payload);
+    static void publish(const String& subtopic, const JsonDocument& doc);
+
+    static void addCommonMetadata(JsonDocument& doc, const String& unit_of_measure, const String& icon, const DeviceClassType device_class, const CategoryType category);
+
+    // Binary Sensor
+    static void publishBinarySensor(JsonDocument& doc, const String& root_device, const String& unique_id_prefix, const String& name, const String& state_topic, const String& payload_on, const String& payload_off, const DeviceClassType device_class, const CategoryType category);
+    static void publishDtuBinarySensor(const String& name, const String& state_topic, const String& payload_on, const String& payload_off, const DeviceClassType device_class, const CategoryType category);
+    static void publishInverterBinarySensor(std::shared_ptr<InverterAbstract> inv, const String& name, const String& state_topic, const String& payload_on, const String& payload_off, const DeviceClassType device_class, const CategoryType category);
+
+    // Sensor
+    static void publishSensor(JsonDocument& doc, const String& root_device, const String& unique_id_prefix, const String& name, const String& state_topic, const String& unit_of_measure, const String& icon, const DeviceClassType device_class, const CategoryType category);
+    static void publishDtuSensor(const String& name, const String& state_topic, const String& unit_of_measure, const String& icon, const DeviceClassType device_class, const CategoryType category);
+    static void publishInverterSensor(std::shared_ptr<InverterAbstract> inv, const String& name, const String& state_topic, const String& unit_of_measure, const String& icon, const DeviceClassType device_class, const CategoryType category);
+
+    static void publishInverterField(std::shared_ptr<InverterAbstract> inv, const ChannelType_t type, const ChannelNum_t channel, const byteAssign_fieldDeviceClass_t fieldType, const bool clear = false);
+    static void publishInverterButton(std::shared_ptr<InverterAbstract> inv, const String& name, const String& state_topic, const String& payload, const String& icon, const DeviceClassType device_class, const CategoryType category);
+    static void publishInverterNumber(std::shared_ptr<InverterAbstract> inv, const String& name, const String& state_topic, const String& command_topic, const int16_t min, const int16_t max, float step, const String& unit_of_measure, const String& icon, const CategoryType category);
 
     static void createInverterInfo(JsonDocument& doc, std::shared_ptr<InverterAbstract> inv);
     static void createDtuInfo(JsonDocument& doc);

diff --git a/include/MqttHandleInverter.h b/include/MqttHandleInverter.h
@@ -5,6 +5,8 @@
 #include <Hoymiles.h>
 #include <TaskSchedulerDeclarations.h>
 #include <espMqttClient.h>
+#include <frozen/map.h>
+#include <frozen/string.h>
 
 class MqttHandleInverterClass {
 public:
@@ -19,7 +21,6 @@ class MqttHandleInverterClass {
 private:
     void loop();
     void publishField(std::shared_ptr<InverterAbstract> inv, const ChannelType_t type, const ChannelNum_t channel, const FieldId_t fieldId);
-    void onMqttMessage(const espMqttClientTypes::MessageProperties& properties, const char* topic, const uint8_t* payload, const size_t len, const size_t index, const size_t total);
 
     Task _loopTask;
 
@@ -41,6 +42,29 @@ class MqttHandleInverterClass {
         FLD_IRR,
         FLD_Q
     };
+
+    enum class Topic : unsigned {
+        LimitPersistentRelative,
+        LimitPersistentAbsolute,
+        LimitNonPersistentRelative,
+        LimitNonPersistentAbsolute,
+        Power,
+        Restart,
+        ResetRfStats,
+    };
+
+    static constexpr frozen::string _cmdtopic = "+/cmd/";
+    static constexpr frozen::map<frozen::string, Topic, 7> _subscriptions = {
+        { "limit_persistent_relative", Topic::LimitPersistentRelative },
+        { "limit_persistent_absolute", Topic::LimitPersistentAbsolute },
+        { "limit_nonpersistent_relative", Topic::LimitNonPersistentRelative },
+        { "limit_nonpersistent_absolute", Topic::LimitNonPersistentAbsolute },
+        { "power", Topic::Power },
+        { "restart", Topic::Restart },
+        { "reset_rf_stats", Topic::ResetRfStats },
+    };
+
+    void onMqttMessage(Topic t, const espMqttClientTypes::MessageProperties& properties, const char* topic, const uint8_t* payload, const size_t len, const size_t index, const size_t total);
 };
 
 extern MqttHandleInverterClass MqttHandleInverter;
diff --git a/include/WebApi_errors.h b/include/WebApi_errors.h
@@ -32,6 +32,7 @@ enum WebApiError {
     InverterChanged,
     InverterDeleted,
     InverterOrdered,
+    InverterStatsResetted,
 
     LimitBase = 5000,
     LimitSerialZero,

diff --git a/include/WebApi_inverter.h b/include/WebApi_inverter.h
@@ -14,4 +14,5 @@ class WebApiInverterClass {
     void onInverterEdit(AsyncWebServerRequest* request);
     void onInverterDelete(AsyncWebServerRequest* request);
     void onInverterOrder(AsyncWebServerRequest* request);
+    void onInverterStatReset(AsyncWebServerRequest* request);
 };
diff --git a/lib/Hoymiles/src/Hoymiles.cpp b/lib/Hoymiles/src/Hoymiles.cpp
@@ -136,16 +136,7 @@ void HoymilesClass::loop()
             if (currentWeekDay != lastWeekDay) {
 
                 for (auto& inv : _inverters) {
-                    // Have to reset the offets first, otherwise it will
-                    // Substract the offset from zero which leads to a high value
-                    inv->Statistics()->resetYieldDayCorrection();
-                    if (inv->getZeroYieldDayOnMidnight()) {
-                        inv->Statistics()->zeroDailyData();
-                    }
-                    if (inv->getClearEventlogOnMidnight()) {
-                        inv->EventLog()->clearBuffer();
-                    }
-                    inv->resetRadioStats();
+                    inv->performDailyTask();
                 }
 
                 lastWeekDay = currentWeekDay;
@@ -204,9 +195,9 @@ std::shared_ptr<InverterAbstract> HoymilesClass::getInverterByPos(const uint8_t
 
 std::shared_ptr<InverterAbstract> HoymilesClass::getInverterBySerial(const uint64_t serial)
 {
-    for (uint8_t i = 0; i < _inverters.size(); i++) {
-        if (_inverters[i]->serial() == serial) {
-            return _inverters[i];
+    for (auto& inv : _inverters) {
+        if (inv->serial() == serial) {
+            return inv;
         }
     }
     return nullptr;
@@ -218,9 +209,7 @@ std::shared_ptr<InverterAbstract> HoymilesClass::getInverterByFragment(const fra
         return nullptr;
     }
 
-    std::shared_ptr<InverterAbstract> inv;
-    for (uint8_t i = 0; i < _inverters.size(); i++) {
-        inv = _inverters[i];
+    for (auto& inv : _inverters) {
         serial_u p;
         p.u64 = inv->serial();
 

diff --git a/lib/Hoymiles/src/HoymilesRadio.cpp b/lib/Hoymiles/src/HoymilesRadio.cpp
@@ -67,23 +67,29 @@ void HoymilesRadio::handleReceivedPackage()
             } else if (verifyResult == FRAGMENT_ALL_MISSING_TIMEOUT) {
                 Hoymiles.getMessageOutput()->println("Nothing received, resend count exeeded");
                 // Statistics: Count RX Fail No Answer
-                inv->RadioStats.RxFailNoAnswer++;
+                if (inv->RadioStats.TxRequestData > 0) {
+                    inv->RadioStats.RxFailNoAnswer++;
+                }
 
                 _commandQueue.pop();
                 _busyFlag = false;
 
             } else if (verifyResult == FRAGMENT_RETRANSMIT_TIMEOUT) {
                 Hoymiles.getMessageOutput()->println("Retransmit timeout");
                 // Statistics: Count RX Fail Partial Answer
-                inv->RadioStats.RxFailPartialAnswer++;
+                if (inv->RadioStats.TxRequestData > 0) {
+                    inv->RadioStats.RxFailPartialAnswer++;
+                }
 
                 _commandQueue.pop();
                 _busyFlag = false;
 
             } else if (verifyResult == FRAGMENT_HANDLE_ERROR) {
                 Hoymiles.getMessageOutput()->println("Packet handling error");
                 // Statistics: Count RX Fail Corrupt Data
-                inv->RadioStats.RxFailCorruptData++;
+                if (inv->RadioStats.TxRequestData > 0) {
+                    inv->RadioStats.RxFailCorruptData++;
+                }
 
                 _commandQueue.pop();
                 _busyFlag = false;
@@ -101,7 +107,9 @@ void HoymilesRadio::handleReceivedPackage()
                 // Successful received all packages
                 Hoymiles.getMessageOutput()->println("Success");
                 // Statistics: Count RX Success
-                inv->RadioStats.RxSuccess++;
+                if (inv->RadioStats.TxRequestData > 0) {
+                    inv->RadioStats.RxSuccess++;
+                }
 
                 _commandQueue.pop();
                 _busyFlag = false;

diff --git a/lib/Hoymiles/src/inverters/InverterAbstract.cpp b/lib/Hoymiles/src/inverters/InverterAbstract.cpp
@@ -273,6 +273,20 @@ uint8_t InverterAbstract::verifyAllFragments(CommandAbstract& cmd)
     return FRAGMENT_OK;
 }
 
+void InverterAbstract::performDailyTask()
+{
+    // Have to reset the offets first, otherwise it will
+    // Substract the offset from zero which leads to a high value
+    Statistics()->resetYieldDayCorrection();
+    if (getZeroYieldDayOnMidnight()) {
+        Statistics()->zeroDailyData();
+    }
+    if (getClearEventlogOnMidnight()) {
+        EventLog()->clearBuffer();
+    }
+    resetRadioStats();
+}
+
 void InverterAbstract::resetRadioStats()
 {
     RadioStats = {};

diff --git a/lib/Hoymiles/src/inverters/InverterAbstract.h b/lib/Hoymiles/src/inverters/InverterAbstract.h
@@ -65,6 +65,8 @@ class InverterAbstract {
     void addRxFragment(const uint8_t fragment[], const uint8_t len);
     uint8_t verifyAllFragments(CommandAbstract& cmd);
 
+    void performDailyTask();
+
     void resetRadioStats();
 
     struct {

diff --git a/pio-scripts/create_factory_bin.py b/pio-scripts/create_factory_bin.py
@@ -18,20 +18,64 @@
 
 Import("env")
 
+env = DefaultEnvironment()
 platform = env.PioPlatform()
 
 import sys
-from os.path import join, getsize
+import csv
+import subprocess
+import shutil
+from os.path import join, getsize, exists, isdir
+from os import listdir
 
 sys.path.append(join(platform.get_package_dir("tool-esptoolpy")))
 import esptool
 
+def esp32_build_filesystem(fs_name, fs_size):
+    filesystem_dir = env.subst("$PROJECT_DATA_DIR")
+    print("Creating %dKiB filesystem with content:" % (int(fs_size, 0)/1024) )
+    if not isdir(filesystem_dir) or not listdir(filesystem_dir):
+        print("No files added -> will NOT create littlefs.bin and NOT overwrite fs partition!")
+        return False
+    # this does not work on GitHub, results in 'mklittlefs: No such file or directory'
+    tool =  shutil.which(env.subst(env["MKFSTOOL"]))
+    if tool is None or not exists(tool):
+        print("Using fallback mklittlefs")
+        tool = "~/.platformio/packages/tool-mklittlefs/mklittlefs"
+
+    cmd = (tool, "-c", filesystem_dir, "-s", fs_size, fs_name)
+    returncode = subprocess.call(cmd, shell=False)
+    print("Return Code:", returncode)
+    return True
+
 def esp32_create_combined_bin(source, target, env):
     print("Generating combined binary for serial flashing")
 
     # The offset from begin of the file where the app0 partition starts
     # This is defined in the partition .csv file
     app_offset = 0x10000
+    fs_offset = -1
+    fs_name = env.subst("$BUILD_DIR/littlefs.bin")
+
+    with open(env.BoardConfig().get("build.partitions")) as csv_file:
+        print("Read partitions from ", env.BoardConfig().get("build.partitions"))
+        csv_reader = csv.reader(csv_file, delimiter=',')
+        line_count = 0
+        for row in csv_reader:
+            if line_count == 0:
+                print(f'{",  ".join(row)}')
+                line_count += 1
+            else:
+                if (len(row) < 4):
+                    continue
+                print(f'{row[0]}   {row[1]}   {row[2]}   {row[3]}   {row[4]}')
+                line_count += 1
+                if(row[0] == 'app0'):
+                    app_offset = int(row[3], base=16)
+                elif(row[0] == 'spiffs'):
+                    partition_size = row[4]
+                    if esp32_build_filesystem(fs_name, partition_size):
+                        fs_offset = int(row[3], base=16)
 
     new_file_name = env.subst("$BUILD_DIR/${PROGNAME}.factory.bin")
     sections = env.subst(env.get("FLASH_EXTRA_IMAGES"))
@@ -77,9 +121,13 @@ def esp32_create_combined_bin(source, target, env):
     print(f" - {hex(app_offset)} | {firmware_name}")
     cmd += [hex(app_offset), firmware_name]
 
+    if fs_offset != -1:
+        print(f" - {hex(fs_offset)} | {fs_name}")
+        cmd += [hex(fs_offset), fs_name]
+
     print('Using esptool.py arguments: %s' % ' '.join(cmd))
 
     esptool.main(cmd)
 
 
-env.AddPostAction("$BUILD_DIR/${PROGNAME}.bin", esp32_create_combined_bin)
+env.AddPostAction("$BUILD_DIR/${PROGNAME}.bin", esp32_create_combined_bin)
diff --git a/platformio.ini b/platformio.ini
@@ -20,6 +20,8 @@ custom_ci_action = generic_esp32_4mb_no_ota,generic_esp32_8mb,generic_esp32s3,ge
 
 framework = arduino
 platform = [email protected]
+platform_packages =
+    platformio/tool-mklittlefs
 
 build_flags =
     -DPIOENV=\"$PIOENV\"