From c241d8c7b6ea0bb8bd8db0344cfa30a54ecf63f9 Mon Sep 17 00:00:00 2001 From: fboundy Date: Sat, 9 Dec 2023 11:46:17 +0000 Subject: [PATCH 1/8] Add daily total id's --- apps/pv_opt/inverters.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/pv_opt/inverters.py b/apps/pv_opt/inverters.py index f81a03a..6afdf60 100644 --- a/apps/pv_opt/inverters.py +++ b/apps/pv_opt/inverters.py @@ -33,6 +33,9 @@ "maximum_dod_percent": "number.solis_battery_minimum_soc", "id_battery_soc": "sensor.solis_battery_soc", "id_consumption": ["sensor.solis_house_load", "sensor.solis_bypass_load"], + "id_consumption_today": "sensor.solis_house_load_today", + "id_grid_import_today": "sensor.solis_grid_import_today", + "id_grid_export_today": "sensor.solis_grid_export_today", "id_grid_import_power": "sensor.solis_grid_import_power", "id_grid_export_power": "sensor.solis_grid_export_power", "id_battery_charge_power": "sensor.solis_battery_input_energy", From 7361e5292573d9d226fa16feb4c7f5316925796c Mon Sep 17 00:00:00 2001 From: fboundy Date: Sat, 9 Dec 2023 11:54:57 +0000 Subject: [PATCH 2/8] Add import and export to grid history if available --- apps/pv_opt/pv_opt.py | 67 +++++++++++++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/apps/pv_opt/pv_opt.py b/apps/pv_opt/pv_opt.py index 1562ea2..24b7419 100644 --- a/apps/pv_opt/pv_opt.py +++ b/apps/pv_opt/pv_opt.py @@ -303,28 +303,58 @@ def _cost_today(self): freq="30T", ) if ( + "id_grid_import_today" in self.config + and "id_grid_export_today" in self.config + ): + grid = ( + pd.concat( + [ + self.hass2df(self.config["id_grid_import_total"], days=1) + .astype(float) + .resample("30T") + .last() + .reindex(index) + .fillna(0) + .reindex(index), + self.hass2df(self.config["id_grid_export_total"], days=1) + .astype(float) + .resample("30T") + .last() + .reindex(index) + .fillna(0), + ], + axis=1, + ) + .set_axis(["grid_import", "grid_export"], axis=1) + .loc[pd.Timestamp.now(tz="UTC").normalize() :] + ) + + elif ( "id_grid_import_power" in self.config and "id_grid_export_power" in self.config ): grid = ( - ( - self.hass2df(self.config["id_grid_import_power"], days=1) - .astype(float) - .resample("30T") - .mean() - .reindex(index) - .fillna(0) - .reindex(index) + pd.concat( + [ + self.hass2df(self.config["id_grid_import_power"], days=1) + .astype(float) + .resample("30T") + .mean() + .reindex(index) + .fillna(0) + .reindex(index), + self.hass2df(self.config["id_grid_export_power"], days=1) + .astype(float) + .resample("30T") + .mean() + .reindex(index) + .fillna(0), + ], + axis=1, ) - - ( - self.hass2df(self.config["id_grid_export_power"], days=1) - .astype(float) - .resample("30T") - .mean() - .reindex(index) - .fillna(0) - ) - ).loc[pd.Timestamp.now(tz="UTC").normalize() :] + .set_axis(["grid_import", "grid_export"], axis=1) + .loc[pd.Timestamp.now(tz="UTC").normalize() :] + ) elif "id_grid_power" in self.config: grid = ( -( @@ -338,7 +368,8 @@ def _cost_today(self): ) ).loc[pd.Timestamp.now(tz="UTC").normalize() :] - # self.log(">>>") + self.log(">>>") + self.log(grid) cost_today = self.contract.net_cost(grid_flow=grid).sum() # self.log(cost_today) return cost_today From 0a30beedce23f95b275956d52facf4edcd319b9c Mon Sep 17 00:00:00 2001 From: fboundy Date: Sat, 9 Dec 2023 12:02:41 +0000 Subject: [PATCH 3/8] Add separate import and export to net_cost --- apps/pv_opt/pvpy.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/apps/pv_opt/pvpy.py b/apps/pv_opt/pvpy.py index a2ce2b7..d9a9788 100644 --- a/apps/pv_opt/pvpy.py +++ b/apps/pv_opt/pvpy.py @@ -436,13 +436,24 @@ def __str__(self): str += f"{tariff.__str__()}\n" return str - def net_cost(self, grid_flow, grid_col="grid"): + def net_cost(self, grid_flow, **kwargs): + grid_import = kwargs.get("grid_import", "grid_import") + grid_export = kwargs.get("grid_export", "grid_export") + grid_col = kwargs.get("grid_col", "grid") start = grid_flow.index[0] end = grid_flow.index[-1] - if isinstance(grid_flow, pd.DataFrame): - grid_flow = grid_flow[grid_col] - grid_imp = grid_flow.clip(0) - grid_exp = grid_flow.clip(upper=0) + if ( + isinstance(grid_flow, pd.DataFrame) + and (grid_export in grid_flow.columns) + and (grid_import in grid_flow.columns) + ): + grid_imp = grid_flow[grid_import] + grid_exp = grid_flow[grid_export] + else: + if isinstance(grid_flow, pd.DataFrame): + grid_flow = grid_flow[grid_col] + grid_imp = grid_flow.clip(0) + grid_exp = grid_flow.clip(upper=0) nc = self.imp.to_df(start, end)["fixed"] nc += self.imp.to_df(start, end)["unit"] * grid_imp / 2000 From 107aa461b83715bb35f0ec7752bf2ffae7135805 Mon Sep 17 00:00:00 2001 From: fboundy Date: Sat, 9 Dec 2023 14:38:24 +0000 Subject: [PATCH 4/8] History from Daily Totals --- apps/pv_opt/pv_opt.py | 47 ++++++++++++++++++++++++++----------------- apps/pv_opt/pvpy.py | 35 +------------------------------- 2 files changed, 30 insertions(+), 52 deletions(-) diff --git a/apps/pv_opt/pv_opt.py b/apps/pv_opt/pv_opt.py index 24b7419..262cf90 100644 --- a/apps/pv_opt/pv_opt.py +++ b/apps/pv_opt/pv_opt.py @@ -306,29 +306,27 @@ def _cost_today(self): "id_grid_import_today" in self.config and "id_grid_export_today" in self.config ): + cols = ["grid_import", "grid_export"] grid = ( pd.concat( [ - self.hass2df(self.config["id_grid_import_total"], days=1) + self.hass2df(self.config[f"id_{col}_today"], days=1) .astype(float) .resample("30T") - .last() + .ffill() .reindex(index) - .fillna(0) - .reindex(index), - self.hass2df(self.config["id_grid_export_total"], days=1) - .astype(float) - .resample("30T") - .last() - .reindex(index) - .fillna(0), + .ffill() + for col in cols ], axis=1, ) - .set_axis(["grid_import", "grid_export"], axis=1) + .set_axis(cols, axis=1) .loc[pd.Timestamp.now(tz="UTC").normalize() :] ) + # grid["dt"] = -grid.index.diff(-1).total_seconds() / 3600 + grid = (-grid.diff(-1).clip(upper=0) * 2000).round(0)[:-1] + self.log(grid) elif ( "id_grid_import_power" in self.config and "id_grid_export_power" in self.config @@ -355,6 +353,7 @@ def _cost_today(self): .set_axis(["grid_import", "grid_export"], axis=1) .loc[pd.Timestamp.now(tz="UTC").normalize() :] ) + elif "id_grid_power" in self.config: grid = ( -( @@ -370,8 +369,9 @@ def _cost_today(self): self.log(">>>") self.log(grid) - cost_today = self.contract.net_cost(grid_flow=grid).sum() - # self.log(cost_today) + # self.log(-grid.diff(-1).clip(upper=0) * 2000) + cost_today = self.contract.net_cost(grid_flow=grid) + self.log(cost_today) return cost_today @ad.app_lock @@ -1354,6 +1354,7 @@ def write_to_hass(self, entity, state, attributes): self.log(f"Couldn't write to entity {entity}: {e}") def write_cost(self, name, entity, cost, df): + # self.log(f">>>{cost.index[0]}") cost_today = self._cost_today() midnight = pd.Timestamp.now(tz="UTC").normalize() + pd.Timedelta("24H") # self.log( @@ -1366,24 +1367,34 @@ def write_cost(self, name, entity, cost, df): ) cols = ["soc", "forced", "import", "export", "grid", "consumption"] + cost = pd.DataFrame(pd.concat([cost_today, cost])).set_axis(["cost"], axis=1) + # cost = pd.DataFrame(cost.groupby(cost.index).sum()) + cost["cumulative_cost"] = cost["cost"].cumsum() + + for d in [df, cost]: + d["period_start"] = ( + d.index.tz_convert(self.tz).strftime("%Y-%m-%dT%H:%M:%S%z").str[:-2] + + ":00" + ) + self.write_to_hass( entity=entity, - state=round((cost.sum() + cost_today) / 100, 2), + state=round((cost["cost"].sum()) / 100, 2), attributes={ "friendly_name": name, "unit_of_measurement": "GBP", "cost_today": round( - (cost.loc[: midnight - pd.Timedelta("30T")].sum() + cost_today) - / 100, + (cost["cost"].loc[: midnight - pd.Timedelta("30T")].sum()) / 100, 2, ), - "cost_tomorrow": round((cost.loc[midnight:].sum()) / 100, 2), + "cost_tomorrow": round((cost["cost"].loc[midnight:].sum()) / 100, 2), } | { col: df[["period_start", col]].to_dict("records") for col in cols if col in df.columns - }, + } + | {"cost": cost[["period_start", "cumulative_cost"]].to_dict("records")}, ) def _write_output(self): diff --git a/apps/pv_opt/pvpy.py b/apps/pv_opt/pvpy.py index d9a9788..5b8f98e 100644 --- a/apps/pv_opt/pvpy.py +++ b/apps/pv_opt/pvpy.py @@ -160,28 +160,15 @@ def to_df(self, start=None, end=None): if "AGILE" in self.name: if self.day_ahead is not None and df.index[-1].day == end.day: # reset the day ahead forecasts if we've got a forecast going into tomorrow - # self.log(">>>Resetting day ahead prices") self.day_ahead = None if pd.Timestamp.now().hour > 11 and df.index[-1].day != end.day: # if it is after 11 but we don't have new Agile prices yet, check for a day-ahead forecast - # self.log(">>>Checking for day ahead prices") if self.day_ahead is None: self.day_ahead = self.get_day_ahead(df.index[0]) - # if self.day_ahead is not None: - # str_log = f">>>Downloaded Day Ahead prices OK. " - # str_log += f"Date range: {self.day_ahead.index[0].strftime(TIME_FORMAT)}" - # str_log += ( - # f" - {self.day_ahead.index[-1].strftime(TIME_FORMAT)}" - # ) - # self.log(str_log) - # else: - # self.log(">>>Failed to get Day Ahead prices OK") if self.day_ahead is not None: self.day_ahead = self.day_ahead.sort_index() - # str_log = ">>>Predicting Agile prices from Day Ahead: " - # str_log += f"End before: {df.index[-1].strftime(TIME_FORMAT)} " mask = (self.day_ahead.index.hour >= 16) & ( self.day_ahead.index.hour < 19 ) @@ -196,27 +183,8 @@ def to_df(self, start=None, end=None): .loc[df.index[-1] :] .iloc[1:] ) - # agile = self.day_ahead.loc[df.index[-1:]].iloc[1:] - - # agile = self.day_ahead.loc[df.index[-1:]].iloc[1:].copy() - # mask = (agile.index.hour >= 16) & (agile.index.hour < 19) - # agile[mask] = agile[mask] * 0.186 + 16.5 - # agile[~mask] = agile[~mask] * 0.229 - 0.6 - - # self.log(f">>{df.index}") - # self.log(f">>>{agile.index}") df = pd.concat([df, agile]) - # str_log += f"End after: {df.index[-1].strftime(TIME_FORMAT)} " - # self.log(str_log) - # else: - # # str_log = ">>> No Day Ahead checks: " - # if self.day_ahead is None: - # str_log += "self.day_ahead is NULL, " - # if pd.Timestamp.now().hour > 11: - # str_log += "hour > 11, " - # str_log += f"Index end day: {df.index[-1].day} End day: {end.day}" - # self.log(str_log) # If the index frequency >30 minutes so we need to just extend it: if ( @@ -313,7 +281,6 @@ def get_day_ahead(self, start): price.index = price.index.tz_localize("CET") price.index = price.index.tz_convert("UTC") price = price[~price.index.duplicated()] - # self.log(f">>> Duplicates: {price.index.has_duplicates}") return price.resample("30T").ffill().loc[start:] @@ -452,6 +419,7 @@ def net_cost(self, grid_flow, **kwargs): else: if isinstance(grid_flow, pd.DataFrame): grid_flow = grid_flow[grid_col] + grid_imp = grid_flow.clip(0) grid_exp = grid_flow.clip(upper=0) @@ -738,7 +706,6 @@ def optimised_force(self, initial_soc, static_flows, contract: Contract, **kwarg else: available[max_slot] = False else: - # self.log(f">>>{str_log}") done = True else: done = True From 2ccb885524b55f2d1a706600765350d6552e1031 Mon Sep 17 00:00:00 2001 From: fboundy Date: Sat, 9 Dec 2023 16:02:18 +0000 Subject: [PATCH 5/8] Add delayed SOC --- apps/pv_opt/pv_opt.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/apps/pv_opt/pv_opt.py b/apps/pv_opt/pv_opt.py index 262cf90..4c099f7 100644 --- a/apps/pv_opt/pv_opt.py +++ b/apps/pv_opt/pv_opt.py @@ -917,6 +917,7 @@ def _state_from_value(self, value): state = value return state + def _expose_configs(self): # for defaults in [DEFAULT_CONFIG, self.inverter.config, self.inverter.brand_config]: for defaults in [DEFAULT_CONFIG]: @@ -1344,6 +1345,21 @@ def _log_inverter_status(self, status): self.log("") def write_to_hass(self, entity, state, attributes): + if not self.entity_exists(entity_id=entity): + self.log( + f"Creating HA Entity {entity}" + ) + id = entity.replace("sensor.", "") + conf = { + "state_topic": f"homeassistant/sensor/{id}/state", + "command_topic": f"homeassistant/domain/{id}/set", + "optimistic": True, + "unique_id": id, + } + + conf_topic = f"homeassistant/self/{id}/config" + self.mqtt.mqtt_publish(conf_topic, dumps(conf), retain=True) + try: self.my_entity = self.get_entity(entity) self.my_entity.set_state(state=state, attributes=attributes) @@ -1446,6 +1462,20 @@ def _write_output(self): }, ) + for offset in [1,4,8,12]: + loc = pd.Timestamp.now()+pd.Timedelta(f"{offset}H") + locs = [loc.floor("30T"), loc.ceil("30T")] + socs = [self.opt.loc[l]'soc' for l in locs] + soc = (loc-locs[0])/(locs[1]-locs[0]) * (socs[1]-socs[0])+socs[0] + entity_id = f"sensor.{self.prefic}_soc_h{offset}" + attributes={ + "friendly_name": f"PV Opt Predicted SOC ({offset} hour delay)", + "unit_of_measurement": "%", + "state_class": "measurement", + "device_class": "battery", + } + self.write_to_hass(entity=entity_id, state=soc, attributes=attributes) + def load_solcast(self): if self.debug: self.log("Getting Solcast data") From de431998e0783a4e5e4dae8b77264ceddeaa87a6 Mon Sep 17 00:00:00 2001 From: fboundy Date: Mon, 11 Dec 2023 12:51:37 +0000 Subject: [PATCH 6/8] Track Optimiser Performance --- apps/pv_opt/pv_opt.py | 40 +++++++++++++++------------------------- 1 file changed, 15 insertions(+), 25 deletions(-) diff --git a/apps/pv_opt/pv_opt.py b/apps/pv_opt/pv_opt.py index 4c099f7..ea46c15 100644 --- a/apps/pv_opt/pv_opt.py +++ b/apps/pv_opt/pv_opt.py @@ -323,10 +323,8 @@ def _cost_today(self): .set_axis(cols, axis=1) .loc[pd.Timestamp.now(tz="UTC").normalize() :] ) - # grid["dt"] = -grid.index.diff(-1).total_seconds() / 3600 - grid = (-grid.diff(-1).clip(upper=0) * 2000).round(0)[:-1] - self.log(grid) + elif ( "id_grid_import_power" in self.config and "id_grid_export_power" in self.config @@ -367,11 +365,7 @@ def _cost_today(self): ) ).loc[pd.Timestamp.now(tz="UTC").normalize() :] - self.log(">>>") - self.log(grid) - # self.log(-grid.diff(-1).clip(upper=0) * 2000) cost_today = self.contract.net_cost(grid_flow=grid) - self.log(cost_today) return cost_today @ad.app_lock @@ -917,7 +911,6 @@ def _state_from_value(self, value): state = value return state - def _expose_configs(self): # for defaults in [DEFAULT_CONFIG, self.inverter.config, self.inverter.brand_config]: for defaults in [DEFAULT_CONFIG]: @@ -1184,11 +1177,15 @@ def optimise(self): f"Plan time: {self.static.index[0].strftime('%d-%b %H:%M')} - {self.static.index[-1].strftime('%d-%b %H:%M')} Initial SOC: {self.initial_soc} Base Cost: {self.base_cost.sum():5.2f} Opt Cost: {self.opt_cost.sum():5.2f}" ) self.log("") - self.log( - f"Optimiser elapsed time {(pd.Timestamp.now()- self.t0).total_seconds():0.2f} seconds" - ) + optimiser_elapsed = round((pd.Timestamp.now() - self.t0).total_seconds(), 1) + self.log(f"Optimiser elapsed time {optimiser_elapsed:0.1f} seconds") self.log("") self.log("") + self.write_to_hass( + entity=f"sensor.{self.prefix}_optimiser_elapsed", + state=optimiser_elapsed, + attributes={"state_class": "measurement"}, + ) self._status("Writing to HA") self._write_output() @@ -1346,9 +1343,7 @@ def _log_inverter_status(self, status): def write_to_hass(self, entity, state, attributes): if not self.entity_exists(entity_id=entity): - self.log( - f"Creating HA Entity {entity}" - ) + self.log(f"Creating HA Entity {entity}") id = entity.replace("sensor.", "") conf = { "state_topic": f"homeassistant/sensor/{id}/state", @@ -1370,12 +1365,8 @@ def write_to_hass(self, entity, state, attributes): self.log(f"Couldn't write to entity {entity}: {e}") def write_cost(self, name, entity, cost, df): - # self.log(f">>>{cost.index[0]}") cost_today = self._cost_today() midnight = pd.Timestamp.now(tz="UTC").normalize() + pd.Timedelta("24H") - # self.log( - # f">>> {cost.loc[:midnight].sum():0.0f} {cost.loc[midnight:].sum():0.0f} {cost.sum():0.0f} {cost.loc[midnight]:0.0f}" - # ) df = df.fillna(0).round(2) df["period_start"] = ( df.index.tz_convert(self.tz).strftime("%Y-%m-%dT%H:%M:%S%z").str[:-2] @@ -1384,7 +1375,6 @@ def write_cost(self, name, entity, cost, df): cols = ["soc", "forced", "import", "export", "grid", "consumption"] cost = pd.DataFrame(pd.concat([cost_today, cost])).set_axis(["cost"], axis=1) - # cost = pd.DataFrame(cost.groupby(cost.index).sum()) cost["cumulative_cost"] = cost["cost"].cumsum() for d in [df, cost]: @@ -1462,13 +1452,13 @@ def _write_output(self): }, ) - for offset in [1,4,8,12]: - loc = pd.Timestamp.now()+pd.Timedelta(f"{offset}H") + for offset in [1, 4, 8, 12]: + loc = pd.Timestamp.now(tz="UTC") + pd.Timedelta(f"{offset}H") locs = [loc.floor("30T"), loc.ceil("30T")] - socs = [self.opt.loc[l]'soc' for l in locs] - soc = (loc-locs[0])/(locs[1]-locs[0]) * (socs[1]-socs[0])+socs[0] - entity_id = f"sensor.{self.prefic}_soc_h{offset}" - attributes={ + socs = [self.opt.loc[l]["soc"] for l in locs] + soc = (loc - locs[0]) / (locs[1] - locs[0]) * (socs[1] - socs[0]) + socs[0] + entity_id = f"sensor.{self.prefix}_soc_h{offset}" + attributes = { "friendly_name": f"PV Opt Predicted SOC ({offset} hour delay)", "unit_of_measurement": "%", "state_class": "measurement", From 5b44163a3416ae51a6fc680878ed83ad9cc2db70 Mon Sep 17 00:00:00 2001 From: fboundy Date: Thu, 14 Dec 2023 08:20:59 +0000 Subject: [PATCH 7/8] FIx bug in grid calculation --- apps/pv_opt/pv_opt.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/pv_opt/pv_opt.py b/apps/pv_opt/pv_opt.py index ea46c15..477179d 100644 --- a/apps/pv_opt/pv_opt.py +++ b/apps/pv_opt/pv_opt.py @@ -323,7 +323,7 @@ def _cost_today(self): .set_axis(cols, axis=1) .loc[pd.Timestamp.now(tz="UTC").normalize() :] ) - grid = (-grid.diff(-1).clip(upper=0) * 2000).round(0)[:-1] + grid = (-grid.diff(-1).clip(upper=0) * 2000).round(0)[:-1].fillna(0) elif ( "id_grid_import_power" in self.config @@ -365,6 +365,7 @@ def _cost_today(self): ) ).loc[pd.Timestamp.now(tz="UTC").normalize() :] + # self.log(grid) cost_today = self.contract.net_cost(grid_flow=grid) return cost_today From 94ab0035de4f1f3b90891b8b0831cdd2b5dac9fa Mon Sep 17 00:00:00 2001 From: fboundy Date: Thu, 14 Dec 2023 08:25:56 +0000 Subject: [PATCH 8/8] Uprev --- apps/pv_opt/pv_opt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/pv_opt/pv_opt.py b/apps/pv_opt/pv_opt.py index 477179d..76b8f3a 100644 --- a/apps/pv_opt/pv_opt.py +++ b/apps/pv_opt/pv_opt.py @@ -23,7 +23,7 @@ # USE_TARIFF = True -VERSION = "3.2.0" +VERSION = "3.2.1" DATE_TIME_FORMAT_LONG = "%Y-%m-%d %H:%M:%S%z" DATE_TIME_FORMAT_SHORT = "%d-%b %H:%M"