diff --git a/doc/rtd/content/04_extensions/sub/01_extensions_org/org/fhswf/MLPro-Int-scikit-learn.rst b/doc/rtd/content/04_extensions/sub/01_extensions_org/org/fhswf/MLPro-Int-scikit-learn.rst index 450b0c8d8..efb78789f 100644 --- a/doc/rtd/content/04_extensions/sub/01_extensions_org/org/fhswf/MLPro-Int-scikit-learn.rst +++ b/doc/rtd/content/04_extensions/sub/01_extensions_org/org/fhswf/MLPro-Int-scikit-learn.rst @@ -10,9 +10,9 @@ MLPro: Integration scikit-learn * - **Topics** - machine-learning, mlpro-extension, online-machine-learning, scikit-learn, wrapper * - **Version** - - v0.1.2 - Alignment with MLPro 1.4.3 + - v0.2.0 - Alignment with MLPro 1.9.4 * - **Last update** - - Thu, 09 May 2024 06:55:53 GMT + - Mon, 02 Dec 2024 16:37:07 GMT * - **URL (GitHub)** - https://github.com/fhswf/MLPro-Int-scikit-learn * - **URL** diff --git a/doc/rtd/content/99_appendices/appendix2/sub/pool/mlpro_oa/streams/tasks/anomaly_predictors/images/MLPro-OA_Anomaly_Predictors_class_diagram.drawio b/doc/rtd/content/99_appendices/appendix2/sub/pool/mlpro_oa/streams/tasks/anomaly_predictors/images/MLPro-OA_Anomaly_Predictors_class_diagram.drawio new file mode 100644 index 000000000..2f75f0f69 --- /dev/null +++ b/doc/rtd/content/99_appendices/appendix2/sub/pool/mlpro_oa/streams/tasks/anomaly_predictors/images/MLPro-OA_Anomaly_Predictors_class_diagram.drawio @@ -0,0 +1 @@ +7R1rd6I49LfMh57t7Dn18BY/aq07nelra2dn2i+eCEGZIlDAqvPrN+GhPCLGCsoo7p4pBAjJfd+bm8sZfzmZ/+MAe3xrqdA44xh1fsZ3zziOZwQJ/cEti6CF5Rk2aBk5uhq2rRr6+m8YNjJh61RXoZu40bMsw9PtZKNimSZUvEQbcBxrlrxNs4zkW20wgpmGvgKMbOsPXfXGQSvH89Lqwheoj8bRqyVRCK5MQHR3OBV3DFRrFmvir874S8eyvOBoMr+EBgZfBJgf14sfxs2r9M/Xf9038L3z7enuv4ugs942jyzn4EDT+3DXmvL1vyH/5fFq3LzrK+/tyRy+hY8w78CYhgAL5+otIgiOHGtqU44gHOk7dDw4J+EXDKNuVxBExAetCfScBbovfEoIB7ZIns5WKBRaYds4hr0lrkBINqNlzyvIoIMQOFsAiiUASjLQazuahUCCJ2hYjn9FeptikuicIf7xf/Emxo1YJGpj45elUfjX73qYwER0F37hRdBNG93ACvY828UNHEFT9e8I+3LSvSM4DAltwXxSA8k8TNOyCTKaloTMpplKpInmzo80F9cGJvFt4Tjxm5zR8ByhDf2PqIWJHX1eP4BwthtnQUQX+gsmNjowh64dgCPbtG5aiduu8eRsxxo50HWpui20yX894hJdBZ5umXsewCME6sJXMeifqQtjMAvQvh2BJll3Haa2pL/tuKIIPky0JAYPPM/Rh1MP+rDsYa0HvbGlnn8uEkt3cIZpcjo0dAXL5tVLe8HrIowN8Tj0iW3ACdIzUN1aKhWJ0A+ij/SKXDDHhFL0JmY93RaDFB8jPd0cQ0f34bwJO2OAJ627afwg7evA9fgolgUSUxhkgFooGklDGGzHHCHZO5aHDB8fWBE8e/GxV4fysyBlPjLlJF3Fpl8F0loDh6KkK+1gUzY2spZx+9ibIKO0y2KRYegjEx0bUMNXsEWtI6+mHTZ7Fob4bIyg3LeBgruZIQ8OtSFr3VQhNmWZEByXS1ggCEgSg9tdz7FeYeKKDxt0JfJy8Cg03TBiN2n+bzmBuDuQ612k3YGs2R837OVWoxX/Cfu087mMnd+FNrKkMXlicyaFOHSljb1UdDY0LOU1iUQDDKHxYLm6bwrxXSeYwRKbN6nrE11Vcc+bsL+8D72+pxvR6zCyQw+c5bbGElQTnvRa1+yClTi+wSIHLPrJCV/tgmPlrLsmsg1GYKXVj4BVMQep4XAeLN3n74iamAbHyAzHt2T/v1ZiJCzHNBiGZ1hRbqJ/pSh0EfXvWlNHgWGXcc859RZRRATJyCLHNfF/bFPY6jUecEbQ2/yaCI7Rc5amuTDxCCI1sIjdYeMO3PXg4ZmGFMNTqyWlB57il+AFK+5ZYv/jDMVnGOo6UA+YozTHmlSXp5iap7I8xXMNXhYkhkdsgP4VymGp/Ldsy1GLGA7y+Ksomhc2R9XcmT4xgAkjEguv8DFCVbAh5KxV/spYN9QbsLCmGNWuB5TX6Kwzthz9N+oWRFyDLjsRHXNS4o4+fjKkdgfiOMVDRN3ssukGuF54DzKtDGC7ehDJw7dMEDp0s2N5HuLmsKOEFZKwKQLTIwrGkqwMWYGKQjJShrIoiLjDkQNUHSZMm6VtEkG2ty/bJcE3/OrHbQxhMo34/TyfZd805RPY9xGHVc2RAVdDQYI/SfiElyM9RbCrpOTrgIEo0AQe7GB8umXwiriZVwzd55OYEsjSEK10R+60opujJ8xC3Qt21XLjP+gzYNjyGMKF9+nZA1HwmsnRQqFGRsMWO2di129xEJmaaMRA9wkOIlaaQdejJ0WBmhQXSTRuR2vyelLbyaSWamm4izTUNI4sDVVpKIlSVaUhyzW4Vtw4qaUhBa80a2m4iRQlalKspDSUa2m4gzRUAZQ1ojSUFBkOtYpKQyRzGnwz5ok3a2lIwSutWhpuIkWZmhQPLQ1frr+/mOqV/b1pCzeK0JpKzdmF0MpgFMdMIpGH4DO2RpYJjKtVa0qArO65sXzMYcT9gp63CGUamHpWkkQIEaZk1CcL/Lx0lyDoEcJy6kLh5fKneqe/9O+efrRa8/foviBskQONEDUbo0YONICnvydTnLaI8AhyUgZIUipKtCbA8gG+zoNHjK/bpjUBxuLBgaqueJbTRo0IGjyLsXGevvrU7+F8jEvurM3E7rNAA7ESBBO34QH31W2A4DkV4iUoy3EbQ+DqitsI+/tcJc0ritXWvBRRmU3alaCZe/5va47bwg/h2SStN5msvuMZkr5rto5IvxL5MLvYdTl4en64Qm3+KfNXyCnocMl86Ljd/SvDOuESJtW6ZVKtClm1ipss9Kxm+AQ6RhoamgRVm1SpHZylxTRErFyReBBxal10vlHfxldKVaiBqUHWwrkCjVoLiyIlEUrraZBW8c5dwx6bat+5+Lclwq9c073vE4IyA003VYQDN0hT6yJkRxI2pIazZues2T1qzMdXwsvCvczuD/fEEdch1+KRyjJNOqyWZUmLWaNqMNBN3RsMUOu5PVAMd+C52ueIqY+bi3u9krmYZSlFOJ9jRtBivHM//e4+z7riAFzr5sJmpxdXhFjhwJma53i2CPGu58ttLMqv0Ykvzjkma/LWeN8S79H+kM324+54B7MJ6Evyf9PxpD96vLh/V7+opK0IPksPgApsb6mtzyMquPQpAvlCA29hw5oCdqYAagVeAAWMbu/u5duON/71Y/Lm2frL7F+JZLz5uB9Y5gC+Q9/XD+SAfzZAnUWyAEEwoIiY37wy8ypwEI3ZGv6CykqGXYXTQuTLXMQypYeWZdQUvSNFc4xMR9FCTsh5J5mWzZgZjKA3AIoydYCyOK+tloIwLXIHll2slEHioSO+7F4ivmRohLH2gkO+2yaKsq1URFgWUptJNz3QFIvNJSVDq3qrBbvRDnGWTVraaRZNOzsxdtYdSYft4yH9+/YTcF8rFYEX2DoCv48IfJR2vlEHsdGDRxGBJzJNNhuEIgJ/usH3XMFDbQQJewzAEkdc5zUUj1Rpf/FXogeTXUur4zGbfJpiMN+kXUsrwKV5u5xoP63Ob3H+9Pjtrjm4b9ntpWFXHat0Be6UcVigmRq5LpR26qHMUja7AS1plyJhlpdt4hexuNwqy6SK6SUSUxu3JRi3eKdcwtikTi9hmaM3btl6oXmjtluKJ3pDh5LACjB0csMOeQI1Lk6HWsOP6LsNP4ZfKalYu/z7kYrULn/r+IUixQaQkxeKpbn0peWx84fxAdDYf+LnG2J0+hy/1p2HnQdni+VZafnvj9++X2m3DtNpA8mQB/P58NtP6vx3vhqrIRKXll6tDashjJD/xM7LIXlgjUmSm/7TbZX0K1/x7WR/qH4Vmily48WT9DqIXJENxJ2yfs0VHNTqlc9uGitLvRIjbNnEjyOLsBHRRAiw5ajNPQTY8kYZ47cnB5iuZjkTpFJqZXTsykhu1cpoLXNQVII6dWW0dQBsj8qIKG6zZvetbuod4CnjW2CCEZZ6q/BXteJd9SrAXuxxhnqTad6i5B8nAonsUtvjcYLLlSj04S5K8sqr/rjT3iMmg9TkZrMhloamFSU7+KcD/xsIUUK/jpcCTj2puxhqaFFSQ2k70bIaEajqQEXwxbtQ/v7bHrzOkKLAFV9POd2lGGyzXItStxSQ70JEdzZ9bXC6+MbYXmNcFIRvkdLeLQLf3AN380uePfec8fu0Z7/xXy8nJGFf47s8fMv8/vBNjicdJqFtrnuxtSx09hx2ho9XK1n4ZA8LWXkFmjamvB1sZ0beqHN2ZvgFldbXW6qUDyvWPuxetmlQh/GWzuhR+LBEiycb46YrlIT46nQ3a+TKIvpKSbRUWMBmDeKICWU2PFdb+rFhYQ185fjLI5WP7iatsVsWuuv0rOKRyjIcHVZLC9kTKmbE4lV1IKogPLO0kagC4pLEsnZZ7q1rIu0B77Tx6CJcViLes0oaRygiPd0NgtLRlrsoKh1X40/6BLmyYGLXsqAgmqDdkFlaGIOwUcHHMnR06PYsByoIOsnF2qHWwJ99b/Smpr+PoVKOb71ZoRTHN52/Qp++Ih+T30tmoc31CTBLob8BV6GDFWPVni9ZHFWyTgF5yHWhghLQeuhKBdlglhbybFiiIFljzzeHnnzb6Mk3no6YqT+yflvpegXkjJxmBomnuVmJDB3atOvD1d/KHXeMr+/bmwxeD113/eta7Hq83IFrRF/N6N+0cUETNJFKmsd1hYO9mMf0BQ6O3zzmsundafP4Pgou1DYyhTLlSquGUJqNzNVFLkpAq7y/HP/hSHl++T7k3O6bcP/yMPr9g2+TU54y1bzs6bKcl69IsY0cq+91xrdxeeo6vrwzObT2WHOfPOQDVZ4NTeblyXPSfv6Iybxbxdrc5bU/zWTma0coDzq0jhBXeNmG3bAqHwSrUfriWSx5cZnKuC59kczf3AEZnKgOaQt4sNUiBFqP+LYT94ktQPKKK+Xr1ktBe/F16ffxnYCzy+/s7KKT207t8ZJl1J+zKsTXHm8JaN3jqlCeiv+Ax1v7tzsjf59LQkTsc+uxj9OkBpPhkggmuqkHG3qDzKl08YOaHHYmB+p82dLiHcKBvtL0Z3jG0WfQDhDv8B/dtaChxGxZzzD1wM7lDHOhGt+n9nh9266S41UvMpbieKULqJxoFXUyW9Tm9kYVuxQdVayiTtxReJAY5cfVIHEOBC2Y96Hq/UcF80YdYye2EQtToENcv55Rpo4TfHIWp8QBE0GCk3zPXtXfVx+gDZqGTroFDY1wX6yJwy/1910zvqUNAj5pU3dwgZ404SiAGMdouqkiltzqcdtnz+TjzLlfrf/T53U9pch2RZSYwmZj3YN9JB3w1ZkD7CQBprWVpnFkbaVKQ0mU0uovru0pRFqg68vSWWzGqiLIFLa1rdLaibYPYrUXLFXErFTJ+wh2RaRKVkk/ZJjr06ezILEWpwkgQyvgJ1wJC9sQ+BWGjgVQdTmsNF5qZr4/m/0i9fIjEnFekspipejjNZWsOVwYs0UbNqu7spc77Bi79f1vctWVOE/bkRRTft2JFuIkM002xJ/hiBNyJPMFy59TipPNOjSJ8myEje+BN3P8G99zarUVhH3a0pulLdwIh7FSPpaX+HHjhTh3Qn4asTjAwTyFvFHH2LUDXISNMO4AloWkcKkC9yS9ATmlVZtNglYtyxsg7pk87HLY3tiMOHeCQ068TzgUm+WNOp6tNIbIxkYTGWPlBxRl6gAFD8jSlq0TS4W466kbBMPa3ZNkwLQ73op2gpfAgOjUsSwvbusiMI5vfUzwV/8DnZJBD4IgFMc/DXeE1eyaWV06eejMhIQNfQ5pUp8+TciYB1cbG9uPH+/PewPRrHYnw1p5AS40Ipg7RA+IkATv0mEbyWMiKfGgMop7aQaFeopw09O74qKLRAugrWpjWELTiNJGjBkDfazdQMepLavEAhQl00t6VdxK38UGz/wsVCVDcoL9Sc2C7EEnGYf+C9EckS0i1DFE92hk8aKZAbCrWpBrlwk9Dj/Mdco9/l/g07sRjf2tJs0Ga+X5byf6ODR/AQ== \ No newline at end of file diff --git a/doc/rtd/content/99_appendices/appendix2/sub/pool/mlpro_oa/streams/tasks/anomaly_predictors/images/MLPro-OA_Anomaly_Predictors_class_diagram.drawio.png b/doc/rtd/content/99_appendices/appendix2/sub/pool/mlpro_oa/streams/tasks/anomaly_predictors/images/MLPro-OA_Anomaly_Predictors_class_diagram.drawio.png new file mode 100644 index 000000000..28da5da27 Binary files /dev/null and b/doc/rtd/content/99_appendices/appendix2/sub/pool/mlpro_oa/streams/tasks/anomaly_predictors/images/MLPro-OA_Anomaly_Predictors_class_diagram.drawio.png differ diff --git a/doc/rtd/content/99_appendices/appendix2/sub/pool/mlpro_oa/streams/tasks/images/.$MLPro-OA_Anomaly_Predictors_class_diagram.drawio.bkp b/doc/rtd/content/99_appendices/appendix2/sub/pool/mlpro_oa/streams/tasks/images/.$MLPro-OA_Anomaly_Predictors_class_diagram.drawio.bkp new file mode 100644 index 000000000..cc0c28b5c --- /dev/null +++ b/doc/rtd/content/99_appendices/appendix2/sub/pool/mlpro_oa/streams/tasks/images/.$MLPro-OA_Anomaly_Predictors_class_diagram.drawio.bkp @@ -0,0 +1,100 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/rtd/content/99_appendices/appendix2/sub/pool/mlpro_oa/streams/tasks/images/MLPro-OA_Anomaly_Predictors_class_diagram.drawio b/doc/rtd/content/99_appendices/appendix2/sub/pool/mlpro_oa/streams/tasks/images/MLPro-OA_Anomaly_Predictors_class_diagram.drawio new file mode 100644 index 000000000..1e2287de5 --- /dev/null +++ b/doc/rtd/content/99_appendices/appendix2/sub/pool/mlpro_oa/streams/tasks/images/MLPro-OA_Anomaly_Predictors_class_diagram.drawio @@ -0,0 +1,121 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/mlpro/oa/streams/tasks/anomalypredictors/__init__.py b/src/mlpro/oa/streams/tasks/anomalypredictors/__init__.py new file mode 100644 index 000000000..563007700 --- /dev/null +++ b/src/mlpro/oa/streams/tasks/anomalypredictors/__init__.py @@ -0,0 +1 @@ +from mlpro.oa.streams.tasks.anomalypredictors.basics import AnomalyPredictor, AnomalyPrediction \ No newline at end of file diff --git a/src/mlpro/oa/streams/tasks/anomalypredictors/basics.py b/src/mlpro/oa/streams/tasks/anomalypredictors/basics.py new file mode 100644 index 000000000..e6d22135a --- /dev/null +++ b/src/mlpro/oa/streams/tasks/anomalypredictors/basics.py @@ -0,0 +1,47 @@ +## ------------------------------------------------------------------------------------------------- +## -- Project : MLPro - The integrative middleware framework for standardized machine learning +## -- Package : mlpro.oa.streams.tasks.anomalypredictors +## -- Module : basics.py +## ------------------------------------------------------------------------------------------------- +## -- History : +## -- yyyy-mm-dd Ver. Auth. Description +## -- 2024-06-04 0.0.0 DA/DS Creation +## -- 2024-08-23 0.1.0 DA/DS Creation +## -- 2024-12-02 0.2.0 DA Refactoring +## ------------------------------------------------------------------------------------------------- + +""" +Ver. 0.2.0 (2024-12-02) + +This module provides basic templates for online anomaly prediction in MLPro. + +""" + + +from mlpro.bf.events import Event +from mlpro.oa.streams import OAStreamTask + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class AnomalyPredictor (OAStreamTask): + """ + ... + """ + + pass + + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class AnomalyPrediction (Event): + """ + ... + """ + + pass diff --git a/src/mlpro/oa/streams/tasks/anomalypredictors/tsf/__init__.py.off b/src/mlpro/oa/streams/tasks/anomalypredictors/tsf/__init__.py.off new file mode 100644 index 000000000..b132f47ef --- /dev/null +++ b/src/mlpro/oa/streams/tasks/anomalypredictors/tsf/__init__.py.off @@ -0,0 +1,3 @@ +from mlpro.oa.streams.tasks.anomalypredictors.tsf.basics import TimeSeriesForcaster, OATimeSeriesForcaster, AnomalyPredictorTSF +from mlpro.oa.streams.tasks.anomalypredictors.tsf.ad_based import AnomalyPredictorAD +from mlpro.oa.streams.tasks.anomalypredictors.tsf.mini_batch_based import MiniBatchManager, StreamMiniBatchManager, OATimeSeriesForcasterMB \ No newline at end of file diff --git a/src/mlpro/oa/streams/tasks/anomalypredictors/tsf/ad_based.py b/src/mlpro/oa/streams/tasks/anomalypredictors/tsf/ad_based.py new file mode 100644 index 000000000..74e9caddf --- /dev/null +++ b/src/mlpro/oa/streams/tasks/anomalypredictors/tsf/ad_based.py @@ -0,0 +1,143 @@ +## ------------------------------------------------------------------------------------------------- +## -- Project : MLPro - The integrative middleware framework for standardized machine learning +## -- Package : mlpro.oa.streams.tasks.anomalypredictors +## -- Module : basics.py +## ------------------------------------------------------------------------------------------------- +## -- History : +## -- yyyy-mm-dd Ver. Auth. Description +## -- 2024-06-04 0.0.0 DA/DS Creation +## -- 2024-08-23 0.1.0 DA/DS Creation +## -- 2024-09-27 0.2.0 DS Creation +## ------------------------------------------------------------------------------------------------- + +""" +Ver. 0.2.0 (2024-09-27) + +This module provides basic templates for online anomaly prediction in MLPro. + +""" + + +from mlpro.bf.ml import Log, Event +from mlpro.bf.math import Function +from mlpro.bf.streams import Log, StreamTask +from mlpro.bf.various import Log +from mlpro.bf.streams import Instance, InstDict +from mlpro.oa.streams.tasks.anomalypredictors.tsf.basics import AnomalyPredictorTSF +from mlpro.oa.streams.tasks.anomalydetectors.basics import Anomaly + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class AnomalyPredictorAD (AnomalyPredictorTSF, Anomaly): + """ + Parameters + ----------- + p_name : str + Optional name of the task. Default is None. + p_range_max : int + Maximum range of asynchonicity. See class Range. Default is Range.C_RANGE_PROCESS. + p_ada : bool + Boolean switch for adaptivitiy. Default = True. + p_buffer_size : int, optional + + p_duplicate_data : bool, optional + If True, instances will be duplicated before processing. Default = False. + p_visualize : bool, optional + Boolean switch for visualisation. Default = False. + p_logging : int + Log level (see constants of class Log). Default: Log.C_LOG_ALL + p_kwargs : dict + Further optional named parameters. + + """ + C_TYPE = 'Anomaly Predictor AD' + + + +## ------------------------------------------------------------------------------------------------- + + def __init__(self, + p_cls_tsf, + p_name: str = None, + p_range_max=StreamTask.C_RANGE_THREAD, + p_ada: bool = True, + p_buffer_size: int = 0, + p_duplicate_data: bool = False, + p_visualize: bool = False, + p_logging=Log.C_LOG_ALL, + **p_kwargs): + + super().__init__(p_name, + p_range_max, + p_ada, + p_buffer_size, + p_duplicate_data, + p_visualize, + p_logging, + **p_kwargs) + + self.p_cls_tsf = p_cls_tsf + self.capture_anomalies = {} + _findings:dict = {}, + + + +## ------------------------------------------------------------------------------------------------- + + def _run(self, p_inst : InstDict): + + for inst_id,(inst_type, inst) in p_inst.entries(): + if inst_type == self.InstTypeNew: + for ano_type, fl in self._findings.entries(): + for finding in fl: + if inst in finding: + self.adapt(p_inst=inst, p_ano_type = ano_type) + else: + self.adapt(p_inst=inst, p_ano_type = None) + + #self.add_tsf(p_ano_type=p_inst.type, p_tsf= timeseriesforcaster) +## ------------------------------------------------------------------------------------------------- + + def _adapt_on_event(self, p_event_id: str, p_event_object: Event) -> bool: + + """ + + Parameters + ---------- + p_event_id + p_event_object + + Returns + ------- + """ + t = type(p_event_object) + + try: + self._findings[t].append(p_event_object.get_instances()) + except: + self._findings[t] = (p_event_object.get_instances()) + + +## ------------------------------------------------------------------------------------------------- + + def get_anomaly(self, p_anomaly): + """ + Process incoming anomaly data from the anomaly detector. + + parameters + ---------- + ad_anomaly + Anomaly data coming from the anomaly detector. + + """ + + self.p_anomaly = p_anomaly + self.captured_anomalies.append(p_anomaly) + + + def is_posiitive_event(self): + pass + + diff --git a/src/mlpro/oa/streams/tasks/anomalypredictors/tsf/basics.py b/src/mlpro/oa/streams/tasks/anomalypredictors/tsf/basics.py new file mode 100644 index 000000000..6c816ace6 --- /dev/null +++ b/src/mlpro/oa/streams/tasks/anomalypredictors/tsf/basics.py @@ -0,0 +1,138 @@ +## ------------------------------------------------------------------------------------------------- +## -- Project : MLPro - The integrative middleware framework for standardized machine learning +## -- Package : mlpro.oa.streams.tasks.anomalypredictors +## -- Module : basics.py +## ------------------------------------------------------------------------------------------------- +## -- History : +## -- yyyy-mm-dd Ver. Auth. Description +## -- 2024-06-04 0.0.0 DA/DS Creation +## -- 2024-08-23 0.1.0 DA/DS Creation +## ------------------------------------------------------------------------------------------------- + +""" +Ver. 0.1.0 (2024-08-23) + +This module provides basic templates for online anomaly prediction in MLPro. + +""" + + +from typing import List +from mlpro.bf.math import Function +from mlpro.bf.math.basics import Element, MSpace +from mlpro.bf.ml import Async, Log, Task +from mlpro.bf.streams import Log, StreamTask +from mlpro.bf.various import Log +from mlpro.oa.streams.tasks.anomalypredictors import AnomalyPredictor, AnomalyPrediction +from mlpro.sl import SLAdaptiveFunction +from mlpro.bf.streams import InstDict +from mlpro.sl.models_eval import Metric + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class TimeSeriesForcaster (Function): + """ + ... + """ + + pass + + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class OATimeSeriesForcaster (TimeSeriesForcaster, SLAdaptiveFunction): + """ + ... + """ + def __init__(self, + p_input_space: MSpace, + p_output_space: MSpace, + p_output_elem_cls=..., + p_threshold=0, + p_ada: bool = True, + p_buffer_size: int = 0, + p_metrics: List[Metric] = ..., + p_score_metric=None, + p_name: str = None, + p_range_max: int = Async.C_RANGE_PROCESS, + p_autorun=Task.C_AUTORUN_NONE, + p_class_shared=None, + p_visualize: bool = False, + p_logging=Log.C_LOG_ALL, + **p_par): + super().__init__(p_input_space, + p_output_space, + p_output_elem_cls, + p_threshold, + p_ada, + p_buffer_size, + p_metrics, + p_score_metric, + p_name, + p_range_max, + p_autorun, + p_class_shared, + p_visualize, + p_logging, + **p_par) + +## ------------------------------------------------------------------------------------------------- + + def _adapt(self, p_input, p_timestamp, p_ano:bool): + pass + + + + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class AnomalyPredictorTSF (AnomalyPredictor): + """ + ... + """ + + + C_TYPE = 'Anomaly Predictor TSF' + +## ------------------------------------------------------------------------------------------------- + def __init__(self, + p_cls_tsf, + p_name: str = None, + p_range_max=StreamTask.C_RANGE_THREAD, + p_ada: bool = True, + p_buffer_size: int = 0, + p_duplicate_data: bool = False, + p_visualize: bool = False, + p_logging=Log.C_LOG_ALL, + **p_kwargs): + super().__init__(p_name, p_range_max, p_ada, p_buffer_size, p_duplicate_data, p_visualize, p_logging, **p_kwargs) + + self.p_cls_tsf = p_cls_tsf + +## ------------------------------------------------------------------------------------------------- + + def _run(self, p_inst : InstDict): + for inst_id, (inst_type, inst) in p_inst.entries(): + if inst_type not in self.known_types: + self._adapt(p_inst=inst, p_ano_type='Unknown') + else: + self._adapt(p_inst=inst, p_ano_type=inst_type) + pass + +## ------------------------------------------------------------------------------------------------- + + def _adapt(self, p_inst , p_ano_type): + pass + +## ------------------------------------------------------------------------------------------------- + + def add_tsf(self, p_ano_type, p_tsf): + pass \ No newline at end of file diff --git a/src/mlpro/oa/streams/tasks/anomalypredictors/tsf/mini_batch_based.py b/src/mlpro/oa/streams/tasks/anomalypredictors/tsf/mini_batch_based.py new file mode 100644 index 000000000..419584ed2 --- /dev/null +++ b/src/mlpro/oa/streams/tasks/anomalypredictors/tsf/mini_batch_based.py @@ -0,0 +1,109 @@ +## ------------------------------------------------------------------------------------------------- +## -- Project : MLPro - The integrative middleware framework for standardized machine learning +## -- Package : mlpro.oa.streams.tasks.anomalypredictors +## -- Module : basics.py +## ------------------------------------------------------------------------------------------------- +## -- History : +## -- yyyy-mm-dd Ver. Auth. Description +## -- 2024-06-04 0.0.0 DA/DS Creation +## -- 2024-08-23 0.1.0 DA/DS Creation +## -- 2024-09-27 0.2.0 DS Creation +## ------------------------------------------------------------------------------------------------- + +""" +Ver. 0.2.0 (2024-09-27) + +This module provides template for managing mini-batches for time series forcasting tasks in MLPro. + +""" + + +from typing import List +from mlpro.bf.math.basics import MSpace +from mlpro.bf.ml import Async, Log, Task +from mlpro.bf.various import Log +from mlpro.oa.streams.tasks.anomalypredictors.tsf.basics import OATimeSeriesForcaster +from mlpro.oa.streams.tasks.anomalydetectors.basics import AnomalyDetector, Anomaly +from mlpro.bf.streams import Instance, Log +from mlpro.sl.models_eval import Metric + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class MiniBatchManager(): + """ + This module implements a template for managing mini-batches for time series forcasting tasks in MLPro. + + Parameters + ---------- + data + Time series data to be split into batches. + batch_size + Size of a mini-batch. + + """ +## -------------------------------------------------------------------------------------------------- + def __init__(self, p_batchno, p_batch_size: int) : + + self.batchno = p_batchno + self.batch_size = p_batch_size + self.data = [] + + +## -------------------------------------------------------------------------------------------------- + def add_data(self, **p_kwargs): + pass + + +## -------------------------------------------------------------------------------------------------- + def _add_data(self, p_anomaly: Anomaly) : + + self.data.append(p_anomaly) + +## -------------------------------------------------------------------------------------------------- + def create_mini_batches(self): + """ + Method to be used to create mini_batches from the data. + + Parameters + ---------- + """ + n_data_points = len(self.data) + + mini_batches = [self.data[i:i + self.p_batch_size] for i in range(0, n_data_points, self.p_batch_size)] + + return mini_batches + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class StreamMiniBatchManager (): + """ + ... + """ +## -------------------------------------------------------------------------------------------------- + def __init__(self) : + pass + + +## -------------------------------------------------------------------------------------------------- + def _add_data(self, p_inst : Instance ): + pass + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +class OATimeSeriesForcasterMB (OATimeSeriesForcaster): + """ + ... + """ + def __init__(self, p_input_space: MSpace, p_output_space: MSpace, p_output_elem_cls=..., p_threshold=0, p_ada: bool = True, p_buffer_size: int = 0, p_metrics: List[Metric] = ..., p_score_metric=None, p_name: str = None, p_range_max: int = Async.C_RANGE_PROCESS, p_autorun=Task.C_AUTORUN_NONE, p_class_shared=None, p_visualize: bool = False, p_logging=Log.C_LOG_ALL, **p_par): + super().__init__(p_input_space, p_output_space, p_output_elem_cls, p_threshold, p_ada, p_buffer_size, p_metrics, p_score_metric, p_name, p_range_max, p_autorun, p_class_shared, p_visualize, p_logging, **p_par) + +## -------------------------------------------------------------------------------------------------- + def _adapt( p_input, p_timestamp): + pass + +## -------------------------------------------------------------------------------------------------- + def _adapt_tsf_mb( p_mini_batch: MiniBatchManager): + pass \ No newline at end of file diff --git a/test/howtos/oa/howto_oa_streams_cbap_001_KMeans_ClusterDriftDetector_InstantaneousDetection_2d.py.off b/test/howtos/oa/howto_oa_streams_cbap_001_KMeans_ClusterDriftDetector_InstantaneousDetection_2d.py.off new file mode 100644 index 000000000..9dad40ce2 --- /dev/null +++ b/test/howtos/oa/howto_oa_streams_cbap_001_KMeans_ClusterDriftDetector_InstantaneousDetection_2d.py.off @@ -0,0 +1,199 @@ +## ------------------------------------------------------------------------------------------------- +## -- Project : MLPro - The integrative middleware framework for standardized machine learning +## -- Package : test.howtos.oa +## -- Module : howto_oa_streams_cbap_001_KMeans_ClusterDriftDetector_InstantaneousDetection_2d.py +## ------------------------------------------------------------------------------------------------- +## -- History : +## -- yyyy-mm-dd Ver. Auth. Description +## -- 2024-08-14 0.0.0 DA Copied from howto_oa_streams_cbad_001_KMeans_....py +## ------------------------------------------------------------------------------------------------- + +""" +Ver. 0.0.0 (2024-08-14) + +This module is done as a part of the Master's Thesis named Online Adaptive Cluster-based Anomaly +Detection, authored by Syamraj Purushamparambil Satheesh, as a part of the course +"Systems Engineering and Engineering Management" in the Department of "Electrical Energy Engineering", +supervised by Dipl.-Info. Detlef Arend. + + +This module demonstrates cluster-based anomlay detection, on synthetic 2-dimensional data generated by +the StreamMLProClusterGenerator, using the ClusterDriftDetector anomaly detection algorithm employing +the KMeans clustering algorithm wrapped from River library, to detect ClusterDrift anomaly. + +You will learn: + +1. Generating synthetic data using the native StreamMLProClusterGenerator. + +2. Creating a workflow and tasks in MLPro-OA. + +3. Clustering the normalized streaming data using the WrRiverKMeans2MLPro, with normalizer as a predecessor. + +4. Detecting drift anomalies in the clustered data using the ClusterDriftDetector. + +In the visualization, the cross hair designating the cluster's centroid becomes'red' in colour when +an anomaly is detected linked to that specific cluster. An overview of the anomalies is displayed on +the screen following the run. + +""" +from mlpro.bf.math import Event +from mlpro.bf.mt import Event +from mlpro.bf.streams.streams import * +from mlpro.bf.various import Log +from mlpro.oa.streams import * +from mlpro_int_river.wrappers.clusteranalyzers.kmeans import WrRiverKMeans2MLPro +from mlpro.oa.streams.tasks.anomalydetectors.cb_detectors.drift_detector import ClusterDriftDetector +from mlpro.oa.streams.tasks.anomalypredictors.tsf.ad_based import AnomalyPredictorAD + + + + +class DevindisPredictor (AnomalyPredictorAD): + + def _adapt_on_event(self, p_event_id: str, p_event_object: Event) -> bool: + + # Set breakpoint here... + pass + + + + +## ------------------------------------------------------------------------------------------------- +## ------------------------------------------------------------------------------------------------- +# 1 Prepare a scenario +class MyScenario(OAStreamScenario): + C_NAME = 'ClusterDriftScenario' + + def _setup(self, p_mode, p_ada: bool, p_visualize: bool, p_logging): + + # 1.1 Get MLPro benchmark Clutser Generator + stream = StreamMLProClusterGenerator(p_num_dim=2, + p_num_instances=5000, + p_num_clusters=3, + p_radii=[100], + p_velocities=[0.0], + p_change_velocities=True, + p_changed_velocities=[0.5, 0.6], + p_points_of_change_velocities=[1000, 1200], + p_num_clusters_for_change_velocities=2, + p_seed=12, + p_logging=p_logging) + + + # 1.2 Set up a stream workflow + + # 1.2.1 Creation of a workflow + workflow = OAStreamWorkflow( p_name='Anomaly Detection', + p_range_max=OAStreamWorkflow.C_RANGE_NONE, + p_ada=p_ada, + p_visualize=p_visualize, + p_logging=p_logging ) + + + # 1.2.2 Creation of tasks and add them to the workflow + + # Cluster Analyzer + task_clusterer = WrRiverKMeans2MLPro( p_name='#1: KMeans@River', + p_n_clusters=3, + p_halflife=0.05, + p_sigma=3, + p_seed=42, + p_visualize=p_visualize, + p_logging=p_logging ) + + workflow.add_task(p_task = task_clusterer) + + # Anomaly Detector + task_anomaly_detector = ClusterDriftDetector( p_clusterer=task_clusterer, + p_with_time_calculation=False, + p_instantaneous_velocity_change_detection=True, + p_min_velocity_threshold=1, + p_initial_skip=400, + p_visualize=p_visualize, + p_logging=p_logging ) + + workflow.add_task(p_task=task_anomaly_detector, p_pred_tasks=[task_clusterer]) + + # Anomaly Predictor + task_anomaly_predictor = DevindisPredictor( p_visualize=p_visualize, + p_logging=p_logging) + + task_anomaly_detector.register_event_handler( p_event_id = 'Anomaly', + p_event_handler = task_anomaly_predictor.adapt_on_event ) + + # 1.3 Return stream and workflow + return stream, workflow + + + +# 2 Prepare for test +if __name__ == "__main__": + cycle_limit = 2000 + logging = Log.C_LOG_ALL + visualize = True + step_rate = 1 +else: + cycle_limit = 5 + logging = Log.C_LOG_NOTHING + visualize = False + step_rate = 1 + + +# 3 Instantiate the stream scenario +myscenario = MyScenario( p_mode=Mode.C_MODE_SIM, + p_cycle_limit=cycle_limit, + p_visualize=visualize, + p_logging=logging ) + +# 4 Reset and run own stream scenario +myscenario.reset() + +if __name__ == "__main__": + myscenario.init_plot( p_plot_settings=PlotSettings( p_view = PlotSettings.C_VIEW_2D, + p_step_rate = step_rate ) ) + input('\nPlease arrange all windows and press ENTER to start stream processing...') + + + +tp_before = datetime.now() +myscenario.run() +tp_after = datetime.now() +tp_delta = tp_after - tp_before +duraction_sec = ( tp_delta.seconds * 1000000 + tp_delta.microseconds + 1 ) / 1000000 +myscenario.log(Log.C_LOG_TYPE_W, 'Duration [sec]:', round(duraction_sec,2), ', Cycles/sec:', round(cycle_limit/duraction_sec,2)) + + + +# 5 Summary +anomalies = myscenario.get_workflow()._tasks[1].get_anomalies() +detected_anomalies= len(anomalies) + +myscenario.log(Log.C_LOG_TYPE_W, '-------------------------------------------------------') +myscenario.log(Log.C_LOG_TYPE_W, '-------------------------------------------------------') +myscenario.log(Log.C_LOG_TYPE_W, 'Here is the recap of the anomaly detector') +myscenario.log(Log.C_LOG_TYPE_W, 'Number of anomalies: ', detected_anomalies ) + +for anomaly in anomalies.values(): + anomaly_name = anomaly.C_NAME + anomaly_id = str(anomaly.id) + clusters_affected = {} + clusters = anomaly.get_clusters() + properties = anomaly.get_properties() + for x in clusters.keys(): + clusters_affected[x] = {} + clusters_affected[x]["velocity"] = properties[x]["velocity"] + clusters_affected[x]["acceleration"] = properties[x]["acceleration"] + + + inst = anomaly.get_instances()[-1].get_id() + myscenario.log(Log.C_LOG_TYPE_W, + 'Anomaly : ', anomaly_name, + '\n Anomaly ID : ', anomaly_id, + '\n Instance ID : ', inst, + '\n Clusters : ', clusters_affected) + +myscenario.log(Log.C_LOG_TYPE_W, '-------------------------------------------------------') +myscenario.log(Log.C_LOG_TYPE_W, '-------------------------------------------------------') + +if __name__ == "__main__": + input('Press ENTER to exit...') \ No newline at end of file