Refactor expandable tasks

drop global variables reduce sample to }bedrock.server.wait minor refactoring raise clear error message on MDX syntax problem rebase on master add section in readme fix issue in cmd arg parse change lineterminator to LF (\n)
cubewise-code · Aug 11, 2023 · 036e1c0 · 036e1c0
1 parent 1c2ce5f
commit 036e1c0
Show file tree

Hide file tree

Showing 7 changed files with 200 additions and 105 deletions.
diff --git a/README.md b/README.md
@@ -17,85 +17,145 @@ Clone or download the RushTI Repository
 
 ## Usage
 
-* Adjust config.ini to match your TM1 environment
-* Create the Tasks.txt file
-* Execute the RushTI.py script: 
+* Adjust `config.ini` to match your TM1 environment
 
-Execution for classical type of tasks file with four arguments: 
-- path to tasks.txt 
-- number of maximum workers to run in parallel
-- 'norm' to select the normal execution mode
-- number of retries per process
-- path to result file
+* Create the `tasks.txt` file
+
+* Execute the `RushTI.py` script: with 2 to 5 arguments, e.g.:
+
+  -`python RushTI.py tasks.txt 2 ` 
+*Executes rushti in normal mode based on tasks.txt with 2 threads*
+
+  -`python RushTI.py tasks.txt 16 OPT`
+*Executes rushti in optimized mode based on tasks.txt with 16 threads*
+
+  -`python RushTI.py tasks.txt 4 NORM 3`
+*Executes rushti in normal mode based on tasks.txt with 4 threads. Allow 3 retries per task*
+
+  -`python RushTI.py tasks.txt 8 opt 2 result.csv`
+*Executes rushti in normal mode based on tasks.txt with 8 threads. Allow 2 retries per task. Write result file to result.csv*
+
+### The Normal Mode
+
+Parallelizes the TI process execution with n max workers while allowing for execution groups through the `wait` key word
 
 ```
-python RushTI.py Tasks_type_classic.txt 16 norm 2 results.csv
+python RushTI.py tasks.txt 16 norm 2 results.csv
 ```
 
-Example of Tasks_type_classic.txt:
+Example of `tasks.txt` in Normal run mode:
 ```
-instance="tm1srv01" process="load.actuals" pMonth=Jan
-wait
-instance="tm1srv01" process="load.actuals" pMonth=Feb
-instance="tm1srv01" process="load.actuals" pMonth=Mar
-instance="tm1srv01" process="load.actuals" pMonth=Apr
-wait
-instance="tm1srv01" process="load.actuals" pMonth=May
+instance="tm1srv01" process="export.actuals" pMonth=Jan
+instance="tm1srv01" process="export.actuals" pMonth=Feb
+instance="tm1srv01" process="export.actuals" pMonth=Mar
+instance="tm1srv01" process="export.actuals" pMonth=Apr
+instance="tm1srv01" process="export.actuals" pMonth=May
+instance="tm1srv01" process="export.actuals" pMonth=Jun
 wait
-instance="tm1srv01" process="load.actuals" pMonth=Jun
-instance="tm1srv01" process="load.actuals" pMonth=Jul
+instance="tm1srv02" process="import.actuals" pMonth=Jan
+instance="tm1srv02" process="import.actuals" pMonth=Feb
+instance="tm1srv02" process="import.actuals" pMonth=Mar
+instance="tm1srv02" process="import.actuals" pMonth=Apr
+instance="tm1srv02" process="import.actuals" pMonth=May
+instance="tm1srv02" process="import.actuals" pMonth=Jun
 wait
-instance="tm1srv01" process="load.actuals" pMonth=Aug
+instance="tm1srv02" process="save.data.all"
 ```
 
-Or
+### The Optimized Mode
+
+Parallelizes the TI process execution with n max workers and allows to define individual dependencies between tasks.
+
+Each task is assigned an id. On all tasks you can define predecessors that must have completed before it can run.
 
-Execution for optimized type of tasks file with four arguments: 
-- path to tasks.txt
-- number of maximum workers to run in parallel
-- 'opt' to specify the optimized execution mode
-- number of retries per process
 
 ```
-python RushTI.py Tasks_type_optimized.txt 16 opt 3 results.csv
+python RushTI.py tasks.txt 16 opt 3 results.csv
 ```
 
-Example of Tasks_type_optimized.txt:
+Example of `tasks.txt` in Optimized Mode:
 ```
-id="1" predecessors="" require_predecessor_success="" instance="tm1srv01" process="load.actuals" pMonth=Jan
-id="2" predecessors="1" require_predecessor_success="1" instance="tm1srv02" process="load.actuals" pMonth=Feb
-id="3" predecessors="1" require_predecessor_success="1" instance="tm1srv01" process="load.actuals" pMonth=Mar
-id="4" predecessors="1" require_predecessor_success="0" instance="tm1srv02" process="load.actuals" pMonth=Apr
-id="5" predecessors="2,3" require_predecessor_success="0" instance="tm1srv01" process="load.actuals" pMonth=May
-id="6" predecessors="4,5" require_predecessor_success="1" instance="tm1srv02" process="load.actuals" pMonth=Jun
-id="7" predecessors="4" require_predecessor_success="0" instance="tm1srv01" process="load.actuals" pMonth=Jul
-id="8" predecessors="6" require_predecessor_success="0" instance="tm1srv02" process="load.actuals" pMonth=Aug
+id="1" predecessors="" require_predecessor_success="" instance="tm1srv01" process="export.actuals" pMonth=Jan
+id="2" predecessors="" require_predecessor_success="" instance="tm1srv01" process="export.actuals" pMonth=Feb
+id="3" predecessors="" require_predecessor_success="" instance="tm1srv01" process="export.actuals" pMonth=Mar
+id="4" predecessors="" require_predecessor_success="" instance="tm1srv01" process="export.actuals" pMonth=Apr
+id="5" predecessors="" require_predecessor_success="" instance="tm1srv01" process="export.actuals" pMonth=May
+id="6" predecessors="" require_predecessor_success="" instance="tm1srv01" process="export.actuals" pMonth=Jun
+id="7" predecessors="1" require_predecessor_success="" instance="tm1srv02" process="load.actuals" pMonth=Jan
+id="8" predecessors="2" require_predecessor_success="" instance="tm1srv02" process="load.actuals" pMonth=Feb
+id="9" predecessors="3" require_predecessor_success="" instance="tm1srv02" process="load.actuals" pMonth=Mar
+id="10" predecessors="4" require_predecessor_success="" instance="tm1srv02" process="load.actuals" pMonth=Apr
+id="11" predecessors="5" require_predecessor_success="" instance="tm1srv02" process="load.actuals" pMonth=May
+id="12" predecessors="6" require_predecessor_success="" instance="tm1srv02" process="load.actuals" pMonth=Jun
+id="13" predecessors="7,8,9,10,11,12" require_predecessor_success="" instance="tm1srv02" process="save.data.all"
 ```
 
-RushTI produces a result file with the following format. 
-If no name is provided the file be named `rushti.csv`
+Please note that the same id can be assigned to multiple tasks.
+
+Both `OPT` and `NORM` now support comments as well as empty lines
+
+
+## The Result File
+
+
+RushTI produces a result file with the below format. If no name is provided the file will be named `rushti.csv`. 
+
+Unlike the `rushti.log` file, 
+the result file is machine-readable and can be consumed with TI after the execution to gain infomation about the execution.
+
+You can use the result file to raise an alert from TI when not all processes were successful.
 
 | PID   |Process Runs|Process Fails|Start|End|Runtime|Overall Success|
 |-------|---------------------|--------------|-------------------|--------------------|--------------|---------------|
 | 10332 |8|0|2023-06-26 15:19:28.778457|2023-06-26 15:19:49.160629|0:00:20.382172|True|
 
-## Running the tests
+## Expandable tasks
 
-No tests yet
+Instead of passing individual arguments per task, you can use the expandable assignment operator `*=*` _(default is `=`)_ 
+to assign all elements from an MDX result to a parameter. 
 
+RushTI will then register one task per element in the MDX result set.
 
-## Built With
+So this:
+```
+instance="tm1" process="export.actuals" pMonth*=*"{[Period].[Jan],[Period].[Feb],[Period].[Mar]}"
+```
+
+Is equivalent to this:
+```
+instance="tm1" process="export.actuals" pMonth=Jan
+instance="tm1" process="export.actuals" pMonth=Feb
+instance="tm1" process="export.actuals" pMonth=Mar
+```
+
+You can use expandable assignment on more than one parameter per line.
+
+Expandable tasks can be used both in `NORM` and `OPT` execution mode.
 
-* [requests](http://docs.python-requests.org/en/master/) - Python HTTP Requests for Humans
-* [TM1py](https://github.com/cubewise-code/TM1py) - A python wrapper for the TM1 REST API
+## Logging
 
-## More about TM1py
-There are lots of things you can do with TM1py:
-* [Upload data from Webservices](https://code.cubewise.com/tm1py-help-content/upload-exchange-rate-from-a-webservice)
-* [Data Science with TM1 and Planning Analytics](https://code.cubewise.com/blog/data-science-with-tm1-planning-analytics)
-* [Cleaning your instance](https://code.cubewise.com/tm1py-help-content/cleanup-your-tm1-application)
+Logging can be configured in `logging_config.ini`.
+By default, RushTI maintains a small log file  `rushti.log` with the most relevant info's about the execution
+
+The log file is helpful for troubleshooting issues and understanding past executions
+
+```
+2023-08-09 14:05:26,506 - 3036 - INFO - RushTI starts. Parameters: ['C:\\RushTI\\RushTI.py', 'tasks.txt', '2'].
+2023-08-09 14:05:30,626 - 3036 - INFO - Executing process: '}bedrock.server.wait' with parameters: {'pLogOutput': '1', 'pWaitSec': '6'} on instance: 'tm1srv01'
+2023-08-09 14:05:36,633 - 3036 - INFO - Execution successful: Process '}bedrock.server.wait' with parameters: {'pLogOutput': '1', 'pWaitSec': '6'} with 0 retries on instance: tm1srv01. Elapsed time: 0:00:06.005861
+...
+2023-08-09 14:05:58,682 - 3036 - INFO - Execution successful: Process '}bedrock.server.wait' with parameters: {'pLogOutput': '1', 'pWaitSec': '4'} with 0 retries on instance: tm1srv01. Elapsed time: 0:00:04.008271
+2023-08-09 14:06:00,692 - 3036 - INFO - Execution successful: Process '}bedrock.server.wait' with parameters: {'pLogOutput': '1', 'pWaitSec': '6'} with 0 retries on instance: tm1srv01. Elapsed time: 0:00:06.016828
+2023-08-09 14:06:00,700 - 3036 - INFO - RushTI ends. 0 fails out of 8 executions. Elapsed time: 0:00:34.191408. Ran with parameters: ['C:\\RushTI\\RushTI.py', 'tasks.txt', '2']
+```
+
+## Need a .exe version of RushTI?
+
+Download RushTI and Use PyInstaller to create the .exe file
+
+## Built With
 
-If you are interested you should check the [TM1py-samples](https://github.com/cubewise-code/TM1py-samples).
+[TM1py](https://github.com/cubewise-code/TM1py) - A python wrapper for the TM1 REST API
 
 
 ## License

diff --git a/Tasks_type_classic_expanded.txt b/Tasks_type_classic_expanded.txt
@@ -0,0 +1,7 @@
+instance="tm1srv01" process="}bedrock.server.wait" pLogOutput=1 pWaitSec=6
+wait
+# comments
+instance="tm1srv01" process="}bedrock.server.wait" pLogOutput=1 pWaitSec*=*"{[Counter].[5], [Counter].[5], [Counter].[9]}"
+wait
+instance="tm1srv01" process="}bedrock.server.wait" pLogOutput=1 pWaitSec*=*"{[Counter].[4], [Counter].[4], [Counter].[4]}"
+instance="tm1srv01" process="}bedrock.server.wait" pLogOutput=1 pWaitSec=6
diff --git a/Tasks_type_classic_mdx.txt b/Tasks_type_classic_mdx.txt
diff --git a/Tasks_type_optimized_expanded.txt b/Tasks_type_optimized_expanded.txt
@@ -0,0 +1,10 @@
+# testing
+id="1" predecessors="" require_predecessor_success="1" instance="tm1srv01" process="}bedrock.server.wait" pLogOutput=1 pWaitSec*=*"{TM1SUBSETALL([Counter].[Counter])}"
+id="2" predecessors="1" require_predecessor_success="1" instance="tm1srv01" process="}bedrock.server.wait" pLogOutput=0 pWaitSec*=*"{[Counter].[9],[Counter].[9]}"
+# with comments
+id="3" predecessors="2" require_predecessor_success="1" instance="tm1srv01" process="}bedrock.server.wait" pWaitSec=7 pLogOutput*=*"{[Counter].[0],[Counter].[1]}"
+id="4" predecessors="3" require_predecessor_success="1" instance="tm1srv01" process="}bedrock.server.wait" pWaitSec=5
+id="5" predecessors="3" require_predecessor_success="1" instance="tm1srv01" process="}bedrock.server.wait" pWaitSec=5
+id="6" predecessors="4,5" require_predecessor_success="1" instance="tm1srv01" process="}bedrock.server.wait" pWaitSec=11
+id="7" predecessors="4" require_predecessor_success="1" instance="tm1srv01" process="}bedrock.server.wait" pWaitSec=6
+id="8" predecessors="6" require_predecessor_success="1" instance="tm1srv01" process="}bedrock.server.wait" pWaitSec=6
diff --git a/Tasks_type_optimized_mdx.txt b/Tasks_type_optimized_mdx.txt
diff --git a/logging_config.ini b/logging_config.ini
@@ -24,4 +24,4 @@ formatter=formatter
 args=('rushti.log', 'a', 5*1024*1024, 10, 'utf-8')
 
 [formatter_formatter]
-format=%(asctime)s - RushTI - %(levelname)s - %(message)s
+format=%(asctime)s - %(process)d - %(levelname)s - %(message)s