Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Solution for Assignment2. #66

Open
wants to merge 14 commits into
base: master
Choose a base branch
from
162 changes: 162 additions & 0 deletions solutions/Abhilasha/Assignment2/BuildAutomationTool.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,162 @@
# -*- coding: utf-8 -*-
"""
Created on Fri Dec 6 14:36:59 2019

@author: Abhilasha
"""

import os
import json
import subprocess
from pathlib import Path
from os.path import relpath
import time


class ActionGraph():

'''loads commands, dependencies and initialises the status for all the actions from all build.json files.'''

def __init__(self, root_dir):

self.root_dir = root_dir
self.all_action_status = {}
self.all_action_commands = {}
self.all_action_dependencies = {}

def create_action_map(self):

for filename in Path(self.root_dir).rglob('*.json'):
kaustubh-karkare marked this conversation as resolved.
Show resolved Hide resolved
rel_location = relpath(filename,self.root_dir)
if len(str(rel_location).split(os.path.sep)) == 1:
rel_location = ''
else:
rel_location = str(rel_location).split(os.path.sep)[0]
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of splitting the same thing twice, you can just share the result.

path_elements = str(rel_location).split(os.path.sep)

Also, you don't just want the first element, you want everything except the last element: path_elements[:-1]

with open(filename) as json_file:
all_data = json.load(json_file)
kaustubh-karkare marked this conversation as resolved.
Show resolved Hide resolved
for data in all_data:
if rel_location != '':
action = rel_location+'/'+data['name']
else:
action = data['name']
self.all_action_status[action] = 'not started'
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Technically, the concept of status is not relevant to a graph. It only matters to the executor and so should not be a property of this object.

if 'command' in data:
self.all_action_commands[action] = data['command']
else:
self.all_action_commands[action] = 'none'
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

self.all_action_commands[action] = data.get('command')

That way, you're not dealing with a "none" string, but a proper None object.

if 'deps' in data:
self.all_action_dependencies[action] = data['deps']
else:
self.all_action_dependencies[action] = 'none'


def get_command_to_be_executed(self, build, action):
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is this build argument used for?

Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

^


if action not in self.all_action_status:
raise Exception('Command not recognized.')
return
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Will never reach this line due to raise on the previous one.

else:
Action_obj = Action(self.all_action_status, self.all_action_commands, self.all_action_dependencies)
Action_obj.get_dependencies(action)
Action_obj.update_dependencies_status(action)
Action_obj.execute_commands()
return


class Action():
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This isn't the correct name.
An action is a specific thing you can execute.
This class is responsible for executing multiple actions.
So "ActionExecutor" might be a better name.



def __init__(self, all_action_status, all_action_commands, all_action_dependencies):

self.all_action_status = all_action_status
self.all_action_commands = all_action_commands
self.all_action_dependencies = all_action_dependencies
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Alternatively, you could just store a single reference to the action_graph, and then do self.graph.action_commands

self.current_action_status = {}
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The action status should be stored in the ActionExecutor, not here, since that is the class that cares about it.

self.current_action_commands = {}
self.current_action_dependencies = {}
self.ongoing_subprocesses = []
self.action_name_for_ongoing_subprocess = []


def get_dependencies(self, action):

'''stores the status, commands, dependencies of the action to be executed and all its dependencies'''

self.current_action_status[action] = self.all_action_status[action]
self.current_action_commands[action] = self.all_action_commands[action]
self.current_action_dependencies[action] = self.all_action_dependencies[action]
if self.all_action_dependencies[action] != 'none':
for name in self.all_action_dependencies[action]:
self.get_dependencies(name)
return


def update_dependencies_status(self, action):

''' updates the status of the actions- ready or processing or done '''

if self.current_action_dependencies[action] == 'none':
self.current_action_status[action] = 'ready'
else:
total_no_of_dependencies = len(self.current_action_dependencies[action])
no_of_dependencies_done = 0;
for dep in self.current_action_dependencies[action]:
if self.current_action_status[dep] == 'not started':
self.update_dependencies_status(dep)
if self.current_action_status[dep] == 'done':
no_of_dependencies_done += 1
if no_of_dependencies_done == total_no_of_dependencies:
self.current_action_status[action] = 'ready'
return



def execute_commands(self):

for name in self.current_action_status:
if(self.current_action_status[name] == 'ready'):
self.current_action_status[name] = 'processing'
cwd = os.getcwd()
if '/' in name:
loc = name[:name.rindex('/')]
else:
loc = ''

if self.current_action_commands[name] != 'none':
if loc != '':
os.chdir(cwd+os.path.sep+loc)
command = str(self.current_action_commands[name])
p = subprocess.Popen(command, shell=True)
os.chdir(cwd)
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can avoid the manual changing of directories this way:

p = subprocess.Popen(command, shell=True, cwd=cwd)

self.ongoing_subprocesses.append(p)
self.action_name_for_ongoing_subprocess.append(name)

else:
self.current_action_status[name] = 'done'
for name in self.current_action_status:
if(self.current_action_status[name] == 'not_started'):
self.update_dependencies_status(name)
self.execute_commands()

while not len(self.ongoing_subprocesses) == 0:

any_subprocess_completed = False
for p, action in zip(self.ongoing_subprocesses, self.action_name_for_ongoing_subprocess):
if not(p.poll() is None):
any_subprocess_completed = True
break
else:
continue

if any_subprocess_completed == True:
self.ongoing_subprocesses.remove(p)
self.action_name_for_ongoing_subprocess.remove(action)
self.current_action_status[action] = 'done'
for name in self.current_action_status:
if(self.current_action_status[name] == 'not started'):
self.update_dependencies_status(name)
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This seems somewhat inefficient. Ie - update_dependencies_status already contains recursion, so will update multiple items anyway. Looping over all of them results in duplicate work.

I think the proper fix is to also keep track of "dependents" (ie - the other direction of dependencies). That way, you just have to update the dependents of the completed action via recursion, and can avoid the loop.

self.execute_commands()
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay, so this line confused me. I suppose it works, but it is non-intuitive.
You want to start the next action (ie - run lines 116 to 139), which is why I think you're calling the method recursively. However, that method will contain its own copy of this while loop, which is unnecessary.
You could make this easier to read by reorganizing the code to something like:

while there are pending actions:
    start any actions that are ready
    while True:
        if any ongoing action is completed:
            break
        time.sleep(1)
    mark new actions as ready

else:
time.sleep(1)
return

93 changes: 93 additions & 0 deletions solutions/Abhilasha/Assignment2/btest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
import unittest
import os
from os import path
import tempfile
import shutil
import BuildAutomationTool as BAT

class TestBuildAutomationTool(unittest.TestCase):

def setUp(self):

self.root = os.getcwd()
self.test_dir = tempfile.TemporaryDirectory()


def TearDown(self):

shutil.rmtree(self.test_dir)


def test_build_test_all(self):

with self.test_dir as tmpdirname:
new_root_dir = tmpdirname+os.path.sep+"code"
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Use os.path.join, instead to directly concatenating.

shutil.copytree(self.root+os.path.sep+"code", new_root_dir )
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

spacing here is inconsistent, did you use a linter?

os.chdir(new_root_dir)
BAT_obj = BAT.ActionGraph(new_root_dir)
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of BAT_obj, why not just call this "action_graph" ?

BAT_obj.create_action_map()
BAT_obj.get_command_to_be_executed('build', 'test_all')
self.assertEqual(path.exists('test.o'), True)
self.assertEqual(path.exists('test_sort_bubble.exe'), True)
self.assertEqual(path.exists('test_sort_quick.exe'), True)
self.assertEqual(path.exists('test_sort_merge.exe'), True)
os.chdir(os.path.join("algorithms"))
self.assertEqual(path.exists('sort_bubble.o'), True)
self.assertEqual(path.exists('sort_quick.o'), True)
self.assertEqual(path.exists('sort_merge.o'), True)
os.chdir(self.root)


def test_clean(self):
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you might want to merge this test with the one above.
Ie - when you copied the code directly, I don't think it contains the .o or .exe file anyway, so executing the clean command does nothing, which doesn't provide that much value, right?


with self.test_dir as tmpdirname:
new_root_dir = tmpdirname+os.path.sep+"code"
shutil.copytree(self.root+os.path.sep+"code", new_root_dir )
os.chdir(new_root_dir)
BAT_obj = BAT.ActionGraph(new_root_dir)
BAT_obj.create_action_map()
BAT_obj.get_command_to_be_executed('build', 'clean')
self.assertEqual(path.exists('test.o'), False)
self.assertEqual(path.exists('test_sort_bubble.exe'), False)
self.assertEqual(path.exists('test_sort_quick.exe'), False)
self.assertEqual(path.exists('test_sort_merge.exe'), False)
os.chdir(os.path.join("algorithms"))
self.assertEqual(path.exists('sort_bubble.o'), False)
self.assertEqual(path.exists('sort_quick.o'), False)
self.assertEqual(path.exists('sort_merge.o'), False)
os.chdir(self.root)


def test_sort_merge(self):

with self.test_dir as tmpdirname:
new_root_dir = tmpdirname+os.path.sep+"code"
shutil.copytree(self.root+os.path.sep+"code", new_root_dir )
os.chdir(new_root_dir)
BAT_obj = BAT.ActionGraph(new_root_dir)
BAT_obj.create_action_map()
BAT_obj.get_command_to_be_executed('build', 'test_sort_merge')
self.assertEqual(path.exists('test.o'), True)
self.assertEqual(path.exists('test_sort_merge.exe'), True)
kaustubh-karkare marked this conversation as resolved.
Show resolved Hide resolved
os.chdir(os.path.join("algorithms"))
self.assertEqual(path.exists('sort_merge.o'), True)
os.chdir(self.root)


def test_invalid_key(self):

with self.test_dir as tmpdirname:
new_root_dir = tmpdirname+os.path.sep+"code"
shutil.copytree(self.root+os.path.sep+"code", new_root_dir )
os.chdir(new_root_dir)
BAT_obj = BAT.ActionGraph(new_root_dir)
BAT_obj.create_action_map()
with self.assertRaisesRegex(Exception, 'Command not recognized.'):
BAT_obj.get_command_to_be_executed('build', 'test_sort_selection')
os.chdir(self.root)


if __name__ == '__main__':

unittest.main()

18 changes: 18 additions & 0 deletions solutions/Abhilasha/Assignment2/code/algorithms/build.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
[
{
"name": "clean",
"command": "rm -f *.o"
},
{
"name": "sort_bubble",
"command": "g++ -c sort_bubble.cpp"
},
{
"name": "sort_merge",
"command": "g++ -c sort_merge.cpp"
},
{
"name": "sort_quick",
"command": "g++ -c sort_quick.cpp"
}
]
4 changes: 4 additions & 0 deletions solutions/Abhilasha/Assignment2/code/algorithms/desktop.ini
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
[ViewState]
Mode=
Vid=
FolderType=Generic
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#include <bits/stdc++.h>
using namespace std;
void sort_function(vector<int> &v)
{
cout<<"sort bubble";
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#include <bits/stdc++.h>
using namespace std;
void sort_function(vector<int> &v)
{
cout<<"sort merge";
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#include <bits/stdc++.h>
using namespace std;
void sort_function(vector<int> &v)
{
cout<<"sort quick";
}
30 changes: 30 additions & 0 deletions solutions/Abhilasha/Assignment2/code/build.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
[
{
"name": "clean",
"deps": ["algorithms/clean"],
"command": "rm -f test.o && rm -f test_*.exe"
},
{
"name": "test",
"command": "g++ -std=c++11 -c test.cpp"
},
{
"name": "test_sort_bubble",
"deps": ["test", "algorithms/sort_bubble"],
"command": "g++ test.o algorithms/sort_bubble.o -o test_sort_bubble.exe && ./test_sort_bubble.exe"
},
{
"name": "test_sort_merge",
"deps": ["test", "algorithms/sort_merge"],
"command": "g++ test.o algorithms/sort_merge.o -o test_sort_merge.exe && ./test_sort_merge.exe"
},
{
"name": "test_sort_quick",
"deps": ["test", "algorithms/sort_quick"],
"command": "g++ test.o algorithms/sort_quick.o -o test_sort_quick.exe && ./test_sort_quick.exe"
},
{
"name": "test_all",
"deps": ["test_sort_bubble", "test_sort_merge", "test_sort_quick"]
}
]
8 changes: 8 additions & 0 deletions solutions/Abhilasha/Assignment2/code/test.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
#include <bits/stdc++.h>
using namespace std;
void sort_function(vector<int> &v);
int main()
{
vector<int> v;
sort_function(v);
}