Merge pull request #34 from TheGhouls/0.4.8

0.4.8

changelog.txt

@@ -1,3 +1,5 @@
+* NEW RELEASES FORMAT HAS REPLACED CHANGELOG *
+
 0.4.5 to 0.4.6
 ==============
 
@@ -182,7 +184,7 @@ Backward compatibility breaks
 as been removed. Transaction now inherit by default from the BaseTransaction class located in the oct-turrets package
 * A new command has been added `oct-pack-turrets` that will create a tar file from a test folder containing all files and configuration
 required for running a turret. This command will generate a tar archive per turret in your project `config.json` file
-## Last update Thu Jul  7 17:30:27 2016 VALETTE Emmanuel
+## Last update Thu Sep  8 15:01:32 2016 VALETTE Emmanuel
 * You can hotplug a turret while a test is running. This turret will appear in the final report
 * The HQ will sleep for 1 second after the creation of the sockets to wait for them
 * Remove the GenericTransaction class in oct.core.generic

oct/__init__.py

@@ -1 +1 @@
-__version__ = '0.4.7'
+__version__ = '0.4.8'

oct/core/hq.py

@@ -29,7 +29,7 @@ def __init__(self, output_dir, config, topic, master=True, *args, **kwargs):
 
         self.result_collector = self.context.socket(zmq.PULL)
         self.external_publisher = self.context.socket(zmq.PUB)
-        self.stats_handler = StatsHandler()
+        self.stats_handler = StatsHandler(config.get('results_database', {}).get('insert_limit', 150))
 
         self._configure_sockets(config)
         self.turrets_manager = TurretsManager(config.get('publish_port', 5000), master)

oct/results/graphs.py

@@ -28,16 +28,18 @@ def resp_graph_raw(dataframe, image_name, dir='./'):
     """
     factor = int(len(dataframe) / 10)
     df = dataframe.reset_index()
-    grp = df.groupby(pd.cut(df.index, np.arange(0, len(df), factor)))
-
     fig = pygal.Dot(stroke=False,
                     x_label_rotation=25,
                     x_title='Elapsed Time In Test (secs)',
                     y_title='Average Response Time (secs)',
                     js=('scripts/pygal-tooltip.min.js',))
-    fig.x_labels = [x for x in grp.first()['epoch']]
-    fig.title = image_name.split('.')[0]
-    fig.add('Time', [x for x in grp.describe()['scriptrun_time'].unstack()['mean'].round(2)])
+    try:
+        grp = df.groupby(pd.cut(df.index, np.arange(0, len(df), factor)))
+        fig.x_labels = [x for x in grp.first()['epoch']]
+        fig.title = image_name.split('.')[0]
+        fig.add('Time', [x for x in grp.describe()['scriptrun_time'].unstack()['mean'].round(2)])
+    except ZeroDivisionError:
+        print("Not enough data for raw graph")
     fig.render_to_file(filename=os.path.join(dir, image_name))
 
 

oct/results/stats_handler.py

@@ -29,8 +29,9 @@ def init_stats(output_dir, config):
 
 class StatsHandler(object):
     """This class will handle results and stats comming from the turrets"""
-    def __init__(self):
+    def __init__(self, insert_limit=150):
         self.results = []
+        self.insert_limit = insert_limit
 
     def write_result(self, data):
         """Write the results received to the database
@@ -40,7 +41,7 @@ def write_result(self, data):
         data['custom_timers'] = ujson.dumps(data['custom_timers'])
         self.results.append(data)
 
-        if len(self.results) >= 490:  # SQLite limit for inser_many is 500
+        if len(self.results) >= 150:  # 150 rows for SQLite default limit
             with db.execution_context():
                 with db.atomic():
                     Result.insert_many(self.results).execute()

oct/results/writer.py

@@ -22,14 +22,17 @@ def set_statics(self):
             return None
         try:
             shutil.copytree(os.path.join(self.templates_dir, 'css'), os.path.join(self.results_dir, 'css'))
-            shutil.copytree(os.path.join(self.templates_dir, 'img'), os.path.join(self.results_dir, 'img'))
             shutil.copytree(os.path.join(self.templates_dir, 'scripts'), os.path.join(self.results_dir, 'scripts'))
             shutil.copytree(os.path.join(self.templates_dir, 'fonts'), os.path.join(self.results_dir, 'fonts'))
         except OSError as e:
             if e.errno == 17:  # File exists
                 print("WARNING : existing output directory for static files, will not replace them")
             else:  # in all other cases, re-raise exceptions
                 raise
+        try:
+            shutil.copytree(os.path.join(self.templates_dir, 'img'), os.path.join(self.results_dir, 'img'))
+        except OSError as e:
+            pass  # img is an optional directory
 
     def write_report(self, template):
         """Write the compiled jinja template to the results file

oct/tools/from_armory.py

@@ -0,0 +1,40 @@
+import os
+import six
+import requests
+
+from oct.utilities.newproject import from_template
+
+
+def download_armory(args):
+    plan_name = args.plan
+
+    results = requests.get("http://armory.theghouls.io/get-plan/%s" % plan_name)
+    plan_name = plan_name.split('/')[-1]
+    if results.status_code != 200:
+        print("No plan found with name %s" % plan_name)
+        print("Status : %s" % results.status_code)
+        return None
+    results = results.json()
+    if not results.get('plans'):
+        print("No plan found with name %s" % plan_name)
+        return
+    download_url = results['plans'][0]['gh_tar_url']
+    file_name = os.path.join("/", "tmp", plan_name + ".tar.gz")
+    r = requests.get(download_url, allow_redirects=True)
+    with open(file_name, 'wb') as f:
+        for chunk in r.iter_content(chunk_size=1024):
+            if chunk:
+                f.write(chunk)
+    args.template = file_name
+    from_template(args)
+    os.remove(file_name)
+
+
+def from_armory(sp):
+    if six.PY2:
+        parser = sp.add_parser('from-armory', help="create project from armory template")
+    else:
+        parser = sp.add_parser('from-armory', help="create project from armory template", aliases=['armory'])
+    parser.add_argument('name', type=str, help="new project name")
+    parser.add_argument('plan')
+    parser.set_defaults(func=download_armory)

oct/utilities/commands.py

@@ -7,6 +7,7 @@
 from oct.utilities.run_device import run_device_command
 from oct.tools.rebuild_results import rebuild_results
 from oct.tools.results_to_csv import results_to_csv
+from oct.tools.from_armory import from_armory
 
 
 PARSERS = [
@@ -15,7 +16,8 @@
     run_command,
     rebuild_results,
     results_to_csv,
-    run_device_command
+    run_device_command,
+    from_armory
 ]
 
 

oct/utilities/newproject.py

@@ -9,7 +9,17 @@
 BASE_DIR = os.path.abspath(os.path.dirname(__file__))
 
 
-def check_template(members):
+def get_members(tar, prefix):
+    if not prefix.endswith('/'):
+        prefix += '/'
+    offset = len(prefix)
+    for tarinfo in tar.getmembers():
+        if tarinfo.name.startswith(prefix):
+            tarinfo.name = tarinfo.name[offset:]
+            yield tarinfo
+
+
+def check_template(members, prefix=None):
 
     required_members = [
         'config.json',
@@ -19,6 +29,8 @@ def check_template(members):
     ]
 
     for fname in required_members:
+        if prefix is not None:
+            fname = os.path.join(prefix, fname)
         assert fname in members, "Required file %s not in archive" % fname
 
     return True
@@ -33,8 +45,9 @@ def from_template(args):
     template = args.template
 
     with tarfile.open(template) as tar:
-        check_template(tar.getnames())
-        tar.extractall(project_name)
+        prefix = os.path.commonprefix(tar.getnames())
+        check_template(tar.getnames(), prefix)
+        tar.extractall(project_name, members=get_members(tar, prefix))
 
 
 def from_oct(args):

oct/utilities/templates/configuration/config.json

@@ -27,6 +27,7 @@
     },
     "results_database": {
         "db_uri": "default",
-        "params": {}
+        "params": {},
+        "insert_limit": "150"
     }
 }

requirements.txt

@@ -9,3 +9,4 @@ pyzmq
 six
 oct-turrets>=0.2.4
 ujson
+requests

setup.py

@@ -47,7 +47,8 @@
         'numpy',
         'jinja2',
         'pandas',
-        'ujson'
+        'ujson',
+        'requests'
     ],
     entry_points={'console_scripts': [
         'oct = oct.utilities.commands:main'

tests/test_tools.py

@@ -63,6 +63,18 @@ def test_rebuild_results_error(self):
         with self.assertRaises(OctConfigurationError):
             main()
 
+    def test_download_armory(self):
+        """From armory command should be able to download and start project from armory"""
+        sys.argv = sys.argv[:1]
+        sys.argv += ['from-armory', "/tmp/test-armory", "karec/armory-wordpress"]
+        main()
+
+    def test_download_armory_fail(self):
+        """From armory command should be able to download and start project from armory"""
+        sys.argv = sys.argv[:1]
+        sys.argv += ['from-armory', "/tmp/test-armory", "karec/bad-rep"]
+        main()
+
     def tearDown(self):
         if os.path.exists(self.output_file):
             os.remove(self.output_file)