Improve token parser

Use docutils parser to parse a docstring when ':' prefix is chosen.
Now docstrings should follow RST guidelines: no indentions are allowed
within token values.
For now, token value supports only a plane text and an enumerate list.
It should be enough for the test plan purpuses.

Fix the #120 issue.

setup.py

@@ -16,7 +16,7 @@
     author='Suresh Thirugn',
     author_email='[email protected]',
     packages=find_packages(),
-    install_requires=['Click', 'termcolor'],
+    install_requires=['Click', 'termcolor', 'docutils'],
     entry_points='''
         [console_scripts]
         testimony=testimony.cli:testimony

testimony/parser.py

@@ -1,6 +1,8 @@
 # coding=utf-8
 """Docstring parser utilities for Testimony."""
 import re
+from docutils.core import publish_string
+from xml.etree import ElementTree
 
 from testimony.constants import DEFAULT_MINIMUM_TOKENS, DEFAULT_TOKENS
 
@@ -18,10 +20,11 @@ def __init__(self, tokens=None, minimum_tokens=None, prefix=':'):
             self.minimum_tokens = DEFAULT_MINIMUM_TOKENS
         else:
             self.minimum_tokens = minimum_tokens
+        self.token_prefix = prefix
         self.minimum_tokens = set(self.minimum_tokens)
         self.tokens = set(self.tokens)
         self.token_regex = re.compile(
-            r'^{0}(\w+):\s+([^{0}]+)(\n|$)'.format(prefix),
+            r'^{0}(\w+):\s+([^{0}]+)(\n|$)'.format(self.token_prefix),
             flags=re.MULTILINE
         )
         if not self.minimum_tokens.issubset(self.tokens):
@@ -51,15 +54,42 @@ def parse(self, docstring=None):
         """
         if docstring is None:
             return {}, {}
+        tokens_dict = {}
         valid_tokens = {}
         invalid_tokens = {}
-        for match in self.token_regex.finditer(docstring):
-            token = match.group(1).strip().lower()
-            value = match.group(2).strip()
+
+        if self.token_prefix == ':':
+            docstring_xml = publish_string(docstring, writer_name='xml')
+            root = ElementTree.fromstring(docstring_xml)
+            tokens = root.findall('./field_list/field')
+            for token in tokens:
+                token_name = token.find('./field_name').text.lower()
+                value_el = token.find('./field_body/')
+                if value_el is None:
+                    invalid_tokens[token_name] = ''
+                    continue
+                if value_el.tag == 'paragraph':
+                    value = value_el.text
+                if value_el.tag == 'enumerated_list':
+                    value_lst = map(lambda elem: elem.text,
+                                    value_el.findall('./list_item/paragraph'))
+                    list_enum = list(enumerate(value_lst, start=1))
+                    steps = map(lambda val: '{}. {}'.format(val[0], val[1]),
+                                list_enum)
+                    value = '\n'.join(steps)
+                tokens_dict[token_name] = value
+        else:
+            for match in self.token_regex.finditer(docstring):
+                token = match.group(1).strip().lower()
+                value = match.group(2).strip()
+                tokens_dict[token] = value
+
+        for token, value in tokens_dict.items():
             if token in self.tokens:
                 valid_tokens[token] = value
             else:
                 invalid_tokens[token] = value
+
         return valid_tokens, invalid_tokens
 
     def validate_tokens(self, tokens):

tests/sample_output.txt

@@ -16,6 +16,7 @@ Setup:
 
 Steps:
  1. Login to the application with valid credentials
+ 2. Add a colon to the steps token: it should appear.
 
 Tags:
  t1, t2, t3
@@ -145,7 +146,8 @@ Setup:
  Global setup
 
 Steps:
- 1. Login to the application with valid username and no password
+ 1. Login to the application with valid username
+ and no password
 
 Test:
  Login with invalid credentials

tests/test_sample.py

@@ -20,9 +20,8 @@ def test_positive_login_1(self):
 
         :Feture: Login - Positive
 
-        :Steps:
-
-        1. Login to the application with valid credentials
+        :Steps: 1. Login to the application with valid credentials
+                2. Add a colon to the steps token: it should appear.
 
         :Assert: Login is successful
 
@@ -50,14 +49,11 @@ def test_positive_login_3(self):
 
         :Feature: Login - Positive
 
-        :Steps:
-
-        1. Login to the application with valid Latin credentials
+        :Steps: 1. Login to the application with valid Latin credentials
 
         :Assert: Login is successful
 
         :Tags: t1
-
         """
         # Code to perform the test
         pass
@@ -68,15 +64,12 @@ def test_positive_login_4(self):
 
         :Feature: Login - Positive
 
-        :Steps:
-
-        1. Login to the application with valid credentials having
-        special characters
+        :Steps: 1. Login to the application with valid credentials having
+                   special characters
 
         :Assert: Activation key is created
 
         :Status: Manual
-
         """
         # Code to perform the test
         pass
@@ -85,9 +78,7 @@ def test_positive_login_4(self):
     def test_negative_login_5(self):
         """Test missing required docstrings
 
-        :Steps:
-
-        1. Login to the application with invalid credentials
+        :Steps: 1. Login to the application with invalid credentials
 
         :BZ: 123456
 
@@ -109,9 +100,7 @@ def test_negative_login_6(self):
 
         :Feature: Login - Negative
 
-        :Steps:
-
-        1. Login to the application with invalid credentials
+        :Steps: 1. Login to the application with invalid credentials
 
         :Assert: Login failed
 
@@ -137,12 +126,10 @@ def test_negative_login_7(self):
 
         :Feature: Login - Negative
 
-        :Steps:
-
-        1. Login to the application with valid username and no password
+        :Steps: 1. Login to the application with valid username
+                   and no password
 
         :Assert: Login failed
-
         """
         # Code to perform the test
         pass