Merge pull request #84 from ThreatResponse/dev-test-setup

Update setup and docs

.gitignore

@@ -46,6 +46,7 @@ nosetests.xml
 coverage.xml
 *,cover
 .hypothesis/
+.pytest_cache/
 
 # Translations
 *.mo

.travis.yml

@@ -1,5 +1,6 @@
 language: python
 python:
+  - "3.4"
   - "3.6"
 cache: pip
 before_install:
@@ -15,19 +16,19 @@ before_install:
 install:
   - pip install -r requirements.txt
   - pip install flake8
-  - python -W ignore::UserWarning:distutils.dist setup.py build
 script:
   - cat aws_ir/_version.py
   - flake8 aws_ir
   - flake8 tests
-  - pytest tests/
+  - python setup.py test
 
 deploy:
   - provider: pypi
     distributions: sdist bdist_wheel
     user: threatresponse
     password:
       secure: 0j3uJew5G/+06gQnKRYDdWfwKYfWuXBFSQXob1T6Q8aFsY7Ats+4HjQE1gC1igU/6XwPCBPUBHVuVC1YCfXQFLbqTAJng8F28Z+ULZLkO78+0qTUbqWXqclKT7ES2VaMRhAyNuxPg76WJ0W5cDq+q9nJ05jJjxdfmPhlEgJNHKpc0W1l5RtXKwTyHawAdxZxhBcuNtopMBuSgB7EN/jy4S5xyKAdNNITLUoVD9HwS9pMWIv5t2crvFZXxyoIDR4ga4bCGCAz1NSdFsSguTQb/EeClItw/BEUuFyv6KkMbkfiCcLJZBdEXEUyshrMIo2smenblK4iYFNS7c25BqMuXsdxe39fgkRFkC41W/JB2WtBvMpMESi7+b18KKR9Y+8Hh+3i+BjiNl2RN8Um8vMaRh6B9jt8+dI7zc6QIohviPeDLif1L7jkU8NpCIGgYDWOy2rGfeWnOz07scoObiZbozlbZw15nWIAIzo4pELt5Ga0voBQ179KJdH88dso25mF9cAKhRezqTzJY1YS7GUKavdugfnL5Cpthyal4ajVrVw5J66cJZmyT2Qmj/uzfTrXxXfFfDALQOiFWea3EbV8qSnf0MkVV4pEixdlnD1jobFZNZLFg1VCNJionSbqZbUccSCLjjKkv05XTjUktJV37V5m41xBm7mRH9PvPgGHCOo=
+    skip_existing: true
     on:
       tags: true
 
@@ -38,5 +39,6 @@ deploy:
     password:
       secure: 3WT5LhsCxkkJReN2Tg2wHwlzFIQ9cVQagIxekf7b/iPDy2M+CGhGC94sct5Rx5J6JSy2yiQfMRAjdvFQMFewM52s1H9iojsOx5uHxAzulEMdMkI7FehKD49d9k0wnBkkIu9cS9/0nvaFCQIiaV/J/5F7U7WXqC0JgZOoJCAIBWrUSjwnbeMlyRsJwfy4jQQY8AqzqewJFlcpfzW0tJKyNMsgRyi+pxT60ihuNrT3oCwzjV/g7CajmD1fq41FwRgSzmFzF3qyCSxEhQ0KOYtUNZ8FzkpxoN8s8CKOBT20B2tUJhMp2wv5f16r4zsFPemA7mpPuZMWlCghJId8ad61reWPs7sxjGgxHXOm0hvBGGrM17vOAM176pahjLlTSN9Bl0A2QyL7fKmQhxnBWT6ZnbVo3jW9cEx2qee1dyR/ybPbMZTatrx9eb9Jdl381/10YMeqkfCm0rrEfLluVVI3Oq0yGvzIRCOgCskzIKwj2bDbor5H2Tu45ht5OUB5inQpFQapRkZ3pc6W4o9c7ONcrhoQVoazEWUAm8hNn7oaR9rd+yY7F0+ojjLZxnOaIYsk11eSeTnVyIesexmHdL0iltuteWZ3wxEIsB8Revx3KxlhudwCvINlD8Jj8L1WdiyIPcEa4QeQF/bNTht6SWFPGC/EbM+jaOAGaagVkdBGQwk=
     skip_cleanup: true
+    skip_existing: true
     on:
       all_branches: true

Dockerfile

@@ -1,11 +1,5 @@
-FROM python:3-onbuild
+FROM python:3.4
 
 RUN mkdir /aws_ir
-
-COPY requirements.txt /aws_ir/requirements.txt
-
-RUN pip install --upgrade pip
-
 ADD . /aws_ir/
-
 WORKDIR /aws_ir

aws_ir/_version.py

@@ -1 +1 @@
-__version__ = '0.3.1'
+__version__ = '0.3.3'

aws_ir/cli.py

@@ -1,10 +1,9 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 import argparse
 import logging
 import os
 import sys
 
-
 import aws_ir
 from aws_ir import __version__
 from aws_ir.libs import case

bin/aws_ir

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 import sys
 import os

docker-compose.yml

@@ -5,7 +5,3 @@ services:
     volumes:
       - ~/.aws:/root/.aws
       - .:/aws_ir/
-    environment:
-      - EDDA_SERVER=192.168.1.83
-      - EDDA_PORT=8080
-      - EDDA_BASE_URL=/api/v2/

docs/development.rst

@@ -300,28 +300,58 @@ Step 1. Fork us on Github.
   # 1. git clone
   [email protected]:<your github here>/aws_ir.git
 
+Step 2. Setup
+
   # 2. setup a virtualenv (must be python3)
   cd aws_ir
-  virtualenv env -p python3
+  python3 -m virtualenv env
 
   # 3. activate the virtualenv
   source env/bin/activate
 
-  # 4. install the requirements
-  pip3 install -r requirements.txt
 
-  # 5. run the test suite (currently pyTest soon to be pyUnit)
-  pytest tests/
+  # 4a. with setuptools
+  pip install -e .
+  python setup.py test
+  python setup.py pytest --addopts='tests/test_cli.py'
+
+  -- or --
+
+  # 4b. with local plugins and pytest-watch
+  point requirements.txt to the local version of aws_ir_plugins `-e ../aws_ir_plugins`
+  .. code-block:: bash
+    pip3 install -r requirements.txt
+    ./bin/aws_ir -h
+    ptw --runner "python setup.py test"
+
+  -- or --
+
+  #4c. Use the docker container
+  .. code-block:: bash
+    docker-compose build aws_ir
+    docker-compose run aws_ir bash
+    pip install -e .
+
 
-  # 6. If all is well you can start developing code!
+Step 3. Develop!
 
-Step 2. Develop!
 
-*Note:* aws_ir does not require an install to run out of a virtualenv.  There
-is a helper script in `bin/aws_ir` that can be called to execute your development
-version of aws_ir.
+*Note:* There is a helper script in `bin/aws_ir` that can be called to execute aws_ir.
 
 When your feature is finished simply open a PR back to us.
 
-Happy developing!  If you have any questions please do file a github issue
+If you have any questions please do file a github issue
 or e-mail [email protected] .
+
+Using testpypi
+*******************************************
+
+.. code-block:: bash
+  pip install --extra-index-url https://test.pypi.org/simple/ aws_ir==0.3.2b165
+
+To use a test build of aws_ir_plugins:
+  in setup.py:
+  - point the required version at aws_ir_plugins==0.0.3b123 (substitute the build you want)
+  - add: dependency_links=['https://test.pypi.org/simple/aws-ir-plugins/']
+
+

docs/installing.rst

@@ -5,42 +5,28 @@ Installation
 System Requirements
 *******************
 
-ThreatResponse now requires python >= 3.4.  It may still work(ish) with Python 2.7 but is not recommended due
-to security problems with cryptography libraries in Python 2.7.
+ThreatResponse requires python >= 3.4.
 
-While aws_ir is written purely in python, some of the libraries used require additional system packages.
-
-Fedora / RHEL Distributions
----------------------------
-
-* python-devel (Python 3.4+)
-* python-pip
-* libffi-devel
-* libssl-devel
-
-Debian Distributions
---------------------
-
-* python-dev (Python 3.4+)
-* python-pip
-* libffi-dev
-* libssl-dev
 
 Installing from PyPi
 ********************
 
 .. code-block:: bash
-
+   $ python3 -m virtualenv env
+   $ source/env/bin/activate
    $ pip install aws_ir
+   $ aws_ir -h
 
 
 Installing From Github
 **********************
 
 .. code-block:: bash
 
+   $ python3 -m virtualenv env
+   $ source/env/bin/activate
    $ pip install git+ssh://[email protected]/ThreatResponse/aws_ir.git@master
-   $ margaritashotgun -h
+   $ aws_ir -h
 
 Local Build and Install
 ***********************
@@ -49,9 +35,10 @@ Local Build and Install
 
    $ git clone https://github.com/ThreatResponse/aws_ir.git
    $ cd aws_ir
-   $ python setup.py
-   $ pip install dist/aws_ir-*.tar.gz
-   $ margaritashotgun -h
+   $ python3 -m virtualenv env
+   $ source/env/bin/activate
+   $ pip install .
+   $ aws_ir -h
 
 Local Execution
 ***************
@@ -62,6 +49,62 @@ In the previous two example dependencies are automatically resolved, if you simp
 
    $ git clone https://github.com/ThreatResponse/aws_ir.git
    $ cd aws_ir
+   $ python3 -m virtualenv env
+   $ source/env/bin/activate
    $ pip install -r requirements.txt
    $ ./bin/aws_ir -h
 
+Using Docker
+************
+
+.. code-block:: bash
+
+   $ git clone https://github.com/ThreatResponse/aws_ir.git
+   $ cd aws_ir
+   $ docker-compose build aws_ir
+   $ docker-compose run aws_ir bash
+   $ pip install .
+
+
+AWS Credentials Using MFA and AssumeRole
+*****************************************
+
+Many users of aws_ir have requested the ability to use the tooling with mfa and
+assumeRole functionality.  While we don't natively support this yet v0.3.0 sets
+the stage to do this natively by switching to boto-session instead of thick clients.
+
+For now if you need to use the tool with MFA we recommend:
+
+`https://pypi.python.org/pypi/awsmfa/0.2.4 <https://pypi.python.org/pypi/awsmfa/0.2.4>`_.
+
+.. code-block:: bash
+
+    aws-mfa \
+    --device arn:aws:iam::12345678:mfa/bobert \
+    -assume-role arn:aws:iam::12345678:role/ResponderRole \
+    --role-session-name \"bobert-ir-session\"
+
+awsmfa takes a set of long lived access keys from a boto profile called [default-long-lived]
+and uses those to generate temporary session tokens that are automatically put into
+the default boto profile.  This ensures that any native tooling that doesn't support
+MFA + AssumeRole can still leverage MFA and short lived credentials for access.
+
+
+Some Linux distributions require additional system packages
+***********************************************************
+
+Fedora / RHEL Distributions
+---------------------------
+
+* python-devel (Python 3.4+)
+* python-pip
+* libffi-devel
+* libssl-devel
+
+Debian Distributions
+--------------------
+
+* python-dev (Python 3.4+)
+* python-pip
+* libffi-dev
+* libssl-dev

docs/quickstart.rst

@@ -7,15 +7,13 @@ First, :doc:`Install aws_ir <installing>`.
 Installation
 ************
 
-Using a python3 virtualenv is highly recommended.
-
 .. code-block:: bash
 
-    $ virtualenv aws_ir_env -p python3
-    $ source/aws_ir_env/bin/activate
-    $ pip3 install aws_ir
+    $ python3 -m virtualenv env
+    $ source/env/bin/activate
+    $ pip install aws_ir
 
-Or see `installing <https://aws_ir.readthedocs.io/en/latest/installing.html>`__.
+For other installation options see: `installing <https://aws_ir.readthedocs.io/en/latest/installing.html>`__.
 
 AWS Credentials
 ***************
@@ -26,37 +24,18 @@ Ensure aws credentials are configured under the user running aws_ir as documente
 Setup Roles with Cloudformation
 *************************************
 
-A cloudformation stack has been provided to setup a group and a responder role.  *Note that this roles has a constraint
-that all your responders use MFA.*
+A cloudformation stack has been provided to setup a group and a responder role.
 
 Simply create the stack available at:
 
 `https://github.com/ThreatResponse/aws_ir/blob/master/cloudformation/responder-role.yml <https://github.com/ThreatResponse/aws_ir/blob/master/cloudformation/responder-role.yml>`_.
 
 Then add all your users to the IncidentResponders group.  After that you're good to go!
 
-AWS Credentials Using MFA and AssumeRole
-*****************************************
-
-Many users of aws_ir have requested the ability to use the tooling with mfa and
-assumeRole functionality.  While we don't natively support this yet v0.3.0 sets
-the stage to do this natively by switching to boto-session instead of thick clients.
-
-For now if you need to use the tool with MFA we recommend:
-
-`https://pypi.python.org/pypi/awsmfa/0.2.4 <https://pypi.python.org/pypi/awsmfa/0.2.4>`_.
-
+*Note that this roles has a constraint that all your responders use MFA.*
 .. code-block:: bash
+  aws:MultiFactorAuthPresent: 'true'
 
-    aws-mfa \
-    --device arn:aws:iam::12345678:mfa/bobert \
-    -assume-role arn:aws:iam::12345678:role/ResponderRole \
-    --role-session-name \"bobert-ir-session\"
-
-awsmfa takes a set of long lived access keys from a boto profile called [default-long-lived]
-and uses those to generate temporary session tokens that are automatically put into
-the default boto profile.  This ensures that any native tooling that doesn't support
-MFA + AssumeRole can still leverage MFA and short lived credentials for access.
 
 Key Compromise
 **************

requirements.txt

@@ -1,17 +1,28 @@
 jinja2
-pytest
-pytest-cov
 boto3>=1.3.0
 progressbar-latest
 gnupg
 requests
 structlog
 pytz
 logutils==0.3.3
+margaritashotgun>=0.4.1
+pluginbase
+
+mock
 moto
+pytest
+pytest-cov
 pytest-moto
-mock
-margaritashotgun
-pluginbase
-aws-ir-plugins
+pytest-watch
+
+### Latest Stable
+aws_ir_plugins==0.0.3
+
+### Test build
+# --extra-index-url https://test.pypi.org/simple
+# aws-ir-plugins==0.0.3b52
+
+### Local
+# -e ../aws_ir_plugins
 

setup.cfg

@@ -1,6 +1,12 @@
+[aliases]
+test=pytest
+
 [metadata]
 description-file = README.rst
 
 [flake8]
 ignore=E402,H238
 max-line-length=99
+
+[tool:pytest]
+[pytest-watch]