Resolve #14

.travis.yml

@@ -1,9 +1,13 @@
+sudo: required
 language: python
-python:
-  - "3.4"
+services:
+  - docker
 # cd into the api project
-before_install: cd api
-install: pip install -r requirements.txt
-# change from /api to /api/api
-before_script: cd api
-script: python manage.py test
+before_install:
+  - make api
+script:
+  - make test-api
+after_success:
+  - docker login -e="$DOCKER_EMAIL" -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" https://gcr.io
+  - docker tag hansard_api gcr.io/hansard-1012/hansard_api:$(git rev-parse HEAD)
+  - docker push gcr.io/hansard-1012/hansard_api:$(git rev-parse HEAD)

Makefile

@@ -0,0 +1,58 @@
+all: iepy api
+
+stop: docker-stop-api docker-stop-iepy
+
+# api ------------------
+
+docker-build-api:
+	docker-compose -f docker-compose-api.yml build
+
+docker-up-api: docker-rm-api docker-build-api
+	docker-compose -f docker-compose-api.yml up -d
+
+docker-stop-api:
+	docker-compose -f docker-compose-api.yml stop
+
+docker-rm-api: docker-stop-api
+	docker-compose -f docker-compose-api.yml rm -f
+
+wait-for-postgres: docker-up-api
+	docker exec hansard_db_1 bash -c "while ! pg_isready; do echo \"$(date) - waiting for database to start\"; sleep 3; done"
+
+migrate-api: docker-up-api wait-for-postgres
+	docker exec hansard_api_1 python api/manage.py migrate
+
+createsuperuser-api: migrate-api
+	docker exec hansard_api_1 bash -c "echo \"from users.models import User; User.objects.create_superuser('admin', '[email protected]', 'sn0wb1rd')\" | python api/manage.py shell"
+
+init-data: migrate-api
+	docker exec -ti hansard_api_1 python api/manage.py init_data
+
+test-api:
+	docker exec -ti hansard_api_1 bash -c "cd api && python manage.py test"
+
+api: createsuperuser-api
+	echo "api running on localhost:8000"
+
+#--------------------
+
+docker-build-iepy:
+	docker-compose -f docker-compose-iepy.yml build
+
+docker-up-iepy: docker-stop-iepy docker-build-iepy
+	docker-compose -f docker-compose-iepy.yml up -d
+
+docker-stop-iepy:
+	docker-compose -f docker-compose-iepy.yml stop
+
+docker-rm-iepy: docker-stop-iepy
+	docker-compose -f docker-compose-iepy.yml rm -f
+
+migrate-iepy: docker-up-iepy
+	docker exec hansard_iepy_1 python /hansard/bin/manage.py migrate
+
+createsuperuser-iepy: migrate-iepy
+	docker exec hansard_iepy_1 bash -c "echo \"from django.contrib.auth.models import User; User.objects.create_superuser('admin', '[email protected]', 'sn0wb1rd')\" | python /hansard/bin/manage.py shell"
+
+iepy: createsuperuser-iepy
+	echo "iepy running on localhost:8001"

api/api/comments/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

api/api/comments/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class CommentsConfig(AppConfig):
+    name = 'comments'

api/api/comments/models.py

@@ -0,0 +1,3 @@
+from django.db import models
+
+# Create your models here.

api/api/comments/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

api/api/comments/views.py

@@ -0,0 +1,3 @@
+from django.shortcuts import render
+
+# Create your views here.

api/api/main/settings.py

@@ -82,12 +82,23 @@
 # Database
 # https://docs.djangoproject.com/en/1.9/ref/settings/#databases
 
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
+if os.environ.get('CONTAINERIZED'):
+    DATABASES = {
+        'default': {
+            'ENGINE': 'django.db.backends.postgresql_psycopg2',
+            'NAME': 'postgres',
+            'USER': 'postgres',
+            'HOST': 'db',
+            'PORT': 5432,
+        }
+    }
+else:
+    DATABASES = {
+        'default': {
+            'ENGINE': 'django.db.backends.sqlite3',
+            'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
+        }
     }
-}
 
 
 # Password validation

api/api/topics/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

api/api/topics/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class TopicsConfig(AppConfig):
+    name = 'topics'

api/api/topics/models.py

@@ -0,0 +1,3 @@
+from django.db import models
+
+# Create your models here.

api/api/topics/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

api/api/topics/views.py

@@ -0,0 +1,3 @@
+from django.shortcuts import render
+
+# Create your views here.

api/requirements.txt

@@ -13,11 +13,16 @@ inflection==0.3.1
 ipdb==0.9.0
 ipython==4.1.2
 ipython-genutils==0.1.0
+nltk==3.2
+numpy==1.11.0
 oauthlib==1.0.3
 path.py==8.1.2
 pexpect==4.0.1
 pickleshare==0.6
+psycopg2==2.6.1
 ptyprocess==0.5.1
+requests==2.9.1
+sets==0.2.0
 simplegeneric==0.8.1
 six==1.10.0
 traitlets==4.2.1

docker-compose-api.yml

@@ -0,0 +1,16 @@
+db:
+  image: postgres:9.6
+api:
+  build: ./api
+  command: python api/manage.py runserver 0.0.0.0:8000
+  ports:
+    - "8000:8000"
+  volumes:
+    - ./api:/app
+  links:
+    - db
+# client:
+#   build: ./client
+#   command: ember server
+#  links:
+#    - api

docker-compose-iepy.yml

@@ -0,0 +1,8 @@
+dbiepy:
+  image: postgres:9.6
+iepy:
+  build: ./iepy
+  ports:
+    - "8001:8001"
+  links:
+    - dbiepy

iepy/Dockerfile

@@ -0,0 +1,21 @@
+from python:3.4
+
+RUN pip install numpy
+
+RUN pip install iepy
+
+RUN apt-get update && apt-get install -y openjdk-7-jre
+
+ENV JAVAHOME=/usr/bin/java
+
+RUN mkdir /root/.config
+
+RUN iepy --download-third-party-data
+
+COPY ./hansard /hansard
+
+RUN pip install psycopg2
+
+EXPOSE 8001
+
+CMD python /hansard/bin/manage.py runserver 0.0.0.0:8001

iepy/docker-compose.yml

@@ -0,0 +1,10 @@
+db:
+  image: postgres
+iepy:
+  build: .
+  ports:
+    - "8000:8000"
+  volumes:
+    - ./hansard:/hansard
+  links:
+    - db

iepy/hansard/__init__.py

@@ -0,0 +1 @@
+from . import rules

iepy/hansard/bin/csv_to_iepy.py

@@ -0,0 +1,28 @@
+"""
+IEPY database loader from csv file
+
+Usage:
+    csv_to_iepy.py <filename>
+    csv_to_iepy.py -h | --help
+
+The <filename> argument can be a .csv file or a .csv.gz file containing the
+corpus in two columns: 'freebase_mid' and 'description'.
+
+Options:
+  -h --help             Show this screen
+  --version             Version number
+"""
+
+import logging
+
+from docopt import docopt
+
+import iepy
+iepy.setup(__file__)
+from iepy.utils import csv_to_iepy
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO, format='%(message)s')
+    opts = docopt(__doc__, version=iepy.__version__)
+    filepath = opts["<filename>"]
+    csv_to_iepy(filepath)

iepy/hansard/bin/gazettes_loader.py

@@ -0,0 +1,76 @@
+"""
+IEPY gazettes loader
+
+Usage:
+    gazettes_loader.py <filename>
+
+
+The <filename> argument can be a .csv file or a .csv.gz file containing the
+gazettes in two columns: 'literal' and 'class'.
+
+
+Options:
+  -h --help             Show this screen
+"""
+
+import sys
+import csv
+import gzip
+import logging
+from operator import itemgetter
+
+from django.db import IntegrityError
+from docopt import docopt
+
+import iepy
+iepy.setup(__file__)
+from iepy.data.models import EntityKind, GazetteItem
+
+logging.basicConfig(level=logging.INFO, format='%(message)s')
+
+
+def add_gazettes_from_csv(filepath):
+    if filepath.endswith(".gz"):
+        fin = gzip.open(filepath, "rt")
+    else:
+        fin = open(filepath, "rt")
+    reader = csv.DictReader(fin)
+
+    expected_fnames = ['literal', 'class']
+    if not set(reader.fieldnames).issuperset(expected_fnames):
+        msg = "Couldn't find the expected field names on the provided csv: {}"
+        sys.exit(msg.format(expected_fnames))
+
+    _create_gazette_entries(
+        itemgetter(*expected_fnames)(line) for line in reader
+    )
+
+
+def _create_gazette_entries(entries_list):
+    kind_cache = {}
+    created = 0
+    for literal, kind_name in entries_list:
+        literal = literal.strip()
+        kind_name = kind_name.strip()
+        kind = kind_cache.get(kind_name)
+        if kind is None:
+            kind, _ = EntityKind.objects.get_or_create(name=kind_name)
+            kind_cache[kind_name] = kind
+        gazette = GazetteItem(text=literal, kind=kind)
+
+        try:
+            gazette.save()
+        except IntegrityError as error:
+            logging.warn(
+                "Gazette '{}' of class '{}' not loaded, literal already existed".format(
+                literal, kind_name))
+            print(error)
+        finally:
+            created += 1
+    print('Created {} new gazette items'.format(created))
+
+
+if __name__ == "__main__":
+    opts = docopt(__doc__, version=iepy.__version__)
+    fname = opts["<filename>"]
+    add_gazettes_from_csv(fname)