Merge branch 'master' into sunu-graph-api

Author: pudo

.travis.yml

@@ -35,8 +35,10 @@ script:
 after_success:
   - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
   # push tag as well
-  - if [[ $TRAVIS_TAG = $TRAVIS_BRANCH ]]; then
+  - if [[ $TRAVIS_BRANCH = "master" ]]; then
       make TAG=latest build-full docker-push ;
+    fi
+  - if [[ $TRAVIS_TAG = $TRAVIS_BRANCH ]]; then
       make TAG=$TRAVIS_TAG build-full docker-push ;
     fi
   - if [[ $TRAVIS_BRANCH = "occrp/production" ]]; then

Dockerfile

@@ -50,8 +50,7 @@ COPY . /aleph
 WORKDIR /aleph
 ENV PYTHONPATH /aleph
 RUN pip install -e /aleph
-RUN cd /usr/local/lib/python3.6/dist-packages && python3 /aleph/setup.py develop
-
+# RUN cd /usr/local/lib/python3.6/dist-packages && python3 /aleph/setup.py develop
 
 # Configure some docker defaults:
 ENV C_FORCE_ROOT=true \

Makefile

@@ -8,13 +8,14 @@ services:
 	$(COMPOSE) up -d --remove-orphans \
 		rabbitmq postgres elasticsearch \
 		convert-document extract-entities \
-		extract-countries recognize-text
+		recognize-text
 
 shell: services    
 	$(DEVDOCKER) /bin/bash
 
 test:
 	$(DEVDOCKER) contrib/test.sh
+	$(COMPOSE) run --rm extract-entities pytest
 
 upgrade: build
 	$(COMPOSE) up -d postgres elasticsearch
@@ -49,18 +50,18 @@ build:
 	docker build --cache-from alephdata/aleph-convert-document -t alephdata/aleph-convert-document:$(TAG) services/convert-document
 	docker build --cache-from alephdata/aleph-recognize-text -t alephdata/aleph-recognize-text:$(TAG) services/recognize-text
 	docker build --cache-from alephdata/aleph-extract-entities -t alephdata/aleph-extract-entities:$(TAG) services/extract-entities
-	docker build --cache-from alephdata/aleph-extract-countries -t alephdata/aleph-extract-countries:$(TAG) services/extract-countries
 
-build-full: build
-	docker build -t alephdata/aleph-ui-production:$(TAG) ui/production
+build-ui:
+	docker build -t alephdata/aleph-ui-production:$(TAG) -f ui/Dockerfile.production ui
+
+build-full: build build-ui
 
 docker-pull:
 	docker pull alephdata/aleph
 	docker pull alephdata/aleph-ui
 	docker pull alephdata/aleph-convert-document
 	docker pull alephdata/aleph-recognize-text
 	docker pull alephdata/aleph-extract-entities
-	docker pull alephdata/aleph-extract-countries
 
 docker-push:
 	docker push alephdata/aleph:$(TAG)
@@ -69,7 +70,6 @@ docker-push:
 	docker push alephdata/aleph-convert-document:$(TAG)
 	docker push alephdata/aleph-recognize-text:$(TAG)
 	docker push alephdata/aleph-extract-entities:$(TAG)
-	docker push alephdata/aleph-extract-countries:$(TAG)
 
 dev: 
 	pip install -q transifex-client bumpversion babel

aleph/analyze/extract_country.py

@@ -18,6 +18,11 @@ class EntityExtractor(EntityAnalyzer, TextIterator, ServiceClientMixin):
         ExtractedEntity.PERSON: DocumentTag.TYPE_PERSON,
         ExtractedEntity.ORGANIZATION: DocumentTag.TYPE_ORGANIZATION,
         ExtractedEntity.COMPANY: DocumentTag.TYPE_ORGANIZATION,
+        ExtractedEntity.PHONE: DocumentTag.TYPE_PHONE,
+        ExtractedEntity.EMAIL: DocumentTag.TYPE_EMAIL,
+        ExtractedEntity.IBAN: DocumentTag.TYPE_IBAN,
+        ExtractedEntity.IPADDRESS: DocumentTag.TYPE_IP,
+        ExtractedEntity.LOCATION: DocumentTag.TYPE_LOCATION
     }
 
     def __init__(self):
@@ -31,10 +36,14 @@ def extract(self, collector, document):
             texts = self.text_iterator(document)
             entities = service.Extract(texts)
             for entity in entities.entities:
+                if entity.type == ExtractedEntity.COUNTRY:
+                    document.add_country(entity.label)
+                if entity.type == ExtractedEntity.LANGUAGE:
+                    document.add_language(entity.label)
                 type_ = self.TYPES.get(entity.type)
-                if type_ is None:
-                    continue
-                collector.emit(entity.label, type_, weight=entity.weight)
+                # log.info('%s: %s', entity.label, type_)
+                if type_ is not None:
+                    collector.emit(entity.label, type_, weight=entity.weight)
             log.info('Extracted %s entities.', len(collector))
         except self.Error as e:
             log.warning("gRPC [%s]: %s", e.code(), e.details())

aleph/analyze/extract_entity.py

@@ -32,7 +32,6 @@ def upgrade_search():
         es.indices.create(index, body=body, ignore=[404, 400])
         # es.indices.put_mapping(index=index, doc_type='doc', body=mapping)
         es.indices.open(index=index, ignore=[400, 404])
-        es.indices.refresh(index=index, ignore=[400, 404])
         es.indices.clear_cache(index=index, ignore=[400, 404])
 
 
@@ -42,4 +41,4 @@ def delete_index():
 
 def clear_index():
     q = {'query': {'match_all': {}}}
-    es.delete_by_query(index=all_indexes(), body=q, refresh=True)
+    es.delete_by_query(index=all_indexes(), body=q)

aleph/analyze/language.py

@@ -1,14 +1,13 @@
 import logging
-import exactitude
 from pprint import pprint  # noqa
 from normality import normalize
+from followthemoney import types
 
 from aleph.core import es
 from aleph.model import Entity, Collection
 from aleph.index.core import collections_index, entities_index, records_index
 from aleph.index.util import query_delete, query_update, unpack_result
-from aleph.index.util import index_safe, index_form, refresh_index
-from aleph.index.util import search_safe
+from aleph.index.util import index_safe, index_form, search_safe
 
 log = logging.getLogger(__name__)
 
@@ -86,19 +85,17 @@ def index_collection(collection):
     if countries is None or not len(countries):
         countries = aggregations['countries']['buckets']
         countries = [c['key'] for c in countries]
-    data['countries'] = exactitude.countries.normalize_set(countries)
+    data['countries'] = types.countries.normalize_set(countries)
 
     languages = collection.languages
     if languages is None or not len(languages):
         languages = aggregations['languages']['buckets']
         languages = [c['key'] for c in languages]
-    data['languages'] = exactitude.languages.normalize_set(languages)
+    data['languages'] = types.languages.normalize_set(languages)
 
     texts.extend([normalize(t, ascii=True) for t in texts])
     data['text'] = index_form(texts)
-    data = index_safe(collections_index(), collection.id, data)
-    refresh_index(index=collections_index())
-    return data
+    return index_safe(collections_index(), collection.id, data)
 
 
 def get_collection(collection_id):
@@ -127,7 +124,6 @@ def delete_collection(collection_id):
     """Delete all documents from a particular collection."""
     q = {'ids': {'values': str(collection_id)}}
     query_delete(collections_index(), q)
-    refresh_index(index=collections_index())
 
 
 def delete_entities(collection_id):
@@ -143,7 +139,6 @@ def delete_documents(collection_id):
     """Delete documents from a collection."""
     records_query = {'term': {'collection_id': collection_id}}
     query_delete(records_index(), records_query)
-    refresh_index(index=records_index())
     query = {'bool': {
         'must': [
             {'term': {'schemata': 'Document'}},

aleph/analyze/regex.py

@@ -1,16 +1,12 @@
 import logging
 from pprint import pprint  # noqa
-from collections import defaultdict
 
-from aleph.core import celery, db
-from aleph.model import Document, DocumentTag
-from aleph.index.core import records_index
+from aleph.core import celery
+from aleph.model import Document
 from aleph.index.records import index_records, clear_records
 from aleph.index.entities import delete_entity, index_single
-from aleph.index.util import refresh_index
 
 log = logging.getLogger(__name__)
-MAX_TAGS_PER_DOCUMENT = 1000
 
 
 @celery.task()
@@ -23,31 +19,11 @@ def index_document_id(document_id):
     index_records(document)
 
 
-def generate_tags(document):
-    """Transform document tag objects into normalized tag snippets."""
-    if document.status == Document.STATUS_PENDING:
-        return []
-    tags = defaultdict(set)
-    q = db.session.query(DocumentTag)
-    q = q.filter(DocumentTag.document_id == document.id)
-    q = q.order_by(DocumentTag.weight.desc())
-    q = q.limit(MAX_TAGS_PER_DOCUMENT)
-    for tag in q.all():
-        type_ = DocumentTag.TYPES[tag.type]
-        values = type_.normalize(tag.text,
-                                 cleaned=True,
-                                 countries=document.countries)
-        if tag.field is not None:
-            tags[tag.field].update(values)
-
-    # pprint(dict(tags))
-    return tags.items()
-
-
 def index_document(document):
     name = document.name
     log.info("Index document [%s]: %s", document.id, name)
-    data = {
+    data = document.to_schema_entity()
+    data.update({
         'status': document.status,
         'content_hash': document.content_hash,
         'foreign_id': document.foreign_id,
@@ -77,7 +53,7 @@ def index_document(document):
         'columns': document.columns,
         'ancestors': document.ancestors,
         'children': document.children.count()
-    }
+    })
 
     texts = list(document.texts)
     texts.extend(document.columns)
@@ -90,17 +66,9 @@ def index_document(document):
             'title': document.parent.title,
         }
 
-    for (field, values) in generate_tags(document):
-        if field not in data:
-            data[field] = list(values)
-        else:
-            data[field].extend(values)
-        texts.extend(values)
-
     return index_single(document, data, texts)
 
 
 def delete_document(document_id):
     clear_records(document_id)
     delete_entity(document_id)
-    refresh_index(index=records_index())