Merge branch 'develop'

ddr/DDR/config.py

@@ -8,7 +8,7 @@
 CONFIG_FILES = [
     '/etc/ddr/ddr.cfg',       '/etc/ddr/local.cfg',
     '/etc/ddr/ddrlocal.cfg',  '/etc/ddr/ddrlocal-local.cfg',
-    '/etc/ddr/idservice.cfg', '/etc/ddr/idservice-local.cfg',
+    '/etc/ddr/ddridservice.cfg', '/etc/ddr/ddridservice-local.cfg',
 ]
 
 class NoConfigError(Exception):
@@ -109,7 +109,7 @@ def _parse_alt_timezones(text):
 WORKBENCH_URL = CONFIG.get('workbench','workbench_url')
 WORKBENCH_USERINFO = CONFIG.get('workbench','workbench_userinfo_url')
 
-IDSERVICE_API_BASE = CONFIG.get('idservice', 'api_base')
+IDSERVICE_API_BASE = CONFIG.get('cmdln', 'idservice_api_base')
 IDSERVICE_LOGIN_URL = IDSERVICE_API_BASE + '/rest-auth/login/'
 IDSERVICE_LOGOUT_URL = IDSERVICE_API_BASE + '/rest-auth/logout/'
 IDSERVICE_USERINFO_URL = IDSERVICE_API_BASE + '/rest-auth/user/'

ddr/DDR/docstore.py

@@ -135,6 +135,14 @@ def all_list_fields():
                 LIST_FIELDS.append(f)
     return LIST_FIELDS
 
+def load_json(path):
+    with open(path, 'r') as f:
+        try:
+            data = json.loads(f.read())
+        except json.errors.JSONDecodeError:
+            raise Exception('simplejson.errors.JSONDecodeError reading %s' % path)
+        return data
+
 class InvalidPage(Exception):
     pass
 class PageNotAnInteger(InvalidPage):
@@ -511,8 +519,7 @@ def _repo_org(self, path, doctype, remove=False):
         seealso DDR.models.common.DDRObject.to_esobject
         """
         # get and validate file
-        with open(path, 'r') as f:
-            data = json.loads(f.read())
+        data = load_json(path)
         if (not (data.get('id') and data.get('repo'))):
             raise Exception('Data file is not well-formed.')
         oi = Identifier(id=data['id'])
@@ -574,8 +581,7 @@ def narrators(self, path):
         @returns: dict
         """
         DOC_TYPE = 'narrator'
-        with open(path, 'r') as f:
-            data = json.loads(f.read())
+        data = load_json(path)
         for document in data['narrators']:
             document['model'] = 'narrator'
             has_published = document.get('has_published', '')
@@ -1212,8 +1218,7 @@ def _make_coll_ent(path):
         p = {'id':None,
              'public':None,
              'status':None,}
-        with open(path, 'r') as f:
-            data = json.loads(f.read())
+        data = load_json(path)
         for field in data:
             fname = field.keys()[0]
             if fname in p.keys():
@@ -1286,14 +1291,13 @@ def _publishable(paths, parents, force=False):
         # TODO knows way too much about JSON data format
         public = None; status = None
         jsonpath = d['identifier'].path_abs('json')
-        with open(jsonpath, 'r') as f:
-            document = json.loads(f.read())
-            for field in document:
-                for k,v in field.iteritems():
-                    if k == 'public':
-                        public = v
-                    if k == 'status':
-                        status = v
+        document = load_json(jsonpath)
+        for field in document:
+            for k,v in field.iteritems():
+                if k == 'public':
+                    public = v
+                if k == 'status':
+                    status = v
         if public and (public not in PUBLIC_OK):
             d['action'] = 'SKIP'
             d['note'] = 'not public'

ddr/DDR/idservice.py

@@ -27,12 +27,12 @@ class IDServiceClient():
     
     >>> from DDR import identifier
     >>> oidentifier = identifier.Identifier('ddr-test')
-    >>> ic.next_id(oidentifier, 'collection')
+    >>> ic.next_object_id(oidentifier, 'collection')
     201,'ddr-test-123'
     >>> cidentifier = identifier.Identifier('ddr-test-123')
-    >>> ic.next_id(cidentifier, 'entity')
+    >>> ic.next_object_id(cidentifier, 'entity')
     201,'ddr-test-123-1'
-    >>> ic.next_id(cidentifier, 'entity')
+    >>> ic.next_object_id(cidentifier, 'entity')
     201,'ddr-test-123-1'
     
     >>> ic.resume('gjost', u'9b68187429be07506dae2d1a493b74afd4ef7c35')

ddr/bin/ddr-idexport

@@ -71,7 +71,10 @@ def main():
     print(collection)
 
     data = [
-         i.id for i in collection.identifiers()
+        i.id
+        for i in collection.identifiers()
+        # only export IDs that idservice can ingest
+        if i.id in identifier.NEXTABLE_MODELS
     ]
 
     with open(path, 'w') as f: