Merge pull request #70 from uc-cdis/feat/version_string

feat(version_string): add optinal version string
uc-cdis · Jan 18, 2018 · 36bc61a · 36bc61a
2 parents 86bda97 + dc8d351
commit 36bc61a
Show file tree

Hide file tree

Showing 10 changed files with 114 additions and 24 deletions.
diff --git a/README.md b/README.md
@@ -97,6 +97,7 @@ Content-Type: application/json
   "form": "object",
   "size": 123,
   "file_name": "abc.txt",
+  "version": "ver_123",
   "urls": ["s3://endpointurl/bucket/key"],
   "hashes": {"md5": "8b9942cf415384b27cadf1f4d2d682e5"}
 }
@@ -107,6 +108,7 @@ Content-Type: application/json
 | form      | Can be one of 'object', 'container', 'multipart' |
 | size      |  File size in bytes (commonly computed via wc -c filename) |
 | file_name |  Optional file name |
+| version | Optional version string |
 | urls      | URLs where the datafile is stored, can be multiple locations both internally and externally |
 | hashes    |  Dictionary is a string:string datastore supporting md5, sha, sha256, sha512 hash types |
 
@@ -189,6 +191,7 @@ HTTP/1.1 200 OK
   "form": "object",
   "size": 123,
   "file_name": "abc.txt",
+  "version": "ver_123",
   "urls": ["s3://endpointurl/bucket/key"],
   "hashes": {"md5": "8b9942cf415384b27cadf1f4d2d682e5"},
   "created_date": "Fri, 17 Nov 2017 06:07:29 GMT",
@@ -204,6 +207,7 @@ HTTP/1.1 200 OK
 | form      | Can be one of 'object', 'container', 'multipart' |
 | size      |  File size in bytes |
 | file_name |  Optional file name |
+| version | Optional version string |
 | urls      | URLs where the datafile is stored, can be multiple locations both internally and externally |
 | hashes    |  Dictionary is a string:string datastore supporting md5, sha, sha256, sha512 hash types |
 | created_date | File created datetime  |
@@ -218,6 +222,7 @@ Content-Type: application/json
   "form": "object",
   "size": 123,
   "file_name": "abc.txt",
+  "version": "ver_123",
   "urls": ["s3://endpointurl/bucket/key"],
   "hashes": {"md5": "8b9942cf415384b27cadf1f4d2d682e5"}
 }
@@ -228,12 +233,13 @@ Content-Type: application/json
 | form      | Can be one of 'object', 'container', 'multipart' |
 | size      |  File size in bytes (commonly computed via wc -c filename) |
 | file_name |  Optional file name |
+| version | Optional version string |
 | urls      | URLs where the datafile is stored, can be multiple locations both internally and externally |
 | hashes    |  Dictionary is a string:string datastore supporting md5, sha, sha256, sha512 hash types |
 
 Curl example:
 ```
-curl http://localhost/index/d5d9a196-f36d-4ab8-bdca-a989e0f21c00? -u test:test -H "Content-type: application/json" -X POST -d '{"form": "object","size": 123,"file_name": "abc.txt", "urls": ["s3://endpointurl/bucket/key"],"hashes": {"md5": "8b9942cf415384b27cadf1f4d2d682e5"}}'
+curl http://localhost/index/d5d9a196-f36d-4ab8-bdca-a989e0f21c00? -u test:test -H "Content-type: application/json" -X POST -d '{"form": "object","size": 123,"file_name": "abc.txt","version": "ver_123","urls": ["s3://endpointurl/bucket/key"],"hashes": {"md5": "8b9942cf415384b27cadf1f4d2d682e5"}}'
 ```
 
 ***Response***
@@ -273,6 +279,7 @@ HTTP/1.1 200 OK
   "form": "object",
   "size": 123,
   "file_name": "abc.txt",
+  "version": "ver_123",
   "urls": ["s3://endpointurl/bucket/key"],
   "hashes": {"md5": "8b9942cf415384b27cadf1f4d2d682e5"},
   "created_date": "Tue, 05 Dec 2017 21:02:59 GMT",
@@ -287,6 +294,7 @@ HTTP/1.1 200 OK
 | form      | Can be one of 'object', 'container', 'multipart' |
 | size      |  File size in bytes |
 | file_name |  File name |
+| version | Version string |
 | urls      | URLs where the datafile is stored, can be multiple locations both internally and externally |
 | hashes    |  Dictionary is a string:string datastore supporting md5, sha, sha256, sha512 hash types |
 | created_date  | File created datetime |
@@ -314,6 +322,7 @@ HTTP/1.1 200 OK
       "rev": "0984a150",
       "size": 123,
       "file_name": "abc.txt",
+      "version": "ver_123",
       "created_date": "Fri, 17 Nov 2017 06:11:18 GMT"
       "updated_date": "Fri, 17 Nov 2017 06:11:18 GMT",
       "urls": ["s3://endpointurl/bucket/key"]
@@ -326,6 +335,7 @@ HTTP/1.1 200 OK
       "hashes": {"md5": "8b9942cf415384b27cadf1f4d2d682e5"},
       "rev": "fec0ce30", "size": 123,
       "file_name": "abc.txt",
+      "version": "ver_123",
       "created_date": "Fri, 17 Nov 2017 06:17:39 GMT",
       "updated_date": "Fri, 17 Nov 2017 06:17:39 GMT",
       "urls": ["s3://endpointurl/bucket/key"]
@@ -341,6 +351,7 @@ HTTP/1.1 200 OK
 | form      | Can be one of 'object', 'container', 'multipart' |
 | size      |  File size in bytes |
 | file_name |  File name |
+| version | Version string |
 | urls      | URLs where the datafile is stored, can be multiple locations both internally and externally |
 | hashes    |  Dictionary is a string:string datastore supporting md5, sha, sha256, sha512 hash types |
 | created_date | File created datetime  |

diff --git a/indexd/index/blueprint.py b/indexd/index/blueprint.py
@@ -69,6 +69,8 @@ def get_index():
 
     file_name = flask.request.args.get('file_name')
 
+    version = flask.request.args.get('version')
+
     hashes = flask.request.args.getlist('hash')
     hashes = {h: v for h, v in map(lambda x: x.split(':', 1), hashes)}
 
@@ -83,6 +85,7 @@ def get_index():
         limit=limit,
         size=size,
         file_name=file_name,
+        version=version,
         urls=urls,
         hashes=hashes,
     )
@@ -93,6 +96,7 @@ def get_index():
         'start': start,
         'size': size,
         'file_name': file_name,
+        'version': version,
         'urls': urls,
         'hashes': hashes,
     }
@@ -178,12 +182,14 @@ def post_index_record():
     hashes = flask.request.json['hashes']
     file_name = flask.request.json.get('file_name')
     metadata = flask.request.json.get('metadata')
+    version = flask.request.json.get('version')
 
     did, rev, baseid = blueprint.index_driver.add(
         form,
         size=size,
         file_name=file_name,
         metadata=metadata,
+        version=version,
         urls=urls,
         hashes=hashes,
     )
@@ -210,13 +216,15 @@ def put_index_record(record):
 
     rev = flask.request.args.get('rev')
     file_name = flask.request.json.get('file_name')
+    version = flask.request.json.get('version')
     urls = flask.request.json.get('urls')
 
     did, baseid, rev = blueprint.index_driver.update(
         record,
         rev,
         file_name=file_name,
-        urls=urls
+        version=version,
+        urls=urls,
     )
 
     ret = {
@@ -255,6 +263,7 @@ def add_index_record_version(record):
     hashes = flask.request.json['hashes']
     file_name = flask.request.json.get('file_name', None)
     metadata = flask.request.json.get('metadata', None)
+    version = flask.request.json.get('version', None)
 
     did, baseid,rev = blueprint.index_driver.add_version(
         record,
@@ -263,6 +272,7 @@ def add_index_record_version(record):
         urls=urls,
         file_name=file_name,
         metadata=metadata,
+        version=version,
         hashes=hashes,
     )
 

diff --git a/indexd/index/driver.py b/indexd/index/driver.py
@@ -17,7 +17,7 @@ def __init__(self, conn, **config):
     @abc.abstractmethod
     def ids(
             self, limit=100, start=None,
-            size=None, urls=None, hashes=None, file_name=None):
+            size=None, urls=None, hashes=None, file_name=None, version=None):
         '''
         Returns a list of records stored by the backend.
         '''
@@ -31,7 +31,7 @@ def hashes_to_urls(self, size, hashes, start=0, limit=100):
         raise NotImplementedError('TODO')
 
     @abc.abstractmethod
-    def add(self, form, size=None, file_name=None, metadata=None, urls=None, hashes=None):
+    def add(self, form, size=None, urls=None, hashes=None, file_name=None, metadata=None, version=None):
         '''
         Creates record for given data.
         '''
@@ -45,7 +45,7 @@ def get(self, did):
         raise NotImplementedError('TODO')
 
     @abc.abstractmethod
-    def update(self, did, rev, urls=None, file_name=None):
+    def update(self, did, rev, urls=None, file_name=None, version=None):
         '''
         Updates record with new values.
         '''
@@ -61,7 +61,7 @@ def delete(self, did, rev):
     @abc.abstractmethod
     def add_version(
             self, did, form, size=None,
-            file_name=None, metadata=None, urls=None, hashes=None):
+            file_name=None,  metadata=None, urls=None, hashes=None, version=None):
         '''
         Add a record version given did
         '''

diff --git a/indexd/index/drivers/alchemy.py b/indexd/index/drivers/alchemy.py
@@ -17,7 +17,7 @@
 from indexd.utils import migrate_database, init_schema_version, is_empty_database
 from sqlalchemy.exc import ProgrammingError
 
-CURRENT_SCHEMA_VERSION = 3
+CURRENT_SCHEMA_VERSION = 4
 Base = declarative_base()
 
 
@@ -57,6 +57,7 @@ class IndexRecord(Base):
     created_date = Column(DateTime, default=datetime.datetime.utcnow)
     updated_date = Column(DateTime, default=datetime.datetime.utcnow)
     file_name = Column(String, index=True)
+    version = Column(String, index=True)
 
     urls = relationship(
         'IndexRecordUrl',
@@ -186,7 +187,7 @@ def session(self):
 
     def ids(
             self, limit=100, start=None,
-            size=None, urls=None, hashes=None, file_name=None):
+            size=None, urls=None, hashes=None, file_name=None, version=None):
         '''
         Returns list of records stored by the backend.
         '''
@@ -202,6 +203,9 @@ def ids(
             if file_name is not None:
                 query = query.filter(IndexRecord.file_name == file_name)
 
+            if version is not None:
+                query = query.filter(IndexRecord.version == version)
+
             if urls is not None and urls:
                 query = query.join(IndexRecord.urls)
                 for u in urls:
@@ -251,7 +255,7 @@ def hashes_to_urls(self, size, hashes, start=0, limit=100):
 
             return [r.url for r in query]
 
-    def add(self, form, size=None, file_name=None, metadata=None, urls=None, hashes=None):
+    def add(self, form, size=None, file_name=None, metadata=None, version=None, urls=None, hashes=None):
         '''
         Creates a new record given size, urls, hashes, metadata, and file name.
         '''
@@ -271,6 +275,8 @@ def add(self, form, size=None, file_name=None, metadata=None, urls=None, hashes=
 
             record.baseid = baseid
             record.file_name = file_name
+            record.version = version
+
 
             did = str(uuid.uuid4())
             record.did, record.rev = did, str(uuid.uuid4())[:8]
@@ -332,7 +338,9 @@ def get(self, did):
 
             form = record.form
             size = record.size
+
             file_name = record.file_name
+            version = record.version
 
             urls = [u.url for u in record.urls]
             hashes = {h.hash_type: h.hash_value for h in record.hashes}
@@ -347,6 +355,7 @@ def get(self, did):
                 'rev': rev,
                 'size': size,
                 'file_name': file_name,
+                'version': version,
                 'urls': urls,
                 'hashes': hashes,
                 'metadata': metadata,
@@ -357,7 +366,7 @@ def get(self, did):
 
         return ret
 
-    def update(self, did, rev, urls=None, file_name=None):
+    def update(self, did, rev, urls=None, file_name=None, version=None):
         '''
         Updates an existing record with new values.
         '''
@@ -380,8 +389,12 @@ def update(self, did, rev, urls=None, file_name=None):
                     did=record.did,
                     url=url
                 ) for url in urls]
+
             if file_name is not None:
                 record.file_name = file_name
+
+            if version is not None:
+                record.version = version
 
             record.rev = str(uuid.uuid4())[:8]
 
@@ -411,7 +424,7 @@ def delete(self, did, rev):
 
     def add_version(
             self, did, form, size=None,
-            file_name=None, metadata=None, urls=None, hashes=None):
+            file_name=None, metadata=None, version=None, urls=None, hashes=None):
         '''
         Add a record version given did
         '''
@@ -441,6 +454,7 @@ def add_version(
             record.form = form
             record.size = size
             record.file_name = file_name
+            record.version = version
 
             record.urls = [IndexRecordUrl(
                 did=record.did,
@@ -492,7 +506,8 @@ def get_all_versions(self, did):
                 form = record.form
 
                 size = record.size
-                file_name =record.file_name
+                file_name = record.file_name
+                version = record.version
                 urls = [u.url for u in record.urls]
                 hashes = {h.hash_type: h.hash_value for h in record.hashes}
                 metadata = {m.key: m.value for m in record.index_metadata}
@@ -506,6 +521,7 @@ def get_all_versions(self, did):
                     'size': size,
                     'file_name': file_name,
                     'metadata': metadata,
+                    'version': version,
                     'urls': urls,
                     'hashes': hashes,
                     'form': form,
@@ -546,7 +562,9 @@ def get_latest_version(self, did):
             form = record.form
             size = record.size
             file_name = record.file_name
+
             metadata = {m.key: m.value for m in record.index_metadata}
+            version = record.version
 
             urls = [u.url for u in record.urls]
             hashes = {h.hash_type: h.hash_value for h in record.hashes}
@@ -560,6 +578,7 @@ def get_latest_version(self, did):
                 'size': size,
                 'file_name': file_name,
                 'metadata': metadata,
+                'version': version,
                 'urls': urls,
                 'hashes': hashes,
                 'form': form,
@@ -679,6 +698,15 @@ def migrate_3(session, **kwargs):
         "CREATE INDEX {tb}__file_name_idx ON {tb} ( file_name )"
         .format(tb=IndexRecord.__tablename__))
 
+def migrate_4(session, **kwargs):
+    session.execute(
+        "ALTER TABLE {} ADD COLUMN version VARCHAR;"
+        .format(IndexRecord.__tablename__))
+
+    session.execute(
+        "CREATE INDEX {tb}__version_idx ON {tb} ( version )"
+        .format(tb=IndexRecord.__tablename__))
+
 # ordered schema migration functions that the index should correspond to
 # CURRENT_SCHEMA_VERSION - 1 when it's written
-SCHEMA_MIGRATION_FUNCTIONS = [migrate_1, migrate_2, migrate_3]
+SCHEMA_MIGRATION_FUNCTIONS = [migrate_1, migrate_2, migrate_3, migrate_4]
diff --git a/indexd/index/schema.py b/indexd/index/schema.py
@@ -32,7 +32,11 @@
     },
     "metadata": {
       "description": "optional metadata of the object",
-      "type": "object",
+      "type": "object"
+    },
+    "version": {
+      "description": "optional version string of the object",
+      "type": "string",
     },
     "urls": {
       "type": "array",
@@ -105,6 +109,9 @@
     "file_name": {
         "type": "string"
     },
+    "version": {
+        "type": "string"
+    },
     "rev": {
       "type": "string",
       "pattern": "^[0-9a-f]{8}$",