Bug fixes and improvements (#28)

- Add warning for outdated local datasets - Fix config migration in case of malformed ~/.data.world - Relax validation of file names in dataset API responses to allow slashes - Improve content in help(datadotworld) - Return newly created dataset URL - Version and README updates in preparation for PyPI release Github Issues: - #27 - #19 - #26 - #24
datadotworld · Apr 14, 2017 · f4d9fef · f4d9fef
1 parent 7414ffc
commit f4d9fef
Show file tree

Hide file tree

Showing 26 changed files with 18,383 additions and 521 deletions.
diff --git a/README.md b/README.md
@@ -6,11 +6,14 @@ A python library for working with data.world datasets
 
 ### Install
 
-This library hasn't yet been added to a central package repository -
-you can install it using `pip` directly from this github repo:
+You can install it using `pip` directly from PyPI
 
+```bash
+pip install datadotworld
 ```
-pip install git+git://github.com/datadotworld/data.world-py.git
+Optionally, you can install the library including pandas support
+```bash
+pip install datadotworld[PANDAS]
 ```
 
 ### Configure

diff --git a/datadotworld/__init__.py b/datadotworld/__init__.py
@@ -1,25 +1,155 @@
-"""
-data.world-py
-Copyright 2017 data.world, Inc.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the
-License.
+# data.world-py
+# Copyright 2017 data.world, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the
+# License.
+#
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+#
+# This product includes software developed at
+# data.world, Inc.(http://data.world/).
 
-You may obtain a copy of the License at
-http://www.apache.org/licenses/LICENSE-2.0
 
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-implied. See the License for the specific language governing
-permissions and limitations under the License.
+"""
+A python library for working with data.world datasets
 
-This product includes software developed at
-data.world, Inc.(http://data.world/).
 """
+
 from __future__ import absolute_import
 
-from datadotworld.datadotworld import load_dataset, query, api_client
+import weakref
+
+from datadotworld.datadotworld import DataDotWorld
+
+__version__ = '1.0.0'
+
+# Convenience top-level functions
+
+__instances = weakref.WeakValueDictionary()
+
+
+def _get_instance(profile):
+    instance = __instances.get(profile)
+    if instance is None:
+        instance = DataDotWorld(profile=profile)
+        __instances[profile] = instance
+    return instance
+
+
+def load_dataset(dataset_key, force_update=False, profile='default'):
+    """
+    Load a dataset from the local filesystem, downloading it from data.world
+    first, if necessary.
+
+    This function returns an object of type `LocalDataset`. The object
+    allows access to metedata via it's `describe()` method and to all the data
+    via three properties `raw_data`, `tables` and `dataframes`, all of which
+    are mappings (dict-like structures).
+
+
+    Parameters
+    ----------
+    dataset_key : str
+        Dataset identifier, in the form of owner/id or of a url
+    force_update : bool
+        Flag, indicating if a new copy of the dataset should be downloaded
+        replacing any previously downloaded copy
+    profile : str, optional
+        Configuration profile (account) to use.
+
+    Returns
+    -------
+    LocalDataset
+        The object representing the dataset
+
+    Raises
+    ------
+    RestApiError
+        If a server error occurs
+
+    Examples
+    --------
+    >>> import datadotworld as dw
+    >>> dataset = dw.load_dataset('jonloyens/an-intro-to-dataworld-dataset')
+    >>> list(dataset.dataframes)
+    ['changelog', 'datadotworldbballstats', 'datadotworldbballteam']
+    """
+    return _get_instance(profile).load_dataset(dataset_key,
+                                               force_update=force_update)
+
+
+def query(dataset_key, query, query_type='sql', profile='default'):
+    """Query an existing dataset
+
+    Parameters
+    ----------
+    dataset_key : str
+        Dataset identifier, in the form of owner/id or of a url
+    query : str
+        SQL or SPARQL query
+    query_type : {'sql', 'sparql'}, optional
+        The type of the query. Must be either 'sql' or 'sparql'.
+    profile : str, optional
+        Configuration profile (account) to use.
+
+    Returns
+    -------
+    Results
+        Object containing the results of the query
+
+    Raises
+    ------
+    RuntimeError
+        If a server error occurs
+
+    Examples
+    --------
+    >>> import datadotworld as dw
+    >>> results = dw.query(
+    ...     'jonloyens/an-intro-to-dataworld-dataset',
+    ...     'SELECT * FROM `DataDotWorldBBallStats`, `DataDotWorldBBallTeam` '
+    ...     'WHERE DataDotWorldBBallTeam.Name = DataDotWorldBBallStats.Name')
+    >>> df = results.dataframe
+    >>> df.shape
+    (8, 6)
+    """
+    return _get_instance(profile).query(dataset_key, query,
+                                        query_type=query_type)
+
+
+def api_client(profile='default'):
+    """Return API client for access to data.world's REST API
+
+    Parameters
+    ----------
+    profile : str, optional
+        Configuration profile (account) to use.
+
+    Returns
+    -------
+    RestApiClient
+        REST API client object
+
+    Examples
+    --------
+    >>> import datadotworld as dw
+    >>> client = dw.api_client()
+    >>> client.get_dataset(
+    ...     'jonloyens/an-intro-to-dataworld-dataset').get('title')
+    'An Intro to data.world Dataset'
+    """
+    return _get_instance(profile).api_client
+
+
+if __name__ == "__main__":
+    import doctest
 
-__version__ = '1.0.0-beta.5'
+    doctest.testmod()
diff --git a/datadotworld/cli.py b/datadotworld/cli.py
@@ -1,23 +1,22 @@
-"""
-data.world-py
-Copyright 2017 data.world, Inc.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the
-License.
-
-You may obtain a copy of the License at
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-implied. See the License for the specific language governing
-permissions and limitations under the License.
-
-This product includes software developed at
-data.world, Inc.(http://data.world/).
-"""
+# data.world-py
+# Copyright 2017 data.world, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the
+# License.
+#
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+#
+# This product includes software developed at
+# data.world, Inc.(http://data.world/).
+
 from __future__ import absolute_import
 
 import click

diff --git a/datadotworld/client/__init__.py b/datadotworld/client/__init__.py
@@ -1,20 +1,18 @@
-"""
-data.world-py
-Copyright 2017 data.world, Inc.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the
-License.
-
-You may obtain a copy of the License at
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-implied. See the License for the specific language governing
-permissions and limitations under the License.
-
-This product includes software developed at
-data.world, Inc.(http://data.world/).
-"""
+# data.world-py
+# Copyright 2017 data.world, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the
+# License.
+#
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+#
+# This product includes software developed at
+# data.world, Inc.(http://data.world/).
diff --git a/datadotworld/client/_swagger/models/file_summary_response.py b/datadotworld/client/_swagger/models/file_summary_response.py
@@ -76,8 +76,6 @@ def name(self, name):
             raise ValueError("Invalid value for `name`, length must be less than or equal to `128`")
         if name is not None and len(name) < 1:
             raise ValueError("Invalid value for `name`, length must be greater than or equal to `1`")
-        if name is not None and not re.search('^[^/]+$', name):
-            raise ValueError("Invalid value for `name`, must be a follow pattern or equal to `/^[^/]+$/`")
 
         self._name = name
 

diff --git a/datadotworld/client/api.py b/datadotworld/client/api.py
@@ -1,23 +1,22 @@
-"""
-data.world-py
-Copyright 2017 data.world, Inc.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the
-License.
-
-You may obtain a copy of the License at
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-implied. See the License for the specific language governing
-permissions and limitations under the License.
-
-This product includes software developed at
-data.world, Inc.(http://data.world/).
-"""
+# data.world-py
+# Copyright 2017 data.world, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the
+# License.
+#
+# You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+#
+# This product includes software developed at
+# data.world, Inc.(http://data.world/).
+
 from __future__ import absolute_import, division
 
 import glob
@@ -118,6 +117,11 @@ def create_dataset(self, owner_id, **kwargs):
         files : dict, optional
             File names and source URLs
 
+        Returns
+        -------
+        str
+            Newly created dataset key
+
         Raises
         ------
         RestApiException
@@ -138,7 +142,10 @@ def create_dataset(self, owner_id, **kwargs):
             kwargs)
 
         try:
-            self._datasets_api.create_dataset(owner_id, request)
+            (_, _, headers) = self._datasets_api.create_dataset_with_http_info(
+                owner_id, request, _return_http_data_only=False)
+            if 'Location' in headers:
+                return headers['Location']
         except _swagger.rest.ApiException as e:
             raise RestApiError(cause=e)
 
@@ -496,7 +503,7 @@ def json(self):
         """
         try:
             return json.loads(self.body)
-        except (json.JSONDecodeError, TypeError):
+        except (ValueError, TypeError):
             return None
 
     def __str__(self):