Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove deprecated logic #80

Draft
wants to merge 23 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from 19 commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ repos:
- types-mock
- types-setuptools
- types-redis
- types-boto
- boto3-stubs
- repo: https://github.com/Quantco/pre-commit-mirrors-pyupgrade
rev: 3.1.0
Expand Down
7 changes: 2 additions & 5 deletions boto_credentials.ini
Original file line number Diff line number Diff line change
@@ -1,11 +1,8 @@
[minio-travis-test]
# AWS minio credentials
access_key=minio
secret_key=miniostorage
access_key=minioadmin
secret_key=minioadmin
janjagusch marked this conversation as resolved.
Show resolved Hide resolved
# AWS minio endpoint
host=127.0.0.1
is_secure=false
port=9000
# Legacy boto config
connect_func=connect_s3
ordinary_calling_format=true
8 changes: 7 additions & 1 deletion docs/changes.rst
Original file line number Diff line number Diff line change
@@ -1,11 +1,17 @@
Changelog
*********

2.0.0
=====
* Removed ``get_store``, ``url2dict``, ``extract_params``, and ``create_store``.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
* Removed ``get_store``, ``url2dict``, ``extract_params``, and ``create_store``.
* Removed ``get_store``, ``url2dict``, ``extract_params``, and ``create_store``.
* Removed `BotoStore` based on deprecated `boto` library.


* ``get_store_from_url`` should be used to create stores from a URL.

1.7.0
=====
* Deprecated ``get_store``, ``url2dict``, and ``extract_params``.

* ``get_store_from_url`` should be used to create stores from a URL
* ``get_store_from_url`` should be used to create stores from a URL.

* Added ``from_url`` and ``from_parsed_url`` to each store.

Expand Down
1 change: 0 additions & 1 deletion environment.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ channels:
- nodefaults
dependencies:
- azure-storage-blob
- boto
- boto3
- docker-compose
- dulwich
Expand Down
7 changes: 1 addition & 6 deletions minimalkv/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,10 @@
VALID_KEY_REGEXP,
VALID_NON_NUM,
)
from minimalkv._get_store import get_store, get_store_from_url
from minimalkv._get_store import get_store_from_url
from minimalkv._key_value_store import KeyValueStore, UrlKeyValueStore
from minimalkv._mixins import CopyMixin, TimeToLiveMixin, UrlMixin
from minimalkv._store_creation import create_store
from minimalkv._store_decoration import decorate_store
from minimalkv._urls import url2dict

try:
import pkg_resources
Expand All @@ -21,15 +19,12 @@

__all__ = [
"CopyMixin",
"create_store",
"decorate_store",
"FOREVER",
"get_store_from_url",
"get_store",
"KeyValueStore",
"NOT_SET",
"TimeToLiveMixin",
"url2dict",
"UrlKeyValueStore",
"UrlMixin",
"VALID_KEY_RE",
Expand Down
101 changes: 2 additions & 99 deletions minimalkv/_get_store.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
from functools import reduce
from typing import Any, Dict, List, Optional, Type
from typing import Dict, List, Optional, Type

from uritools import SplitResult, urisplit

from minimalkv._key_value_store import KeyValueStore
from minimalkv._urls import url2dict


def get_store_from_url(
Expand Down Expand Up @@ -73,8 +72,7 @@ def get_store_from_url(
scheme = scheme_parts[0]

if scheme not in scheme_to_store:
# If we can't find the scheme, we fall back to the old creation methods
return get_store(**url2dict(url))
raise NotImplementedError
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TODO: We need to implement the _from_parsed_url class method for all stores before removing all the deprecated logic.

TODO:: Put this into a separate PR.


store_cls_from_url = scheme_to_store[scheme]
if store_cls is not None and store_cls_from_url != store_cls:
Expand Down Expand Up @@ -142,98 +140,3 @@ def _extract_wrappers(parsed_url: SplitResult) -> List[str]:
)

return scheme_wrappers + fragment_wrappers


def get_store(
type: str, create_if_missing: bool = True, **params: Any
) -> KeyValueStore:
"""Return a storage object according to the ``type`` and additional parameters.

The ``type`` must be one of the types below, where each allows requires different
parameters:

* ``"azure"``: Returns a ``minimalkv.azure.AzureBlockBlobStorage``. Parameters are
``"account_name"``, ``"account_key"``, ``"container"``, ``"use_sas"`` and ``"create_if_missing"`` (default: ``True``).
``"create_if_missing"`` has to be ``False`` if ``"use_sas"`` is set. When ``"use_sas"`` is set,
``"account_key"`` is interpreted as Shared Access Signature (SAS) token.FIRE
``"max_connections"``: Maximum number of network connections used by one store (default: ``2``).
``"socket_timeout"``: maximum timeout value in seconds (socket_timeout: ``200``).
``"max_single_put_size"``: max_single_put_size is the largest size upload supported in a single put call.
``"max_block_size"``: maximum block size is maximum size of the blocks(maximum size is <= 100MB)
* ``"s3"``: Returns a plain ``minimalkv.net.botostore.BotoStore``.
Parameters must include ``"host"``, ``"bucket"``, ``"access_key"``, ``"secret_key"``.
Optional parameters are

- ``"force_bucket_suffix"`` (default: ``True``). If set, it is ensured that
the bucket name ends with ``-<access_key>``
by appending this string if necessary;
If ``False``, the bucket name is used as-is.
- ``"create_if_missing"`` (default: ``True`` ). If set, creates the bucket if it does not exist;
otherwise, try to retrieve the bucket and fail with an ``IOError``.
* ``"hs3"`` returns a variant of ``minimalkv.net.botostore.BotoStore`` that allows "/" in the key name.
The parameters are the same as for ``"s3"``
* ``"gcs"``: Returns a ``minimalkv.net.gcstore.GoogleCloudStore``. Parameters are
``"credentials"``, ``"bucket_name"``, ``"bucket_creation_location"``, ``"project"`` and ``"create_if_missing"`` (default: ``True``).

- ``"credentials"``: either the path to a credentials.json file or a *google.auth.credentials.Credentials* object
- ``"bucket_name"``: Name of the bucket the blobs are stored in.
- ``"project"``: The name of the GCStorage project. If a credentials JSON is passed then it contains the project name
and this parameter will be ignored.
- ``"create_if_missing"``: [optional] Create new bucket to store blobs in if ``"bucket_name"`` doesn't exist yet. (default: ``True``).
- ``"bucket_creation_location"``: [optional] If a new bucket is created (create_if_missing=True), the location it will be created in.
If ``None`` then GCloud uses a default location.
* ``"hgcs"``: Like ``"gcs"`` but "/" are allowed in the keynames.
* ``"fs"``: Returns a ``minimalkv.fs.FilesystemStore``. Specify the base path as "path" parameter.
* ``"hfs"`` returns a variant of ``minimalkv.fs.FilesystemStore`` that allows "/" in the key name.
The parameters are the same as for ``"file"``.
* ``"memory"``: Returns a DictStore. Doesn't take any parameters
* ``"redis"``: Returns a RedisStore. Constructs a StrictRedis using params as kwargs.
See StrictRedis documentation for details.

Parameters
----------
type : str
Type of storage to open, with optional storage decorators.
create_if_missing : bool, optional, default = True
Create the "root" of the storage (Azure container, parent directory, S3 bucket,
etc.). Has no effect for stores where this makes no sense, like ``redis`` or
``memory``.
kwargs
Parameters specific to the store type.

Returns
-------
store: KeyValueStore
Key value store of type ``type`` as described in ``kwargs`` parameters.

"""
from minimalkv._store_creation import create_store
from minimalkv._store_decoration import decorate_store

# split off old-style wrappers, if any:
parts = type.split("+")
type = parts.pop(0)
decorators = list(reversed(parts))

# find new-style wrappers, if any:
wrapspec = params.pop("wrap", "")
wrappers = list(wrapspec.split("+")) if wrapspec else []

# can't have both:
if wrappers:
if decorators:
raise ValueError(
"Adding store wrappers via store type as well as via wrap parameter are not allowed. Preferably use wrap."
)
decorators = wrappers

# create_if_missing is a universal parameter, so it's part of the function signature
# it can be safely ignored by stores where 'creating' makes no sense.
params["create_if_missing"] = create_if_missing

store = create_store(type, params)

# apply wrappers/decorators:
wrapped_store = reduce(decorate_store, decorators, store)

return wrapped_store
19 changes: 0 additions & 19 deletions minimalkv/_hstores.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
from minimalkv.memory.redisstore import RedisStore
from minimalkv.net.azurestore import AzureBlockBlobStore
from minimalkv.net.boto3store import Boto3Store
from minimalkv.net.botostore import BotoStore
from minimalkv.net.gcstore import GoogleCloudStore
from minimalkv.net.s3fsstore import S3FSStore

Expand All @@ -23,24 +22,6 @@ class HAzureBlockBlobStore(ExtendedKeyspaceMixin, AzureBlockBlobStore): # noqa
pass


class HBotoStore(ExtendedKeyspaceMixin, BotoStore): # noqa D
def size(self, key: str) -> bytes:
"""Get size of data at key in bytes.

Parameters
----------
key : str
Key of data.

Returns
-------
size : int
Size of value at key in bytes.
"""
k = self.bucket.lookup(self.prefix + key)
return k.size


class HS3FSStore(ExtendedKeyspaceMixin, S3FSStore): # noqa D
pass

Expand Down
Loading