Merge pull request #4 from lightly-ai/develop

Release 1.0.0 - Develop to Master

.gitignore

@@ -1,3 +1,4 @@
 **__pycache__**
 **build
-**settings.json
+**settings.json
+lightning_logs/

lightly/api/communication.py

@@ -9,7 +9,7 @@
 
 from lightly.api.utils import getenv
 SERVER_LOCATION = getenv('LIGHTLY_SERVER_LOCATION',
-                         'https://api.whattolabel.com')
+                         'https://api.lightly.ai')
 
 
 def _post_request(dst_url, data=None, json=None,

lightly/cli/config/config.yaml

@@ -7,19 +7,12 @@ embeddings: ''                # Path to csv file which holds embeddings.
 checkpoint: ''                # Path to a model checkpoint. If left empty, a pre-trained model
                               # will be used.
 
-### deprecated!
-# The following arguments are deprecated and will be removed
-# in the next version.
-from_folder: ''               # Path to input directory which holds images (replaced by input_dir).
-path_to_folder: ''            # Path to input directory which holds images (replaced by input_dir).
-path_to_embeddings: ''        # Path to csv file which holds embeddings (replaced by embeddings).
-
 ### platform
 # The following arguments are required for requests to the
 # Lightly platform.
 token: ''                     # User access token to the platform.
 dataset_id: ''                # Identifier of the dataset on the platform
-upload: 'thumbnails'          # Whether to upload full images, thumbnails only, or metadata only.
+upload: 'full'                # Whether to upload full images, thumbnails only, or metadata only.
                               # Must be one of ['full', 'thumbnails', 'none']
 embedding_name: 'default'     # Name of the embedding to be used on the platform.
 emb_upload_bsz: 32            # Number of embeddings which are uploaded in a single batch.

lightly/cli/download_cli.py

@@ -20,33 +20,19 @@
 
 
 def _download_cli(cfg, is_cli_call=True):
-    '''Download all samples in a given tag.
-       If input_dir and output_dir are specified, a copy of all images
-       in the tag will be stored in the output directory.
-
-    Args:
-        cfg['input_dir']: (str, optional)
-            Path to folder which holds all images from the dataset
-        cfg['output_dir']: (str, optional)
-            Path to folder where the copied images will be stored
-        cfg['tag_name']: (str) Name of the requested tag
-        cfg['dataset_id']: (str) Dataset identifier on the platform
-        cfg['token']: (str) Token which grants access to the platform
-
-    '''
 
     tag_name = cfg['tag_name']
     dataset_id = cfg['dataset_id']
     token = cfg['token']
 
     if not tag_name:
         print('Please specify a tag name')
-        print('For help, try: lightly-upload --help')
+        print('For help, try: lightly-download --help')
         return
 
     if not token or not dataset_id:
         print('Please specify your access token and dataset id')
-        print('For help, try: lightly-upload --help')
+        print('For help, try: lightly-download --help')
         return
 
     # get all samples in the queried tag
@@ -62,6 +48,9 @@ def _download_cli(cfg, is_cli_call=True):
     with open(cfg['tag_name'] + '.txt', 'w') as f:
         for item in samples:
             f.write("%s\n" % item)
+    msg = 'The list of files in tag {} is stored at: '.format(cfg['tag_name'])
+    msg += os.path.join(os.getcwd(), cfg['tag_name'] + '.txt')
+    print(msg)
 
     if cfg['input_dir'] and cfg['output_dir']:
         # "name.jpg" -> "/name.jpg" to prevent bugs like this:

lightly/cli/embed_cli.py

@@ -26,22 +26,6 @@
 
 
 def _embed_cli(cfg, is_cli_call=True):
-    """Use the trained self-supervised model to embed samples from your dataset.
-
-    Args:
-        cfg[data]: (str) Name of the dataset
-        cfg[root]: (str) Directory where the dataset should be stored
-        cfg[checkpoint]: (str) Path to the lightning checkpoint
-        cfg[download]: (bool) Whether to download the dataset
-        cfg[input_dir]: (str) If specified, the dataset is loaded \
-            from the folder
-
-    Returns:
-        embeddings: (np.ndarray) A d-dimensional embedding \
-            for each data sample
-        labels: (np.ndarray) Data labels, 0 if there are no labels
-        filenames: (List[str]) File name of each data sample
-    """
 
     data = cfg['data']
     checkpoint = cfg['checkpoint']

lightly/cli/train_cli.py

@@ -26,20 +26,6 @@
 
 
 def _train_cli(cfg, is_cli_call=True):
-    """Train a self-supervised model on the image dataset of your choice.
-
-    Args:
-        cfg[data]: (str)
-            Name of the dataset (to download use cifar10 or cifar100)
-        cfg[root]: (str) Directory where the dataset should be stored
-        cfg[download]: (bool) Whether to download the dataset
-        cfg[input_dir]: (str)
-            If specified, the dataset is loaded from the folder
-
-    Returns:
-        checkpoint: (str) Path to checkpoint of the best model during training
-
-    """
 
     data = cfg['data']
     download = cfg['download']

lightly/cli/upload_cli.py

@@ -16,15 +16,6 @@
 
 
 def _upload_cli(cfg, is_cli_call=True):
-    '''Upload your image dataset and/or embeddings to the Lightly platform.
-
-    Args:
-        cfg['input_dir']: (str) Path to folder which holds images to upload
-        cfg['embeddings']: (str) Path to csv file which holds embeddings to upload
-        cfg['dataset_id']: (str) Dataset identifier on the platform
-        cfg['token']: (str) Token which grants acces to the platform
-
-    '''
 
     input_dir = cfg['input_dir']
     if input_dir and is_cli_call:

tests/api/test_CreateInitialTag.py

@@ -21,7 +21,7 @@ def setup(self, psuccess=1.):
 
         # set up url
         self.dst_url = os.getenvb(b'LIGHTLY_SERVER_LOCATION',
-                                  b'https://api.whattolabel.com').decode()
+                                  b'https://api.lightly.ai').decode()
         # route
         self.dst_url += f'/users/datasets/{self.dataset_id}/tags'
 

tests/api/test_Get.py

@@ -18,7 +18,7 @@ def setup(self, psuccess=1.):
 
         # set up url
         self.dst_url = os.getenvb(b'LIGHTLY_SERVER_LOCATION',
-                                  b'https://api.whattolabel.com').decode()
+                                  b'https://api.lightly.ai').decode()
         # route
         self.dst_url += '/sample/route/to/get'
         # query

tests/api/test_GetLatestVersion.py

@@ -20,7 +20,7 @@ def setup(self, psuccess=1.):
 
         # set up url
         self.dst_url = os.getenvb(b'LIGHTLY_SERVER_LOCATION',
-                                  b'https://api.whattolabel.com').decode()
+                                  b'https://api.lightly.ai').decode()
         # route
         self.dst_url += '/pip/version'
         self.version = '0.0.0'

tests/api/test_GetPresignedURL.py

@@ -22,7 +22,7 @@ def setup(self, n_data=1000, psuccess=1.):
 
         # set up url
         self.dst_url = os.getenvb(b'LIGHTLY_SERVER_LOCATION',
-                                  b'https://api.whattolabel.com').decode()
+                                  b'https://api.lightly.ai').decode()
         # route
         self.dst_url += f'/users/datasets/{self.dataset_id}'
         self.dst_url += f'/samples/{self.sample_id}/writeurl'

tests/api/test_GetSamples.py

@@ -21,15 +21,15 @@ def setup(self, mode='tag_exists', psuccess=1.):
         self.token = '123'
 
         self.tag_url = os.getenvb(b'LIGHTLY_SERVER_LOCATION',
-                                  b'https://api.whattolabel.com').decode()
+                                  b'https://api.lightly.ai').decode()
         self.tag_url += f'/users/datasets/{self.dataset_id}/tags/'
         self.tags = [
             {'name': 'initial-tag', '_id': '123'},
             {'name': 'test-tag', '_id': '456'},
         ]
 
         self.dst_url = os.getenvb(b'LIGHTLY_SERVER_LOCATION',
-                                  b'https://api.whattolabel.com').decode()
+                                  b'https://api.lightly.ai').decode()
         self.dst_url += f'/users/datasets/{self.dataset_id}/tags/123/download'
         self.samples = 'sample_1.jpg\nsample_2.jpg'
 

tests/api/test_GetTags.py

@@ -25,7 +25,7 @@ def setup(self, psuccess=1.):
 
         # set up url
         self.dst_url = os.getenvb(b'LIGHTLY_SERVER_LOCATION',
-                                  b'https://api.whattolabel.com').decode()
+                                  b'https://api.lightly.ai').decode()
         # route
         self.dst_url += f'/users/datasets/{self.dataset_id}/tags'
         # query

tests/api/test_Post.py

@@ -18,7 +18,7 @@ def setup(self, psuccess=1.):
 
         # set up url
         self.dst_url = os.getenvb(b'LIGHTLY_SERVER_LOCATION',
-                                  b'https://api.whattolabel.com').decode()
+                                  b'https://api.lightly.ai').decode()
         # route
         self.dst_url += '/sample/route/to/post'
 

tests/api/test_Put.py

@@ -18,7 +18,7 @@ def setup(self, psuccess=1.):
 
         # set up url
         self.dst_url = os.getenvb(b'LIGHTLY_SERVER_LOCATION',
-                                  b'https://api.whattolabel.com').decode()
+                                  b'https://api.lightly.ai').decode()
         # route
         self.dst_url += '/sample/route/to/put'
 

tests/api/test_UploadEmbedding.py

@@ -28,7 +28,7 @@ def setup(self, psuccess=1.):
         }
 
         self.dst_url = os.getenvb(b'LIGHTLY_SERVER_LOCATION',
-                                  b'https://api.whattolabel.com').decode()
+                                  b'https://api.lightly.ai').decode()
         self.dst_url += f'/users/datasets/{self.dataset_id}/embeddings'
 
     def callback_1(self, request):

tests/api/test_UploadEmbeddings.py

@@ -21,7 +21,7 @@ def setup(self, n_data=1000):
 
         # set up url
         self.dst_url = os.getenvb(b'LIGHTLY_SERVER_LOCATION',
-                                  b'https://api.whattolabel.com').decode()
+                                  b'https://api.lightly.ai').decode()
         self.emb_url = f'{self.dst_url}/users/datasets/{self.dataset_id}/embeddings'
         self.tag_url = f'{self.dst_url}/users/datasets/{self.dataset_id}/tags/?token={self.token}'
 

tests/api/test_UploadImages.py

@@ -18,7 +18,7 @@ def setup(self, n_data=1000):
         self.dataset_id = 'XYZ'
         self.token = 'secret'
         self.dst_url = os.getenvb(b'LIGHTLY_SERVER_LOCATION',
-                                  b'https://api.whattolabel.com').decode()
+                                  b'https://api.lightly.ai').decode()
 
         self.gettag_url = f'{self.dst_url}/users/datasets/{self.dataset_id}/tags/?token={self.token}'
         self.sample_url = f'{self.dst_url}/users/datasets/{self.dataset_id}/samples/'

tests/api/test_UploadSample.py

@@ -28,7 +28,7 @@ def setup(self, psuccess=1.):
         }
 
         self.dst_url = os.getenvb(b'LIGHTLY_SERVER_LOCATION',
-                                  b'https://api.whattolabel.com').decode()
+                                  b'https://api.lightly.ai').decode()
         self.dst_url += f'/users/datasets/{self.dataset_id}/samples/'
 
     def callback(self, request):

tests/models/test_ModelsSimCLR.py

@@ -15,7 +15,7 @@ def setUp(self):
             'resnet-101',
             'resnet-152'
         ]
-        self.batch_size = 4
+        self.batch_size = 2
         self.input_tensor = torch.rand((self.batch_size, 3, 32, 32))
 
     def test_create_variations_cpu(self):
@@ -59,8 +59,8 @@ def test_feature_dim_configurable(self):
     def test_variations_input_dimension(self):
         device = 'cuda' if torch.cuda.is_available() else 'cpu'
         for model_name in self.resnet_variants:
-            for input_width in [16, 32, 64, 256]:
-                for input_height in [16, 32, 64, 256]:
+            for input_width in [16, 32, 64, 128]:
+                for input_height in [16, 32, 64, 128]:
                     model = ResNetSimCLR(name=model_name).to(device)
 
                     input_tensor = torch.rand((self.batch_size,

tox.ini

@@ -10,7 +10,7 @@ whitelist_externals = make
                       pip
 
 passenv = *
-setenv = LIGHTLY_SERVER_LOCATION = https://api-dev.whattolabel.com 
+setenv = LIGHTLY_SERVER_LOCATION = https://api-dev.lightly.ai
 
 commands = 
            pip install torch==1.5.1+cu101 torchvision==0.6.1+cu101 -f https://download.pytorch.org/whl/torch_stable.html
@@ -25,7 +25,7 @@ whitelist_externals = make
                       pip
 
 passenv = *
-setenv = LIGHTLY_SERVER_LOCATION = https://api-dev.whattolabel.com 
+setenv = LIGHTLY_SERVER_LOCATION = https://api-dev.lightly.ai
          CUDA_VISIBLE_DEVICES = -1
 
 commands =