Code Review 8/29/18

01_create_subsets.py

@@ -0,0 +1,126 @@
+import argparse
+import logging
+import os
+from csv import DictWriter
+
+from data.avc.subsets import get_subset_split
+from log import init_console_logger
+
+LOGGER = logging.getLogger('data')
+LOGGER.setLevel(logging.DEBUG)
+
+
+def write_subset_file(path, subset_list):
+    with open(path, 'w') as f:
+        field_names = list(subset_list[0].keys())
+        writer = DictWriter(f, field_names)
+        writer.writeheader()
+
+        for item in subset_list:
+            item = dict(item)
+            item['labels'] = ';'.join(item['labels'])
+            writer.writerow(item)
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description='Creates CSVs containing a train-valid-test split for the given dataset')
+
+    parser.add_argument('-vr',
+                        '--valid-ratio',
+                        dest='valid_ratio',
+                        action='store',
+                        type=float,
+                        default=0.1,
+                        help='Ratio of dataset used for validation set')
+
+    parser.add_argument('-tr',
+                        '--test-ratio',
+                        dest='test_ratio',
+                        action='store',
+                        type=float,
+                        default=0.1,
+                        help='Ratio of dataset used for test set')
+
+    parser.add_argument('-rs',
+                        '--random-seed',
+                        dest='random_seed',
+                        action='store',
+                        type=int,
+                        default=12345678,
+                        help='Random seed used for generating split')
+
+    parser.add_argument('-o',
+                        '--ontology-path',
+                        dest='ontology_path',
+                        action='store',
+                        type=str,
+                        default=os.path.join(os.path.dirname(__file__), 'resources/ontology.json'),
+                        help='Path to AudioSet ontology')
+
+    parser.add_argument('-mp',
+                        '--metadata-path',
+                        dest='metadata_path',
+                        action='store',
+                        type=str,
+                        help='Path to metadata csv file(s). Accepts a glob string.')
+
+    parser.add_argument('-fp',
+                        '--filter-path',
+                        dest='filter_path',
+                        action='store',
+                        type=str,
+                        help='Path to filter csv file(s).')
+
+
+    parser.add_argument('-r',
+                        '--random-state',
+                        dest='random_state',
+                        action='store',
+                        type=int,
+                        default=20171021,
+                        help='Random seed used to set the RNG state')
+
+    parser.add_argument('data_dir',
+                        action='store',
+                        type=str,
+                        help='Path to directory where data files are stored')
+
+    parser.add_argument('output_dir',
+                        action='store',
+                        type=str,
+                        help='Path to directory where output files will be stored')
+
+    parser.add_argument('filename_prefix',
+                        action='store',
+                        type=str,
+                        help='Path to directory where output files will be stored')
+
+    return parser.parse_args()
+
+
+if __name__ == '__main__':
+    init_console_logger(LOGGER, verbose=True)
+
+    args = parse_arguments()
+
+    train_list, valid_list, test_list \
+        = get_subset_split(args.data_dir,
+                           valid_ratio=args.valid_ratio,
+                           test_ratio=args.test_ratio,
+                           random_state=args.random_state,
+                           metadata_path=args.metadata_path,
+                           filter_path=args.filter_path,
+                           ontology_path=args.ontology_path)
+
+    output_dir = args.output_dir
+    filename_prefix = args.filename_prefix
+    train_subset_path = os.path.join(output_dir, filename_prefix + '_train.csv')
+    valid_subset_path = os.path.join(output_dir, filename_prefix + '_valid.csv')
+    test_subset_path = os.path.join(output_dir, filename_prefix + '_test.csv')
+
+    if not os.path.isdir(output_dir):
+        os.makedirs(output_dir)
+
+    write_subset_file(train_subset_path, train_list)
+    write_subset_file(valid_subset_path, valid_list)
+    write_subset_file(test_subset_path, test_list)

02_generate_samples.py

@@ -0,0 +1,143 @@
+import argparse
+import logging
+import math
+from functools import partial
+
+import multiprocessing_logging
+
+from data.avc.sample import sample_and_save
+from data.utils import map_iterate_in_parallel
+from log import init_console_logger
+
+LOGGER = logging.getLogger('sampling')
+LOGGER.setLevel(logging.DEBUG)
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser(description='Pre-sample videos and audios for L3 model.')
+    parser.add_argument('-bs',
+                        '--batch-size',
+                        dest='batch_size',
+                        action='store',
+                        type=int,
+                        default=64,
+                        help='Number of examples per training batch')
+
+    parser.add_argument('-ns',
+                        '--num-streamers',
+                        dest='num_streamers',
+                        action='store',
+                        type=int,
+                        default=64,
+                        help='Number of training pescador streamers that can be open concurrently')
+
+    parser.add_argument('-mr',
+                        '--mux-rate',
+                        dest='mux_rate',
+                        action='store',
+                        type=float,
+                        default=2.0,
+                        help='Poisson distribution parameter for determining number of training samples to take from a streamer')
+
+    parser.add_argument('-a',
+                        '--augment',
+                        dest='augment',
+                        action='store_true',
+                        default=False,
+                        help='If True, performs data augmentation on audio and images')
+
+    parser.add_argument('-pc',
+                        '--precompute',
+                        dest='precompute',
+                        action='store_true',
+                        default=False,
+                        help='If True, streamer precompute samples')
+
+    parser.add_argument('-nd',
+                        '--num-distractors',
+                        dest='num_distractors',
+                        action='store',
+                        type=int,
+                        default=1,
+                        help='Number of distractors for generating examples')
+
+    parser.add_argument('-im',
+                        '--include-metadata',
+                        dest='include_metadata',
+                        action='store_true',
+                        help='If True, includes additional metadata in h5 files')
+
+    parser.add_argument('-mv',
+                        '--max-videos',
+                        dest='max_videos',
+                        action='store',
+                        type=int,
+                        help='Maximum number of videos to use for generating examples. If not specified, all videos will be used')
+
+    parser.add_argument('-r',
+                        '--random-state',
+                        dest='random_state',
+                        action='store',
+                        type=int,
+                        default=20171021,
+                        help='Random seed used to set the RNG state')
+
+    parser.add_argument('-n',
+                        '--num-workers',
+                        dest='num_workers',
+                        action='store',
+                        type=int,
+                        default=4,
+                        help='Number of multiprocessing workers used to download videos')
+
+    parser.add_argument('-v',
+                        '--verbose',
+                        dest='verbose',
+                        action='store_true',
+                        default=False,
+                        help='Logs verbose info')
+
+
+    parser.add_argument('subset_path',
+                        action='store',
+                        type=str,
+                        help='Path to subset file')
+
+    parser.add_argument('num_samples',
+                        action='store',
+                        type=int,
+                        help='(Minimum) number of samples to generate')
+
+    parser.add_argument('output_dir',
+                        action='store',
+                        type=str,
+                        help='Path to directory where output files will be stored')
+
+    args = parser.parse_args()
+
+    init_console_logger(LOGGER, verbose=args.verbose)
+    multiprocessing_logging.install_mp_handler()
+
+    # Just round up for now
+    num_workers = args.num_workers
+    batch_size = args.batch_size
+    batches_per_worker = int(math.ceil(args.num_samples / (num_workers * batch_size)))
+
+    worker_func = partial(sample_and_save,
+        subset_path=args.subset_path,
+        num_batches=batches_per_worker,
+        output_dir=args.output_dir,
+        num_streamers=args.num_streamers,
+        batch_size=batch_size,
+        random_state=args.random_state,
+        precompute=args.precompute,
+        num_distractors=args.num_distractors,
+        augment=args.augment,
+        rate=args.mux_rate,
+        max_videos=args.max_videos,
+        include_metadata=args.include_metadata)
+
+    map_iterate_in_parallel(range(num_workers), worker_func,
+                            processes=num_workers)
+
+    LOGGER.info('Done!')