Fixed compatibility with existing framework

data/omniglot/preprocess.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+set -x
 
 # download data and convert to .json format
 if [ ! -d "data/all_data" ] || [ ! "$(ls -A data/all_data)" ]; then

data/omniglot/preprocess/data_to_json.py

@@ -7,9 +7,10 @@
 from collections import defaultdict
 
 image_size = (28, 28)
+status_update_after = 5000 # images processed
 
 user_class = dict()
-user_data = defaultdict(list)
+user_data = defaultdict(dict)
 
 parent_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 image_paths = os.path.join(parent_path, 'data', 'raw_data',  'images_*', '*', '*', '*.png')
@@ -22,14 +23,20 @@
     img = Image.open(character_file).resize(image_size, resample=Image.LANCZOS)
     flattened_img = np.array(img.convert('L')).flatten() / 255.
 
-    user_data[user_id].append(flattened_img.tolist())
     if user_id not in user_class:
         user_class[user_id] = character_class
+        user_data[user_id]['x'] = list()
+        user_data[user_id]['y'] = list()
+    user_data[user_id]['x'].append(flattened_img.tolist())
+    user_data[user_id]['y'].append(user_id)
 
-    if (i+1) % 1000 == 0:
+    if (i+1) % status_update_after == 0:
         print ("{} images converted".format(i+1))
 
-all_data = { 'users': list(user_class.items()), 'user_data': user_data }
+all_data = dict()
+all_data['users'] = list(user_class.keys())
+all_data['num_samples'] = [ len(user_data[x]['x']) for x in all_data['users'] ]
+all_data['user_data'] = user_data
 
 file_name = 'all_data.json'
 file_path = os.path.join(parent_path, 'data', 'all_data', file_name)