Upgrading generate_yamls.py for easier configuration automation.

README.md

@@ -89,6 +89,78 @@ For example, the configuration file in [conf/dynomite.yml](conf/dynomite.yml)
 Finally, to make writing syntactically correct configuration files easier, dynomite provides a command-line argument -t or --test-conf that can be used to test the YAML configuration file for any syntax error.
 
 
+### Configuration YAML Generator
+
+The utility [generate_yams.py](https://github.com/Netflix/dynomite/blob/dev/scripts/dynomite/generate_yamls.py) automates
+.yaml configuration files creation, needed for every node.
+
+The following usage examples show how to create different cluster configuration
+with the command line.
+
+**Usage example 1**: 
+- 2 Datacenters (usa / europe)
+- Each having 1 Rack (usa_rack1 / europe_rack1)
+    - One node per Rack, external ip's should be used (1.1.1.2 / 1.1.1.3)
+
+
+```
+python generate_yamls.py 1.1.1.2:europe_rack1:europe 1.1.1.3:usa_rack1:usa -o mycluster
+``` 
+
+Will generate two yml files outputing the files to 'mycluster' folder
+
+More commands:
+```
+python generate_yamls.py --help
+
+usage:
+        Dynomite Configuration YAML Generator
+
+        Script for generating Dynomite yaml configuration files for distribution with every node.
+        generated yaml files will be outputted for each node, named as {ipaddress}.yml
+        so cluster wide can be easily configured
+
+
+       [-h] [-cp CLIENT_PORT] [-o OUTPUT_DIR] [-sp SERVER_PORT]
+       [-pp PEER_PORT] [-rc {DC_QUORUM,DC_ONE,DC_SAFE_QUORUM}]
+       [-sso {datacenter,none,rack}] [--redis] [--mem]
+       nodes [nodes ...]
+
+positional arguments:
+  nodes                 Usage: <script> publicIp:rack_name:datacenter
+                        publicIp:rack_name:datacenter ... outputs one yaml
+                        file per input node(for a single rack) restrict
+                        generation of the confs for all hosts per rack and not
+                        across rack.
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -cp CLIENT_PORT       Client port to use (The client port Dynomite provides
+                        instead of directly accessing redis or memcache)
+                        Default is: 8102 Your redis or memcache clients should
+                        connect to this port
+  -o OUTPUT_DIR         Output directory for the YAML files, if does not exist
+                        will be created, Default is current directory (.)
+  -sp SERVER_PORT       The port your redis or memcache will run locally on
+                        each node, assuming it's uniform, (default is: 6379)
+  -pp PEER_PORT         The port Dynamo clients will use to communicate with
+                        each other, (default is: 8101)
+  -rc {DC_QUORUM,DC_ONE,DC_SAFE_QUORUM}
+                        Sets the read_consistency of the cluster operation
+                        mode (default is: DC_ONE)
+  -sso {datacenter,none,rack}
+                        Type of communication between Dynomite nodes, Must be
+                        one of 'none', 'rack', 'datacenter', or 'all' (default
+                        is: datacenter)
+  --redis               Sets the data_store property to use redis (0), Default
+                        is 0.
+  --mem                 Sets the data_store property to use memcache (1),
+                        Default is 0.
+
+``` 
+
+
+
 ## License
 
 Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0

scripts/dynomite/generate_yamls.py

@@ -9,52 +9,195 @@
 '''
 
 import yaml, sys
+import argparse
+import os
 
 APPNAME='dyn_o_mite'
-CLIENT_PORT='8102'
-DYN_PEER_PORT=8101
+DEFAULT_CLIENT_PORT='8102'
+DYN_PEER_PORT= '8101'
 MEMCACHE_PORT='11211'
+REDIS_PORT='6379'
 MAX_TOKEN = 4294967295
 
-DEFAULT_DC = 'default_dc'
-
-# generate the equidistant tokens for the number of nodes given. max 4294967295
-token_map = dict()
-token_item = (MAX_TOKEN // (len(sys.argv) -1))
-for i in range(1, len(sys.argv)):
-    node = sys.argv[i]
-    token_value = (token_item * i)
-    if token_value > MAX_TOKEN:
-        token_value = MAX_TOKEN
-
-    token_map[node] = token_value
-
-for k,v in token_map.items():
-    # get the peers ready, and yank the current one from the dict
-    dyn_seeds_map = token_map.copy()
-    del dyn_seeds_map[k]
-    dyn_seeds = []
-    for y,z in dyn_seeds_map.items():
-        key = y.split(':')
-        dyn_seeds.append(key[0] + ':' + str(DYN_PEER_PORT) + ':' + key[1] + ':' + DEFAULT_DC + ':' + str(z));
-
-    ip_dc = k.split(':');
-    data = {
-        'listen': '0.0.0.0:' + CLIENT_PORT,
-        'timeout': 150000,
-        'servers': ['127.0.0.1:' + MEMCACHE_PORT + ':1'],
-        'dyn_seed_provider': 'simple_provider',
-
-        'dyn_port': DYN_PEER_PORT,
-        'dyn_listen': '0.0.0.0:' + str(DYN_PEER_PORT),
-        'datacenter': DEFAULT_DC,
-        'rack': ip_dc[1],
-        'tokens': v,
-        'dyn_seeds': dyn_seeds,
-        }
-
-    outer = {APPNAME: data}
-
-    file_name = ip_dc[0] + '.yml'
-    with open(file_name, 'w') as outfile:
-        outfile.write( yaml.dump(outer, default_flow_style=False) )
+SECURE_SERVER_OPTION = "datacenter"
+READ_CONSISTENCY = "DC_ONE"
+
+
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser("""
+        Dynomite Configuration YAML Generator
+
+        Script for generating Dynomite yaml configuration files for distribution with every node.
+        generated yaml files will be outputted for each node, named as {ipaddress}.yml
+        so cluster wide can be easily configured
+
+    """)
+
+    parser.add_argument('nodes', type=str, nargs='+',
+                        help="""
+            Usage: <script> publicIp:rack_name:datacenter publicIp:rack_name:datacenter ...
+
+            outputs one yaml file per input node(for a single rack)
+            restrict generation of the confs for all hosts per rack and not across rack.
+        """)
+
+
+    parser.add_argument(
+        '-cp',
+        dest='client_port',
+        type=str,
+        default=DEFAULT_CLIENT_PORT,
+        help="""
+            Client port to use (The client port Dynomite provides instead of directly accessing redis or memcache) 
+            Default is: {}
+
+            Your redis or memcache clients should connect to this port
+        """.format(DEFAULT_CLIENT_PORT)
+    )
+
+    parser.add_argument(
+        '-o',
+        dest='output_dir',
+        type=str,
+        default='./',
+        help="""
+            Output directory for the YAML files, if does not exist will be created, Default is current directory (.)
+        """
+    )
+
+    parser.add_argument(
+        '-sp',
+        dest='server_port',
+        type=str,
+        default=REDIS_PORT,
+        help="""
+            The port your redis or memcache will run locally on each node, assuming it's uniform, (default is: {})
+        """.format(REDIS_PORT)
+    )
+
+    parser.add_argument(
+        '-pp',
+        dest='peer_port',
+        type=str,
+        default=DYN_PEER_PORT,
+        help="""
+            The port Dynamo clients will use to communicate with each other, (default is: {})
+        """.format(DYN_PEER_PORT)
+    )
+
+    parser.add_argument(
+        '-rc',
+        dest='read_consistency',
+        type=str,
+        default=READ_CONSISTENCY,
+        choices=set(('DC_ONE', 'DC_QUORUM', 'DC_SAFE_QUORUM')),
+        help="""
+            Sets the read_consistency of the cluster operation mode (default is: DC_ONLY)
+        """
+    )
+
+    parser.add_argument(
+        '-sso',
+        dest='secure_server_option',
+        type=str,
+        default=SECURE_SERVER_OPTION,
+        choices=set(('none', 'rack', 'datacenter')),
+        help="""
+            Type of communication between Dynomite nodes, Must be one of 'none', 'rack', 'datacenter', or 'all' (default is: {})
+        """.format(SECURE_SERVER_OPTION)
+    )
+
+
+    parser.add_argument(
+        '--redis',
+        action="store_true",
+        default=True,
+        help="""
+            Sets the data_store property to use redis (0), Default is 0.
+        """
+    )
+
+    parser.add_argument(
+        '--mem',
+        action="store_true",
+        default=False,
+        help="""
+            Sets the data_store property to use memcache (1), Default is 0.
+        """
+    )
+
+
+    args = parser.parse_args()
+
+
+    dir = os.path.dirname(__file__)
+
+    output_path = args.output_dir
+
+    if os.path.isabs(dir):
+        print "path exists:{}".format(dir)
+    else:
+        output_path = os.path.join(dir, args.output_dir)
+        if (not os.path.exists(output_path)):
+            os.makedirs(output_path)
+            print "creating output path %s" % output_path
+
+
+
+
+    if (args.mem):
+        data_store = 1
+    else:
+        data_store = 0
+
+    # generate the equidistant tokens for the number of nodes given. max 4294967295
+    token_map = dict()
+    total_nodes = len(args.nodes)
+    token_item = (MAX_TOKEN // total_nodes)
+
+    #print "token_item:{}".format(token_item)
+    for i in range(0, total_nodes):
+        node = args.nodes[i]
+        #print "Iterating node file ... > {}.yml".format(node)
+        token_value = (token_item * (i+1))
+        if token_value > MAX_TOKEN:
+            token_value = MAX_TOKEN
+
+        token_map[node] = token_value
+
+    for k,v in token_map.items():
+        # get the peers ready, and yank the current one from the dict
+        #print "k:{} v:{}".format(k,v)
+        dyn_seeds_map = token_map.copy()
+        del dyn_seeds_map[k]
+        dyn_seeds = []
+        for y,z in dyn_seeds_map.items():
+            key = y.split(':')
+            dyn_seeds.append(key[0] + ':' + str(args.peer_port) + ':' + key[1] + ':' + key[2] + ':' + str(z))
+
+        ip_dc = k.split(':')
+        #print ip_dc
+        data = {
+            'data_store': data_store,
+            'listen': '0.0.0.0:' + args.client_port,
+            'timeout': 150000,
+            'servers': ['127.0.0.1:' + args.server_port + ':1'],
+            'dyn_seed_provider': 'simple_provider',
+            'read_consistency': 'DC_ONE',
+            'secure_server_option': args.secure_server_option,
+            'dyn_port': args.peer_port,
+            'dyn_listen': '0.0.0.0:' + args.peer_port,
+            'datacenter': ip_dc[2],
+            'rack': """{}""".format(ip_dc[1]),
+            'tokens': """{}""".format(v),
+            'dyn_seeds': dyn_seeds,
+            }
+
+        outer = {APPNAME: data}
+
+        file_name = """{}.yml""".format(os.path.join(output_path, ip_dc[0]))
+
+        with open(file_name, 'w') as outfile:
+            outfile.write( yaml.dump(outer, default_flow_style=False))