mesos-slave.json

{
  "AWSTemplateFormatVersion" : "2010-09-09",

  "Description" : "Launches a pool of Mesos Slaves",

  "Parameters" : {
    "InstanceAmi" : {
      "Description" : "Mesos AMI",
      "Type" : "String"
    },
    "InstanceType" : {
      "Description" : "EC2 instance type",
      "Type" : "String",
      "Default" : "t2.micro",
      "AllowedValues" : [
        "t2.micro", "t2.small", "t2.medium",
        "m3.medium", "m3.large", "m3.xlarge", "m3.2xlarge",
        "c3.large", "c3.xlarge", "c3.2xlarge", "c3.4xlarge", "c3.8xlarge",
        "c4.large", "c4.xlarge", "c4.2xlarge", "c4.4xlarge", "c4.8xlarge",
        "r3.large", "r3.xlarge", "r3.2xlarge", "r3.4xlarge", "r3.8xlarge",
        "i2.xlarge", "i2.2xlarge", "i2.4xlarge", "i2.8xlarge",
        "hs1.8xlarge", "g2.2xlarge"
      ],
      "ConstraintDescription" : "must be a valid, HVM-compatible EC2 instance type."
    },
    "KeyName" : {
      "Description" : "Name of an existing EC2 KeyPair to enable SSH access to the instances",
      "Type" : "AWS::EC2::KeyPair::KeyName"
    },
    "ClusterId" : {
      "Description" : "Mesos cluster ID",
      "Type" : "String"
    },
    "ClusterSize" : {
      "Description" : "Number of nodes to launch",
      "Type" : "Number"
    },
    "ExhibitorDiscoveryUrl" : {
      "Description" : "Exhibitor URL that returns active ZooKeeper nodes ('getCluster') (e.g., http://<exhibitor_node>/exhibitor/v1/cluster/list)",
      "Type" : "String"
    },
    "ZkClientSecurityGroup" : {
      "Description" : "Security group ID for ZK clients (should grant access to ZK nodes)",
      "Type" : "AWS::EC2::SecurityGroup::Id"
    },
    "MesosMasterSecurityGroup" : {
      "Description" : "Security group ID of Mesos Master nodes. If not passed, you'll need to grant network access on your own",
      "Type" : "AWS::EC2::SecurityGroup::Id",
      "Default" : ""
    },
    "DockerCredentials" : {
      "Description" : "JSON string to be saved as .dockercfg",
      "Type" : "String",
      "Default" : "{}"
    },
    "LogstashConfig" : {
      "Description" : "(optional) Config string for Logstash (to read task logs, you can use 'input { file { path => \"/tmp/mesos/slaves/*/frameworks/*/executors/*/runs/latest/stdout\" }')",
      "Type" : "String",
      "Default" : ""
    },
    "ConfigScriptS3Bucket" : {
      "Description" : "(optional) S3 bucket of an executable to download and run after other boot-time provisioning (e.g., \"my-bucket\").",
      "Type" : "String",
      "Default" : ""
    },
    "ConfigScriptS3Path" : {
      "Description" : "(optional) Path within ConfigScriptS3Bucket of an executable to download and run after other boot-time provisioning (e.g., \"/path/to/script\").",
      "Type" : "String",
      "Default" : ""
    },
    "MesosResources" : {
      "Description" : "(optional) Resources to allocate to Mesos (as passed to --resources). To expose ports 8000-9000, for instance, you can use 'ports(*):[8000-9000, 31000-32000]'",
      "Type" : "String",
      "Default" : ""
    },
    "AdminSecurityGroup" : {
      "Description" : "Existing security group that should be granted administrative access to Mesos (e.g., 'sg-123456')",
      "Type" : "AWS::EC2::SecurityGroup::Id"
    },
    "Subnets" : {
      "Description" : "List of VPC subnet IDs for the cluster. Must match the passed AvailabilityZones.",
      "Type" : "List<AWS::EC2::Subnet::Id>"
    },
    "VpcId" : {
      "Description" : "VPC associated with the provided subnets",
      "Type" : "AWS::EC2::VPC::Id"
    },
    "AvailabilityZones": {
      "Description" : "(Optional) If passed, only launch nodes in these AZs (e.g., 'us-east-1a,us-east-1b'). Must match the passed Subnets.",
      "Type" : "CommaDelimitedList",
      "Default" : ""
    },
    "StackCreationTimeout" : {
      "Description" : "Timeout on initial stack creation",
      "Type" : "String",
      "Default": "PT10M"
    }
  },

  "Conditions" : {
    "EnableLogstashCondition" : { "Fn::Not" : [{ "Fn::Equals" : [{ "Ref" : "LogstashConfig" }, ""] }] },
    "EnableConfigScript" : { "Fn::And" : [
      { "Fn::Not" : [{ "Fn::Equals" : [{ "Ref" : "ConfigScriptS3Bucket" }, ""] }]},
      { "Fn::Not" : [{ "Fn::Equals" : [{ "Ref" : "ConfigScriptS3Path" }, ""] }]}
    ]},
    "UseAllAvailabilityZones" : { "Fn::Equals" : [{ "Fn::Join" : ["", { "Ref" : "AvailabilityZones"} ]}, ""]},
    "GrantNetworkAccess" : { "Fn::Not" : [{ "Fn::Equals" : [{ "Ref" : "MesosMasterSecurityGroup"}, ""]}]}
  },

  "Resources" : {
    "SlaveRole": {
      "Type": "AWS::IAM::Role",
      "Properties": {
        "AssumeRolePolicyDocument": {
          "Version" : "2012-10-17",
          "Statement": [ {
            "Effect": "Allow",
            "Principal": {
              "Service": [ "ec2.amazonaws.com" ]
            },
            "Action": [ "sts:AssumeRole" ]
          } ]
        },
        "Path": "/",
        "Policies": [ {
          "PolicyName": "root",
          "PolicyDocument": {
            "Version" : "2012-10-17",
            "Statement": [ {
              "Effect": "Allow",
              "Action": [
                "s3:GetObject"
              ],
              "Resource": [
                { "Fn::Join" : ["", ["arn:aws:s3:::", { "Ref" : "ConfigScriptS3Bucket" }, { "Ref" : "ConfigScriptS3Path" }]]}
              ]
            } ]
          }
        } ]
      }
    },

    "SlaveInstanceProfile": {
      "Type": "AWS::IAM::InstanceProfile",
      "Properties": {
        "Path": "/",
        "Roles": [ {
          "Ref": "SlaveRole"
        } ]
      }
    },

    "ServerGroup" : {
      "Type" : "AWS::AutoScaling::AutoScalingGroup",
      "Properties" : {
        "AvailabilityZones" : {
          "Fn::If" : [
            "UseAllAvailabilityZones",
            { "Fn::GetAZs": "AWS::Region" },
            {"Ref" : "AvailabilityZones"}
          ]
        },
        "LaunchConfigurationName" : { "Ref" : "LaunchConfig" },
        "MinSize" : { "Ref" : "ClusterSize" },
        "MaxSize" : { "Ref" : "ClusterSize" },
        "DesiredCapacity" : { "Ref" : "ClusterSize" },
        "VPCZoneIdentifier" : { "Ref" : "Subnets" },
        "Tags" : [
          {
            "Key" : "role",
            "Value" : "mesos-slave",
            "PropagateAtLaunch" : "true"
          }
        ]
      },
      "CreationPolicy" : {
        "ResourceSignal" : {
          "Timeout" : { "Ref" : "StackCreationTimeout" }
        }
      }
    },

    "LaunchConfig" : {
      "Type" : "AWS::AutoScaling::LaunchConfiguration",
      "Metadata" : {
        "AWS::CloudFormation::Init" : {
          "config": {
            "packages" : {
              "python": {
                "awscli": []
              }
            },
            "files" : {
              "/usr/local/bin/zk-list-nodes" : {
                "content" : { "Fn::Join" : ["", [
                  "#!/bin/bash -e\n",
                  "curl -s {{zk_discovery_url}} | python -c '",
                    "import sys, json;",
                    "j=json.load(sys.stdin);",
                    "servers=[\":\".join([s, str(j[\"port\"])]) for s in j[\"servers\"]];",
                    "print \",\".join(servers)'"
                ]]},
                "context" : {
                  "zk_discovery_url"  : { "Ref" : "ExhibitorDiscoveryUrl" }
                },
                "mode"    : "000755",
                "owner"   : "root",
                "group"   : "root"
              },

              "/usr/local/bin/configure-mesos-slave" : {
                "content" : { "Fn::Join" : ["\n", [
                  "#!/bin/bash -e",
                  "echo \"zk://`/usr/local/bin/zk-list-nodes`/{{mesos_cluster_id}}\" > /etc/mesos/zk",
                  "echo \"`/usr/bin/ec2metadata --local-ipv4`\" > /etc/mesos-slave/hostname",
                  "echo \"{{executor_registration_timeout}}\" > /etc/mesos-slave/MESOS_EXECUTOR_REGISTRATION_TIMEOUT",
                  "echo \"{{resources}}\" > /etc/mesos-slave/resources", ""
                ]]},
                "context" : {
                  "mesos_cluster_id": { "Ref": "ClusterId" },
                  "executor_registration_timeout": "5mins",
                  "resources": { "Ref": "MesosResources" }
                },
                "mode"    : "000755",
                "owner"   : "root",
                "group"   : "root"
              },

              "/etc/cron.d/mesos": {
                "content" : { "Fn::Join" : ["\n", [
                  "# Refresh mesos config every minute, but don't restart",
                  "* * * * * root /usr/local/bin/configure-mesos-slave",
                  ""
                ]]},
                "mode"    : "000644",
                "owner"   : "root",
                "group"   : "root"
              },

              "/root/.dockercfg": {
                "content" : { "Ref" : "DockerCredentials" },
                "mode"    : "000600",
                "owner"   : "root",
                "group"   : "root"
              },

              "/etc/default/docker": {
                "content" : "DOCKER_OPTS=\"-g $(readlink -f /var/lib/docker)\"\n",
                "mode"    : "000644",
                "owner"   : "root",
                "group"   : "root"
              }
            }
          }
        }
      },
      "Properties" : {
        "KeyName" : { "Ref" : "KeyName" },
        "ImageId" : { "Ref" : "InstanceAmi" },
        "SecurityGroups" : [ { "Ref" : "SecurityGroup" }, { "Ref" : "ZkClientSecurityGroup" }, { "Ref" : "AdminSecurityGroup" } ],
        "AssociatePublicIpAddress": "false",
        "IamInstanceProfile" : { "Ref" : "SlaveInstanceProfile" },
        "InstanceType" : { "Ref" : "InstanceType" },
        "UserData"       : { "Fn::Base64" : { "Fn::Join" : ["", [
          "#!/bin/bash -ex\n",

          "# Helper function\n",
          "function error_exit\n",
          "{\n",
          "  cfn-signal -e 1 -r \"$1\" ",
              " --resource ServerGroup",
              " --stack ", { "Ref" : "AWS::StackName" },
              " --region ", { "Ref" : "AWS::Region" }, "\n",
          "  exit 1\n",
          "}\n",

          "# Process CloudFormation init definitions\n",
          "cfn-init -s ", { "Ref" : "AWS::StackName" }, " -r LaunchConfig ",
          "    --region ", { "Ref" : "AWS::Region" }, " || error_exit 'Failed to run cfn-init'\n",

          "# Move docker dir to bigger device\n",
          "sudo service docker stop\n",
          "sudo mv /var/lib/docker /mnt/docker\n",
          "sudo ln -s /mnt/docker /var/lib/docker\n",
          "sudo service docker start\n",

          "# Move Mesos workdir to bigger device\n",
          "sudo mkdir /mnt/mesos\n",
          "ln -s /mnt/mesos /tmp/mesos\n",

          { "Fn::If" : [
            "EnableLogstashCondition",
            { "Fn::Join" : ["", [
              "echo '", { "Ref" : "LogstashConfig" },"' > /opt/logstash/logstash.conf\n",
              "runit-service create logstash '/opt/logstash/bin/logstash -f /opt/logstash/logstash.conf'\n",
              "echo '/root' > /etc/sv/logstash/env/HOME\n",
              "runit-service enable logstash\n"
            ]]},
            ""
          ]},

          "# Configure, enable, and start mesos-slave\n",
          "sudo /usr/local/bin/configure-mesos-slave\n",
          "sudo mesos slave "
          " --master=zk://`zk-list-nodes`/" { "Ref": "ClusterId" },
          " --hostname=`ec2metadata --local-ipv4`"
          " --executor_registration_timeout=5mins"
          " --containerizers=docker\n",

          { "Fn::If" : [
            "EnableConfigScript",
            { "Fn::Join" : ["", [
              "# Download and run the user-provided config script \n",
              "aws --region ", { "Ref" : "AWS::Region" },
              "  s3 cp s3://", { "Ref" : "ConfigScriptS3Bucket" }, { "Ref" : "ConfigScriptS3Path" }, " /tmp/cfn-user-config\n",
              "chmod +x /tmp/cfn-user-config\n",
              "/tmp/cfn-user-config || error_exit \n"
            ]]},
            ""
          ]},

          "# All is well so signal success\n",
          "cfn-signal -e 0 -r \"Stack setup complete\" ",
              " --resource ServerGroup",
              " --stack ", { "Ref" : "AWS::StackName" },
              " --region ", { "Ref" : "AWS::Region" }, "\n",

          "#EOF"
        ]]}}
      }
    },

    "SecurityGroup" : {
      "Type" : "AWS::EC2::SecurityGroup",
      "Properties" : {
        "GroupDescription" : "Mesos Slaves",
        "VpcId" : { "Ref" : "VpcId" }
      }
    },

    "MasterToSlaveIngress" : {
      "Type" : "AWS::EC2::SecurityGroupIngress",
      "Condition" : "GrantNetworkAccess",
      "Properties" : {
        "GroupId" : { "Ref" : "SecurityGroup" },
        "IpProtocol" : "tcp",
        "FromPort" : "5051",
        "ToPort" : "5051",
        "SourceSecurityGroupId" : { "Ref" : "MesosMasterSecurityGroup" }
      }
    },

    "SlaveToMasterIngress" : {
      "Type" : "AWS::EC2::SecurityGroupIngress",
      "Condition" : "GrantNetworkAccess",
      "Properties" : {
        "GroupId" : { "Ref" : "MesosMasterSecurityGroup" },
        "IpProtocol" : "tcp",
        "FromPort" : "5050",
        "ToPort" : "5050",
        "SourceSecurityGroupId" : { "Ref" : "SecurityGroup" }
      }
    }
  }
}