From 677ddb4d8a2c805a2b126b5db856797f41adc474 Mon Sep 17 00:00:00 2001
From: ikemsley <ikemsley@alertlogic.com>
Date: Wed, 13 Sep 2017 15:48:25 +0100
Subject: [PATCH 01/24] Change o365msgs to aicspmsgs

---
 O365WebHook/ingest.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/O365WebHook/ingest.js b/O365WebHook/ingest.js
index 0f4261e..f312842 100644
--- a/O365WebHook/ingest.js
+++ b/O365WebHook/ingest.js
@@ -39,7 +39,7 @@ class Ingest extends m_alServiceC.AlServiceC {
             },
             body : data
         };
-        return this.post(`/data/o365msgs`, payload);
+        return this.post(`/data/aicspmsgs`, payload);
     }
 }
 

From a62a9db75ddd22ba9d8b79f51ed9718b02c13749 Mon Sep 17 00:00:00 2001
From: ikemsley <ikemsley@alertlogic.com>
Date: Mon, 18 Sep 2017 10:18:12 +0100
Subject: [PATCH 02/24] Change urls for creating access key in README

---
 README.md | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index fbf1689..4d1c5c9 100644
--- a/README.md
+++ b/README.md
@@ -46,7 +46,7 @@ In order to install O365 Log collector:
 Login and get an authentication token from the Alert Logic Cloud Insight product [AIMS API](https://console.product.dev.alertlogic.com/api/aims/).  For example, from the command line use [curl](https://en.wikipedia.org/wiki/CURL) as follows (where `<user>` is your CloudInsight user and `<pwd>` is your CloudInsight password):
 
 ```
-curl -X POST -v -u '<user>:<pwd>' https://api.product.dev.alertlogic.com/aims/v1/authenticate
+curl -X POST -v -u '<user>:<pwd>' https://api.global-services.global.alertlogic.com/aims/v1/authenticate
 ```
 
 Make a note of the following fields returned in the response:
@@ -58,7 +58,7 @@ Make a note of the following fields returned in the response:
 Use the authentication token returned in the response to create access keys for the Azure application deployed in the next section.  For example, issue the following curl command (where `<TOKEN>` is the auth token, `<ACCOUNT_ID>` is the account id, and `<USER_ID>` is the user id returned above):
 
 ```
-curl -X POST -H "x-aims-auth-token: <TOKEN>" https://api.product.dev.alertlogic.com/aims/v1/<ACCOUNT_ID>/users/<USER_ID>/access_keys
+curl -X POST -H "x-aims-auth-token: <TOKEN>" https://api.global-services.global.alertlogic.com/aims/v1/<ACCOUNT_ID>/users/<USER_ID>/access_keys
 ```
 
 An example of a successful response is:
@@ -69,6 +69,13 @@ An example of a successful response is:
 
 Make a note of the `access_key_id` and `secret_key` values for use in the deployment steps below.
 
+**Note:** Only five access keys can be created per user.  If you get a "limit exceeded" response you will need to
+delete some keys in order to create new ones.  Use the following command to delete access keys:
+
+```
+curl -X POST -H "x-aims-auth-token: <TOKEN>" https://api.global-services.global.alertlogic.com/aims/v1/<ACCOUNT_ID>/users/<USER_ID>/access_keys/<ACCESS_KEY_ID>
+```
+
 
 ### Download and Deploy the Custom ARM Template in an Azure Subscription
 

From fa796d3b90e5206dbb662f9bbf0607cfcdaa0a83 Mon Sep 17 00:00:00 2001
From: Tomas Dosoudil <tomdos@users.noreply.github.com>
Date: Tue, 19 Sep 2017 10:39:20 +0100
Subject: [PATCH 03/24] use raw link to template.json instead of blob (#24)

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 4d1c5c9..777eafe 100644
--- a/README.md
+++ b/README.md
@@ -79,7 +79,7 @@ curl -X POST -H "x-aims-auth-token: <TOKEN>" https://api.global-services.global.
 
 ### Download and Deploy the Custom ARM Template in an Azure Subscription
 
-1. **TODO: it is possible to use URI deployment without downloading a file.** Download an ARM [template](https://github.com/alertlogic/o365-collector/blob/master/template.json)
+1. **TODO: it is possible to use URI deployment without downloading a file.** Download an ARM [template](https://github.com/alertlogic/o365-collector/raw/master/template.json)
 1. Log into [Azure portal](https://portal.azure.com). **Note**, In order to perform steps below you should have an acive Azure subscription, to find out visit [Azure subscriptions blade](https://portal.azure.com/#blade/Microsoft_Azure_Billing/SubscriptionsBlade)
 1. Go to [Customer Deployment](https://portal.azure.com/#create/Microsoft.Template) page. Type in `deploy` in a seach query located on top of Azure Web UI and select `Deploy a custom template`.
 1. Click `Build your own template in the editor` and load the file previously downloaded on step 1 above.
@@ -102,7 +102,7 @@ You can obtain it from _Azure_ -> _AD_ -> _App registrations_ -> _Your app name_
 ## Using Azure CLI to deploy a template
 
 1. Follow the installation steps up to (but not including) [Download and Deploy the Custom ARM Template in an Azure Subscription](#download_and_deploy_the_custom_arm_template_in_an_azure_subscription) above.
-1. Download [ARM template](template.json) locally
+1. Download [ARM template](https://github.com/alertlogic/o365-collector/raw/master/template.json) locally
 1. Create a resource group with name "ResourceGroupName" in location "West US" by executing following command
 
 ```

From dc0eb18a2ba1157dc0738fa3754dd2d9a1e92586 Mon Sep 17 00:00:00 2001
From: Tomas Dosoudil <tomdos@users.noreply.github.com>
Date: Wed, 20 Sep 2017 10:44:41 +0100
Subject: [PATCH 04/24] cli deployment (#31)

bla

bla
---
 README.md | 46 +++++++++++++++++++++++++---------------------
 1 file changed, 25 insertions(+), 21 deletions(-)

diff --git a/README.md b/README.md
index 777eafe..209d1a6 100644
--- a/README.md
+++ b/README.md
@@ -17,7 +17,7 @@ Installation requires the following steps:
 1. Download and deploy a custom ARM template to Microsoft Azure to create functions for collecting and managing O365 log data
 1. Verify that installation was successful using Alertlogic CloudDefender UI.
 
-### Register a New O365 Web Application in O365
+## Register a New O365 Web Application in O365
 
 In order to install O365 Log collector:
 
@@ -32,7 +32,7 @@ In order to install O365 Log collector:
 
 1. After application is created select it and make a note of `Application Id`, for example, `a261478c-84fb-42f9-84c2-de050a4babe3`
 
-### Set Up the Required Active Directory Security Permissions
+## Set Up the Required Active Directory Security Permissions
 
 1. On the `Settings` panel and select `Required permissions` and click `+Add`
 1. Hit `Select an API` and chose `Office 365 Management APIs`, click `Select`
@@ -41,7 +41,7 @@ In order to install O365 Log collector:
 1. On the `Settings` panel of the application and select `Keys`.
 1. Enter key `Description` and `Duration` and click `Save`. **Note**, please save the key value, it is needed later during template deployment.
 
-### Create an Alert Logic Access Key
+## Create an Alert Logic Access Key
 
 Login and get an authentication token from the Alert Logic Cloud Insight product [AIMS API](https://console.product.dev.alertlogic.com/api/aims/).  For example, from the command line use [curl](https://en.wikipedia.org/wiki/CURL) as follows (where `<user>` is your CloudInsight user and `<pwd>` is your CloudInsight password):
 
@@ -76,8 +76,9 @@ delete some keys in order to create new ones.  Use the following command to dele
 curl -X POST -H "x-aims-auth-token: <TOKEN>" https://api.global-services.global.alertlogic.com/aims/v1/<ACCOUNT_ID>/users/<USER_ID>/access_keys/<ACCESS_KEY_ID>
 ```
 
+## Function deployment
 
-### Download and Deploy the Custom ARM Template in an Azure Subscription
+### Deploy via the Custom ARM Template in an Azure Subscription
 
 1. **TODO: it is possible to use URI deployment without downloading a file.** Download an ARM [template](https://github.com/alertlogic/o365-collector/raw/master/template.json)
 1. Log into [Azure portal](https://portal.azure.com). **Note**, In order to perform steps below you should have an acive Azure subscription, to find out visit [Azure subscriptions blade](https://portal.azure.com/#blade/Microsoft_Azure_Billing/SubscriptionsBlade)
@@ -95,28 +96,31 @@ You can obtain it from _Azure_ -> _AD_ -> _App registrations_ -> _Your app name_
 1. Once deployment is finished go to `Resource groups` blade and select a resource group used for the deployment on step 3 above.
 1. Select `Access Control (IAM)` and add `Website Contributor` role to AD application identity created above.
 
-### Verify the Installation
+### Deploy via Azure CLI
 
-1. Log into Alertlogic CloudDefender and navigate into `Log Manager -> Sources` page. Check new O365 log source (with a name provided on step 15) has been created and source status is `ok`.
-
-## Using Azure CLI to deploy a template
-
-1. Follow the installation steps up to (but not including) [Download and Deploy the Custom ARM Template in an Azure Subscription](#download_and_deploy_the_custom_arm_template_in_an_azure_subscription) above.
-1. Download [ARM template](https://github.com/alertlogic/o365-collector/raw/master/template.json) locally
-1. Create a resource group with name "ResourceGroupName" in location "West US" by executing following command
-
-```
-az group create -n ResourceGroupName -l "West US"
-```
+You can use either [Azure Cloud Shell](https://docs.microsoft.com/en-gb/azure/cloud-shell/quickstart#start-cloud-shell) or
+local installation of [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli?view=azure-cli-latest).
 
-Deploy a template with the application name "ApplicationName" using following command, during its execution enter required parameters when asked
-
-```
-az group deployment create -g ResourceGroupName -n ApplicationName --template-file template.json
-```
+1. Create a resource group with name "AlertLogicCollect" in location "Central US" by executing following command
+    ```
+    az group create --name AlertLogicCollect --location "Central US"
+    ```
+1. Once created go to `Resource groups` blade and select the resource group.
+1. Select `Access Control (IAM)` and add `Website Contributor` role to AD application identity created above.
+1. Deploy a template by using following command, during its execution enter required parameters when asked
+    ```
+    az group deployment create \
+        --name AlertLogicCollect \
+        --resource-group AlertLogicCollect \
+        --template-uri "https://raw.githubusercontent.com/alertlogic/o365-collector/master/template.json"
+    ```
 
 Wait until it is deployed successfully.
 
+## Verify the Installation
+
+1. Go to `Azure Apps` and choose your function. The last log under `Functions-> Master-> Monitor` should have OK status and should not contain any error messages.
+1. Log into Alertlogic CloudDefender and navigate into `Log Manager -> Sources` page. Check new O365 log source (with a name provided on step 15) has been created and source status is `ok`.
 
 # How It Works
 

From 8f7ae3f043bc99e52457ef44a132cc5747ceb408 Mon Sep 17 00:00:00 2001
From: Konstantin Kuzmin <kkuzmin@alertlogic.com>
Date: Tue, 19 Sep 2017 19:34:39 +0100
Subject: [PATCH 05/24] Fill log batch with correct metadata.

---
 O365WebHook/ingest_proto.js        | 62 +++++++++++++++---------------
 local_dev/o365webhook_local_dev.js | 10 ++---
 2 files changed, 35 insertions(+), 37 deletions(-)

diff --git a/O365WebHook/ingest_proto.js b/O365WebHook/ingest_proto.js
index 0b8f5c9..2498e7a 100644
--- a/O365WebHook/ingest_proto.js
+++ b/O365WebHook/ingest_proto.js
@@ -13,6 +13,7 @@ const protobuf = require('protobufjs');
 const async = require('async');
 const Long = require('long');
 const path = require('path');
+const crypto = require('crypto');
 
 // FIXME - protobuf load
 // We have to load PROTO_DEF every invocation. Maybe the solution can to to use
@@ -43,20 +44,28 @@ module.exports.setMessage = function(context, root, content, callback) {
 
 module.exports.setHostMetadata = function(context, root, content, callback) {
     var hostmetaType = root.lookupType('host_metadata.metadata');
-
+    var hostmetaData = getHostmeta(context, root);
+    var meta = {
+        hostUuid : process.env.O365_COLLECTOR_ID,
+        data : hostmetaData,
+        dataChecksum : new Buffer('')
+    };
+    var sha = crypto.createHash('sha1');
+    var hashPayload = hostmetaType.encode(meta).finish();
+    hashValue = sha.update(hashPayload).digest();
+    
     var metadataPayload = {
-        // FIXME - we need to calculate checksum properly
-        dataChecksum: new Buffer.from([234,104,231,10,12,60,139,208,204,230,
-                          236,248,60,113,61,93,52,49,18,194]),
-        timestamp: Math.floor(Date.now() / 1000),
-        data: dummyMetadataDict(context, root)
+        hostUuid : process.env.O365_COLLECTOR_ID,
+        dataChecksum : hashValue,
+        timestamp : Math.floor(Date.now() / 1000),
+        data : hostmetaData
     };
 
     build(hostmetaType, metadataPayload, function(err, buf) {
         if (err)
             context.log('Error: Unable to build host_metadata.');
 
-        callback(err, buf);
+        return callback(err, buf);
     });
 };
 
@@ -74,7 +83,7 @@ module.exports.setBatch = function(context, root, metadata, messages, callback)
         if (err)
             context.log('Error: Unable to build collected_batch.');
 
-        callback(err, buf);
+        return callback(err, buf);
     });
 };
 
@@ -90,14 +99,14 @@ module.exports.setBatchList = function(context, root, batches, callback) {
         if (err)
             context.log('Error: Unable to build collected_batch_list.');
 
-        callback(err, buf);
+        return callback(err, buf);
     });
 };
 
 module.exports.encode = function(context, root, batchList, callback) {
     var batchListType = root.lookupType('common_proto.collected_batch_list');
     var buf = batchListType.encode(batchList).finish();
-    callback(null, buf);
+    return callback(null, buf);
 };
 
 
@@ -110,7 +119,7 @@ function build(type, payload, callback) {
 
     var payloadCreated = type.create(payload);
 
-    callback(null, payloadCreated);
+    return callback(null, payloadCreated);
 }
 
 
@@ -142,41 +151,32 @@ function parseMessage(context, root, memo, content, callback) {
             context.log('Error: Unable to build collected_message.');
 
         memo.push(buf);
-        callback(err, memo);
+        return callback(err, memo);
     });
 }
 
-// TODO - Fill Metadata dictionary with some dummy content.
-// FIXME - we need to use some real data in metadata
-function dummyMetadataDict(context, root) {
+function getHostmeta(context, root) {
     var dictType = root.lookupType('alc_dict.dict');
     var elemType = root.lookupType('alc_dict.elem');
     var valueType = root.lookupType('alc_dict.value');
 
-    var val1 = {str: 'standalone'};
-    var valPayload1 = buildSync(valueType, val1);
-
-    var val2 = {str: '454712-mnimn2.syd.intensive.int'};
-    var valPayload2 = buildSync(valueType, val2);
-
-    var elem1 = {
+    var hostTypeVal = {str: 'azure_fun'};
+    var hostTypeElem = {
         key: 'host_type',
-        value: val1
+        value: hostTypeVal
     };
-    var elemPayload1 = buildSync(elemType, elem1);
 
-    var elem2 = {
+    var localHostnameVal = {str: process.env.WEBSITE_HOSTNAME};
+    var localHostnameElem = {
         key: 'local_hostname',
-        value: val2
+        value: localHostnameVal
     };
-    var elemPayload2 = buildSync(elemType, elem2);
 
     var dict = {
-        elem: [elem1, elem2]
+        elem: [localHostnameElem, hostTypeElem]
     };
-    var dictPayload = buildSync(dictType, dict);
-
-    return dictPayload;
+    
+    return buildSync(dictType, dict);
 }
 
 
diff --git a/local_dev/o365webhook_local_dev.js b/local_dev/o365webhook_local_dev.js
index d16dc03..5d95846 100644
--- a/local_dev/o365webhook_local_dev.js
+++ b/local_dev/o365webhook_local_dev.js
@@ -69,13 +69,11 @@ var debugEvent = {
             //   "contentExpiration": "2017-08-22T22:47:57.748Z"
             // },
            {
+    "contentUri": "https://manage.office.com/api/v1.0/bf8d32d3-1c13-4487-af02-80dba2236485/activity/feed/audit/20170919042652571002899$20170919042652571002899$audit_exchange$Audit_Exchange",
+    "contentId": "20170919042652571002899$20170919042652571002899$audit_exchange$Audit_Exchange",
     "contentType": "Audit.Exchange",
-    "contentId": "20170905123301777014849$20170905123301777014849$audit_exchange$Audit_Exchange$IsFromNotification",
-    "contentUri": "https://manage.office.com/api/v1.0/bf8d32d3-1c13-4487-af02-80dba2236485/activity/feed/audit/20170905123301777014849$20170905123301777014849$audit_exchange$Audit_Exchange$IsFromNotification",
-    "notificationStatus": "Succeeded",
-    "contentCreated": "2017-09-05T12:37:27.163Z",
-    "notificationSent": "2017-09-05T12:37:27.163Z",
-    "contentExpiration": "2017-09-12T12:33:01.777Z"
+    "contentCreated": "2017-09-19T04:26:52.571Z",
+    "contentExpiration": "2017-09-26T04:26:52.571Z"
   }
         ]
 };

From 2703b8aa099b6b170776990bc9e83ef0aac9fe36 Mon Sep 17 00:00:00 2001
From: Konstantin Kuzmin <kkuzmin@alertlogic.com>
Date: Tue, 19 Sep 2017 19:35:31 +0100
Subject: [PATCH 06/24] Refactor.

---
 Master/o365collector.js | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/Master/o365collector.js b/Master/o365collector.js
index f7faf38..955a3c9 100644
--- a/Master/o365collector.js
+++ b/Master/o365collector.js
@@ -23,7 +23,10 @@ exports.checkRegister = function (context, AlertlogicMasterTimer, azcollectSvc,
     } else {
         // Collector is not registered.
         azcollectSvc.register_o365().then(resp => {
-            m_appSettings.updateAppsettings({O365_COLLECTOR_ID: resp.source.id}, 
+            let newSettings = {
+                O365_COLLECTOR_ID: resp.source.id
+            };
+            m_appSettings.updateAppsettings(newSettings, 
                 function(settingsError) {
                     if (settingsError) {
                         return callback(settingsError);

From effdc1ae2100a6466b7e5a3121ca76067fc57fd2 Mon Sep 17 00:00:00 2001
From: Konstantin Kuzmin <kkuzmin@alertlogic.com>
Date: Wed, 20 Sep 2017 12:21:10 +0100
Subject: [PATCH 07/24] Use o365 host id for log batches.

---
 Master/o365collector.js            |  6 ++++--
 O365WebHook/ingest_proto.js        | 12 ++++--------
 local_dev/o365webhook_local_dev.js | 11 ++---------
 3 files changed, 10 insertions(+), 19 deletions(-)

diff --git a/Master/o365collector.js b/Master/o365collector.js
index 955a3c9..79e1ef2 100644
--- a/Master/o365collector.js
+++ b/Master/o365collector.js
@@ -17,15 +17,17 @@ const m_o365mgmnt = require('../lib/o365_mgmnt');
 
 
 exports.checkRegister = function (context, AlertlogicMasterTimer, azcollectSvc, callback) {
-    if (process.env.O365_COLLECTOR_ID) {
+    if (process.env.O365_COLLECTOR_ID && process.env.O365_HOST_ID) {
         context.log('DEBUG: Reuse collector id', process.env.O365_COLLECTOR_ID);
         return callback(null, process.env.O365_COLLECTOR_ID);
     } else {
         // Collector is not registered.
         azcollectSvc.register_o365().then(resp => {
             let newSettings = {
-                O365_COLLECTOR_ID: resp.source.id
+                O365_COLLECTOR_ID: resp.source.id,
+                O365_HOST_ID: resp.source.host.id
             };
+            context.log(newSettings);
             m_appSettings.updateAppsettings(newSettings, 
                 function(settingsError) {
                     if (settingsError) {
diff --git a/O365WebHook/ingest_proto.js b/O365WebHook/ingest_proto.js
index 2498e7a..c85d5b5 100644
--- a/O365WebHook/ingest_proto.js
+++ b/O365WebHook/ingest_proto.js
@@ -46,7 +46,7 @@ module.exports.setHostMetadata = function(context, root, content, callback) {
     var hostmetaType = root.lookupType('host_metadata.metadata');
     var hostmetaData = getHostmeta(context, root);
     var meta = {
-        hostUuid : process.env.O365_COLLECTOR_ID,
+        hostUuid : process.env.O365_HOST_ID,
         data : hostmetaData,
         dataChecksum : new Buffer('')
     };
@@ -55,7 +55,7 @@ module.exports.setHostMetadata = function(context, root, content, callback) {
     hashValue = sha.update(hashPayload).digest();
     
     var metadataPayload = {
-        hostUuid : process.env.O365_COLLECTOR_ID,
+        hostUuid : process.env.O365_HOST_ID,
         dataChecksum : hashValue,
         timestamp : Math.floor(Date.now() / 1000),
         data : hostmetaData
@@ -160,18 +160,14 @@ function getHostmeta(context, root) {
     var elemType = root.lookupType('alc_dict.elem');
     var valueType = root.lookupType('alc_dict.value');
 
-    var hostTypeVal = {str: 'azure_fun'};
     var hostTypeElem = {
         key: 'host_type',
-        value: hostTypeVal
+        value: {str: 'azure_fun'}
     };
-
-    var localHostnameVal = {str: process.env.WEBSITE_HOSTNAME};
     var localHostnameElem = {
         key: 'local_hostname',
-        value: localHostnameVal
+        value: {str: process.env.WEBSITE_HOSTNAME}
     };
-
     var dict = {
         elem: [localHostnameElem, hostTypeElem]
     };
diff --git a/local_dev/o365webhook_local_dev.js b/local_dev/o365webhook_local_dev.js
index 5d95846..44eb5ad 100644
--- a/local_dev/o365webhook_local_dev.js
+++ b/local_dev/o365webhook_local_dev.js
@@ -58,7 +58,7 @@ var debugEvent = {
             //   "contentCreated": "2017-08-13T23:03:56.050Z",
             //   "notificationSent": "2017-08-13T23:03:56.050Z",
             //   "contentExpiration": "2017-08-20T16:44:49.279Z"
-            // }
+            // },
             // {
             //   "contentType": "Audit.AzureActiveDirectory",
             //   "contentId": "20170815224757748004716$20170815224757748004716$audit_azureactivedirectory$Audit_AzureActiveDirectory$IsFromNotification",
@@ -67,14 +67,7 @@ var debugEvent = {
             //   "contentCreated": "2017-08-17T12:31:25.653Z",
             //   "notificationSent": "2017-08-17T12:31:25.653Z",
             //   "contentExpiration": "2017-08-22T22:47:57.748Z"
-            // },
-           {
-    "contentUri": "https://manage.office.com/api/v1.0/bf8d32d3-1c13-4487-af02-80dba2236485/activity/feed/audit/20170919042652571002899$20170919042652571002899$audit_exchange$Audit_Exchange",
-    "contentId": "20170919042652571002899$20170919042652571002899$audit_exchange$Audit_Exchange",
-    "contentType": "Audit.Exchange",
-    "contentCreated": "2017-09-19T04:26:52.571Z",
-    "contentExpiration": "2017-09-26T04:26:52.571Z"
-  }
+            // }
         ]
 };
 

From 2337597ddeef984cadc9dfbe0b163a1730900b60 Mon Sep 17 00:00:00 2001
From: Konstantin Kuzmin <kkuzmin@alertlogic.com>
Date: Wed, 20 Sep 2017 16:43:18 +0100
Subject: [PATCH 08/24] Validate webhook address during Master checkin. Add
 tests.

---
 Master/o365collector.js |   8 ++-
 package.json            |   7 ++-
 test/master_test.js     |  61 ++++++++++++++++++++
 test/mock.js            | 122 ++++++++++++++++++++++++++++++++++++++++
 4 files changed, 195 insertions(+), 3 deletions(-)
 create mode 100644 test/master_test.js
 create mode 100644 test/mock.js

diff --git a/Master/o365collector.js b/Master/o365collector.js
index f7faf38..29c13b7 100644
--- a/Master/o365collector.js
+++ b/Master/o365collector.js
@@ -14,9 +14,11 @@ const async = require('async');
 const m_alServiceC = require('../lib/al_servicec');
 const m_appSettings = require('./appsettings');
 const m_o365mgmnt = require('../lib/o365_mgmnt');
+const functionJson = require('./function.json');
 
 
 exports.checkRegister = function (context, AlertlogicMasterTimer, azcollectSvc, callback) {
+    console.log(functionJson);
     if (process.env.O365_COLLECTOR_ID) {
         context.log('DEBUG: Reuse collector id', process.env.O365_COLLECTOR_ID);
         return callback(null, process.env.O365_COLLECTOR_ID);
@@ -80,13 +82,17 @@ exports.checkin = function (context, AlertlogicMasterTimer, azcollectSvc, callba
 var _checkEnableAuditStreams = function(context, listedStreams, callback) {
     try {
         let o365AuditStreams = JSON.parse(process.env.O365_CONTENT_STREAMS);
+        // TODO: take webhook path from O365Webhook/function.json
+        let webhookURL = 'https://' + process.env.WEBSITE_HOSTNAME +
+            '/api/o365/webhook';
         async.map(o365AuditStreams,
             function(stream, asyncCallback) {
                 let currentStream = listedStreams.find(
                         obj => obj.contentType === stream);
                 if (currentStream && currentStream.status === 'enabled' &&
                     currentStream.webhook && 
-                    currentStream.webhook.status === 'enabled') {
+                    currentStream.webhook.status === 'enabled' &&
+                    currentStream.webhook.address === webhookURL) {
                     context.log('DEBUG: Stream already enabled', stream);
                     return asyncCallback(null, stream);
                 } else {
diff --git a/package.json b/package.json
index 39a64ec..763d48d 100644
--- a/package.json
+++ b/package.json
@@ -16,10 +16,13 @@
     "local-ad": "node ./local_dev/ad_token_local_dev.js",
     "local-version": "node ./local_dev/version_local_dev.js",
     "lint": "jshint --exclude \"./node_modules/*\" **/*.js",
-    "test": "npm run lint"
+    "test": "npm run lint && mocha"
   },
   "devDependencies": {
     "jshint": "^2.9.5",
-    "pre-commit": "^1.2.2"
+    "mocha": "^3.5.3",
+    "pre-commit": "^1.2.2",
+    "rewire": "^2.5.2",
+    "sinon": "^3.3.0"
   }
 }
diff --git a/test/master_test.js b/test/master_test.js
new file mode 100644
index 0000000..9839161
--- /dev/null
+++ b/test/master_test.js
@@ -0,0 +1,61 @@
+/* -----------------------------------------------------------------------------
+ * @copyright (C) 2017, Alert Logic, Inc
+ * @doc
+ * 
+ * Unit tests for Master function
+ * 
+ * @end
+ * -----------------------------------------------------------------------------
+ */
+ 
+var assert = require('assert');
+var rewire = require('rewire');
+var sinon = require('sinon');
+
+var testMock = require('./mock');
+var m_o365mgmnt = require('../lib/o365_mgmnt');
+
+var o365collector = rewire('../Master/o365collector');
+
+describe('Master Function Units', function() {
+    var private_checkEnableAuditStreams;
+    var subscriptionsStartStub;
+    
+    before(function() {
+        private_checkEnableAuditStreams = o365collector.__get__('_checkEnableAuditStreams');
+        subscriptionsStartStub = sinon.stub(m_o365mgmnt, 'subscriptionsStart').callsFake(
+            function fakeFn(contentType, webhook, callback) {
+                return callback(null);
+        });
+    });
+    after(function() {
+        subscriptionsStartStub.restore();
+    });
+            
+    describe('_checkEnableAuditStreams()', function() {
+        it('checks already enabled streams with proper webhook configs', function(done) {
+            process.env.O365_CONTENT_STREAMS = 
+                '["Audit.AzureActiveDirectory", "Audit.Exchange", "Audit.SharePoint", "Audit.General"]';
+            private_checkEnableAuditStreams(testMock.context, testMock.allEnabledStreams, function(err, streams){
+                if (err)
+                    return done(err);
+                
+                assert.equal(4, streams.length);
+                assert.equal(0, subscriptionsStartStub.callCount);
+                done();
+            });
+        });
+        
+        it('checks subscriptionStart is called for already enabled webhooks if web app is being reinstalled', function(done) {
+            process.env.O365_CONTENT_STREAMS = 
+                '["Audit.AzureActiveDirectory", "Audit.Exchange", "Audit.SharePoint", "Audit.General"]';
+            private_checkEnableAuditStreams(testMock.context, testMock.oneOldEnabledStream, function(err, streams){
+                if (err)
+                    return done(err);
+                
+                assert.equal(1, subscriptionsStartStub.callCount);
+                done();
+            });
+        });
+    });
+});
diff --git a/test/mock.js b/test/mock.js
new file mode 100644
index 0000000..37aa181
--- /dev/null
+++ b/test/mock.js
@@ -0,0 +1,122 @@
+/* -----------------------------------------------------------------------------
+ * @copyright (C) 2017, Alert Logic, Inc
+ * @doc
+ * 
+ * Mocks for unit testing.
+ * 
+ * @end
+ * -----------------------------------------------------------------------------
+ */
+process.env.WEBSITE_HOSTNAME = 'kkuzmin-app-o365.azurewebsites.net';
+process.env.O365_CONTENT_STREAMS = '["Audit.AzureActiveDirectory", "Audit.Exchange", "Audit.SharePoint", "Audit.General"]';
+process.env.TMP = '/tmp/';
+process.env.APP_SUBSCRIPTION_ID = 'subscription-id';
+process.env.CUSTOMCONNSTR_APP_CLIENT_ID = 'client-id';
+process.env.CUSTOMCONNSTR_APP_CLIENT_SECRET = 'client-secret';
+process.env.APP_TENANT_ID = 'test.onmicrosoft.com';
+process.env.O365_TENANT_ID = 'test.onmicrosoft.com';
+
+var context = {
+    invocationId: 'ID',
+    bindings: {
+    },
+    log: function () {
+        var util = require('util');
+        var val = util.format.apply(null, arguments);
+        console.log(val);
+    },
+    done: function () {
+        console.log('Test response:');
+    },
+    res: null
+};
+
+var allEnabledStreams = [
+  {
+    "contentType": "Audit.AzureActiveDirectory",
+    "status": "enabled",
+    "webhook": {
+      "authId": null,
+      "address": "https://kkuzmin-app-o365.azurewebsites.net/api/o365/webhook",
+      "expiration": "",
+      "status": "enabled"
+    }
+  },
+  {
+    "contentType": "Audit.Exchange",
+    "status": "enabled",
+    "webhook": {
+      "authId": null,
+      "address": "https://kkuzmin-app-o365.azurewebsites.net/api/o365/webhook",
+      "expiration": "",
+      "status": "enabled"
+    }
+  },
+  {
+    "contentType": "Audit.General",
+    "status": "enabled",
+    "webhook": {
+      "authId": null,
+      "address": "https://kkuzmin-app-o365.azurewebsites.net/api/o365/webhook",
+      "expiration": "",
+      "status": "enabled"
+    }
+  },
+  {
+    "contentType": "Audit.SharePoint",
+    "status": "enabled",
+    "webhook": {
+      "authId": null,
+      "address": "https://kkuzmin-app-o365.azurewebsites.net/api/o365/webhook",
+      "expiration": "",
+      "status": "enabled"
+    }
+  }
+];
+
+var oneOldEnabledStream = [
+  {
+    "contentType": "Audit.AzureActiveDirectory",
+    "status": "enabled",
+    "webhook": {
+      "authId": null,
+      "address": "https://old-app.azurewebsites.net/api/o365/webhook",
+      "expiration": "",
+      "status": "enabled"
+    }
+  },
+  {
+    "contentType": "Audit.Exchange",
+    "status": "enabled",
+    "webhook": {
+      "authId": null,
+      "address": "https://kkuzmin-app-o365.azurewebsites.net/api/o365/webhook",
+      "expiration": "",
+      "status": "enabled"
+    }
+  },
+  {
+    "contentType": "Audit.General",
+    "status": "enabled",
+    "webhook": {
+      "authId": null,
+      "address": "https://kkuzmin-app-o365.azurewebsites.net/api/o365/webhook",
+      "expiration": "",
+      "status": "enabled"
+    }
+  },
+  {
+    "contentType": "Audit.SharePoint",
+    "status": "enabled",
+    "webhook": {
+      "authId": null,
+      "address": "https://kkuzmin-app-o365.azurewebsites.net/api/o365/webhook",
+      "expiration": "",
+      "status": "enabled"
+    }
+  }
+];
+
+exports.allEnabledStreams = allEnabledStreams;
+exports.oneOldEnabledStream = oneOldEnabledStream;
+exports.context = context;

From fb0eaa9e7e346d787a743134980c5aa97c9490ce Mon Sep 17 00:00:00 2001
From: Konstantin Kuzmin <kkuzmin@alertlogic.com>
Date: Thu, 21 Sep 2017 10:17:32 +0100
Subject: [PATCH 09/24] Code cleanup

---
 Master/o365collector.js | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/Master/o365collector.js b/Master/o365collector.js
index 29c13b7..39c1363 100644
--- a/Master/o365collector.js
+++ b/Master/o365collector.js
@@ -14,11 +14,9 @@ const async = require('async');
 const m_alServiceC = require('../lib/al_servicec');
 const m_appSettings = require('./appsettings');
 const m_o365mgmnt = require('../lib/o365_mgmnt');
-const functionJson = require('./function.json');
 
 
 exports.checkRegister = function (context, AlertlogicMasterTimer, azcollectSvc, callback) {
-    console.log(functionJson);
     if (process.env.O365_COLLECTOR_ID) {
         context.log('DEBUG: Reuse collector id', process.env.O365_COLLECTOR_ID);
         return callback(null, process.env.O365_COLLECTOR_ID);
@@ -96,10 +94,6 @@ var _checkEnableAuditStreams = function(context, listedStreams, callback) {
                     context.log('DEBUG: Stream already enabled', stream);
                     return asyncCallback(null, stream);
                 } else {
-                    // TODO: take webhook path from O365Webhook/function.json
-                    let webhookURL = 'https://' + 
-                        process.env.WEBSITE_HOSTNAME +
-                        '/api/o365/webhook';
                     let webhook = { webhook : {
                         address : webhookURL,
                         expiration : ""

From f3b6b97970bab978a9687dd2aa5d9c746ebe2620 Mon Sep 17 00:00:00 2001
From: Konstantin Kuzmin <kkuzmin@alertlogic.com>
Date: Thu, 21 Sep 2017 10:51:58 +0100
Subject: [PATCH 10/24] Use funpack. Add default values. Fix repo name.

Fix readme and template

Fix template
---
 README.md     | 10 +++++-----
 template.json | 13 ++++++++++---
 2 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index 209d1a6..9b4354a 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,4 @@
-# o365-collector
+# azure-collector
 
 Alert Logic Office 365 Log Collector
 
@@ -80,7 +80,7 @@ curl -X POST -H "x-aims-auth-token: <TOKEN>" https://api.global-services.global.
 
 ### Deploy via the Custom ARM Template in an Azure Subscription
 
-1. **TODO: it is possible to use URI deployment without downloading a file.** Download an ARM [template](https://github.com/alertlogic/o365-collector/raw/master/template.json)
+1. **TODO: it is possible to use URI deployment without downloading a file.** Download an ARM [template](https://github.com/alertlogic/azure-collector/raw/master/template.json)
 1. Log into [Azure portal](https://portal.azure.com). **Note**, In order to perform steps below you should have an acive Azure subscription, to find out visit [Azure subscriptions blade](https://portal.azure.com/#blade/Microsoft_Azure_Billing/SubscriptionsBlade)
 1. Go to [Customer Deployment](https://portal.azure.com/#create/Microsoft.Template) page. Type in `deploy` in a seach query located on top of Azure Web UI and select `Deploy a custom template`.
 1. Click `Build your own template in the editor` and load the file previously downloaded on step 1 above.
@@ -112,7 +112,7 @@ local installation of [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/ins
     az group deployment create \
         --name AlertLogicCollect \
         --resource-group AlertLogicCollect \
-        --template-uri "https://raw.githubusercontent.com/alertlogic/o365-collector/master/template.json"
+        --template-uri "https://raw.githubusercontent.com/alertlogic/azure-collector/master/template.json"
     ```
 
 Wait until it is deployed successfully.
@@ -169,8 +169,8 @@ A notification contains a link to the actual lo data which is retrieved by the `
 
 # Local Development
 
-1. Clone repo `git clone git@github.com:alertlogic/o365-collector.git`
-1. `cd o365-collector`
+1. Clone repo `git clone git@github.com:alertlogic/azure-collector.git`
+1. `cd azure-collector`
 1. Run `./local_dev/setup.sh`
 1. Edit `./local_dev/dev_config.js`
 1. Run Master function locally: `npm run local-master`
diff --git a/template.json b/template.json
index a2386ec..01a026d 100644
--- a/template.json
+++ b/template.json
@@ -3,10 +3,13 @@
     "contentVersion": "1.0.0.0",
     "parameters": {
         "Name": {
-            "type": "String"
+            "type": "String",
+            "defaultValue": "AlertlogicCollectorApp"
         },
         "Storage Name": {
-            "type": "String"
+            "type": "String",
+            "defaultValue": "AlertlogicCollectorStorage"
+            
         },
         "Alertlogic Access Key Id": {
             "type": "String"
@@ -44,7 +47,7 @@
         },
         "Repository URL": {
             "type": "String",
-            "defaultValue": "https://github.com/alertlogic/o365-collector.git"
+            "defaultValue": "https://github.com/alertlogic/azure-collector.git"
         },
         "Repository Branch": {
             "type": "String",
@@ -80,6 +83,10 @@
                             "name": "FUNCTIONS_EXTENSION_VERSION",
                             "value": "~1"
                         },
+                        {
+                            "name": "SCM_USE_FUNCPACK",
+                            "value": "1"
+                        },
                         {
                             "name": "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING",
                             "value": "[concat('DefaultEndpointsProtocol=https;AccountName=',parameters('Storage Name'),';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('Storage Name')), '2015-06-15').key1)]"

From 0de1945ac9c12cd22b80aae47cdf49a479e8b415 Mon Sep 17 00:00:00 2001
From: Konstantin Kuzmin <kkuzmin@alertlogic.com>
Date: Thu, 21 Sep 2017 11:22:04 +0100
Subject: [PATCH 11/24] Fix

---
 README.md     | 2 +-
 template.json | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 9b4354a..6787c78 100644
--- a/README.md
+++ b/README.md
@@ -110,7 +110,7 @@ local installation of [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/ins
 1. Deploy a template by using following command, during its execution enter required parameters when asked
     ```
     az group deployment create \
-        --name AlertLogicCollect \
+        --name AlertLogicCollectorApp \
         --resource-group AlertLogicCollect \
         --template-uri "https://raw.githubusercontent.com/alertlogic/azure-collector/master/template.json"
     ```
diff --git a/template.json b/template.json
index 01a026d..d5c7505 100644
--- a/template.json
+++ b/template.json
@@ -8,7 +8,7 @@
         },
         "Storage Name": {
             "type": "String",
-            "defaultValue": "AlertlogicCollectorStorage"
+            "defaultValue": "alertlogiccollectorstorage"
             
         },
         "Alertlogic Access Key Id": {

From f5f47fc6bd5a86b7c5cdb012682c6964f0b7d9bf Mon Sep 17 00:00:00 2001
From: Konstantin Kuzmin <kkuzmin@alertlogic.com>
Date: Thu, 21 Sep 2017 11:24:31 +0100
Subject: [PATCH 12/24] Fix default storage name

---
 template.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/template.json b/template.json
index d5c7505..5d53ada 100644
--- a/template.json
+++ b/template.json
@@ -8,7 +8,7 @@
         },
         "Storage Name": {
             "type": "String",
-            "defaultValue": "alertlogiccollectorstorage"
+            "defaultValue": "alcollectorstorage"
             
         },
         "Alertlogic Access Key Id": {

From 5f4cce791dc173be76cc59f9ee37d70c45c7f020 Mon Sep 17 00:00:00 2001
From: Konstantin Kuzmin <kkuzmin@alertlogic.com>
Date: Thu, 21 Sep 2017 15:26:41 +0100
Subject: [PATCH 13/24] Address comments

---
 README.md     | 6 ++----
 template.json | 2 +-
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/README.md b/README.md
index 6787c78..7dd7851 100644
--- a/README.md
+++ b/README.md
@@ -110,7 +110,7 @@ local installation of [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/ins
 1. Deploy a template by using following command, during its execution enter required parameters when asked
     ```
     az group deployment create \
-        --name AlertLogicCollectorApp \
+        --name AlertLogicCollector \
         --resource-group AlertLogicCollect \
         --template-uri "https://raw.githubusercontent.com/alertlogic/azure-collector/master/template.json"
     ```
@@ -165,7 +165,7 @@ The `Collector` function exposes an HTTP API endpoint `https://<app-name>/o365/w
 ]
 ```
 
-A notification contains a link to the actual lo data which is retrieved by the `Collector`, wrapped into a protobuf structure [TBD link]() and is sent into Alertlogic Ingest service.
+A notification contains a link to the actual data which is retrieved by the `Collector`, wrapped into a protobuf structure [TBD link]() and is sent into Alertlogic Ingest service.
 
 # Local Development
 
@@ -193,8 +193,6 @@ You can obtain it from _Azure_ -> _AD_ -> _App registrations_ -> _Your app name_
 # Known Issues/ Open Questions
 
 - Sometimes deployments fail after siteSync action. We need better updater to handle that in order not to wait for 12 hours for the next update attempt.
-- Put correct metadata into log batches.
-- Initial Azure Function deployment may take up to 45 minutes.
 
 # Useful Links
 
diff --git a/template.json b/template.json
index 5d53ada..fa241f5 100644
--- a/template.json
+++ b/template.json
@@ -4,7 +4,7 @@
     "parameters": {
         "Name": {
             "type": "String",
-            "defaultValue": "AlertlogicCollectorApp"
+            "defaultValue": "AlertlogicCollector"
         },
         "Storage Name": {
             "type": "String",

From f37dc46fbb5e94b10f69286be628c868d9405008 Mon Sep 17 00:00:00 2001
From: Konstantin Kuzmin <kkuzmin@alertlogic.com>
Date: Thu, 21 Sep 2017 15:42:49 +0100
Subject: [PATCH 14/24] Address comments

---
 template.json | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/template.json b/template.json
index fa241f5..d1b4352 100644
--- a/template.json
+++ b/template.json
@@ -4,12 +4,10 @@
     "parameters": {
         "Name": {
             "type": "String",
-            "defaultValue": "AlertlogicCollector"
+            "defaultValue": "AlertLogicCollector"
         },
         "Storage Name": {
-            "type": "String",
-            "defaultValue": "alcollectorstorage"
-            
+            "type": "String"
         },
         "Alertlogic Access Key Id": {
             "type": "String"

From 255d7d6978c5e495bddb5fdc342e60b93f88e3a8 Mon Sep 17 00:00:00 2001
From: Konstantin Kuzmin <kkuzmin@alertlogic.com>
Date: Thu, 21 Sep 2017 17:00:38 +0100
Subject: [PATCH 15/24] Return errors into context.done()

---
 Master/index.js  | 2 +-
 Updater/index.js | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Master/index.js b/Master/index.js
index 601a3e5..2248c63 100644
--- a/Master/index.js
+++ b/Master/index.js
@@ -61,6 +61,6 @@ module.exports = function (context, AlertlogicMasterTimer) {
         if (error) {
             context.log('ERROR: Master error ', error);
         }
-        context.done();
+        context.done(error);
     });
 };
diff --git a/Updater/index.js b/Updater/index.js
index f60c85f..e87b11e 100644
--- a/Updater/index.js
+++ b/Updater/index.js
@@ -30,7 +30,7 @@ module.exports = function (context, AlertlogicUpdaterTimer) {
             siteSync(context, adToken, function(syncError) {
                 if (syncError) {
                     context.log('Site sync failed: ', syncError);
-                    context.done();
+                    context.done(syncError);
                 } else {
                     context.log('Site sync OK');
                     context.done();

From 9af55156db75a568d8c250cc9fffd390f612eba8 Mon Sep 17 00:00:00 2001
From: ikemsley <ikemsley@gmail.com>
Date: Fri, 22 Sep 2017 16:30:25 +0100
Subject: [PATCH 16/24] US63918 - Create RBAC role for app identity via ARM
 template (#6)

* US63918 - Create RBAC role for app identity via ARM template

* US63918 - Create RBAC role for app identity via ARM template

* US63918 - Create RBAC role for app identity via ARM template

* US63918 - Create RBAC role for app identity via ARM template

* US63918 - Create RBAC role for app identity via ARM template

* US63918 - Create RBAC role for app identity via ARM template
---
 README.md     | 31 ++++++++++++++++++++++---------
 template.json | 18 +++++++++++++++++-
 2 files changed, 39 insertions(+), 10 deletions(-)

diff --git a/README.md b/README.md
index 7dd7851..fce4f29 100644
--- a/README.md
+++ b/README.md
@@ -40,6 +40,11 @@ In order to install O365 Log collector:
 1. On `Required permissions` panel click `Required permissions` button and confirm the selection. **Note**, only AD tenant admin can grant permisions to an Azure AD application.
 1. On the `Settings` panel of the application and select `Keys`.
 1. Enter key `Description` and `Duration` and click `Save`. **Note**, please save the key value, it is needed later during template deployment.
+1. Save the `Application ID` and `Service Principal Id` for use below. To get the `Service Principal Id`, navigate to the `Registered App` blade, 
+click on the link under `Managed application in local directory`.  Then click `Properties`.  The `Service Principal Id`
+is labled `Object ID` on the properties page.  **Caution** This is not the same `Object ID` listed in the `Properties` blade reached 
+by clicking `Settings` or `All Settings` from the `Registered app`.  It is also not the `Object ID` shown on the `Registered app`
+blade itself.   
 
 ## Create an Alert Logic Access Key
 
@@ -86,15 +91,23 @@ curl -X POST -H "x-aims-auth-token: <TOKEN>" https://api.global-services.global.
 1. Click `Build your own template in the editor` and load the file previously downloaded on step 1 above.
 1. Click `Save` button.
 2. Fill in required template parameters and click the `Purchase` button to start a deployment. I.e.:
-   - `APP_TENANT_ID` - The GUID of the tenant e.g. `alazurealertlogic.onmicrosoft.com`
-   - `CUSTOMCONNSTR_APP_CLIENT_ID` - The GUID of your application that created the subscription.
-You can obtain it from _Azure_ -> _AD_ -> _App registrations_ -> _Your app name_
-   - `CUSTOMCONNSTR_APP_CLIENT_SECRET` - A secret key of your application from _App Registrations_.
-   - `CUSTOMCONNSTR_APP_CI_ACCESS_KEY_ID` - `access_key_id` returned from AIMs [above](#create_an_alert_logic_access_key).
-   - `CUSTOMCONNSTR_APP_CI_SECRET_KEY`- `secret_key` returned from AIMs [above](#create_an_alert_logic_access_key).
-
-1. Once deployment is finished go to `Resource groups` blade and select a resource group used for the deployment on step 3 above.
-1. Select `Access Control (IAM)` and add `Website Contributor` role to AD application identity created above.
+   - `Name` - Any name
+   - `Storage Name` - Any Storage Account name (that does not currently exist)
+   - `Alertlogic Access Key Id` - `access_key_id` returned from AIMs [above](#create_an_alert_logic_access_key)
+   - `Alertlogic Secret Key` - `secret_key` returned from AIMs [above](#create_an_alert_logic_access_key)
+   - `Alertlogic API endpoint` - usually `api.global-services.global.alertlogic.com` 
+   - `Alertlogic Data Residency` - usually `default`
+   - `Office365 Content Streams` - The list of streams you would like to collect.  Valid values are:
+        - ["Audit.AzureActiveDirectory","Audit.Exchange","Audit.SharePoint","Audit.General", "DLP.All"]
+   - `Office365 Tenant Id` - The GUID of the tenant e.g. `alazurealertlogic.onmicrosoft.com`
+   - `Service Principal Id` - The `Object ID` of the application that created the subscription.
+   You can obtain it from _Azure_ -> _AD_ -> _App registrations_ -> _Your app name_ -> Link under 
+_Managed application in local directory_ -> _Properties_ -> _Object ID_
+   - `App Client Id` - The GUID of your application that created the subscription.
+                     You can obtain it from _Azure_ -> _AD_ -> _App registrations_ -> _Your app name_
+   - `App Client Secret` - The secret key of your application from _App Registrations_
+   - `Repository URL` - must be `https://github.com/alertlogic/azure-collector.git`
+   - `Repository Branch` - should usually be `master`
 
 ### Deploy via Azure CLI
 
diff --git a/template.json b/template.json
index d1b4352..54d8382 100644
--- a/template.json
+++ b/template.json
@@ -37,6 +37,9 @@
         "Office365 Tenant Id": {
             "type": "String"
         },
+        "Service Principal Id": {
+            "type": "String"
+        },
         "App Client Id": {
             "type": "String"
         },
@@ -55,8 +58,11 @@
     "variables": {
         "location": "[resourceGroup().location]",
         "resourceGroupName": "[resourceGroup().name]",
+        "resourceGroupId": "[resourceGroup().id]",
+        "roleAssignmentId": "[guid(uniqueString( resourceGroup().id, deployment().name ))]",
         "subscriptionId": "[split(subscription().id, '/')[2]]",
-        "tenantId": "[subscription().tenantId]"
+        "tenantId": "[subscription().tenantId]",
+        "contributor": "[concat('/subscriptions/', subscription().subscriptionId, '/providers/Microsoft.Authorization/roleDefinitions/', 'b24988ac-6180-42a0-ab88-20f7382dd24c')]"
     },
     "resources": [
         {
@@ -178,6 +184,16 @@
             "properties": {
                 "accountType": "Standard_LRS"
             }
+        },
+        {
+            "type": "Microsoft.Authorization/roleAssignments",
+            "name": "[variables('roleAssignmentId')]",
+            "apiVersion": "2015-07-01",
+            "properties": {
+                "roleDefinitionId": "[variables('contributor')]",
+                "principalId": "[parameters('Service Principal Id')]",
+                "scope": "[variables('resourceGroupId')]"
+            }
         }
     ]
 }

From 0c237e12336b5763517ecaaeca1eec9099618822 Mon Sep 17 00:00:00 2001
From: Tomas Dosoudil <tomdos@users.noreply.github.com>
Date: Mon, 25 Sep 2017 15:26:16 +0100
Subject: [PATCH 17/24] cleanup outdated docs (#9)

---
 azure_doc.md | 32 --------------------------------
 1 file changed, 32 deletions(-)
 delete mode 100644 azure_doc.md

diff --git a/azure_doc.md b/azure_doc.md
deleted file mode 100644
index 940327a..0000000
--- a/azure_doc.md
+++ /dev/null
@@ -1,32 +0,0 @@
-# Azure unofficial doc
-This document should contain clarification of official documentation and behavior of Azure in practice.
-
-## Office 365 Management Activity API reference
-The documentation about [Office 365 API](https://msdn.microsoft.com/en-us/office-365/office-365-management-activity-api-reference) 
-contains several either mistakes or behavior we could not observer. 
-
-### Notification failure and retry
-The notification shall retry delivery in case of failure. If it encounter excessive failures it shall increase the time between retries. 
-That may potentially lead to disabling the webhook. However we could not observer any message redelivery neither disabling the webhook. 
-The webhook was invoked 8 times and contained 19 messages (contents) in total. 
-
-### Endpoint with startTime and endTime
-Either both must be present or omitted and the difference between each of them can not be larger than 24h. It is incorrect and it doesn't 
-work with both parameters. It worked with `startTime` only and the difference between `startTime` and now can be more than 24h.
-
-
-## Azure Function
-
-### Return status
-There should be at least three ways how to specify return status (code) according this 
-(doc)[https://docs.microsoft.com/en-us/azure/azure-functions/functions-reference-node]. However the only way which works is:
-```
-context.res.headers = {};
-context.res.status = 500;
-```
-
-### Status in Invocation log
-The status (either red cross or green tick) most likely indicates whether the function crashes or finishes with `context.done()`. It doesn't 
-reflect return status code.
-
-

From ec919a9ffce30bd6788884ac40ab283c646e4383 Mon Sep 17 00:00:00 2001
From: kkuzmin <kkuzmin@alertlogic.com>
Date: Mon, 25 Sep 2017 17:13:04 +0100
Subject: [PATCH 18/24] Checkins and updates time distribution (#7)

* Better Master cron
Use post deployment action hooks

* Use seconds value for master trigger schedule.

* Fix schedule
---
 PostDeploymentActions/updateMasterTimer.ps1  | 8 ++++++++
 PostDeploymentActions/updateUpdaterTimer.ps1 | 9 +++++++++
 template.json                                | 4 ++++
 3 files changed, 21 insertions(+)
 create mode 100755 PostDeploymentActions/updateMasterTimer.ps1
 create mode 100755 PostDeploymentActions/updateUpdaterTimer.ps1

diff --git a/PostDeploymentActions/updateMasterTimer.ps1 b/PostDeploymentActions/updateMasterTimer.ps1
new file mode 100755
index 0000000..456aa2c
--- /dev/null
+++ b/PostDeploymentActions/updateMasterTimer.ps1
@@ -0,0 +1,8 @@
+$date = Get-Date
+$min = ($date.Minute + 1) % 15
+$sec = $date.Second
+$new_schedule = "$sec $min-59/15 * * * *"
+Write-Output "Updating Master timer trigger with ($new_schedule)."
+$master_function = Get-Content '..\\wwwroot\\Master\\function.json' -raw | ConvertFrom-Json
+$master_function.bindings | % {if($_.name -eq 'AlertlogicMasterTimer'){$_.schedule=$new_schedule}}
+$master_function | ConvertTo-Json  | set-content '..\\wwwroot\\Master\\function.json'
\ No newline at end of file
diff --git a/PostDeploymentActions/updateUpdaterTimer.ps1 b/PostDeploymentActions/updateUpdaterTimer.ps1
new file mode 100755
index 0000000..af874c4
--- /dev/null
+++ b/PostDeploymentActions/updateUpdaterTimer.ps1
@@ -0,0 +1,9 @@
+$randH = Get-Random -minimum 0 -maximum 11
+$randM = Get-Random -minimum 0 -maximum 59
+$randS = Get-Random -minimum 0 -maximum 59
+$randH12 = $randH + 12
+$new_schedule = "$randS $randM $randH,$randH12 * * *"
+Write-Output "Updating Updater timer trigger with ($new_schedule)".
+$master_function = Get-Content '..\\wwwroot\\Updater\\function.json' -raw | ConvertFrom-Json
+$master_function.bindings | % {if($_.name -eq 'AlertlogicUpdaterTimer'){$_.schedule=$new_schedule}}
+$master_function | ConvertTo-Json  | set-content '..\\wwwroot\\Updater\\function.json'
\ No newline at end of file
diff --git a/template.json b/template.json
index 54d8382..ac94579 100644
--- a/template.json
+++ b/template.json
@@ -91,6 +91,10 @@
                             "name": "SCM_USE_FUNCPACK",
                             "value": "1"
                         },
+                        {
+                            "name": "SCM_POST_DEPLOYMENT_ACTIONS_PATH",
+                            "value": "PostDeploymentActions"
+                        },
                         {
                             "name": "WEBSITE_CONTENTAZUREFILECONNECTIONSTRING",
                             "value": "[concat('DefaultEndpointsProtocol=https;AccountName=',parameters('Storage Name'),';AccountKey=',listKeys(resourceId('Microsoft.Storage/storageAccounts', parameters('Storage Name')), '2015-06-15').key1)]"

From 5808f669ed1a6e92ca8f8801ee995015fc9add33 Mon Sep 17 00:00:00 2001
From: kkuzmin <kkuzmin@alertlogic.com>
Date: Mon, 25 Sep 2017 17:20:20 +0100
Subject: [PATCH 19/24] Use built-in azure fun log levels. (#8)

* Use built-in azure fun log levels.

* Refactor local_dev

* Add warning message for oversized payload.

* Fir to use  context.log.warn()

* Print info ok in webhook.
---
 Master/endpoints.js                |  4 ++--
 Master/index.js                    |  8 ++++----
 Master/o365collector.js            |  5 ++---
 O365WebHook/index.js               |  4 ++--
 O365WebHook/ingest_proto.js        | 12 ++++++------
 O365WebHook/o365content.js         |  7 +++++--
 Updater/index.js                   |  6 +++---
 local_dev/master_local_dev.js      | 19 +++++++++++++++----
 local_dev/o365webhook_local_dev.js | 28 +++++++++++++++-------------
 local_dev/updater_local_dev.js     | 19 +++++++++++++++----
 test/mock.js                       | 20 ++++++++++++++++----
 11 files changed, 85 insertions(+), 47 deletions(-)

diff --git a/Master/endpoints.js b/Master/endpoints.js
index 1514c57..2a39cf0 100644
--- a/Master/endpoints.js
+++ b/Master/endpoints.js
@@ -16,8 +16,8 @@ const m_appSettings = require('./appsettings');
 
 exports.checkUpdate = function (context, AlertlogicMasterTimer, callback) {
     if (process.env.APP_INGEST_ENDPOINT && process.env.APP_AZCOLLECT_ENDPOINT) {
-        context.log('DEBUG: Reuse Ingest endpoint', process.env.APP_INGEST_ENDPOINT);
-        context.log('DEBUG: Reuse Azcollect endpoint', process.env.APP_AZCOLLECT_ENDPOINT);
+        context.log.verbose('Reuse Ingest endpoint', process.env.APP_INGEST_ENDPOINT);
+        context.log.verbose('Reuse Azcollect endpoint', process.env.APP_AZCOLLECT_ENDPOINT);
         return callback(null);
     } else {
         // Endpoint settings do not exist. Update them.
diff --git a/Master/index.js b/Master/index.js
index 2248c63..05b95be 100644
--- a/Master/index.js
+++ b/Master/index.js
@@ -28,7 +28,7 @@ module.exports = function (context, AlertlogicMasterTimer) {
                     if (endpointsError) {
                         return asyncCallback(endpointsError);
                     }
-                    context.log('INFO: Alertlogic endpoints updated.');
+                    context.log.info('Alertlogic endpoints updated.');
                     return asyncCallback(null);
             });
         },
@@ -41,7 +41,7 @@ module.exports = function (context, AlertlogicMasterTimer) {
                     if (azcollectError) {
                         return asyncCallback(azcollectError);
                     }
-                    context.log('INFO: O365 source registered', collectorId);
+                    context.log.info('O365 source registered', collectorId);
                     return asyncCallback(null, azcollectSvc);
                 });
         },
@@ -52,14 +52,14 @@ module.exports = function (context, AlertlogicMasterTimer) {
                     if (azcollectError) {
                         return asyncCallback(`Checkin failed ${azcollectError}`);
                     }
-                    context.log('INFO: O365 source checkin OK', checkinResp);
+                    context.log.info('O365 source checkin OK', checkinResp);
                     return asyncCallback(null);
                 });
         }
     ],
     function(error, results) {
         if (error) {
-            context.log('ERROR: Master error ', error);
+            context.log.error('Master error ', error);
         }
         context.done(error);
     });
diff --git a/Master/o365collector.js b/Master/o365collector.js
index 4b3777f..986fa33 100644
--- a/Master/o365collector.js
+++ b/Master/o365collector.js
@@ -18,7 +18,7 @@ const m_o365mgmnt = require('../lib/o365_mgmnt');
 
 exports.checkRegister = function (context, AlertlogicMasterTimer, azcollectSvc, callback) {
     if (process.env.O365_COLLECTOR_ID && process.env.O365_HOST_ID) {
-        context.log('DEBUG: Reuse collector id', process.env.O365_COLLECTOR_ID);
+        context.log.verbose('Reuse collector id', process.env.O365_COLLECTOR_ID);
         return callback(null, process.env.O365_COLLECTOR_ID);
     } else {
         // Collector is not registered.
@@ -27,7 +27,6 @@ exports.checkRegister = function (context, AlertlogicMasterTimer, azcollectSvc,
                 O365_COLLECTOR_ID: resp.source.id,
                 O365_HOST_ID: resp.source.host.id
             };
-            context.log(newSettings);
             m_appSettings.updateAppsettings(newSettings, 
                 function(settingsError) {
                     if (settingsError) {
@@ -96,7 +95,7 @@ var _checkEnableAuditStreams = function(context, listedStreams, callback) {
                     currentStream.webhook && 
                     currentStream.webhook.status === 'enabled' &&
                     currentStream.webhook.address === webhookURL) {
-                    context.log('DEBUG: Stream already enabled', stream);
+                    context.log.verbose('Stream already enabled', stream);
                     return asyncCallback(null, stream);
                 } else {
                     let webhook = { webhook : {
diff --git a/O365WebHook/index.js b/O365WebHook/index.js
index 4545426..0b2d091 100644
--- a/O365WebHook/index.js
+++ b/O365WebHook/index.js
@@ -25,12 +25,12 @@ module.exports = function (context, event) {
         return m_o365content.processNotifications(context, eventBody,
             function(err) {
                 if (err) {
-                    context.log(`ERROR: ${err}`);
+                    context.log.error(`${err}`);
                     context.res.headers = {};
                     context.res.status = 500;
                     context.done();
                 } else {
-                    context.log('Debug: Success!');
+                    context.log.info('OK!');
                     context.done();
                 }
             });
diff --git a/O365WebHook/ingest_proto.js b/O365WebHook/ingest_proto.js
index c85d5b5..7b0bd23 100644
--- a/O365WebHook/ingest_proto.js
+++ b/O365WebHook/ingest_proto.js
@@ -21,7 +21,7 @@ const crypto = require('crypto');
 module.exports.load = function(context, callback) {
     protobuf.load(getCommonProtoPath(), function(err, root) {
         if (err)
-            context.log('Error: Unable to load proto files.', err);
+            context.log.error('Unable to load proto files.', err);
 
         callback(err, root);
     });
@@ -34,7 +34,7 @@ module.exports.setMessage = function(context, root, content, callback) {
         },
         function(err, result) {
             if (err)
-                context.log('Error: Unable to build messages.');
+                context.log.error('Unable to build messages.');
 
             callback(err, result);
         }
@@ -63,7 +63,7 @@ module.exports.setHostMetadata = function(context, root, content, callback) {
 
     build(hostmetaType, metadataPayload, function(err, buf) {
         if (err)
-            context.log('Error: Unable to build host_metadata.');
+            context.log.error('Unable to build host_metadata.');
 
         return callback(err, buf);
     });
@@ -81,7 +81,7 @@ module.exports.setBatch = function(context, root, metadata, messages, callback)
 
     build(batchType, batchPayload, function(err, buf) {
         if (err)
-            context.log('Error: Unable to build collected_batch.');
+            context.log.error('Unable to build collected_batch.');
 
         return callback(err, buf);
     });
@@ -97,7 +97,7 @@ module.exports.setBatchList = function(context, root, batches, callback) {
 
     build(batchListType, batchListPayload, function(err, buf) {
         if (err)
-            context.log('Error: Unable to build collected_batch_list.');
+            context.log.error('Unable to build collected_batch_list.');
 
         return callback(err, buf);
     });
@@ -148,7 +148,7 @@ function parseMessage(context, root, memo, content, callback) {
 
     build(messageType, messagePayload, function(err, buf) {
         if (err)
-            context.log('Error: Unable to build collected_message.');
+            context.log.error('Unable to build collected_message.');
 
         memo.push(buf);
         return callback(err, memo);
diff --git a/O365WebHook/o365content.js b/O365WebHook/o365content.js
index 94581c3..13ec585 100644
--- a/O365WebHook/o365content.js
+++ b/O365WebHook/o365content.js
@@ -83,7 +83,7 @@ function parseContent(context, parsedContent, callback) {
 
             var creationTime;
             if (item.CreationTime == undefined) {
-                context.log('WARNING: Unable to parse CreationTime from content.');
+                context.log.warn('Unable to parse CreationTime from content.');
                 creationTime = Math.floor(Date.now() / 1000);
             }
             else {
@@ -105,7 +105,7 @@ function parseContent(context, parsedContent, callback) {
             if (err) {
                 return callback(`Content parsing failure. ${err}`);
             } else {
-                context.log('DEBUG: parsedData: ', result);
+                context.log.verbose('parsedData: ', result);
                 return callback(null, result);
             }
         }
@@ -152,6 +152,9 @@ function sendToIngest(context, content, callback) {
                 if (err) {
                     return callback(`Unable to compress. ${err}`);
                 } else {
+                    if (compressed.byteLength > 700000)
+                        context.log.warn(`Compressed log batch length`,
+                            `(${compressed.byteLength}) exceeds maximum allowed value.`);
                     return g_ingestc.sendO365Data(compressed)
                         .then(resp => {
                             return callback(null, resp);
diff --git a/Updater/index.js b/Updater/index.js
index e87b11e..9b32a68 100644
--- a/Updater/index.js
+++ b/Updater/index.js
@@ -23,16 +23,16 @@ module.exports = function (context, AlertlogicUpdaterTimer) {
 
     requestNewToken(context, creds, function(tokenError, adToken) {
         if (tokenError) {
-            context.log('Error getting AD token: ',
+            context.log.error('Error getting AD token: ',
                 tokenError.statusCode, tokenError.statusMessage);
             context.done();
         } else {
             siteSync(context, adToken, function(syncError) {
                 if (syncError) {
-                    context.log('Site sync failed: ', syncError);
+                    context.log.error('Site sync failed: ', syncError);
                     context.done(syncError);
                 } else {
-                    context.log('Site sync OK');
+                    context.log.info('Site sync OK');
                     context.done();
                 }
             });
diff --git a/local_dev/master_local_dev.js b/local_dev/master_local_dev.js
index eee3bca..6abd5b4 100644
--- a/local_dev/master_local_dev.js
+++ b/local_dev/master_local_dev.js
@@ -1,3 +1,5 @@
+const util = require('util');
+
 var devConfig = require('./dev_config');
 var azureFunction = require('../Master/index');
 
@@ -39,10 +41,19 @@ var debugContext = {
     bindings: {
         req
     },
-    log: function () {
-        var util = require('util');
-        var val = util.format.apply(null, arguments);
-        console.log(val);
+    log: {
+        error : function() {
+            return console.log('ERROR:', util.format.apply(null, arguments));
+        },
+        warn : function() {
+            return console.log('WARNING:', util.format.apply(null, arguments));
+        },
+        info : function() {
+            return console.log('INFO:', util.format.apply(null, arguments));
+        },
+        verbose : function() {
+            return console.log('VERBOSE:', util.format.apply(null, arguments));
+        }
     },
     done: function () {
         console.log('Response:', this.res);
diff --git a/local_dev/o365webhook_local_dev.js b/local_dev/o365webhook_local_dev.js
index 44eb5ad..30478bf 100644
--- a/local_dev/o365webhook_local_dev.js
+++ b/local_dev/o365webhook_local_dev.js
@@ -1,3 +1,5 @@
+const util = require('util');
+
 var devConfig = require('./dev_config');
 var azureFunction = require('../O365WebHook/index');
 
@@ -31,10 +33,19 @@ var debugContext = {
     bindings: {
         req
     },
-    log: function () {
-        var util = require('util');
-        var val = util.format.apply(null, arguments);
-        console.log(val);
+    log: {
+        error : function() {
+            return console.log('ERROR:', util.format.apply(null, arguments));
+        },
+        warn : function() {
+            return console.log('WARNING:', util.format.apply(null, arguments));
+        },
+        info : function() {
+            return console.log('INFO:', util.format.apply(null, arguments));
+        },
+        verbose : function() {
+            return console.log('VERBOSE:', util.format.apply(null, arguments));
+        }
     },
     done: function () {
         console.log('Response:', this.res);
@@ -58,15 +69,6 @@ var debugEvent = {
             //   "contentCreated": "2017-08-13T23:03:56.050Z",
             //   "notificationSent": "2017-08-13T23:03:56.050Z",
             //   "contentExpiration": "2017-08-20T16:44:49.279Z"
-            // },
-            // {
-            //   "contentType": "Audit.AzureActiveDirectory",
-            //   "contentId": "20170815224757748004716$20170815224757748004716$audit_azureactivedirectory$Audit_AzureActiveDirectory$IsFromNotification",
-            //   "contentUri": "https://manage.office.com/api/v1.0/bf8d32d3-1c13-4487-af02-80dba2236485/activity/feed/audit/20170815224757748004716$20170815224757748004716$audit_azureactivedirectory$Audit_AzureActiveDirectory$IsFromNotification",
-            //   "notificationStatus": "Failed",
-            //   "contentCreated": "2017-08-17T12:31:25.653Z",
-            //   "notificationSent": "2017-08-17T12:31:25.653Z",
-            //   "contentExpiration": "2017-08-22T22:47:57.748Z"
             // }
         ]
 };
diff --git a/local_dev/updater_local_dev.js b/local_dev/updater_local_dev.js
index 3581890..c31d407 100644
--- a/local_dev/updater_local_dev.js
+++ b/local_dev/updater_local_dev.js
@@ -1,3 +1,5 @@
+const util = require('util');
+
 var devConfig = require('./dev_config');
 var azureFunction = require('../Updater/index');
 
@@ -39,10 +41,19 @@ var debugContext = {
     bindings: {
         req
     },
-    log: function () {
-        var util = require('util');
-        var val = util.format.apply(null, arguments);
-        console.log(val);
+    log: {
+        error : function() {
+            return console.log('ERROR:', util.format.apply(null, arguments));
+        },
+        warn : function() {
+            return console.log('WARNING:', util.format.apply(null, arguments));
+        },
+        info : function() {
+            return console.log('INFO:', util.format.apply(null, arguments));
+        },
+        verbose : function() {
+            return console.log('VERBOSE:', util.format.apply(null, arguments));
+        }
     },
     done: function () {
         console.log('Response:', this.res);
diff --git a/test/mock.js b/test/mock.js
index 37aa181..8db594e 100644
--- a/test/mock.js
+++ b/test/mock.js
@@ -7,6 +7,9 @@
  * @end
  * -----------------------------------------------------------------------------
  */
+ 
+const util = require('util');
+
 process.env.WEBSITE_HOSTNAME = 'kkuzmin-app-o365.azurewebsites.net';
 process.env.O365_CONTENT_STREAMS = '["Audit.AzureActiveDirectory", "Audit.Exchange", "Audit.SharePoint", "Audit.General"]';
 process.env.TMP = '/tmp/';
@@ -20,10 +23,19 @@ var context = {
     invocationId: 'ID',
     bindings: {
     },
-    log: function () {
-        var util = require('util');
-        var val = util.format.apply(null, arguments);
-        console.log(val);
+    log: {
+        error : function() {
+            return console.log('ERROR:', util.format.apply(null, arguments));
+        },
+        warn : function() {
+            return console.log('WARNING:', util.format.apply(null, arguments));
+        },
+        info : function() {
+            return console.log('INFO:', util.format.apply(null, arguments));
+        },
+        verbose : function() {
+            return console.log('VERBOSE:', util.format.apply(null, arguments));
+        }
     },
     done: function () {
         console.log('Test response:');

From 1f025685582dfbfac7e78fd0cafa8b84cb977025 Mon Sep 17 00:00:00 2001
From: kkuzmin <kkuzmin@alertlogic.com>
Date: Mon, 25 Sep 2017 17:20:31 +0100
Subject: [PATCH 20/24] Fix readme (#10)

* Fix readme

* Fix typos
---
 README.md | 27 ++++++++++++++-------------
 1 file changed, 14 insertions(+), 13 deletions(-)

diff --git a/README.md b/README.md
index fce4f29..d4e8953 100644
--- a/README.md
+++ b/README.md
@@ -5,14 +5,14 @@ Alert Logic Office 365 Log Collector
 
 # Overview
 
-This repo contains Azure Web application Node.js source code and an ARM template for setting up a data collector in Azure which will collect and forward Office 365 log data to the Alert Logic Cloud Defender Log Manager (LM) feature.
+This repository contains Azure Web application Node.js source code and an ARM template for setting up a data collector in Azure which will collect and forward Office 365 log data to the Alert Logic Cloud Defender Log Manager (LM) feature.
 
 # Installation
 
 Installation requires the following steps:
 
 1. Register a new O365 web application in O365 portal for collecting O365 logs.
-1. Set up the required Active Directory security permissions for the application to authorize it to read threat intelligence data and activity reports for your orgaization.
+1. Set up the required Active Directory security permissions for the application to authorize it to read threat intelligence data and activity reports for your organization.
 1. Create an Access Key that will allow the application to connect to the Alert Logic Cloud Defender and Cloud Insight backend.
 1. Download and deploy a custom ARM template to Microsoft Azure to create functions for collecting and managing O365 log data
 1. Verify that installation was successful using Alertlogic CloudDefender UI.
@@ -37,18 +37,18 @@ In order to install O365 Log collector:
 1. On the `Settings` panel and select `Required permissions` and click `+Add`
 1. Hit `Select an API` and chose `Office 365 Management APIs`, click `Select`
 1. In `Application permissions` select `Read service health information for your organization`, `Read activity data for your organization`, `Read threat intelligence data for your organization` and `Read activity reports for your organization`. Click `Select` and `Done` buttons.
-1. On `Required permissions` panel click `Required permissions` button and confirm the selection. **Note**, only AD tenant admin can grant permisions to an Azure AD application.
+1. On `Required permissions` panel click `Required permissions` button and confirm the selection. **Note**, only AD tenant admin can grant permissions to an Azure AD application.
 1. On the `Settings` panel of the application and select `Keys`.
 1. Enter key `Description` and `Duration` and click `Save`. **Note**, please save the key value, it is needed later during template deployment.
 1. Save the `Application ID` and `Service Principal Id` for use below. To get the `Service Principal Id`, navigate to the `Registered App` blade, 
 click on the link under `Managed application in local directory`.  Then click `Properties`.  The `Service Principal Id`
-is labled `Object ID` on the properties page.  **Caution** This is not the same `Object ID` listed in the `Properties` blade reached 
+is labeled `Object ID` on the properties page.  **Caution** This is not the same `Object ID` listed in the `Properties` blade reached 
 by clicking `Settings` or `All Settings` from the `Registered app`.  It is also not the `Object ID` shown on the `Registered app`
 blade itself.   
 
 ## Create an Alert Logic Access Key
 
-Login and get an authentication token from the Alert Logic Cloud Insight product [AIMS API](https://console.product.dev.alertlogic.com/api/aims/).  For example, from the command line use [curl](https://en.wikipedia.org/wiki/CURL) as follows (where `<user>` is your CloudInsight user and `<pwd>` is your CloudInsight password):
+Login and get an authentication token from the Alert Logic Cloud Insight product [AIMS API](https://console.product.dev.alertlogic.com/api/aims/).  From the command line use [curl](https://en.wikipedia.org/wiki/CURL) as follows (where `<user>` is your CloudInsight user and `<pwd>` is your CloudInsight password):
 
 ```
 curl -X POST -v -u '<user>:<pwd>' https://api.global-services.global.alertlogic.com/aims/v1/authenticate
@@ -60,7 +60,7 @@ Make a note of the following fields returned in the response:
    * ACCOUNT ID
    * TOKEN
 
-Use the authentication token returned in the response to create access keys for the Azure application deployed in the next section.  For example, issue the following curl command (where `<TOKEN>` is the auth token, `<ACCOUNT_ID>` is the account id, and `<USER_ID>` is the user id returned above):
+Use the authentication token returned in the response to create access keys for the Azure application deployed in the next section.  Issue the following curl command (where `<TOKEN>` is the auth token, `<ACCOUNT_ID>` is the account id, and `<USER_ID>` is the user id returned above):
 
 ```
 curl -X POST -H "x-aims-auth-token: <TOKEN>" https://api.global-services.global.alertlogic.com/aims/v1/<ACCOUNT_ID>/users/<USER_ID>/access_keys
@@ -83,11 +83,12 @@ curl -X POST -H "x-aims-auth-token: <TOKEN>" https://api.global-services.global.
 
 ## Function deployment
 
+Log into [Azure portal](https://portal.azure.com). **Note**, In order to perform steps below you should have an active Azure subscription, to find out visit [Azure subscriptions blade](https://portal.azure.com/#blade/Microsoft_Azure_Billing/SubscriptionsBlade)
+
 ### Deploy via the Custom ARM Template in an Azure Subscription
 
-1. **TODO: it is possible to use URI deployment without downloading a file.** Download an ARM [template](https://github.com/alertlogic/azure-collector/raw/master/template.json)
-1. Log into [Azure portal](https://portal.azure.com). **Note**, In order to perform steps below you should have an acive Azure subscription, to find out visit [Azure subscriptions blade](https://portal.azure.com/#blade/Microsoft_Azure_Billing/SubscriptionsBlade)
-1. Go to [Customer Deployment](https://portal.azure.com/#create/Microsoft.Template) page. Type in `deploy` in a seach query located on top of Azure Web UI and select `Deploy a custom template`.
+1. Download an ARM [template](https://github.com/alertlogic/azure-collector/raw/master/template.json)
+1. Go to [Customer Deployment](https://portal.azure.com/#create/Microsoft.Template) page. Type in `deploy` in a search query located on top of Azure Web UI and select `Deploy a custom template`.
 1. Click `Build your own template in the editor` and load the file previously downloaded on step 1 above.
 1. Click `Save` button.
 2. Fill in required template parameters and click the `Purchase` button to start a deployment. I.e.:
@@ -133,13 +134,13 @@ Wait until it is deployed successfully.
 ## Verify the Installation
 
 1. Go to `Azure Apps` and choose your function. The last log under `Functions-> Master-> Monitor` should have OK status and should not contain any error messages.
-1. Log into Alertlogic CloudDefender and navigate into `Log Manager -> Sources` page. Check new O365 log source (with a name provided on step 15) has been created and source status is `ok`.
+1. Log into Alertlogic CloudDefender and navigate into `Log Manager -> Sources` page. Check new O365 log source (with a name provided during `az group deployment create` above) has been created and source status is `ok`.
 
 # How It Works
 
 ## Master Function
 
-The `Master` function is a timer trigger function which is responcible for:
+The `Master` function is a timer trigger function which is responsible for:
 - registering the Azure web app In Alertlogic backend;
 - reporting health-checks to the backed;
 - performing log source configuration updates, which happen via Alertlogic UI.
@@ -189,13 +190,13 @@ A notification contains a link to the actual data which is retrieved by the `Col
 1. Run Master function locally: `npm run local-master`
 1. Run Updater function locally: `npm run local-updater`
 1. Run O365WebHook function locally: `npm run local-o365webhook`
-1. Run `npm test` in order to perform code analisys.
+1. Run `npm test` in order to perform code analysis.
 
 Please use the following [code style](https://github.com/airbnb/javascript) as much as possible.
 
 ## Setting environment in dev_config.js
 
-- `process.env.APP_TENANT_ID` - The GUID of the tenant ie. 'alazurealertlogic.onmicrosoft.com'
+- `process.env.APP_TENANT_ID` - The GUID of the tenant i.e. 'alazurealertlogic.onmicrosoft.com'
 - `process.env.CUSTOMCONNSTR_APP_CLIENT_ID` - The GUID of your application that created the subscription.
 You can obtain it from _Azure_ -> _AD_ -> _App registrations_ -> _Your app name_
 - `process.env.CUSTOMCONNSTR_APP_CLIENT_SECRET` - A secret key of your application from _App Registrations_.

From b9b08bba17aba5823dd4c38731a3455d43b3dde4 Mon Sep 17 00:00:00 2001
From: kkuzmin <kkuzmin@alertlogic.com>
Date: Mon, 25 Sep 2017 17:29:00 +0100
Subject: [PATCH 21/24] Release 1.0.0

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index 763d48d..bc6a54e 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "azure_collector",
-  "version": "0.0.1",
+  "version": "1.0.0",
   "dependencies": {
     "async": "*",
     "azure": "^2.0.0-preview",

From e900ca3ecd6ccb115d080c914d658b80fc43fdbe Mon Sep 17 00:00:00 2001
From: Konstantin Kuzmin <kkuzmin@alertlogic.com>
Date: Mon, 25 Sep 2017 18:14:16 +0100
Subject: [PATCH 22/24] Return errors in webhook and updater.

---
 O365WebHook/index.js | 2 +-
 Updater/index.js     | 2 +-
 package.json         | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/O365WebHook/index.js b/O365WebHook/index.js
index 0b2d091..bcc85d4 100644
--- a/O365WebHook/index.js
+++ b/O365WebHook/index.js
@@ -28,7 +28,7 @@ module.exports = function (context, event) {
                     context.log.error(`${err}`);
                     context.res.headers = {};
                     context.res.status = 500;
-                    context.done();
+                    context.done(err);
                 } else {
                     context.log.info('OK!');
                     context.done();
diff --git a/Updater/index.js b/Updater/index.js
index 9b32a68..c31c556 100644
--- a/Updater/index.js
+++ b/Updater/index.js
@@ -25,7 +25,7 @@ module.exports = function (context, AlertlogicUpdaterTimer) {
         if (tokenError) {
             context.log.error('Error getting AD token: ',
                 tokenError.statusCode, tokenError.statusMessage);
-            context.done();
+            context.done(tokenError);
         } else {
             siteSync(context, adToken, function(syncError) {
                 if (syncError) {
diff --git a/package.json b/package.json
index 763d48d..bc6a54e 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "azure_collector",
-  "version": "0.0.1",
+  "version": "1.0.0",
   "dependencies": {
     "async": "*",
     "azure": "^2.0.0-preview",

From fe9780815f4825ff5b2ac8218b7b41b4a975fa50 Mon Sep 17 00:00:00 2001
From: Konstantin Kuzmin <kkuzmin@alertlogic.com>
Date: Mon, 25 Sep 2017 18:17:41 +0100
Subject: [PATCH 23/24] Revert package version.

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index bc6a54e..763d48d 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "azure_collector",
-  "version": "1.0.0",
+  "version": "0.0.1",
   "dependencies": {
     "async": "*",
     "azure": "^2.0.0-preview",

From c31913148cb951a951d443daec166e4465d84b71 Mon Sep 17 00:00:00 2001
From: kkuzmin <kkuzmin@alertlogic.com>
Date: Mon, 25 Sep 2017 20:05:41 +0100
Subject: [PATCH 24/24] Release 1.0.1

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index bc6a54e..61897ef 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "azure_collector",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "dependencies": {
     "async": "*",
     "azure": "^2.0.0-preview",