Merge pull request #449 from ddps-lab/azure-collector

update azure collecotor avoid 429 error

collector/spot-dataset/azure/lambda/current_collector/load_price.py

@@ -11,9 +11,9 @@
 
 price_list = []
 response_dict = {}
-MAX_SKIP = 2000
-SKIP_NUM_LIST = [i*100 for i in range(AZURE_CONST.MAX_SKIP)]
+SKIP_NUM_LIST = [i*1000 for i in range(AZURE_CONST.MAX_SKIP)]
 event = threading.Event()
+lock = threading.Lock()
 
 
 # get instancetier from armSkuName
@@ -74,7 +74,9 @@ def get_price(skip_num):
 
         return
 
+    lock.acquire()
     price_list.extend(price_data)
+    lock.release()
 
     return
 
@@ -135,5 +137,6 @@ def collect_price_with_multithreading():
 
     price_df = pd.DataFrame(price_list)
     savings_df = preprocessing_price(price_df)
+    savings_df = savings_df.drop_duplicates(subset=['InstanceTier', 'InstanceType', 'Region'], keep='first')
 
     return savings_df

const_config.py

@@ -104,15 +104,15 @@ def SERVER_SAVE_FILENAME():
 
     @constant
     def GET_PRICE_URL():
-        return "https://prices.azure.com:443/api/retail/prices?$filter=serviceName%20eq%20%27Virtual%20Machines%27%20and%20priceType%20eq%20%27Consumption%27%20and%20unitOfMeasure%20eq%20%271%20Hour%27&$skip="
+        return "https://prices.azure.com:443/api/retail/prices?$filter=serviceName eq 'Virtual Machines' and priceType eq 'Consumption' and unitOfMeasure eq '1 Hour' and  contains(productName, 'Windows') eq false and contains(meterName, 'Low Priority') eq false  and contains(meterName, 'Expired') eq false and contains(location, 'Gov') eq false and contains(location, 'ATT') eq false &$skip="
 
     @constant
     def FILTER_LOCATIONS():
         return ['GOV', 'EUAP', 'ATT', 'SLV', '']
 
     @constant
     def MAX_SKIP():
-        return 2000
+        return 200
 
 class GcpCollector(object):
     @constant