feat: modify task unit count

Signed-off-by: ImMin5 <[email protected]>
cloudforet-io · Oct 25, 2023 · e979177 · e979177
1 parent 128f5b7
commit e979177
Show file tree

Hide file tree

Showing 4 changed files with 12 additions and 6 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -1,4 +1,4 @@
-FROM cloudforet/python-core:2
+FROM cloudforet/python-core:1.12
 
 ENV PYTHONUNBUFFERED 1
 ENV SPACEONE_PORT 50051

diff --git a/src/cloudforet/cost_analysis/manager/cost_manager.py b/src/cloudforet/cost_analysis/manager/cost_manager.py
@@ -32,8 +32,9 @@ def get_data(self, options, secret_data, schema, task_options):
             parameters = self._make_parameters(_start, _end, options)
 
             start_time = time.time()
-            _LOGGER.info(f'[get_data] start to collect data from {_start} to {_end}')
+            _LOGGER.info(f'[get_data] {tenant_ids} start to collect data from {_start} to {_end}')
             for idx, tenant_id in enumerate(tenant_ids):
+                _LOGGER.info(f'[get_data] #{idx+1} {tenant_id} tenant start to collect data from {_start} to {_end}')
                 scope = self._make_scope(secret_data, task_options, collect_scope, tenant_id)
                 blobs = self.azure_cm_connector.begin_create_operation(scope, parameters)
 
@@ -181,7 +182,7 @@ def _get_tenant_ids(task_options, collect_scope):
         if 'tenant_id' in task_options:
             tenant_ids.append(task_options['tenant_id'])
         elif collect_scope == 'customer_tenant_id':
-            tenant_ids.append(task_options['customer_tenant'])
+            tenant_ids.append(task_options['customer_tenants'])
         else:
             tenant_ids.append('EA Agreement')
         return tenant_ids

diff --git a/src/cloudforet/cost_analysis/manager/job_manager.py b/src/cloudforet/cost_analysis/manager/job_manager.py
@@ -1,4 +1,5 @@
 import logging
+import math
 from datetime import datetime, timedelta
 from dateutil.relativedelta import relativedelta
 
@@ -10,6 +11,7 @@
 
 _LOGGER = logging.getLogger(__name__)
 
+_TASK_LIST_SIZE = 4
 
 class JobManager(BaseManager):
 
@@ -35,11 +37,14 @@ def get_tasks(self, options, secret_data, schema, start, last_synchronized_at, d
                 if len(customer_tenants) == 0:
                     raise ERROR_EMPTY_CUSTOMER_TENANTS(customer_tenants=customer_tenants)
 
-                for customer_tenant in customer_tenants:
+                tenant_size = math.ceil(len(customer_tenants)/_TASK_LIST_SIZE)
+                divided_customer_tenants = [customer_tenants[i:i + tenant_size] for i in range(0, len(customer_tenants), tenant_size)]
+
+                for divided_customer_tenant in divided_customer_tenants:
                     tasks.append({'task_options': {'start': start_month,
                                                    'account_agreement_type': billing_account_agreement_type,
                                                    'collect_scope': 'customer_tenant_id',
-                                                   'customer_tenant': customer_tenant}})
+                                                   'customer_tenants': divided_customer_tenant}})
                     changed.append({'start': start_month})
             else:
                 tasks = [{'task_options': {'start': start_month, 'account_agreement_type': billing_account_agreement_type,

diff --git a/src/cloudforet/cost_analysis/model/job_model.py b/src/cloudforet/cost_analysis/model/job_model.py
@@ -10,7 +10,7 @@ class TaskOptionsWithSubscription(Model):
 
 
 class TaskOptionsWithCustomerTenant(Model):
-    customer_tenant = StringType(serialize_when_none=False)
+    customer_tenants = ListType(StringType, serialize_when_none=False)
 
 
 class TaskOptions(TaskOptionsWithSubscription, TaskOptionsWithCustomerTenant):