From 3d9a9a82bed6f62e4a8c87bf4802ba0a519f0c7d Mon Sep 17 00:00:00 2001 From: aishwa8141 Date: Mon, 11 Jan 2021 12:04:12 +0530 Subject: [PATCH 1/7] added collection-certificate-generator config --- .../helm_charts/datapipeline_jobs/values.j2 | 44 +++++-------------- 1 file changed, 11 insertions(+), 33 deletions(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 35ee19b090..0a94b22fbd 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -192,13 +192,13 @@ post-publish-processor: jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 -certificate-generator: - certificate-generator: |+ +collection-certificate-generator: + collection-certificate-generator: |+ include file("/data/flink/conf/base-config.conf") kafka { input.topic = {{ env_name }}.generate.certificate.request output.failed.topic = {{ env_name }}.generate.certificate.failed - output.post.certificate.processor.topic = {{ env_name }}.post.certificate.process.request + output.audit.topic = {{ env_name }}.telemetry.raw groupId = {{ env_name }}-generator-certificate-group } task { @@ -211,33 +211,6 @@ certificate-generator: enc-service { basePath = "{{ enc_service_base_url }}" } - cert_domain_url = "{{ cert_domain_url }}" - cert_container_name = "{{ cert_container_name }}" - cert_cloud_storage_type = "{{ cert_cloud_storage_type }}" - cert_azure_storage_secret = "{{ cert_azure_storage_secret }}" - cert_azure_storage_key = "{{ cert_azure_storage_key }}" - - flink-conf: |+ - jobmanager.memory.flink.size: {{ flink_job_names['certificate-generator'].jobmanager_memory }} - taskmanager.memory.flink.size: {{ flink_job_names['certificate-generator'].taskmanager_memory }} - taskmanager.numberOfTaskSlots: {{ flink_job_names['certificate-generator'].taskslots }} - parallelism.default: 1 - jobmanager.execution.failover-strategy: region - taskmanager.memory.network.fraction: 0.1 - -post-certificate-processor: - post-certificate-processor: |+ - include file("/data/flink/conf/base-config.conf") - kafka { - input.topic = {{ env_name }}.post.certificate.process.request - output.audit.topic = {{ env_name }}.telemetry.raw - output.failed.topic = {{ env_name }}.post.certificate.process.failed - groupId = {{ env_name }}-post-certificate-process-group - } - task { - consumer.parallelism = {{ post_certificate_processor_consumer_parallelism }} - parallelism = {{ post_certificate_processor_parallelism }} - } lms-cassandra { keyspace = "{{ middleware_course_keyspace }}" enrollment.table = "{{ middleware_user_enrollment_table }}" @@ -246,11 +219,16 @@ post-certificate-processor: learner-service { basePath = "{{ learner_service_base_url }}" } + cert_domain_url = "{{ cert_domain_url }}" + cert_container_name = "{{ cert_container_name }}" + cert_cloud_storage_type = "{{ cert_cloud_storage_type }}" + cert_azure_storage_secret = "{{ cert_azure_storage_secret }}" + cert_azure_storage_key = "{{ cert_azure_storage_key }}" flink-conf: |+ - jobmanager.memory.flink.size: {{ flink_job_names['post-publish-processor'].jobmanager_memory }} - taskmanager.memory.flink.size: {{ flink_job_names['post-publish-processor'].taskmanager_memory }} - taskmanager.numberOfTaskSlots: {{ flink_job_names['post-publish-processor'].taskslots }} + jobmanager.memory.flink.size: {{ flink_job_names['collection-certificate-generator'].jobmanager_memory }} + taskmanager.memory.flink.size: {{ flink_job_names['collection-certificate-generator'].taskmanager_memory }} + taskmanager.numberOfTaskSlots: {{ flink_job_names['collection-certificate-generator'].taskslots }} parallelism.default: 1 jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 From b82279a875889023118d93e3af58123bc9afdc74 Mon Sep 17 00:00:00 2001 From: aishwa8141 Date: Mon, 11 Jan 2021 12:13:48 +0530 Subject: [PATCH 2/7] added collection-certificate-generator config --- .../ansible/roles/flink-jobs-deploy/defaults/main.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 36fb9c771e..15d7f55a02 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -125,20 +125,13 @@ flink_job_names: taskmanager_memory: 1024m taskslots: 1 cpu_requests: 0.3 - certificate-generator: + collection-certificate-generator: job_class_name: 'org.sunbird.job.task.CertificateGeneratorStreamTask' replica: 1 jobmanager_memory: 1024m taskmanager_memory: 1024m taskslots: 1 cpu_requests: 0.3 - post-certificate-processor: - job_class_name: 'org.sunbird.job.task.PostCertificateProcessorStreamTask' - replica: 1 - jobmanager_memory: 1024m - taskmanager_memory: 1024m - taskslots: 1 - cpu_requests: 0.3 certificate-pre-processor: job_class_name: 'org.sunbird.job.task.CertificatePreProcessorStreamTask' replica: 1 From 4974880105e7a521478ca127ea6a2e065e224cf6 Mon Sep 17 00:00:00 2001 From: aishwa8141 Date: Mon, 11 Jan 2021 13:47:49 +0530 Subject: [PATCH 3/7] added collection-certificate-generator config --- .../ansible/roles/flink-jobs-deploy/defaults/main.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 15d7f55a02..8edd2ed781 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -67,13 +67,9 @@ post_publish_processor_parallelism: 1 post_publish_event_router_parallelism: 1 -### Certificate Job related Vars -certificate_generator_consumer_parallelism: 1 -certificate_generator_parallelism: 1 - -### Post Certificate Processor Job related Vars -post_certificate_processor_consumer_parallelism: 1 -post_certificate_processor_parallelism: 1 +### Collection Generator Job related Vars +collection-certificate-generator_consumer_parallelism: 1 +collection-certificate-generator_parallelism: 1 middleware_user_enrollment_table: "user_enrolments" middleware_course_batch_table: "course_batch" From d4bd74846623ee1a59f8bbe1220e3a02cd46d281 Mon Sep 17 00:00:00 2001 From: aishwa8141 Date: Wed, 13 Jan 2021 13:58:09 +0530 Subject: [PATCH 4/7] added collection-certificate-generator config --- kubernetes/helm_charts/datapipeline_jobs/values.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 0a94b22fbd..714fac3c45 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -202,8 +202,8 @@ collection-certificate-generator: groupId = {{ env_name }}-generator-certificate-group } task { - consumer.parallelism = {{ certificate_generator_consumer_parallelism }} - parallelism = {{ certificate_generator_parallelism }} + consumer.parallelism = {{ collection-certificate-generator_consumer_parallelism }} + parallelism = {{ collection-certificate-generator_parallelism }} } cert-reg { basePath = "{{ cert_reg_service_base_url }}" From 4fd8e7f313afbf7c73549390873d9c17fcf4235e Mon Sep 17 00:00:00 2001 From: aishwa8141 Date: Wed, 13 Jan 2021 13:59:42 +0530 Subject: [PATCH 5/7] added collection-certificate-generator config --- kubernetes/helm_charts/datapipeline_jobs/values.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 714fac3c45..d0754b2ba1 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -202,8 +202,8 @@ collection-certificate-generator: groupId = {{ env_name }}-generator-certificate-group } task { - consumer.parallelism = {{ collection-certificate-generator_consumer_parallelism }} - parallelism = {{ collection-certificate-generator_parallelism }} + consumer.parallelism = {{ collection_certificate-generator_consumer_parallelism }} + parallelism = {{ collection_certificate-generator_parallelism }} } cert-reg { basePath = "{{ cert_reg_service_base_url }}" From 513d300bc81a6f64d09c6975432ed70d49437752 Mon Sep 17 00:00:00 2001 From: aishwa8141 Date: Wed, 13 Jan 2021 14:00:06 +0530 Subject: [PATCH 6/7] added collection-certificate-generator config --- kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 8edd2ed781..f0c502cd94 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -68,8 +68,8 @@ post_publish_event_router_parallelism: 1 ### Collection Generator Job related Vars -collection-certificate-generator_consumer_parallelism: 1 -collection-certificate-generator_parallelism: 1 +collection_certificate-generator_consumer_parallelism: 1 +collection_certificate-generator_parallelism: 1 middleware_user_enrollment_table: "user_enrolments" middleware_course_batch_table: "course_batch" From 612032401e766faaca72d1a12356ffdd0aa91ef9 Mon Sep 17 00:00:00 2001 From: aishwa8141 Date: Wed, 13 Jan 2021 14:01:27 +0530 Subject: [PATCH 7/7] added collection-certificate-generator config --- kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml | 4 ++-- kubernetes/helm_charts/datapipeline_jobs/values.j2 | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index f0c502cd94..338953ac10 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -68,8 +68,8 @@ post_publish_event_router_parallelism: 1 ### Collection Generator Job related Vars -collection_certificate-generator_consumer_parallelism: 1 -collection_certificate-generator_parallelism: 1 +collection_certificate_generator_consumer_parallelism: 1 +collection_certificate_generator_parallelism: 1 middleware_user_enrollment_table: "user_enrolments" middleware_course_batch_table: "course_batch" diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index d0754b2ba1..a4208ea069 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -202,8 +202,8 @@ collection-certificate-generator: groupId = {{ env_name }}-generator-certificate-group } task { - consumer.parallelism = {{ collection_certificate-generator_consumer_parallelism }} - parallelism = {{ collection_certificate-generator_parallelism }} + consumer.parallelism = {{ collection_certificate_generator_consumer_parallelism }} + parallelism = {{ collection_certificate_generator_parallelism }} } cert-reg { basePath = "{{ cert_reg_service_base_url }}"