diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 36fb9c771e..338953ac10 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -67,13 +67,9 @@ post_publish_processor_parallelism: 1 post_publish_event_router_parallelism: 1 -### Certificate Job related Vars -certificate_generator_consumer_parallelism: 1 -certificate_generator_parallelism: 1 - -### Post Certificate Processor Job related Vars -post_certificate_processor_consumer_parallelism: 1 -post_certificate_processor_parallelism: 1 +### Collection Generator Job related Vars +collection_certificate_generator_consumer_parallelism: 1 +collection_certificate_generator_parallelism: 1 middleware_user_enrollment_table: "user_enrolments" middleware_course_batch_table: "course_batch" @@ -125,20 +121,13 @@ flink_job_names: taskmanager_memory: 1024m taskslots: 1 cpu_requests: 0.3 - certificate-generator: + collection-certificate-generator: job_class_name: 'org.sunbird.job.task.CertificateGeneratorStreamTask' replica: 1 jobmanager_memory: 1024m taskmanager_memory: 1024m taskslots: 1 cpu_requests: 0.3 - post-certificate-processor: - job_class_name: 'org.sunbird.job.task.PostCertificateProcessorStreamTask' - replica: 1 - jobmanager_memory: 1024m - taskmanager_memory: 1024m - taskslots: 1 - cpu_requests: 0.3 certificate-pre-processor: job_class_name: 'org.sunbird.job.task.CertificatePreProcessorStreamTask' replica: 1 diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 35ee19b090..a4208ea069 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -192,18 +192,18 @@ post-publish-processor: jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1 -certificate-generator: - certificate-generator: |+ +collection-certificate-generator: + collection-certificate-generator: |+ include file("/data/flink/conf/base-config.conf") kafka { input.topic = {{ env_name }}.generate.certificate.request output.failed.topic = {{ env_name }}.generate.certificate.failed - output.post.certificate.processor.topic = {{ env_name }}.post.certificate.process.request + output.audit.topic = {{ env_name }}.telemetry.raw groupId = {{ env_name }}-generator-certificate-group } task { - consumer.parallelism = {{ certificate_generator_consumer_parallelism }} - parallelism = {{ certificate_generator_parallelism }} + consumer.parallelism = {{ collection_certificate_generator_consumer_parallelism }} + parallelism = {{ collection_certificate_generator_parallelism }} } cert-reg { basePath = "{{ cert_reg_service_base_url }}" @@ -211,33 +211,6 @@ certificate-generator: enc-service { basePath = "{{ enc_service_base_url }}" } - cert_domain_url = "{{ cert_domain_url }}" - cert_container_name = "{{ cert_container_name }}" - cert_cloud_storage_type = "{{ cert_cloud_storage_type }}" - cert_azure_storage_secret = "{{ cert_azure_storage_secret }}" - cert_azure_storage_key = "{{ cert_azure_storage_key }}" - - flink-conf: |+ - jobmanager.memory.flink.size: {{ flink_job_names['certificate-generator'].jobmanager_memory }} - taskmanager.memory.flink.size: {{ flink_job_names['certificate-generator'].taskmanager_memory }} - taskmanager.numberOfTaskSlots: {{ flink_job_names['certificate-generator'].taskslots }} - parallelism.default: 1 - jobmanager.execution.failover-strategy: region - taskmanager.memory.network.fraction: 0.1 - -post-certificate-processor: - post-certificate-processor: |+ - include file("/data/flink/conf/base-config.conf") - kafka { - input.topic = {{ env_name }}.post.certificate.process.request - output.audit.topic = {{ env_name }}.telemetry.raw - output.failed.topic = {{ env_name }}.post.certificate.process.failed - groupId = {{ env_name }}-post-certificate-process-group - } - task { - consumer.parallelism = {{ post_certificate_processor_consumer_parallelism }} - parallelism = {{ post_certificate_processor_parallelism }} - } lms-cassandra { keyspace = "{{ middleware_course_keyspace }}" enrollment.table = "{{ middleware_user_enrollment_table }}" @@ -246,11 +219,16 @@ post-certificate-processor: learner-service { basePath = "{{ learner_service_base_url }}" } + cert_domain_url = "{{ cert_domain_url }}" + cert_container_name = "{{ cert_container_name }}" + cert_cloud_storage_type = "{{ cert_cloud_storage_type }}" + cert_azure_storage_secret = "{{ cert_azure_storage_secret }}" + cert_azure_storage_key = "{{ cert_azure_storage_key }}" flink-conf: |+ - jobmanager.memory.flink.size: {{ flink_job_names['post-publish-processor'].jobmanager_memory }} - taskmanager.memory.flink.size: {{ flink_job_names['post-publish-processor'].taskmanager_memory }} - taskmanager.numberOfTaskSlots: {{ flink_job_names['post-publish-processor'].taskslots }} + jobmanager.memory.flink.size: {{ flink_job_names['collection-certificate-generator'].jobmanager_memory }} + taskmanager.memory.flink.size: {{ flink_job_names['collection-certificate-generator'].taskmanager_memory }} + taskmanager.numberOfTaskSlots: {{ flink_job_names['collection-certificate-generator'].taskslots }} parallelism.default: 1 jobmanager.execution.failover-strategy: region taskmanager.memory.network.fraction: 0.1