From 4fd3b75c61b0af71964d46bc119e75e98d147fda Mon Sep 17 00:00:00 2001 From: santhosh-tg <93243580+santhosh-tg@users.noreply.github.com> Date: Thu, 5 Oct 2023 14:03:05 +0530 Subject: [PATCH] merge release-6.0.0 to 7.0.0 (#3908) * Fix syntax (#3894) * Updated vars for cron schedule (#3895) * Updated vars for cron schedule * Update main.yml * [ED-2793] Fix: release-6.0.0 installation issues (#3900) * Update Jenkins version * Fix job parameter * Fix error: You need to install jmespath prior to running json_query filter * Update main.yml with cassandra (#3845) (#3901) Co-authored-by: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> * ED-2838: ##Release-6.0.0 new variables (#3899) * ED-2838 Release-6.0.0 new variables * Added with comments * ED-2838 jenkins josb update and new variables * added storage endpoint detials * [ED-2793] Fix installation issues (#3905) * Update maven repo url * Add missing jobs * Fix docker creds secrets issue when having specialcharacters --------- Co-authored-by: Raghupathi Guduri Co-authored-by: gohilamariappan <41056032+gohilamariappan@users.noreply.github.com> Co-authored-by: Prasath Sivasubramaniyan --- ansible/roles/cassandra/defaults/main.yml | 4 +- .../ml-analytics-service/defaults/main.yml | 5 + .../roles/ml-analytics-service/tasks/main.yml | 11 +- deploy/jenkins/jenkins-server-setup.sh | 7 +- .../jobs/InquiryFlinkJob/config.xml | 108 ++++++++++++ .../Build/jobs/Core/jobs/Analytics/config.xml | 31 +++- .../jobs/Core/jobs/Assessment/config.xml | 55 ++++-- .../Build/jobs/Core/jobs/Content/config.xml | 18 ++ .../Build/jobs/Core/jobs/Learner/config.xml | 20 ++- .../jobs/Build/jobs/Core/jobs/Lms/config.xml | 18 ++ .../jobs/AnalyticsCore/config.xml | 26 ++- .../jobs/CoreDataProducts/config.xml | 23 ++- .../jobs/FlinkJobs/config.xml | 18 ++ .../jobs/InquiryFlinkJob/config.xml | 77 +++++++++ .../Lern/jobs/LernDataProducts/config.xml | 25 ++- .../jobs/Lern/jobs/LernFlinkJobs/config.xml | 18 ++ .../jobs/InquiryFlinkJob/config.xml | 149 ++++++++++++++++ .../Kubernetes/InquiryUploadSchema/config.xml | 163 ++++++++++++++++++ .../jobs/ApplicationElasticSearch/config.xml | 4 +- kubernetes/ansible/bootstrap.yaml | 2 +- kubernetes/ansible/bootstrap_minimal.yaml | 2 +- kubernetes/ansible/namespace.yml | 2 +- .../templates/configmap.yaml | 2 +- .../ansible/inventory/dev/Core/common.yml | 9 + .../ansible/inventory/dev/Core/secrets.yml | 2 +- .../dev/KnowledgePlatform/common.yml | 8 +- 26 files changed, 756 insertions(+), 51 deletions(-) create mode 100644 deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml create mode 100644 deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml create mode 100644 deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml create mode 100644 deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Kubernetes/InquiryUploadSchema/config.xml diff --git a/ansible/roles/cassandra/defaults/main.yml b/ansible/roles/cassandra/defaults/main.yml index 33c02c17bf..8f1f0f7951 100644 --- a/ansible/roles/cassandra/defaults/main.yml +++ b/ansible/roles/cassandra/defaults/main.yml @@ -1,5 +1,5 @@ user_home: "/home/{{ ansible_ssh_user }}/" -cassandra_repo: 'deb http://www.apache.org/dist/cassandra/debian 311x main' -cassandra_repo_key: 'https://www.apache.org/dist/cassandra/KEYS' +cassandra_repo: 'deb https://debian.cassandra.apache.org 311x main' +cassandra_repo_key: 'https://downloads.apache.org/cassandra/KEYS' resource_crunch: "yes" # make this value yes if heap size should be quarter of the Server Memory when you are deploying multiple databases, else make this value no to have 50% of the value as per the best practice of cassandra cassandra_home_directory: "/var/lib/cassandra" diff --git a/ansible/roles/ml-analytics-service/defaults/main.yml b/ansible/roles/ml-analytics-service/defaults/main.yml index 15ef432e43..a67906b1f8 100755 --- a/ansible/roles/ml-analytics-service/defaults/main.yml +++ b/ansible/roles/ml-analytics-service/defaults/main.yml @@ -133,3 +133,8 @@ ml_analytics_reports_store: "{{ cloud_service_provider }}" ml_analytics_reports_container: "{{ cloud_storage_privatereports_bucketname }}" ml_analytics_driver_memory: "{{ ml_analytics_default_driver_memory | default('5g') }}" ml_analytics_executor_memory: "{{ ml_analytics_default_executor_memory | default('5g') }}" +ml_analytics_batch_cron_minute: "{{ ml_batch_cron_minute | default('30') }}" +ml_analytics_batch_cron_hour: "{{ ml_batch_cron_hour | default('18') }}" +ml_analytics_nvsk_cron_minute: "{{ ml_nvsk_cron_minute | default('30') }}" +ml_analytics_nvsk_cron_hour: "{{ ml_nvsk_cron_hour | default('7') }}" +ml_analytics_nvsk_cron_weekday: "{{ ml_nvsk_cron_weekday | default('4') }}" diff --git a/ansible/roles/ml-analytics-service/tasks/main.yml b/ansible/roles/ml-analytics-service/tasks/main.yml index 6f37d6b3ea..7fbd3fe1ba 100755 --- a/ansible/roles/ml-analytics-service/tasks/main.yml +++ b/ansible/roles/ml-analytics-service/tasks/main.yml @@ -26,6 +26,7 @@ - python3-virtualenv - zip - unzip + - acl state: present - name: CHANGE THE OWNERSHIP FOR THIS {{ BASEPATH }} DIRECTORY @@ -135,15 +136,15 @@ cron: name: "Run Batch Ingestion Job" user: "{{ USER }}" - minute: "30" - hour: "18" + minute: "{{ ml_analytics_batch_cron_minute }}" + hour: "{{ ml_analytics_batch_cron_hour }}" job: "{{ BASEPATH }}/ml-analytics-service/run.sh > {{ BASEPATH }}/ml-analytics-service/crontab_job.log" - name: CREATE THE CRON NVSK Data Upload cron: name: "NVSK Data Upload JOB" user: "{{ USER }}" - minute: "30" - hour: "7" - weekday: "4" + minute: "{{ ml_analytics_nvsk_cron_minute }}" + hour: "{{ ml_analytics_nvsk_cron_hour }}" + weekday: "{{ ml_analytics_nvsk_cron_weekday }}" job: "{{ BASEPATH }}/ml-analytics-service/run_weekly.sh > {{ BASEPATH }}/ml-analytics-service/nvsk_data_weekly.logs" diff --git a/deploy/jenkins/jenkins-server-setup.sh b/deploy/jenkins/jenkins-server-setup.sh index d96e3b4228..2c7516a57a 100755 --- a/deploy/jenkins/jenkins-server-setup.sh +++ b/deploy/jenkins/jenkins-server-setup.sh @@ -12,10 +12,10 @@ echo -e "\n\e[0;32m${bold}Installating JDK8${normal}\n" apt-get install -y openjdk-8-jdk echo -e "\n\e[0;32m${bold}Installating Jenkins${normal}" -wget -q -O - https://pkg.jenkins.io/debian-stable/jenkins.io.key | apt-key add - +wget -q -O - https://pkg.jenkins.io/debian-stable/jenkins.io-2023.key | apt-key add - apt-add-repository "deb https://pkg.jenkins.io/debian-stable binary/" apt-get update -apt-get install -y jenkins=2.319.3 +apt-get install -y jenkins=2.346.3 echo -e "\n\e[0;32m${bold}Installating PIP${normal}" apt-get install -y python-pip @@ -115,6 +115,7 @@ su jenkins bash -c "curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34. echo -e "\n\e[0;32m${bold}Installing jmespath${normal}" sudo apt install -y python3-jmespath +sudo apt install python-jmespath #python2 #### Kubernetes Tools #### @@ -155,7 +156,7 @@ rm openjdk-11.0.2_linux-x64_bin.tar.gz #Install maven 3.6.3 echo -e "\n\e[0;32m${bold}Installating maven 3.6.3${normal}" -wget https://downloads.apache.org/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz +wget https://archive.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz tar -xf apache-maven-3.6.3-bin.tar.gz mv apache-maven-3.6.3 /opt/ mv /opt/apache-maven-3.6.3/bin/mvn /opt/apache-maven-3.6.3/bin/mvn3.6 diff --git a/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml new file mode 100644 index 0000000000..28b220559d --- /dev/null +++ b/deploy/jenkins/jobs/ArtifactUpload/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml @@ -0,0 +1,108 @@ + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + 10 + -1 + 1 + + + + + false + false + + + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + Build/KnowledgePlatform/InquiryFlinkJob + false + + + image_tag + <font color=darkgreen size=2><b>OPTIONAL: Specify the tag to upload a specific image version to the container registry.</b></font> + + false + + + artifact_source + <font color=dimgray size=2><b> +ArtifactRepo - Push the docker image to container registry. +</b></font> + + + ArtifactRepo + + + + + + + 0 + 0 + + false + project + false + + + + + + + + Build/KnowledgePlatform/InquiryFlinkJob + + SUCCESS + 0 + BLUE + true + + + + + + + + 2 + + + https://github.com/Sunbird-inQuiry/data-pipeline.git + + + + + ${inquiry_pipeline_branch_or_tag} + + + false + + + + true + false + + 0 + false + + + + kubernetes/pipelines/upload/Jenkinsfile + false + + + false + diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Analytics/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Analytics/config.xml index 5bc1fe4d34..c6304ef3b4 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Analytics/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Analytics/config.xml @@ -35,6 +35,30 @@ refs/heads/${public_repo_branch} true + + CLOUD_STORE_GROUP_ID + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk group id</li> +</b></font> + org.sunbird + false + + + CLOUD_STORE_ARTIFACT_ID + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk artifact id</li> +</b></font> + cloud-store-sdk_2.12 + false + + + CLOUD_STORE_VERSION + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk version</li> +</b></font> + 1.4.0 + false + @@ -48,12 +72,7 @@ - - - H/15 * * * * - false - - + diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Assessment/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Assessment/config.xml index d2c038b5fa..b4deb5e46b 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Assessment/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Assessment/config.xml @@ -1,6 +1,6 @@ - + hudson.model.ParametersDefinitionProperty com.sonyericsson.rebuild.RebuildSettings @@ -19,26 +19,47 @@ - + false false - github_release_tag + inquiry_release_tag <font style="color:dimgray;font-size:14px;"><b> <li>To build from a tag, use refs/tags/github_tag</li> <li>To build from a branch, use refs/heads/github_branch</li> -<li>The default value of ${public_repo_branch} will be the release / tag version set in global configuration</li> -<li>To build from a differnt branch, replace the ${public_repo_branch} with your branch</li> +<li>The default value of ${inquiry_service_build_branch_or_tag} will be the release / tag version set in global configuration</li> +<li>To build from a differnt branch, replace the ${inquiry_service_build_branch_or_tag} with your branch</li> </b></font> - refs/heads/${public_repo_branch} + refs/heads/${inquiry_service_build_branch_or_tag} true + + core_release_tag + <font style="color:dimgray;font-size:14px;"><b> +<li>Provide this value to build core components from Knowlg BB</li> +<li>To build from a tag, use refs/tags/github_tag</li> +<li>To build from a branch, use refs/heads/github_branch</li> +<li>The default value of ${inquiry_core_build_branch_or_tag} will be the release / tag version set in global configuration</li> +<li>To build from a differnt branch, replace the ${inquiry_core_build_branch_or_tag} with your branch</li> +</b></font> + refs/heads/${inquiry_core_build_branch_or_tag} + false + + + core_repo_link + Knowlg Core Repo + + + https://github.com/project-sunbird/knowledge-platform.git + + + - + 0 0 @@ -49,31 +70,29 @@ - - - H/15 * * * * - false - - + - - + + 2 - https://github.com/project-sunbird/knowledge-platform.git + https://github.com/Sunbird-inQuiry/inquiry-api-service.git - ${github_release_tag} + ${inquiry_release_tag} + false + + build/assessment-service/Jenkinsfile false false - \ No newline at end of file + diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Content/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Content/config.xml index 7b47e1972e..f3f5d6477c 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Content/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Content/config.xml @@ -36,6 +36,24 @@ refs/heads/${public_repo_branch} true + + CLOUD_STORE_GROUP_ID + Set the Cloud store sdk group id. e.g. org.sunbird + ${cloud_store_group_id} + false + + + CLOUD_STORE_ARTIFACT_ID + Set the Cloud store sdk artifact id. e.g. cloud-store-sdk_2.12 + ${cloud_store_artifact_id} + false + + + CLOUD_STORE_VERSION + Set the Cloud store sdk version. e.g. 1.4.6 + ${cloud_store_version} + false + diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Learner/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Learner/config.xml index 2cdac260ad..a12dd235f1 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Learner/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Learner/config.xml @@ -36,6 +36,24 @@ refs/heads/${public_repo_branch} true + + cloud_store_group_id + Set the Cloud store sdk group id. e.g. org.sunbird + ${cloud_store_group_id} + false + + + cloud_store_artifact_id + Set the Cloud store sdk artifact id. e.g. cloud-store-sdk + ${cloud_store_artifact_id} + false + + + cloud_store_version + Set the Cloud store sdk version. e.g. 1.4.6 + ${cloud_store_version} + false + @@ -51,7 +69,7 @@ - H/15 * * * * + false diff --git a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Lms/config.xml b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Lms/config.xml index 89116c8406..4eecb95a61 100644 --- a/deploy/jenkins/jobs/Build/jobs/Core/jobs/Lms/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Core/jobs/Lms/config.xml @@ -36,6 +36,24 @@ refs/heads/${public_repo_branch} true + + cloud_store_group_id + Set the Cloud store sdk group id. e.g. org.sunbird + ${cloud_store_group_id} + false + + + cloud_store_artifact_id + Set the Cloud store sdk artifact id. e.g. cloud-store-sdk + ${cloud_store_artifact_id} + false + + + cloud_store_version + Set the Cloud store sdk version. e.g 1.4.6 + ${cloud_store_version} + false + diff --git a/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/AnalyticsCore/config.xml b/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/AnalyticsCore/config.xml index 2b9e72c45c..5c202a759c 100644 --- a/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/AnalyticsCore/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/AnalyticsCore/config.xml @@ -36,6 +36,30 @@ refs/heads/${public_repo_branch} true + + CLOUD_STORE_GROUP_ID + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk group id</li> +</b></font> + org.sunbird + false + + + CLOUD_STORE_ARTIFACT_ID + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk artifact id</li> +</b></font> + cloud-store-sdk_2.12 + false + + + CLOUD_STORE_VERSION + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk version</li> +</b></font> + 1.4.0 + false + @@ -51,7 +75,7 @@ - H/15 * * * * + false diff --git a/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/CoreDataProducts/config.xml b/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/CoreDataProducts/config.xml index a458418b24..8ff4dc5426 100644 --- a/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/CoreDataProducts/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/DataPipeline/jobs/CoreDataProducts/config.xml @@ -27,6 +27,27 @@ github_release_tag + <font color=red size=2><b>CAUTION: If the value is blank, latest code will be built. Specify github tag name to build from a tag.</b></font> + false + + + CLOUD_STORE_GROUP_ID + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk group id</li> +</b></font> + org.sunbird + false + + + CLOUD_STORE_ARTIFACT_ID + <font style="color:dimgray;font-size:14px;"><b> +<li>Mention the cloud storage sdk artifact id</li> +</b></font> + cloud-store-sdk_2.12 + false + + + CLOUD_STORE_VERSION <font style="color:dimgray;font-size:14px;"><b> <li>To build from a tag, use refs/tags/github_tag</li> <li>To build from a branch, use refs/heads/github_branch</li> @@ -51,7 +72,7 @@ - H/15 * * * * + false diff --git a/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/FlinkJobs/config.xml b/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/FlinkJobs/config.xml index 11f30a0d33..62a0cd1f68 100644 --- a/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/FlinkJobs/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/FlinkJobs/config.xml @@ -36,6 +36,24 @@ refs/heads/${public_repo_branch} true + + CLOUD_STORE_GROUP_ID + Set the Cloud store sdk group id. e.g. org.sunbird + ${cloud_store_group_id} + false + + + CLOUD_STORE_ARTIFACT_ID + Set the Cloud store sdk artifact id. e.g. cloud-store-sdk_2.12 + ${cloud_store_artifact_id} + false + + + CLOUD_STORE_VERSION + Set the Cloud store sdk version. e.g. 1.4.6 + ${cloud_store_version} + false + diff --git a/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml b/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml new file mode 100644 index 0000000000..93b0eba8f7 --- /dev/null +++ b/deploy/jenkins/jobs/Build/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml @@ -0,0 +1,77 @@ + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + 10 + -1 + 1 + + + + + false + false + + + + + inquiry_release_tag + <font style="color:dimgray;font-size:14px;"><b> +<li>To build from a tag, use refs/tags/github_tag</li> +<li>To build from a branch, use refs/heads/github_branch</li> +<li>The default value of ${inquiry_pipeline_branch_or_tag} will be the release / tag version set in global configuration</li> +<li>To build from a differnt branch, replace the ${inquiry_pipeline_branch_or_tag} with your branch</li> +</b></font> + refs/heads/${inquiry_pipeline_branch_or_tag} + true + + + + + 0 + 0 + + false + project + false + + + + + + + + + + 2 + + + https://github.com/Sunbird-inQuiry/data-pipeline.git + + + + + ${inquiry_release_tag} + + + false + + + + kubernetes/pipelines/build/Jenkinsfile + false + + + false + diff --git a/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernDataProducts/config.xml b/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernDataProducts/config.xml index 088b9fa27f..9b997d2ce1 100644 --- a/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernDataProducts/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernDataProducts/config.xml @@ -36,6 +36,24 @@ refs/heads/${public_repo_branch} true + + cloud_store_group_id + Set the Cloud store sdk group id. e.g. org.sunbird + ${cloud_store_group_id} + false + + + cloud_store_artifact_id + Set the Cloud store sdk artifact id. e.g. cloud-store-sdk_2.12 + ${cloud_store_artifact_id} + false + + + cloud_store_version + Set the Cloud store sdk version. e.g. 1.4.6 + ${cloud_store_version} + false + @@ -49,12 +67,7 @@ - - - H/15 * * * * - false - - + diff --git a/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernFlinkJobs/config.xml b/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernFlinkJobs/config.xml index df82bece0b..f8041941e2 100644 --- a/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernFlinkJobs/config.xml +++ b/deploy/jenkins/jobs/Build/jobs/Lern/jobs/LernFlinkJobs/config.xml @@ -36,6 +36,24 @@ refs/heads/${public_repo_branch} true + + cloud_store_group_id + Set the Cloud store sdk group id. e.g. org.sunbird + ${cloud_store_group_id} + false + + + cloud_store_artifact_id + Set the Cloud store sdk artifact id. e.g. cloud-store-sdk_2.12 + ${cloud_store_artifact_id} + false + + + cloud_store_version + Set the Cloud store sdk version. e.g. 1.4.6 + ${cloud_store_version} + false + diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml new file mode 100644 index 0000000000..af888dbe75 --- /dev/null +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/KnowledgePlatform/jobs/InquiryFlinkJob/config.xml @@ -0,0 +1,149 @@ + + + + + hudson.model.ParametersDefinitionProperty + com.sonyericsson.rebuild.RebuildSettings + + + + + false + + + + -1 + 10 + -1 + 2 + + + + + false + false + + + + + private_branch + + choice-parameter-2544395024638227 + 1 + + true + + + + true + + + InquiryFlinkJob + Deploy/dev/KnowledgePlatform/InquiryFlinkJob + + + ET_FORMATTED_HTML + true + + + inquiry_release_tag + <font style="color:dimgray;font-size:14px;"><b> +<li>To build from a tag, use refs/tags/github_tag</li> +<li>To build from a branch, use refs/heads/github_branch</li> +<li>The default value of ${inquiry_pipeline_branch_or_tag} will be the release / tag version set in global configuration</li> +<li>To build from a differnt branch, replace the ${inquiry_pipeline_branch_or_tag} with your branch</li> +</b></font> + refs/heads/${inquiry_pipeline_branch_or_tag} + false + + + image_tag + <font color=red size=2><b>CAUTION: If the value is blank, image tag will be taken from the latest metadata.json.</b></font> + + false + + + absolute_job_path + <font color=dimgray size=2><b>Do not change this value! The metadata.json will be copied from this job.</b></font> + ArtifactUpload/dev/KnowledgePlatform/InquiryFlinkJob + false + + + job_names_to_deploy + <font color=green size=2><b>Choose the job names to deploy. Multi-selection is available.</b></font> + choice-parameter-1273072434092073 + 1 + + true + + + + InquiryFlinkJob + Deploy/dev/KnowledgePlatform/InquiryFlinkJob + + + PT_MULTI_SELECT + false + 1 + + + + + 0 + 0 + + false + project + false + + + + + + + + + + 2 + + + https://github.com/Sunbird-inQuiry/data-pipeline.git + + + + + ${inquiry_release_tag} + + + false + + + + true + false + + 0 + false + + + + kubernetes/pipelines/deploy/Jenkinsfile + false + + + false + diff --git a/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Kubernetes/InquiryUploadSchema/config.xml b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Kubernetes/InquiryUploadSchema/config.xml new file mode 100644 index 0000000000..7441f81065 --- /dev/null +++ b/deploy/jenkins/jobs/Deploy/jobs/dev/jobs/Kubernetes/InquiryUploadSchema/config.xml @@ -0,0 +1,163 @@ + + + + false + + + + -1 + -1 + -1 + 5 + + + + + false + false + + + + + private_branch + + choice-parameter-2544395024638227 + 1 + + true + + + + true + + + InquiryUploadSchema + Deploy/dev/Kubernetes/InquiryUploadSchema + + + ET_FORMATTED_HTML + true + + + branch_or_tag + + choice-parameter-2620434998790477 + 1 + + true + + + + true + + + InquiryUploadSchema + Deploy/dev/Kubernetes/InquiryUploadSchema + + + ET_FORMATTED_HTML + true + + + schema_repo + <font color=dimgray size=2><b>Enter the repo url from which schema folder to be uploaded</b></font> + + + https://github.com/Sunbird-inQuiry/inquiry-api-service.git + https://github.com/project-sunbird/knowledge-platform.git + + + + + schema_repo_branch_or_tag + <font color=dimgray size=2><b>Enter the branch or tag for schema repo</b></font> + + false + + + source_folder + + choice-parameter-5348290139923778 + 1 + + true + + + + InquiryUploadSchema + Deploy/dev/Kubernetes/InquiryUploadSchema + + + PT_MULTI_SELECT + false + 1 + + + + + 0 + 0 + + false + project + false + + + + + + + + + + 2 + + + https://github.com/project-sunbird/sunbird-devops.git + + + + + ${branch_or_tag} + + + false + + + + true + false + + 0 + false + + + + pipelines/upload/schema/knowledge-platform/schema.Jenkinsfile + false + + + false + diff --git a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/ApplicationElasticSearch/config.xml b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/ApplicationElasticSearch/config.xml index 859b804b88..f669caa862 100644 --- a/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/ApplicationElasticSearch/config.xml +++ b/deploy/jenkins/jobs/Provision/jobs/dev/jobs/Core/jobs/ApplicationElasticSearch/config.xml @@ -26,7 +26,7 @@ @@ -114,4 +114,4 @@ return """<b>This parameter is not used</b>""" false - \ No newline at end of file + diff --git a/kubernetes/ansible/bootstrap.yaml b/kubernetes/ansible/bootstrap.yaml index 0fa7e0acfe..72f1beacff 100644 --- a/kubernetes/ansible/bootstrap.yaml +++ b/kubernetes/ansible/bootstrap.yaml @@ -39,7 +39,7 @@ - name: "Tagging {{ namespace }} to enable istio" shell: "kubectl label namespaces {{ namespace }} istio-injection=enabled --overwrite" - name: Creating docker secrets - shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ namespace }} --docker-server {{ vault_docker_registry_url }} --docker-username {{ vault_docker_registry_user }} --docker-password {{ vault_docker_registry_password }} --dry-run=client -o=yaml | kubectl apply -f -" + shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ namespace }} --docker-server {{ vault_docker_registry_url }} --docker-username {{ vault_docker_registry_user }} --docker-password '{{ vault_docker_registry_password }}' --dry-run=client -o=yaml | kubectl apply -f -" - name: Installing reloader for configmaps reload shell: helm upgrade --install --atomic reloader ../helm_charts/core/reloader --namespace "{{ namespace }}" -f /tmp/helm_vars.yaml roles: diff --git a/kubernetes/ansible/bootstrap_minimal.yaml b/kubernetes/ansible/bootstrap_minimal.yaml index 44170246ae..cdb145423e 100644 --- a/kubernetes/ansible/bootstrap_minimal.yaml +++ b/kubernetes/ansible/bootstrap_minimal.yaml @@ -30,7 +30,7 @@ - logging ignore_errors: yes - name: Creating docker secrets - shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ item }} --docker-server {{ vault_docker_registry_url }} --docker-username {{ vault_docker_registry_user }} --docker-password {{ vault_docker_registry_password }} --dry-run=client -o=yaml | kubectl apply -f -" + shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ item }} --docker-server {{ vault_docker_registry_url }} --docker-username {{ vault_docker_registry_user }} --docker-password '{{ vault_docker_registry_password }}' --dry-run=client -o=yaml | kubectl apply -f -" when: imagepullsecrets|length > 0 with_items: - "{{ bootstrap_namespace.split(',') }}" diff --git a/kubernetes/ansible/namespace.yml b/kubernetes/ansible/namespace.yml index bc62234c2c..bc38fab7d8 100644 --- a/kubernetes/ansible/namespace.yml +++ b/kubernetes/ansible/namespace.yml @@ -28,7 +28,7 @@ - "{{ bootstrap_namespace.split(',') }}" ignore_errors: yes - name: Creating docker secrets - shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ item }} --docker-server {{ vault_docker_registry_url }} --docker-username {{ vault_docker_registry_user }} --docker-password {{ vault_docker_registry_password }} --dry-run=client -o=yaml | kubectl apply -f -" + shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ item }} --docker-server {{ vault_docker_registry_url }} --docker-username {{ vault_docker_registry_user }} --docker-password '{{ vault_docker_registry_password }}' --dry-run=client -o=yaml | kubectl apply -f -" when: imagepullsecrets|length > 0 with_items: - "{{ bootstrap_namespace.split(',') }}" diff --git a/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml b/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml index b59cebe9a0..4cb69fd90f 100644 --- a/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml +++ b/kubernetes/helm_charts/core/nginx-private-ingress/templates/configmap.yaml @@ -40,7 +40,7 @@ data: listen 80; listen [::]:80; {{- if and .Values.nginx_private_ingress_ip (ne .Values.csp "oci") }} - server_name: {{ .Values.nginx_private_ingress_ip }}; + server_name {{ .Values.nginx_private_ingress_ip }}; {{- end }} resolver {{ .Values.kube_dns_ip }}; diff --git a/private_repo/ansible/inventory/dev/Core/common.yml b/private_repo/ansible/inventory/dev/Core/common.yml index 3386020d08..da2d8a9694 100644 --- a/private_repo/ansible/inventory/dev/Core/common.yml +++ b/private_repo/ansible/inventory/dev/Core/common.yml @@ -488,3 +488,12 @@ ml_analytics_cname_url: # Reference value - https://obj.dev.sunbirded.org/samiks ml_analytics_client_id : project-sunbird-dev-client # Keycloak client group ml_analytics_username : reportAdmin1 # Username of a user which has roles of "PROGRAM_MANAGER", "PROGRAM_DESIGNER", "REPORT_ADMIN", "REPORT_VIEWER" ml_analytics_createdBy : fb85a044-d9eb-479b-a55a-faf1bfaea14d # Unique system generated user UUID which is the same user as above +ml_container: samiksha + +# provide the s3 compatible endpoint +# for AWS +# cloud_private_storage_account_endpoint: "https://s3.{{ cloud_public_storage_region }}.amazonaws.com" +# for OCI +#cloud_private_storage_account_endpoint: "https://.compat.objectstorage.{{cloud_public_storage_region}}.oraclecloud.com" +cloud_private_storage_account_endpoint: "{{ cloud_public_storage_endpoint }}" # Leave Blank for Azure +bb_name: "" #Leave Blank diff --git a/private_repo/ansible/inventory/dev/Core/secrets.yml b/private_repo/ansible/inventory/dev/Core/secrets.yml index 2e16859454..c7f3e5b7e4 100644 --- a/private_repo/ansible/inventory/dev/Core/secrets.yml +++ b/private_repo/ansible/inventory/dev/Core/secrets.yml @@ -237,4 +237,4 @@ lp_vault_youtube_api_key: # youtube api token if you want # Examples values added below ml_analytics_authorization_access_token : 'Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJpc3MiOiJkOTY3NzRjYzXXXXXXXXXXXXXQ4Y2RiOWQ2Mzg0OSJ9.n4hXxKxl_698yeZPSWhXeGvMOb4esfgPadIZe8jZ0Z4' # Bearer auth token which has access to APIs as defined in the release note ml_analytics_password : Test@123 # Password of a user which has roles of "PROGRAM_MANAGER", "PROGRAM_DESIGNER", "REPORT_ADMIN", "REPORT_VIEWER" matching with supplied username to variable ml_analytics_username -ml_analytics_client_secret : fd241dce-4ZZ9-47e1-97cf-1c7de7a44216 # Keycloak client secret for creating tokens, mapped to client id supplied to variable ml_analytics_client_id +ml_analytics_client_secret : fd241dce-4ZZ9-47e1-97cf-1c7de7a44216 # Keycloak client secret for creating tokens, mapped to client id supplied to variable ml_analytics_client_id \ No newline at end of file diff --git a/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml b/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml index ce4e93b37e..d87400b63c 100644 --- a/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml +++ b/private_repo/ansible/inventory/dev/KnowledgePlatform/common.yml @@ -213,4 +213,10 @@ postgres: db_admin_password: "{{dp_vault_pgdb_admin_password}}" # graylog -send_logs_to_graylog: true # filebeat agents will send logs to graylog instead of ES \ No newline at end of file +send_logs_to_graylog: true # filebeat agents will send logs to graylog instead of ES + +druid_storage_type: "" # Your cloud service provider name. Supported values are aws, azure, gcloud + +# Optional variables - Can be left blank +cloud_storage_report_verfication_bucketname: "" +dp_storage_endpoint_config: ""