From d705159c55afde994d73a9eb287028c8ca98db04 Mon Sep 17 00:00:00 2001 From: Kumar Gauraw Date: Wed, 4 Oct 2023 17:12:58 +0530 Subject: [PATCH 1/2] Issue #IQ-559 feat: changes for inQuiry --- ansible/functional-tests.yml | 8 + ansible/inquiry_upload-schema.yml | 15 + .../roles/functional-tests/defaults/main.yml | 0 ansible/roles/functional-tests/tasks/main.yml | 18 + ansible/roles/kong-api/defaults/main.yml | 514 ++++++++++++++++++ .../assessment-service_application.conf | 22 +- pipelines/functional-tests/JenkinsFile | 56 ++ .../knowledge-platform/schema.Jenkinsfile | 55 ++ 8 files changed, 686 insertions(+), 2 deletions(-) create mode 100644 ansible/functional-tests.yml create mode 100644 ansible/inquiry_upload-schema.yml create mode 100644 ansible/roles/functional-tests/defaults/main.yml create mode 100644 ansible/roles/functional-tests/tasks/main.yml create mode 100644 pipelines/functional-tests/JenkinsFile create mode 100644 pipelines/upload/schema/knowledge-platform/schema.Jenkinsfile diff --git a/ansible/functional-tests.yml b/ansible/functional-tests.yml new file mode 100644 index 0000000000..2bcf45bfa3 --- /dev/null +++ b/ansible/functional-tests.yml @@ -0,0 +1,8 @@ +- hosts: local + become: yes + gather_facts: false + vars_files: + - ['{{inventory_dir}}/secrets.yml'] + roles: + - functional-tests + run_once: true \ No newline at end of file diff --git a/ansible/inquiry_upload-schema.yml b/ansible/inquiry_upload-schema.yml new file mode 100644 index 0000000000..32ebde956d --- /dev/null +++ b/ansible/inquiry_upload-schema.yml @@ -0,0 +1,15 @@ +- hosts: local + become: yes + gather_facts: no + vars_files: + - "{{inventory_dir}}/secrets.yml" + environment: + AZURE_STORAGE_ACCOUNT: "{{ sunbird_public_storage_account_name }}" + AZURE_STORAGE_SAS_TOKEN: "{{ sunbird_public_storage_account_sas }}" + tasks: + - name: upload batch + command: "az storage blob upload-batch --destination {{ plugin_container_name }}/schemas/local/{{ item }} --source {{ source_path }}/{{ item }}" + with_items: + - "{{ source_name.split(',') }}" + async: 3600 + poll: 10 diff --git a/ansible/roles/functional-tests/defaults/main.yml b/ansible/roles/functional-tests/defaults/main.yml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ansible/roles/functional-tests/tasks/main.yml b/ansible/roles/functional-tests/tasks/main.yml new file mode 100644 index 0000000000..eb02115389 --- /dev/null +++ b/ansible/roles/functional-tests/tasks/main.yml @@ -0,0 +1,18 @@ +- name: Create directories + file: + path: functional-tests-files + state: directory + owner: jenkins + group: jenkins +- name: Copy Test Script File + copy: + src: "{{ source_name }}/ft_question_questionset_{{ file_version }}.json" + dest: functional-tests-files/ft_question_questionset_{{ file_version }}.json +- name: Template Env File + template: + src: "{{ source_name }}/env_question_questionset_{{ file_version }}.json" + dest: functional-tests-files/env_question_questionset_{{ file_version }}.json +- name: Trigger Functional Test + shell: + cmd: newman run ft_question_questionset_{{ file_version }}.json -e env_question_questionset_{{ file_version }}.json --reporters cli,htmlextra,junit --reporter-htmlextra-export newman/report.html --reporter-junit-export newman/junit_report.xml ; chown -R jenkins:jenkins newman + chdir: functional-tests-files diff --git a/ansible/roles/kong-api/defaults/main.yml b/ansible/roles/kong-api/defaults/main.yml index 6090bfff0f..cf51212d15 100644 --- a/ansible/roles/kong-api/defaults/main.yml +++ b/ansible/roles/kong-api/defaults/main.yml @@ -10132,3 +10132,517 @@ kong_apis: config.required: false config.enabled: false #Lern release-5.3.0 + +## inQuiry release-6.0.0 + +- name: questionCreateV2 + uris: "{{ question_prefix }}/v2/create" + upstream_url: "{{ assessment_service_url }}/question/v5/create" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - contentCreate + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: questionReadV2 + uris: "{{ question_prefix }}/v2/read" + upstream_url: "{{ assessment_service_url }}/question/v5/read" + strip_uri: true + plugins: + - name: cors + - "{{ statsd_pulgin }}" + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: ip + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: questionPrivateReadV2 + uris: "{{ question_prefix }}/v2/private/read" + upstream_url: "{{ assessment_service_url }}/question/v5/private/read" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - contentAccess + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: questionUpdateV2 + uris: "{{ question_prefix }}/v2/update" + upstream_url: "{{ assessment_service_url }}/question/v5/update" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - contentUpdate + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: questionRetireV2 + uris: "{{ question_prefix }}/v2/retire" + upstream_url: "{{ assessment_service_url }}/question/v5/retire" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - contentAdmin + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: questionReviewV2 + uris: "{{ question_prefix }}/v2/review" + upstream_url: "{{ assessment_service_url }}/question/v5/review" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - contentAdmin + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: questionPublishV2 + uris: "{{ question_prefix }}/v2/publish" + upstream_url: "{{ assessment_service_url }}/question/v5/publish" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - contentCreate + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: questionListV2 + uris: "{{ question_prefix }}/v2/list" + upstream_url: "{{ assessment_service_url }}/question/v5/list" + strip_uri: true + plugins: + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: ip + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: questionRejectV2 + uris: "{{ question_prefix }}/v2/reject" + upstream_url: "{{ assessment_service_url }}/question/v5/reject" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - contentAdmin + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: copyQuestionV2 + uris: "{{ question_prefix }}/v2/copy" + upstream_url: "{{ assessment_service_url }}/question/v5/copy" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - contentCreate + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: questionSetCreateV2 + uris: "{{ questionset_prefix }}/v2/create" + upstream_url: "{{ assessment_service_url }}/questionset/v5/create" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - contentCreate + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: questionSetReadV2 + uris: "{{ questionset_prefix }}/v2/read" + upstream_url: "{{ assessment_service_url }}/questionset/v5/read" + strip_uri: true + plugins: + - name: cors + - "{{ statsd_pulgin }}" + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: ip + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: questionSetPrivateReadV2 + uris: "{{ questionset_prefix }}/v2/private/read" + upstream_url: "{{ assessment_service_url }}/questionset/v5/private/read" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - contentAccess + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: questionSetUpdateV2 + uris: "{{ questionset_prefix }}/v2/update" + upstream_url: "{{ assessment_service_url }}/questionset/v5/update" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - contentUpdate + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: questionSetAddQuestionV2 + uris: "{{ questionset_prefix }}/v2/add" + upstream_url: "{{ assessment_service_url }}/questionset/v5/add" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentUpdate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: questionSetRemoveQuestionV2 + uris: "{{ questionset_prefix }}/v2/remove" + upstream_url: "{{ assessment_service_url }}/questionset/v5/remove" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - 'contentUpdate' + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: questionSetRetireV2 + uris: "{{ questionset_prefix }}/v2/retire" + upstream_url: "{{ assessment_service_url }}/questionset/v5/retire" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - contentAdmin + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: questionSetReviewV2 + uris: "{{ questionset_prefix }}/v2/review" + upstream_url: "{{ assessment_service_url }}/questionset/v5/review" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - contentAdmin + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: questionSetPublishV2 + uris: "{{ questionset_prefix }}/v2/publish" + upstream_url: "{{ assessment_service_url }}/questionset/v5/publish" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - contentAdmin + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: questionSetUpdateHierarchyV2 + uris: "{{ questionset_prefix }}/v2/hierarchy/update" + upstream_url: "{{ assessment_service_url }}/questionset/v5/hierarchy/update" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - contentUpdate + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: questionSetReadHierarchyV2 + uris: "{{ questionset_prefix }}/v2/hierarchy" + upstream_url: "{{ assessment_service_url }}/questionset/v5/hierarchy" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - anonymousContentAccess + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: questionSetRejectV2 + uris: "{{ questionset_prefix }}/v2/reject" + upstream_url: "{{ assessment_service_url }}/questionset/v5/reject" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - contentAdmin + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: questionImportAPIV2 + uris: "{{ question_prefix }}/v2/import" + upstream_url: "{{ assessment_service_url }}/question/v5/import" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - contentCreate + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: questionsetImportAPIV2 + uris: "{{ questionset_prefix }}/v2/import" + upstream_url: "{{ assessment_service_url }}/questionset/v5/import" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - contentCreate + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + +- name: copyQuestionSetV2 + uris: "{{ questionset_prefix }}/v2/copy" + upstream_url: "{{ assessment_service_url }}/questionset/v5/copy" + strip_uri: true + plugins: + - name: jwt + - name: cors + - "{{ statsd_pulgin }}" + - name: acl + config.whitelist: + - contentCreate + - name: rate-limiting + config.policy: local + config.hour: "{{ medium_rate_limit_per_hour }}" + config.limit_by: credential + - name: request-size-limiting + config.allowed_payload_size: "{{ medium_request_size_limit }}" + - name: opa-checks + config.required: false + config.enabled: false + diff --git a/ansible/roles/stack-sunbird/templates/assessment-service_application.conf b/ansible/roles/stack-sunbird/templates/assessment-service_application.conf index d5e7723a19..3e127b78ca 100644 --- a/ansible/roles/stack-sunbird/templates/assessment-service_application.conf +++ b/ansible/roles/stack-sunbird/templates/assessment-service_application.conf @@ -89,6 +89,18 @@ akka { nr-of-instances = 5 dispatcher = actors-dispatcher } + /questionV5Actor + { + router = smallest-mailbox-pool + nr-of-instances = 5 + dispatcher = actors-dispatcher + } + /questionSetV5Actor + { + router = smallest-mailbox-pool + nr-of-instances = 5 + dispatcher = actors-dispatcher + } } } } @@ -419,17 +431,23 @@ assessment.copy.props_to_remove=["downloadUrl", "artifactUrl", "variants", "LastPublishedBy", "rejectReasons", "rejectComment", "gradeLevel", "subject", "medium", "board", "topic", "purpose", "subtopic", "contentCredits", "owner", "collaborators", "creators", "contributors", "badgeAssertions", "dialcodes", - "concepts", "keywords", "reservedDialcodes", "dialcodeRequired", "leafNodes", "sYS_INTERNAL_LAST_UPDATED_ON", "prevStatus", "lastPublishedBy", "streamingUrl"] + "concepts", "keywords", "reservedDialcodes", "dialcodeRequired", "leafNodes", "sYS_INTERNAL_LAST_UPDATED_ON", "prevStatus", "lastPublishedBy", "streamingUrl", "publish_type", "migrationVersion"] cloud_storage_container: "{{ cloud_storage_content_bucketname }}" cloudstorage { metadata.replace_absolute_path={{ cloudstorage_replace_absolute_path | default('false') }} metadata.list={{ cloudstorage_metadata_list }} - relative_path_prefix="{{ cloudstorage_relative_path_prefix | default('CLOUD_STORAGE_BASE_PATH') }}" + relative_path_prefix="{{ cloudstorage_relative_path_prefix_content | default('CLOUD_STORAGE_BASE_PATH') }}" read_base_path="{{ cloudstorage_base_path }}" write_base_path={{ valid_cloudstorage_base_urls }} } #Inquiry-release-5.6.0 question.list.limit={{ question_list_api_request_limit | default('20') }} + +# V5 API Configurations +v5_supported_qumlVersions=[1.1] +v5_default_qumlVersion=1.1 + + diff --git a/pipelines/functional-tests/JenkinsFile b/pipelines/functional-tests/JenkinsFile new file mode 100644 index 0000000000..a841bc21f3 --- /dev/null +++ b/pipelines/functional-tests/JenkinsFile @@ -0,0 +1,56 @@ +@Library('deploy-conf') _ +node() { + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" + + stage('checkout public repo') { + folder = new File("$WORKSPACE/.git") + if (folder.exists()) + { + println "Found .git folder. Clearing it.." + sh'git clean -fxd' + } + checkout scm + } + + ansiColor('xterm') { + values = [:] + currentWs = sh(returnStdout: true, script: 'pwd').trim() + envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() + module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() + jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() + stage('deploy artifact'){ + sh """ + rm -rf test_repo + git clone ${params.test_repo} -b ${params.test_repo_branch_or_tag} test_repo + """ + ansiblePlaybook = "${currentWs}/ansible/functional-tests.yml" + ansibleExtraArgs = "--extra-vars \" source_name=${currentWs}/test_repo/functional-tests/${params.file_version} inquiry_api_version=${params.inquiry_api_version} file_version=${params.file_version} inquiry_host_url=${params.inquiry_host_url} inquiry_channel_id=${params.inquiry_channel_id} \" --vault-password-file /var/lib/jenkins/secrets/vault-pass" + values.put('currentWs', currentWs) + values.put('env', envDir) + values.put('module', module) + values.put('jobName', jobName) + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + println values + ansible_playbook_run(values) + currentBuild.result = 'SUCCESS' + currentBuild.description = "Private: ${params.private_branch}, Public: ${params.branch_or_tag}, test_repo_branch_or_tag: ${params.test_repo_branch_or_tag}" + archiveArtifacts "ansible/functional-tests-files/newman/report.html" + } + } + summary() + } + catch (err) { + currentBuild.result = 'FAILURE' + throw err + } + finally { + slack_notify(currentBuild.result) + email_notify() + } +} \ No newline at end of file diff --git a/pipelines/upload/schema/knowledge-platform/schema.Jenkinsfile b/pipelines/upload/schema/knowledge-platform/schema.Jenkinsfile new file mode 100644 index 0000000000..eb0bd8ed27 --- /dev/null +++ b/pipelines/upload/schema/knowledge-platform/schema.Jenkinsfile @@ -0,0 +1,55 @@ +@Library('deploy-conf') _ +node() { + try { + String ANSI_GREEN = "\u001B[32m" + String ANSI_NORMAL = "\u001B[0m" + String ANSI_BOLD = "\u001B[1m" + String ANSI_RED = "\u001B[31m" + String ANSI_YELLOW = "\u001B[33m" + + stage('checkout public repo') { + folder = new File("$WORKSPACE/.git") + if (folder.exists()) + { + println "Found .git folder. Clearing it.." + sh'git clean -fxd' + } + checkout scm + } + + ansiColor('xterm') { + values = [:] + currentWs = sh(returnStdout: true, script: 'pwd').trim() + envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() + module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() + jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() + stage('deploy artifact'){ + sh """ + rm -rf schema_repo + git clone ${params.schema_repo} -b ${params.schema_repo_branch_or_tag} schema_repo + """ + ansiblePlaybook = "${currentWs}/ansible/inquiry_upload-schema.yml" + ansibleExtraArgs = "--extra-vars \" source_name=${params.source_folder} source_path=${currentWs}/schema_repo/schemas \" --vault-password-file /var/lib/jenkins/secrets/vault-pass" + values.put('currentWs', currentWs) + values.put('env', envDir) + values.put('module', module) + values.put('jobName', jobName) + values.put('ansiblePlaybook', ansiblePlaybook) + values.put('ansibleExtraArgs', ansibleExtraArgs) + println values + ansible_playbook_run(values) + currentBuild.result = 'SUCCESS' + currentBuild.description = "Private: ${params.private_branch}, Public: ${params.branch_or_tag}, schema_repo_branch_or_tag: ${params.schema_repo_branch_or_tag}" + } + } + summary() + } + catch (err) { + currentBuild.result = 'FAILURE' + throw err + } + finally { + slack_notify(currentBuild.result) + email_notify() + } +} From a56bd7ccf44ef4b7406defbf437f847f8e68d775 Mon Sep 17 00:00:00 2001 From: Kumar Gauraw Date: Mon, 9 Oct 2023 18:33:53 +0530 Subject: [PATCH 2/2] Issue #IQ-559 fix: updated cassandra script and deleted schema upload --- ansible/inquiry_upload-schema.yml | 15 ----- .../templates/inquiry_alter_table.cql | 1 + .../templates/inquiry_create_table.cql | 44 +++++++++++++++ .../knowledge-platform/schema.Jenkinsfile | 55 ------------------- 4 files changed, 45 insertions(+), 70 deletions(-) delete mode 100644 ansible/inquiry_upload-schema.yml create mode 100644 ansible/roles/cassandra-db-update/templates/inquiry_alter_table.cql create mode 100644 ansible/roles/cassandra-db-update/templates/inquiry_create_table.cql delete mode 100644 pipelines/upload/schema/knowledge-platform/schema.Jenkinsfile diff --git a/ansible/inquiry_upload-schema.yml b/ansible/inquiry_upload-schema.yml deleted file mode 100644 index 32ebde956d..0000000000 --- a/ansible/inquiry_upload-schema.yml +++ /dev/null @@ -1,15 +0,0 @@ -- hosts: local - become: yes - gather_facts: no - vars_files: - - "{{inventory_dir}}/secrets.yml" - environment: - AZURE_STORAGE_ACCOUNT: "{{ sunbird_public_storage_account_name }}" - AZURE_STORAGE_SAS_TOKEN: "{{ sunbird_public_storage_account_sas }}" - tasks: - - name: upload batch - command: "az storage blob upload-batch --destination {{ plugin_container_name }}/schemas/local/{{ item }} --source {{ source_path }}/{{ item }}" - with_items: - - "{{ source_name.split(',') }}" - async: 3600 - poll: 10 diff --git a/ansible/roles/cassandra-db-update/templates/inquiry_alter_table.cql b/ansible/roles/cassandra-db-update/templates/inquiry_alter_table.cql new file mode 100644 index 0000000000..4924c1136a --- /dev/null +++ b/ansible/roles/cassandra-db-update/templates/inquiry_alter_table.cql @@ -0,0 +1 @@ +ALTER TABLE {{ question_keyspace_name }}.question_data ADD (outcomeDeclaration text, feedback text); \ No newline at end of file diff --git a/ansible/roles/cassandra-db-update/templates/inquiry_create_table.cql b/ansible/roles/cassandra-db-update/templates/inquiry_create_table.cql new file mode 100644 index 0000000000..c4543734e3 --- /dev/null +++ b/ansible/roles/cassandra-db-update/templates/inquiry_create_table.cql @@ -0,0 +1,44 @@ +CREATE KEYSPACE IF NOT EXISTS {{ hierarchy_keyspace_name }} WITH replication = { + 'class': 'SimpleStrategy', + 'replication_factor': '1' +}; + +CREATE KEYSPACE IF NOT EXISTS {{ question_keyspace_name }} WITH replication = { + 'class': 'SimpleStrategy', + 'replication_factor': '1' +}; + +CREATE TABLE IF NOT EXISTS {{ hierarchy_keyspace_name }}.questionset_hierarchy ( + identifier text, + hierarchy text, + instructions text, + outcomeDeclaration text, + PRIMARY KEY (identifier) +); + +CREATE TABLE IF NOT EXISTS {{ question_keyspace_name }}.question_data ( + identifier text, + body blob, + editorState text, + answer blob, + solutions text, + instructions text, + hints text, + media text, + responseDeclaration text, + interactions text, + outcomeDeclaration text, + feedback text, + PRIMARY KEY (identifier) +); + +{% if groups['cassandra-node-2'] is defined %} +ALTER KEYSPACE {{ hierarchy_keyspace_name }} WITH replication = { + 'class': 'NetworkTopologyStrategy', + 'datacenter1' : 2 +}; +ALTER KEYSPACE {{ question_keyspace_name }} WITH replication = { + 'class': 'NetworkTopologyStrategy', + 'datacenter1' : 2 +}; +{% endif %} \ No newline at end of file diff --git a/pipelines/upload/schema/knowledge-platform/schema.Jenkinsfile b/pipelines/upload/schema/knowledge-platform/schema.Jenkinsfile deleted file mode 100644 index eb0bd8ed27..0000000000 --- a/pipelines/upload/schema/knowledge-platform/schema.Jenkinsfile +++ /dev/null @@ -1,55 +0,0 @@ -@Library('deploy-conf') _ -node() { - try { - String ANSI_GREEN = "\u001B[32m" - String ANSI_NORMAL = "\u001B[0m" - String ANSI_BOLD = "\u001B[1m" - String ANSI_RED = "\u001B[31m" - String ANSI_YELLOW = "\u001B[33m" - - stage('checkout public repo') { - folder = new File("$WORKSPACE/.git") - if (folder.exists()) - { - println "Found .git folder. Clearing it.." - sh'git clean -fxd' - } - checkout scm - } - - ansiColor('xterm') { - values = [:] - currentWs = sh(returnStdout: true, script: 'pwd').trim() - envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim() - module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim() - jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim() - stage('deploy artifact'){ - sh """ - rm -rf schema_repo - git clone ${params.schema_repo} -b ${params.schema_repo_branch_or_tag} schema_repo - """ - ansiblePlaybook = "${currentWs}/ansible/inquiry_upload-schema.yml" - ansibleExtraArgs = "--extra-vars \" source_name=${params.source_folder} source_path=${currentWs}/schema_repo/schemas \" --vault-password-file /var/lib/jenkins/secrets/vault-pass" - values.put('currentWs', currentWs) - values.put('env', envDir) - values.put('module', module) - values.put('jobName', jobName) - values.put('ansiblePlaybook', ansiblePlaybook) - values.put('ansibleExtraArgs', ansibleExtraArgs) - println values - ansible_playbook_run(values) - currentBuild.result = 'SUCCESS' - currentBuild.description = "Private: ${params.private_branch}, Public: ${params.branch_or_tag}, schema_repo_branch_or_tag: ${params.schema_repo_branch_or_tag}" - } - } - summary() - } - catch (err) { - currentBuild.result = 'FAILURE' - throw err - } - finally { - slack_notify(currentBuild.result) - email_notify() - } -}