Merge pull request #1105 from G33tha/release-2.10.0

added aws cloud storage type for learning and samza job deploy

ansible/roles/learning-service/defaults/main.yml

@@ -50,3 +50,9 @@ tomcat_init_mem: -Xms1024m
 tomcat_max_mem: -Xmx4096m
 search_index_host: "{{ groups['composite-search-cluster']|join(':9200,')}}:9200"
 compositesearch_index_name: "compositesearch"
+
+
+### if cloud storage type is AWS
+aws_access_key: ""
+aws_secret_key: ""
+aws_s3_bucket: ""

ansible/roles/learning-service/templates/application.conf.j2

@@ -229,11 +229,17 @@ learning.content.type.not.copied.list=["Asset"]
 #Youtube License Validation Regex Pattern
 youtube.license.regex.pattern=["\\?vi?=([^&]*)", "watch\\?.*v=([^&]*)", "(?:embed|vi?)/([^/?]*)","^([A-Za-z0-9\\-\\_]*)"]
 
-#Azure Storage details
 cloud_storage_type="{{ cloud_store }}"
+{% if cloud_store|lower == 'aws' %}
+aws_storage_key="{{ aws_access_key }}"
+aws_storage_secret="{{ aws_secret_key }}"
+aws_storage_container="{{ aws_s3_bucket }}"
+{% else %}
+#Azure Storage details
 azure_storage_key="{{sunbird_public_storage_account_name}}"
 azure_storage_secret="{{sunbird_public_storage_account_key}}"
 azure_storage_container="{{ azure_public_container }}"
+{% endif %}
 
 installation.id="{{ instance_name }}"
 

ansible/roles/samza-jobs/defaults/main.yml

@@ -43,3 +43,9 @@ course_progress_batch_size: 100
 itemset_generate_pdf: true
 auto_creator_container_count: 1
 content_streaming_enabled: true
+
+
+### if cloud storage type is AWS
+aws_access_key: ""
+aws_secret_key: ""
+aws_s3_bucket: ""

ansible/roles/samza-jobs/tasks/deploy.yml

@@ -66,15 +66,14 @@
   register: config_files
   when: "{{item|changed}}"
   with_items: "{{ (extract_dir|default({})).results|default([]) }}"
-
+    
 - name: update environment specific details in new job configs
   replace: dest="{{item[1].stdout}}" regexp="{{item[0].key}}" replace="{{item[0].value}}"
   when: "{{item[1]|changed}}"
   with_nested:
     - [{key: "__yarn_host__", value: "{{__yarn_host__}}"}, {key: "__yarn_port__", value: "{{__yarn_port__}}"}, {key: "__env__", value: "{{env}}" }, {key: "__env_name__", value: "{{env_name}}" }, {key: "__zookeepers__", value: "{{zookeepers}}"}, {key: "__kafka_brokers__", value: "{{kafka_brokers}}"}, {key: "__delayInMilliSeconds__", value: "{{delayInMilliSeconds}}" }, {key: "__retryTimeInMilliSeconds__", value: "{{retryTimeInMilliSeconds}}" }, {key: "__bypass_reverse_search__", value: "{{bypass_reverse_search}}" }, {key: "__retryBackoffBaseInSeconds__", value: "{{retry_backoff_base_in_seconds}}" }, {key: "__retryLimit__", value: "{{retry_limit}}" }, {key: "__retryLimitEnable__", value: "{{retry_limit_enable}}" },  {key: "__google_api_key__", value: "{{google_api_key}}" }, {key: "__searchServiceEndpoint__", value: "{{search_service_endpoint}}" }, {key: "__objectDenormalizationAdditionalConfig__", value: "{{object_denormalization_additional_config}}" },{key: "__audit_es_host__", value: "{{audit_es_host}}"}, {key: "__search_es_host__", value: "{{search_es_host}}"}, {key: "__redis_host__", value: "{{redis_host}}"}, {key: "__redis_port__", value: "{{redis_port}}"}, {key: "__environment_id__", value: "{{environment_id}}"}, {key: "__graph_passport_key__", value: "{{graph_passport_key}}"}, {key: "__lp_bolt_url__", value: "{{lp_bolt_url}}"}, {key: "__lp_bolt_read_url__", value: "{{lp_bolt_read_url}}"}, {key: "__lp_bolt_write_url__", value: "{{lp_bolt_write_url}}"}, {key: "__other_bolt_url__", value: "{{other_bolt_url}}"}, {key: "__other_bolt_read_url__", value: "{{other_bolt_read_url}}"}, {key: "__other_bolt_write_url__", value: "{{other_bolt_write_url}}"}, {key: "__mw_shard_id__", value: "{{mw_shard_id}}"}, {key: "__lp_url__", value: "{{lp_url}}"}, {key: "__cloud_storage_config_environment__", value: "{{cloud_storage_config_environment}}"}, {key: "__google_vision_tagging__", value: "{{google_vision_tagging}}"}, {key: "__lp_tmpfile_location__", value: "{{lp_tmpfile_location}}"}, {key: "__esRouterAdditionalConfig__", value: "{{es_router_additional_config}}"},{key: "__esRouterSecondaryAdditionalConfig__", value: "{{es_router_additional_secondary_config}}"},{key: "__es_port__", value: "{{es_port}}"}, {key: "__keyspace_name__", value: "{{content_keyspace_name}}"}, {key: "__collection_fullecar_disable__", value: "{{collection_fullecar_disable}}"},{key: "__max_iteration_count_for_samza_job__", value: "{{max_iteration_count_for_samza_job}}"},{key: "__cloud_storage_type__", value: "{{cloud_store}}"},{key: "__azure_storage_key__", value: "{{sunbird_public_storage_account_name}}"},{key: "__azure_storage_secret__", value: "{{sunbird_public_storage_account_key}}"},{key: "__azure_storage_container__", value: "{{azure_public_container}}"},{key: "__content_media_base_url__", value: "{{content_media_base_url}}"}, {key: "__plugin_media_base_url__", value: "{{plugin_media_base_url}}"}, {key: "__installation_id__", value: "{{instance_name}}"}, {key: "__content_media_base_url__", value: "{{content_media_base_url}}"}, {key: "__hierarchy_keyspace_name__", value: "{{hierarchy_keyspace_name}}"},  {key: "__composite_search_indexer_container_count__", value: "{{composite_search_indexer_container_count}}"},{key: "__cassandra_lp_connection__", value: "{{lp_cassandra_connection}}"}, {key: "__cassandra_lpa_connection__", value: "{{dp_cassandra_connection}}"}, {key: "__streaming_mime_type__", value: "{{streaming_mime_type}}"}, {key: "__cassandra_sunbird_connection__", value: "{{core_cassandra_connection}}"}, {key: "__cloud_upload_retry_count__", value: "{{cloud_upload_retry_count}}"}, {key: "__compositesearch_index_name__", value: "{{compositesearch_index_name}}"},{key: "__publish_pipeline_container_count__", value: "{{publish_pipeline_container_count}}"},{key: "__yarn_container_memory_mb__", value: "{{publish_yarn_container_memory_mb}}"},{key: "__youtube_api_key__", value: "{{youtube_api_key}}"},{key: "__kp_learning_service_base_url__", value: "{{kp_learning_service_base_url}}"},{key: "__sunbird_installation__", value: "{{sunbird_platform_installation}}"},  {key: "__search_lms_es_host__", value: "{{search_lms_es_host}}"},{key: "__dial_image_storage_container__", value: "{{dial_image_storage_container}}"},{key: "__dial_base_url__", value: "{{dial_base_url}}"},{key: "__learner_service_base_url__", value: "{{learner_service_base_url}}"},{key: "__cert_service_base_url__", value: "{{cert_service_base_url}}"},{key: "__certificate_base_path__", value: "{{certificate_base_path}}"},{key: "__kp_content_service_base_url__", value: "{{kp_content_service_base_url}}"},{key: "__kp_print_service_base_url__", value: "{{kp_print_service_base_url}}"},{key: "__cert_reg_service_base_url__", value: "{{cert_reg_service_base_url}}"},{key: "__kp_search_service_base_url__", value: "{{kp_search_service_base_url}}"},{key: "__samza_coordinator_replication_factor__", value: "{{samza_coordinator_replication_factor}}"},{key: "__samza_checkpoint_replication_factor__", value: "{{samza_checkpoint_replication_factor}}"},{key: "__course_batch_updater_container_count__", value: "{{course_batch_updater_container_count}}"},{key: "__course_certificate_generator_container_count__", value: "{{course_certificate_generator_container_count}}"},{key: "__course_progress_batch_size__", value: "{{course_progress_batch_size}}"},{key: "__itemset_generate_pdf__", value: "{{itemset_generate_pdf}}"},{key: "__auto_creator_container_count__", value: "{{auto_creator_container_count}}"},{key: "__content_streaming_enabled__", value: "{{content_streaming_enabled}}"}]
     - "{{ (config_files|default({})).results|default([]) }}"
 
-
 - name: Create directory for additional config
   file: path={{object_denormalization_additional_config_dir}} owner=hduser group=hadoop recurse=yes state=directory