Issue #0000 fix: DialcodeUsageSummaryModel fix to ignore SEARCH event…

…s with boolean value for dialcodes field.
project-sunbird · Jan 4, 2019 · 544a31c · 544a31c
1 parent 07aa2d8
commit 544a31c
Show file tree

Hide file tree

Showing 4 changed files with 9 additions and 5 deletions.
diff --git a/...es/batch-models/src/main/scala/org/ekstep/analytics/model/DialcodeUsageSummaryModel.scala b/...es/batch-models/src/main/scala/org/ekstep/analytics/model/DialcodeUsageSummaryModel.scala
@@ -4,7 +4,7 @@ import org.apache.spark.SparkContext
 import org.apache.spark.rdd.RDD
 import org.ekstep.analytics.framework._
 import org.ekstep.analytics.framework.conf.AppConf
-import org.ekstep.analytics.framework.util.CommonUtil
+import org.ekstep.analytics.framework.util.{CommonUtil, JSONUtils, JobLogger}
 
 import scala.collection.mutable.ListBuffer
 
@@ -27,9 +27,12 @@ object DialcodeUsageSummaryModel extends IBatchModelTemplate[V3Event, V3Event, D
 
             if (filters("dialcodes").isInstanceOf[String]) {
                 dialcode += filters.getOrElse("dialcodes", "").asInstanceOf[String]
-            } else {
+            } else if (filters("dialcodes").isInstanceOf[List[String]]) {
                 dialcode ++= filters.getOrElse("dialcodes", List()).asInstanceOf[List[String]]
             }
+            else {
+                JobLogger.log("ignore event with dialcodes other than string and list. mid: " + event.mid, None, Level.INFO)
+            }
             (dialcode.toList, event.ets, event.context.channel)
         })
 

diff --git a/...les/batch-models/src/test/resources/dialcode-usage-summary/withSameDialcodeAndChannel.log b/...les/batch-models/src/test/resources/dialcode-usage-summary/withSameDialcodeAndChannel.log
@@ -1,2 +1,3 @@
 {"eid":"SEARCH","ets":1542175922000,"ver":"3.0","mid":"LP.1542175922142.13b8c7e0-f488-44bd-916e-4f98aebc5336","actor":{"id":"org.ekstep.learning.platform","type":"System"},"context":{"channel":"01235953109336064029413","pdata":{"id":"prod.ntp.learning.platform","pid":"search-service","ver":"1.0"},"env":"search","did":"53a63adfdc9c2c5252e3cc965b5189347e2d9c58"},"edata":{"size":1,"query":"","filters":{"dialcodes":["2Q8WDW"],"contentType":["TextBook","TextBookUnit"],"objectType":["Content","ContentImage"],"status":["Live"],"compatibilityLevel":{"max":4,"min":1},"channel":{"ne":["0124433024890224640","0124446042259128320","0124487522476933120","0125840271570288640","0124453662635048969"]},"framework":{},"mimeType":{},"resourceType":{}},"sort":{},"type":"content","topn":[{"identifier":"do_312594668076343296224224"}]},"cdata":[{"id":"prod.diksha.app","type":"AppId"}],"flags":{"tv_processed":true,"dd_processed":true},"type":"events","syncts":1542175923089,"@timestamp":"2018-11-14T06:12:03.089Z","ts":"2018-11-14T06:12:02.142+0000"}
-{"eid":"SEARCH","ets":1542175922111,"ver":"3.0","mid":"LP.1542175922142.13b8c7e0-f488-44bd-916e-4f98aebc5336","actor":{"id":"org.ekstep.learning.platform","type":"System"},"context":{"channel":"01235953109336064029413","pdata":{"id":"prod.ntp.learning.platform","pid":"search-service","ver":"1.0"},"env":"search","did":"53a63adfdc9c2c5252e3cc965b5189347e2d9c58"},"edata":{"size":1,"query":"","filters":{"dialcodes":["2Q8WDW"],"contentType":["TextBook","TextBookUnit"],"objectType":["Content","ContentImage"],"status":["Live"],"compatibilityLevel":{"max":4,"min":1},"channel":{"ne":["0124433024890224640","0124446042259128320","0124487522476933120","0125840271570288640","0124453662635048969"]},"framework":{},"mimeType":{},"resourceType":{}},"sort":{},"type":"content","topn":[{"identifier":"do_312594668076343296224224"}]},"cdata":[{"id":"prod.diksha.app","type":"AppId"}],"flags":{"tv_processed":true,"dd_processed":true},"type":"events","syncts":1542175923089,"@timestamp":"2018-11-14T06:12:03.089Z","ts":"2018-11-14T06:12:02.142+0000"}
+{"eid":"SEARCH","ets":1542175922111,"ver":"3.0","mid":"LP.1542175922142.13b8c7e0-f488-44bd-916e-4f98aebc5336","actor":{"id":"org.ekstep.learning.platform","type":"System"},"context":{"channel":"01235953109336064029413","pdata":{"id":"prod.ntp.learning.platform","pid":"search-service","ver":"1.0"},"env":"search","did":"53a63adfdc9c2c5252e3cc965b5189347e2d9c58"},"edata":{"size":1,"query":"","filters":{"dialcodes":["2Q8WDW"],"contentType":["TextBook","TextBookUnit"],"objectType":["Content","ContentImage"],"status":["Live"],"compatibilityLevel":{"max":4,"min":1},"channel":{"ne":["0124433024890224640","0124446042259128320","0124487522476933120","0125840271570288640","0124453662635048969"]},"framework":{},"mimeType":{},"resourceType":{}},"sort":{},"type":"content","topn":[{"identifier":"do_312594668076343296224224"}]},"cdata":[{"id":"prod.diksha.app","type":"AppId"}],"flags":{"tv_processed":true,"dd_processed":true},"type":"events","syncts":1542175923089,"@timestamp":"2018-11-14T06:12:03.089Z","ts":"2018-11-14T06:12:02.142+0000"}
+{"eid":"SEARCH","ets":1546453795939,"ver":"3.0","mid":"LP.1546453795939.79953458-410c-4a84-93ce-440c00cfa7ce","actor":{"id":"org.ekstep.learning.platform","type":"System"},"context":{"channel":"in.ekstep","pdata":{"id":"prod.ntp.learning.platform","ver":"1.0","pid":"search-service"},"env":"search"},"edata":{"type":"content","duration":0,"score":0,"rating":0.0,"index":0,"query":"","sort":{},"topn":[],"filters":{"channel":{"ne":["0124433024890224640","0124446042259128320","0124487522476933120","0125840271570288640","0124453662635048969"]},"resourceType":{},"framework":{},"mimeType":{},"contentType":{},"objectType":["Content","ContentImage"],"dialcodes":true},"size":0},"@timestamp":"2019-01-02T18:29:56.694Z"}
diff --git a/...dels/src/test/scala/org/ekstep/analytics/job/summarizer/TestDialcodeUsageSummarizer.scala b/...dels/src/test/scala/org/ekstep/analytics/job/summarizer/TestDialcodeUsageSummarizer.scala
@@ -8,7 +8,7 @@ class TestDialcodeUsageSummarizer extends SparkSpec(null) {
 
   "DialcodeUsageSummarizer" should "execute the job and won't throw any Exception" in {
 
-    val config = JobConfig(Fetcher("local", None, Option(Array(Query(None, None, None, None, None, None, None, None, None, Option("src/test/resources/dialcode-usage-summary/telemetry_test_data.log"))))), null, null, "org.ekstep.analytics.model.DialcodeUsageSummaryModel", None, Option(Array(Dispatcher("console", Map("printEvent" -> false.asInstanceOf[AnyRef])))), Option(10), Option("TestDialcodeUsageSummarizer"), Option(true))
+    val config = JobConfig(Fetcher("local", None, Option(Array(Query(None, None, None, None, None, None, None, None, None, Option("src/test/resources/dialcode-usage-summary/withSameDialcodeAndChannel.log"))))), null, null, "org.ekstep.analytics.model.DialcodeUsageSummaryModel", None, Option(Array(Dispatcher("console", Map("printEvent" -> false.asInstanceOf[AnyRef])))), Option(10), Option("TestDialcodeUsageSummarizer"), Option(true))
     DialcodeUsageSummarizer.main(JSONUtils.serialize(config))(Option(sc));
   }
 }
diff --git a/...atch-models/src/test/scala/org/ekstep/analytics/model/TestDialcodeUsageSummaryModel.scala b/...atch-models/src/test/scala/org/ekstep/analytics/model/TestDialcodeUsageSummaryModel.scala
@@ -6,7 +6,7 @@ case class DialUsageMetric(total_dial_scans: Int, first_scan: Long, last_scan: L
 
 class TestDialcodeUsageSummaryModel extends SparkSpec(null) {
 
-  "DialcodeUsageSummaryModel" should "aggregate based on channel and dialcode" in {
+  "DialcodeUsageSummaryModel" should "aggregate based on channel & dialcode and also ignore events with boolean value for dialcodes field" in {
     val rdd1 = loadFile[V3Event]("src/test/resources/dialcode-usage-summary/withSameDialcodeAndChannel.log")
     val me0 = DialcodeUsageSummaryModel.execute(rdd1, None)
     val events = me0.collect()