Merge pull request #3104 from ProjectSidewalk/develop

v7.10.0

Gruntfile.js

@@ -72,7 +72,8 @@ module.exports = function(grunt) {
                     'public/javascripts/SVValidate/src/zoom/*.js',
                     'public/javascripts/common/Panomarker.js',
                     'public/javascripts/common/UtilitiesSidewalk.js',
-                    'public/javascripts/common/GSVInfoPopover.js'
+                    'public/javascripts/common/GSVInfoPopover.js',
+                    'public/javascripts/common/MissionStartTutorial.js'
                 ],
                 dest: 'public/javascripts/SVValidate/build/SVValidate.js'
             },

LICENSE.md

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2016 Makeability Lab
+Copyright (c) 2016-2022 Makeability Lab
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

app/controllers/AdminController.scala

@@ -13,6 +13,7 @@ import formats.json.LabelFormat
 import formats.json.TaskFormats._
 import formats.json.UserRoleSubmissionFormats._
 import formats.json.LabelFormat._
+import javassist.NotFoundException
 import models.attribute.{GlobalAttribute, GlobalAttributeTable}
 import models.audit.{AuditTaskInteractionTable, AuditTaskTable, AuditedStreetWithTimestamp, InteractionWithLabel}
 import models.daos.slick.DBTableDefinitions.UserTable
@@ -23,12 +24,15 @@ import models.mission.MissionTable
 import models.region.RegionCompletionTable
 import models.street.StreetEdgeTable
 import models.user._
+import models.utils.CommonUtils.METERS_TO_MILES
 import play.api.libs.json.{JsArray, JsError, JsObject, JsValue, Json}
 import play.extras.geojson
 import play.api.mvc.BodyParsers
 import play.api.Play
 import play.api.Play.current
 import play.api.cache.EhCachePlugin
+import play.api.i18n.Messages
+
 import javax.naming.AuthenticationException
 import scala.concurrent.Future
 
@@ -70,8 +74,15 @@ class AdminController @Inject() (implicit val env: Environment[User, SessionAuth
   def userProfile(username: String) = UserAwareAction.async { implicit request =>
     if (isAdmin(request.identity)) {
       UserTable.find(username) match {
-        case Some(user) => Future.successful(Ok(views.html.admin.user("Project Sidewalk", request.identity, Some(user))))
-        case _ => Future.successful(Ok(views.html.admin.user("Project Sidewalk", request.identity)))
+        case Some(user) =>
+          // Get distance audited by the user. Convert meters to km if using metric system, to miles if using IS.
+          val auditedDistance: Float = {
+            val userId: UUID = UUID.fromString(user.userId)
+            if (Messages("measurement.system") == "metric") AuditTaskTable.getDistanceAudited(userId) / 1000F
+            else AuditTaskTable.getDistanceAudited(userId) * METERS_TO_MILES
+          }
+          Future.successful(Ok(views.html.admin.user("Project Sidewalk", request.identity.get, user, auditedDistance)))
+        case _ => Future.failed(new NotFoundException("Username not found."))
       }
     } else {
       Future.failed(new AuthenticationException("User is not an administrator"))
@@ -268,7 +279,7 @@ class AdminController @Inject() (implicit val env: Environment[User, SessionAuth
           }
           val featureCollection = Json.obj("type" -> "FeatureCollection", "features" -> features)
           Future.successful(Ok(featureCollection))
-        case _ => Future.successful(Ok(views.html.admin.user("Project Sidewalk", request.identity)))
+        case _ => Future.failed(new NotFoundException("Username not found."))
       }
     } else {
       Future.failed(new AuthenticationException("User is not an administrator"))
@@ -295,7 +306,7 @@ class AdminController @Inject() (implicit val env: Environment[User, SessionAuth
           }
           val featureCollection = Json.obj("type" -> "FeatureCollection", "features" -> features)
           Future.successful(Ok(featureCollection))
-        case _ => Future.successful(Ok(views.html.admin.user("Project Sidewalk", request.identity)))
+        case _ => Future.failed(new NotFoundException("Username not found."))
       }
     } else {
       Future.failed(new AuthenticationException("User is not an administrator"))
@@ -322,7 +333,7 @@ class AdminController @Inject() (implicit val env: Environment[User, SessionAuth
         case Some(user) =>
           val tasksWithLabels = AuditTaskTable.selectTasksWithLabels(UUID.fromString(user.userId)).map(x => Json.toJson(x))
           Future.successful(Ok(JsArray(tasksWithLabels)))
-        case _ => Future.successful(Ok(views.html.admin.user("Project Sidewalk", request.identity)))
+        case _ => Future.failed(new NotFoundException("Username not found."))
       }
     } else {
       Future.failed(new AuthenticationException("User is not an administrator"))

app/controllers/TaskController.scala

@@ -8,8 +8,10 @@ import com.mohiva.play.silhouette.api.{Environment, Silhouette}
 import com.mohiva.play.silhouette.impl.authenticators.SessionAuthenticator
 import com.vividsolutions.jts.geom._
 import controllers.headers.ProvidesHeader
+import controllers.helper.ControllerUtils.sendSciStarterContributions
 import formats.json.TaskSubmissionFormats._
 import models.amt.AMTAssignmentTable
+import models.audit.AuditTaskInteractionTable.secondsAudited
 import models.audit._
 import models.daos.slick.DBTableDefinitions.{DBUser, UserTable}
 import models.gsv.{GSVData, GSVDataTable, GSVLink, GSVLinkTable}
@@ -19,9 +21,12 @@ import models.region._
 import models.street.StreetEdgePriorityTable.streetPrioritiesFromIds
 import models.street.{StreetEdgePriority, StreetEdgePriorityTable}
 import models.user.{User, UserCurrentRegionTable}
-import play.api.Logger
+import models.utils.CommonUtils.ordered
+import play.api.Play.current
+import play.api.{Logger, Play}
 import play.api.libs.json._
 import play.api.mvc._
+import scala.collection.mutable.ListBuffer
 import scala.concurrent.Future
 
 /**
@@ -174,6 +179,7 @@ class TaskController @Inject() (implicit val env: Environment[User, SessionAuthe
    * Helper function that updates database with all data submitted through the audit page.
    */
   def processAuditTaskSubmissions(submission: Seq[AuditTaskSubmission], remoteAddress: String, identity: Option[User]) = {
+    var newLabels: ListBuffer[(Int, Timestamp)] = ListBuffer()
     val returnValues: Seq[TaskPostReturnValue] = for (data <- submission) yield {
       val userOption: Option[User] = identity
       val streetEdgeId: Int = data.auditTask.streetEdgeId
@@ -259,10 +265,13 @@ class TaskController @Inject() (implicit val env: Environment[User, SessionAuthe
               }
             }
 
-            LabelTable.save(Label(0, auditTaskId, missionId, label.gsvPanoramaId, labelTypeId,
+            val newLabelId: Int = LabelTable.save(Label(0, auditTaskId, missionId, label.gsvPanoramaId, labelTypeId,
               label.photographerHeading, label.photographerPitch, label.panoramaLat, label.panoramaLng, label.deleted,
               label.temporaryLabelId, timeCreated, label.tutorial, calculatedStreetEdgeId, 0, 0, 0, None,
               label.severity, label.temporary, label.description))
+
+            newLabels += ((newLabelId, timeCreated))
+            newLabelId
         }
 
         // Insert label points.
@@ -347,6 +356,14 @@ class TaskController @Inject() (implicit val env: Environment[User, SessionAuthe
       TaskPostReturnValue(auditTaskId, data.auditTask.streetEdgeId, possibleNewMission, switchToValidation, updatedStreets)
     }
 
+    // Send contributions to SciStarter so that it can be recorded in their user dashboard there.
+    val eligibleUser: Boolean = List("Registered", "Administrator", "Owner").contains(identity.get.role.getOrElse(""))
+    val envType: String = Play.configuration.getString("environment-type").get
+    if (newLabels.nonEmpty && envType == "prod" && eligibleUser) {
+      val timeSpent: Float = secondsAudited(identity.get.userId.toString, newLabels.map(_._1).min, newLabels.map(_._2).max)
+      val scistarterResponse: Future[Int] = sendSciStarterContributions(identity.get.email, newLabels.length, timeSpent)
+    }
+
     Future.successful(Ok(Json.obj(
       "audit_task_id" -> returnValues.head.auditTaskId,
       "street_edge_id" -> returnValues.head.streetEdgeId,

app/controllers/UserProfileController.scala

@@ -10,13 +10,14 @@ import com.vividsolutions.jts.geom.Coordinate
 import controllers.headers.ProvidesHeader
 import formats.json.LabelFormat.labelMetadataUserDashToJson
 import models.audit.{AuditTaskTable, StreetEdgeWithAuditStatus}
-import models.mission.MissionTable
 import models.user.UserOrgTable
 import models.label.{LabelLocation, LabelTable, LabelValidationTable}
 import models.user.{User, WebpageActivity, WebpageActivityTable}
+import models.utils.CommonUtils.METERS_TO_MILES
 import play.api.libs.json.{JsObject, JsValue, Json}
 import play.extras.geojson
 import play.api.i18n.Messages
+
 import scala.concurrent.Future
 
 /**
@@ -39,10 +40,10 @@ class UserProfileController @Inject() (implicit val env: Environment[User, Sessi
       val timestamp: Timestamp = new Timestamp(Instant.now.toEpochMilli)
       val ipAddress: String = request.remoteAddress
       WebpageActivityTable.save(WebpageActivity(0, user.userId.toString, ipAddress, "Visit_UserDashboard", timestamp))
-      // Get distance audited by the user. If using metric units, convert from miles to kilometers.
+      // Get distance audited by the user. Convert meters to km if using metric system, to miles if using IS.
       val auditedDistance: Float = {
-        if (Messages("measurement.system") == "metric") MissionTable.getDistanceAudited(user.userId) * 1.60934.toFloat
-        else MissionTable.getDistanceAudited(user.userId)
+        if (Messages("measurement.system") == "metric") AuditTaskTable.getDistanceAudited(user.userId) / 1000F
+        else AuditTaskTable.getDistanceAudited(user.userId) * METERS_TO_MILES
       }
       Future.successful(Ok(views.html.userProfile(s"Project Sidewalk", Some(user), auditedDistance)))
     }
@@ -200,10 +201,10 @@ class UserProfileController @Inject() (implicit val env: Environment[User, Sessi
     request.identity match {
       case Some(user) =>
         val userId: UUID = user.userId
-        // Get distance audited by the user. If using metric units, convert from miles to kilometers.
+        // Get distance audited by the user. Convert meters to km if using metric system, to miles if using IS.
         val auditedDistance: Float = {
-          if (Messages("measurement.system") == "metric") MissionTable.getDistanceAudited(userId) * 1.60934.toFloat
-          else MissionTable.getDistanceAudited(userId)
+          if (Messages("measurement.system") == "metric") AuditTaskTable.getDistanceAudited(userId) / 1000F
+          else AuditTaskTable.getDistanceAudited(userId) * METERS_TO_MILES
         }
         Future.successful(Ok(Json.obj(
           "distance_audited" -> auditedDistance,

app/controllers/ValidationTaskController.scala

@@ -6,6 +6,7 @@ import javax.inject.Inject
 import com.mohiva.play.silhouette.api.{Environment, Silhouette}
 import com.mohiva.play.silhouette.impl.authenticators.SessionAuthenticator
 import controllers.headers.ProvidesHeader
+import controllers.helper.ControllerUtils.sendSciStarterContributions
 import formats.json.ValidationTaskSubmissionFormats._
 import models.amt.AMTAssignmentTable
 import models.label._
@@ -14,12 +15,13 @@ import models.mission.{Mission, MissionTable}
 import models.user.{User, UserStatTable}
 import models.validation._
 import play.api.libs.json._
-import play.api.Logger
+import play.api.{Logger, Play}
 import play.api.mvc._
 import scala.concurrent.Future
 import scala.collection.mutable.ListBuffer
 import formats.json.CommentSubmissionFormats._
 import formats.json.LabelFormat
+import play.api.Play.current
 import java.time.Instant
 
 /**
@@ -97,6 +99,16 @@ class ValidationTaskController @Inject() (implicit val env: Environment[User, Se
       }
     }
 
+    // Send contributions to SciStarter so that it can be recorded in their user dashboard there.
+    val labels: Seq[LabelValidationSubmission] = submission.flatMap(_.labels)
+    val eligibleUser: Boolean = List("Registered", "Administrator", "Owner").contains(identity.get.role.getOrElse(""))
+    val envType: String = Play.configuration.getString("environment-type").get
+    if (labels.nonEmpty && envType == "prod" && eligibleUser) {
+      // Cap time for each validation at 1 minute.
+      val timeSpent: Float = labels.map(l => Math.min(l.endTimestamp - l.startTimestamp, 60000)).sum / 1000F
+      val scistarterResponse: Future[Int] = sendSciStarterContributions(identity.get.email, labels.length, timeSpent)
+    }
+
     // If this user is a turker who has just finished 3 validation missions, switch them to auditing.
     val switchToAuditing = userOption.isDefined &&
       userOption.get.role.getOrElse("") == "Turker" &&

app/controllers/helper/ControllerUtils.scala

@@ -1,11 +1,21 @@
 package controllers.helper
 
-import play.api.Play
+import play.api.{Logger, Play}
 import play.api.Play.current
 import play.api.mvc.Request
+import scala.concurrent.{ExecutionContext, Future}
 import scala.util.matching.Regex
+import org.apache.http.NameValuePair
+import org.apache.http.client.entity.UrlEncodedFormEntity
+import org.apache.http.client.methods.HttpPost
+import org.apache.http.impl.client.DefaultHttpClient
+import org.apache.http.message.BasicNameValuePair
+import java.io.InputStream
+import java.util
 
 object ControllerUtils {
+    implicit val context: ExecutionContext = play.api.libs.concurrent.Execution.Implicits.defaultContext
+
     /**
      * Returns true if the user is on mobile, false if the user is not on mobile.
      */
@@ -18,4 +28,45 @@ object ControllerUtils {
             }
         })
     }
+
+    def sha256Hash(text: String) : String = String.format("%064x", new java.math.BigInteger(1, java.security.MessageDigest.getInstance("SHA-256").digest(text.getBytes("UTF-8"))))
+
+    /**
+     * Send a POST request to SciStarter to record the user's contributions.
+     *
+     * @param email The email address of the user who contributed. Will be hashed in POST request.
+     * @param contributions Number of contributions. Either number of labels created or number of labels validated.
+     * @param timeSpent Total time spent on those contributions.
+     * @return Response code from the API request.
+     */
+    def sendSciStarterContributions(email: String, contributions: Int, timeSpent: Float): Future[Int] = Future {
+        // Get the SciStarter API key, throw an error if not found.
+        val apiKey: Option[String] = Play.configuration.getString("scistarter-api-key")
+        if (apiKey.isEmpty) {
+            Logger.error("SciStarter API key not found.")
+            throw new Exception("SciStarter API key not found.")
+        }
+
+        // Set up the URL and POST request data with hashed email and amount of contribution.
+        val hashedEmail: String = sha256Hash(email)
+        val url: String = s"https://scistarter.org/api/participation/hashed/project-sidewalk?key=${apiKey.get}"
+        val post: HttpPost = new HttpPost(url)
+        val client: DefaultHttpClient = new DefaultHttpClient
+        val nameValuePairs = new util.ArrayList[NameValuePair](1)
+        nameValuePairs.add(new BasicNameValuePair("hashed", hashedEmail));
+        nameValuePairs.add(new BasicNameValuePair("type", "classification"));
+        nameValuePairs.add(new BasicNameValuePair("count", contributions.toString));
+        nameValuePairs.add(new BasicNameValuePair("duration", (timeSpent / contributions).toString));
+        post.setEntity(new UrlEncodedFormEntity(nameValuePairs));
+
+        // Make API call, logging any errors.
+        try {
+            val response = client.execute(post)
+            response.getStatusLine.getStatusCode
+        } catch {
+            case e: Exception =>
+                Logger.warn(e.getMessage)
+                throw e
+        }
+    }
 }

app/models/audit/AuditTaskInteractionTable.scala

@@ -5,6 +5,7 @@ import models.utils.MyPostgresDriver.simple._
 import play.api.Play.current
 import play.api.libs.json.{JsObject, Json}
 import play.extras.geojson
+import java.sql.Timestamp
 import scala.slick.jdbc.{GetResult, StaticQuery => Q}
 import scala.slick.lifted.ForeignKeyQuery
 
@@ -235,4 +236,39 @@ object AuditTaskInteractionTable {
          |WHERE diff < '00:05:00.000' AND diff > '00:00:00.000';""".stripMargin
     ).first
   }
+
+  /**
+   * Calculate the time spent auditing by the given user for a specified time range, starting at a label creation time.
+   *
+   * To do this, we take the important events from the audit_task_interaction table, get the difference between each
+   * consecutive timestamp, filter out the timestamp diffs that are greater than five minutes, and then sum those time
+   * diffs.
+   *
+   * @param userId
+   * @param timeRangeStartLabelId Label_id for the label whose `time_created` field marks the start of the time range.
+   * @param timeRangeEnd A timestamp representing the end of the time range; should be the time when a label was placed.
+   * @return
+   */
+  def secondsAudited(userId: String, timeRangeStartLabelId: Int, timeRangeEnd: Timestamp): Float = db.withSession { implicit session =>
+    Q.queryNA[Float](
+      s"""SELECT extract( epoch FROM SUM(diff) ) AS seconds_contributed
+         |FROM (
+         |    SELECT (timestamp - LAG(timestamp, 1) OVER(PARTITION BY user_id ORDER BY timestamp)) AS diff
+         |    FROM audit_task_interaction
+         |    INNER JOIN audit_task ON audit_task.audit_task_id = audit_task_interaction.audit_task_id
+         |    WHERE action IN ('ViewControl_MouseDown', 'LabelingCanvas_MouseDown')
+         |        AND audit_task.user_id = '$userId'
+         |        AND audit_task_interaction.timestamp < '$timeRangeEnd'
+         |        AND audit_task_interaction.timestamp > (
+         |            SELECT COALESCE(MAX(time_created), TIMESTAMP 'epoch')
+         |            FROM label
+         |            INNER JOIN audit_task ON label.audit_task_id = audit_task.audit_task_id
+         |            WHERE audit_task.user_id = '$userId'
+         |                AND label.label_id < $timeRangeStartLabelId
+         |    )
+         |) "time_diffs"
+         |WHERE diff < '00:05:00.000' AND diff > '00:00:00.000';""".stripMargin
+    ).first
+  }
+
 }

app/models/audit/AuditTaskTable.scala

@@ -351,6 +351,17 @@ object AuditTaskTable {
     _streetEdges.list.groupBy(_.streetEdgeId).map(_._2.head).toList
   }
 
+  /**
+    * Gets total distance audited by a user in meters.
+    */
+  def getDistanceAudited(userId: UUID): Float = db.withSession { implicit session =>
+    completedTasks
+      .filter(_.userId === userId.toString)
+      .innerJoin(streetEdges).on(_.streetEdgeId === _.streetEdgeId)
+      .map(_._2.geom.transform(26918).length)
+      .sum.run.getOrElse(0F)
+  }
+
   /**
     * Get the sum of the line distance of all streets in the region that the user has not audited.
     */

app/models/mission/MissionTable.scala

@@ -98,7 +98,6 @@ object MissionTable {
   }
   val validationMissions = missions.filter(_.missionTypeId === validationMissionTypeId)
 
-  val METERS_TO_MILES: Float = 0.000621371F
 
   // Distances for first few missions: 250 ft, 250 ft, then 500 ft for all remaining.
   val distancesForFirstAuditMissions: List[Float] = List(76.2F, 76.2F)
@@ -329,15 +328,6 @@ object MissionTable {
     missions.filter(m => m.userId === userId.toString && m.completed).map(_.pay).sum.run.getOrElse(0.0D)
   }
 
-  /**
-    * Gets total distance audited by a user in miles.
-    *
-    * @param userId the UUID of the user
-    */
-  def getDistanceAudited(userId: UUID): Float = db.withSession { implicit session =>
-    missions.filter(_.userId === userId.toString).map(_.distanceProgress).sum.run.getOrElse(0F) * METERS_TO_MILES
-  }
-
   /**
     * Gets meters audited by a user in their current mission, if it exists.
     */