From e23bbda6c42487060e08a28771bca95fd35c7c17 Mon Sep 17 00:00:00 2001 From: Jake Lee Kennedy Date: Wed, 29 Jan 2025 17:01:04 +0000 Subject: [PATCH 1/3] delete old api code --- commercial/app/AppLoader.scala | 11 - commercial/app/CommercialLifecycle.scala | 172 -------------- commercial/app/RefreshJob.scala | 35 --- commercial/app/controllers/Analytics.scala | 32 --- .../controllers/CommercialControllers.scala | 9 - .../app/controllers/JobsController.scala | 26 --- commercial/app/controllers/Multi.scala | 66 ------ .../controllers/TrafficDriverController.scala | 52 ----- .../controllers/TravelOffersController.scala | 26 --- commercial/app/model/data/Firehose.scala | 30 --- commercial/app/model/feeds/FeedFetcher.scala | 85 ------- commercial/app/model/feeds/FeedMetaData.scala | 29 --- commercial/app/model/feeds/FeedParser.scala | 49 ---- commercial/app/model/feeds/FeedReader.scala | 128 ---------- commercial/app/model/feeds/S3FeedStore.scala | 13 -- .../app/model/merchandise/Merchandise.scala | 221 ------------------ .../model/merchandise/MerchandiseAgent.scala | 31 --- .../app/model/merchandise/TrafficDriver.scala | 36 --- .../model/merchandise/jobs/Industries.scala | 56 ----- .../model/merchandise/jobs/JobsAgent.scala | 46 ---- .../app/model/merchandise/jobs/JobsFeed.scala | 35 --- .../travel/TravelOffersAgent.scala | 36 --- .../merchandise/travel/TravelOffersApi.scala | 33 --- 23 files changed, 1257 deletions(-) delete mode 100644 commercial/app/CommercialLifecycle.scala delete mode 100644 commercial/app/RefreshJob.scala delete mode 100644 commercial/app/controllers/Analytics.scala delete mode 100644 commercial/app/controllers/JobsController.scala delete mode 100644 commercial/app/controllers/Multi.scala delete mode 100644 commercial/app/controllers/TrafficDriverController.scala delete mode 100644 commercial/app/controllers/TravelOffersController.scala delete mode 100644 commercial/app/model/data/Firehose.scala delete mode 100644 commercial/app/model/feeds/FeedFetcher.scala delete mode 100644 commercial/app/model/feeds/FeedMetaData.scala delete mode 100644 commercial/app/model/feeds/FeedParser.scala delete mode 100644 commercial/app/model/feeds/FeedReader.scala delete mode 100644 commercial/app/model/feeds/S3FeedStore.scala delete mode 100644 commercial/app/model/merchandise/Merchandise.scala delete mode 100644 commercial/app/model/merchandise/MerchandiseAgent.scala delete mode 100644 commercial/app/model/merchandise/TrafficDriver.scala delete mode 100644 commercial/app/model/merchandise/jobs/Industries.scala delete mode 100644 commercial/app/model/merchandise/jobs/JobsAgent.scala delete mode 100644 commercial/app/model/merchandise/jobs/JobsFeed.scala delete mode 100644 commercial/app/model/merchandise/travel/TravelOffersAgent.scala delete mode 100644 commercial/app/model/merchandise/travel/TravelOffersApi.scala diff --git a/commercial/app/AppLoader.scala b/commercial/app/AppLoader.scala index 994dcd796627..84718599b128 100644 --- a/commercial/app/AppLoader.scala +++ b/commercial/app/AppLoader.scala @@ -1,12 +1,8 @@ import org.apache.pekko.actor.{ActorSystem => PekkoActorSystem} import app.{FrontendApplicationLoader, FrontendBuildInfo, FrontendComponents} import com.softwaremill.macwire._ -import commercial.CommercialLifecycle import commercial.controllers.{CommercialControllers, HealthCheck} import commercial.model.capi.CapiAgent -import commercial.model.feeds.{FeedsFetcher, FeedsParser} -import commercial.model.merchandise.jobs.{Industries, JobsAgent} -import commercial.model.merchandise.travel.TravelOffersAgent import common.CloudWatchMetricsLifecycle import common.dfp.DfpAgentLifecycle import conf.switches.SwitchboardLifecycle @@ -38,13 +34,7 @@ trait CommercialServices { lazy val capiHttpClient: HttpClient = wire[CapiHttpClient] lazy val contentApiClient = wire[ContentApiClient] - lazy val travelOffersAgent = wire[TravelOffersAgent] - lazy val jobsAgent = wire[JobsAgent] lazy val capiAgent = wire[CapiAgent] - lazy val industries = wire[Industries] - - lazy val feedsFetcher = wire[FeedsFetcher] - lazy val feedsParser = wire[FeedsParser] } trait AppComponents extends FrontendComponents with CommercialControllers with CommercialServices { @@ -53,7 +43,6 @@ trait AppComponents extends FrontendComponents with CommercialControllers with C lazy val healthCheck = wire[HealthCheck] override lazy val lifecycleComponents = List( - wire[CommercialLifecycle], wire[DfpAgentLifecycle], wire[SwitchboardLifecycle], wire[CloudWatchMetricsLifecycle], diff --git a/commercial/app/CommercialLifecycle.scala b/commercial/app/CommercialLifecycle.scala deleted file mode 100644 index 958c3ae3da01..000000000000 --- a/commercial/app/CommercialLifecycle.scala +++ /dev/null @@ -1,172 +0,0 @@ -package commercial - -import java.util.concurrent.Executors -import commercial.model.merchandise.jobs.Industries -import app.LifecycleComponent -import commercial.model.feeds._ -import common.LoggingField._ -import common.{GuLogging, JobScheduler, PekkoAsync} -import metrics.MetricUploader -import play.api.inject.ApplicationLifecycle - -import scala.concurrent.{ExecutionContext, ExecutionContextExecutorService, Future} -import scala.util.control.NonFatal - -object CommercialMetrics { - val metrics = MetricUploader("Commercial") -} - -class CommercialLifecycle( - appLifecycle: ApplicationLifecycle, - jobs: JobScheduler, - pekkoAsync: PekkoAsync, - feedsFetcher: FeedsFetcher, - feedsParser: FeedsParser, - industries: Industries, -) extends LifecycleComponent - with GuLogging { - - // This class does work that should be kept separate from the EC used to serve requests - implicit private val ec: ExecutionContextExecutorService = ExecutionContext.fromExecutorService( - Executors.newFixedThreadPool(10), - ) - - appLifecycle.addStopHook { () => - Future { - stop() - } - } - - private def toLogFields( - feed: String, - action: String, - success: Boolean, - maybeDuration: Option[Long] = None, - maybeSize: Option[Int] = None, - ) = { - - val prefix = "commercial-feed" - - val defaultFields: List[LogField] = List( - prefix -> feed, - s"$prefix-action" -> action, - s"$prefix-result" -> (if (success) "success" else "failure"), - ) - - val optionalFields: List[Option[LogField]] = List( - maybeDuration map (LogFieldLong(s"$prefix-duration", _)), - maybeSize map (LogFieldLong(s"$prefix-size", _)), - ) - - defaultFields ::: optionalFields.flatten - } - - private val refreshJobs: List[RefreshJob] = List( - new IndustriesRefresh(industries, jobs), - ) - - override def start(): Unit = { - - def delayedStartSchedule(delayedStart: Int = 0, refreshStep: Int = 15) = s"0 $delayedStart/$refreshStep * * * ?" - - def fetchFeed(fetcher: FeedFetcher): Future[Unit] = { - - val feedName = fetcher.feedMetaData.name - - val msgPrefix = s"Fetching $feedName feed" - log.info(s"$msgPrefix from ${fetcher.feedMetaData.url} ...") - val eventualResponse = fetcher.fetch() - eventualResponse.failed.foreach { - case e: SwitchOffException => - log.warn(s"$msgPrefix failed: ${e.getMessage}") - case NonFatal(e) => - logErrorWithCustomFields(s"$msgPrefix failed: ${e.getMessage}", e, toLogFields(feedName, "fetch", false)) - } - eventualResponse.foreach { response => - S3FeedStore.put(feedName, response.feed) - logInfoWithCustomFields( - s"$msgPrefix succeeded in ${response.duration}", - toLogFields(feedName, "fetch", true, Some(response.duration.toSeconds)), - ) - } - eventualResponse.map(_ => ()) - } - - def parseFeed[T](parser: FeedParser[T]): Future[Unit] = { - - val feedName = parser.feedMetaData.name - - val msgPrefix = s"Parsing $feedName feed" - log.info(s"$msgPrefix ...") - val parsedFeed = parser.parse(S3FeedStore.get(parser.feedMetaData.name)) - parsedFeed.failed.foreach { - case e: SwitchOffException => - log.warn(s"$msgPrefix failed: ${e.getMessage}") - case NonFatal(e) => - logErrorWithCustomFields(s"$msgPrefix failed: ${e.getMessage}", e, toLogFields(feedName, "parse", false)) - } - parsedFeed.foreach { feed => - logInfoWithCustomFields( - s"$msgPrefix succeeded: parsed ${feed.contents.size} $feedName in ${feed.parseDuration}", - toLogFields(feedName, "parse", true, Some(feed.parseDuration.toSeconds), Some(feed.contents.size)), - ) - } - parsedFeed.map(_ => ()) - } - - def mkJobName(feedName: String, task: String): String = s"${feedName.replaceAll("/", "-")}-$task-job" - - val jobRefreshStep = 15 - - for (fetcher <- feedsFetcher.all) { - val feedName = fetcher.feedMetaData.name - val jobName = mkJobName(feedName, "fetch") - jobs.deschedule(jobName) - jobs.scheduleEveryNMinutes(jobName, jobRefreshStep) { - fetchFeed(fetcher) - } - } - - for (parser <- feedsParser.all) { - val feedName = parser.feedMetaData.name - val jobName = mkJobName(feedName, "parse") - jobs.deschedule(jobName) - jobs.scheduleEveryNMinutes(jobName, jobRefreshStep) { - parseFeed(parser) - } - } - - val refreshJobDelay = jobRefreshStep / refreshJobs.size - - refreshJobs.zipWithIndex foreach { case (job, i) => - job.start(delayedStartSchedule(delayedStart = i * refreshJobDelay, refreshStep = jobRefreshStep)) - } - - pekkoAsync.after1s { - - industries.refresh().failed.foreach { case NonFatal(e) => - log.warn(s"Failed to refresh job industries: ${e.getMessage}") - } - - for (fetcher <- feedsFetcher.all) { - fetchFeed(fetcher) - } - - for (parser <- feedsParser.all) { - parseFeed(parser) - } - } - } - - def stop(): Unit = { - refreshJobs foreach (_.stop()) - - for (fetcher <- feedsFetcher.all) { - jobs.deschedule(s"${fetcher.feedMetaData.name}FetchJob") - } - - for (parser <- feedsParser.all) { - jobs.deschedule(s"${parser.feedMetaData.name}ParseJob") - } - } -} diff --git a/commercial/app/RefreshJob.scala b/commercial/app/RefreshJob.scala deleted file mode 100644 index 381c04f0f658..000000000000 --- a/commercial/app/RefreshJob.scala +++ /dev/null @@ -1,35 +0,0 @@ -package commercial - -import common.{JobScheduler, GuLogging} -import commercial.model.merchandise.jobs.Industries - -import scala.concurrent.ExecutionContext - -trait RefreshJob extends GuLogging { - - def name: String - def jobs: JobScheduler - - protected def refresh(): Unit - - def start(schedule: String): Unit = { - jobs.deschedule(s"${name}RefreshJob") - - log.info(s"$name refresh on schedule $schedule") - jobs.schedule(s"${name}RefreshJob", schedule) { - refresh() - } - } - - def stop(): Unit = { - jobs.deschedule(s"${name}RefreshJob") - } -} - -class IndustriesRefresh(industries: Industries, val jobs: JobScheduler)(implicit executionContext: ExecutionContext) - extends RefreshJob { - - val name: String = "Industries" - - def refresh(): Unit = industries.refresh() -} diff --git a/commercial/app/controllers/Analytics.scala b/commercial/app/controllers/Analytics.scala deleted file mode 100644 index c68d7ea10095..000000000000 --- a/commercial/app/controllers/Analytics.scala +++ /dev/null @@ -1,32 +0,0 @@ -package commercial.controllers - -import commercial.model.data.Firehose -import conf.Configuration.environment.isProd -import conf.switches.Switch -import model.Cached.WithoutRevalidationResult -import model.{CacheTime, Cached, TinyResponse} -import play.api.Logger -import play.api.libs.json.Json -import play.api.libs.json.Json.prettyPrint -import play.api.mvc._ - -import scala.concurrent.ExecutionContext -import scala.util.control.NonFatal - -object Analytics extends Results { - - def storeJsonBody[A](switch: Switch, streamName: => String, log: Logger)( - analytics: String, - )(implicit ec: ExecutionContext, request: Request[A]): Result = { - if (switch.isSwitchedOn) { - if (isProd) { - val result = Firehose.stream(streamName)(analytics) - result.failed foreach { case NonFatal(e) => - log.error(s"Failed to put '$analytics'", e) - } - } else log.info(prettyPrint(Json.parse(analytics))) - TinyResponse.noContent() - } else - Cached(CacheTime.NotFound)(WithoutRevalidationResult(NotFound)) - } -} diff --git a/commercial/app/controllers/CommercialControllers.scala b/commercial/app/controllers/CommercialControllers.scala index 9c6e973876bc..0cf2d17cefc6 100644 --- a/commercial/app/controllers/CommercialControllers.scala +++ b/commercial/app/controllers/CommercialControllers.scala @@ -2,8 +2,6 @@ package commercial.controllers import com.softwaremill.macwire._ import commercial.model.capi.CapiAgent -import commercial.model.merchandise.jobs.JobsAgent -import commercial.model.merchandise.travel.TravelOffersAgent import contentapi.ContentApiClient import model.ApplicationContext import play.api.mvc.ControllerComponents @@ -11,17 +9,10 @@ import play.api.mvc.ControllerComponents trait CommercialControllers { def contentApiClient: ContentApiClient def capiAgent: CapiAgent - def travelOffersAgent: TravelOffersAgent - def jobsAgent: JobsAgent def controllerComponents: ControllerComponents implicit def appContext: ApplicationContext lazy val contentApiOffersController = wire[ContentApiOffersController] - lazy val creativeTestPage = wire[CreativeTestPage] lazy val hostedContentController = wire[HostedContentController] - lazy val jobsController = wire[JobsController] - lazy val multi = wire[Multi] - lazy val travelOffersController = wire[TravelOffersController] - lazy val trafficDriverController = wire[TrafficDriverController] lazy val piggybackPixelController = wire[PiggybackPixelController] lazy val adsDotTextFileController = wire[AdsDotTextViewController] lazy val passbackController = wire[PassbackController] diff --git a/commercial/app/controllers/JobsController.scala b/commercial/app/controllers/JobsController.scala deleted file mode 100644 index a62afb453c17..000000000000 --- a/commercial/app/controllers/JobsController.scala +++ /dev/null @@ -1,26 +0,0 @@ -package commercial.controllers - -import common.JsonComponent -import commercial.model.Segment -import commercial.model.merchandise.Job -import commercial.model.merchandise.jobs.JobsAgent -import model.Cached -import play.api.mvc._ -import scala.concurrent.duration._ - -class JobsController(jobsAgent: JobsAgent, val controllerComponents: ControllerComponents) - extends BaseController - with implicits.Requests { - - implicit val codec: Codec = Codec.utf_8 - - private def jobSample(specificIds: Seq[String], segment: Segment): Seq[Job] = - (jobsAgent.specificJobs(specificIds) ++ jobsAgent.jobsTargetedAt(segment)).distinct.take(2) - - def getJobs: Action[AnyContent] = - Action { implicit request => - Cached(60.seconds) { - JsonComponent.fromWritable(jobSample(specificIds, segment)) - } - } -} diff --git a/commercial/app/controllers/Multi.scala b/commercial/app/controllers/Multi.scala deleted file mode 100644 index 4cef3ba6e294..000000000000 --- a/commercial/app/controllers/Multi.scala +++ /dev/null @@ -1,66 +0,0 @@ -package commercial.controllers - -import commercial.model.Segment -import commercial.model.merchandise.jobs.JobsAgent -import commercial.model.merchandise.travel.TravelOffersAgent -import commercial.model.merchandise.Merchandise -import common.{JsonComponent} -import model.Cached -import play.api.libs.json.{JsArray, Json} -import play.api.mvc._ - -class Multi( - travelOffersAgent: TravelOffersAgent, - jobsAgent: JobsAgent, - val controllerComponents: ControllerComponents, -) extends BaseController - with implicits.Requests { - - private def multiSample( - offerTypes: Seq[String], - offerIds: Seq[Option[String]], - segment: Segment, - ): Seq[Merchandise] = { - val components: Seq[(String, Option[String])] = offerTypes zip offerIds - - components flatMap { - case ("Job", Some(jobId)) => - jobsAgent.specificJobs(Seq(jobId)) - - case ("Job", None) => - jobsAgent.jobsTargetedAt(segment) - - case ("Travel", Some(travelId)) => - travelOffersAgent.specificTravelOffers(Seq(travelId)) - - case ("Travel", None) => - travelOffersAgent.offersTargetedAt(segment) - - case _ => Nil - } - } - - def getMulti(): Action[AnyContent] = - Action { implicit request => - val offerTypes: Seq[String] = request.getParameters("offerTypes") - - val offerIds: Seq[Option[String]] = request.getParameters("offerIds") map { slotId => - if (slotId.trim.isEmpty) None else Some(slotId) - } - - val contents: Seq[Merchandise] = multiSample(offerTypes, offerIds, segment) - - if (offerTypes.size == contents.size) { - Cached(componentMaxAge) { - JsonComponent.fromWritable(JsArray((offerTypes zip contents).map { case (contentType, content) => - Json.obj( - "type" -> contentType, - "value" -> Json.toJson(content)(Merchandise.merchandiseWrites), - ) - })) - } - } else { - Cached(componentMaxAge)(jsonFormat.nilResult) - } - } -} diff --git a/commercial/app/controllers/TrafficDriverController.scala b/commercial/app/controllers/TrafficDriverController.scala deleted file mode 100644 index 47001564d845..000000000000 --- a/commercial/app/controllers/TrafficDriverController.scala +++ /dev/null @@ -1,52 +0,0 @@ -package commercial.controllers - -import commercial.model.capi.CapiAgent -import commercial.model.merchandise.TrafficDriver -import common.{Edition, ImplicitControllerExecutionContext, JsonComponent, GuLogging} -import contentapi.ContentApiClient -import model.{Cached, ContentType} -import play.api.mvc._ - -import scala.concurrent.Future -import scala.concurrent.duration.DurationInt -import scala.util.control.NonFatal - -class TrafficDriverController( - contentApiClient: ContentApiClient, - capiAgent: CapiAgent, - val controllerComponents: ControllerComponents, -) extends BaseController - with ImplicitControllerExecutionContext - with implicits.Requests - with GuLogging { - - // Request information about the article from cAPI. - private def retrieveContent()(implicit request: Request[AnyContent]): Future[Option[ContentType]] = { - - val content: Future[Option[model.ContentType]] = - capiAgent.contentByShortUrls(specificIds).map(_.headOption) - - content.failed.foreach { case NonFatal(e) => - log.error( - s"Looking up content by short URL failed: ${e.getMessage}", - ) - } - - content - - } - - // Build model from cAPI data and return as JSON, or empty if nothing found. - def renderJson(): Action[AnyContent] = - Action.async { implicit request => - retrieveContent().map { - case None => Cached(componentNilMaxAge) { jsonFormat.nilResult } - case Some(content) => - Cached(60.seconds) { - JsonComponent.fromWritable(TrafficDriver.fromContent(content, Edition(request))) - } - } - - } - -} diff --git a/commercial/app/controllers/TravelOffersController.scala b/commercial/app/controllers/TravelOffersController.scala deleted file mode 100644 index 81c67f7fdcf5..000000000000 --- a/commercial/app/controllers/TravelOffersController.scala +++ /dev/null @@ -1,26 +0,0 @@ -package commercial.controllers - -import commercial.model.Segment -import commercial.model.merchandise.travel.TravelOffersAgent -import common.JsonComponent -import model.Cached -import commercial.model.merchandise.TravelOffer -import play.api.libs.json.Json -import play.api.mvc._ - -import scala.concurrent.duration._ - -class TravelOffersController(travelOffersAgent: TravelOffersAgent, val controllerComponents: ControllerComponents) - extends BaseController - with implicits.Requests { - - private def travelSample(specificIds: Seq[String], segment: Segment): Seq[TravelOffer] = - (travelOffersAgent.specificTravelOffers(specificIds) ++ travelOffersAgent.offersTargetedAt(segment)).distinct - - def getTravel: Action[AnyContent] = - Action { implicit request => - Cached(60.seconds) { - JsonComponent.fromWritable(travelSample(specificIds, segment)) - } - } -} diff --git a/commercial/app/model/data/Firehose.scala b/commercial/app/model/data/Firehose.scala deleted file mode 100644 index 2db4a5fc2c9c..000000000000 --- a/commercial/app/model/data/Firehose.scala +++ /dev/null @@ -1,30 +0,0 @@ -package commercial.model.data - -import java.nio.ByteBuffer -import java.nio.charset.Charset - -import awswrappers.kinesisfirehose._ -import com.amazonaws.services.kinesisfirehose.model.{PutRecordRequest, PutRecordResult, Record} -import com.amazonaws.services.kinesisfirehose.{AmazonKinesisFirehoseAsync, AmazonKinesisFirehoseAsyncClientBuilder} -import conf.Configuration.aws.{mandatoryCredentials, region} - -import scala.concurrent.Future - -object Firehose { - - private lazy val firehose: AmazonKinesisFirehoseAsync = { - AmazonKinesisFirehoseAsyncClientBuilder - .standard() - .withCredentials(mandatoryCredentials) - .withRegion(region) - .build() - } - - private val charset = Charset.forName("UTF-8") - - def stream(streamName: String)(data: String): Future[PutRecordResult] = { - val record = new Record().withData(ByteBuffer.wrap(s"$data\n".getBytes(charset))) - val request = new PutRecordRequest().withDeliveryStreamName(streamName).withRecord(record) - firehose.putRecordFuture(request) - } -} diff --git a/commercial/app/model/feeds/FeedFetcher.scala b/commercial/app/model/feeds/FeedFetcher.scala deleted file mode 100644 index 20d73f06e4ea..000000000000 --- a/commercial/app/model/feeds/FeedFetcher.scala +++ /dev/null @@ -1,85 +0,0 @@ -package commercial.model.feeds - -import java.lang.System.currentTimeMillis - -import conf.Configuration -import play.api.libs.json.{JsArray, Json} -import play.api.libs.ws.{WSClient, WSResponse} - -import scala.concurrent.duration.{Duration, _} -import scala.concurrent.{ExecutionContext, Future} -import scala.language.postfixOps -import scala.util.control.NonFatal - -trait FeedFetcher { - - def wsClient: WSClient - def feedMetaData: FeedMetaData - def fetch()(implicit executionContext: ExecutionContext): Future[FetchResponse] - def fetch(feedMetaData: FeedMetaData)(implicit executionContext: ExecutionContext): Future[FetchResponse] = { - - def body(response: WSResponse): String = response.bodyAsBytes.decodeString(feedMetaData.responseEncoding) - - def contentType(response: WSResponse): String = response.header("Content-Type") getOrElse "application/octet-stream" - - val start = currentTimeMillis() - - val futureResponse = wsClient - .url(feedMetaData.url) - .withQueryStringParameters(feedMetaData.parameters.toSeq: _*) - .withRequestTimeout(feedMetaData.timeout) - .get() - - futureResponse map { response => - if (response.status == 200) { - FetchResponse( - Feed(body(response), contentType(response)), - Duration(currentTimeMillis() - start, MILLISECONDS), - ) - } else { - throw FetchException(response.status, response.statusText) - } - } recoverWith { case NonFatal(e) => - Future.failed(e) - } - } -} - -class SingleFeedFetcher(val wsClient: WSClient)(val feedMetaData: FeedMetaData) extends FeedFetcher { - - def fetch()(implicit executionContext: ExecutionContext): Future[FetchResponse] = { - fetch(feedMetaData) - } -} - -class FeedsFetcher(wsClient: WSClient) { - - private val jobs: Option[FeedFetcher] = { - Configuration.commercial.jobsUrl map { url => - new SingleFeedFetcher(wsClient)(JobsFeedMetaData(url)) - } - } - - private val travelOffers: Option[FeedFetcher] = - Configuration.commercial.travelFeedUrl map { url => - new SingleFeedFetcher(wsClient)(TravelOffersFeedMetaData(url)) - } - - val all: Seq[FeedFetcher] = Seq(travelOffers, jobs).flatten - -} - -object ResponseEncoding { - - val iso88591 = "ISO-8859-1" - val utf8 = "UTF-8" - - val default = iso88591 -} - -case class FetchResponse(feed: Feed, duration: Duration) -case class Feed(content: String, contentType: String) - -final case class FetchException(status: Int, message: String) extends Exception(s"HTTP status $status: $message") - -final case class SwitchOffException(switchName: String) extends Exception(s"$switchName switch is off") diff --git a/commercial/app/model/feeds/FeedMetaData.scala b/commercial/app/model/feeds/FeedMetaData.scala deleted file mode 100644 index 19a80d0b4fb6..000000000000 --- a/commercial/app/model/feeds/FeedMetaData.scala +++ /dev/null @@ -1,29 +0,0 @@ -package commercial.model.feeds - -import commercial.model.feeds.ResponseEncoding.utf8 -import conf.switches.{Switch, Switches} - -import scala.concurrent.duration.{Duration, _} - -sealed trait FeedMetaData { - - def name: String - def url: String - def parameters: Map[String, String] = Map.empty - def timeout: Duration = 2.seconds - def responseEncoding: String = ResponseEncoding.default -} - -case class JobsFeedMetaData(override val url: String) extends FeedMetaData { - - val name = "jobs" - - override val responseEncoding = utf8 -} - -case class TravelOffersFeedMetaData(url: String) extends FeedMetaData { - - override def name: String = "travel-offers" - - override val timeout = 10.seconds -} diff --git a/commercial/app/model/feeds/FeedParser.scala b/commercial/app/model/feeds/FeedParser.scala deleted file mode 100644 index c73a342e0f54..000000000000 --- a/commercial/app/model/feeds/FeedParser.scala +++ /dev/null @@ -1,49 +0,0 @@ -package commercial.model.feeds - -import commercial.model.merchandise.jobs.JobsAgent -import commercial.model.merchandise.travel.TravelOffersAgent -import conf.Configuration -import commercial.model.merchandise.{Job, TravelOffer} - -import scala.concurrent.{ExecutionContext, Future} -import scala.concurrent.duration.Duration - -sealed trait FeedParser[+T] { - - def feedMetaData: FeedMetaData - def parse(feedContent: => Option[String]): Future[ParsedFeed[T]] -} - -class FeedsParser( - travelOffersAgent: TravelOffersAgent, - jobsAgent: JobsAgent, -)(implicit executionContext: ExecutionContext) { - - private val jobs: Option[FeedParser[Job]] = { - Configuration.commercial.jobsUrl map { url => - new FeedParser[Job] { - - val feedMetaData = JobsFeedMetaData(url) - - def parse(feedContent: => Option[String]) = jobsAgent.refresh(feedMetaData, feedContent) - } - } - } - - private val travelOffers: Option[FeedParser[TravelOffer]] = { - Configuration.commercial.travelFeedUrl map { url => - new FeedParser[TravelOffer] { - - val feedMetaData = TravelOffersFeedMetaData(url) - - def parse(feedContent: => Option[String]) = travelOffersAgent.refresh(feedMetaData, feedContent) - } - } - } - - val all = Seq(jobs, travelOffers).flatten -} - -case class ParsedFeed[+T](contents: Seq[T], parseDuration: Duration) - -case class MissingFeedException(feedName: String) extends Exception(s"Missing feed: $feedName") diff --git a/commercial/app/model/feeds/FeedReader.scala b/commercial/app/model/feeds/FeedReader.scala deleted file mode 100644 index 0e70ea738041..000000000000 --- a/commercial/app/model/feeds/FeedReader.scala +++ /dev/null @@ -1,128 +0,0 @@ -package commercial.model.feeds - -import commercial.CommercialMetrics -import common.GuLogging -import conf.switches.Switch -import play.api.libs.json.{JsValue, Json} -import play.api.libs.ws.{WSClient, WSRequest, WSResponse, WSSignatureCalculator} - -import scala.concurrent.duration.Duration -import scala.concurrent.{ExecutionContext, Future} -import scala.util.control.NonFatal -import scala.util.{Failure, Success, Try} -import scala.xml.{Elem, XML} - -class FeedReader(wsClient: WSClient) extends GuLogging { - - def read[T]( - request: FeedRequest, - signature: Option[WSSignatureCalculator] = None, - validResponseStatuses: Seq[Int] = Seq(200), - )(parse: String => T)(implicit ec: ExecutionContext): Future[T] = { - - def readUrl(): Future[T] = { - - def recordLoad(duration: Long): Unit = { - val feedName: String = request.feedName.toLowerCase.replaceAll("\\s+", "-") - val key: String = s"$feedName-feed-load-time" - CommercialMetrics.metrics.put(Map(s"$key" -> duration.toDouble)) - } - - val start: Long = System.currentTimeMillis - - val requestHolder: WSRequest = { - val unsignedRequestHolder: WSRequest = wsClient - .url(request.url) - .withQueryStringParameters(request.parameters.toSeq: _*) - .withRequestTimeout(request.timeout) - signature.foldLeft(unsignedRequestHolder) { (soFar, calc) => - soFar.sign(calc) - } - } - val futureResponse: Future[WSResponse] = requestHolder.get() - - val contents: Future[T] = futureResponse map { response => - response.status match { - case status if validResponseStatuses.contains(status) => - recordLoad(System.currentTimeMillis - start) - val body: String = response.bodyAsBytes.decodeString(request.responseEncoding) - - Try(parse(body)) match { - case Success(parsedBody) => parsedBody - case Failure(throwable) => - log.error(s"Could not parse body: (Body: $body)", throwable) - throw throwable - } - - case invalid => - log.error(s"Invalid status code: ${response.status} (Response StatusText: ${response.statusText}") - throw FeedReadException(request, response.status, response.statusText) - } - } - - contents.failed.foreach { case NonFatal(e) => - log.error(s"Failed to fetch feed contents.", e) - recordLoad(-1) - } - - contents - } - - readUrl() - } - - def readSeq[T](request: FeedRequest)(parse: String => Seq[T])(implicit ec: ExecutionContext): Future[Seq[T]] = { - val contents = read(request)(parse) - - contents foreach { items => - log.info(s"Loaded ${items.size} ${request.feedName} from ${request.url}") - } - - contents.failed.foreach { - case e: FeedSwitchOffException => log.warn(e.getMessage) - case NonFatal(e) => log.error(s"Failed to read feed ${request.feedName} with URL ${request.url}", e) - } - - contents - } - - def readSeqFromXml[T](request: FeedRequest)(parse: Elem => Seq[T])(implicit ec: ExecutionContext): Future[Seq[T]] = { - readSeq(request) { body => - parse(XML.loadString(body)) - } - } - - def readSeqFromJson[T]( - request: FeedRequest, - )(parse: JsValue => Seq[T])(implicit ec: ExecutionContext): Future[Seq[T]] = { - readSeq(request) { body => - parse(Json.parse(body)) - } - } -} - -case class FeedRequest( - feedName: String, - url: String, - parameters: Map[String, String] = Map.empty, - responseEncoding: String, - timeout: Duration, -) - -case class FeedSwitchOffException(feedName: String) extends Exception { - override val getMessage: String = s"Reading $feedName feed failed: Switch is off" -} - -case class FeedReadException(request: FeedRequest, statusCode: Int, statusText: String) extends Exception { - override val getMessage: String = - s"Reading ${request.feedName} feed from ${request.url} failed: $statusCode: $statusText" -} - -case class FeedParseException(request: FeedRequest, causeMessage: String) extends Exception { - override val getMessage: String = - s"Parsing ${request.feedName} feed from ${request.url} failed: $causeMessage" -} - -case class FeedMissingConfigurationException(feedName: String) extends Exception { - override val getMessage: String = s"Missing configuration for $feedName feed" -} diff --git a/commercial/app/model/feeds/S3FeedStore.scala b/commercial/app/model/feeds/S3FeedStore.scala deleted file mode 100644 index ef943f623223..000000000000 --- a/commercial/app/model/feeds/S3FeedStore.scala +++ /dev/null @@ -1,13 +0,0 @@ -package commercial.model.feeds - -import conf.Configuration.commercial.merchandisingFeedsLatest -import services.S3 - -object S3FeedStore { - - private def key(feedName: String): String = s"$merchandisingFeedsLatest/$feedName" - - def put(feedName: String, feed: Feed): Unit = S3.putPrivate(key(feedName), value = feed.content, feed.contentType) - - def get(feedName: String): Option[String] = S3.get(key(feedName)) -} diff --git a/commercial/app/model/merchandise/Merchandise.scala b/commercial/app/model/merchandise/Merchandise.scala deleted file mode 100644 index 7594fe3c9d83..000000000000 --- a/commercial/app/model/merchandise/Merchandise.scala +++ /dev/null @@ -1,221 +0,0 @@ -package commercial.model.merchandise - -import commercial.model.OptString -import commercial.model.capi.{CapiImages, ImageInfo} -import model.ImageElement -import jobs.Industries -import org.apache.commons.lang.{StringEscapeUtils, StringUtils} -import org.joda.time.DateTime -import org.joda.time.format.DateTimeFormat -import play.api.libs.json.JodaWrites._ -import org.jsoup.Jsoup -import org.jsoup.nodes.{Document, Element} -import play.api.libs.functional.syntax._ -import play.api.libs.json._ -import views.support.Item300 - -import scala.util.Try -import scala.util.control.NonFatal -import scala.xml.Node - -sealed trait Merchandise - -case class TravelOffer( - id: String, - title: String, - offerUrl: String, - imageUrl: String, - fromPrice: Option[Double], - earliestDeparture: DateTime, - keywordIdSuffixes: Seq[String], - countries: Seq[String], - category: Option[String], - tags: Seq[String], - duration: Option[Int], - position: Int, -) extends Merchandise { - - val durationInWords: String = duration map { - case 1 => "1 night" - case multiple => s"$multiple nights" - } getOrElse "" - - def formattedPrice: Option[String] = - fromPrice map { price => - if (price % 1 == 0) - f"£$price%,.0f" - else - f"£$price%,.2f" - } -} - -case class Member(username: String, gender: Gender, age: Int, profilePhoto: String, location: String) - extends Merchandise { - - val profileId: Option[String] = profilePhoto match { - case Member.IdPattern(id) => Some(id) - case _ => None - } - - val profileUrl: String = s"https://soulmates.theguardian.com/${profileId.map(id => s"landing/$id") getOrElse ""}" -} - -case class MemberPair(member1: Member, member2: Member) extends Merchandise - -case class Job( - id: Int, - title: String, - shortDescription: String, - locationDescription: Option[String], - recruiterName: String, - recruiterPageUrl: Option[String], - recruiterLogoUrl: String, - sectorIds: Seq[Int], - salaryDescription: String, - keywordIdSuffixes: Seq[String] = Nil, -) extends Merchandise { - - val shortSalaryDescription = StringUtils.abbreviate(salaryDescription, 25).replace("...", "…") - - def listingUrl: String = s"https://jobs.theguardian.com/job/$id" - - val industries: Seq[String] = - Industries.sectorIdIndustryMap.filter { case (sectorId, name) => sectorIds.contains(sectorId) }.values.toSeq - - val mainIndustry: Option[String] = industries.headOption -} - -sealed trait Gender { - val name: String -} - -object Gender { - - def fromName(name: String): Gender = - name match { - case Woman.name => Woman - case _ => Man - } -} - -case object Woman extends Gender { - val name: String = "Woman" -} - -case object Man extends Gender { - val name: String = "Man" -} - -object Merchandise { - - val merchandiseWrites: Writes[Merchandise] = new Writes[Merchandise] { - def writes(m: Merchandise) = - m match { - case j: Job => Json.toJson(j) - case m: Member => Json.toJson(m) - case p: MemberPair => - Json.obj( - "member1" -> Json.toJson(p.member1), - "member2" -> Json.toJson(p.member2), - ) - case t: TravelOffer => Json.toJson(t) - } - } -} - -object TravelOffer { - - def fromXml(xml: Node): TravelOffer = { - - def text(nodeSelector: String): String = (xml \ nodeSelector).text.trim - def tagText(group: String): Seq[String] = (xml \\ "tag").filter(hasGroupAttr(_, group)).map(_.text.trim) - - def parseInt(s: String) = Try(s.toInt).toOption - - def parseDouble(s: String) = - try { - Some(s.toDouble) - } catch { - case NonFatal(_) => None - } - - def hasGroupAttr(elem: Node, value: String) = elem.attribute("group") exists (_.text == value) - - TravelOffer( - id = text("@vibeid"), - title = text("name"), - offerUrl = text("url"), - imageUrl = text("image"), - fromPrice = parseDouble(text("@fromprice")), - earliestDeparture = DateTime.parse(text("@earliestdeparture")), - keywordIdSuffixes = Nil, - countries = tagText("Country").distinct, - category = tagText("Holiday Type").headOption, - tags = Nil, - duration = parseInt(text("@duration")), - position = -1, - ) - } - - implicit val travelOfferWrites: Writes[TravelOffer] = Json.writes[TravelOffer] -} - -object Member { - val IdPattern = """.*/([\da-f]+)/.*""".r - - implicit val genderReads: Reads[Gender] = JsPath.read[String].map(Gender.fromName) - implicit val genderWrites: Writes[Gender] = Writes[Gender](gender => JsString(gender.name)) - - implicit val memberReads: Reads[Member] = - ( - (JsPath \ "username").read[String] and - (JsPath \ "gender").read[Gender] and - (JsPath \ "age").read[Int] and - (JsPath \ "profile_photo").read[String] and - (JsPath \ "location").read[String].map(locations => locations.split(",").head) - )(Member.apply _) - - implicit val memberWrites: Writes[Member] = new Writes[Member] { - - def writes(member: Member): JsValue = { - Json.obj( - "username" -> member.username, - "gender" -> member.gender, - "age" -> member.age, - "profile_photo" -> member.profilePhoto, - "location" -> member.location, - "profile_id" -> member.profileId, - "profile_url" -> member.profileUrl, - ) - } - } -} - -object Job { - - def fromXml(xml: Node): Job = - Job( - id = (xml \ "JobID").text.toInt, - title = (xml \ "JobTitle").text, - shortDescription = StringEscapeUtils.unescapeHtml((xml \ "ShortJobDescription").text), - locationDescription = OptString((xml \ "LocationDescription").text), - recruiterName = (xml \ "RecruiterName").text, - recruiterPageUrl = OptString((xml \ "RecruiterPageUrl").text), - recruiterLogoUrl = (xml \ "RecruiterLogoURL").text, - sectorIds = (xml \ "Sectors" \ "Sector") map (_.text.toInt), - salaryDescription = (xml \ "SalaryDescription").text, - ) - - implicit val jobWrites: Writes[Job] = new Writes[Job] { - def writes(j: Job) = - Json.obj( - "id" -> j.id, - "title" -> j.title, - "listingUrl" -> j.listingUrl, - "recruiterLogoUrl" -> j.recruiterLogoUrl, - "recruiterName" -> j.recruiterName, - "locationDescription" -> j.locationDescription, - "shortSalaryDescription" -> j.shortSalaryDescription, - ) - } -} diff --git a/commercial/app/model/merchandise/MerchandiseAgent.scala b/commercial/app/model/merchandise/MerchandiseAgent.scala deleted file mode 100644 index aff3c2efca95..000000000000 --- a/commercial/app/model/merchandise/MerchandiseAgent.scala +++ /dev/null @@ -1,31 +0,0 @@ -package commercial.model.merchandise - -import commercial.model.Segment -import common.{Box, GuLogging} - -import scala.concurrent.Future -import scala.util.Random - -trait MerchandiseAgent[T] extends GuLogging { - - private lazy val agent = Box[Seq[T]](Nil) - - def available: Seq[T] = agent() - - def updateAvailableMerchandise(freshMerchandise: Seq[T]): Future[Seq[T]] = { - agent.alter { oldMerchandise => - if (freshMerchandise.nonEmpty) { - freshMerchandise - } else { - log.warn("Using old merchandise as there is no fresh merchandise") - oldMerchandise - } - } - } - - def getTargetedMerchandise(segment: Segment, default: Seq[T])(targeting: T => Boolean): Seq[T] = { - val targeted = Random.shuffle(available filter targeting) - if (targeted.isEmpty) default else targeted - } - -} diff --git a/commercial/app/model/merchandise/TrafficDriver.scala b/commercial/app/model/merchandise/TrafficDriver.scala deleted file mode 100644 index 663fd3d50d6d..000000000000 --- a/commercial/app/model/merchandise/TrafficDriver.scala +++ /dev/null @@ -1,36 +0,0 @@ -package commercial.model.merchandise - -import commercial.model.capi.{CapiImages, ImageInfo} -import common.Edition -import model.ContentType -import play.api.libs.json.{Json, Writes} - -case class TrafficDriver( - articleHeadline: String, - articleUrl: String, - articleText: Option[String], - articleImage: ImageInfo, - edition: String, -) - -object TrafficDriver { - - def fromContent(contentType: ContentType, edition: Edition): TrafficDriver = { - - val content = contentType.content - val imageInfo = CapiImages.buildImageData(content.trail.trailPicture) - - TrafficDriver( - content.trail.headline, - content.metadata.webUrl, - content.trail.fields.trailText, - imageInfo, - edition.id, - ) - - } - - implicit val writesTrafficDriver: Writes[TrafficDriver] = - Json.writes[TrafficDriver] - -} diff --git a/commercial/app/model/merchandise/jobs/Industries.scala b/commercial/app/model/merchandise/jobs/Industries.scala deleted file mode 100644 index cbc0e9594cf4..000000000000 --- a/commercial/app/model/merchandise/jobs/Industries.scala +++ /dev/null @@ -1,56 +0,0 @@ -package commercial.model.merchandise.jobs - -import common.Box -import commercial.model.capi.Lookup -import contentapi.ContentApiClient - -import scala.concurrent.{ExecutionContext, Future} - -object Industries { - - // note, these are ordered by importance - val sectorIdIndustryMap = Map[Int, String]( - (111, "Charities"), - (286, "Social care"), - (127, "Education"), - (166, "Government & Politics"), - (196, "Housing"), - (223, "Marketing & PR"), - (184, "Health"), - (235, "Media"), - (218, "Legal"), - (101, "Arts & heritage"), - (149, "Finance & Accounting"), - (141, "Environment"), - (211, "Technology"), - (124, "Construction"), - (137, "Engineering"), - (142, "Design"), - (158, "General"), - (204, "Hospitality"), - (219, "Leisure"), - (244, "Recruitment"), - (245, "Retail & FMCG"), - (259, "Science"), - (294, "Travel & transport"), - (343, "Skilled Trade"), - (350, "Social Enterprise"), - ) -} - -class Industries(contentApiClient: ContentApiClient) { - - private val lookup = new Lookup(contentApiClient) - private lazy val industryKeywordIds = Box(Map.empty[Int, Seq[String]]) - - def refresh()(implicit executionContext: ExecutionContext): Future[Iterable[Map[Int, Seq[String]]]] = - Future.sequence { - Industries.sectorIdIndustryMap map { case (id, name) => - lookup.keyword(name) flatMap { keywords => - industryKeywordIds.alter(_.updated(id, keywords.map(_.id))) - } - } - } - - def forIndustry(id: Int): Seq[String] = industryKeywordIds().getOrElse(id, Nil) -} diff --git a/commercial/app/model/merchandise/jobs/JobsAgent.scala b/commercial/app/model/merchandise/jobs/JobsAgent.scala deleted file mode 100644 index 33adacc2a40e..000000000000 --- a/commercial/app/model/merchandise/jobs/JobsAgent.scala +++ /dev/null @@ -1,46 +0,0 @@ -package commercial.model.merchandise.jobs - -import commercial.model.Segment -import commercial.model.capi.Keyword -import commercial.model.feeds.{FeedMetaData, ParsedFeed} -import commercial.model.merchandise.{Job, MerchandiseAgent} - -import scala.concurrent.{ExecutionContext, Future} - -class JobsAgent(allIndustries: Industries) extends MerchandiseAgent[Job] { - - def jobsTargetedAt(segment: Segment): Seq[Job] = { - def defaultJobs = available filter (_.industries.contains("Media")) - getTargetedMerchandise(segment, defaultJobs) { job => - Keyword.idSuffixesIntersect(segment.context.keywords, job.keywordIdSuffixes) - } - } - - def specificJobs(jobIdStrings: Seq[String]): Seq[Job] = { - val jobIds = jobIdStrings map (_.toInt) - available filter (job => jobIds contains job.id) - } - - def refresh(feedMetaData: FeedMetaData, feedContent: => Option[String])(implicit - executionContext: ExecutionContext, - ): Future[ParsedFeed[Job]] = { - - def withKeywords(parsedFeed: Future[ParsedFeed[Job]]): Future[ParsedFeed[Job]] = { - parsedFeed map { feed => - val jobs = feed.contents map { job => - val jobKeywordIds = job.sectorIds.flatMap(allIndustries.forIndustry).distinct - job.copy(keywordIdSuffixes = jobKeywordIds map Keyword.getIdSuffix) - } - ParsedFeed(jobs, feed.parseDuration) - } - } - - val parsedFeed = withKeywords(JobsFeed.parsedJobs(feedMetaData, feedContent)) - - parsedFeed foreach { feed => - updateAvailableMerchandise(feed.contents) - } - - parsedFeed - } -} diff --git a/commercial/app/model/merchandise/jobs/JobsFeed.scala b/commercial/app/model/merchandise/jobs/JobsFeed.scala deleted file mode 100644 index a731f4c68de7..000000000000 --- a/commercial/app/model/merchandise/jobs/JobsFeed.scala +++ /dev/null @@ -1,35 +0,0 @@ -package commercial.model.merchandise.jobs - -import java.lang.System.currentTimeMillis -import java.util.concurrent.TimeUnit.MILLISECONDS - -import commercial.model.feeds.{FeedMetaData, MissingFeedException, ParsedFeed, SwitchOffException} -import common.GuLogging -import commercial.model.merchandise.Job - -import scala.concurrent.{ExecutionContext, Future} -import scala.concurrent.duration.Duration -import scala.util.control.NonFatal -import scala.xml.{Elem, XML} - -object JobsFeed extends GuLogging { - - def parse(xml: Elem): Seq[Job] = - for { - jobXml <- xml \\ "Job" - if (jobXml \ "RecruiterLogoURL").nonEmpty - if (jobXml \ "RecruiterName").text != "THE GUARDIAN MASTERCLASSES" - } yield Job.fromXml(jobXml) - - def parsedJobs(feedMetaData: FeedMetaData, feedContent: => Option[String])(implicit - executionContext: ExecutionContext, - ): Future[ParsedFeed[Job]] = { - val start = currentTimeMillis - feedContent map { body => - val parsed = parse(XML.loadString(body.dropWhile(_ != '<'))) - Future(ParsedFeed(parsed, Duration(currentTimeMillis - start, MILLISECONDS))) - } getOrElse { - Future.failed(MissingFeedException(feedMetaData.name)) - } - } -} diff --git a/commercial/app/model/merchandise/travel/TravelOffersAgent.scala b/commercial/app/model/merchandise/travel/TravelOffersAgent.scala deleted file mode 100644 index dd759e61bf70..000000000000 --- a/commercial/app/model/merchandise/travel/TravelOffersAgent.scala +++ /dev/null @@ -1,36 +0,0 @@ -package commercial.model.merchandise.travel - -import commercial.model.Segment -import commercial.model.capi.Keyword -import commercial.model.feeds.{FeedMetaData, ParsedFeed} -import commercial.model.merchandise.{MerchandiseAgent, TravelOffer} -import contentapi.ContentApiClient - -import scala.concurrent.{ExecutionContext, Future} - -class TravelOffersAgent(contentApiClient: ContentApiClient) extends MerchandiseAgent[TravelOffer] { - - def offersTargetedAt(segment: Segment): Seq[TravelOffer] = { - val defaultOffers = available.sortBy(_.position).take(4) - getTargetedMerchandise(segment, defaultOffers)(offer => - Keyword.idSuffixesIntersect(segment.context.keywords, offer.keywordIdSuffixes), - ) - } - - def specificTravelOffers(offerIdStrings: Seq[String]): Seq[TravelOffer] = { - offerIdStrings flatMap (offerId => available find (_.id == offerId)) - } - - def refresh(feedMetaData: FeedMetaData, feedContent: => Option[String])(implicit - executionContext: ExecutionContext, - ): Future[ParsedFeed[TravelOffer]] = { - - val parsedFeed: Future[ParsedFeed[TravelOffer]] = TravelOffersApi.parseOffers(feedMetaData, feedContent) - - parsedFeed map { offers => - updateAvailableMerchandise(offers.contents) - } - - parsedFeed - } -} diff --git a/commercial/app/model/merchandise/travel/TravelOffersApi.scala b/commercial/app/model/merchandise/travel/TravelOffersApi.scala deleted file mode 100644 index ff4310ae6f05..000000000000 --- a/commercial/app/model/merchandise/travel/TravelOffersApi.scala +++ /dev/null @@ -1,33 +0,0 @@ -package commercial.model.merchandise.travel - -import java.lang.System.currentTimeMillis - -import commercial.model.feeds.{FeedMetaData, MissingFeedException, ParsedFeed, SwitchOffException} -import common.GuLogging -import commercial.model.merchandise.TravelOffer -import org.joda.time.format.DateTimeFormat - -import scala.concurrent.{ExecutionContext, Future} -import scala.concurrent.duration._ -import scala.util.control.NonFatal -import scala.xml.{Elem, XML} - -object TravelOffersApi extends GuLogging { - - private val dateFormat = DateTimeFormat.forPattern("dd-MMM-yyyy") - - def parse(xml: Elem): Seq[TravelOffer] = (xml \\ "product") map TravelOffer.fromXml - - def parseOffers(feedMetaData: FeedMetaData, feedContent: => Option[String])(implicit - executionContext: ExecutionContext, - ): Future[ParsedFeed[TravelOffer]] = { - val start = System.currentTimeMillis - feedContent map { body => - val elems = XML.loadString(body) - val parsed = parse(elems) - Future(ParsedFeed(parsed, Duration(currentTimeMillis - start, MILLISECONDS))) - } getOrElse { - Future.failed(MissingFeedException(feedMetaData.name)) - } - } -} From 08ad55265c48bb2e393826faebc350283aba521f Mon Sep 17 00:00:00 2001 From: Jake Lee Kennedy Date: Mon, 3 Feb 2025 10:02:03 +0000 Subject: [PATCH 2/3] remove from preview app --- preview/app/AppLoader.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/preview/app/AppLoader.scala b/preview/app/AppLoader.scala index 73b9df1b38c5..4664d0728689 100644 --- a/preview/app/AppLoader.scala +++ b/preview/app/AppLoader.scala @@ -4,7 +4,6 @@ import app.{FrontendApplicationLoader, FrontendComponents, LifecycleComponent} import com.amazonaws.regions.Regions import com.amazonaws.services.s3.AmazonS3ClientBuilder import com.softwaremill.macwire._ -import commercial.CommercialLifecycle import commercial.controllers.CommercialControllers import commercial.targeting.TargetingLifecycle import common.dfp.FaciaDfpAgentLifecycle @@ -49,7 +48,6 @@ trait PreviewLifecycleComponents def standaloneLifecycleComponents: List[LifecycleComponent] = List( - wire[CommercialLifecycle], wire[OnwardJourneyLifecycle], wire[ConfigAgentLifecycle], wire[FaciaDfpAgentLifecycle], From b7cc6a7d62f2d40c92510c9eb68c599cd2ff86c8 Mon Sep 17 00:00:00 2001 From: Jake Lee Kennedy Date: Tue, 4 Feb 2025 17:16:15 +0000 Subject: [PATCH 3/3] delete old tests --- commercial/test/model/jobs/Fixtures.scala | 135 ------------------ commercial/test/model/jobs/JobTest.scala | 48 ------- commercial/test/model/jobs/JobsFeedTest.scala | 23 --- .../test/model/travel/TravelOfferTest.scala | 81 ----------- .../test/test/CommercialTestSuite.scala | 2 - 5 files changed, 289 deletions(-) delete mode 100644 commercial/test/model/jobs/Fixtures.scala delete mode 100644 commercial/test/model/jobs/JobTest.scala delete mode 100644 commercial/test/model/jobs/JobsFeedTest.scala delete mode 100644 commercial/test/model/travel/TravelOfferTest.scala diff --git a/commercial/test/model/jobs/Fixtures.scala b/commercial/test/model/jobs/Fixtures.scala deleted file mode 100644 index b6e84a71c68a..000000000000 --- a/commercial/test/model/jobs/Fixtures.scala +++ /dev/null @@ -1,135 +0,0 @@ -package commercial.model.merchandise.jobs - -import commercial.model.merchandise.Job - -object Fixtures { - - val xml = - """ - | - | - | 1058606 - | Area Management Training Programme (Graduate Area Manager) - | ALDI - | We're only looking for outstanding individuals for the Aldi Management Programme. - | - | 149 - | 158 - | 245 - | - | http://jobs.theguardian.com/getasset/?uiAssetID=833D7672-6B21-4D81-BCE1-9CEF11CCEA21 - | Unpaid Voluntary Work - | Rotherham/Sheffield - | http://jobs.theguardian.com/employer/196643/ - | - | - | 4302057 - | Listen Laugh & Learn in Worcestershire - | MOTOR NEURONE DISEASE ASSOCIATION - | The role of Association Visitor is a highly rewarding one with many opportunities for developing your own skills and knowledge, whilst being part of a team. Professional qualifications or personal experience of MND are not necessary... - | - | 111 - | 112 - | 121 - | 286 - | 600117 - | 600190 - | - | Unpaid Voluntary Work - | Rotherham/Sheffield - | http://jobs.theguardian.com/employer/196644/ - | - | - | 4365671 - | Female Youth Work Volunteer in sports - | CATCH 22 - | The Active Women's project helps young women access sports activities in their local area in Southampton. Volunteers will support the running of these sessions, as well as supporting the young women in accessing the sessions. - | - | 111 - | 112 - | 115 - | 219 - | 222 - | 600118 - | 600119 - | - | http://jobs.theguardian.com/getasset/?uiAssetID=1D428A1F-65B4-4ADE-8CA3-91F2AE7423E3 - | Unpaid Voluntary Work - | Rotherham/Sheffield - | http://jobs.theguardian.com/employer/196645/ - | - | - | 4411510 - | Air Traffic Controller - | RAF CAREERS - | Control some of the world&rsquo;s most modern aircraft using radar & communications equipment - | - | 166 - | 308 - | - | http://jobs.theguardian.com/getasset/?uiAssetID=14357B35-BBA3-4F96-ADF3-F9ABC82C894C - | Unpaid Voluntary Work - | Rotherham/Sheffield - | http://jobs.theguardian.com/employer/196646/ - | - | - | 6061077 - | Uncovering the city: Urban writing seminar with Bradley L. Garrett - | THE GUARDIAN MASTERCLASSES - | In this stimulating seminar you’ll learn how to delve beneath the surface of cities to find stories that readers and editors will love. - | - | 101 - | 141 - | 147 - | 235 - | 237 - | 239 - | 240 - | 241 - | 600114 - | - | http://jobs.theguardian.com/getasset/?uiAssetID=59188A47-8B25-4F56-9F19-4BB278DB3269 - | na - | Kings Cross, Central London - | http://jobs.theguardian.com/employer/230273/ - | - | - | """.stripMargin - - val jobs = List( - Job( - 1058606, - "Area Management Training Programme (Graduate Area Manager)", - "We're only looking for outstanding individuals for the Aldi Management Programme.", - Some("Rotherham/Sheffield"), - "ALDI", - Some("http://jobs.theguardian.com/employer/196643/"), - "http://jobs.theguardian.com/getasset/?uiAssetID=833D7672-6B21-4D81-BCE1-9CEF11CCEA21", - Seq(149, 158, 245), - "Unpaid Voluntary Work", - ), - Job( - 4365671, - "Female Youth Work Volunteer in sports", - "The Active Women's project helps young women access sports activities in their local area in Southampton. Volunteers will support the running of these sessions, as well as supporting the young women in accessing the sessions.", - Some("Rotherham/Sheffield"), - "CATCH 22", - Some("http://jobs.theguardian.com/employer/196645/"), - "http://jobs.theguardian.com/getasset/?uiAssetID=1D428A1F-65B4-4ADE-8CA3-91F2AE7423E3", - Seq(111, 112, 115, 219, 222, 600118, 600119), - "Unpaid Voluntary Work", - ), - Job( - 4411510, - "Air Traffic Controller", - "Control some of the world’s most modern aircraft using radar & communications equipment", - Some("Rotherham/Sheffield"), - "RAF CAREERS", - Some("http://jobs.theguardian.com/employer/196646/"), - "http://jobs.theguardian.com/getasset/?uiAssetID=14357B35-BBA3-4F96-ADF3-F9ABC82C894C", - Seq(166, 308), - "Unpaid Voluntary Work", - ), - ) - -} diff --git a/commercial/test/model/jobs/JobTest.scala b/commercial/test/model/jobs/JobTest.scala deleted file mode 100644 index ddabcb288d8a..000000000000 --- a/commercial/test/model/jobs/JobTest.scala +++ /dev/null @@ -1,48 +0,0 @@ -package commercial.model.merchandise.jobs - -import commercial.model.merchandise.Job -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.DoNotDiscover -import org.scalatest.matchers.should.Matchers -import test.ConfiguredTestSuite - -@DoNotDiscover class JobTest extends AnyFlatSpec with Matchers with ConfiguredTestSuite { - - "mainIndustry" should "give a value for jobs" in { - val job = Job(1, "title", "desc", None, "recruiter", None, "logo", Seq(218), "Unpaid Voluntary Work") - - job.mainIndustry should be(Some("Legal")) - } - - "mainIndustry" should "not give a value for jobs without an industry sector" in { - val job = Job(1, "title", "desc", None, "recruiter", None, "logo", Nil, "Unpaid Voluntary Work") - - job.mainIndustry should be(None) - } - - "mainIndustry" should "respect the industry order" in { - val job = Job(1, "title", "desc", None, "recruiter", None, "logo", Seq(124, 111, 101), "Unpaid Voluntary Work") - - job.mainIndustry should be(Some("Arts & heritage")) - } - - "shortSalaryDescription" should "be in correct format" in { - val job1 = - Job(1, "title", "desc", None, "recruiter", None, "logo", Seq(124, 111, 101), "Unpaid Voluntary Work") - val job2 = - Job( - 2, - "title", - "desc", - None, - "recruiter", - None, - "logo", - Seq(124, 111, 101), - "Unpaid Voluntary Work Without Bonus", - ) - - job1.shortSalaryDescription should be("Unpaid Voluntary Work") - job2.shortSalaryDescription should be("Unpaid Voluntary Work …") - } -} diff --git a/commercial/test/model/jobs/JobsFeedTest.scala b/commercial/test/model/jobs/JobsFeedTest.scala deleted file mode 100644 index 779a07cdaf32..000000000000 --- a/commercial/test/model/jobs/JobsFeedTest.scala +++ /dev/null @@ -1,23 +0,0 @@ -package commercial.model.merchandise.jobs - -import commercial.model.merchandise.Job -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -import scala.xml.XML - -class JobsFeedTest extends AnyFlatSpec with Matchers { - - "parse" should "parse all jobs in XML feed" in { - val jobs = JobsFeed.parse(XML.loadString(Fixtures.xml)) - - jobs should be(Fixtures.jobs) - } - - "parsed Jobs" should "figure out their industries" in { - val jobs: Seq[Job] = JobsFeed.parse(XML.loadString(Fixtures.xml)) - - jobs.head.mainIndustry should be(Some("Finance & Accounting")) - } - -} diff --git a/commercial/test/model/travel/TravelOfferTest.scala b/commercial/test/model/travel/TravelOfferTest.scala deleted file mode 100644 index 105362ebe529..000000000000 --- a/commercial/test/model/travel/TravelOfferTest.scala +++ /dev/null @@ -1,81 +0,0 @@ -package commercial.model.merchandise.travel - -import commercial.model.merchandise.TravelOffer -import org.joda.time.DateTime -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers - -import scala.xml.Elem - -class TravelOfferTest extends AnyFlatSpec with Matchers { - - private val xml: Elem = - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - "fromXml" should "generate a Travel Offer from a valid XML element" in { - TravelOffer.fromXml(xml) shouldBe TravelOffer( - id = "a08878776d1429a5109064d64b5fda05", - title = "New Zealand - Land of the Long White Cloud", - offerUrl = - "https://holidays.theguardian.com/collection/new-zealand---land-of-the-long-white-cloud-a08878776d1429a5109064d64b5fda05", - imageUrl = "https://holidays.theguardian.com/tourimg.php?a=img&img=92b6c4fcb13dab1da639805c87aea979", - fromPrice = Some(3999.0), - earliestDeparture = DateTime.parse("2016-10-14"), - keywordIdSuffixes = Nil, - countries = Seq("New Zealand"), - category = Some("Escorted tours"), - tags = Nil, - duration = Some(21), - position = -1, - ) - } -} diff --git a/commercial/test/test/CommercialTestSuite.scala b/commercial/test/test/CommercialTestSuite.scala index aae9ce1c4efb..eb619706ef18 100644 --- a/commercial/test/test/CommercialTestSuite.scala +++ b/commercial/test/test/CommercialTestSuite.scala @@ -1,13 +1,11 @@ package commercial.test import commercial.model.capi.LookupTest -import commercial.model.merchandise.jobs import org.scalatest.Suites import test.SingleServerSuite class CommercialTestSuite extends Suites( - new jobs.JobTest, new LookupTest, ) with SingleServerSuite {}