Merge pull request #4036 from PatrickGoRaft/quarterly-2021

Quarterly 2021

common/src/main/resources/reference.conf

@@ -126,15 +126,19 @@ hmda {
   countyLoanLimit {
     2018 {
       fields.filename = "FullCountyLoanLimitList2018.txt"
-      fields.filename = ${?COUNTY_LOAN_LIMIT_FILENAME}
+      fields.filename = ${?COUNTY_LOAN_LIMIT_FILENAME_2018}
     }
     2019 {
       fields.filename = "FullCountyLoanLimitList2019.txt"
-      fields.filename = ${?COUNTY_LOAN_LIMIT_FILENAME}
+      fields.filename = ${?COUNTY_LOAN_LIMIT_FILENAME_2019}
     }
     2020 {
       fields.filename = "FullCountyLoanLimitList2020.txt"
-      fields.filename = ${?COUNTY_LOAN_LIMIT_FILENAME}
+      fields.filename = ${?COUNTY_LOAN_LIMIT_FILENAME_2020}
+    }
+    2021 {
+      fields.filename = "FullCountyLoanLimitList2021.txt"
+      fields.filename = ${?COUNTY_LOAN_LIMIT_FILENAME_2021}
     }
   }
 

common/src/main/scala/hmda/census/records/CensusRecords.scala

@@ -88,6 +88,10 @@ object CensusRecords {
         indexedTract2019
       case 2020 =>
         indexedTract2020
+      case 2021 =>
+        indexedTract2020
+      case _ =>
+        indexedTract2020
     }
   }
 

hmda-analytics/src/main/resources/application.conf

@@ -48,15 +48,29 @@ hmda {
     larDeletion = ${?HMDA-LAR-DELETE}
     historyInsertion = true
     historyInsertion = ${?HMDA-HISTORY-INSERT}
+    2021 {
+      tsTableNameQ1 = "ts2021_q1"
+      tsTableNameQ1 = ${?TS_TABLE_NAME_2021_Q1}
+      tsTableNameQ2 = "ts2021_q2"
+      tsTableNameQ2 = ${?TS_TABLE_NAME_2021_Q2}
+      tsTableNameQ3 = "ts2021_q3"
+      tsTableNameQ3 = ${?TS_TABLE_NAME_2021_Q1}
+      larTableNameQ1 = "lar2021_q1"
+      larTableNameQ1 = ${?LAR_TABLE_NAME_2021_Q1}
+      larTableNameQ2 = "lar2021_q2"
+      larTableNameQ2 = ${?LAR_TABLE_NAME_2021_Q2}
+      larTableNameQ3 = "lar2021_q3"
+      larTableNameQ3 = ${?LAR_TABLE_NAME_2021_Q3}
+    }
     2020 {
       tsTableName = "transmittalsheet2020"
       tsTableName = ${?TS_TABLE_NAME_2020}
       tsTableNameQ1 = "ts2020_q1"
       tsTableNameQ1 = ${?TS_TABLE_NAME_2020_Q1}
       tsTableNameQ2 = "ts2020_q2"
-      tsTableNameQ2 = ${?TS_TABLE_NAME_2020_Q1}
+      tsTableNameQ2 = ${?TS_TABLE_NAME_2020_Q2}
       tsTableNameQ3 = "ts2020_q3"
-      tsTableNameQ3 = ${?TS_TABLE_NAME_2020_Q1}
+      tsTableNameQ3 = ${?TS_TABLE_NAME_2020_Q3}
       larTableName = "loanapplicationregister2020"
       larTableName = ${?LAR_TABLE_NAME_2020}
       larTableNameQ1 = "lar2020_q1"

hmda-analytics/src/main/scala/hmda/analytics/HmdaAnalyticsApp.scala

@@ -79,25 +79,41 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo
   val tsTableName2020Q2  = config.getString("hmda.analytics.2020.tsTableNameQ2")
   val tsTableName2020Q3  = config.getString("hmda.analytics.2020.tsTableNameQ3")
 
+  //2021 TS
+  val tsTableName2021Q1  = config.getString("hmda.analytics.2021.tsTableNameQ1")
+  val tsTableName2021Q2  = config.getString("hmda.analytics.2021.tsTableNameQ2")
+  val tsTableName2021Q3  = config.getString("hmda.analytics.2021.tsTableNameQ3")
+
   //2020 LAR
   val larTableName2020 = config.getString("hmda.analytics.2020.larTableName")
   val larTableName2020Q1 = config.getString("hmda.analytics.2020.larTableNameQ1")
   val larTableName2020Q2 = config.getString("hmda.analytics.2020.larTableNameQ2")
   val larTableName2020Q3 = config.getString("hmda.analytics.2020.larTableNameQ3")
 
+  //2021 Quarterly
+  val larTableName2021Q1 = config.getString("hmda.analytics.2021.larTableNameQ1")
+  val larTableName2021Q2 = config.getString("hmda.analytics.2021.larTableNameQ2")
+  val larTableName2021Q3 = config.getString("hmda.analytics.2021.larTableNameQ3")
+
 
   val transmittalSheetRepository2018 = new TransmittalSheetRepository(dbConfig, tsTableName2018)
   val transmittalSheetRepository2019 = new TransmittalSheetRepository(dbConfig, tsTableName2019)
   val transmittalSheetRepository2020 = new TransmittalSheetRepository(dbConfig, tsTableName2020)
   val transmittalSheetRepository2020Q1 = new TransmittalSheetRepository(dbConfig, tsTableName2020Q1)
   val transmittalSheetRepository2020Q2 = new TransmittalSheetRepository(dbConfig, tsTableName2020Q2)
   val transmittalSheetRepository2020Q3 = new TransmittalSheetRepository(dbConfig, tsTableName2020Q3)
+  val transmittalSheetRepository2021Q1 = new TransmittalSheetRepository(dbConfig, tsTableName2021Q1)
+  val transmittalSheetRepository2021Q2 = new TransmittalSheetRepository(dbConfig, tsTableName2021Q2)
+  val transmittalSheetRepository2021Q3 = new TransmittalSheetRepository(dbConfig, tsTableName2021Q3)
   val larRepository2018              = new LarRepository(dbConfig, larTableName2018)
   val larRepository2019              = new LarRepository(dbConfig, larTableName2019)
   val larRepository2020              = new LarRepository(dbConfig, larTableName2020)
   val larRepository2020Q1              = new LarRepository(dbConfig, larTableName2020Q1)
   val larRepository2020Q2              = new LarRepository(dbConfig, larTableName2020Q2)
   val larRepository2020Q3              = new LarRepository(dbConfig, larTableName2020Q3)
+  val larRepository2021Q1              = new LarRepository(dbConfig, larTableName2021Q1)
+  val larRepository2021Q2              = new LarRepository(dbConfig, larTableName2021Q2)
+  val larRepository2021Q3              = new LarRepository(dbConfig, larTableName2021Q3)
   val submissionHistoryRepository    = new SubmissionHistoryRepository(dbConfig, histTableName)
 
   val consumerSettings: ConsumerSettings[String, String] =
@@ -153,6 +169,9 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo
               case Period(2020, Some("Q2")) => transmittalSheetRepository2020Q2.deleteByLeiAndQuarter(lei = ts.lei)
               case Period(2020, Some("Q3")) => transmittalSheetRepository2020Q3.deleteByLeiAndQuarter(lei = ts.lei)
               case Period(2020, None) => transmittalSheetRepository2020.deleteByLeiAndQuarter(lei = ts.lei)
+              case Period(2021, Some("Q1")) => transmittalSheetRepository2021Q1.deleteByLeiAndQuarter(lei = ts.lei)
+              case Period(2021, Some("Q2")) => transmittalSheetRepository2021Q2.deleteByLeiAndQuarter(lei = ts.lei)
+              case Period(2021, Some("Q3")) => transmittalSheetRepository2021Q3.deleteByLeiAndQuarter(lei = ts.lei)
               case _ => throw new IllegalArgumentException(s"Unable to discern period from $submissionId to delete TS rows.")
             }
           } yield delete
@@ -215,6 +234,18 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo
                 transmittalSheetRepository2020.insert(
                   ts.copy(isQuarterly = Some(true), signDate = Some(signdate.getOrElse(0L)))
                 )
+              case Period(2021, Some("Q1")) =>
+                transmittalSheetRepository2021Q1.insert(
+                  ts.copy(isQuarterly = Some(true), signDate = Some(signdate.getOrElse(0L)))
+                )
+              case Period(2021, Some("Q2")) =>
+                transmittalSheetRepository2021Q2.insert(
+                  ts.copy(isQuarterly = Some(true), signDate = Some(signdate.getOrElse(0L)))
+                )
+              case Period(2021, Some("Q3")) =>
+                transmittalSheetRepository2021Q3.insert(
+                  ts.copy(isQuarterly = Some(true), signDate = Some(signdate.getOrElse(0L)))
+                )
               case _ =>
                 throw new IllegalArgumentException(s"Unable to discern period from $submissionId to insert TS rows.")
             }
@@ -243,6 +274,9 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo
               case Period(2020, Some("Q2")) => larRepository2020Q2.deletebyLeiAndQuarter(lar.larIdentifier.LEI)
               case Period(2020, Some("Q3")) => larRepository2020Q3.deletebyLeiAndQuarter(lar.larIdentifier.LEI)
               case Period(2020, None) => larRepository2020.deletebyLeiAndQuarter(lar.larIdentifier.LEI)
+              case Period(2021, Some("Q1")) => larRepository2021Q1.deletebyLeiAndQuarter(lar.larIdentifier.LEI)
+              case Period(2021, Some("Q2")) => larRepository2021Q2.deletebyLeiAndQuarter(lar.larIdentifier.LEI)
+              case Period(2021, Some("Q3")) => larRepository2021Q3.deletebyLeiAndQuarter(lar.larIdentifier.LEI)
               case _ => throw new IllegalArgumentException(s"Unable to discern period from $submissionId to delete LAR rows.")
 
             }
@@ -285,6 +319,18 @@ object HmdaAnalyticsApp extends App with TransmittalSheetComponent with LarCompo
                 larRepository2020.insert(
                   LarConverter(lar = lar, 2020, isQuarterly = true)
                 )
+              case Period(2021, Some("Q1")) =>
+                larRepository2021Q1.insert(
+                  LarConverter(lar = lar, 2021, isQuarterly = true)
+                )
+              case Period(2021, Some("Q2")) =>
+                larRepository2021Q2.insert(
+                  LarConverter(lar = lar, 2021, isQuarterly = true)
+                )
+              case Period(2021, Some("Q3")) =>
+                larRepository2021Q3.insert(
+                  LarConverter(lar = lar, 2021, isQuarterly = true)
+                )
               case _ =>
                 throw new IllegalArgumentException(s"Unable to discern period from $submissionId to insert LAR rows.")
             }

hmda-analytics/src/main/scala/hmda/analytics/query/LarConverter.scala

@@ -27,6 +27,9 @@ object LarConverter {
       config.getString("hmda.countyLoanLimit.2019.fields.filename")
   val countyLoanLimitFileName2020 =
       config.getString("hmda.countyLoanLimit.2020.fields.filename")
+
+  val countyLoanLimitFileName2021 =
+    config.getString("hmda.countyLoanLimit.2021.fields.filename")
 
   val countyLoanLimits2018: Seq[CountyLoanLimit] =
     parseCountyLoanLimitFile(countyLoanLimitFileName2018)
@@ -35,17 +38,25 @@ object LarConverter {
   val countyLoanLimits2020: Seq[CountyLoanLimit] =
     parseCountyLoanLimitFile(countyLoanLimitFileName2020)
 
+  val countyLoanLimits2021: Seq[CountyLoanLimit] =
+    parseCountyLoanLimitFile(countyLoanLimitFileName2021)
+
   val overallLoanLimit2018 = overallLoanLimits(countyLoanLimits2018)
   val overallLoanLimit2019 = overallLoanLimits(countyLoanLimits2019)
   val overallLoanLimit2020 = overallLoanLimits(countyLoanLimits2020)
+  val overallLoanLimit2021 = overallLoanLimits(countyLoanLimits2021)
+
 
   val countyLoanLimitsByCounty2018 = countyLoansLimitByCounty(countyLoanLimits2018)
   val countyLoanLimitsByCounty2019 = countyLoansLimitByCounty(countyLoanLimits2019)
   val countyLoanLimitsByCounty2020 = countyLoansLimitByCounty(countyLoanLimits2020)
+  val countyLoanLimitsByCounty2021 = countyLoansLimitByCounty(countyLoanLimits2021)
+
 
   val countyLoanLimitsByState2018 = countyLoansLimitByState(countyLoanLimits2018)
   val countyLoanLimitsByState2019 = countyLoansLimitByState(countyLoanLimits2019)
   val countyLoanLimitsByState2020 = countyLoansLimitByState(countyLoanLimits2020)
+  val countyLoanLimitsByState2021 = countyLoansLimitByState(countyLoanLimits2021)
 
   def apply(
     lar: LoanApplicationRegister,
@@ -199,6 +210,8 @@ object LarConverter {
       case 2018 => countyLoanLimitsByCounty2018
       case 2019 => countyLoanLimitsByCounty2019
       case 2020 => countyLoanLimitsByCounty2020
+      case 2021 => countyLoanLimitsByCounty2021
+
     }
   }
 
@@ -207,6 +220,8 @@ object LarConverter {
       case 2018 => countyLoanLimitsByState2018
       case 2019 => countyLoanLimitsByState2019
       case 2020 => countyLoanLimitsByState2020
+      case 2021 => countyLoanLimitsByState2021
+
     }
   }
 
@@ -215,6 +230,8 @@ object LarConverter {
       case 2018 => overallLoanLimit2018
       case 2019 => overallLoanLimit2019
       case 2020 => overallLoanLimit2020
+      case 2021 => overallLoanLimit2021
+
     }
   }
 

hmda-data-publisher/src/main/resources/application.conf

@@ -69,6 +69,13 @@ akka {
       TsSchedulerQuarterly2020.expression =   "0 0 0 23 4 ? 2079"
       TsSchedulerQuarterly2020.expression =   ${?TS_Scheduler_Quarterly_2020}
 
+      // 2021  Quarterly - private
+      LarSchedulerQuarterly2021.expression =  "0 0 0 23 4 ? 2079"
+      LarSchedulerQuarterly2021.expression =  ${?LAR_Scheduler_Quarterly_2021}
+
+      TsSchedulerQuarterly2021.expression =   "0 0 0 23 4 ? 2079"
+      TsSchedulerQuarterly2021.expression =   ${?TS_Scheduler_Quarterly_2021}
+
     }
   }
 }
@@ -246,6 +253,56 @@ pg-tables {
   lar2020Q3QATableName = "qa_lar_table_2020_q3"
   lar2020Q3QATableName = ${?LAR_2020_Q3_QA_TABLE}
 
+  //2021 PG Tables
+  lar2021TableName ="loanapplicationregister2021"
+  lar2021TableName =${?LAR_2021_TABLE}
+  mlar2021TableName ="modifiedlar2021"
+  mlar2021TableName =${?MODIFIED_LAR_2021_TABLE}
+  panel2021TableName = "institutions2021"
+  panel2021TableName = ${?PANEL_2021_TABLE}
+  ts2021TableName = "transmittalsheet2021"
+  ts2021TableName = ${?TS_2021_TABLE}
+
+  ts2021Q1TableName = "ts2021_q1"
+  ts2021Q1TableName = ${?TS_2021_Q1_TABLE}
+  ts2021Q2TableName = "ts2021_q2"
+  ts2021Q2TableName = ${?TS_2021_Q2_TABLE}
+  ts2021Q3TableName = "ts2021_q3"
+  ts2021Q4TableName = ${?TS_2021_Q3_TABLE}
+
+  lar2021Q1TableName ="lar2021_q1"
+  lar2021Q1TableName =${?LAR_2021_Q1_TABLE}
+  lar2021Q2TableName ="lar2021_q2"
+  lar2021Q2TableName =${?LAR_2021_Q2_TABLE}
+  lar2021Q3TableName ="lar2021_q3"
+  lar2021Q3TableName =${?LAR_2021_Q3_TABLE}
+
+  //2021 PG Tables for QA
+  mlar2021QATableName = "qa_mlar_table_2021"
+  mlar2021QATableName = ${?MODIFIED_LAR_2021_QA_TABLE}
+  panel2021QATableName = "qa_panel_table_2021"
+  panel2021QATableName = ${?PANEL_2021_QA_TABLE}
+
+  ts2021QATableName = "qa_ts_table_2021"
+  ts2021QATableName = ${?TS_2021_QA_TABLE}
+  ts2021Q1QATableName = "qa_ts_table_2021_q1"
+  ts2021Q1QATableName = ${?TS_2021_Q1_QATABLE}
+  ts2021Q2QATableName = "qa_ts_table_2021_q2"
+  ts2021Q2QATableName = ${?TS_2021_Q2_QATABLE}
+  ts2021Q3QATableName = "qa_ts_table_2021_q3"
+  ts2021Q3QATableName = ${?TS_2021_Q3_QATABLE}
+
+  lar2021QATableName = "qa_lar_table_2021"
+  lar2021QATableName = ${?LAR_2021_QA_TABLE}
+  lar2021QALoanLimitTableName = "qa_lar_loanlimit_table_2021"
+  lar2021QALoanLimitTableName = ${?LAR_LOANLIMIT_2021_QA_TABLE}
+  lar2021Q1QATableName = "qa_lar_table_2021_q1"
+  lar2021Q1QATableName = ${?LAR_2021_Q1_QA_TABLE}
+  lar2021Q2QATableName = "qa_lar_table_2021_q2"
+  lar2021Q2QATableName = ${?LAR_2021_Q2_QA_TABLE}
+  lar2021Q3QATableName = "qa_lar_table_2021_q3"
+  lar2021Q3QATableName = ${?LAR_2021_Q3_QA_TABLE}
+
   //Common PG Email Table
   emailTableName ="institutions_emails_2018"
   emailTableName =${?EMAIL_TABLE}

hmda-data-publisher/src/main/scala/hmda/publisher/api/DataPublisherHttpApi.scala

@@ -44,12 +44,14 @@ private class DataPublisherHttpApi(
       case Schedules.LarSchedulerLoanLimit2019 => larScheduler
       case Schedules.LarSchedulerLoanLimit2020 => larScheduler
       case Schedules.LarSchedulerQuarterly2020 => larScheduler
+      case Schedules.LarSchedulerQuarterly2021 => larScheduler
       case Schedules.TsPublicScheduler2018     => tsPublicScheduler
       case Schedules.TsPublicScheduler2019     => tsPublicScheduler
       case Schedules.TsScheduler2018           => tsScheduler
       case Schedules.TsScheduler2019           => tsScheduler
       case Schedules.TsScheduler2020           => tsScheduler
       case Schedules.TsSchedulerQuarterly2020  => tsScheduler
+      case Schedules.TsSchedulerQuarterly2021  => tsScheduler
     }
     receiver ! msg
   }

hmda-data-publisher/src/main/scala/hmda/publisher/helper/PGTableNameLoader.scala

@@ -59,6 +59,34 @@ trait PGTableNameLoader {
   val lar2020Q2QATableName: String = pgTableConfig.getString("lar2020Q2QATableName")
   val lar2020Q3QATableName: String = pgTableConfig.getString("lar2020Q3QATableName")
 
+  //2021 table names
+  val mlar2021TableName: String   = pgTableConfig.getString("mlar2021TableName")
+  val mlar2021QATableName: String = pgTableConfig.getString("mlar2021QATableName")
+
+  val panel2021TableName: String   = pgTableConfig.getString("panel2021TableName")
+  val panel2021QATableName: String = pgTableConfig.getString("panel2021QATableName")
+
+  val ts2021TableName: String   = pgTableConfig.getString("ts2021TableName")
+  val ts2021Q1TableName: String = pgTableConfig.getString("ts2021Q1TableName")
+  val ts2021Q2TableName: String = pgTableConfig.getString("ts2021Q2TableName")
+  val ts2021Q3TableName: String = pgTableConfig.getString("ts2021Q3TableName")
+
+  val ts2021QATableName: String   = pgTableConfig.getString("ts2021QATableName")
+  val ts2021Q1QATableName: String = pgTableConfig.getString("ts2021Q1QATableName")
+  val ts2021Q2QATableName: String = pgTableConfig.getString("ts2021Q2QATableName")
+  val ts2021Q3QATableName: String = pgTableConfig.getString("ts2021Q3QATableName")
+
+  val lar2021TableName: String   = pgTableConfig.getString("lar2021TableName")
+  val lar2021QALoanLimitTableName: String = pgTableConfig.getString("lar2021QALoanLimitTableName")
+  val lar2021Q1TableName: String = pgTableConfig.getString("lar2021Q1TableName")
+  val lar2021Q2TableName: String = pgTableConfig.getString("lar2021Q2TableName")
+  val lar2021Q3TableName: String = pgTableConfig.getString("lar2021Q3TableName")
+
+  val lar2021QATableName: String   = pgTableConfig.getString("lar2021QATableName")
+  val lar2021Q1QATableName: String = pgTableConfig.getString("lar2021Q1QATableName")
+  val lar2021Q2QATableName: String = pgTableConfig.getString("lar2021Q2QATableName")
+  val lar2021Q3QATableName: String = pgTableConfig.getString("lar2021Q3QATableName")
+
   //common table names
   val emailTableName: String = pgTableConfig.getString("emailTableName")
 

hmda-data-publisher/src/main/scala/hmda/publisher/helper/QuarterTimeBarrier.scala

@@ -2,6 +2,7 @@ package hmda.publisher.helper
 
 import java.time.{Clock, LocalDate}
 
+import hmda.publisher.HmdaDataPublisherApp.log
 import hmda.publisher.validation.PublishingGuard.Period
 import hmda.util.BankFilterUtils.config
 import hmda.util.Filer
@@ -10,9 +11,12 @@ class QuarterTimeBarrier(clock: Clock) {
 
   def runIfStillRelevant[T](quarter: Period.Quarter)(thunk: => T): Option[T] = {
     val now = LocalDate.now(clock)
-    if (now.isBefore(QuarterTimeBarrier.getEndDateForQuarter(quarter).plusDays(8))) {
+    if ((now.isAfter(QuarterTimeBarrier.getFirstDateForQuarter(quarter)) &&
+      now.isBefore(QuarterTimeBarrier.getEndDateForQuarter(quarter).plusDays(8)))||
+      now.isEqual(QuarterTimeBarrier.getFirstDateForQuarter(quarter))) {
       Some(thunk)
     } else {
+      log.info("Data Publisher QuarterTimeBarrier, quarterly filing closed for: " + quarter + "\n")
       None
     }
   }
@@ -28,6 +32,20 @@ object QuarterTimeBarrier {
       case Period.y2020Q1 => LocalDate.ofYearDay(2020,rulesConfig.qf.q1.endDayOfYear)
       case Period.y2020Q2 => LocalDate.ofYearDay(2020,rulesConfig.qf.q2.endDayOfYear)
       case Period.y2020Q3 => LocalDate.ofYearDay(2020,rulesConfig.qf.q3.endDayOfYear)
+      case Period.y2021Q1 => LocalDate.ofYearDay(2021,rulesConfig.qf.q1.endDayOfYear)
+      case Period.y2021Q2 => LocalDate.ofYearDay(2021,rulesConfig.qf.q2.endDayOfYear)
+      case Period.y2021Q3 => LocalDate.ofYearDay(2021,rulesConfig.qf.q3.endDayOfYear)
+    }
+  }
+
+  def getFirstDateForQuarter(quarter: Period.Quarter): LocalDate = {
+    quarter match {
+      case Period.y2020Q1 => LocalDate.ofYearDay(2020,rulesConfig.qf.q1.startDayOfYear)
+      case Period.y2020Q2 => LocalDate.ofYearDay(2020,rulesConfig.qf.q2.startDayOfYear)
+      case Period.y2020Q3 => LocalDate.ofYearDay(2020,rulesConfig.qf.q3.startDayOfYear)
+      case Period.y2021Q1 => LocalDate.ofYearDay(2021,rulesConfig.qf.q1.startDayOfYear)
+      case Period.y2021Q2 => LocalDate.ofYearDay(2021,rulesConfig.qf.q2.startDayOfYear)
+      case Period.y2021Q3 => LocalDate.ofYearDay(2021,rulesConfig.qf.q3.startDayOfYear)
     }
   }