Migration to MySQL 5.7 - ensure all group by clauses are valid

fineract-provider/src/main/java/org/apache/fineract/accounting/glaccount/service/GLAccountReadPlatformServiceImpl.java

@@ -127,7 +127,7 @@ public List<GLAccountData> retrieveAllGLAccounts(final Integer accountClassifica
             if (associationParametersData.isRunningBalanceRequired()) {
                 sql = sql + " and gl_j.id in (select t1.id from (select t2.account_id, max(t2.id) as id from "
                         + "(select id, max(entry_date) as entry_date, account_id from acc_gl_journal_entry where is_running_balance_calculated = 1 "
-                        + "group by account_id desc) t3 inner join acc_gl_journal_entry t2 on t2.account_id = t3.account_id and t2.entry_date = t3.entry_date "
+                        + "group by account_id desc, id) t3 inner join acc_gl_journal_entry t2 on t2.account_id = t3.account_id and t2.entry_date = t3.entry_date "
                         + "group by t2.account_id desc) t1)";
             }
         }

fineract-provider/src/main/java/org/apache/fineract/mix/service/XBRLResultServiceImpl.java

@@ -148,7 +148,7 @@ private String getAccountSql(final Date startDate, final Date endDate) {
                 + " and acc_gl_journal_entry.entry_date > " + startDate +
                 // "and (acc_gl_journal_entry.office_id=${branch} or ${branch}=1) "
                 // +
-                " group by name " + "order by glcode) credits " + "on debits.glcode=credits.glcode;";
+                " group by name, glcode " + "order by glcode) credits " + "on debits.glcode=credits.glcode;";
         return sql;
     }
 

fineract-provider/src/main/java/org/apache/fineract/organisation/staff/service/StaffReadPlatformServiceImpl.java

@@ -282,7 +282,7 @@ public Object[] hasAssociatedItems(final Long staffId){
                       " left outer join m_loan l on staff.id = l.loan_officer_id and l.loan_status_id < ? " +
                       " left outer join m_savings_account s on c.staff_id = s.field_officer_id and s.status_enum < ? "+ 
                       " where  staff.id  = ? "+
-                      " group by staff.id";
+                      " group by staff.id, client, grp, loan, sav";
 
 
 		List<Map<String, Object>> result = this.jdbcTemplate.queryForList(

fineract-provider/src/main/java/org/apache/fineract/portfolio/collectionsheet/service/CollectionSheetReadPlatformServiceImpl.java

@@ -263,10 +263,12 @@ public String collectionSheetSchema(final boolean isCenterCollection) {
 
             sql.append("and (gp.status_enum = 300 or (gp.status_enum = 600 and gp.closedon_date >= :dueDate)) ")
                     .append("and (cl.status_enum = 300 or (cl.status_enum = 600 and cl.closedon_date >= :dueDate)) ")
-                    .append("GROUP BY gp.id ,cl.id , ln.id ORDER BY gp.id , cl.id , ln.id ").append(") loandata ")
+                    .append("GROUP BY gp.id, cl.id, ln.id, ca.attendance_type_enum ORDER BY gp.id , cl.id , ln.id ")
+                    .append(") loandata ")
                     .append("LEFT JOIN m_loan_charge lc ON lc.loan_id = loandata.loanId AND lc.is_paid_derived = 0 AND lc.is_active = 1 ")
                     .append("AND ( lc.due_for_collection_as_of_date  <= :dueDate OR lc.charge_time_enum = 1) ")
-                    .append("GROUP BY loandata.groupId, ").append("loandata.clientId, ").append("loandata.loanId ")
+                    .append("GROUP BY loandata.groupId, loandata.clientId, loandata.loanId ")
+                    .append(", loandata.principalDue, loandata.interestDue, loandata.feeDue, loandata.attendanceTypeId ")
                     .append("ORDER BY loandata.groupId, ").append("loandata.clientId, ").append("loandata.loanId ");
 
             return sql.toString();

fineract-provider/src/main/java/org/apache/fineract/portfolio/group/service/CenterReadPlatformServiceImpl.java

@@ -575,7 +575,7 @@ public Collection<StaffCenterData> retriveAllCentersByMeetingDate(final Long off
         if (staffId != null) {
             sql += " and g.staff_id=? ";
             sql += "and lrs.duedate<='" + passeddate + "' and l.loan_type_enum=3";
-            sql += " group by c.id,ci.id";
+            sql += " group by c.id, ci.id, g.account_no, g.external_id, g.status_enum, g.activation_date, g.hierarchy";
             centerDataArray = this.jdbcTemplate.query(sql, centerCalendarMapper, new Object[] { passeddate, passeddate, passeddate, passeddate,
                     passeddate, passeddate, officeId, staffId });
         } else {

fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/guarantor/service/GuarantorReadPlatformServiceImpl.java

@@ -74,7 +74,7 @@ public List<GuarantorData> retrieveGuarantorsForValidLoan(final Long loanId) {
     public List<GuarantorData> retrieveGuarantorsForLoan(final Long loanId) {
         final GuarantorMapper rm = new GuarantorMapper();
         String sql = "select " + rm.schema();
-        sql += " where loan_id = ?  group by g.id,gfd.id";
+        sql += " where loan_id = ?  group by g.id,gfd.id, gt.id";
         final List<GuarantorData> guarantorDatas = this.jdbcTemplate.query(sql, rm,
                 new Object[] { AccountAssociationType.GUARANTOR_ACCOUNT_ASSOCIATION.getValue(),
                         loanId });
@@ -91,7 +91,7 @@ public List<GuarantorData> retrieveGuarantorsForLoan(final Long loanId) {
     public GuarantorData retrieveGuarantor(final Long loanId, final Long guarantorId) {
         final GuarantorMapper rm = new GuarantorMapper();
         String sql = "select " + rm.schema();
-        sql += " where g.loan_id = ? and g.id = ? group by g.id,gfd.id";
+        sql += " where g.loan_id = ? and g.id = ? group by g.id, gfd.id, gt.id";
         final GuarantorData guarantorData = this.jdbcTemplate.queryForObject(sql, rm,
                 new Object[] { AccountAssociationType.GUARANTOR_ACCOUNT_ASSOCIATION.getValue(),
                         loanId, guarantorId });

fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/service/LoanChargeReadPlatformServiceImpl.java

@@ -398,7 +398,7 @@ public LoanChargeData mapRow(final ResultSet rs, @SuppressWarnings("unused") fin
 
     private Collection<LoanInstallmentChargeData> retrieveInstallmentLoanChargesForAccrual(Long loanChargeId) {
         final LoanInstallmentChargeAccrualMapper rm = new LoanInstallmentChargeAccrualMapper();
-        String sql = "select " + rm.schema() + " where lic.loan_charge_id= ?  group by lsi.installment";
+        String sql = "select " + rm.schema() + " where lic.loan_charge_id= ?  group by lsi.installment, lsi.duedate, lic.amount_outstanding_derived, lic.amount, lic.is_paid_derived, lic.amount_waived_derived, lic.waived";
         Collection<LoanInstallmentChargeData> chargeDatas = this.jdbcTemplate.query(sql, rm,
                 new Object[] { LoanTransactionType.ACCRUAL.getValue(), loanChargeId });
         final Map<Integer, LoanInstallmentChargeData> installmentChargeDatas = new HashMap<>();

fineract-provider/src/main/java/org/apache/fineract/portfolio/loanaccount/service/LoanReadPlatformServiceImpl.java

@@ -1530,7 +1530,7 @@ public Integer retriveLoanCounter(final Long clientId, Long productId) {
     @Override
     public Collection<DisbursementData> retrieveLoanDisbursementDetails(final Long loanId) {
         final LoanDisbursementDetailMapper rm = new LoanDisbursementDetailMapper();
-        final String sql = "select " + rm.schema() + " where dd.loan_id=? group by dd.id order by dd.expected_disburse_date";
+        final String sql = "select " + rm.schema() + " where dd.loan_id=? group by dd.id, lc.amount_waived_derived order by dd.expected_disburse_date";
         return this.jdbcTemplate.query(sql, rm, new Object[] { loanId });
     }
 

fineract-provider/src/main/java/org/apache/fineract/portfolio/savings/service/DepositAccountReadPlatformServiceImpl.java

@@ -523,7 +523,7 @@ public Collection<Map<String, Object>> retriveDataForRDScheduleCreation() {
         sb.append(" inner join m_calendar mc  on mc.id = mci.calendar_id and mc.calendar_type_enum = ?");
         sb.append(" inner join m_mandatory_savings_schedule ms on ms.savings_account_id = dat.savings_account_id and ms.duedate > ?");
         sb.append(" where dat.deposit_period is null");
-        sb.append(" group by ms.savings_account_id");
+        sb.append(" group by ms.savings_account_id, rd.mandatory_recommended_deposit_amount, mc.recurrence");
 
         return this.jdbcTemplate.queryForList(sb.toString(), SavingsAccountStatusType.ACTIVE.getValue(),
                 CalendarEntityType.SAVINGS.getValue(), CalendarType.COLLECTION.getValue(),

fineract-provider/src/main/java/org/apache/fineract/portfolio/search/service/SearchReadPlatformServiceImpl.java

@@ -262,7 +262,7 @@ public String schema(final AdHocQuerySearchConditions searchConditions, final Ma
                 }
             }
 
-            sql.append(" group by mo.id) a ");
+            sql.append(" group by mo.id, mp.name) a ");
 
             // update isWhereClauseAdded to false to add filters for derived
             // table

fineract-provider/src/main/java/org/apache/fineract/spm/service/ScorecardReadPlatformServiceImpl.java

@@ -115,7 +115,7 @@ Collection<ScorecardData> updateScorecardValues(Collection<ScorecardData> scorec
     public Collection<ScorecardData> retrieveScorecardBySurvey(Long surveyId) {
         this.context.authenticatedUser();
         ScorecardMapper scm = new ScorecardMapper();
-        String sql = "select " + scm.schema() + " where sc.survey_id = ? " + " group by sc.survey_id, sc.client_id ";
+        String sql = "select " + scm.schema() + " where sc.survey_id = ? " + " group by sc.survey_id, sc.client_id, sc.id ";
         Collection<ScorecardData> scorecardDatas = this.jdbcTemplate.query(sql, scm, new Object[] { surveyId });
         updateScorecardValues(scorecardDatas);
         return scorecardDatas;
@@ -125,7 +125,7 @@ public Collection<ScorecardData> retrieveScorecardBySurvey(Long surveyId) {
     public Collection<ScorecardData> retrieveScorecardByClient(Long clientId) {
         this.context.authenticatedUser();
         ScorecardMapper scm = new ScorecardMapper();
-        String sql = "select " + scm.schema() + " where sc.client_id = ? " + " group by sc.survey_id, sc.client_id ";
+        String sql = "select " + scm.schema() + " where sc.client_id = ? " + " group by sc.survey_id, sc.client_id, sc.id ";
         Collection<ScorecardData> scorecardDatas = this.jdbcTemplate.query(sql, scm, new Object[] { clientId });
         updateScorecardValues(scorecardDatas);
         return scorecardDatas;
@@ -135,7 +135,7 @@ public Collection<ScorecardData> retrieveScorecardByClient(Long clientId) {
     public Collection<ScorecardData> retrieveScorecardBySurveyAndClient(Long surveyId, Long clientId) {
         this.context.authenticatedUser();
         ScorecardMapper scm = new ScorecardMapper();
-        String sql = "select " + scm.schema() + " where sc.survey_id = ? and sc.client_id = ? " + " group by sc.survey_id, sc.client_id ";
+        String sql = "select " + scm.schema() + " where sc.survey_id = ? and sc.client_id = ? " + " group by sc.survey_id, sc.client_id, sc.id ";
         Collection<ScorecardData> scorecardDatas = this.jdbcTemplate.query(sql, scm, new Object[] { surveyId, clientId });
         updateScorecardValues(scorecardDatas);
         return scorecardDatas;

fineract-provider/src/main/resources/sql/migrations/core_db/V118_7__move_withdrawal_annual_fee_to_charges.sql

@@ -49,7 +49,7 @@ begin
 
 	declare cursor1 cursor for
       select sa.currency_code, sa.withdrawal_fee_type_enum, sa.withdrawal_fee_amount
-      from m_savings_account sa where sa.withdrawal_fee_amount is not null and sa.withdrawal_fee_amount > 0 group by sa.currency_code, sa.withdrawal_fee_type_enum;
+      from m_savings_account sa where sa.withdrawal_fee_amount is not null and sa.withdrawal_fee_amount > 0 group by sa.currency_code, sa.withdrawal_fee_type_enum, sa.withdrawal_fee_amount;
 
 	declare cursor2 cursor for
       select sa.id, sa.withdrawal_fee_amount from m_savings_account sa where sa.currency_code=v_currency_code and sa.withdrawal_fee_type_enum=v_withdrawal_fee_type_enum and sa.withdrawal_fee_amount is not null and sa.withdrawal_fee_amount > 0;
@@ -164,7 +164,7 @@ begin
 
 	declare cursor1 cursor for
       select sa.currency_code, sa.annual_fee_amount
-      from m_savings_account sa where sa.annual_fee_amount is not null and sa.annual_fee_on_month is not null and sa.annual_fee_on_day is not null group by sa.currency_code;
+      from m_savings_account sa where sa.annual_fee_amount is not null and sa.annual_fee_on_month is not null and sa.annual_fee_on_day is not null group by sa.currency_code, sa.annual_fee_amount;
 
 	declare cursor2 cursor for
       select sa.id, sa.annual_fee_amount, sa.annual_fee_on_month, sa.annual_fee_on_day, sa.annual_fee_next_due_date from m_savings_account sa where sa.currency_code=v_currency_code and sa.annual_fee_amount is not null and sa.annual_fee_on_month is not null and sa.annual_fee_on_day is not null;

fineract-provider/src/main/resources/sql/migrations/core_db/V145__add_force_password_reset_in_c_configuration.sql

@@ -19,10 +19,10 @@
 
 INSERT INTO `c_configuration` (`id`, `name`, `value`, `enabled`) VALUES (NULL, 'force-password-reset-days', '0', '0');
 
-ALTER TABLE  `m_appuser` ADD  `last_time_password_updated` DATE NOT NULL ,
+ALTER TABLE  `m_appuser` ADD  `last_time_password_updated` DATE NOT NULL DEFAULT '1970-01-01',
 ADD INDEX (  `last_time_password_updated` ) ;
 
-UPDATE  `m_appuser` SET  `last_time_password_updated` =  NOW() WHERE  `m_appuser`.`last_time_password_updated` ='0000-00-00';
+UPDATE  `m_appuser` SET  `last_time_password_updated` =  NOW() WHERE  `m_appuser`.`last_time_password_updated` ='1970-01-01';
 
 CREATE TABLE IF NOT EXISTS `m_appuser_previous_password` (
   `id` bigint(20) NOT NULL AUTO_INCREMENT,

fineract-provider/src/main/resources/sql/migrations/core_db/V313__multi_rescheduling_script.sql

@@ -123,6 +123,7 @@ mlrr.loan_id,
  where mlt.reshedule_request_id is not null
  and (mlt.term_type =8 or mlt.term_type = 7)
  and mlt.reshedule_request_id = mlrr.id
+ group by mlt.id
  order by mlt.term_type desc limit 1) parent_id
 from m_loan_reschedule_request mlrr 
 where (mlrr.grace_on_interest is not null or mlrr.grace_on_principal is not null));

fineract-provider/src/main/resources/sql/migrations/core_db/V343__scheduled_email_campaign_status.sql

@@ -0,0 +1,20 @@
++--
++-- Licensed to the Apache Software Foundation (ASF) under one
++-- or more contributor license agreements. See the NOTICE file
++-- distributed with this work for additional information
++-- regarding copyright ownership. The ASF licenses this file
++-- to you under the Apache License, Version 2.0 (the
++-- "License"); you may not use this file except in compliance
++-- with the License. You may obtain a copy of the License at
++--
++-- http://www.apache.org/licenses/LICENSE-2.0
++--
++-- Unless required by applicable law or agreed to in writing,
++-- software distributed under the License is distributed on an
++-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
++-- KIND, either express or implied. See the License for the
++-- specific language governing permissions and limitations
++-- under the License.
++--
++
++ALTER TABLE `scheduled_email_campaign` ADD `previous_run_status` VARCHAR(10) NULL;