From 0efde1bbde8429e5fad82ea92f3a0567a289d213 Mon Sep 17 00:00:00 2001 From: rkodev <43806892+rkodev@users.noreply.github.com> Date: Mon, 15 Jul 2019 19:21:24 +0300 Subject: [PATCH 1/5] Added an intermediate state during sync of locally processed events --- gradle.properties | 2 +- .../repository/EventClientRepository.java | 26 ++++++++++++++++--- .../sync/ClientProcessorForJava.java | 15 +++++++++++ 3 files changed, 38 insertions(+), 5 deletions(-) diff --git a/gradle.properties b/gradle.properties index 21e28a4c5..403d131da 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ -VERSION_NAME=1.6.45-SNAPSHOT +VERSION_NAME=1.6.46-SNAPSHOT VERSION_CODE=1 GROUP=org.smartregister POM_SETTING_DESCRIPTION=OpenSRP Client Core Application diff --git a/opensrp-app/src/main/java/org/smartregister/repository/EventClientRepository.java b/opensrp-app/src/main/java/org/smartregister/repository/EventClientRepository.java index 2ef79bf1a..3281b3133 100644 --- a/opensrp-app/src/main/java/org/smartregister/repository/EventClientRepository.java +++ b/opensrp-app/src/main/java/org/smartregister/repository/EventClientRepository.java @@ -39,6 +39,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; + import static org.smartregister.AllConstants.ROWID; /** @@ -1262,7 +1263,7 @@ public List getEventsByBaseEntityIdsAndSyncStatus(String syncStatus * * @param lastRowId * @return JsonData which contains a {@link JSONArray} and the maximum row id in the array - * of {@link Event}s returned. This enables this method to be called again for the consequent batches + * of {@link Event}s returned. This enables this method to be called again for the consequent batches */ @Nullable public JsonData getEvents(long lastRowId, int limit) { @@ -1337,8 +1338,8 @@ private JSONObject getEventObject(Cursor cursor, long rowId) throws JSONExceptio * * @param lastRowId * @return JsonData which contains a {@link JSONArray} and the maximum row id in the array - * of {@link Client}s returned or {@code null} if no records match the conditions or an exception occurred. - * This enables this method to be called again for the consequent batches + * of {@link Client}s returned or {@code null} if no records match the conditions or an exception occurred. + * This enables this method to be called again for the consequent batches */ @Nullable public JsonData getClients(long lastRowId, int limit) { @@ -1418,7 +1419,7 @@ public void addorUpdateClient(String baseEntityId, JSONObject jsonObject, String } public void addEvent(String baseEntityId, JSONObject jsonObject) {//Backward compatibility - addEvent(baseEntityId, jsonObject, BaseRepository.TYPE_Unsynced); + addEvent(baseEntityId, jsonObject, BaseRepository.TYPE_Task_Unprocessed); } public void addEvent(String baseEntityId, JSONObject jsonObject, String syncStatus) { @@ -1468,6 +1469,23 @@ public void addEvent(String baseEntityId, JSONObject jsonObject, String syncStat } } + public void markEventAsProcessed(String formSubmissionId, String eventID) { + try { + + ContentValues values = new ContentValues(); + values.put(event_column.formSubmissionId.name(), formSubmissionId); + values.put(event_column.syncStatus.name(), StringUtils.isNotBlank(eventID) ? BaseRepository.TYPE_Synced : BaseRepository.TYPE_Unsynced); + values.put(ROWID, getMaxRowId(Table.event) + 1); + + getWritableDatabase().update(Table.event.name(), + values, + event_column.formSubmissionId.name() + " = ?", + new String[]{formSubmissionId}); + + } catch (Exception e) { + Log.e(getClass().getName(), "Exception", e); + } + } public void markEventAsSynced(String formSubmissionId) { try { diff --git a/opensrp-app/src/main/java/org/smartregister/sync/ClientProcessorForJava.java b/opensrp-app/src/main/java/org/smartregister/sync/ClientProcessorForJava.java index 7ae722095..c81cd2522 100644 --- a/opensrp-app/src/main/java/org/smartregister/sync/ClientProcessorForJava.java +++ b/opensrp-app/src/main/java/org/smartregister/sync/ClientProcessorForJava.java @@ -21,6 +21,7 @@ import org.smartregister.domain.jsonmapping.JsonMapping; import org.smartregister.domain.jsonmapping.Rule; import org.smartregister.domain.jsonmapping.Table; +import org.smartregister.repository.BaseRepository; import org.smartregister.repository.DetailsRepository; import org.smartregister.util.AssetHandler; @@ -73,10 +74,24 @@ public synchronized void processClient(List eventClientList) throws if (eventClient.getClient() != null) { processEvent(eventClient.getEvent(), eventClient.getClient(), clientClassification); } + + completeProcessing(eventClient); } } } + public void completeProcessing(EventClient eventClient) { + if (eventClient == null) + return; + + Event event = eventClient.getEvent(); + if (event == null) + return; + + CoreLibrary.getInstance().context() + .getEventClientRepository().markEventAsProcessed(event.getFormSubmissionId(), event.getEventId()); + } + public Boolean processEvent(Event event, Client client, ClientClassification clientClassification) throws Exception { try { if (event.getCreator() != null) { From 276625b0345629c9c447a515c4f83f8578d70d82 Mon Sep 17 00:00:00 2001 From: rkodev <43806892+rkodev@users.noreply.github.com> Date: Mon, 15 Jul 2019 19:26:37 +0300 Subject: [PATCH 2/5] Codacy fix --- .../main/java/org/smartregister/sync/ClientProcessorForJava.java | 1 - 1 file changed, 1 deletion(-) diff --git a/opensrp-app/src/main/java/org/smartregister/sync/ClientProcessorForJava.java b/opensrp-app/src/main/java/org/smartregister/sync/ClientProcessorForJava.java index c81cd2522..f32e65e63 100644 --- a/opensrp-app/src/main/java/org/smartregister/sync/ClientProcessorForJava.java +++ b/opensrp-app/src/main/java/org/smartregister/sync/ClientProcessorForJava.java @@ -21,7 +21,6 @@ import org.smartregister.domain.jsonmapping.JsonMapping; import org.smartregister.domain.jsonmapping.Rule; import org.smartregister.domain.jsonmapping.Table; -import org.smartregister.repository.BaseRepository; import org.smartregister.repository.DetailsRepository; import org.smartregister.util.AssetHandler; From fa3894b407b16df70c3fde0549976e525de2cb67 Mon Sep 17 00:00:00 2001 From: rkodev <43806892+rkodev@users.noreply.github.com> Date: Tue, 16 Jul 2019 12:07:27 +0300 Subject: [PATCH 3/5] - Added test - Updated event type --- .../repository/BaseRepository.java | 1 + .../repository/EventClientRepository.java | 8 +++-- .../sync/ClientProcessorForJava.java | 11 +++---- .../repository/EventClientRepositoryTest.java | 15 ++++++++- .../sync/ClientProcessorForJavaTest.java | 32 +++++++++++++++++++ 5 files changed, 56 insertions(+), 11 deletions(-) create mode 100644 opensrp-app/src/test/java/org/smartregister/sync/ClientProcessorForJavaTest.java diff --git a/opensrp-app/src/main/java/org/smartregister/repository/BaseRepository.java b/opensrp-app/src/main/java/org/smartregister/repository/BaseRepository.java index 26774bbe3..c3e30dcbe 100644 --- a/opensrp-app/src/main/java/org/smartregister/repository/BaseRepository.java +++ b/opensrp-app/src/main/java/org/smartregister/repository/BaseRepository.java @@ -25,6 +25,7 @@ public class BaseRepository { public static String TYPE_Valid = "Valid"; public static String TYPE_InValid = "Invalid"; public static String TYPE_Task_Unprocessed = "task_unprocessed"; + public static String TYPE_Unprocessed = "unprocessed"; public static String TYPE_Created = "Created"; protected static final String ORDER_BY = " order by "; diff --git a/opensrp-app/src/main/java/org/smartregister/repository/EventClientRepository.java b/opensrp-app/src/main/java/org/smartregister/repository/EventClientRepository.java index 3281b3133..0ba5d9417 100644 --- a/opensrp-app/src/main/java/org/smartregister/repository/EventClientRepository.java +++ b/opensrp-app/src/main/java/org/smartregister/repository/EventClientRepository.java @@ -40,6 +40,8 @@ import java.util.List; import java.util.Map; +import timber.log.Timber; + import static org.smartregister.AllConstants.ROWID; /** @@ -1419,7 +1421,7 @@ public void addorUpdateClient(String baseEntityId, JSONObject jsonObject, String } public void addEvent(String baseEntityId, JSONObject jsonObject) {//Backward compatibility - addEvent(baseEntityId, jsonObject, BaseRepository.TYPE_Task_Unprocessed); + addEvent(baseEntityId, jsonObject, BaseRepository.TYPE_Unprocessed); } public void addEvent(String baseEntityId, JSONObject jsonObject, String syncStatus) { @@ -1465,7 +1467,7 @@ public void addEvent(String baseEntityId, JSONObject jsonObject, String syncStat } } catch (Exception e) { - Log.e(getClass().getName(), "Exception", e); + Timber.e(e); } } @@ -1483,7 +1485,7 @@ public void markEventAsProcessed(String formSubmissionId, String eventID) { new String[]{formSubmissionId}); } catch (Exception e) { - Log.e(getClass().getName(), "Exception", e); + Timber.e(e); } } diff --git a/opensrp-app/src/main/java/org/smartregister/sync/ClientProcessorForJava.java b/opensrp-app/src/main/java/org/smartregister/sync/ClientProcessorForJava.java index f32e65e63..19c22f8e6 100644 --- a/opensrp-app/src/main/java/org/smartregister/sync/ClientProcessorForJava.java +++ b/opensrp-app/src/main/java/org/smartregister/sync/ClientProcessorForJava.java @@ -73,17 +73,11 @@ public synchronized void processClient(List eventClientList) throws if (eventClient.getClient() != null) { processEvent(eventClient.getEvent(), eventClient.getClient(), clientClassification); } - - completeProcessing(eventClient); } } } - public void completeProcessing(EventClient eventClient) { - if (eventClient == null) - return; - - Event event = eventClient.getEvent(); + public void completeProcessing(Event event) { if (event == null) return; @@ -93,6 +87,9 @@ public void completeProcessing(EventClient eventClient) { public Boolean processEvent(Event event, Client client, ClientClassification clientClassification) throws Exception { try { + // mark event as processed regardless of any errors + completeProcessing(event); + if (event.getCreator() != null) { Log.i(TAG, "EVENT from openmrs"); } diff --git a/opensrp-app/src/test/java/org/smartregister/repository/EventClientRepositoryTest.java b/opensrp-app/src/test/java/org/smartregister/repository/EventClientRepositoryTest.java index d981f3f18..269654297 100644 --- a/opensrp-app/src/test/java/org/smartregister/repository/EventClientRepositoryTest.java +++ b/opensrp-app/src/test/java/org/smartregister/repository/EventClientRepositoryTest.java @@ -272,7 +272,7 @@ public void fetchEventClientsByRowId() throws Exception { MatrixCursor matrixCursor = new MatrixCursor(new String[]{"rowid", "json"}); JSONArray eventArray = new JSONArray(ClientData.eventJsonArray); for (int i = 0; i < eventArray.length(); i++) { - matrixCursor.addRow(new String[]{(i+1) + "", eventArray.getJSONObject(i).toString()}); + matrixCursor.addRow(new String[]{(i + 1) + "", eventArray.getJSONObject(i).toString()}); } Mockito.when(sqliteDatabase.rawQuery(org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.any(Object[].class))).thenReturn(matrixCursor); @@ -318,4 +318,17 @@ public void assertcreateTableCallsExecSql() { Mockito.verify(sqliteDatabase, Mockito.times(count)).execSQL(Mockito.anyString()); } + /** + * Events manually saved should always have an unprocessed status + */ + @Test + public void testAddEventDefaultStatus() { + EventClientRepository eventClientRepository = Mockito.spy(new EventClientRepository(repository)); + String baseEntityId = "12345"; + JSONObject jsonObject = Mockito.mock(JSONObject.class); + + eventClientRepository.addEvent(baseEntityId, jsonObject); + + Mockito.verify(eventClientRepository).addEvent(baseEntityId, jsonObject, BaseRepository.TYPE_Unprocessed); + } } \ No newline at end of file diff --git a/opensrp-app/src/test/java/org/smartregister/sync/ClientProcessorForJavaTest.java b/opensrp-app/src/test/java/org/smartregister/sync/ClientProcessorForJavaTest.java new file mode 100644 index 000000000..4bcc2f76d --- /dev/null +++ b/opensrp-app/src/test/java/org/smartregister/sync/ClientProcessorForJavaTest.java @@ -0,0 +1,32 @@ +package org.smartregister.sync; + +import android.content.Context; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.smartregister.BaseUnitTest; +import org.smartregister.domain.db.Event; + +public class ClientProcessorForJavaTest extends BaseUnitTest { + + @Mock + private Context context; + + + @Before + public void setUp() throws Exception { + MockitoAnnotations.initMocks(this); + } + + @Test + public void testProcessEventMarksItSaved() throws Exception { + ClientProcessorForJava clientProcessorForJava = Mockito.spy(new ClientProcessorForJava(context)); + Event mockEvent = Mockito.mock(Event.class); + clientProcessorForJava.processEvent(mockEvent, null, null); + + Mockito.verify(clientProcessorForJava).completeProcessing(mockEvent); + } +} From ec54fd11f491c74b80444c1d469e65ad5d9c45b8 Mon Sep 17 00:00:00 2001 From: rkodev <43806892+rkodev@users.noreply.github.com> Date: Wed, 17 Jul 2019 16:49:12 +0300 Subject: [PATCH 4/5] - Include all unprocessed events to be synced when uploading - Only update unprocessed events when marking it as processed --- .../repository/EventClientRepository.java | 12 ++++++------ .../smartregister/sync/ClientProcessorForJava.java | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/opensrp-app/src/main/java/org/smartregister/repository/EventClientRepository.java b/opensrp-app/src/main/java/org/smartregister/repository/EventClientRepository.java index 0ba5d9417..5b3bf8b6f 100644 --- a/opensrp-app/src/main/java/org/smartregister/repository/EventClientRepository.java +++ b/opensrp-app/src/main/java/org/smartregister/repository/EventClientRepository.java @@ -798,7 +798,7 @@ public Map getUnSyncedEvents(int limit) { + Table.event.name() + " where " + event_column.syncStatus - + " = ? and length(" + + " in (? , ?) and length(" + event_column.json + ")>2 order by " + event_column.updatedAt @@ -806,7 +806,7 @@ public Map getUnSyncedEvents(int limit) { + limit; Cursor cursor = null; try { - cursor = getWritableDatabase().rawQuery(query, new String[]{BaseRepository.TYPE_Unsynced}); + cursor = getWritableDatabase().rawQuery(query, new String[]{BaseRepository.TYPE_Unsynced, BaseRepository.TYPE_Unprocessed}); while (cursor.moveToNext()) { String jsonEventStr = (cursor.getString(0)); @@ -1471,18 +1471,18 @@ public void addEvent(String baseEntityId, JSONObject jsonObject, String syncStat } } - public void markEventAsProcessed(String formSubmissionId, String eventID) { + public void markEventAsProcessed(String formSubmissionId) { try { ContentValues values = new ContentValues(); values.put(event_column.formSubmissionId.name(), formSubmissionId); - values.put(event_column.syncStatus.name(), StringUtils.isNotBlank(eventID) ? BaseRepository.TYPE_Synced : BaseRepository.TYPE_Unsynced); + values.put(event_column.syncStatus.name(), BaseRepository.TYPE_Unsynced); values.put(ROWID, getMaxRowId(Table.event) + 1); getWritableDatabase().update(Table.event.name(), values, - event_column.formSubmissionId.name() + " = ?", - new String[]{formSubmissionId}); + event_column.formSubmissionId.name() + " = ? and " + event_column.syncStatus.name() + " = ? ", + new String[]{formSubmissionId, BaseRepository.TYPE_Unprocessed}); } catch (Exception e) { Timber.e(e); diff --git a/opensrp-app/src/main/java/org/smartregister/sync/ClientProcessorForJava.java b/opensrp-app/src/main/java/org/smartregister/sync/ClientProcessorForJava.java index 19c22f8e6..bac8a6298 100644 --- a/opensrp-app/src/main/java/org/smartregister/sync/ClientProcessorForJava.java +++ b/opensrp-app/src/main/java/org/smartregister/sync/ClientProcessorForJava.java @@ -82,7 +82,7 @@ public void completeProcessing(Event event) { return; CoreLibrary.getInstance().context() - .getEventClientRepository().markEventAsProcessed(event.getFormSubmissionId(), event.getEventId()); + .getEventClientRepository().markEventAsProcessed(event.getFormSubmissionId()); } public Boolean processEvent(Event event, Client client, ClientClassification clientClassification) throws Exception { From 5758458c2f4df5469b4e776a440853061c5d939d Mon Sep 17 00:00:00 2001 From: rkodev <43806892+rkodev@users.noreply.github.com> Date: Wed, 17 Jul 2019 17:30:10 +0300 Subject: [PATCH 5/5] Added java docs --- .../smartregister/repository/EventClientRepository.java | 7 +++++-- .../org/smartregister/sync/ClientProcessorForJava.java | 5 +++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/opensrp-app/src/main/java/org/smartregister/repository/EventClientRepository.java b/opensrp-app/src/main/java/org/smartregister/repository/EventClientRepository.java index 5b3bf8b6f..a16bbadac 100644 --- a/opensrp-app/src/main/java/org/smartregister/repository/EventClientRepository.java +++ b/opensrp-app/src/main/java/org/smartregister/repository/EventClientRepository.java @@ -1471,11 +1471,15 @@ public void addEvent(String baseEntityId, JSONObject jsonObject, String syncStat } } + /** + * Flag an event as locally processed. + * This method only updates locally created and processed events and prevents reprocessing locally + * @param formSubmissionId + */ public void markEventAsProcessed(String formSubmissionId) { try { ContentValues values = new ContentValues(); - values.put(event_column.formSubmissionId.name(), formSubmissionId); values.put(event_column.syncStatus.name(), BaseRepository.TYPE_Unsynced); values.put(ROWID, getMaxRowId(Table.event) + 1); @@ -1493,7 +1497,6 @@ public void markEventAsSynced(String formSubmissionId) { try { ContentValues values = new ContentValues(); - values.put(event_column.formSubmissionId.name(), formSubmissionId); values.put(event_column.syncStatus.name(), BaseRepository.TYPE_Synced); values.put(ROWID, getMaxRowId(Table.event) + 1); diff --git a/opensrp-app/src/main/java/org/smartregister/sync/ClientProcessorForJava.java b/opensrp-app/src/main/java/org/smartregister/sync/ClientProcessorForJava.java index bac8a6298..d6851fcf4 100644 --- a/opensrp-app/src/main/java/org/smartregister/sync/ClientProcessorForJava.java +++ b/opensrp-app/src/main/java/org/smartregister/sync/ClientProcessorForJava.java @@ -77,6 +77,11 @@ public synchronized void processClient(List eventClientList) throws } } + /** + * Call this method to flag the event as processed in the local repository. + * All events valid or otherwise must be flagged to avoid re-processing + * @param event + */ public void completeProcessing(Event event) { if (event == null) return;