From 55e4a96249a23fc76ebd854525c09e9dd735a8c0 Mon Sep 17 00:00:00 2001 From: Nathan Hui Date: Fri, 9 Feb 2024 19:15:16 -0800 Subject: [PATCH] fix: Fixes looping behavior and optimizes upload sizes --- src/cellular/dataUpload.cpp | 149 +++++++++++++++++++++++++---------- src/cellular/dataUpload.hpp | 21 +---- src/cellular/recorder.cpp | 27 ++++--- src/cellular/recorder.hpp | 12 +-- src/cli/conio.hpp | 2 +- src/debug/recorder_debug.cpp | 2 +- src/fileCLI/fileCLI.cpp | 2 +- src/product.hpp | 29 ++++++- 8 files changed, 160 insertions(+), 84 deletions(-) diff --git a/src/cellular/dataUpload.cpp b/src/cellular/dataUpload.cpp index 341cdd75..fe2d626b 100644 --- a/src/cellular/dataUpload.cpp +++ b/src/cellular/dataUpload.cpp @@ -15,78 +15,143 @@ void DataUpload::init(void) { - SF_OSAL_printf("Entering SYSTEM_STATE_DATA_UPLOAD\n"); + SF_OSAL_printf("Entering SYSTEM_STATE_DATA_UPLOAD" __NL__); this->initSuccess = 1; - if(sf::cloud::wait_connect(30000)) + if (sf::cloud::wait_connect(SF_CELL_SIGNAL_TIMEOUT_MS)) { this->initSuccess = 0; } } -STATES_e DataUpload::run(void) +STATES_e DataUpload::can_upload(void) { - - - uint8_t dataEncodeBuffer[DATA_UPLOAD_MAX_BLOCK_LEN]; - char dataPublishBuffer[DATA_UPLOAD_MAX_UPLOAD_LEN]; - char publishName[DU_PUBLISH_ID_NAME_LEN + 1]; - int nBytesToEncode; - size_t nBytesToSend; - - if (!this->initSuccess) + if (!pSystemDesc->pRecorder->hasData()) { - SF_OSAL_printf("Failed to init\n"); return STATE_DEEP_SLEEP; } - memset(dataEncodeBuffer, 0, DATA_UPLOAD_MAX_BLOCK_LEN); - nBytesToEncode = pSystemDesc->pRecorder->getLastPacket(dataEncodeBuffer, DATA_UPLOAD_MAX_BLOCK_LEN, publishName, DU_PUBLISH_ID_NAME_LEN); - if (-1 == nBytesToEncode) + if (!sf::cloud::is_connected()) { - SF_OSAL_printf("Failed to retrive data\n"); - return STATE_CLI; + if (sf::cloud::wait_connect(SF_CELL_SIGNAL_TIMEOUT_MS)) + { + // Lost connection and failed to reconnect + return STATE_DEEP_SLEEP; + } } - SF_OSAL_printf("Publish ID: %s\n", publishName); - if (nBytesToEncode % 4 != 0) + if (pSystemDesc->pWaterSensor->getCurrentStatus()) { - nBytesToEncode += 4 - (nBytesToEncode % 4); + // In water + return STATE_DEPLOYED; } - SF_OSAL_printf("Got %d bytes to encode\n", nBytesToEncode); - memset(dataPublishBuffer, 0, DATA_UPLOAD_MAX_UPLOAD_LEN); - nBytesToSend = DATA_UPLOAD_MAX_UPLOAD_LEN; - urlsafe_b64_encode(dataEncodeBuffer, nBytesToEncode, dataPublishBuffer, &nBytesToSend); - - SF_OSAL_printf("Got %u bytes to upload\n", nBytesToSend); - - if (!sf::cloud::is_connected()) + if (pSystemDesc->pBattery->getVCell() < SF_BATTERY_UPLOAD_VOLTAGE) { - // we're not connected! abort - SF_OSAL_printf("not connected :("); - return STATE_CLI; + return STATE_DEEP_SLEEP; } + // Don't change current state, continue looping + return STATE_UPLOAD; +} - SF_OSAL_printf("Data: %s", dataPublishBuffer); +STATES_e DataUpload::run(void) +{ + uint8_t binary_packet_buffer[SF_PACKET_SIZE]; + char ascii_record_buffer[SF_RECORD_SIZE + 1]; + char publishName[DU_PUBLISH_ID_NAME_LEN + 1]; + int nBytesToEncode; + size_t nBytesToSend; + STATES_e next_state; + int retval; - int sucsess = sf::cloud::publish_blob(publishName, dataPublishBuffer); - if (!sucsess) + if (!this->initSuccess) { - SF_OSAL_printf("Failed to upload data %d" __NL__, sucsess); + SF_OSAL_printf("Failed to init\n"); + return STATE_DEEP_SLEEP; } - SF_OSAL_printf("Uploaded record %s\n", dataPublishBuffer); - Particle.process(); - - pSystemDesc->pRecorder->popLastPacket(nBytesToEncode); + while ((next_state = can_upload()) == STATE_UPLOAD) + { + memset(binary_packet_buffer, 0, SF_PACKET_SIZE); + nBytesToEncode = pSystemDesc->pRecorder->getLastPacket(binary_packet_buffer, SF_PACKET_SIZE, publishName, DU_PUBLISH_ID_NAME_LEN); + switch (nBytesToEncode) + { + case -2: + // We already know that there is data, but we aren't able to retrieve + // it. This can indicate recorder failure. + FLOG_AddError(FLOG_UPL_OPEN_FAIL, 0); + return STATE_DEEP_SLEEP; + case -1: + case -3: + // Either active session (bug) or buffer overflow (bug) + SF_OSAL_printf("Failed to retrieve data: %d" __NL__, nBytesToEncode); + return STATE_CLI; + default: + break; + } + + SF_OSAL_printf("Publish ID: %s" __NL__, publishName); + + // 32-bit alignment? 4 byte alignment? + if (nBytesToEncode % 4 != 0) + { + nBytesToEncode += 4 - (nBytesToEncode % 4); + } + SF_OSAL_printf("Got %d bytes to encode" __NL__, nBytesToEncode); + + // Encode + memset(ascii_record_buffer, 0, SF_RECORD_SIZE + 1); + nBytesToSend = SF_RECORD_SIZE + 1; + if (retval = urlsafe_b64_encode(binary_packet_buffer, nBytesToEncode, ascii_record_buffer, &nBytesToSend)) + { + // size limit violation is bug + SF_OSAL_printf("Failed to encode: %d" __NL__, retval); + return STATE_CLI; + } + + SF_OSAL_printf("Got %u bytes to upload" __NL__, nBytesToSend); + + SF_OSAL_printf("Data: %s" __NL__, ascii_record_buffer); + + switch ((retval = sf::cloud::publish_blob(publishName, ascii_record_buffer))) + { + default: + case sf::cloud::OVERSIZE_DATA: + case sf::cloud::OVERSIZE_NAME: + SF_OSAL_printf("Failed to publish: %d" __NL__, retval); + return STATE_CLI; + case sf::cloud::NOT_CONNECTED: + FLOG_AddError(FLOG_UPL_CONNECT_FAIL, 1); + return STATE_DEEP_SLEEP; + case sf::cloud::PUBLISH_FAIL: + FLOG_AddError(FLOG_UPL_PUB_FAIL, 0); + return STATE_DEEP_SLEEP; + case sf::cloud::SUCCESS: + break; + } + + SF_OSAL_printf("Uploaded record" __NL__); + + Particle.process(); + + switch (pSystemDesc->pRecorder->popLastPacket(nBytesToEncode)) + { + case -1: + case -2: + // This is a bug + SF_OSAL_printf("Session failure" __NL__); + return STATE_CLI; + case 0: + break; + } + } + return next_state; - return STATE_CLI; } void DataUpload::exit(void) { - if(sf::cloud::wait_disconnect(5000)) + if (sf::cloud::wait_disconnect(5000)) { FLOG_AddError(FLOG_CELL_DISCONN_FAIL, 0); } diff --git a/src/cellular/dataUpload.hpp b/src/cellular/dataUpload.hpp index 690b6f60..dac4bafc 100644 --- a/src/cellular/dataUpload.hpp +++ b/src/cellular/dataUpload.hpp @@ -10,23 +10,7 @@ * RGB LED state is handled by the system theme. */ -#if SF_UPLOAD_ENCODING == SF_UPLOAD_BASE85 -/** - * How many bytes to store chunks of data in on the SPI flash. - * - * 496 * 5/4 (base85 encoding compression rate) = 620 which is less than the 622 - * bytes which is the maximum size of publish events. - */ -#define DATA_UPLOAD_MAX_BLOCK_LEN 496 -#elif SF_UPLOAD_ENCODING == SF_UPLOAD_BASE64 || SF_UPLOAD_ENCODING == SF_UPLOAD_BASE64URL -/** - * How many bytes to store chunks of data in on the SPI flash. - * - * 466 * 4/3 (base64 encoding compression rate) = 621 which is less than the 622 - * bytes which is the maximum size of publish events. - */ -#define DATA_UPLOAD_MAX_BLOCK_LEN 466 -#endif + /** * @brief Number of bytes to buffer for upload @@ -38,7 +22,7 @@ * @brief Minimum time between publish * */ -#define DATA_UPLOAD_MIN_PUBLISH_TIME_MS 1000 +#define DATA_UPLOAD_MIN_PUBLISH_TIME_MS SF_UPLOAD_MS_PER_TRANSMIT /** * @brief Publish Event Name Length (not including NULL terminator) @@ -63,5 +47,6 @@ class DataUpload : public Task{ int initSuccess; system_tick_t lastConnectTime; STATES_e exitState(void); + STATES_e can_upload(void); }; #endif \ No newline at end of file diff --git a/src/cellular/recorder.cpp b/src/cellular/recorder.cpp index 3f202353..81848599 100644 --- a/src/cellular/recorder.cpp +++ b/src/cellular/recorder.cpp @@ -301,15 +301,15 @@ int Recorder::getLastPacket(void* pBuffer, } current_length = session.getLength(); - if (current_length % REC_MAX_PACKET_SIZE == 0) + if (current_length % SF_PACKET_SIZE == 0) { // only full packets available - bytes_to_read = REC_MAX_PACKET_SIZE; + bytes_to_read = SF_PACKET_SIZE; } else { // partial packet - bytes_to_read = current_length % REC_MAX_PACKET_SIZE; + bytes_to_read = current_length % SF_PACKET_SIZE; } #ifdef REC_DEBUG @@ -328,7 +328,7 @@ int Recorder::getLastPacket(void* pBuffer, session.seek(newLength); bytesRead = session.read(pBuffer, bytes_to_read); snprintf((char*)pName, nameLen, "Sfin-%s-%s-%d", pSystemDesc->deviceID, - name, newLength / REC_MAX_PACKET_SIZE); + name, newLength / SF_PACKET_SIZE); session.close(); return bytesRead; } @@ -337,7 +337,8 @@ int Recorder::getLastPacket(void* pBuffer, * @brief Trims the last block with specified length from the recorder * * @param len Length of block to trim - * @return int 1 if successful, otherwise 0 + * @return int 0 if successful, otherwise error code. -1 if another session is + * already open. -2 if previous session unopenable. */ int Recorder::popLastPacket(size_t len) { @@ -362,19 +363,19 @@ int Recorder::popLastPacket(size_t len) #ifdef REC_DEBUG SF_OSAL_printf("Failed to open last session" __NL__); #endif - return -1; + return -2; } current_length = session.getLength(); - if (current_length % REC_MAX_PACKET_SIZE == 0) + if (current_length % SF_PACKET_SIZE == 0) { // only full packets available - bytes_to_pop = REC_MAX_PACKET_SIZE; + bytes_to_pop = SF_PACKET_SIZE; } else { // partial packet - bytes_to_pop = current_length % REC_MAX_PACKET_SIZE; + bytes_to_pop = current_length % SF_PACKET_SIZE; } newLength = current_length - bytes_to_pop; @@ -388,7 +389,7 @@ int Recorder::popLastPacket(size_t len) session.truncate(newLength); session.close(); } - return 1; + return 0; } int Recorder::setSessionTime(uint32_t session_time) @@ -473,16 +474,16 @@ int Recorder::putBytes(const void* pData, size_t nBytes) FLOG_AddError(FLOG_REC_SESSION_CLOSED, 0); return 1; } - if (nBytes > (REC_MAX_PACKET_SIZE - this->dataIdx)) + if (nBytes > (SF_PACKET_SIZE - this->dataIdx)) { // data will not fit, flush and clear // pad 0 - for (; this->dataIdx < REC_MAX_PACKET_SIZE; this->dataIdx++) + for (; this->dataIdx < SF_PACKET_SIZE; this->dataIdx++) { this->dataBuffer[this->dataIdx] = 0; } // SF_OSAL_printf("Flushing\n"); - this->pSession->write(this->dataBuffer, REC_MAX_PACKET_SIZE); + this->pSession->write(this->dataBuffer, SF_PACKET_SIZE); memset(this->dataBuffer, 0, REC_MEMORY_BUFFER_SIZE); this->dataIdx = 0; diff --git a/src/cellular/recorder.hpp b/src/cellular/recorder.hpp index 3ae28318..593146c8 100644 --- a/src/cellular/recorder.hpp +++ b/src/cellular/recorder.hpp @@ -15,14 +15,13 @@ */ #define REC_SESSION_NAME_MAX_LEN 64 -#define REC_MEMORY_BUFFER_SIZE 512 -#if SF_UPLOAD_ENCODING == SF_UPLOAD_BASE85 -#define REC_MAX_PACKET_SIZE 496 -#elif SF_UPLOAD_ENCODING == SF_UPLOAD_BASE64 || SF_UPLOAD_ENCODING == SF_UPLOAD_BASE64URL -#define REC_MAX_PACKET_SIZE 466 +#define REC_MEMORY_BUFFER_SIZE 1024 +#if REC_MEMORY_BUFFER_SIZE < SF_PACKET_SIZE +#error REC_MEMORY_BUFFER_SIZE < SF_PACKET_SIZE #endif + class Recorder { public: @@ -60,7 +59,8 @@ class Recorder * @brief Trims the last block with specified length from the recorder * * @param len Length of block to trim - * @return int 1 if successful, otherwise 0 + * @return int 0 if successful, otherwise error code. -1 if another session + * is already open. -2 if previous session unopenable. */ int popLastPacket(size_t len); /** diff --git a/src/cli/conio.hpp b/src/cli/conio.hpp index 3ed7188f..aae5f026 100644 --- a/src/cli/conio.hpp +++ b/src/cli/conio.hpp @@ -3,7 +3,7 @@ #include -#define SF_OSAL_PRINTF_BUFLEN 1024 +#define SF_OSAL_PRINTF_BUFLEN 1536 #ifdef __cplusplus extern "C" diff --git a/src/debug/recorder_debug.cpp b/src/debug/recorder_debug.cpp index 4d2f04fd..6201f4ea 100644 --- a/src/debug/recorder_debug.cpp +++ b/src/debug/recorder_debug.cpp @@ -182,7 +182,7 @@ void REC_testPopLastPacket(void) Recorder* pRecorder = pSystemDesc->pRecorder; int retval; - retval = pRecorder->popLastPacket(REC_MAX_PACKET_SIZE); + retval = pRecorder->popLastPacket(SF_PACKET_SIZE); SF_OSAL_printf("Returned %d" __NL__, retval); } \ No newline at end of file diff --git a/src/fileCLI/fileCLI.cpp b/src/fileCLI/fileCLI.cpp index bfce5b89..6e0ea9c7 100644 --- a/src/fileCLI/fileCLI.cpp +++ b/src/fileCLI/fileCLI.cpp @@ -401,7 +401,7 @@ void base85dump(int fp, size_t file_len) size_t encodedLen = 0; for (file_idx = 0; file_idx < file_len; file_idx += bytes_read) { - bytes_read = read(fp, byte_buffer, SF_BLOCK_SIZE); + bytes_read = read(fp, byte_buffer, SF_PACKET_SIZE); #if SF_UPLOAD_ENCODING == SF_UPLOAD_BASE85 encodedLen = bintob85(encoded_buffer, byte_buffer, bytes_read) - encodedBuffer; #elif SF_UPLOAD_ENCODING == SF_UPLOAD_BASE64 diff --git a/src/product.hpp b/src/product.hpp index a338f763..fea77879 100644 --- a/src/product.hpp +++ b/src/product.hpp @@ -147,7 +147,11 @@ * */ #define SF_UPLOAD_REATTEMPT_DELAY_SEC 600 - + /** + * @brief Milliseconds between transmit attempts + * + */ +#define SF_UPLOAD_MS_PER_TRANSMIT 1000 /** * @brief how many ms is a GPS data point valid for a given data log * @@ -210,11 +214,32 @@ #define SF_UPLOAD_ENCODING SF_UPLOAD_BASE64URL +#if SF_UPLOAD_ENCODING == SF_UPLOAD_BASE85 + /** + * How many bytes to store chunks of data in on the SPI flash. + * + * 816 * 5/4 (base85 encoding compression rate) = 1020 which is less than 1024 + * bytes which is the maximum size of publish events. + */ +#define SF_PACKET_SIZE 816 +#define SF_RECORD_SIZE 1020 +#elif SF_UPLOAD_ENCODING == SF_UPLOAD_BASE64 || SF_UPLOAD_ENCODING == SF_UPLOAD_BASE64URL + /** + * How many bytes to store chunks of data in on the SPI flash. + * + * 768 * 4/3 (base64 encoding compression rate) = 1024 which is the maximum size + * of publish events. + */ +#define SF_PACKET_SIZE 768 +#define SF_RECORD_SIZE 1024 +#endif + + + #define SF_SERIAL_SPEED 9600 #define SF_CLI_MAX_CMD_LEN 100 -#define SF_BLOCK_SIZE 496 #define SF_NAME_MAX 64