From 779820ed3322b87fb49ceae44ac7447b90a94b22 Mon Sep 17 00:00:00 2001 From: Dirk Mueller Date: Sat, 5 Oct 2019 20:47:55 +0200 Subject: [PATCH 1/2] Cleanup base64::encode functions The implementation choice here using libb64 is generally good as it is a relatively fast implementation, however the adaptation to use PROGMEM for the translation function was a bad choice, as reading randomly PROGMEM with byte-wide access is very very very slow. Doing a naive if-snake is between 20% and 55% faster and uses less flash (about 120 bytes less) and also for reasons I don't understand 8 bytes less data RAM (maybe the removal of static?). In addition the base64::encode function was allocating for larger input a huge amount of memory (twice the total size). we can reduce that by doing a chunk-wise conversation to base64. --- cores/esp8266/base64.cpp | 49 ++++++++++++++++++-------------- cores/esp8266/base64.h | 20 +++++++------ cores/esp8266/libb64/cencode.cpp | 19 +++++++++---- 3 files changed, 53 insertions(+), 35 deletions(-) diff --git a/cores/esp8266/base64.cpp b/cores/esp8266/base64.cpp index f0d0793529..271fca4f9c 100644 --- a/cores/esp8266/base64.cpp +++ b/cores/esp8266/base64.cpp @@ -24,7 +24,6 @@ #include "Arduino.h" extern "C" { -#include "libb64/cdecode.h" #include "libb64/cencode.h" } #include "base64.h" @@ -35,14 +34,19 @@ extern "C" { * @param length size_t * @return String */ -String base64::encode(const uint8_t * data, size_t length, bool doNewLines) { +String base64::encode(const uint8_t * data, size_t length, bool doNewLines) +{ + String base64; + // base64 needs more size then the source data, use cencode.h macros - size_t size = ((doNewLines ? base64_encode_expected_len(length) - : base64_encode_expected_len_nonewlines(length)) + 1); - char * buffer = (char *) malloc(size); - if(buffer) { + size_t size = ((doNewLines ? base64_encode_expected_len( length ) + : base64_encode_expected_len_nonewlines( length )) + 1); + + if (base64.reserve(size)) + { + base64_encodestate _state; - if(doNewLines) + if (doNewLines) { base64_init_encodestate(&_state); } @@ -50,22 +54,23 @@ String base64::encode(const uint8_t * data, size_t length, bool doNewLines) { { base64_init_encodestate_nonewlines(&_state); } - int len = base64_encode_block((const char *) &data[0], length, &buffer[0], &_state); - len = base64_encode_blockend((buffer + len), &_state); - String base64 = String(buffer); - free(buffer); - return base64; + constexpr size_t BUFSIZE = 48; + char buf[BUFSIZE + 1 /* newline */ + 1 /* NUL */]; + for (size_t len = 0; len < length; len += BUFSIZE * 3 / 4) + { + size_t blocklen = base64_encode_block((const char*) data + len, + std::min( BUFSIZE * 3 / 4, length - len ), buf, &_state); + buf[blocklen] = '\0'; + base64 += buf; + } + if (base64_encode_blockend(buf, &_state)) + base64 += buf; + } + else + { + base64 = F("-FAIL-"); } - return String("-FAIL-"); -} -/** - * convert input data to base64 - * @param text const String& - * @return String - */ -String base64::encode(const String& text, bool doNewLines) { - return base64::encode((const uint8_t *) text.c_str(), text.length(), doNewLines); + return base64; } - diff --git a/cores/esp8266/base64.h b/cores/esp8266/base64.h index 1d6e22fac7..20f00f81fd 100644 --- a/cores/esp8266/base64.h +++ b/cores/esp8266/base64.h @@ -25,14 +25,18 @@ #ifndef CORE_BASE64_H_ #define CORE_BASE64_H_ -class base64 { - public: - // NOTE: The default behaviour of backend (lib64) - // is to add a newline every 72 (encoded) characters output. - // This may 'break' longer uris and json variables - static String encode(const uint8_t * data, size_t length, bool doNewLines = true); - static String encode(const String& text, bool doNewLines = true); - private: +class base64 +{ +public: + // NOTE: The default behaviour of backend (lib64) + // is to add a newline every 72 (encoded) characters output. + // This may 'break' longer uris and json variables + static String encode(const uint8_t * data, size_t length, bool doNewLines = true); + static String inline encode(const String& text, bool doNewLines = true) + { + return encode( (const uint8_t *) text.c_str(), text.length(), doNewLines ); + } +private: }; diff --git a/cores/esp8266/libb64/cencode.cpp b/cores/esp8266/libb64/cencode.cpp index ae8f90a831..d86df31af2 100755 --- a/cores/esp8266/libb64/cencode.cpp +++ b/cores/esp8266/libb64/cencode.cpp @@ -5,7 +5,6 @@ This is part of the libb64 project, and has been placed in the public domain. For details, see http://sourceforge.net/projects/libb64 */ -#include #include "cencode.h" extern "C" { @@ -23,10 +22,20 @@ void base64_init_encodestate_nonewlines(base64_encodestate* state_in){ state_in->stepsnewline = -1; } -char base64_encode_value(char value_in){ - static const char encoding[] PROGMEM = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - if (value_in > 63) return '='; - return pgm_read_byte( &encoding[(int)value_in] ); +char base64_encode_value(const char n) { + char r; + + if (n < 26) + r = n + 'A'; + else if (n < 26 + 26) + r = n - 26 + 'a'; + else if (n < 26 + 26 + 10 ) + r = n - 26 - 26 + '0'; + else if (n == 62 ) + r = '+'; + else + r = '/'; + return r; } int base64_encode_block(const char* plaintext_in, int length_in, char* code_out, base64_encodestate* state_in){ From da26de2959f09bfde9c1ea84c8ba46a7d09cb496 Mon Sep 17 00:00:00 2001 From: Dirk Mueller Date: Sat, 5 Oct 2019 20:59:58 +0200 Subject: [PATCH 2/2] Create authorisation base64 encoded string without newlines Rather than first creating a string with newlines and then stripping it away in the fast path of constructing the query, we can call the right method and trust that the result does not have newlines anymore. --- libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp index a05b9dd6f0..1aec2a5ea2 100644 --- a/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp +++ b/libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp @@ -317,7 +317,7 @@ bool HTTPClient::beginInternal(String url, const char* expectedProtocol) // auth info String auth = host.substring(0, index); host.remove(0, index + 1); // remove auth part including @ - _base64Authorization = base64::encode(auth); + _base64Authorization = base64::encode(auth, false /* doNewLines */); } // get port @@ -502,7 +502,7 @@ void HTTPClient::setAuthorization(const char * user, const char * password) String auth = user; auth += ":"; auth += password; - _base64Authorization = base64::encode(auth); + _base64Authorization = base64::encode(auth, false /* doNewLines */); } } @@ -514,6 +514,7 @@ void HTTPClient::setAuthorization(const char * auth) { if(auth) { _base64Authorization = auth; + _base64Authorization.replace(String('\n'), emptyString); } } @@ -1241,7 +1242,6 @@ bool HTTPClient::sendHeader(const char * type) } if(_base64Authorization.length()) { - _base64Authorization.replace("\n", ""); header += F("Authorization: Basic "); header += _base64Authorization; header += "\r\n";