From 9428c55053f77fb58ab2c5b3b866393684f3d0e0 Mon Sep 17 00:00:00 2001 From: Alex Hultman Date: Sat, 8 Oct 2022 08:16:55 +0200 Subject: [PATCH] Use new chunked encoding parser --- examples/Crc32.cpp | 4 + src/ChunkedEncoding.h | 4 +- src/HttpParser.h | 165 +++++++++--------------------------------- 3 files changed, 42 insertions(+), 131 deletions(-) diff --git a/examples/Crc32.cpp b/examples/Crc32.cpp index f8e6e0b..9ca0ce9 100644 --- a/examples/Crc32.cpp +++ b/examples/Crc32.cpp @@ -5,6 +5,10 @@ * be hashed with crc32 and sent back in the response. This example also shows how to deal with * aborted requests. */ +/* curl -H "Transfer-Encoding: chunked" --data-binary @video.mp4 http://localhost:3000 */ +/* curl --data-binary @video.mp4 http://localhost:3000 */ +/* crc32 video.mp4 */ + /* Note that uWS::SSLApp({options}) is the same as uWS::App() when compiled without SSL support */ #include diff --git a/src/ChunkedEncoding.h b/src/ChunkedEncoding.h index 5b04e0b..dde4af5 100644 --- a/src/ChunkedEncoding.h +++ b/src/ChunkedEncoding.h @@ -38,7 +38,7 @@ namespace uWS { /* Consume everything higher than 32 */ while (data.length() && data.data()[0] > 32) { - unsigned char digit = data.data()[0]; + unsigned char digit = (unsigned char)data.data()[0]; if (digit >= 'a') { digit -= ('a' - '9') - 1; } @@ -153,7 +153,7 @@ namespace uWS { emitSoon = data; } } - decChunkSize(state, data.length()); + decChunkSize(state, (unsigned int) data.length()); state |= STATE_IS_CHUNKED; // new: decrease data by its size (bug) data.remove_prefix(data.length()); // ny bug fix för getNextChunk diff --git a/src/HttpParser.h b/src/HttpParser.h index b11b6a3..b1c84cb 100644 --- a/src/HttpParser.h +++ b/src/HttpParser.h @@ -26,6 +26,7 @@ #include #include #include "MoveOnlyFunction.h" +#include "ChunkedEncoding.h" #include "BloomFilter.h" #include "ProxyParser.h" @@ -150,109 +151,11 @@ struct HttpParser { private: std::string fallback; + /* This guy really has only 30 bits since we reserve two highest bits to chunked encoding parsing state */ unsigned int remainingStreamingBytes = 0; - /* This guy should be encoded in remainingStreamingBytes as highest bit */ - //bool chunked = false; - - /* We do not support any trailer other than empty ones for now */ - unsigned int consumeTrailer(char *data, unsigned int length) { - if (length >= 2) { - if (data[0] == '\r' && data[1] == '\n') { - return 2; - } - } - return 0; - } - - /* Returns [bytes consumed, chunk to emit, isFin] */ - std::tuple consumeChunked(char *data, unsigned int length) { - char *start = data; - char *end = data + length; - - /* Can't consume nothing */ - if (!length) { - return {0, std::string_view(), false}; - } - - /* If first byte is CR, then skip two bytes */ - if (data[0] == '\r') { - if (length > 2) { - data += 2; - } else { - /* We must have at least 3 bytes */ - return {0, std::string_view(), false}; - } - } - - /* Find next \r */ - char *numberEnd = data; - while (numberEnd < end && *numberEnd != '\r') numberEnd++; - /* Break if we cannot find \r */ - if (numberEnd == end) { - return {0, std::string_view(), false}; - } - - unsigned int toEmit = 0; - if (&numberEnd[1] != end && numberEnd[1] == '\n') { - /* The following cannot fail */ - toEmit = toUnsignedIntegerHex(std::string_view(data, (unsigned int)(numberEnd - data))); - data = numberEnd + 2; - - /* Skip reading data if we got the null chunk */ - if (toEmit == 0) { - /* Consume trailer then as well */ - data += consumeTrailer(data, (unsigned int)(end - data)); - - /* Emit the empty chunk with fin = true */ - return {data - start, std::string_view(nullptr, 0), true}; - } - - /* We cannot emit everything and must return what we have without knowing if it is fin or not */ - if (data + toEmit >= end) { - // todo: set remainingBytesStream and chunked - // todo: emit what we can and enter slow path - return {0, std::string_view(), false}; - } - - /* Emit everything now */ - std::string_view emittableChunk(data, toEmit); - data += toEmit; - bool fin = false; - - /* Consume the two \r\n following the whole chunk */ - data += 2; - if (data >= end) { - /* We could emit the data still here, only without knowing if it is fin */ - return {0, std::string_view(), false}; - } - - /* Check if this is fin */ - if ((data + 3 < end) && !memcmp(data, "0\r\n", 3)) { - fin = true; - data += 3; - - /* Consume trailer then as well */ - data += consumeTrailer(data, (unsigned int)(end - data)); - } - - /* Finally emit this chunk */ - return {data - start, emittableChunk, fin}; - } - - return {0, std::string_view(), false}; - } - const size_t MAX_FALLBACK_SIZE = 1024 * 4; - static unsigned int toUnsignedIntegerHex(std::string_view str) { - unsigned int unsignedIntegerValue = 0; - for (char c : str) { - unsignedIntegerValue = unsignedIntegerValue * 16u + ((unsigned int) c - (unsigned int) '0'); - } - return unsignedIntegerValue; - } - static unsigned int toUnsignedInteger(std::string_view str) { unsigned int unsignedIntegerValue = 0; for (char c : str) { @@ -403,24 +306,18 @@ private: } } else { /* We are not GET and we have no content-length, so assume transfer-encoding: chunked */ - /* Todo: care about consume minimally */ - while (true) { - auto [consumed, chunk, fin] = consumeChunked(data, length); - if (!consumed) { - break; + remainingStreamingBytes = STATE_IS_CHUNKED; + /* If consume minimally, we do not want to consume anything but we want to mark this as being chunked */ + if (!CONSUME_MINIMALLY) { + /* Go ahead and parse it (todo: better heuristics for emitting FIN to the app level) */ + std::string_view dataToConsume(data, length); + for (auto chunk : uWS::ChunkIterator(&dataToConsume, &remainingStreamingBytes)) { + dataHandler(user, chunk, chunk.length() == 0); } - /* If we consumed something and we either have fin to report, or a chunk, then emit to app */ - if (consumed && (chunk.length() || fin)) { - dataHandler(user, chunk, fin); - } - - data += consumed; - length -= consumed; + unsigned int consumed = (length - (unsigned int) dataToConsume.length()); + data = (char *) dataToConsume.data(); + length = (unsigned int) dataToConsume.length(); consumedTotal += consumed; - - if (fin) { - break; - } } } } else { @@ -444,22 +341,32 @@ public: if (remainingStreamingBytes) { - // this is exactly the same as below! - // todo: refactor this - if (remainingStreamingBytes >= length) { - void *returnedUser = dataHandler(user, std::string_view(data, length), remainingStreamingBytes == length); - remainingStreamingBytes -= length; - return returnedUser; + /* It's either chunked or with a content-length */ + if (isParsingChunkedEncoding(remainingStreamingBytes)) { + std::string_view dataToConsume(data, length); + for (auto chunk : uWS::ChunkIterator(&dataToConsume, &remainingStreamingBytes)) { + dataHandler(user, chunk, chunk.length() == 0); + } + data = (char *) dataToConsume.data(); + length = (unsigned int) dataToConsume.length(); } else { - void *returnedUser = dataHandler(user, std::string_view(data, remainingStreamingBytes), true); - - data += remainingStreamingBytes; - length -= remainingStreamingBytes; - - remainingStreamingBytes = 0; - - if (returnedUser != user) { + // this is exactly the same as below! + // todo: refactor this + if (remainingStreamingBytes >= length) { + void *returnedUser = dataHandler(user, std::string_view(data, length), remainingStreamingBytes == length); + remainingStreamingBytes -= length; return returnedUser; + } else { + void *returnedUser = dataHandler(user, std::string_view(data, remainingStreamingBytes), true); + + data += remainingStreamingBytes; + length -= remainingStreamingBytes; + + remainingStreamingBytes = 0; + + if (returnedUser != user) { + return returnedUser; + } } }