From a21446176da093921a28caac6dbc26f217392514 Mon Sep 17 00:00:00 2001 From: Alex Hultman Date: Fri, 7 Oct 2022 12:47:29 +0200 Subject: [PATCH] Add new ChunkedEncoding parser and test --- src/ChunkedEncoding.h | 121 ++++++++++++++++++++++++++++++++++++++ tests/ChunkedEncoding.cpp | 79 +++++++++++++++++++++++++ tests/Makefile | 4 +- 3 files changed, 203 insertions(+), 1 deletion(-) create mode 100644 src/ChunkedEncoding.h create mode 100644 tests/ChunkedEncoding.cpp diff --git a/src/ChunkedEncoding.h b/src/ChunkedEncoding.h new file mode 100644 index 0000000..4ed8031 --- /dev/null +++ b/src/ChunkedEncoding.h @@ -0,0 +1,121 @@ +/* + * Authored by Alex Hultman, 2018-2022. + * Intellectual property of third-party. + + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + + * http://www.apache.org/licenses/LICENSE-2.0 + + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef UWS_CHUNKEDENCODING_H +#define UWS_CHUNKEDENCODING_H + +/* Independent chunked encoding parser, used by HttpParser. */ + +#include +#include +#include +#include +#include "MoveOnlyFunction.h" +#include + +namespace uWS { + + uint32_t STATE_HAS_SIZE = 0x80000000; + uint32_t STATE_IS_CHUNKED = 0x40000000; + uint32_t STATE_SIZE_MASK = 0x3FFFFFFF; + + /* Reads hex number until CR or out of data to consume. Updates state. Returns bytes consumed. */ + void consumeHexNumber(std::string_view &data, unsigned int &state) { + /* Consume everything higher than 32 */ + while (data.length() && data.data()[0] > 32) { + + unsigned char digit = data.data()[0]; + if (digit >= 'a') { + digit -= ('a' - '9') - 1; + } + + state = state * 16u + ((unsigned int) digit - (unsigned int) '0'); + data.remove_prefix(1); + } + /* Consume everything not /n */ + while (data.length() && data.data()[0] != '\n') { + data.remove_prefix(1); + } + /* Now we stand on \n so consume it and enable size */ + if (data.length()) { + //printf("size of chunk is %d\n", state); + state += 2; // include the two last /r/n + state |= STATE_HAS_SIZE; + data.remove_prefix(1); + } + } + + unsigned int chunkSize(unsigned int state) { + return state & STATE_SIZE_MASK; + } + + void decChunkSize(unsigned int &state, unsigned int by) { + state = (state & ~STATE_SIZE_MASK) | (chunkSize(state) - by); + } + + bool hasChunkSize(unsigned int state) { + return state & STATE_HAS_SIZE; + } + + // bättre interface + std::optional getNextChunk(std::string_view data, unsigned int &state); + + /* Consumes as much as possible, emitting chunks using cb. Changes passed state for later resumption. Returns number of bytes consumed. */ + inline unsigned int consumeChunkedEncoding(std::string_view data, unsigned int &state, MoveOnlyFunction cb) { + + std::string_view originalData = data; + + while (data.length()) { + + if (!hasChunkSize(state)) { + consumeHexNumber(data, state); + if (hasChunkSize(state) && chunkSize(state) == 2) { + cb(std::string_view(nullptr, 0)); + } + continue; + } + + // do we have data to emit all? + if (data.length() >= chunkSize(state)) { + // emit all but 2 bytes then reset state to 0 and goto beginning + // not fin + if (chunkSize(state) > 2) { + cb(std::string_view(data.data(), chunkSize(state) - 2)); + } + data.remove_prefix(chunkSize(state)); + state = 0; + continue; + } else { + /* We will consume all our input data */ + if (chunkSize(state) > 2) { + unsigned int maximalAppEmit = chunkSize(state) - 2; + if (data.length() > maximalAppEmit) { + cb(data.substr(0, maximalAppEmit)); + } else { + cb(data); + } + } + decChunkSize(state, data.length()); + return originalData.length(); + } + } + + return originalData.length() - data.length(); + } +} + +#endif // UWS_CHUNKEDENCODING_H diff --git a/tests/ChunkedEncoding.cpp b/tests/ChunkedEncoding.cpp new file mode 100644 index 0000000..96230ea --- /dev/null +++ b/tests/ChunkedEncoding.cpp @@ -0,0 +1,79 @@ +#include +#include +#include +#include +#include +#include + +#include "../src/ChunkedEncoding.h" + +void runTest(unsigned int maxConsume) { + /* A list of chunks */ + std::vector chunks = { + "Hello there I am the first segment", + "Why hello there", + "", + "I am last?", + "And I am a little longer but it doesn't matter", + "" + }; + + /* Encode them in chunked encoding */ + std::stringstream ss; + for (std::string_view chunk : chunks) { + + // if length is 0 then append trailer also + + ss << std::hex << chunk.length() << "\r\n" << chunk << "\r\n"; + + // every null chunk is followed by an empty trailer + if (chunk.length() == 0) { + //ss << "\r\n"; + } + } + + /* Consume them, checking that we get what we expect */ + std::string buffer = ss.str(); + unsigned int state = 0; + std::string_view chunkEncoded = buffer; + + unsigned int consumed = UINT_MAX; + int chunkOffset = 0; + while (consumed) { + /* Consume up to maxConsume */ + std::string_view indata = chunkEncoded.substr(0, std::min(maxConsume, chunkEncoded.length())); + + consumed = uWS::consumeChunkedEncoding(indata, state, [&](std::string_view chunk) { + /* Print for logging */ + std::cout << "<" << chunk << ">"; + + if (!chunk.length() && chunks[chunkOffset].length()) { + std::cout << "We got emitted an empty chunk but expected a non-empty one" << std::endl; + std::abort(); + } + + if (chunks[chunkOffset].starts_with(chunk)) { + chunks[chunkOffset].remove_prefix(chunk.length()); + if (!chunks[chunkOffset].length()) { + chunkOffset++; + } + } else { + std::cerr << "Chunk does not match! Should be <" << chunks[chunkOffset] << ">" << std::endl; + std::abort(); + } + }); + if (consumed != indata.length()) { + std::cerr << "Chunk parser did not consume exactly the bytes passed!" << std::endl; + std::abort(); + } + chunkEncoded.remove_prefix(consumed); + } +} + +int main() { + for (int i = 0; i < 1000; i++) { + runTest(i); + } + + std::cout << "ALL BRUTEFORCE DONE" << std::endl; +} \ No newline at end of file diff --git a/tests/Makefile b/tests/Makefile index 93d72bb..6e5655a 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,4 +1,6 @@ default: + $(CXX) -std=c++20 -fsanitize=address -g ChunkedEncoding.cpp -o ChunkedEncoding + ./ChunkedEncoding $(CXX) -std=c++17 -fsanitize=address TopicTree.cpp -o TopicTree ./TopicTree $(CXX) -std=c++17 -fsanitize=address HttpRouter.cpp -o HttpRouter @@ -8,4 +10,4 @@ default: $(CXX) -std=c++17 -fsanitize=address ExtensionsNegotiator.cpp -o ExtensionsNegotiator ./ExtensionsNegotiator $(CXX) -std=c++17 -fsanitize=address HttpParser.cpp -o HttpParser - ./HttpParser \ No newline at end of file + ./HttpParser