talksik/uWebSockets
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Experimental: rewrite extensions negotiation

Browse Source
This commit is contained in:
Alex Hultman
2021-01-11 07:27:40 +01:00
parent a86a500f3b
commit 7265ffe480
8 changed files with 236 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/HttpResponse.h
+26 -54
View File
@@ -214,67 +214,39 @@ public:
writeHeader("Sec-WebSocket-Protocol", secWebSocketProtocol.substr(0, secWebSocketProtocol.find(',')));
}
/* Negotiate compression, we may use a smaller compression window than we negotiate */
/* Negotiate compression */
bool perMessageDeflate = false;
/* We are always allowed to share compressor, if perMessageDeflate */
int compressOptions = webSocketContextData->compression & SHARED_COMPRESSOR;
if (webSocketContextData->compression != DISABLED) {
if (secWebSocketExtensions.length()) {
/* We never support client context takeover (the client cannot compress with a sliding window). */
unsigned int wantedOptions = PERMESSAGE_DEFLATE | CLIENT_NO_CONTEXT_TAKEOVER;
CompressOptions compressOptions = CompressOptions::DISABLED;
if (secWebSocketExtensions.length() && webSocketContextData->compression != DISABLED) {
/* Shared compressor is the default */
if (webSocketContextData->compression == SHARED_COMPRESSOR) {
/* Disable per-socket compressor */
wantedOptions |= SERVER_NO_CONTEXT_TAKEOVER;
}
/* We always want shared inflation */
int wantedInflationWindow = 0;
/* isServer = true */
ExtensionsNegotiator<true> extensionsNegotiator(wantedOptions);
extensionsNegotiator.readOffer(secWebSocketExtensions);
/* Map from selected compressor */
int wantedCompressionWindow = (webSocketContextData->compression & 0xFF00) >> 8;
/* Todo: remove these mid string copies */
std::string offer = extensionsNegotiator.generateOffer();
if (offer.length()) {
auto [negCompression, negCompressionWindow, negInflationWindow, negResponse] =
uWS::negotiateCompression(true, wantedCompressionWindow, wantedInflationWindow,
secWebSocketExtensions);
/* Todo: this is a quick fix that should be properly moved to ExtensionsNegotiator */
if (webSocketContextData->compression & DEDICATED_COMPRESSOR &&
webSocketContextData->compression != DEDICATED_COMPRESSOR_256KB) {
/* 3kb, 4kb is 9, 256 is 15 (default) */
int maxServerWindowBits = 9;
switch (webSocketContextData->compression) {
case DEDICATED_COMPRESSOR_8KB:
maxServerWindowBits = 10;
break;
case DEDICATED_COMPRESSOR_16KB:
maxServerWindowBits = 11;
break;
case DEDICATED_COMPRESSOR_32KB:
maxServerWindowBits = 12;
break;
case DEDICATED_COMPRESSOR_64KB:
maxServerWindowBits = 13;
break;
case DEDICATED_COMPRESSOR_128KB:
maxServerWindowBits = 14;
break;
}
offer += "; server_max_window_bits=";
offer += std::to_string(maxServerWindowBits);
if (negCompression) {
perMessageDeflate = true;
/* Map from windowBits to compressor */
if (negCompressionWindow == 0) {
compressOptions = CompressOptions::SHARED_COMPRESSOR;
} else {
compressOptions = (CompressOptions) ((uint32_t) (negCompressionWindow << 8)
| (uint32_t) (negCompressionWindow - 7));
/* If we are dedicated and have the 3kb then correct any 4kb to 3kb,
* (they both share the windowBits = 9) */
if (webSocketContextData->compression == DEDICATED_COMPRESSOR_3KB) {
compressOptions = DEDICATED_COMPRESSOR_3KB;
}
writeHeader("Sec-WebSocket-Extensions", offer);
}
/* Did we negotiate permessage-deflate? */
if (extensionsNegotiator.getNegotiatedOptions() & PERMESSAGE_DEFLATE) {
perMessageDeflate = true;
}
/* Is the server allowed to compress with a sliding window? */
if (!(extensionsNegotiator.getNegotiatedOptions() & SERVER_NO_CONTEXT_TAKEOVER)) {
compressOptions = webSocketContextData->compression;
}
writeHeader("Sec-WebSocket-Extensions", negResponse);
}
}
@@ -372,7 +344,7 @@ public:
/* Write an HTTP header with unsigned int value */
HttpResponse *writeHeader(std::string_view key, uint64_t value) {
writeStatus(HTTP_200_OK);
Super::write(key.data(), (int) key.length());
Super::write(": ", 2);
writeUnsigned64(value);
src/PerMessageDeflate.h
+16 -45
View File
@@ -1,5 +1,5 @@
/*
* Authored by Alex Hultman, 2018-2020.
* Authored by Alex Hultman, 2018-2021.
* Intellectual property of third-party.
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -22,23 +22,20 @@
/* We always define these options no matter if ZLIB is enabled or not */
namespace uWS {
/* Compress options (really more like PerMessageDeflateOptions) */
enum CompressOptions : int {
/* Compression disabled */
/* Compressor mode is HIGH8(windowBits), LOW8(memLevel) */
enum CompressOptions : uint32_t {
DISABLED = 0,
/* We compress using a shared non-sliding window. No added memory usage, worse compression. */
SHARED_COMPRESSOR = 1,
/* We compress using a dedicated sliding window. Major memory usage added, better compression of similarly repeated messages. */
DEDICATED_COMPRESSOR = 2,
/* Flags for limiting memory usage of dedicated compressor */
DEDICATED_COMPRESSOR_3KB = 2 | 4,
DEDICATED_COMPRESSOR_4KB = 2 | 8,
DEDICATED_COMPRESSOR_8KB = 2 | 16,
DEDICATED_COMPRESSOR_16KB = 2 | 32,
DEDICATED_COMPRESSOR_32KB = 2 | 64,
DEDICATED_COMPRESSOR_64KB = 2 | 128,
DEDICATED_COMPRESSOR_128KB = 2 | 256,
DEDICATED_COMPRESSOR_256KB = 2 | 512
DEDICATED_COMPRESSOR_3KB = 9 << 8 | 1,
DEDICATED_COMPRESSOR_4KB = 9 << 8 | 2,
DEDICATED_COMPRESSOR_8KB = 10 << 8 | 3,
DEDICATED_COMPRESSOR_16KB = 11 << 8 | 4,
DEDICATED_COMPRESSOR_32KB = 12 << 8 | 5,
DEDICATED_COMPRESSOR_64KB = 13 << 8 | 6,
DEDICATED_COMPRESSOR_128KB = 14 << 8 | 7,
DEDICATED_COMPRESSOR_256KB = 15 << 8 | 8,
/* Same as 256kb */
DEDICATED_COMPRESSOR = 15 << 8 | 8
};
}
@@ -94,40 +91,14 @@ struct ZlibContext {
struct DeflationStream {
z_stream deflationStream = {};
DeflationStream(int compressOptions) {
DeflationStream(CompressOptions compressOptions) {
/* Sliding inflator should be about 44kb by default, less than compressor */
/* Memory usage is given by 2 ^ (windowBits + 2) + 2 ^ (memLevel + 9) */
int windowBits = -15, memLevel = 8;
int windowBits = -(int) ((compressOptions & 0xFF00) >> 8), memLevel = compressOptions & 0x00FF;
if (compressOptions == DEDICATED_COMPRESSOR_3KB) {
windowBits = -9;
memLevel = 1;
} else if (compressOptions == DEDICATED_COMPRESSOR_4KB) {
windowBits = -9;
memLevel = 2;
} else if (compressOptions == DEDICATED_COMPRESSOR_8KB) {
windowBits = -10;
memLevel = 3;
} else if (compressOptions == DEDICATED_COMPRESSOR_16KB) {
windowBits = -11;
memLevel = 4;
} else if (compressOptions == DEDICATED_COMPRESSOR_32KB) {
windowBits = -12;
memLevel = 5;
} else if (compressOptions == DEDICATED_COMPRESSOR_64KB) {
windowBits = -13;
memLevel = 6;
} else if (compressOptions == DEDICATED_COMPRESSOR_128KB) {
windowBits = -14;
memLevel = 7;
} else if (compressOptions == DEDICATED_COMPRESSOR_256KB) {
windowBits = -15;
memLevel = 8;
}
/* DEDICATED_COMPRESSOR_256KB is the same as DEDICATED_COMPRESSOR */
//printf("windowBits: %d, memLevel: %d\n", windowBits, memLevel);
deflateInit2(&deflationStream, 1, Z_DEFLATED, windowBits, memLevel, Z_DEFAULT_STRATEGY);
}
src/WebSocket.h
+1 -1
View File
@@ -34,7 +34,7 @@ struct WebSocket : AsyncSocket<SSL> {
private:
typedef AsyncSocket<SSL> Super;
void *init(bool perMessageDeflate, int compressOptions, std::string &&backpressure) {
void *init(bool perMessageDeflate, CompressOptions compressOptions, std::string &&backpressure) {
new (us_socket_ext(SSL, (us_socket_t *) this)) WebSocketData(perMessageDeflate, compressOptions, std::move(backpressure));
return this;
}
src/WebSocketContextData.h
+1 -1
View File
@@ -60,7 +60,7 @@ public:
unsigned int idleTimeout = 0;
/* We do need these for async upgrade */
int compression;
CompressOptions compression;
/* There needs to be a maxBackpressure which will force close everything over that limit */
size_t maxBackpressure = 0;
src/WebSocketData.h
+2 -2
View File
@@ -49,11 +49,11 @@ private:
/* We could be a subscriber */
Subscriber *subscriber = nullptr;
public:
WebSocketData(bool perMessageDeflate, int compressOptions, std::string &&backpressure) : AsyncSocketData<false>(std::move(backpressure)), WebSocketState<true>() {
WebSocketData(bool perMessageDeflate, CompressOptions compressOptions, std::string &&backpressure) : AsyncSocketData<false>(std::move(backpressure)), WebSocketState<true>() {
compressionStatus = perMessageDeflate ? ENABLED : DISABLED;
/* Initialize the dedicated sliding window */
if (perMessageDeflate && (compressOptions & CompressOptions::DEDICATED_COMPRESSOR)) {
if (perMessageDeflate && (compressOptions != CompressOptions::SHARED_COMPRESSOR)) {
deflationStream = new DeflationStream(compressOptions);
}
}
src/WebSocketExtensions.h
+123 -51
View File
@@ -1,5 +1,5 @@
/*
* Authored by Alex Hultman, 2018-2020.
* Authored by Alex Hultman, 2018-2021.
* Intellectual property of third-party.
* Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,26 +18,32 @@
#ifndef UWS_WEBSOCKETEXTENSIONS_H
#define UWS_WEBSOCKETEXTENSIONS_H
/* There is a new, huge bug scenario that needs to be fixed:
* pub/sub does not support being in DEDICATED_COMPRESSOR-mode while having
* some clients downgraded to SHARED_COMPRESSOR - we cannot allow the client to
* demand a downgrade to SHARED_COMPRESSOR (yet) until we fix that scenario in pub/sub */
// #define UWS_ALLOW_SHARED_AND_DEDICATED_COMPRESSOR_MIX
#include <climits>
#include <cctype>
#include <string>
#include <string_view>
#include <tuple>
namespace uWS {
enum Options : unsigned int {
NO_OPTIONS = 0,
PERMESSAGE_DEFLATE = 1,
SERVER_NO_CONTEXT_TAKEOVER = 2, // remove this
CLIENT_NO_CONTEXT_TAKEOVER = 4, // remove this
NO_DELAY = 8,
SLIDING_DEFLATE_WINDOW = 16
};
enum ExtensionTokens {
/* Standard permessage-deflate tokens */
TOK_PERMESSAGE_DEFLATE = 1838,
TOK_SERVER_NO_CONTEXT_TAKEOVER = 2807,
TOK_CLIENT_NO_CONTEXT_TAKEOVER = 2783,
TOK_SERVER_MAX_WINDOW_BITS = 2372,
TOK_CLIENT_MAX_WINDOW_BITS = 2348
TOK_CLIENT_MAX_WINDOW_BITS = 2348,
/* Non-standard alias for Safari */
TOK_X_WEBKIT_DEFLATE_FRAME = 2149,
TOK_NO_CONTEXT_TAKEOVER = 2049,
TOK_MAX_WINDOW_BITS = 1614
};
struct ExtensionsParser {
@@ -45,12 +51,18 @@ private:
int *lastInteger = nullptr;
public:
/* Standard */
bool perMessageDeflate = false;
bool serverNoContextTakeover = false;
bool clientNoContextTakeover = false;
int serverMaxWindowBits = 0;
int clientMaxWindowBits = 0;
/* Non-standard Safari */
bool xWebKitDeflateFrame = false;
bool noContextTakeover = false;
int maxWindowBits = 0;
int getToken(const char *&in, const char *stop) {
while (in != stop && !isalnum(*in)) {
in++;
@@ -78,12 +90,28 @@ public:
ExtensionsParser(const char *data, size_t length) {
const char *stop = data + length;
int token = 1;
for (; token && token != TOK_PERMESSAGE_DEFLATE; token = getToken(data, stop));
/* Ignore anything before permessage-deflate or x-webkit-deflate-frame */
for (; token && token != TOK_PERMESSAGE_DEFLATE && token != TOK_X_WEBKIT_DEFLATE_FRAME; token = getToken(data, stop));
/* What protocol are we going to use? */
perMessageDeflate = (token == TOK_PERMESSAGE_DEFLATE);
xWebKitDeflateFrame = (token == TOK_X_WEBKIT_DEFLATE_FRAME);
while ((token = getToken(data, stop))) {
switch (token) {
case TOK_X_WEBKIT_DEFLATE_FRAME:
/* Duplicates not allowed/supported */
return;
case TOK_NO_CONTEXT_TAKEOVER:
noContextTakeover = true;
break;
case TOK_MAX_WINDOW_BITS:
maxWindowBits = 1;
lastInteger = &maxWindowBits;
break;
case TOK_PERMESSAGE_DEFLATE:
/* Duplicates not allowed/supported */
return;
case TOK_SERVER_NO_CONTEXT_TAKEOVER:
serverNoContextTakeover = true;
@@ -109,60 +137,104 @@ public:
}
};
template <bool isServer>
struct ExtensionsNegotiator {
protected:
unsigned int options;
/* Takes what we (the server) wants, returns what we got */
std::tuple<bool, int, int, std::string_view> negotiateCompression(bool wantCompression, int wantedCompressionWindow, int wantedInflationWindow, std::string_view offer) {
public:
ExtensionsNegotiator(unsigned int wantedOptions) {
options = wantedOptions;
/* If we don't want compression then we are done here */
if (!wantCompression) {
return {false, 0, 0, ""};
}
std::string generateOffer() {
std::string extensionsOffer;
if (options & Options::PERMESSAGE_DEFLATE) {
extensionsOffer += "permessage-deflate";
ExtensionsParser ep(offer.data(), offer.length());
if (options & Options::CLIENT_NO_CONTEXT_TAKEOVER) {
extensionsOffer += "; client_no_context_takeover";
static thread_local std::string response;
response = "";
int compressionWindow = wantedCompressionWindow;
int inflationWindow = wantedInflationWindow;
bool compression = false;
if (ep.xWebKitDeflateFrame) {
/* We now have compression */
compression = true;
response = "x-webkit-deflate-frame";
/* If the other peer has DEMANDED us no sliding window,
* we cannot compress with anything other than shared compressor */
if (ep.noContextTakeover) {
/* We must fail here right now (fix pub/sub) */
#ifndef UWS_ALLOW_SHARED_AND_DEDICATED_COMPRESSOR_MIX
if (wantedCompressionWindow != 0) {
return {false, 0, 0, ""};
}
#endif
/* It is questionable sending this improves anything */
/*if (options & Options::SERVER_NO_CONTEXT_TAKEOVER) {
extensionsOffer += "; server_no_context_takeover";
}*/
compressionWindow = 0;
}
return extensionsOffer;
}
/* If the other peer has DEMANDED us to use a limited sliding window,
* we have to limit out compression sliding window */
if (ep.maxWindowBits && ep.maxWindowBits < compressionWindow) {
compressionWindow = ep.maxWindowBits;
}
void readOffer(std::string_view offer) {
if (isServer) {
ExtensionsParser extensionsParser(offer.data(), offer.length());
if ((options & PERMESSAGE_DEFLATE) && extensionsParser.perMessageDeflate) {
if (extensionsParser.clientNoContextTakeover || (options & CLIENT_NO_CONTEXT_TAKEOVER)) {
options |= CLIENT_NO_CONTEXT_TAKEOVER;
}
/* We leave this option for us to read even if the client did not send it */
if (extensionsParser.serverNoContextTakeover) {
options |= SERVER_NO_CONTEXT_TAKEOVER;
}/* else {
options &= ~SERVER_NO_CONTEXT_TAKEOVER;
}*/
/* We decide our own inflation sliding window (and their compression sliding window) */
if (wantedInflationWindow < 15) {
if (!wantedInflationWindow) {
response += "; no_context_takeover";
} else {
options &= ~PERMESSAGE_DEFLATE;
response += "; max_window_bits=" + std::to_string(wantedInflationWindow);
}
}
} else if (ep.perMessageDeflate) {
/* We now have compression */
compression = true;
response = "permessage-deflate";
if (ep.clientNoContextTakeover) {
inflationWindow = 0;
} else if (ep.clientMaxWindowBits && ep.clientMaxWindowBits != 1) {
inflationWindow = std::min<int>(ep.clientMaxWindowBits, inflationWindow);
}
/* Whatever we have now, write */
if (inflationWindow < 15) {
if (!inflationWindow || !ep.clientMaxWindowBits) {
response += "; client_no_context_takeover";
inflationWindow = 0;
} else {
response += "; client_max_window_bits=" + std::to_string(inflationWindow);
}
}
/* This block basically lets the client lower it */
if (ep.serverNoContextTakeover) {
/* This is an important (temporary) fix since we haven't allowed
* these two modes to mix, and pub/sub will not handle this case (yet) */
#ifdef UWS_ALLOW_SHARED_AND_DEDICATED_COMPRESSOR_MIX
compressionWindow = 0;
#endif
} else if (ep.serverMaxWindowBits) {
compressionWindow = std::min<int>(ep.serverMaxWindowBits, compressionWindow);
}
/* Whatever we have now, write */
if (compressionWindow < 15) {
if (!compressionWindow) {
response += "; server_no_context_takeover";
} else {
response += "; server_max_window_bits=" + std::to_string(compressionWindow);
}
} else {
// todo!
}
}
unsigned int getNegotiatedOptions() {
return options;
/* A final sanity check (this check does not actually catch too high values!) */
if ((compressionWindow && compressionWindow < 8) || compressionWindow > 15 || (inflationWindow && inflationWindow < 8) || inflationWindow > 15) {
return {false, 0, 0, ""};
}
};
return {compression, compressionWindow, inflationWindow, response};
}
}
tests/ExtensionsNegotiator.cpp
+64
View File
@@ -0,0 +1,64 @@
/* This is a temporary fix since we do not support this mode with pub/sub yet */
#define UWS_ALLOW_SHARED_AND_DEDICATED_COMPRESSOR_MIX
#include "../src/WebSocketExtensions.h"
#include <iostream>
void testNegotiation(bool wantCompression, int wantedCompressionWindow, int wantedInflationWindow, std::string_view offer,
bool negCompression, int negCompressionWindow, int negInflationWindow, std::string_view negResponse) {
auto [compression, compressionWindow, inflationWindow, response] = uWS::negotiateCompression(wantCompression, wantedCompressionWindow, wantedInflationWindow, offer);
if (compression == negCompression && compressionWindow == negCompressionWindow && inflationWindow == negInflationWindow && response == negResponse) {
std::cout << "PASS" << std::endl;
} else {
std::cout << "FAIL: <" << response << "> is not expected <" << negResponse << ">" << std::endl;
}
}
int main() {
/* Both parties must indicate compression for it to negotiate */
testNegotiation(false, 15, 15, "permessage-deflate", false, 0, 0, "");
testNegotiation(false, 15, 15, "x-webkit-deflate-frame", false, 0, 0, "");
testNegotiation(true, 15, 15, "", false, 15, 15, "");
testNegotiation(true, 15, 15, "", false, 15, 15, "");
/* client_max_window_bits can only be used if the client indicates support */
testNegotiation(true, 15, 11, "permessage-deflate; ", true, 15, 0, "permessage-deflate; client_no_context_takeover");
testNegotiation(true, 15, 0, "permessage-deflate; ", true, 15, 0, "permessage-deflate; client_no_context_takeover");
testNegotiation(true, 15, 11, "permessage-deflate; client_max_window_bits=14", true, 15, 11, "permessage-deflate; client_max_window_bits=11");
testNegotiation(true, 15, 11, "permessage-deflate; client_max_window_bits=9", true, 15, 9, "permessage-deflate; client_max_window_bits=9");
/* server_max_window_bits can always be used */
testNegotiation(true, 0, 15, "permessage-deflate; ", true, 0, 15, "permessage-deflate; server_no_context_takeover");
testNegotiation(true, 8, 15, "permessage-deflate; ", true, 8, 15, "permessage-deflate; server_max_window_bits=8");
testNegotiation(true, 15, 15, "permessage-deflate; server_max_window_bits=8", true, 8, 15, "permessage-deflate; server_max_window_bits=8");
testNegotiation(true, 11, 15, "permessage-deflate; server_max_window_bits=14", true, 11, 15, "permessage-deflate; server_max_window_bits=11");
/* x-webkit-deflate-frame has no particular rules */
testNegotiation(true, 11, 15, "x-webkit-deflate-frame; no_context_takeover; max_window_bits=8", true, 0, 15, "x-webkit-deflate-frame");
testNegotiation(true, 11, 12, "x-webkit-deflate-frame; no_context_takeover; max_window_bits=8", true, 0, 12, "x-webkit-deflate-frame; max_window_bits=12");
testNegotiation(true, 11, 12, "x-webkit-deflate-frame; max_window_bits=8", true, 8, 12, "x-webkit-deflate-frame; max_window_bits=12");
testNegotiation(true, 15, 0, "x-webkit-deflate-frame; max_window_bits=15", true, 15, 0, "x-webkit-deflate-frame; no_context_takeover");
/* Defaults */
testNegotiation(true, 15, 15, "x-webkit-deflate-frame", true, 15, 15, "x-webkit-deflate-frame");
testNegotiation(true, 15, 15, "permessage-deflate", true, 15, 15, "permessage-deflate");
/* Fail on invalid values */
testNegotiation(true, 15, 15, "x-webkit-deflate-frame; max_window_bits=3", false, 0, 0, "");
/* This one doesn't fail, but at least ignores the too high value */
testNegotiation(true, 15, 15, "x-webkit-deflate-frame; max_window_bits=16", true, 15, 15, "x-webkit-deflate-frame");
testNegotiation(true, 15, 15, "permessage-deflate; server_max_window_bits=3", false, 0, 0, "");
testNegotiation(true, 15, 15, "permessage-deflate; client_max_window_bits=3", false, 0, 0, "");
/* Same here; these won't fail but just be ignored */
testNegotiation(true, 15, 15, "permessage-deflate; server_max_window_bits=17", true, 15, 15, "permessage-deflate");
testNegotiation(true, 15, 15, "permessage-deflate; client_max_window_bits=17", true, 15, 15, "permessage-deflate");
std::cout << "ALL PASS" << std::endl;
}
tests/Makefile
+3 -1
View File
@@ -4,4 +4,6 @@ default:
$(CXX) -std=c++17 -fsanitize=address HttpRouter.cpp -o HttpRouter
./HttpRouter
$(CXX) -std=c++17 -fsanitize=address BloomFilter.cpp -o BloomFilter
./BloomFilter
./BloomFilter
$(CXX) -std=c++17 -fsanitize=address ExtensionsNegotiator.cpp -o ExtensionsNegotiator
./ExtensionsNegotiator