From 7b208ab30babcc64bfbc66bd16b114896cf46995 Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Mon, 1 Jul 2019 18:32:12 -0500 Subject: [PATCH 1/2] Added literal data for content, added 'transfer' override. --- local/include/core/HttpReplay.h | 32 +++++++++- local/src/core/HttpReplay.cc | 100 +++++++++++++++++++++++------- local/src/server/replay-server.cc | 15 +++-- 3 files changed, 116 insertions(+), 31 deletions(-) diff --git a/local/include/core/HttpReplay.h b/local/include/core/HttpReplay.h index c4c19e0..d483162 100644 --- a/local/include/core/HttpReplay.h +++ b/local/include/core/HttpReplay.h @@ -62,6 +62,9 @@ static const std::string YAML_HTTP_METHOD_KEY{"method"}; static const std::string YAML_HTTP_URL_KEY{"url"}; static const std::string YAML_CONTENT_KEY{"content"}; static const std::string YAML_CONTENT_LENGTH_KEY{"size"}; +static const std::string YAML_CONTENT_DATA_KEY{"data"}; +static const std::string YAML_CONTENT_ENCODING_KEY{"encoding"}; +static const std::string YAML_CONTENT_TRANSFER_KEY{"transfer"}; static constexpr size_t MAX_HDR_SIZE = 131072; // Max our ATS is configured for static constexpr size_t MAX_DRAIN_BUFFER_SIZE = 1 << 20; @@ -279,7 +282,10 @@ class HttpHeader { unsigned _status = 0; TextView _reason; - unsigned _content_size = 0; + /// If @a content_size is valid but not @a content_data, synthesize the content. + /// This is split instead of @c TextView because these get set independently during load. + char const* _content_data; ///< Literal data for the content. + size_t _content_size = 0; ///< Length of the content. TextView _method; TextView _http_version; std::string _url; @@ -300,6 +306,9 @@ class HttpHeader { static void global_init(); + /// Precomputed content buffer. + static swoc::MemSpan _content; + protected: class Binding : public swoc::bwf::NameBinding { using BufferWriter = swoc::BufferWriter; @@ -334,10 +343,27 @@ class HttpHeader { */ static TextView localize(TextView text); + /// Encoding for input text. + enum class Encoding { + TEXT, ///< Plain text, no encoding. + URI //< URI encoded. + }; + + /** Convert @a name to a localized view. + * + * @param name Text to localize. + * @param enc Type of decoding to perform before localization. + * @return The localized view, or @a name if localization is frozen and @a + * name is not found. + * + * @a name will be localized if string localization is not frozen, or @a name + * is already localized. @a enc specifies the text is encoded and needs to be + * decoded before localization. + */ + static TextView localize(TextView text, Encoding enc); + static NameSet _names; static swoc::MemArena _arena; - /// Precomputed content buffer. - static swoc::MemSpan _content; }; // YAML support utilities. diff --git a/local/src/core/HttpReplay.cc b/local/src/core/HttpReplay.cc index 3239dd2..1c91b2c 100644 --- a/local/src/core/HttpReplay.cc +++ b/local/src/core/HttpReplay.cc @@ -49,6 +49,11 @@ swoc::TextView HttpHeader::FIELD_CONTENT_LENGTH; swoc::TextView HttpHeader::FIELD_TRANSFER_ENCODING; std::bitset<600> HttpHeader::STATUS_NO_CONTENT; +// libswoc tweaks not yet ported upstream. +namespace swoc { +template uintmax_t svto_radix(TextView && src) { return svto_radix(src); } +} // namespace swoc + namespace { [[maybe_unused]] bool INITIALIZED = []() -> bool { HttpHeader::global_init(); @@ -392,13 +397,13 @@ swoc::Errata HttpHeader::transmit_body(Stream &stream) const { Info("Transmit {} byte body {}{}.", _content_size, swoc::bwf::If(_content_length_p, "[CL]"), swoc::bwf::If(_chunked_p, "[chunked]")); - if (_content_size || (_status && !STATUS_NO_CONTENT[_status])) { + if (_content_size > 0 || (_status && !STATUS_NO_CONTENT[_status])) { + TextView content { _content_data, _content_size }; if (_chunked_p) { ChunkCodex codex; - std::tie(n, ec) = - codex.transmit(stream, {_content.data(), _content_size}); + std::tie(n, ec) = codex.transmit(stream, content); } else { - n = stream.write({_content.data(), _content_size}); + n = stream.write(content); ec = std::error_code(errno, std::system_category()); if (!_content_length_p) { // no content-length, must close to signal end // of body. @@ -411,12 +416,10 @@ swoc::Errata HttpHeader::transmit_body(Stream &stream) const { swoc::bwf::If(_chunked_p, " [chunked]"), n, _content_size, ec); } - } else if (!_content_size && _status && !STATUS_NO_CONTENT[_status]) { - // Go ahead and close the connection if it is not specified - if (!_chunked_p && !_content_length_p) { - Info("No CL or TE, status {} - closing.", _status); - stream.close(); - } + } else if (_content_size == 0 && _status && !STATUS_NO_CONTENT[_status] && !_chunked_p && !_content_length_p) { + // There's no body but the status expects one, so signal no body with EOS. + Info("No CL or TE, status {} - closing.", _status); + stream.close(); } return errata; @@ -670,19 +673,6 @@ swoc::Errata HttpHeader::load(YAML::Node const &node) { } } - if (node[YAML_CONTENT_KEY]) { - auto content_node{node[YAML_CONTENT_KEY]}; - if (content_node.IsMap()) { - if (content_node[YAML_CONTENT_LENGTH_KEY]) { - _content_size = - swoc::svtou(content_node[YAML_CONTENT_LENGTH_KEY].Scalar()); - } - } else { - errata.error(R"("{}" node at {} is not a map.)", YAML_CONTENT_KEY, - content_node.Mark()); - } - } - if (node[YAML_HDR_KEY]) { auto hdr_node{node[YAML_HDR_KEY]}; if (hdr_node[YAML_FIELDS_KEY]) { @@ -698,6 +688,49 @@ swoc::Errata HttpHeader::load(YAML::Node const &node) { } } + // Do this after header so it can override transfer encoding. + if (auto content_node { node[YAML_CONTENT_KEY] } ; content_node ) { + if (content_node.IsMap()) { + if (auto xf_node { content_node[YAML_CONTENT_TRANSFER_KEY] } ; xf_node ) { + TextView xf { xf_node.Scalar() }; + if (0 == strcasecmp("chunked"_tv, xf)) { + _chunked_p = true; + } else if (0 == strcasecmp("plain"_tv, xf)) { + _chunked_p = false; + } else { + errata.error(R"(Invalid value "{}" for "{}" key at {} in "{}" node at )" + , xf, YAML_CONTENT_TRANSFER_KEY, xf_node.Mark() + , YAML_CONTENT_KEY, content_node.Mark()); + } + } + if (auto data_node { content_node[YAML_CONTENT_DATA_KEY] } ; data_node) { + Encoding enc { Encoding::TEXT }; + if (auto enc_node { content_node[YAML_CONTENT_ENCODING_KEY] } ; enc_node) { + if (0 == strcasecmp("uri"_tv, enc_node.Scalar())) { + enc = Encoding::URI; + } else { + errata.error(R"(Unknown encoding "{}" at {}.)", enc_node.Scalar(), enc_node.Mark()); + } + } + TextView content { this->localize(data_node.Scalar(), enc) }; + _content_data = content.data(); + _content_size = content.size(); + if (content_node[YAML_CONTENT_LENGTH_KEY]) { + errata.info(R"(The "{}" key is ignored if "{}" is present at {}.)", YAML_CONTENT_LENGTH_KEY + , YAML_CONTENT_DATA_KEY, content_node.Mark()); + } + } else if (auto size_node { content_node[YAML_CONTENT_LENGTH_KEY]} ; size_node) { + _content_size = swoc::svtou(size_node.Scalar()); + } else { + errata.error(R"("{}" node at {} does not have a "{}" or "{}" key as required.)", YAML_CONTENT_KEY + , node.Mark(), YAML_CONTENT_LENGTH_KEY, YAML_CONTENT_DATA_KEY ); + } + } else { + errata.error(R"("{}" node at {} is not a map.)", YAML_CONTENT_KEY, + content_node.Mark()); + } + } + if (0 == _status && !_method) { errata.error( R"(HTTP header at {} has neither a status as a response nor a method as a request.)", @@ -731,6 +764,27 @@ swoc::TextView HttpHeader::localize(TextView text) { return text; } +swoc::TextView HttpHeader::localize(TextView text, Encoding enc) { + if (Encoding::URI == enc) { + auto span{_arena.require(text.size()).remnant().rebind()}; + auto spot = text.begin(), limit = text.end(); + char *dst = span.begin(); + while (spot < limit) { + if (*spot == '%' && + (spot + 1 < limit && isxdigit(spot[1]) && (spot + 2 < limit && isxdigit(spot[2])))) { + *dst++ = swoc::svto_radix<16>(TextView{spot + 1, spot + 3}); + spot += 3; + } else { + *dst++ = *spot++; + } + } + TextView text { span.data(), dst }; + _arena.alloc(text.size()); + return text; + } + return self_type::localize(text); +} + swoc::Rv HttpHeader::parse_request(swoc::TextView data) { swoc::Rv zret; diff --git a/local/src/server/replay-server.cc b/local/src/server/replay-server.cc index 15a8bb5..402c631 100644 --- a/local/src/server/replay-server.cc +++ b/local/src/server/replay-server.cc @@ -386,16 +386,21 @@ void Engine::command_run() { errata.clear(); } - // After this, any string expected to be localized that isn't is an error, - // so lock down the local string storage to avoid locking and report an - // error instead if not found. + // After this, any string expected to be localized that isn't is an error, so lock down the local + // string storage to avoid runtime locking and report an error instead if not found. HttpHeader::_frozen = true; size_t max_content_length = 0; for (auto const &[key, txn] : Transactions) { - max_content_length = - std::max(max_content_length, txn._rsp._content_size); + if (txn._rsp._content_data == nullptr) { // don't check responses with literal content. + max_content_length = std::max(max_content_length, txn._rsp._content_size); + } } HttpHeader::set_max_content_length(max_content_length); + for (auto &[key, txn] : Transactions) { + if (txn._rsp._content_data == nullptr) { // fill in from static content. + txn._rsp._content_data = txn._rsp._content.data(); + } + } std::cout << "Ready with " << Transactions.size() << " transactions." << std::endl; From a69321d450fa147fea3bdfcc6a515f1c1437f9fb Mon Sep 17 00:00:00 2001 From: "Alan M. Carroll" Date: Tue, 2 Jul 2019 09:11:51 -0500 Subject: [PATCH 2/2] Fix for "plain" text encoding. --- local/src/core/HttpReplay.cc | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/local/src/core/HttpReplay.cc b/local/src/core/HttpReplay.cc index 1c91b2c..ebe0697 100644 --- a/local/src/core/HttpReplay.cc +++ b/local/src/core/HttpReplay.cc @@ -362,7 +362,7 @@ swoc::Errata HttpHeader::update_content_length(swoc::TextView method) { _content_length_p = false; // Some methods ignore the Content-Length for the current transaction if (strcasecmp(method, "HEAD") == 0) { - // Don't try chuned encoding later + // Don't try chunked encoding later _content_size = 0; _content_length_p = true; } else { @@ -706,10 +706,13 @@ swoc::Errata HttpHeader::load(YAML::Node const &node) { if (auto data_node { content_node[YAML_CONTENT_DATA_KEY] } ; data_node) { Encoding enc { Encoding::TEXT }; if (auto enc_node { content_node[YAML_CONTENT_ENCODING_KEY] } ; enc_node) { - if (0 == strcasecmp("uri"_tv, enc_node.Scalar())) { + TextView text { enc_node.Scalar() }; + if (0 == strcasecmp("uri"_tv, text)) { enc = Encoding::URI; + } else if (0 == strcasecmp("plain"_tv, text)) { + enc = Encoding::TEXT; } else { - errata.error(R"(Unknown encoding "{}" at {}.)", enc_node.Scalar(), enc_node.Mark()); + errata.error(R"(Unknown encoding "{}" at {}.)", text, enc_node.Mark()); } } TextView content { this->localize(data_node.Scalar(), enc) };