Make http client connection buffer size configurable (#843)

umennel · deanberris · commit 5a1c8413d32b · 2018-05-28T20:59:44.000+10:00
Fixes #842
diff --git a/boost/network/protocol/http/client.hpp b/boost/network/protocol/http/client.hpp
@@ -21,7 +21,6 @@
 #include <string>
 
 #include <boost/network/protocol/http/client/facade.hpp>
-#include <boost/network/protocol/http/client/macros.hpp>
 #include <boost/network/protocol/http/client/options.hpp>
 
 namespace boost {
diff --git a/boost/network/protocol/http/client/async_impl.hpp b/boost/network/protocol/http/client/async_impl.hpp
@@ -13,6 +13,7 @@
 #include <functional>
 #include <boost/asio/io_service.hpp>
 #include <boost/asio/strand.hpp>
+#include <boost/network/protocol/http/client/macros.hpp>
 #include <boost/network/protocol/http/traits/connection_policy.hpp>
 
 namespace boost {
@@ -31,7 +32,10 @@ struct async_client
   typedef typename resolver<Tag>::type resolver_type;
   typedef typename string<Tag>::type string_type;
 
-  typedef typename std::array<typename char_<Tag>::type, 1024>::const_iterator const_iterator;
+  typedef
+      typename std::array<typename char_<Tag>::type,
+                          BOOST_NETWORK_HTTP_CLIENT_CONNECTION_BUFFER_SIZE>::
+          const_iterator const_iterator;
   typedef iterator_range<const_iterator> char_const_range;
 
   typedef std::function<void(char_const_range,
diff --git a/boost/network/protocol/http/client/connection/async_base.hpp b/boost/network/protocol/http/client/connection/async_base.hpp
@@ -31,7 +31,10 @@ struct async_connection_base {
   typedef typename string<Tag>::type string_type;
   typedef basic_request<Tag> request;
   typedef basic_response<Tag> response;
-  typedef typename std::array<typename char_<Tag>::type, 1024>::const_iterator const_iterator;
+  typedef
+      typename std::array<typename char_<Tag>::type,
+                          BOOST_NETWORK_HTTP_CLIENT_CONNECTION_BUFFER_SIZE>::
+          const_iterator const_iterator;
   typedef iterator_range<const_iterator> char_const_range;
   typedef std::function<void(char_const_range const &, boost::system::error_code const &)>
       body_callback_function_type;
diff --git a/boost/network/protocol/http/client/connection/async_normal.hpp b/boost/network/protocol/http/client/connection/async_normal.hpp
@@ -38,19 +38,20 @@ namespace network {
 namespace http {
 namespace impl {
 
-template <class Tag>
+template <class buffer_type>
 struct chunk_encoding_parser {
+  typedef typename buffer_type::const_iterator const_iterator;
+  typedef boost::iterator_range<const_iterator> char_const_range;
+
   chunk_encoding_parser() : state(state_t::header), chunk_size(0) {}
 
   enum class state_t { header, header_end, data, data_end };
 
   state_t state;
   size_t chunk_size;
-  std::array<typename char_<Tag>::type, 1024> buffer;
+  buffer_type buffer;
 
-  void update_chunk_size(
-      boost::iterator_range<typename std::array<
-          typename char_<Tag>::type, 1024>::const_iterator> const &range) {
+  void update_chunk_size(char_const_range const &range) {
     if (range.empty()) return;
     std::stringstream ss;
     ss << std::hex << range;
@@ -60,11 +61,7 @@ struct chunk_encoding_parser {
     chunk_size = (chunk_size << (range.size() * 4)) | size;
   }
 
-  boost::iterator_range<
-      typename std::array<typename char_<Tag>::type, 1024>::const_iterator>
-      operator()(
-          boost::iterator_range<typename std::array<
-              typename char_<Tag>::type, 1024>::const_iterator> const &range) {
+  char_const_range operator()(char_const_range const &range) {
     auto iter = boost::begin(range);
     auto begin = iter;
     auto pos = boost::begin(buffer);
@@ -147,6 +144,7 @@ struct http_async_connection
   typedef typename delegate_factory<Tag>::type delegate_factory_type;
   typedef typename delegate_factory_type::connection_delegate_ptr
       connection_delegate_ptr;
+  typedef chunk_encoding_parser<typename protocol_base::buffer_type> chunk_encoding_parser_type;
 
   http_async_connection(resolver_type& resolver, resolve_function resolve,
                         bool follow_redirect, int timeout,
@@ -484,13 +482,14 @@ struct http_async_connection
             } else {
               string_type body_string;
               if (this->is_chunk_encoding && remove_chunk_markers_) {
-                for (size_t i = 0; i < this->partial_parsed.size(); i += 1024) {
+                const auto parse_buffer_size = parse_chunk_encoding.buffer.size();
+                for (size_t i = 0; i < this->partial_parsed.size(); i += parse_buffer_size) {
                   auto range = parse_chunk_encoding(boost::make_iterator_range(
-                      static_cast<typename std::array<typename char_<Tag>::type, 1024>::const_iterator>(
-                          this->partial_parsed.data()) + i,
-                      static_cast<typename std::array<typename char_<Tag>::type, 1024>::const_iterator>(
-                          this->partial_parsed.data()) +
-                          std::min(i + 1024, this->partial_parsed.size())));
+                      static_cast<
+                          typename chunk_encoding_parser_type::const_iterator>(this->partial_parsed.data()) + i,
+                      static_cast<
+                          typename chunk_encoding_parser_type::const_iterator>(this->partial_parsed.data()) +
+                          std::min(i + parse_buffer_size, this->partial_parsed.size())));
                   body_string.append(boost::begin(range), boost::end(range));
                 }
                 this->partial_parsed.clear();
@@ -602,7 +601,7 @@ struct http_async_connection
   connection_delegate_ptr delegate_;
   boost::asio::streambuf command_streambuf;
   string_type method;
-  chunk_encoding_parser<Tag> parse_chunk_encoding;
+  chunk_encoding_parser_type parse_chunk_encoding;
 };
 
 }  // namespace impl
diff --git a/boost/network/protocol/http/client/connection/async_protocol_handler.hpp b/boost/network/protocol/http/client/connection/async_protocol_handler.hpp
@@ -13,6 +13,7 @@
 #include <array>
 #include <boost/logic/tribool.hpp>
 #include <boost/network/detail/debug.hpp>
+#include <boost/network/protocol/http/client/macros.hpp>
 #include <boost/network/protocol/http/algorithms/linearize.hpp>
 #include <boost/network/protocol/http/parser/incremental.hpp>
 #include <boost/network/protocol/http/request_parser.hpp>
@@ -400,8 +401,8 @@ struct http_async_protocol_handler {
   }
 
   typedef response_parser<Tag> response_parser_type;
-  // TODO(dberris): make 1024 go away and become a configurable value.
-  typedef std::array<typename char_<Tag>::type, 1024> buffer_type;
+  typedef std::array<typename char_<Tag>::type,
+                     BOOST_NETWORK_HTTP_CLIENT_CONNECTION_BUFFER_SIZE> buffer_type;
 
   response_parser_type response_parser_;
   boost::promise<string_type> version_promise;
diff --git a/boost/network/protocol/http/client/facade.hpp b/boost/network/protocol/http/client/facade.hpp
@@ -40,7 +40,10 @@ class basic_client_facade {
   /** The response type. This models the HTTP Response concept.*/
   typedef basic_response<Tag> response;
 
-  typedef typename std::array<typename char_<Tag>::type, 1024>::const_iterator const_iterator;
+  typedef
+      typename std::array<typename char_<Tag>::type,
+                          BOOST_NETWORK_HTTP_CLIENT_CONNECTION_BUFFER_SIZE>::
+          const_iterator const_iterator;
   typedef iterator_range<const_iterator> char_const_range;
 
   /**
diff --git a/boost/network/protocol/http/client/macros.hpp b/boost/network/protocol/http/client/macros.hpp
@@ -10,11 +10,21 @@
 #include <array>
 #include <system_error>
 
+#ifndef BOOST_NETWORK_HTTP_CLIENT_CONNECTION_BUFFER_SIZE
+/**
+ * We define the buffer size for each connection that we will use on the client
+ * side.
+ */
+#define BOOST_NETWORK_HTTP_CLIENT_CONNECTION_BUFFER_SIZE 4096uL
+#endif
+
 #ifndef BOOST_NETWORK_HTTP_BODY_CALLBACK
-#define BOOST_NETWORK_HTTP_BODY_CALLBACK(function_name, range_name,        \
-                                         error_name)                       \
-  void function_name(boost::iterator_range<std::array<char, 1024>::const_iterator> (range_name), \
-                     boost::system::error_code const& (error_name))
+#define BOOST_NETWORK_HTTP_BODY_CALLBACK(function_name, range_name, error_name)\
+  void function_name(                                                          \
+      boost::iterator_range<                                                   \
+          std::array<char, BOOST_NETWORK_HTTP_CLIENT_CONNECTION_BUFFER_SIZE>:: \
+              const_iterator>(range_name),                                     \
+      boost::system::error_code const&(error_name))
 #endif
 
 #endif /* BOOST_NETWORK_PROTOCOL_HTTP_CLIENT_MACROS_HPP_20110430 */
diff --git a/boost/network/protocol/http/policies/async_connection.hpp b/boost/network/protocol/http/policies/async_connection.hpp
@@ -30,7 +30,10 @@ struct async_connection_policy : resolver_policy<Tag>::type {
   typedef typename resolver_base::resolve_function resolve_function;
   typedef typename resolver_base::resolve_completion_function
       resolve_completion_function;
-  typedef typename std::array<typename char_<Tag>::type, 1024>::const_iterator const_iterator;
+  typedef
+      typename std::array<typename char_<Tag>::type,
+                          BOOST_NETWORK_HTTP_CLIENT_CONNECTION_BUFFER_SIZE>::
+          const_iterator const_iterator;
   typedef iterator_range<const_iterator> char_const_range;
   typedef std::function<void(char_const_range,
                              boost::system::error_code const&)>
