1 /**
2 	A simple HTTP/1.1 client implementation.
3 
4 	Copyright: © 2012-2014 Sönke Ludwig
5 	License: Subject to the terms of the MIT license, as written in the included LICENSE.txt file.
6 	Authors: Sönke Ludwig, Jan Krüger
7 */
8 module vibe.http.client;
9 
10 public import vibe.core.net;
11 public import vibe.http.common;
12 public import vibe.inet.url;
13 
14 import vibe.container.dictionarylist;
15 import vibe.container.internal.utilallocator;
16 import vibe.container.ringbuffer : RingBuffer;
17 import vibe.core.connectionpool;
18 import vibe.core.core;
19 import vibe.core.log;
20 import vibe.data.json;
21 import vibe.inet.message;
22 import vibe.inet.url;
23 import vibe.stream.counting;
24 import vibe.stream.tls;
25 import vibe.stream.operations;
26 import vibe.stream.wrapper : createConnectionProxyStream;
27 import vibe.stream.zlib;
28 import vibe.internal.freelistref;
29 import vibe.internal.interfaceproxy : InterfaceProxy, interfaceProxy;
30 
31 import core.exception : AssertError;
32 import std.algorithm : splitter;
33 import std.array;
34 import std.conv;
35 import std.encoding : sanitize;
36 import std.exception;
37 import std.format;
38 import std.string;
39 import std.typecons;
40 import std.datetime;
41 import std.socket : AddressFamily;
42 
43 version(Posix)
44 {
45 	version = UnixSocket;
46 }
47 
48 
49 /**************************************************************************************************/
50 /* Public functions                                                                               */
51 /**************************************************************************************************/
52 @safe:
53 
54 /**
55 	Performs a synchronous HTTP request on the specified URL.
56 
57 	The requester parameter allows to customize the request and to specify the request body for
58 	non-GET requests before it is sent. A response object is then returned or passed to the
59 	responder callback synchronously.
60 
61 	This function is a low-level HTTP client facility. It will not perform automatic redirect,
62 	caching or similar tasks. For a high-level download facility (similar to cURL), see the
63 	`vibe.inet.urltransfer` module.
64 
65 	Note that it is highly recommended to use one of the overloads that take a responder callback,
66 	as they can avoid some memory allocations and are safe against accidentally leaving stale
67 	response objects (objects whose response body wasn't fully read). For the returning overloads
68 	of the function it is recommended to put a `scope(exit)` right after the call in which
69 	`HTTPClientResponse.dropBody` is called to avoid this.
70 
71 	See_also: `vibe.inet.urltransfer.download`
72 */
73 HTTPClientResponse requestHTTP(string url, scope void delegate(scope HTTPClientRequest req) requester = null, const(HTTPClientSettings) settings = defaultSettings)
74 {
75 	return requestHTTP(URL.parse(url), requester, settings);
76 }
77 /// ditto
78 HTTPClientResponse requestHTTP(URL url, scope void delegate(scope HTTPClientRequest req) requester = null, const(HTTPClientSettings) settings = defaultSettings)
79 {
80 	auto cli = connectHTTP(url, settings);
81 	auto res = cli.request(
82 		(scope req){ httpRequesterDg(req, url, settings, requester); },
83 	);
84 
85 	// make sure the connection stays locked if the body still needs to be read
86 	if( res.m_client ) res.lockedConnection = cli;
87 
88 	logTrace("Returning HTTPClientResponse for conn %s", () @trusted { return cast(void*)res.lockedConnection.__conn; } ());
89 	return res;
90 }
91 /// ditto
92 void requestHTTP(string url, scope void delegate(scope HTTPClientRequest req) requester, scope void delegate(scope HTTPClientResponse req) responder, const(HTTPClientSettings) settings = defaultSettings)
93 {
94 	requestHTTP(URL(url), requester, responder, settings);
95 }
96 /// ditto
97 void requestHTTP(URL url, scope void delegate(scope HTTPClientRequest req) requester, scope void delegate(scope HTTPClientResponse req) responder, const(HTTPClientSettings) settings = defaultSettings)
98 {
99 	auto cli = connectHTTP(url, settings);
100 	cli.request(
101 		(scope req){ httpRequesterDg(req, url, settings, requester); },
102 		responder
103 	);
104 	assert(!cli.m_requesting, "HTTP client still requesting after return!?");
105 	assert(!cli.m_responding, "HTTP client still responding after return!?");
106 }
107 
108 private bool isTLSRequired(in URL url, in HTTPClientSettings settings)
109 {
110 	version(UnixSocket) {
111 		enforce(url.schema == "http" || url.schema == "https" || url.schema == "http+unix" || url.schema == "https+unix", "URL schema must be http(s) or http(s)+unix.");
112 	} else {
113 		enforce(url.schema == "http" || url.schema == "https", "URL schema must be http(s).");
114 	}
115 	enforce(url.host.length > 0, "URL must contain a host name.");
116 	bool use_tls;
117 
118 	if (settings.proxyURL.schema !is null)
119 		use_tls = settings.proxyURL.schema == "https";
120 	else
121 	{
122 		version(UnixSocket)
123 			use_tls = url.schema == "https" || url.schema == "https+unix";
124 		else
125 			use_tls = url.schema == "https";
126 	}
127 
128 	return use_tls;
129 }
130 
131 private void httpRequesterDg(scope HTTPClientRequest req, in URL url, in HTTPClientSettings settings, scope void delegate(scope HTTPClientRequest req) requester)
132 {
133 	import std.algorithm.searching : canFind;
134 	import vibe.http.internal.basic_auth_client: addBasicAuth;
135 
136 	if (url.localURI.length) {
137 		assert(url.path.absolute, "Request URL path must be absolute.");
138 		req.requestURL = url.localURI;
139 	}
140 
141 	if (settings.proxyURL.schema !is null)
142 		req.requestURL = url.toString(); // proxy exception to the URL representation
143 
144 	// IPv6 addresses need to be put into brackets
145 	auto hoststr = url.host.canFind(':') ? "["~url.host~"]" : url.host;
146 
147 	// Provide port number when it is not the default one (RFC2616 section 14.23)
148 	if (url.port && url.port != url.defaultPort)
149 		req.headers["Host"] = format("%s:%d", hoststr, url.port);
150 	else
151 		req.headers["Host"] = hoststr;
152 
153 	if ("authorization" !in req.headers && url.username != "")
154 		req.addBasicAuth(url.username, url.password);
155 
156 	if (requester) () @trusted { requester(req); } ();
157 }
158 
159 /** Posts a simple JSON request. Note that the server www.example.org does not
160 	exists, so there will be no meaningful result.
161 */
162 unittest {
163 	import vibe.core.log;
164 	import vibe.http.client;
165 	import vibe.stream.operations;
166 
167 	void test()
168 	{
169 		requestHTTP("http://www.example.org/",
170 			(scope req) {
171 				req.method = HTTPMethod.POST;
172 				//req.writeJsonBody(["name": "My Name"]);
173 			},
174 			(scope res) {
175 				logInfo("Response: %s", res.bodyReader.readAllUTF8());
176 			}
177 		);
178 	}
179 }
180 
181 
182 /**
183 	Returns a HTTPClient proxy object that is connected to the specified host.
184 
185 	Internally, a connection pool is used to reuse already existing connections. Note that
186 	usually requestHTTP should be used for making requests instead of manually using a
187 	HTTPClient to do so.
188 */
189 auto connectHTTP(string host, ushort port = 0, bool use_tls = false, const(HTTPClientSettings) settings = null)
190 {
191 	auto sttngs = settings ? settings : defaultSettings;
192 
193 	if (port == 0) port = use_tls ? 443 : 80;
194 	auto ckey = ConnInfo(host, sttngs.tlsPeerName, port, use_tls, sttngs.proxyURL.host, sttngs.proxyURL.port, sttngs.networkInterface);
195 
196 	ConnectionPool!HTTPClient pool;
197 	s_connections.opApply((ref c) @safe {
198 		if (c[0] == ckey)
199 			pool = c[1];
200 		return 0;
201 	});
202 
203 	if (!pool) {
204 		logDebug("Create HTTP client pool %s(%s):%s %s proxy %s:%d", host, sttngs.tlsPeerName, port, use_tls, sttngs.proxyURL.host, sttngs.proxyURL.port);
205 		pool = new ConnectionPool!HTTPClient({
206 				auto ret = new HTTPClient;
207 				ret.connect(host, port, use_tls, sttngs);
208 				return ret;
209 			});
210 		if (s_connections.full) s_connections.removeFront();
211 		s_connections.put(tuple(ckey, pool));
212 	}
213 
214 	return pool.lockConnection();
215 }
216 
217 /// Ditto
218 auto connectHTTP(URL url, const(HTTPClientSettings) settings = null)
219 {
220 	const use_tls = isTLSRequired(url, settings);
221 	return connectHTTP(url.getFilteredHost, url.port, use_tls, settings);
222 }
223 
224 static ~this()
225 {
226 	foreach (ci; s_connections) {
227 		ci[1].removeUnused((conn) {
228 			conn.disconnect();
229 		});
230 	}
231 }
232 
233 private struct ConnInfo { string host; string tlsPeerName; ushort port; bool useTLS; string proxyIP; ushort proxyPort; NetworkAddress bind_addr; }
234 private static RingBuffer!(Tuple!(ConnInfo, ConnectionPool!HTTPClient), 16) s_connections;
235 
236 
237 /**************************************************************************************************/
238 /* Public types                                                                                   */
239 /**************************************************************************************************/
240 
241 /**
242 	Defines an HTTP/HTTPS proxy request or a connection timeout for an HTTPClient.
243 */
244 class HTTPClientSettings {
245 	URL proxyURL;
246 	Duration defaultKeepAliveTimeout = 10.seconds;
247 
248 	/// Timeout for establishing a connection to the server
249 	Duration connectTimeout = Duration.max;
250 
251 	/// Timeout during read operations on the underyling transport
252 	Duration readTimeout = Duration.max;
253 
254 	/// Forces a specific network interface to use for outgoing connections.
255 	NetworkAddress networkInterface = anyAddress;
256 
257 	/// Can be used to force looking up IPv4/IPv6 addresses for host names.
258 	AddressFamily dnsAddressFamily = AddressFamily.UNSPEC;
259 
260 	/** Allows to customize the TLS context before connecting to a server.
261 
262 		Note that this overrides a callback set with `HTTPClient.setTLSContextSetup`.
263 	*/
264 	void delegate(TLSContext ctx) @safe nothrow tlsContextSetup;
265 
266 	/**
267 		TLS Peer name override.
268 
269 		Allows to customize the tls peer name sent to server during the TLS connection setup (SNI)
270 	*/
271 	string tlsPeerName;
272 
273 	@property HTTPClientSettings dup()
274 	const @safe {
275 		auto ret = new HTTPClientSettings;
276 		ret.proxyURL = this.proxyURL;
277 		ret.connectTimeout = this.connectTimeout;
278 		ret.readTimeout = this.readTimeout;
279 		ret.networkInterface = this.networkInterface;
280 		ret.dnsAddressFamily = this.dnsAddressFamily;
281 		ret.tlsContextSetup = this.tlsContextSetup;
282 		ret.tlsPeerName = this.tlsPeerName;
283 		return ret;
284 	}
285 }
286 
287 ///
288 unittest {
289 	void test() {
290 
291 		HTTPClientSettings settings = new HTTPClientSettings;
292 		settings.proxyURL = URL.parse("http://proxyuser:proxypass@192.168.2.50:3128");
293 		settings.defaultKeepAliveTimeout = 0.seconds; // closes connection immediately after receiving the data.
294 		requestHTTP("http://www.example.org",
295 					(scope req){
296 			req.method = HTTPMethod.GET;
297 		},
298 		(scope res){
299 			logInfo("Headers:");
300 			foreach (key, ref value; res.headers.byKeyValue) {
301 				logInfo("%s: %s", key, value);
302 			}
303 			logInfo("Response: %s", res.bodyReader.readAllUTF8());
304 		}, settings);
305 
306 	}
307 }
308 
309 version (Have_vibe_core)
310 unittest { // test connect timeout
311 	import std.conv : to;
312 	import vibe.core.stream : pipe, nullSink;
313 
314 	HTTPClientSettings settings = new HTTPClientSettings;
315 	settings.connectTimeout = 50.msecs;
316 
317 	// Use an IP address that is guaranteed to be unassigned globally to force
318 	// a timeout (see RFC 3330)
319 	auto cli = connectHTTP("192.0.2.0", 80, false, settings);
320 	auto timer = setTimer(500.msecs, { assert(false, "Connect timeout occurred too late"); });
321 	scope (exit) timer.stop();
322 
323 	try {
324 		cli.request(
325 			(scope req) { assert(false, "Expected no connection"); },
326 			(scope res) { assert(false, "Expected no response"); }
327 		);
328 		assert(false, "Response read expected to fail due to timeout");
329 	} catch(Exception e) {}
330 }
331 
332 unittest { // test read timeout
333 	import std.conv : to;
334 	import vibe.core.stream : pipe, nullSink;
335 
336 	version (VibeLibasyncDriver) {
337 		logInfo("Skipping HTTP client read timeout test due to buggy libasync driver.");
338 	} else {
339 		HTTPClientSettings settings = new HTTPClientSettings;
340 		settings.readTimeout = 50.msecs;
341 
342 		auto l = listenTCP(0, (conn) {
343 			try conn.pipe(nullSink);
344 			catch (Exception e) assert(false, e.msg);
345 			conn.close();
346 		}, "127.0.0.1");
347 
348 		auto cli = connectHTTP("127.0.0.1", l.bindAddress.port, false, settings);
349 		auto timer = setTimer(500.msecs, { assert(false, "Read timeout occurred too late"); });
350 		scope (exit) {
351 			timer.stop();
352 			l.stopListening();
353 			cli.disconnect();
354 			sleep(10.msecs); // allow the read connection end to fully close
355 		}
356 
357 		try {
358 			cli.request(
359 				(scope req) { req.method = HTTPMethod.GET; },
360 				(scope res) { assert(false, "Expected no response"); }
361 			);
362 			assert(false, "Response read expected to fail due to timeout");
363 		} catch(Exception e) {}
364 	}
365 }
366 
367 
368 /**
369 	Implementation of a HTTP 1.0/1.1 client with keep-alive support.
370 
371 	Note that it is usually recommended to use requestHTTP for making requests as that will use a
372 	pool of HTTPClient instances to keep the number of connection establishments low while not
373 	blocking requests from different tasks.
374 */
375 final class HTTPClient {
376 	@safe:
377 
378 	enum maxHeaderLineLength = 4096;
379 
380 	private {
381 		Rebindable!(const(HTTPClientSettings)) m_settings;
382 		string m_server;
383 		string m_tlsPeerName;
384 		ushort m_port;
385 		bool m_useTLS;
386 		TCPConnection m_conn;
387 		InterfaceProxy!Stream m_stream;
388 		TLSStream m_tlsStream;
389 		TLSContext m_tls;
390 		static __gshared m_userAgent = "vibe.d/"~vibeVersionString~" (HTTPClient, +http://vibed.org/)";
391 		static __gshared void function(TLSContext) ms_tlsSetup;
392 		bool m_requesting = false, m_responding = false;
393 		SysTime m_keepAliveLimit;
394 		Duration m_keepAliveTimeout;
395 	}
396 
397 	/** Get the current settings for the HTTP client. **/
398 	@property const(HTTPClientSettings) settings() const {
399 		return m_settings;
400 	}
401 
402 	/**
403 		Sets the default user agent string for new HTTP requests.
404 	*/
405 	static void setUserAgentString(string str) @trusted { m_userAgent = str; }
406 
407 	/**
408 		Sets a callback that will be called for every TLS context that is created.
409 
410 		Setting such a callback is useful for adjusting the validation parameters
411 		of the TLS context.
412 	*/
413 	static void setTLSSetupCallback(void function(TLSContext) @safe func) @trusted { ms_tlsSetup = func; }
414 
415 	/**
416 		Sets up this HTTPClient to connect to a specific server.
417 
418 		This method may only be called if any previous connection has been closed.
419 
420 		The actual connection is deferred until a request is initiated (using `HTTPClient.request`).
421 	*/
422 	void connect(string server, ushort port = 80, bool use_tls = false, const(HTTPClientSettings) settings = defaultSettings)
423 	{
424 		assert(!m_conn);
425 		assert(port != 0);
426 		disconnect();
427 		m_conn = TCPConnection.init;
428 		m_settings = settings;
429 		m_keepAliveTimeout = settings.defaultKeepAliveTimeout;
430 		m_keepAliveLimit = Clock.currTime(UTC()) + m_keepAliveTimeout;
431 		m_server = server;
432 		m_tlsPeerName = settings.tlsPeerName.length ? settings.tlsPeerName : server;
433 		m_port = port;
434 		m_useTLS = use_tls;
435 		if (use_tls) {
436 			m_tls = createTLSContext(TLSContextKind.client);
437 			// this will be changed to trustedCert once a proper root CA store is available by default
438 			m_tls.peerValidationMode = TLSPeerValidationMode.none;
439 			if (settings.tlsContextSetup) settings.tlsContextSetup(m_tls);
440 			else () @trusted { if (ms_tlsSetup) ms_tlsSetup(m_tls); } ();
441 		}
442 	}
443 
444 	/**
445 		Forcefully closes the TCP connection.
446 
447 		Before calling this method, be sure that no request is currently being processed.
448 	*/
449 	void disconnect()
450 	nothrow {
451 		if (m_conn) {
452 			version (Have_vibe_core) {}
453 			else scope(failure) assert(false);
454 
455 			if (m_conn.connected) {
456 				try m_stream.finalize();
457 				catch (Exception e) logDebug("Failed to finalize connection stream when closing HTTP client connection: %s", e.msg);
458 				m_conn.close();
459 			}
460 
461 			if (m_useTLS) () @trusted { return destroy(m_stream); } ();
462 			m_stream = InterfaceProxy!Stream.init;
463 			() @trusted { return destroy(m_conn); } ();
464 			m_conn = TCPConnection.init;
465 		}
466 	}
467 
468 	private void doProxyRequest(T, U)(ref T res, U requester, ref bool close_conn, ref bool has_body)
469 	@trusted { // scope new
470 		import std.conv : to;
471 		scope request_allocator = createRequestAllocator();
472 		scope (exit) freeRequestAllocator(request_allocator);
473 
474 		res.dropBody();
475 		scope(failure)
476 			res.disconnect();
477 		if (res.statusCode != 407) {
478 			throw new HTTPStatusException(HTTPStatus.internalServerError, "Proxy returned Proxy-Authenticate without a 407 status code.");
479 		}
480 
481 		// send the request again with the proxy authentication information if available
482 		if (m_settings.proxyURL.username is null) {
483 			throw new HTTPStatusException(HTTPStatus.proxyAuthenticationRequired, "Proxy Authentication Required.");
484 		}
485 
486 		m_responding = false;
487 		close_conn = false;
488 		bool found_proxy_auth;
489 
490 		foreach (string proxyAuth; res.headers.getAll("Proxy-Authenticate"))
491 		{
492 			if (proxyAuth.length >= "Basic".length && proxyAuth[0.."Basic".length] == "Basic")
493 			{
494 				found_proxy_auth = true;
495 				break;
496 			}
497 		}
498 
499 		if (!found_proxy_auth)
500 		{
501 			throw new HTTPStatusException(HTTPStatus.notAcceptable, "The Proxy Server didn't allow Basic Authentication");
502 		}
503 
504 		SysTime connected_time;
505 		has_body = doRequestWithRetry(requester, true, close_conn, connected_time);
506 		m_responding = true;
507 
508 		static if (is(T == HTTPClientResponse))
509 			res = new HTTPClientResponse(this, close_conn);
510 		else
511 			res = scoped!HTTPClientResponse(this, close_conn);
512 
513 		res.initialize(has_body, request_allocator, connected_time);
514 
515 		if (res.headers.get("Proxy-Authenticate", null) !is null){
516 			res.dropBody();
517 			throw new HTTPStatusException(HTTPStatus.proxyAuthenticationRequired, "Proxy Authentication Failed.");
518 		}
519 
520 	}
521 
522 	/**
523 		Performs a HTTP request.
524 
525 		`requester` is called first to populate the request with headers and the desired
526 		HTTP method and version. After a response has been received it is then passed
527 		to the caller which can in turn read the reponse body. Any part of the body
528 		that has not been processed will automatically be consumed and dropped.
529 
530 		Note that the `requester` callback might be invoked multiple times in the event
531 		that a request has to be resent due to a connection failure.
532 
533 		Also note that the second form of this method (returning a `HTTPClientResponse`) is
534 		not recommended to use as it may accidentially block a HTTP connection when
535 		only part of the response body was read and also requires a heap allocation
536 		for the response object. The callback based version on the other hand uses
537 		a stack allocation and guarantees that the request has been fully processed
538 		once it has returned.
539 	*/
540 	void request(scope void delegate(scope HTTPClientRequest req) requester, scope void delegate(scope HTTPClientResponse) responder)
541 	@trusted { // scope new
542 		scope request_allocator = createRequestAllocator();
543 		scope (exit) freeRequestAllocator(request_allocator);
544 
545 		scope (failure) {
546 			m_responding = false;
547 			disconnect();
548 		}
549 
550 		bool close_conn;
551 		SysTime connected_time;
552 		bool has_body = doRequestWithRetry(requester, false, close_conn, connected_time);
553 
554 		m_responding = true;
555 		auto res = scoped!HTTPClientResponse(this, close_conn);
556 		res.initialize(has_body, request_allocator, connected_time);
557 
558 		// proxy implementation
559 		if (res.headers.get("Proxy-Authenticate", null) !is null) {
560 			doProxyRequest(res, requester, close_conn, has_body);
561 		}
562 
563 		Exception user_exception;
564 		while (true)
565 		{
566 			try responder(res);
567 			catch (Exception e) {
568 				logDebug("Error while handling response: %s", e.toString().sanitize());
569 				user_exception = e;
570 			}
571 			if (res.statusCode < 200) {
572 				// just an informational status -> read and handle next response
573 				if (m_responding) res.dropBody();
574 				if (m_conn) {
575 					res = scoped!HTTPClientResponse(this, close_conn);
576 					res.initialize(has_body, request_allocator, connected_time);
577 					continue;
578 				}
579 			}
580 			if (m_responding) {
581 				logDebug("Failed to handle the complete response of the server - disconnecting.");
582 				res.disconnect();
583 			}
584 			assert(!m_responding, "Still in responding state after finalizing the response!?");
585 
586 			if (user_exception || res.headers.get("Connection") == "close")
587 				disconnect();
588 			break;
589 		}
590 		if (user_exception) throw user_exception;
591 	}
592 
593 	/// ditto
594 	HTTPClientResponse request(scope void delegate(HTTPClientRequest) requester)
595 	{
596 		bool close_conn;
597 		SysTime connected_time;
598 		scope (failure) {
599 			m_responding = false;
600 			disconnect();
601 		}
602 		bool has_body = doRequestWithRetry(requester, false, close_conn, connected_time);
603 		m_responding = true;
604 		auto res = new HTTPClientResponse(this, close_conn);
605 		res.initialize(has_body, () @trusted { return vibeThreadAllocator(); } (), connected_time);
606 
607 		// proxy implementation
608 		if (res.headers.get("Proxy-Authenticate", null) !is null) {
609 			doProxyRequest(res, requester, close_conn, has_body);
610 		}
611 
612 		return res;
613 	}
614 
615 	private bool doRequestWithRetry(scope void delegate(HTTPClientRequest req) requester, bool confirmed_proxy_auth /* basic only */, out bool close_conn, out SysTime connected_time)
616 	{
617 		if (m_conn && m_conn.connected && Clock.currTime(UTC()) > m_keepAliveLimit){
618 			logDebug("Disconnected to avoid timeout");
619 			disconnect();
620 		}
621 
622 		// check if this isn't the first request on a connection
623 		bool is_persistent_request = m_conn && m_conn.connected;
624 
625 		// retry the request if the connection gets closed prematurely and this is a persistent request
626 		bool has_body;
627 		foreach (i; 0 .. is_persistent_request ? 2 : 1) {
628 		 	connected_time = Clock.currTime(UTC());
629 
630 			close_conn = false;
631 			has_body = doRequest(requester, close_conn, false, connected_time);
632 
633 			logTrace("HTTP client waiting for response");
634 			if (!m_stream.empty) break;
635 		}
636 		return has_body;
637 	}
638 
639 	private bool doRequest(scope void delegate(HTTPClientRequest req) requester, ref bool close_conn, bool confirmed_proxy_auth = false /* basic only */, SysTime connected_time = Clock.currTime(UTC()))
640 	{
641 		assert(!m_requesting, "Interleaved HTTP client requests detected!");
642 		assert(!m_responding, "Interleaved HTTP client request/response detected!");
643 
644 		m_requesting = true;
645 		scope(exit) m_requesting = false;
646 
647 		if (!m_conn || !m_conn.connected || m_conn.waitForDataEx(0.seconds) == WaitForDataStatus.noMoreData) {
648 			if (m_conn)
649 				disconnect(); // make sure all resources are freed
650 
651 			if (m_settings.proxyURL.host !is null){
652 
653 				enum AddressType {
654 					IPv4,
655 					IPv6,
656 					Host
657 				}
658 
659 				static AddressType getAddressType(string host){
660 					import std.regex : regex, Captures, Regex, matchFirst;
661 
662 					static IPv4Regex = regex(`^\s*((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))\s*$`, ``);
663 					static IPv6Regex = regex(`^\s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?\s*$`, ``);
664 
665 					if (!matchFirst(host, IPv4Regex).empty)
666 					{
667 						return AddressType.IPv4;
668 					}
669 					else if (!matchFirst(host, IPv6Regex).empty)
670 					{
671 						return AddressType.IPv6;
672 					}
673 					else
674 					{
675 						return AddressType.Host;
676 					}
677 				}
678 
679 				import std.functional : memoize;
680 				alias findAddressType = memoize!getAddressType;
681 
682 				bool use_dns;
683 				if (() @trusted { return findAddressType(m_settings.proxyURL.host); } () == AddressType.Host)
684 				{
685 					use_dns = true;
686 				}
687 
688 				NetworkAddress proxyAddr = resolveHost(m_settings.proxyURL.host, m_settings.dnsAddressFamily, use_dns,
689 					m_settings.connectTimeout);
690 				proxyAddr.port = m_settings.proxyURL.port;
691 				m_conn = connectTCPWithTimeout(proxyAddr, m_settings.networkInterface, m_settings.connectTimeout);
692 			}
693 			else {
694 				version(UnixSocket)
695 				{
696 					import core.sys.posix.sys.un;
697 					import core.sys.posix.sys.socket;
698 					import std.regex : regex, Captures, Regex, matchFirst, ctRegex;
699 					import core.stdc.string : strcpy;
700 
701 					NetworkAddress addr;
702 					if (m_server[0] == '/')
703 					{
704 						addr.family = AF_UNIX;
705 						sockaddr_un* s = addr.sockAddrUnix();
706 						enforce(s.sun_path.length > m_server.length, "Unix sockets cannot have that long a name.");
707 						s.sun_family = AF_UNIX;
708 						() @trusted { strcpy(cast(char*)s.sun_path.ptr,m_server.toStringz()); } ();
709 					} else
710 					{
711 						addr = resolveHost(m_server, m_settings.dnsAddressFamily, true, m_settings.connectTimeout);
712 						addr.port = m_port;
713 					}
714 					m_conn = connectTCPWithTimeout(addr, m_settings.networkInterface, m_settings.connectTimeout);
715 				} else
716 				{
717 					auto addr = resolveHost(m_server, m_settings.dnsAddressFamily, true, m_settings.connectTimeout);
718 					addr.port = m_port;
719 					m_conn = connectTCPWithTimeout(addr, m_settings.networkInterface, m_settings.connectTimeout);
720 				}
721 			}
722 
723 			if (m_settings.readTimeout != Duration.max)
724 				m_conn.readTimeout = m_settings.readTimeout;
725 
726 			m_stream = m_conn;
727 			if (m_useTLS) {
728 				try m_tlsStream = createTLSStream(m_conn, m_tls, TLSStreamState.connecting, m_tlsPeerName, m_conn.remoteAddress);
729 				catch (Exception e) {
730 					m_conn.close();
731 					m_conn = TCPConnection.init;
732 					throw e;
733 				}
734 				m_stream = m_tlsStream;
735 			}
736 		}
737 
738 		return () @trusted { // scoped
739 			auto req = scoped!HTTPClientRequest(m_stream, m_conn);
740 			if (m_useTLS)
741 				req.m_peerCertificate = m_tlsStream.peerCertificate;
742 
743 			req.headers["User-Agent"] = m_userAgent;
744 			if (m_settings.proxyURL.host !is null){
745 				req.headers["Proxy-Connection"] = "keep-alive";
746 				if (confirmed_proxy_auth)
747 				{
748 					import std.base64;
749 					ubyte[] user_pass = cast(ubyte[])(m_settings.proxyURL.username ~ ":" ~ m_settings.proxyURL.password);
750 
751 					req.headers["Proxy-Authorization"] = "Basic " ~ cast(string) Base64.encode(user_pass);
752 				}
753 			}
754 			else {
755 				req.headers["Connection"] = "keep-alive";
756 			}
757 			req.headers["Accept-Encoding"] = "gzip, deflate";
758 			req.headers["Host"] = m_server;
759 			requester(req);
760 
761 			if (req.httpVersion == HTTPVersion.HTTP_1_0)
762 				close_conn = true;
763 			else  if (m_settings.proxyURL.host !is null)
764 				close_conn = req.headers.get("Proxy-Connection", "keep-alive") != "keep-alive";
765 			else
766 				close_conn = req.headers.get("Connection", "keep-alive") != "keep-alive";
767 
768 			req.finalize();
769 
770 			return req.method != HTTPMethod.HEAD;
771 		} ();
772 	}
773 }
774 
775 private auto connectTCPWithTimeout(NetworkAddress addr, NetworkAddress bind_address, Duration timeout)
776 {
777 	return connectTCP(addr, bind_address, timeout);
778 }
779 
780 /**
781 	Represents a HTTP client request (as sent to the server).
782 */
783 final class HTTPClientRequest : HTTPRequest {
784 	import vibe.internal.array : FixedAppender;
785 
786 	private {
787 		InterfaceProxy!OutputStream m_bodyWriter;
788 		FreeListRef!ChunkedOutputStream m_chunkedStream;
789 		bool m_headerWritten = false;
790 		FixedAppender!(string, 22) m_contentLengthBuffer;
791 		TCPConnection m_rawConn;
792 		TLSCertificateInformation m_peerCertificate;
793 	}
794 
795 
796 	/// private
797 	this(InterfaceProxy!Stream conn, TCPConnection raw_conn)
798 	{
799 		super(conn);
800 		m_rawConn = raw_conn;
801 	}
802 
803 	@property NetworkAddress localAddress() const { return m_rawConn.localAddress; }
804 	@property NetworkAddress remoteAddress() const { return m_rawConn.remoteAddress; }
805 
806 	@property ref inout(TLSCertificateInformation) peerCertificate() inout { return m_peerCertificate; }
807 
808 	/**
809 		Accesses the Content-Length header of the request.
810 
811 		Negative values correspond to an unset Content-Length header.
812 	*/
813 	@property long contentLength() const { return headers.get("Content-Length", "-1").to!long(); }
814 	/// ditto
815 	@property void contentLength(long value)
816 	{
817 		if (value >= 0) headers["Content-Length"] = clengthString(value);
818 		else if ("Content-Length" in headers) headers.remove("Content-Length");
819 	}
820 
821 	/**
822 		Writes the whole request body at once using raw bytes.
823 	*/
824 	void writeBody(RandomAccessStream data)
825 	{
826 		writeBody(data, data.size - data.tell());
827 	}
828 	/// ditto
829 	void writeBody(InputStream data)
830 	{
831 		data.pipe(bodyWriter);
832 		finalize();
833 	}
834 	/// ditto
835 	void writeBody(InputStream data, ulong length)
836 	{
837 		headers["Content-Length"] = clengthString(length);
838 		data.pipe(bodyWriter, length);
839 		finalize();
840 	}
841 	/// ditto
842 	void writeBody(in ubyte[] data, string content_type = null)
843 	{
844 		if( content_type != "" ) headers["Content-Type"] = content_type;
845 		headers["Content-Length"] = clengthString(data.length);
846 		bodyWriter.write(data);
847 		finalize();
848 	}
849 
850 	/**
851 		Writes the request body as JSON data.
852 	*/
853 	void writeJsonBody(T)(T data, bool allow_chunked = false)
854 	{
855 		import vibe.stream.wrapper : streamOutputRange;
856 
857 		headers["Content-Type"] = "application/json; charset=UTF-8";
858 
859 		// set an explicit content-length field if chunked encoding is not allowed
860 		if (!allow_chunked) {
861 			import vibe.internal.rangeutil;
862 			long length = 0;
863 			auto counter = () @trusted { return RangeCounter(&length); } ();
864 			() @trusted { serializeToJson(counter, data); } ();
865 			headers["Content-Length"] = clengthString(length);
866 		}
867 
868 		auto rng = streamOutputRange!1024(bodyWriter);
869 		() @trusted { serializeToJson(&rng, data); } ();
870 		rng.flush();
871 		finalize();
872 	}
873 
874 	/** Writes the request body as form data.
875 	*/
876 	void writeFormBody(T)(T key_value_map)
877 	{
878 		import vibe.inet.webform : formEncode;
879 		import vibe.stream.wrapper : streamOutputRange;
880 
881 		import vibe.internal.rangeutil;
882 		long length = 0;
883 		auto counter = () @trusted { return RangeCounter(&length); } ();
884 		counter.formEncode(key_value_map);
885 		headers["Content-Length"] = clengthString(length);
886 		headers["Content-Type"] = "application/x-www-form-urlencoded";
887 		auto dst = streamOutputRange!1024(bodyWriter);
888 		() @trusted { return &dst; } ().formEncode(key_value_map);
889 	}
890 
891 	///
892 	unittest {
893 		void test(HTTPClientRequest req) {
894 			req.writeFormBody(["foo": "bar"]);
895 		}
896 	}
897 
898 	void writePart(MultiPart part)
899 	{
900 		assert(false, "TODO");
901 	}
902 
903 	/**
904 		An output stream suitable for writing the request body.
905 
906 		The first retrieval will cause the request header to be written, make sure
907 		that all headers are set up in advance.s
908 	*/
909 	@property InterfaceProxy!OutputStream bodyWriter()
910 	{
911 		if (m_bodyWriter) return m_bodyWriter;
912 
913 		assert(!m_headerWritten, "Trying to write request body after body was already written.");
914 
915 		if (httpVersion != HTTPVersion.HTTP_1_0
916 			&& "Content-Length" !in headers && "Transfer-Encoding" !in headers
917 			&& headers.get("Connection", "") != "close")
918 		{
919 			headers["Transfer-Encoding"] = "chunked";
920 		}
921 
922 		writeHeader();
923 		m_bodyWriter = m_conn;
924 
925 		if (headers.get("Transfer-Encoding", null) == "chunked") {
926 			m_chunkedStream = createChunkedOutputStreamFL(m_bodyWriter);
927 			m_bodyWriter = m_chunkedStream;
928 		}
929 
930 		return m_bodyWriter;
931 	}
932 
933 	private void writeHeader()
934 	{
935 		import vibe.stream.wrapper;
936 
937 		assert(!m_headerWritten, "HTTPClient tried to write headers twice.");
938 		m_headerWritten = true;
939 
940 		auto output = streamOutputRange!1024(m_conn);
941 
942 		formattedWrite(() @trusted { return &output; } (), "%s %s %s\r\n", httpMethodString(method), requestURL, getHTTPVersionString(httpVersion));
943 		logTrace("--------------------");
944 		logTrace("HTTP client request:");
945 		logTrace("--------------------");
946 		logTrace("%s", this);
947 		foreach (k, v; headers.byKeyValue) {
948 			() @trusted { formattedWrite(&output, "%s: %s\r\n", k, v); } ();
949 			logTrace("%s: %s", k, v);
950 		}
951 		output.put("\r\n");
952 		logTrace("--------------------");
953 	}
954 
955 	private void finalize()
956 	{
957 		// test if already finalized
958 		if (m_headerWritten && !m_bodyWriter)
959 			return;
960 
961 		// force the request to be sent
962 		if (!m_headerWritten) writeHeader();
963 		else {
964 			bodyWriter.flush();
965 			if (m_chunkedStream) {
966 				m_bodyWriter.finalize();
967 				m_conn.flush();
968 			}
969 			m_bodyWriter = typeof(m_bodyWriter).init;
970 			m_conn = typeof(m_conn).init;
971 		}
972 	}
973 
974 	private string clengthString(ulong len)
975 	{
976 		m_contentLengthBuffer.clear();
977 		() @trusted { formattedWrite(&m_contentLengthBuffer, "%s", len); } ();
978 		return () @trusted { return m_contentLengthBuffer.data; } ();
979 	}
980 }
981 
982 
983 /**
984 	Represents a HTTP client response (as received from the server).
985 */
986 final class HTTPClientResponse : HTTPResponse {
987 	@safe:
988 
989 	private {
990 		HTTPClient m_client;
991 		LockedConnection!HTTPClient lockedConnection;
992 		FreeListRef!LimitedInputStream m_limitedInputStream;
993 		FreeListRef!ChunkedInputStream m_chunkedInputStream;
994 		FreeListRef!ZlibInputStream m_zlibInputStream;
995 		FreeListRef!EndCallbackInputStream m_endCallback;
996 		InterfaceProxy!InputStream m_bodyReader;
997 		bool m_closeConn;
998 		int m_maxRequests;
999 	}
1000 
1001 	/// Contains the keep-alive 'max' parameter, indicates how many requests a client can
1002 	/// make before the server closes the connection.
1003 	@property int maxRequests() const {
1004 		return m_maxRequests;
1005 	}
1006 
1007 
1008 	/// All cookies that shall be set on the client for this request
1009 	override @property ref DictionaryList!Cookie cookies() {
1010 		if ("Set-Cookie" in this.headers && m_cookies.length == 0) {
1011 			foreach (cookieString; this.headers.getAll("Set-Cookie")) {
1012 				auto cookie = parseHTTPCookie(cookieString);
1013 				if (cookie[0].length)
1014 					m_cookies[cookie[0]] = cookie[1];
1015 			}
1016 		}
1017 		return m_cookies;
1018 	}
1019 
1020 	/// private
1021 	this(HTTPClient client, bool close_conn)
1022 	nothrow {
1023 		m_client = client;
1024 		m_closeConn = close_conn;
1025 	}
1026 
1027 	private void initialize(Allocator)(bool has_body, Allocator alloc, SysTime connected_time = Clock.currTime(UTC()))
1028 	{
1029 		scope(failure) finalize(true);
1030 
1031 		// read and parse status line ("HTTP/#.# #[ $]\r\n")
1032 		logTrace("HTTP client reading status line");
1033 		string stln = () @trusted { return cast(string)m_client.m_stream.readLine(HTTPClient.maxHeaderLineLength, "\r\n", alloc); } ();
1034 		logTrace("stln: %s", stln);
1035 		this.httpVersion = parseHTTPVersion(stln);
1036 
1037 		enforce(stln.startsWith(" "));
1038 		stln = stln[1 .. $];
1039 		this.statusCode = parse!int(stln);
1040 		if( stln.length > 0 ){
1041 			enforce(stln.startsWith(" "));
1042 			stln = stln[1 .. $];
1043 			this.statusPhrase = stln;
1044 		}
1045 
1046 		// read headers until an empty line is hit
1047 		parseRFC5322Header(m_client.m_stream, this.headers, HTTPClient.maxHeaderLineLength, alloc, false);
1048 
1049 		logTrace("---------------------");
1050 		logTrace("HTTP client response:");
1051 		logTrace("---------------------");
1052 		logTrace("%s", this);
1053 		foreach (k, v; this.headers.byKeyValue)
1054 			logTrace("%s: %s", k, v);
1055 		logTrace("---------------------");
1056 		Duration server_timeout;
1057 		bool has_server_timeout;
1058 		if (auto pka = "Keep-Alive" in this.headers) {
1059 			foreach(s; splitter(*pka, ',')){
1060 				auto pair = s.splitter('=');
1061 				auto name = pair.front.strip();
1062 				pair.popFront();
1063 				if (icmp(name, "timeout") == 0) {
1064 					has_server_timeout = true;
1065 					server_timeout = pair.front.to!int().seconds;
1066 				} else if (icmp(name, "max") == 0) {
1067 					m_maxRequests = pair.front.to!int();
1068 				}
1069 			}
1070 		}
1071 		Duration elapsed = Clock.currTime(UTC()) - connected_time;
1072 		if (this.headers.get("Connection") == "close") {
1073 			// this header will trigger m_client.disconnect() in m_client.doRequest() when it goes out of scope
1074 		} else if (has_server_timeout && m_client.m_keepAliveTimeout > server_timeout) {
1075 			m_client.m_keepAliveLimit = Clock.currTime(UTC()) + server_timeout - elapsed;
1076 		} else if (this.httpVersion == HTTPVersion.HTTP_1_1) {
1077 			m_client.m_keepAliveLimit = Clock.currTime(UTC()) + m_client.m_keepAliveTimeout;
1078 		}
1079 
1080 		if (!has_body) finalize();
1081 	}
1082 
1083 	~this()
1084 	{
1085 		debug if (m_client) {
1086 			import core.stdc.stdio;
1087 			printf("WARNING: HTTPClientResponse not fully processed before being finalized\n");
1088 		}
1089 	}
1090 
1091 	/**
1092 		An input stream suitable for reading the response body.
1093 	*/
1094 	@property InterfaceProxy!InputStream bodyReader()
1095 	{
1096 		if( m_bodyReader ) return m_bodyReader;
1097 
1098 		assert (m_client, "Response was already read or no response body, may not use bodyReader.");
1099 
1100 		// prepare body the reader
1101 		if (auto pte = "Transfer-Encoding" in this.headers) {
1102 			enforce(*pte == "chunked");
1103 			m_chunkedInputStream = createChunkedInputStreamFL(m_client.m_stream);
1104 			m_bodyReader = this.m_chunkedInputStream;
1105 		} else if (auto pcl = "Content-Length" in this.headers) {
1106 			m_limitedInputStream = createLimitedInputStreamFL(m_client.m_stream, to!ulong(*pcl));
1107 			m_bodyReader = m_limitedInputStream;
1108 		} else if (isKeepAliveResponse) {
1109 			m_limitedInputStream = createLimitedInputStreamFL(m_client.m_stream, 0);
1110 			m_bodyReader = m_limitedInputStream;
1111 		} else {
1112 			m_bodyReader = m_client.m_stream;
1113 		}
1114 
1115 		if( auto pce = "Content-Encoding" in this.headers ){
1116 			if( *pce == "deflate" ){
1117 				m_zlibInputStream = createDeflateInputStreamFL(m_bodyReader);
1118 				m_bodyReader = m_zlibInputStream;
1119 			} else if( *pce == "gzip" || *pce == "x-gzip"){
1120 				m_zlibInputStream = createGzipInputStreamFL(m_bodyReader);
1121 				m_bodyReader = m_zlibInputStream;
1122 			}
1123 			else enforce(*pce == "identity" || *pce == "", "Unsuported content encoding: "~*pce);
1124 		}
1125 
1126 		// be sure to free resouces as soon as the response has been read
1127 		m_endCallback = createEndCallbackInputStreamFL(m_bodyReader, &this.finalize);
1128 		m_bodyReader = m_endCallback;
1129 
1130 		return m_bodyReader;
1131 	}
1132 
1133 	/**
1134 		Provides unsafe means to read raw data from the connection.
1135 
1136 		No transfer decoding and no content decoding is done on the data.
1137 
1138 		Not that the provided delegate must read the whole stream,
1139 		as the state of the response is unknown after raw bytes have been
1140 		taken. Failure to read the right amount of data will lead to
1141 		protocol corruption in later requests.
1142 	*/
1143 	void readRawBody(scope void delegate(scope InterfaceProxy!InputStream stream) @safe del)
1144 	{
1145 		assert(!m_bodyReader, "May not mix use of readRawBody and bodyReader.");
1146 		del(interfaceProxy!InputStream(m_client.m_stream));
1147 		finalize();
1148 	}
1149 	/// ditto
1150 	static if (!is(InputStream == InterfaceProxy!InputStream))
1151 	void readRawBody(scope void delegate(scope InputStream stream) @safe del)
1152 	{
1153 		import vibe.internal.interfaceproxy : asInterface;
1154 
1155 		assert(!m_bodyReader, "May not mix use of readRawBody and bodyReader.");
1156 		del(m_client.m_stream.asInterface!(.InputStream));
1157 		finalize();
1158 	}
1159 
1160 	/**
1161 		Reads the whole response body and tries to parse it as JSON.
1162 	*/
1163 	Json readJson(){
1164 		auto bdy = bodyReader.readAllUTF8();
1165 		return () @trusted { return parseJson(bdy); } ();
1166 	}
1167 
1168 	/**
1169 		Reads and discards the response body.
1170 	*/
1171 	void dropBody()
1172 	{
1173 		if (m_client) {
1174 			if( bodyReader.empty ){
1175 				finalize();
1176 			} else {
1177 				bodyReader.pipe(nullSink);
1178 				assert(!lockedConnection.__conn);
1179 			}
1180 		}
1181 	}
1182 
1183 	/**
1184 		Forcefully terminates the connection regardless of the current state.
1185 
1186 		Note that this will only actually disconnect if the request has not yet
1187 		been fully processed. If the whole body was already read, the
1188 		connection is not owned by the current request operation anymore and
1189 		cannot be accessed. Use a "Connection: close" header instead in this
1190 		case to let the server close the connection.
1191 	*/
1192 	void disconnect()
1193 	{
1194 		finalize(true);
1195 	}
1196 
1197 	/**
1198 		Switches the connection to a new protocol and returns the resulting ConnectionStream.
1199 
1200 		The caller caller gets ownership of the ConnectionStream and is responsible
1201 		for closing it.
1202 
1203 		Notice:
1204 			When using the overload that returns a `ConnectionStream`, the caller
1205 			must make sure that the stream is not used after the
1206 			`HTTPClientRequest` has been destroyed.
1207 
1208 		Params:
1209 			new_protocol = The protocol to which the connection is expected to
1210 				upgrade. Should match the Upgrade header of the request. If an
1211 				empty string is passed, the "Upgrade" header will be ignored and
1212 				should be checked by other means.
1213 	*/
1214 	ConnectionStream switchProtocol(string new_protocol)
1215 	{
1216 		enforce(statusCode == HTTPStatus.switchingProtocols, "Server did not send a 101 - Switching Protocols response");
1217 		string *resNewProto = "Upgrade" in headers;
1218 		enforce(resNewProto, "Server did not send an Upgrade header");
1219 		enforce(!new_protocol.length || !icmp(*resNewProto, new_protocol),
1220 			"Expected Upgrade: " ~ new_protocol ~", received Upgrade: " ~ *resNewProto);
1221 		auto stream = createConnectionProxyStream!(typeof(m_client.m_stream), typeof(m_client.m_conn))(m_client.m_stream, m_client.m_conn);
1222 		m_closeConn = true; // cannot reuse connection for further requests!
1223 		return stream;
1224 	}
1225 	/// ditto
1226 	void switchProtocol(string new_protocol, scope void delegate(ConnectionStream str) @safe del)
1227 	{
1228 		enforce(statusCode == HTTPStatus.switchingProtocols, "Server did not send a 101 - Switching Protocols response");
1229 		string *resNewProto = "Upgrade" in headers;
1230 		enforce(resNewProto, "Server did not send an Upgrade header");
1231 		enforce(!new_protocol.length || !icmp(*resNewProto, new_protocol),
1232 			"Expected Upgrade: " ~ new_protocol ~", received Upgrade: " ~ *resNewProto);
1233 		auto stream = createConnectionProxyStream(m_client.m_stream, m_client.m_conn);
1234 		scope (exit) () @trusted { destroy(stream); } ();
1235 		m_closeConn = true;
1236 		del(stream);
1237 	}
1238 
1239 	private @property isKeepAliveResponse()
1240 	const {
1241 		string conn;
1242 		if (this.httpVersion == HTTPVersion.HTTP_1_0) {
1243 			// Workaround for non-standard-conformant servers - for example see #1780
1244 			auto pcl = "Content-Length" in this.headers;
1245 			if (pcl) conn = this.headers.get("Connection", "close");
1246 			else return false; // can't use keepalive when no content length is set
1247 		}
1248 		else conn = this.headers.get("Connection", "keep-alive");
1249 		return icmp(conn, "close") != 0;
1250 	}
1251 
1252 	private void finalize()
1253 	{
1254 		finalize(m_closeConn);
1255 	}
1256 
1257 	private void finalize(bool disconnect)
1258 	{
1259 		// ignore duplicate and too early calls to finalize
1260 		// (too early happesn for empty response bodies)
1261 		if (!m_client) return;
1262 
1263 		auto cli = m_client;
1264 		m_client = null;
1265 		cli.m_responding = false;
1266 		destroy(m_endCallback);
1267 		destroy(m_zlibInputStream);
1268 		destroy(m_chunkedInputStream);
1269 		destroy(m_limitedInputStream);
1270 		if (disconnect) cli.disconnect();
1271 		destroy(lockedConnection);
1272 	}
1273 }
1274 
1275 /** Returns clean host string. In case of unix socket it performs urlDecode on host. */
1276 package auto getFilteredHost(URL url)
1277 {
1278 	version(UnixSocket)
1279 	{
1280 		import vibe.textfilter.urlencode : urlDecode;
1281 		if (url.schema == "https+unix" || url.schema == "http+unix")
1282 			return urlDecode(url.host);
1283 		else
1284 			return url.host;
1285 	} else
1286 		return url.host;
1287 }
1288 
1289 // This object is a placeholder and should to never be modified.
1290 package @property const(HTTPClientSettings) defaultSettings()
1291 @trusted nothrow {
1292 	__gshared HTTPClientSettings ret = new HTTPClientSettings;
1293 	return ret;
1294 }