The OpenD Programming Language

1 // Copyright 2013-2022, Adam D. Ruppe.
2 
3 // FIXME: websocket proxy support
4 // FIXME: ipv6 support
5 
6 // FIXME: headers are supposed to be case insensitive. ugh.
7 
8 /++
9 	This is version 2 of my http/1.1 client implementation.
10 
11 
12 	It has no dependencies for basic operation, but does require OpenSSL
13 	libraries (or compatible) to support HTTPS. This dynamically loaded
14 	on-demand (meaning it won't be loaded if you don't use it, but if you do
15 	use it, the openssl dynamic libraries must be found in the system search path).
16 
17 	On Windows, you can bundle the openssl dlls with your exe and they will be picked
18 	up when distributed.
19 
20 	You can compile with `-version=without_openssl` to entirely disable ssl support.
21 
22 	http2.d, despite its name, does NOT implement HTTP/2.0, but this
23 	shouldn't matter for 99.9% of usage, since all servers will continue
24 	to support HTTP/1.1 for a very long time.
25 
26 	History:
27 		Automatic `100 Continue` handling was added on September 28, 2021. It doesn't
28 		set the Expect header, so it isn't supposed to happen, but plenty of web servers
29 		don't follow the standard anyway.
30 
31 		A dependency on [arsd.core] was added on March 19, 2023 (dub v11.0). Previously,
32 		module was stand-alone. You will have add the `core.d` file from the arsd repo
33 		to your build now if you are managing the files and builds yourself.
34 
35 		The benefits of this dependency include some simplified implementation code which
36 		makes it easier for me to add more api conveniences, better exceptions with more
37 		information, and better event loop integration with other arsd modules beyond
38 		just the simpledisplay adapters available previously. The new integration can
39 		also make things like heartbeat timers easier for you to code.
40 +/
41 module arsd.http2;
42 
43 ///
44 unittest {
45 	import arsd.http2;
46 
47 	void main() {
48 		auto client = new HttpClient();
49 
50 		auto request = client.request(Uri("http://dlang.org/"));
51 		auto response = request.waitForCompletion();
52 
53 		import std.stdio;
54 		writeln(response.contentText);
55 		writeln(response.code, " ", response.codeText);
56 		writeln(response.contentType);
57 	}
58 
59 	version(arsd_http2_integration_test) main(); // exclude from docs
60 }
61 
62 /+
63 // arsd core is now default but you can opt out for a lil while
64 version(no_arsd_core) {
65 
66 } else {
67 	version=use_arsd_core;
68 }
69 +/
70 
71 static import arsd.core;
72 
73 // FIXME: I think I want to disable sigpipe here too.
74 
75 import arsd.core : encodeUriComponent, decodeUriComponent;
76 
77 debug(arsd_http2_verbose) debug=arsd_http2;
78 
79 debug(arsd_http2) import std.stdio : writeln;
80 
81 version=arsd_http_internal_implementation;
82 
83 version(without_openssl) {}
84 else {
85 version=use_openssl;
86 version=with_openssl;
87 version(older_openssl) {} else
88 version=newer_openssl;
89 }
90 
91 version(arsd_http_winhttp_implementation) {
92 	pragma(lib, "winhttp")
93 	import core.sys.windows.winhttp;
94 	// FIXME: alter the dub package file too
95 
96 	// https://github.com/curl/curl/blob/master/lib/vtls/schannel.c
97 	// https://docs.microsoft.com/en-us/windows/win32/secauthn/creating-an-schannel-security-context
98 
99 
100 	// https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpreaddata
101 	// https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpsendrequest
102 	// https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpopenrequest
103 	// https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpconnect
104 }
105 
106 
107 
108 /++
109 	Demonstrates core functionality, using the [HttpClient],
110 	[HttpRequest] (returned by [HttpClient.navigateTo|client.navigateTo]),
111 	and [HttpResponse] (returned by [HttpRequest.waitForCompletion|request.waitForCompletion]).
112 
113 +/
114 unittest {
115 	import arsd.http2;
116 
117 	void main() {
118 		auto client = new HttpClient();
119 		auto request = client.navigateTo(Uri("http://dlang.org/"));
120 		auto response = request.waitForCompletion();
121 
122 		string returnedHtml = response.contentText;
123 	}
124 }
125 
126 private __gshared bool defaultVerifyPeer_ = true;
127 
128 void defaultVerifyPeer(bool v) {
129 	defaultVerifyPeer_ = v;
130 }
131 
132 debug import std.stdio;
133 
134 import std.socket;
135 import core.time;
136 
137 // FIXME: check Transfer-Encoding: gzip always
138 
139 version(with_openssl) {
140 	//pragma(lib, "crypto");
141 	//pragma(lib, "ssl");
142 }
143 
144 /+
145 HttpRequest httpRequest(string method, string url, ubyte[] content, string[string] content) {
146 	return null;
147 }
148 +/
149 
150 /**
151 	auto request = get("http://arsdnet.net/");
152 	request.send();
153 
154 	auto response = get("http://arsdnet.net/").waitForCompletion();
155 */
156 HttpRequest get(string url) {
157 	auto client = new HttpClient();
158 	auto request = client.navigateTo(Uri(url));
159 	return request;
160 }
161 
162 /**
163 	Do not forget to call `waitForCompletion()` on the returned object!
164 */
165 HttpRequest post(string url, string[string] req) {
166 	auto client = new HttpClient();
167 	ubyte[] bdata;
168 	foreach(k, v; req) {
169 		if(bdata.length)
170 			bdata ~= cast(ubyte[]) "&";
171 		bdata ~= cast(ubyte[]) encodeUriComponent(k);
172 		bdata ~= cast(ubyte[]) "=";
173 		bdata ~= cast(ubyte[]) encodeUriComponent(v);
174 	}
175 	auto request = client.request(Uri(url), HttpVerb.POST, bdata, "application/x-www-form-urlencoded");
176 	return request;
177 }
178 
179 /// gets the text off a url. basic operation only.
180 string getText(string url) {
181 	auto request = get(url);
182 	auto response = request.waitForCompletion();
183 	return cast(string) response.content;
184 }
185 
186 /+
187 ubyte[] getBinary(string url, string[string] cookies = null) {
188 	auto hr = httpRequest("GET", url, null, cookies);
189 	if(hr.code != 200)
190 		throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url));
191 	return hr.content;
192 }
193 
194 /**
195 	Gets a textual document, ignoring headers. Throws on non-text or error.
196 */
197 string get(string url, string[string] cookies = null) {
198 	auto hr = httpRequest("GET", url, null, cookies);
199 	if(hr.code != 200)
200 		throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url));
201 	if(hr.contentType.indexOf("text/") == -1)
202 		throw new Exception(hr.contentType ~ " is bad content for conversion to string");
203 	return cast(string) hr.content;
204 
205 }
206 
207 string post(string url, string[string] args, string[string] cookies = null) {
208 	string content;
209 
210 	foreach(name, arg; args) {
211 		if(content.length)
212 			content ~= "&";
213 		content ~= encodeUriComponent(name) ~ "=" ~ encodeUriComponent(arg);
214 	}
215 
216 	auto hr = httpRequest("POST", url, cast(ubyte[]) content, cookies, ["Content-Type: application/x-www-form-urlencoded"]);
217 	if(hr.code != 200)
218 		throw new Exception(format("HTTP answered %d instead of 200", hr.code));
219 	if(hr.contentType.indexOf("text/") == -1)
220 		throw new Exception(hr.contentType ~ " is bad content for conversion to string");
221 
222 	return cast(string) hr.content;
223 }
224 
225 +/
226 
227 ///
228 struct HttpResponse {
229 	/++
230 		The HTTP response code, if the response was completed, or some value < 100 if it was aborted or failed.
231 
232 		Code 0 - initial value, nothing happened
233 		Code 1 - you called request.abort
234 		Code 2 - connection refused
235 		Code 3 - connection succeeded, but server disconnected early
236 		Code 4 - server sent corrupted response (or this code has a bug and processed it wrong)
237 		Code 5 - request timed out
238 
239 		Code >= 100 - a HTTP response
240 	+/
241 	int code;
242 	string codeText; ///
243 
244 	string httpVersion; ///
245 
246 	string statusLine; ///
247 
248 	string contentType; /// The *full* content type header. See also [contentTypeMimeType] and [contentTypeCharset].
249 	string location; /// The location header
250 
251 	/++
252 
253 		History:
254 			Added December 5, 2020 (version 9.1)
255 	+/
256 	bool wasSuccessful() {
257 		return code >= 200 && code < 400;
258 	}
259 
260 	/++
261 		Returns the mime type part of the [contentType] header.
262 
263 		History:
264 			Added July 25, 2022 (version 10.9)
265 	+/
266 	string contentTypeMimeType() {
267 		auto idx = contentType.indexOf(";");
268 		if(idx == -1)
269 			return contentType;
270 
271 		return contentType[0 .. idx].strip;
272 	}
273 
274 	/// the charset out of content type, if present. `null` if not.
275 	string contentTypeCharset() {
276 		auto idx = contentType.indexOf("charset=");
277 		if(idx == -1)
278 			return null;
279 		auto c = contentType[idx + "charset=".length .. $].strip;
280 		if(c.length)
281 			return c;
282 		return null;
283 	}
284 
285 	/++
286 		Names and values of cookies set in the response.
287 
288 		History:
289 			Prior to July 5, 2021 (dub v10.2), this was a public field instead of a property. I did
290 			not consider this a breaking change since the intended use is completely compatible with the
291 			property, and it was not actually implemented properly before anyway.
292 	+/
293 	@property string[string] cookies() const {
294 		string[string] ret;
295 		foreach(cookie; cookiesDetails)
296 			ret[cookie.name] = cookie.value;
297 		return ret;
298 	}
299 	/++
300 		The full parsed-out information of cookies set in the response.
301 
302 		History:
303 			Added July 5, 2021 (dub v10.2).
304 	+/
305 	@property CookieHeader[] cookiesDetails() inout {
306 		CookieHeader[] ret;
307 		foreach(header; headers) {
308 			if(auto content = header.isHttpHeader("set-cookie")) {
309 				// format: name=value, value might be double quoted. it MIGHT be url encoded, but im not going to attempt that since the RFC is silent.
310 				// then there's optionally ; attr=value after that. attributes need not have a value
311 
312 				CookieHeader cookie;
313 
314 				auto remaining = content;
315 
316 				cookie_name:
317 				foreach(idx, ch; remaining) {
318 					if(ch == '=') {
319 						cookie.name = remaining[0 .. idx].idup_if_needed;
320 						remaining = remaining[idx + 1 .. $];
321 						break;
322 					}
323 				}
324 
325 				cookie_value:
326 
327 				{
328 					auto idx = remaining.indexOf(";");
329 					if(idx == -1) {
330 						cookie.value = remaining.idup_if_needed;
331 						remaining = remaining[$..$];
332 					} else {
333 						cookie.value = remaining[0 .. idx].idup_if_needed;
334 						remaining = remaining[idx + 1 .. $].stripLeft;
335 					}
336 
337 					if(cookie.value.length > 2 && cookie.value[0] == '"' && cookie.value[$-1] == '"')
338 						cookie.value = cookie.value[1 .. $ - 1];
339 				}
340 
341 				cookie_attributes:
342 
343 				while(remaining.length) {
344 					string name;
345 					foreach(idx, ch; remaining) {
346 						if(ch == '=') {
347 							name = remaining[0 .. idx].idup_if_needed;
348 							remaining = remaining[idx + 1 .. $];
349 
350 							string value;
351 
352 							foreach(idx2, ch2; remaining) {
353 								if(ch2 == ';') {
354 									value = remaining[0 .. idx2].idup_if_needed;
355 									remaining = remaining[idx2 + 1 .. $].stripLeft;
356 									break;
357 								}
358 							}
359 
360 							if(value is null) {
361 								value = remaining.idup_if_needed;
362 								remaining = remaining[$ .. $];
363 							}
364 
365 							cookie.attributes[name] = value;
366 							continue cookie_attributes;
367 						} else if(ch == ';') {
368 							name = remaining[0 .. idx].idup_if_needed;
369 							remaining = remaining[idx + 1 .. $].stripLeft;
370 							cookie.attributes[name] = "";
371 							continue cookie_attributes;
372 						}
373 					}
374 
375 					if(remaining.length) {
376 						cookie.attributes[remaining.idup_if_needed] = "";
377 						remaining = remaining[$..$];
378 
379 					}
380 				}
381 
382 				ret ~= cookie;
383 			}
384 		}
385 		return ret;
386 	}
387 
388 	string[] headers; /// Array of all headers returned.
389 	string[string] headersHash; ///
390 
391 	ubyte[] content; /// The raw content returned in the response body.
392 	string contentText; /// [content], but casted to string (for convenience)
393 
394 	alias responseText = contentText; // just cuz I do this so often.
395 	//alias body = content;
396 
397 	/++
398 		returns `new Document(this.contentText)`. Requires [arsd.dom].
399 	+/
400 	auto contentDom()() {
401 		import arsd.dom;
402 		return new Document(this.contentText);
403 
404 	}
405 
406 	/++
407 		returns `var.fromJson(this.contentText)`. Requires [arsd.jsvar].
408 	+/
409 	auto contentJson()() {
410 		import arsd.jsvar;
411 		return var.fromJson(this.contentText);
412 	}
413 
414 	HttpRequestParameters requestParameters; ///
415 
416 	LinkHeader[] linksStored;
417 	bool linksLazilyParsed;
418 
419 	HttpResponse deepCopy() const {
420 		HttpResponse h = cast(HttpResponse) this;
421 		h.headers = h.headers.dup;
422 		h.headersHash = h.headersHash.dup;
423 		h.content = h.content.dup;
424 		h.linksStored = h.linksStored.dup;
425 		return h;
426 	}
427 
428 	/// Returns links header sorted by "rel" attribute.
429 	/// It returns a new array on each call.
430 	LinkHeader[string] linksHash() {
431 		auto links = this.links();
432 		LinkHeader[string] ret;
433 		foreach(link; links)
434 			ret[link.rel] = link;
435 		return ret;
436 	}
437 
438 	/// Returns the Link header, parsed.
439 	LinkHeader[] links() {
440 		if(linksLazilyParsed)
441 			return linksStored;
442 		linksLazilyParsed = true;
443 		LinkHeader[] ret;
444 
445 		auto hdrPtr = "link" in headersHash;
446 		if(hdrPtr is null)
447 			return ret;
448 
449 		auto header = *hdrPtr;
450 
451 		LinkHeader current;
452 
453 		while(header.length) {
454 			char ch = header[0];
455 
456 			if(ch == '<') {
457 				// read url
458 				header = header[1 .. $];
459 				size_t idx;
460 				while(idx < header.length && header[idx] != '>')
461 					idx++;
462 				current.url = header[0 .. idx];
463 				header = header[idx .. $];
464 			} else if(ch == ';') {
465 				// read attribute
466 				header = header[1 .. $];
467 				header = header.stripLeft;
468 
469 				size_t idx;
470 				while(idx < header.length && header[idx] != '=')
471 					idx++;
472 
473 				string name = header[0 .. idx];
474 				if(idx + 1 < header.length)
475 					header = header[idx + 1 .. $];
476 				else
477 					header = header[$ .. $];
478 
479 				string value;
480 
481 				if(header.length && header[0] == '"') {
482 					// quoted value
483 					header = header[1 .. $];
484 					idx = 0;
485 					while(idx < header.length && header[idx] != '\"')
486 						idx++;
487 					value = header[0 .. idx];
488 					header = header[idx .. $];
489 
490 				} else if(header.length) {
491 					// unquoted value
492 					idx = 0;
493 					while(idx < header.length && header[idx] != ',' && header[idx] != ' ' && header[idx] != ';')
494 						idx++;
495 
496 					value = header[0 .. idx];
497 					header = header[idx .. $].stripLeft;
498 				}
499 
500 				name = name.toLower;
501 				if(name == "rel")
502 					current.rel = value;
503 				else
504 					current.attributes[name] = value;
505 
506 			} else if(ch == ',') {
507 				// start another
508 				ret ~= current;
509 				current = LinkHeader.init;
510 			} else if(ch == ' ' || ch == '\n' || ch == '\r' || ch == '\t') {
511 				// ignore
512 			}
513 
514 			if(header.length)
515 				header = header[1 .. $];
516 		}
517 
518 		ret ~= current;
519 
520 		linksStored = ret;
521 
522 		return ret;
523 	}
524 }
525 
526 /+
527 	headerName MUST be all lower case and NOT have the colon on it
528 
529 	returns slice of the input thing after the header name
530 +/
531 private inout(char)[] isHttpHeader(inout(char)[] thing, const(char)[] headerName) {
532 	foreach(idx, ch; thing) {
533 		if(idx < headerName.length) {
534 			if(headerName[idx] == '-' && ch != '-')
535 				return null;
536 			if((ch | ' ') != headerName[idx])
537 				return null;
538 		} else if(idx == headerName.length) {
539 			if(ch != ':')
540 				return null;
541 		} else {
542 			return thing[idx .. $].strip;
543 		}
544 	}
545 	return null;
546 }
547 
548 private string idup_if_needed(string s) { return s; }
549 private string idup_if_needed(const(char)[] s) { return s.idup; }
550 
551 unittest {
552 	assert("Cookie: foo=bar".isHttpHeader("cookie") == "foo=bar");
553 	assert("cookie: foo=bar".isHttpHeader("cookie") == "foo=bar");
554 	assert("cOOkie: foo=bar".isHttpHeader("cookie") == "foo=bar");
555 	assert("Set-Cookie: foo=bar".isHttpHeader("set-cookie") == "foo=bar");
556 	assert(!"".isHttpHeader("cookie"));
557 }
558 
559 ///
560 struct LinkHeader {
561 	string url; ///
562 	string rel; ///
563 	string[string] attributes; /// like title, rev, media, whatever attributes
564 }
565 
566 /++
567 	History:
568 		Added July 5, 2021
569 +/
570 struct CookieHeader {
571 	string name;
572 	string value;
573 	string[string] attributes;
574 
575 	// max-age
576 	// expires
577 	// httponly
578 	// secure
579 	// samesite
580 	// path
581 	// domain
582 	// partitioned ?
583 
584 	// also want cookiejar features here with settings to save session cookies or not
585 
586 	// storing in file: http://kb.mozillazine.org/Cookies.txt (second arg in practice true if first arg starts with . it seems)
587 	// or better yet sqlite: http://kb.mozillazine.org/Cookies.sqlite
588 	// should be able to import/export from either upon request
589 }
590 
591 import std.string;
592 static import std.algorithm;
593 import std.conv;
594 import std.range;
595 
596 
597 private AddressFamily family(string unixSocketPath) {
598 	if(unixSocketPath.length)
599 		return AddressFamily.UNIX;
600 	else // FIXME: what about ipv6?
601 		return AddressFamily.INET;
602 }
603 
604 version(Windows)
605 private class UnixAddress : Address {
606 	this(string) {
607 		throw new Exception("No unix address support on this system in lib yet :(");
608 	}
609 	override sockaddr* name() { assert(0); }
610 	override const(sockaddr)* name() const { assert(0); }
611 	override int nameLen() const { assert(0); }
612 }
613 
614 
615 // Copy pasta from cgi.d, then stripped down. unix path thing added tho
616 /++
617 	Represents a URI. It offers named access to the components and relative uri resolution, though as a user of the library, you'd mostly just construct it like `Uri("http://example.com/index.html")`.
618 +/
619 struct Uri {
620 	alias toString this; // blargh idk a url really is a string, but should it be implicit?
621 
622 	// scheme://userinfo@host:port/path?query#fragment
623 
624 	string scheme; /// e.g. "http" in "http://example.com/"
625 	string userinfo; /// the username (and possibly a password) in the uri
626 	string host; /// the domain name
627 	int port; /// port number, if given. Will be zero if a port was not explicitly given
628 	string path; /// e.g. "/folder/file.html" in "http://example.com/folder/file.html"
629 	string query; /// the stuff after the ? in a uri
630 	string fragment; /// the stuff after the # in a uri.
631 
632 	/// Breaks down a uri string to its components
633 	this(string uri) {
634 		size_t lastGoodIndex;
635 		foreach(char ch; uri) {
636 			if(ch > 127) {
637 				break;
638 			}
639 			lastGoodIndex++;
640 		}
641 
642 		string replacement = uri[0 .. lastGoodIndex];
643 		foreach(char ch; uri[lastGoodIndex .. $]) {
644 			if(ch > 127) {
645 				// need to percent-encode any non-ascii in it
646 				char[3] buffer;
647 				buffer[0] = '%';
648 
649 				auto first = ch / 16;
650 				auto second = ch % 16;
651 				first += (first >= 10) ? ('A'-10) : '0';
652 				second += (second >= 10) ? ('A'-10) : '0';
653 
654 				buffer[1] = cast(char) first;
655 				buffer[2] = cast(char) second;
656 
657 				replacement ~= buffer[];
658 			} else {
659 				replacement ~= ch;
660 			}
661 		}
662 
663 		reparse(replacement);
664 	}
665 
666 	/// Returns `port` if set, otherwise if scheme is https 443, otherwise always 80
667 	int effectivePort() const @property nothrow pure @safe @nogc {
668 		return port != 0 ? port
669 			: scheme == "https" ? 443 : 80;
670 	}
671 
672 	private string unixSocketPath = null;
673 	/// Indicates it should be accessed through a unix socket instead of regular tcp. Returns new version without modifying this object.
674 	Uri viaUnixSocket(string path) const {
675 		Uri copy = this;
676 		copy.unixSocketPath = path;
677 		return copy;
678 	}
679 
680 	/// Goes through a unix socket in the abstract namespace (linux only). Returns new version without modifying this object.
681 	version(linux)
682 	Uri viaAbstractSocket(string path) const {
683 		Uri copy = this;
684 		copy.unixSocketPath = "\0" ~ path;
685 		return copy;
686 	}
687 
688 	private void reparse(string uri) {
689 		// from RFC 3986
690 		// the ctRegex triples the compile time and makes ugly errors for no real benefit
691 		// it was a nice experiment but just not worth it.
692 		// enum ctr = ctRegex!r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?";
693 		/*
694 			Captures:
695 				0 = whole url
696 				1 = scheme, with :
697 				2 = scheme, no :
698 				3 = authority, with //
699 				4 = authority, no //
700 				5 = path
701 				6 = query string, with ?
702 				7 = query string, no ?
703 				8 = anchor, with #
704 				9 = anchor, no #
705 		*/
706 		// Yikes, even regular, non-CT regex is also unacceptably slow to compile. 1.9s on my computer!
707 		// instead, I will DIY and cut that down to 0.6s on the same computer.
708 		/*
709 
710 				Note that authority is
711 					user:password@domain:port
712 				where the user:password@ part is optional, and the :port is optional.
713 
714 				Regex translation:
715 
716 				Scheme cannot have :, /, ?, or # in it, and must have one or more chars and end in a :. It is optional, but must be first.
717 				Authority must start with //, but cannot have any other /, ?, or # in it. It is optional.
718 				Path cannot have any ? or # in it. It is optional.
719 				Query must start with ? and must not have # in it. It is optional.
720 				Anchor must start with # and can have anything else in it to end of string. It is optional.
721 		*/
722 
723 		this = Uri.init; // reset all state
724 
725 		// empty uri = nothing special
726 		if(uri.length == 0) {
727 			return;
728 		}
729 
730 		size_t idx;
731 
732 		scheme_loop: foreach(char c; uri[idx .. $]) {
733 			switch(c) {
734 				case ':':
735 				case '/':
736 				case '?':
737 				case '#':
738 					break scheme_loop;
739 				default:
740 			}
741 			idx++;
742 		}
743 
744 		if(idx == 0 && uri[idx] == ':') {
745 			// this is actually a path! we skip way ahead
746 			goto path_loop;
747 		}
748 
749 		if(idx == uri.length) {
750 			// the whole thing is a path, apparently
751 			path = uri;
752 			return;
753 		}
754 
755 		if(idx > 0 && uri[idx] == ':') {
756 			scheme = uri[0 .. idx];
757 			idx++;
758 		} else {
759 			// we need to rewind; it found a / but no :, so the whole thing is prolly a path...
760 			idx = 0;
761 		}
762 
763 		if(idx + 2 < uri.length && uri[idx .. idx + 2] == "//") {
764 			// we have an authority....
765 			idx += 2;
766 
767 			auto authority_start = idx;
768 			authority_loop: foreach(char c; uri[idx .. $]) {
769 				switch(c) {
770 					case '/':
771 					case '?':
772 					case '#':
773 						break authority_loop;
774 					default:
775 				}
776 				idx++;
777 			}
778 
779 			auto authority = uri[authority_start .. idx];
780 
781 			auto idx2 = authority.indexOf("@");
782 			if(idx2 != -1) {
783 				userinfo = authority[0 .. idx2];
784 				authority = authority[idx2 + 1 .. $];
785 			}
786 
787 			if(authority.length && authority[0] == '[') {
788 				// ipv6 address special casing
789 				idx2 = authority.indexOf(']');
790 				if(idx2 != -1) {
791 					auto end = authority[idx2 + 1 .. $];
792 					if(end.length && end[0] == ':')
793 						idx2 = idx2 + 1;
794 					else
795 						idx2 = -1;
796 				}
797 			} else {
798 				idx2 = authority.indexOf(":");
799 			}
800 
801 			if(idx2 == -1) {
802 				port = 0; // 0 means not specified; we should use the default for the scheme
803 				host = authority;
804 			} else {
805 				host = authority[0 .. idx2];
806 				if(idx2 + 1 < authority.length)
807 					port = to!int(authority[idx2 + 1 .. $]);
808 				else
809 					port = 0;
810 			}
811 		}
812 
813 		path_loop:
814 		auto path_start = idx;
815 
816 		foreach(char c; uri[idx .. $]) {
817 			if(c == '?' || c == '#')
818 				break;
819 			idx++;
820 		}
821 
822 		path = uri[path_start .. idx];
823 
824 		if(idx == uri.length)
825 			return; // nothing more to examine...
826 
827 		if(uri[idx] == '?') {
828 			idx++;
829 			auto query_start = idx;
830 			foreach(char c; uri[idx .. $]) {
831 				if(c == '#')
832 					break;
833 				idx++;
834 			}
835 			query = uri[query_start .. idx];
836 		}
837 
838 		if(idx < uri.length && uri[idx] == '#') {
839 			idx++;
840 			fragment = uri[idx .. $];
841 		}
842 
843 		// uriInvalidated = false;
844 	}
845 
846 	private string rebuildUri() const {
847 		string ret;
848 		if(scheme.length)
849 			ret ~= scheme ~ ":";
850 		if(userinfo.length || host.length)
851 			ret ~= "//";
852 		if(userinfo.length)
853 			ret ~= userinfo ~ "@";
854 		if(host.length)
855 			ret ~= host;
856 		if(port)
857 			ret ~= ":" ~ to!string(port);
858 
859 		ret ~= path;
860 
861 		if(query.length)
862 			ret ~= "?" ~ query;
863 
864 		if(fragment.length)
865 			ret ~= "#" ~ fragment;
866 
867 		// uri = ret;
868 		// uriInvalidated = false;
869 		return ret;
870 	}
871 
872 	/// Converts the broken down parts back into a complete string
873 	string toString() const {
874 		// if(uriInvalidated)
875 			return rebuildUri();
876 	}
877 
878 	/// Returns a new absolute Uri given a base. It treats this one as
879 	/// relative where possible, but absolute if not. (If protocol, domain, or
880 	/// other info is not set, the new one inherits it from the base.)
881 	///
882 	/// Browsers use a function like this to figure out links in html.
883 	Uri basedOn(in Uri baseUrl) const {
884 		Uri n = this; // copies
885 		if(n.scheme == "data")
886 			return n;
887 		// n.uriInvalidated = true; // make sure we regenerate...
888 
889 		// userinfo is not inherited... is this wrong?
890 
891 		// if anything is given in the existing url, we don't use the base anymore.
892 		if(n.scheme.empty) {
893 			n.scheme = baseUrl.scheme;
894 			if(n.host.empty) {
895 				n.host = baseUrl.host;
896 				if(n.port == 0) {
897 					n.port = baseUrl.port;
898 					if(n.path.length > 0 && n.path[0] != '/') {
899 						auto b = baseUrl.path[0 .. baseUrl.path.lastIndexOf("/") + 1];
900 						if(b.length == 0)
901 							b = "/";
902 						n.path = b ~ n.path;
903 					} else if(n.path.length == 0) {
904 						n.path = baseUrl.path;
905 					}
906 				}
907 			}
908 		}
909 
910 		n.removeDots();
911 
912 		// if still basically talking to the same thing, we should inherit the unix path
913 		// too since basically the unix path is saying for this service, always use this override.
914 		if(n.host == baseUrl.host && n.scheme == baseUrl.scheme && n.port == baseUrl.port)
915 			n.unixSocketPath = baseUrl.unixSocketPath;
916 
917 		return n;
918 	}
919 
920 	/++
921 		Resolves ../ and ./ parts of the path. Used in the implementation of [basedOn] and you could also use it to normalize things.
922 	+/
923 	void removeDots() {
924 		auto parts = this.path.split("/");
925 		string[] toKeep;
926 		foreach(part; parts) {
927 			if(part == ".") {
928 				continue;
929 			} else if(part == "..") {
930 				//if(toKeep.length > 1)
931 					toKeep = toKeep[0 .. $-1];
932 				//else
933 					//toKeep = [""];
934 				continue;
935 			} else {
936 				//if(toKeep.length && toKeep[$-1].length == 0 && part.length == 0)
937 					//continue; // skip a `//` situation
938 				toKeep ~= part;
939 			}
940 		}
941 
942 		auto path = toKeep.join("/");
943 		if(path.length && path[0] != '/')
944 			path = "/" ~ path;
945 
946 		this.path = path;
947 	}
948 }
949 
950 /*
951 void main(string args[]) {
952 	write(post("http://arsdnet.net/bugs.php", ["test" : "hey", "again" : "what"]));
953 }
954 */
955 
956 ///
957 struct BasicAuth {
958 	string username; ///
959 	string password; ///
960 }
961 
962 class ProxyException : Exception {
963 	this(string msg) {super(msg); }
964 }
965 
966 /**
967 	Represents a HTTP request. You usually create these through a [HttpClient].
968 
969 
970 	---
971 	auto request = new HttpRequest(); // note that when there's no associated client, some features may not work
972 	// normally you'd instead do `new HttpClient(); client.request(...)`
973 	// set any properties here
974 
975 	// synchronous usage
976 	auto reply = request.perform();
977 
978 	// async usage, type 1:
979 	request.send();
980 	request2.send();
981 
982 	// wait until the first one is done, with the second one still in-flight
983 	auto response = request.waitForCompletion();
984 
985 	// async usage, type 2:
986 	request.onDataReceived = (HttpRequest hr) {
987 		if(hr.state == HttpRequest.State.complete) {
988 			// use hr.responseData
989 		}
990 	};
991 	request.send(); // send, using the callback
992 
993 	// before terminating, be sure you wait for your requests to finish!
994 
995 	request.waitForCompletion();
996 	---
997 */
998 class HttpRequest {
999 
1000 	/// Automatically follow a redirection?
1001 	bool followLocation = false;
1002 
1003 	/++
1004 		Maximum number of redirections to follow (used only if [followLocation] is set to true). Will resolve with an error if a single request has more than this number of redirections. The default value is currently 10, but may change without notice. If you need a specific value, be sure to call this function.
1005 
1006 		If you want unlimited redirects, call it with `int.max`. If you set it to 0 but set [followLocation] to `true`, any attempt at redirection will abort the request. To disable automatically following redirection, set [followLocation] to `false` so you can process the 30x code yourself as a completed request.
1007 
1008 		History:
1009 			Added July 27, 2022 (dub v10.9)
1010 	+/
1011 	void setMaximumNumberOfRedirects(int max = 10) {
1012 		maximumNumberOfRedirectsRemaining = max;
1013 	}
1014 
1015 	private int maximumNumberOfRedirectsRemaining;
1016 
1017 	/++
1018 		Set to `true` to automatically retain cookies in the associated [HttpClient] from this request.
1019 		Note that you must have constructed the request from a `HttpClient` or at least passed one into the
1020 		constructor for this to have any effect.
1021 
1022 		Bugs:
1023 			See [HttpClient.retainCookies] for important caveats.
1024 
1025 		History:
1026 			Added July 5, 2021 (dub v10.2)
1027 	+/
1028 	bool retainCookies = false;
1029 
1030 	private HttpClient client;
1031 
1032 	this() {
1033 	}
1034 
1035 	///
1036 	this(HttpClient client, Uri where, HttpVerb method, ICache cache = null, Duration timeout = 10.seconds, string proxy = null) {
1037 		this.client = client;
1038 		populateFromInfo(where, method);
1039 		setTimeout(timeout);
1040 		this.cache = cache;
1041 		this.proxy = proxy;
1042 
1043 		setMaximumNumberOfRedirects();
1044 	}
1045 
1046 
1047 	/// ditto
1048 	this(Uri where, HttpVerb method, ICache cache = null, Duration timeout = 10.seconds, string proxy = null) {
1049 		this(null, where, method, cache, timeout, proxy);
1050 	}
1051 
1052 	/++
1053 		Sets the timeout from inactivity on the request. This is the amount of time that passes with no send or receive activity on the request before it fails with "request timed out" error.
1054 
1055 		History:
1056 			Added March 31, 2021
1057 	+/
1058 	void setTimeout(Duration timeout) {
1059 		this.requestParameters.timeoutFromInactivity = timeout;
1060 		this.timeoutFromInactivity = MonoTime.currTime + this.requestParameters.timeoutFromInactivity;
1061 	}
1062 
1063 	/++
1064 		Set to `true` to gzip the request body when sending to the server. This is often not supported, and thus turned off
1065 		by default.
1066 
1067 
1068 		If a server doesn't support this, you MAY get an http error or it might just do the wrong thing.
1069 		By spec, it is supposed to be code "415 Unsupported Media Type", but there's no guarantee they
1070 		will do that correctly since many servers will simply have never considered this possibility. Request
1071 		compression is quite rare, so before using this, ensure your server supports it by checking its documentation
1072 		or asking its administrator. (Or running a test, but remember, it might just do the wrong thing and not issue
1073 		an appropriate error, or the config may change in the future.)
1074 
1075 		History:
1076 			Added August 6, 2024 (dub v11.5)
1077 	+/
1078 	void gzipBody(bool want) {
1079 		this.requestParameters.gzipBody = want;
1080 	}
1081 
1082 	private MonoTime timeoutFromInactivity;
1083 
1084 	private Uri where;
1085 
1086 	private ICache cache;
1087 
1088 	/++
1089 		Proxy to use for this request. It should be a URL or `null`.
1090 
1091 		This must be sent before you call [send].
1092 
1093 		History:
1094 			Added April 12, 2021 (dub v9.5)
1095 	+/
1096 	string proxy;
1097 
1098 	/++
1099 		For https connections, if this is `true`, it will fail to connect if the TLS certificate can not be
1100 		verified. Setting this to `false` will skip this check and allow the connection to continue anyway.
1101 
1102 		When the [HttpRequest] is constructed from a [HttpClient], it will inherit the value from the client
1103 		instead of using the `= true` here. You can change this value any time before you call [send] (which
1104 		is done implicitly if you call [waitForCompletion]).
1105 
1106 		History:
1107 			Added April 5, 2022 (dub v10.8)
1108 
1109 			Prior to this, it always used the global (but undocumented) `defaultVerifyPeer` setting, and sometimes
1110 			even if it was true, it would skip the verification. Now, it always respects this local setting.
1111 	+/
1112 	bool verifyPeer = true;
1113 
1114 
1115 	/// Final url after any redirections
1116 	string finalUrl;
1117 
1118 	void populateFromInfo(Uri where, HttpVerb method) {
1119 		auto parts = where.basedOn(this.where);
1120 		this.where = parts;
1121 		finalUrl = where.toString();
1122 		requestParameters.method = method;
1123 		requestParameters.unixSocketPath = where.unixSocketPath;
1124 		requestParameters.host = parts.host;
1125 		requestParameters.port = cast(ushort) parts.effectivePort;
1126 		requestParameters.ssl = parts.scheme == "https";
1127 		requestParameters.uri = parts.path.length ? parts.path : "/";
1128 		if(parts.query.length) {
1129 			requestParameters.uri ~= "?";
1130 			requestParameters.uri ~= parts.query;
1131 		}
1132 	}
1133 
1134 	~this() {
1135 	}
1136 
1137 	ubyte[] sendBuffer;
1138 
1139 	HttpResponse responseData;
1140 	private HttpClient parentClient;
1141 
1142 	size_t bodyBytesSent;
1143 	size_t bodyBytesReceived;
1144 
1145 	State state_;
1146 	final State state() { return state_; }
1147 	final State state(State s) {
1148 		assert(state_ != State.complete);
1149 		return state_ = s;
1150 	}
1151 	/// Called when data is received. Check the state to see what data is available.
1152 	void delegate(HttpRequest) onDataReceived;
1153 
1154 	enum State {
1155 		/// The request has not yet been sent
1156 		unsent,
1157 
1158 		/// The send() method has been called, but no data is
1159 		/// sent on the socket yet because the connection is busy.
1160 		pendingAvailableConnection,
1161 
1162 		/// connect has been called, but we're waiting on word of success
1163 		connecting,
1164 
1165 		/// connecting a ssl, needing this
1166 		sslConnectPendingRead,
1167 		/// ditto
1168 		sslConnectPendingWrite,
1169 
1170 		/// The headers are being sent now
1171 		sendingHeaders,
1172 
1173 		// FIXME: allow Expect: 100-continue and separate the body send
1174 
1175 		/// The body is being sent now
1176 		sendingBody,
1177 
1178 		/// The request has been sent but we haven't received any response yet
1179 		waitingForResponse,
1180 
1181 		/// We have received some data and are currently receiving headers
1182 		readingHeaders,
1183 
1184 		/// All headers are available but we're still waiting on the body
1185 		readingBody,
1186 
1187 		/// The request is complete.
1188 		complete,
1189 
1190 		/// The request is aborted, either by the abort() method, or as a result of the server disconnecting
1191 		aborted
1192 	}
1193 
1194 	/// Sends now and waits for the request to finish, returning the response.
1195 	HttpResponse perform() {
1196 		send();
1197 		return waitForCompletion();
1198 	}
1199 
1200 	/// Sends the request asynchronously.
1201 	void send() {
1202 		sendPrivate(true);
1203 	}
1204 
1205 	private void sendPrivate(bool advance) {
1206 		if(state != State.unsent && state != State.aborted)
1207 			return; // already sent
1208 
1209 		if(cache !is null) {
1210 			auto res = cache.getCachedResponse(this.requestParameters);
1211 			if(res !is null) {
1212 				state = State.complete;
1213 				responseData = (*res).deepCopy();
1214 				return;
1215 			}
1216 		}
1217 
1218 		if(this.where.scheme == "data") {
1219 			void error(string content) {
1220 				responseData.code = 400;
1221 				responseData.codeText = "Bad Request";
1222 				responseData.contentType = "text/plain";
1223 				responseData.content = cast(ubyte[]) content;
1224 				responseData.contentText = content;
1225 				state = State.complete;
1226 				return;
1227 			}
1228 
1229 			auto thing = this.where.path;
1230 			// format is: type,data
1231 			// type can have ;base64
1232 			auto comma = thing.indexOf(",");
1233 			if(comma == -1)
1234 				return error("Invalid data uri, no comma found");
1235 
1236 			auto type = thing[0 .. comma];
1237 			auto data = thing[comma + 1 .. $];
1238 			if(type.length == 0)
1239 				type = "text/plain";
1240 
1241 			auto bdata = cast(ubyte[]) decodeUriComponent(data);
1242 
1243 			if(type.indexOf(";base64") != -1) {
1244 				import std.base64;
1245 				try {
1246 					bdata = Base64.decode(bdata);
1247 				} catch(Exception e) {
1248 					return error(e.msg);
1249 				}
1250 			}
1251 
1252 			responseData.code = 200;
1253 			responseData.codeText = "OK";
1254 			responseData.contentType = type;
1255 			responseData.content = bdata;
1256 			responseData.contentText = cast(string) responseData.content;
1257 			state = State.complete;
1258 			return;
1259 		}
1260 
1261 		string headers;
1262 
1263 		headers ~= to!string(requestParameters.method);
1264 		headers ~= " ";
1265 		if(proxy.length && !requestParameters.ssl) {
1266 			// if we're doing a http proxy, we need to send a complete, absolute uri
1267 			// so reconstruct it
1268 			headers ~= "http://";
1269 			headers ~= requestParameters.host;
1270 			if(requestParameters.port != 80) {
1271 				headers ~= ":";
1272 				headers ~= to!string(requestParameters.port);
1273 			}
1274 		}
1275 
1276 		headers ~= requestParameters.uri;
1277 
1278 		if(requestParameters.useHttp11)
1279 			headers ~= " HTTP/1.1\r\n";
1280 		else
1281 			headers ~= " HTTP/1.0\r\n";
1282 
1283 		// the whole authority section is supposed to be there, but curl doesn't send if default port
1284 		// so I'll copy what they do
1285 		headers ~= "Host: ";
1286 		headers ~= requestParameters.host;
1287 		if(requestParameters.port != 80 && requestParameters.port != 443) {
1288 			headers ~= ":";
1289 			headers ~= to!string(requestParameters.port);
1290 		}
1291 		headers ~= "\r\n";
1292 
1293 		bool specSaysRequestAlwaysHasBody =
1294 			requestParameters.method == HttpVerb.POST ||
1295 			requestParameters.method == HttpVerb.PUT ||
1296 			requestParameters.method == HttpVerb.PATCH;
1297 
1298 		if(requestParameters.userAgent.length)
1299 			headers ~= "User-Agent: "~requestParameters.userAgent~"\r\n";
1300 		if(requestParameters.contentType.length)
1301 			headers ~= "Content-Type: "~requestParameters.contentType~"\r\n";
1302 		if(requestParameters.authorization.length)
1303 			headers ~= "Authorization: "~requestParameters.authorization~"\r\n";
1304 		if(requestParameters.bodyData.length || specSaysRequestAlwaysHasBody)
1305 			headers ~= "Content-Length: "~to!string(requestParameters.bodyData.length)~"\r\n";
1306 		if(requestParameters.acceptGzip)
1307 			headers ~= "Accept-Encoding: gzip\r\n";
1308 		if(requestParameters.keepAlive)
1309 			headers ~= "Connection: keep-alive\r\n";
1310 
1311 		string cookieHeader;
1312 		foreach(name, value; requestParameters.cookies) {
1313 			if(cookieHeader is null)
1314 				cookieHeader = "Cookie: ";
1315 			else
1316 				cookieHeader ~= "; ";
1317 			cookieHeader ~= name;
1318 			cookieHeader ~= "=";
1319 			cookieHeader ~= value;
1320 		}
1321 
1322 		if(cookieHeader !is null) {
1323 			cookieHeader ~= "\r\n";
1324 			headers ~= cookieHeader;
1325 		}
1326 
1327 		foreach(header; requestParameters.headers)
1328 			headers ~= header ~ "\r\n";
1329 
1330 		const(ubyte)[] bodyToSend = requestParameters.bodyData;
1331 		if(requestParameters.gzipBody) {
1332 			headers ~= "Content-Encoding: gzip\r\n";
1333 			auto c = new Compress(HeaderFormat.gzip);
1334 
1335 			auto data = c.compress(bodyToSend);
1336 			data ~= c.flush();
1337 			bodyToSend = cast(ubyte[]) data;
1338 		}
1339 
1340 		headers ~= "\r\n";
1341 
1342 		// FIXME: separate this for 100 continue
1343 		sendBuffer = cast(ubyte[]) headers ~ bodyToSend;
1344 
1345 		// import std.stdio; writeln("******* ", cast(string) sendBuffer);
1346 
1347 		responseData = HttpResponse.init;
1348 		responseData.requestParameters = requestParameters;
1349 		bodyBytesSent = 0;
1350 		bodyBytesReceived = 0;
1351 		state = State.pendingAvailableConnection;
1352 
1353 		bool alreadyPending = false;
1354 		foreach(req; pending)
1355 			if(req is this) {
1356 				alreadyPending = true;
1357 				break;
1358 			}
1359 		if(!alreadyPending) {
1360 			pending ~= this;
1361 		}
1362 
1363 		if(advance)
1364 			HttpRequest.advanceConnections(requestParameters.timeoutFromInactivity);
1365 	}
1366 
1367 
1368 	/// Waits for the request to finish or timeout, whichever comes first.
1369 	HttpResponse waitForCompletion() {
1370 		while(state != State.aborted && state != State.complete) {
1371 			if(state == State.unsent) {
1372 				send();
1373 				continue;
1374 			}
1375 			if(auto err = HttpRequest.advanceConnections(requestParameters.timeoutFromInactivity)) {
1376 				switch(err) {
1377 					case 1: throw new Exception("HttpRequest.advanceConnections returned 1: all connections timed out");
1378 					case 2: throw new Exception("HttpRequest.advanceConnections returned 2: nothing to do");
1379 					case 3: continue; // EINTR
1380 					default: throw new Exception("HttpRequest.advanceConnections got err " ~ to!string(err));
1381 				}
1382 			}
1383 		}
1384 
1385 		if(state == State.complete && responseData.code >= 200)
1386 			if(cache !is null)
1387 				cache.cacheResponse(this.requestParameters, this.responseData);
1388 
1389 		return responseData;
1390 	}
1391 
1392 	/// Aborts this request.
1393 	void abort() {
1394 		this.state = State.aborted;
1395 		this.responseData.code = 1;
1396 		this.responseData.codeText = "request.abort called";
1397 		// the actual cancellation happens in the event loop
1398 	}
1399 
1400 	HttpRequestParameters requestParameters; ///
1401 
1402 	version(arsd_http_winhttp_implementation) {
1403 		public static void resetInternals() {
1404 
1405 		}
1406 
1407 		static assert(0, "implementation not finished");
1408 	}
1409 
1410 
1411 	version(arsd_http_internal_implementation) {
1412 
1413 	/++
1414 		Changes the limit of number of open, inactive sockets. Reusing connections can provide a significant
1415 		performance improvement, but the operating system can also impose a global limit on the number of open
1416 		sockets and/or files that you don't want to run into. This lets you choose a balance right for you.
1417 
1418 
1419 		When the total number of cached, inactive sockets approaches this maximum, it will check for ones closed by the
1420 		server first. If there are none already closed by the server, it will select sockets at random from its connection
1421 		cache and close them to make room for the new ones.
1422 
1423 		Please note:
1424 
1425 		$(LIST
1426 			* there is always a limit of six open sockets per domain, per the common practice suggested by the http standard
1427 			* the limit given here is thread-local. If you run multiple http clients/requests from multiple threads, don't set this too high or you might bump into the global limit from the OS.
1428 			* setting this too low can waste connections because the server might close them, but they will never be garbage collected since my current implementation won't check for dead connections except when it thinks it is running close to the limit.
1429 		)
1430 
1431 		Setting it just right for your use case may provide an up to 10x performance boost.
1432 
1433 		This implementation is subject to change. If it does, I'll document it, but may not bump the version number.
1434 
1435 		History:
1436 			Added August 10, 2022 (dub v10.9)
1437 	+/
1438 	static void setConnectionCacheSize(int max = 32) {
1439 		connectionCacheSize = max;
1440 	}
1441 
1442 	private static {
1443 		// we manage the actual connections. When a request is made on a particular
1444 		// host, we try to reuse connections. We may open more than one connection per
1445 		// host to do parallel requests.
1446 		//
1447 		// The key is the *domain name* and the port. Multiple domains on the same address will have separate connections.
1448 		Socket[][string] socketsPerHost;
1449 
1450 		// only one request can be active on a given socket (at least HTTP < 2.0) so this is that
1451 		HttpRequest[Socket] activeRequestOnSocket;
1452 		HttpRequest[] pending; // and these are the requests that are waiting
1453 
1454 		int cachedSockets;
1455 		int connectionCacheSize = 32;
1456 
1457 		/+
1458 			This is a somewhat expensive, but essential operation. If it isn't used in a heavy
1459 			application, you'll risk running out of file descriptors.
1460 		+/
1461 		void cleanOldSockets() {
1462 			static struct CloseCandidate {
1463 				string key;
1464 				Socket socket;
1465 			}
1466 
1467 			CloseCandidate[36] closeCandidates;
1468 			int closeCandidatesPosition;
1469 
1470 			outer: foreach(key, sockets; socketsPerHost) {
1471 				foreach(socket; sockets) {
1472 					if(socket in activeRequestOnSocket)
1473 						continue; // it is still in use; we can't close it
1474 
1475 					closeCandidates[closeCandidatesPosition++] = CloseCandidate(key, socket);
1476 					if(closeCandidatesPosition == closeCandidates.length)
1477 						break outer;
1478 				}
1479 			}
1480 
1481 			auto cc = closeCandidates[0 .. closeCandidatesPosition];
1482 
1483 			if(cc.length == 0)
1484 				return; // no candidates to even examine
1485 
1486 			// has the server closed any of these? if so, we also close and drop them
1487 			static SocketSet readSet = null;
1488 			if(readSet is null)
1489 				readSet = new SocketSet();
1490 			readSet.reset();
1491 
1492 			foreach(candidate; cc) {
1493 				readSet.add(candidate.socket);
1494 			}
1495 
1496 			int closeCount;
1497 
1498 			auto got = Socket.select(readSet, null, null, 0.msecs /* timeout, want it small since we just checking for eof */);
1499 			if(got > 0) {
1500 				foreach(ref candidate; cc) {
1501 					if(readSet.isSet(candidate.socket)) {
1502 						// if we can read when it isn't in use, that means eof; the
1503 						// server closed it.
1504 						candidate.socket.close();
1505 						loseSocketByKey(candidate.key, candidate.socket);
1506 						closeCount++;
1507 					}
1508 				}
1509 				debug(arsd_http2) writeln(closeCount, " from inactivity");
1510 			} else {
1511 				// and if not, of the remaining ones, close a few just at random to bring us back beneath the arbitrary limit.
1512 
1513 				while(cc.length > 0 && (cachedSockets - closeCount) > connectionCacheSize) {
1514 					import std.random;
1515 					auto idx = uniform(0, cc.length);
1516 
1517 					cc[idx].socket.close();
1518 					loseSocketByKey(cc[idx].key, cc[idx].socket);
1519 
1520 					cc[idx] = cc[$ - 1];
1521 					cc = cc[0 .. $-1];
1522 					closeCount++;
1523 				}
1524 				debug(arsd_http2) writeln(closeCount, " from randomness");
1525 			}
1526 
1527 			cachedSockets -= closeCount;
1528 		}
1529 
1530 		void loseSocketByKey(string key, Socket s) {
1531 			if(auto list = key in socketsPerHost) {
1532 				for(int a = 0; a < (*list).length; a++) {
1533 					if((*list)[a] is s) {
1534 
1535 						for(int b = a; b < (*list).length - 1; b++)
1536 							(*list)[b] = (*list)[b+1];
1537 						(*list) = (*list)[0 .. $-1];
1538 						break;
1539 					}
1540 				}
1541 			}
1542 		}
1543 
1544 		void loseSocket(string host, ushort port, bool ssl, Socket s) {
1545 			import std.string;
1546 			auto key = format("http%s://%s:%s", ssl ? "s" : "", host, port);
1547 
1548 			loseSocketByKey(key, s);
1549 		}
1550 
1551 		Socket getOpenSocketOnHost(string proxy, string host, ushort port, bool ssl, string unixSocketPath, bool verifyPeer) {
1552 			Socket openNewConnection() {
1553 				Socket socket;
1554 				if(ssl) {
1555 					version(with_openssl) {
1556 						loadOpenSsl();
1557 						socket = new SslClientSocket(family(unixSocketPath), SocketType.STREAM, host, verifyPeer);
1558 						socket.blocking = false;
1559 					} else
1560 						throw new Exception("SSL not compiled in");
1561 				} else {
1562 					socket = new Socket(family(unixSocketPath), SocketType.STREAM);
1563 					socket.blocking = false;
1564 				}
1565 
1566 				socket.setOption(SocketOptionLevel.TCP, SocketOption.TCP_NODELAY, 1);
1567 
1568 				// FIXME: connect timeout?
1569 				if(unixSocketPath) {
1570 					import std.stdio; writeln(cast(ubyte[]) unixSocketPath);
1571 					socket.connect(new UnixAddress(unixSocketPath));
1572 				} else {
1573 					// FIXME: i should prolly do ipv6 if available too.
1574 					if(host.length == 0) // this could arguably also be an in contract since it is user error, but the exception is good enough
1575 						throw new Exception("No host given for request");
1576 					if(proxy.length) {
1577 						if(proxy.indexOf("//") == -1)
1578 							proxy = "http://" ~ proxy;
1579 						auto proxyurl = Uri(proxy);
1580 
1581 						//auto proxyhttps = proxyurl.scheme == "https";
1582 						enum proxyhttps = false; // this isn't properly implemented and might never be necessary anyway so meh
1583 
1584 						// the precise types here are important to help with overload
1585 						// resolution of the devirtualized call!
1586 						Address pa = new InternetAddress(proxyurl.host, proxyurl.port ? cast(ushort) proxyurl.port : 80);
1587 
1588 						debug(arsd_http2) writeln("using proxy ", pa.toString());
1589 
1590 						if(proxyhttps) {
1591 							socket.connect(pa);
1592 						} else {
1593 							// the proxy never actually starts TLS, but if the request is tls then we need to CONNECT then upgrade the connection
1594 							// using the parent class functions let us bypass the encryption
1595 							socket.Socket.connect(pa);
1596 						}
1597 
1598 						socket.blocking = true; // FIXME total hack to simplify the code here since it isn't really using the event loop yet
1599 
1600 						string message;
1601 						if(ssl) {
1602 							auto hostName =  host ~ ":" ~ to!string(port);
1603 							message = "CONNECT " ~ hostName ~ " HTTP/1.1\r\n";
1604 							message ~= "Host: " ~ hostName ~ "\r\n";
1605 							if(proxyurl.userinfo.length) {
1606 								import std.base64;
1607 								message ~= "Proxy-Authorization: Basic " ~ Base64.encode(cast(ubyte[]) proxyurl.userinfo) ~ "\r\n";
1608 							}
1609 							message ~= "\r\n";
1610 
1611 							// FIXME: what if proxy times out? should be reasonably fast too.
1612 							if(proxyhttps) {
1613 								socket.send(message, SocketFlags.NONE);
1614 							} else {
1615 								socket.Socket.send(message, SocketFlags.NONE);
1616 							}
1617 
1618 							ubyte[1024] recvBuffer;
1619 							// and last time
1620 							ptrdiff_t rcvGot;
1621 							if(proxyhttps) {
1622 								rcvGot = socket.receive(recvBuffer[], SocketFlags.NONE);
1623 								// bool verifyPeer = true;
1624 								//(cast(OpenSslSocket)socket).freeSsl();
1625 								//(cast(OpenSslSocket)socket).initSsl(verifyPeer, host);
1626 							} else {
1627 								rcvGot = socket.Socket.receive(recvBuffer[], SocketFlags.NONE);
1628 							}
1629 
1630 							if(rcvGot == -1)
1631 								throw new ProxyException("proxy receive error");
1632 							auto got = cast(string) recvBuffer[0 .. rcvGot];
1633 							auto expect = "HTTP/1.1 200";
1634 							if(got.length < expect.length || (got[0 .. expect.length] != expect && got[0 .. expect.length] != "HTTP/1.0 200"))
1635 								throw new ProxyException("Proxy rejected request: " ~ got[0 .. expect.length <= got.length ? expect.length : got.length]);
1636 
1637 							if(proxyhttps) {
1638 								//(cast(OpenSslSocket)socket).do_ssl_connect();
1639 							} else {
1640 								(cast(OpenSslSocket)socket).do_ssl_connect();
1641 							}
1642 						} else {
1643 						}
1644 					} else {
1645 						socket.connect(new InternetAddress(host, port));
1646 					}
1647 				}
1648 
1649 				debug(arsd_http2) writeln("opening to ", host, ":", port, " ", cast(void*) socket, " ssl=", ssl);
1650 				assert(socket.handle() !is socket_t.init);
1651 				return socket;
1652 			}
1653 
1654 			// import std.stdio; writeln(cachedSockets);
1655 			if(cachedSockets > connectionCacheSize)
1656 				cleanOldSockets();
1657 
1658 			import std.string;
1659 			auto key = format("http%s://%s:%s", ssl ? "s" : "", host, port);
1660 
1661 			if(auto hostListing = key in socketsPerHost) {
1662 				// try to find an available socket that is already open
1663 				foreach(socket; *hostListing) {
1664 					if(socket !in activeRequestOnSocket) {
1665 						// let's see if it has closed since we last tried
1666 						// e.g. a server timeout or something. If so, we need
1667 						// to lose this one and immediately open a new one.
1668 						static SocketSet readSet = null;
1669 						if(readSet is null)
1670 							readSet = new SocketSet();
1671 						readSet.reset();
1672 						assert(socket !is null);
1673 						assert(socket.handle() !is socket_t.init, socket is null ? "null" : socket.toString());
1674 						readSet.add(socket);
1675 						auto got = Socket.select(readSet, null, null, 0.msecs /* timeout, want it small since we just checking for eof */);
1676 						if(got > 0) {
1677 							// we can read something off this... but there aren't
1678 							// any active requests. Assume it is EOF and open a new one
1679 
1680 							socket.close();
1681 							loseSocket(host, port, ssl, socket);
1682 							goto openNew;
1683 						}
1684 						cachedSockets--;
1685 						return socket;
1686 					}
1687 				}
1688 
1689 				// if not too many already open, go ahead and do a new one
1690 				if((*hostListing).length < 6) {
1691 					auto socket = openNewConnection();
1692 					(*hostListing) ~= socket;
1693 					return socket;
1694 				} else
1695 					return null; // too many, you'll have to wait
1696 			}
1697 
1698 			openNew:
1699 
1700 			auto socket = openNewConnection();
1701 			socketsPerHost[key] ~= socket;
1702 			return socket;
1703 		}
1704 
1705 		// stuff used by advanceConnections
1706 		SocketSet readSet;
1707 		SocketSet writeSet;
1708 		private ubyte[] reusableBuffer;
1709 
1710 		/+
1711 			Generic event loop registration:
1712 
1713 				handle, operation (read/write), buffer (on posix it *might* be stack if a select loop), timeout (in real time), callback when op completed.
1714 
1715 				....basically Windows style. Then it translates internally.
1716 
1717 				It should tell the thing if the buffer is reused or not
1718 		+/
1719 
1720 
1721 		/++
1722 			This is made public for rudimentary event loop integration, but is still
1723 			basically an internal detail. Try not to use it if you have another way.
1724 
1725 			This does a single iteration of the internal select()-based processing loop.
1726 
1727 
1728 			Future directions:
1729 				I want to merge the internal use of [WebSocket.eventLoop] with this;
1730 				[advanceConnections] does just one run on the loop, whereas eventLoop
1731 				runs it until all connections are closed. But they'd both process both
1732 				pending http requests and active websockets.
1733 
1734 				After that, I want to be able to integrate in other event loops too.
1735 				One might be to simply to reactor callbacks, then perhaps Windows overlapped
1736 				i/o (that's just going to be tricky to retrofit into the existing select()-based
1737 				code). It could then go fiber just by calling the resume function too.
1738 
1739 				The hard part is ensuring I keep this file stand-alone while offering these
1740 				things.
1741 
1742 				This `advanceConnections` call will probably continue to work now that it is
1743 				public, but it may not be wholly compatible with all the future features; you'd
1744 				have to pick either the internal event loop or an external one you integrate, but not
1745 				mix them.
1746 
1747 			History:
1748 				This has been included in the library since almost day one, but
1749 				it was private until April 13, 2021 (dub v9.5).
1750 
1751 			Params:
1752 				maximumTimeout = the maximum time it will wait in select(). It may return much sooner than this if a connection timed out in the mean time.
1753 				automaticallyRetryOnInterruption = internally loop on EINTR.
1754 
1755 			Returns:
1756 
1757 				0 = no error, work may remain so you should call `advanceConnections` again when you can
1758 
1759 				1 = passed `maximumTimeout` reached with no work done, yet requests are still in the queue. You may call `advanceConnections` again.
1760 
1761 				2 = no work to do, no point calling it again unless you've added new requests. Your program may exit if you have nothing to add since it means everything requested is now done.
1762 
1763 				3 = EINTR occurred on select(), you should check your interrupt flags if you set a signal handler, then call `advanceConnections` again if you aren't exiting. Only occurs if `automaticallyRetryOnInterruption` is set to `false` (the default when it is called externally).
1764 
1765 				any other value should be considered a non-recoverable error if you want to be forward compatible as I reserve the right to add more values later.
1766 		+/
1767 		public int advanceConnections(Duration maximumTimeout = 10.seconds, bool automaticallyRetryOnInterruption = false) {
1768 			debug(arsd_http2_verbose) writeln("advancing");
1769 			if(readSet is null)
1770 				readSet = new SocketSet();
1771 			if(writeSet is null)
1772 				writeSet = new SocketSet();
1773 
1774 			if(reusableBuffer is null)
1775 				reusableBuffer = new ubyte[](32 * 1024);
1776 			ubyte[] buffer = reusableBuffer;
1777 
1778 			HttpRequest[16] removeFromPending;
1779 			size_t removeFromPendingCount = 0;
1780 
1781 			bool hadAbortedRequest;
1782 
1783 			// are there pending requests? let's try to send them
1784 			foreach(idx, pc; pending) {
1785 				if(removeFromPendingCount == removeFromPending.length)
1786 					break;
1787 
1788 				if(pc.state == HttpRequest.State.aborted) {
1789 					removeFromPending[removeFromPendingCount++] = pc;
1790 					hadAbortedRequest = true;
1791 					continue;
1792 				}
1793 
1794 				Socket socket;
1795 
1796 				try {
1797 					socket = getOpenSocketOnHost(pc.proxy, pc.requestParameters.host, pc.requestParameters.port, pc.requestParameters.ssl, pc.requestParameters.unixSocketPath, pc.verifyPeer);
1798 				} catch(ProxyException e) {
1799 					// connection refused or timed out (I should disambiguate somehow)...
1800 					pc.state = HttpRequest.State.aborted;
1801 
1802 					pc.responseData.code = 2;
1803 					pc.responseData.codeText = e.msg ~ " from " ~ pc.proxy;
1804 
1805 					hadAbortedRequest = true;
1806 
1807 					removeFromPending[removeFromPendingCount++] = pc;
1808 					continue;
1809 
1810 				} catch(SocketException e) {
1811 					// connection refused or timed out (I should disambiguate somehow)...
1812 					pc.state = HttpRequest.State.aborted;
1813 
1814 					pc.responseData.code = 2;
1815 					pc.responseData.codeText = pc.proxy.length ? ("connection failed to proxy " ~ pc.proxy) : "connection failed";
1816 
1817 					hadAbortedRequest = true;
1818 
1819 					removeFromPending[removeFromPendingCount++] = pc;
1820 					continue;
1821 				} catch(Exception e) {
1822 					// connection failed due to other user error or SSL (i should disambiguate somehow)...
1823 					pc.state = HttpRequest.State.aborted;
1824 
1825 					pc.responseData.code = 2;
1826 					pc.responseData.codeText = e.msg;
1827 
1828 					hadAbortedRequest = true;
1829 
1830 					removeFromPending[removeFromPendingCount++] = pc;
1831 					continue;
1832 
1833 				}
1834 
1835 				if(socket !is null) {
1836 					activeRequestOnSocket[socket] = pc;
1837 					assert(pc.sendBuffer.length);
1838 					pc.state = State.connecting;
1839 
1840 					removeFromPending[removeFromPendingCount++] = pc;
1841 				}
1842 			}
1843 
1844 			import std.algorithm : remove;
1845 			foreach(rp; removeFromPending[0 .. removeFromPendingCount])
1846 				pending = pending.remove!((a) => a is rp)();
1847 
1848 			tryAgain:
1849 
1850 			Socket[16] inactive;
1851 			int inactiveCount = 0;
1852 			void killInactives() {
1853 				foreach(s; inactive[0 .. inactiveCount]) {
1854 					debug(arsd_http2) writeln("removing socket from active list ", cast(void*) s);
1855 					activeRequestOnSocket.remove(s);
1856 					cachedSockets++;
1857 				}
1858 			}
1859 
1860 
1861 			readSet.reset();
1862 			writeSet.reset();
1863 
1864 			bool hadOne = false;
1865 
1866 			auto minTimeout = maximumTimeout;
1867 			auto now = MonoTime.currTime;
1868 
1869 			// active requests need to be read or written to
1870 			foreach(sock, request; activeRequestOnSocket) {
1871 
1872 				if(request.state == State.aborted) {
1873 					inactive[inactiveCount++] = sock;
1874 					sock.close();
1875 					loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock);
1876 					hadAbortedRequest = true;
1877 					continue;
1878 				}
1879 
1880 				// check the other sockets just for EOF, if they close, take them out of our list,
1881 				// we'll reopen if needed upon request.
1882 				readSet.add(sock);
1883 				hadOne = true;
1884 
1885 				Duration timeo;
1886 				if(request.timeoutFromInactivity <= now)
1887 					timeo = 0.seconds;
1888 				else
1889 					timeo = request.timeoutFromInactivity - now;
1890 
1891 				if(timeo < minTimeout)
1892 					minTimeout = timeo;
1893 
1894 				if(request.state == State.connecting || request.state == State.sslConnectPendingWrite || request.state == State.sendingHeaders || request.state == State.sendingBody) {
1895 					writeSet.add(sock);
1896 					hadOne = true;
1897 				}
1898 			}
1899 
1900 			if(!hadOne) {
1901 				if(hadAbortedRequest) {
1902 					killInactives();
1903 					return 0; // something got aborted, that's progress
1904 				}
1905 				return 2; // automatic timeout, nothing to do
1906 			}
1907 
1908 			auto selectGot = Socket.select(readSet, writeSet, null, minTimeout);
1909 			if(selectGot == 0) { /* timeout */
1910 				now = MonoTime.currTime;
1911 				bool anyWorkDone = false;
1912 				foreach(sock, request; activeRequestOnSocket) {
1913 
1914 					if(request.timeoutFromInactivity <= now) {
1915 						request.state = HttpRequest.State.aborted;
1916 						request.responseData.code = 5;
1917 						if(request.state == State.connecting)
1918 							request.responseData.codeText = "Connect timed out";
1919 						else
1920 							request.responseData.codeText = "Request timed out";
1921 
1922 						inactive[inactiveCount++] = sock;
1923 						sock.close();
1924 						loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock);
1925 						anyWorkDone = true;
1926 					}
1927 				}
1928 				killInactives();
1929 				return anyWorkDone ? 0 : 1;
1930 				// return 1; was an error to time out but now im making it on the individual request
1931 			} else if(selectGot == -1) { /* interrupted */
1932 				/*
1933 				version(Posix) {
1934 					import core.stdc.errno;
1935 					if(errno != EINTR)
1936 						throw new Exception("select error: " ~ to!string(errno));
1937 				}
1938 				*/
1939 				if(automaticallyRetryOnInterruption)
1940 					goto tryAgain;
1941 				else
1942 					return 3;
1943 			} else { /* ready */
1944 
1945 				void sslProceed(HttpRequest request, SslClientSocket s) {
1946 					try {
1947 						auto code = s.do_ssl_connect();
1948 						switch(code) {
1949 							case 0:
1950 								request.state = State.sendingHeaders;
1951 							break;
1952 							case SSL_ERROR_WANT_READ:
1953 								request.state = State.sslConnectPendingRead;
1954 							break;
1955 							case SSL_ERROR_WANT_WRITE:
1956 								request.state = State.sslConnectPendingWrite;
1957 							break;
1958 							default:
1959 								assert(0);
1960 						}
1961 					} catch(Exception e) {
1962 						request.state = State.aborted;
1963 
1964 						request.responseData.code = 2;
1965 						request.responseData.codeText = e.msg;
1966 						inactive[inactiveCount++] = s;
1967 						s.close();
1968 						loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, s);
1969 					}
1970 				}
1971 
1972 
1973 				foreach(sock, request; activeRequestOnSocket) {
1974 					// always need to try to send first in part because http works that way but
1975 					// also because openssl will sometimes leave something ready to read even if we haven't
1976 					// sent yet (probably leftover data from the crypto negotiation) and if that happens ssl
1977 					// is liable to block forever hogging the connection and not letting it send...
1978 					if(request.state == State.connecting)
1979 					if(writeSet.isSet(sock) || readSet.isSet(sock)) {
1980 						import core.stdc.stdint;
1981 						int32_t error;
1982 						int retopt = sock.getOption(SocketOptionLevel.SOCKET, SocketOption.ERROR, error);
1983 						if(retopt < 0 || error != 0) {
1984 							request.state = State.aborted;
1985 
1986 							request.responseData.code = 2;
1987 							try {
1988 								request.responseData.codeText = "connection failed - " ~ formatSocketError(error);
1989 							} catch(Exception e) {
1990 								request.responseData.codeText = "connection failed";
1991 							}
1992 							inactive[inactiveCount++] = sock;
1993 							sock.close();
1994 							loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock);
1995 							continue;
1996 						} else {
1997 							if(auto s = cast(SslClientSocket) sock) {
1998 								sslProceed(request, s);
1999 								continue;
2000 							} else {
2001 								request.state = State.sendingHeaders;
2002 							}
2003 						}
2004 					}
2005 
2006 					if(request.state == State.sslConnectPendingRead)
2007 					if(readSet.isSet(sock)) {
2008 						sslProceed(request, cast(SslClientSocket) sock);
2009 						continue;
2010 					}
2011 					if(request.state == State.sslConnectPendingWrite)
2012 					if(writeSet.isSet(sock)) {
2013 						sslProceed(request, cast(SslClientSocket) sock);
2014 						continue;
2015 					}
2016 
2017 					if(request.state == State.sendingHeaders || request.state == State.sendingBody)
2018 					if(writeSet.isSet(sock)) {
2019 						request.timeoutFromInactivity = MonoTime.currTime + request.requestParameters.timeoutFromInactivity;
2020 						assert(request.sendBuffer.length);
2021 						auto sent = sock.send(request.sendBuffer);
2022 						debug(arsd_http2_verbose) writeln(cast(void*) sock, "<send>", cast(string) request.sendBuffer, "</send>");
2023 						if(sent <= 0) {
2024 							if(wouldHaveBlocked())
2025 								continue;
2026 
2027 							request.state = State.aborted;
2028 
2029 							request.responseData.code = 3;
2030 							request.responseData.codeText = "send failed to server: " ~ lastSocketError(sock);
2031 							inactive[inactiveCount++] = sock;
2032 							sock.close();
2033 							loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock);
2034 							continue;
2035 
2036 						}
2037 						request.sendBuffer = request.sendBuffer[sent .. $];
2038 						if(request.sendBuffer.length == 0) {
2039 							request.state = State.waitingForResponse;
2040 
2041 							debug(arsd_http2_verbose) writeln("all sent");
2042 						}
2043 					}
2044 
2045 
2046 					if(readSet.isSet(sock)) {
2047 						keep_going:
2048 						request.timeoutFromInactivity = MonoTime.currTime + request.requestParameters.timeoutFromInactivity;
2049 						auto got = sock.receive(buffer);
2050 						debug(arsd_http2_verbose) { if(got < 0) writeln(lastSocketError); else writeln("====PACKET ",got,"=====",cast(string)buffer[0 .. got],"===/PACKET==="); }
2051 						if(got < 0) {
2052 							if(wouldHaveBlocked())
2053 								continue;
2054 							debug(arsd_http2) writeln("receive error");
2055 							if(request.state != State.complete) {
2056 								request.state = State.aborted;
2057 
2058 								request.responseData.code = 3;
2059 								request.responseData.codeText = "receive error from server: " ~ lastSocketError(sock);
2060 							}
2061 							inactive[inactiveCount++] = sock;
2062 							sock.close();
2063 							loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock);
2064 						} else if(got == 0) {
2065 							// remote side disconnected
2066 							debug(arsd_http2) writeln("remote disconnect");
2067 							if(request.state != State.complete) {
2068 								request.state = State.aborted;
2069 
2070 								request.responseData.code = 3;
2071 								request.responseData.codeText = "server disconnected";
2072 							}
2073 							inactive[inactiveCount++] = sock;
2074 							sock.close();
2075 							loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock);
2076 						} else {
2077 							// data available
2078 							bool stillAlive;
2079 
2080 							try {
2081 								stillAlive = request.handleIncomingData(buffer[0 .. got]);
2082 								/+
2083 									state needs to be set and public
2084 									requestData.content/contentText needs to be around
2085 									you need to be able to clear the content and keep processing for things like event sources.
2086 									also need to be able to abort it.
2087 
2088 									and btw it should prolly just have evnet source as a pre-packaged thing.
2089 								+/
2090 							} catch (Exception e) {
2091 								debug(arsd_http2_verbose) { import std.stdio; writeln(e); }
2092 								request.state = HttpRequest.State.aborted;
2093 								request.responseData.code = 4;
2094 								request.responseData.codeText = e.msg;
2095 
2096 								inactive[inactiveCount++] = sock;
2097 								sock.close();
2098 								loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock);
2099 							}
2100 
2101 							if(!stillAlive || request.state == HttpRequest.State.complete || request.state == HttpRequest.State.aborted) {
2102 								//import std.stdio; writeln(cast(void*) sock, " ", stillAlive, " ", request.state);
2103 								inactive[inactiveCount++] = sock;
2104 							// reuse the socket for another pending request, if we can
2105 							}
2106 						}
2107 
2108 						if(request.onDataReceived)
2109 							request.onDataReceived(request);
2110 
2111 						version(with_openssl)
2112 						if(auto s = cast(SslClientSocket) sock) {
2113 							// select doesn't handle the case with stuff
2114 							// left in the ssl buffer so i'm checking it separately
2115 							if(s.dataPending()) {
2116 								goto keep_going;
2117 							}
2118 						}
2119 					}
2120 				}
2121 			}
2122 
2123 			killInactives();
2124 
2125 			// we've completed a request, are there any more pending connection? if so, send them now
2126 
2127 			return 0;
2128 		}
2129 	}
2130 
2131 	public static void resetInternals() {
2132 		socketsPerHost = null;
2133 		activeRequestOnSocket = null;
2134 		pending = null;
2135 
2136 	}
2137 
2138 	struct HeaderReadingState {
2139 		bool justSawLf;
2140 		bool justSawCr;
2141 		bool atStartOfLine = true;
2142 		bool readingLineContinuation;
2143 	}
2144 	HeaderReadingState headerReadingState;
2145 
2146 	struct BodyReadingState {
2147 		bool isGzipped;
2148 		bool isDeflated;
2149 
2150 		bool isChunked;
2151 		int chunkedState;
2152 
2153 		// used for the chunk size if it is chunked
2154 		int contentLengthRemaining;
2155 	}
2156 	BodyReadingState bodyReadingState;
2157 
2158 	bool closeSocketWhenComplete;
2159 
2160 	import std.zlib;
2161 	UnCompress uncompress;
2162 
2163 	const(ubyte)[] leftoverDataFromLastTime;
2164 
2165 	bool handleIncomingData(scope const ubyte[] dataIn) {
2166 		bool stillAlive = true;
2167 		debug(arsd_http2) writeln("handleIncomingData, state: ", state);
2168 		if(state == State.waitingForResponse) {
2169 			state = State.readingHeaders;
2170 			headerReadingState = HeaderReadingState.init;
2171 			bodyReadingState = BodyReadingState.init;
2172 		}
2173 
2174 		const(ubyte)[] data;
2175 		if(leftoverDataFromLastTime.length)
2176 			data = leftoverDataFromLastTime ~ dataIn[];
2177 		else
2178 			data = dataIn[];
2179 
2180 		if(state == State.readingHeaders) {
2181 			void parseLastHeader() {
2182 				assert(responseData.headers.length);
2183 				if(responseData.headers.length == 1) {
2184 					responseData.statusLine = responseData.headers[0];
2185 					import std.algorithm;
2186 					auto parts = responseData.statusLine.splitter(" ");
2187 					responseData.httpVersion = parts.front;
2188 					parts.popFront();
2189 					if(parts.empty)
2190 						throw new Exception("Corrupted response, bad status line");
2191 					responseData.code = to!int(parts.front());
2192 					parts.popFront();
2193 					responseData.codeText = "";
2194 					while(!parts.empty) {
2195 						// FIXME: this sucks!
2196 						responseData.codeText ~= parts.front();
2197 						parts.popFront();
2198 						if(!parts.empty)
2199 							responseData.codeText ~= " ";
2200 					}
2201 				} else {
2202 					// parse the new header
2203 					auto header = responseData.headers[$-1];
2204 
2205 					auto colon = header.indexOf(":");
2206 					if(colon < 0 || colon >= header.length)
2207 						return;
2208 					auto name = toLower(header[0 .. colon]);
2209 					auto value = header[colon + 1 .. $].strip; // skip colon and strip whitespace
2210 
2211 					switch(name) {
2212 						case "connection":
2213 							if(value == "close")
2214 								closeSocketWhenComplete = true;
2215 						break;
2216 						case "content-type":
2217 							responseData.contentType = value;
2218 						break;
2219 						case "location":
2220 							responseData.location = value;
2221 						break;
2222 						case "content-length":
2223 							bodyReadingState.contentLengthRemaining = to!int(value);
2224 							// preallocate the buffer for a bit of a performance boost
2225 							responseData.content.reserve(bodyReadingState.contentLengthRemaining);
2226 						break;
2227 						case "transfer-encoding":
2228 							// note that if it is gzipped, it zips first, then chunks the compressed stream.
2229 							// so we should always dechunk first, then feed into the decompressor
2230 							if(value == "chunked")
2231 								bodyReadingState.isChunked = true;
2232 							else throw new Exception("Unknown Transfer-Encoding: " ~ value);
2233 						break;
2234 						case "content-encoding":
2235 							if(value == "gzip") {
2236 								bodyReadingState.isGzipped = true;
2237 								uncompress = new UnCompress();
2238 							} else if(value == "deflate") {
2239 								bodyReadingState.isDeflated = true;
2240 								uncompress = new UnCompress();
2241 							} else throw new Exception("Unknown Content-Encoding: " ~ value);
2242 						break;
2243 						case "set-cookie":
2244 							// handled elsewhere fyi
2245 						break;
2246 						default:
2247 							// ignore
2248 					}
2249 
2250 					responseData.headersHash[name] = value;
2251 				}
2252 			}
2253 
2254 			size_t position = 0;
2255 			for(position = 0; position < data.length; position++) {
2256 				if(headerReadingState.readingLineContinuation) {
2257 					if(data[position] == ' ' || data[position] == '\t')
2258 						continue;
2259 					headerReadingState.readingLineContinuation = false;
2260 				}
2261 
2262 				if(headerReadingState.atStartOfLine) {
2263 					headerReadingState.atStartOfLine = false;
2264 					// FIXME it being \r should never happen... and i don't think it does
2265 					if(data[position] == '\r' || data[position] == '\n') {
2266 						// done with headers
2267 
2268 						position++; // skip the \r
2269 
2270 						if(responseData.headers.length)
2271 							parseLastHeader();
2272 
2273 						if(responseData.code >= 100 && responseData.code < 200) {
2274 							// "100 Continue" - we should continue uploading request data at this point
2275 							// "101 Switching Protocols" - websocket, not expected here...
2276 							// "102 Processing" - server still working, keep the connection alive
2277 							// "103 Early Hints" - can have useful Link headers etc
2278 							//
2279 							// and other unrecognized ones can just safely be skipped
2280 
2281 							// FIXME: the headers shouldn't actually be reset; 103 Early Hints
2282 							// can give useful headers we want to keep
2283 
2284 							responseData.headers = null;
2285 							headerReadingState.atStartOfLine = true;
2286 
2287 							continue; // the \n will be skipped by the for loop advance
2288 						}
2289 
2290 						if(this.requestParameters.method == HttpVerb.HEAD)
2291 							state = State.complete;
2292 						else
2293 							state = State.readingBody;
2294 
2295 						// skip the \n before we break
2296 						position++;
2297 
2298 						break;
2299 					} else if(data[position] == ' ' || data[position] == '\t') {
2300 						// line continuation, ignore all whitespace and collapse it into a space
2301 						headerReadingState.readingLineContinuation = true;
2302 						responseData.headers[$-1] ~= ' ';
2303 					} else {
2304 						// new header
2305 						if(responseData.headers.length)
2306 							parseLastHeader();
2307 						responseData.headers ~= "";
2308 					}
2309 				}
2310 
2311 				if(data[position] == '\r') {
2312 					headerReadingState.justSawCr = true;
2313 					continue;
2314 				} else
2315 					headerReadingState.justSawCr = false;
2316 
2317 				if(data[position] == '\n') {
2318 					headerReadingState.justSawLf = true;
2319 					headerReadingState.atStartOfLine = true;
2320 					continue;
2321 				} else
2322 					headerReadingState.justSawLf = false;
2323 
2324 				responseData.headers[$-1] ~= data[position];
2325 			}
2326 
2327 			data = data[position .. $];
2328 		}
2329 
2330 		if(state == State.readingBody) {
2331 			if(bodyReadingState.isChunked) {
2332 				// read the hex length, stopping at a \r\n, ignoring everything between the new line but after the first non-valid hex character
2333 				// read binary data of that length. it is our content
2334 				// repeat until a zero sized chunk
2335 				// then read footers as headers.
2336 
2337 				start_over:
2338 				for(int a = 0; a < data.length; a++) {
2339 					final switch(bodyReadingState.chunkedState) {
2340 						case 0: // reading hex
2341 							char c = data[a];
2342 							if((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) {
2343 								// just keep reading
2344 							} else {
2345 								int power = 1;
2346 								bodyReadingState.contentLengthRemaining = 0;
2347 								if(a == 0)
2348 									break; // just wait for more data
2349 								assert(a != 0, cast(string) data);
2350 								for(int b = a-1; b >= 0; b--) {
2351 									char cc = data[b];
2352 									if(cc >= 'a' && cc <= 'z')
2353 										cc -= 0x20;
2354 									int val = 0;
2355 									if(cc >= '0' && cc <= '9')
2356 										val = cc - '0';
2357 									else
2358 										val = cc - 'A' + 10;
2359 
2360 									assert(val >= 0 && val <= 15, to!string(val));
2361 									bodyReadingState.contentLengthRemaining += power * val;
2362 									power *= 16;
2363 								}
2364 								debug(arsd_http2_verbose) writeln("Chunk length: ", bodyReadingState.contentLengthRemaining);
2365 								bodyReadingState.chunkedState = 1;
2366 								data = data[a + 1 .. $];
2367 								goto start_over;
2368 							}
2369 						break;
2370 						case 1: // reading until end of line
2371 							char c = data[a];
2372 							if(c == '\n') {
2373 								if(bodyReadingState.contentLengthRemaining == 0)
2374 									bodyReadingState.chunkedState = 5;
2375 								else
2376 									bodyReadingState.chunkedState = 2;
2377 							}
2378 							data = data[a + 1 .. $];
2379 							goto start_over;
2380 						case 2: // reading data
2381 							auto can = a + bodyReadingState.contentLengthRemaining;
2382 							if(can > data.length)
2383 								can = cast(int) data.length;
2384 
2385 							auto newData = data[a .. can];
2386 							data = data[can .. $];
2387 
2388 							//if(bodyReadingState.isGzipped || bodyReadingState.isDeflated)
2389 							//	responseData.content ~= cast(ubyte[]) uncompress.uncompress(data[a .. can]);
2390 							//else
2391 								responseData.content ~= newData;
2392 
2393 							bodyReadingState.contentLengthRemaining -= newData.length;
2394 							debug(arsd_http2_verbose) writeln("clr: ", bodyReadingState.contentLengthRemaining, " " , a, " ", can);
2395 							assert(bodyReadingState.contentLengthRemaining >= 0);
2396 							if(bodyReadingState.contentLengthRemaining == 0) {
2397 								bodyReadingState.chunkedState = 3;
2398 							} else {
2399 								// will continue grabbing more
2400 							}
2401 							goto start_over;
2402 						case 3: // reading 13/10
2403 							assert(data[a] == 13);
2404 							bodyReadingState.chunkedState++;
2405 							data = data[a + 1 .. $];
2406 							goto start_over;
2407 						case 4: // reading 10 at end of packet
2408 							assert(data[a] == 10);
2409 							data = data[a + 1 .. $];
2410 							bodyReadingState.chunkedState = 0;
2411 							goto start_over;
2412 						case 5: // reading footers
2413 							//goto done; // FIXME
2414 
2415 							int footerReadingState = 0;
2416 							int footerSize;
2417 
2418 							while(footerReadingState != 2 && a < data.length) {
2419 								// import std.stdio; writeln(footerReadingState, " ", footerSize, " ", data);
2420 								switch(footerReadingState) {
2421 									case 0:
2422 										if(data[a] == 13)
2423 											footerReadingState++;
2424 										else
2425 											footerSize++;
2426 									break;
2427 									case 1:
2428 										if(data[a] == 10) {
2429 											if(footerSize == 0) {
2430 												// all done, time to break
2431 												footerReadingState++;
2432 
2433 											} else {
2434 												// actually had a footer, try to read another
2435 												footerReadingState = 0;
2436 												footerSize = 0;
2437 											}
2438 										} else {
2439 											throw new Exception("bad footer thing");
2440 										}
2441 									break;
2442 									default:
2443 										assert(0);
2444 								}
2445 
2446 								a++;
2447 							}
2448 
2449 							if(footerReadingState != 2)
2450 								break start_over; // haven't hit the end of the thing yet
2451 
2452 							bodyReadingState.chunkedState = 0;
2453 							data = data[a .. $];
2454 
2455 							if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) {
2456 								auto n = uncompress.uncompress(responseData.content);
2457 								n ~= uncompress.flush();
2458 								responseData.content = cast(ubyte[]) n;
2459 							}
2460 
2461 							//	responseData.content ~= cast(ubyte[]) uncompress.flush();
2462 							responseData.contentText = cast(string) responseData.content;
2463 
2464 							goto done;
2465 					}
2466 				}
2467 
2468 			} else {
2469 				//if(bodyReadingState.isGzipped || bodyReadingState.isDeflated)
2470 				//	responseData.content ~= cast(ubyte[]) uncompress.uncompress(data);
2471 				//else
2472 					responseData.content ~= data;
2473 				//assert(data.length <= bodyReadingState.contentLengthRemaining, format("%d <= %d\n%s", data.length, bodyReadingState.contentLengthRemaining, cast(string)data));
2474 				{
2475 					int use = cast(int) data.length;
2476 					if(use > bodyReadingState.contentLengthRemaining)
2477 						use = bodyReadingState.contentLengthRemaining;
2478 					bodyReadingState.contentLengthRemaining -= use;
2479 					data = data[use .. $];
2480 				}
2481 				if(bodyReadingState.contentLengthRemaining == 0) {
2482 					if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) {
2483 						// import std.stdio; writeln(responseData.content.length, " ", responseData.content[0 .. 2], " .. ", responseData.content[$-2 .. $]);
2484 						auto n = uncompress.uncompress(responseData.content);
2485 						n ~= uncompress.flush();
2486 						responseData.content = cast(ubyte[]) n;
2487 						responseData.contentText = cast(string) responseData.content;
2488 						//responseData.content ~= cast(ubyte[]) uncompress.flush();
2489 					} else {
2490 						responseData.contentText = cast(string) responseData.content;
2491 					}
2492 
2493 					done:
2494 
2495 					if(retainCookies && client !is null) {
2496 						client.retainCookies(responseData);
2497 					}
2498 
2499 					if(followLocation && responseData.location.length) {
2500 						if(maximumNumberOfRedirectsRemaining <= 0) {
2501 							throw new Exception("Maximum number of redirects exceeded");
2502 						} else {
2503 							maximumNumberOfRedirectsRemaining--;
2504 						}
2505 
2506 						static bool first = true;
2507 						//version(DigitalMars) if(!first) asm { int 3; }
2508 						debug(arsd_http2) writeln("redirecting to ", responseData.location);
2509 						populateFromInfo(Uri(responseData.location), HttpVerb.GET);
2510 						//import std.stdio; writeln("redirected to ", responseData.location);
2511 						first = false;
2512 						responseData = HttpResponse.init;
2513 						headerReadingState = HeaderReadingState.init;
2514 						bodyReadingState = BodyReadingState.init;
2515 						if(client !is null) {
2516 							// FIXME: this won't clear cookies that were cleared in another request
2517 							client.populateCookies(this); // they might have changed in the previous redirection cycle!
2518 						}
2519 						state = State.unsent;
2520 						stillAlive = false;
2521 						sendPrivate(false);
2522 					} else {
2523 						state = State.complete;
2524 						// FIXME
2525 						//if(closeSocketWhenComplete)
2526 							//socket.close();
2527 					}
2528 				}
2529 			}
2530 		}
2531 
2532 		if(data.length)
2533 			leftoverDataFromLastTime = data.dup;
2534 		else
2535 			leftoverDataFromLastTime = null;
2536 
2537 		return stillAlive;
2538 	}
2539 
2540 	}
2541 }
2542 
2543 /++
2544 	Waits for the first of the given requests to be either aborted or completed.
2545 	Returns the first one in that state, or `null` if the operation was interrupted
2546 	or reached the given timeout before any completed. (If it returns null even before
2547 	the timeout, it might be because the user pressed ctrl+c, so you should consider
2548 	checking if you should cancel the operation. If not, you can simply call it again
2549 	with the same arguments to start waiting again.)
2550 
2551 	You MUST check for null, even if you don't specify a timeout!
2552 
2553 	Note that if an individual request times out before any others request, it will
2554 	return that timed out request, since that counts as completion.
2555 
2556 	If the return is not null, you should call `waitForCompletion` on the given request
2557 	to get the response out. It will not have to wait since it is guaranteed to be
2558 	finished when returned by this function; that will just give you the cached response.
2559 
2560 	(I thought about just having it return the response, but tying a response back to
2561 	a request is harder than just getting the original request object back and taking
2562 	the response out of it.)
2563 
2564 	Please note: if a request in the set has already completed or been aborted, it will
2565 	always return the first one it sees upon calling the function. You may wish to remove
2566 	them from the list before calling the function.
2567 
2568 	History:
2569 		Added December 24, 2021 (dub v10.5)
2570 +/
2571 HttpRequest waitForFirstToComplete(Duration timeout, HttpRequest[] requests...) {
2572 
2573 	foreach(request; requests) {
2574 		if(request.state == HttpRequest.State.unsent)
2575 			request.send();
2576 		else if(request.state == HttpRequest.State.complete)
2577 			return request;
2578 		else if(request.state == HttpRequest.State.aborted)
2579 			return request;
2580 	}
2581 
2582 	while(true) {
2583 		if(auto err = HttpRequest.advanceConnections(timeout)) {
2584 			switch(err) {
2585 				case 1: return null;
2586 				case 2: throw new Exception("HttpRequest.advanceConnections returned 2: nothing to do");
2587 				case 3: return null;
2588 				default: throw new Exception("HttpRequest.advanceConnections got err " ~ to!string(err));
2589 			}
2590 		}
2591 
2592 		foreach(request; requests) {
2593 			if(request.state == HttpRequest.State.aborted || request.state == HttpRequest.State.complete) {
2594 				request.waitForCompletion();
2595 				return request;
2596 			}
2597 		}
2598 
2599 	}
2600 }
2601 
2602 /// ditto
2603 HttpRequest waitForFirstToComplete(HttpRequest[] requests...) {
2604 	return waitForFirstToComplete(1.weeks, requests);
2605 }
2606 
2607 /++
2608 	An input range that runs [waitForFirstToComplete] but only returning each request once.
2609 	Before you loop over it, you can set some properties to customize behavior.
2610 
2611 	If it times out or is interrupted, it will prematurely run empty. You can set the delegate
2612 	to process this.
2613 
2614 	Implementation note: each iteration through the loop does a O(n) check over each item remaining.
2615 	This shouldn't matter, but if it does become an issue for you, let me know.
2616 
2617 	History:
2618 		Added December 24, 2021 (dub v10.5)
2619 +/
2620 struct HttpRequestsAsTheyComplete {
2621 	/++
2622 		Seeds it with an overall timeout and the initial requests.
2623 		It will send all the requests before returning, then will process
2624 		the responses as they come.
2625 
2626 		Please note that it modifies the array of requests you pass in! It
2627 		will keep a reference to it and reorder items on each call of popFront.
2628 		You might want to pass a duplicate if you have another purpose for your
2629 		array and don't want to see it shuffled.
2630 	+/
2631 	this(Duration timeout, HttpRequest[] requests) {
2632 		remainingRequests = requests;
2633 		this.timeout = timeout;
2634 		popFront();
2635 	}
2636 
2637 	/++
2638 		You can set this delegate to decide how to handle an interruption. Returning true
2639 		from this will keep working. Returning false will terminate the loop.
2640 
2641 		If this is null, an interruption will always terminate the loop.
2642 
2643 		Note that interruptions can be caused by the garbage collector being triggered by
2644 		another thread as well as by user action. If you don't set a SIGINT handler, it
2645 		might be reasonable to always return true here.
2646 	+/
2647 	bool delegate() onInterruption;
2648 
2649 	private HttpRequest[] remainingRequests;
2650 
2651 	/// The timeout you set in the constructor. You can change it if you want.
2652 	Duration timeout;
2653 
2654 	/++
2655 		Adds another request to the work queue. It is safe to call this from inside the loop
2656 		as you process other requests.
2657 	+/
2658 	void appendRequest(HttpRequest request) {
2659 		remainingRequests ~= request;
2660 	}
2661 
2662 	/++
2663 		If the loop exited, it might be due to an interruption or a time out. If you like, you
2664 		can call this to pick up the work again,
2665 
2666 		If it returns `false`, the work is indeed all finished and you should not re-enter the loop.
2667 
2668 		---
2669 		auto range = HttpRequestsAsTheyComplete(10.seconds, your_requests);
2670 		process_loop: foreach(req; range) {
2671 			// process req
2672 		}
2673 		// make sure we weren't interrupted because the user requested we cancel!
2674 		// but then try to re-enter the range if possible
2675 		if(!user_quit && range.reenter()) {
2676 			// there's still something unprocessed in there
2677 			// range.reenter returning true means it is no longer
2678 			// empty, so we should try to loop over it again
2679 			goto process_loop; // re-enter the loop
2680 		}
2681 		---
2682 	+/
2683 	bool reenter() {
2684 		if(remainingRequests.length == 0)
2685 			return false;
2686 		empty = false;
2687 		popFront();
2688 		return true;
2689 	}
2690 
2691 	/// Standard range primitives. I reserve the right to change the variables to read-only properties in the future without notice.
2692 	HttpRequest front;
2693 
2694 	/// ditto
2695 	bool empty;
2696 
2697 	/// ditto
2698 	void popFront() {
2699 		resume:
2700 		if(remainingRequests.length == 0) {
2701 			empty = true;
2702 			return;
2703 		}
2704 
2705 		front = waitForFirstToComplete(timeout, remainingRequests);
2706 
2707 		if(front is null) {
2708 			if(onInterruption) {
2709 				if(onInterruption())
2710 					goto resume;
2711 			}
2712 			empty = true;
2713 			return;
2714 		}
2715 		foreach(idx, req; remainingRequests) {
2716 			if(req is front) {
2717 				remainingRequests[idx] = remainingRequests[$ - 1];
2718 				remainingRequests = remainingRequests[0 .. $ - 1];
2719 				return;
2720 			}
2721 		}
2722 	}
2723 }
2724 
2725 //
2726 struct HttpRequestParameters {
2727 	// FIXME: implement these
2728 	//Duration timeoutTotal; // the whole request must finish in this time or else it fails,even if data is still trickling in
2729 	Duration timeoutFromInactivity; // if there's no activity in this time it dies. basically the socket receive timeout
2730 
2731 	// debugging
2732 	bool useHttp11 = true; ///
2733 	bool acceptGzip = true; ///
2734 	bool keepAlive = true; ///
2735 
2736 	// the request itself
2737 	HttpVerb method; ///
2738 	string host; ///
2739 	ushort port; ///
2740 	string uri; ///
2741 
2742 	bool ssl; ///
2743 
2744 	string userAgent; ///
2745 	string authorization; ///
2746 
2747 	string[string] cookies; ///
2748 
2749 	string[] headers; /// do not duplicate host, content-length, content-type, or any others that have a specific property
2750 
2751 	string contentType; ///
2752 	ubyte[] bodyData; ///
2753 
2754 	string unixSocketPath; ///
2755 
2756 	bool gzipBody; ///
2757 }
2758 
2759 interface IHttpClient {
2760 
2761 }
2762 
2763 ///
2764 enum HttpVerb {
2765 	///
2766 	GET,
2767 	///
2768 	HEAD,
2769 	///
2770 	POST,
2771 	///
2772 	PUT,
2773 	///
2774 	DELETE,
2775 	///
2776 	OPTIONS,
2777 	///
2778 	TRACE,
2779 	///
2780 	CONNECT,
2781 	///
2782 	PATCH,
2783 	///
2784 	MERGE
2785 }
2786 
2787 /++
2788 	Supported file formats for [HttpClient.setClientCert]. These are loaded by OpenSSL
2789 	in the current implementation.
2790 
2791 	History:
2792 		Added February 3, 2022 (dub v10.6)
2793 +/
2794 enum CertificateFileFormat {
2795 	guess, /// try to guess the format from the file name and/or contents
2796 	pem, /// the files are specifically in PEM format
2797 	der /// the files are specifically in DER format
2798 }
2799 
2800 /++
2801 	HttpClient keeps cookies, location, and some other state to reuse connections, when possible, like a web browser.
2802 	You can use it as your entry point to make http requests.
2803 
2804 	See the example on [arsd.http2#examples].
2805 +/
2806 class HttpClient {
2807 	/* Protocol restrictions, useful to disable when debugging servers */
2808 	bool useHttp11 = true; ///
2809 	bool acceptGzip = true; ///
2810 	bool keepAlive = true; ///
2811 
2812 	/++
2813 		Sets the client certificate used as a log in identifier on https connections.
2814 		The certificate and key must be unencrypted at this time and both must be in
2815 		the same file format.
2816 
2817 		Bugs:
2818 			The current implementation sets the filenames into a static variable,
2819 			meaning it is shared across all clients and connections.
2820 
2821 			Errors in the cert or key are only reported if the server reports an
2822 			authentication failure. Make sure you are passing correct filenames
2823 			and formats of you do see a failure.
2824 
2825 		History:
2826 			Added February 2, 2022 (dub v10.6)
2827 	+/
2828 	void setClientCertificate(string certFilename, string keyFilename, CertificateFileFormat certFormat = CertificateFileFormat.guess) {
2829 		this.certFilename = certFilename;
2830 		this.keyFilename = keyFilename;
2831 		this.certFormat = certFormat;
2832 	}
2833 
2834 	/++
2835 		Sets whether [HttpRequest]s created through this object (with [navigateTo], [request], etc.), will have the
2836 		value of [HttpRequest.verifyPeer] of true or false upon construction.
2837 
2838 		History:
2839 			Added April 5, 2022 (dub v10.8). Previously, there was an undocumented global value used.
2840 	+/
2841 	bool defaultVerifyPeer = true;
2842 
2843 	/++
2844 		Adds a header to be automatically appended to each request created through this client.
2845 
2846 		If you add duplicate headers, it will add multiple copies.
2847 
2848 		You should NOT use this to add headers that can be set through other properties like [userAgent], [authorization], or [setCookie].
2849 
2850 		History:
2851 			Added July 12, 2023
2852 	+/
2853 	void addDefaultHeader(string key, string value) {
2854 		defaultHeaders ~= key ~ ": " ~ value;
2855 	}
2856 
2857 	private string[] defaultHeaders;
2858 
2859 	// FIXME: getCookies api
2860 	// FIXME: an easy way to download files
2861 
2862 	// FIXME: try to not make these static
2863 	private static string certFilename;
2864 	private static string keyFilename;
2865 	private static CertificateFileFormat certFormat;
2866 
2867 	///
2868 	@property Uri location() {
2869 		return currentUrl;
2870 	}
2871 
2872 	/++
2873 		Default timeout for requests created on this client.
2874 
2875 		History:
2876 			Added March 31, 2021
2877 	+/
2878 	Duration defaultTimeout = 10.seconds;
2879 
2880 	/++
2881 		High level function that works similarly to entering a url
2882 		into a browser.
2883 
2884 		Follows locations, retain cookies, updates the current url, etc.
2885 	+/
2886 	HttpRequest navigateTo(Uri where, HttpVerb method = HttpVerb.GET) {
2887 		currentUrl = where.basedOn(currentUrl);
2888 		currentDomain = where.host;
2889 
2890 		auto request = this.request(currentUrl, method);
2891 		request.followLocation = true;
2892 		request.retainCookies = true;
2893 
2894 		return request;
2895 	}
2896 
2897 	/++
2898 		Creates a request without updating the current url state. If you want to save cookies, either call [retainCookies] with the response yourself
2899 		or set [HttpRequest.retainCookies|request.retainCookies] to `true` on the returned object. But see important implementation shortcomings on [retainCookies].
2900 
2901 		To upload files, you can use the [FormData] overload.
2902 	+/
2903 	HttpRequest request(Uri uri, HttpVerb method = HttpVerb.GET, ubyte[] bodyData = null, string contentType = null) {
2904 		string proxyToUse = getProxyFor(uri);
2905 
2906 		auto request = new HttpRequest(this, uri, method, cache, defaultTimeout, proxyToUse);
2907 
2908 		request.verifyPeer = this.defaultVerifyPeer;
2909 
2910 		request.requestParameters.userAgent = userAgent;
2911 		request.requestParameters.authorization = authorization;
2912 
2913 		request.requestParameters.useHttp11 = this.useHttp11;
2914 		request.requestParameters.acceptGzip = this.acceptGzip;
2915 		request.requestParameters.keepAlive = this.keepAlive;
2916 
2917 		request.requestParameters.bodyData = bodyData;
2918 		request.requestParameters.contentType = contentType;
2919 
2920 		request.requestParameters.headers = this.defaultHeaders;
2921 
2922 		populateCookies(request);
2923 
2924 		return request;
2925 	}
2926 
2927 	/// ditto
2928 	HttpRequest request(Uri uri, FormData fd, HttpVerb method = HttpVerb.POST) {
2929 		return request(uri, method, fd.toBytes, fd.contentType);
2930 	}
2931 
2932 
2933 	private void populateCookies(HttpRequest request) {
2934 		// FIXME: what about expiration and the like? or domain/path checks? or Secure checks?
2935 		// FIXME: is uri.host correct? i think it should include port number too. what fun.
2936 		if(auto cookies = ""/*uri.host*/ in this.cookies) {
2937 			foreach(cookie; *cookies)
2938 				request.requestParameters.cookies[cookie.name] = cookie.value;
2939 		}
2940 	}
2941 
2942 	private Uri currentUrl;
2943 	private string currentDomain;
2944 	private ICache cache;
2945 
2946 	/++
2947 
2948 	+/
2949 	this(ICache cache = null) {
2950 		this.defaultVerifyPeer = .defaultVerifyPeer_;
2951 		this.cache = cache;
2952 		loadDefaultProxy();
2953 	}
2954 
2955 	/++
2956 		Loads the system-default proxy. Note that the constructor does this automatically
2957 		so you should rarely need to call this explicitly.
2958 
2959 		The environment variables are used, if present, on all operating systems.
2960 
2961 		History:
2962 			no_proxy support added April 13, 2022
2963 
2964 			Added April 12, 2021 (included in dub v9.5)
2965 
2966 		Bugs:
2967 			On Windows, it does NOT currently check the IE settings, but I do intend to
2968 			implement that in the future. When I do, it will be classified as a bug fix,
2969 			NOT a breaking change.
2970 	+/
2971 	void loadDefaultProxy() {
2972 		import std.process;
2973 		httpProxy = environment.get("http_proxy", environment.get("HTTP_PROXY", null));
2974 		httpsProxy = environment.get("https_proxy", environment.get("HTTPS_PROXY", null));
2975 		auto noProxy = environment.get("no_proxy", environment.get("NO_PROXY", null));
2976 		if (noProxy.length) {
2977 			proxyIgnore = noProxy.split(",");
2978 			foreach (ref rule; proxyIgnore)
2979 				rule = rule.strip;
2980 		}
2981 
2982 		// FIXME: on Windows, I should use the Internet Explorer proxy settings
2983 	}
2984 
2985 	/++
2986 		Checks if the given uri should be proxied according to the httpProxy, httpsProxy, proxyIgnore
2987 		variables and returns either httpProxy, httpsProxy or null.
2988 
2989 		If neither `httpProxy` or `httpsProxy` are set this always returns `null`. Same if `proxyIgnore`
2990 		contains `*`.
2991 
2992 		DNS is not resolved for proxyIgnore IPs, only IPs match IPs and hosts match hosts.
2993 	+/
2994 	string getProxyFor(Uri uri) {
2995 		string proxyToUse;
2996 		switch(uri.scheme) {
2997 			case "http":
2998 				proxyToUse = httpProxy;
2999 			break;
3000 			case "https":
3001 				proxyToUse = httpsProxy;
3002 			break;
3003 			default:
3004 				proxyToUse = null;
3005 		}
3006 
3007 		if (proxyToUse.length) {
3008 			foreach (ignore; proxyIgnore) {
3009 				if (matchProxyIgnore(ignore, uri)) {
3010 					return null;
3011 				}
3012 			}
3013 		}
3014 
3015 		return proxyToUse;
3016 	}
3017 
3018 	/// Returns -1 on error, otherwise the IP as uint. Parsing is very strict.
3019 	private static long tryParseIPv4(scope const(char)[] s) nothrow {
3020 		import std.algorithm : findSplit, all;
3021 		import std.ascii : isDigit;
3022 
3023 		static int parseNum(scope const(char)[] num) nothrow {
3024 			if (num.length < 1 || num.length > 3 || !num.representation.all!isDigit)
3025 				return -1;
3026 			try {
3027 				auto ret = num.to!int;
3028 				return ret > 255 ? -1 : ret;
3029 			} catch (Exception) {
3030 				assert(false);
3031 			}
3032 		}
3033 
3034 		if (s.length < "0.0.0.0".length || s.length > "255.255.255.255".length)
3035 			return -1;
3036 		auto firstPair = s.findSplit(".");
3037 		auto secondPair = firstPair[2].findSplit(".");
3038 		auto thirdPair = secondPair[2].findSplit(".");
3039 		auto a = parseNum(firstPair[0]);
3040 		auto b = parseNum(secondPair[0]);
3041 		auto c = parseNum(thirdPair[0]);
3042 		auto d = parseNum(thirdPair[2]);
3043 		if (a < 0 || b < 0 || c < 0 || d < 0)
3044 			return -1;
3045 		return (cast(uint)a << 24) | (b << 16) | (c << 8) | (d);
3046 	}
3047 
3048 	unittest {
3049 		assert(tryParseIPv4("0.0.0.0") == 0);
3050 		assert(tryParseIPv4("127.0.0.1") == 0x7f000001);
3051 		assert(tryParseIPv4("162.217.114.56") == 0xa2d97238);
3052 		assert(tryParseIPv4("256.0.0.1") == -1);
3053 		assert(tryParseIPv4("0.0.0.-2") == -1);
3054 		assert(tryParseIPv4("0.0.0.a") == -1);
3055 		assert(tryParseIPv4("0.0.0") == -1);
3056 		assert(tryParseIPv4("0.0.0.0.0") == -1);
3057 	}
3058 
3059 	/++
3060 		Returns true if the given no_proxy rule matches the uri.
3061 
3062 		Invalid IP ranges are silently ignored and return false.
3063 
3064 		See $(LREF proxyIgnore).
3065 	+/
3066 	static bool matchProxyIgnore(scope const(char)[] rule, scope const Uri uri) nothrow {
3067 		import std.algorithm;
3068 		import std.ascii : isDigit;
3069 		import std.uni : sicmp;
3070 
3071 		string uriHost = uri.host;
3072 		if (uriHost.length && uriHost[$ - 1] == '.')
3073 			uriHost = uriHost[0 .. $ - 1];
3074 
3075 		if (rule == "*")
3076 			return true;
3077 		while (rule.length && rule[0] == '.') rule = rule[1 .. $];
3078 
3079 		static int parsePort(scope const(char)[] portStr) nothrow {
3080 			if (portStr.length < 1 || portStr.length > 5 || !portStr.representation.all!isDigit)
3081 				return -1;
3082 			try {
3083 				return portStr.to!int;
3084 			} catch (Exception) {
3085 				assert(false, "to!int should succeed");
3086 			}
3087 		}
3088 
3089 		if (sicmp(rule, uriHost) == 0
3090 			|| (uriHost.length > rule.length
3091 				&& sicmp(rule, uriHost[$ - rule.length .. $]) == 0
3092 				&& uriHost[$ - rule.length - 1] == '.'))
3093 			return true;
3094 
3095 		if (rule.startsWith("[")) { // IPv6
3096 			// below code is basically nothrow lastIndexOfAny("]:")
3097 			ptrdiff_t lastColon = cast(ptrdiff_t) rule.length - 1;
3098 			while (lastColon >= 0) {
3099 				if (rule[lastColon] == ']' || rule[lastColon] == ':')
3100 					break;
3101 				lastColon--;
3102 			}
3103 			if (lastColon == -1)
3104 				return false; // malformed
3105 
3106 			if (rule[lastColon] == ':') { // match with port
3107 				auto port = parsePort(rule[lastColon + 1 .. $]);
3108 				if (port != -1) {
3109 					if (uri.effectivePort != port.to!int)
3110 						return false;
3111 					return uriHost == rule[0 .. lastColon];
3112 				}
3113 			}
3114 			// exact match of host already done above
3115 		} else {
3116 			auto slash = rule.lastIndexOfNothrow('/');
3117 			if (slash == -1) { // no IP range
3118 				auto colon = rule.lastIndexOfNothrow(':');
3119 				auto host = colon == -1 ? rule : rule[0 .. colon];
3120 				auto port = colon != -1 ? parsePort(rule[colon + 1 .. $]) : -1;
3121 				auto ip = tryParseIPv4(host);
3122 				if (ip == -1) { // not an IPv4, test for host with port
3123 					return port != -1
3124 						&& uri.effectivePort == port
3125 						&& uriHost == host;
3126 				} else {
3127 					// perform IPv4 equals
3128 					auto other = tryParseIPv4(uriHost);
3129 					if (other == -1)
3130 						return false; // rule == IPv4, uri != IPv4
3131 					if (port != -1)
3132 						return uri.effectivePort == port
3133 							&& uriHost == host;
3134 					else
3135 						return uriHost == host;
3136 				}
3137 			} else {
3138 				auto maskStr = rule[slash + 1 .. $];
3139 				auto ip = tryParseIPv4(rule[0 .. slash]);
3140 				if (ip == -1)
3141 					return false;
3142 				if (maskStr.length && maskStr.length < 3 && maskStr.representation.all!isDigit) {
3143 					// IPv4 range match
3144 					int mask;
3145 					try {
3146 						mask = maskStr.to!int;
3147 					} catch (Exception) {
3148 						assert(false);
3149 					}
3150 
3151 					auto other = tryParseIPv4(uriHost);
3152 					if (other == -1)
3153 						return false; // rule == IPv4, uri != IPv4
3154 
3155 					if (mask == 0) // matches all
3156 						return true;
3157 					if (mask > 32) // matches none
3158 						return false;
3159 
3160 					auto shift = 32 - mask;
3161 					return cast(uint)other >> shift
3162 						== cast(uint)ip >> shift;
3163 				}
3164 			}
3165 		}
3166 		return false;
3167 	}
3168 
3169 	unittest {
3170 		assert(matchProxyIgnore("0.0.0.0/0", Uri("http://127.0.0.1:80/a")));
3171 		assert(matchProxyIgnore("0.0.0.0/0", Uri("http://127.0.0.1/a")));
3172 		assert(!matchProxyIgnore("0.0.0.0/0", Uri("https://dlang.org/a")));
3173 		assert(matchProxyIgnore("*", Uri("https://dlang.org/a")));
3174 		assert(matchProxyIgnore("127.0.0.0/8", Uri("http://127.0.0.1:80/a")));
3175 		assert(matchProxyIgnore("127.0.0.0/8", Uri("http://127.0.0.1/a")));
3176 		assert(matchProxyIgnore("127.0.0.1", Uri("http://127.0.0.1:1234/a")));
3177 		assert(!matchProxyIgnore("127.0.0.1:80", Uri("http://127.0.0.1:1234/a")));
3178 		assert(!matchProxyIgnore("127.0.0.1/8", Uri("http://localhost/a"))); // no DNS resolution / guessing
3179 		assert(!matchProxyIgnore("0.0.0.0/1", Uri("http://localhost/a"))
3180 			&& !matchProxyIgnore("128.0.0.0/1", Uri("http://localhost/a"))); // no DNS resolution / guessing 2
3181 		foreach (m; 1 .. 32) {
3182 			assert(matchProxyIgnore(text("127.0.0.1/", m), Uri("http://127.0.0.1/a")));
3183 			assert(!matchProxyIgnore(text("127.0.0.1/", m), Uri("http://128.0.0.1/a")));
3184 			bool expectedMatch = m <= 24;
3185 			assert(expectedMatch == matchProxyIgnore(text("127.0.1.0/", m), Uri("http://127.0.1.128/a")), m.to!string);
3186 		}
3187 		assert(matchProxyIgnore("localhost", Uri("http://localhost/a")));
3188 		assert(matchProxyIgnore("localhost", Uri("http://foo.localhost/a")));
3189 		assert(matchProxyIgnore("localhost", Uri("http://foo.localhost./a")));
3190 		assert(matchProxyIgnore(".localhost", Uri("http://localhost/a")));
3191 		assert(matchProxyIgnore(".localhost", Uri("http://foo.localhost/a")));
3192 		assert(matchProxyIgnore(".localhost", Uri("http://foo.localhost./a")));
3193 		assert(!matchProxyIgnore("foo.localhost", Uri("http://localhost/a")));
3194 		assert(matchProxyIgnore("foo.localhost", Uri("http://foo.localhost/a")));
3195 		assert(matchProxyIgnore("foo.localhost", Uri("http://foo.localhost./a")));
3196 		assert(!matchProxyIgnore("bar.localhost", Uri("http://localhost/a")));
3197 		assert(!matchProxyIgnore("bar.localhost", Uri("http://foo.localhost/a")));
3198 		assert(!matchProxyIgnore("bar.localhost", Uri("http://foo.localhost./a")));
3199 		assert(!matchProxyIgnore("bar.localhost", Uri("http://bbar.localhost./a")));
3200 		assert(matchProxyIgnore("[::1]", Uri("http://[::1]/a")));
3201 		assert(!matchProxyIgnore("[::1]", Uri("http://[::2]/a")));
3202 		assert(matchProxyIgnore("[::1]:80", Uri("http://[::1]/a")));
3203 		assert(!matchProxyIgnore("[::1]:443", Uri("http://[::1]/a")));
3204 		assert(!matchProxyIgnore("[::1]:80", Uri("https://[::1]/a")));
3205 		assert(matchProxyIgnore("[::1]:443", Uri("https://[::1]/a")));
3206 		assert(matchProxyIgnore("google.com", Uri("https://GOOGLE.COM/a")));
3207 	}
3208 
3209 	/++
3210 		Proxies to use for requests. The [HttpClient] constructor will set these to the system values,
3211 		then you can reset it to `null` if you want to override and not use the proxy after all, or you
3212 		can set it after construction to whatever.
3213 
3214 		The proxy from the client will be automatically set to the requests performed through it. You can
3215 		also override on a per-request basis by creating the request and setting the `proxy` field there
3216 		before sending it.
3217 
3218 		History:
3219 			Added April 12, 2021 (included in dub v9.5)
3220 	+/
3221 	string httpProxy;
3222 	/// ditto
3223 	string httpsProxy;
3224 	/++
3225 		List of hosts or ips, optionally including a port, where not to proxy.
3226 
3227 		Each entry may be one of the following formats:
3228 		- `127.0.0.1` (IPv4, any port)
3229 		- `127.0.0.1:1234` (IPv4, specific port)
3230 		- `127.0.0.1/8` (IPv4 range / CIDR block, any port)
3231 		- `[::1]` (IPv6, any port)
3232 		- `[::1]:1234` (IPv6, specific port)
3233 		- `*` (all hosts and ports, basically don't proxy at all anymore)
3234 		- `.domain.name`, `domain.name` (don't proxy the specified domain,
3235 			leading dots are stripped and subdomains are also not proxied)
3236 		- `.domain.name:1234`, `domain.name:1234` (same as above, with specific port)
3237 
3238 		No DNS resolution or regex is done in this list.
3239 
3240 		See https://about.gitlab.com/blog/2021/01/27/we-need-to-talk-no-proxy/
3241 
3242 		History:
3243 			Added April 13, 2022
3244 	+/
3245 	string[] proxyIgnore;
3246 
3247 	/// See [retainCookies] for important caveats.
3248 	void setCookie(string name, string value, string domain = null) {
3249 		CookieHeader ch;
3250 
3251 		ch.name = name;
3252 		ch.value = value;
3253 
3254 		setCookie(ch, domain);
3255 	}
3256 
3257 	/// ditto
3258 	void setCookie(CookieHeader ch, string domain = null) {
3259 		if(domain is null)
3260 			domain = currentDomain;
3261 
3262 		// FIXME: figure all this out or else cookies liable to get too long, in addition to the overwriting and oversharing issues in long scraping sessions
3263 		cookies[""/*domain*/] ~= ch;
3264 	}
3265 
3266 	/++
3267 		[HttpClient] does NOT automatically store cookies. You must explicitly retain them from a response by calling this method.
3268 
3269 		Examples:
3270 			---
3271 			import arsd.http2;
3272 			void main() {
3273 				auto client = new HttpClient();
3274 				auto setRequest = client.request(Uri("http://arsdnet.net/cgi-bin/cookies/set"));
3275 				auto setResponse = setRequest.waitForCompletion();
3276 
3277 				auto request = client.request(Uri("http://arsdnet.net/cgi-bin/cookies/get"));
3278 				auto response = request.waitForCompletion();
3279 
3280 				// the cookie wasn't explicitly retained, so the server echos back nothing
3281 				assert(response.responseText.length == 0);
3282 
3283 				// now keep the cookies from our original set
3284 				client.retainCookies(setResponse);
3285 
3286 				request = client.request(Uri("http://arsdnet.net/cgi-bin/cookies/get"));
3287 				response = request.waitForCompletion();
3288 
3289 				// now it matches
3290 				assert(response.responseText.length && response.responseText == setResponse.cookies["example-cookie"]);
3291 			}
3292 			---
3293 
3294 		Bugs:
3295 			It does NOT currently implement domain / path / secure separation nor cookie expiration. It assumes that if you call this function, you're ok with it.
3296 
3297 			You may want to use separate HttpClient instances if any sharing is unacceptable at this time.
3298 
3299 		History:
3300 			Added July 5, 2021 (dub v10.2)
3301 	+/
3302 	void retainCookies(HttpResponse fromResponse) {
3303 		foreach(name, value; fromResponse.cookies)
3304 			setCookie(name, value);
3305 	}
3306 
3307 	///
3308 	void clearCookies(string domain = null) {
3309 		if(domain is null)
3310 			cookies = null;
3311 		else
3312 			cookies[domain] = null;
3313 	}
3314 
3315 	// If you set these, they will be pre-filled on all requests made with this client
3316 	string userAgent = "D arsd.html2"; ///
3317 	string authorization; ///
3318 
3319 	/* inter-request state */
3320 	private CookieHeader[][string] cookies;
3321 }
3322 
3323 private ptrdiff_t lastIndexOfNothrow(T)(scope T[] arr, T value) nothrow
3324 {
3325 	ptrdiff_t ret = cast(ptrdiff_t)arr.length - 1;
3326 	while (ret >= 0) {
3327 		if (arr[ret] == value)
3328 			return ret;
3329 		ret--;
3330 	}
3331 	return ret;
3332 }
3333 
3334 interface ICache {
3335 	/++
3336 		The client is about to make the given `request`. It will ALWAYS pass it to the cache object first so you can decide if you want to and can provide a response. You should probably check the appropriate headers to see if you should even attempt to look up on the cache (HttpClient does NOT do this to give maximum flexibility to the cache implementor).
3337 
3338 		Return null if the cache does not provide.
3339 	+/
3340 	const(HttpResponse)* getCachedResponse(HttpRequestParameters request);
3341 
3342 	/++
3343 		The given request has received the given response. The implementing class needs to decide if it wants to cache or not. Return true if it was added, false if you chose not to.
3344 
3345 		You may wish to examine headers, etc., in making the decision. The HttpClient will ALWAYS pass a request/response to this.
3346 	+/
3347 	bool cacheResponse(HttpRequestParameters request, HttpResponse response);
3348 }
3349 
3350 /+
3351 // / Provides caching behavior similar to a real web browser
3352 class HttpCache : ICache {
3353 	const(HttpResponse)* getCachedResponse(HttpRequestParameters request) {
3354 		return null;
3355 	}
3356 }
3357 
3358 // / Gives simple maximum age caching, ignoring the actual http headers
3359 class SimpleCache : ICache {
3360 	const(HttpResponse)* getCachedResponse(HttpRequestParameters request) {
3361 		return null;
3362 	}
3363 }
3364 +/
3365 
3366 /++
3367 	A pseudo-cache to provide a mock server. Construct one of these,
3368 	populate it with test responses, and pass it to [HttpClient] to
3369 	do a network-free test.
3370 
3371 	You should populate it with the [populate] method. Any request not
3372 	pre-populated will return a "server refused connection" response.
3373 +/
3374 class HttpMockProvider : ICache {
3375 	/+ +
3376 
3377 	+/
3378 	version(none)
3379 	this(Uri baseUrl, string defaultResponseContentType) {
3380 
3381 	}
3382 
3383 	this() {}
3384 
3385 	HttpResponse defaultResponse;
3386 
3387 	/// Implementation of the ICache interface. Hijacks all requests to return a pre-populated response or "server disconnected".
3388 	const(HttpResponse)* getCachedResponse(HttpRequestParameters request) {
3389 		import std.conv;
3390 		auto defaultPort = request.ssl ? 443 : 80;
3391 		string identifier = text(
3392 			request.method, " ",
3393 			request.ssl ? "https" : "http", "://",
3394 			request.host,
3395 			(request.port && request.port != defaultPort) ? (":" ~ to!string(request.port)) : "",
3396 			request.uri
3397 		);
3398 
3399 		if(auto res = identifier in population)
3400 			return res;
3401 		return &defaultResponse;
3402 	}
3403 
3404 	/// Implementation of the ICache interface. We never actually cache anything here since it is all about mock responses, not actually caching real data.
3405 	bool cacheResponse(HttpRequestParameters request, HttpResponse response) {
3406 		return false;
3407 	}
3408 
3409 	/++
3410 		Convenience method to populate simple responses. For more complex
3411 		work, use one of the other overloads where you build complete objects
3412 		yourself.
3413 
3414 		Params:
3415 			request = a verb and complete URL to mock as one string.
3416 			For example "GET http://example.com/". If you provide only
3417 			a partial URL, it will be based on the `baseUrl` you gave
3418 			in the `HttpMockProvider` constructor.
3419 
3420 			responseCode = the HTTP response code, like 200 or 404.
3421 
3422 			response = the response body as a string. It is assumed
3423 			to be of the `defaultResponseContentType` you passed in the
3424 			`HttpMockProvider` constructor.
3425 	+/
3426 	void populate(string request, int responseCode, string response) {
3427 
3428 		// FIXME: absolute-ize the URL in the request
3429 
3430 		HttpResponse r;
3431 		r.code = responseCode;
3432 		r.codeText = getHttpCodeText(r.code);
3433 
3434 		r.content = cast(ubyte[]) response;
3435 		r.contentText = response;
3436 
3437 		population[request] = r;
3438 	}
3439 
3440 	version(none)
3441 	void populate(string method, string url, HttpResponse response) {
3442 		// FIXME
3443 	}
3444 
3445 	private HttpResponse[string] population;
3446 }
3447 
3448 // modified from the one in cgi.d to just have the text
3449 private static string getHttpCodeText(int code) pure nothrow @nogc {
3450 	switch(code) {
3451 		// this module's proprietary extensions
3452 		case 0: return null;
3453 		case 1: return "request.abort called";
3454 		case 2: return "connection failed";
3455 		case 3: return "server disconnected";
3456 		case 4: return "exception thrown"; // actually should be some other thing
3457 		case 5: return "Request timed out";
3458 
3459 		// * * * standard ones * * *
3460 
3461 		// 1xx skipped since they shouldn't happen
3462 
3463 		//
3464 		case 200: return "OK";
3465 		case 201: return "Created";
3466 		case 202: return "Accepted";
3467 		case 203: return "Non-Authoritative Information";
3468 		case 204: return "No Content";
3469 		case 205: return "Reset Content";
3470 		//
3471 		case 300: return "Multiple Choices";
3472 		case 301: return "Moved Permanently";
3473 		case 302: return "Found";
3474 		case 303: return "See Other";
3475 		case 307: return "Temporary Redirect";
3476 		case 308: return "Permanent Redirect";
3477 		//
3478 		case 400: return "Bad Request";
3479 		case 403: return "Forbidden";
3480 		case 404: return "Not Found";
3481 		case 405: return "Method Not Allowed";
3482 		case 406: return "Not Acceptable";
3483 		case 409: return "Conflict";
3484 		case 410: return "Gone";
3485 		//
3486 		case 500: return "Internal Server Error";
3487 		case 501: return "Not Implemented";
3488 		case 502: return "Bad Gateway";
3489 		case 503: return "Service Unavailable";
3490 		//
3491 		default: assert(0, "Unsupported http code");
3492 	}
3493 }
3494 
3495 
3496 ///
3497 struct HttpCookie {
3498 	string name; ///
3499 	string value; ///
3500 	string domain; ///
3501 	string path; ///
3502 	//SysTime expirationDate; ///
3503 	bool secure; ///
3504 	bool httpOnly; ///
3505 }
3506 
3507 // FIXME: websocket
3508 
3509 version(testing)
3510 void main() {
3511 	import std.stdio;
3512 	auto client = new HttpClient();
3513 	auto request = client.navigateTo(Uri("http://localhost/chunked.php"));
3514 	request.send();
3515 	auto request2 = client.navigateTo(Uri("http://dlang.org/"));
3516 	request2.send();
3517 
3518 	{
3519 	auto response = request2.waitForCompletion();
3520 	//write(cast(string) response.content);
3521 	}
3522 
3523 	auto response = request.waitForCompletion();
3524 	write(cast(string) response.content);
3525 
3526 	writeln(HttpRequest.socketsPerHost);
3527 }
3528 
3529 string lastSocketError(Socket sock) {
3530 	import std.socket;
3531 	version(use_openssl) {
3532 		if(auto s = cast(OpenSslSocket) sock)
3533 			if(s.lastSocketError.length)
3534 				return s.lastSocketError;
3535 	}
3536 	return std.socket.lastSocketError();
3537 }
3538 
3539 // From sslsocket.d, but this is the maintained version!
3540 version(use_openssl) {
3541 	alias SslClientSocket = OpenSslSocket;
3542 
3543 	// CRL = Certificate Revocation List
3544 	static immutable string[] sslErrorCodes = [
3545 		"OK (code 0)",
3546 		"Unspecified SSL/TLS error (code 1)",
3547 		"Unable to get TLS issuer certificate (code 2)",
3548 		"Unable to get TLS CRL (code 3)",
3549 		"Unable to decrypt TLS certificate signature (code 4)",
3550 		"Unable to decrypt TLS CRL signature (code 5)",
3551 		"Unable to decode TLS issuer public key (code 6)",
3552 		"TLS certificate signature failure (code 7)",
3553 		"TLS CRL signature failure (code 8)",
3554 		"TLS certificate not yet valid (code 9)",
3555 		"TLS certificate expired (code 10)",
3556 		"TLS CRL not yet valid (code 11)",
3557 		"TLS CRL expired (code 12)",
3558 		"TLS error in certificate not before field (code 13)",
3559 		"TLS error in certificate not after field (code 14)",
3560 		"TLS error in CRL last update field (code 15)",
3561 		"TLS error in CRL next update field (code 16)",
3562 		"TLS system out of memory (code 17)",
3563 		"TLS certificate is self-signed (code 18)",
3564 		"Self-signed certificate in TLS chain (code 19)",
3565 		"Unable to get TLS issuer certificate locally (code 20)",
3566 		"Unable to verify TLS leaf signature (code 21)",
3567 		"TLS certificate chain too long (code 22)",
3568 		"TLS certificate was revoked (code 23)",
3569 		"TLS CA is invalid (code 24)",
3570 		"TLS error: path length exceeded (code 25)",
3571 		"TLS error: invalid purpose (code 26)",
3572 		"TLS error: certificate untrusted (code 27)",
3573 		"TLS error: certificate rejected (code 28)",
3574 	];
3575 
3576 	string getOpenSslErrorCode(long error) {
3577 		if(error == 62)
3578 			return "TLS certificate host name mismatch";
3579 
3580 		if(error < 0 || error >= sslErrorCodes.length)
3581 			return "SSL/TLS error code " ~ to!string(error);
3582 		return sslErrorCodes[cast(size_t) error];
3583 	}
3584 
3585 	struct SSL;
3586 	struct SSL_CTX;
3587 	struct SSL_METHOD;
3588 	struct X509_STORE_CTX;
3589 	enum SSL_VERIFY_NONE = 0;
3590 	enum SSL_VERIFY_PEER = 1;
3591 
3592 	// copy it into the buf[0 .. size] and return actual length you read.
3593 	// rwflag == 0 when reading, 1 when writing.
3594 	extern(C) alias pem_password_cb = int function(char* buffer, int bufferSize, int rwflag, void* userPointer);
3595 	extern(C) alias print_errors_cb = int function(const char*, size_t, void*);
3596 	extern(C) alias client_cert_cb = int function(SSL *ssl, X509 **x509, EVP_PKEY **pkey);
3597 	extern(C) alias keylog_cb = void function(SSL*, char*);
3598 
3599 	struct X509;
3600 	struct X509_STORE;
3601 	struct EVP_PKEY;
3602 	struct X509_VERIFY_PARAM;
3603 
3604 	import core.stdc.config;
3605 
3606 	enum SSL_ERROR_WANT_READ = 2;
3607 	enum SSL_ERROR_WANT_WRITE = 3;
3608 
3609 	struct ossllib {
3610 		__gshared static extern(C) {
3611 			/* these are only on older openssl versions { */
3612 				int function() SSL_library_init;
3613 				void function() SSL_load_error_strings;
3614 				SSL_METHOD* function() SSLv23_client_method;
3615 			/* } */
3616 
3617 			void function(ulong, void*) OPENSSL_init_ssl;
3618 
3619 			SSL_CTX* function(const SSL_METHOD*) SSL_CTX_new;
3620 			SSL* function(SSL_CTX*) SSL_new;
3621 			int function(SSL*, int) SSL_set_fd;
3622 			int function(SSL*) SSL_connect;
3623 			int function(SSL*, const void*, int) SSL_write;
3624 			int function(SSL*, void*, int) SSL_read;
3625 			@trusted nothrow @nogc int function(SSL*) SSL_shutdown;
3626 			void function(SSL*) SSL_free;
3627 			void function(SSL_CTX*) SSL_CTX_free;
3628 
3629 			int function(const SSL*) SSL_pending;
3630 			int function (const SSL *ssl, int ret) SSL_get_error;
3631 
3632 			void function(SSL*, int, void*) SSL_set_verify;
3633 
3634 			void function(SSL*, int, c_long, void*) SSL_ctrl;
3635 
3636 			SSL_METHOD* function() SSLv3_client_method;
3637 			SSL_METHOD* function() TLS_client_method;
3638 
3639 			void function(SSL_CTX*, void function(SSL*, char* line)) SSL_CTX_set_keylog_callback;
3640 
3641 			int function(SSL_CTX*) SSL_CTX_set_default_verify_paths;
3642 
3643 			X509_STORE* function(SSL_CTX*) SSL_CTX_get_cert_store;
3644 			c_long function(const SSL* ssl) SSL_get_verify_result;
3645 
3646 			X509_VERIFY_PARAM* function(const SSL*) SSL_get0_param;
3647 
3648 			/+
3649 			SSL_CTX_load_verify_locations
3650 			SSL_CTX_set_client_CA_list
3651 			+/
3652 
3653 			// client cert things
3654 			void function (SSL_CTX *ctx, int function(SSL *ssl, X509 **x509, EVP_PKEY **pkey)) SSL_CTX_set_client_cert_cb;
3655 		}
3656 	}
3657 
3658 	struct eallib {
3659 		__gshared static extern(C) {
3660 			/* these are only on older openssl versions { */
3661 				void function() OpenSSL_add_all_ciphers;
3662 				void function() OpenSSL_add_all_digests;
3663 			/* } */
3664 
3665 			const(char)* function(int) OpenSSL_version;
3666 
3667 			void function(ulong, void*) OPENSSL_init_crypto;
3668 
3669 			void function(print_errors_cb, void*) ERR_print_errors_cb;
3670 
3671 			void function(X509*) X509_free;
3672 			int function(X509_STORE*, X509*) X509_STORE_add_cert;
3673 
3674 
3675 			X509* function(FILE *fp, X509 **x, pem_password_cb *cb, void *u) PEM_read_X509;
3676 			EVP_PKEY* function(FILE *fp, EVP_PKEY **x, pem_password_cb *cb, void* userPointer) PEM_read_PrivateKey;
3677 
3678 			EVP_PKEY* function(FILE *fp, EVP_PKEY **a) d2i_PrivateKey_fp;
3679 			X509* function(FILE *fp, X509 **x) d2i_X509_fp;
3680 
3681 			X509* function(X509** a, const(ubyte*)* pp, c_long length) d2i_X509;
3682 			int function(X509* a, ubyte** o) i2d_X509;
3683 
3684 			int function(X509_VERIFY_PARAM* a, const char* b, size_t l) X509_VERIFY_PARAM_set1_host;
3685 
3686 			X509* function(X509_STORE_CTX *ctx) X509_STORE_CTX_get_current_cert;
3687 			int function(X509_STORE_CTX *ctx) X509_STORE_CTX_get_error;
3688 		}
3689 	}
3690 
3691 	struct OpenSSL {
3692 		static:
3693 
3694 		template opDispatch(string name) {
3695 			auto opDispatch(T...)(T t) {
3696 				static if(__traits(hasMember, ossllib, name)) {
3697 					auto ptr = __traits(getMember, ossllib, name);
3698 				} else static if(__traits(hasMember, eallib, name)) {
3699 					auto ptr = __traits(getMember, eallib, name);
3700 				} else static assert(0);
3701 
3702 				if(ptr is null)
3703 					throw new Exception(name ~ " not loaded");
3704 				return ptr(t);
3705 			}
3706 		}
3707 
3708 		// macros in the original C
3709 		SSL_METHOD* SSLv23_client_method() {
3710 			if(ossllib.SSLv23_client_method)
3711 				return ossllib.SSLv23_client_method();
3712 			else
3713 				return ossllib.TLS_client_method();
3714 		}
3715 
3716 		void SSL_set_tlsext_host_name(SSL* a, const char* b) {
3717 			if(ossllib.SSL_ctrl)
3718 				return ossllib.SSL_ctrl(a, 55 /*SSL_CTRL_SET_TLSEXT_HOSTNAME*/, 0 /*TLSEXT_NAMETYPE_host_name*/, cast(void*) b);
3719 			else throw new Exception("SSL_set_tlsext_host_name not loaded");
3720 		}
3721 
3722 		// special case
3723 		@trusted nothrow @nogc int SSL_shutdown(SSL* a) {
3724 			if(ossllib.SSL_shutdown)
3725 				return ossllib.SSL_shutdown(a);
3726 			assert(0);
3727 		}
3728 
3729 		void SSL_CTX_keylog_cb_func(SSL_CTX* ctx, keylog_cb func) {
3730 			// this isn't in openssl 1.0 and is non-essential, so it is allowed to fail.
3731 			if(ossllib.SSL_CTX_set_keylog_callback)
3732 				ossllib.SSL_CTX_set_keylog_callback(ctx, func);
3733 			//else throw new Exception("SSL_CTX_keylog_cb_func not loaded");
3734 		}
3735 
3736 	}
3737 
3738 	extern(C)
3739 	int collectSslErrors(const char* ptr, size_t len, void* user) @trusted {
3740 		string* s = cast(string*) user;
3741 
3742 		(*s) ~= ptr[0 .. len];
3743 
3744 		return 0;
3745 	}
3746 
3747 
3748 	private __gshared void* ossllib_handle;
3749 	version(Windows)
3750 		private __gshared void* oeaylib_handle;
3751 	else
3752 		alias oeaylib_handle = ossllib_handle;
3753 	version(Posix)
3754 		private import core.sys.posix.dlfcn;
3755 	else version(Windows)
3756 		private import core.sys.windows.windows;
3757 
3758 	import core.stdc.stdio;
3759 
3760 	private __gshared Object loadSslMutex = new Object;
3761 	private __gshared bool sslLoaded = false;
3762 
3763 	void loadOpenSsl() {
3764 		if(sslLoaded)
3765 			return;
3766 	synchronized(loadSslMutex) {
3767 
3768 		version(Posix) {
3769 			version(OSX) {
3770 				static immutable string[] ossllibs = [
3771 					"libssl.46.dylib",
3772 					"libssl.44.dylib",
3773 					"libssl.43.dylib",
3774 					"libssl.35.dylib",
3775 					"libssl.1.1.dylib",
3776 					"libssl.dylib",
3777 					"/usr/local/opt/openssl/lib/libssl.1.0.0.dylib",
3778 				];
3779 			} else {
3780 				static immutable string[] ossllibs = [
3781 					"libssl.so.3",
3782 					"libssl.so.1.1",
3783 					"libssl.so.1.0.2",
3784 					"libssl.so.1.0.1",
3785 					"libssl.so.1.0.0",
3786 					"libssl.so",
3787 				];
3788 			}
3789 
3790 			foreach(lib; ossllibs) {
3791 				ossllib_handle = dlopen(lib.ptr, RTLD_NOW);
3792 				if(ossllib_handle !is null) break;
3793 			}
3794 		} else version(Windows) {
3795 			version(X86_64) {
3796 				ossllib_handle = LoadLibraryW("libssl-1_1-x64.dll"w.ptr);
3797 				oeaylib_handle = LoadLibraryW("libcrypto-1_1-x64.dll"w.ptr);
3798 			}
3799 
3800 			static immutable wstring[] ossllibs = [
3801 				"libssl-3-x64.dll"w,
3802 				"libssl-3.dll"w,
3803 				"libssl-1_1.dll"w,
3804 				"libssl32.dll"w,
3805 			];
3806 
3807 			if(ossllib_handle is null)
3808 			foreach(lib; ossllibs) {
3809 				ossllib_handle = LoadLibraryW(lib.ptr);
3810 				if(ossllib_handle !is null) break;
3811 			}
3812 
3813 			static immutable wstring[] eaylibs = [
3814 				"libcrypto-3-x64.dll"w,
3815 				"libcrypto-3.dll"w,
3816 				"libcrypto-1_1.dll"w,
3817 				"libeay32.dll",
3818 			];
3819 
3820 			if(oeaylib_handle is null)
3821 			foreach(lib; eaylibs) {
3822 				oeaylib_handle = LoadLibraryW(lib.ptr);
3823 				if (oeaylib_handle !is null) break;
3824 			}
3825 
3826 			if(ossllib_handle is null) {
3827 				ossllib_handle = LoadLibraryW("ssleay32.dll"w.ptr);
3828 				oeaylib_handle = ossllib_handle;
3829 			}
3830 		}
3831 
3832 		if(ossllib_handle is null)
3833 			throw new Exception("libssl library not found");
3834 		if(oeaylib_handle is null)
3835 			throw new Exception("libeay32 library not found");
3836 
3837 		foreach(memberName; __traits(allMembers, ossllib)) {
3838 			alias t = typeof(__traits(getMember, ossllib, memberName));
3839 			version(Posix)
3840 				__traits(getMember, ossllib, memberName) = cast(t) dlsym(ossllib_handle, memberName);
3841 			else version(Windows) {
3842 				__traits(getMember, ossllib, memberName) = cast(t) GetProcAddress(ossllib_handle, memberName);
3843 			}
3844 		}
3845 
3846 		foreach(memberName; __traits(allMembers, eallib)) {
3847 			alias t = typeof(__traits(getMember, eallib, memberName));
3848 			version(Posix)
3849 				__traits(getMember, eallib, memberName) = cast(t) dlsym(oeaylib_handle, memberName);
3850 			else version(Windows) {
3851 				__traits(getMember, eallib, memberName) = cast(t) GetProcAddress(oeaylib_handle, memberName);
3852 			}
3853 		}
3854 
3855 
3856 		if(ossllib.SSL_library_init)
3857 			ossllib.SSL_library_init();
3858 		else if(ossllib.OPENSSL_init_ssl)
3859 			ossllib.OPENSSL_init_ssl(0, null);
3860 		else throw new Exception("couldn't init openssl");
3861 
3862 		if(eallib.OpenSSL_add_all_ciphers) {
3863 			eallib.OpenSSL_add_all_ciphers();
3864 			if(eallib.OpenSSL_add_all_digests is null)
3865 				throw new Exception("no add digests");
3866 			eallib.OpenSSL_add_all_digests();
3867 		} else if(eallib.OPENSSL_init_crypto)
3868 			eallib.OPENSSL_init_crypto(0 /*OPENSSL_INIT_ADD_ALL_CIPHERS and ALL_DIGESTS together*/, null);
3869 		else throw new Exception("couldn't init crypto openssl");
3870 
3871 		if(ossllib.SSL_load_error_strings)
3872 			ossllib.SSL_load_error_strings();
3873 		else if(ossllib.OPENSSL_init_ssl)
3874 			ossllib.OPENSSL_init_ssl(0x00200000L, null);
3875 		else throw new Exception("couldn't load openssl errors");
3876 
3877 		sslLoaded = true;
3878 	}
3879 	}
3880 
3881 	/+
3882 		// I'm just gonna let the OS clean this up on process termination because otherwise SSL_free
3883 		// might have trouble being run from the GC after this module is unloaded.
3884 	shared static ~this() {
3885 		if(ossllib_handle) {
3886 			version(Windows) {
3887 				FreeLibrary(oeaylib_handle);
3888 				FreeLibrary(ossllib_handle);
3889 			} else version(Posix)
3890 				dlclose(ossllib_handle);
3891 			ossllib_handle = null;
3892 		}
3893 		ossllib.tupleof = ossllib.tupleof.init;
3894 	}
3895 	+/
3896 
3897 	//pragma(lib, "crypto");
3898 	//pragma(lib, "ssl");
3899 	extern(C)
3900 	void write_to_file(SSL* ssl, char* line)
3901 	{
3902 		import std.stdio;
3903 		import std.string;
3904 		import std.process : environment;
3905 		string logfile = environment.get("SSLKEYLOGFILE");
3906 		if (logfile !is null)
3907 		{
3908 			auto f = std.stdio.File(logfile, "a+");
3909 			f.writeln(fromStringz(line));
3910 			f.close();
3911 		}
3912 	}
3913 
3914 	class OpenSslSocket : Socket {
3915 		private SSL* ssl;
3916 		private SSL_CTX* ctx;
3917 		private void initSsl(bool verifyPeer, string hostname) {
3918 			ctx = OpenSSL.SSL_CTX_new(OpenSSL.SSLv23_client_method());
3919 			assert(ctx !is null);
3920 
3921 			debug OpenSSL.SSL_CTX_keylog_cb_func(ctx, &write_to_file);
3922 			ssl = OpenSSL.SSL_new(ctx);
3923 
3924 			if(hostname.length) {
3925 				OpenSSL.SSL_set_tlsext_host_name(ssl, toStringz(hostname));
3926 				if(verifyPeer)
3927 					OpenSSL.X509_VERIFY_PARAM_set1_host(OpenSSL.SSL_get0_param(ssl), hostname.ptr, hostname.length);
3928 			}
3929 
3930 			if(verifyPeer) {
3931 				OpenSSL.SSL_CTX_set_default_verify_paths(ctx);
3932 
3933 				version(Windows) {
3934 					loadCertificatesFromRegistry(ctx);
3935 				}
3936 
3937 				OpenSSL.SSL_set_verify(ssl, SSL_VERIFY_PEER, &verifyCertificateFromRegistryArsdHttp);
3938 			} else
3939 				OpenSSL.SSL_set_verify(ssl, SSL_VERIFY_NONE, null);
3940 
3941 			OpenSSL.SSL_set_fd(ssl, cast(int) this.handle); // on win64 it is necessary to truncate, but the value is never large anyway see http://openssl.6102.n7.nabble.com/Sockets-windows-64-bit-td36169.html
3942 
3943 
3944 			OpenSSL.SSL_CTX_set_client_cert_cb(ctx, &cb);
3945 		}
3946 
3947 		extern(C)
3948 		static int cb(SSL* ssl, X509** x509, EVP_PKEY** pkey) {
3949 			if(HttpClient.certFilename.length && HttpClient.keyFilename.length) {
3950 				FILE* fpCert = fopen((HttpClient.certFilename ~ "\0").ptr, "rb");
3951 				if(fpCert is null)
3952 					return 0;
3953 				scope(exit)
3954 					fclose(fpCert);
3955 				FILE* fpKey = fopen((HttpClient.keyFilename ~ "\0").ptr, "rb");
3956 				if(fpKey is null)
3957 					return 0;
3958 				scope(exit)
3959 					fclose(fpKey);
3960 
3961 				with(CertificateFileFormat)
3962 				final switch(HttpClient.certFormat) {
3963 					case guess:
3964 						if(HttpClient.certFilename.endsWith(".pem") || HttpClient.keyFilename.endsWith(".pem"))
3965 							goto case pem;
3966 						else
3967 							goto case der;
3968 					case pem:
3969 						*x509 = OpenSSL.PEM_read_X509(fpCert, null, null, null);
3970 						*pkey = OpenSSL.PEM_read_PrivateKey(fpKey, null, null, null);
3971 					break;
3972 					case der:
3973 						*x509 = OpenSSL.d2i_X509_fp(fpCert, null);
3974 						*pkey = OpenSSL.d2i_PrivateKey_fp(fpKey, null);
3975 					break;
3976 				}
3977 
3978 				return 1;
3979 			}
3980 
3981 			return 0;
3982 		}
3983 
3984 		final bool dataPending() {
3985 			return OpenSSL.SSL_pending(ssl) > 0;
3986 		}
3987 
3988 		@trusted
3989 		override void connect(Address to) {
3990 			super.connect(to);
3991 			if(blocking) {
3992 				do_ssl_connect();
3993 			}
3994 		}
3995 
3996 		private string lastSocketError;
3997 
3998 		@trusted
3999 		// returns true if it is finished, false if it would have blocked, throws if there's an error
4000 		int do_ssl_connect() {
4001 			if(OpenSSL.SSL_connect(ssl) == -1) {
4002 
4003 				auto errCode = OpenSSL.SSL_get_error(ssl, -1);
4004 				if(errCode == SSL_ERROR_WANT_READ || errCode == SSL_ERROR_WANT_WRITE) {
4005 					return errCode;
4006 				}
4007 
4008 				string str;
4009 				OpenSSL.ERR_print_errors_cb(&collectSslErrors, &str);
4010 
4011 				auto err = OpenSSL.SSL_get_verify_result(ssl);
4012 				this.lastSocketError = str ~ " " ~ getOpenSslErrorCode(err);
4013 
4014 				throw new Exception("Secure connect failed: " ~ getOpenSslErrorCode(err));
4015 			} else this.lastSocketError = null;
4016 
4017 			return 0;
4018 		}
4019 
4020 		@trusted
4021 		override ptrdiff_t send(scope const(void)[] buf, SocketFlags flags) {
4022 		//import std.stdio;writeln(cast(string) buf);
4023 			debug(arsd_http2_verbose) writeln("ssl writing ", buf.length);
4024 			auto retval = OpenSSL.SSL_write(ssl, buf.ptr, cast(uint) buf.length);
4025 
4026 			// don't need to throw anymore since it is checked elsewhere
4027 			// code useful sometimes for debugging hence commenting instead of deleting
4028 			if(retval == -1) {
4029 				string str;
4030 				OpenSSL.ERR_print_errors_cb(&collectSslErrors, &str);
4031 				this.lastSocketError = str;
4032 
4033 				// throw new Exception("ssl send failed " ~ str);
4034 			} else this.lastSocketError = null;
4035 			return retval;
4036 
4037 		}
4038 		override ptrdiff_t send(scope const(void)[] buf) {
4039 			return send(buf, SocketFlags.NONE);
4040 		}
4041 		@trusted
4042 		override ptrdiff_t receive(scope void[] buf, SocketFlags flags) {
4043 
4044 			debug(arsd_http2_verbose) writeln("ssl_read before");
4045 			auto retval = OpenSSL.SSL_read(ssl, buf.ptr, cast(int)buf.length);
4046 			debug(arsd_http2_verbose) writeln("ssl_read after");
4047 
4048 			// don't need to throw anymore since it is checked elsewhere
4049 			// code useful sometimes for debugging hence commenting instead of deleting
4050 			if(retval == -1) {
4051 
4052 				string str;
4053 				OpenSSL.ERR_print_errors_cb(&collectSslErrors, &str);
4054 				this.lastSocketError = str;
4055 
4056 				// throw new Exception("ssl receive failed " ~ str);
4057 			} else this.lastSocketError = null;
4058 			return retval;
4059 		}
4060 		override ptrdiff_t receive(scope void[] buf) {
4061 			return receive(buf, SocketFlags.NONE);
4062 		}
4063 
4064 		this(AddressFamily af, SocketType type = SocketType.STREAM, string hostname = null, bool verifyPeer = true) {
4065 			version(Windows) __traits(getMember, this, "_blocking") = true; // lol longstanding phobos bug setting this to false on init
4066 			super(af, type);
4067 			initSsl(verifyPeer, hostname);
4068 		}
4069 
4070 		override void close() scope {
4071 			if(ssl) OpenSSL.SSL_shutdown(ssl);
4072 			super.close();
4073 		}
4074 
4075 		this(socket_t sock, AddressFamily af, string hostname, bool verifyPeer = true) {
4076 			super(sock, af);
4077 			initSsl(verifyPeer, hostname);
4078 		}
4079 
4080 		void freeSsl() {
4081 			if(ssl is null)
4082 				return;
4083 			OpenSSL.SSL_free(ssl);
4084 			OpenSSL.SSL_CTX_free(ctx);
4085 			ssl = null;
4086 			ctx = null;
4087 		}
4088 
4089 		~this() {
4090 			freeSsl();
4091 		}
4092 	}
4093 }
4094 
4095 
4096 /++
4097 	An experimental component for working with REST apis. Note that it
4098 	is a zero-argument template, so to create one, use `new HttpApiClient!()(args..)`
4099 	or you will get "HttpApiClient is used as a type" compile errors.
4100 
4101 	This will probably not work for you yet, and I might change it significantly.
4102 
4103 	Requires [arsd.jsvar].
4104 
4105 
4106 	Here's a snippet to create a pull request on GitHub to Phobos:
4107 
4108 	---
4109 	auto github = new HttpApiClient!()("https://api.github.com/", "your personal api token here");
4110 
4111 	// create the arguments object
4112 	// see: https://developer.github.com/v3/pulls/#create-a-pull-request
4113 	var args = var.emptyObject;
4114 	args.title = "My Pull Request";
4115 	args.head = "yourusername:" ~ branchName;
4116 	args.base = "master";
4117 	// note it is ["body"] instead of .body because `body` is a D keyword
4118 	args["body"] = "My cool PR is opened by the API!";
4119 	args.maintainer_can_modify = true;
4120 
4121 	/+
4122 		Fun fact, you can also write that:
4123 
4124 		var args = [
4125 			"title": "My Pull Request".var,
4126 			"head": "yourusername:" ~ branchName.var,
4127 			"base" : "master".var,
4128 			"body" : "My cool PR is opened by the API!".var,
4129 			"maintainer_can_modify": true.var
4130 		];
4131 
4132 		Note the .var constructor calls in there. If everything is the same type, you actually don't need that, but here since there's strings and bools, D won't allow the literal without explicit constructors to align them all.
4133 	+/
4134 
4135 	// this translates to `repos/dlang/phobos/pulls` and sends a POST request,
4136 	// containing `args` as json, then immediately grabs the json result and extracts
4137 	// the value `html_url` from it. `prUrl` is typed `var`, from arsd.jsvar.
4138 	auto prUrl = github.rest.repos.dlang.phobos.pulls.POST(args).result.html_url;
4139 
4140 	writeln("Created: ", prUrl);
4141 	---
4142 
4143 	Why use this instead of just building the URL? Well, of course you can! This just makes
4144 	it a bit more convenient than string concatenation and manages a few headers for you.
4145 
4146 	Subtypes could potentially add static type checks too.
4147 +/
4148 class HttpApiClient() {
4149 	import arsd.jsvar;
4150 
4151 	HttpClient httpClient;
4152 
4153 	alias HttpApiClientType = typeof(this);
4154 
4155 	string urlBase;
4156 	string oauth2Token;
4157 	string submittedContentType;
4158 	string authType = "Bearer";
4159 
4160 	/++
4161 		Params:
4162 
4163 		urlBase = The base url for the api. Tends to be something like `https://api.example.com/v2/` or similar.
4164 		oauth2Token = the authorization token for the service. You'll have to get it from somewhere else.
4165 		submittedContentType = the content-type of POST, PUT, etc. bodies.
4166 		httpClient = an injected http client, or null if you want to use a default-constructed one
4167 
4168 		History:
4169 			The `httpClient` param was added on December 26, 2020.
4170 	+/
4171 	this(string urlBase, string oauth2Token, string submittedContentType = "application/json", HttpClient httpClient = null) {
4172 		if(httpClient is null)
4173 			this.httpClient = new HttpClient();
4174 		else
4175 			this.httpClient = httpClient;
4176 
4177 		assert(urlBase[0] == 'h');
4178 		assert(urlBase[$-1] == '/');
4179 
4180 		this.urlBase = urlBase;
4181 		this.oauth2Token = oauth2Token;
4182 		this.submittedContentType = submittedContentType;
4183 	}
4184 
4185 	///
4186 	static struct HttpRequestWrapper {
4187 		HttpApiClientType apiClient; ///
4188 		HttpRequest request; ///
4189 		HttpResponse _response;
4190 
4191 		///
4192 		this(HttpApiClientType apiClient, HttpRequest request) {
4193 			this.apiClient = apiClient;
4194 			this.request = request;
4195 		}
4196 
4197 		/// Returns the full [HttpResponse] object so you can inspect the headers
4198 		@property HttpResponse response() {
4199 			if(_response is HttpResponse.init)
4200 				_response = request.waitForCompletion();
4201 			return _response;
4202 		}
4203 
4204 		/++
4205 			Returns the parsed JSON from the body of the response.
4206 
4207 			Throws on non-2xx responses.
4208 		+/
4209 		var result() {
4210 			return apiClient.throwOnError(response);
4211 		}
4212 
4213 		alias request this;
4214 	}
4215 
4216 	///
4217 	HttpRequestWrapper request(string uri, HttpVerb requestMethod = HttpVerb.GET, ubyte[] bodyBytes = null) {
4218 		if(uri[0] == '/')
4219 			uri = uri[1 .. $];
4220 
4221 		auto u = Uri(uri).basedOn(Uri(urlBase));
4222 
4223 		auto req = httpClient.navigateTo(u, requestMethod);
4224 
4225 		if(oauth2Token.length)
4226 			req.requestParameters.headers ~= "Authorization: "~ authType ~" " ~ oauth2Token;
4227 		req.requestParameters.contentType = submittedContentType;
4228 		req.requestParameters.bodyData = bodyBytes;
4229 
4230 		return HttpRequestWrapper(this, req);
4231 	}
4232 
4233 	///
4234 	var throwOnError(HttpResponse res) {
4235 		if(res.code < 200 || res.code >= 300)
4236 			throw new Exception(res.codeText ~ " " ~ res.contentText);
4237 
4238 		var response = var.fromJson(res.contentText);
4239 		if(response.errors) {
4240 			throw new Exception(response.errors.toJson());
4241 		}
4242 
4243 		return response;
4244 	}
4245 
4246 	///
4247 	@property RestBuilder rest() {
4248 		return RestBuilder(this, null, null);
4249 	}
4250 
4251 	// hipchat.rest.room["Tech Team"].history
4252         // gives: "/room/Tech%20Team/history"
4253 	//
4254 	// hipchat.rest.room["Tech Team"].history("page", "12)
4255 	///
4256 	static struct RestBuilder {
4257 		HttpApiClientType apiClient;
4258 		string[] pathParts;
4259 		string[2][] queryParts;
4260 		this(HttpApiClientType apiClient, string[] pathParts, string[2][] queryParts) {
4261 			this.apiClient = apiClient;
4262 			this.pathParts = pathParts;
4263 			this.queryParts = queryParts;
4264 		}
4265 
4266 		RestBuilder _SELF() {
4267 			return this;
4268 		}
4269 
4270 		/// The args are so you can call opCall on the returned
4271 		/// object, despite @property being broken af in D.
4272 		RestBuilder opDispatch(string str, T)(string n, T v) {
4273 			return RestBuilder(apiClient, pathParts ~ str, queryParts ~ [n, to!string(v)]);
4274 		}
4275 
4276 		///
4277 		RestBuilder opDispatch(string str)() {
4278 			return RestBuilder(apiClient, pathParts ~ str, queryParts);
4279 		}
4280 
4281 
4282 		///
4283 		RestBuilder opIndex(string str) {
4284 			return RestBuilder(apiClient, pathParts ~ str, queryParts);
4285 		}
4286 		///
4287 		RestBuilder opIndex(var str) {
4288 			return RestBuilder(apiClient, pathParts ~ str.get!string, queryParts);
4289 		}
4290 		///
4291 		RestBuilder opIndex(int i) {
4292 			return RestBuilder(apiClient, pathParts ~ to!string(i), queryParts);
4293 		}
4294 
4295 		///
4296 		RestBuilder opCall(T)(string name, T value) {
4297 			return RestBuilder(apiClient, pathParts, queryParts ~ [name, to!string(value)]);
4298 		}
4299 
4300 		///
4301 		string toUri() {
4302 			string result;
4303 			foreach(idx, part; pathParts) {
4304 				if(idx)
4305 					result ~= "/";
4306 				result ~= encodeUriComponent(part);
4307 			}
4308 			result ~= "?";
4309 			foreach(idx, part; queryParts) {
4310 				if(idx)
4311 					result ~= "&";
4312 				result ~= encodeUriComponent(part[0]);
4313 				result ~= "=";
4314 				result ~= encodeUriComponent(part[1]);
4315 			}
4316 
4317 			return result;
4318 		}
4319 
4320 		///
4321 		final HttpRequestWrapper GET() { return _EXECUTE(HttpVerb.GET, this.toUri(), ToBytesResult.init); }
4322 		/// ditto
4323 		final HttpRequestWrapper DELETE() { return _EXECUTE(HttpVerb.DELETE, this.toUri(), ToBytesResult.init); }
4324 
4325 		// need to be able to send: JSON, urlencoded, multipart/form-data, and raw stuff.
4326 		/// ditto
4327 		final HttpRequestWrapper POST(T...)(T t) { return _EXECUTE(HttpVerb.POST, this.toUri(), toBytes(t)); }
4328 		/// ditto
4329 		final HttpRequestWrapper PATCH(T...)(T t) { return _EXECUTE(HttpVerb.PATCH, this.toUri(), toBytes(t)); }
4330 		/// ditto
4331 		final HttpRequestWrapper PUT(T...)(T t) { return _EXECUTE(HttpVerb.PUT, this.toUri(), toBytes(t)); }
4332 
4333 		struct ToBytesResult {
4334 			ubyte[] bytes;
4335 			string contentType;
4336 		}
4337 
4338 		private ToBytesResult toBytes(T...)(T t) {
4339 			import std.conv : to;
4340 			static if(T.length == 0)
4341 				return ToBytesResult(null, null);
4342 			else static if(T.length == 1 && is(T[0] == var))
4343 				return ToBytesResult(cast(ubyte[]) t[0].toJson(), "application/json"); // json data
4344 			else static if(T.length == 1 && (is(T[0] == string) || is(T[0] == ubyte[])))
4345 				return ToBytesResult(cast(ubyte[]) t[0], null); // raw data
4346 			else static if(T.length == 1 && is(T[0] : FormData))
4347 				return ToBytesResult(t[0].toBytes, t[0].contentType);
4348 			else static if(T.length > 1 && T.length % 2 == 0 && is(T[0] == string)) {
4349 				// string -> value pairs for a POST request
4350 				string answer;
4351 				foreach(idx, val; t) {
4352 					static if(idx % 2 == 0) {
4353 						if(answer.length)
4354 							answer ~= "&";
4355 						answer ~= encodeUriComponent(val); // it had better be a string! lol
4356 						answer ~= "=";
4357 					} else {
4358 						answer ~= encodeUriComponent(to!string(val));
4359 					}
4360 				}
4361 
4362 				return ToBytesResult(cast(ubyte[]) answer, "application/x-www-form-urlencoded");
4363 			}
4364 			else
4365 				static assert(0); // FIXME
4366 
4367 		}
4368 
4369 		HttpRequestWrapper _EXECUTE(HttpVerb verb, string uri, ubyte[] bodyBytes) {
4370 			return apiClient.request(uri, verb, bodyBytes);
4371 		}
4372 
4373 		HttpRequestWrapper _EXECUTE(HttpVerb verb, string uri, ToBytesResult tbr) {
4374 			auto r = apiClient.request(uri, verb, tbr.bytes);
4375 			if(tbr.contentType !is null)
4376 				r.requestParameters.contentType = tbr.contentType;
4377 			return r;
4378 		}
4379 	}
4380 }
4381 
4382 
4383 // see also: arsd.cgi.encodeVariables
4384 /++
4385 	Creates a multipart/form-data object that is suitable for file uploads and other kinds of POST.
4386 
4387 	It has a set of names and values of mime components. Names can be repeated. They will be presented in the same order in which you add them. You will mostly want to use the [append] method.
4388 
4389 	You can pass this directly to [HttpClient.request].
4390 
4391 	Based on: https://developer.mozilla.org/en-US/docs/Web/API/FormData
4392 
4393 	---
4394 		auto fd = new FormData();
4395 		// add some data, plain string first
4396 		fd.append("name", "Adam");
4397 		// then a file
4398 		fd.append("photo", std.file.read("adam.jpg"), "image/jpeg", "adam.jpg");
4399 
4400 		// post it!
4401 		auto client = new HttpClient();
4402 		client.request(Uri("http://example.com/people"), fd).waitForCompletion();
4403 	---
4404 
4405 	History:
4406 		Added June 8, 2018
4407 +/
4408 class FormData {
4409 	static struct MimePart {
4410 		string name;
4411 		const(void)[] data;
4412 		string contentType;
4413 		string filename;
4414 	}
4415 
4416 	private MimePart[] parts;
4417 	private string boundary = "0016e64be86203dd36047610926a"; // FIXME
4418 
4419 	/++
4420 		Appends the given entry to the request. This can be a simple key/value pair of strings or file uploads.
4421 
4422 		For a simple key/value pair, leave `contentType` and `filename` as `null`.
4423 
4424 		For file uploads, please note that many servers require filename be given for a file upload and it may not allow you to put in a path. I suggest using [std.path.baseName] to strip off path information from a file you are loading.
4425 
4426 		The `contentType` is generally verified by servers for file uploads.
4427 	+/
4428 	void append(string key, const(void)[] value, string contentType = null, string filename = null) {
4429 		parts ~= MimePart(key, value, contentType, filename);
4430 	}
4431 
4432 	/++
4433 		Deletes any entries from the set with the given key.
4434 
4435 		History:
4436 			Added June 7, 2023 (dub v11.0)
4437 	+/
4438 	void deleteKey(string key) {
4439 		MimePart[] newParts;
4440 		foreach(part; parts)
4441 			if(part.name != key)
4442 				newParts ~= part;
4443 		parts = newParts;
4444 	}
4445 
4446 	/++
4447 		Returns the first entry with the given key, or `MimePart.init` if there is nothing.
4448 
4449 		History:
4450 			Added June 7, 2023 (dub v11.0)
4451 	+/
4452 	MimePart get(string key) {
4453 		foreach(part; parts)
4454 			if(part.name == key)
4455 				return part;
4456 		return MimePart.init;
4457 	}
4458 
4459 	/++
4460 		Returns the all entries with the given key.
4461 
4462 		History:
4463 			Added June 7, 2023 (dub v11.0)
4464 	+/
4465 	MimePart[] getAll(string key) {
4466 		MimePart[] answer;
4467 		foreach(part; parts)
4468 			if(part.name == key)
4469 				answer ~= part;
4470 		return answer;
4471 	}
4472 
4473 	/++
4474 		Returns true if the given key exists in the set.
4475 
4476 		History:
4477 			Added June 7, 2023 (dub v11.0)
4478 	+/
4479 	bool has(string key) {
4480 		return get(key).name == key;
4481 	}
4482 
4483 	/++
4484 		Sets the given key to the given value if it exists, or appends it if it doesn't.
4485 
4486 		You probably want [append] instead.
4487 
4488 		See_Also:
4489 			[append]
4490 
4491 		History:
4492 			Added June 7, 2023 (dub v11.0)
4493 	+/
4494 	void set(string key, const(void)[] value, string contentType, string filename) {
4495 		foreach(ref part; parts)
4496 			if(part.name == key) {
4497 				part.data = value;
4498 				part.contentType = contentType;
4499 				part.filename = filename;
4500 				return;
4501 			}
4502 
4503 		append(key, value, contentType, filename);
4504 	}
4505 
4506 	/++
4507 		Returns all the current entries in the object.
4508 
4509 		History:
4510 			Added June 7, 2023 (dub v11.0)
4511 	+/
4512 	MimePart[] entries() {
4513 		return parts;
4514 	}
4515 
4516 	// FIXME:
4517 	// keys iterator
4518 	// values iterator
4519 
4520 	/++
4521 		Gets the content type header that should be set in the request. This includes the type and boundary that is applicable to the [toBytes] method.
4522 	+/
4523 	string contentType() {
4524 		return "multipart/form-data; boundary=" ~ boundary;
4525 	}
4526 
4527 	/++
4528 		Returns bytes applicable for the body of this request. Use the [contentType] method to get the appropriate content type header with the right boundary.
4529 	+/
4530 	ubyte[] toBytes() {
4531 		string data;
4532 
4533 		foreach(part; parts) {
4534 			data ~= "--" ~ boundary ~ "\r\n";
4535 			data ~= "Content-Disposition: form-data; name=\""~part.name~"\"";
4536 			if(part.filename !is null)
4537 				data ~= "; filename=\""~part.filename~"\"";
4538 			data ~= "\r\n";
4539 			if(part.contentType !is null)
4540 				data ~= "Content-Type: " ~ part.contentType ~ "\r\n";
4541 			data ~= "\r\n";
4542 
4543 			data ~= cast(string) part.data;
4544 
4545 			data ~= "\r\n";
4546 		}
4547 
4548 		data ~= "--" ~ boundary ~ "--\r\n";
4549 
4550 		return cast(ubyte[]) data;
4551 	}
4552 }
4553 
4554 private bool bicmp(in ubyte[] item, in char[] search) {
4555 	if(item.length != search.length) return false;
4556 
4557 	foreach(i; 0 .. item.length) {
4558 		ubyte a = item[i];
4559 		ubyte b = search[i];
4560 		if(a >= 'A' && a <= 'Z')
4561 			a += 32;
4562 		//if(b >= 'A' && b <= 'Z')
4563 			//b += 32;
4564 		if(a != b)
4565 			return false;
4566 	}
4567 
4568 	return true;
4569 }
4570 
4571 /++
4572 	WebSocket client, based on the browser api, though also with other api options.
4573 
4574 	---
4575 		import arsd.http2;
4576 
4577 		void main() {
4578 			auto ws = new WebSocket(Uri("ws://...."));
4579 
4580 			ws.onmessage = (in char[] msg) {
4581 				ws.send("a reply");
4582 			};
4583 
4584 			ws.connect();
4585 
4586 			WebSocket.eventLoop();
4587 		}
4588 	---
4589 
4590 	Symbol_groups:
4591 		foundational =
4592 			Used with all API styles.
4593 
4594 		browser_api =
4595 			API based on the standard in the browser.
4596 
4597 		event_loop_integration =
4598 			Integrating with external event loops is done through static functions. You should
4599 			call these BEFORE doing anything else with the WebSocket module or class.
4600 
4601 			$(PITFALL NOT IMPLEMENTED)
4602 			---
4603 				WebSocket.setEventLoopProxy(arsd.simpledisplay.EventLoop.proxy.tupleof);
4604 				// or something like that. it is not implemented yet.
4605 			---
4606 			$(PITFALL NOT IMPLEMENTED)
4607 
4608 		blocking_api =
4609 			The blocking API is best used when you only need basic functionality with a single connection.
4610 
4611 			---
4612 			WebSocketFrame msg;
4613 			do {
4614 				// FIXME good demo
4615 			} while(msg);
4616 			---
4617 
4618 			Or to check for blocks before calling:
4619 
4620 			---
4621 			try_to_process_more:
4622 			while(ws.isMessageBuffered()) {
4623 				auto msg = ws.waitForNextMessage();
4624 				// process msg
4625 			}
4626 			if(ws.isDataPending()) {
4627 				ws.lowLevelReceive();
4628 				goto try_to_process_more;
4629 			} else {
4630 				// nothing ready, you can do other things
4631 				// or at least sleep a while before trying
4632 				// to process more.
4633 				if(ws.readyState == WebSocket.OPEN) {
4634 					Thread.sleep(1.seconds);
4635 					goto try_to_process_more;
4636 				}
4637 			}
4638 			---
4639 
4640 +/
4641 class WebSocket {
4642 	private Uri uri;
4643 	private string[string] cookies;
4644 
4645 	private string host;
4646 	private ushort port;
4647 	private bool ssl;
4648 
4649 	// used to decide if we mask outgoing msgs
4650 	private bool isClient;
4651 
4652 	private MonoTime timeoutFromInactivity;
4653 	private MonoTime nextPing;
4654 
4655 	/++
4656 		wss://echo.websocket.org
4657 	+/
4658 	/// Group: foundational
4659 	this(Uri uri, Config config = Config.init)
4660 		//in (uri.scheme == "ws" || uri.scheme == "wss")
4661 		in { assert(uri.scheme == "ws" || uri.scheme == "wss"); } do
4662 	{
4663 		this.uri = uri;
4664 		this.config = config;
4665 
4666 		this.receiveBuffer = new ubyte[](config.initialReceiveBufferSize);
4667 
4668 		host = uri.host;
4669 		ssl = uri.scheme == "wss";
4670 		port = cast(ushort) (uri.port ? uri.port : ssl ? 443 : 80);
4671 
4672 		if(ssl) {
4673 			version(with_openssl) {
4674 				loadOpenSsl();
4675 				socket = new SslClientSocket(family(uri.unixSocketPath), SocketType.STREAM, host, config.verifyPeer);
4676 			} else
4677 				throw new Exception("SSL not compiled in");
4678 		} else
4679 			socket = new Socket(family(uri.unixSocketPath), SocketType.STREAM);
4680 
4681 		socket.setOption(SocketOptionLevel.TCP, SocketOption.TCP_NODELAY, 1);
4682 		cookies = config.cookies;
4683 	}
4684 
4685 	/++
4686 
4687 	+/
4688 	/// Group: foundational
4689 	void connect() {
4690 		this.isClient = true;
4691 
4692 		socket.blocking = false;
4693 
4694 		if(uri.unixSocketPath)
4695 			socket.connect(new UnixAddress(uri.unixSocketPath));
4696 		else
4697 			socket.connect(new InternetAddress(host, port)); // FIXME: ipv6 support...
4698 
4699 
4700 		auto readSet = new SocketSet();
4701 		auto writeSet = new SocketSet();
4702 
4703 		readSet.reset();
4704 		writeSet.reset();
4705 
4706 		readSet.add(socket);
4707 		writeSet.add(socket);
4708 
4709 		auto selectGot = Socket.select(readSet, writeSet, null, config.timeoutFromInactivity);
4710 		if(selectGot == -1) {
4711 			// interrupted
4712 
4713 			throw new Exception("Websocket connection interrupted - retry might succeed");
4714 		} else if(selectGot == 0) {
4715 			// time out
4716 			socket.close();
4717 			throw new Exception("Websocket connection timed out");
4718 		} else {
4719 			if(writeSet.isSet(socket) || readSet.isSet(socket)) {
4720 				import core.stdc.stdint;
4721 				int32_t error;
4722 				int retopt = socket.getOption(SocketOptionLevel.SOCKET, SocketOption.ERROR, error);
4723 				if(retopt < 0 || error != 0) {
4724 					socket.close();
4725 					throw new Exception("Websocket connection failed - " ~ formatSocketError(error));
4726 				} else {
4727 					// FIXME: websocket handshake could and really should be async too.
4728 					socket.blocking = true; // just convenience
4729 					if(auto s = cast(SslClientSocket) socket) {
4730 						s.do_ssl_connect();
4731 					} else {
4732 						// we're ready
4733 					}
4734 				}
4735 			}
4736 		}
4737 
4738 		auto uri = this.uri.path.length ? this.uri.path : "/";
4739 		if(this.uri.query.length) {
4740 			uri ~= "?";
4741 			uri ~= this.uri.query;
4742 		}
4743 
4744 		// the headers really shouldn't be bigger than this, at least
4745 		// the chunks i need to process
4746 		ubyte[4096] bufferBacking = void;
4747 		ubyte[] buffer = bufferBacking[];
4748 		size_t pos;
4749 
4750 		void append(in char[][] items...) {
4751 			foreach(what; items) {
4752 				if((pos + what.length) > buffer.length) {
4753 					buffer.length += 4096;
4754 				}
4755 				buffer[pos .. pos + what.length] = cast(ubyte[]) what[];
4756 				pos += what.length;
4757 			}
4758 		}
4759 
4760 		append("GET ", uri, " HTTP/1.1\r\n");
4761 		append("Host: ", this.uri.host, "\r\n");
4762 
4763 		append("Upgrade: websocket\r\n");
4764 		append("Connection: Upgrade\r\n");
4765 		append("Sec-WebSocket-Version: 13\r\n");
4766 
4767 		// FIXME: randomize this
4768 		append("Sec-WebSocket-Key: x3JEHMbDL1EzLkh9GBhXDw==\r\n");
4769 		if(cookies.length > 0) {
4770 			append("Cookie: ");
4771 			bool first=true;
4772 			foreach(k,v;cookies) {
4773 				if(first) first = false;
4774 				else append("; ");
4775 				append(k);
4776 				append("=");
4777 				append(v);
4778 			}
4779 			append("\r\n");
4780 		}
4781 		/*
4782 		//This is equivalent but has dependencies
4783 		import std.format;
4784 		import std.algorithm : map;
4785 		append(format("cookie: %-(%s %)\r\n",cookies.byKeyValue.map!(t=>format("%s=%s",t.key,t.value))));
4786 		*/
4787 
4788 		if(config.protocol.length)
4789 			append("Sec-WebSocket-Protocol: ", config.protocol, "\r\n");
4790 		if(config.origin.length)
4791 			append("Origin: ", config.origin, "\r\n");
4792 
4793 		foreach(h; config.additionalHeaders) {
4794 			append(h);
4795 			append("\r\n");
4796 		}
4797 
4798 		append("\r\n");
4799 
4800 		auto remaining = buffer[0 .. pos];
4801 		//import std.stdio; writeln(host, " " , port, " ", cast(string) remaining);
4802 		while(remaining.length) {
4803 			auto r = socket.send(remaining);
4804 			if(r < 0)
4805 				throw new Exception(lastSocketError(socket));
4806 			if(r == 0)
4807 				throw new Exception("unexpected connection termination");
4808 			remaining = remaining[r .. $];
4809 		}
4810 
4811 		// the response shouldn't be especially large at this point, just
4812 		// headers for the most part. gonna try to get it in the stack buffer.
4813 		// then copy stuff after headers, if any, to the frame buffer.
4814 		ubyte[] used;
4815 
4816 		void more() {
4817 			auto r = socket.receive(buffer[used.length .. $]);
4818 
4819 			if(r < 0)
4820 				throw new Exception(lastSocketError(socket));
4821 			if(r == 0)
4822 				throw new Exception("unexpected connection termination");
4823 			//import std.stdio;writef("%s", cast(string) buffer[used.length .. used.length + r]);
4824 
4825 			used = buffer[0 .. used.length + r];
4826 		}
4827 
4828 		more();
4829 
4830 		import std.algorithm;
4831 		if(!used.startsWith(cast(ubyte[]) "HTTP/1.1 101"))
4832 			throw new Exception("didn't get a websocket answer");
4833 		// skip the status line
4834 		while(used.length && used[0] != '\n')
4835 			used = used[1 .. $];
4836 
4837 		if(used.length == 0)
4838 			throw new Exception("Remote server disconnected or didn't send enough information");
4839 
4840 		if(used.length < 1)
4841 			more();
4842 
4843 		used = used[1 .. $]; // skip the \n
4844 
4845 		if(used.length == 0)
4846 			more();
4847 
4848 		// checks on the protocol from ehaders
4849 		bool isWebsocket;
4850 		bool isUpgrade;
4851 		const(ubyte)[] protocol;
4852 		const(ubyte)[] accept;
4853 
4854 		while(used.length) {
4855 			if(used.length >= 2 && used[0] == '\r' && used[1] == '\n') {
4856 				used = used[2 .. $];
4857 				break; // all done
4858 			}
4859 			int idxColon;
4860 			while(idxColon < used.length && used[idxColon] != ':')
4861 				idxColon++;
4862 			if(idxColon == used.length)
4863 				more();
4864 			auto idxStart = idxColon + 1;
4865 			while(idxStart < used.length && used[idxStart] == ' ')
4866 				idxStart++;
4867 			if(idxStart == used.length)
4868 				more();
4869 			auto idxEnd = idxStart;
4870 			while(idxEnd < used.length && used[idxEnd] != '\r')
4871 				idxEnd++;
4872 			if(idxEnd == used.length)
4873 				more();
4874 
4875 			auto headerName = used[0 .. idxColon];
4876 			auto headerValue = used[idxStart .. idxEnd];
4877 
4878 			// move past this header
4879 			used = used[idxEnd .. $];
4880 			// and the \r\n
4881 			if(2 <= used.length)
4882 				used = used[2 .. $];
4883 
4884 			if(headerName.bicmp("upgrade")) {
4885 				if(headerValue.bicmp("websocket"))
4886 					isWebsocket = true;
4887 			} else if(headerName.bicmp("connection")) {
4888 				if(headerValue.bicmp("upgrade"))
4889 					isUpgrade = true;
4890 			} else if(headerName.bicmp("sec-websocket-accept")) {
4891 				accept = headerValue;
4892 			} else if(headerName.bicmp("sec-websocket-protocol")) {
4893 				protocol = headerValue;
4894 			}
4895 
4896 			if(!used.length) {
4897 				more();
4898 			}
4899 		}
4900 
4901 
4902 		if(!isWebsocket)
4903 			throw new Exception("didn't answer as websocket");
4904 		if(!isUpgrade)
4905 			throw new Exception("didn't answer as upgrade");
4906 
4907 
4908 		// FIXME: check protocol if config requested one
4909 		// FIXME: check accept for the right hash
4910 
4911 		receiveBuffer[0 .. used.length] = used[];
4912 		receiveBufferUsedLength = used.length;
4913 
4914 		readyState_ = OPEN;
4915 
4916 		if(onopen)
4917 			onopen();
4918 
4919 		nextPing = MonoTime.currTime + config.pingFrequency.msecs;
4920 		timeoutFromInactivity = MonoTime.currTime + config.timeoutFromInactivity;
4921 
4922 		registerActiveSocket(this);
4923 	}
4924 
4925 	/++
4926 		Is data pending on the socket? Also check [isMessageBuffered] to see if there
4927 		is already a message in memory too.
4928 
4929 		If this returns `true`, you can call [lowLevelReceive], then try [isMessageBuffered]
4930 		again.
4931 	+/
4932 	/// Group: blocking_api
4933 	public bool isDataPending(Duration timeout = 0.seconds) {
4934 		static SocketSet readSet;
4935 		if(readSet is null)
4936 			readSet = new SocketSet();
4937 
4938 		version(with_openssl)
4939 		if(auto s = cast(SslClientSocket) socket) {
4940 			// select doesn't handle the case with stuff
4941 			// left in the ssl buffer so i'm checking it separately
4942 			if(s.dataPending()) {
4943 				return true;
4944 			}
4945 		}
4946 
4947 		readSet.reset();
4948 
4949 		readSet.add(socket);
4950 
4951 		//tryAgain:
4952 		auto selectGot = Socket.select(readSet, null, null, timeout);
4953 		if(selectGot == 0) { /* timeout */
4954 			// timeout
4955 			return false;
4956 		} else if(selectGot == -1) { /* interrupted */
4957 			return false;
4958 		} else { /* ready */
4959 			if(readSet.isSet(socket)) {
4960 				return true;
4961 			}
4962 		}
4963 
4964 		return false;
4965 	}
4966 
4967 	private void llsend(ubyte[] d) {
4968 		if(readyState == CONNECTING)
4969 			throw new Exception("WebSocket not connected when trying to send. Did you forget to call connect(); ?");
4970 			//connect();
4971 			//import std.stdio; writeln("LLSEND: ", d);
4972 		while(d.length) {
4973 			auto r = socket.send(d);
4974 			if(r < 0 && wouldHaveBlocked()) {
4975 				// FIXME: i should register for a write wakeup
4976 				version(use_arsd_core) assert(0);
4977 				import core.thread;
4978 				Thread.sleep(1.msecs);
4979 				continue;
4980 			}
4981 			//import core.stdc.errno; import std.stdio; writeln(errno);
4982 			if(r <= 0) {
4983 				// import std.stdio; writeln(GetLastError());
4984 				throw new Exception("Socket send failed");
4985 			}
4986 			d = d[r .. $];
4987 		}
4988 	}
4989 
4990 	private void llclose() {
4991 		// import std.stdio; writeln("LLCLOSE");
4992 		socket.shutdown(SocketShutdown.SEND);
4993 	}
4994 
4995 	/++
4996 		Waits for more data off the low-level socket and adds it to the pending buffer.
4997 
4998 		Returns `true` if the connection is still active.
4999 	+/
5000 	/// Group: blocking_api
5001 	public bool lowLevelReceive() {
5002 		if(readyState == CONNECTING)
5003 			throw new Exception("WebSocket not connected when trying to receive. Did you forget to call connect(); ?");
5004 		if (receiveBufferUsedLength == receiveBuffer.length)
5005 		{
5006 			if (receiveBuffer.length == config.maximumReceiveBufferSize)
5007 				throw new Exception("Maximum receive buffer size exhausted");
5008 
5009 			import std.algorithm : min;
5010 			receiveBuffer.length = min(receiveBuffer.length + config.initialReceiveBufferSize,
5011 				config.maximumReceiveBufferSize);
5012 		}
5013 		auto r = socket.receive(receiveBuffer[receiveBufferUsedLength .. $]);
5014 		if(r == 0)
5015 			return false;
5016 		if(r < 0 && wouldHaveBlocked())
5017 			return true;
5018 		if(r <= 0) {
5019 			//import std.stdio; writeln(WSAGetLastError());
5020 			return false;
5021 		}
5022 		receiveBufferUsedLength += r;
5023 		return true;
5024 	}
5025 
5026 	private Socket socket;
5027 
5028 	/* copy/paste section { */
5029 
5030 	private int readyState_;
5031 	private ubyte[] receiveBuffer;
5032 	private size_t receiveBufferUsedLength;
5033 
5034 	private Config config;
5035 
5036 	enum CONNECTING = 0; /// Socket has been created. The connection is not yet open.
5037 	enum OPEN = 1; /// The connection is open and ready to communicate.
5038 	enum CLOSING = 2; /// The connection is in the process of closing.
5039 	enum CLOSED = 3; /// The connection is closed or couldn't be opened.
5040 
5041 	/++
5042 
5043 	+/
5044 	/// Group: foundational
5045 	static struct Config {
5046 		/++
5047 			These control the size of the receive buffer.
5048 
5049 			It starts at the initial size, will temporarily
5050 			balloon up to the maximum size, and will reuse
5051 			a buffer up to the likely size.
5052 
5053 			Anything larger than the maximum size will cause
5054 			the connection to be aborted and an exception thrown.
5055 			This is to protect you against a peer trying to
5056 			exhaust your memory, while keeping the user-level
5057 			processing simple.
5058 		+/
5059 		size_t initialReceiveBufferSize = 4096;
5060 		size_t likelyReceiveBufferSize = 4096; /// ditto
5061 		size_t maximumReceiveBufferSize = 10 * 1024 * 1024; /// ditto
5062 
5063 		/++
5064 			Maximum combined size of a message.
5065 		+/
5066 		size_t maximumMessageSize = 10 * 1024 * 1024;
5067 
5068 		string[string] cookies; /// Cookies to send with the initial request. cookies[name] = value;
5069 		string origin; /// Origin URL to send with the handshake, if desired.
5070 		string protocol; /// the protocol header, if desired.
5071 
5072 		/++
5073 			Additional headers to put in the HTTP request. These should be formatted `Name: value`, like for example:
5074 
5075 			---
5076 			Config config;
5077 			config.additionalHeaders ~= "Authorization: Bearer your_auth_token_here";
5078 			---
5079 
5080 			History:
5081 				Added February 19, 2021 (included in dub version 9.2)
5082 		+/
5083 		string[] additionalHeaders;
5084 
5085 		/++
5086 			Amount of time (in msecs) of idleness after which to send an automatic ping
5087 
5088 			Please note how this interacts with [timeoutFromInactivity] - a ping counts as activity that
5089 			keeps the socket alive.
5090 		+/
5091 		int pingFrequency = 5000;
5092 
5093 		/++
5094 			Amount of time to disconnect when there's no activity. Note that automatic pings will keep the connection alive; this timeout only occurs if there's absolutely nothing, including no responses to websocket ping frames. Since the default [pingFrequency] is only seconds, this one minute should never elapse unless the connection is actually dead.
5095 
5096 			The one thing to keep in mind is if your program is busy and doesn't check input, it might consider this a time out since there's no activity. The reason is that your program was busy rather than a connection failure, but it doesn't care. You should avoid long processing periods anyway though!
5097 
5098 			History:
5099 				Added March 31, 2021 (included in dub version 9.4)
5100 		+/
5101 		Duration timeoutFromInactivity = 1.minutes;
5102 
5103 		/++
5104 			For https connections, if this is `true`, it will fail to connect if the TLS certificate can not be
5105 			verified. Setting this to `false` will skip this check and allow the connection to continue anyway.
5106 
5107 			History:
5108 				Added April 5, 2022 (dub v10.8)
5109 
5110 				Prior to this, it always used the global (but undocumented) `defaultVerifyPeer` setting, and sometimes
5111 				even if it was true, it would skip the verification. Now, it always respects this local setting.
5112 		+/
5113 		bool verifyPeer = true;
5114 	}
5115 
5116 	/++
5117 		Returns one of [CONNECTING], [OPEN], [CLOSING], or [CLOSED].
5118 	+/
5119 	int readyState() {
5120 		return readyState_;
5121 	}
5122 
5123 	/++
5124 		Closes the connection, sending a graceful teardown message to the other side.
5125 		If you provide no arguments, it sends code 1000, normal closure. If you provide
5126 		a code, you should also provide a short reason string.
5127 
5128 		Params:
5129 			code = reason code.
5130 
5131 			0-999 are invalid.
5132 			1000-2999 are defined by the RFC. [https://www.rfc-editor.org/rfc/rfc6455.html#section-7.4.1]
5133 				1000 - normal finish
5134 				1001 - endpoint going away
5135 				1002 - protocol error
5136 				1003 - unacceptable data received (e.g. binary message when you can't handle it)
5137 				1004 - reserved
5138 				1005 - missing status code (should not be set except by implementations)
5139 				1006 - abnormal connection closure (should only be set by implementations)
5140 				1007 - inconsistent data received (i.e. utf-8 decode error in text message)
5141 				1008 - policy violation
5142 				1009 - received message too big
5143 				1010 - client aborting due to required extension being unsupported by the server
5144 				1011 - server had unexpected failure
5145 				1015 - reserved for TLS handshake failure
5146 			3000-3999 are to be registered with IANA.
5147 			4000-4999 are private-use custom codes depending on the application. These are what you'd most commonly set here.
5148 
5149 			reason = <= 123 bytes of human-readable reason text, used for logs and debugging
5150 
5151 		History:
5152 			The default `code` was changed to 1000 on January 9, 2023. Previously it was 0,
5153 			but also ignored anyway.
5154 
5155 			On May 11, 2024, the optional arguments were changed to overloads since if you provide a code, you should also provide a reason.
5156 	+/
5157 	/// Group: foundational
5158 	void close() {
5159 		close(1000, null);
5160 	}
5161 
5162 	/// ditto
5163 	void close(int code, string reason)
5164 		//in (reason.length < 123)
5165 		in { assert(reason.length <= 123); } do
5166 	{
5167 		if(readyState_ != OPEN)
5168 			return; // it cool, we done
5169 		WebSocketFrame wss;
5170 		wss.fin = true;
5171 		wss.masked = this.isClient;
5172 		wss.opcode = WebSocketOpcode.close;
5173 		wss.data = [ubyte((code >> 8) & 0xff), ubyte(code & 0xff)] ~ cast(ubyte[]) reason.dup;
5174 		wss.send(&llsend);
5175 
5176 		readyState_ = CLOSING;
5177 
5178 		closeCalled = true;
5179 
5180 		llclose();
5181 	}
5182 
5183 	deprecated("If you provide a code, please also provide a reason string") void close(int code) {
5184 		close(code, null);
5185 	}
5186 
5187 
5188 	private bool closeCalled;
5189 
5190 	/++
5191 		Sends a ping message to the server. This is done automatically by the library if you set a non-zero [Config.pingFrequency], but you can also send extra pings explicitly as well with this function.
5192 	+/
5193 	/// Group: foundational
5194 	void ping(in ubyte[] data = null) {
5195 		WebSocketFrame wss;
5196 		wss.fin = true;
5197 		wss.masked = this.isClient;
5198 		wss.opcode = WebSocketOpcode.ping;
5199 		if(data !is null) wss.data = data.dup;
5200 		wss.send(&llsend);
5201 	}
5202 
5203 	/++
5204 		Sends a pong message to the server. This is normally done automatically in response to pings.
5205 	+/
5206 	/// Group: foundational
5207 	void pong(in ubyte[] data = null) {
5208 		WebSocketFrame wss;
5209 		wss.fin = true;
5210 		wss.masked = this.isClient;
5211 		wss.opcode = WebSocketOpcode.pong;
5212 		if(data !is null) wss.data = data.dup;
5213 		wss.send(&llsend);
5214 	}
5215 
5216 	/++
5217 		Sends a text message through the websocket.
5218 	+/
5219 	/// Group: foundational
5220 	void send(in char[] textData) {
5221 		WebSocketFrame wss;
5222 		wss.fin = true;
5223 		wss.masked = this.isClient;
5224 		wss.opcode = WebSocketOpcode.text;
5225 		wss.data = cast(ubyte[]) textData.dup;
5226 		wss.send(&llsend);
5227 	}
5228 
5229 	/++
5230 		Sends a binary message through the websocket.
5231 	+/
5232 	/// Group: foundational
5233 	void send(in ubyte[] binaryData) {
5234 		WebSocketFrame wss;
5235 		wss.masked = this.isClient;
5236 		wss.fin = true;
5237 		wss.opcode = WebSocketOpcode.binary;
5238 		wss.data = cast(ubyte[]) binaryData.dup;
5239 		wss.send(&llsend);
5240 	}
5241 
5242 	/++
5243 		Waits for and returns the next complete message on the socket.
5244 
5245 		Note that the onmessage function is still called, right before
5246 		this returns.
5247 	+/
5248 	/// Group: blocking_api
5249 	public WebSocketFrame waitForNextMessage() {
5250 		do {
5251 			auto m = processOnce();
5252 			if(m.populated)
5253 				return m;
5254 		} while(lowLevelReceive());
5255 
5256 		return WebSocketFrame.init; // FIXME? maybe.
5257 	}
5258 
5259 	/++
5260 		Tells if [waitForNextMessage] would block.
5261 	+/
5262 	/// Group: blocking_api
5263 	public bool waitForNextMessageWouldBlock() {
5264 		checkAgain:
5265 		if(isMessageBuffered())
5266 			return false;
5267 		if(!isDataPending())
5268 			return true;
5269 		while(isDataPending())
5270 			if(lowLevelReceive() == false)
5271 				return false;
5272 		goto checkAgain;
5273 	}
5274 
5275 	/++
5276 		Is there a message in the buffer already?
5277 		If `true`, [waitForNextMessage] is guaranteed to return immediately.
5278 		If `false`, check [isDataPending] as the next step.
5279 	+/
5280 	/// Group: blocking_api
5281 	public bool isMessageBuffered() {
5282 		ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength];
5283 		auto s = d;
5284 		if(d.length) {
5285 			auto orig = d;
5286 			auto m = WebSocketFrame.read(d);
5287 			// that's how it indicates that it needs more data
5288 			if(d !is orig)
5289 				return true;
5290 		}
5291 
5292 		return false;
5293 	}
5294 
5295 	private ubyte continuingType;
5296 	private ubyte[] continuingData;
5297 	//private size_t continuingDataLength;
5298 
5299 	private WebSocketFrame processOnce() {
5300 		ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength];
5301 		auto s = d;
5302 		// FIXME: handle continuation frames more efficiently. it should really just reuse the receive buffer.
5303 		WebSocketFrame m;
5304 		if(d.length) {
5305 			auto orig = d;
5306 			m = WebSocketFrame.read(d);
5307 			// that's how it indicates that it needs more data
5308 			if(d is orig)
5309 				return WebSocketFrame.init;
5310 			m.unmaskInPlace();
5311 			switch(m.opcode) {
5312 				case WebSocketOpcode.continuation:
5313 					if(continuingData.length + m.data.length > config.maximumMessageSize)
5314 						throw new Exception("message size exceeded");
5315 
5316 					continuingData ~= m.data;
5317 					if(m.fin) {
5318 						if(ontextmessage)
5319 							ontextmessage(cast(char[]) continuingData);
5320 						if(onbinarymessage)
5321 							onbinarymessage(continuingData);
5322 
5323 						continuingData = null;
5324 					}
5325 				break;
5326 				case WebSocketOpcode.text:
5327 					if(m.fin) {
5328 						if(ontextmessage)
5329 							ontextmessage(m.textData);
5330 					} else {
5331 						continuingType = m.opcode;
5332 						//continuingDataLength = 0;
5333 						continuingData = null;
5334 						continuingData ~= m.data;
5335 					}
5336 				break;
5337 				case WebSocketOpcode.binary:
5338 					if(m.fin) {
5339 						if(onbinarymessage)
5340 							onbinarymessage(m.data);
5341 					} else {
5342 						continuingType = m.opcode;
5343 						//continuingDataLength = 0;
5344 						continuingData = null;
5345 						continuingData ~= m.data;
5346 					}
5347 				break;
5348 				case WebSocketOpcode.close:
5349 
5350 					//import std.stdio; writeln("closed ", cast(string) m.data);
5351 
5352 					ushort code = CloseEvent.StandardCloseCodes.noStatusCodePresent;
5353 					const(char)[] reason;
5354 
5355 					if(m.data.length >= 2) {
5356 						code = (m.data[0] << 8) | m.data[1];
5357 						reason = (cast(char[]) m.data[2 .. $]);
5358 					}
5359 
5360 					if(onclose)
5361 						onclose(CloseEvent(code, reason, true));
5362 
5363 					// if we receive one and haven't sent one back we're supposed to echo it back and close.
5364 					if(!closeCalled)
5365 						close(code, reason.idup);
5366 
5367 					readyState_ = CLOSED;
5368 
5369 					unregisterActiveSocket(this);
5370 					socket.close();
5371 				break;
5372 				case WebSocketOpcode.ping:
5373 					// import std.stdio; writeln("ping received ", m.data);
5374 					pong(m.data);
5375 				break;
5376 				case WebSocketOpcode.pong:
5377 					// import std.stdio; writeln("pong received ", m.data);
5378 					// just really references it is still alive, nbd.
5379 				break;
5380 				default: // ignore though i could and perhaps should throw too
5381 			}
5382 		}
5383 
5384 		if(d.length) {
5385 			m.data = m.data.dup();
5386 		}
5387 
5388 		import core.stdc.string;
5389 		memmove(receiveBuffer.ptr, d.ptr, d.length);
5390 		receiveBufferUsedLength = d.length;
5391 
5392 		return m;
5393 	}
5394 
5395 	private void autoprocess() {
5396 		// FIXME
5397 		do {
5398 			processOnce();
5399 		} while(lowLevelReceive());
5400 	}
5401 
5402 	/++
5403 		Arguments for the close event. The `code` and `reason` are provided from the close message on the websocket, if they are present. The spec says code 1000 indicates a normal, default reason close, but reserves the code range from 3000-5000 for future definition; the 3000s can be registered with IANA and the 4000's are application private use. The `reason` should be user readable, but not displayed to the end user. `wasClean` is true if the server actually sent a close event, false if it just disconnected.
5404 
5405 		$(PITFALL
5406 			The `reason` argument references a temporary buffer and there's no guarantee it will remain valid once your callback returns. It may be freed and will very likely be overwritten. If you want to keep the reason beyond the callback, make sure you `.idup` it.
5407 		)
5408 
5409 		History:
5410 			Added March 19, 2023 (dub v11.0).
5411 	+/
5412 	static struct CloseEvent {
5413 		ushort code;
5414 		const(char)[] reason;
5415 		bool wasClean;
5416 
5417 		string extendedErrorInformationUnstable;
5418 
5419 		/++
5420 			See https://www.rfc-editor.org/rfc/rfc6455#section-7.4.1 for details.
5421 		+/
5422 		enum StandardCloseCodes {
5423 			purposeFulfilled = 1000,
5424 			goingAway = 1001,
5425 			protocolError = 1002,
5426 			unacceptableData = 1003, // e.g. got text message when you can only handle binary
5427 			Reserved = 1004,
5428 			noStatusCodePresent = 1005, // not set by endpoint.
5429 			abnormalClosure = 1006, // not set by endpoint. closed without a Close control. FIXME: maybe keep a copy of errno around for these
5430 			inconsistentData = 1007, // e.g. utf8 validation failed
5431 			genericPolicyViolation = 1008,
5432 			messageTooBig = 1009,
5433 			clientRequiredExtensionMissing = 1010, // only the client should send this
5434 			unnexpectedCondition = 1011,
5435 			unverifiedCertificate = 1015, // not set by client
5436 		}
5437 
5438 		string toString() {
5439 			return cast(string) (arsd.core.toStringInternal(code) ~ ": " ~ reason);
5440 		}
5441 	}
5442 
5443 	/++
5444 		The `CloseEvent` you get references a temporary buffer that may be overwritten after your handler returns. If you want to keep it or the `event.reason` member, remember to `.idup` it.
5445 
5446 		History:
5447 			The `CloseEvent` was changed to a [arsd.core.FlexibleDelegate] on March 19, 2023 (dub v11.0). Before that, `onclose` was a public member of type `void delegate()`. This change means setters still work with or without the [CloseEvent] argument.
5448 
5449 			Your onclose method is now also called on abnormal terminations. Check the `wasClean` member of the `CloseEvent` to know if it came from a close frame or other cause.
5450 	+/
5451 	arsd.core.FlexibleDelegate!(void delegate(CloseEvent event)) onclose;
5452 	void delegate() onerror; ///
5453 	void delegate(in char[]) ontextmessage; ///
5454 	void delegate(in ubyte[]) onbinarymessage; ///
5455 	void delegate() onopen; ///
5456 
5457 	/++
5458 
5459 	+/
5460 	/// Group: browser_api
5461 	void onmessage(void delegate(in char[]) dg) {
5462 		ontextmessage = dg;
5463 	}
5464 
5465 	/// ditto
5466 	void onmessage(void delegate(in ubyte[]) dg) {
5467 		onbinarymessage = dg;
5468 	}
5469 
5470 	/* } end copy/paste */
5471 
5472 	// returns true if still active
5473 	private static bool readyToRead(WebSocket sock) {
5474 		sock.timeoutFromInactivity = MonoTime.currTime + sock.config.timeoutFromInactivity;
5475 		if(!sock.lowLevelReceive()) {
5476 			sock.readyState_ = CLOSED;
5477 
5478 			if(sock.onerror)
5479 				sock.onerror();
5480 
5481 			if(sock.onclose)
5482 				sock.onclose(CloseEvent(CloseEvent.StandardCloseCodes.abnormalClosure, "Connection lost", false, lastSocketError(sock.socket)));
5483 
5484 			unregisterActiveSocket(sock);
5485 			sock.socket.close();
5486 			return false;
5487 		}
5488 		while(sock.processOnce().populated) {}
5489 		return true;
5490 	}
5491 
5492 	// returns true if still active, false if not
5493 	private static bool timeoutAndPingCheck(WebSocket sock, MonoTime now, Duration* minimumTimeoutForSelect) {
5494 		auto diff = sock.timeoutFromInactivity - now;
5495 		if(diff <= 0.msecs) {
5496 			// it timed out
5497 			if(sock.onerror)
5498 				sock.onerror();
5499 
5500 			if(sock.onclose)
5501 				sock.onclose(CloseEvent(CloseEvent.StandardCloseCodes.abnormalClosure, "Connection timed out", false, null));
5502 
5503 			sock.readyState_ = CLOSED;
5504 			unregisterActiveSocket(sock);
5505 			sock.socket.close();
5506 			return false;
5507 		}
5508 
5509 		if(minimumTimeoutForSelect && diff < *minimumTimeoutForSelect)
5510 			*minimumTimeoutForSelect = diff;
5511 
5512 		diff = sock.nextPing - now;
5513 
5514 		if(diff <= 0.msecs) {
5515 			//sock.send(`{"action": "ping"}`);
5516 			sock.ping();
5517 			sock.nextPing = now + sock.config.pingFrequency.msecs;
5518 		} else {
5519 			if(minimumTimeoutForSelect && diff < *minimumTimeoutForSelect)
5520 				*minimumTimeoutForSelect = diff;
5521 		}
5522 
5523 		return true;
5524 	}
5525 
5526 	/*
5527 	const int bufferedAmount // amount pending
5528 	const string extensions
5529 
5530 	const string protocol
5531 	const string url
5532 	*/
5533 
5534 	static {
5535 		/++
5536 			Runs an event loop with all known websockets on this thread until all websockets
5537 			are closed or unregistered, or until you call [exitEventLoop], or set `*localLoopExited`
5538 			to false (please note it may take a few seconds until it checks that flag again; it may
5539 			not exit immediately).
5540 
5541 			History:
5542 				The `localLoopExited` parameter was added August 22, 2022 (dub v10.9)
5543 
5544 			See_Also:
5545 				[addToSimpledisplayEventLoop]
5546 		+/
5547 		void eventLoop(shared(bool)* localLoopExited = null) {
5548 			import core.atomic;
5549 			atomicOp!"+="(numberOfEventLoops, 1);
5550 			scope(exit) {
5551 				if(atomicOp!"-="(numberOfEventLoops, 1) <= 0)
5552 					loopExited = false; // reset it so we can reenter
5553 			}
5554 
5555 			version(use_arsd_core) {
5556 				loopExited = false;
5557 
5558 				import arsd.core;
5559 				getThisThreadEventLoop().run(() => WebSocket.activeSockets.length == 0 || loopExited || (localLoopExited !is null && *localLoopExited == true));
5560 			} else {
5561 				static SocketSet readSet;
5562 
5563 				if(readSet is null)
5564 					readSet = new SocketSet();
5565 
5566 				loopExited = false;
5567 
5568 				outermost: while(!loopExited && (localLoopExited is null || (*localLoopExited == false))) {
5569 					readSet.reset();
5570 
5571 					Duration timeout = 3.seconds;
5572 
5573 					auto now = MonoTime.currTime;
5574 					bool hadAny;
5575 					foreach(sock; activeSockets) {
5576 						if(!timeoutAndPingCheck(sock, now, &timeout))
5577 							continue outermost;
5578 
5579 						readSet.add(sock.socket);
5580 						hadAny = true;
5581 					}
5582 
5583 					if(!hadAny) {
5584 						// import std.stdio; writeln("had none");
5585 						return;
5586 					}
5587 
5588 					tryAgain:
5589 						// import std.stdio; writeln(timeout);
5590 					auto selectGot = Socket.select(readSet, null, null, timeout);
5591 					if(selectGot == 0) { /* timeout */
5592 						// timeout
5593 						continue; // it will be handled at the top of the loop
5594 					} else if(selectGot == -1) { /* interrupted */
5595 						goto tryAgain;
5596 					} else {
5597 						foreach(sock; activeSockets) {
5598 							if(readSet.isSet(sock.socket)) {
5599 								if(!readyToRead(sock))
5600 									continue outermost;
5601 								selectGot--;
5602 								if(selectGot <= 0)
5603 									break;
5604 							}
5605 						}
5606 					}
5607 				}
5608 			}
5609 		}
5610 
5611 		private static shared(int) numberOfEventLoops;
5612 
5613 		private __gshared bool loopExited;
5614 		/++
5615 			Exits all running [WebSocket.eventLoop]s next time they loop around. You can call this from a signal handler or another thread.
5616 
5617 			Please note they may not loop around to check the flag for several seconds. Any new event loops will exit immediately until
5618 			all current ones are closed. Once all event loops are exited, the flag is cleared and you can start the loop again.
5619 
5620 			This function is likely to be deprecated in the future due to its quirks and imprecise name.
5621 		+/
5622 		void exitEventLoop() {
5623 			loopExited = true;
5624 		}
5625 
5626 		WebSocket[] activeSockets;
5627 
5628 		void registerActiveSocket(WebSocket s) {
5629 			// ensure it isn't already there...
5630 			assert(s !is null);
5631 			if(s.registered)
5632 				return;
5633 			s.activeSocketArrayIndex = activeSockets.length;
5634 			activeSockets ~= s;
5635 			s.registered = true;
5636 			version(use_arsd_core) {
5637 				version(Posix)
5638 				s.unregisterToken = arsd.core.getThisThreadEventLoop().addCallbackOnFdReadable(s.socket.handle, new arsd.core.CallbackHelper(() { s.readyToRead(s); }));
5639 			}
5640 		}
5641 		void unregisterActiveSocket(WebSocket s) {
5642 			version(use_arsd_core) {
5643 				s.unregisterToken.unregister();
5644 			}
5645 
5646 			auto i = s.activeSocketArrayIndex;
5647 			assert(activeSockets[i] is s);
5648 
5649 			activeSockets[i] = activeSockets[$-1];
5650 			activeSockets[i].activeSocketArrayIndex = i;
5651 			activeSockets = activeSockets[0 .. $-1];
5652 			activeSockets.assumeSafeAppend();
5653 			s.registered = false;
5654 		}
5655 	}
5656 
5657 	private bool registered;
5658 	private size_t activeSocketArrayIndex;
5659 	version(use_arsd_core) {
5660 		static import arsd.core;
5661 		arsd.core.ICoreEventLoop.UnregisterToken unregisterToken;
5662 	}
5663 }
5664 
5665 private template imported(string mod) {
5666 	mixin(`import imported = ` ~ mod ~ `;`);
5667 }
5668 
5669 /++
5670 	Warning: you should call this AFTER websocket.connect or else it might throw on connect because the function sets nonblocking mode and the connect function doesn't handle that well (it throws on the "would block" condition in that function. easier to just do that first)
5671 +/
5672 template addToSimpledisplayEventLoop() {
5673 	import arsd.simpledisplay;
5674 	void addToSimpledisplayEventLoop(WebSocket ws, imported!"arsd.simpledisplay".SimpleWindow window) {
5675 		version(use_arsd_core)
5676 			return; // already done implicitly
5677 
5678 		version(Windows)
5679 		auto event = WSACreateEvent();
5680 		// FIXME: supposed to close event too
5681 
5682 		void midprocess() {
5683 			version(Windows)
5684 				ResetEvent(event);
5685 			if(!ws.lowLevelReceive()) {
5686 				ws.readyState_ = WebSocket.CLOSED;
5687 				WebSocket.unregisterActiveSocket(ws);
5688 				ws.socket.close();
5689 				return;
5690 			}
5691 			while(ws.processOnce().populated) {}
5692 		}
5693 
5694 		version(Posix) {
5695 			auto reader = new PosixFdReader(&midprocess, ws.socket.handle);
5696 		} else version(none) {
5697 			if(WSAAsyncSelect(ws.socket.handle, window.hwnd, WM_USER + 150, FD_CLOSE | FD_READ))
5698 				throw new Exception("WSAAsyncSelect");
5699 
5700                         window.handleNativeEvent = delegate int(HWND hwnd, UINT msg, WPARAM wParam, LPARAM lParam) {
5701                                 if(hwnd !is window.impl.hwnd)
5702                                         return 1; // we don't care...
5703                                 switch(msg) {
5704                                         case WM_USER + 150: // socket activity
5705                                                 switch(LOWORD(lParam)) {
5706                                                         case FD_READ:
5707                                                         case FD_CLOSE:
5708 								midprocess();
5709                                                         break;
5710                                                         default:
5711                                                                 // nothing
5712                                                 }
5713                                         break;
5714                                         default: return 1; // not handled, pass it on
5715                                 }
5716                                 return 0;
5717                         };
5718 
5719 		} else version(Windows) {
5720 			ws.socket.blocking = false; // the WSAEventSelect does this anyway and doing it here lets phobos know about it.
5721 			//CreateEvent(null, 0, 0, null);
5722 			if(!event) {
5723 				throw new Exception("WSACreateEvent");
5724 			}
5725 			if(WSAEventSelect(ws.socket.handle, event, 1/*FD_READ*/ | (1<<5)/*FD_CLOSE*/)) {
5726 				//import std.stdio; writeln(WSAGetLastError());
5727 				throw new Exception("WSAEventSelect");
5728 			}
5729 
5730 			auto handle = new WindowsHandleReader(&midprocess, event);
5731 
5732 			/+
5733 			static class Ready {}
5734 
5735 			Ready thisr = new Ready;
5736 
5737 			justCommunication.addEventListener((Ready r) {
5738 				if(r is thisr)
5739 					midprocess();
5740 			});
5741 
5742 			import core.thread;
5743 			auto thread = new Thread({
5744 				while(true) {
5745 					WSAWaitForMultipleEvents(1, &event, true, -1/*WSA_INFINITE*/, false);
5746 					justCommunication.postEvent(thisr);
5747 				}
5748 			});
5749 			thread.isDaemon = true;
5750 			thread.start;
5751 			+/
5752 
5753 		} else static assert(0, "unsupported OS");
5754 	}
5755 }
5756 
5757 version(Windows) {
5758         import core.sys.windows.windows;
5759         import core.sys.windows.winsock2;
5760 }
5761 
5762 version(none) {
5763         extern(Windows) int WSAAsyncSelect(SOCKET, HWND, uint, int);
5764         enum int FD_CLOSE = 1 << 5;
5765         enum int FD_READ = 1 << 0;
5766         enum int WM_USER = 1024;
5767 }
5768 
5769 version(Windows) {
5770 	import core.stdc.config;
5771 	extern(Windows)
5772 	int WSAEventSelect(SOCKET, HANDLE /* to an Event */, c_long);
5773 
5774 	extern(Windows)
5775 	HANDLE WSACreateEvent();
5776 
5777 	extern(Windows)
5778 	DWORD WSAWaitForMultipleEvents(DWORD, HANDLE*, BOOL, DWORD, BOOL);
5779 }
5780 
5781 /* copy/paste from cgi.d */
5782 public {
5783 	enum WebSocketOpcode : ubyte {
5784 		continuation = 0,
5785 		text = 1,
5786 		binary = 2,
5787 		// 3, 4, 5, 6, 7 RESERVED
5788 		close = 8,
5789 		ping = 9,
5790 		pong = 10,
5791 		// 11,12,13,14,15 RESERVED
5792 	}
5793 
5794 	public struct WebSocketFrame {
5795 		private bool populated;
5796 		bool fin;
5797 		bool rsv1;
5798 		bool rsv2;
5799 		bool rsv3;
5800 		WebSocketOpcode opcode; // 4 bits
5801 		bool masked;
5802 		ubyte lengthIndicator; // don't set this when building one to send
5803 		ulong realLength; // don't use when sending
5804 		ubyte[4] maskingKey; // don't set this when sending
5805 		ubyte[] data;
5806 
5807 		static WebSocketFrame simpleMessage(WebSocketOpcode opcode, in void[] data) {
5808 			WebSocketFrame msg;
5809 			msg.fin = true;
5810 			msg.opcode = opcode;
5811 			msg.data = cast(ubyte[]) data.dup; // it is mutated below when masked, so need to be cautious and copy it, sigh
5812 
5813 			return msg;
5814 		}
5815 
5816 		private void send(scope void delegate(ubyte[]) llsend) {
5817 			ubyte[64] headerScratch;
5818 			int headerScratchPos = 0;
5819 
5820 			realLength = data.length;
5821 
5822 			{
5823 				ubyte b1;
5824 				b1 |= cast(ubyte) opcode;
5825 				b1 |= rsv3 ? (1 << 4) : 0;
5826 				b1 |= rsv2 ? (1 << 5) : 0;
5827 				b1 |= rsv1 ? (1 << 6) : 0;
5828 				b1 |= fin  ? (1 << 7) : 0;
5829 
5830 				headerScratch[0] = b1;
5831 				headerScratchPos++;
5832 			}
5833 
5834 			{
5835 				headerScratchPos++; // we'll set header[1] at the end of this
5836 				auto rlc = realLength;
5837 				ubyte b2;
5838 				b2 |= masked ? (1 << 7) : 0;
5839 
5840 				assert(headerScratchPos == 2);
5841 
5842 				if(realLength > 65535) {
5843 					// use 64 bit length
5844 					b2 |= 0x7f;
5845 
5846 					// FIXME: double check endinaness
5847 					foreach(i; 0 .. 8) {
5848 						headerScratch[2 + 7 - i] = rlc & 0x0ff;
5849 						rlc >>>= 8;
5850 					}
5851 
5852 					headerScratchPos += 8;
5853 				} else if(realLength > 125) {
5854 					// use 16 bit length
5855 					b2 |= 0x7e;
5856 
5857 					// FIXME: double check endinaness
5858 					foreach(i; 0 .. 2) {
5859 						headerScratch[2 + 1 - i] = rlc & 0x0ff;
5860 						rlc >>>= 8;
5861 					}
5862 
5863 					headerScratchPos += 2;
5864 				} else {
5865 					// use 7 bit length
5866 					b2 |= realLength & 0b_0111_1111;
5867 				}
5868 
5869 				headerScratch[1] = b2;
5870 			}
5871 
5872 			//assert(!masked, "masking key not properly implemented");
5873 			if(masked) {
5874 				import std.random;
5875 				foreach(ref item; maskingKey)
5876 					item = uniform(ubyte.min, ubyte.max);
5877 				headerScratch[headerScratchPos .. headerScratchPos + 4] = maskingKey[];
5878 				headerScratchPos += 4;
5879 
5880 				// we'll just mask it in place...
5881 				int keyIdx = 0;
5882 				foreach(i; 0 .. data.length) {
5883 					data[i] = data[i] ^ maskingKey[keyIdx];
5884 					if(keyIdx == 3)
5885 						keyIdx = 0;
5886 					else
5887 						keyIdx++;
5888 				}
5889 			}
5890 
5891 			//writeln("SENDING ", headerScratch[0 .. headerScratchPos], data);
5892 			llsend(headerScratch[0 .. headerScratchPos]);
5893 			if(data.length)
5894 				llsend(data);
5895 		}
5896 
5897 		static WebSocketFrame read(ref ubyte[] d) {
5898 			WebSocketFrame msg;
5899 
5900 			auto orig = d;
5901 
5902 			WebSocketFrame needsMoreData() {
5903 				d = orig;
5904 				return WebSocketFrame.init;
5905 			}
5906 
5907 			if(d.length < 2)
5908 				return needsMoreData();
5909 
5910 			ubyte b = d[0];
5911 
5912 			msg.populated = true;
5913 
5914 			msg.opcode = cast(WebSocketOpcode) (b & 0x0f);
5915 			b >>= 4;
5916 			msg.rsv3 = b & 0x01;
5917 			b >>= 1;
5918 			msg.rsv2 = b & 0x01;
5919 			b >>= 1;
5920 			msg.rsv1 = b & 0x01;
5921 			b >>= 1;
5922 			msg.fin = b & 0x01;
5923 
5924 			b = d[1];
5925 			msg.masked = (b & 0b1000_0000) ? true : false;
5926 			msg.lengthIndicator = b & 0b0111_1111;
5927 
5928 			d = d[2 .. $];
5929 
5930 			if(msg.lengthIndicator == 0x7e) {
5931 				// 16 bit length
5932 				msg.realLength = 0;
5933 
5934 				if(d.length < 2) return needsMoreData();
5935 
5936 				foreach(i; 0 .. 2) {
5937 					msg.realLength |= d[0] << ((1-i) * 8);
5938 					d = d[1 .. $];
5939 				}
5940 			} else if(msg.lengthIndicator == 0x7f) {
5941 				// 64 bit length
5942 				msg.realLength = 0;
5943 
5944 				if(d.length < 8) return needsMoreData();
5945 
5946 				foreach(i; 0 .. 8) {
5947 					msg.realLength |= ulong(d[0]) << ((7-i) * 8);
5948 					d = d[1 .. $];
5949 				}
5950 			} else {
5951 				// 7 bit length
5952 				msg.realLength = msg.lengthIndicator;
5953 			}
5954 
5955 			if(msg.masked) {
5956 
5957 				if(d.length < 4) return needsMoreData();
5958 
5959 				msg.maskingKey = d[0 .. 4];
5960 				d = d[4 .. $];
5961 			}
5962 
5963 			if(msg.realLength > d.length) {
5964 				return needsMoreData();
5965 			}
5966 
5967 			msg.data = d[0 .. cast(size_t) msg.realLength];
5968 			d = d[cast(size_t) msg.realLength .. $];
5969 
5970 			return msg;
5971 		}
5972 
5973 		void unmaskInPlace() {
5974 			if(this.masked) {
5975 				int keyIdx = 0;
5976 				foreach(i; 0 .. this.data.length) {
5977 					this.data[i] = this.data[i] ^ this.maskingKey[keyIdx];
5978 					if(keyIdx == 3)
5979 						keyIdx = 0;
5980 					else
5981 						keyIdx++;
5982 				}
5983 			}
5984 		}
5985 
5986 		char[] textData() {
5987 			return cast(char[]) data;
5988 		}
5989 	}
5990 }
5991 
5992 private extern(C)
5993 int verifyCertificateFromRegistryArsdHttp(int preverify_ok, X509_STORE_CTX* ctx) {
5994 	version(Windows) {
5995 		if(preverify_ok)
5996 			return 1;
5997 
5998 		auto err_cert = OpenSSL.X509_STORE_CTX_get_current_cert(ctx);
5999 		auto err = OpenSSL.X509_STORE_CTX_get_error(ctx);
6000 
6001 		if(err == 62)
6002 			return 0; // hostname mismatch is an error we can trust; that means OpenSSL already found the certificate and rejected it
6003 
6004 		auto len = OpenSSL.i2d_X509(err_cert, null);
6005 		if(len == -1)
6006 			return 0;
6007 		ubyte[] buffer = new ubyte[](len);
6008 		auto ptr = buffer.ptr;
6009 		len = OpenSSL.i2d_X509(err_cert, &ptr);
6010 		if(len != buffer.length)
6011 			return 0;
6012 
6013 
6014 		CERT_CHAIN_PARA thing;
6015 		thing.cbSize = thing.sizeof;
6016 		auto context = CertCreateCertificateContext(X509_ASN_ENCODING, buffer.ptr, cast(int) buffer.length);
6017 		if(context is null)
6018 			return 0;
6019 		scope(exit) CertFreeCertificateContext(context);
6020 
6021 		PCCERT_CHAIN_CONTEXT chain;
6022 		if(CertGetCertificateChain(null, context, null, null, &thing, 0, null, &chain)) {
6023 			scope(exit)
6024 				CertFreeCertificateChain(chain);
6025 
6026 			DWORD errorStatus = chain.TrustStatus.dwErrorStatus;
6027 
6028 			if(errorStatus == 0)
6029 				return 1; // Windows approved it, OK carry on
6030 			// otherwise, sustain OpenSSL's original ruling
6031 		}
6032 
6033 		return 0;
6034 	} else {
6035 		return preverify_ok;
6036 	}
6037 }
6038 
6039 
6040 version(Windows) {
6041 	pragma(lib, "crypt32");
6042 	import core.sys.windows.wincrypt;
6043 	extern(Windows) {
6044 		PCCERT_CONTEXT CertEnumCertificatesInStore(HCERTSTORE hCertStore, PCCERT_CONTEXT pPrevCertContext);
6045 		// BOOL CertGetCertificateChain(HCERTCHAINENGINE hChainEngine, PCCERT_CONTEXT pCertContext, LPFILETIME pTime, HCERTSTORE hAdditionalStore, PCERT_CHAIN_PARA pChainPara, DWORD dwFlags, LPVOID pvReserved, PCCERT_CHAIN_CONTEXT *ppChainContext);
6046 		PCCERT_CONTEXT CertCreateCertificateContext(DWORD dwCertEncodingType, const BYTE *pbCertEncoded, DWORD cbCertEncoded);
6047 	}
6048 
6049 	void loadCertificatesFromRegistry(SSL_CTX* ctx) {
6050 		auto store = CertOpenSystemStore(0, "ROOT");
6051 		if(store is null) {
6052 			// import std.stdio; writeln("failed");
6053 			return;
6054 		}
6055 		scope(exit)
6056 			CertCloseStore(store, 0);
6057 
6058 		X509_STORE* ssl_store = OpenSSL.SSL_CTX_get_cert_store(ctx);
6059 		PCCERT_CONTEXT c;
6060 		while((c = CertEnumCertificatesInStore(store, c)) !is null) {
6061 			FILETIME na = c.pCertInfo.NotAfter;
6062 			SYSTEMTIME st;
6063 			FileTimeToSystemTime(&na, &st);
6064 
6065 			/+
6066 			_CRYPTOAPI_BLOB i = cast() c.pCertInfo.Issuer;
6067 
6068 			char[256] buffer;
6069 			auto p = CertNameToStrA(X509_ASN_ENCODING, &i, CERT_SIMPLE_NAME_STR, buffer.ptr, cast(int) buffer.length);
6070 			import std.stdio; writeln(buffer[0 .. p]);
6071 			+/
6072 
6073 			if(st.wYear <= 2021) {
6074 				// see: https://www.openssl.org/blog/blog/2021/09/13/LetsEncryptRootCertExpire/
6075 				continue; // no point keeping an expired root cert and it can break Let's Encrypt anyway
6076 			}
6077 
6078 			const(ubyte)* thing = c.pbCertEncoded;
6079 			auto x509 = OpenSSL.d2i_X509(null, &thing, c.cbCertEncoded);
6080 			if (x509) {
6081 				auto success = OpenSSL.X509_STORE_add_cert(ssl_store, x509);
6082 				//if(!success)
6083 					//writeln("FAILED HERE");
6084 				OpenSSL.X509_free(x509);
6085 			} else {
6086 				//writeln("FAILED");
6087 			}
6088 		}
6089 
6090 		CertFreeCertificateContext(c);
6091 
6092 		// import core.stdc.stdio; printf("%s\n", OpenSSL.OpenSSL_version(0));
6093 	}
6094 
6095 
6096 	// because i use the FILE* in PEM_read_X509 and friends
6097 	// gotta use this to bridge the MS C runtime functions
6098 	// might be able to just change those to only use the BIO versions
6099 	// instead
6100 
6101 	// only on MS C runtime
6102 	version(CRuntime_Microsoft) {} else version=no_openssl_applink;
6103 
6104 	version(no_openssl_applink) {} else {
6105 		private extern(C) {
6106 			void _open();
6107 			void _read();
6108 			void _write();
6109 			void _lseek();
6110 			void _close();
6111 			int _fileno(FILE*);
6112 			int _setmode(int, int);
6113 		}
6114 	export extern(C) void** OPENSSL_Applink() {
6115 		import core.stdc.stdio;
6116 
6117 		static extern(C) void* app_stdin() { return cast(void*) stdin; }
6118 		static extern(C) void* app_stdout() { return cast(void*) stdout; }
6119 		static extern(C) void* app_stderr() { return cast(void*) stderr; }
6120 		static extern(C) int app_feof(FILE* fp) { return feof(fp); }
6121 		static extern(C) int app_ferror(FILE* fp) { return ferror(fp); }
6122 		static extern(C) void app_clearerr(FILE* fp) { return clearerr(fp); }
6123 		static extern(C) int app_fileno(FILE* fp) { return _fileno(fp); }
6124 		static extern(C) int app_fsetmod(FILE* fp, char mod) {
6125 			return _setmode(_fileno(fp), mod == 'b' ? _O_BINARY : _O_TEXT);
6126 		}
6127 
6128 		static immutable void*[] table = [
6129 			cast(void*) 22, // applink max
6130 
6131 			&app_stdin,
6132 			&app_stdout,
6133 			&app_stderr,
6134 			&fprintf,
6135 			&fgets,
6136 			&fread,
6137 			&fwrite,
6138 			&app_fsetmod,
6139 			&app_feof,
6140 			&fclose,
6141 
6142 			&fopen,
6143 			&fseek,
6144 			&ftell,
6145 			&fflush,
6146 			&app_ferror,
6147 			&app_clearerr,
6148 			&app_fileno,
6149 
6150 			&_open,
6151 			&_read,
6152 			&_write,
6153 			&_lseek,
6154 			&_close,
6155 		];
6156 		static assert(table.length == 23);
6157 
6158 		return cast(void**) table.ptr;
6159 	}
6160 	}
6161 }
6162 
6163 unittest {
6164 	auto client = new HttpClient();
6165 	auto response = client.navigateTo(Uri("data:,Hello%2C%20World%21")).waitForCompletion();
6166 	assert(response.contentTypeMimeType == "text/plain", response.contentType);
6167 	assert(response.contentText == "Hello, World!", response.contentText);
6168 
6169 	response = client.navigateTo(Uri("data:text/plain;base64,SGVsbG8sIFdvcmxkIQ==")).waitForCompletion();
6170 	assert(response.contentTypeMimeType == "text/plain", response.contentType);
6171 	assert(response.contentText == "Hello, World!", response.contentText);
6172 
6173 	response = client.navigateTo(Uri("data:text/html,%3Ch1%3EHello%2C%20World%21%3C%2Fh1%3E")).waitForCompletion();
6174 	assert(response.contentTypeMimeType == "text/html", response.contentType);
6175 	assert(response.contentText == "<h1>Hello, World!</h1>", response.contentText);
6176 }
6177 
6178 version(arsd_http2_unittests)
6179 unittest {
6180 	import core.thread;
6181 
6182 	static void server() {
6183 		import std.socket;
6184 		auto socket = new TcpSocket();
6185 		socket.setOption(SocketOptionLevel.SOCKET, SocketOption.REUSEADDR, true);
6186 		socket.bind(new InternetAddress(12346));
6187 		socket.listen(1);
6188 		auto s = socket.accept();
6189 		socket.close();
6190 
6191 		ubyte[1024] thing;
6192 		auto g = s.receive(thing[]);
6193 
6194 		/+
6195 		string response = "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 9\r\n\r\nHello!!??";
6196 		auto packetSize = 2;
6197 		+/
6198 
6199 		auto packetSize = 1;
6200 		string response = "HTTP/1.1 200 OK\r\nConnection: close\r\nTransfer-Encoding: chunked\r\n\r\n6\r\nHello!\r\n0\r\n\r\n";
6201 
6202 		while(response.length) {
6203 			s.send(response[0 .. packetSize]);
6204 			response = response[packetSize .. $];
6205 			//import std.stdio; writeln(response);
6206 		}
6207 
6208 		s.close();
6209 	}
6210 
6211 	auto thread = new Thread(&server);
6212 	thread.start;
6213 
6214 	Thread.sleep(200.msecs);
6215 
6216 	auto response = get("http://localhost:12346/").waitForCompletion;
6217 	assert(response.code == 200);
6218 	//import std.stdio; writeln(response);
6219 
6220 	foreach(site; ["https://dlang.org/", "http://arsdnet.net", "https://phobos.dpldocs.info"]) {
6221 		response = get(site).waitForCompletion;
6222 		assert(response.code == 200);
6223 	}
6224 
6225 	thread.join;
6226 }
6227 
6228 /+
6229 	so the url params are arguments. it knows the request
6230 	internally. other params are properties on the req
6231 
6232 	names may have different paths... those will just add ForSomething i think.
6233 
6234 	auto req = api.listMergeRequests
6235 	req.page = 10;
6236 
6237 	or
6238 		req.page(1)
6239 		.bar("foo")
6240 
6241 	req.execute();
6242 
6243 
6244 	everything in the response is nullable access through the
6245 	dynamic object, just with property getters there. need to make
6246 	it static generated tho
6247 
6248 	other messages may be: isPresent and getDynamic
6249 
6250 
6251 	AND/OR what about doing it like the rails objects
6252 
6253 	BroadcastMessage.get(4)
6254 	// various properties
6255 
6256 	// it lists what you updated
6257 
6258 	BroadcastMessage.foo().bar().put(5)
6259 +/