1 // Copyright 2013-2022, Adam D. Ruppe. 2 3 // FIXME: websocket proxy support 4 // FIXME: ipv6 support 5 6 // FIXME: headers are supposed to be case insensitive. ugh. 7 8 /++ 9 This is version 2 of my http/1.1 client implementation. 10 11 12 It has no dependencies for basic operation, but does require OpenSSL 13 libraries (or compatible) to support HTTPS. This dynamically loaded 14 on-demand (meaning it won't be loaded if you don't use it, but if you do 15 use it, the openssl dynamic libraries must be found in the system search path). 16 17 On Windows, you can bundle the openssl dlls with your exe and they will be picked 18 up when distributed. 19 20 You can compile with `-version=without_openssl` to entirely disable ssl support. 21 22 http2.d, despite its name, does NOT implement HTTP/2.0, but this 23 shouldn't matter for 99.9% of usage, since all servers will continue 24 to support HTTP/1.1 for a very long time. 25 26 History: 27 Automatic `100 Continue` handling was added on September 28, 2021. It doesn't 28 set the Expect header, so it isn't supposed to happen, but plenty of web servers 29 don't follow the standard anyway. 30 31 A dependency on [arsd.core] was added on March 19, 2023 (dub v11.0). Previously, 32 module was stand-alone. You will have add the `core.d` file from the arsd repo 33 to your build now if you are managing the files and builds yourself. 34 35 The benefits of this dependency include some simplified implementation code which 36 makes it easier for me to add more api conveniences, better exceptions with more 37 information, and better event loop integration with other arsd modules beyond 38 just the simpledisplay adapters available previously. The new integration can 39 also make things like heartbeat timers easier for you to code. 40 +/ 41 module arsd.http2; 42 43 /// 44 unittest { 45 import arsd.http2; 46 47 void main() { 48 auto client = new HttpClient(); 49 50 auto request = client.request(Uri("http://dlang.org/")); 51 auto response = request.waitForCompletion(); 52 53 import std.stdio; 54 writeln(response.contentText); 55 writeln(response.code, " ", response.codeText); 56 writeln(response.contentType); 57 } 58 59 version(arsd_http2_integration_test) main(); // exclude from docs 60 } 61 62 /+ 63 // arsd core is now default but you can opt out for a lil while 64 version(no_arsd_core) { 65 66 } else { 67 version=use_arsd_core; 68 } 69 +/ 70 71 static import arsd.core; 72 73 // FIXME: I think I want to disable sigpipe here too. 74 75 import arsd.core : encodeUriComponent, decodeUriComponent; 76 77 debug(arsd_http2_verbose) debug=arsd_http2; 78 79 debug(arsd_http2) import std.stdio : writeln; 80 81 version=arsd_http_internal_implementation; 82 83 version(without_openssl) {} 84 else { 85 version=use_openssl; 86 version=with_openssl; 87 version(older_openssl) {} else 88 version=newer_openssl; 89 } 90 91 version(arsd_http_winhttp_implementation) { 92 pragma(lib, "winhttp") 93 import core.sys.windows.winhttp; 94 // FIXME: alter the dub package file too 95 96 // https://github.com/curl/curl/blob/master/lib/vtls/schannel.c 97 // https://docs.microsoft.com/en-us/windows/win32/secauthn/creating-an-schannel-security-context 98 99 100 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpreaddata 101 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpsendrequest 102 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpopenrequest 103 // https://docs.microsoft.com/en-us/windows/win32/api/winhttp/nf-winhttp-winhttpconnect 104 } 105 106 107 108 /++ 109 Demonstrates core functionality, using the [HttpClient], 110 [HttpRequest] (returned by [HttpClient.navigateTo|client.navigateTo]), 111 and [HttpResponse] (returned by [HttpRequest.waitForCompletion|request.waitForCompletion]). 112 113 +/ 114 unittest { 115 import arsd.http2; 116 117 void main() { 118 auto client = new HttpClient(); 119 auto request = client.navigateTo(Uri("http://dlang.org/")); 120 auto response = request.waitForCompletion(); 121 122 string returnedHtml = response.contentText; 123 } 124 } 125 126 private __gshared bool defaultVerifyPeer_ = true; 127 128 void defaultVerifyPeer(bool v) { 129 defaultVerifyPeer_ = v; 130 } 131 132 debug import std.stdio; 133 134 import std.socket; 135 import core.time; 136 137 // FIXME: check Transfer-Encoding: gzip always 138 139 version(with_openssl) { 140 //pragma(lib, "crypto"); 141 //pragma(lib, "ssl"); 142 } 143 144 /+ 145 HttpRequest httpRequest(string method, string url, ubyte[] content, string[string] content) { 146 return null; 147 } 148 +/ 149 150 /** 151 auto request = get("http://arsdnet.net/"); 152 request.send(); 153 154 auto response = get("http://arsdnet.net/").waitForCompletion(); 155 */ 156 HttpRequest get(string url) { 157 auto client = new HttpClient(); 158 auto request = client.navigateTo(Uri(url)); 159 return request; 160 } 161 162 /** 163 Do not forget to call `waitForCompletion()` on the returned object! 164 */ 165 HttpRequest post(string url, string[string] req) { 166 auto client = new HttpClient(); 167 ubyte[] bdata; 168 foreach(k, v; req) { 169 if(bdata.length) 170 bdata ~= cast(ubyte[]) "&"; 171 bdata ~= cast(ubyte[]) encodeUriComponent(k); 172 bdata ~= cast(ubyte[]) "="; 173 bdata ~= cast(ubyte[]) encodeUriComponent(v); 174 } 175 auto request = client.request(Uri(url), HttpVerb.POST, bdata, "application/x-www-form-urlencoded"); 176 return request; 177 } 178 179 /// gets the text off a url. basic operation only. 180 string getText(string url) { 181 auto request = get(url); 182 auto response = request.waitForCompletion(); 183 return cast(string) response.content; 184 } 185 186 /+ 187 ubyte[] getBinary(string url, string[string] cookies = null) { 188 auto hr = httpRequest("GET", url, null, cookies); 189 if(hr.code != 200) 190 throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url)); 191 return hr.content; 192 } 193 194 /** 195 Gets a textual document, ignoring headers. Throws on non-text or error. 196 */ 197 string get(string url, string[string] cookies = null) { 198 auto hr = httpRequest("GET", url, null, cookies); 199 if(hr.code != 200) 200 throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url)); 201 if(hr.contentType.indexOf("text/") == -1) 202 throw new Exception(hr.contentType ~ " is bad content for conversion to string"); 203 return cast(string) hr.content; 204 205 } 206 207 string post(string url, string[string] args, string[string] cookies = null) { 208 string content; 209 210 foreach(name, arg; args) { 211 if(content.length) 212 content ~= "&"; 213 content ~= encodeUriComponent(name) ~ "=" ~ encodeUriComponent(arg); 214 } 215 216 auto hr = httpRequest("POST", url, cast(ubyte[]) content, cookies, ["Content-Type: application/x-www-form-urlencoded"]); 217 if(hr.code != 200) 218 throw new Exception(format("HTTP answered %d instead of 200", hr.code)); 219 if(hr.contentType.indexOf("text/") == -1) 220 throw new Exception(hr.contentType ~ " is bad content for conversion to string"); 221 222 return cast(string) hr.content; 223 } 224 225 +/ 226 227 /// 228 struct HttpResponse { 229 /++ 230 The HTTP response code, if the response was completed, or some value < 100 if it was aborted or failed. 231 232 Code 0 - initial value, nothing happened 233 Code 1 - you called request.abort 234 Code 2 - connection refused 235 Code 3 - connection succeeded, but server disconnected early 236 Code 4 - server sent corrupted response (or this code has a bug and processed it wrong) 237 Code 5 - request timed out 238 239 Code >= 100 - a HTTP response 240 +/ 241 int code; 242 string codeText; /// 243 244 string httpVersion; /// 245 246 string statusLine; /// 247 248 string contentType; /// The *full* content type header. See also [contentTypeMimeType] and [contentTypeCharset]. 249 string location; /// The location header 250 251 /++ 252 253 History: 254 Added December 5, 2020 (version 9.1) 255 +/ 256 bool wasSuccessful() { 257 return code >= 200 && code < 400; 258 } 259 260 /++ 261 Returns the mime type part of the [contentType] header. 262 263 History: 264 Added July 25, 2022 (version 10.9) 265 +/ 266 string contentTypeMimeType() { 267 auto idx = contentType.indexOf(";"); 268 if(idx == -1) 269 return contentType; 270 271 return contentType[0 .. idx].strip; 272 } 273 274 /// the charset out of content type, if present. `null` if not. 275 string contentTypeCharset() { 276 auto idx = contentType.indexOf("charset="); 277 if(idx == -1) 278 return null; 279 auto c = contentType[idx + "charset=".length .. $].strip; 280 if(c.length) 281 return c; 282 return null; 283 } 284 285 /++ 286 Names and values of cookies set in the response. 287 288 History: 289 Prior to July 5, 2021 (dub v10.2), this was a public field instead of a property. I did 290 not consider this a breaking change since the intended use is completely compatible with the 291 property, and it was not actually implemented properly before anyway. 292 +/ 293 @property string[string] cookies() const { 294 string[string] ret; 295 foreach(cookie; cookiesDetails) 296 ret[cookie.name] = cookie.value; 297 return ret; 298 } 299 /++ 300 The full parsed-out information of cookies set in the response. 301 302 History: 303 Added July 5, 2021 (dub v10.2). 304 +/ 305 @property CookieHeader[] cookiesDetails() inout { 306 CookieHeader[] ret; 307 foreach(header; headers) { 308 if(auto content = header.isHttpHeader("set-cookie")) { 309 // format: name=value, value might be double quoted. it MIGHT be url encoded, but im not going to attempt that since the RFC is silent. 310 // then there's optionally ; attr=value after that. attributes need not have a value 311 312 CookieHeader cookie; 313 314 auto remaining = content; 315 316 cookie_name: 317 foreach(idx, ch; remaining) { 318 if(ch == '=') { 319 cookie.name = remaining[0 .. idx].idup_if_needed; 320 remaining = remaining[idx + 1 .. $]; 321 break; 322 } 323 } 324 325 cookie_value: 326 327 { 328 auto idx = remaining.indexOf(";"); 329 if(idx == -1) { 330 cookie.value = remaining.idup_if_needed; 331 remaining = remaining[$..$]; 332 } else { 333 cookie.value = remaining[0 .. idx].idup_if_needed; 334 remaining = remaining[idx + 1 .. $].stripLeft; 335 } 336 337 if(cookie.value.length > 2 && cookie.value[0] == '"' && cookie.value[$-1] == '"') 338 cookie.value = cookie.value[1 .. $ - 1]; 339 } 340 341 cookie_attributes: 342 343 while(remaining.length) { 344 string name; 345 foreach(idx, ch; remaining) { 346 if(ch == '=') { 347 name = remaining[0 .. idx].idup_if_needed; 348 remaining = remaining[idx + 1 .. $]; 349 350 string value; 351 352 foreach(idx2, ch2; remaining) { 353 if(ch2 == ';') { 354 value = remaining[0 .. idx2].idup_if_needed; 355 remaining = remaining[idx2 + 1 .. $].stripLeft; 356 break; 357 } 358 } 359 360 if(value is null) { 361 value = remaining.idup_if_needed; 362 remaining = remaining[$ .. $]; 363 } 364 365 cookie.attributes[name] = value; 366 continue cookie_attributes; 367 } else if(ch == ';') { 368 name = remaining[0 .. idx].idup_if_needed; 369 remaining = remaining[idx + 1 .. $].stripLeft; 370 cookie.attributes[name] = ""; 371 continue cookie_attributes; 372 } 373 } 374 375 if(remaining.length) { 376 cookie.attributes[remaining.idup_if_needed] = ""; 377 remaining = remaining[$..$]; 378 379 } 380 } 381 382 ret ~= cookie; 383 } 384 } 385 return ret; 386 } 387 388 string[] headers; /// Array of all headers returned. 389 string[string] headersHash; /// 390 391 ubyte[] content; /// The raw content returned in the response body. 392 string contentText; /// [content], but casted to string (for convenience) 393 394 alias responseText = contentText; // just cuz I do this so often. 395 //alias body = content; 396 397 /++ 398 returns `new Document(this.contentText)`. Requires [arsd.dom]. 399 +/ 400 auto contentDom()() { 401 import arsd.dom; 402 return new Document(this.contentText); 403 404 } 405 406 /++ 407 returns `var.fromJson(this.contentText)`. Requires [arsd.jsvar]. 408 +/ 409 auto contentJson()() { 410 import arsd.jsvar; 411 return var.fromJson(this.contentText); 412 } 413 414 HttpRequestParameters requestParameters; /// 415 416 LinkHeader[] linksStored; 417 bool linksLazilyParsed; 418 419 HttpResponse deepCopy() const { 420 HttpResponse h = cast(HttpResponse) this; 421 h.headers = h.headers.dup; 422 h.headersHash = h.headersHash.dup; 423 h.content = h.content.dup; 424 h.linksStored = h.linksStored.dup; 425 return h; 426 } 427 428 /// Returns links header sorted by "rel" attribute. 429 /// It returns a new array on each call. 430 LinkHeader[string] linksHash() { 431 auto links = this.links(); 432 LinkHeader[string] ret; 433 foreach(link; links) 434 ret[link.rel] = link; 435 return ret; 436 } 437 438 /// Returns the Link header, parsed. 439 LinkHeader[] links() { 440 if(linksLazilyParsed) 441 return linksStored; 442 linksLazilyParsed = true; 443 LinkHeader[] ret; 444 445 auto hdrPtr = "link" in headersHash; 446 if(hdrPtr is null) 447 return ret; 448 449 auto header = *hdrPtr; 450 451 LinkHeader current; 452 453 while(header.length) { 454 char ch = header[0]; 455 456 if(ch == '<') { 457 // read url 458 header = header[1 .. $]; 459 size_t idx; 460 while(idx < header.length && header[idx] != '>') 461 idx++; 462 current.url = header[0 .. idx]; 463 header = header[idx .. $]; 464 } else if(ch == ';') { 465 // read attribute 466 header = header[1 .. $]; 467 header = header.stripLeft; 468 469 size_t idx; 470 while(idx < header.length && header[idx] != '=') 471 idx++; 472 473 string name = header[0 .. idx]; 474 if(idx + 1 < header.length) 475 header = header[idx + 1 .. $]; 476 else 477 header = header[$ .. $]; 478 479 string value; 480 481 if(header.length && header[0] == '"') { 482 // quoted value 483 header = header[1 .. $]; 484 idx = 0; 485 while(idx < header.length && header[idx] != '\"') 486 idx++; 487 value = header[0 .. idx]; 488 header = header[idx .. $]; 489 490 } else if(header.length) { 491 // unquoted value 492 idx = 0; 493 while(idx < header.length && header[idx] != ',' && header[idx] != ' ' && header[idx] != ';') 494 idx++; 495 496 value = header[0 .. idx]; 497 header = header[idx .. $].stripLeft; 498 } 499 500 name = name.toLower; 501 if(name == "rel") 502 current.rel = value; 503 else 504 current.attributes[name] = value; 505 506 } else if(ch == ',') { 507 // start another 508 ret ~= current; 509 current = LinkHeader.init; 510 } else if(ch == ' ' || ch == '\n' || ch == '\r' || ch == '\t') { 511 // ignore 512 } 513 514 if(header.length) 515 header = header[1 .. $]; 516 } 517 518 ret ~= current; 519 520 linksStored = ret; 521 522 return ret; 523 } 524 } 525 526 /+ 527 headerName MUST be all lower case and NOT have the colon on it 528 529 returns slice of the input thing after the header name 530 +/ 531 private inout(char)[] isHttpHeader(inout(char)[] thing, const(char)[] headerName) { 532 foreach(idx, ch; thing) { 533 if(idx < headerName.length) { 534 if(headerName[idx] == '-' && ch != '-') 535 return null; 536 if((ch | ' ') != headerName[idx]) 537 return null; 538 } else if(idx == headerName.length) { 539 if(ch != ':') 540 return null; 541 } else { 542 return thing[idx .. $].strip; 543 } 544 } 545 return null; 546 } 547 548 private string idup_if_needed(string s) { return s; } 549 private string idup_if_needed(const(char)[] s) { return s.idup; } 550 551 unittest { 552 assert("Cookie: foo=bar".isHttpHeader("cookie") == "foo=bar"); 553 assert("cookie: foo=bar".isHttpHeader("cookie") == "foo=bar"); 554 assert("cOOkie: foo=bar".isHttpHeader("cookie") == "foo=bar"); 555 assert("Set-Cookie: foo=bar".isHttpHeader("set-cookie") == "foo=bar"); 556 assert(!"".isHttpHeader("cookie")); 557 } 558 559 /// 560 struct LinkHeader { 561 string url; /// 562 string rel; /// 563 string[string] attributes; /// like title, rev, media, whatever attributes 564 } 565 566 /++ 567 History: 568 Added July 5, 2021 569 +/ 570 struct CookieHeader { 571 string name; 572 string value; 573 string[string] attributes; 574 575 // max-age 576 // expires 577 // httponly 578 // secure 579 // samesite 580 // path 581 // domain 582 // partitioned ? 583 584 // also want cookiejar features here with settings to save session cookies or not 585 586 // storing in file: http://kb.mozillazine.org/Cookies.txt (second arg in practice true if first arg starts with . it seems) 587 // or better yet sqlite: http://kb.mozillazine.org/Cookies.sqlite 588 // should be able to import/export from either upon request 589 } 590 591 import std.string; 592 static import std.algorithm; 593 import std.conv; 594 import std.range; 595 596 597 private AddressFamily family(string unixSocketPath) { 598 if(unixSocketPath.length) 599 return AddressFamily.UNIX; 600 else // FIXME: what about ipv6? 601 return AddressFamily.INET; 602 } 603 604 version(Windows) 605 private class UnixAddress : Address { 606 this(string) { 607 throw new Exception("No unix address support on this system in lib yet :("); 608 } 609 override sockaddr* name() { assert(0); } 610 override const(sockaddr)* name() const { assert(0); } 611 override int nameLen() const { assert(0); } 612 } 613 614 615 // Copy pasta from cgi.d, then stripped down. unix path thing added tho 616 /++ 617 Represents a URI. It offers named access to the components and relative uri resolution, though as a user of the library, you'd mostly just construct it like `Uri("http://example.com/index.html")`. 618 +/ 619 struct Uri { 620 alias toString this; // blargh idk a url really is a string, but should it be implicit? 621 622 // scheme://userinfo@host:port/path?query#fragment 623 624 string scheme; /// e.g. "http" in "http://example.com/" 625 string userinfo; /// the username (and possibly a password) in the uri 626 string host; /// the domain name 627 int port; /// port number, if given. Will be zero if a port was not explicitly given 628 string path; /// e.g. "/folder/file.html" in "http://example.com/folder/file.html" 629 string query; /// the stuff after the ? in a uri 630 string fragment; /// the stuff after the # in a uri. 631 632 /// Breaks down a uri string to its components 633 this(string uri) { 634 size_t lastGoodIndex; 635 foreach(char ch; uri) { 636 if(ch > 127) { 637 break; 638 } 639 lastGoodIndex++; 640 } 641 642 string replacement = uri[0 .. lastGoodIndex]; 643 foreach(char ch; uri[lastGoodIndex .. $]) { 644 if(ch > 127) { 645 // need to percent-encode any non-ascii in it 646 char[3] buffer; 647 buffer[0] = '%'; 648 649 auto first = ch / 16; 650 auto second = ch % 16; 651 first += (first >= 10) ? ('A'-10) : '0'; 652 second += (second >= 10) ? ('A'-10) : '0'; 653 654 buffer[1] = cast(char) first; 655 buffer[2] = cast(char) second; 656 657 replacement ~= buffer[]; 658 } else { 659 replacement ~= ch; 660 } 661 } 662 663 reparse(replacement); 664 } 665 666 /// Returns `port` if set, otherwise if scheme is https 443, otherwise always 80 667 int effectivePort() const @property nothrow pure @safe @nogc { 668 return port != 0 ? port 669 : scheme == "https" ? 443 : 80; 670 } 671 672 private string unixSocketPath = null; 673 /// Indicates it should be accessed through a unix socket instead of regular tcp. Returns new version without modifying this object. 674 Uri viaUnixSocket(string path) const { 675 Uri copy = this; 676 copy.unixSocketPath = path; 677 return copy; 678 } 679 680 /// Goes through a unix socket in the abstract namespace (linux only). Returns new version without modifying this object. 681 version(linux) 682 Uri viaAbstractSocket(string path) const { 683 Uri copy = this; 684 copy.unixSocketPath = "\0" ~ path; 685 return copy; 686 } 687 688 private void reparse(string uri) { 689 // from RFC 3986 690 // the ctRegex triples the compile time and makes ugly errors for no real benefit 691 // it was a nice experiment but just not worth it. 692 // enum ctr = ctRegex!r"^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?"; 693 /* 694 Captures: 695 0 = whole url 696 1 = scheme, with : 697 2 = scheme, no : 698 3 = authority, with // 699 4 = authority, no // 700 5 = path 701 6 = query string, with ? 702 7 = query string, no ? 703 8 = anchor, with # 704 9 = anchor, no # 705 */ 706 // Yikes, even regular, non-CT regex is also unacceptably slow to compile. 1.9s on my computer! 707 // instead, I will DIY and cut that down to 0.6s on the same computer. 708 /* 709 710 Note that authority is 711 user:password@domain:port 712 where the user:password@ part is optional, and the :port is optional. 713 714 Regex translation: 715 716 Scheme cannot have :, /, ?, or # in it, and must have one or more chars and end in a :. It is optional, but must be first. 717 Authority must start with //, but cannot have any other /, ?, or # in it. It is optional. 718 Path cannot have any ? or # in it. It is optional. 719 Query must start with ? and must not have # in it. It is optional. 720 Anchor must start with # and can have anything else in it to end of string. It is optional. 721 */ 722 723 this = Uri.init; // reset all state 724 725 // empty uri = nothing special 726 if(uri.length == 0) { 727 return; 728 } 729 730 size_t idx; 731 732 scheme_loop: foreach(char c; uri[idx .. $]) { 733 switch(c) { 734 case ':': 735 case '/': 736 case '?': 737 case '#': 738 break scheme_loop; 739 default: 740 } 741 idx++; 742 } 743 744 if(idx == 0 && uri[idx] == ':') { 745 // this is actually a path! we skip way ahead 746 goto path_loop; 747 } 748 749 if(idx == uri.length) { 750 // the whole thing is a path, apparently 751 path = uri; 752 return; 753 } 754 755 if(idx > 0 && uri[idx] == ':') { 756 scheme = uri[0 .. idx]; 757 idx++; 758 } else { 759 // we need to rewind; it found a / but no :, so the whole thing is prolly a path... 760 idx = 0; 761 } 762 763 if(idx + 2 < uri.length && uri[idx .. idx + 2] == "//") { 764 // we have an authority.... 765 idx += 2; 766 767 auto authority_start = idx; 768 authority_loop: foreach(char c; uri[idx .. $]) { 769 switch(c) { 770 case '/': 771 case '?': 772 case '#': 773 break authority_loop; 774 default: 775 } 776 idx++; 777 } 778 779 auto authority = uri[authority_start .. idx]; 780 781 auto idx2 = authority.indexOf("@"); 782 if(idx2 != -1) { 783 userinfo = authority[0 .. idx2]; 784 authority = authority[idx2 + 1 .. $]; 785 } 786 787 if(authority.length && authority[0] == '[') { 788 // ipv6 address special casing 789 idx2 = authority.indexOf(']'); 790 if(idx2 != -1) { 791 auto end = authority[idx2 + 1 .. $]; 792 if(end.length && end[0] == ':') 793 idx2 = idx2 + 1; 794 else 795 idx2 = -1; 796 } 797 } else { 798 idx2 = authority.indexOf(":"); 799 } 800 801 if(idx2 == -1) { 802 port = 0; // 0 means not specified; we should use the default for the scheme 803 host = authority; 804 } else { 805 host = authority[0 .. idx2]; 806 if(idx2 + 1 < authority.length) 807 port = to!int(authority[idx2 + 1 .. $]); 808 else 809 port = 0; 810 } 811 } 812 813 path_loop: 814 auto path_start = idx; 815 816 foreach(char c; uri[idx .. $]) { 817 if(c == '?' || c == '#') 818 break; 819 idx++; 820 } 821 822 path = uri[path_start .. idx]; 823 824 if(idx == uri.length) 825 return; // nothing more to examine... 826 827 if(uri[idx] == '?') { 828 idx++; 829 auto query_start = idx; 830 foreach(char c; uri[idx .. $]) { 831 if(c == '#') 832 break; 833 idx++; 834 } 835 query = uri[query_start .. idx]; 836 } 837 838 if(idx < uri.length && uri[idx] == '#') { 839 idx++; 840 fragment = uri[idx .. $]; 841 } 842 843 // uriInvalidated = false; 844 } 845 846 private string rebuildUri() const { 847 string ret; 848 if(scheme.length) 849 ret ~= scheme ~ ":"; 850 if(userinfo.length || host.length) 851 ret ~= "//"; 852 if(userinfo.length) 853 ret ~= userinfo ~ "@"; 854 if(host.length) 855 ret ~= host; 856 if(port) 857 ret ~= ":" ~ to!string(port); 858 859 ret ~= path; 860 861 if(query.length) 862 ret ~= "?" ~ query; 863 864 if(fragment.length) 865 ret ~= "#" ~ fragment; 866 867 // uri = ret; 868 // uriInvalidated = false; 869 return ret; 870 } 871 872 /// Converts the broken down parts back into a complete string 873 string toString() const { 874 // if(uriInvalidated) 875 return rebuildUri(); 876 } 877 878 /// Returns a new absolute Uri given a base. It treats this one as 879 /// relative where possible, but absolute if not. (If protocol, domain, or 880 /// other info is not set, the new one inherits it from the base.) 881 /// 882 /// Browsers use a function like this to figure out links in html. 883 Uri basedOn(in Uri baseUrl) const { 884 Uri n = this; // copies 885 if(n.scheme == "data") 886 return n; 887 // n.uriInvalidated = true; // make sure we regenerate... 888 889 // userinfo is not inherited... is this wrong? 890 891 // if anything is given in the existing url, we don't use the base anymore. 892 if(n.scheme.empty) { 893 n.scheme = baseUrl.scheme; 894 if(n.host.empty) { 895 n.host = baseUrl.host; 896 if(n.port == 0) { 897 n.port = baseUrl.port; 898 if(n.path.length > 0 && n.path[0] != '/') { 899 auto b = baseUrl.path[0 .. baseUrl.path.lastIndexOf("/") + 1]; 900 if(b.length == 0) 901 b = "/"; 902 n.path = b ~ n.path; 903 } else if(n.path.length == 0) { 904 n.path = baseUrl.path; 905 } 906 } 907 } 908 } 909 910 n.removeDots(); 911 912 // if still basically talking to the same thing, we should inherit the unix path 913 // too since basically the unix path is saying for this service, always use this override. 914 if(n.host == baseUrl.host && n.scheme == baseUrl.scheme && n.port == baseUrl.port) 915 n.unixSocketPath = baseUrl.unixSocketPath; 916 917 return n; 918 } 919 920 /++ 921 Resolves ../ and ./ parts of the path. Used in the implementation of [basedOn] and you could also use it to normalize things. 922 +/ 923 void removeDots() { 924 auto parts = this.path.split("/"); 925 string[] toKeep; 926 foreach(part; parts) { 927 if(part == ".") { 928 continue; 929 } else if(part == "..") { 930 //if(toKeep.length > 1) 931 toKeep = toKeep[0 .. $-1]; 932 //else 933 //toKeep = [""]; 934 continue; 935 } else { 936 //if(toKeep.length && toKeep[$-1].length == 0 && part.length == 0) 937 //continue; // skip a `//` situation 938 toKeep ~= part; 939 } 940 } 941 942 auto path = toKeep.join("/"); 943 if(path.length && path[0] != '/') 944 path = "/" ~ path; 945 946 this.path = path; 947 } 948 } 949 950 /* 951 void main(string args[]) { 952 write(post("http://arsdnet.net/bugs.php", ["test" : "hey", "again" : "what"])); 953 } 954 */ 955 956 /// 957 struct BasicAuth { 958 string username; /// 959 string password; /// 960 } 961 962 class ProxyException : Exception { 963 this(string msg) {super(msg); } 964 } 965 966 /** 967 Represents a HTTP request. You usually create these through a [HttpClient]. 968 969 970 --- 971 auto request = new HttpRequest(); // note that when there's no associated client, some features may not work 972 // normally you'd instead do `new HttpClient(); client.request(...)` 973 // set any properties here 974 975 // synchronous usage 976 auto reply = request.perform(); 977 978 // async usage, type 1: 979 request.send(); 980 request2.send(); 981 982 // wait until the first one is done, with the second one still in-flight 983 auto response = request.waitForCompletion(); 984 985 // async usage, type 2: 986 request.onDataReceived = (HttpRequest hr) { 987 if(hr.state == HttpRequest.State.complete) { 988 // use hr.responseData 989 } 990 }; 991 request.send(); // send, using the callback 992 993 // before terminating, be sure you wait for your requests to finish! 994 995 request.waitForCompletion(); 996 --- 997 */ 998 class HttpRequest { 999 1000 /// Automatically follow a redirection? 1001 bool followLocation = false; 1002 1003 /++ 1004 Maximum number of redirections to follow (used only if [followLocation] is set to true). Will resolve with an error if a single request has more than this number of redirections. The default value is currently 10, but may change without notice. If you need a specific value, be sure to call this function. 1005 1006 If you want unlimited redirects, call it with `int.max`. If you set it to 0 but set [followLocation] to `true`, any attempt at redirection will abort the request. To disable automatically following redirection, set [followLocation] to `false` so you can process the 30x code yourself as a completed request. 1007 1008 History: 1009 Added July 27, 2022 (dub v10.9) 1010 +/ 1011 void setMaximumNumberOfRedirects(int max = 10) { 1012 maximumNumberOfRedirectsRemaining = max; 1013 } 1014 1015 private int maximumNumberOfRedirectsRemaining; 1016 1017 /++ 1018 Set to `true` to automatically retain cookies in the associated [HttpClient] from this request. 1019 Note that you must have constructed the request from a `HttpClient` or at least passed one into the 1020 constructor for this to have any effect. 1021 1022 Bugs: 1023 See [HttpClient.retainCookies] for important caveats. 1024 1025 History: 1026 Added July 5, 2021 (dub v10.2) 1027 +/ 1028 bool retainCookies = false; 1029 1030 private HttpClient client; 1031 1032 this() { 1033 } 1034 1035 /// 1036 this(HttpClient client, Uri where, HttpVerb method, ICache cache = null, Duration timeout = 10.seconds, string proxy = null) { 1037 this.client = client; 1038 populateFromInfo(where, method); 1039 setTimeout(timeout); 1040 this.cache = cache; 1041 this.proxy = proxy; 1042 1043 setMaximumNumberOfRedirects(); 1044 } 1045 1046 1047 /// ditto 1048 this(Uri where, HttpVerb method, ICache cache = null, Duration timeout = 10.seconds, string proxy = null) { 1049 this(null, where, method, cache, timeout, proxy); 1050 } 1051 1052 /++ 1053 Sets the timeout from inactivity on the request. This is the amount of time that passes with no send or receive activity on the request before it fails with "request timed out" error. 1054 1055 History: 1056 Added March 31, 2021 1057 +/ 1058 void setTimeout(Duration timeout) { 1059 this.requestParameters.timeoutFromInactivity = timeout; 1060 this.timeoutFromInactivity = MonoTime.currTime + this.requestParameters.timeoutFromInactivity; 1061 } 1062 1063 /++ 1064 Set to `true` to gzip the request body when sending to the server. This is often not supported, and thus turned off 1065 by default. 1066 1067 1068 If a server doesn't support this, you MAY get an http error or it might just do the wrong thing. 1069 By spec, it is supposed to be code "415 Unsupported Media Type", but there's no guarantee they 1070 will do that correctly since many servers will simply have never considered this possibility. Request 1071 compression is quite rare, so before using this, ensure your server supports it by checking its documentation 1072 or asking its administrator. (Or running a test, but remember, it might just do the wrong thing and not issue 1073 an appropriate error, or the config may change in the future.) 1074 1075 History: 1076 Added August 6, 2024 (dub v11.5) 1077 +/ 1078 void gzipBody(bool want) { 1079 this.requestParameters.gzipBody = want; 1080 } 1081 1082 private MonoTime timeoutFromInactivity; 1083 1084 private Uri where; 1085 1086 private ICache cache; 1087 1088 /++ 1089 Proxy to use for this request. It should be a URL or `null`. 1090 1091 This must be sent before you call [send]. 1092 1093 History: 1094 Added April 12, 2021 (dub v9.5) 1095 +/ 1096 string proxy; 1097 1098 /++ 1099 For https connections, if this is `true`, it will fail to connect if the TLS certificate can not be 1100 verified. Setting this to `false` will skip this check and allow the connection to continue anyway. 1101 1102 When the [HttpRequest] is constructed from a [HttpClient], it will inherit the value from the client 1103 instead of using the `= true` here. You can change this value any time before you call [send] (which 1104 is done implicitly if you call [waitForCompletion]). 1105 1106 History: 1107 Added April 5, 2022 (dub v10.8) 1108 1109 Prior to this, it always used the global (but undocumented) `defaultVerifyPeer` setting, and sometimes 1110 even if it was true, it would skip the verification. Now, it always respects this local setting. 1111 +/ 1112 bool verifyPeer = true; 1113 1114 1115 /// Final url after any redirections 1116 string finalUrl; 1117 1118 void populateFromInfo(Uri where, HttpVerb method) { 1119 auto parts = where.basedOn(this.where); 1120 this.where = parts; 1121 finalUrl = where.toString(); 1122 requestParameters.method = method; 1123 requestParameters.unixSocketPath = where.unixSocketPath; 1124 requestParameters.host = parts.host; 1125 requestParameters.port = cast(ushort) parts.effectivePort; 1126 requestParameters.ssl = parts.scheme == "https"; 1127 requestParameters.uri = parts.path.length ? parts.path : "/"; 1128 if(parts.query.length) { 1129 requestParameters.uri ~= "?"; 1130 requestParameters.uri ~= parts.query; 1131 } 1132 } 1133 1134 ~this() { 1135 } 1136 1137 ubyte[] sendBuffer; 1138 1139 HttpResponse responseData; 1140 private HttpClient parentClient; 1141 1142 size_t bodyBytesSent; 1143 size_t bodyBytesReceived; 1144 1145 State state_; 1146 final State state() { return state_; } 1147 final State state(State s) { 1148 assert(state_ != State.complete); 1149 return state_ = s; 1150 } 1151 /// Called when data is received. Check the state to see what data is available. 1152 void delegate(HttpRequest) onDataReceived; 1153 1154 enum State { 1155 /// The request has not yet been sent 1156 unsent, 1157 1158 /// The send() method has been called, but no data is 1159 /// sent on the socket yet because the connection is busy. 1160 pendingAvailableConnection, 1161 1162 /// connect has been called, but we're waiting on word of success 1163 connecting, 1164 1165 /// connecting a ssl, needing this 1166 sslConnectPendingRead, 1167 /// ditto 1168 sslConnectPendingWrite, 1169 1170 /// The headers are being sent now 1171 sendingHeaders, 1172 1173 // FIXME: allow Expect: 100-continue and separate the body send 1174 1175 /// The body is being sent now 1176 sendingBody, 1177 1178 /// The request has been sent but we haven't received any response yet 1179 waitingForResponse, 1180 1181 /// We have received some data and are currently receiving headers 1182 readingHeaders, 1183 1184 /// All headers are available but we're still waiting on the body 1185 readingBody, 1186 1187 /// The request is complete. 1188 complete, 1189 1190 /// The request is aborted, either by the abort() method, or as a result of the server disconnecting 1191 aborted 1192 } 1193 1194 /// Sends now and waits for the request to finish, returning the response. 1195 HttpResponse perform() { 1196 send(); 1197 return waitForCompletion(); 1198 } 1199 1200 /// Sends the request asynchronously. 1201 void send() { 1202 sendPrivate(true); 1203 } 1204 1205 private void sendPrivate(bool advance) { 1206 if(state != State.unsent && state != State.aborted) 1207 return; // already sent 1208 1209 if(cache !is null) { 1210 auto res = cache.getCachedResponse(this.requestParameters); 1211 if(res !is null) { 1212 state = State.complete; 1213 responseData = (*res).deepCopy(); 1214 return; 1215 } 1216 } 1217 1218 if(this.where.scheme == "data") { 1219 void error(string content) { 1220 responseData.code = 400; 1221 responseData.codeText = "Bad Request"; 1222 responseData.contentType = "text/plain"; 1223 responseData.content = cast(ubyte[]) content; 1224 responseData.contentText = content; 1225 state = State.complete; 1226 return; 1227 } 1228 1229 auto thing = this.where.path; 1230 // format is: type,data 1231 // type can have ;base64 1232 auto comma = thing.indexOf(","); 1233 if(comma == -1) 1234 return error("Invalid data uri, no comma found"); 1235 1236 auto type = thing[0 .. comma]; 1237 auto data = thing[comma + 1 .. $]; 1238 if(type.length == 0) 1239 type = "text/plain"; 1240 1241 auto bdata = cast(ubyte[]) decodeUriComponent(data); 1242 1243 if(type.indexOf(";base64") != -1) { 1244 import std.base64; 1245 try { 1246 bdata = Base64.decode(bdata); 1247 } catch(Exception e) { 1248 return error(e.msg); 1249 } 1250 } 1251 1252 responseData.code = 200; 1253 responseData.codeText = "OK"; 1254 responseData.contentType = type; 1255 responseData.content = bdata; 1256 responseData.contentText = cast(string) responseData.content; 1257 state = State.complete; 1258 return; 1259 } 1260 1261 string headers; 1262 1263 headers ~= to!string(requestParameters.method); 1264 headers ~= " "; 1265 if(proxy.length && !requestParameters.ssl) { 1266 // if we're doing a http proxy, we need to send a complete, absolute uri 1267 // so reconstruct it 1268 headers ~= "http://"; 1269 headers ~= requestParameters.host; 1270 if(requestParameters.port != 80) { 1271 headers ~= ":"; 1272 headers ~= to!string(requestParameters.port); 1273 } 1274 } 1275 1276 headers ~= requestParameters.uri; 1277 1278 if(requestParameters.useHttp11) 1279 headers ~= " HTTP/1.1\r\n"; 1280 else 1281 headers ~= " HTTP/1.0\r\n"; 1282 1283 // the whole authority section is supposed to be there, but curl doesn't send if default port 1284 // so I'll copy what they do 1285 headers ~= "Host: "; 1286 headers ~= requestParameters.host; 1287 if(requestParameters.port != 80 && requestParameters.port != 443) { 1288 headers ~= ":"; 1289 headers ~= to!string(requestParameters.port); 1290 } 1291 headers ~= "\r\n"; 1292 1293 bool specSaysRequestAlwaysHasBody = 1294 requestParameters.method == HttpVerb.POST || 1295 requestParameters.method == HttpVerb.PUT || 1296 requestParameters.method == HttpVerb.PATCH; 1297 1298 if(requestParameters.userAgent.length) 1299 headers ~= "User-Agent: "~requestParameters.userAgent~"\r\n"; 1300 if(requestParameters.contentType.length) 1301 headers ~= "Content-Type: "~requestParameters.contentType~"\r\n"; 1302 if(requestParameters.authorization.length) 1303 headers ~= "Authorization: "~requestParameters.authorization~"\r\n"; 1304 if(requestParameters.bodyData.length || specSaysRequestAlwaysHasBody) 1305 headers ~= "Content-Length: "~to!string(requestParameters.bodyData.length)~"\r\n"; 1306 if(requestParameters.acceptGzip) 1307 headers ~= "Accept-Encoding: gzip\r\n"; 1308 if(requestParameters.keepAlive) 1309 headers ~= "Connection: keep-alive\r\n"; 1310 1311 string cookieHeader; 1312 foreach(name, value; requestParameters.cookies) { 1313 if(cookieHeader is null) 1314 cookieHeader = "Cookie: "; 1315 else 1316 cookieHeader ~= "; "; 1317 cookieHeader ~= name; 1318 cookieHeader ~= "="; 1319 cookieHeader ~= value; 1320 } 1321 1322 if(cookieHeader !is null) { 1323 cookieHeader ~= "\r\n"; 1324 headers ~= cookieHeader; 1325 } 1326 1327 foreach(header; requestParameters.headers) 1328 headers ~= header ~ "\r\n"; 1329 1330 const(ubyte)[] bodyToSend = requestParameters.bodyData; 1331 if(requestParameters.gzipBody) { 1332 headers ~= "Content-Encoding: gzip\r\n"; 1333 auto c = new Compress(HeaderFormat.gzip); 1334 1335 auto data = c.compress(bodyToSend); 1336 data ~= c.flush(); 1337 bodyToSend = cast(ubyte[]) data; 1338 } 1339 1340 headers ~= "\r\n"; 1341 1342 // FIXME: separate this for 100 continue 1343 sendBuffer = cast(ubyte[]) headers ~ bodyToSend; 1344 1345 // import std.stdio; writeln("******* ", cast(string) sendBuffer); 1346 1347 responseData = HttpResponse.init; 1348 responseData.requestParameters = requestParameters; 1349 bodyBytesSent = 0; 1350 bodyBytesReceived = 0; 1351 state = State.pendingAvailableConnection; 1352 1353 bool alreadyPending = false; 1354 foreach(req; pending) 1355 if(req is this) { 1356 alreadyPending = true; 1357 break; 1358 } 1359 if(!alreadyPending) { 1360 pending ~= this; 1361 } 1362 1363 if(advance) 1364 HttpRequest.advanceConnections(requestParameters.timeoutFromInactivity); 1365 } 1366 1367 1368 /// Waits for the request to finish or timeout, whichever comes first. 1369 HttpResponse waitForCompletion() { 1370 while(state != State.aborted && state != State.complete) { 1371 if(state == State.unsent) { 1372 send(); 1373 continue; 1374 } 1375 if(auto err = HttpRequest.advanceConnections(requestParameters.timeoutFromInactivity)) { 1376 switch(err) { 1377 case 1: throw new Exception("HttpRequest.advanceConnections returned 1: all connections timed out"); 1378 case 2: throw new Exception("HttpRequest.advanceConnections returned 2: nothing to do"); 1379 case 3: continue; // EINTR 1380 default: throw new Exception("HttpRequest.advanceConnections got err " ~ to!string(err)); 1381 } 1382 } 1383 } 1384 1385 if(state == State.complete && responseData.code >= 200) 1386 if(cache !is null) 1387 cache.cacheResponse(this.requestParameters, this.responseData); 1388 1389 return responseData; 1390 } 1391 1392 /// Aborts this request. 1393 void abort() { 1394 this.state = State.aborted; 1395 this.responseData.code = 1; 1396 this.responseData.codeText = "request.abort called"; 1397 // the actual cancellation happens in the event loop 1398 } 1399 1400 HttpRequestParameters requestParameters; /// 1401 1402 version(arsd_http_winhttp_implementation) { 1403 public static void resetInternals() { 1404 1405 } 1406 1407 static assert(0, "implementation not finished"); 1408 } 1409 1410 1411 version(arsd_http_internal_implementation) { 1412 1413 /++ 1414 Changes the limit of number of open, inactive sockets. Reusing connections can provide a significant 1415 performance improvement, but the operating system can also impose a global limit on the number of open 1416 sockets and/or files that you don't want to run into. This lets you choose a balance right for you. 1417 1418 1419 When the total number of cached, inactive sockets approaches this maximum, it will check for ones closed by the 1420 server first. If there are none already closed by the server, it will select sockets at random from its connection 1421 cache and close them to make room for the new ones. 1422 1423 Please note: 1424 1425 $(LIST 1426 * there is always a limit of six open sockets per domain, per the common practice suggested by the http standard 1427 * the limit given here is thread-local. If you run multiple http clients/requests from multiple threads, don't set this too high or you might bump into the global limit from the OS. 1428 * setting this too low can waste connections because the server might close them, but they will never be garbage collected since my current implementation won't check for dead connections except when it thinks it is running close to the limit. 1429 ) 1430 1431 Setting it just right for your use case may provide an up to 10x performance boost. 1432 1433 This implementation is subject to change. If it does, I'll document it, but may not bump the version number. 1434 1435 History: 1436 Added August 10, 2022 (dub v10.9) 1437 +/ 1438 static void setConnectionCacheSize(int max = 32) { 1439 connectionCacheSize = max; 1440 } 1441 1442 private static { 1443 // we manage the actual connections. When a request is made on a particular 1444 // host, we try to reuse connections. We may open more than one connection per 1445 // host to do parallel requests. 1446 // 1447 // The key is the *domain name* and the port. Multiple domains on the same address will have separate connections. 1448 Socket[][string] socketsPerHost; 1449 1450 // only one request can be active on a given socket (at least HTTP < 2.0) so this is that 1451 HttpRequest[Socket] activeRequestOnSocket; 1452 HttpRequest[] pending; // and these are the requests that are waiting 1453 1454 int cachedSockets; 1455 int connectionCacheSize = 32; 1456 1457 /+ 1458 This is a somewhat expensive, but essential operation. If it isn't used in a heavy 1459 application, you'll risk running out of file descriptors. 1460 +/ 1461 void cleanOldSockets() { 1462 static struct CloseCandidate { 1463 string key; 1464 Socket socket; 1465 } 1466 1467 CloseCandidate[36] closeCandidates; 1468 int closeCandidatesPosition; 1469 1470 outer: foreach(key, sockets; socketsPerHost) { 1471 foreach(socket; sockets) { 1472 if(socket in activeRequestOnSocket) 1473 continue; // it is still in use; we can't close it 1474 1475 closeCandidates[closeCandidatesPosition++] = CloseCandidate(key, socket); 1476 if(closeCandidatesPosition == closeCandidates.length) 1477 break outer; 1478 } 1479 } 1480 1481 auto cc = closeCandidates[0 .. closeCandidatesPosition]; 1482 1483 if(cc.length == 0) 1484 return; // no candidates to even examine 1485 1486 // has the server closed any of these? if so, we also close and drop them 1487 static SocketSet readSet = null; 1488 if(readSet is null) 1489 readSet = new SocketSet(); 1490 readSet.reset(); 1491 1492 foreach(candidate; cc) { 1493 readSet.add(candidate.socket); 1494 } 1495 1496 int closeCount; 1497 1498 auto got = Socket.select(readSet, null, null, 0.msecs /* timeout, want it small since we just checking for eof */); 1499 if(got > 0) { 1500 foreach(ref candidate; cc) { 1501 if(readSet.isSet(candidate.socket)) { 1502 // if we can read when it isn't in use, that means eof; the 1503 // server closed it. 1504 candidate.socket.close(); 1505 loseSocketByKey(candidate.key, candidate.socket); 1506 closeCount++; 1507 } 1508 } 1509 debug(arsd_http2) writeln(closeCount, " from inactivity"); 1510 } else { 1511 // and if not, of the remaining ones, close a few just at random to bring us back beneath the arbitrary limit. 1512 1513 while(cc.length > 0 && (cachedSockets - closeCount) > connectionCacheSize) { 1514 import std.random; 1515 auto idx = uniform(0, cc.length); 1516 1517 cc[idx].socket.close(); 1518 loseSocketByKey(cc[idx].key, cc[idx].socket); 1519 1520 cc[idx] = cc[$ - 1]; 1521 cc = cc[0 .. $-1]; 1522 closeCount++; 1523 } 1524 debug(arsd_http2) writeln(closeCount, " from randomness"); 1525 } 1526 1527 cachedSockets -= closeCount; 1528 } 1529 1530 void loseSocketByKey(string key, Socket s) { 1531 if(auto list = key in socketsPerHost) { 1532 for(int a = 0; a < (*list).length; a++) { 1533 if((*list)[a] is s) { 1534 1535 for(int b = a; b < (*list).length - 1; b++) 1536 (*list)[b] = (*list)[b+1]; 1537 (*list) = (*list)[0 .. $-1]; 1538 break; 1539 } 1540 } 1541 } 1542 } 1543 1544 void loseSocket(string host, ushort port, bool ssl, Socket s) { 1545 import std.string; 1546 auto key = format("http%s://%s:%s", ssl ? "s" : "", host, port); 1547 1548 loseSocketByKey(key, s); 1549 } 1550 1551 Socket getOpenSocketOnHost(string proxy, string host, ushort port, bool ssl, string unixSocketPath, bool verifyPeer) { 1552 Socket openNewConnection() { 1553 Socket socket; 1554 if(ssl) { 1555 version(with_openssl) { 1556 loadOpenSsl(); 1557 socket = new SslClientSocket(family(unixSocketPath), SocketType.STREAM, host, verifyPeer); 1558 socket.blocking = false; 1559 } else 1560 throw new Exception("SSL not compiled in"); 1561 } else { 1562 socket = new Socket(family(unixSocketPath), SocketType.STREAM); 1563 socket.blocking = false; 1564 } 1565 1566 socket.setOption(SocketOptionLevel.TCP, SocketOption.TCP_NODELAY, 1); 1567 1568 // FIXME: connect timeout? 1569 if(unixSocketPath) { 1570 import std.stdio; writeln(cast(ubyte[]) unixSocketPath); 1571 socket.connect(new UnixAddress(unixSocketPath)); 1572 } else { 1573 // FIXME: i should prolly do ipv6 if available too. 1574 if(host.length == 0) // this could arguably also be an in contract since it is user error, but the exception is good enough 1575 throw new Exception("No host given for request"); 1576 if(proxy.length) { 1577 if(proxy.indexOf("//") == -1) 1578 proxy = "http://" ~ proxy; 1579 auto proxyurl = Uri(proxy); 1580 1581 //auto proxyhttps = proxyurl.scheme == "https"; 1582 enum proxyhttps = false; // this isn't properly implemented and might never be necessary anyway so meh 1583 1584 // the precise types here are important to help with overload 1585 // resolution of the devirtualized call! 1586 Address pa = new InternetAddress(proxyurl.host, proxyurl.port ? cast(ushort) proxyurl.port : 80); 1587 1588 debug(arsd_http2) writeln("using proxy ", pa.toString()); 1589 1590 if(proxyhttps) { 1591 socket.connect(pa); 1592 } else { 1593 // the proxy never actually starts TLS, but if the request is tls then we need to CONNECT then upgrade the connection 1594 // using the parent class functions let us bypass the encryption 1595 socket.Socket.connect(pa); 1596 } 1597 1598 socket.blocking = true; // FIXME total hack to simplify the code here since it isn't really using the event loop yet 1599 1600 string message; 1601 if(ssl) { 1602 auto hostName = host ~ ":" ~ to!string(port); 1603 message = "CONNECT " ~ hostName ~ " HTTP/1.1\r\n"; 1604 message ~= "Host: " ~ hostName ~ "\r\n"; 1605 if(proxyurl.userinfo.length) { 1606 import std.base64; 1607 message ~= "Proxy-Authorization: Basic " ~ Base64.encode(cast(ubyte[]) proxyurl.userinfo) ~ "\r\n"; 1608 } 1609 message ~= "\r\n"; 1610 1611 // FIXME: what if proxy times out? should be reasonably fast too. 1612 if(proxyhttps) { 1613 socket.send(message, SocketFlags.NONE); 1614 } else { 1615 socket.Socket.send(message, SocketFlags.NONE); 1616 } 1617 1618 ubyte[1024] recvBuffer; 1619 // and last time 1620 ptrdiff_t rcvGot; 1621 if(proxyhttps) { 1622 rcvGot = socket.receive(recvBuffer[], SocketFlags.NONE); 1623 // bool verifyPeer = true; 1624 //(cast(OpenSslSocket)socket).freeSsl(); 1625 //(cast(OpenSslSocket)socket).initSsl(verifyPeer, host); 1626 } else { 1627 rcvGot = socket.Socket.receive(recvBuffer[], SocketFlags.NONE); 1628 } 1629 1630 if(rcvGot == -1) 1631 throw new ProxyException("proxy receive error"); 1632 auto got = cast(string) recvBuffer[0 .. rcvGot]; 1633 auto expect = "HTTP/1.1 200"; 1634 if(got.length < expect.length || (got[0 .. expect.length] != expect && got[0 .. expect.length] != "HTTP/1.0 200")) 1635 throw new ProxyException("Proxy rejected request: " ~ got[0 .. expect.length <= got.length ? expect.length : got.length]); 1636 1637 if(proxyhttps) { 1638 //(cast(OpenSslSocket)socket).do_ssl_connect(); 1639 } else { 1640 (cast(OpenSslSocket)socket).do_ssl_connect(); 1641 } 1642 } else { 1643 } 1644 } else { 1645 socket.connect(new InternetAddress(host, port)); 1646 } 1647 } 1648 1649 debug(arsd_http2) writeln("opening to ", host, ":", port, " ", cast(void*) socket, " ssl=", ssl); 1650 assert(socket.handle() !is socket_t.init); 1651 return socket; 1652 } 1653 1654 // import std.stdio; writeln(cachedSockets); 1655 if(cachedSockets > connectionCacheSize) 1656 cleanOldSockets(); 1657 1658 import std.string; 1659 auto key = format("http%s://%s:%s", ssl ? "s" : "", host, port); 1660 1661 if(auto hostListing = key in socketsPerHost) { 1662 // try to find an available socket that is already open 1663 foreach(socket; *hostListing) { 1664 if(socket !in activeRequestOnSocket) { 1665 // let's see if it has closed since we last tried 1666 // e.g. a server timeout or something. If so, we need 1667 // to lose this one and immediately open a new one. 1668 static SocketSet readSet = null; 1669 if(readSet is null) 1670 readSet = new SocketSet(); 1671 readSet.reset(); 1672 assert(socket !is null); 1673 assert(socket.handle() !is socket_t.init, socket is null ? "null" : socket.toString()); 1674 readSet.add(socket); 1675 auto got = Socket.select(readSet, null, null, 0.msecs /* timeout, want it small since we just checking for eof */); 1676 if(got > 0) { 1677 // we can read something off this... but there aren't 1678 // any active requests. Assume it is EOF and open a new one 1679 1680 socket.close(); 1681 loseSocket(host, port, ssl, socket); 1682 goto openNew; 1683 } 1684 cachedSockets--; 1685 return socket; 1686 } 1687 } 1688 1689 // if not too many already open, go ahead and do a new one 1690 if((*hostListing).length < 6) { 1691 auto socket = openNewConnection(); 1692 (*hostListing) ~= socket; 1693 return socket; 1694 } else 1695 return null; // too many, you'll have to wait 1696 } 1697 1698 openNew: 1699 1700 auto socket = openNewConnection(); 1701 socketsPerHost[key] ~= socket; 1702 return socket; 1703 } 1704 1705 // stuff used by advanceConnections 1706 SocketSet readSet; 1707 SocketSet writeSet; 1708 private ubyte[] reusableBuffer; 1709 1710 /+ 1711 Generic event loop registration: 1712 1713 handle, operation (read/write), buffer (on posix it *might* be stack if a select loop), timeout (in real time), callback when op completed. 1714 1715 ....basically Windows style. Then it translates internally. 1716 1717 It should tell the thing if the buffer is reused or not 1718 +/ 1719 1720 1721 /++ 1722 This is made public for rudimentary event loop integration, but is still 1723 basically an internal detail. Try not to use it if you have another way. 1724 1725 This does a single iteration of the internal select()-based processing loop. 1726 1727 1728 Future directions: 1729 I want to merge the internal use of [WebSocket.eventLoop] with this; 1730 [advanceConnections] does just one run on the loop, whereas eventLoop 1731 runs it until all connections are closed. But they'd both process both 1732 pending http requests and active websockets. 1733 1734 After that, I want to be able to integrate in other event loops too. 1735 One might be to simply to reactor callbacks, then perhaps Windows overlapped 1736 i/o (that's just going to be tricky to retrofit into the existing select()-based 1737 code). It could then go fiber just by calling the resume function too. 1738 1739 The hard part is ensuring I keep this file stand-alone while offering these 1740 things. 1741 1742 This `advanceConnections` call will probably continue to work now that it is 1743 public, but it may not be wholly compatible with all the future features; you'd 1744 have to pick either the internal event loop or an external one you integrate, but not 1745 mix them. 1746 1747 History: 1748 This has been included in the library since almost day one, but 1749 it was private until April 13, 2021 (dub v9.5). 1750 1751 Params: 1752 maximumTimeout = the maximum time it will wait in select(). It may return much sooner than this if a connection timed out in the mean time. 1753 automaticallyRetryOnInterruption = internally loop on EINTR. 1754 1755 Returns: 1756 1757 0 = no error, work may remain so you should call `advanceConnections` again when you can 1758 1759 1 = passed `maximumTimeout` reached with no work done, yet requests are still in the queue. You may call `advanceConnections` again. 1760 1761 2 = no work to do, no point calling it again unless you've added new requests. Your program may exit if you have nothing to add since it means everything requested is now done. 1762 1763 3 = EINTR occurred on select(), you should check your interrupt flags if you set a signal handler, then call `advanceConnections` again if you aren't exiting. Only occurs if `automaticallyRetryOnInterruption` is set to `false` (the default when it is called externally). 1764 1765 any other value should be considered a non-recoverable error if you want to be forward compatible as I reserve the right to add more values later. 1766 +/ 1767 public int advanceConnections(Duration maximumTimeout = 10.seconds, bool automaticallyRetryOnInterruption = false) { 1768 debug(arsd_http2_verbose) writeln("advancing"); 1769 if(readSet is null) 1770 readSet = new SocketSet(); 1771 if(writeSet is null) 1772 writeSet = new SocketSet(); 1773 1774 if(reusableBuffer is null) 1775 reusableBuffer = new ubyte[](32 * 1024); 1776 ubyte[] buffer = reusableBuffer; 1777 1778 HttpRequest[16] removeFromPending; 1779 size_t removeFromPendingCount = 0; 1780 1781 bool hadAbortedRequest; 1782 1783 // are there pending requests? let's try to send them 1784 foreach(idx, pc; pending) { 1785 if(removeFromPendingCount == removeFromPending.length) 1786 break; 1787 1788 if(pc.state == HttpRequest.State.aborted) { 1789 removeFromPending[removeFromPendingCount++] = pc; 1790 hadAbortedRequest = true; 1791 continue; 1792 } 1793 1794 Socket socket; 1795 1796 try { 1797 socket = getOpenSocketOnHost(pc.proxy, pc.requestParameters.host, pc.requestParameters.port, pc.requestParameters.ssl, pc.requestParameters.unixSocketPath, pc.verifyPeer); 1798 } catch(ProxyException e) { 1799 // connection refused or timed out (I should disambiguate somehow)... 1800 pc.state = HttpRequest.State.aborted; 1801 1802 pc.responseData.code = 2; 1803 pc.responseData.codeText = e.msg ~ " from " ~ pc.proxy; 1804 1805 hadAbortedRequest = true; 1806 1807 removeFromPending[removeFromPendingCount++] = pc; 1808 continue; 1809 1810 } catch(SocketException e) { 1811 // connection refused or timed out (I should disambiguate somehow)... 1812 pc.state = HttpRequest.State.aborted; 1813 1814 pc.responseData.code = 2; 1815 pc.responseData.codeText = pc.proxy.length ? ("connection failed to proxy " ~ pc.proxy) : "connection failed"; 1816 1817 hadAbortedRequest = true; 1818 1819 removeFromPending[removeFromPendingCount++] = pc; 1820 continue; 1821 } catch(Exception e) { 1822 // connection failed due to other user error or SSL (i should disambiguate somehow)... 1823 pc.state = HttpRequest.State.aborted; 1824 1825 pc.responseData.code = 2; 1826 pc.responseData.codeText = e.msg; 1827 1828 hadAbortedRequest = true; 1829 1830 removeFromPending[removeFromPendingCount++] = pc; 1831 continue; 1832 1833 } 1834 1835 if(socket !is null) { 1836 activeRequestOnSocket[socket] = pc; 1837 assert(pc.sendBuffer.length); 1838 pc.state = State.connecting; 1839 1840 removeFromPending[removeFromPendingCount++] = pc; 1841 } 1842 } 1843 1844 import std.algorithm : remove; 1845 foreach(rp; removeFromPending[0 .. removeFromPendingCount]) 1846 pending = pending.remove!((a) => a is rp)(); 1847 1848 tryAgain: 1849 1850 Socket[16] inactive; 1851 int inactiveCount = 0; 1852 void killInactives() { 1853 foreach(s; inactive[0 .. inactiveCount]) { 1854 debug(arsd_http2) writeln("removing socket from active list ", cast(void*) s); 1855 activeRequestOnSocket.remove(s); 1856 cachedSockets++; 1857 } 1858 } 1859 1860 1861 readSet.reset(); 1862 writeSet.reset(); 1863 1864 bool hadOne = false; 1865 1866 auto minTimeout = maximumTimeout; 1867 auto now = MonoTime.currTime; 1868 1869 // active requests need to be read or written to 1870 foreach(sock, request; activeRequestOnSocket) { 1871 1872 if(request.state == State.aborted) { 1873 inactive[inactiveCount++] = sock; 1874 sock.close(); 1875 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1876 hadAbortedRequest = true; 1877 continue; 1878 } 1879 1880 // check the other sockets just for EOF, if they close, take them out of our list, 1881 // we'll reopen if needed upon request. 1882 readSet.add(sock); 1883 hadOne = true; 1884 1885 Duration timeo; 1886 if(request.timeoutFromInactivity <= now) 1887 timeo = 0.seconds; 1888 else 1889 timeo = request.timeoutFromInactivity - now; 1890 1891 if(timeo < minTimeout) 1892 minTimeout = timeo; 1893 1894 if(request.state == State.connecting || request.state == State.sslConnectPendingWrite || request.state == State.sendingHeaders || request.state == State.sendingBody) { 1895 writeSet.add(sock); 1896 hadOne = true; 1897 } 1898 } 1899 1900 if(!hadOne) { 1901 if(hadAbortedRequest) { 1902 killInactives(); 1903 return 0; // something got aborted, that's progress 1904 } 1905 return 2; // automatic timeout, nothing to do 1906 } 1907 1908 auto selectGot = Socket.select(readSet, writeSet, null, minTimeout); 1909 if(selectGot == 0) { /* timeout */ 1910 now = MonoTime.currTime; 1911 bool anyWorkDone = false; 1912 foreach(sock, request; activeRequestOnSocket) { 1913 1914 if(request.timeoutFromInactivity <= now) { 1915 request.state = HttpRequest.State.aborted; 1916 request.responseData.code = 5; 1917 if(request.state == State.connecting) 1918 request.responseData.codeText = "Connect timed out"; 1919 else 1920 request.responseData.codeText = "Request timed out"; 1921 1922 inactive[inactiveCount++] = sock; 1923 sock.close(); 1924 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1925 anyWorkDone = true; 1926 } 1927 } 1928 killInactives(); 1929 return anyWorkDone ? 0 : 1; 1930 // return 1; was an error to time out but now im making it on the individual request 1931 } else if(selectGot == -1) { /* interrupted */ 1932 /* 1933 version(Posix) { 1934 import core.stdc.errno; 1935 if(errno != EINTR) 1936 throw new Exception("select error: " ~ to!string(errno)); 1937 } 1938 */ 1939 if(automaticallyRetryOnInterruption) 1940 goto tryAgain; 1941 else 1942 return 3; 1943 } else { /* ready */ 1944 1945 void sslProceed(HttpRequest request, SslClientSocket s) { 1946 try { 1947 auto code = s.do_ssl_connect(); 1948 switch(code) { 1949 case 0: 1950 request.state = State.sendingHeaders; 1951 break; 1952 case SSL_ERROR_WANT_READ: 1953 request.state = State.sslConnectPendingRead; 1954 break; 1955 case SSL_ERROR_WANT_WRITE: 1956 request.state = State.sslConnectPendingWrite; 1957 break; 1958 default: 1959 assert(0); 1960 } 1961 } catch(Exception e) { 1962 request.state = State.aborted; 1963 1964 request.responseData.code = 2; 1965 request.responseData.codeText = e.msg; 1966 inactive[inactiveCount++] = s; 1967 s.close(); 1968 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, s); 1969 } 1970 } 1971 1972 1973 foreach(sock, request; activeRequestOnSocket) { 1974 // always need to try to send first in part because http works that way but 1975 // also because openssl will sometimes leave something ready to read even if we haven't 1976 // sent yet (probably leftover data from the crypto negotiation) and if that happens ssl 1977 // is liable to block forever hogging the connection and not letting it send... 1978 if(request.state == State.connecting) 1979 if(writeSet.isSet(sock) || readSet.isSet(sock)) { 1980 import core.stdc.stdint; 1981 int32_t error; 1982 int retopt = sock.getOption(SocketOptionLevel.SOCKET, SocketOption.ERROR, error); 1983 if(retopt < 0 || error != 0) { 1984 request.state = State.aborted; 1985 1986 request.responseData.code = 2; 1987 try { 1988 request.responseData.codeText = "connection failed - " ~ formatSocketError(error); 1989 } catch(Exception e) { 1990 request.responseData.codeText = "connection failed"; 1991 } 1992 inactive[inactiveCount++] = sock; 1993 sock.close(); 1994 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 1995 continue; 1996 } else { 1997 if(auto s = cast(SslClientSocket) sock) { 1998 sslProceed(request, s); 1999 continue; 2000 } else { 2001 request.state = State.sendingHeaders; 2002 } 2003 } 2004 } 2005 2006 if(request.state == State.sslConnectPendingRead) 2007 if(readSet.isSet(sock)) { 2008 sslProceed(request, cast(SslClientSocket) sock); 2009 continue; 2010 } 2011 if(request.state == State.sslConnectPendingWrite) 2012 if(writeSet.isSet(sock)) { 2013 sslProceed(request, cast(SslClientSocket) sock); 2014 continue; 2015 } 2016 2017 if(request.state == State.sendingHeaders || request.state == State.sendingBody) 2018 if(writeSet.isSet(sock)) { 2019 request.timeoutFromInactivity = MonoTime.currTime + request.requestParameters.timeoutFromInactivity; 2020 assert(request.sendBuffer.length); 2021 auto sent = sock.send(request.sendBuffer); 2022 debug(arsd_http2_verbose) writeln(cast(void*) sock, "<send>", cast(string) request.sendBuffer, "</send>"); 2023 if(sent <= 0) { 2024 if(wouldHaveBlocked()) 2025 continue; 2026 2027 request.state = State.aborted; 2028 2029 request.responseData.code = 3; 2030 request.responseData.codeText = "send failed to server: " ~ lastSocketError(sock); 2031 inactive[inactiveCount++] = sock; 2032 sock.close(); 2033 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 2034 continue; 2035 2036 } 2037 request.sendBuffer = request.sendBuffer[sent .. $]; 2038 if(request.sendBuffer.length == 0) { 2039 request.state = State.waitingForResponse; 2040 2041 debug(arsd_http2_verbose) writeln("all sent"); 2042 } 2043 } 2044 2045 2046 if(readSet.isSet(sock)) { 2047 keep_going: 2048 request.timeoutFromInactivity = MonoTime.currTime + request.requestParameters.timeoutFromInactivity; 2049 auto got = sock.receive(buffer); 2050 debug(arsd_http2_verbose) { if(got < 0) writeln(lastSocketError); else writeln("====PACKET ",got,"=====",cast(string)buffer[0 .. got],"===/PACKET==="); } 2051 if(got < 0) { 2052 if(wouldHaveBlocked()) 2053 continue; 2054 debug(arsd_http2) writeln("receive error"); 2055 if(request.state != State.complete) { 2056 request.state = State.aborted; 2057 2058 request.responseData.code = 3; 2059 request.responseData.codeText = "receive error from server: " ~ lastSocketError(sock); 2060 } 2061 inactive[inactiveCount++] = sock; 2062 sock.close(); 2063 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 2064 } else if(got == 0) { 2065 // remote side disconnected 2066 debug(arsd_http2) writeln("remote disconnect"); 2067 if(request.state != State.complete) { 2068 request.state = State.aborted; 2069 2070 request.responseData.code = 3; 2071 request.responseData.codeText = "server disconnected"; 2072 } 2073 inactive[inactiveCount++] = sock; 2074 sock.close(); 2075 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 2076 } else { 2077 // data available 2078 bool stillAlive; 2079 2080 try { 2081 stillAlive = request.handleIncomingData(buffer[0 .. got]); 2082 /+ 2083 state needs to be set and public 2084 requestData.content/contentText needs to be around 2085 you need to be able to clear the content and keep processing for things like event sources. 2086 also need to be able to abort it. 2087 2088 and btw it should prolly just have evnet source as a pre-packaged thing. 2089 +/ 2090 } catch (Exception e) { 2091 debug(arsd_http2_verbose) { import std.stdio; writeln(e); } 2092 request.state = HttpRequest.State.aborted; 2093 request.responseData.code = 4; 2094 request.responseData.codeText = e.msg; 2095 2096 inactive[inactiveCount++] = sock; 2097 sock.close(); 2098 loseSocket(request.requestParameters.host, request.requestParameters.port, request.requestParameters.ssl, sock); 2099 } 2100 2101 if(!stillAlive || request.state == HttpRequest.State.complete || request.state == HttpRequest.State.aborted) { 2102 //import std.stdio; writeln(cast(void*) sock, " ", stillAlive, " ", request.state); 2103 inactive[inactiveCount++] = sock; 2104 // reuse the socket for another pending request, if we can 2105 } 2106 } 2107 2108 if(request.onDataReceived) 2109 request.onDataReceived(request); 2110 2111 version(with_openssl) 2112 if(auto s = cast(SslClientSocket) sock) { 2113 // select doesn't handle the case with stuff 2114 // left in the ssl buffer so i'm checking it separately 2115 if(s.dataPending()) { 2116 goto keep_going; 2117 } 2118 } 2119 } 2120 } 2121 } 2122 2123 killInactives(); 2124 2125 // we've completed a request, are there any more pending connection? if so, send them now 2126 2127 return 0; 2128 } 2129 } 2130 2131 public static void resetInternals() { 2132 socketsPerHost = null; 2133 activeRequestOnSocket = null; 2134 pending = null; 2135 2136 } 2137 2138 struct HeaderReadingState { 2139 bool justSawLf; 2140 bool justSawCr; 2141 bool atStartOfLine = true; 2142 bool readingLineContinuation; 2143 } 2144 HeaderReadingState headerReadingState; 2145 2146 struct BodyReadingState { 2147 bool isGzipped; 2148 bool isDeflated; 2149 2150 bool isChunked; 2151 int chunkedState; 2152 2153 // used for the chunk size if it is chunked 2154 int contentLengthRemaining; 2155 } 2156 BodyReadingState bodyReadingState; 2157 2158 bool closeSocketWhenComplete; 2159 2160 import std.zlib; 2161 UnCompress uncompress; 2162 2163 const(ubyte)[] leftoverDataFromLastTime; 2164 2165 bool handleIncomingData(scope const ubyte[] dataIn) { 2166 bool stillAlive = true; 2167 debug(arsd_http2) writeln("handleIncomingData, state: ", state); 2168 if(state == State.waitingForResponse) { 2169 state = State.readingHeaders; 2170 headerReadingState = HeaderReadingState.init; 2171 bodyReadingState = BodyReadingState.init; 2172 } 2173 2174 const(ubyte)[] data; 2175 if(leftoverDataFromLastTime.length) 2176 data = leftoverDataFromLastTime ~ dataIn[]; 2177 else 2178 data = dataIn[]; 2179 2180 if(state == State.readingHeaders) { 2181 void parseLastHeader() { 2182 assert(responseData.headers.length); 2183 if(responseData.headers.length == 1) { 2184 responseData.statusLine = responseData.headers[0]; 2185 import std.algorithm; 2186 auto parts = responseData.statusLine.splitter(" "); 2187 responseData.httpVersion = parts.front; 2188 parts.popFront(); 2189 if(parts.empty) 2190 throw new Exception("Corrupted response, bad status line"); 2191 responseData.code = to!int(parts.front()); 2192 parts.popFront(); 2193 responseData.codeText = ""; 2194 while(!parts.empty) { 2195 // FIXME: this sucks! 2196 responseData.codeText ~= parts.front(); 2197 parts.popFront(); 2198 if(!parts.empty) 2199 responseData.codeText ~= " "; 2200 } 2201 } else { 2202 // parse the new header 2203 auto header = responseData.headers[$-1]; 2204 2205 auto colon = header.indexOf(":"); 2206 if(colon < 0 || colon >= header.length) 2207 return; 2208 auto name = toLower(header[0 .. colon]); 2209 auto value = header[colon + 1 .. $].strip; // skip colon and strip whitespace 2210 2211 switch(name) { 2212 case "connection": 2213 if(value == "close") 2214 closeSocketWhenComplete = true; 2215 break; 2216 case "content-type": 2217 responseData.contentType = value; 2218 break; 2219 case "location": 2220 responseData.location = value; 2221 break; 2222 case "content-length": 2223 bodyReadingState.contentLengthRemaining = to!int(value); 2224 // preallocate the buffer for a bit of a performance boost 2225 responseData.content.reserve(bodyReadingState.contentLengthRemaining); 2226 break; 2227 case "transfer-encoding": 2228 // note that if it is gzipped, it zips first, then chunks the compressed stream. 2229 // so we should always dechunk first, then feed into the decompressor 2230 if(value == "chunked") 2231 bodyReadingState.isChunked = true; 2232 else throw new Exception("Unknown Transfer-Encoding: " ~ value); 2233 break; 2234 case "content-encoding": 2235 if(value == "gzip") { 2236 bodyReadingState.isGzipped = true; 2237 uncompress = new UnCompress(); 2238 } else if(value == "deflate") { 2239 bodyReadingState.isDeflated = true; 2240 uncompress = new UnCompress(); 2241 } else throw new Exception("Unknown Content-Encoding: " ~ value); 2242 break; 2243 case "set-cookie": 2244 // handled elsewhere fyi 2245 break; 2246 default: 2247 // ignore 2248 } 2249 2250 responseData.headersHash[name] = value; 2251 } 2252 } 2253 2254 size_t position = 0; 2255 for(position = 0; position < data.length; position++) { 2256 if(headerReadingState.readingLineContinuation) { 2257 if(data[position] == ' ' || data[position] == '\t') 2258 continue; 2259 headerReadingState.readingLineContinuation = false; 2260 } 2261 2262 if(headerReadingState.atStartOfLine) { 2263 headerReadingState.atStartOfLine = false; 2264 // FIXME it being \r should never happen... and i don't think it does 2265 if(data[position] == '\r' || data[position] == '\n') { 2266 // done with headers 2267 2268 position++; // skip the \r 2269 2270 if(responseData.headers.length) 2271 parseLastHeader(); 2272 2273 if(responseData.code >= 100 && responseData.code < 200) { 2274 // "100 Continue" - we should continue uploading request data at this point 2275 // "101 Switching Protocols" - websocket, not expected here... 2276 // "102 Processing" - server still working, keep the connection alive 2277 // "103 Early Hints" - can have useful Link headers etc 2278 // 2279 // and other unrecognized ones can just safely be skipped 2280 2281 // FIXME: the headers shouldn't actually be reset; 103 Early Hints 2282 // can give useful headers we want to keep 2283 2284 responseData.headers = null; 2285 headerReadingState.atStartOfLine = true; 2286 2287 continue; // the \n will be skipped by the for loop advance 2288 } 2289 2290 if(this.requestParameters.method == HttpVerb.HEAD) 2291 state = State.complete; 2292 else 2293 state = State.readingBody; 2294 2295 // skip the \n before we break 2296 position++; 2297 2298 break; 2299 } else if(data[position] == ' ' || data[position] == '\t') { 2300 // line continuation, ignore all whitespace and collapse it into a space 2301 headerReadingState.readingLineContinuation = true; 2302 responseData.headers[$-1] ~= ' '; 2303 } else { 2304 // new header 2305 if(responseData.headers.length) 2306 parseLastHeader(); 2307 responseData.headers ~= ""; 2308 } 2309 } 2310 2311 if(data[position] == '\r') { 2312 headerReadingState.justSawCr = true; 2313 continue; 2314 } else 2315 headerReadingState.justSawCr = false; 2316 2317 if(data[position] == '\n') { 2318 headerReadingState.justSawLf = true; 2319 headerReadingState.atStartOfLine = true; 2320 continue; 2321 } else 2322 headerReadingState.justSawLf = false; 2323 2324 responseData.headers[$-1] ~= data[position]; 2325 } 2326 2327 data = data[position .. $]; 2328 } 2329 2330 if(state == State.readingBody) { 2331 if(bodyReadingState.isChunked) { 2332 // read the hex length, stopping at a \r\n, ignoring everything between the new line but after the first non-valid hex character 2333 // read binary data of that length. it is our content 2334 // repeat until a zero sized chunk 2335 // then read footers as headers. 2336 2337 start_over: 2338 for(int a = 0; a < data.length; a++) { 2339 final switch(bodyReadingState.chunkedState) { 2340 case 0: // reading hex 2341 char c = data[a]; 2342 if((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) { 2343 // just keep reading 2344 } else { 2345 int power = 1; 2346 bodyReadingState.contentLengthRemaining = 0; 2347 if(a == 0) 2348 break; // just wait for more data 2349 assert(a != 0, cast(string) data); 2350 for(int b = a-1; b >= 0; b--) { 2351 char cc = data[b]; 2352 if(cc >= 'a' && cc <= 'z') 2353 cc -= 0x20; 2354 int val = 0; 2355 if(cc >= '0' && cc <= '9') 2356 val = cc - '0'; 2357 else 2358 val = cc - 'A' + 10; 2359 2360 assert(val >= 0 && val <= 15, to!string(val)); 2361 bodyReadingState.contentLengthRemaining += power * val; 2362 power *= 16; 2363 } 2364 debug(arsd_http2_verbose) writeln("Chunk length: ", bodyReadingState.contentLengthRemaining); 2365 bodyReadingState.chunkedState = 1; 2366 data = data[a + 1 .. $]; 2367 goto start_over; 2368 } 2369 break; 2370 case 1: // reading until end of line 2371 char c = data[a]; 2372 if(c == '\n') { 2373 if(bodyReadingState.contentLengthRemaining == 0) 2374 bodyReadingState.chunkedState = 5; 2375 else 2376 bodyReadingState.chunkedState = 2; 2377 } 2378 data = data[a + 1 .. $]; 2379 goto start_over; 2380 case 2: // reading data 2381 auto can = a + bodyReadingState.contentLengthRemaining; 2382 if(can > data.length) 2383 can = cast(int) data.length; 2384 2385 auto newData = data[a .. can]; 2386 data = data[can .. $]; 2387 2388 //if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) 2389 // responseData.content ~= cast(ubyte[]) uncompress.uncompress(data[a .. can]); 2390 //else 2391 responseData.content ~= newData; 2392 2393 bodyReadingState.contentLengthRemaining -= newData.length; 2394 debug(arsd_http2_verbose) writeln("clr: ", bodyReadingState.contentLengthRemaining, " " , a, " ", can); 2395 assert(bodyReadingState.contentLengthRemaining >= 0); 2396 if(bodyReadingState.contentLengthRemaining == 0) { 2397 bodyReadingState.chunkedState = 3; 2398 } else { 2399 // will continue grabbing more 2400 } 2401 goto start_over; 2402 case 3: // reading 13/10 2403 assert(data[a] == 13); 2404 bodyReadingState.chunkedState++; 2405 data = data[a + 1 .. $]; 2406 goto start_over; 2407 case 4: // reading 10 at end of packet 2408 assert(data[a] == 10); 2409 data = data[a + 1 .. $]; 2410 bodyReadingState.chunkedState = 0; 2411 goto start_over; 2412 case 5: // reading footers 2413 //goto done; // FIXME 2414 2415 int footerReadingState = 0; 2416 int footerSize; 2417 2418 while(footerReadingState != 2 && a < data.length) { 2419 // import std.stdio; writeln(footerReadingState, " ", footerSize, " ", data); 2420 switch(footerReadingState) { 2421 case 0: 2422 if(data[a] == 13) 2423 footerReadingState++; 2424 else 2425 footerSize++; 2426 break; 2427 case 1: 2428 if(data[a] == 10) { 2429 if(footerSize == 0) { 2430 // all done, time to break 2431 footerReadingState++; 2432 2433 } else { 2434 // actually had a footer, try to read another 2435 footerReadingState = 0; 2436 footerSize = 0; 2437 } 2438 } else { 2439 throw new Exception("bad footer thing"); 2440 } 2441 break; 2442 default: 2443 assert(0); 2444 } 2445 2446 a++; 2447 } 2448 2449 if(footerReadingState != 2) 2450 break start_over; // haven't hit the end of the thing yet 2451 2452 bodyReadingState.chunkedState = 0; 2453 data = data[a .. $]; 2454 2455 if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) { 2456 auto n = uncompress.uncompress(responseData.content); 2457 n ~= uncompress.flush(); 2458 responseData.content = cast(ubyte[]) n; 2459 } 2460 2461 // responseData.content ~= cast(ubyte[]) uncompress.flush(); 2462 responseData.contentText = cast(string) responseData.content; 2463 2464 goto done; 2465 } 2466 } 2467 2468 } else { 2469 //if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) 2470 // responseData.content ~= cast(ubyte[]) uncompress.uncompress(data); 2471 //else 2472 responseData.content ~= data; 2473 //assert(data.length <= bodyReadingState.contentLengthRemaining, format("%d <= %d\n%s", data.length, bodyReadingState.contentLengthRemaining, cast(string)data)); 2474 { 2475 int use = cast(int) data.length; 2476 if(use > bodyReadingState.contentLengthRemaining) 2477 use = bodyReadingState.contentLengthRemaining; 2478 bodyReadingState.contentLengthRemaining -= use; 2479 data = data[use .. $]; 2480 } 2481 if(bodyReadingState.contentLengthRemaining == 0) { 2482 if(bodyReadingState.isGzipped || bodyReadingState.isDeflated) { 2483 // import std.stdio; writeln(responseData.content.length, " ", responseData.content[0 .. 2], " .. ", responseData.content[$-2 .. $]); 2484 auto n = uncompress.uncompress(responseData.content); 2485 n ~= uncompress.flush(); 2486 responseData.content = cast(ubyte[]) n; 2487 responseData.contentText = cast(string) responseData.content; 2488 //responseData.content ~= cast(ubyte[]) uncompress.flush(); 2489 } else { 2490 responseData.contentText = cast(string) responseData.content; 2491 } 2492 2493 done: 2494 2495 if(retainCookies && client !is null) { 2496 client.retainCookies(responseData); 2497 } 2498 2499 if(followLocation && responseData.location.length) { 2500 if(maximumNumberOfRedirectsRemaining <= 0) { 2501 throw new Exception("Maximum number of redirects exceeded"); 2502 } else { 2503 maximumNumberOfRedirectsRemaining--; 2504 } 2505 2506 static bool first = true; 2507 //version(DigitalMars) if(!first) asm { int 3; } 2508 debug(arsd_http2) writeln("redirecting to ", responseData.location); 2509 populateFromInfo(Uri(responseData.location), HttpVerb.GET); 2510 //import std.stdio; writeln("redirected to ", responseData.location); 2511 first = false; 2512 responseData = HttpResponse.init; 2513 headerReadingState = HeaderReadingState.init; 2514 bodyReadingState = BodyReadingState.init; 2515 if(client !is null) { 2516 // FIXME: this won't clear cookies that were cleared in another request 2517 client.populateCookies(this); // they might have changed in the previous redirection cycle! 2518 } 2519 state = State.unsent; 2520 stillAlive = false; 2521 sendPrivate(false); 2522 } else { 2523 state = State.complete; 2524 // FIXME 2525 //if(closeSocketWhenComplete) 2526 //socket.close(); 2527 } 2528 } 2529 } 2530 } 2531 2532 if(data.length) 2533 leftoverDataFromLastTime = data.dup; 2534 else 2535 leftoverDataFromLastTime = null; 2536 2537 return stillAlive; 2538 } 2539 2540 } 2541 } 2542 2543 /++ 2544 Waits for the first of the given requests to be either aborted or completed. 2545 Returns the first one in that state, or `null` if the operation was interrupted 2546 or reached the given timeout before any completed. (If it returns null even before 2547 the timeout, it might be because the user pressed ctrl+c, so you should consider 2548 checking if you should cancel the operation. If not, you can simply call it again 2549 with the same arguments to start waiting again.) 2550 2551 You MUST check for null, even if you don't specify a timeout! 2552 2553 Note that if an individual request times out before any others request, it will 2554 return that timed out request, since that counts as completion. 2555 2556 If the return is not null, you should call `waitForCompletion` on the given request 2557 to get the response out. It will not have to wait since it is guaranteed to be 2558 finished when returned by this function; that will just give you the cached response. 2559 2560 (I thought about just having it return the response, but tying a response back to 2561 a request is harder than just getting the original request object back and taking 2562 the response out of it.) 2563 2564 Please note: if a request in the set has already completed or been aborted, it will 2565 always return the first one it sees upon calling the function. You may wish to remove 2566 them from the list before calling the function. 2567 2568 History: 2569 Added December 24, 2021 (dub v10.5) 2570 +/ 2571 HttpRequest waitForFirstToComplete(Duration timeout, HttpRequest[] requests...) { 2572 2573 foreach(request; requests) { 2574 if(request.state == HttpRequest.State.unsent) 2575 request.send(); 2576 else if(request.state == HttpRequest.State.complete) 2577 return request; 2578 else if(request.state == HttpRequest.State.aborted) 2579 return request; 2580 } 2581 2582 while(true) { 2583 if(auto err = HttpRequest.advanceConnections(timeout)) { 2584 switch(err) { 2585 case 1: return null; 2586 case 2: throw new Exception("HttpRequest.advanceConnections returned 2: nothing to do"); 2587 case 3: return null; 2588 default: throw new Exception("HttpRequest.advanceConnections got err " ~ to!string(err)); 2589 } 2590 } 2591 2592 foreach(request; requests) { 2593 if(request.state == HttpRequest.State.aborted || request.state == HttpRequest.State.complete) { 2594 request.waitForCompletion(); 2595 return request; 2596 } 2597 } 2598 2599 } 2600 } 2601 2602 /// ditto 2603 HttpRequest waitForFirstToComplete(HttpRequest[] requests...) { 2604 return waitForFirstToComplete(1.weeks, requests); 2605 } 2606 2607 /++ 2608 An input range that runs [waitForFirstToComplete] but only returning each request once. 2609 Before you loop over it, you can set some properties to customize behavior. 2610 2611 If it times out or is interrupted, it will prematurely run empty. You can set the delegate 2612 to process this. 2613 2614 Implementation note: each iteration through the loop does a O(n) check over each item remaining. 2615 This shouldn't matter, but if it does become an issue for you, let me know. 2616 2617 History: 2618 Added December 24, 2021 (dub v10.5) 2619 +/ 2620 struct HttpRequestsAsTheyComplete { 2621 /++ 2622 Seeds it with an overall timeout and the initial requests. 2623 It will send all the requests before returning, then will process 2624 the responses as they come. 2625 2626 Please note that it modifies the array of requests you pass in! It 2627 will keep a reference to it and reorder items on each call of popFront. 2628 You might want to pass a duplicate if you have another purpose for your 2629 array and don't want to see it shuffled. 2630 +/ 2631 this(Duration timeout, HttpRequest[] requests) { 2632 remainingRequests = requests; 2633 this.timeout = timeout; 2634 popFront(); 2635 } 2636 2637 /++ 2638 You can set this delegate to decide how to handle an interruption. Returning true 2639 from this will keep working. Returning false will terminate the loop. 2640 2641 If this is null, an interruption will always terminate the loop. 2642 2643 Note that interruptions can be caused by the garbage collector being triggered by 2644 another thread as well as by user action. If you don't set a SIGINT handler, it 2645 might be reasonable to always return true here. 2646 +/ 2647 bool delegate() onInterruption; 2648 2649 private HttpRequest[] remainingRequests; 2650 2651 /// The timeout you set in the constructor. You can change it if you want. 2652 Duration timeout; 2653 2654 /++ 2655 Adds another request to the work queue. It is safe to call this from inside the loop 2656 as you process other requests. 2657 +/ 2658 void appendRequest(HttpRequest request) { 2659 remainingRequests ~= request; 2660 } 2661 2662 /++ 2663 If the loop exited, it might be due to an interruption or a time out. If you like, you 2664 can call this to pick up the work again, 2665 2666 If it returns `false`, the work is indeed all finished and you should not re-enter the loop. 2667 2668 --- 2669 auto range = HttpRequestsAsTheyComplete(10.seconds, your_requests); 2670 process_loop: foreach(req; range) { 2671 // process req 2672 } 2673 // make sure we weren't interrupted because the user requested we cancel! 2674 // but then try to re-enter the range if possible 2675 if(!user_quit && range.reenter()) { 2676 // there's still something unprocessed in there 2677 // range.reenter returning true means it is no longer 2678 // empty, so we should try to loop over it again 2679 goto process_loop; // re-enter the loop 2680 } 2681 --- 2682 +/ 2683 bool reenter() { 2684 if(remainingRequests.length == 0) 2685 return false; 2686 empty = false; 2687 popFront(); 2688 return true; 2689 } 2690 2691 /// Standard range primitives. I reserve the right to change the variables to read-only properties in the future without notice. 2692 HttpRequest front; 2693 2694 /// ditto 2695 bool empty; 2696 2697 /// ditto 2698 void popFront() { 2699 resume: 2700 if(remainingRequests.length == 0) { 2701 empty = true; 2702 return; 2703 } 2704 2705 front = waitForFirstToComplete(timeout, remainingRequests); 2706 2707 if(front is null) { 2708 if(onInterruption) { 2709 if(onInterruption()) 2710 goto resume; 2711 } 2712 empty = true; 2713 return; 2714 } 2715 foreach(idx, req; remainingRequests) { 2716 if(req is front) { 2717 remainingRequests[idx] = remainingRequests[$ - 1]; 2718 remainingRequests = remainingRequests[0 .. $ - 1]; 2719 return; 2720 } 2721 } 2722 } 2723 } 2724 2725 // 2726 struct HttpRequestParameters { 2727 // FIXME: implement these 2728 //Duration timeoutTotal; // the whole request must finish in this time or else it fails,even if data is still trickling in 2729 Duration timeoutFromInactivity; // if there's no activity in this time it dies. basically the socket receive timeout 2730 2731 // debugging 2732 bool useHttp11 = true; /// 2733 bool acceptGzip = true; /// 2734 bool keepAlive = true; /// 2735 2736 // the request itself 2737 HttpVerb method; /// 2738 string host; /// 2739 ushort port; /// 2740 string uri; /// 2741 2742 bool ssl; /// 2743 2744 string userAgent; /// 2745 string authorization; /// 2746 2747 string[string] cookies; /// 2748 2749 string[] headers; /// do not duplicate host, content-length, content-type, or any others that have a specific property 2750 2751 string contentType; /// 2752 ubyte[] bodyData; /// 2753 2754 string unixSocketPath; /// 2755 2756 bool gzipBody; /// 2757 } 2758 2759 interface IHttpClient { 2760 2761 } 2762 2763 /// 2764 enum HttpVerb { 2765 /// 2766 GET, 2767 /// 2768 HEAD, 2769 /// 2770 POST, 2771 /// 2772 PUT, 2773 /// 2774 DELETE, 2775 /// 2776 OPTIONS, 2777 /// 2778 TRACE, 2779 /// 2780 CONNECT, 2781 /// 2782 PATCH, 2783 /// 2784 MERGE 2785 } 2786 2787 /++ 2788 Supported file formats for [HttpClient.setClientCert]. These are loaded by OpenSSL 2789 in the current implementation. 2790 2791 History: 2792 Added February 3, 2022 (dub v10.6) 2793 +/ 2794 enum CertificateFileFormat { 2795 guess, /// try to guess the format from the file name and/or contents 2796 pem, /// the files are specifically in PEM format 2797 der /// the files are specifically in DER format 2798 } 2799 2800 /++ 2801 HttpClient keeps cookies, location, and some other state to reuse connections, when possible, like a web browser. 2802 You can use it as your entry point to make http requests. 2803 2804 See the example on [arsd.http2#examples]. 2805 +/ 2806 class HttpClient { 2807 /* Protocol restrictions, useful to disable when debugging servers */ 2808 bool useHttp11 = true; /// 2809 bool acceptGzip = true; /// 2810 bool keepAlive = true; /// 2811 2812 /++ 2813 Sets the client certificate used as a log in identifier on https connections. 2814 The certificate and key must be unencrypted at this time and both must be in 2815 the same file format. 2816 2817 Bugs: 2818 The current implementation sets the filenames into a static variable, 2819 meaning it is shared across all clients and connections. 2820 2821 Errors in the cert or key are only reported if the server reports an 2822 authentication failure. Make sure you are passing correct filenames 2823 and formats of you do see a failure. 2824 2825 History: 2826 Added February 2, 2022 (dub v10.6) 2827 +/ 2828 void setClientCertificate(string certFilename, string keyFilename, CertificateFileFormat certFormat = CertificateFileFormat.guess) { 2829 this.certFilename = certFilename; 2830 this.keyFilename = keyFilename; 2831 this.certFormat = certFormat; 2832 } 2833 2834 /++ 2835 Sets whether [HttpRequest]s created through this object (with [navigateTo], [request], etc.), will have the 2836 value of [HttpRequest.verifyPeer] of true or false upon construction. 2837 2838 History: 2839 Added April 5, 2022 (dub v10.8). Previously, there was an undocumented global value used. 2840 +/ 2841 bool defaultVerifyPeer = true; 2842 2843 /++ 2844 Adds a header to be automatically appended to each request created through this client. 2845 2846 If you add duplicate headers, it will add multiple copies. 2847 2848 You should NOT use this to add headers that can be set through other properties like [userAgent], [authorization], or [setCookie]. 2849 2850 History: 2851 Added July 12, 2023 2852 +/ 2853 void addDefaultHeader(string key, string value) { 2854 defaultHeaders ~= key ~ ": " ~ value; 2855 } 2856 2857 private string[] defaultHeaders; 2858 2859 // FIXME: getCookies api 2860 // FIXME: an easy way to download files 2861 2862 // FIXME: try to not make these static 2863 private static string certFilename; 2864 private static string keyFilename; 2865 private static CertificateFileFormat certFormat; 2866 2867 /// 2868 @property Uri location() { 2869 return currentUrl; 2870 } 2871 2872 /++ 2873 Default timeout for requests created on this client. 2874 2875 History: 2876 Added March 31, 2021 2877 +/ 2878 Duration defaultTimeout = 10.seconds; 2879 2880 /++ 2881 High level function that works similarly to entering a url 2882 into a browser. 2883 2884 Follows locations, retain cookies, updates the current url, etc. 2885 +/ 2886 HttpRequest navigateTo(Uri where, HttpVerb method = HttpVerb.GET) { 2887 currentUrl = where.basedOn(currentUrl); 2888 currentDomain = where.host; 2889 2890 auto request = this.request(currentUrl, method); 2891 request.followLocation = true; 2892 request.retainCookies = true; 2893 2894 return request; 2895 } 2896 2897 /++ 2898 Creates a request without updating the current url state. If you want to save cookies, either call [retainCookies] with the response yourself 2899 or set [HttpRequest.retainCookies|request.retainCookies] to `true` on the returned object. But see important implementation shortcomings on [retainCookies]. 2900 2901 To upload files, you can use the [FormData] overload. 2902 +/ 2903 HttpRequest request(Uri uri, HttpVerb method = HttpVerb.GET, ubyte[] bodyData = null, string contentType = null) { 2904 string proxyToUse = getProxyFor(uri); 2905 2906 auto request = new HttpRequest(this, uri, method, cache, defaultTimeout, proxyToUse); 2907 2908 request.verifyPeer = this.defaultVerifyPeer; 2909 2910 request.requestParameters.userAgent = userAgent; 2911 request.requestParameters.authorization = authorization; 2912 2913 request.requestParameters.useHttp11 = this.useHttp11; 2914 request.requestParameters.acceptGzip = this.acceptGzip; 2915 request.requestParameters.keepAlive = this.keepAlive; 2916 2917 request.requestParameters.bodyData = bodyData; 2918 request.requestParameters.contentType = contentType; 2919 2920 request.requestParameters.headers = this.defaultHeaders; 2921 2922 populateCookies(request); 2923 2924 return request; 2925 } 2926 2927 /// ditto 2928 HttpRequest request(Uri uri, FormData fd, HttpVerb method = HttpVerb.POST) { 2929 return request(uri, method, fd.toBytes, fd.contentType); 2930 } 2931 2932 2933 private void populateCookies(HttpRequest request) { 2934 // FIXME: what about expiration and the like? or domain/path checks? or Secure checks? 2935 // FIXME: is uri.host correct? i think it should include port number too. what fun. 2936 if(auto cookies = ""/*uri.host*/ in this.cookies) { 2937 foreach(cookie; *cookies) 2938 request.requestParameters.cookies[cookie.name] = cookie.value; 2939 } 2940 } 2941 2942 private Uri currentUrl; 2943 private string currentDomain; 2944 private ICache cache; 2945 2946 /++ 2947 2948 +/ 2949 this(ICache cache = null) { 2950 this.defaultVerifyPeer = .defaultVerifyPeer_; 2951 this.cache = cache; 2952 loadDefaultProxy(); 2953 } 2954 2955 /++ 2956 Loads the system-default proxy. Note that the constructor does this automatically 2957 so you should rarely need to call this explicitly. 2958 2959 The environment variables are used, if present, on all operating systems. 2960 2961 History: 2962 no_proxy support added April 13, 2022 2963 2964 Added April 12, 2021 (included in dub v9.5) 2965 2966 Bugs: 2967 On Windows, it does NOT currently check the IE settings, but I do intend to 2968 implement that in the future. When I do, it will be classified as a bug fix, 2969 NOT a breaking change. 2970 +/ 2971 void loadDefaultProxy() { 2972 import std.process; 2973 httpProxy = environment.get("http_proxy", environment.get("HTTP_PROXY", null)); 2974 httpsProxy = environment.get("https_proxy", environment.get("HTTPS_PROXY", null)); 2975 auto noProxy = environment.get("no_proxy", environment.get("NO_PROXY", null)); 2976 if (noProxy.length) { 2977 proxyIgnore = noProxy.split(","); 2978 foreach (ref rule; proxyIgnore) 2979 rule = rule.strip; 2980 } 2981 2982 // FIXME: on Windows, I should use the Internet Explorer proxy settings 2983 } 2984 2985 /++ 2986 Checks if the given uri should be proxied according to the httpProxy, httpsProxy, proxyIgnore 2987 variables and returns either httpProxy, httpsProxy or null. 2988 2989 If neither `httpProxy` or `httpsProxy` are set this always returns `null`. Same if `proxyIgnore` 2990 contains `*`. 2991 2992 DNS is not resolved for proxyIgnore IPs, only IPs match IPs and hosts match hosts. 2993 +/ 2994 string getProxyFor(Uri uri) { 2995 string proxyToUse; 2996 switch(uri.scheme) { 2997 case "http": 2998 proxyToUse = httpProxy; 2999 break; 3000 case "https": 3001 proxyToUse = httpsProxy; 3002 break; 3003 default: 3004 proxyToUse = null; 3005 } 3006 3007 if (proxyToUse.length) { 3008 foreach (ignore; proxyIgnore) { 3009 if (matchProxyIgnore(ignore, uri)) { 3010 return null; 3011 } 3012 } 3013 } 3014 3015 return proxyToUse; 3016 } 3017 3018 /// Returns -1 on error, otherwise the IP as uint. Parsing is very strict. 3019 private static long tryParseIPv4(scope const(char)[] s) nothrow { 3020 import std.algorithm : findSplit, all; 3021 import std.ascii : isDigit; 3022 3023 static int parseNum(scope const(char)[] num) nothrow { 3024 if (num.length < 1 || num.length > 3 || !num.representation.all!isDigit) 3025 return -1; 3026 try { 3027 auto ret = num.to!int; 3028 return ret > 255 ? -1 : ret; 3029 } catch (Exception) { 3030 assert(false); 3031 } 3032 } 3033 3034 if (s.length < "0.0.0.0".length || s.length > "255.255.255.255".length) 3035 return -1; 3036 auto firstPair = s.findSplit("."); 3037 auto secondPair = firstPair[2].findSplit("."); 3038 auto thirdPair = secondPair[2].findSplit("."); 3039 auto a = parseNum(firstPair[0]); 3040 auto b = parseNum(secondPair[0]); 3041 auto c = parseNum(thirdPair[0]); 3042 auto d = parseNum(thirdPair[2]); 3043 if (a < 0 || b < 0 || c < 0 || d < 0) 3044 return -1; 3045 return (cast(uint)a << 24) | (b << 16) | (c << 8) | (d); 3046 } 3047 3048 unittest { 3049 assert(tryParseIPv4("0.0.0.0") == 0); 3050 assert(tryParseIPv4("127.0.0.1") == 0x7f000001); 3051 assert(tryParseIPv4("162.217.114.56") == 0xa2d97238); 3052 assert(tryParseIPv4("256.0.0.1") == -1); 3053 assert(tryParseIPv4("0.0.0.-2") == -1); 3054 assert(tryParseIPv4("0.0.0.a") == -1); 3055 assert(tryParseIPv4("0.0.0") == -1); 3056 assert(tryParseIPv4("0.0.0.0.0") == -1); 3057 } 3058 3059 /++ 3060 Returns true if the given no_proxy rule matches the uri. 3061 3062 Invalid IP ranges are silently ignored and return false. 3063 3064 See $(LREF proxyIgnore). 3065 +/ 3066 static bool matchProxyIgnore(scope const(char)[] rule, scope const Uri uri) nothrow { 3067 import std.algorithm; 3068 import std.ascii : isDigit; 3069 import std.uni : sicmp; 3070 3071 string uriHost = uri.host; 3072 if (uriHost.length && uriHost[$ - 1] == '.') 3073 uriHost = uriHost[0 .. $ - 1]; 3074 3075 if (rule == "*") 3076 return true; 3077 while (rule.length && rule[0] == '.') rule = rule[1 .. $]; 3078 3079 static int parsePort(scope const(char)[] portStr) nothrow { 3080 if (portStr.length < 1 || portStr.length > 5 || !portStr.representation.all!isDigit) 3081 return -1; 3082 try { 3083 return portStr.to!int; 3084 } catch (Exception) { 3085 assert(false, "to!int should succeed"); 3086 } 3087 } 3088 3089 if (sicmp(rule, uriHost) == 0 3090 || (uriHost.length > rule.length 3091 && sicmp(rule, uriHost[$ - rule.length .. $]) == 0 3092 && uriHost[$ - rule.length - 1] == '.')) 3093 return true; 3094 3095 if (rule.startsWith("[")) { // IPv6 3096 // below code is basically nothrow lastIndexOfAny("]:") 3097 ptrdiff_t lastColon = cast(ptrdiff_t) rule.length - 1; 3098 while (lastColon >= 0) { 3099 if (rule[lastColon] == ']' || rule[lastColon] == ':') 3100 break; 3101 lastColon--; 3102 } 3103 if (lastColon == -1) 3104 return false; // malformed 3105 3106 if (rule[lastColon] == ':') { // match with port 3107 auto port = parsePort(rule[lastColon + 1 .. $]); 3108 if (port != -1) { 3109 if (uri.effectivePort != port.to!int) 3110 return false; 3111 return uriHost == rule[0 .. lastColon]; 3112 } 3113 } 3114 // exact match of host already done above 3115 } else { 3116 auto slash = rule.lastIndexOfNothrow('/'); 3117 if (slash == -1) { // no IP range 3118 auto colon = rule.lastIndexOfNothrow(':'); 3119 auto host = colon == -1 ? rule : rule[0 .. colon]; 3120 auto port = colon != -1 ? parsePort(rule[colon + 1 .. $]) : -1; 3121 auto ip = tryParseIPv4(host); 3122 if (ip == -1) { // not an IPv4, test for host with port 3123 return port != -1 3124 && uri.effectivePort == port 3125 && uriHost == host; 3126 } else { 3127 // perform IPv4 equals 3128 auto other = tryParseIPv4(uriHost); 3129 if (other == -1) 3130 return false; // rule == IPv4, uri != IPv4 3131 if (port != -1) 3132 return uri.effectivePort == port 3133 && uriHost == host; 3134 else 3135 return uriHost == host; 3136 } 3137 } else { 3138 auto maskStr = rule[slash + 1 .. $]; 3139 auto ip = tryParseIPv4(rule[0 .. slash]); 3140 if (ip == -1) 3141 return false; 3142 if (maskStr.length && maskStr.length < 3 && maskStr.representation.all!isDigit) { 3143 // IPv4 range match 3144 int mask; 3145 try { 3146 mask = maskStr.to!int; 3147 } catch (Exception) { 3148 assert(false); 3149 } 3150 3151 auto other = tryParseIPv4(uriHost); 3152 if (other == -1) 3153 return false; // rule == IPv4, uri != IPv4 3154 3155 if (mask == 0) // matches all 3156 return true; 3157 if (mask > 32) // matches none 3158 return false; 3159 3160 auto shift = 32 - mask; 3161 return cast(uint)other >> shift 3162 == cast(uint)ip >> shift; 3163 } 3164 } 3165 } 3166 return false; 3167 } 3168 3169 unittest { 3170 assert(matchProxyIgnore("0.0.0.0/0", Uri("http://127.0.0.1:80/a"))); 3171 assert(matchProxyIgnore("0.0.0.0/0", Uri("http://127.0.0.1/a"))); 3172 assert(!matchProxyIgnore("0.0.0.0/0", Uri("https://dlang.org/a"))); 3173 assert(matchProxyIgnore("*", Uri("https://dlang.org/a"))); 3174 assert(matchProxyIgnore("127.0.0.0/8", Uri("http://127.0.0.1:80/a"))); 3175 assert(matchProxyIgnore("127.0.0.0/8", Uri("http://127.0.0.1/a"))); 3176 assert(matchProxyIgnore("127.0.0.1", Uri("http://127.0.0.1:1234/a"))); 3177 assert(!matchProxyIgnore("127.0.0.1:80", Uri("http://127.0.0.1:1234/a"))); 3178 assert(!matchProxyIgnore("127.0.0.1/8", Uri("http://localhost/a"))); // no DNS resolution / guessing 3179 assert(!matchProxyIgnore("0.0.0.0/1", Uri("http://localhost/a")) 3180 && !matchProxyIgnore("128.0.0.0/1", Uri("http://localhost/a"))); // no DNS resolution / guessing 2 3181 foreach (m; 1 .. 32) { 3182 assert(matchProxyIgnore(text("127.0.0.1/", m), Uri("http://127.0.0.1/a"))); 3183 assert(!matchProxyIgnore(text("127.0.0.1/", m), Uri("http://128.0.0.1/a"))); 3184 bool expectedMatch = m <= 24; 3185 assert(expectedMatch == matchProxyIgnore(text("127.0.1.0/", m), Uri("http://127.0.1.128/a")), m.to!string); 3186 } 3187 assert(matchProxyIgnore("localhost", Uri("http://localhost/a"))); 3188 assert(matchProxyIgnore("localhost", Uri("http://foo.localhost/a"))); 3189 assert(matchProxyIgnore("localhost", Uri("http://foo.localhost./a"))); 3190 assert(matchProxyIgnore(".localhost", Uri("http://localhost/a"))); 3191 assert(matchProxyIgnore(".localhost", Uri("http://foo.localhost/a"))); 3192 assert(matchProxyIgnore(".localhost", Uri("http://foo.localhost./a"))); 3193 assert(!matchProxyIgnore("foo.localhost", Uri("http://localhost/a"))); 3194 assert(matchProxyIgnore("foo.localhost", Uri("http://foo.localhost/a"))); 3195 assert(matchProxyIgnore("foo.localhost", Uri("http://foo.localhost./a"))); 3196 assert(!matchProxyIgnore("bar.localhost", Uri("http://localhost/a"))); 3197 assert(!matchProxyIgnore("bar.localhost", Uri("http://foo.localhost/a"))); 3198 assert(!matchProxyIgnore("bar.localhost", Uri("http://foo.localhost./a"))); 3199 assert(!matchProxyIgnore("bar.localhost", Uri("http://bbar.localhost./a"))); 3200 assert(matchProxyIgnore("[::1]", Uri("http://[::1]/a"))); 3201 assert(!matchProxyIgnore("[::1]", Uri("http://[::2]/a"))); 3202 assert(matchProxyIgnore("[::1]:80", Uri("http://[::1]/a"))); 3203 assert(!matchProxyIgnore("[::1]:443", Uri("http://[::1]/a"))); 3204 assert(!matchProxyIgnore("[::1]:80", Uri("https://[::1]/a"))); 3205 assert(matchProxyIgnore("[::1]:443", Uri("https://[::1]/a"))); 3206 assert(matchProxyIgnore("google.com", Uri("https://GOOGLE.COM/a"))); 3207 } 3208 3209 /++ 3210 Proxies to use for requests. The [HttpClient] constructor will set these to the system values, 3211 then you can reset it to `null` if you want to override and not use the proxy after all, or you 3212 can set it after construction to whatever. 3213 3214 The proxy from the client will be automatically set to the requests performed through it. You can 3215 also override on a per-request basis by creating the request and setting the `proxy` field there 3216 before sending it. 3217 3218 History: 3219 Added April 12, 2021 (included in dub v9.5) 3220 +/ 3221 string httpProxy; 3222 /// ditto 3223 string httpsProxy; 3224 /++ 3225 List of hosts or ips, optionally including a port, where not to proxy. 3226 3227 Each entry may be one of the following formats: 3228 - `127.0.0.1` (IPv4, any port) 3229 - `127.0.0.1:1234` (IPv4, specific port) 3230 - `127.0.0.1/8` (IPv4 range / CIDR block, any port) 3231 - `[::1]` (IPv6, any port) 3232 - `[::1]:1234` (IPv6, specific port) 3233 - `*` (all hosts and ports, basically don't proxy at all anymore) 3234 - `.domain.name`, `domain.name` (don't proxy the specified domain, 3235 leading dots are stripped and subdomains are also not proxied) 3236 - `.domain.name:1234`, `domain.name:1234` (same as above, with specific port) 3237 3238 No DNS resolution or regex is done in this list. 3239 3240 See https://about.gitlab.com/blog/2021/01/27/we-need-to-talk-no-proxy/ 3241 3242 History: 3243 Added April 13, 2022 3244 +/ 3245 string[] proxyIgnore; 3246 3247 /// See [retainCookies] for important caveats. 3248 void setCookie(string name, string value, string domain = null) { 3249 CookieHeader ch; 3250 3251 ch.name = name; 3252 ch.value = value; 3253 3254 setCookie(ch, domain); 3255 } 3256 3257 /// ditto 3258 void setCookie(CookieHeader ch, string domain = null) { 3259 if(domain is null) 3260 domain = currentDomain; 3261 3262 // FIXME: figure all this out or else cookies liable to get too long, in addition to the overwriting and oversharing issues in long scraping sessions 3263 cookies[""/*domain*/] ~= ch; 3264 } 3265 3266 /++ 3267 [HttpClient] does NOT automatically store cookies. You must explicitly retain them from a response by calling this method. 3268 3269 Examples: 3270 --- 3271 import arsd.http2; 3272 void main() { 3273 auto client = new HttpClient(); 3274 auto setRequest = client.request(Uri("http://arsdnet.net/cgi-bin/cookies/set")); 3275 auto setResponse = setRequest.waitForCompletion(); 3276 3277 auto request = client.request(Uri("http://arsdnet.net/cgi-bin/cookies/get")); 3278 auto response = request.waitForCompletion(); 3279 3280 // the cookie wasn't explicitly retained, so the server echos back nothing 3281 assert(response.responseText.length == 0); 3282 3283 // now keep the cookies from our original set 3284 client.retainCookies(setResponse); 3285 3286 request = client.request(Uri("http://arsdnet.net/cgi-bin/cookies/get")); 3287 response = request.waitForCompletion(); 3288 3289 // now it matches 3290 assert(response.responseText.length && response.responseText == setResponse.cookies["example-cookie"]); 3291 } 3292 --- 3293 3294 Bugs: 3295 It does NOT currently implement domain / path / secure separation nor cookie expiration. It assumes that if you call this function, you're ok with it. 3296 3297 You may want to use separate HttpClient instances if any sharing is unacceptable at this time. 3298 3299 History: 3300 Added July 5, 2021 (dub v10.2) 3301 +/ 3302 void retainCookies(HttpResponse fromResponse) { 3303 foreach(name, value; fromResponse.cookies) 3304 setCookie(name, value); 3305 } 3306 3307 /// 3308 void clearCookies(string domain = null) { 3309 if(domain is null) 3310 cookies = null; 3311 else 3312 cookies[domain] = null; 3313 } 3314 3315 // If you set these, they will be pre-filled on all requests made with this client 3316 string userAgent = "D arsd.html2"; /// 3317 string authorization; /// 3318 3319 /* inter-request state */ 3320 private CookieHeader[][string] cookies; 3321 } 3322 3323 private ptrdiff_t lastIndexOfNothrow(T)(scope T[] arr, T value) nothrow 3324 { 3325 ptrdiff_t ret = cast(ptrdiff_t)arr.length - 1; 3326 while (ret >= 0) { 3327 if (arr[ret] == value) 3328 return ret; 3329 ret--; 3330 } 3331 return ret; 3332 } 3333 3334 interface ICache { 3335 /++ 3336 The client is about to make the given `request`. It will ALWAYS pass it to the cache object first so you can decide if you want to and can provide a response. You should probably check the appropriate headers to see if you should even attempt to look up on the cache (HttpClient does NOT do this to give maximum flexibility to the cache implementor). 3337 3338 Return null if the cache does not provide. 3339 +/ 3340 const(HttpResponse)* getCachedResponse(HttpRequestParameters request); 3341 3342 /++ 3343 The given request has received the given response. The implementing class needs to decide if it wants to cache or not. Return true if it was added, false if you chose not to. 3344 3345 You may wish to examine headers, etc., in making the decision. The HttpClient will ALWAYS pass a request/response to this. 3346 +/ 3347 bool cacheResponse(HttpRequestParameters request, HttpResponse response); 3348 } 3349 3350 /+ 3351 // / Provides caching behavior similar to a real web browser 3352 class HttpCache : ICache { 3353 const(HttpResponse)* getCachedResponse(HttpRequestParameters request) { 3354 return null; 3355 } 3356 } 3357 3358 // / Gives simple maximum age caching, ignoring the actual http headers 3359 class SimpleCache : ICache { 3360 const(HttpResponse)* getCachedResponse(HttpRequestParameters request) { 3361 return null; 3362 } 3363 } 3364 +/ 3365 3366 /++ 3367 A pseudo-cache to provide a mock server. Construct one of these, 3368 populate it with test responses, and pass it to [HttpClient] to 3369 do a network-free test. 3370 3371 You should populate it with the [populate] method. Any request not 3372 pre-populated will return a "server refused connection" response. 3373 +/ 3374 class HttpMockProvider : ICache { 3375 /+ + 3376 3377 +/ 3378 version(none) 3379 this(Uri baseUrl, string defaultResponseContentType) { 3380 3381 } 3382 3383 this() {} 3384 3385 HttpResponse defaultResponse; 3386 3387 /// Implementation of the ICache interface. Hijacks all requests to return a pre-populated response or "server disconnected". 3388 const(HttpResponse)* getCachedResponse(HttpRequestParameters request) { 3389 import std.conv; 3390 auto defaultPort = request.ssl ? 443 : 80; 3391 string identifier = text( 3392 request.method, " ", 3393 request.ssl ? "https" : "http", "://", 3394 request.host, 3395 (request.port && request.port != defaultPort) ? (":" ~ to!string(request.port)) : "", 3396 request.uri 3397 ); 3398 3399 if(auto res = identifier in population) 3400 return res; 3401 return &defaultResponse; 3402 } 3403 3404 /// Implementation of the ICache interface. We never actually cache anything here since it is all about mock responses, not actually caching real data. 3405 bool cacheResponse(HttpRequestParameters request, HttpResponse response) { 3406 return false; 3407 } 3408 3409 /++ 3410 Convenience method to populate simple responses. For more complex 3411 work, use one of the other overloads where you build complete objects 3412 yourself. 3413 3414 Params: 3415 request = a verb and complete URL to mock as one string. 3416 For example "GET http://example.com/". If you provide only 3417 a partial URL, it will be based on the `baseUrl` you gave 3418 in the `HttpMockProvider` constructor. 3419 3420 responseCode = the HTTP response code, like 200 or 404. 3421 3422 response = the response body as a string. It is assumed 3423 to be of the `defaultResponseContentType` you passed in the 3424 `HttpMockProvider` constructor. 3425 +/ 3426 void populate(string request, int responseCode, string response) { 3427 3428 // FIXME: absolute-ize the URL in the request 3429 3430 HttpResponse r; 3431 r.code = responseCode; 3432 r.codeText = getHttpCodeText(r.code); 3433 3434 r.content = cast(ubyte[]) response; 3435 r.contentText = response; 3436 3437 population[request] = r; 3438 } 3439 3440 version(none) 3441 void populate(string method, string url, HttpResponse response) { 3442 // FIXME 3443 } 3444 3445 private HttpResponse[string] population; 3446 } 3447 3448 // modified from the one in cgi.d to just have the text 3449 private static string getHttpCodeText(int code) pure nothrow @nogc { 3450 switch(code) { 3451 // this module's proprietary extensions 3452 case 0: return null; 3453 case 1: return "request.abort called"; 3454 case 2: return "connection failed"; 3455 case 3: return "server disconnected"; 3456 case 4: return "exception thrown"; // actually should be some other thing 3457 case 5: return "Request timed out"; 3458 3459 // * * * standard ones * * * 3460 3461 // 1xx skipped since they shouldn't happen 3462 3463 // 3464 case 200: return "OK"; 3465 case 201: return "Created"; 3466 case 202: return "Accepted"; 3467 case 203: return "Non-Authoritative Information"; 3468 case 204: return "No Content"; 3469 case 205: return "Reset Content"; 3470 // 3471 case 300: return "Multiple Choices"; 3472 case 301: return "Moved Permanently"; 3473 case 302: return "Found"; 3474 case 303: return "See Other"; 3475 case 307: return "Temporary Redirect"; 3476 case 308: return "Permanent Redirect"; 3477 // 3478 case 400: return "Bad Request"; 3479 case 403: return "Forbidden"; 3480 case 404: return "Not Found"; 3481 case 405: return "Method Not Allowed"; 3482 case 406: return "Not Acceptable"; 3483 case 409: return "Conflict"; 3484 case 410: return "Gone"; 3485 // 3486 case 500: return "Internal Server Error"; 3487 case 501: return "Not Implemented"; 3488 case 502: return "Bad Gateway"; 3489 case 503: return "Service Unavailable"; 3490 // 3491 default: assert(0, "Unsupported http code"); 3492 } 3493 } 3494 3495 3496 /// 3497 struct HttpCookie { 3498 string name; /// 3499 string value; /// 3500 string domain; /// 3501 string path; /// 3502 //SysTime expirationDate; /// 3503 bool secure; /// 3504 bool httpOnly; /// 3505 } 3506 3507 // FIXME: websocket 3508 3509 version(testing) 3510 void main() { 3511 import std.stdio; 3512 auto client = new HttpClient(); 3513 auto request = client.navigateTo(Uri("http://localhost/chunked.php")); 3514 request.send(); 3515 auto request2 = client.navigateTo(Uri("http://dlang.org/")); 3516 request2.send(); 3517 3518 { 3519 auto response = request2.waitForCompletion(); 3520 //write(cast(string) response.content); 3521 } 3522 3523 auto response = request.waitForCompletion(); 3524 write(cast(string) response.content); 3525 3526 writeln(HttpRequest.socketsPerHost); 3527 } 3528 3529 string lastSocketError(Socket sock) { 3530 import std.socket; 3531 version(use_openssl) { 3532 if(auto s = cast(OpenSslSocket) sock) 3533 if(s.lastSocketError.length) 3534 return s.lastSocketError; 3535 } 3536 return std.socket.lastSocketError(); 3537 } 3538 3539 // From sslsocket.d, but this is the maintained version! 3540 version(use_openssl) { 3541 alias SslClientSocket = OpenSslSocket; 3542 3543 // CRL = Certificate Revocation List 3544 static immutable string[] sslErrorCodes = [ 3545 "OK (code 0)", 3546 "Unspecified SSL/TLS error (code 1)", 3547 "Unable to get TLS issuer certificate (code 2)", 3548 "Unable to get TLS CRL (code 3)", 3549 "Unable to decrypt TLS certificate signature (code 4)", 3550 "Unable to decrypt TLS CRL signature (code 5)", 3551 "Unable to decode TLS issuer public key (code 6)", 3552 "TLS certificate signature failure (code 7)", 3553 "TLS CRL signature failure (code 8)", 3554 "TLS certificate not yet valid (code 9)", 3555 "TLS certificate expired (code 10)", 3556 "TLS CRL not yet valid (code 11)", 3557 "TLS CRL expired (code 12)", 3558 "TLS error in certificate not before field (code 13)", 3559 "TLS error in certificate not after field (code 14)", 3560 "TLS error in CRL last update field (code 15)", 3561 "TLS error in CRL next update field (code 16)", 3562 "TLS system out of memory (code 17)", 3563 "TLS certificate is self-signed (code 18)", 3564 "Self-signed certificate in TLS chain (code 19)", 3565 "Unable to get TLS issuer certificate locally (code 20)", 3566 "Unable to verify TLS leaf signature (code 21)", 3567 "TLS certificate chain too long (code 22)", 3568 "TLS certificate was revoked (code 23)", 3569 "TLS CA is invalid (code 24)", 3570 "TLS error: path length exceeded (code 25)", 3571 "TLS error: invalid purpose (code 26)", 3572 "TLS error: certificate untrusted (code 27)", 3573 "TLS error: certificate rejected (code 28)", 3574 ]; 3575 3576 string getOpenSslErrorCode(long error) { 3577 if(error == 62) 3578 return "TLS certificate host name mismatch"; 3579 3580 if(error < 0 || error >= sslErrorCodes.length) 3581 return "SSL/TLS error code " ~ to!string(error); 3582 return sslErrorCodes[cast(size_t) error]; 3583 } 3584 3585 struct SSL; 3586 struct SSL_CTX; 3587 struct SSL_METHOD; 3588 struct X509_STORE_CTX; 3589 enum SSL_VERIFY_NONE = 0; 3590 enum SSL_VERIFY_PEER = 1; 3591 3592 // copy it into the buf[0 .. size] and return actual length you read. 3593 // rwflag == 0 when reading, 1 when writing. 3594 extern(C) alias pem_password_cb = int function(char* buffer, int bufferSize, int rwflag, void* userPointer); 3595 extern(C) alias print_errors_cb = int function(const char*, size_t, void*); 3596 extern(C) alias client_cert_cb = int function(SSL *ssl, X509 **x509, EVP_PKEY **pkey); 3597 extern(C) alias keylog_cb = void function(SSL*, char*); 3598 3599 struct X509; 3600 struct X509_STORE; 3601 struct EVP_PKEY; 3602 struct X509_VERIFY_PARAM; 3603 3604 import core.stdc.config; 3605 3606 enum SSL_ERROR_WANT_READ = 2; 3607 enum SSL_ERROR_WANT_WRITE = 3; 3608 3609 struct ossllib { 3610 __gshared static extern(C) { 3611 /* these are only on older openssl versions { */ 3612 int function() SSL_library_init; 3613 void function() SSL_load_error_strings; 3614 SSL_METHOD* function() SSLv23_client_method; 3615 /* } */ 3616 3617 void function(ulong, void*) OPENSSL_init_ssl; 3618 3619 SSL_CTX* function(const SSL_METHOD*) SSL_CTX_new; 3620 SSL* function(SSL_CTX*) SSL_new; 3621 int function(SSL*, int) SSL_set_fd; 3622 int function(SSL*) SSL_connect; 3623 int function(SSL*, const void*, int) SSL_write; 3624 int function(SSL*, void*, int) SSL_read; 3625 @trusted nothrow @nogc int function(SSL*) SSL_shutdown; 3626 void function(SSL*) SSL_free; 3627 void function(SSL_CTX*) SSL_CTX_free; 3628 3629 int function(const SSL*) SSL_pending; 3630 int function (const SSL *ssl, int ret) SSL_get_error; 3631 3632 void function(SSL*, int, void*) SSL_set_verify; 3633 3634 void function(SSL*, int, c_long, void*) SSL_ctrl; 3635 3636 SSL_METHOD* function() SSLv3_client_method; 3637 SSL_METHOD* function() TLS_client_method; 3638 3639 void function(SSL_CTX*, void function(SSL*, char* line)) SSL_CTX_set_keylog_callback; 3640 3641 int function(SSL_CTX*) SSL_CTX_set_default_verify_paths; 3642 3643 X509_STORE* function(SSL_CTX*) SSL_CTX_get_cert_store; 3644 c_long function(const SSL* ssl) SSL_get_verify_result; 3645 3646 X509_VERIFY_PARAM* function(const SSL*) SSL_get0_param; 3647 3648 /+ 3649 SSL_CTX_load_verify_locations 3650 SSL_CTX_set_client_CA_list 3651 +/ 3652 3653 // client cert things 3654 void function (SSL_CTX *ctx, int function(SSL *ssl, X509 **x509, EVP_PKEY **pkey)) SSL_CTX_set_client_cert_cb; 3655 } 3656 } 3657 3658 struct eallib { 3659 __gshared static extern(C) { 3660 /* these are only on older openssl versions { */ 3661 void function() OpenSSL_add_all_ciphers; 3662 void function() OpenSSL_add_all_digests; 3663 /* } */ 3664 3665 const(char)* function(int) OpenSSL_version; 3666 3667 void function(ulong, void*) OPENSSL_init_crypto; 3668 3669 void function(print_errors_cb, void*) ERR_print_errors_cb; 3670 3671 void function(X509*) X509_free; 3672 int function(X509_STORE*, X509*) X509_STORE_add_cert; 3673 3674 3675 X509* function(FILE *fp, X509 **x, pem_password_cb *cb, void *u) PEM_read_X509; 3676 EVP_PKEY* function(FILE *fp, EVP_PKEY **x, pem_password_cb *cb, void* userPointer) PEM_read_PrivateKey; 3677 3678 EVP_PKEY* function(FILE *fp, EVP_PKEY **a) d2i_PrivateKey_fp; 3679 X509* function(FILE *fp, X509 **x) d2i_X509_fp; 3680 3681 X509* function(X509** a, const(ubyte*)* pp, c_long length) d2i_X509; 3682 int function(X509* a, ubyte** o) i2d_X509; 3683 3684 int function(X509_VERIFY_PARAM* a, const char* b, size_t l) X509_VERIFY_PARAM_set1_host; 3685 3686 X509* function(X509_STORE_CTX *ctx) X509_STORE_CTX_get_current_cert; 3687 int function(X509_STORE_CTX *ctx) X509_STORE_CTX_get_error; 3688 } 3689 } 3690 3691 struct OpenSSL { 3692 static: 3693 3694 template opDispatch(string name) { 3695 auto opDispatch(T...)(T t) { 3696 static if(__traits(hasMember, ossllib, name)) { 3697 auto ptr = __traits(getMember, ossllib, name); 3698 } else static if(__traits(hasMember, eallib, name)) { 3699 auto ptr = __traits(getMember, eallib, name); 3700 } else static assert(0); 3701 3702 if(ptr is null) 3703 throw new Exception(name ~ " not loaded"); 3704 return ptr(t); 3705 } 3706 } 3707 3708 // macros in the original C 3709 SSL_METHOD* SSLv23_client_method() { 3710 if(ossllib.SSLv23_client_method) 3711 return ossllib.SSLv23_client_method(); 3712 else 3713 return ossllib.TLS_client_method(); 3714 } 3715 3716 void SSL_set_tlsext_host_name(SSL* a, const char* b) { 3717 if(ossllib.SSL_ctrl) 3718 return ossllib.SSL_ctrl(a, 55 /*SSL_CTRL_SET_TLSEXT_HOSTNAME*/, 0 /*TLSEXT_NAMETYPE_host_name*/, cast(void*) b); 3719 else throw new Exception("SSL_set_tlsext_host_name not loaded"); 3720 } 3721 3722 // special case 3723 @trusted nothrow @nogc int SSL_shutdown(SSL* a) { 3724 if(ossllib.SSL_shutdown) 3725 return ossllib.SSL_shutdown(a); 3726 assert(0); 3727 } 3728 3729 void SSL_CTX_keylog_cb_func(SSL_CTX* ctx, keylog_cb func) { 3730 // this isn't in openssl 1.0 and is non-essential, so it is allowed to fail. 3731 if(ossllib.SSL_CTX_set_keylog_callback) 3732 ossllib.SSL_CTX_set_keylog_callback(ctx, func); 3733 //else throw new Exception("SSL_CTX_keylog_cb_func not loaded"); 3734 } 3735 3736 } 3737 3738 extern(C) 3739 int collectSslErrors(const char* ptr, size_t len, void* user) @trusted { 3740 string* s = cast(string*) user; 3741 3742 (*s) ~= ptr[0 .. len]; 3743 3744 return 0; 3745 } 3746 3747 3748 private __gshared void* ossllib_handle; 3749 version(Windows) 3750 private __gshared void* oeaylib_handle; 3751 else 3752 alias oeaylib_handle = ossllib_handle; 3753 version(Posix) 3754 private import core.sys.posix.dlfcn; 3755 else version(Windows) 3756 private import core.sys.windows.windows; 3757 3758 import core.stdc.stdio; 3759 3760 private __gshared Object loadSslMutex = new Object; 3761 private __gshared bool sslLoaded = false; 3762 3763 void loadOpenSsl() { 3764 if(sslLoaded) 3765 return; 3766 synchronized(loadSslMutex) { 3767 3768 version(Posix) { 3769 version(OSX) { 3770 static immutable string[] ossllibs = [ 3771 "libssl.46.dylib", 3772 "libssl.44.dylib", 3773 "libssl.43.dylib", 3774 "libssl.35.dylib", 3775 "libssl.1.1.dylib", 3776 "libssl.dylib", 3777 "/usr/local/opt/openssl/lib/libssl.1.0.0.dylib", 3778 ]; 3779 } else { 3780 static immutable string[] ossllibs = [ 3781 "libssl.so.3", 3782 "libssl.so.1.1", 3783 "libssl.so.1.0.2", 3784 "libssl.so.1.0.1", 3785 "libssl.so.1.0.0", 3786 "libssl.so", 3787 ]; 3788 } 3789 3790 foreach(lib; ossllibs) { 3791 ossllib_handle = dlopen(lib.ptr, RTLD_NOW); 3792 if(ossllib_handle !is null) break; 3793 } 3794 } else version(Windows) { 3795 version(X86_64) { 3796 ossllib_handle = LoadLibraryW("libssl-1_1-x64.dll"w.ptr); 3797 oeaylib_handle = LoadLibraryW("libcrypto-1_1-x64.dll"w.ptr); 3798 } 3799 3800 static immutable wstring[] ossllibs = [ 3801 "libssl-3-x64.dll"w, 3802 "libssl-3.dll"w, 3803 "libssl-1_1.dll"w, 3804 "libssl32.dll"w, 3805 ]; 3806 3807 if(ossllib_handle is null) 3808 foreach(lib; ossllibs) { 3809 ossllib_handle = LoadLibraryW(lib.ptr); 3810 if(ossllib_handle !is null) break; 3811 } 3812 3813 static immutable wstring[] eaylibs = [ 3814 "libcrypto-3-x64.dll"w, 3815 "libcrypto-3.dll"w, 3816 "libcrypto-1_1.dll"w, 3817 "libeay32.dll", 3818 ]; 3819 3820 if(oeaylib_handle is null) 3821 foreach(lib; eaylibs) { 3822 oeaylib_handle = LoadLibraryW(lib.ptr); 3823 if (oeaylib_handle !is null) break; 3824 } 3825 3826 if(ossllib_handle is null) { 3827 ossllib_handle = LoadLibraryW("ssleay32.dll"w.ptr); 3828 oeaylib_handle = ossllib_handle; 3829 } 3830 } 3831 3832 if(ossllib_handle is null) 3833 throw new Exception("libssl library not found"); 3834 if(oeaylib_handle is null) 3835 throw new Exception("libeay32 library not found"); 3836 3837 foreach(memberName; __traits(allMembers, ossllib)) { 3838 alias t = typeof(__traits(getMember, ossllib, memberName)); 3839 version(Posix) 3840 __traits(getMember, ossllib, memberName) = cast(t) dlsym(ossllib_handle, memberName); 3841 else version(Windows) { 3842 __traits(getMember, ossllib, memberName) = cast(t) GetProcAddress(ossllib_handle, memberName); 3843 } 3844 } 3845 3846 foreach(memberName; __traits(allMembers, eallib)) { 3847 alias t = typeof(__traits(getMember, eallib, memberName)); 3848 version(Posix) 3849 __traits(getMember, eallib, memberName) = cast(t) dlsym(oeaylib_handle, memberName); 3850 else version(Windows) { 3851 __traits(getMember, eallib, memberName) = cast(t) GetProcAddress(oeaylib_handle, memberName); 3852 } 3853 } 3854 3855 3856 if(ossllib.SSL_library_init) 3857 ossllib.SSL_library_init(); 3858 else if(ossllib.OPENSSL_init_ssl) 3859 ossllib.OPENSSL_init_ssl(0, null); 3860 else throw new Exception("couldn't init openssl"); 3861 3862 if(eallib.OpenSSL_add_all_ciphers) { 3863 eallib.OpenSSL_add_all_ciphers(); 3864 if(eallib.OpenSSL_add_all_digests is null) 3865 throw new Exception("no add digests"); 3866 eallib.OpenSSL_add_all_digests(); 3867 } else if(eallib.OPENSSL_init_crypto) 3868 eallib.OPENSSL_init_crypto(0 /*OPENSSL_INIT_ADD_ALL_CIPHERS and ALL_DIGESTS together*/, null); 3869 else throw new Exception("couldn't init crypto openssl"); 3870 3871 if(ossllib.SSL_load_error_strings) 3872 ossllib.SSL_load_error_strings(); 3873 else if(ossllib.OPENSSL_init_ssl) 3874 ossllib.OPENSSL_init_ssl(0x00200000L, null); 3875 else throw new Exception("couldn't load openssl errors"); 3876 3877 sslLoaded = true; 3878 } 3879 } 3880 3881 /+ 3882 // I'm just gonna let the OS clean this up on process termination because otherwise SSL_free 3883 // might have trouble being run from the GC after this module is unloaded. 3884 shared static ~this() { 3885 if(ossllib_handle) { 3886 version(Windows) { 3887 FreeLibrary(oeaylib_handle); 3888 FreeLibrary(ossllib_handle); 3889 } else version(Posix) 3890 dlclose(ossllib_handle); 3891 ossllib_handle = null; 3892 } 3893 ossllib.tupleof = ossllib.tupleof.init; 3894 } 3895 +/ 3896 3897 //pragma(lib, "crypto"); 3898 //pragma(lib, "ssl"); 3899 extern(C) 3900 void write_to_file(SSL* ssl, char* line) 3901 { 3902 import std.stdio; 3903 import std.string; 3904 import std.process : environment; 3905 string logfile = environment.get("SSLKEYLOGFILE"); 3906 if (logfile !is null) 3907 { 3908 auto f = std.stdio.File(logfile, "a+"); 3909 f.writeln(fromStringz(line)); 3910 f.close(); 3911 } 3912 } 3913 3914 class OpenSslSocket : Socket { 3915 private SSL* ssl; 3916 private SSL_CTX* ctx; 3917 private void initSsl(bool verifyPeer, string hostname) { 3918 ctx = OpenSSL.SSL_CTX_new(OpenSSL.SSLv23_client_method()); 3919 assert(ctx !is null); 3920 3921 debug OpenSSL.SSL_CTX_keylog_cb_func(ctx, &write_to_file); 3922 ssl = OpenSSL.SSL_new(ctx); 3923 3924 if(hostname.length) { 3925 OpenSSL.SSL_set_tlsext_host_name(ssl, toStringz(hostname)); 3926 if(verifyPeer) 3927 OpenSSL.X509_VERIFY_PARAM_set1_host(OpenSSL.SSL_get0_param(ssl), hostname.ptr, hostname.length); 3928 } 3929 3930 if(verifyPeer) { 3931 OpenSSL.SSL_CTX_set_default_verify_paths(ctx); 3932 3933 version(Windows) { 3934 loadCertificatesFromRegistry(ctx); 3935 } 3936 3937 OpenSSL.SSL_set_verify(ssl, SSL_VERIFY_PEER, &verifyCertificateFromRegistryArsdHttp); 3938 } else 3939 OpenSSL.SSL_set_verify(ssl, SSL_VERIFY_NONE, null); 3940 3941 OpenSSL.SSL_set_fd(ssl, cast(int) this.handle); // on win64 it is necessary to truncate, but the value is never large anyway see http://openssl.6102.n7.nabble.com/Sockets-windows-64-bit-td36169.html 3942 3943 3944 OpenSSL.SSL_CTX_set_client_cert_cb(ctx, &cb); 3945 } 3946 3947 extern(C) 3948 static int cb(SSL* ssl, X509** x509, EVP_PKEY** pkey) { 3949 if(HttpClient.certFilename.length && HttpClient.keyFilename.length) { 3950 FILE* fpCert = fopen((HttpClient.certFilename ~ "\0").ptr, "rb"); 3951 if(fpCert is null) 3952 return 0; 3953 scope(exit) 3954 fclose(fpCert); 3955 FILE* fpKey = fopen((HttpClient.keyFilename ~ "\0").ptr, "rb"); 3956 if(fpKey is null) 3957 return 0; 3958 scope(exit) 3959 fclose(fpKey); 3960 3961 with(CertificateFileFormat) 3962 final switch(HttpClient.certFormat) { 3963 case guess: 3964 if(HttpClient.certFilename.endsWith(".pem") || HttpClient.keyFilename.endsWith(".pem")) 3965 goto case pem; 3966 else 3967 goto case der; 3968 case pem: 3969 *x509 = OpenSSL.PEM_read_X509(fpCert, null, null, null); 3970 *pkey = OpenSSL.PEM_read_PrivateKey(fpKey, null, null, null); 3971 break; 3972 case der: 3973 *x509 = OpenSSL.d2i_X509_fp(fpCert, null); 3974 *pkey = OpenSSL.d2i_PrivateKey_fp(fpKey, null); 3975 break; 3976 } 3977 3978 return 1; 3979 } 3980 3981 return 0; 3982 } 3983 3984 final bool dataPending() { 3985 return OpenSSL.SSL_pending(ssl) > 0; 3986 } 3987 3988 @trusted 3989 override void connect(Address to) { 3990 super.connect(to); 3991 if(blocking) { 3992 do_ssl_connect(); 3993 } 3994 } 3995 3996 private string lastSocketError; 3997 3998 @trusted 3999 // returns true if it is finished, false if it would have blocked, throws if there's an error 4000 int do_ssl_connect() { 4001 if(OpenSSL.SSL_connect(ssl) == -1) { 4002 4003 auto errCode = OpenSSL.SSL_get_error(ssl, -1); 4004 if(errCode == SSL_ERROR_WANT_READ || errCode == SSL_ERROR_WANT_WRITE) { 4005 return errCode; 4006 } 4007 4008 string str; 4009 OpenSSL.ERR_print_errors_cb(&collectSslErrors, &str); 4010 4011 auto err = OpenSSL.SSL_get_verify_result(ssl); 4012 this.lastSocketError = str ~ " " ~ getOpenSslErrorCode(err); 4013 4014 throw new Exception("Secure connect failed: " ~ getOpenSslErrorCode(err)); 4015 } else this.lastSocketError = null; 4016 4017 return 0; 4018 } 4019 4020 @trusted 4021 override ptrdiff_t send(scope const(void)[] buf, SocketFlags flags) { 4022 //import std.stdio;writeln(cast(string) buf); 4023 debug(arsd_http2_verbose) writeln("ssl writing ", buf.length); 4024 auto retval = OpenSSL.SSL_write(ssl, buf.ptr, cast(uint) buf.length); 4025 4026 // don't need to throw anymore since it is checked elsewhere 4027 // code useful sometimes for debugging hence commenting instead of deleting 4028 if(retval == -1) { 4029 string str; 4030 OpenSSL.ERR_print_errors_cb(&collectSslErrors, &str); 4031 this.lastSocketError = str; 4032 4033 // throw new Exception("ssl send failed " ~ str); 4034 } else this.lastSocketError = null; 4035 return retval; 4036 4037 } 4038 override ptrdiff_t send(scope const(void)[] buf) { 4039 return send(buf, SocketFlags.NONE); 4040 } 4041 @trusted 4042 override ptrdiff_t receive(scope void[] buf, SocketFlags flags) { 4043 4044 debug(arsd_http2_verbose) writeln("ssl_read before"); 4045 auto retval = OpenSSL.SSL_read(ssl, buf.ptr, cast(int)buf.length); 4046 debug(arsd_http2_verbose) writeln("ssl_read after"); 4047 4048 // don't need to throw anymore since it is checked elsewhere 4049 // code useful sometimes for debugging hence commenting instead of deleting 4050 if(retval == -1) { 4051 4052 string str; 4053 OpenSSL.ERR_print_errors_cb(&collectSslErrors, &str); 4054 this.lastSocketError = str; 4055 4056 // throw new Exception("ssl receive failed " ~ str); 4057 } else this.lastSocketError = null; 4058 return retval; 4059 } 4060 override ptrdiff_t receive(scope void[] buf) { 4061 return receive(buf, SocketFlags.NONE); 4062 } 4063 4064 this(AddressFamily af, SocketType type = SocketType.STREAM, string hostname = null, bool verifyPeer = true) { 4065 version(Windows) __traits(getMember, this, "_blocking") = true; // lol longstanding phobos bug setting this to false on init 4066 super(af, type); 4067 initSsl(verifyPeer, hostname); 4068 } 4069 4070 override void close() scope { 4071 if(ssl) OpenSSL.SSL_shutdown(ssl); 4072 super.close(); 4073 } 4074 4075 this(socket_t sock, AddressFamily af, string hostname, bool verifyPeer = true) { 4076 super(sock, af); 4077 initSsl(verifyPeer, hostname); 4078 } 4079 4080 void freeSsl() { 4081 if(ssl is null) 4082 return; 4083 OpenSSL.SSL_free(ssl); 4084 OpenSSL.SSL_CTX_free(ctx); 4085 ssl = null; 4086 ctx = null; 4087 } 4088 4089 ~this() { 4090 freeSsl(); 4091 } 4092 } 4093 } 4094 4095 4096 /++ 4097 An experimental component for working with REST apis. Note that it 4098 is a zero-argument template, so to create one, use `new HttpApiClient!()(args..)` 4099 or you will get "HttpApiClient is used as a type" compile errors. 4100 4101 This will probably not work for you yet, and I might change it significantly. 4102 4103 Requires [arsd.jsvar]. 4104 4105 4106 Here's a snippet to create a pull request on GitHub to Phobos: 4107 4108 --- 4109 auto github = new HttpApiClient!()("https://api.github.com/", "your personal api token here"); 4110 4111 // create the arguments object 4112 // see: https://developer.github.com/v3/pulls/#create-a-pull-request 4113 var args = var.emptyObject; 4114 args.title = "My Pull Request"; 4115 args.head = "yourusername:" ~ branchName; 4116 args.base = "master"; 4117 // note it is ["body"] instead of .body because `body` is a D keyword 4118 args["body"] = "My cool PR is opened by the API!"; 4119 args.maintainer_can_modify = true; 4120 4121 /+ 4122 Fun fact, you can also write that: 4123 4124 var args = [ 4125 "title": "My Pull Request".var, 4126 "head": "yourusername:" ~ branchName.var, 4127 "base" : "master".var, 4128 "body" : "My cool PR is opened by the API!".var, 4129 "maintainer_can_modify": true.var 4130 ]; 4131 4132 Note the .var constructor calls in there. If everything is the same type, you actually don't need that, but here since there's strings and bools, D won't allow the literal without explicit constructors to align them all. 4133 +/ 4134 4135 // this translates to `repos/dlang/phobos/pulls` and sends a POST request, 4136 // containing `args` as json, then immediately grabs the json result and extracts 4137 // the value `html_url` from it. `prUrl` is typed `var`, from arsd.jsvar. 4138 auto prUrl = github.rest.repos.dlang.phobos.pulls.POST(args).result.html_url; 4139 4140 writeln("Created: ", prUrl); 4141 --- 4142 4143 Why use this instead of just building the URL? Well, of course you can! This just makes 4144 it a bit more convenient than string concatenation and manages a few headers for you. 4145 4146 Subtypes could potentially add static type checks too. 4147 +/ 4148 class HttpApiClient() { 4149 import arsd.jsvar; 4150 4151 HttpClient httpClient; 4152 4153 alias HttpApiClientType = typeof(this); 4154 4155 string urlBase; 4156 string oauth2Token; 4157 string submittedContentType; 4158 string authType = "Bearer"; 4159 4160 /++ 4161 Params: 4162 4163 urlBase = The base url for the api. Tends to be something like `https://api.example.com/v2/` or similar. 4164 oauth2Token = the authorization token for the service. You'll have to get it from somewhere else. 4165 submittedContentType = the content-type of POST, PUT, etc. bodies. 4166 httpClient = an injected http client, or null if you want to use a default-constructed one 4167 4168 History: 4169 The `httpClient` param was added on December 26, 2020. 4170 +/ 4171 this(string urlBase, string oauth2Token, string submittedContentType = "application/json", HttpClient httpClient = null) { 4172 if(httpClient is null) 4173 this.httpClient = new HttpClient(); 4174 else 4175 this.httpClient = httpClient; 4176 4177 assert(urlBase[0] == 'h'); 4178 assert(urlBase[$-1] == '/'); 4179 4180 this.urlBase = urlBase; 4181 this.oauth2Token = oauth2Token; 4182 this.submittedContentType = submittedContentType; 4183 } 4184 4185 /// 4186 static struct HttpRequestWrapper { 4187 HttpApiClientType apiClient; /// 4188 HttpRequest request; /// 4189 HttpResponse _response; 4190 4191 /// 4192 this(HttpApiClientType apiClient, HttpRequest request) { 4193 this.apiClient = apiClient; 4194 this.request = request; 4195 } 4196 4197 /// Returns the full [HttpResponse] object so you can inspect the headers 4198 @property HttpResponse response() { 4199 if(_response is HttpResponse.init) 4200 _response = request.waitForCompletion(); 4201 return _response; 4202 } 4203 4204 /++ 4205 Returns the parsed JSON from the body of the response. 4206 4207 Throws on non-2xx responses. 4208 +/ 4209 var result() { 4210 return apiClient.throwOnError(response); 4211 } 4212 4213 alias request this; 4214 } 4215 4216 /// 4217 HttpRequestWrapper request(string uri, HttpVerb requestMethod = HttpVerb.GET, ubyte[] bodyBytes = null) { 4218 if(uri[0] == '/') 4219 uri = uri[1 .. $]; 4220 4221 auto u = Uri(uri).basedOn(Uri(urlBase)); 4222 4223 auto req = httpClient.navigateTo(u, requestMethod); 4224 4225 if(oauth2Token.length) 4226 req.requestParameters.headers ~= "Authorization: "~ authType ~" " ~ oauth2Token; 4227 req.requestParameters.contentType = submittedContentType; 4228 req.requestParameters.bodyData = bodyBytes; 4229 4230 return HttpRequestWrapper(this, req); 4231 } 4232 4233 /// 4234 var throwOnError(HttpResponse res) { 4235 if(res.code < 200 || res.code >= 300) 4236 throw new Exception(res.codeText ~ " " ~ res.contentText); 4237 4238 var response = var.fromJson(res.contentText); 4239 if(response.errors) { 4240 throw new Exception(response.errors.toJson()); 4241 } 4242 4243 return response; 4244 } 4245 4246 /// 4247 @property RestBuilder rest() { 4248 return RestBuilder(this, null, null); 4249 } 4250 4251 // hipchat.rest.room["Tech Team"].history 4252 // gives: "/room/Tech%20Team/history" 4253 // 4254 // hipchat.rest.room["Tech Team"].history("page", "12) 4255 /// 4256 static struct RestBuilder { 4257 HttpApiClientType apiClient; 4258 string[] pathParts; 4259 string[2][] queryParts; 4260 this(HttpApiClientType apiClient, string[] pathParts, string[2][] queryParts) { 4261 this.apiClient = apiClient; 4262 this.pathParts = pathParts; 4263 this.queryParts = queryParts; 4264 } 4265 4266 RestBuilder _SELF() { 4267 return this; 4268 } 4269 4270 /// The args are so you can call opCall on the returned 4271 /// object, despite @property being broken af in D. 4272 RestBuilder opDispatch(string str, T)(string n, T v) { 4273 return RestBuilder(apiClient, pathParts ~ str, queryParts ~ [n, to!string(v)]); 4274 } 4275 4276 /// 4277 RestBuilder opDispatch(string str)() { 4278 return RestBuilder(apiClient, pathParts ~ str, queryParts); 4279 } 4280 4281 4282 /// 4283 RestBuilder opIndex(string str) { 4284 return RestBuilder(apiClient, pathParts ~ str, queryParts); 4285 } 4286 /// 4287 RestBuilder opIndex(var str) { 4288 return RestBuilder(apiClient, pathParts ~ str.get!string, queryParts); 4289 } 4290 /// 4291 RestBuilder opIndex(int i) { 4292 return RestBuilder(apiClient, pathParts ~ to!string(i), queryParts); 4293 } 4294 4295 /// 4296 RestBuilder opCall(T)(string name, T value) { 4297 return RestBuilder(apiClient, pathParts, queryParts ~ [name, to!string(value)]); 4298 } 4299 4300 /// 4301 string toUri() { 4302 string result; 4303 foreach(idx, part; pathParts) { 4304 if(idx) 4305 result ~= "/"; 4306 result ~= encodeUriComponent(part); 4307 } 4308 result ~= "?"; 4309 foreach(idx, part; queryParts) { 4310 if(idx) 4311 result ~= "&"; 4312 result ~= encodeUriComponent(part[0]); 4313 result ~= "="; 4314 result ~= encodeUriComponent(part[1]); 4315 } 4316 4317 return result; 4318 } 4319 4320 /// 4321 final HttpRequestWrapper GET() { return _EXECUTE(HttpVerb.GET, this.toUri(), ToBytesResult.init); } 4322 /// ditto 4323 final HttpRequestWrapper DELETE() { return _EXECUTE(HttpVerb.DELETE, this.toUri(), ToBytesResult.init); } 4324 4325 // need to be able to send: JSON, urlencoded, multipart/form-data, and raw stuff. 4326 /// ditto 4327 final HttpRequestWrapper POST(T...)(T t) { return _EXECUTE(HttpVerb.POST, this.toUri(), toBytes(t)); } 4328 /// ditto 4329 final HttpRequestWrapper PATCH(T...)(T t) { return _EXECUTE(HttpVerb.PATCH, this.toUri(), toBytes(t)); } 4330 /// ditto 4331 final HttpRequestWrapper PUT(T...)(T t) { return _EXECUTE(HttpVerb.PUT, this.toUri(), toBytes(t)); } 4332 4333 struct ToBytesResult { 4334 ubyte[] bytes; 4335 string contentType; 4336 } 4337 4338 private ToBytesResult toBytes(T...)(T t) { 4339 import std.conv : to; 4340 static if(T.length == 0) 4341 return ToBytesResult(null, null); 4342 else static if(T.length == 1 && is(T[0] == var)) 4343 return ToBytesResult(cast(ubyte[]) t[0].toJson(), "application/json"); // json data 4344 else static if(T.length == 1 && (is(T[0] == string) || is(T[0] == ubyte[]))) 4345 return ToBytesResult(cast(ubyte[]) t[0], null); // raw data 4346 else static if(T.length == 1 && is(T[0] : FormData)) 4347 return ToBytesResult(t[0].toBytes, t[0].contentType); 4348 else static if(T.length > 1 && T.length % 2 == 0 && is(T[0] == string)) { 4349 // string -> value pairs for a POST request 4350 string answer; 4351 foreach(idx, val; t) { 4352 static if(idx % 2 == 0) { 4353 if(answer.length) 4354 answer ~= "&"; 4355 answer ~= encodeUriComponent(val); // it had better be a string! lol 4356 answer ~= "="; 4357 } else { 4358 answer ~= encodeUriComponent(to!string(val)); 4359 } 4360 } 4361 4362 return ToBytesResult(cast(ubyte[]) answer, "application/x-www-form-urlencoded"); 4363 } 4364 else 4365 static assert(0); // FIXME 4366 4367 } 4368 4369 HttpRequestWrapper _EXECUTE(HttpVerb verb, string uri, ubyte[] bodyBytes) { 4370 return apiClient.request(uri, verb, bodyBytes); 4371 } 4372 4373 HttpRequestWrapper _EXECUTE(HttpVerb verb, string uri, ToBytesResult tbr) { 4374 auto r = apiClient.request(uri, verb, tbr.bytes); 4375 if(tbr.contentType !is null) 4376 r.requestParameters.contentType = tbr.contentType; 4377 return r; 4378 } 4379 } 4380 } 4381 4382 4383 // see also: arsd.cgi.encodeVariables 4384 /++ 4385 Creates a multipart/form-data object that is suitable for file uploads and other kinds of POST. 4386 4387 It has a set of names and values of mime components. Names can be repeated. They will be presented in the same order in which you add them. You will mostly want to use the [append] method. 4388 4389 You can pass this directly to [HttpClient.request]. 4390 4391 Based on: https://developer.mozilla.org/en-US/docs/Web/API/FormData 4392 4393 --- 4394 auto fd = new FormData(); 4395 // add some data, plain string first 4396 fd.append("name", "Adam"); 4397 // then a file 4398 fd.append("photo", std.file.read("adam.jpg"), "image/jpeg", "adam.jpg"); 4399 4400 // post it! 4401 auto client = new HttpClient(); 4402 client.request(Uri("http://example.com/people"), fd).waitForCompletion(); 4403 --- 4404 4405 History: 4406 Added June 8, 2018 4407 +/ 4408 class FormData { 4409 static struct MimePart { 4410 string name; 4411 const(void)[] data; 4412 string contentType; 4413 string filename; 4414 } 4415 4416 private MimePart[] parts; 4417 private string boundary = "0016e64be86203dd36047610926a"; // FIXME 4418 4419 /++ 4420 Appends the given entry to the request. This can be a simple key/value pair of strings or file uploads. 4421 4422 For a simple key/value pair, leave `contentType` and `filename` as `null`. 4423 4424 For file uploads, please note that many servers require filename be given for a file upload and it may not allow you to put in a path. I suggest using [std.path.baseName] to strip off path information from a file you are loading. 4425 4426 The `contentType` is generally verified by servers for file uploads. 4427 +/ 4428 void append(string key, const(void)[] value, string contentType = null, string filename = null) { 4429 parts ~= MimePart(key, value, contentType, filename); 4430 } 4431 4432 /++ 4433 Deletes any entries from the set with the given key. 4434 4435 History: 4436 Added June 7, 2023 (dub v11.0) 4437 +/ 4438 void deleteKey(string key) { 4439 MimePart[] newParts; 4440 foreach(part; parts) 4441 if(part.name != key) 4442 newParts ~= part; 4443 parts = newParts; 4444 } 4445 4446 /++ 4447 Returns the first entry with the given key, or `MimePart.init` if there is nothing. 4448 4449 History: 4450 Added June 7, 2023 (dub v11.0) 4451 +/ 4452 MimePart get(string key) { 4453 foreach(part; parts) 4454 if(part.name == key) 4455 return part; 4456 return MimePart.init; 4457 } 4458 4459 /++ 4460 Returns the all entries with the given key. 4461 4462 History: 4463 Added June 7, 2023 (dub v11.0) 4464 +/ 4465 MimePart[] getAll(string key) { 4466 MimePart[] answer; 4467 foreach(part; parts) 4468 if(part.name == key) 4469 answer ~= part; 4470 return answer; 4471 } 4472 4473 /++ 4474 Returns true if the given key exists in the set. 4475 4476 History: 4477 Added June 7, 2023 (dub v11.0) 4478 +/ 4479 bool has(string key) { 4480 return get(key).name == key; 4481 } 4482 4483 /++ 4484 Sets the given key to the given value if it exists, or appends it if it doesn't. 4485 4486 You probably want [append] instead. 4487 4488 See_Also: 4489 [append] 4490 4491 History: 4492 Added June 7, 2023 (dub v11.0) 4493 +/ 4494 void set(string key, const(void)[] value, string contentType, string filename) { 4495 foreach(ref part; parts) 4496 if(part.name == key) { 4497 part.data = value; 4498 part.contentType = contentType; 4499 part.filename = filename; 4500 return; 4501 } 4502 4503 append(key, value, contentType, filename); 4504 } 4505 4506 /++ 4507 Returns all the current entries in the object. 4508 4509 History: 4510 Added June 7, 2023 (dub v11.0) 4511 +/ 4512 MimePart[] entries() { 4513 return parts; 4514 } 4515 4516 // FIXME: 4517 // keys iterator 4518 // values iterator 4519 4520 /++ 4521 Gets the content type header that should be set in the request. This includes the type and boundary that is applicable to the [toBytes] method. 4522 +/ 4523 string contentType() { 4524 return "multipart/form-data; boundary=" ~ boundary; 4525 } 4526 4527 /++ 4528 Returns bytes applicable for the body of this request. Use the [contentType] method to get the appropriate content type header with the right boundary. 4529 +/ 4530 ubyte[] toBytes() { 4531 string data; 4532 4533 foreach(part; parts) { 4534 data ~= "--" ~ boundary ~ "\r\n"; 4535 data ~= "Content-Disposition: form-data; name=\""~part.name~"\""; 4536 if(part.filename !is null) 4537 data ~= "; filename=\""~part.filename~"\""; 4538 data ~= "\r\n"; 4539 if(part.contentType !is null) 4540 data ~= "Content-Type: " ~ part.contentType ~ "\r\n"; 4541 data ~= "\r\n"; 4542 4543 data ~= cast(string) part.data; 4544 4545 data ~= "\r\n"; 4546 } 4547 4548 data ~= "--" ~ boundary ~ "--\r\n"; 4549 4550 return cast(ubyte[]) data; 4551 } 4552 } 4553 4554 private bool bicmp(in ubyte[] item, in char[] search) { 4555 if(item.length != search.length) return false; 4556 4557 foreach(i; 0 .. item.length) { 4558 ubyte a = item[i]; 4559 ubyte b = search[i]; 4560 if(a >= 'A' && a <= 'Z') 4561 a += 32; 4562 //if(b >= 'A' && b <= 'Z') 4563 //b += 32; 4564 if(a != b) 4565 return false; 4566 } 4567 4568 return true; 4569 } 4570 4571 /++ 4572 WebSocket client, based on the browser api, though also with other api options. 4573 4574 --- 4575 import arsd.http2; 4576 4577 void main() { 4578 auto ws = new WebSocket(Uri("ws://....")); 4579 4580 ws.onmessage = (in char[] msg) { 4581 ws.send("a reply"); 4582 }; 4583 4584 ws.connect(); 4585 4586 WebSocket.eventLoop(); 4587 } 4588 --- 4589 4590 Symbol_groups: 4591 foundational = 4592 Used with all API styles. 4593 4594 browser_api = 4595 API based on the standard in the browser. 4596 4597 event_loop_integration = 4598 Integrating with external event loops is done through static functions. You should 4599 call these BEFORE doing anything else with the WebSocket module or class. 4600 4601 $(PITFALL NOT IMPLEMENTED) 4602 --- 4603 WebSocket.setEventLoopProxy(arsd.simpledisplay.EventLoop.proxy.tupleof); 4604 // or something like that. it is not implemented yet. 4605 --- 4606 $(PITFALL NOT IMPLEMENTED) 4607 4608 blocking_api = 4609 The blocking API is best used when you only need basic functionality with a single connection. 4610 4611 --- 4612 WebSocketFrame msg; 4613 do { 4614 // FIXME good demo 4615 } while(msg); 4616 --- 4617 4618 Or to check for blocks before calling: 4619 4620 --- 4621 try_to_process_more: 4622 while(ws.isMessageBuffered()) { 4623 auto msg = ws.waitForNextMessage(); 4624 // process msg 4625 } 4626 if(ws.isDataPending()) { 4627 ws.lowLevelReceive(); 4628 goto try_to_process_more; 4629 } else { 4630 // nothing ready, you can do other things 4631 // or at least sleep a while before trying 4632 // to process more. 4633 if(ws.readyState == WebSocket.OPEN) { 4634 Thread.sleep(1.seconds); 4635 goto try_to_process_more; 4636 } 4637 } 4638 --- 4639 4640 +/ 4641 class WebSocket { 4642 private Uri uri; 4643 private string[string] cookies; 4644 4645 private string host; 4646 private ushort port; 4647 private bool ssl; 4648 4649 // used to decide if we mask outgoing msgs 4650 private bool isClient; 4651 4652 private MonoTime timeoutFromInactivity; 4653 private MonoTime nextPing; 4654 4655 /++ 4656 wss://echo.websocket.org 4657 +/ 4658 /// Group: foundational 4659 this(Uri uri, Config config = Config.init) 4660 //in (uri.scheme == "ws" || uri.scheme == "wss") 4661 in { assert(uri.scheme == "ws" || uri.scheme == "wss"); } do 4662 { 4663 this.uri = uri; 4664 this.config = config; 4665 4666 this.receiveBuffer = new ubyte[](config.initialReceiveBufferSize); 4667 4668 host = uri.host; 4669 ssl = uri.scheme == "wss"; 4670 port = cast(ushort) (uri.port ? uri.port : ssl ? 443 : 80); 4671 4672 if(ssl) { 4673 version(with_openssl) { 4674 loadOpenSsl(); 4675 socket = new SslClientSocket(family(uri.unixSocketPath), SocketType.STREAM, host, config.verifyPeer); 4676 } else 4677 throw new Exception("SSL not compiled in"); 4678 } else 4679 socket = new Socket(family(uri.unixSocketPath), SocketType.STREAM); 4680 4681 socket.setOption(SocketOptionLevel.TCP, SocketOption.TCP_NODELAY, 1); 4682 cookies = config.cookies; 4683 } 4684 4685 /++ 4686 4687 +/ 4688 /// Group: foundational 4689 void connect() { 4690 this.isClient = true; 4691 4692 socket.blocking = false; 4693 4694 if(uri.unixSocketPath) 4695 socket.connect(new UnixAddress(uri.unixSocketPath)); 4696 else 4697 socket.connect(new InternetAddress(host, port)); // FIXME: ipv6 support... 4698 4699 4700 auto readSet = new SocketSet(); 4701 auto writeSet = new SocketSet(); 4702 4703 readSet.reset(); 4704 writeSet.reset(); 4705 4706 readSet.add(socket); 4707 writeSet.add(socket); 4708 4709 auto selectGot = Socket.select(readSet, writeSet, null, config.timeoutFromInactivity); 4710 if(selectGot == -1) { 4711 // interrupted 4712 4713 throw new Exception("Websocket connection interrupted - retry might succeed"); 4714 } else if(selectGot == 0) { 4715 // time out 4716 socket.close(); 4717 throw new Exception("Websocket connection timed out"); 4718 } else { 4719 if(writeSet.isSet(socket) || readSet.isSet(socket)) { 4720 import core.stdc.stdint; 4721 int32_t error; 4722 int retopt = socket.getOption(SocketOptionLevel.SOCKET, SocketOption.ERROR, error); 4723 if(retopt < 0 || error != 0) { 4724 socket.close(); 4725 throw new Exception("Websocket connection failed - " ~ formatSocketError(error)); 4726 } else { 4727 // FIXME: websocket handshake could and really should be async too. 4728 socket.blocking = true; // just convenience 4729 if(auto s = cast(SslClientSocket) socket) { 4730 s.do_ssl_connect(); 4731 } else { 4732 // we're ready 4733 } 4734 } 4735 } 4736 } 4737 4738 auto uri = this.uri.path.length ? this.uri.path : "/"; 4739 if(this.uri.query.length) { 4740 uri ~= "?"; 4741 uri ~= this.uri.query; 4742 } 4743 4744 // the headers really shouldn't be bigger than this, at least 4745 // the chunks i need to process 4746 ubyte[4096] bufferBacking = void; 4747 ubyte[] buffer = bufferBacking[]; 4748 size_t pos; 4749 4750 void append(in char[][] items...) { 4751 foreach(what; items) { 4752 if((pos + what.length) > buffer.length) { 4753 buffer.length += 4096; 4754 } 4755 buffer[pos .. pos + what.length] = cast(ubyte[]) what[]; 4756 pos += what.length; 4757 } 4758 } 4759 4760 append("GET ", uri, " HTTP/1.1\r\n"); 4761 append("Host: ", this.uri.host, "\r\n"); 4762 4763 append("Upgrade: websocket\r\n"); 4764 append("Connection: Upgrade\r\n"); 4765 append("Sec-WebSocket-Version: 13\r\n"); 4766 4767 // FIXME: randomize this 4768 append("Sec-WebSocket-Key: x3JEHMbDL1EzLkh9GBhXDw==\r\n"); 4769 if(cookies.length > 0) { 4770 append("Cookie: "); 4771 bool first=true; 4772 foreach(k,v;cookies) { 4773 if(first) first = false; 4774 else append("; "); 4775 append(k); 4776 append("="); 4777 append(v); 4778 } 4779 append("\r\n"); 4780 } 4781 /* 4782 //This is equivalent but has dependencies 4783 import std.format; 4784 import std.algorithm : map; 4785 append(format("cookie: %-(%s %)\r\n",cookies.byKeyValue.map!(t=>format("%s=%s",t.key,t.value)))); 4786 */ 4787 4788 if(config.protocol.length) 4789 append("Sec-WebSocket-Protocol: ", config.protocol, "\r\n"); 4790 if(config.origin.length) 4791 append("Origin: ", config.origin, "\r\n"); 4792 4793 foreach(h; config.additionalHeaders) { 4794 append(h); 4795 append("\r\n"); 4796 } 4797 4798 append("\r\n"); 4799 4800 auto remaining = buffer[0 .. pos]; 4801 //import std.stdio; writeln(host, " " , port, " ", cast(string) remaining); 4802 while(remaining.length) { 4803 auto r = socket.send(remaining); 4804 if(r < 0) 4805 throw new Exception(lastSocketError(socket)); 4806 if(r == 0) 4807 throw new Exception("unexpected connection termination"); 4808 remaining = remaining[r .. $]; 4809 } 4810 4811 // the response shouldn't be especially large at this point, just 4812 // headers for the most part. gonna try to get it in the stack buffer. 4813 // then copy stuff after headers, if any, to the frame buffer. 4814 ubyte[] used; 4815 4816 void more() { 4817 auto r = socket.receive(buffer[used.length .. $]); 4818 4819 if(r < 0) 4820 throw new Exception(lastSocketError(socket)); 4821 if(r == 0) 4822 throw new Exception("unexpected connection termination"); 4823 //import std.stdio;writef("%s", cast(string) buffer[used.length .. used.length + r]); 4824 4825 used = buffer[0 .. used.length + r]; 4826 } 4827 4828 more(); 4829 4830 import std.algorithm; 4831 if(!used.startsWith(cast(ubyte[]) "HTTP/1.1 101")) 4832 throw new Exception("didn't get a websocket answer"); 4833 // skip the status line 4834 while(used.length && used[0] != '\n') 4835 used = used[1 .. $]; 4836 4837 if(used.length == 0) 4838 throw new Exception("Remote server disconnected or didn't send enough information"); 4839 4840 if(used.length < 1) 4841 more(); 4842 4843 used = used[1 .. $]; // skip the \n 4844 4845 if(used.length == 0) 4846 more(); 4847 4848 // checks on the protocol from ehaders 4849 bool isWebsocket; 4850 bool isUpgrade; 4851 const(ubyte)[] protocol; 4852 const(ubyte)[] accept; 4853 4854 while(used.length) { 4855 if(used.length >= 2 && used[0] == '\r' && used[1] == '\n') { 4856 used = used[2 .. $]; 4857 break; // all done 4858 } 4859 int idxColon; 4860 while(idxColon < used.length && used[idxColon] != ':') 4861 idxColon++; 4862 if(idxColon == used.length) 4863 more(); 4864 auto idxStart = idxColon + 1; 4865 while(idxStart < used.length && used[idxStart] == ' ') 4866 idxStart++; 4867 if(idxStart == used.length) 4868 more(); 4869 auto idxEnd = idxStart; 4870 while(idxEnd < used.length && used[idxEnd] != '\r') 4871 idxEnd++; 4872 if(idxEnd == used.length) 4873 more(); 4874 4875 auto headerName = used[0 .. idxColon]; 4876 auto headerValue = used[idxStart .. idxEnd]; 4877 4878 // move past this header 4879 used = used[idxEnd .. $]; 4880 // and the \r\n 4881 if(2 <= used.length) 4882 used = used[2 .. $]; 4883 4884 if(headerName.bicmp("upgrade")) { 4885 if(headerValue.bicmp("websocket")) 4886 isWebsocket = true; 4887 } else if(headerName.bicmp("connection")) { 4888 if(headerValue.bicmp("upgrade")) 4889 isUpgrade = true; 4890 } else if(headerName.bicmp("sec-websocket-accept")) { 4891 accept = headerValue; 4892 } else if(headerName.bicmp("sec-websocket-protocol")) { 4893 protocol = headerValue; 4894 } 4895 4896 if(!used.length) { 4897 more(); 4898 } 4899 } 4900 4901 4902 if(!isWebsocket) 4903 throw new Exception("didn't answer as websocket"); 4904 if(!isUpgrade) 4905 throw new Exception("didn't answer as upgrade"); 4906 4907 4908 // FIXME: check protocol if config requested one 4909 // FIXME: check accept for the right hash 4910 4911 receiveBuffer[0 .. used.length] = used[]; 4912 receiveBufferUsedLength = used.length; 4913 4914 readyState_ = OPEN; 4915 4916 if(onopen) 4917 onopen(); 4918 4919 nextPing = MonoTime.currTime + config.pingFrequency.msecs; 4920 timeoutFromInactivity = MonoTime.currTime + config.timeoutFromInactivity; 4921 4922 registerActiveSocket(this); 4923 } 4924 4925 /++ 4926 Is data pending on the socket? Also check [isMessageBuffered] to see if there 4927 is already a message in memory too. 4928 4929 If this returns `true`, you can call [lowLevelReceive], then try [isMessageBuffered] 4930 again. 4931 +/ 4932 /// Group: blocking_api 4933 public bool isDataPending(Duration timeout = 0.seconds) { 4934 static SocketSet readSet; 4935 if(readSet is null) 4936 readSet = new SocketSet(); 4937 4938 version(with_openssl) 4939 if(auto s = cast(SslClientSocket) socket) { 4940 // select doesn't handle the case with stuff 4941 // left in the ssl buffer so i'm checking it separately 4942 if(s.dataPending()) { 4943 return true; 4944 } 4945 } 4946 4947 readSet.reset(); 4948 4949 readSet.add(socket); 4950 4951 //tryAgain: 4952 auto selectGot = Socket.select(readSet, null, null, timeout); 4953 if(selectGot == 0) { /* timeout */ 4954 // timeout 4955 return false; 4956 } else if(selectGot == -1) { /* interrupted */ 4957 return false; 4958 } else { /* ready */ 4959 if(readSet.isSet(socket)) { 4960 return true; 4961 } 4962 } 4963 4964 return false; 4965 } 4966 4967 private void llsend(ubyte[] d) { 4968 if(readyState == CONNECTING) 4969 throw new Exception("WebSocket not connected when trying to send. Did you forget to call connect(); ?"); 4970 //connect(); 4971 //import std.stdio; writeln("LLSEND: ", d); 4972 while(d.length) { 4973 auto r = socket.send(d); 4974 if(r < 0 && wouldHaveBlocked()) { 4975 // FIXME: i should register for a write wakeup 4976 version(use_arsd_core) assert(0); 4977 import core.thread; 4978 Thread.sleep(1.msecs); 4979 continue; 4980 } 4981 //import core.stdc.errno; import std.stdio; writeln(errno); 4982 if(r <= 0) { 4983 // import std.stdio; writeln(GetLastError()); 4984 throw new Exception("Socket send failed"); 4985 } 4986 d = d[r .. $]; 4987 } 4988 } 4989 4990 private void llclose() { 4991 // import std.stdio; writeln("LLCLOSE"); 4992 socket.shutdown(SocketShutdown.SEND); 4993 } 4994 4995 /++ 4996 Waits for more data off the low-level socket and adds it to the pending buffer. 4997 4998 Returns `true` if the connection is still active. 4999 +/ 5000 /// Group: blocking_api 5001 public bool lowLevelReceive() { 5002 if(readyState == CONNECTING) 5003 throw new Exception("WebSocket not connected when trying to receive. Did you forget to call connect(); ?"); 5004 if (receiveBufferUsedLength == receiveBuffer.length) 5005 { 5006 if (receiveBuffer.length == config.maximumReceiveBufferSize) 5007 throw new Exception("Maximum receive buffer size exhausted"); 5008 5009 import std.algorithm : min; 5010 receiveBuffer.length = min(receiveBuffer.length + config.initialReceiveBufferSize, 5011 config.maximumReceiveBufferSize); 5012 } 5013 auto r = socket.receive(receiveBuffer[receiveBufferUsedLength .. $]); 5014 if(r == 0) 5015 return false; 5016 if(r < 0 && wouldHaveBlocked()) 5017 return true; 5018 if(r <= 0) { 5019 //import std.stdio; writeln(WSAGetLastError()); 5020 return false; 5021 } 5022 receiveBufferUsedLength += r; 5023 return true; 5024 } 5025 5026 private Socket socket; 5027 5028 /* copy/paste section { */ 5029 5030 private int readyState_; 5031 private ubyte[] receiveBuffer; 5032 private size_t receiveBufferUsedLength; 5033 5034 private Config config; 5035 5036 enum CONNECTING = 0; /// Socket has been created. The connection is not yet open. 5037 enum OPEN = 1; /// The connection is open and ready to communicate. 5038 enum CLOSING = 2; /// The connection is in the process of closing. 5039 enum CLOSED = 3; /// The connection is closed or couldn't be opened. 5040 5041 /++ 5042 5043 +/ 5044 /// Group: foundational 5045 static struct Config { 5046 /++ 5047 These control the size of the receive buffer. 5048 5049 It starts at the initial size, will temporarily 5050 balloon up to the maximum size, and will reuse 5051 a buffer up to the likely size. 5052 5053 Anything larger than the maximum size will cause 5054 the connection to be aborted and an exception thrown. 5055 This is to protect you against a peer trying to 5056 exhaust your memory, while keeping the user-level 5057 processing simple. 5058 +/ 5059 size_t initialReceiveBufferSize = 4096; 5060 size_t likelyReceiveBufferSize = 4096; /// ditto 5061 size_t maximumReceiveBufferSize = 10 * 1024 * 1024; /// ditto 5062 5063 /++ 5064 Maximum combined size of a message. 5065 +/ 5066 size_t maximumMessageSize = 10 * 1024 * 1024; 5067 5068 string[string] cookies; /// Cookies to send with the initial request. cookies[name] = value; 5069 string origin; /// Origin URL to send with the handshake, if desired. 5070 string protocol; /// the protocol header, if desired. 5071 5072 /++ 5073 Additional headers to put in the HTTP request. These should be formatted `Name: value`, like for example: 5074 5075 --- 5076 Config config; 5077 config.additionalHeaders ~= "Authorization: Bearer your_auth_token_here"; 5078 --- 5079 5080 History: 5081 Added February 19, 2021 (included in dub version 9.2) 5082 +/ 5083 string[] additionalHeaders; 5084 5085 /++ 5086 Amount of time (in msecs) of idleness after which to send an automatic ping 5087 5088 Please note how this interacts with [timeoutFromInactivity] - a ping counts as activity that 5089 keeps the socket alive. 5090 +/ 5091 int pingFrequency = 5000; 5092 5093 /++ 5094 Amount of time to disconnect when there's no activity. Note that automatic pings will keep the connection alive; this timeout only occurs if there's absolutely nothing, including no responses to websocket ping frames. Since the default [pingFrequency] is only seconds, this one minute should never elapse unless the connection is actually dead. 5095 5096 The one thing to keep in mind is if your program is busy and doesn't check input, it might consider this a time out since there's no activity. The reason is that your program was busy rather than a connection failure, but it doesn't care. You should avoid long processing periods anyway though! 5097 5098 History: 5099 Added March 31, 2021 (included in dub version 9.4) 5100 +/ 5101 Duration timeoutFromInactivity = 1.minutes; 5102 5103 /++ 5104 For https connections, if this is `true`, it will fail to connect if the TLS certificate can not be 5105 verified. Setting this to `false` will skip this check and allow the connection to continue anyway. 5106 5107 History: 5108 Added April 5, 2022 (dub v10.8) 5109 5110 Prior to this, it always used the global (but undocumented) `defaultVerifyPeer` setting, and sometimes 5111 even if it was true, it would skip the verification. Now, it always respects this local setting. 5112 +/ 5113 bool verifyPeer = true; 5114 } 5115 5116 /++ 5117 Returns one of [CONNECTING], [OPEN], [CLOSING], or [CLOSED]. 5118 +/ 5119 int readyState() { 5120 return readyState_; 5121 } 5122 5123 /++ 5124 Closes the connection, sending a graceful teardown message to the other side. 5125 If you provide no arguments, it sends code 1000, normal closure. If you provide 5126 a code, you should also provide a short reason string. 5127 5128 Params: 5129 code = reason code. 5130 5131 0-999 are invalid. 5132 1000-2999 are defined by the RFC. [https://www.rfc-editor.org/rfc/rfc6455.html#section-7.4.1] 5133 1000 - normal finish 5134 1001 - endpoint going away 5135 1002 - protocol error 5136 1003 - unacceptable data received (e.g. binary message when you can't handle it) 5137 1004 - reserved 5138 1005 - missing status code (should not be set except by implementations) 5139 1006 - abnormal connection closure (should only be set by implementations) 5140 1007 - inconsistent data received (i.e. utf-8 decode error in text message) 5141 1008 - policy violation 5142 1009 - received message too big 5143 1010 - client aborting due to required extension being unsupported by the server 5144 1011 - server had unexpected failure 5145 1015 - reserved for TLS handshake failure 5146 3000-3999 are to be registered with IANA. 5147 4000-4999 are private-use custom codes depending on the application. These are what you'd most commonly set here. 5148 5149 reason = <= 123 bytes of human-readable reason text, used for logs and debugging 5150 5151 History: 5152 The default `code` was changed to 1000 on January 9, 2023. Previously it was 0, 5153 but also ignored anyway. 5154 5155 On May 11, 2024, the optional arguments were changed to overloads since if you provide a code, you should also provide a reason. 5156 +/ 5157 /// Group: foundational 5158 void close() { 5159 close(1000, null); 5160 } 5161 5162 /// ditto 5163 void close(int code, string reason) 5164 //in (reason.length < 123) 5165 in { assert(reason.length <= 123); } do 5166 { 5167 if(readyState_ != OPEN) 5168 return; // it cool, we done 5169 WebSocketFrame wss; 5170 wss.fin = true; 5171 wss.masked = this.isClient; 5172 wss.opcode = WebSocketOpcode.close; 5173 wss.data = [ubyte((code >> 8) & 0xff), ubyte(code & 0xff)] ~ cast(ubyte[]) reason.dup; 5174 wss.send(&llsend); 5175 5176 readyState_ = CLOSING; 5177 5178 closeCalled = true; 5179 5180 llclose(); 5181 } 5182 5183 deprecated("If you provide a code, please also provide a reason string") void close(int code) { 5184 close(code, null); 5185 } 5186 5187 5188 private bool closeCalled; 5189 5190 /++ 5191 Sends a ping message to the server. This is done automatically by the library if you set a non-zero [Config.pingFrequency], but you can also send extra pings explicitly as well with this function. 5192 +/ 5193 /// Group: foundational 5194 void ping(in ubyte[] data = null) { 5195 WebSocketFrame wss; 5196 wss.fin = true; 5197 wss.masked = this.isClient; 5198 wss.opcode = WebSocketOpcode.ping; 5199 if(data !is null) wss.data = data.dup; 5200 wss.send(&llsend); 5201 } 5202 5203 /++ 5204 Sends a pong message to the server. This is normally done automatically in response to pings. 5205 +/ 5206 /// Group: foundational 5207 void pong(in ubyte[] data = null) { 5208 WebSocketFrame wss; 5209 wss.fin = true; 5210 wss.masked = this.isClient; 5211 wss.opcode = WebSocketOpcode.pong; 5212 if(data !is null) wss.data = data.dup; 5213 wss.send(&llsend); 5214 } 5215 5216 /++ 5217 Sends a text message through the websocket. 5218 +/ 5219 /// Group: foundational 5220 void send(in char[] textData) { 5221 WebSocketFrame wss; 5222 wss.fin = true; 5223 wss.masked = this.isClient; 5224 wss.opcode = WebSocketOpcode.text; 5225 wss.data = cast(ubyte[]) textData.dup; 5226 wss.send(&llsend); 5227 } 5228 5229 /++ 5230 Sends a binary message through the websocket. 5231 +/ 5232 /// Group: foundational 5233 void send(in ubyte[] binaryData) { 5234 WebSocketFrame wss; 5235 wss.masked = this.isClient; 5236 wss.fin = true; 5237 wss.opcode = WebSocketOpcode.binary; 5238 wss.data = cast(ubyte[]) binaryData.dup; 5239 wss.send(&llsend); 5240 } 5241 5242 /++ 5243 Waits for and returns the next complete message on the socket. 5244 5245 Note that the onmessage function is still called, right before 5246 this returns. 5247 +/ 5248 /// Group: blocking_api 5249 public WebSocketFrame waitForNextMessage() { 5250 do { 5251 auto m = processOnce(); 5252 if(m.populated) 5253 return m; 5254 } while(lowLevelReceive()); 5255 5256 return WebSocketFrame.init; // FIXME? maybe. 5257 } 5258 5259 /++ 5260 Tells if [waitForNextMessage] would block. 5261 +/ 5262 /// Group: blocking_api 5263 public bool waitForNextMessageWouldBlock() { 5264 checkAgain: 5265 if(isMessageBuffered()) 5266 return false; 5267 if(!isDataPending()) 5268 return true; 5269 while(isDataPending()) 5270 if(lowLevelReceive() == false) 5271 return false; 5272 goto checkAgain; 5273 } 5274 5275 /++ 5276 Is there a message in the buffer already? 5277 If `true`, [waitForNextMessage] is guaranteed to return immediately. 5278 If `false`, check [isDataPending] as the next step. 5279 +/ 5280 /// Group: blocking_api 5281 public bool isMessageBuffered() { 5282 ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; 5283 auto s = d; 5284 if(d.length) { 5285 auto orig = d; 5286 auto m = WebSocketFrame.read(d); 5287 // that's how it indicates that it needs more data 5288 if(d !is orig) 5289 return true; 5290 } 5291 5292 return false; 5293 } 5294 5295 private ubyte continuingType; 5296 private ubyte[] continuingData; 5297 //private size_t continuingDataLength; 5298 5299 private WebSocketFrame processOnce() { 5300 ubyte[] d = receiveBuffer[0 .. receiveBufferUsedLength]; 5301 auto s = d; 5302 // FIXME: handle continuation frames more efficiently. it should really just reuse the receive buffer. 5303 WebSocketFrame m; 5304 if(d.length) { 5305 auto orig = d; 5306 m = WebSocketFrame.read(d); 5307 // that's how it indicates that it needs more data 5308 if(d is orig) 5309 return WebSocketFrame.init; 5310 m.unmaskInPlace(); 5311 switch(m.opcode) { 5312 case WebSocketOpcode.continuation: 5313 if(continuingData.length + m.data.length > config.maximumMessageSize) 5314 throw new Exception("message size exceeded"); 5315 5316 continuingData ~= m.data; 5317 if(m.fin) { 5318 if(ontextmessage) 5319 ontextmessage(cast(char[]) continuingData); 5320 if(onbinarymessage) 5321 onbinarymessage(continuingData); 5322 5323 continuingData = null; 5324 } 5325 break; 5326 case WebSocketOpcode.text: 5327 if(m.fin) { 5328 if(ontextmessage) 5329 ontextmessage(m.textData); 5330 } else { 5331 continuingType = m.opcode; 5332 //continuingDataLength = 0; 5333 continuingData = null; 5334 continuingData ~= m.data; 5335 } 5336 break; 5337 case WebSocketOpcode.binary: 5338 if(m.fin) { 5339 if(onbinarymessage) 5340 onbinarymessage(m.data); 5341 } else { 5342 continuingType = m.opcode; 5343 //continuingDataLength = 0; 5344 continuingData = null; 5345 continuingData ~= m.data; 5346 } 5347 break; 5348 case WebSocketOpcode.close: 5349 5350 //import std.stdio; writeln("closed ", cast(string) m.data); 5351 5352 ushort code = CloseEvent.StandardCloseCodes.noStatusCodePresent; 5353 const(char)[] reason; 5354 5355 if(m.data.length >= 2) { 5356 code = (m.data[0] << 8) | m.data[1]; 5357 reason = (cast(char[]) m.data[2 .. $]); 5358 } 5359 5360 if(onclose) 5361 onclose(CloseEvent(code, reason, true)); 5362 5363 // if we receive one and haven't sent one back we're supposed to echo it back and close. 5364 if(!closeCalled) 5365 close(code, reason.idup); 5366 5367 readyState_ = CLOSED; 5368 5369 unregisterActiveSocket(this); 5370 socket.close(); 5371 break; 5372 case WebSocketOpcode.ping: 5373 // import std.stdio; writeln("ping received ", m.data); 5374 pong(m.data); 5375 break; 5376 case WebSocketOpcode.pong: 5377 // import std.stdio; writeln("pong received ", m.data); 5378 // just really references it is still alive, nbd. 5379 break; 5380 default: // ignore though i could and perhaps should throw too 5381 } 5382 } 5383 5384 if(d.length) { 5385 m.data = m.data.dup(); 5386 } 5387 5388 import core.stdc.string; 5389 memmove(receiveBuffer.ptr, d.ptr, d.length); 5390 receiveBufferUsedLength = d.length; 5391 5392 return m; 5393 } 5394 5395 private void autoprocess() { 5396 // FIXME 5397 do { 5398 processOnce(); 5399 } while(lowLevelReceive()); 5400 } 5401 5402 /++ 5403 Arguments for the close event. The `code` and `reason` are provided from the close message on the websocket, if they are present. The spec says code 1000 indicates a normal, default reason close, but reserves the code range from 3000-5000 for future definition; the 3000s can be registered with IANA and the 4000's are application private use. The `reason` should be user readable, but not displayed to the end user. `wasClean` is true if the server actually sent a close event, false if it just disconnected. 5404 5405 $(PITFALL 5406 The `reason` argument references a temporary buffer and there's no guarantee it will remain valid once your callback returns. It may be freed and will very likely be overwritten. If you want to keep the reason beyond the callback, make sure you `.idup` it. 5407 ) 5408 5409 History: 5410 Added March 19, 2023 (dub v11.0). 5411 +/ 5412 static struct CloseEvent { 5413 ushort code; 5414 const(char)[] reason; 5415 bool wasClean; 5416 5417 string extendedErrorInformationUnstable; 5418 5419 /++ 5420 See https://www.rfc-editor.org/rfc/rfc6455#section-7.4.1 for details. 5421 +/ 5422 enum StandardCloseCodes { 5423 purposeFulfilled = 1000, 5424 goingAway = 1001, 5425 protocolError = 1002, 5426 unacceptableData = 1003, // e.g. got text message when you can only handle binary 5427 Reserved = 1004, 5428 noStatusCodePresent = 1005, // not set by endpoint. 5429 abnormalClosure = 1006, // not set by endpoint. closed without a Close control. FIXME: maybe keep a copy of errno around for these 5430 inconsistentData = 1007, // e.g. utf8 validation failed 5431 genericPolicyViolation = 1008, 5432 messageTooBig = 1009, 5433 clientRequiredExtensionMissing = 1010, // only the client should send this 5434 unnexpectedCondition = 1011, 5435 unverifiedCertificate = 1015, // not set by client 5436 } 5437 5438 string toString() { 5439 return cast(string) (arsd.core.toStringInternal(code) ~ ": " ~ reason); 5440 } 5441 } 5442 5443 /++ 5444 The `CloseEvent` you get references a temporary buffer that may be overwritten after your handler returns. If you want to keep it or the `event.reason` member, remember to `.idup` it. 5445 5446 History: 5447 The `CloseEvent` was changed to a [arsd.core.FlexibleDelegate] on March 19, 2023 (dub v11.0). Before that, `onclose` was a public member of type `void delegate()`. This change means setters still work with or without the [CloseEvent] argument. 5448 5449 Your onclose method is now also called on abnormal terminations. Check the `wasClean` member of the `CloseEvent` to know if it came from a close frame or other cause. 5450 +/ 5451 arsd.core.FlexibleDelegate!(void delegate(CloseEvent event)) onclose; 5452 void delegate() onerror; /// 5453 void delegate(in char[]) ontextmessage; /// 5454 void delegate(in ubyte[]) onbinarymessage; /// 5455 void delegate() onopen; /// 5456 5457 /++ 5458 5459 +/ 5460 /// Group: browser_api 5461 void onmessage(void delegate(in char[]) dg) { 5462 ontextmessage = dg; 5463 } 5464 5465 /// ditto 5466 void onmessage(void delegate(in ubyte[]) dg) { 5467 onbinarymessage = dg; 5468 } 5469 5470 /* } end copy/paste */ 5471 5472 // returns true if still active 5473 private static bool readyToRead(WebSocket sock) { 5474 sock.timeoutFromInactivity = MonoTime.currTime + sock.config.timeoutFromInactivity; 5475 if(!sock.lowLevelReceive()) { 5476 sock.readyState_ = CLOSED; 5477 5478 if(sock.onerror) 5479 sock.onerror(); 5480 5481 if(sock.onclose) 5482 sock.onclose(CloseEvent(CloseEvent.StandardCloseCodes.abnormalClosure, "Connection lost", false, lastSocketError(sock.socket))); 5483 5484 unregisterActiveSocket(sock); 5485 sock.socket.close(); 5486 return false; 5487 } 5488 while(sock.processOnce().populated) {} 5489 return true; 5490 } 5491 5492 // returns true if still active, false if not 5493 private static bool timeoutAndPingCheck(WebSocket sock, MonoTime now, Duration* minimumTimeoutForSelect) { 5494 auto diff = sock.timeoutFromInactivity - now; 5495 if(diff <= 0.msecs) { 5496 // it timed out 5497 if(sock.onerror) 5498 sock.onerror(); 5499 5500 if(sock.onclose) 5501 sock.onclose(CloseEvent(CloseEvent.StandardCloseCodes.abnormalClosure, "Connection timed out", false, null)); 5502 5503 sock.readyState_ = CLOSED; 5504 unregisterActiveSocket(sock); 5505 sock.socket.close(); 5506 return false; 5507 } 5508 5509 if(minimumTimeoutForSelect && diff < *minimumTimeoutForSelect) 5510 *minimumTimeoutForSelect = diff; 5511 5512 diff = sock.nextPing - now; 5513 5514 if(diff <= 0.msecs) { 5515 //sock.send(`{"action": "ping"}`); 5516 sock.ping(); 5517 sock.nextPing = now + sock.config.pingFrequency.msecs; 5518 } else { 5519 if(minimumTimeoutForSelect && diff < *minimumTimeoutForSelect) 5520 *minimumTimeoutForSelect = diff; 5521 } 5522 5523 return true; 5524 } 5525 5526 /* 5527 const int bufferedAmount // amount pending 5528 const string extensions 5529 5530 const string protocol 5531 const string url 5532 */ 5533 5534 static { 5535 /++ 5536 Runs an event loop with all known websockets on this thread until all websockets 5537 are closed or unregistered, or until you call [exitEventLoop], or set `*localLoopExited` 5538 to false (please note it may take a few seconds until it checks that flag again; it may 5539 not exit immediately). 5540 5541 History: 5542 The `localLoopExited` parameter was added August 22, 2022 (dub v10.9) 5543 5544 See_Also: 5545 [addToSimpledisplayEventLoop] 5546 +/ 5547 void eventLoop(shared(bool)* localLoopExited = null) { 5548 import core.atomic; 5549 atomicOp!"+="(numberOfEventLoops, 1); 5550 scope(exit) { 5551 if(atomicOp!"-="(numberOfEventLoops, 1) <= 0) 5552 loopExited = false; // reset it so we can reenter 5553 } 5554 5555 version(use_arsd_core) { 5556 loopExited = false; 5557 5558 import arsd.core; 5559 getThisThreadEventLoop().run(() => WebSocket.activeSockets.length == 0 || loopExited || (localLoopExited !is null && *localLoopExited == true)); 5560 } else { 5561 static SocketSet readSet; 5562 5563 if(readSet is null) 5564 readSet = new SocketSet(); 5565 5566 loopExited = false; 5567 5568 outermost: while(!loopExited && (localLoopExited is null || (*localLoopExited == false))) { 5569 readSet.reset(); 5570 5571 Duration timeout = 3.seconds; 5572 5573 auto now = MonoTime.currTime; 5574 bool hadAny; 5575 foreach(sock; activeSockets) { 5576 if(!timeoutAndPingCheck(sock, now, &timeout)) 5577 continue outermost; 5578 5579 readSet.add(sock.socket); 5580 hadAny = true; 5581 } 5582 5583 if(!hadAny) { 5584 // import std.stdio; writeln("had none"); 5585 return; 5586 } 5587 5588 tryAgain: 5589 // import std.stdio; writeln(timeout); 5590 auto selectGot = Socket.select(readSet, null, null, timeout); 5591 if(selectGot == 0) { /* timeout */ 5592 // timeout 5593 continue; // it will be handled at the top of the loop 5594 } else if(selectGot == -1) { /* interrupted */ 5595 goto tryAgain; 5596 } else { 5597 foreach(sock; activeSockets) { 5598 if(readSet.isSet(sock.socket)) { 5599 if(!readyToRead(sock)) 5600 continue outermost; 5601 selectGot--; 5602 if(selectGot <= 0) 5603 break; 5604 } 5605 } 5606 } 5607 } 5608 } 5609 } 5610 5611 private static shared(int) numberOfEventLoops; 5612 5613 private __gshared bool loopExited; 5614 /++ 5615 Exits all running [WebSocket.eventLoop]s next time they loop around. You can call this from a signal handler or another thread. 5616 5617 Please note they may not loop around to check the flag for several seconds. Any new event loops will exit immediately until 5618 all current ones are closed. Once all event loops are exited, the flag is cleared and you can start the loop again. 5619 5620 This function is likely to be deprecated in the future due to its quirks and imprecise name. 5621 +/ 5622 void exitEventLoop() { 5623 loopExited = true; 5624 } 5625 5626 WebSocket[] activeSockets; 5627 5628 void registerActiveSocket(WebSocket s) { 5629 // ensure it isn't already there... 5630 assert(s !is null); 5631 if(s.registered) 5632 return; 5633 s.activeSocketArrayIndex = activeSockets.length; 5634 activeSockets ~= s; 5635 s.registered = true; 5636 version(use_arsd_core) { 5637 version(Posix) 5638 s.unregisterToken = arsd.core.getThisThreadEventLoop().addCallbackOnFdReadable(s.socket.handle, new arsd.core.CallbackHelper(() { s.readyToRead(s); })); 5639 } 5640 } 5641 void unregisterActiveSocket(WebSocket s) { 5642 version(use_arsd_core) { 5643 s.unregisterToken.unregister(); 5644 } 5645 5646 auto i = s.activeSocketArrayIndex; 5647 assert(activeSockets[i] is s); 5648 5649 activeSockets[i] = activeSockets[$-1]; 5650 activeSockets[i].activeSocketArrayIndex = i; 5651 activeSockets = activeSockets[0 .. $-1]; 5652 activeSockets.assumeSafeAppend(); 5653 s.registered = false; 5654 } 5655 } 5656 5657 private bool registered; 5658 private size_t activeSocketArrayIndex; 5659 version(use_arsd_core) { 5660 static import arsd.core; 5661 arsd.core.ICoreEventLoop.UnregisterToken unregisterToken; 5662 } 5663 } 5664 5665 private template imported(string mod) { 5666 mixin(`import imported = ` ~ mod ~ `;`); 5667 } 5668 5669 /++ 5670 Warning: you should call this AFTER websocket.connect or else it might throw on connect because the function sets nonblocking mode and the connect function doesn't handle that well (it throws on the "would block" condition in that function. easier to just do that first) 5671 +/ 5672 template addToSimpledisplayEventLoop() { 5673 import arsd.simpledisplay; 5674 void addToSimpledisplayEventLoop(WebSocket ws, imported!"arsd.simpledisplay".SimpleWindow window) { 5675 version(use_arsd_core) 5676 return; // already done implicitly 5677 5678 version(Windows) 5679 auto event = WSACreateEvent(); 5680 // FIXME: supposed to close event too 5681 5682 void midprocess() { 5683 version(Windows) 5684 ResetEvent(event); 5685 if(!ws.lowLevelReceive()) { 5686 ws.readyState_ = WebSocket.CLOSED; 5687 WebSocket.unregisterActiveSocket(ws); 5688 ws.socket.close(); 5689 return; 5690 } 5691 while(ws.processOnce().populated) {} 5692 } 5693 5694 version(Posix) { 5695 auto reader = new PosixFdReader(&midprocess, ws.socket.handle); 5696 } else version(none) { 5697 if(WSAAsyncSelect(ws.socket.handle, window.hwnd, WM_USER + 150, FD_CLOSE | FD_READ)) 5698 throw new Exception("WSAAsyncSelect"); 5699 5700 window.handleNativeEvent = delegate int(HWND hwnd, UINT msg, WPARAM wParam, LPARAM lParam) { 5701 if(hwnd !is window.impl.hwnd) 5702 return 1; // we don't care... 5703 switch(msg) { 5704 case WM_USER + 150: // socket activity 5705 switch(LOWORD(lParam)) { 5706 case FD_READ: 5707 case FD_CLOSE: 5708 midprocess(); 5709 break; 5710 default: 5711 // nothing 5712 } 5713 break; 5714 default: return 1; // not handled, pass it on 5715 } 5716 return 0; 5717 }; 5718 5719 } else version(Windows) { 5720 ws.socket.blocking = false; // the WSAEventSelect does this anyway and doing it here lets phobos know about it. 5721 //CreateEvent(null, 0, 0, null); 5722 if(!event) { 5723 throw new Exception("WSACreateEvent"); 5724 } 5725 if(WSAEventSelect(ws.socket.handle, event, 1/*FD_READ*/ | (1<<5)/*FD_CLOSE*/)) { 5726 //import std.stdio; writeln(WSAGetLastError()); 5727 throw new Exception("WSAEventSelect"); 5728 } 5729 5730 auto handle = new WindowsHandleReader(&midprocess, event); 5731 5732 /+ 5733 static class Ready {} 5734 5735 Ready thisr = new Ready; 5736 5737 justCommunication.addEventListener((Ready r) { 5738 if(r is thisr) 5739 midprocess(); 5740 }); 5741 5742 import core.thread; 5743 auto thread = new Thread({ 5744 while(true) { 5745 WSAWaitForMultipleEvents(1, &event, true, -1/*WSA_INFINITE*/, false); 5746 justCommunication.postEvent(thisr); 5747 } 5748 }); 5749 thread.isDaemon = true; 5750 thread.start; 5751 +/ 5752 5753 } else static assert(0, "unsupported OS"); 5754 } 5755 } 5756 5757 version(Windows) { 5758 import core.sys.windows.windows; 5759 import core.sys.windows.winsock2; 5760 } 5761 5762 version(none) { 5763 extern(Windows) int WSAAsyncSelect(SOCKET, HWND, uint, int); 5764 enum int FD_CLOSE = 1 << 5; 5765 enum int FD_READ = 1 << 0; 5766 enum int WM_USER = 1024; 5767 } 5768 5769 version(Windows) { 5770 import core.stdc.config; 5771 extern(Windows) 5772 int WSAEventSelect(SOCKET, HANDLE /* to an Event */, c_long); 5773 5774 extern(Windows) 5775 HANDLE WSACreateEvent(); 5776 5777 extern(Windows) 5778 DWORD WSAWaitForMultipleEvents(DWORD, HANDLE*, BOOL, DWORD, BOOL); 5779 } 5780 5781 /* copy/paste from cgi.d */ 5782 public { 5783 enum WebSocketOpcode : ubyte { 5784 continuation = 0, 5785 text = 1, 5786 binary = 2, 5787 // 3, 4, 5, 6, 7 RESERVED 5788 close = 8, 5789 ping = 9, 5790 pong = 10, 5791 // 11,12,13,14,15 RESERVED 5792 } 5793 5794 public struct WebSocketFrame { 5795 private bool populated; 5796 bool fin; 5797 bool rsv1; 5798 bool rsv2; 5799 bool rsv3; 5800 WebSocketOpcode opcode; // 4 bits 5801 bool masked; 5802 ubyte lengthIndicator; // don't set this when building one to send 5803 ulong realLength; // don't use when sending 5804 ubyte[4] maskingKey; // don't set this when sending 5805 ubyte[] data; 5806 5807 static WebSocketFrame simpleMessage(WebSocketOpcode opcode, in void[] data) { 5808 WebSocketFrame msg; 5809 msg.fin = true; 5810 msg.opcode = opcode; 5811 msg.data = cast(ubyte[]) data.dup; // it is mutated below when masked, so need to be cautious and copy it, sigh 5812 5813 return msg; 5814 } 5815 5816 private void send(scope void delegate(ubyte[]) llsend) { 5817 ubyte[64] headerScratch; 5818 int headerScratchPos = 0; 5819 5820 realLength = data.length; 5821 5822 { 5823 ubyte b1; 5824 b1 |= cast(ubyte) opcode; 5825 b1 |= rsv3 ? (1 << 4) : 0; 5826 b1 |= rsv2 ? (1 << 5) : 0; 5827 b1 |= rsv1 ? (1 << 6) : 0; 5828 b1 |= fin ? (1 << 7) : 0; 5829 5830 headerScratch[0] = b1; 5831 headerScratchPos++; 5832 } 5833 5834 { 5835 headerScratchPos++; // we'll set header[1] at the end of this 5836 auto rlc = realLength; 5837 ubyte b2; 5838 b2 |= masked ? (1 << 7) : 0; 5839 5840 assert(headerScratchPos == 2); 5841 5842 if(realLength > 65535) { 5843 // use 64 bit length 5844 b2 |= 0x7f; 5845 5846 // FIXME: double check endinaness 5847 foreach(i; 0 .. 8) { 5848 headerScratch[2 + 7 - i] = rlc & 0x0ff; 5849 rlc >>>= 8; 5850 } 5851 5852 headerScratchPos += 8; 5853 } else if(realLength > 125) { 5854 // use 16 bit length 5855 b2 |= 0x7e; 5856 5857 // FIXME: double check endinaness 5858 foreach(i; 0 .. 2) { 5859 headerScratch[2 + 1 - i] = rlc & 0x0ff; 5860 rlc >>>= 8; 5861 } 5862 5863 headerScratchPos += 2; 5864 } else { 5865 // use 7 bit length 5866 b2 |= realLength & 0b_0111_1111; 5867 } 5868 5869 headerScratch[1] = b2; 5870 } 5871 5872 //assert(!masked, "masking key not properly implemented"); 5873 if(masked) { 5874 import std.random; 5875 foreach(ref item; maskingKey) 5876 item = uniform(ubyte.min, ubyte.max); 5877 headerScratch[headerScratchPos .. headerScratchPos + 4] = maskingKey[]; 5878 headerScratchPos += 4; 5879 5880 // we'll just mask it in place... 5881 int keyIdx = 0; 5882 foreach(i; 0 .. data.length) { 5883 data[i] = data[i] ^ maskingKey[keyIdx]; 5884 if(keyIdx == 3) 5885 keyIdx = 0; 5886 else 5887 keyIdx++; 5888 } 5889 } 5890 5891 //writeln("SENDING ", headerScratch[0 .. headerScratchPos], data); 5892 llsend(headerScratch[0 .. headerScratchPos]); 5893 if(data.length) 5894 llsend(data); 5895 } 5896 5897 static WebSocketFrame read(ref ubyte[] d) { 5898 WebSocketFrame msg; 5899 5900 auto orig = d; 5901 5902 WebSocketFrame needsMoreData() { 5903 d = orig; 5904 return WebSocketFrame.init; 5905 } 5906 5907 if(d.length < 2) 5908 return needsMoreData(); 5909 5910 ubyte b = d[0]; 5911 5912 msg.populated = true; 5913 5914 msg.opcode = cast(WebSocketOpcode) (b & 0x0f); 5915 b >>= 4; 5916 msg.rsv3 = b & 0x01; 5917 b >>= 1; 5918 msg.rsv2 = b & 0x01; 5919 b >>= 1; 5920 msg.rsv1 = b & 0x01; 5921 b >>= 1; 5922 msg.fin = b & 0x01; 5923 5924 b = d[1]; 5925 msg.masked = (b & 0b1000_0000) ? true : false; 5926 msg.lengthIndicator = b & 0b0111_1111; 5927 5928 d = d[2 .. $]; 5929 5930 if(msg.lengthIndicator == 0x7e) { 5931 // 16 bit length 5932 msg.realLength = 0; 5933 5934 if(d.length < 2) return needsMoreData(); 5935 5936 foreach(i; 0 .. 2) { 5937 msg.realLength |= d[0] << ((1-i) * 8); 5938 d = d[1 .. $]; 5939 } 5940 } else if(msg.lengthIndicator == 0x7f) { 5941 // 64 bit length 5942 msg.realLength = 0; 5943 5944 if(d.length < 8) return needsMoreData(); 5945 5946 foreach(i; 0 .. 8) { 5947 msg.realLength |= ulong(d[0]) << ((7-i) * 8); 5948 d = d[1 .. $]; 5949 } 5950 } else { 5951 // 7 bit length 5952 msg.realLength = msg.lengthIndicator; 5953 } 5954 5955 if(msg.masked) { 5956 5957 if(d.length < 4) return needsMoreData(); 5958 5959 msg.maskingKey = d[0 .. 4]; 5960 d = d[4 .. $]; 5961 } 5962 5963 if(msg.realLength > d.length) { 5964 return needsMoreData(); 5965 } 5966 5967 msg.data = d[0 .. cast(size_t) msg.realLength]; 5968 d = d[cast(size_t) msg.realLength .. $]; 5969 5970 return msg; 5971 } 5972 5973 void unmaskInPlace() { 5974 if(this.masked) { 5975 int keyIdx = 0; 5976 foreach(i; 0 .. this.data.length) { 5977 this.data[i] = this.data[i] ^ this.maskingKey[keyIdx]; 5978 if(keyIdx == 3) 5979 keyIdx = 0; 5980 else 5981 keyIdx++; 5982 } 5983 } 5984 } 5985 5986 char[] textData() { 5987 return cast(char[]) data; 5988 } 5989 } 5990 } 5991 5992 private extern(C) 5993 int verifyCertificateFromRegistryArsdHttp(int preverify_ok, X509_STORE_CTX* ctx) { 5994 version(Windows) { 5995 if(preverify_ok) 5996 return 1; 5997 5998 auto err_cert = OpenSSL.X509_STORE_CTX_get_current_cert(ctx); 5999 auto err = OpenSSL.X509_STORE_CTX_get_error(ctx); 6000 6001 if(err == 62) 6002 return 0; // hostname mismatch is an error we can trust; that means OpenSSL already found the certificate and rejected it 6003 6004 auto len = OpenSSL.i2d_X509(err_cert, null); 6005 if(len == -1) 6006 return 0; 6007 ubyte[] buffer = new ubyte[](len); 6008 auto ptr = buffer.ptr; 6009 len = OpenSSL.i2d_X509(err_cert, &ptr); 6010 if(len != buffer.length) 6011 return 0; 6012 6013 6014 CERT_CHAIN_PARA thing; 6015 thing.cbSize = thing.sizeof; 6016 auto context = CertCreateCertificateContext(X509_ASN_ENCODING, buffer.ptr, cast(int) buffer.length); 6017 if(context is null) 6018 return 0; 6019 scope(exit) CertFreeCertificateContext(context); 6020 6021 PCCERT_CHAIN_CONTEXT chain; 6022 if(CertGetCertificateChain(null, context, null, null, &thing, 0, null, &chain)) { 6023 scope(exit) 6024 CertFreeCertificateChain(chain); 6025 6026 DWORD errorStatus = chain.TrustStatus.dwErrorStatus; 6027 6028 if(errorStatus == 0) 6029 return 1; // Windows approved it, OK carry on 6030 // otherwise, sustain OpenSSL's original ruling 6031 } 6032 6033 return 0; 6034 } else { 6035 return preverify_ok; 6036 } 6037 } 6038 6039 6040 version(Windows) { 6041 pragma(lib, "crypt32"); 6042 import core.sys.windows.wincrypt; 6043 extern(Windows) { 6044 PCCERT_CONTEXT CertEnumCertificatesInStore(HCERTSTORE hCertStore, PCCERT_CONTEXT pPrevCertContext); 6045 // BOOL CertGetCertificateChain(HCERTCHAINENGINE hChainEngine, PCCERT_CONTEXT pCertContext, LPFILETIME pTime, HCERTSTORE hAdditionalStore, PCERT_CHAIN_PARA pChainPara, DWORD dwFlags, LPVOID pvReserved, PCCERT_CHAIN_CONTEXT *ppChainContext); 6046 PCCERT_CONTEXT CertCreateCertificateContext(DWORD dwCertEncodingType, const BYTE *pbCertEncoded, DWORD cbCertEncoded); 6047 } 6048 6049 void loadCertificatesFromRegistry(SSL_CTX* ctx) { 6050 auto store = CertOpenSystemStore(0, "ROOT"); 6051 if(store is null) { 6052 // import std.stdio; writeln("failed"); 6053 return; 6054 } 6055 scope(exit) 6056 CertCloseStore(store, 0); 6057 6058 X509_STORE* ssl_store = OpenSSL.SSL_CTX_get_cert_store(ctx); 6059 PCCERT_CONTEXT c; 6060 while((c = CertEnumCertificatesInStore(store, c)) !is null) { 6061 FILETIME na = c.pCertInfo.NotAfter; 6062 SYSTEMTIME st; 6063 FileTimeToSystemTime(&na, &st); 6064 6065 /+ 6066 _CRYPTOAPI_BLOB i = cast() c.pCertInfo.Issuer; 6067 6068 char[256] buffer; 6069 auto p = CertNameToStrA(X509_ASN_ENCODING, &i, CERT_SIMPLE_NAME_STR, buffer.ptr, cast(int) buffer.length); 6070 import std.stdio; writeln(buffer[0 .. p]); 6071 +/ 6072 6073 if(st.wYear <= 2021) { 6074 // see: https://www.openssl.org/blog/blog/2021/09/13/LetsEncryptRootCertExpire/ 6075 continue; // no point keeping an expired root cert and it can break Let's Encrypt anyway 6076 } 6077 6078 const(ubyte)* thing = c.pbCertEncoded; 6079 auto x509 = OpenSSL.d2i_X509(null, &thing, c.cbCertEncoded); 6080 if (x509) { 6081 auto success = OpenSSL.X509_STORE_add_cert(ssl_store, x509); 6082 //if(!success) 6083 //writeln("FAILED HERE"); 6084 OpenSSL.X509_free(x509); 6085 } else { 6086 //writeln("FAILED"); 6087 } 6088 } 6089 6090 CertFreeCertificateContext(c); 6091 6092 // import core.stdc.stdio; printf("%s\n", OpenSSL.OpenSSL_version(0)); 6093 } 6094 6095 6096 // because i use the FILE* in PEM_read_X509 and friends 6097 // gotta use this to bridge the MS C runtime functions 6098 // might be able to just change those to only use the BIO versions 6099 // instead 6100 6101 // only on MS C runtime 6102 version(CRuntime_Microsoft) {} else version=no_openssl_applink; 6103 6104 version(no_openssl_applink) {} else { 6105 private extern(C) { 6106 void _open(); 6107 void _read(); 6108 void _write(); 6109 void _lseek(); 6110 void _close(); 6111 int _fileno(FILE*); 6112 int _setmode(int, int); 6113 } 6114 export extern(C) void** OPENSSL_Applink() { 6115 import core.stdc.stdio; 6116 6117 static extern(C) void* app_stdin() { return cast(void*) stdin; } 6118 static extern(C) void* app_stdout() { return cast(void*) stdout; } 6119 static extern(C) void* app_stderr() { return cast(void*) stderr; } 6120 static extern(C) int app_feof(FILE* fp) { return feof(fp); } 6121 static extern(C) int app_ferror(FILE* fp) { return ferror(fp); } 6122 static extern(C) void app_clearerr(FILE* fp) { return clearerr(fp); } 6123 static extern(C) int app_fileno(FILE* fp) { return _fileno(fp); } 6124 static extern(C) int app_fsetmod(FILE* fp, char mod) { 6125 return _setmode(_fileno(fp), mod == 'b' ? _O_BINARY : _O_TEXT); 6126 } 6127 6128 static immutable void*[] table = [ 6129 cast(void*) 22, // applink max 6130 6131 &app_stdin, 6132 &app_stdout, 6133 &app_stderr, 6134 &fprintf, 6135 &fgets, 6136 &fread, 6137 &fwrite, 6138 &app_fsetmod, 6139 &app_feof, 6140 &fclose, 6141 6142 &fopen, 6143 &fseek, 6144 &ftell, 6145 &fflush, 6146 &app_ferror, 6147 &app_clearerr, 6148 &app_fileno, 6149 6150 &_open, 6151 &_read, 6152 &_write, 6153 &_lseek, 6154 &_close, 6155 ]; 6156 static assert(table.length == 23); 6157 6158 return cast(void**) table.ptr; 6159 } 6160 } 6161 } 6162 6163 unittest { 6164 auto client = new HttpClient(); 6165 auto response = client.navigateTo(Uri("data:,Hello%2C%20World%21")).waitForCompletion(); 6166 assert(response.contentTypeMimeType == "text/plain", response.contentType); 6167 assert(response.contentText == "Hello, World!", response.contentText); 6168 6169 response = client.navigateTo(Uri("data:text/plain;base64,SGVsbG8sIFdvcmxkIQ==")).waitForCompletion(); 6170 assert(response.contentTypeMimeType == "text/plain", response.contentType); 6171 assert(response.contentText == "Hello, World!", response.contentText); 6172 6173 response = client.navigateTo(Uri("data:text/html,%3Ch1%3EHello%2C%20World%21%3C%2Fh1%3E")).waitForCompletion(); 6174 assert(response.contentTypeMimeType == "text/html", response.contentType); 6175 assert(response.contentText == "<h1>Hello, World!</h1>", response.contentText); 6176 } 6177 6178 version(arsd_http2_unittests) 6179 unittest { 6180 import core.thread; 6181 6182 static void server() { 6183 import std.socket; 6184 auto socket = new TcpSocket(); 6185 socket.setOption(SocketOptionLevel.SOCKET, SocketOption.REUSEADDR, true); 6186 socket.bind(new InternetAddress(12346)); 6187 socket.listen(1); 6188 auto s = socket.accept(); 6189 socket.close(); 6190 6191 ubyte[1024] thing; 6192 auto g = s.receive(thing[]); 6193 6194 /+ 6195 string response = "HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 9\r\n\r\nHello!!??"; 6196 auto packetSize = 2; 6197 +/ 6198 6199 auto packetSize = 1; 6200 string response = "HTTP/1.1 200 OK\r\nConnection: close\r\nTransfer-Encoding: chunked\r\n\r\n6\r\nHello!\r\n0\r\n\r\n"; 6201 6202 while(response.length) { 6203 s.send(response[0 .. packetSize]); 6204 response = response[packetSize .. $]; 6205 //import std.stdio; writeln(response); 6206 } 6207 6208 s.close(); 6209 } 6210 6211 auto thread = new Thread(&server); 6212 thread.start; 6213 6214 Thread.sleep(200.msecs); 6215 6216 auto response = get("http://localhost:12346/").waitForCompletion; 6217 assert(response.code == 200); 6218 //import std.stdio; writeln(response); 6219 6220 foreach(site; ["https://dlang.org/", "http://arsdnet.net", "https://phobos.dpldocs.info"]) { 6221 response = get(site).waitForCompletion; 6222 assert(response.code == 200); 6223 } 6224 6225 thread.join; 6226 } 6227 6228 /+ 6229 so the url params are arguments. it knows the request 6230 internally. other params are properties on the req 6231 6232 names may have different paths... those will just add ForSomething i think. 6233 6234 auto req = api.listMergeRequests 6235 req.page = 10; 6236 6237 or 6238 req.page(1) 6239 .bar("foo") 6240 6241 req.execute(); 6242 6243 6244 everything in the response is nullable access through the 6245 dynamic object, just with property getters there. need to make 6246 it static generated tho 6247 6248 other messages may be: isPresent and getDynamic 6249 6250 6251 AND/OR what about doing it like the rails objects 6252 6253 BroadcastMessage.get(4) 6254 // various properties 6255 6256 // it lists what you updated 6257 6258 BroadcastMessage.foo().bar().put(5) 6259 +/