The OpenD Programming Language

1 /++
2 	OBSOLETE: Old version of my http implementation. Do not use this, instead use [arsd.http2].
3 
4 	I no longer work on this, use http2.d instead.
5 +/
6 /*deprecated*/ module arsd.http; // adrdox apparently loses the comment above with deprecated, i need to fix that over there.
7 
8 import std.socket;
9 
10 // FIXME: check Transfer-Encoding: gzip always
11 
12 version(with_openssl) {
13 	pragma(lib, "crypto");
14 	pragma(lib, "ssl");
15 }
16 
17 ubyte[] getBinary(string url, string[string] cookies = null) {
18 	auto hr = httpRequest("GET", url, null, cookies);
19 	if(hr.code != 200)
20 		throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url));
21 	return hr.content;
22 }
23 
24 /**
25 	Gets a textual document, ignoring headers. Throws on non-text or error.
26 */
27 string get(string url, string[string] cookies = null) {
28 	auto hr = httpRequest("GET", url, null, cookies);
29 	if(hr.code != 200)
30 		throw new Exception(format("HTTP answered %d instead of 200 on %s", hr.code, url));
31 	if(hr.contentType.indexOf("text/") == -1)
32 		throw new Exception(hr.contentType ~ " is bad content for conversion to string");
33 	return cast(string) hr.content;
34 
35 }
36 
37 static import std.uri;
38 
39 string post(string url, string[string] args, string[string] cookies = null) {
40 	string content;
41 
42 	foreach(name, arg; args) {
43 		if(content.length)
44 			content ~= "&";
45 		content ~= std.uri.encode(name) ~ "=" ~ std.uri.encode(arg);
46 	}
47 
48 	auto hr = httpRequest("POST", url, cast(ubyte[]) content, cookies, ["Content-Type: application/x-www-form-urlencoded"]);
49 	if(hr.code != 200)
50 		throw new Exception(format("HTTP answered %d instead of 200", hr.code));
51 	if(hr.contentType.indexOf("text/") == -1)
52 		throw new Exception(hr.contentType ~ " is bad content for conversion to string");
53 
54 	return cast(string) hr.content;
55 }
56 
57 struct HttpResponse {
58 	int code;
59 	string contentType;
60 	string[string] cookies;
61 	string[] headers;
62 	ubyte[] content;
63 }
64 
65 import std.string;
66 static import std.algorithm;
67 import std.conv;
68 
69 struct UriParts {
70 	string original;
71 	string method;
72 	string host;
73 	ushort port;
74 	string path;
75 
76 	bool useHttps;
77 
78 	this(string uri) {
79 		original = uri;
80 
81 		if(uri[0 .. 8] == "https://")
82 			useHttps = true;
83 		else
84 		if(uri[0..7] != "http://")
85 			throw new Exception("You must use an absolute, http or https URL.");
86 
87 		version(with_openssl) {} else
88 		if(useHttps)
89 			throw new Exception("openssl support not compiled in try -version=with_openssl");
90 
91 		int start = useHttps ? 8 : 7;
92 
93 		auto posSlash = uri[start..$].indexOf("/");
94 		if(posSlash != -1)
95 			posSlash += start;
96 
97 		if(posSlash == -1)
98 			posSlash = uri.length;
99 
100 		auto posColon = uri[start..$].indexOf(":");
101 		if(posColon != -1)
102 			posColon += start;
103 
104 		if(useHttps)
105 			port = 443;
106 		else
107 			port = 80;
108 
109 		if(posColon != -1 && posColon < posSlash) {
110 			host = uri[start..posColon];
111 			port = to!ushort(uri[posColon+1..posSlash]);
112 		} else
113 			host = uri[start..posSlash];
114 
115 		path = uri[posSlash..$];
116 		if(path == "")
117 			path = "/";
118 	}
119 }
120 
121 HttpResponse httpRequest(string method, string uri, const(ubyte)[] content = null, string[string] cookies = null, string[] headers = null) {
122 	import std.socket;
123 
124 	auto u = UriParts(uri);
125 	// auto f = openNetwork(u.host, u.port);
126 	auto f = new TcpSocket();
127 	f.connect(new InternetAddress(u.host, u.port));
128 
129 	void delegate(string) write = (string d) {
130 		f.send(d);
131 	};
132 
133 	char[4096] readBuffer; // rawRead actually blocks until it can fill up the whole buffer... which is broken as far as http goes so one char at a time i guess. slow lol
134 	char[] delegate() read = () {
135 		size_t num = f.receive(readBuffer);
136 		return readBuffer[0..num];
137 	};
138 
139 	version(with_openssl) {
140 		import deimos.openssl.ssl;
141 		SSL* ssl;
142 		SSL_CTX* ctx;
143 		if(u.useHttps) {
144 			void sslAssert(bool ret){
145 				if (!ret){
146 					throw new Exception("SSL_ERROR");
147 				}
148 			}
149 			SSL_library_init();
150 			OpenSSL_add_all_algorithms();
151 			SSL_load_error_strings();
152 
153 			ctx = SSL_CTX_new(SSLv3_client_method());
154 			sslAssert(!(ctx is null));
155 
156 			ssl = SSL_new(ctx);
157 			SSL_set_fd(ssl, f.handle);
158 			sslAssert(SSL_connect(ssl) != -1);
159 
160 			write = (string d) {
161 				SSL_write(ssl, d.ptr, cast(uint)d.length);
162 			};
163 
164 			read = () {
165 				auto len = SSL_read(ssl, readBuffer.ptr, readBuffer.length);
166 				return readBuffer[0 .. len];
167 			};
168 		}
169 	}
170 
171 
172 	HttpResponse response = doHttpRequestOnHelpers(write, read, method, uri, content, cookies, headers, u.useHttps);
173 
174 	version(with_openssl) {
175 		if(u.useHttps) {
176 			SSL_free(ssl);
177 			SSL_CTX_free(ctx);
178 		}
179 	}
180 
181 	return response;
182 }
183 
184 /**
185 	Executes a generic http request, returning the full result. The correct formatting
186 	of the parameters are the caller's responsibility. Content-Length is added automatically,
187 	but YOU must give Content-Type!
188 */
189 HttpResponse doHttpRequestOnHelpers(void delegate(string) write, char[] delegate() read, string method, string uri, const(ubyte)[] content = null, string[string] cookies = null, string[] headers = null, bool https = false)
190 	in {
191 		assert(method == "POST" || method == "GET");
192 	}
193 do {
194 	auto u = UriParts(uri);
195 
196 
197 
198 
199 
200 	write(format("%s %s HTTP/1.1\r\n", method, u.path));
201 	write(format("Host: %s\r\n", u.host));
202 	write(format("Connection: close\r\n"));
203 	if(content !is null)
204 		write(format("Content-Length: %d\r\n", content.length));
205 
206 	if(cookies !is null) {
207 		string cookieHeader = "Cookie: ";
208 		bool first = true;
209 		foreach(k, v; cookies) {
210 			if(first)
211 				first = false;
212 			else
213 				cookieHeader ~= "; ";
214 			cookieHeader ~= std.uri.encodeComponent(k) ~ "=" ~ std.uri.encodeComponent(v);
215 		}
216 
217 		write(format("%s\r\n", cookieHeader));
218 	}
219 
220 	if(headers !is null)
221 		foreach(header; headers)
222 			write(format("%s\r\n", header));
223 	write("\r\n");
224 	if(content !is null)
225 		write(cast(string) content);
226 
227 
228 	string buffer;
229 
230 	string readln() {
231 		auto idx = buffer.indexOf("\r\n");
232 		if(idx == -1) {
233 			auto more = read();
234 			if(more.length == 0) { // end of file or something
235 				auto ret = buffer;
236 				buffer = null;
237 				return ret;
238 			}
239 			buffer ~= more;
240 			return readln();
241 		}
242 		auto ret = buffer[0 .. idx + 2]; // + the \r\n
243 		if(idx + 2 < buffer.length)
244 			buffer = buffer[idx + 2 .. $];
245 		else
246 			buffer = null;
247 		return ret;
248 	}
249 
250 	HttpResponse hr;
251  cont:
252 	string l = readln();
253 	if(l[0..9] != "HTTP/1.1 ")
254 		throw new Exception("Not talking to a http server");
255 
256 	hr.code = to!int(l[9..12]); // HTTP/1.1 ### OK
257 
258 	if(hr.code == 100) { // continue
259 		do {
260 			l = readln();
261 		} while(l.length > 1);
262 
263 		goto cont;
264 	}
265 
266 	bool chunked = false;
267 
268 	auto line = readln();
269 	while(line.length) {
270 		if(line.strip.length == 0)
271 			break;
272 		hr.headers ~= line;
273 		if(line.startsWith("Content-Type: "))
274 			hr.contentType = line[14..$-1];
275 		if(line.startsWith("Set-Cookie: ")) {
276 			auto hdr = line["Set-Cookie: ".length .. $-1];
277 			auto semi = hdr.indexOf(";");
278 			if(semi != -1)
279 				hdr = hdr[0 .. semi];
280 
281 			auto equal = hdr.indexOf("=");
282 			string name, value;
283 			if(equal == -1) {
284 				name = hdr;
285 				// doesn't this mean erase the cookie?
286 			} else {
287 				name = hdr[0 .. equal];
288 				value = hdr[equal + 1 .. $];
289 			}
290 
291 			name = std.uri.decodeComponent(name);
292 			value = std.uri.decodeComponent(value);
293 
294 			hr.cookies[name] = value;
295 		}
296 		if(line.startsWith("Transfer-Encoding: chunked"))
297 			chunked = true;
298 		line = readln();
299 	}
300 
301 	// there might be leftover stuff in the line buffer
302 	ubyte[] response = cast(ubyte[]) buffer.dup;
303 	auto part = read();
304 	while(part.length) {
305 		response ~= part;
306 		part = read();
307 	}
308 
309 	if(chunked) {
310 		// read the hex length, stopping at a \r\n, ignoring everything between the new line but after the first non-valid hex character
311 		// read binary data of that length. it is our content
312 		// repeat until a zero sized chunk
313 		// then read footers as headers.
314 
315 		int state = 0;
316 		int size;
317 		int start = 0;
318 		for(int a = 0; a < response.length; a++) {
319 			final switch(state) {
320 				case 0: // reading hex
321 					char c = response[a];
322 					if((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) {
323 						// just keep reading
324 					} else {
325 						int power = 1;
326 						size = 0;
327 						for(int b = a-1; b >= start; b--) {
328 							char cc = response[b];
329 							if(cc >= 'a' && cc <= 'z')
330 								cc -= 0x20;
331 							int val = 0;
332 							if(cc >= '0' && cc <= '9')
333 								val = cc - '0';
334 							else
335 								val = cc - 'A' + 10;
336 
337 							size += power * val;
338 							power *= 16;
339 						}
340 						state++;
341 						continue;
342 					}
343 				break;
344 				case 1: // reading until end of line
345 					char c = response[a];
346 					if(c == '\n') {
347 						if(size == 0)
348 							state = 3;
349 						else
350 							state = 2;
351 					}
352 				break;
353 				case 2: // reading data
354 					hr.content ~= response[a..a+size];
355 					a += size;
356 					a+= 1; // skipping a 13 10
357 					start = a + 1;
358 					state = 0;
359 				break;
360 				case 3: // reading footers
361 					goto done; // FIXME
362 			}
363 		}
364 	} else
365 		hr.content = response;
366 	done:
367 
368 	return hr;
369 }
370 
371 
372 /*
373 void main(string args[]) {
374 	write(post("http://arsdnet.net/bugs.php", ["test" : "hey", "again" : "what"]));
375 }
376 */
377 
378 version(none):
379 
380 struct Url {
381 	string url;
382 }
383 
384 struct BasicAuth {
385 	string username;
386 	string password;
387 }
388 
389 /*
390 	When you send something, it creates a request
391 	and sends it asynchronously. The request object
392 
393 	auto request = new HttpRequest();
394 	// set any properties here
395 
396 	// synchronous usage
397 	auto reply = request.perform();
398 
399 	// async usage, type 1:
400 	request.send();
401 	request2.send();
402 
403 	// wait until the first one is done, with the second one still in-flight
404 	auto response = request.waitForCompletion();
405 
406 
407 	// async usage, type 2:
408 	request.onDataReceived = (HttpRequest hr) {
409 		if(hr.state == HttpRequest.State.complete) {
410 			// use hr.responseData
411 		}
412 	};
413 	request.send(); // send, using the callback
414 
415 	// before terminating, be sure you wait for your requests to finish!
416 
417 	request.waitForCompletion();
418 
419 */
420 
421 class HttpRequest {
422 	private static {
423 		// we manage the actual connections. When a request is made on a particular
424 		// host, we try to reuse connections. We may open more than one connection per
425 		// host to do parallel requests.
426 		//
427 		// The key is the *domain name*. Multiple domains on the same address will have separate connections.
428 		Socket[][string] socketsPerHost;
429 
430 		// only one request can be active on a given socket (at least HTTP < 2.0) so this is that
431 		HttpRequest[Socket] activeRequestOnSocket;
432 		HttpRequest[] pending; // and these are the requests that are waiting
433 
434 		SocketSet readSet;
435 
436 
437 		void advanceConnections() {
438 			if(readSet is null)
439 				readSet = new SocketSet();
440 
441 			// are there pending requests? let's try to send them
442 
443 			readSet.reset();
444 
445 			// active requests need to be read or written to
446 			foreach(sock, request; activeRequestOnSocket)
447 				readSet.add(sock);
448 
449 			// check the other sockets just for EOF, if they close, take them out of our list,
450 			// we'll reopen if needed upon request.
451 
452 			auto got = Socket.select(readSet, writeSet, null, 10.seconds /* timeout */);
453 			if(got == 0) /* timeout */
454 				{}
455 			else
456 			if(got == -1) /* interrupted */
457 				{}
458 			else /* ready */
459 				{}
460 
461 			// call select(), do what needs to be done
462 			// no requests are active, send the ones pending connection now
463 			// we've completed a request, are there any more pending connection? if so, send them now
464 
465 			auto readSet = new SocketSet();
466 		}
467 	}
468 
469 	this() {
470 		addConnection(this);
471 	}
472 
473 	~this() {
474 		removeConnection(this);
475 	}
476 
477 	HttpResponse responseData;
478 	HttpRequestParameters parameters;
479 	private HttpClient parentClient;
480 
481 	size_t bodyBytesSent;
482 	size_t bodyBytesReceived;
483 
484 	State state;
485 	/// Called when data is received. Check the state to see what data is available.
486 	void delegate(AsynchronousHttpRequest) onDataReceived;
487 
488 	enum State {
489 		/// The request has not yet been sent
490 		unsent,
491 
492 		/// The send() method has been called, but no data is
493 		/// sent on the socket yet because the connection is busy.
494 		pendingAvailableConnection,
495 
496 		/// The headers are being sent now
497 		sendingHeaders,
498 
499 		/// The body is being sent now
500 		sendingBody,
501 
502 		/// The request has been sent but we haven't received any response yet
503 		waitingForResponse,
504 
505 		/// We have received some data and are currently receiving headers
506 		readingHeaders,
507 
508 		/// All headers are available but we're still waiting on the body
509 		readingBody,
510 
511 		/// The request is complete.
512 		complete,
513 
514 		/// The request is aborted, either by the abort() method, or as a result of the server disconnecting
515 		aborted
516 	}
517 
518 	/// Sends now and waits for the request to finish, returning the response.
519 	HttpResponse perform() {
520 		send();
521 		return waitForCompletion();
522 	}
523 
524 	/// Sends the request asynchronously.
525 	void send() {
526 		if(state != State.unsent && state != State.aborted)
527 			return; // already sent
528 
529 		responseData = HttpResponse.init;
530 		bodyBytesSent = 0;
531 		bodyBytesReceived = 0;
532 		state = State.pendingAvailableConnection;
533 
534 		HttpResponse.advanceConnections();
535 	}
536 
537 
538 	/// Waits for the request to finish or timeout, whichever comes furst.
539 	HttpResponse waitForCompletion() {
540 		while(state != State.aborted && state != State.complete)
541 			HttpResponse.advanceConnections();
542 		return responseData;
543 	}
544 
545 	/// Aborts this request.
546 	/// Due to the nature of the HTTP protocol, aborting one request will result in all subsequent requests made on this same connection to be aborted as well.
547 	void abort() {
548 		parentClient.close();
549 	}
550 }
551 
552 struct HttpRequestParameters {
553 	Duration timeout;
554 
555 	// debugging
556 	bool useHttp11 = true;
557 	bool acceptGzip = true;
558 
559 	// the request itself
560 	HttpVerb method;
561 	string host;
562 	string uri;
563 
564 	string userAgent;
565 
566 	string[string] cookies;
567 
568 	string[] headers; /// do not duplicate host, content-length, content-type, or any others that have a specific property
569 
570 	string contentType;
571 	ubyte[] bodyData;
572 }
573 
574 interface IHttpClient {
575 
576 }
577 
578 enum HttpVerb { GET, HEAD, POST, PUT, DELETE, OPTIONS, TRACE, CONNECT }
579 
580 /*
581 	Usage:
582 
583 	auto client = new HttpClient("localhost", 80);
584 	// relative links work based on the current url
585 	client.get("foo/bar");
586 	client.get("baz"); // gets foo/baz
587 
588 	auto request = client.get("rofl");
589 	auto response = request.waitForCompletion();
590 */
591 
592 /// HttpClient keeps cookies, location, and some other state to reuse connections, when possible, like a web browser.
593 class HttpClient {
594 	/* Protocol restrictions, useful to disable when debugging servers */
595 	bool useHttp11 = true;
596 	bool useGzip = true;
597 
598 	/// Automatically follow a redirection?
599 	bool followLocation = false;
600 
601 	@property Url location() {
602 		return currentUrl;
603 	}
604 
605 	/// High level function that works similarly to entering a url
606 	/// into a browser.
607 	///
608 	/// Follows locations, updates the current url.
609 	AsynchronousHttpRequest navigateTo(Url where) {
610 		currentUrl = where.basedOn(currentUrl);
611 		assert(0);
612 	}
613 
614 	private Url currentUrl;
615 
616 	this() {
617 
618 	}
619 
620 	this(Url url) {
621 		open(url);
622 	}
623 
624 	this(string host, ushort port = 80, bool useSsl = false) {
625 		open(host, port);
626 	}
627 
628 	// FIXME: add proxy
629 	// FIXME: some kind of caching
630 
631 	void open(Url url) {
632 
633 	}
634 
635 	void open(string host, ushort port = 80, bool useSsl = false) {
636 
637 	}
638 
639 	void close() {
640 		socket.close();
641 	}
642 
643 	void setCookie(string name, string value) {
644 
645 	}
646 
647 	void clearCookies() {
648 
649 	}
650 
651 	HttpResponse sendSynchronously() {
652 		auto request = sendAsynchronously();
653 		return request.waitForCompletion();
654 	}
655 
656 	AsynchronousHttpRequest sendAsynchronously() {
657 
658 	}
659 
660 	string method;
661 	string host;
662 	ushort port;
663 	string uri;
664 
665 	string[] headers;
666 	ubyte[] requestBody;
667 
668 	string userAgent;
669 
670 	/* inter-request state */
671 	string[string] cookies;
672 }
673 
674 // FIXME: websocket