The OpenD Programming Language

1 /// My old curl wrapper. Use [arsd.http2] instead on newer projects, or [std.net.curl] in Phobos if you don't trust my homemade implementations :)
2 module arsd.curl;
3 
4 // see this for info on making a curl.lib on windows:
5 // http://stackoverflow.com/questions/7933845/where-is-curl-lib-for-dmd
6 
7 pragma(lib, "curl");
8 
9 import std.string;
10 extern(C) {
11 	struct CURL;
12 	struct curl_slist;
13 
14 	alias int CURLcode;
15 	alias int CURLoption;
16 
17 	enum int CURLOPT_URL = 10002;
18 	enum int CURLOPT_WRITEFUNCTION = 20011;
19 	enum int CURLOPT_WRITEDATA = 10001;
20 	enum int CURLOPT_POSTFIELDS = 10015;
21 	enum int CURLOPT_POSTFIELDSIZE = 60;
22 	enum int CURLOPT_POST = 47;
23 	enum int CURLOPT_HTTPHEADER = 10023;
24 	enum int CURLOPT_USERPWD = 0x00002715;
25 
26 	enum int CURLOPT_VERBOSE = 41;
27 
28 //	enum int CURLOPT_COOKIE = 22;
29 	enum int CURLOPT_COOKIEFILE = 10031;
30 	enum int CURLOPT_COOKIEJAR = 10082;
31 
32 	enum int CURLOPT_SSL_VERIFYPEER = 64;
33 
34 	enum int CURLOPT_FOLLOWLOCATION = 52;
35 
36 	CURL* curl_easy_init();
37 	void curl_easy_cleanup(CURL* handle);
38 	CURLcode curl_easy_perform(CURL* curl);
39 
40 	void curl_global_init(int flags);
41 
42 	enum int CURL_GLOBAL_ALL = 0b1111;
43 
44 	CURLcode curl_easy_setopt(CURL* handle, CURLoption option, ...);
45 	curl_slist* curl_slist_append(curl_slist*, const char*);
46 	void curl_slist_free_all(curl_slist*);
47 
48 	// size is size of item, count is how many items
49 	size_t write_data(void* buffer, size_t size, size_t count, void* user) {
50 		string* str = cast(string*) user;
51 		char* data = cast(char*) buffer;
52 
53 		assert(size == 1);
54 
55 		*str ~= data[0..count];
56 
57 		return count;
58 	}
59 
60 	char* curl_easy_strerror(CURLcode  errornum );
61 }
62 /*
63 struct CurlOptions {
64 	string username;
65 	string password;
66 }
67 */
68 
69 string getDigestString(string s) {
70 	import std.digest.md;
71 	import std.digest;
72 	auto hash = md5Of(s);
73 	auto a = toHexString(hash);
74 	return a.idup;
75 }
76 //import std.md5;
77 import std.file;
78 /// this automatically caches to a local file for the given time. it ignores the expires header in favor of your time to keep.
79 version(linux)
80 string cachedCurl(string url, int maxCacheHours) {
81 	string res;
82 
83 	auto cacheFile = "/tmp/arsd-curl-cache-" ~ getDigestString(url);
84 
85 	import std.datetime;
86 
87 	if(!std.file.exists(cacheFile) || std.file.timeLastModified(cacheFile) < Clock.currTime() - dur!"hours"(maxCacheHours)) {
88 		res = curl(url);
89 		std.file.write(cacheFile, res);
90 	} else {
91 		res = cast(string) std.file.read(cacheFile);
92 	}
93 
94 	return res;
95 }
96 
97 
98 string curl(string url, string data = null, string contentType = "application/x-www-form-urlencoded") {
99 	return curlAuth(url, data, null, null, contentType);
100 }
101 
102 string curlCookie(string cookieFile, string url, string data = null, string contentType = "application/x-www-form-urlencoded") {
103 	return curlAuth(url, data, null, null, contentType, null, null, cookieFile);
104 }
105 
106 string curlAuth(string url, string data = null, string username = null, string password = null, string contentType = "application/x-www-form-urlencoded", string methodOverride = null, string[] customHeaders = null, string cookieJar = null) {
107 	CURL* curl = curl_easy_init();
108 	if(curl is null)
109 		throw new Exception("curl init");
110 	scope(exit)
111 		curl_easy_cleanup(curl);
112 
113 	string ret;
114 
115 	int res;
116 
117 	debug(arsd_curl_verbose)
118 		curl_easy_setopt(curl, CURLOPT_VERBOSE, 1);
119 
120 	res = curl_easy_setopt(curl, CURLOPT_URL, std..string.toStringz(url));
121 	if(res != 0) throw new CurlException(res);
122 	if(username !is null) {
123 		res = curl_easy_setopt(curl, CURLOPT_USERPWD, std..string.toStringz(username ~ ":" ~ password));
124 		if(res != 0) throw new CurlException(res);
125 	}
126 	res = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, &write_data);
127 	if(res != 0) throw new CurlException(res);
128 	res = curl_easy_setopt(curl, CURLOPT_WRITEDATA, &ret);
129 	if(res != 0) throw new CurlException(res);
130 
131 	curl_slist* headers = null;
132 	//if(data !is null)
133 	//	contentType = "";
134 	if(contentType.length)
135 	headers = curl_slist_append(headers, toStringz("Content-Type: " ~ contentType));
136 
137 	foreach(h; customHeaders) {
138 		headers = curl_slist_append(headers, toStringz(h));
139 	}
140 	scope(exit)
141 		curl_slist_free_all(headers);
142 
143 	if(data) {
144 		res = curl_easy_setopt(curl, CURLOPT_POSTFIELDS, data.ptr);
145 		if(res != 0) throw new CurlException(res);
146 		res = curl_easy_setopt(curl, CURLOPT_POSTFIELDSIZE, data.length);
147 		if(res != 0) throw new CurlException(res);
148 	}
149 
150 	res = curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
151 	if(res != 0) throw new CurlException(res);
152 
153 	if(cookieJar !is null) {
154 		res = curl_easy_setopt(curl, CURLOPT_COOKIEJAR, toStringz(cookieJar));
155 		if(res != 0) throw new CurlException(res);
156 		res = curl_easy_setopt(curl, CURLOPT_COOKIEFILE, toStringz(cookieJar));
157 		if(res != 0) throw new CurlException(res);
158 	} else {
159 		// just want to enable cookie parsing for location 3xx thingies.
160 		// some crappy sites will give you an endless runaround if they can't
161 		// place their fucking tracking cookies.
162 		res = curl_easy_setopt(curl, CURLOPT_COOKIEFILE, toStringz("lol totally not here"));
163 	}
164 
165 	res = curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0);
166 	if(res != 0) throw new CurlException(res);
167 	//res = curl_easy_setopt(curl, 81, 0); // FIXME verify host
168 	//if(res != 0) throw new CurlException(res);
169 
170 	version(no_curl_follow) {} else {
171 		res = curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
172 		if(res != 0) throw new CurlException(res);
173 	}
174 
175 	if(methodOverride !is null) {
176 		switch(methodOverride) {
177 			default: assert(0);
178 			case "POST":
179 				res = curl_easy_setopt(curl, CURLOPT_POST, 1);
180 			break;
181 			case "GET":
182 				//curl_easy_setopt(curl, CURLOPT_POST, 0);
183 			break;
184 		}
185 	}
186 
187 	auto failure = curl_easy_perform(curl);
188 	if(failure != 0)
189 		throw new CurlException(failure, "\nURL" ~ url);
190 
191 	return ret;
192 }
193 
194 class CurlException : Exception {
195 	this(CURLcode code, string msg = null, string file = __FILE__, int line = __LINE__) @system {
196 		string message = file ~ ":" ~ to!string(line) ~ " (" ~ to!string(code) ~ ") ";
197 
198 		auto strerror = curl_easy_strerror(code);
199 
200 		while(*strerror) {
201 			message ~= *strerror;
202 			strerror++;
203 		}
204 
205 		super(message ~ msg);
206 	}
207 }
208 
209 
210 import std.conv;