The OpenD Programming Language

1 module requests.base;
2 
3 import requests.streams;
4 import requests.utils;
5 import requests.uri;
6 import requests.connmanager;
7 
8 import std.format;
9 import std.datetime;
10 import core.time;
11 import std.stdio;
12 import std.algorithm;
13 import std.string;
14 import std.exception;
15 import std.bitmanip;
16 import std.conv;
17 import std.typecons;
18 
19 /++ 
20     Interface to provide user info and http headers requred for server auth.
21 +/
22 public interface Auth {
23     string[string] authHeaders(string domain); /// Create headers for authentication
24     string         userName();                 /// Returns user name
25     string         password();                 /// Returns user password
26 }
27 /**
28  * Basic authentication.
29  * Adds $(B Authorization: Basic) header to request
30  * Example:
31  * ---
32  * import requests;
33  * void main() {
34  * rq = Request();
35  * rq.authenticator = new BasicAuthentication("user", "passwd");
36  * rs = rq.get("http://httpbin.org/basic-auth/user/passwd");
37  * }
38  * ---
39 */
40 public class BasicAuthentication: Auth {
41     private {
42         string   _username, _password;
43         string[] _domains;
44     }
45     /// Constructor.
46     /// Params:
47     /// username = username
48     /// password = password
49     /// domains = not used now
50     ///
51     this(string username, string password, string[] domains = []) {
52         _username = username;
53         _password = password;
54         _domains = domains;
55     }
56     /// create Basic Auth header
57     override string[string] authHeaders(string domain) {
58         import std.base64;
59         string[string] auth;
60         auth["Authorization"] = "Basic " ~ to!string(Base64.encode(cast(ubyte[])"%s:%s".format(_username, _password)));
61         return auth;
62     }
63     /// returns username
64     override string userName() {
65         return _username;
66     }
67     /// return user password
68     override string password() {
69         return _password;
70     }
71 }
72 
73 /**
74  * Struct to send multiple files in POST request.
75  */
76 public struct PostFile {
77     /// Path to the file to send.
78     string fileName;
79     /// Name of the field (if empty - send file base name)
80     string fieldName;
81     /// contentType of the file if not empty
82     string contentType;
83 }
84 ///
85 /// This is File-like interface for sending data to multipart forms
86 ///
87 public interface FiniteReadable {
88     /// size of the content
89     abstract ulong  getSize();
90     /// file-like read()
91     abstract ubyte[] read();
92 }
93 ///
94 /// Helper to create form elements from File.
95 /// Params:
96 /// name = name of the field in form
97 /// f = opened std.stio.File to send to server
98 /// parameters = optional parameters (most important are "filename" and "Content-Type")
99 ///
100 public auto formData(string name, File f, string[string] parameters = null) {
101     return MultipartForm.FormData(name, new FormDataFile(f), parameters);
102 }
103 ///
104 /// Helper to create form elements from ubyte[].
105 /// Params:
106 /// name = name of the field in form
107 /// b = data to send to server
108 /// parameters = optional parameters (can be "filename" and "Content-Type")
109 ///
110 public auto formData(string name, ubyte[] b, string[string] parameters = null) {
111     return MultipartForm.FormData(name, new FormDataBytes(b), parameters);
112 }
113 public auto formData(string name, string b, string[string] parameters = null) {
114     return MultipartForm.FormData(name, new FormDataBytes(b.dup.representation), parameters);
115 }
116 
117 private immutable uint defaultBufferSize = 12*1024;
118 /// Class to provide FiniteReadable from user-provided ubyte[]
119 public class FormDataBytes : FiniteReadable {
120     private {
121         ulong   _size;
122         ubyte[] _data;
123         size_t  _offset;
124         bool    _exhausted;
125     }
126     /// constructor from ubyte[]
127     this(ubyte[] data) {
128         _data = data;
129         _size = data.length;
130     }
131     final override ulong getSize() {
132         return _size;
133     }
134     final override ubyte[] read() {
135         enforce( !_exhausted, "You can't read froum exhausted source" );
136         size_t toRead = min(defaultBufferSize, _size - _offset);
137         auto result = _data[_offset.._offset+toRead];
138         _offset += toRead;
139         if ( toRead == 0 ) {
140             _exhausted = true;
141         }
142         return result;
143     }
144 }
145 /// Class to provide FiniteReadable from File
146 public class FormDataFile : FiniteReadable {
147     import  std.file;
148     private {
149         File    _fileHandle;
150         ulong   _fileSize;
151         size_t  _processed;
152         bool    _exhausted;
153     }
154     /// constructor from File object
155     this(File file) {
156         import std.file;
157         _fileHandle = file;
158         _fileSize = std.file.getSize(file.name);
159     }
160     final override ulong getSize() pure nothrow @safe {
161         return _fileSize;
162     }
163     final override ubyte[] read() {
164         enforce( !_exhausted, "You can't read froum exhausted source" );
165         auto b = new ubyte[defaultBufferSize];
166         auto r = _fileHandle.rawRead(b);
167         auto toRead = min(r.length, _fileSize - _processed);
168         if ( toRead == 0 ) {
169             _exhausted = true;
170         }
171         _processed += toRead;
172         return r[0..toRead];
173     }
174 }
175 ///
176 /// This struct used to bulld POST's to forms.
177 /// Each part have name and data. data is something that can be read-ed and have size.
178 /// For example this can be string-like object (wrapped for reading) or opened File.
179 ///
180 public struct MultipartForm {
181     package struct FormData {
182         FiniteReadable  input;
183         string          name;
184         string[string]  parameters;
185         this(string name, FiniteReadable i, string[string] parameters = null) {
186             this.input = i;
187             this.name = name;
188             this.parameters = parameters;
189         }
190     }
191 
192     package FormData[] _sources;
193     auto add(FormData d) {
194         _sources ~= d;
195         return this;
196     }
197     auto add(string name, FiniteReadable i, string[string]parameters = null) {
198         _sources ~= FormData(name, i, parameters);
199         return this;
200     }
201     bool empty() const
202     {
203         return _sources.length == 0;
204     }
205 }
206 ///
207 
208 /// General type exception from Request
209 public class RequestException: Exception {
210     this(string msg, string file = __FILE__, size_t line = __LINE__, Throwable next = null) @safe pure nothrow {
211         super(msg, file, line, next);
212     }
213 }
214 
215 /**
216     ReceiveAsRange is InputRange used to supply client with data from response.
217     Each popFront fetch next data portion.
218 */
219 public struct ReceiveAsRange {
220     bool empty() {
221         return data.length == 0;
222     }
223     ubyte[] front() {
224         return data;
225     }
226     void popFront() {
227         if ( read ) {
228             // get new portion
229             data = read();
230         } else {
231             // we can't read any new data
232             data.length = 0;
233         }
234     }
235     package {
236         bool                    activated;
237         ubyte[]                 data;
238         /// HTTP or FTP module set up delegate read() for reading next portion of data.
239         ubyte[]                 delegate() read;
240         RefCounted!ConnManager  cm;
241     }
242 }
243 
244 /**
245     Response
246 */
247 public class Response {
248     package {
249         /// Server status code
250         ushort           _code;
251         /// Response body
252         Buffer!ubyte     _responseBody;
253         /// Response headers
254         string[string]   _responseHeaders;
255         /// Initial URI
256         URI              _uri;
257         /// Final URI. Can differ from uri() if request go through redirections.
258         URI              _finalURI;
259         /// stream range stored here
260         ReceiveAsRange   _receiveAsRange;
261         SysTime          _startedAt,
262                          _connectedAt,
263                          _requestSentAt,
264                          _finishedAt;
265         /// Length of received content
266         long             _contentReceived;
267         /// Server-supplied content length (can be -1 when unknown)
268         long             _contentLength = -1;
269         mixin(Setter!ushort("code"));
270         mixin(Setter!URI("uri"));
271         mixin(Setter!URI("finalURI"));
272     }
273     mixin(Getter("code"));
274     mixin(Getter("contentReceived"));
275     mixin(Getter("contentLength"));
276     mixin(Getter("uri"));
277     mixin(Getter("finalURI"));
278 
279     @property auto getStats() const pure @safe {
280         import std.typecons: Tuple;
281         alias statTuple = Tuple!(Duration, "connectTime",
282             Duration, "sendTime",
283             Duration, "recvTime");
284         statTuple stat;
285         stat.connectTime = _connectedAt - _startedAt;
286         stat.sendTime = _requestSentAt - _connectedAt;
287         stat.recvTime = _finishedAt - _requestSentAt;
288         return stat;
289     }
290     @property auto ref responseBody() @safe nothrow {
291         return _responseBody;
292     }
293     @property auto ref responseHeaders() pure @safe nothrow {
294         return _responseHeaders;
295     }
296     @property auto ref receiveAsRange() pure @safe nothrow {
297         return _receiveAsRange;
298     }
299     /// string representation of response
300     override string toString() const {
301         return "Response(%d, %s)".format(_code, _finalURI.uri());
302     }
303     /// format response to string (hpPqsBTUS).
304     /**
305 
306         %h - remote hostname
307 
308         %p - remote port
309 
310         %P - remote path
311 
312         %q - query parameters
313 
314         %s - string representation
315 
316         %B - received bytes
317 
318         %T - resquest total time
319 
320         %U - request uri()
321 
322         %S - status code
323     */
324     string format(string fmt) const {
325         import std.array;
326         auto a = appender!string();
327         auto f = FormatSpec!char(fmt);
328         while (f.writeUpToNextSpec(a)) {
329             switch (f.spec) {
330                 case 'h':
331                     // Remote hostname.
332                     a.put(_uri.host);
333                     break;
334                 case 'p':
335                     // Remote port.
336                     a.put("%d".format(_uri.port));
337                     break;
338                 case 'P':
339                     // Path.
340                     a.put(_uri.path);
341                     break;
342                 case 'q':
343                     // query parameters supplied with url.
344                     a.put(_uri.query);
345                     break;
346                 case 's':
347                     a.put("Response(%d, %s)".format(_code, _finalURI.uri()));
348                     break;
349                 case 'B': // received bytes
350                     a.put("%d".format(_responseBody.length));
351                     break;
352                 case 'T': // request total time, ms
353                     a.put("%d".format((_finishedAt - _startedAt).total!"msecs"));
354                     break;
355                 case 'U':
356                     a.put(_uri.uri());
357                     break;
358                 case 'S':
359                     a.put("%d".format(_code));
360                     break;
361                 default:
362                     throw new FormatException("Unknown Response format specifier: %" ~ f.spec);
363             }
364         }
365         return a.data();
366     }
367 }
368 
369 struct _UH {
370     // flags for each important header, added by user using addHeaders
371     mixin(bitfields!(
372     bool, "Host", 1,
373     bool, "UserAgent", 1,
374     bool, "ContentLength", 1,
375     bool, "Connection", 1,
376     bool, "AcceptEncoding", 1,
377     bool, "ContentType", 1,
378     bool, "Cookie", 1,
379     uint, "", 1
380     ));
381 }
382