mir.ion.conv source code

1 /++
2 Conversion utilities.
3 +/
4 module mir.ion.conv;
5 
6 public import mir.ion.internal.stage3: mir_json2ion;
7 import mir.ion.stream: IonValueStream;
8 
9 private enum dip1000 = __traits(compiles, ()@nogc { throw new Exception(""); });
10 
11 /++
12 Serialize value to binary ion data and deserialize it back to requested type.
13 Uses GC allocated string tables.
14 +/
15 template serde(T)
16     if (!is(immutable T == immutable IonValueStream))
17 {
18     import mir.serde: SerdeTarget;
19 
20     ///
21     T serde(V)(auto scope ref const V value, int serdeTarget = SerdeTarget.ion)
22     {
23         T target;
24         serde(target, value, serdeTarget);
25         return target;
26     }
27 
28     ///
29     void serde(V)(scope ref T target, auto ref scope const V value, int serdeTarget = SerdeTarget.ion)
30         if (!is(immutable V == immutable IonValueStream))
31     {
32         import mir.ion.exception;
33         import mir.deser.ion: deserializeIon;
34         import mir.ion.internal.data_holder: ionPrefix;
35         import mir.ser: serializeValue;
36         import mir.ser.ion: ionSerializer;
37         import mir.ion.symbol_table: IonSymbolTable, removeSystemSymbols, IonSystemSymbolTable_v1;
38         import mir.ion.value: IonValue, IonDescribedValue, IonList;
39         import mir.serde: serdeGetSerializationKeysRecurse;
40         import mir.utility: _expect;
41 
42         enum nMax = 4096;
43         enum keys = serdeGetSerializationKeysRecurse!V.removeSystemSymbols;
44 
45 
46         import mir.appender : scopedBuffer;
47         auto symbolTableBuffer = scopedBuffer!(const(char)[]);
48 
49         auto table = () @trusted { IonSymbolTable!false ret = void; ret.initializeNull; return ret; }();
50         auto serializer = ionSerializer!(nMax * 8, keys, false);
51         serializer.initialize(table);
52         serializeValue(serializer, value);
53         serializer.finalize;
54 
55         scope const(const(char)[])[] symbolTable;
56 
57         // use runtime table
58         if (table.initialized)
59         {
60             symbolTableBuffer.put(IonSystemSymbolTable_v1);
61             foreach (IonErrorCode error, scope IonDescribedValue symbolValue; IonList(table.unfinilizedKeysData))
62             {
63                 assert(!error);
64                 (()@trusted => symbolTableBuffer.put(symbolValue.trustedGet!(const(char)[])))();
65             }
66             symbolTable = symbolTableBuffer.data;
67         }
68         else
69         {
70             static immutable compileTimeTable = IonSystemSymbolTable_v1 ~ keys;
71             symbolTable = compileTimeTable;
72         }
73         auto ionValue = ()@trusted {return serializer.data.IonValue.describe();}();
74         return deserializeIon!T(target, symbolTable, ionValue);
75     }
76 
77     /// ditto
78     void serde()(scope ref T target, scope IonValueStream stream, int serdeTarget = SerdeTarget.ion)
79         if (!is(immutable V == immutable IonValueStream))
80     {
81         import mir.deser.ion: deserializeIon;
82         return deserializeIon!T(target, stream.data);
83     }
84 }
85 
86 /// ditto
87 template serde(T)
88     if (is(T == IonValueStream))
89 {
90     ///
91     import mir.serde: SerdeTarget;
92     T serde(V)(auto ref scope const V value, int serdeTarget = SerdeTarget.ion)
93         if (!is(immutable V == immutable IonValueStream))
94     {
95         import mir.ser.ion: serializeIon;
96         return serializeIon(value, serdeTarget).IonValueStream;
97     }
98 }
99 
100 
101 ///
102 version(mir_ion_test)
103 @safe
104 unittest {
105     import mir.ion.stream: IonValueStream;
106     import mir.algebraic_alias.json: JsonAlgebraic;
107     static struct S
108     {
109         double a;
110         string s;
111     }
112     auto s = S(12.34, "str");
113     assert(s.serde!JsonAlgebraic.serde!S == s);
114     assert(s.serde!IonValueStream.serde!S == s);
115 }
116 
117 @safe pure
118 version(mir_ion_test)
119 unittest {
120     static struct S
121     {
122         double a;
123         string s;
124     }
125     auto s = S(12.34, "str");
126     assert(s.serde!S == s);
127     assert(s.serde!IonValueStream.serde!S == s);
128 }
129 
130 /++
131 Converts JSON Value Stream to binary Ion data.
132 +/
133 immutable(ubyte)[] json2ion(scope const(char)[] text)
134     @trusted pure
135 {
136     pragma(inline, false);
137     import mir.ion.exception: ionErrorMsg, IonParserMirException;
138 
139     immutable(ubyte)[] ret;
140     mir_json2ion(text, (error, data)
141     {
142         if (error.code)
143             throw new IonParserMirException(error.code.ionErrorMsg, error.location);
144         ret = data.idup;
145     });
146     return ret;
147 }
148 
149 ///
150 @safe pure
151 version(mir_ion_test) unittest
152 {
153     import mir.test;
154     static immutable ubyte[] data = [0xe0, 0x01, 0x00, 0xea, 0xe9, 0x81, 0x83, 0xd6, 0x87, 0xb4, 0x81, 0x62, 0x81, 0x61, 0xd6, 0x8b, 0x21, 0x01, 0x8a, 0x21, 0x02];
155     `{"a":1,"b":2}`.json2ion.should == data;
156 }
157 
158 /++
159 Convert an JSON value to a Ion Value Stream.
160 
161 This function is the @nogc version of json2ion.
162 Params:
163     text = The JSON to convert
164     appender = A buffer that will receive the Ion binary data
165 +/
166 void json2ion(Appender)(scope const(char)[] text, scope ref Appender appender)
167     @trusted pure @nogc
168 {
169     import mir.ion.exception: ionErrorMsg, ionException, IonMirException;
170     import mir.ion.internal.data_holder: ionPrefix;
171 
172     mir_json2ion(text, (error, data)
173     {
174         if (error.code)
175         {
176             enum nogc = __traits(compiles, (const(ubyte)[] data, scope ref Appender appender) @nogc { appender.put(data); });
177             static if (!nogc || dip1000)
178             {
179                 throw new IonMirException(error.code.ionErrorMsg, ". location = ", error.location, ", last input key = ", error.key);
180             }
181             else
182             {
183                 throw error.code.ionException;
184             }
185         }
186         appender.put(data);
187     });
188 }
189 
190 ///
191 @safe pure
192 version(mir_ion_test) unittest
193 {
194     import mir.test;
195     import mir.appender : scopedBuffer;
196     static immutable ubyte[] data = [0xe0, 0x01, 0x00, 0xea, 0xe9, 0x81, 0x83, 0xd6, 0x87, 0xb4, 0x81, 0x62, 0x81, 0x61, 0xd6, 0x8b, 0x21, 0x01, 0x8a, 0x21, 0x02];
197     auto buf = scopedBuffer!ubyte;
198     json2ion(` { "a" : 1, "b" : 2 } `, buf);
199     buf.data.should == data;
200 }
201 
202 /++
203 Converts JSON Value Stream to binary Ion data wrapped to $(SUBREF stream, IonValueStream).
204 +/
205 IonValueStream json2ionStream(scope const(char)[] text)
206     @trusted pure
207 {
208     return text.json2ion.IonValueStream;
209 }
210 
211 ///
212 @safe pure
213 version(mir_ion_test) unittest
214 {
215     static immutable ubyte[] data = [0xe0, 0x01, 0x00, 0xea, 0xe9, 0x81, 0x83, 0xd6, 0x87, 0xb4, 0x81, 0x62, 0x81, 0x61, 0xd6, 0x8b, 0x21, 0x01, 0x8a, 0x21, 0x02];
216     assert(`{"a":1,"b":2}`.json2ionStream.data == data);
217 }
218 
219 /++
220 Converts Ion Value Stream data to JSON text.
221 
222 The function performs `data.IonValueStream.serializeJson`.
223 +/
224 string ion2json(scope const(ubyte)[] data)
225     @safe pure
226 {
227     pragma(inline, false);
228     import mir.ser.json: serializeJson;
229     return data.IonValueStream.serializeJson;
230 }
231 
232 ///
233 @safe pure
234 version(mir_ion_test) unittest
235 {
236     static immutable ubyte[] data = [0xe0, 0x01, 0x00, 0xea, 0xe9, 0x81, 0x83, 0xd6, 0x87, 0xb4, 0x81, 0x61, 0x81, 0x62, 0xd6, 0x8a, 0x21, 0x01, 0x8b, 0x21, 0x02];
237     assert(data.ion2json == `{"a":1,"b":2}`);
238     // static assert(data.ion2json == `{"a":1,"b":2}`);
239 }
240 
241 version(mir_ion_test) unittest
242 {
243     assert("".json2ion.ion2text == "");
244 }
245 
246 /++
247 Converts Ion Value Stream data to JSON text
248 
249 The function performs `data.IonValueStream.serializeJsonPretty`.
250 +/
251 string ion2jsonPretty(scope const(ubyte)[] data)
252     @safe pure
253 {
254     pragma(inline, false);
255     import mir.ser.json: serializeJsonPretty;
256     return data.IonValueStream.serializeJsonPretty;
257 }
258 
259 ///
260 @safe pure
261 version(mir_ion_test) unittest
262 {
263     static immutable ubyte[] data = [0xe0, 0x01, 0x00, 0xea, 0xe9, 0x81, 0x83, 0xd6, 0x87, 0xb4, 0x81, 0x61, 0x81, 0x62, 0xd6, 0x8a, 0x21, 0x01, 0x8b, 0x21, 0x02];
264     assert(data.ion2jsonPretty == "{\n\t\"a\": 1,\n\t\"b\": 2\n}");
265     // static assert(data.ion2jsonPretty == "{\n\t\"a\": 1,\n\t\"b\": 2\n}");
266 }
267 
268 /++
269 Convert an Ion Text value to a Ion data.
270 Params:
271     text = The text to convert
272 Returns:
273     An array containing the Ion Text value as an Ion data.
274 +/
275 immutable(ubyte)[] text2ion(scope const(char)[] text)
276     @trusted pure
277 {
278     import mir.ion.internal.data_holder: ionPrefix;
279     import mir.ion.symbol_table: IonSymbolTable;
280     import mir.ion.internal.data_holder: ionPrefix;
281     import mir.ser.ion : ionSerializer;
282     import mir.serde : SerdeTarget;
283     import mir.deser.text : IonTextDeserializer;
284     enum nMax = 4096;
285 
286     IonSymbolTable!true table;
287     auto serializer = ionSerializer!(nMax * 8, null, true);
288     serializer.initialize(table);
289 
290     auto deser = IonTextDeserializer!(typeof(serializer))(&serializer);
291     deser(text);
292     serializer.finalize;
293 
294     if (table.initialized)
295     {
296         table.finalize;
297         return cast(immutable) (ionPrefix ~ table.data ~ serializer.data);
298     }
299     else
300     {
301         return cast(immutable) (ionPrefix ~ serializer.data);
302     }
303 }
304 ///
305 @safe pure
306 version(mir_ion_test) unittest
307 {
308     static immutable ubyte[] data = [0xe0, 0x01, 0x00, 0xea, 0xe9, 0x81, 0x83, 0xd6, 0x87, 0xb4, 0x81, 0x61, 0x81, 0x62, 0xd6, 0x8a, 0x21, 0x01, 0x8b, 0x21, 0x02];
309     assert(`{"a":1,"b":2}`.text2ion == data);
310     static assert(`{"a":1,"b":2}`.text2ion == data);
311     enum s = `{a:2.232323e2, b:2.1,}`.text2ion;
312 }
313 
314 /++
315 Converts Ion Text Value Stream to binary Ion data wrapped to $(SUBREF stream, IonValueStream).
316 +/
317 IonValueStream text2ionStream(scope const(char)[] text)
318     @trusted pure
319 {
320     return text.text2ion.IonValueStream;
321 }
322 
323 ///
324 @safe pure
325 version(mir_ion_test) unittest
326 {
327     static immutable ubyte[] data = [0xe0, 0x01, 0x00, 0xea, 0xe9, 0x81, 0x83, 0xd6, 0x87, 0xb4, 0x81, 0x61, 0x81, 0x62, 0xd6, 0x8a, 0x21, 0x01, 0x8b, 0x21, 0x02];
328     assert(`{a:1,b:2}`.text2ionStream.data == data);
329 }
330 
331 /++
332 Convert an Ion Text value to a Ion Value Stream.
333 
334 This function is the @nogc version of text2ion.
335 Params:
336     text = The text to convert
337     appender = A buffer that will receive the Ion binary data
338 +/
339 void text2ion(Appender)(scope const(char)[] text, scope ref Appender appender)
340     @trusted
341 {
342     import mir.ion.internal.data_holder: ionPrefix;
343     import mir.ion.symbol_table: IonSymbolTable;
344     import mir.ion.internal.data_holder: ionPrefix;
345     import mir.ser.ion : ionSerializer;
346     import mir.serde : SerdeTarget;
347     import mir.deser.text : IonTextDeserializer;
348     enum nMax = 4096;
349     IonSymbolTable!false table = void;
350     table.initialize;
351 
352     auto serializer = ionSerializer!(nMax * 8, null, false);
353     serializer.initialize(table);
354 
355     auto deser = IonTextDeserializer!(typeof(serializer))(&serializer);
356 
357     deser(text);
358     serializer.finalize;
359 
360     appender.put(ionPrefix);
361     if (table.initialized)
362     {
363         table.finalize;
364         appender.put(table.data);
365     }
366     appender.put(serializer.data);
367 }
368 ///
369 @safe pure @nogc
370 version(mir_ion_test) unittest
371 {
372     import mir.appender : scopedBuffer;
373     static immutable data = [0xe0, 0x01, 0x00, 0xea, 0xe9, 0x81, 0x83, 0xd6, 0x87, 0xb4, 0x81, 0x61, 0x81, 0x62, 0xd6, 0x8a, 0x21, 0x01, 0x8b, 0x21, 0x02];
374     auto buf = scopedBuffer!ubyte;
375     text2ion("{\n\ta: 1,\n\tb: 2\n}", buf);
376     assert(buf.data == data);
377 }
378 
379 /++
380 Converts Ion Value Stream data to text.
381 
382 The function performs `data.IonValueStream.serializeText`.
383 +/
384 string ion2text(scope const(ubyte)[] data)
385     @safe pure
386 {
387     pragma(inline, false);
388     import mir.ser.text: serializeText;
389     return data.IonValueStream.serializeText;
390 }
391 
392 ///
393 @safe pure
394 version(mir_ion_test) unittest
395 {
396     static immutable ubyte[] data = [0xe0, 0x01, 0x00, 0xea, 0xe9, 0x81, 0x83, 0xd6, 0x87, 0xb4, 0x81, 0x61, 0x81, 0x62, 0xd6, 0x8a, 0x21, 0x01, 0x8b, 0x21, 0x02];
397     assert(data.ion2text == `{a:1,b:2}`);
398     // static assert(data.ion2text == `{a:1,b:2}`);
399 }
400 
401 ///
402 @safe pure
403 version(mir_ion_test) unittest
404 {
405     static immutable ubyte[] data = [0xe0, 0x01, 0x00, 0xea, 0xea, 0x81, 0x83, 0xde, 0x86, 0x87, 0xb4, 0x83, 0x55, 0x53, 0x44, 0xe6, 0x81, 0x8a, 0x53, 0xc1, 0x04, 0xd2];
406     assert(data.ion2text == `USD::123.4`);
407     // static assert(data.ion2text == `USD::123.4`);
408 }
409 
410 // 
411 
412 /++
413 Converts Ion Value Stream data to text
414 
415 The function performs `data.IonValueStream.serializeTextPretty`.
416 +/
417 string ion2textPretty(scope const(ubyte)[] data)
418     @safe pure
419 {
420     pragma(inline, false);
421     import mir.ser.text: serializeTextPretty;
422     return data.IonValueStream.serializeTextPretty;
423 }
424 
425 ///
426 @safe pure
427 version(mir_ion_test) unittest
428 {
429     static immutable ubyte[] data = [0xe0, 0x01, 0x00, 0xea, 0xe9, 0x81, 0x83, 0xd6, 0x87, 0xb4, 0x81, 0x61, 0x81, 0x62, 0xd6, 0x8a, 0x21, 0x01, 0x8b, 0x21, 0x02];
430     assert(data.ion2textPretty == "{\n\ta: 1,\n\tb: 2\n}");
431     // static assert(data.ion2textPretty == "{\n\ta: 1,\n\tb: 2\n}");
432 }
433 
434 void msgpack2ion(Appender)(scope const(ubyte)[] data, scope ref Appender appender)
435     @trusted
436 {
437     import mir.ion.internal.data_holder: ionPrefix;
438     import mir.ion.symbol_table: IonSymbolTable;
439     import mir.ion.internal.data_holder: ionPrefix;
440     import mir.ser.ion : ionSerializer;
441     import mir.serde : SerdeTarget;
442     import mir.deser.msgpack : MsgpackValueStream;
443     enum nMax = 4096;
444 
445     IonSymbolTable!false table = void;
446     table.initialize;
447     auto serializer = ionSerializer!(nMax * 8, null, false);
448     serializer.initialize(table);
449 
450     data.MsgpackValueStream.serialize(serializer);
451     serializer.finalize;
452 
453     appender.put(ionPrefix);
454     if (table.initialized)
455     {
456         table.finalize;
457         appender.put(table.data);
458     }
459     appender.put(serializer.data);
460 }
461 
462 /++
463 Converts MessagePack binary data to Ion binary data.
464 +/
465 @safe pure
466 immutable(ubyte)[] msgpack2ion()(scope const(ubyte)[] data)
467 {
468     import mir.appender : scopedBuffer;
469     auto buf = scopedBuffer!ubyte;
470     data.msgpack2ion(buf);
471     return buf.data.idup;
472 }
473 
474 @safe pure @nogc
475 version(mir_ion_test) unittest
476 {
477     import mir.appender : scopedBuffer;
478     import mir.deser.ion : deserializeIon;
479     static struct S
480     {
481         bool compact;
482         int schema;
483     }
484 
485     auto buf = scopedBuffer!ubyte();
486     static immutable ubyte[] data = [0x82, 0xa7, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0xc3, 0xa6, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x04];
487     data.msgpack2ion(buf);
488     assert(buf.data.deserializeIon!S == S(true, 4));
489 }
490   
491 @safe pure
492 version(mir_ion_test) unittest
493 {
494     static immutable testStrings = [
495         "2018-01-02T03:04:05Z",
496         "2018-01-02T03:04:05.678901234Z",
497         "2038-01-19T03:14:07.999999999Z",
498         "2038-01-19T03:14:08Z",
499         "2038-01-19T03:14:08.000000001Z",
500         "2106-02-07T06:28:15Z",
501         "2106-02-07T06:28:15.999999999Z",
502         "2106-02-07T06:28:16.000000000Z",
503         "2514-05-30T01:53:03.999999999Z",
504         "2514-05-30T01:53:04.000000000Z",
505         "1969-12-31T23:59:59.000000000Z",
506         "1969-12-31T23:59:59.999999999Z",
507         "1970-01-01T00:00:00Z",
508         "1970-01-01T00:00:00.000000001Z",
509         "1970-01-01T00:00:01Z",
510         "1899-12-31T23:59:59.999999999Z",
511         "1900-01-01T00:00:00.000000000Z",
512         "9999-12-31T23:59:59.999999999Z",
513     ];
514 
515     static immutable ubyte[][] testData = [
516         [0xd6, 0xff, 0x5a, 0x4a, 0xf6, 0xa5],
517         [0xd7, 0xff, 0xa1, 0xdc, 0xd7, 0xc8, 0x5a, 0x4a, 0xf6, 0xa5],
518         [0xd7, 0xff, 0xee, 0x6b, 0x27, 0xfc, 0x7f, 0xff, 0xff, 0xff],
519         [0xd6, 0xff, 0x80, 0x00, 0x00, 0x00],
520         [0xd7, 0xff, 0x00, 0x00, 0x00, 0x04, 0x80, 0x00, 0x00, 0x00],
521         [0xd6, 0xff, 0xff, 0xff, 0xff, 0xff],
522         [0xd7, 0xff, 0xee, 0x6b, 0x27, 0xfc, 0xff, 0xff, 0xff, 0xff],
523         [0xd7, 0xff, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00],
524         [0xd7, 0xff, 0xee, 0x6b, 0x27, 0xff, 0xff, 0xff, 0xff, 0xff],
525         [0xc7, 0x0c, 0xff, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00],
526         [0xc7, 0x0c, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff],
527         [0xc7, 0x0c, 0xff, 0x3b, 0x9a, 0xc9, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff],
528         [0xd6, 0xff, 0x00, 0x00, 0x00, 0x00],
529         [0xd7, 0xff, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00],
530         [0xd6, 0xff, 0x00, 0x00, 0x00, 0x01],
531         [0xc7, 0x0c, 0xff, 0x3b, 0x9a, 0xc9, 0xff, 0xff, 0xff, 0xff, 0xff, 0x7c, 0x55, 0x81, 0x7f],
532         [0xc7, 0x0c, 0xff, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x7c, 0x55, 0x81, 0x80],
533         [0xc7, 0x0c, 0xff, 0x3b, 0x9a, 0xc9, 0xff, 0x00, 0x00, 0x00, 0x3a, 0xff, 0xf4, 0x41, 0x7f],
534     ];
535 
536     foreach (i, ts; testStrings)
537     {
538         import mir.ser.ion : serializeIon;
539         import mir.timestamp : Timestamp;
540         auto mp = testData[i];
541         auto ion = mp.msgpack2ion;
542         import mir.test;
543         ion.ion2text.should == ts;
544         ts.Timestamp.serializeIon.ion2msgpack.should == mp;
545     }
546 }
547 
548 /++
549 Converts Ion binary data to MessagePack binary data.
550 +/
551 @safe pure
552 immutable(ubyte)[] ion2msgpack()(scope const(ubyte)[] data)
553 {
554     import mir.ser.msgpack: serializeMsgpack;
555     return data.IonValueStream.serializeMsgpack;
556 }
557 
558 @safe pure
559 version(mir_ion_test) unittest
560 {
561     import mir.test;
562     foreach(text; [
563         `null`,
564         `true`,
565         `1`,
566         `-2`,
567         `3.0`,
568         `2001-01-02T03:04:05Z`,
569         `[]`,
570         `[1,-2,3.0]`,
571         `[null,true,[1,-2,3.0],2001-01-02T03:04:05Z]`,
572         `{}`,
573         `{d:2001-01-02T03:04:05Z}`,
574     ])
575         text.text2ion.ion2msgpack.msgpack2ion.ion2text.should == text;
576 }
The OpenD Programming Language
