Coverage report: /development/source/library/org/datagraph/spocq-shard/src/core/encoding/sparql-results-turtle.lisp
| Kind | Covered | All | % |
| expression | 254 | 844 | 30.1 |
| branch | 21 | 98 | 21.4 |
Key
Not instrumented
Conditionalized out
Executed
Not executed
Both branches taken
One branch taken
Neither branch taken
1
;;; -*- Mode: lisp; Syntax: ansi-common-lisp; Base: 10; Package: org.datagraph.spocq.implementation; -*-
3
(in-package :org.datagraph.spocq.implementation)
5
;;; results as turtle; serializer only
8
;;; writing sparql results and terms
10
(defgeneric write-rdf-turtle (results stream)
11
(:documentation "Encode the result field to the stream as ntriples.
12
The results must have three or four dimensions - if four, then treat the elemnents as quads.
13
Allow the first element to be a list of dimensions.
14
The remaining entries aer solutions sets. if a variable is unbound, the solution element is etf:nil.")
16
(:method ((results t) (stream t))
17
(spocq.e:request-error "Invalid turtle result field: ~s" results))
19
(:method ((results list-solution-field) (stream stream))
20
(write-rdf-turtle (cons (solution-field-dimensions results)
21
(solution-field-solutions results))
24
(:method ((results cons) (stream stream))
25
(let* ((variables (if (every #'variable-p (first results))
27
(solutions (if (eq variables (first results)) (rest results) results))
29
(start (or (response-offset) 0))
32
(unless (typep (length variables) '(integer 3 4))
33
(spocq.e:request-error "Invalid turtle field: ~s ..." variables))
35
(case (length (first solutions))
37
(t (triple-dimensions)))))
39
(setf solutions (nthcdr start solutions))
42
(loop for solution in solutions
43
with last-subject = nil
44
if (and end (>= index end))
46
else do (flet ((emit (term)
47
(encode-turtle-object term stream)
48
(write-char #\space stream)))
49
(destructuring-bind (subject predicate object &optional graph) solution
50
(declare (ignore graph))
51
(when (and subject predicate object)
53
(cond ((equalp last-subject subject)
54
(format stream " ;~% "))
56
(format stream " .~%")
59
(setf last-subject subject)
64
(write-char #\. stream)
66
(incf-stat *statements-returned* index)
69
(:method ((results solution-generator) (stream stream))
70
(let* ((dimensions (solution-generator-dimensions results))
71
(channel (solution-generator-channel results))
72
(base-width (length dimensions))
74
(start (or (response-offset) 0))
77
(case (length dimensions)
78
(0 ) ;; corresponds to a unit table
80
(do-pages (page channel)
81
(when (and end (>= index end))
83
(if (>= (+ index (array-dimension page 0)) start)
84
(cond ((= base-width (array-dimension page 1))
85
(trace-data write-rdf-ntriples dimensions (term-value-field page))
87
(write-rdf-field-turtle last-subject page stream index start end)))
89
(log-warn "field width mismatch: ~s : ~s."
90
dimensions (array-dimension page 1))
91
(incf index (array-dimension page 0))))
92
; otherwise skip the entire page
93
(incf index (array-dimension page 0))))
94
(when last-subject (write-char #\. stream))
96
(incf-stat *statements-returned* index))
98
;; this should not happen, as other arity should not parse
99
(spocq.e:request-error "Invalid turtle field: ~s ..." dimensions)))
102
(defun write-rdf-field-turtle (last-subject page stream &optional (index 0) (start 0) end)
103
(unless (typep (array-dimension page 1) '(integer 3 4)) ()
104
(spocq.e:request-error "Invalid turtle field dimensions: ~s ..." (array-dimensions page)))
105
(flet ((term-aspect-encoder (term-type term-literal term-language-tag term-datatype)
106
(encode-turtle-term-aspects term-type term-literal term-language-tag term-datatype stream)))
107
(declare (dynamic-extent #'term-aspect-encoder))
108
(let ((term-deconstructor (repository-term-deconstructor *transaction*)))
109
(flet ((emit (term-id)
111
(-1 (write-string "<urn:dydra:default>" stream))
112
(-2 (write-string "<urn:dydra:named>" stream))
113
(t (funcall term-deconstructor #'term-aspect-encoder *transaction* term-id)))
114
(write-char #\space stream)))
115
(dotimes (page-index (array-dimension page 0))
116
(when (>= index start)
117
(when (and end (>= index end))
119
(let ((subject (aref page page-index 0))
120
(predicate (aref page page-index 1))
121
(object (aref page page-index 2)))
122
(when (not (zerop (* subject predicate object)))
124
(cond ((= last-subject subject)
125
(format stream " ;~% "))
127
(format stream " .~%")
130
(setf last-subject subject)
138
(defun cl-user::format-turtle-character-data (stream string &optional option arg)
139
"Emit a character data string to an utf-8 encoded stream with escaping.
140
This means the terminator, whitespace-format, and constrol characters.
141
All else is passed to the stream for utf-8 encoding."
142
(declare (ignore option arg))
143
(loop for char across string
145
(#\" (write-string "\\\"" stream))
146
(#\\ (write-string "\\\\" stream))
147
;;(#\/ (write-string "\\/" stream))
148
(#\backspace (write-string "\\b" stream))
149
(#\page (write-string "\\f" stream))
150
(#\linefeed (write-string "\\n" stream))
151
(#\return (write-string "\\r" stream))
152
(#\tab (write-string "\\t" stream))
153
(t (let ((code (char-code char)))
154
(if (<= code #x1f) ; C0 control : http://en.wikipedia.org/wiki/Control_character
155
(format stream "\\u~4,'0x" code)
156
(write-char char stream)))))))
159
(defgeneric cl-user::format-turtle-iri-namestring (stream term &optional option arg)
160
(:documentation "Emit an IRI namestring to an utf-8 encoded stream with escaping.
161
For an IRI, this means the terminator, whitespace-format, and constrol characters.
162
All else is passed to the stream for utf-8 encoding.")
164
(:method (stream (object string) &optional option arg)
165
(declare (ignore option arg))
166
(labels ((encode-unicode-char-escape (char)
167
(encode-unicode-charcode-escape (char-code char)))
168
(encode-unicode-charcode-escape (code)
169
(format stream "\\u~4,'0x" code))
170
(encode-urlencode-char-escape (char)
171
(encode-urlencode-charcode-escape (char-code char)))
172
(encode-urlencode-charcode-escape (code)
173
(format stream "%~2,'0x" code)))
174
(loop for char across object
177
(encode-urlencode-char-escape char))
178
((#\< #\> #\" #\| #\^ #\` #\\)
179
(encode-unicode-char-escape char))
180
(t (let ((code (char-code char)))
181
(if (<= code #x1f) ; C0 control : http://en.wikipedia.org/wiki/Control_character
182
(encode-unicode-charcode-escape code)
183
(write-char char stream))))))))
185
(:method (stream (object spocq:iri) &optional option arg)
186
(declare (ignore option arg))
187
(cl-user::format-turtle-iri-namestring stream (spocq:iri-lexical-form object)))
189
(:method (stream (object symbol) &optional option arg)
190
(declare (ignore option arg))
191
;; vocabulary iri should not require escaping
192
(cl-user::format-turtle-iri-namestring stream (symbol-uri-namestring object))))
194
;;; (format *trace-output* ">~/format-json-character-data/<" (concatenate 'string "asdf.\"." `(,(code-char 31)) ".qwer"))
196
(defun cl-user::format-turtle-external-utf8-string (stream %string &optional option arg)
197
(declare (ignore option arg))
198
(stream-write-turtle-external-utf8-string stream %string))
200
(defgeneric stream-write-turtle-external-utf8-string (stream %string)
201
(:documentation "Given an utf-8 encoded stream and a utf-8 byte sequence as an external string,
202
emit the string to the stream wih escaping for string data.")
204
(:method ((stream amqp:channel) %string)
205
(macrolet ((emit (byte) `(amqp.i::amqp-stream-write-byte stream ,byte)))
206
(flet ((emit-non-zero (byte)
208
(return-from stream-write-turtle-external-utf8-string)
212
;; All Unicode characters may be placed within the
213
;; quotation marks except for the characters that must be escaped:
214
;; quotation mark, reverse solidus, and the control characters (U+0000
216
(loop (let ((byte (cffi:mem-ref %string :uint8 i)))
217
(declare (type (integer 0 255) byte))
218
(when (zerop byte) (return))
219
(cond ((= 0 (logand #x80 byte))
222
(emit #.(char-code #\\))
223
(emit #.(char-code #\")))
225
(emit #.(char-code #\\))
226
(emit #.(char-code #\\)))
228
(emit #.(char-code #\\))
229
(emit #.(char-code #\t)))
230
(#.(char-code #\return)
231
(emit #.(char-code #\\))
232
(emit #.(char-code #\r)))
233
(#.(char-code #\linefeed)
234
(emit #.(char-code #\\))
235
(emit #.(char-code #\n)))
238
(format stream "\\u~4,'0x" byte)
241
((= #xc0 (logand #xe0 byte))
243
(emit-non-zero (cffi:mem-ref %string :uint8 (incf i)))
245
((= #xe0 (logand #xf0 byte))
247
(emit-non-zero (cffi:mem-ref %string :uint8 (incf i)))
248
(emit-non-zero (cffi:mem-ref %string :uint8 (incf i)))
250
((= #xf0 (logand #xf8 byte))
252
(emit-non-zero (cffi:mem-ref %string :uint8 (incf i)))
253
(emit-non-zero (cffi:mem-ref %string :uint8 (incf i)))
254
(emit-non-zero (cffi:mem-ref %string :uint8 (incf i)))
257
(log-error "Illegal UTF-8 data: x~2,'0x." byte)
259
(:method ((stream stream) %string)
260
(cl-user::format-turtle-character-data stream (cffi:foreign-string-to-lisp %string))))
263
(defun cl-user::format-turtle-external-utf8-iri-namestring (stream %string &optional option arg)
264
(declare (ignore option arg))
265
(stream-write-turtle-external-utf8-iri-namestring stream %string))
267
(defgeneric stream-write-turtle-external-utf8-iri-namestring (stream %string)
268
(:documentation "Given an utf-8 encoded stream and a utf-8 byte sequence for an external iri namestring,
269
emit the string to the stream wih escaping for iri data.")
271
(:method ((stream amqp:channel) %string)
272
(macrolet ((emit (byte) `(amqp.i::amqp-stream-write-byte stream ,byte)))
273
(flet ((emit-non-zero (byte)
275
(return-from stream-write-turtle-external-utf8-iri-namestring)
279
;; All Unicode characters may be placed within the
280
;; quotation marks except for the characters that must be escaped:
281
;; quotation mark, reverse solidus, and the control characters (U+0000
283
(loop (let ((byte (cffi:mem-ref %string :uint8 i)))
284
(declare (type (integer 0 255) byte))
285
(when (zerop byte) (return))
286
(cond ((= 0 (logand #x80 byte))
289
(emit #.(char-code #\\))
290
(emit #.(char-code #\>)))
292
(emit #.(char-code #\\))
293
(emit #.(char-code #\\)))
295
(emit #.(char-code #\\))
296
(emit #.(char-code #\t)))
297
(#.(char-code #\return)
298
(emit #.(char-code #\\))
299
(emit #.(char-code #\r)))
300
(#.(char-code #\linefeed)
301
(emit #.(char-code #\\))
302
(emit #.(char-code #\n)))
305
(format stream "\\u~4,'0x" byte)
308
((= #xc0 (logand #xe0 byte))
310
(emit-non-zero (cffi:mem-ref %string :uint8 (incf i)))
312
((= #xe0 (logand #xf0 byte))
314
(emit-non-zero (cffi:mem-ref %string :uint8 (incf i)))
315
(emit-non-zero (cffi:mem-ref %string :uint8 (incf i)))
317
((= #xf0 (logand #xf8 byte))
319
(emit-non-zero (cffi:mem-ref %string :uint8 (incf i)))
320
(emit-non-zero (cffi:mem-ref %string :uint8 (incf i)))
321
(emit-non-zero (cffi:mem-ref %string :uint8 (incf i)))
324
(log-error "Illegal UTF-8 data: x~2,'0x." byte)
326
(:method ((stream stream) %string)
327
(cl-user::format-turtle-iri-namestring stream (cffi:foreign-string-to-lisp %string))))
330
(defun cl-user::format-turtle-term (stream term &optional option arg)
331
(declare (ignore option arg))
332
(encode-turtle-object term stream))
334
(defgeneric encode-turtle-object (object stream)
335
(:method ((object function) stream)
336
(funcall object stream))
338
(:method ((object t) (stream t))
339
(error "no encoding defined for object '~a' of type ~a for text/turtle."
340
object (type-of object)))
342
(:method ((object spocq:blank-node) stream)
343
(format stream "_:~@[~a_~]~/format-turtle-character-data/"
344
(blank-node-prefix) (spocq:blank-node-label object)))
346
(:method ((object spocq:boolean) stream)
347
(format stream "\"~a\"^^<http://www.w3.org/2001/XMLSchema#boolean>"
348
(term-lexical-form object)))
350
(:method ((eql t) stream)
351
(write-string "\"true\"^^<http://www.w3.org/2001/XMLSchema#boolean>" stream))
353
(:method ((eql null) stream)
354
(write-string "\"false\"^^<http://www.w3.org/2001/XMLSchema#boolean>" stream))
356
(:method ((object spocq:date) stream)
357
(format stream "\"~/format-turtle-character-data/\"^^<http://www.w3.org/2001/XMLSchema#date>"
358
(term-lexical-form object)))
360
(:method ((object spocq:date-time) stream)
361
(format stream "\"~/format-turtle-character-data/\"^^<http://www.w3.org/2001/XMLSchema#dateTime>"
362
(term-lexical-form object)))
364
(:method ((object spocq:day-time-duration) stream)
365
(format stream "\"~/format-turtle-character-data/\"^^<http://www.w3.org/2001/XMLSchema#dayTimeDuration>"
366
(term-lexical-form object)))
368
(:method ((object spocq:decimal) stream)
369
(if (typep object *expand-literal-values*)
370
(format stream "\"~a\"^^<http://www.w3.org/2001/XMLSchema#decimal>" (term-lexical-form object))
371
(format stream "~a" (term-lexical-form object))))
373
(:method ((object double-float) stream)
374
(if (or (eql object double-float-nan)
375
(eql object double-float-positive-infinity)
376
(eql object double-float-negative-infinity))
378
(let ((*read-default-float-format* 'double-float))
379
(if (typep object *expand-literal-values*)
380
(format stream "\"~f\"^^<http://www.w3.org/2001/XMLSchema#double>" object)
381
(format stream "~f" object)))))
383
(:method ((object spocq:duration) stream)
384
(format stream "\"~/format-turtle-character-data/\"^^<http://www.w3.org/2001/XMLSchema#duration>"
385
(term-lexical-form object)))
387
(:method ((object spocq:g-day) stream)
388
(format stream "\"~/format-turtle-character-data/\"^^<http://www.w3.org/2001/XMLSchema#gDay>"
389
(term-lexical-form object)))
391
(:method ((object spocq:g-month) stream)
392
(format stream "\"~/format-turtle-character-data/\"^^<http://www.w3.org/2001/XMLSchema#gMonth>"
393
(term-lexical-form object)))
395
(:method ((object spocq:g-month-day) stream)
396
(format stream "\"~/format-turtle-character-data/\"^^<http://www.w3.org/2001/XMLSchema#gMonthDay>"
397
(term-lexical-form object)))
399
(:method ((object spocq:g-year) stream)
400
(format stream "\"~/format-turtle-character-data/\"^^<http://www.w3.org/2001/XMLSchema#g-year>"
401
(term-lexical-form object)))
403
(:method ((object spocq:g-year-month) stream)
404
(format stream "\"~/format-turtle-character-data/\"^^<http://www.w3.org/2001/XMLSchema#gYearMonth>"
405
(term-lexical-form object)))
407
(:method ((object spocq:integer) stream)
408
(if (typep object *expand-literal-values*)
409
(format stream "\"~a\"^^<http://www.w3.org/2001/XMLSchema#integer>" (term-lexical-form object))
410
(format stream "~a" (term-lexical-form object))))
412
(:method ((object integer) stream)
413
(cond ((typep object *expand-literal-values*)
414
(format stream "\"~a\"^^<~a>"
416
(if *encode-object-subtypes*
418
((signed-byte 16) "http://www.w3.org/2001/XMLSchema#short")
419
((signed-byte 32) "http://www.w3.org/2001/XMLSchema#int")
420
(t "http://www.w3.org/2001/XMLSchema#integer"))
421
"http://www.w3.org/2001/XMLSchema#integer")))
423
(format stream "~d" object))))
425
(:method ((object spocq:iri) stream)
426
(format stream "<~/format-turtle-iri-namestring/>" (spocq:iri-lexical-form object)))
428
(:method ((object spocq:plain-literal) stream)
429
(format stream "\"~/format-turtle-character-data/\"@~a"
430
(spocq:literal-lexical-form object)
431
(spocq:plain-literal-language-tag object)))
433
(:method ((object symbol) stream)
434
(let ((uri-namestring (symbol-uri-namestring object)))
435
(cond (uri-namestring
436
(format stream "<~/format-turtle-iri-namestring/>" uri-namestring))
437
(*encode-turtle-object-as-variable-p*
438
(if (variable-p object)
439
(format stream "_:~@[~a_~]_~a" (blank-node-prefix) object)
440
(format stream "\"~a\"" object)))
442
(call-next-method)))))
444
(:method ((object string) stream)
445
(if (typep object *expand-literal-values*)
446
(format stream "\"~/format-turtle-character-data/\"^^<http://www.w3.org/2001/XMLSchema#string>" object)
447
(format stream "\"~/format-turtle-character-data/\"" object)))
449
(:method ((object spocq:unbound-variable) stream)
452
(:method ((object spocq:atomic-typed-literal) stream)
453
(format stream "\"~/format-turtle-character-data/\"^^<~/format-turtle-iri-namestring/>"
454
(spocq:literal-lexical-form object)
455
(spocq:literal-datatype-uri object)))
457
(:method ((object single-float) stream)
458
(if (or (eql object nan) (eql object +inf) (eql object -inf))
460
(let ((*read-default-float-format* 'single-float))
461
(if (typep object *expand-literal-values*)
462
(format stream "\"~f\"^^<http://www.w3.org/2001/XMLSchema#float>" object)
463
(format stream "~f" object)))))
465
(:method ((object rational) stream)
466
(let ((*read-default-float-format* 'single-float))
467
(if (typep object *expand-literal-values*)
468
(format stream "\"~f\"^^<http://www.w3.org/2001/XMLSchema#decimal>" (float object 1.0s0))
469
(format stream "~f" (float object 1.0s0)))))
471
(:method ((object spocq:time) stream)
472
(format stream "\"~/format-turtle-character-data/\"^^<http://www.w3.org/2001/XMLSchema#time>"
473
(term-lexical-form object)))
475
(:method ((object spocq:unsupported-typed-literal) stream)
476
(format stream "\"~/format-turtle-character-data/\"^^<~/format-turtle-iri-namestring/>"
477
(spocq:literal-lexical-form object)
478
(spocq:unsupported-typed-literal-datatype-uri object)))
480
(:method ((object spocq:year-month-duration) stream)
481
(format stream "\"~/format-turtle-character-data/\"^^<http://www.w3.org/2001/XMLSchema#yearMonthDuration>"
482
(term-lexical-form object))))
485
(defun encode-turtle-term-number (term-number stream)
486
"Given a TERM-NUMBER, fetch the respective term and encode that to the stream.
487
In general, this operates on both transient and persistent terms, but in practice, in particular for
488
application/sparql-results+term-number, it is called for transient terms only."
490
(flet ((term-aspect-encoder (term-type term-literal term-language-tag term-datatype)
491
(encode-turtle-term-aspects term-type term-literal term-language-tag term-datatype stream)))
492
(declare (dynamic-extent #'term-aspect-encoder))
493
(repository-call-with-numbered-term-aspects #'term-aspect-encoder *transaction* term-number)))
496
(defun encode-turtle-term (%term stream)
497
(encode-turtle-term-aspects (cffi:foreign-slot-value %term 'rdfcache::term 'rdfcache::type)
498
(cffi:foreign-slot-value %term 'rdfcache::term 'rdfcache::value)
499
(cffi:foreign-slot-value %term 'rdfcache::term 'rdfcache::language)
500
(cffi:foreign-slot-value %term 'rdfcache::term 'rdfcache::datatype)
503
(defun encode-turtle-term-aspects (term-type term-literal term-language-tag term-datatype stream)
506
;; (write-string "0" stream)
508
(:node ; encode a blank node
509
(format stream "_:~@[~a_~]" (blank-node-prefix))
510
(stream-write-turtle-external-utf8-string stream term-literal))
511
(:literal ; encode a typed or language-tagged literal
512
(cond ((not (cffi:null-pointer-p term-language-tag))
513
(format stream "\"~/format-turtle-external-utf8-string/\"@~/format-turtle-external-utf8-string/"
514
term-literal term-language-tag))
515
((and (not (cffi:null-pointer-p term-datatype))
516
(not (%string-equal term-datatype (|%http://www.w3.org/2001/XMLSchema#string|))))
517
;; in order to encode tsv, this would need to be extended to abbreviate numbers
518
;; that requires getting enough of the datatype iri to differentiate...
519
(format stream "\"~/format-turtle-external-utf8-string/\"^^<~/format-turtle-external-utf8-iri-namestring/>"
520
term-literal term-datatype))
521
((typep "" *expand-literal-values*)
522
(format stream "\"~/format-turtle-external-utf8-string/\"^^<http://www.w3.org/2001/XMLSchema#string>"
525
(format stream "\"~/format-turtle-external-utf8-string/\"" term-literal))))
527
(format stream "<~/format-turtle-external-utf8-iri-namestring/>" term-literal))))
532
(defmethod send-response-message (operation (message t) (stream t) (content-type mime:text/turtle))
533
"Given a MESSAGE, and a STREAM with the text/turtle CONTENT-TYPE, fall-back encode as ntriples"
534
(when *encoding-trace-output*
535
(setf stream (make-broadcast-stream *encoding-trace-output* stream)))
536
(let ((*package* *spocq-reader-package*))
537
(write-rdf-turtle message stream)))
540
(write-rdf-turtle '((?::a ?::s ?::z)
541
(<http://example/1> <http://example/2> "3")
542
(<http://example/1> <http://example/4> "5")
543
(<http://example/1> <http://example/4a> "5a")
544
(<http://example/6> <http://example/7> "8")
547
(test-sparql "select ?s ?p ?o where {graph ?g {?s ?p ?o}} limit 30" :repository-id "james/system" :response-content-type mime:text/turtle)