Coverage report: /development/source/library/org/datagraph/spocq-shard/src/core/statistics.lisp

KindCoveredAll%
expression0198 0.0
branch012 0.0
Key
Not instrumented
Conditionalized out
Executed
Not executed
 
Both branches taken
One branch taken
Neither branch taken
1
 ;;; -*- Mode: lisp; Syntax: ansi-common-lisp; Base: 10; Package: org.datagraph.spocq.implementation; -*-
2
 
3
 (in-package :org.datagraph.spocq.implementation)
4
 
5
 
6
 (defun pipe-usage-log-query (stream &key start-time end-time repository host
7
                                     (repository-id (error "repository-id is required."))
8
                                     (content-type mime:application/VND.DYDRA.SPARQL-RESULTS+HTML+TABLE)
9
                                     (offset 0) (limit nil))
10
   "extract the query constraints from environment variables - for cgi support, with over-rides from
11
  explicit command-line parameters; construct a log query; encode the output to standard-out as an
12
  html document with the properties for respective individual queries in a table."
13
   
14
   (let* ((base-uri (format nil "http://~a/" host))
15
          (repository (typecase repository
16
                        (null nil)
17
                        (iri repository)
18
                        (string (unless (string-equal repository "http://" :end1 (min (length repository) 7))
19
                                  (setf repository (concatenate 'string base-uri repository)))
20
                                (intern-iri repository))))
21
          (query-expression (construct-usage-log-query :start-time start-time
22
                                                       :end-time end-time
23
                                                       :repository repository
24
                                                       :offset offset
25
                                                       :limit (or limit 256)))
26
          (query (make-query :id (format nil "~a" (uuid:make-v1-uuid))
27
                             :sse-expression query-expression
28
                             :content-type content-type
29
                             :repository-id repository-id)))
30
     (cond ((equalp repository <http://example.com/sparql>)
31
            (format stream "<html><body><pre>~%")
32
            (pprint-sse query-expression stream)
33
            (format stream "~%</pre></body></html>~%"))
34
           (t
35
            (setf (task-property query :query-parameters)
36
                  `(:limit ,limit :offset ,offset :host ,host :end-time ,end-time :repository ,(when repository (|dydra|:|iri-path| repository))))
37
            (setf (task-property query :styles)
38
                  `(?::|timestamp| "width: 10em"
39
                                   ?::|taskId| "width: 20em; text-align: center;"
40
                                   ?::|repository| "width: 16em; text-align: center;"
41
                                   ?::|query_time| "width: 6em; text-align: right;"
42
                                   ?::|algebra_operations| "width: 6em; text-align:right;"
43
                                   ?::|user_id| "width: 24em"))
44
            (pipe-query query stream)))))
45
 
46
 
47
 (defun construct-usage-log-query (&key start-time end-time repository limit offset)
48
   (let* ((pattern `(spocq.a:|bgp|
49
                     (spocq.a:|triple| ?::|taskId| <urn:dydra:timestamp> ?::|time|)
50
                     (spocq.a:|triple| ?::|taskId| <urn:dydra:query_time> ?::|query_time|)
51
                     (spocq.a:|triple| ?::|taskId| <urn:dydra:algebra_operations> ?::|algebra_operations|)
52
                     (spocq.a:|triple| ?::|taskId| <urn:dydra:user_id> ?::|user_id|)
53
                     ;;(spocq.a:|triple| ?::|taskId| <urn:dydra:run_time> ?::|run_time|)
54
                     ;;(spocq.a:|triple| ?::|taskId| <urn:dydra:real_time> ?::|real_time|)
55
                     ;;(spocq.a:|triple| ?::|taskId| <urn:dydra:match_responses> ?::|match_responses|)
56
                     ;;(spocq.a:|triple| ?::|taskId| <urn:dydra:match_requests> ?::|match_requests|)
57
                     ))
58
          (filtered-pattern
59
           (if start-time
60
             (if end-time
61
               `(spocq.a:|filter| ,pattern
62
                                  (spocq.a:&& (spocq.a:>= ?::|time| ,(spocq.e:date-time start-time))
63
                                              (spocq.a:< ?::|time| ,(spocq.e:date-time end-time))))
64
               `(spocq.a:|filter| ,pattern
65
                                  (spocq.a:>= ?::|time| ,(spocq.e:date-time start-time))))
66
             (if end-time
67
               `(spocq.a:|filter| ,pattern
68
                                  (spocq.a:< ?::|time| ,(spocq.e:date-time end-time)))
69
               pattern)))
70
          ;; note, that the operations are ordered to filter and order prior to
71
          ;; extending with the computed bindings
72
          (base-query `(spocq.a:|select|
73
                        (spocq.a:|order|
74
                         (spocq.a:|graph| ,(or repository '?::|repository|)
75
                                          ,filtered-pattern)
76
                         ((spocq.a:|desc| ?::|time|)))
77
                        ((?::|timestamp| (|dydra|:|format| ?::|time| "%Y-%m-%dT%H:%i:%s"))
78
                         (?::|taskId| (|dydra|:|urn-uuid-string| ?::|taskId|))
79
                         (?::|repository| ,(if repository (|dydra|:|iri-path| repository) '(|dydra|:|iri-path| ?::|repository|)))
80
                         ?::|query_time|
81
                         ?::|algebra_operations|
82
                         ?::|user_id|
83
                         ;;?::|run_time|
84
                         ;;?::|real_time|
85
                         ;;?::|match_requests|
86
                         ;;?::|match_responses|
87
                         ))))
88
     (if (or limit offset)
89
       `(spocq.a:|slice| ,base-query
90
                         ,@(when offset `(:offset ,offset))
91
                         ,@(when limit `(:count ,limit)))
92
       base-query)))
93
 
94
 
95
 ;;; (expand-query (construct-usage-log-query :repository "de4.dydra.com") )
96
 ;;; statistics
97
 
98
 (defun pipe-usage-chart-query (stream &key start-time end-time repository host
99
                                               (repository-id (error "repository-id is required."))
100
                                               (content-type mime:application/VND.DYDRA.SPARQL-RESULTS+HTML+D3)
101
                                               (offset 0) (limit nil))
102
   "extract the query constraints from environment variables - for cgi support, with over-rides from
103
  explicit command-line parameters; construct a statistics query; encode the output to standard-out as
104
  an html document with the periodic aggregated values in a graphic display."
105
   
106
   (let* ((base-uri (format nil "http://~a/" host))
107
          (repository (typecase repository
108
                        (null nil)
109
                        (iri repository)
110
                        (string (unless (string-equal repository "http://" :end1 (min (length repository) 7))
111
                                  (setf repository (concatenate 'string base-uri repository)))
112
                                (intern-iri repository))))
113
          (query-expression (construct-usage-chart-query :start-time start-time
114
                                                              :end-time end-time
115
                                                              :repository repository
116
                                                              :offset offset
117
                                                              :limit (or limit 4096)))
118
          (query (make-query :id (format nil "~a" (uuid:make-v1-uuid))
119
                             :sse-expression query-expression
120
                             :content-type content-type
121
                             :repository-id repository-id)))
122
                          
123
     (cond ((equalp repository <http://example.com/sparql>)
124
            (format stream "<html><body><pre>~%")
125
            (pprint-sse query-expression stream)
126
            (format stream "~%</pre></body></html>~%"))
127
           (t
128
            (setf (task-property query :query-parameters) `(:limit ,limit :offset ,offset :host ,host :end-time ,end-time :repository ,(when repository (|dydra|:|iri-path| repository))))
129
            (pipe-query query stream)))))
130
 
131
 
132
 (defun construct-usage-chart-query (&key start-time end-time repository limit offset)
133
   (let* ((pattern `(spocq.a:|bgp|
134
                     (spocq.a:|triple| ?::|taskId| <urn:dydra:timestamp> ?::|timestamp|)
135
                     (spocq.a:|triple| ?::|taskId| <urn:dydra:query_time> ?::|query_time|)
136
                     (spocq.a:|triple| ?::|taskId| <urn:dydra:run_time> ?::|run_time|)
137
                     (spocq.a:|triple| ?::|taskId| <urn:dydra:real_time> ?::|real_time|)
138
                     (spocq.a:|triple| ?::|taskId| <urn:dydra:algebra_operations> ?::|algebra_operations|)
139
                     (spocq.a:|triple| ?::|taskId| <urn:dydra:match_requests> ?::|match_requests|)
140
                     (spocq.a:|triple| ?::|taskId| <urn:dydra:match_responses> ?::|match_responses|)
141
                     ))
142
          (filtered-pattern
143
           (if start-time
144
             (if end-time
145
               `(spocq.a:|filter| ,pattern
146
                                  (spocq.a:&& (spocq.a:>= ?::|timestamp| ,start-time) (spocq.a:< ?::|timestamp| ,end-time)))
147
               `(spocq.a:|filter| ,pattern
148
                                  (spocq.a:>= ?::|timestamp| ,start-time)))
149
             (if end-time
150
               `(spocq.a:|filter| ,pattern
151
                                  (spocq.a:< ?::|timestamp| ,end-time))
152
               pattern)))
153
          (base-query `(spocq.a:|order|
154
                        (spocq.a:|select|
155
                         (spocq.a:|select|
156
                          (spocq.a:|graph| ,(or repository '?::|repository|)
157
                                           ,filtered-pattern)
158
                          ((?::|timestamp| (|http://dydra.com#|:|format| ?::|timestamp| "%Y-%m-%dT%H:%i:00Z"))
159
                           ;; non-numeric ,@(unless repository '(?::|repository|))
160
                            ?::|query_time| ?::|run_time| ?::|real_time| ?::|algebra_operations| ?::|match_requests| ?::|match_responses|))
161
                         (:GROUP-BY (?::|timestamp|)
162
                                    ?::|timestamp|
163
                                    ;;(?::|count| (spocq.a:|count| ?::|timestamp|))
164
                                    ;; non-numeric ,(if repository `(?::|repository| (spocq.a:|sample| ,repository)) '?::|repository|)
165
                                    (?::|query_avg| (spocq.a:|avg| ?::|query_time|))
166
                                    (?::|query_max| (spocq.a:|max| ?::|query_time|))
167
                                    (?::|operations_avg| (spocq.a:|avg| ?::|algebra_operations|))
168
                                    (?::|operations_max| (spocq.a:|max| ?::|algebra_operations|))
169
                                    (?::|response_avg| (spocq.a:|avg| ?::|match_responses|))
170
                                    (?::|response_max| (spocq.a:|max| ?::|match_responses|))
171
                                    (?::|request_avg| (spocq.a:|avg| ?::|match_requests|))
172
                                    (?::|request_max| (spocq.a:|max| ?::|match_requests|))
173
                                    (?::|real_avg| (spocq.a:|avg| ?::|real_time|))
174
                                    (?::|real_max| (spocq.a:|max| ?::|real_time|))
175
                                    (?::|run_avg| (spocq.a:/ (spocq.a:|avg| ?::|run_time|) 1000000))
176
                                    (?::|run_max| (spocq.a:/ (spocq.a:|max| ?::|run_time|) 1000000))))
177
                        ((spocq.a:|desc| ?::|timestamp|)))))
178
     (if (or limit offset)
179
       `(spocq.a:|slice| ,base-query
180
                         ,@(when offset `(:offset ,offset))
181
                         ,@(when limit `(:count ,limit)))
182
       base-query)))
183
                         
184
 ;;; (construct-temporal-statistics-query :start-time (spocq.e:date-time "2013-01-01T00:00:00Z"))
185
 ;;; (construct-temporal-statistics-query :end-time (spocq.e:date-time "2013-01-01T00:00:00Z"))
186
 ;;; (construct-temporal-statistics-query :start-time (spocq.e:date-time "2013-01-01T00:00:00Z") :end-time (spocq.e:date-time "2013-01-01T00:00:01Z"))
187
 ;;; (construct-temporal-statistics-query)
188
 ;;; (construct-temporal-statistics-query :repository <http://dydra.com/a/r>)
189
 
190
 
191
 #|
192
 
193
 (pipe-usage-log-query *standard-output*
194
                       :repository-id "statistics/de4.dydra.com"
195
                       :offset 100 :limit 100
196
                       :start-time (spocq.e:date-time "2013-03-15T12:00:00Z")
197
                       :repository <http://de4.dydra.com/heltnormalt/dk-stage>
198
                       :content-type mime:application/sparql-results+json)
199
 
200
 ;;; remove old entries, with start_time
201
 ;;;
202
 ;;; count w/ start_time
203
 (pipe-query (make-query :id (format nil "~a" (uuid:make-v1-uuid))
204
                             :sse-expression (parse-sparql "
205
 select count(?p) where {graph ?g {?s <urn:dydra:start_time> ?time .
206
                                  ?s ?p ?o . } }")
207
                             :content-type mime:application/sparql-results+json
208
                             :repository-id "statistics/dev.dydra.com")
209
              *trace-output*)
210
 
211
 ;; latest date
212
 (pipe-query (make-query :id (format nil "~a" (uuid:make-v1-uuid))
213
                             :sse-expression (parse-sparql "
214
 select ?time where {graph ?g {?s <urn:dydra:start_time> ?time . } } order by desc (?time) limit 1")
215
                             :content-type mime:application/sparql-results+json
216
                             :repository-id "statistics/dev.dydra.com")
217
              *trace-output*)
218
 
219
 ;;; delete
220
 (pipe-query (make-query :id (format nil "~a" (uuid:make-v1-uuid))
221
                             :sse-expression (parse-sparql "
222
 delete { graph ?g { ?s ?p ?o } }
223
 where {graph ?g {?s <urn:dydra:start_time> ?time . ?s ?p ?o . } }")
224
                             :content-type mime:application/sparql-results+json
225
                             :repository-id "statistics/dev.dydra.com")
226
              *trace-output*)
227
 
228
 
229
 ;;; count w/ timestamp
230
 (pipe-query (make-query :id (format nil "~a" (uuid:make-v1-uuid))
231
                             :sse-expression (parse-sparql "
232
 select count(*)
233
 where {
234
   graph ?g {
235
     ?s <urn:dydra:timestamp> ?timestamp .
236
     ?s <urn:dydra:query_time> ?query_time .
237
     ?s <urn:dydra:algebra_operations> ?algebra_operations .
238
     ?s <urn:dydra:user_id> ?user_tag .
239
  } }")
240
                             :content-type mime:application/sparql-results+json
241
                             :repository-id "statistics/de4.dydra.com")
242
              *trace-output*)
243
 
244
 (pipe-query (make-query :id (format nil "~a" (uuid:make-v1-uuid))
245
                             :sse-expression (parse-sparql "
246
 select count(?p) where {graph ?g {?s <urn:dydra:timestamp> ?time .
247
                                  ?s ?p ?o . } }")
248
                             :content-type mime:application/sparql-results+json
249
                             :repository-id "statistics/de4.dydra.com")
250
              *trace-output*)
251
 
252
 
253
 (pipe-query (make-query :id (format nil "~a" (uuid:make-v1-uuid))
254
                             :sse-expression (parse-sparql "select count(*) where {?s ?p ?o}")
255
                             :content-type mime:application/sparql-results+json
256
                             :repository-id "statistics/de4.dydra.com")
257
              *trace-output*)
258
 
259
 (pipe-query (make-query :id (format nil "~a" (uuid:make-v1-uuid))
260
                             :sse-expression (parse-sparql "select count(*) where {graph ?g {?s <urn:dydra:start_time> ?o } }")
261
                             :content-type mime:application/sparql-results+json
262
                             :repository-id "statistics/de4.dydra.com")
263
              *trace-output*)
264
 
265
 (pipe-query (make-query :id (format nil "~a" (uuid:make-v1-uuid))
266
                             :sse-expression (parse-sparql "select ?o where {graph ?g {?s <urn:dydra:start_time> ?o } } order by desc(?o) limit 2")
267
                             :content-type mime:application/sparql-results+json
268
                             :repository-id "statistics/de4.dydra.com")
269
              *trace-output*)
270
 
271
 (pipe-query (make-query :id (format nil "~a" (uuid:make-v1-uuid))
272
                             :sse-expression (parse-sparql "
273
 select (count (?o) as ?count) where {graph ?g {?s <urn:dydra:start_time> ?o . filter (?o > '2013-03-12T12:00:00Z'^^<http://www.w3.org/2001/XMLSchema#dateTime>) } }")
274
                             :content-type mime:application/sparql-results+json
275
                             :repository-id "statistics/de4.dydra.com")
276
              *trace-output*)
277
 
278
 (pipe-query (make-query :id (format nil "~a" (uuid:make-v1-uuid))
279
                             :sse-expression (parse-sparql "
280
 select count(*) where {graph ?g {?s <urn:dydra:start_time> ?time .
281
                                        filter (?time > '2013-03-12T12:00:00Z'^^<http://www.w3.org/2001/XMLSchema#dateTime>)
282
                                        filter (?time < '2013-03-18T17:00:00Z'^^<http://www.w3.org/2001/XMLSchema#dateTime>)
283
 
284
                                    } }")
285
                             :content-type mime:application/sparql-results+json
286
                             :repository-id "statistics/de4.dydra.com")
287
              *trace-output*)
288
 
289
 (pipe-query (make-query :id (format nil "~a" (uuid:make-v1-uuid))
290
                             :sse-expression (parse-sparql "
291
 select distinct ?p where {graph ?g { ?s ?p ?time .
292
                                      ?s <urn:dydra:start_time> '2013-03-15T08:13:48Z'^^<http://www.w3.org/2001/XMLSchema#dateTime> .
293
                                    } }")
294
                             :content-type mime:application/sparql-results+json
295
                             :repository-id "statistics/de4.dydra.com")
296
              *trace-output*)
297
 
298
 (pipe-query (make-query :id (format nil "~a" (uuid:make-v1-uuid))
299
                             :sse-expression (parse-sparql "
300
  select (<http://dydra.com#format>(?start_time, '%Y-%m-%dT%H:%i') as ?timestamp)
301
         ?run_time ?real_time  
302
         ?match_responses ?match_requests
303
  where {graph ?g { ?taskId <urn:dydra:algebra_operations> ?algebra_operations .
304
                     ?taskId <urn:dydra:end_time> ?end_time .
305
                      ?taskId <urn:dydra:start_time> ?start_time .
306
                      ?taskId <urn:dydra:run_time> ?run_time .
307
                      ?taskId <urn:dydra:real_time> ?real_time .
308
                      ?taskId <urn:dydra:match_responses> ?match_responses .
309
                      ?taskId <urn:dydra:match_requests> ?match_requests .
310
                                      ?taskId <urn:dydra:start_time> '2013-03-15T08:13:48Z'^^<http://www.w3.org/2001/XMLSchema#dateTime> .
311
                                    } }")
312
                             :content-type mime:application/sparql-results+json
313
                             :repository-id "statistics/de4.dydra.com")
314
              *trace-output*)
315
 
316
 (pipe-query (make-query :id (format nil "~a" (uuid:make-v1-uuid))
317
                             :sse-expression (parse-sparql "
318
 select  * where {graph <http://de4.dydra.com/heltnormalt/dk-stage> { ?taskId <urn:dydra:algebra_operations> ?algebra_operations .
319
                      ?taskId <urn:dydra:end_time> ?end_time .
320
                      ?taskId <urn:dydra:start_time> ?start_time .
321
                      ?taskId <urn:dydra:run_time> ?run_time .
322
                      ?taskId <urn:dydra:real_time> ?real_time .
323
                      ?taskId <urn:dydra:match_responses> ?match_responses .
324
                      ?taskId <urn:dydra:match_requests> ?match_requests .
325
                      filter ( ?start_time >= '2013-03-15T12:00:00Z'^^<http://www.w3.org/2001/XMLSchema#dateTime>)
326
                                    } } offset 0")
327
                             :content-type mime:application/sparql-results+json
328
                             :repository-id "statistics/de4.dydra.com")
329
              *trace-output*)
330
 
331
 (pipe-usage-chart-query *standard-output*
332
                              :repository-id "statistics/de4.dydra.com"
333
                              :start-time (spocq.e:date-time "2013-03-15T12:00:00Z")
334
                              :repository <http://de4.dydra.com/heltnormalt/dk-stage>
335
                              :content-type mime:application/VND.DYDRA.SPARQL-RESULTS+HTML+D3)
336
 
337
 
338
 
339
 (main-usage-chart :repository-id "statistics/de4.dydra.com" :response-content-type mime:application/VND.DYDRA.SPARQL-RESULTS+HTML+D3
340
                          :query-parameters "start-time=2013-03-15T12:00:00Z")
341
 |#