Coverage report: /development/source/library/org/datagraph/spocq-shard/src/odbc/materialized-view.lisp
| Kind | Covered | All | % |
| expression | 632 | 2258 | 28.0 |
| branch | 26 | 172 | 15.1 |
Key
Not instrumented
Conditionalized out
Executed
Not executed
Both branches taken
One branch taken
Neither branch taken
1
;;; -*- Mode: lisp; Syntax: ansi-common-lisp; Base: 10; Package: org.datagraph.spocq.implementation; -*-3
(in-package :org.datagraph.spocq.implementation)4
;;; 20170220 now in build5
;;; (load #p"patches/sql-truncate-exp.lisp")6
;;; (load #p"patches/materialized-view.lisp")9
(:documentation "Materialized views based (odbc) sql processing."10
"Resource api definitions establish the correspondence between sparql predicates and sql relation attributes.11
This controls the translation between BGP expressions and SQL, with support for projection and equality13
The values are retained as integer term numbers, which avoids externalization/internalization overhead,14
but limits comparisons to 'same term'.15
Any value-based filters must be placed outside of the BGP.17
The process is enabled by a combination of factors18
- repository configuration or a creation request argument which specifies the class as materialized-repository;19
- by a BGP which specifies a static or dynamically bound type, single subject and constant predicates;20
- by a corresponding table in the SQL database to hold the respective class extent. (default schema: views)21
# sudo -u postgres psql "${RAILS_ENV}" 22
# create schema views;26
When enabled, an api definition plays two roles:27
- it controls materialization, whereby the api definition translates into an exhaustive optional query,28
which projects the (class x property) extensions to be inserted into the relational cache.29
views are not created on-demand and the observed creation rates is limited, as 10^3 rows per second.30
- it controls general query interpretation, in which bgp forms are translated into sql queries on the33
Resource api definitions are computed on-the-fly and retained in a repository-local cache.35
The materialization happens on a per-type basis - with allowance to include more than one class in a36
table. It is not feasible to materialize all classes into a single table as the arity is too high:37
for the example wikipedia dataset, there are several hundred properties and the full nested optional38
materialization query never completes.")41
(defparameter *materialized-view-db* nil)43
(defvar *bgp-compilation-mode* :rdf44
"Specifies the target storage form:45
:rdf compiles as (either joined or nested) triple-store scans46
:sql compiles as a query against an sql view")48
(defvar *materialized-repository-revision-id* "00000000-0000-0000-0000-111111111111")51
(defclass materialized-repository (multi-api-repository lmdb-repository)53
:initform nil :initarg :source-repository54
:accessor repository-source-repository55
:documentation "indicates the source for a materialized view.")57
:initform nil :initarg :view-database58
:accessor repository-view-database59
:documentation "Specify the database to hold the materialized view.")61
:initform 'materialized-transaction :allocation :class)63
:initform 'materialized-repository-revision :allocation :class))64
(:documentation "A materialized-repository combines a base rdf repository with a collection of65
relational tables in which the material representation of individual classes is cached.66
The collection is managed according to constant resource type identifiers - in the simple implementation67
without provision for inheritance, as specified by corresponding api definitions."))69
(defclass materialized-repository-revision (repository-revision materialized-repository)72
(defmethod initialize-instance :before ((instance materialized-repository-revision)73
&key repeat-limit repeat-interval window-interval max-revision-ordinal74
min-revision-ordinal max-revision-record min-revision-record)75
;; just declare them legal even though that have no meaning for this revision76
(declare (ignore repeat-limit repeat-interval window-interval max-revision-ordinal77
min-revision-ordinal max-revision-record min-revision-record)))79
(defclass sql-storage (rlmdb:repository)80
((rlmdb.i::transaction-class81
:initform 'rlmdb:transaction :allocation :class)86
:initarg :view-database87
:reader storage-view-database))88
(:documentation "Specialize rlmdb:repository in order to includes the meta database.89
Provide information about the sql storage"))90
(defclass postgres-storage (sql-storage)92
(defclass mysql-storage (sql-storage)96
(defclass internal-materialized-repository (materialized-repository) ;; not rdfcache99
:initform 'internal-materialized-repository-revision)101
:initform 'postgres-storage :allocation :class))102
(:documentation "The materialization variation where the relation contains term numbers"))104
(defclass internal-materialized-repository-revision (materialized-repository-revision internal-materialized-repository)107
(defmethod compute-instance-model ((repository internal-materialized-repository) (as t) &rest args)108
(declare (ignore args))109
`(("@type" . "repository")110
("name" . ,(repository-name repository))111
("location" . ,(resource-uri repository))112
("identifier" . ,(repository-identifier repository))113
("description" . ,(repository-description repository))))115
(defclass view-repository (materialized-repository)117
:initarg :view :initform nil118
:reader get-materialized-repository-view :writer (setf materialized-repository-view))120
:initarg :view-name :initform nil121
:accessor materialized-repository-view-name)))123
(defclass view-database-repository (view-repository)125
:initarg :view-table-name126
:reader repository-view-table-name)))129
(defclass internal-view-repository (view-database-repository internal-materialized-repository)132
:initform 'internal-view-repository-revision))133
(:documentation "extend internal materialized repository to combine it with a specific view.134
This adds the dimensions for the sql query - both the projection attributes and the request parameters.135
It changes the execution not to compile the query but instead to execute a select directly for136
the given arguments."))138
(defmethod repository-is-view ((repository internal-view-repository))141
(defmethod compute-instance-model ((repository internal-view-repository) (as t) &rest args)142
(declare (ignore args))143
`(("@type" . "repository")144
("name" . ,(repository-name repository))145
("location" . ,(resource-uri repository))146
("identifier" . ,(repository-identifier repository))147
("description" . ,(repository-description repository))))149
(defclass internal-view-repository-revision (internal-materialized-repository-revision internal-view-repository)152
(defclass external-materialized-repository (materialized-repository)155
:initform 'external-materialized-repository-revision)157
:initform 'postgres-storage :allocation :class))158
(:documentation "The materialization variation where the relation contains external database values"))160
(defclass external-materialized-repository-revision (materialized-repository-revision external-materialized-repository)164
(defgeneric view-repository-p (designator)165
(:method ((repository repository))166
(view-repository-p (repository-id repository)))167
(:method ((repository view-repository))169
(:method ((object t))171
(:method ((repository-id string))172
(let ((id-type (repository-id-type repository-id)))173
(when (and id-type (subtypep id-type 'view-repository))175
(:method ((location pathname))178
(pushnew (list *class.internal-view-repository* (cl-ppcre:create-scanner "[^/]+/[^_]__[^_]+__view"))179
*repository-id-type-map*182
(defun attribute-combinations (seq)184
(flet ((note-combination (c)185
(unless (loop for known in result187
when (null (set-exclusive-or c known))189
(push (copy-list c) result))))190
(declare (dynamic-extent #'note-combination))191
(dotimes (i (length seq))192
(alexandria:map-combinations #'note-combination seq :length (1+ i))))194
;;; (attribute-combinations '(a s d))196
(defgeneric compute-repository-view-table-name (repository account-name repository-name view-name)197
(:method ((repository view-database-repository) account-name repository-name view-name)198
(format nil "~a_~a_~a" account-name repository-name view-name)))201
(defmethod initialize-instance :before ((instance view-database-repository) &rest initargs)205
;;; !!! this ignores the source repository which may have been specififed in an admin request207
(defmethod initialize-instance ((instance view-database-repository) &rest initargs208
&key (id (error "id is required"))209
(source-repository nil) (view-name nil view-name-s))210
(let ((source-repository-name nil)213
(cond ((and source-repository view-name-s)214
(multiple-value-bind (account-name repository-name) (parse-repository-id source-repository)215
(setf source-repository-name repository-name)216
(setf source-account account-name)))217
((and (setf source-repository (rlmdb:get-metadata-property id "source-repository"))218
(setf view-name (rlmdb:get-metadata-property id "view-name")))219
(multiple-value-bind (id-account-name id-repository-name) (parse-repository-id source-repository)220
(setf source-repository-name id-repository-name)221
(setf source-account id-account-name)))223
(multiple-value-bind (id-account-name id-repository-name id-view-name)224
(parse-view-repository-id id)225
(unless (and id-account-name id-repository-name id-view-name)226
(spocq.e:request-error "Invalid view repository id ~s." id))227
(setf source-account id-account-name)228
(setf source-repository-name id-repository-name)229
(unless view-name-s (setf view-name id-view-name))230
(setf source-repository231
(make-repository-id :account-name id-account-name :repository-name id-repository-name)))))232
(setf table-name (compute-repository-view-table-name instance234
source-repository-name236
(apply #'call-next-method instance237
:source-repository source-repository238
:account (account source-account)239
:view-database (spocq::make-postgresql-uri :ID NIL240
:USER *postgres-user*241
:PASSWORD *postgres-password*242
:AUTHORITY *postgres-authority*244
:DATABASE *MYSQL-DATABASE*245
:PARAMETERS `((:TABLE . ,table-name)246
(:SCHEMA . "views")))248
:view-table-name table-name251
(defmethod repository-revision-write-date ((repository materialized-repository) (revision t))254
(defgeneric materialized-repository-view (repository)256
"Recompute the view and cache on each reference.257
Otherwise a change to the view text is not reflected in regenerated258
materialized cache content.")259
(:method ((repository view-repository))260
(or (get-materialized-repository-view repository)261
(setf (materialized-repository-view repository)262
(compute-materialized-repository-view repository)))))264
(defgeneric compute-materialized-repository-view (repository)265
(:method ((repository view-repository))266
(let* ((view-sparql (or (repository-view (repository-source-repository repository)267
(materialized-repository-view-name repository))268
(spocq.e:resource-not-found-error :identifier (repository-id repository))))269
(view-parameters (sparql-query-parameters view-sparql)) ;; operates on text270
(index-parameters (attribute-combinations view-parameters)))271
(unless view-parameters272
(spocq.e:request-error "A materialized view requires parameters: ~a:~%~a"273
(repository-id repository) view-sparql))274
(let ((view (make-internal-materialized-view :name (materialized-repository-view-name repository)275
:repository (repository (repository-source-repository repository))277
:parameters index-parameters)))278
(unless (null (set-difference view-parameters (view-dimensions view)))279
(spocq.e:request-error "A materialized view must project all parameters: ~a:~%~a"280
(repository-id repository) view-sparql))283
;;; (compute-materialized-repository-view (gethash "openrdf-sesame/foaf__types__view" *repositories*))286
(defmethod initialize-repository-storage ((repository-id string) (prototype view-repository)287
&key source-repository view-name)289
(let ((account-name nil)290
(repository-name nil))291
(multiple-value-bind (id-account-name id-repository-name id-view-name)292
(parse-view-repository-id repository-id)293
(cond (id-account-name294
(setf account-name id-account-name295
repository-name id-repository-name)296
(if (and view-name id-view-name)297
(unless (equalp view-name id-view-name)298
(log-warn "initialize-repository-storage: ~s: ~s supersedes ~s"299
repository-id view-name id-view-name))300
(setf view-name id-view-name)))302
(multiple-value-setq (account-name repository-name)303
(parse-repository-id repository-id)))))304
(cond ((and account-name repository-name source-repository view-name)305
(rlmdb:put-metadata repository-id306
(list "source-repository" source-repository307
"view-name" view-name)))309
(spocq.e:request-error "Invalid view repository id ~s . ~s" repository-id view-name)))312
(defmethod repository-wildcard-term ((repository external-materialized-repository))315
(defmethod repository-object-term-number ((repository materialized-repository) (object t))316
(rlmdb:value-term-number object))318
(defmethod repository-term-number-object ((repository materialized-repository) (id t))319
(rlmdb:term-number-value id))321
(defmethod repository-write-date ((repository materialized-repository))324
(defmethod repository-write-date ((repository internal-materialized-repository))325
(let ((timestamp (rlmdb:get-metadata-timestamp repository)))327
(timeline-location-universal-time timestamp))))331
(defclass materialized-view (view)334
(defgeneric view-construct-pattern (view)335
(:method ((view materialized-view))336
(case (view-operation view)337
((spocq.a:|select| spocq.a:|distinct|) nil)338
(spocq.a:|construct| (third (view-sse-expression view)))339
(t (error "invalid materialized-view: ~a: ~a" (view-name view) (view-operation view))))))341
(defclass external-materialized-view (materialized-view)343
(:documentation "An external materialized view mediates view access to an external materialized repository"))345
(defclass internal-materialized-view (materialized-view)347
(:documentation "An internal materialized view mediates access to an internal materialized repository"))349
(defmethod initialize-instance ((instance internal-materialized-view)350
&key (parameters nil) name query)352
(spocq.e:request-error "A materialized view requires parameters: ~a:%~a" name query))356
(defun make-internal-materialized-view (&rest args)357
(apply #'make-instance 'internal-materialized-view args))359
(defstruct (view-solution-generator (:include solution-generator))361
(defmethod copy-generator :around ((generator view-solution-generator) &key (view (view-solution-generator-view generator))363
(apply #'call-next-method generator :view view))364
(defmethod copy-generator ((generator view-solution-generator) &rest args)365
(apply #'make-view-solution-generator args))369
(defclass materialized-transaction (rdfcache-transaction)372
(defclass odbc-api (materialized-resource-api)374
:initarg :predicates :initform nil375
:reader resource-api-predicates))377
"An odbc api specializes a materialized resource api to mediate an odbc relation.378
the attribute definitins are drawn from the relation schema metadata and bgps are compiled379
for external materializations to convert on the basis of those types."))381
(defmethod initialize-instance ((instance odbc-api) &rest initargs382
&key input-definitions383
(predicates (mapcar #'first input-definitions)))384
(apply #'call-next-method instance385
:predicates predicates388
(defun sql-internal-type (external-type &optional constraints)389
(cond ((equalp external-type "timestamp") 'spocq:date-time)390
((equalp external-type "date") 'spocq:date-time)391
((equalp external-type "uuid") 'spocq:uuid)392
((equalp external-type "numeric") 'number)393
((equalp external-type "text") 'string)394
(t (clsql-sys::compute-lisp-type-from-specified-type external-type constraints))))396
(defgeneric decode-material-value (string internal-type db external-type)397
(:method ((null null) internal-type db external-type)398
(spocq:make-unbound-variable nil))399
(:method ((string string) internal-type db external-type)401
(spocq:uuid (intern-uuid string))402
(spocq:date-time (when (and (> (length string) 11) (case (char string 10) ((#\space #\t) t) (t nil)))403
(setf (char string 10) #\T))404
(spocq.e:date-time string))405
(t (clsql-sys::read-sql-value string internal-type db external-type))))406
(:method ((value t) (internal-type t) (db t) (external-type t))409
(defgeneric intern-material-value (transaction term-value)410
(:documentation "yield the respective term number")411
(:method ((transaction rdfcache-transaction) value)412
(rdfcache-object-term-number transaction value)))417
(defmacro with-materialized-repository-database ((store location &rest args) repository &body body)418
(let ((op (gensym "materialized-op-")))419
`(flet ((,op (,store ,location)420
(declare (ignorable ,store ,location))422
(declare (dynamic-extent #',op))423
(call-with-materialized-repository-database #',op ,repository ,@args))))425
(defgeneric call-with-materialized-repository-database (function repository &key verbose)426
(:method ((function t) (repository materialized-repository) &rest args)427
(apply #'call-with-materialized-repository-database function (repository-view-database repository) args))429
(:method ((function t) (database-designator spocq:mysql-uri) &key (verbose *sql-verbose*))430
(clsql:with-database (*materialized-view-db*431
(list (spocq:mysql-uri-authority database-designator)432
(spocq:mysql-uri-database database-designator)433
(spocq:mysql-uri-user database-designator)434
(spocq:mysql-uri-password database-designator))435
:if-exists :new ;; :old436
:database-type :mysql438
(flet ((sql-operation ()439
(funcall function *materialized-view-db* database-designator)))441
(unwind-protect (progn (clsql:start-sql-recording :database *materialized-view-db*)443
(clsql:stop-sql-recording :database *materialized-view-db*))446
(:method ((function t) (database-designator spocq:postgresql-uri) &key (verbose *sql-verbose*))447
(let ((schema (spocq:postgresql-uri-schema database-designator)))448
(clsql:with-database (*materialized-view-db*449
(list (spocq:postgresql-uri-authority database-designator)450
(spocq:postgresql-uri-database database-designator)451
(spocq:postgresql-uri-user database-designator)452
(spocq:postgresql-uri-password database-designator))453
:if-exists :new ;; :old454
:database-type :postgresql-socket456
(flet ((sql-operation ()458
(clsql-sys:database-execute-command (format nil "SET search_path TO ~a" schema)459
*materialized-view-db*))460
(funcall function *materialized-view-db* database-designator)))462
(unwind-protect (progn (clsql:start-sql-recording :database *materialized-view-db*)464
(clsql:stop-sql-recording :database *materialized-view-db*))465
(sql-operation)))))))469
(defmethod resolve-repository-revision-id ((repository materialized-repository) &key revision if-does-not-exist)470
(declare (ignore if-does-not-exist))473
(unless (or (null revision) (string-equal revision "HEAD"))474
(error "repository revision not found: ~s ~s" repository revision))475
*materialized-repository-revision-id*)478
#+(or) ; these must allow for read/write transaction in order to intern external terms from the external source479
(defmethod transaction-open ((transaction materialized-transaction) &rest args)480
(declare (ignore args) (dynamic-extent args))483
(defmethod transaction-close ((transaction materialized-transaction) (disposition t))489
(defmethod compute-repository-revision ((repository materialized-repository) (revision string) &rest args &key491
;; always specific a materialized revision, to allow other methods to observe themode492
(revision-class 'materialized-repository-revision))493
"For a materialized repository, specialize the revision instance class to494
introduce materialization in the query processing."495
(declare (ignore reference if-does-not-exist))496
(apply #'call-next-method repository revision497
:revision-class revision-class501
(defmethod repository-resource-api :around ((revision t) name)502
(print (compute-applicable-methods #'repository-resource-api (list revision name)))505
(defmethod repository-resource-api ((revision materialized-repository-revision) name)506
(repository-resource-api (repository-revision-reference revision) name))507
(defmethod repository-resource-apis ((revision materialized-repository-revision))508
(repository-resource-apis (repository-revision-reference revision)))509
(defmethod repository-view-database ((revision materialized-repository-revision))510
(repository-view-database (repository-revision-reference revision)))513
(defmethod repository-resource-api ((repository materialized-repository) (name null))514
(repository-resource-api repository (repository-name repository)))516
(defmethod repository-resource-api ((repository materialized-repository) (name t))517
(or (call-next-method)518
(setf (repository-resource-api repository name)519
(resource-retrieve-resource-api repository name))))521
(defgeneric repository-database-type (repository)522
(:method ((repository materialized-repository))523
(repository-database-type (repository-view-database repository)))524
(:method ((repository spocq:postgresql-uri))526
(:method ((repository spocq:mysql-uri))529
(defmethod spocq.e:repository-clear-graph ((repository materialized-repository) (graph (eql :all))530
&key &allow-other-keys)531
(clear-materialized-view (repository-view-database repository) (repository-name repository))532
(update-materialized-view repository (resource-retrieve-resource-api repository (repository-name repository))))535
(defmethod spocq.e:repository-clear-graph ((repository internal-materialized-repository) (graph (eql :all))536
&key &allow-other-keys)537
"clearing all graphs in an internal materialized epository causes it to be regenerated.538
compute the sql update respective the view query, generate it into a temporary table,539
drop any existing table and swap the new table into place."540
(let* ((view (materialized-repository-view repository)))541
(update-materialized-view repository view)))544
#+(or) ;; if it is necessary to construct the view in stages, rename and replace it.545
(defmethod spocq.e:repository-clear-graph ((repository internal-materialized-repository) (graph (eql :all))546
&key &allow-other-keys)547
"clearing all graphs in an internal materialized epository causes it to be regenerated.548
compute the sql update respective the view query, generate it into a temporary table,549
drop any existing table and swap the new table into place."550
(let* ((view (materialized-repository-view repository))551
(view-name (view-name view))552
(tmp-view (clone-instance view :name (concatenate 'string view-name "_tmp"))))553
(setf (view-query view) nil554
(view-dimensions view) nil)555
(let* ((tmp-name (view-name tmp-view))556
(table-name (clsql-sys::sql-expression :table view-name))557
(tmp-table-name (clsql-sys::sql-expression :table tmp-name))558
(db (repository-view-database repository)))561
(update-materialized-view repository tmp-view)562
(clsql:with-transaction (:database db)563
(when (clsql:table-exists-p table-name :database db :owner :all)564
(clsql:drop-table table-name :database db))565
(clsql:rename-table tmp-table-name table-name :database db)))))567
(defmethod spocq.e:repository-clear-graph ((repository materialized-repository) (graph t)568
&key &allow-other-keys)572
(defmethod clear-materialized-view ((designator spocq:mysql-uri) (table string))573
(with-materialized-repository-database (schema-db location) designator574
(declare (ignore location))575
(let* ((query (format nil "truncate ~a" table)))576
(clsql-sys:query query :database schema-db))))579
(defmethod clear-materialized-view ((designator spocq:postgresql-uri) (table string))580
(with-materialized-repository-database (schema-db location) designator581
(declare (ignore location))582
(let* ((query (format nil "truncate ~a" table)))583
(clsql-sys:query query :database schema-db))))586
(defgeneric compute-materialization-sparql (api)587
(:documentation "Generate a select which enumerates all resources from a graph, respective the api.588
Use the required properties as the base and extend that with all optionals.589
Project all object variables plus that for the subject.590
The projection order is (resource . (ordered by attribute name)), in order that the insert not need to shuffle.")591
(:method ((api resource-api))592
(let* ((input-defs (resource-api-input-definitions api))593
(output-defs (resource-api-output-definitions api))594
(full-defs (sort (remove-duplicates (append input-defs output-defs) :test #'equalp :key #'first)597
(variables (loop for (predicate attribute . nil) in full-defs598
collect (case predicate599
(|rdf|:|type| '?::_type)600
(t (cons-variable attribute)))))601
(node-variable (resource-api-resource-variable api))602
;; augment required properties with the type to ensure a left-join base603
(requireds (loop for (predicate nil . properties) in full-defs604
for optional = (getf properties :optional nil)605
for variable in variables606
unless optional collect `(spocq.a:|triple| ,node-variable ,predicate ,variable)))607
(optionals (loop for (predicate nil . properties) in full-defs608
for optional = (getf properties :optional nil)609
for variable in variables610
when optional collect `(spocq.a:|triple| ,node-variable ,predicate ,variable)))611
(read-query `(spocq.a:|bgp| ,@requireds)) )613
"API ~a must have a required base to be materialized." api)614
(loop for optional in optionals615
do (setf read-query `(spocq.a:|leftjoin| ,read-query (spocq.a:|bgp| ,optional))))616
(setf read-query `(spocq.a:|select| ,read-query ,(cons node-variable variables)))619
(defgeneric compute-materialized-sparql (api)620
(:documentation "Generate a select which enumerates all resources from a relation, respective the api.621
Include all properties in a single bgp.622
Project all object variables plus that for the subject resource.623
The projection order is (resource . (ordered by attribute name)), in order that the insert not need to shuffle.")624
(:method ((api resource-api))625
(let* ((input-defs (resource-api-input-definitions api))626
(output-defs (resource-api-output-definitions api))627
(full-defs (sort (remove-duplicates (append input-defs output-defs) :test #'equalp :key #'first)630
(variables (loop for (predicate attribute . nil) in full-defs631
collect (case predicate632
(|rdf|:|type| '?::_type)633
(t (cons-variable attribute)))))634
(node-variable (resource-api-resource-variable api))635
;; no type constraint636
(requireds (loop for (predicate nil &key optional &allow-other-keys) in full-defs637
for variable in variables638
unless optional collect `(spocq.a:|triple| ,node-variable ,predicate ,variable)))639
(optionals (loop for (predicate nil &key optional &allow-other-keys) in full-defs640
for variable in variables641
when optional collect `(spocq.a:|triple| ,node-variable ,predicate ,variable)))642
(read-query `(spocq.a:|bgp| ,@(append requireds optionals))))643
(setf read-query `(spocq.a:|select| ,read-query ,(cons node-variable variables)))647
(defgeneric resource-api-attributes (api db)648
(:documentation "return sql attribute instances for the api field definitions.")649
(:method ((repository repository) (db clsql:database))650
(resource-api-attributes (repository-resource-api repository '?::_resourceId) db))651
(:method ((api resource-api) (db clsql:database))652
(let* ((input-defs (resource-api-input-definitions api))653
(full-defs (remove-duplicates (append input-defs (resource-api-output-definitions api))654
:test #'equalp :key #'first))655
(attributes (sort (loop for (nil attribute . nil) in full-defs collect attribute) #'string-lessp)))656
(loop for attribute in attributes657
collect (list (clsql-sys::sql-expression :attribute (clsql-sys::database-identifier attribute db))660
(:method ((dimensions list) (db clsql:database))661
(loop for attribute in dimensions662
collect (list (clsql-sys::sql-expression :attribute (clsql-sys::database-identifier attribute db))666
(defgeneric ensure-materialized-view (store api)667
(:documentation "Ensure that the cache database includes a view which represents the extension of the668
given resource as a relation. If no table corresponds to the source repository's view, create one.669
values : string : the view name")670
(:method ((repository materialized-repository) (named-for t))671
(ensure-materialized-view repository (repository-resource-api repository named-for)))672
(:method ((repository materialized-repository) (api materialized-resource-api))673
(let ((view-db-designator (repository-view-database repository))674
(view-name (resource-api-view-name api)))675
(with-materialized-repository-database (view-db location) view-db-designator676
(unless (clsql:table-exists-p view-name :database view-db :owner :all)677
(destructuring-bind (table-name attributes) (rest (compute-materialized-view-definition repository api view-db))678
(clsql:create-table table-name attributes682
(:method ((repository materialized-repository) (view view))683
"create the table. create also any indices indicated for possible parameters."684
(let* ((view-db-designator (repository-view-database repository))685
(table-namestring (qualified-view-name view))686
;;(table-name (clsql-sys::sql-expression :table table-namestring))687
(view-parameters (view-parameters view)))688
(flet ((compute-index-name (table-name parameter-set)689
(format nil "~a_~{_~a~}" table-name (mapcar #'string-downcase parameter-set))))690
(with-materialized-repository-database (view-db location) view-db-designator691
(destructuring-bind (sql-table-name attributes) (rest (compute-materialized-view-definition repository view view-db))692
(unless (when (clsql:table-exists-p sql-table-name :database view-db :owner :all)693
;; if the attributes and the indeces matche the new definition, keep the table694
;; otherwise drop the table and its indices695
(cond ((and (every #'string-equal696
(mapcar #'(lambda (attr-def)697
(CLSQL-SYS::UNESCAPED-DATABASE-IDENTIFIER (clsql:sql-name (first attr-def))))699
(clsql:list-attributes sql-table-name :database view-db :owner :all))700
(let ((indexes (clsql:list-indexes :on sql-table-name :database view-db :owner :all))701
(parameter-indexes (loop for parameter-set in view-parameters702
collect (compute-index-name table-namestring parameter-set))))703
(null (set-exclusive-or indexes parameter-indexes :test #'string-equal)))))705
(clsql:drop-table sql-table-name :database view-db)707
(log-notice "ensure-materialized-view: create-table: ~s, db: ~s" table-namestring view-db)708
(clsql:create-table sql-table-name attributes :database view-db)709
;; if the generation time becomes an issue, this step should be delayed until the table is filled.710
;; likewise, drop and later recreate the index around a regeneration.711
;; see https://stackoverflow.com/questions/12206600/how-to-speed-up-insertion-performance-in-postgresql712
(loop for index-dimensions in view-parameters713
for index-attribute-names = (mapcar #'string-downcase index-dimensions)714
for index-name = (compute-index-name table-namestring index-attribute-names)715
do (clsql:create-index index-name :on table-namestring :attributes index-attribute-names716
:database view-db))))717
table-namestring)))))719
(defgeneric probe-materialized-view (store api)720
(:documentation "Test whether the database includes a view which represents the extension of the721
given resource as a relation. 723
(:method ((repository materialized-repository) (named-for t))724
(probe-materialized-view repository (repository-resource-api repository named-for)))725
(:method ((repository materialized-repository) (api materialized-resource-api))726
(let ((db (repository-view-database repository))727
(view-name (resource-api-view-name api)))728
(clsql:table-exists-p view-name :database db :owner :all))))730
(defgeneric compute-materialized-view-definition (store api db)731
(:documentation "Compute the table expression for a view which represents the extension of the732
given resource as a relation.733
In use, thiese are not evaluated. Their components are passed as argument to the operator.")734
(:method ((repository materialized-repository) (named-for t) (db t))735
(compute-materialized-view-definition repository (repository-resource-api repository named-for) db))736
(:method ((repository materialized-repository) (api materialized-resource-api) (db CLSQL-SYS:DATABASE))737
(let* ((view-name (resource-api-view-name api))738
(columns (resource-api-attributes api db)))739
`(clsql:create-table ,(clsql-sys::sql-expression :table view-name)740
,(cons (list (clsql-sys::sql-expression :attribute (clsql-sys::database-identifier "_resourceId" db))744
(:method ((repository materialized-repository) (view view) (db CLSQL-SYS:DATABASE))745
(let* ((table-namestring (string-downcase (qualified-view-name view)))746
(view-dimensions (view-dimensions view))747
(columns (resource-api-attributes view-dimensions db)))748
`(clsql:create-table ,(clsql-sys::sql-expression :table table-namestring)752
(defgeneric update-materialized-view (database api &key channel-size-limit field-page-length)753
(:documentation "Given a repository as a materialized wrapper, update its relation.754
Generate and run the respective materialization query, which enumerates the respective class,755
Generate the corresponding sql to insert the content into the truncated view table.")756
(:method ((repository-id string) (named-for t) &rest args)757
(apply #'update-materialized-view (repository repository-id) named-for args))758
(:method ((repository internal-materialized-repository) (named-for t) &rest args)759
(apply #'update-materialized-view repository (repository-resource-api repository named-for) args))761
(:method ((repository internal-materialized-repository) (api materialized-resource-api) &key762
((:channel-size-limit *channel-size-limit*) 2)763
((:field-page-length *field-page-length*) 32))764
"enumerate the api resourece extension from the repository and insert each765
solution into the material view"766
(let* ((database (repository-view-database repository))767
(read-query (compute-materialization-sparql api))768
(view (ensure-materialized-view repository api))770
(autocommit (clsql-sys:database-autocommit database))771
(*bgp-compilation-mode* :rdf))772
(clsql:truncate-table view)773
(flet ((numeric-solutions (field)774
;;;INSERT INTO your table VALUES (1,2), (5,5), ...;775
(let ((values (loop with length = (array-dimension field 0)776
with width = (array-dimension field 1)777
for row from 0 below length779
collect (loop for column from 0 below width collect (aref field row column)))))780
(clsql:insert-values :into view :values values :database database))782
(declare (dynamic-extent #'numeric-solutions))784
(progn (clsql:set-autocommit nil :database database)785
;; run the materialization query including all extensions786
;;!!! this will cause problems if it requires non-standard metadata, eg. for entialment787
(run-sparql read-query788
:repository-id (repository-id repository)789
:solution-handler #'numeric-solutions790
;; without this, it materializes the entire repository respective the api projection791
:dynamic-bindings (list '(?::_type) (resource-api-resource api))792
;; :library-path ? where would that be found ?793
:agent (system-agent))794
(clsql:set-autocommit autocommit :database database)))797
(:method ((repository internal-materialized-repository) (view view) &key798
((:channel-size-limit *channel-size-limit*) 2)799
((:field-page-length *field-page-length*) 512))800
"Ensure that the view table exists.801
If it does not yet exist, create it.802
If it exists, check that the schema agrees."803
(let* ((view-db-designator (repository-view-database repository))804
(view-query (view-query view))806
(case (view-operation view)807
((spocq.a:|select| spocq.a:|distinct|) (view-sse-expression view))808
(spocq.a:|construct| (second (view-sse-expression view)))809
(t (error "invalid materialized-view: ~a: ~a" (view-name view) (view-operation view)))))810
(view-dimensions (view-dimensions view))811
(table-namestring (qualified-view-name view))814
(insert-row-limit 4096)816
(*bgp-compilation-mode* :rdf)818
(ensure-materialized-view repository view)819
(with-materialized-repository-database (view-db location) view-db-designator820
(flet ((numeric-solutions (field)821
(unless (= (length view-dimensions) (array-dimension field 1))822
(error "dimension mismatch: ~a" (list (length view-dimensions) (array-dimension field 1))))823
;;;INSERT INTO your table VALUES (1,2), (5,5), ...;824
(loop with length = (array-dimension field 0)825
with width = (array-dimension field 1)826
for row from 0 below length829
(incf insert-row-count)830
(push (loop for column from 0 below width collect (aref field row column)) values)831
(when (>= insert-row-count insert-row-limit)832
(clsql:insert-values :into table-namestring :values values :database view-db)834
(setf insert-row-count 0))))835
(when (zerop (mod count #.(* 32 512)))839
(declare (dynamic-extent #'numeric-solutions))840
(log-notice "update-materialized-view: ~s" table-namestring)841
(clsql:with-transaction (:database view-db)842
(clsql:truncate-table table-namestring :database view-db)845
(setf autocommit (clsql:set-autocommit nil :database view-db))846
;;!!! see above re extensions847
(apply #'run-sparql sse-expression848
:request-content view-query849
:repository-id (repository-source-repository repository)850
:solution-handler #'numeric-solutions851
;; without this, it materializes the entire repository respective the api projection853
:agent (system-agent) ;; use system agent854
;; take the dataset freom the environment or from the query text855
(case *dataset-source*856
(:query (view-options view))858
(list* :dataset-graphs *dataset-graphs* (view-options view)))))859
(when (plusp insert-row-count)860
(clsql:insert-values :into table-namestring :values values :database view-db)))861
(clsql:set-autocommit autocommit :database view-db)))862
(log-notice "update-materialized-view: complete: ~a x~a" table-namestring count)864
(view-dimensions view)))))))866
(defmethod compile-query-for-repository ((query query) (repository internal-view-repository))867
"Use the query expression dimensions and its request arguments to control the sql retrieval"868
(let* ((view (materialized-repository-view repository))869
(view-name (view-name view))870
(table-namestring (qualified-view-name view))871
(arguments (query-dynamic-bindings query))872
(query-dimensions (expression-dimensions (query-sse-expression query)))873
(view-dimensions (view-dimensions view))874
(projection-dimensions (sort (if query-dimensions875
(intersection view-dimensions query-dimensions)876
(copy-list view-dimensions))878
(offset (shiftf *response-offset* 0)) ;; these should apply just when this is the outermost query879
(limit (shiftf *response-limit* nil))880
(where (let ((constraints (loop for dimension in (first arguments)881
for dca = (string-downcase dimension)882
for value in (rest arguments)883
;; constraints are between an argument dimension and its value884
collect (clsql:sql-operation '=885
(sql-attribute dca table-namestring)886
(rdfcache-object-term-number *transaction* value)))))887
;; (print (list :constraints constraints))888
(if (rest constraints)889
(reduce #'(lambda (c1 c2) (clsql:sql-operation 'and c1 c2)) constraints)890
(first constraints))))892
(sql (apply #'clsql:sql-operation 'select894
;; output dimension map is orded by dimension895
(loop for attribute in projection-dimensions896
collect (sql-attribute attribute table-namestring))897
(list* :from (clsql:sql-expression :table table-namestring)900
(append (when limit `(:limit ,limit))901
(when offset `(:offset ,offset)))))))902
(sql `(clsql:sql-operation 'select903
,@(loop for attribute in projection-dimensions904
for dca = (string-downcase attribute)905
collect `(sql-attribute ',dca ,table-namestring))906
:from (clsql:sql-expression :table ,table-namestring)909
,@(when limit `(:limit ,limit))910
,@(when offset `(:offset ,offset))))911
(view-clause `(spocq.a:|view| (materialized-repository-view (repository ,(repository-id repository)))912
,sql ,projection-dimensions))913
(pattern (view-construct-pattern view))915
;; the initialization function generates just the sql select916
;; as per the view attributes and the query bindings918
(with-task-environment (:task query :normal-disposition :continue)920
`(spocq.a:|construct| ,view-clause ,pattern)924
(with-task-environment (:task query :normal-disposition :continue)926
(log-debug "compile-query view ~a: ~s (~s)" (task-id query) view-name table-namestring)927
(setf (query-patterns query) nil)928
(setf (task-operator-count query) 1)929
;; compile the query form and collect the agp instances as a side-effect930
(let ((initialization-function 931
(spocq-compile lambda-expression)))932
(setf (task-initialization-function query) initialization-function)933
(generate-accounting-note :compile :task query)934
(values initialization-function935
lambda-expression))))))938
(defmacro spocq.a:|view| (view sql dimensions)939
`(spocq.e::view ,view ,sql ',dimensions))941
(defun spocq.e::view (view sql dimensions)942
(view-generator view sql dimensions))944
(defun view-generator (view query projection-dimensions &key end start)945
(let ((result-channel (make-channel :name (list 'spocq.a:|view| (task-id *query*))946
:dimensions projection-dimensions947
:size (effective-channel-size :start start :end end)948
:page-length (effective-page-length :start start :end end))))949
(make-view-solution-generator :operator 'spocq.a:|view|951
:dimensions projection-dimensions952
:expression (list #'run-materialized-view-thread result-channel query)953
:channel result-channel956
(defun run-materialized-view-thread (destination query)957
(let* ((*thread-operations* (cons (list 'spocq.a::|view| (repository-id *repository*)) *thread-operations*)))958
(process-materialized-view destination *repository* query))962
(defgeneric process-materialized-view (destination view-repository sql-query)963
(:documentation "Given a destination and a materialized source repository, 964
initiate the delegated query, accept the results and emit them to the destination.965
DESTINATION : (or channel function)966
VIEW-REPOSITORY : internal-materialized-repository967
SQL-QUERY : (or string sql-query-object)")969
(:method ((solution-destination array-page-channel) (repository internal-materialized-repository) sql-query)970
(let* ((view-db-designator (repository-view-database repository))971
(repository-id (repository-id repository))972
(revision-id (repository-revision-id *task*))973
(transaction *transaction*)974
(channel-dimensions (channel-dimensions solution-destination))976
(result-page-length (channel-page-length solution-destination))977
(result-page-width (channel-page-width solution-destination))978
(result-index result-page-length)980
(trace-bgp view.match.start repository-id revision-id transaction channel-dimensions)982
(labels ((coerce-to-term-number (value)984
(integer ; a materialized term987
(rdfcache-object-term-number transaction (round (* value 100))))989
(rdfcache-object-term-number transaction value))))990
(collect-solution (solution)991
(trace-bgp view.collect-solution solution)992
(next-solution-location)993
(locally (declare (type (simple-array fixnum (* *)) result-page)994
(type fixnum result-index)995
(type (cons fixnum *) solution)996
(optimize (SPEED 3) (SAFETY 0)))997
(loop for column-index from 0 below result-page-width998
for value in solution999
for integer-value = (coerce-to-term-number value)1000
do (setf (aref result-page result-index column-index) integer-value))))1001
(next-solution-location ()1002
;; return a page (possible newly created) and the next free location in that page1004
(when (>= (incf result-index) result-page-length)1005
(when result-page (put-result result-page))1006
(setf result-page (new-field-page solution-destination result-page-length result-page-width)1008
(values result-page result-index))1009
(complete-solutions ()1010
(trace-bgp view.match.complete-solutions result-count)1011
(incf *match-requests* 1)1012
(incf *match-responses* result-count)1013
(log-debug "view count: ~s: solutions: ~s" repository-id result-count)1015
(let ((page-result-count (1+ result-index)))1016
(when (< page-result-count result-page-length)1018
(adjust-page result-page (list page-result-count result-page-width)))))1019
(put-result result-page))1020
(complete-field solution-destination)1021
(incf-stat *solutions-constructed* result-count)1022
(return-from :view-scan result-count))1024
(trace-bgp view.put solution-destination channel-dimensions page)1025
(put-field-page solution-destination page)1026
(unless (task-active-p *query*)1027
(complete-field solution-destination)1028
(return-from :view-scan result-count))))1029
(declare (dynamic-extent #'collect-solution))1030
(with-materialized-repository-database (view-db location) view-db-designator1031
(clsql-sys::map-query-for-effect #'collect-solution1035
(complete-solutions))))))1039
(defgeneric compute-materialized-extension (database api)1040
(:documentation "Given an repository as a materialized wrapper, return the api extension")1041
(:method ((repository internal-materialized-repository) (api-name t))1042
"Given the api name, lookup the api definitiaon and continue."1043
(let ((api (resource-api api-name)))1045
(compute-materialized-extension repository api))1047
(log-warn "unknown api: ~a" api-name)1049
(:method ((repository internal-materialized-repository) (api resource-api))1050
"enumerate the api resource extension from the repository and insert each1051
solution into the maerial view"1052
(let* ((read-query (compute-materialization-sparql api)))1053
;;!!! see above re extensions1054
(run-sparql read-query1055
:repository-id (repository-id repository)1056
:agent (system-agent)))))1058
(defun compute-resource-api-definition (repository type-resource-id1059
&key (class 'materialized-resource-api)1060
(view-name (if (variable-p type-resource-id)1061
(repository-id repository)1062
(iri-sql-name type-resource-id)))1063
(operation <http://stage.dydra.com/pg-view>)1064
(operation-class <http://www.w3.org/ns/hydra/core#PostgresView>)1065
(input-class <http://data.dydra.com/argsIn>)1066
(output-class <http://data.dydra.com/resultOut>)1067
((:bgp-compilation-mode *bgp-compilation-mode*) :rdf))1068
"Compute an api definition by enumerating the properties associated with nodes of the given type.1069
Universal properites are deemed inputs and all others are optional outputs."1070
(setf repository (repository repository))1071
;; (print (list :compute-resource-api-definition *bgp-compilation-mode*))1072
(let* ((resource-count (caar (run-sparql-internal `(spocq.a:|select|1073
(spocq.a:|bgp| (spocq.a:|triple| ?::|node| |rdf|:|type| ,type-resource-id))1074
((?::c (spocq.a:|count| ?::|node| :DISTINCT spocq.a:|distinct|))))1075
:repository-id (repository-id repository)1076
:agent (system-agent))))1077
(counts (run-sparql-internal `(spocq.a:|select|1078
(spocq.a:|bgp| (spocq.a:|triple| ?::|node| |rdf|:|type| ,type-resource-id)1079
(spocq.a:|triple| ?::|node| ?::p ?::o))1080
(:group-by (?::p) ?::p (?::c (spocq.a:|count| ?::|node| :DISTINCT spocq.a:|distinct|))))1081
:repository-id (repository-id repository)1082
:agent (system-agent)))1083
(required (loop for (property property-count) in counts1084
when (= property-count resource-count)1085
collect (list property property-count)))1086
(optional (loop for (property property-count) in counts1087
when (/= property-count resource-count)1088
collect (list property property-count))))1089
(when (plusp resource-count)1091
:resource ,type-resource-id1092
:view-name ,view-name1093
:operation ,operation1094
:operation-class ,operation-class1095
:input-class ,input-class1096
:output-class ,output-class1097
:input-definitions ; this without the resource identifier, as the subject has no property1098
,(sort (loop for (property property-count) in required1099
;; collect (list property (iri-local-part property) :selectivity 1))1100
collect (list property (iri-sql-name property) :selectivity 1))1101
#'string-lessp :key #'second)1103
,(sort (loop for (property property-count) in optional1104
collect (list property (iri-sql-name property) :optional t1105
:selectivity (float (/ property-count resource-count))))1106
#'string-lessp :key #'second)))))1108
(defun compute-resource-api (repository resource)1109
(let ((definition (compute-resource-api-definition repository resource)))1111
(apply #'make-instance definition))))1113
;;; (compute-resource-api-definition <http://schema.org/SportsTeam> :repository-id "james/dbpedia-en-short-extract")1115
;;; bgp -> view match function generation1117
(defmethod compute-agp-pattern-lambda ((agp agp) (repository materialized-repository))1118
"Inspect BGP forms for a single constant type terms which matches an api definition,1119
Where a match occurs, compile the BGP against the API definition, respective the input1120
and output variables, as an SQL form.1121
If the respective SLQL table exists and contains content, execute the query1122
directly against the SQL table."1123
(let ((pattern-forms (rest (agp-form agp)))1124
(pattern-subject nil)1127
(if (and (eq *bgp-compilation-mode* :sql)1128
(setf pattern-subject (bgp-pattern-subject pattern-forms))1129
(setf pattern-type (bgp-pattern-type pattern-forms))1130
(notany #'variable-p (bgp-pattern-predicates pattern-forms))1131
;; allow a dynamic binding exists, which should yield the term number1132
;; as well as the term value in the query bgp pattern1133
(setf api (repository-resource-api repository pattern-type))1134
(probe-materialized-view repository api))1135
(let ((lambda (compute-view-lambda repository pattern-forms1137
:graph (agp-graph agp)1138
:base-dimensions (agp-propagated-base-dimensions agp)1139
:projection-dimensions (agp-projection-dimensions agp)1140
:dataset-graphs (agp-dataset-graphs agp)1141
:variables (if (variable-p (agp-graph agp))1142
(cons (agp-graph agp) (agp-variables agp))1143
(agp-variables agp))1144
:dynamic-variables (first (query-dynamic-bindings (agp-query agp))))))1145
(when (and *trace-pattern-forms*1146
(log-level-qualifies? :trace))1147
(log-trace "agp materialized: patterns: ~a lambda ~a" pattern-forms lambda))1149
(call-next-method))))1151
(defmethod compute-repository-pattern-partitions ((repository internal-materialized-repository) (patterns t))1152
(if (eq *bgp-compilation-mode* :sql)1154
(call-next-method)))1156
(defmethod compute-match-scan-partitions ((repository internal-materialized-repository) (patterns t) &key graph)1157
(declare (ignore graph))1158
(if (eq *bgp-compilation-mode* :sql)1160
(call-next-method)))1162
(defmethod compute-repository-pattern-partitions ((repository external-materialized-repository) (patterns t))1165
(defmethod compute-match-scan-partitions ((repository external-materialized-repository) (patterns t) &key graph)1166
(declare (ignore graph))1169
(defgeneric compute-materialized-view-sql (repository api bgp-pattern-body)1170
(:documentation "Given a resource api and a bgp pattern body, generate the sql form which1171
would perform the equivalent query against the materialized view as described by the api.1172
context : (or materialized-repository resource-api)1173
bgp-pattern-body : (list statement-pattern)1174
values : sql-operation1176
it recognizes any predicates which are present in i/o mappings and use those to1177
associate constants and dynamic variables with constraints and otherwise variables1178
with projection attributes.1179
in addition, if the the subject is an unbound variable it is added to the output mapping1180
with '#resourceId', while if it is bound sip or dynamically or a constant, it is added to1181
the constraints on that attribute.")1183
(:method ((repository t) (context null) pattern-body)1184
(let* ((api (resource-api (bgp-pattern-type pattern-body))))1186
(compute-materialized-view-sql repository api pattern-body))))1188
(:method ((repository t) (context internal-materialized-repository) pattern-body)1189
(compute-materialized-view-sql repository (repository-resource-api context pattern-body) pattern-body))1191
(:method ((repository t) (context t) (pattern-body t))1192
(let* ((api (resource-api context)))1194
(compute-materialized-view-sql repository api pattern-body))))1196
(:method ((repository t) (api resource-api) (pattern-api resource-api))1197
(compute-materialized-view-sql repository api (rest (second (compute-materialized-sparql pattern-api)))))1199
(:method ((repository t) (api resource-api) (pattern-body list))1200
(let* ((patterns (remove-if-not #'elementary-bgp-statement-form-p pattern-body))1201
(subject (statement-subject (first patterns)))1202
(slice (rest (assoc 'spocq.a:|slice| pattern-body)))1203
(pattern-bindings (compute-pattern-bindings patterns))1204
(filter (second (assoc 'spocq.a:|filter| pattern-body)))1205
(view-name (resource-api-view-name api))1206
(input-attribute-map1207
;; collect all pattern bindings - both variables and constant terms,1208
;; which are defined as inputs1209
(loop for (predicate . object) in pattern-bindings1210
for attribute = (resource-api-input-attribute api predicate object)1212
collect (cons object attribute)))1213
(output-attribute-map1214
;; do the same for otputs, but in this case it makes sense for variables only1215
(loop for (predicate . object) in pattern-bindings1216
for attribute = (resource-api-output-attribute api predicate object)1218
collect (cons object attribute))))1219
(if (and (variable-p subject) (not (boundp subject)))1220
(setf output-attribute-map (acons subject "_resourceId" output-attribute-map))1221
(setf input-attribute-map (acons subject "_resourceId" input-attribute-map)))1222
;; (print (list :input-attribute-map input-attribute-map))1223
;; (print (list :output-attribute-map output-attribute-map))1224
(let* ((projection-dimensions (expression-variables pattern-body))1225
(constraint-map (loop for entry in (append input-attribute-map output-attribute-map)1226
for (object . attribute) = entry1227
unless (and (variable-p object) (not (boundp object)))1229
(projection-map (loop with map = (append input-attribute-map output-attribute-map)1230
for variable in projection-dimensions1231
for entry = (assoc variable map)1234
;; (print (list :constraint-map constraint-map))1235
;; (print (list :projection-map projection-map))1236
(let* ((input-variables ())1237
;; construct the actual sql1238
(where (let ((constraints (loop for (object . attribute) in constraint-map1239
;; constraints are between an attribute and either a constant or a dynamic variable1240
collect (clsql:sql-operation '=1241
(sql-attribute attribute view-name)1242
(if (variable-p object)1243
;; capture as a variable, to be encoded in sql as its term number1244
(rest (or (assoc object input-variables)1245
(first (setf input-variables1246
(acons object (sql-variable object)1247
input-variables)))))1248
(rdfcache-object-term-number *transaction* object))))))1249
;; (print (list :constraints constraints))1250
(if (rest constraints)1251
(reduce #'(lambda (c1 c2) (clsql:sql-operation 'and c1 c2)) constraints)1252
(first constraints)))))1254
;; any variable reference in a filter must be replaced with the respective attribute1255
(flet ((variable-to-attribute (object)1256
(if (variable-p object)1257
(let* ((predicate (first (rassoc object pattern-bindings)))1258
(attribute (resource-api-output-attribute api predicate object)))1260
(sql-attribute attribute)1261
(sql-variable object)))1263
(setf where (clsql:sql-operation :and where (compute-sql-expression (map-tree #'variable-to-attribute filter))))))1264
(setf projection-map (sort projection-map #'string-lessp :key #'first))1265
(let ((sql (apply #'clsql:sql-operation 'select1267
;; output dimension map is orded by dimension1268
(loop for (nil . attribute) in projection-map1269
collect (sql-attribute attribute view-name))1270
(list* :from (clsql:sql-expression :table view-name)1273
(append (when (first slice) `(:limit ,(first slice)))1274
(when (second slice) `(:offset ,(second slice)))))))))1275
;;; do not need to cache attribute-dimension relation as this is never part of1276
;;; of the generation for a join1280
(defmethod compute-view-lambda ((context internal-materialized-repository)1281
bgp-pattern-body &key1282
(api (repository-resource-api context bgp-pattern-body))1283
(base-dimensions ())1286
(variables (if (variable-p graph)1287
(cons graph (expression-variables bgp-pattern-body))1288
(expression-variables bgp-pattern-body)))1289
(projection-dimensions variables)1290
(dynamic-variables ()))1291
(declare (ignore dataset-graphs dynamic-variables))1292
(assert (null (assoc 'spocq.a::|equivalents| bgp-pattern-body)) ()1293
"equivalents not permitted in matierialized view pattern: ~s" bgp-pattern-body)1294
(let* ((query (compute-materialized-view-sql context api bgp-pattern-body))1295
(projection-variable-count (length projection-dimensions)))1296
;; (print projection-dimensions)1297
;; (print query) ;; (setf projection-dimensions (remove '?::_RESOURCEID projection-dimensions))1298
`(lambda (solution-destination ,@(when base-dimensions '(source)))1299
(assert-argument-types view-match1300
(solution-destination (or channel function))1301
,@(when base-dimensions '((source (or channel function)))))1302
(let* ((repository-id (repository-id *repository*))1303
(revision-id (repository-revision-id *task*))1304
(transaction *transaction*)1305
(*thread-operations* (cons (list 'spocq.a::|view| ',bgp-pattern-body) *thread-operations*))1307
(result-page-length (channel-page-length solution-destination))1308
(result-index *field-page-length*)1310
;; te revision id is not observed (yet)1311
(trace-bgp view.match.start repository-id revision-id transaction)1313
(labels ((collect-solution (solution)1314
(trace-bgp view.collect-solution solution)1315
(destructuring-bind ,projection-dimensions solution1316
(next-solution-location)1317
(locally (declare (type (simple-array fixnum (* ,projection-variable-count)) result-page)1318
(type fixnum result-index)1319
(optimize ,@*field-optimization*))1320
(setf ,@(loop for var-index from 01321
for variable in projection-dimensions1322
nconc `((aref result-page result-index ,var-index) ,variable)))))1324
(next-solution-location ()1325
;; return a page (possible newly created) and the next free location in that page1327
(when (>= (incf result-index) result-page-length)1328
(when result-page (put-result result-page))1329
(setf result-page (new-field-page solution-destination result-page-length ,projection-variable-count)1331
(values result-page result-index))1332
(complete-solutions ()1333
(trace-bgp view.match.complete-solutions result-count)1334
(incf *match-requests* 1)1335
(incf *match-responses* result-count)1336
(log-debug "view counts: ~s: solutions: ~s" repository-id result-count)1338
(let ((page-result-count (1+ result-index)))1339
(when (< page-result-count result-page-length)1341
(adjust-page result-page (list page-result-count ,projection-variable-count)))))1342
(put-result result-page))1343
(complete-field solution-destination)1344
(incf-stat *solutions-constructed* result-count)1345
(return-from :view-scan result-count))1347
(trace-bgp bgp.put solution-destination ',projection-dimensions page)1348
(put-field-page solution-destination page)1349
(unless (task-active-p *query*)1350
(complete-field solution-destination)1351
(return-from :view-scan result-count))))1352
(declare (dynamic-extent #'collect-solution))1353
(clsql-sys::map-query-for-effect #'collect-solution ,query (repository-view-database *repository*) :auto)1354
(complete-solutions)1356
(clsql:map-query nil #'collect-solution ,query1357
:database (repository-view-database *repository*)1358
:result-types ',(make-list (length projection-dimensions) :initial-element 'integer))))))))1361
(defmethod compute-materialized-view-sql ((repository external-materialized-repository) (api t) (pattern-body list))1362
"The bgp matching operator must include provisions to externalize any filters and to intern the results"1363
(let* ((patterns (remove-if-not #'elementary-bgp-statement-form-p pattern-body))1364
(subject (statement-subject (first patterns)))1365
(slice-offset (second (assoc 'spocq.a:|slice| pattern-body)))1366
(slice-count (third (assoc 'spocq.a:|slice| pattern-body)))1367
(pattern-bindings (compute-pattern-bindings patterns))1368
(filter (second (assoc 'spocq.a:|filter| pattern-body)))1369
(view-name (resource-api-view-name api))1370
(view-identifier (sql-ident-table view-name))1371
#+(or)(database-type (repository-database-type repository))1372
(input-attribute-map1373
;; collect all pattern bindings - both variables and constant terms,1374
;; which are defined as inputs1375
(loop for (predicate . object) in pattern-bindings1376
for attribute = (resource-api-input-attribute api predicate object)1378
collect (cons object attribute)))1379
(output-attribute-map1380
;; do the same for otputs, but in this case it makes sense for variables only1381
(loop for (predicate . object) in pattern-bindings1382
for attribute = (resource-api-output-attribute api predicate object)1384
collect (cons object attribute)))1385
;; take as the subject attribute the first attribute which is a key, but not the record id,1386
;; with otherwise the id as the default.1387
(key (loop for (property attribute . options) in (resource-api-input-definitions api)1389
when (getf options :is-key)1390
do (if (string-equal "id" attribute)1391
(setf id-key attribute)1393
finally (return id-key)))1394
(*sql-environment* (make-sql-environment :as (gentemp (sql-ident-table-name view-identifier)))))1395
(when (and (variable-p subject) (not (boundp subject)) key)1396
(let ((key-attribute key))1397
(setf output-attribute-map (acons subject key-attribute output-attribute-map))1398
(setf input-attribute-map (acons subject key-attribute input-attribute-map))))1399
(loop for (predicate . variable) in pattern-bindings1400
for attribute = (when (variable-p variable)1401
(resource-api-output-attribute api predicate variable))1403
do (setf (sql-variable-attribute variable *sql-environment*)1404
(sql-attribute attribute (sql-ident-table-name view-identifier))))1405
; (print (list :input-attribute-map input-attribute-map))1406
; (print (list :output-attribute-map output-attribute-map))1407
(let* ((projection-dimensions (expression-variables pattern-body))1408
(constraint-map (loop for entry in (remove-duplicates (append input-attribute-map output-attribute-map)1410
for (object . attribute) = entry1411
unless (and (variable-p object) (not (boundp object)))1413
(projection-map (loop with map = (append input-attribute-map output-attribute-map)1414
for variable in projection-dimensions1415
for entry = (assoc variable map)1418
;; (print (list :constraint-map constraint-map))1419
;; (print (list :projection-map projection-map))1420
(labels ((variable-to-attribute (object)1421
(if (variable-p object)1422
(let* ((predicate (first (rassoc object pattern-bindings)))1423
(attribute (resource-api-output-attribute api predicate object)))1425
(sql-attribute attribute)1426
(sql-variable object)))1428
(when (> (length projection-map) (if (boundp subject) 0 1))1429
(let* ((input-variables ())1430
;; construct the actual sql1431
(where (let ((constraints (loop for (object . attribute) in constraint-map1432
;; constraints are between an attribute and either a constant or a dynamic variable1433
collect (clsql:sql-operation '=1434
(sql-attribute attribute view-name)1435
(if (variable-p object)1436
;; capture as a variable, to be passed to sql as its object value1437
(rest (or (assoc object input-variables)1438
(first (setf input-variables1439
(acons object (sql-variable object)1440
input-variables)))))1441
(compute-sql-expression object)1443
(clsql-sys:database-output-sql-as-type1444
(resource-api-input-type api attribute)1446
(repository-view-database repository)1448
;; (print (list :constraints constraints))1449
(if (rest constraints)1450
(reduce #'(lambda (c1 c2) (clsql:sql-operation 'and c1 c2)) constraints)1451
(first constraints)))))1453
;; any variable reference in a filter must be replaced with the respective attribute1456
(clsql:sql-operation :and where (compute-sql-expression (map-tree #'variable-to-attribute filter)))1457
(compute-sql-expression (map-tree #'variable-to-attribute filter)))))1458
(setf projection-map (sort projection-map #'string-lessp :key #'first))1459
(let ((sql (apply #'clsql:sql-operation 'select1461
;; output dimension map is orded by dimension1462
(loop for (nil . attribute) in projection-map1463
collect (sql-attribute attribute view-name))1464
(list* :from (clsql:sql-expression :table view-name)1467
(append (when slice-count `(:limit ,slice-count))1468
(when slice-offset `(:offset ,slice-offset))))))))1469
;;; do not need to cache attribute-dimension relation as this is never part of1470
;;; of the generation for a join1471
(values sql patterns input-attribute-map output-attribute-map constraint-map))))))))1474
(defmethod compute-bgp-lambda ((repository external-materialized-repository) pattern &rest args &key &allow-other-keys)1475
(apply #'compute-view-lambda repository pattern args))1477
(defmethod compute-view-lambda ((context external-materialized-repository)1478
bgp-pattern-body &key1479
(api (repository-resource-api context bgp-pattern-body))1480
(base-dimensions ())1483
(variables (if (variable-p graph)1484
(cons graph (expression-variables bgp-pattern-body))1485
(expression-variables bgp-pattern-body)))1486
(projection-dimensions variables)1487
(dynamic-variables ())1489
"Compute the pattern lambda for an external materialized view.1490
In this case the function must map between internal term identifiers and external term values."1491
(declare (ignore environment))1492
(declare (ignore dataset-graphs dynamic-variables))1493
(assert (null (assoc 'spocq.a::|equivalents| bgp-pattern-body)) ()1494
"equivalents not permitted in materialized view pattern: ~s" bgp-pattern-body)1495
(labels ((compute-null-field ()1496
(log-warn "NULL materialied query: ~s . ~s" context bgp-pattern-body)1497
`(lambda (solution-destination ,@(when base-dimensions '(source)))1498
,@(when base-dimensions '((declare (ignore source))))1499
(log-debug "null materialized bgp: ~s" (repository-id *repository*))1500
(complete-field solution-destination)1502
(compute-predicate-scan ()1503
;; compute a pseudo table scan which actually just emits the predicates defined for the view1504
;; set solution content to accounts for the s-p-o 'p' variable and the1505
;; projection dimensions1506
(let* ((predicates (resource-api-predicates api))1507
(variable (third (find-if #'triple-form-p bgp-pattern-body)))1508
(variable-index (position variable projection-dimensions))1509
(slice-offset (second (assoc 'spocq.a:|slice| bgp-pattern-body)))1510
(slice-count (third (assoc 'spocq.a:|slice| bgp-pattern-body))))1512
(setf predicates (subseq predicates (min slice-offset (length predicates)))))1514
(setf predicates (subseq predicates 0 (min slice-count (length predicates)))))1515
`(lambda (solution-destination ,@(when base-dimensions '(source)))1516
,@(when base-dimensions '((declare (ignore source))))1517
(let* ((repository-id (repository-id *repository*))1518
(transaction *transaction*)1519
(*thread-operations* (cons (list 'spocq.a::|view| ',bgp-pattern-body) *thread-operations*))1521
(result-page-length (channel-page-length solution-destination))1522
(result-page-width ,(length projection-dimensions))1523
(result-index *field-page-length*)1526
(labels ((next-solution-location ()1527
;; return a page (possible newly created) and the next free location in that page1529
(when (>= (incf result-index) result-page-length)1530
(when result-page (put-result result-page))1531
(setf result-page (new-field-page solution-destination result-page-length result-page-width)1533
(collect-predicate (predicate)1534
(next-solution-location)1535
(locally (declare (type (simple-array fixnum (* ,(length projection-dimensions))) result-page)1536
(type fixnum result-index)1537
(optimize ,@*field-optimization*))1538
(let ((term-number (intern-material-value transaction predicate)))1539
(loop for index below ,(length projection-dimensions)1540
do (setf (aref result-page result-index index)1541
(if (= index ,variable-index) term-number 0))))))1542
(complete-solutions ()1543
(trace-bgp view.match.complete-solutions result-count)1544
(incf *match-requests* 1)1545
(incf *match-responses* result-count)1546
(log-debug "view counts: ~s: solutions: ~s" repository-id result-count)1548
(let ((page-result-count (1+ result-index)))1549
(when (< page-result-count result-page-length)1551
(adjust-page result-page (list page-result-count result-page-width)))))1552
(put-result result-page))1553
(complete-field solution-destination)1554
(incf-stat *solutions-constructed* result-count)1555
(return-from :view-scan result-count))1557
(trace-bgp bgp.put solution-destination ',projection-dimensions page)1558
(put-field-page solution-destination page)1559
(unless (task-active-p *query*)1560
(complete-field solution-destination)1561
(return-from :view-scan result-count))))1562
(loop for predicate in ',predicates do (collect-predicate predicate))1563
(complete-solutions)))))))1564
(compute-sql-map-field (query input-map)1565
(let* ((projection-variable-count (length projection-dimensions)))1566
(flet ((variable-attribute (v) (rest (assoc v input-map))))1567
;; (print projection-dimensions)1568
;; (print query) ;; (setf projection-dimensions (remove '?::_RESOURCEID projection-dimensions))1569
`(lambda (solution-destination ,@(when base-dimensions '(source)))1570
(assert-argument-types view-match1571
(solution-destination (or channel function))1572
,@(when base-dimensions '((source (or channel function)))))1573
(let* ((repository-id (repository-id *repository*))1574
(revision-id (repository-revision-id *task*))1575
(transaction *transaction*)1576
(*thread-operations* (cons (list 'spocq.a::|view| ',bgp-pattern-body) *thread-operations*))1578
(result-page-length (channel-page-length solution-destination))1579
(result-index *field-page-length*)1581
;; te revision id is not observed (yet)1582
(trace-bgp view.match.start repository-id revision-id transaction)1583
(with-materialized-repository-database (%db designator) *repository*1585
(labels ((collect-solution (solution)1586
;; internal soution values1587
(trace-bgp view.collect-solution solution)1588
(destructuring-bind ,projection-dimensions solution1589
(next-solution-location)1590
(locally (declare (type (simple-array fixnum (* ,projection-variable-count)) result-page)1591
(type fixnum result-index)1592
(optimize ,@*field-optimization*))1593
(setf ,@(loop for var-index from 01594
for variable in projection-dimensions1595
for attribute = (variable-attribute variable)1596
for external-type = (resource-api-input-type api attribute)1597
for internal-type = (sql-internal-type external-type)1598
nconc `((aref result-page result-index ,var-index)1600
(intern-material-value transaction1601
(decode-material-value ,variable ',internal-type %db ',external-type))1603
(next-solution-location ()1604
;; return a page (possible newly created) and the next free location in that page1606
(when (>= (incf result-index) result-page-length)1607
(when result-page (put-result result-page))1608
(setf result-page (new-field-page solution-destination result-page-length ,projection-variable-count)1610
(complete-solutions ()1611
(trace-bgp view.match.complete-solutions result-count)1612
(incf *match-requests* 1)1613
(incf *match-responses* result-count)1614
(log-debug "view counts: ~s: solutions: ~s" repository-id result-count)1616
(let ((page-result-count (1+ result-index)))1617
(when (< page-result-count result-page-length)1619
(adjust-page result-page (list page-result-count ,projection-variable-count)))))1620
(put-result result-page))1621
(complete-field solution-destination)1622
(incf-stat *solutions-constructed* result-count)1623
(return-from :view-scan result-count))1625
(trace-bgp bgp.put solution-destination ',projection-dimensions page)1626
(put-field-page solution-destination page)1627
(unless (task-active-p *query*)1628
(complete-field solution-destination)1629
(return-from :view-scan result-count))))1630
(declare (dynamic-extent #'collect-solution))1631
(clsql-sys::map-query-for-effect #'collect-solution ,query %db :auto)1632
(complete-solutions))))))))))1633
(if (and (= (count-if #'triple-form-p bgp-pattern-body) 1)1634
(variable-p (third (find-if #'triple-form-p bgp-pattern-body))))1635
(compute-predicate-scan)1636
(multiple-value-bind (query patterns input-map output-map)1637
(compute-materialized-view-sql context api bgp-pattern-body)1638
(declare (ignore patterns output-map))1639
(unless (loop for var in projection-dimensions1640
unless (assoc var input-map)1643
(spocq.e:request-error "some projection variable is not an attribute: ~s: ~s" projection-dimensions input-map))1645
(compute-sql-map-field query input-map)1646
(compute-null-field))))))1648
;;; (compute-view-lambda (repository "operations/transaction_events") '((spocq.a:|triple| ?::s <http://dydra.com/operations#timestamp_start> ?::ts)))1649
;;; (compute-view-lambda (repository "operations/query_events") '((spocq.a:|triple| ?::s <http://dydra.com/operations#timestamp> ?::ts)))1650
;;; (compute-view-lambda (repository "operations/query_events") '((spocq.a:|triple| ?::s ?::p ?::ts)))1652
(defun compute-sql-slot-name (name)1653
(setq name (string-upcase name))1654
(let ((name- (substitute #\- #\_ name)))1655
(cond ((or (find-symbol name :spocq.i)1656
(find-symbol name- :spocq.i)))1658
(warn "unknown slot name: ~s/~s" name name-)1659
(intern name- :spocq.i)))))1661
(defun compute-sql-type-name (name)1662
(setq name (string-upcase name))1663
(cond ((equalp name "BPCHAR") :VARCHAR)1664
((equalp name "datetime") :timestamp)1665
((equalp name "uuid") :uuid)1666
((equalp name "text") :varchar)1667
((find-symbol name :spocq.i))1668
((find-symbol name :clsql-sys))1669
((find-symbol name :keyword))1671
(log-warn "unknown type name: ~s" name)1672
(intern name :spocq.i))))1674
(defmethod decode-api-declaration ((encoding (eql :odbc)) (declaration cons) resource-name)1675
(decode-api-declaration :sql declaration resource-name))1677
(defmethod decode-api-declaration ((encoding (eql :postgresql)) (declaration cons) resource-name)1678
(decode-api-declaration :sql declaration resource-name))1680
(defmethod decode-api-declaration ((encoding (eql :mysql)) (declaration cons) resource-name)1681
(decode-api-declaration :sql declaration resource-name))1683
(defmethod decode-api-declaration ((encoding (eql :sql)) (declaration cons) resource-name)1684
"Translate an odbc schema declaration into API definitions.1685
As expressed, the declaration comprises the description for a single class."1688
(loop for element in declaration1689
for (table attname typname attlen is-key) = element1690
for property = (merge-and-intern-iri (concatenate 'string "#" attname) :base-iri resource-name)1691
for slot-name = (compute-sql-slot-name attname)1692
for external-type = (let ((type (compute-sql-type-name typname)))1693
(or (find type '(uuid clsql-sys::int2 clsql-sys::int4 clsql-sys::int8 CLSQL-SYS::TIMESTAMP)1694
:test #'string-equal)1695
(if (typep attlen '(integer 1)) (list type attlen) type)))1696
for map-entry = (list* property attname :type external-type :slot-name slot-name (when is-key '(:is-key t)))1698
do (assert (equalp table table-name) ()1699
"api declaration must comprise a single class: ~s" declaration)1700
else do (setf table-name table)1702
(push map-entry (getf initargs :input-definitions))1703
(push map-entry (getf initargs :output-definitions))))1704
(apply #'make-odbc-api1705
:resource resource-name1707
:operation-class 'hydra::|ODBCView|1708
:input-class nil :output-class nil1710
:view-name table-name1713
(defun make-odbc-api (&rest args)1714
(apply #'make-instance 'odbc-api args))1717
(defmethod resource-retrieve-resource-api ((repository external-materialized-repository) (name t))1718
(resource-retrieve-resource-api repository (iri-local-part name)))1720
(defmethod resource-retrieve-resource-api ((repository external-materialized-repository) (table string))1721
(resource-retrieve-resource-api (repository-view-database repository) table))1724
(defmethod resource-retrieve-resource-api ((designator spocq:mysql-uri) (table string))1725
"For a mysql api, extract the table from the resource identifier, retrieve the table schema and decode it."1726
(with-materialized-repository-database (schema-db location) designator1727
(let* ((schema-name (spocq:mysql-uri-schema location))1728
(schema-query (format nil "SELECT table_name,column_name,data_type,character_maximum_length,column_key FROM `INFORMATION_SCHEMA`.`columns` WHERE table_name = '~a' and table_schema = '~a';"1730
(base-iri (if (member table '("accounts" "import_events") :test #'equalp)1731
(merge-and-intern-iri "/operations" :base-iri (site-namespace))1733
(declaration (clsql-sys:query schema-query :database schema-db)))1734
(when (equalp table "accounts")1735
(setf declaration (remove-if #'(lambda (def) (member (second def) '("authentication_token" "encrypted_password" "reset_password_token")1739
(decode-api-declaration :sql declaration base-iri)1740
(spocq.e:resource-not-found-error :identifier designator)))))1742
(defmethod resource-retrieve-resource-api ((designator spocq:postgresql-uri) (table string))1743
"For a postgres api, extract the table from the resource identifier, retrieve the table schema and decode it."1744
(with-materialized-repository-database (schema-db location) designator1745
(let* ((schema-query (format nil "SELECT pg_class.relname,pg_attribute.attname,pg_type.typname,pg_attribute.attlen FROM pg_type,pg_class,pg_attribute WHERE pg_class.oid=pg_attribute.attrelid AND pg_class.relname='~a' AND pg_attribute.atttypid=pg_type.oid AND pg_attribute.attnum > 0;"1747
;; for the type as well: format_type(a.atttypid, a.atttypmod) AS data_type1748
;; for primaries AND i.indisprimary1749
(index-query (format nil "SELECT a.attname FROM pg_index i JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey) WHERE i.indrelid = '~a'::regclass;"1751
(schema-name (spocq:postgresql-uri-schema location))1752
(base-iri (if (member table '("transaction_events" "query_events") :test #'equalp)1753
(merge-and-intern-iri (concatenate 'string "/" schema-name) :base-iri (site-namespace))1755
(schema (clsql-sys:query schema-query :database schema-db))1756
(keys (loop for (key) in (clsql-sys:query index-query :database schema-db) collect key))1757
(declaration (loop for attribute-decl in schema1758
for name = (second attribute-decl)1759
collect (append attribute-decl1760
(when (find name keys :test #'string-equal) '(t))))))1762
(decode-api-declaration :sql declaration base-iri)1763
(spocq.e:resource-not-found-error :identifier designator)))))1766
;;; (resource-retrieve-resource-api (repository "operations/accounts") <http://example.org#accounts>)1767
;;; (resource-retrieve-resource-api (repository "operations/import_events") <http://example.org#import_events>)1768
;;; (resource-retrieve-resource-api (repository "operations/transaction_events") <http://example.org#transaction_events>)1769
;;; (resource-retrieve-resource-api (repository "operations/query_events") <http://example.org#query_events>)1771
;;; (resource-retrieve-resource-api <postgresql://postgres:postgres@localhost/test?schema=accounting> "query_events")1774
;;; not used in standard compilation as that control-flow uses compute-agp-pattern-lambda1775
(defun compute-materialized-view-function (context bgp-pattern-body &rest args &key base-dimensions)1776
(declare (ignore base-dimensions)1777
(dynamic-extent args))1778
(let* ((key (list 'compute-materialized-view-function bgp-pattern-body))1779
(function (get-aspect-cache key)))1781
(let* ((lambda (apply #'compute-view-lambda context bgp-pattern-body args))1782
(function (spocq-compile lambda)))1783
(setf (get-aspect-cache key) function)1784
(values function lambda)))))1786
(defmethod repository-service-description ((repository materialized-repository))1787
(let* ((api (resource-retrieve-resource-api (repository-view-database repository) (repository-name repository)))1788
(solutions (compute-repository-service-description-solutions api)))1789
(make-service-description :solutions solutions :timestamp (get-universal-time))))1795
(run-sparql-internal "select count(*) where {?s ?p ?o}" :repository-id "jhacker/test")1796
(run-sparql-internal "select * where {?s ?p ?o}" :repository-id "jhacker/test")1798
(COMPUTE-MATERIALIZED-VIEW-SQL1799
(repository "operations/import_events")1800
(resource-retrieve-resource-api (repository "operations/import_events") <http://example.org#import_events>)1801
'((ORG.DATAGRAPH.SPOCQ.ALGEBRA:|triple| ?::?11802
<http://dydra.com/operations#job_id>1804
(ORG.DATAGRAPH.SPOCQ.ALGEBRA:|triple| ?::?11805
<http://dydra.com/operations#created_at>1807
(ORG.DATAGRAPH.SPOCQ.ALGEBRA:|triple| ?::?11808
<http://dydra.com/operations#agent_key>1810
(ORG.DATAGRAPH.SPOCQ.ALGEBRA:|triple| ?::?11811
<http://dydra.com/operations#repository_key>1813
(ORG.DATAGRAPH.SPOCQ.ALGEBRA:|triple| ?::?11814
<http://dydra.com/operations#account_key>1818
(COMPUTE-MATERIALIZED-VIEW-SQL1819
(repository "operations/import_events")1820
(resource-retrieve-resource-api (repository "operations/import_events") <http://example.org#import_events>)1821
'((ORG.DATAGRAPH.SPOCQ.ALGEBRA:|triple| ?::?11822
<http://dydra.com/operations#job_id>1824
(ORG.DATAGRAPH.SPOCQ.ALGEBRA:|triple| ?::?11825
<http://dydra.com/operations#repository_key>1829
(parse-sparql "prefix ops: <http://dydra.com/operations#>1830
prefix dc: <http://purl.org/dc/elements/1.1/>1831
select ?uuid ?agent ?account ?repository ?timestamp1833
[ <http://dydra.com/operations#job_id> ?uuid ;1834
<http://dydra.com/operations#repository_key> 'plm' ]1835
} order by desc (?timestamp) limit 20 1838
(ORG.DATAGRAPH.SPOCQ.ALGEBRA:|bgp|1839
(ORG.DATAGRAPH.SPOCQ.ALGEBRA:|triple| ?::?11840
<http://dydra.com/operations#job_id>1842
(ORG.DATAGRAPH.SPOCQ.ALGEBRA:|triple| ?::?11843
<http://dydra.com/operations#created_at>1845
(ORG.DATAGRAPH.SPOCQ.ALGEBRA:|triple| ?::?11846
<http://dydra.com/operations#agent_key>1848
(ORG.DATAGRAPH.SPOCQ.ALGEBRA:|triple| ?::?11849
<http://dydra.com/operations#repository_key>1851
(ORG.DATAGRAPH.SPOCQ.ALGEBRA:|triple| ?::?11852
<http://dydra.com/operations#account_key>