Coverage report: /development/source/library/org/datagraph/spocq-shard/src/store/rdfcache/term-number-field-cache.lisp

Kind	Covered	All	%
expression	0	789	0.0
branch	0	84	0.0

Key

Not instrumented

Conditionalized out

Executed

Not executed

Both branches taken

One branch taken

Neither branch taken

1

;;; -*- Mode: lisp; Syntax: ansi-common-lisp; Base: 10; Package: org.datagraph.spocq.implementation; -*-

2

3

(in-package :org.datagraph.spocq.implementation)

4

5

6

(defmethod initialize-instance :after ((instance term-number-field-cache) &key statements)

7

"index the given filed by spoc"

8

(flet ((add-statement (index)

9

(flet ((index-statement (stmt hash key1 &optional key2)

10

(if key2

11

                       (let ((key1-hash (or (gethash key1 hash) (setf (gethash key1 hash) (make-repository-index)))))

12

(push stmt (gethash key2 key1-hash)))

13

(push stmt (gethash key1 hash)))))

14

(let ((subject (aref statements index 0))

15

(predicate (aref statements index 1))

16

(object (aref statements index 2))

17

(graph (aref statements index 3)))

18

(index-statement index (cache-index-ps instance) predicate subject)

19

(index-statement index (cache-index-po instance) predicate object)

20

(index-statement index (cache-index-s instance) subject)

21

(index-statement index (cache-index-p instance) predicate)

22

(index-statement index (cache-index-o instance) object)

23

(index-statement index (cache-index-c instance) graph)))))

24

(loop for index below (array-dimension statements 0)

25

do (add-statement index))

26

(setf (slot-value instance 'statements) statements)))

27

28

(defmethod repository-make-transaction ((repository term-number-field-cache) &key &allow-other-keys)

29

(repository-transaction repository))

30

31

(defmethod repository-open-transaction ((repository term-number-field-cache) (transaction transaction))

32

t)

33

34

 (defmethod repository-close-transaction ((repository term-number-field-cache) (transaction transaction) disposition)

35

t)

36

37

(defmethod update-repository-revision-id ((repository term-number-field-cache))

38

"")

39

40

 (defmethod read-repository-pattern-count ((repository term-number-field-cache) subject predicate object context)

41

(flet ((coerce-term (term)

42

(if (or (null term) (spocq:blank-node-p term) (variable-p term))

43

rdfcache:*wildcard-term-number*

44

(rdfcache-object-term-number nil term))))

45

(let ((context (cond ((null context) RDFCACHE:*DEFAULT-CONTEXT-NUMBER*)

46

((variable-p context) rdfcache:*wildcard-term-number*)

47

(t (rdfcache-object-term-number nil context))))

48

(subject (coerce-term subject))

49

(predicate (coerce-term predicate))

50

(object (coerce-term object)))

51

(count-term-number-field repository context subject predicate object))))

52

53

(defmethod REPOSITORY-WILDCARD-TERM ((repository term-number-field-cache))

54

(REPOSITORY-WILDCARD-TERM (repository-reference repository)))

55

56

(defmethod REPOSITORY-NAMED-CONTEXTS-TERM-NUMBER ((repository term-number-field-cache))

57

(REPOSITORY-NAMED-CONTEXTS-TERM-NUMBER (repository-reference repository)) )

58

59

(defmethod REPOSITORY-default-CONTEXT-TERM-NUMBER ((repository term-number-field-cache))

60

(REPOSITORY-default-CONTEXT-TERM-NUMBER (repository-reference repository)))

61

62

(defmethod repository-object-term-number ((repository term-number-field-cache) term)

63

(repository-object-term-number (repository-reference repository) term))

64

65

66

;;; hash cache implementation

67

68

(defmacro spoc-term-number-case ((subject predicate object context)

69

&key spoc spo (spo- spo) spc (sp-c spc) sp (sp-- sp)

70

soc (s-oc soc) so (s-o- so) sc (s--c sc) s (s--- s)

71

poc (-poc poc) po (-po- po) pc (-p-c pc) p (-p-- p)

72

oc (--oc oc) o (--o- o) c (---c c) all (---- all))

73

"Consists of a sequence of forms, each identified by a combination of statement components.

74

The arguments are a mediator, s sequence of constituent variable and a matching series of

75

  constituent forms. If the mediator is not the constant nil, the variables are bound to the respective

76

  repository value. If nil, to the forms direct value. Then that clause is evaluated which indicates the

77

non-null constituents. if no constituent is present, control passes to the :---- clause."

78

79

`(let ((.flags. (logior (if (= wildcard-term-number ,subject) 0 #b1000)

80

(if (= wildcard-term-number ,predicate) 0 #b0100)

81

(if (= wildcard-term-number ,object) 0 #b0010)

82

(if (<= ,context 0) 0 #b0001))))

83

(ecase .flags.

84

(#b1111 ,spoc)

85

(#b1110 ,spo-)

86

(#b1101 ,sp-c)

87

(#b1100 ,sp--)

88

(#b1011 ,s-oc)

89

(#b1010 ,s-o-)

90

(#b1001 ,s--c)

91

(#b1000 ,s---)

92

(#b0111 ,-poc)

93

(#b0110 ,-po-)

94

(#b0101 ,-p-c)

95

(#b0100 ,-p--)

96

(#b0011 ,--oc)

97

(#b0010 ,--o-)

98

(#b0001 ,---c)

99

(#b0000 ,----))))

100

101

102

103

 (defmethod de.setf.resource:map-statements* (continuation (db term-number-field-cache) subject predicate object graph)

104

"The indexed repository tries to focus iteration over the respective index.

105

The indices are at three level, so the context constrain requires a test against the

106

matched statements' sources."

107

; (declare (dynamic-extent continuation))

108

(let ((statements (cache-statements db))

109

(wildcard-term-number (REPOSITORY-WILDCARD-TERM db))

110

(default-context-term-number (repository-default-context-term-number db))

111

(named-contexts-term-number (repository-named-contexts-term-number db)))

112

(labels ((triple-subject (index) (aref statements index 0))

113

(triple-predicate (index) (aref statements index 1))

114

(triple-object (index) (aref statements index 2))

115

(triple-graph (index) (aref statements index 3))

116

(continue-with-statement (index)

117

(let ((statement-graph (triple-graph index)))

118

(when (or (= wildcard-term-number graph)

119

                            (and (= named-contexts-term-number graph) (> statement-graph 0))

120

                            (and (= default-context-term-number graph) (= statement-graph default-context-term-number)))

121

                    (funcall continuation (triple-graph index) (triple-subject index) (triple-predicate index) (triple-object index)))))

122

(filter-statements-by-object-by-graph (statements)

123

(dolist (statement statements)

124

(when (and (= (triple-object statement) object)

125

(= (triple-graph statement) graph))

126

(continue-with-statement statement))))

127

(filter-statements-by-object (statements)

128

(dolist (statement statements)

129

(when (= (triple-object statement) object)

130

(continue-with-statement statement))))

131

(filter-statements-by-graph (statements)

132

(dolist (statement statements)

133

(when (= (triple-graph statement) graph)

134

(continue-with-statement statement))))

135

(map-list (statements)

136

(dolist (statement statements)

137

(continue-with-statement statement)))

138

(map-index (statements)

139

(loop for index below (array-dimension statements 0)

140

do (continue-with-statement index)))

141

(gethash-if (key hash)

142

(when hash (gethash key hash))))

143

144

(spoc-term-number-case (subject predicate object graph)

145

                  :spoc (filter-statements-by-object-by-graph (gethash-if subject (gethash predicate (cache-index-ps db))))

146

                  :spo  (filter-statements-by-object (gethash-if subject (gethash predicate (cache-index-ps db))))

147

                  :spc  (filter-statements-by-graph (gethash-if subject (gethash predicate (cache-index-ps db))))

148

                  :sp   (map-list (gethash-if subject (gethash predicate (cache-index-ps db))))

149

                  :soc  (filter-statements-by-object-by-graph (gethash subject (cache-index-s db)))

150

:so (filter-statements-by-object (gethash subject (cache-index-s db)))

151

:sc (filter-statements-by-graph (gethash subject (cache-index-s db)))

152

:s (map-list (gethash subject (cache-index-s db)))

153

                  :poc  (filter-statements-by-graph (gethash-if object (gethash predicate (cache-index-po db))))

154

                  :po   (map-list (gethash-if object (gethash predicate (cache-index-po db))))

155

:pc (filter-statements-by-graph (gethash predicate (cache-index-p db)))

156

:p (map-list (gethash predicate (cache-index-p db)))

157

:oc (filter-statements-by-graph (gethash object (cache-index-o db)))

158

:o (map-list (gethash object (cache-index-o db)))

159

:c (map-list (gethash graph (cache-index-c db)))

160

:all (map-index (cache-statements db))))))

161

162

(defun match-term-number-field (db graph subject predicate object continuation)

163

(de.setf.resource:map-statements* continuation db subject predicate object graph))

164

165

(defun count-term-number-field (db graph subject predicate object)

166

(let ((count 0))

167

(flet ((count-continuation (context subjkect predicate object)

168

(declare (ignore context subjkect predicate object))

169

(incf count)))

170

(declare (dynamic-extent #'count-continuation))

171

(de.setf.resource:map-statements* #'count-continuation db subject predicate object graph))

172

count))

173

174

175

176

(defmethod compute-bgp-lambda ((repository term-number-field-cache) body &key

177

                                (base-dimensions ())          ; initial solution field variables

178

(projection-dimensions (expression-dimensions body))

179

(wildcard-term (repository-wildcard-term repository))

180

                                (default-context-term (repository-default-context-term repository))

181

                                (named-contexts-term (repository-named-contexts-term repository))

182

                                graph    ; if in a graph clause, then either a variable or a literal

183

(dataset-graphs nil)

184

                                (graphs (cond ((null graph)       (dataset-default-graphs dataset-graphs))

185

                                              ((variable-p graph) (dataset-named-graphs dataset-graphs))

186

                                              ((typep graph 'iri) (list graph))

187

(t

188

                                               (error "Invalid dataset specification: graph ~s, dataset ~s."

189

                                                      graph dataset-graphs))))

190

(variables (if (variable-p graph)

191

                                             (cons graph (expression-variables body))

192

(expression-variables body)))

193

(dynamic-variables ())

194

(transaction (repository-transaction repository))

195

(trace *compute-bgp-lambda.trace*)

196

environment)

197

   "Generate a matching operator from a BGP for the rdfcache store. The graph matching requirements are

198

described in graph.lisp. They are implemented with the following logic:

199

200

  The graph argument is taken from the agp's graph field. If this bgp is in the scope of a graph clause, that

201

  term noted by the macro-expansion for spocq.a:|graph|. Otherwise, it is an autonomous graph pattern, for

202

  which there is no graph term. The default-graphs and named-graphs terms were either present in the context

203

  query - as specified either in the rquest headers or in the request expression, or were present in the body

204

as spocq.a::|from| and spocq.a::|from-named| forms.

205

206

If the graph is a variable, the bgp applies to named graphs:

207

- if named-graphs is non-null, the match maps the variable over that set, while

208

  - if named-graphs is null, no constraint is applied to the graph variable and it ranges query/next results.

209

If graph is a literal, a variable is introduced, bound to that value at the outset.

210

If graph is null, the bgp applies to 'default' graphs:

211

  - If default-graphs is non-null, a variable is introduced and the match maps it over that set, while

212

  - if default-graphs is null, a variable is introduced, bound to the repository's default graph designator.

213

214

The iteration operator propagates the accumulated bindings over a sequence of core single quad pattern

215

  matches. Each of these starts with a query for which each term is either wild, or a foreign term which

216

  either represents a constant value from the original bgp expression, or carries over a matched term from a

217

  preceeding statement match. The process starts with initial bindings - the graph and/or a solution from an

218

  initial field and augments the bindings as the matches proceed bin binding to the foreign pointers from

219

the successive matches as permitted by a given next.

220

221

This version evolved from the initial rdfcache interface primarily by removing all term structures.

222

As the term identifers are bout retrieved _and_ provided from the query constants, there is no need

223

  to represent the terms' internal structure at the interface to the store. Which means none of the stack definitions

224

for the foreign objects and nothing to substitute in the patterns.

225

  It also eliminates the (broken) mp code, also because it is not clear how to handle thread coordination in

226

a way which does not compromise rdfcache:next iteration rate."

227

(declare (ignore environment))

228

(when (assoc 'spocq.a:|quad| body)

229

(error "invalid bpg pattern: ~s" body))

230

;; use the dimensions provided in the call

231

;; (setf projection-dimensions (expression-dimensions body))

232

(let* ((named-graphs (dataset-named-graphs dataset-graphs))

233

(default-graphs (dataset-default-graphs dataset-graphs))

234

;; if the query had a literal graph term, provide a variable

235

(graph-variable (if (variable-p graph) graph 'graph))

236

;; note if the graph variable is among the initial solutions

237

(initial-solution-graph-variable (find graph base-dimensions))

238

;; indicate whether the graph needs to be set on first query

239

;; (set-graph-variable (and (variable-p graph) (not initial-solution-graph-variable)))

240

          ;; extract just the expressions wto be included in the matching and bindings propagation code

241

(iteration-patterns (remove-if-not #'bgp-pattern-form-p body))

242

(id (second (assoc 'spocq.a::|id| body)))

243

;; extract and consolidate anu declaration clauses

244

          (declarations (reduce #'append (mapcar #'rest (remove 'spocq.a::|declare| body :test-not #'eq :key #'first))))

245

          ;; get the a-list of variables which were inferred to be related through a sameTerm filter constraint

246

;; in order to mirror any binding/setting

247

(equivalents (rest (assoc 'spocq.a::|equivalents| body)))

248

(triple-pattern-count (count 'spocq.a:|triple| iteration-patterns :key #'first))

249

(match-counters (loop for i from 0 below triple-pattern-count collect (gensym "COUNTER")))

250

(temp-match-counters match-counters)

251

(slice-offset (second (assoc 'spocq.a:|slice| body)))

252

(slice-count (third (assoc 'spocq.a:|slice| body)))

253

(projection-variable-count (length projection-dimensions))

254

(equivalent-variables (mapcar #'first equivalents))

255

(collection-variables (difference-dimensions projection-dimensions equivalent-variables))

256

(blank-nodes (expression-blank-nodes iteration-patterns))

257

(blank-node-map (loop for node in blank-nodes collect (cons node (cons-variable))))

258

(default-context-term-number (repository-object-term-number repository default-context-term))

259

(named-contexts-term-number (repository-object-term-number repository named-contexts-term)))

260

(declare (ignore declarations))

261

262

(labels ((intern-if-constant (object)

263

(cond ((variable-p object)

264

object)

265

((spocq:blank-node-p object)

266

(or (rest (assoc object blank-node-map))

267

(error "lost a blank-node: ~a" object)))

268

((property-path-p object)

269

(repository-intern-property-path transaction object))

270

(t

271

(rdfcache-object-term-number transaction object))))

272

(compute-filter-continuation (triples accumulated-variables)

273

(let* ((form (first triples))

274

(cc (compute-next-continuation (rest triples) accumulated-variables))

275

(test-expression (second form))

276

(test-variables (expression-variables test-expression))

277

                       (test-expression `(handler-case (ebv ,test-expression) (error () nil))))

278

(if test-variables

279

                    (let ((test-aliases (loop for variable in test-variables collect (make-symbol (symbol-name variable)))))

280

`(when ((lambda ,test-aliases

281

                                (symbol-macrolet ,(loop for variable in test-variables

282

                                                        for alias in test-aliases

283

                                                        collect `(,variable (rdfcache-term-number-object ,transaction ,alias)))

284

                                  (trace-bgp bgp.filter ',test-expression ',test-variables (list ,@test-aliases) (list ,@test-variables))

285

,test-expression))

286

,@test-variables)

287

,cc))

288

`(when ,test-expression ,cc))))

289

(compute-statement-continuation (triples accumulated-variables)

290

(let* ((form (first triples))

291

                       ;; the first match must always use the graph term - either a constant from/from-named iri

292

;; or one of the wild-card, designators default/named/all.

293

                       ;; if wild, then the result matters, but if constant, then the result does not.

294

                       ;; all subsequent iterations use the result from the first match and do not need to rebind

295

(first-iteration-p (eq triples iteration-patterns))

296

(last-triple-p (null (assoc 'spocq.a:|triple| (rest triples))))

297

                       ;; set the graph from each result for the outermost iteration only. that means:

298

                       ;; trying w/o this: nb. in contrast to the term-pointer version, which sets the pointer

299

                       ;; once only, the term-id based version must retrieve the id each iteration

300

                       (match-result-arguments (loop for term in (cons graph-variable (statement-terms form))

301

                                                     collect (cond ((or (not (variable-p term)) (member term accumulated-variables))

302

                                                                    (gensym "constant"))

303

                                                                   ((and (eq term graph-variable)

304

                                                                         (not first-iteration-p))

305

                                                                    (gensym "constant"))

306

(t

307

                                                                    term))))

308

(uniqued-result-arguments (loop for vars on match-result-arguments

309

                                                       for v in match-result-arguments

310

                                                       collect (if (member v (rest vars))

311

                                                                 (gensym)

312

                                                                 v))))

313

(labels ((local-variable-p (v) (member v match-result-arguments))

314

(wild-variable-p (v) (and (local-variable-p v)

315

                                                     (not (eq v graph)))))

316

                    (let* ((match-pattern (loop for term in (cons graph-variable (statement-terms form))

317

                                                collect (if (wild-variable-p term) wildcard-term term)))

318

                           (concrete-match-pattern (let ((graph-term (first match-pattern)))

319

                                                     (cons graph-term (loop for term in (rest match-pattern)

320

                                                                            collect (if (eq term graph-term)

321

                                                                                      `(max 0 ,term)

322

                                                                                      term)))))

323

                           (accumulated-variables (append accumulated-variables (remove-if-not #'symbol-package match-result-arguments)))

324

(body (cond ((property-path-p (third match-pattern))

325

                                        (error "Property paths are not supported for repository: ~s: ~s." repository (third match-pattern)))

326

327

                                       ((extension-operator-p (third match-pattern))

328

                                        (error "Extension operators are not supported for repository: ~s: ~s." repository (third match-pattern)))

329

                                       ((or (member wildcard-term match-pattern)

330

graph

331

                                            (not (= default-context-term-number rdfcache:*default-context-number*)))

332

                                        ; if variables are in the statement pattern

333

                                        ; or the graph itself is a variable

334

                                        ; or the default graph comrpises the named graphs

335

                                        ; iterate over all matched bindings statement pattern

336

337

                                        (let* ((path-continuation (gensym "MATCH-CONTINUATION"))

338

                                               (ignored-variables (remove-if #'symbol-package uniqued-result-arguments))

339

                                               (match-aliases (loop for v in match-result-arguments

340

                                                                    for v-eq = (rassoc v equivalents)

341

                                                                    when v-eq

342

                                                                    collect v-eq

343

                                                                    and do (setf equivalents (remove v-eq equivalents))))

344

                                               (continuation-form (compute-next-continuation (rest triples) accumulated-variables)))

345

346

;; constrain aliased variables

347

                                          (unless (equal uniqued-result-arguments match-result-arguments)

348

(setf continuation-form

349

                                                  `(when (and ,@(loop for v in match-result-arguments

350

                                                                      for u in uniqued-result-arguments

351

                                                                      unless (eq u v)

352

                                                                      collect `(= ,u ,v)))

353

                                                     ,continuation-form)))

354

(when match-aliases

355

(setf continuation-form

356

                                                  `(let ,(loop for (alias . v) in match-aliases

357

                                                               collect `(,alias ,v))

358

                                                     ,continuation-form)))

359

                                          `(flet ((,path-continuation (,@uniqued-result-arguments)

360

                                                    ,@(when ignored-variables `((declare (ignorable ,@ignored-variables))))

361

                                                    (trace-bgp bgp.field-cache-matched ,@uniqued-result-arguments)

362

                                                    (unless (task-active-p *query*)

363

                                                      (log-debug "bgp premature completion: ~a" *query*)

364

                                                      (complete-solutions))

365

                                                    ,(if (and slice-offset last-triple-p)

366

                                                       `(when (minusp (decf solution-offset))

367

                                                          ,continuation-form)

368

                                                       continuation-form)))

369

                                             (declare (dynamic-extent #',path-continuation))

370

                                             (trace-bgp bgp.field-cache-to-match ',concrete-match-pattern)

371

                                             (match-term-number-field ,repository ,@concrete-match-pattern #',path-continuation))))

372

(t

373

;; just count

374

                                        (let ((continuation-form (compute-next-continuation (rest triples) accumulated-variables))

375

                                              (count-form `(count-term-number-field ,repository ,@concrete-match-pattern)))

376

                                          (unless (equal uniqued-result-arguments match-result-arguments)

377

(setf continuation-form

378

                                                  `(when (and ,@(loop for v in match-result-arguments

379

                                                                      for u in uniqued-result-arguments

380

                                                                      unless (eq u v)

381

                                                                      collect `(= ,u ,v)))

382

                                                     ,continuation-form)))

383

                                          (if (and slice-offset last-triple-p)

384

;; constant slice

385

`(progn (incf match-requests)

386

                                                    (let ((count ,count-form))

387

                                                      (when (and (plusp count)

388

                                                                 (not (plusp (decf solution-offset count))))

389

                                                        (incf match-results)

390

                                                        (incf ,(pop temp-match-counters))

391

                                                        ,continuation-form)))

392

;; constant match

393

`(progn (incf match-requests)

394

                                                    (when (plusp ,count-form)

395

                                                      (incf match-results)

396

                                                      (incf ,(pop temp-match-counters))

397

                                                      ,continuation-form))))))))

398

body))))

399

400

(compute-base-continuation ()

401

;; set the next solution row

402

(let ((collect-form `(collect-solution ,@collection-variables)))

403

(if slice-count

404

`(progn

405

(when (minusp (decf solution-count)) (complete-solutions))

406

,collect-form)

407

collect-form)))

408

409

(compute-next-continuation (triples accumulated-variables)

410

(if triples

411

(let* ((form (first triples))) ; (triple ?s ?p ?o)

412

(ecase (first form)

413

(spocq.a:|triple|

414

;; compule a triple/quad into a query/next continuation

415

(compute-statement-continuation triples accumulated-variables))

416

(spocq.a:|filter|

417

                       ;; compile a filter into a constraint on the solutions which pass through it

418

(compute-filter-continuation triples accumulated-variables))))

419

(compute-base-continuation))))

420

421

(setf iteration-patterns (loop for (tag . spo) in iteration-patterns

422

collect (ecase tag

423

(spocq.a:|triple|

424

                                                 (destructuring-bind (s p o) spo

425

                                                   `(,tag ,(intern-if-constant s)

426

                                                          ,(if (extension-operator-p p)

427

428

                                                             (intern-if-constant p))

429

                                                          ,(intern-if-constant o))))

430

                                                (spocq.a:|filter| (cons tag spo)))))

431

(setf graphs (mapcar #'intern-if-constant graphs))

432

(setf default-graphs (mapcar #'intern-if-constant default-graphs))

433

(setf named-graphs (mapcar #'intern-if-constant named-graphs))

434

       (setf equivalents (loop for (var . value-or-var) in equivalents collect (cons var (intern-if-constant value-or-var))))

435

436

;; allow for just the graph as a variable for literal bgp patterns?

437

(when (variable-p graph)

438

(setf projection-dimensions (union-dimensions (list graph) projection-dimensions)))

439

       (let* (;; the interface operator accepts the repository, and optionally an initial spo solution field, and/or a graph sequence

440

              ;; this wraps an operator which iterates outer over the spo field to bind the starting values for those variables

441

              ;; for each pass and then iterates secondarily over the graph field to perform the pattern matches.

442

;;

443

;; the top-level match form starts with all patterns, _no_ bound variables

444

              ;; corresponding to the patterns. not even the solution variables are bound initially, as they may themselves

445

;; start out as wild cards.

446

(base-bindings (if (or graphs (not graph))

447

                               (union-dimensions (list graph-variable) base-dimensions)

448

base-dimensions))

449

(initial-bindings (union-dimensions base-bindings dynamic-variables))

450

;; this cannot work in this way, as it causes all variables to

451

;; be handled as if they are pre-bound, with the consequence, that

452

;; they are matched, but never extractedd from the cursor...

453

;; (initial-bindings (union-dimensions base-bindings variables))

454

(triple-matching-form (compute-next-continuation iteration-patterns initial-bindings)))

455

456

(when trace

457

           (format *trace-output* "~&*compute-bgp-lambda.trace*~%body ~s~%graph ~s~%named ~s~%default ~s~%graphs ~s~%initial-solution-graph-variable ~s~%graph-variable ~s"

458

                   body graph named-graphs default-graphs graphs initial-solution-graph-variable graph-variable))

459

         ;; the actual query operator takes one of two forms. iff base dimensions are included, it is

460

         ;; generated to expect a solution field source, and to iterate the entire operator over the field's solutions.

461

;; without base dimensions, it runs autonomously.

462

         ;; the combination makes it possible to construct query processing both as reduction through bottom-up combination

463

;; and as solution combination, and even as some combination of the two

464

         ;; where evaluation materialized all intermediate solutions, it did not matter that the reduction order agree

465

         ;; with the matching order. the intermediate matching results were cached for delayed use as combination arguments.

466

         ;; where the solution data flows form one operator tot he next, with the intent to limit the space required for

467

         ;; intermediate fields by streaming results, the contradictory evaluation order would still require buffered

468

;; materialization. for example an expression of the form

469

;;

470

;; (join (?a ?b ?c) (join (?b ?c) (?c)))

471

;;

472

;; would reduce with data flow

473

;;

474

;; ((?b ?c) . (?c) . (?a ?b ?c))

475

;;

476

;; but match with data flow

477

;; (?c) -> (?b ?c) -> (?a ?b ?c)

478

;;

479

;; which entail inverted match orders. the question is, how to enable both?

480

481

(let ((query-lambda

482

;;'(lambda (channel)

483

;; (put-field-page channel nil)

484

;; 0)

485

;;#+(or)

486

`(lambda (bgp-destination ,@(when base-dimensions '(source)))

487

(declare (optimize ,@*field-optimization*))

488

(assert-argument-types bgp-match

489

(bgp-destination (or channel function))

490

,@(when base-dimensions '((source (or channel function)))))

491

;; ,@(when base-dimensions `((print (cons source ',base-dimensions))))

492

(let* ((repository-id (repository-id *repository*))

493

                          (*thread-operations* (cons (list 'spocq.a:|bgp| ',body) *thread-operations*))

494

(match-requests 0)

495

(match-results 0)

496

,@(when slice-offset `((solution-offset ,slice-offset)))

497

,@(when slice-count `((solution-count ,slice-count)))

498

(result-page nil)

499

(result-page-length (channel-page-length bgp-destination))

500

(result-index *field-page-length*)

501

(result-count 0)

502

(last-graph-id 0)

503

(graph-ids-read (make-term-id-cache :single-thread t))

504

(*wildcard-identifier* ,wildcard-term)

505

(*default-context-identifier* ,default-context-term-number)

506

,@(loop for var in match-counters collect `(,var 0))

507

                          ,@(loop for var in (intersection (set-difference dynamic-variables base-bindings) variables)

508

                                  collect `(,var (query-binding-term-number ',var)))

509

                          ;; this allows to specify dynamic variables retrospectively, but suppresses

510

;; extracting them from the cursor

511

#+(or)

512

,@(loop for var in (set-difference variables base-bindings)

513

                                  collect `(,var (query-binding-term-number ',var))))

514

(declare (ignorable repository-id)) ; if just paths

515

(block :bgp-match

516

(labels ((coerce-to-term-id (term-number)

517

(typecase term-number

518

(fixnum term-number)

519

                                     (t (log-warn "bgp match result not a term number: ~s." term-number)

520

+null-term-id+)))

521

(collect-solution ,collection-variables

522

                                  (trace-bgp bgp-match.collect-solution ,@collection-variables)

523

(incf result-count)

524

,(if projection-dimensions

525

                                     `(let ,(loop for (alias . equivalent) in equivalents         ; bind left-over constants

526

                                                  collect (list alias equivalent))

527

(next-solution-location)

528

                                        (locally (declare (type (simple-array fixnum (* ,projection-variable-count)) result-page)

529

                                                          (type fixnum result-index)

530

                                                          (ftype (function (t) fixnum) coerce-to-term-id)

531

                                                          (optimize ,@*field-optimization*))

532

                                          (setf ,@(loop for var-index from 0

533

                                                        for variable in projection-dimensions

534

                                                        nconc `((aref result-page result-index ,var-index)

535

                                                                (coerce-to-term-id ,variable))))))

536

'(next-solution-location)))

537

(next-solution-location ()

538

                                  ;; return a page (possible newly created) and the next free location in that page

539

                                  (when (>= (incf result-index) result-page-length)

540

(when result-page (put-result result-page))

541

                                    (setf result-page (new-field-page bgp-destination result-page-length ,projection-variable-count)

542

result-index 0))

543

(values result-page result-index))

544

(complete-solutions ()

545

                                  (trace-bgp bgp-match.complete-solutions result-count)

546

(incf *match-requests* match-requests)

547

(incf *match-responses* match-results)

548

                                  (log-debug "bgp matches+counts: ~s: requests: ~s, matches: (~s ~s), solutions: ~s"

549

                                             repository-id match-requests match-results (list ,@match-counters) result-count)

550

(when result-page

551

(let ((page-result-count (1+ result-index)))

552

                                      (when (< page-result-count result-page-length)

553

(setf result-page

554

                                              (adjust-page result-page (list page-result-count ,projection-variable-count)))))

555

(put-result result-page))

556

(complete-field bgp-destination)

557

;; (print (list :bgp result-count :complete))

558

(incf-stat *solutions-constructed* result-count)

559

(return-from :bgp-match result-count))

560

(put-result (page)

561

                                  (trace-bgp bgp-match.enqueue bgp-destination ',projection-dimensions page)

562

(put-field-page bgp-destination page)

563

(unless (task-active-p *query*)

564

(complete-field bgp-destination)

565

(return-from :bgp-match result-count))))

566

(trace-bgp bgp.start ',id (task-id *query*)

567

repository-id ',projection-dimensions)

568

(incf-stat *algebra-operations*)

569

;; (print (list :in-bgp repository repository-id))

570

                         ;; unless there are paths present (actually zero-length paths is the issue)

571

                           ;; require an id to perform the query. otherwise there can be no result - skip it

572

(unwind-protect

573

                               (progn ,(let ((iterate-over-graphs `(flet ((match-patterns (,graph-variable)

574

                                                                            ;; if the graph is to be matched, it is rebound in the matching form

575

                                                                            ;; (declare (ignorable ,graph-variable))

576

                                                                            (when (and ,graph-variable

577

                                                                                       (not (eql ,graph-variable last-graph-id)))

578

                                                                              (setf last-graph-id ,graph-variable)

579

                                                                              (setf (gethash ,graph-variable graph-ids-read) t))

580

                                                                            ,triple-matching-form))

581

                                                                     ,(cond (graphs

582

                                                                             ;; if the dataset was specified, use it and respect any initial solution constraint

583

                                                                             (if initial-solution-graph-variable

584

                                                                               `(progn ,@(loop for graph in graphs

585

                                                                                               collect `(cond ((eql ,wildcard-term initial-solution-graph-variable)

586

                                                                                                               (match-patterns ,graph))

587

                                                                                                              ((spocq.e:equal ,initial-solution-graph-variable ,graph)

588

                                                                                                               (return (match-patterns ,graph))))))

589

                                                                               `(progn ,@(loop for graph in graphs

590

                                                                                               collect `(match-patterns ,graph)))))

591

                                                                            (graph

592

                                                                              ;; if in a graph clause, if there was no dataset declared,

593

                                                                              ;; either use an initial soution binding or search.

594

                                                                              ;; otherwise, the effective dataset includes no named graph

595

                                                                              ;; see rdf-sparql-query 8.2.1

596

                                                                              (if initial-solution-graph-variable

597

                                                                                `(match-patterns ,graph)

598

                                                                                (if default-graphs

599

nil

600

                                                                                  `(match-patterns ,named-contexts-term-number))))

601

(t

602

                                                                             ;; outside of a graph clause, the analogous applies

603

                                                                             ;; if there is no dataset clause, use the default graph.

604

                                                                             ;; otherwise the default graph is empty.

605

                                                                             (if named-graphs

606

nil

607

                                                                               `(match-patterns ,default-context-term-number)))))))

608

(if base-dimensions

609

`(do-pages (page source)

610

                                              ,(let ((macros (loop for variable in base-dimensions

611

                                                                   for i from 0

612

                                                                   collect `(,variable (aref page page-index ,i)))))

613

                                                 `(locally (declare (type (simple-array fixnum (* ,(length base-dimensions))) page)

614

                                                                    (optimize ,@*field-optimization*))

615

                                                    (assert (and (typep page '(simple-array fixnum))

616

                                                                 (= (array-dimension page 1) ,(length base-dimensions)))

617

()

618

                                                            "Invalid propagated page: ~s" page)

619

                                                    (trace-data bgp-match.dequeue bgp-destination ',base-dimensions page (term-value-field page))

620

                                                    ;; (let ((*print-array* nil)) (print page))

621

                                                    (loop for page-index from 0 below (array-dimension page 0)

622

                                                          do (symbol-macrolet ,macros

623

                                                               ;; (print (list page-index ,@(loop for variable in base-dimensions append (list  (list 'quote variable) variable))))

624

                                                               ,iterate-over-graphs)))))

625

iterate-over-graphs))

626

(complete-solutions))))

627

(log-warn "incomplete bgp: ~s." ',id)

628

result-count)))))

629

(log-debug "query [~a] bgp [~a] lambda: ~s"

630

(task-id *query*) id query-lambda)

631

query-lambda)))))

632

633