Coverage report: /development/source/library/org/datagraph/spocq-shard/src/spocq-server/streams/model.lisp

Kind	Covered	All	%
expression	0	506	0.0
branch	0	30	0.0

Key

Not instrumented

Conditionalized out

Executed

Not executed

Both branches taken

One branch taken

Neither branch taken

1

 ;;; -*- Mode: lisp; Syntax: ansi-common-lisp; Base: 10; Package: org.datagraph.spocq.server.implementation; -*-

2

;;; (load #p"patches/model.lisp")

3

4

(in-package :org.datagraph.spocq.server.implementation)

5

6

(:documentation "RDF stream semantics.

7

8

This file implements rdf stream processing model as a specialized repository

9

and operators which manage the relation between the assertion content and

10

temporal properties.

11

12

the model and operations evolved from the original rdf stream processing definitions

13

(http://streamreasoning.github.io/RSP-QL/RSP_Requirements_Design_Document/)

14

through a draft of a version extended for transactional stores.

15

16

17

the core model is the 'element' which associates graph data with an

18

'application' and/or 'transaction' time. the element is modeled as a

19

sequence of triple with associated time and transaction identifiers.

20

21

")

22

23

(defpackage :rdf-stream-processing

24

(:use )

25

(:nicknames :rsp)

26

(:export

27

:associative

28

:associative-structure

29

:associative-tree

30

:container

31

:encode-element

32

:format-term

33

:multivalent-structure

34

:graph

35

:dataset

36

:element-identifier

37

:element-temporal-predicate

38

:element-temporal-identifier

39

:element-transactional-predicate

40

:element-transactional-identifier

41

:is-element-of

42

:time-series-graph

43

:persistent-structure

44

:repository

45

:sequence

46

:sequential

47

:solitary

48

:store

49

:stream

50

:structure

51

:structure-associative

52

:structure-sequential

53

:triple

54

55

:cardinality

56

:content

57

:equal

58

:order

59

:name

60

:names

61

:name-map

62

:named-content

63

:statements

64

:default

65

:named

66

67

:is-associative

68

:is-associative-tree

69

:is-dataset

70

:is-graph

71

:is-member-of

72

:is-persistent-structure

73

:is-repository

74

:is-sequence

75

:is-structure

76

:is-solitary

77

:is-store

78

:is-time-series-graph

79

80

:has-default

81

:has-named

82

:default-part

83

:named-part

84

:content-union

85

86

:timestamp-triple

87

:timestamp-predicate

88

89

:precedes

90

:follows

91

:has-least

92

:has-greatest

93

:least

94

:greatest

95

:is-least-of

96

:is-greatest-of

97

:predecessor

98

:successor

99

100

:static-part

101

:dynamic-part

102

:has-static

103

:has-dynamic

104

:initial-version

105

:terminal-version

106

:is-version-of

107

:has-next

108

:has-previous

109

:next

110

:previous

111

:version-names

112

:is-version-name-of

113

:version-content

114

115

:store-transformation

116

:rename

117

:window

118

:extract

119

120

:is-stream

121

122

:tree-union

123

:cons

124

:revise

125

:append

126

:eval

127

:dataset-union

128

:insert-data

129

:create

130

:apply

131

))

132

133

(defun rsp:format-term (stream object &optional colon at)

134

(declare (ignore colon at))

135

(spocq.i::encode-turtle-object object stream))

136

137

;;; maps

138

139

(defgeneric apply-map (map name)

140

(:method ((map function) name) (funcall map name))

141

(:method ((map list) name) (rest (assoc name map :test #'equalp)))

142

(:method ((map hash-table) name) (gethash name map)))

143

144

(defgeneric map-content (function context) )

145

146

(defgeneric exclude (map name)

147

(:method ((map list) name) (remove name map :test #'equalp :key #'first)))

148

149

(defgeneric augment (map name value)

150

(:method ((map list) name value) (acons name value map)))

151

152

(defgeneric restrict (map names)

153

(:method ((map list) names)

154

(flet ((test-name (name) (find name names :test #'equalp)))

155

(declare (dynamic-extent #'test-name))

156

(remove-if #'test-name map :key #'first))))

157

158

(defgeneric rsp:equal (object1 object2)

159

(:method ((o1 t) (o2 t))

160

(equalp o1 o2)))

161

162

(defun set-equal (m1 m2 &key (test #'equal))

163

(= (length (intersection m1 m2 :test test))

164

(length m1)

165

(length m2)))

166

167

168

;;; classes

169

(defun rsp:triple (s p o)

170

`(spocq.a:|triple| ,s ,p ,o))

171

172

(defclass rsp:structure ()

173

()

174

(:documentation "A structure is the abstract root for all combinations of

175

graphs - that is, statements, and names. It comprises associative and

176

sequential combinations, in variations and order hierarchies:"))

177

178

(defclass rsp:associative-structure (rsp:structure)

179

()

180

(:documentation "An associative stricture is an arrangement of names and graphs.

181

It begins at order-0, which is a simple graph, progresses to order-1, which is a simple

182

dataset and evolves through higher orders to represent repositories and stores."))

183

184

(defclass rsp:container ()

185

((content :initform nil :initarg :content

186

:accessor rsp:content))

187

(:documentation "A container encapsulates unspecified content."))

188

189

(defclass rsp:graph (rsp:associative-structure rsp:container)

190

((content :accessor rsp:statements))

191

(:documentation "A graph comprises a set of RDF statements (triples)."))

192

193

(defclass rsp:associative (rsp:associative-structure)

194

((names :initform nil

195

:initarg :names

196

:accessor rsp:names)

197

(name-map :initform nil

198

:initarg :name-map

199

:accessor rsp:name-map)))

200

201

(defclass rsp:associative-tree (rsp:associative)

202

())

203

204

(defclass rsp:dataset (rsp:associative-tree)

205

())

206

207

(defclass rsp:sequence (rdf-stream-container)

208

())

209

210

(defclass rsp:multivalent-structure (rsp:structure)

211

((associative :initform nil

212

:initarg :associative

213

:accessor rsp:structure-associative)

214

(sequential :initform nil

215

:initarg :sequential

216

:accessor rsp:structure-sequential)))

217

218

(defclass rsp:time-series-graph (rsp:dataset)

219

())

220

221

(defclass rsp:persistent-structure (rsp:structure)

222

((static-part :initform nil

223

:accessor rsp:static-part)

224

(dynamic-part :initform nil

225

:accessor rsp:dynamic-part))

226

(:documentation "A parsistent structure combines a static and a dynamic part

227

as the working representation of versioned streaming data.

228

There are two specialized forms, a repository, which adds a time stamp predicate

229

and a store, which comprises repositories."))

230

231

(defclass rsp:repository (rsp:persistent-structure)

232

((timestamp-predicate

233

:initform |prov|:|atTime|

234

:accessor rsp:timestamp-predicate))

235

(:documentation "A repository adds a timestamp predicate to a persistent

236

structure to associate temporal values with its versioned content."))

237

238

(defclass rsp:store (rsp:persistent-structure)

239

()

240

(:documentation "A repository is a versioned collection of repositories."))

241

242

(defclass rsp:stream (rsp:graph)

243

())

244

245

246

;;; structure

247

248

(defmethod rsp:equal ((s1 rsp:structure) (s2 rsp:structure))

249

t)

250

251

252

;;; graph

253

254

(defun rsp:graph (content)

255

(make-instance 'rsp:graph :content content))

256

257

(defmethod print-object ((object rsp:graph) stream)

258

(print-unreadable-object (object stream :identity t :type t)

259

(format stream " x~d" (length (rsp:content object)))))

260

261

(defmethod rsp:equal ((s1 rsp:graph) (s2 rsp:graph))

262

(and (call-next-method)

263

(= (length (intersection (rsp:content s1) (rsp:content s2)

264

:test #'rsp:equal))

265

(length (rsp:content s1))

266

(length (rsp:content s2)))))

267

268

269

;;; associative

270

271

(defun rsp:associative (names name-map)

272

(make-instance 'rsp:associative

273

:names names :name-map name-map))

274

275

(defmethod rsp:equal ((s1 rsp:associative) (s2 rsp:associative))

276

(and (call-next-method)

277

(set-equal (rsp:names s1) (rsp:names s2))

278

(set-equal (rsp:name-map s1) (rsp:name-map s2))))

279

280

(defgeneric rsp:is-associative (object)

281

(:method ((object rsp:associative)) t)

282

(:method ((object t)) nil))

283

284

(defmethod apply-map ((structure rsp:associative) name)

285

(apply-map (rsp:name-map structure) name))

286

287

(defgeneric rsp:named-content (name associative)

288

(:method ((name t) (object rsp:associative))

289

(apply-map (rsp:name-map object) name)))

290

291

(defgeneric rsp:default-part (structure)

292

(:method ((structure t))

293

nil)

294

(:method ((structure rsp:associative))

295

(apply-map (rsp:name-map structure) <urn:dydra:default>)))

296

297

(defgeneric rsp:named-part (structure)

298

(:method ((structure t))

299

nil)

300

(:method ((structure rsp:associative))

301

(rsp:associative (rsp:names structure)

302

(exclude (rsp:name-map structure) <urn:dydra:default>))))

303

304

(defgeneric rsp:cardinality (structure)

305

(:method ((object t))

306

nil)

307

(:method ((structure rsp:associative))

308

(length (rsp:names structure))))

309

310

(defun rsp:solitary (name structure)

311

(rsp:associative (list name) (acons name structure nil)))

312

313

(defgeneric rsp:is-solitary (object)

314

(:method ((object t))

315

nil)

316

(:method ((structure rsp:associative))

317

(when (rsp:is-associative structure)

318

(and (= (rsp:cardinality structure) 1)

319

(null (rsp:default-part structure))))))

320

321

322

;;; associative-tree

323

324

(defun rsp:associative-tree (names name-map)

325

(make-instance 'rsp:associative-tree

326

:names names :name-map name-map))

327

328

(defgeneric rsp:is-associative-tree (object)

329

(:method ((object rsp:associative-tree)) t)

330

(:method ((object t)) nil))

331

332

333

;;; dataset

334

335

(defun rsp:dataset (names name-map)

336

(make-instance 'rsp:dataset

337

:names names :name-map name-map))

338

339

(defgeneric rsp:is-dataset (object)

340

(:method ((object rsp:dataset)) t)

341

(:method ((object t)) nil))

342

343

344

;;; time-series-graph

345

346

(defun rsp:tree-union (graph triple)

347

(cons triple graph))

348

349

(defun rsp:time-series-graph (default-graph timestamp-triple named-graph)

350

(destructuring-bind (s p o) (spocq.i::triple-terms timestamp-triple)

351

(declare (ignore o))

352

(let ((name (cons-blank-node)))

353

(rsp:multivalent-structure (rsp:associative (list name)

354

                                                   (rsp:multivalent-structure

355

                                                    (rsp:associative (list s)

356

                                                                     (acons name named-graph nil))

357

                                                    (rsp:tree-union default-graph timestamp-triple)))

358

                                  (rsp:graph (list (rsp:triple name <http://www.w3.org/ns/rsp#usesTimestampPredicate> p)))))))

359

360

(defgeneric rsp:is-time-series-graph (object)

361

(:method ((object rsp:time-series-graph)) t)

362

(:method ((object t)) nil))

363

364

;;; sequence

365

366

(defun rsp:sequence (content)

367

(make-instance 'rsp:sequence :content content))

368

369

(defgeneric rsp:is-sequence (object)

370

(:method ((object rsp:sequence))

371

t)

372

(:method ((object t))

373

nil))

374

375

376

(defgeneric rsp:is-member-of (element sequence)

377

(:method (element (sequence rsp:sequence))

378

(find element (rsp:content sequence))))

379

380

(defgeneric rsp:has-greatest (sequence)

381

(:method ((sequence rsp:sequence))

382

(not (null (rsp:content sequence)))))

383

384

(defgeneric rsp:has-least (sequence)

385

(:method ((sequence rsp:sequence))

386

(not (null (rsp:content sequence)))))

387

388

(defgeneric rsp:precedes (element1 element2 sequence)

389

(:method (element1 element2 (sequence rsp:sequence))

390

(rsp:precedes element1 element2 (rsp:content sequence)))

391

(:method (element1 element2 (sequence t))

392

(equal element1 (rsp:predecessor element2 sequence))))

393

394

(defgeneric rsp:follows (element1 element2 sequence)

395

(:method (element1 element2 (sequence rsp:sequence))

396

(rsp:follows element1 element2 (rsp:content sequence)))

397

(:method (element1 element2 (sequence t))

398

(equal element1 (rsp:successor element2 sequence))))

399

400

(defgeneric rsp:least (sequence)

401

(:method ((sequence rsp:sequence))

402

(rsp:least (rsp:content sequence)))

403

(:method ((sequence vector))

404

(when (plusp (length sequence))

405

(aref sequence 0)))

406

(:method ((sequence list))

407

(first sequence)))

408

409

(defgeneric rsp:greatest (sequence)

410

(:method ((sequence rsp:sequence))

411

(rsp:greatest (rsp:content sequence)))

412

(:method ((sequence vector))

413

(when (plusp (length sequence))

414

(aref sequence (1- (length sequence)))))

415

(:method ((sequence list))

416

(first (last sequence))))

417

418

(defgeneric rsp:is-least-of (element sequence)

419

(:method (element (sequence rsp:sequence))

420

(rsp:is-least-of element (rsp:content sequence)))

421

(:method (element (sequence t))

422

(eql element (rsp:least sequence))))

423

424

(defgeneric rsp:is-greatest-of (element sequence)

425

(:method (element (sequence rsp:sequence))

426

(rsp:is-greatest-of element (rsp:content sequence)))

427

(:method (element (sequence t))

428

(eql element (rsp:greatest sequence))))

429

430

(defgeneric rsp:predecessor (element sequence)

431

(:method (element (sequence rsp:sequence))

432

(rsp:predecessor element (rsp:content sequence)))

433

(:method (element (sequence vector))

434

(let ((position (position element sequence)))

435

(when (and position (> position 0))

436

(elt sequence (1- position)))))

437

(:method (element (sequence list))

438

(loop with predecessor = nil

439

for member in sequence

440

if (equal element member)

441

return predecessor

442

else do (setf predecessor member))))

443

444

(defgeneric rsp:successor (element sequence)

445

(:method (element (sequence rsp:sequence))

446

(rsp:successor element (rsp:content sequence)))

447

(:method (element (sequence vector))

448

(let ((position (position element sequence)))

449

(when (and position (< position (1- (length sequence))))

450

(aref sequence (1+ position)))))

451

(:method (element (sequence list))

452

(loop for sequence on sequence

453

when (equal element (first sequence))

454

return (second sequence))))

455

456

457

;;; data-structure

458

459

(defun rsp:multivalent-structure (associative sequential)

460

(make-instance 'rsp:multivalent-structure

461

:associative associative

462

:sequential sequential))

463

464

;;; manipulation

465

466

(defgeneric rsp:cons (element sequence)

467

(:method ((element t) (sequence rsp:sequence))

468

(clone-instance sequence :content (rsp:cons element (rsp:content sequence))))

469

(:method ((element t) (sequence list))

470

(cons element sequence))

471

(:method ((element t) (sequence vector))

472

(setf sequence (adjust-array sequence (list (1+ (length sequence))))

473

(aref sequence (1- (length sequence))) element)

474

sequence))

475

476

(defgeneric rsp:revise (operator sequence)

477

(:method ((operator function) (sequence rsp:sequence))

478

(rsp:cons (funcall operator (rsp:greatest sequence))

479

sequence)))

480

481

482

;;; base update operators

483

484

(defmethod repository-add-graph (transaction

485

(from-graph sequence-property-path)

486

(to-graph sequence-property-path)

487

&key if-does-not-exist)

488

(declare (ignore if-does-not-exist))

489

spocq.a:|true|)

490

491

(defmethod repository-clear-graph (transaction (graph-designator sequence-property-path)

492

&key if-does-not-exist)

493

(declare (ignore if-does-not-exist))

494

spocq.a:|true|)

495

496

(defmethod repository-copy-graph (transaction*

497

(from-graph sequence-property-path)

498

(to-graph sequence-property-path)

499

&key if-does-not-exist)

500

(declare (ignore if-does-not-exist))

501

spocq.a:|true|)

502

503

(defmethod repository-create-graph (transaction (graph-designator sequence-property-path)

504

&key if-exists)

505

(declare (ignore if-exists))

506

spocq.a:|true|)

507

508

509

(defmethod repository-delete-graph (transaction (graph-designator sequence-property-path)

510

                                                 &key if-does-not-exist)

511

(declare (ignore if-does-not-exist))

512

spocq.a:|true|)

513

514

515

(defmethod spocq.e:delete-data ((structure rsp:structure))

516

(repository-delete-field (task-repository *query*) structure)

517

spocq.a:|true|)

518

519

(defmethod repository-delete-field (transaction (structure rsp:structure))

520

spocq.a:|true|)

521

522

;; delete where is not extended

523

524

525

;;; specialize insert data for the respective structure forms

526

;;; each isolates the respective component and perform the operation oon that

527

(defmethod spocq.e:insert-data ((data rsp:container))

528

(repository-insert-field (task-repository *query*) data)

529

spocq.a:|true|)

530

531

(defmethod repository-insert-field (transaction (data rsp:container))

532

"Insert the container-content into the dataset corresponding to the transaction"

533

(repository-insert-field transaction (rsp:content data)))

534

535

536

(defmethod spocq.e:load-graph (location (graph-designator sequence-property-path) &key (verbose t))

537

(declare (ignore verbose))

538

spocq.a:|true|)

539

540

;;; spocq.e:modify

541

;;; devolves to compute-modify-operators

542

;;; which computes functions which operate on the solution field sets

543

;;; that operator must be extended for structures

544

545

(defmethod repository-move-graph (transaction*

546

(from-graph sequence-property-path)

547

(to-graph sequence-property-path)

548

&key if-does-not-exist)

549

(declare (ignore if-does-not-exist))

550

spocq.a:|true|)

551

552

;;; persistent-structure

553

554

(defgeneric rsp:content-union (dataset )

555

)

556

557

558

(defclass rsp::element (rsp:graph) ;!!! could delegate to the graph element rather than inheriting

559

((identifier

560

:initform nil

561

:accessor rsp::element-identifier)))

562

563

(defclass rsp::temporal-element (rsp::element)

564

((temporal-identifier

565

:initform (error "temporal-identifier is required")

566

:accessor rsp::element-temporal-identifier)

567

(temporal-predicate

568

:initform (error "temporal-predicate is required")

569

:accessor rsp::element-temporal-predicate)))

570

(defclass rsp::transactional-element (rsp::element)

571

((transactional-identifier

572

:initform (error "transactional-identifier is required")

573

:accessor rsp::element-transactional-identifier)

574

(transactional-predicate

575

:initform (error "transactional-predicate is required")

576

:accessor rsp::element-transactional-predicate)))

577

(defclass rsp::stream-element (rsp::temporal-element rsp::transactional-element)

578

())

579

580

(defgeneric rsp::encode-element (element media-type stream)

581

(:documentation "encode the element graph, including temporal and transactional identifiers")

582

(:method ((element rsp::temporal-element) (media-type mime:rdf) (stream t))

583

(rsp:encode-element `(spocq:quad ,(rsp:element-identifier element)

584

,(rsp:element-temporal-predicate element)

585

,(rsp:element-temporal-identifier element))

586

media-type

587

stream)

588

(call-next-method))

589

(:method ((element rsp::temporal-element) (media-type mime:rdf) (stream t))

590

(rsp:encode-element `(spocq:quad ,(rsp:element-identifier element)

591

                                      ,(rsp:element-transactional-predicate element)

592

                                      ,(rsp:element-transactional-identifier element))

593

media-type

594

stream)

595

(call-next-method))

596

(:method ((element rsp::element) (media-type mime:application/n-quads) stream)

597

(loop for statement in (rsp:content element)

598

do (rsp::encode-element statement media-type stream)))

599

(:method ((element rsp::element) (media-type mime:application/trig) stream)

600

(format stream "~/rsp:format-term/ {" (rsp::element-identifier element))

601

(loop for statement in (rsp:content element) ;

602

do (rsp::encode-element statement media-type stream))

603

(write-string "}" stream))

604

)

605

606

607

(defgeneric rsp:is-persistent-structure (object)

608

(:method ((instance rsp:persistent-structure))

609

t)

610

(:method ((instance t))

611

nil))

612

613

(defgeneric rsp:has-static (structure)

614

(:method ((structure rsp:persistent-structure))

615

(spocq.i::test-rlmdb-pattern-match (rsp:static-part structure) 0 0 0 0)))

616

617

(defgeneric rsp:has-dynamic (structure)

618

(:method ((structure rsp:persistent-structure))

619

(spocq.i::test-rlmdb-pattern-match (rsp:dynamic-part structure) 0 0 0 0)))

620

621

(defgeneric rsp:is-element-of (element structure)

622

(:method-combination and)

623

(:method and ((element rsp::temporal-element) (structure rsp:persistent-structure))

624

)

625

(:method and ((element rsp::transactional-element) (structure rsp:persistent-structure))

626

(equalp (run-sparql `(spocq.a:|ask|

627

(spocq.a:|bgp| ,(rsp:statements element)))

628

:agent spocq.i::*agent*

629

:repository (rsp:dynamic-part structure)

630

                         :revision-identifier (rsp::element-transactional-identifier element))

631

spocq:|true|))

632

(:method and ((element rsp::temporal-element) (structure rsp:persistent-structure))

633

(equalp (run-sparql `(spocq.a:|ask|

634

(spocq.a:|bgp| ,(rsp:statements element)))

635

:agent spocq.i::*agent*

636

:repository (rsp:dynamic-part structure)

637

:temporal-identifier (rsp::element-temporal-identifier element))

638

spocq:|true|))

639

(:method and ((element rsp::stream-element) (structure rsp:persistent-structure))

640

(equalp (run-sparql `(spocq.a:|ask|

641

(spocq.a:|bgp| ,(rsp:statements element)))

642

:agent spocq.i::*agent*

643

:repository (rsp:dynamic-part structure)

644

:temporal-identifier (rsp::element-temporal-identifier element)

645

                         :revision-identifier (rsp::element-transactional-identifier element))

646

spocq:|true|))

647

)

648

649

650

651

652

653

654

655

656

657

658

659