Coverage report: /development/source/library/org/datagraph/spocq-shard/src/core/libraries/rdfhdt.lisp

Kind	Covered	All	%
expression	4	539	0.7
branch	0	34	0.0

Key

Not instrumented

Conditionalized out

Executed

Not executed

Both branches taken

One branch taken

Neither branch taken

1

;;;

2

3

(in-package :common-lisp-user)

4

5

(cffi:define-foreign-library hdt_spocq

6

(:unix "libhdt_lisp.so"))

7

;;; (cffi:load-foreign-library #p"/opt/spocq/libhdt_lisp.so")

8

9

(cl:defpackage :hdt

10

(:use :cffi)

11

(:export

12

;; library operators

13

;; libhdt

14

15

;; libhdt_lisp : nm libhdt_spocq.so | fgrep ' T ' | cut -d ' ' -f 3 | sort

16

:%id-to-string

17

:%string-to-id

18

:%create-triple-id

19

:%delete-triple-id

20

:%set-triple-id-all

21

:%get-triple-id-object

22

:%set-triple-id-object

23

:%create-triple-string

24

:%delete-triple-string

25

:%get-triple-id-subject

26

:%set-triple-id-subject

27

:%set-triple-string-all

28

:%get-triple-id-predicate

29

:%set-triple-id-predicate

30

:%get-triple-string-object

31

:%set-triple-string-object

32

:%create-id-triple-id-cursor

33

:%get-triple-string-subject

34

:%set-triple-string-subject

35

:%triple-id-to-triple-string

36

:%triple-string-to-triple-id

37

:%get-triple-string-predicate

38

:%set-triple-string-predicate

39

:%term-literal-term-number

40

:%term-number-term-literal

41

:%call-with-id-string

42

43

:basic-hdt

44

:%map-indexed-hdt-progress-listener

45

:%delete-hdt

46

:map-indexed-hdt

47

:%create-spo-id-triple-id-cursor

48

:%create-triple-id-triple-id-cursor

49

:%create-spo-string-triple-id-cursor

50

:%create-spo-string-triple-string-cursor

51

:%create-triple-string-triple-string-cursor

52

53

:dictionary

54

:%triples-get-number-of-elements

55

:%triple-id-cursor-get-number-of-elements

56

:%id-cursor-close

57

:%id-cursor-count

58

:%id-cursor-estimated-num-results

59

:%id-cursor-has-next

60

:%id-cursor-next

61

:%id-cursor-next-matrix

62

:%string-cursor-close

63

:%string-cursor-estimated-num-results

64

:%string-cursor-has-next

65

:%string-cursor-next

66

67

:triple-id-string-iterator

68

:triple-string

69

:%map-subject-numbers

70

:%map-subjects

71

:%map-predicate-numbers

72

:%map-object-numbers

73

:%map-repository-statements

74

:%map-repository-statements*

75

:triples

76

:subject

77

:predicate

78

:object

79

:cursor

80

:triple

81

:triple-id

82

83

:%dictionary-string-to-id

84

:%hdt-search*

85

:%hdt-search

86

:%statement-count

87

:%pattern-count

88

:triple-component-role

89

:file-map

90

91

:%hdt-get-triples

92

:%hdt-get-dictionary

93

:%hdt-statement-count

94

:%hdt-pattern-count

95

:%hdt-read-id-pattern-count

96

:%hdt-read-term-pattern-count

97

:%hdt-map-triple-strings

98

:%hdt-map-triple-strings*

99

:%hdt-map-triple-ids

100

:%hdt-get-n-subjects

101

:%hdt-get-n-properties

102

:%hdt-get-n-objects

103

))

104

105

(cl:in-package :hdt)

106

107

(cl:eval-when (:compile-toplevel :load-toplevel :execute)

108

(cl:defun cffi::warn-if-kw-or-belongs-to-cl (name) (cl:declare (cl:ignore name)) cl:nil))

109

110

(cffi:defctype :size_t :ullong "std::size_t")

111

112

(cffi:defcenum triple-component-role

113

:subject

114

:predicate

115

:object)

116

117

(cffi:defcstruct basic-hdt )

118

(cffi:defcstruct dictionary )

119

(cffi:defcstruct triples )

120

(cffi:defcstruct file-map )

121

(cffi:defcstruct cursor )

122

(cffi:defcstruct triple-id )

123

(cffi:defcstruct triple-string )

124

(cffi:defcstruct iterator-triple-id )

125

(cffi:defcstruct iterator-triple-string)

126

;;; libhdt_lisp

127

128

(cffi:defcfun ("_Z10idToStringPN3hdt10DictionaryEmNS_19TripleComponentRoleE" %id-to-string) :string

129

(dictionary :pointer)

130

(term-id :size_t)

131

(component hdt:triple-component-role))

132

 (cffi:defcfun ("_Z16callWithidStringPFmPKcmEPN3hdt10DictionaryEmNS3_19TripleComponentRoleE" hdt::%call-with-id-string) :size_t

133

(continuation :pointer)

134

(dictionary :pointer)

135

(term-id :size_t)

136

(component hdt:triple-component-role))

137

138

139

140

(cffi:defcfun ("_Z10stringToIdPN3hdt10DictionaryEPcNS_19TripleComponentRoleE" %string-to-id) :size_t

141

(dictionary :pointer)

142

(term-value :string)

143

(component hdt:triple-component-role))

144

(cffi:defcfun ("_Z14createTripleIDmmm" %create-triple-id) (:pointer (:struct triple-id))

145

(subject :size_t)

146

(predicate :size_t)

147

(object :size_t)

148

(component hdt:triple-component-role))

149

(cffi:defcfun ("_Z14deleteTripleIDPN3hdt8TripleIDE" %delete-triple-id) :void

150

(triple (:pointer (:struct triple-id))))

151

(cffi:defcfun ("_Z14setTripleIDAllPN3hdt8TripleIDEmmm" %set-triple-id-all) (:pointer (:struct triple-id))

152

(triple (:pointer (:struct triple-id)))

153

(subject :size_t)

154

(predicate :size_t)

155

(object :size_t))

156

(cffi:defcfun ("_Z17getTripleIDObjectPN3hdt8TripleIDE" %get-triple-id-object) :size_t

157

(triple (:pointer (:struct triple-id))))

158

(cffi:defcfun ("_Z17setTripleIDObjectPN3hdt8TripleIDEm" %set-triple-id-object) :size_t

159

(triple (:pointer (:struct triple-id)))

160

(object :size_t))

161

(cffi:defcfun ("_Z18createTripleStringPcS_S_" %create-triple-string) (:pointer (:struct triple-string))

162

(subject :string)

163

(predicate :string)

164

(object :string))

165

(cffi:defcfun ("_Z18deleteTripleStringPN3hdt12TripleStringE" %delete-triple-string) :void

166

(triple (:pointer (:struct triple-string))))

167

(cffi:defcfun ("_Z18getTripleIDSubjectPN3hdt8TripleIDE" %get-triple-id-subject) :size_t

168

(triple (:pointer (:struct triple-id))))

169

(cffi:defcfun ("_Z18setTripleIDSubjectPN3hdt8TripleIDEm" %set-triple-id-subject) :size_t

170

(triple (:pointer (:struct triple-id)))

171

(subject :size_t))

172

 (cffi:defcfun ("_Z18setTripleStringAllPN3hdt12TripleStringEPcS2_S2_" %set-triple-string-all) (:pointer (:struct triple-string))

173

(triple (:pointer (:struct triple-string)))

174

(subject :string)

175

(predicate :string)

176

(object :string))

177

178

 (cffi:defcfun ("_Z25createSPOIDTripleIDCursorPN3hdt3HDTEmmm" %create-spo-id-triple-id-cursor) (:pointer (:struct iterator-triple-id))

179

(hdt (:pointer (:struct basic-hdt)))

180

(subject :size_t)

181

(predicate :size_t)

182

(object :size_t))

183

(cffi:defcfun ("_Z23iteratorTripleIDHasNextPN3hdt16IteratorTripleIDE" %id-cursor-has-next) :boolean

184

(cursor (:pointer (:struct iterator-triple-id))))

185

 (cffi:defcfun ("_Z20iteratorTripleIDNextPN3hdt16IteratorTripleIDE" %id-cursor-next) (:pointer (:struct triple-id))

186

(cursor (:pointer (:struct cursor))))

187

 (cffi:defcfun ("_Z35iteratorTripleIDEstimatedNumResultsPN3hdt16IteratorTripleIDE" %id-cursor-estimated-num-results) :size_t

188

(cursor (:pointer (:struct iterator-triple-id))))

189

(cffi:defcfun ("_Z27iteratorTripleIDCursorClosePN3hdt16IteratorTripleIDE" %id-cursor-close) :void

190

(cursor (:pointer (:struct iterator-triple-id))))

191

192

(cffi:defcfun ("_Z20getTripleIDPredicatePN3hdt8TripleIDE" %get-triple-id-predicate) :size_t

193

(triple (:pointer (:struct triple-id))))

194

(cffi:defcfun ("_Z20setTripleIDPredicatePN3hdt8TripleIDEm" %set-triple-id-predicate) :size_t

195

(triple (:pointer (:struct triple-id)))

196

(subject :size_t))

197

198

199

(cffi:defcfun ("_Z21getTripleStringObjectPN3hdt12TripleStringE" %get-triple-string-object) :string

200

(triple-string (:pointer (:struct triple-string))))

201

(cffi:defcfun ("_Z21setTripleStringObjectPN3hdt12TripleStringEPc" %set-triple-string-object) :string

202

(triple-string (:pointer (:struct triple-string)))

203

(subject :string))

204

(cffi:defcfun ("_Z22getTripleStringSubjectPN3hdt12TripleStringE" %get-triple-string-subject) :string

205

(triple (:pointer (:struct triple-string))))

206

(cffi:defcfun ("_Z22setTripleStringSubjectPN3hdt12TripleStringEPc" %set-triple-string-subject) :string

207

(triple (:pointer (:struct triple-string)))

208

(subject :string))

209

210

211

212

 (cffi:defcfun ("_Z22tripleIDtoTripleStringPN3hdt3HDTEPKNS_8TripleIDEPNS_12TripleStringE" %triple-id-to-triple-string) (:pointer (:struct triple-string))

213

(hdt (:pointer (:struct basic-hdt)))

214

(triple-id (:pointer (:struct triple-string)))

215

(triple-string (:pointer (:struct triple-string))))

216

 (cffi:defcfun ("_Z22tripleStringtoTripleIDPN3hdt3HDTEPKNS_12TripleStringEPNS_8TripleIDE" %triple-string-to-triple-id) (:pointer (:struct triple-id))

217

(hdt (:pointer (:struct basic-hdt)))

218

(triple-string (:pointer (:struct triple-string)))

219

(triple-id (:pointer (:struct triple-string))))

220

221

(cffi:defcfun ("_Z24deleteTripleStringCursorPN3hdt20IteratorTripleStringE" %string-cursor-close) :void

222

(cursor (:pointer (:struct iterator-triple-string))))

223

224

(cffi:defcfun ("_Z24getTripleStringPredicatePN3hdt12TripleStringE" %get-triple-string-predicate) :string

225

(triple (:pointer (:struct triple-string))))

226

(cffi:defcfun ("_Z24setTripleStringPredicatePN3hdt12TripleStringEPc" %set-triple-string-predicate) :string

227

(triple (:pointer (:struct triple-string)))

228

(subject :string))

229

230

(cffi:defcfun ("_Z26iteratorTripleIDNextMatrixPN3hdt16IteratorTripleIDEPmi" %id-cursor-next-matrix) :int

231

(cursor (:pointer (:struct iterator-triple-id)))

232

(matrix :pointer)

233

(size :int))

234

235

(cffi:defcfun ("_Z21iteratorTripleIDCountPN3hdt16IteratorTripleIDE" %id-cursor-count) :int

236

(cursor (:pointer (:struct iterator-triple-id))))

237

238

(cffi:defcfun ("_Z25tripleStringCursorHasNextPN3hdt20IteratorTripleStringE" %string-cursor-has-next) :boolean

239

(cursor (:pointer (:struct iterator-triple-string))))

240

241

 (cffi:defcfun ("_ZN3hdt22TripleIDStringIterator19estimatedNumResultsEv" %string-cursor-estimated-num-results) :size_t

242

(cursor (:pointer (:struct iterator-triple-string))))

243

244

245

246

247

 (cffi:defcfun ("_Z28createTripleIDTripleIDCursorPN3hdt3HDTEPNS_8TripleIDE" %create-triple-id-triple-id-cursor) (:pointer (:struct iterator-triple-id))

248

(hdt (:pointer (:struct basic-hdt)))

249

(triple (:pointer (:struct triple-id))))

250

 (cffi:defcfun ("_Z29createSPOStringTripleIDCursorPN3hdt3HDTEPcS2_S2_" %create-spo-string-triple-id-cursor) (:pointer (:struct iterator-triple-id))

251

(hdt (:pointer (:struct basic-hdt)))

252

(subject :string)

253

(predicate :string)

254

(object :string))

255

 (cffi:defcfun ("_Z33createSPOStringTripleStringCursorPN3hdt3HDTEPcS2_S2_" %create-spo-string-triple-string-cursor) (:pointer (:struct iterator-triple-string))

256

(hdt (:pointer (:struct basic-hdt)))

257

(subject :string)

258

(predicate :string)

259

(object :string))

260

 (cffi:defcfun ("_Z36createTripleStringTripleStringCursorPN3hdt3HDTEPNS_12TripleStringE" %create-triple-string-triple-string-cursor) (:pointer (:struct iterator-triple-string))

261

(hdt (:pointer (:struct basic-hdt)))

262

(triple (:pointer (:struct triple-string))))

263

264

#|

265

(cffi:defcfun ("_ZN3hdt27BitmapTriplesIterator7hasNextEv" %id-cursor-has-next) :bool

266

(cursor (:pointer (:struct cursor))))

267

(cffi:defcfun ("_ZN3hdt22TripleIDStringIterator7hasNextEv" %string-cursor-has-next) :bool

268

(cursor (:pointer (:struct cursor))))

269

|#

270

271

272

(cffi:defcfun ("_ZN3hdt22TripleIDStringIterator4nextEv" %string-cursor-next) (:pointer (:struct triple-string))

273

(cursor (:pointer (:struct cursor))))

274

275

276

 (cffi:defcfun ("_ZN3hdt10HDTManager13mapIndexedHDTEPKcPNS_16ProgressListenerE" hdt:%map-indexed-hdt-progress-listener) (:pointer (:struct basic-hdt))

277

(pathname :string)

278

(listener :pointer))

279

(cffi:defcfun ("_Z9deleteHDTPN3hdt3HDTE" %delete-hdt) :void

280

(hdt (:pointer (:struct basic-hdt))))

281

282

#+(or)

283

 (cffi:defcfun ("_ZN3hdt21FourSectionDictionary10stringToIdERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEENS_19TripleComponentRoleE" hdt:%dictionary-string-to-id) :long

284

(dictionary :pointer)

285

(string :string)

286

(component hdt:triple-component-role))

287

288

 (cffi:defcfun ("_ZN3hdt21FourSectionDictionary10idToStringB5cxx11EmNS_19TripleComponentRoleE" hdt:%dictionary-string-to-id) :string

289

(dictionary :pointer)

290

(term-number :size_t)

291

(component hdt:triple-component-role))

292

293

(cffi:defcfun ("_ZN3hdt8BasicHDT13getDictionaryEv" %hdt-get-dictionary) :pointer (hdt :pointer))

294

(cffi:defcfun ("_ZN3hdt8BasicHDT10getTriplesEv" %hdt-get-triples) :pointer (hdt :pointer))

295

296

(cffi:defcfun ("_Z15getHDTNsubjectsPN3hdt3HDTE" %hdt-get-n-subjects) :size_t (hdt :pointer))

297

(cffi:defcfun ("_Z17getHDTNpropertiesPN3hdt3HDTE" %hdt-get-n-properties) :size_t (hdt :pointer))

298

(cffi:defcfun ("_Z14getHDTNobjectsPN3hdt3HDTE" %hdt-get-n-objects) :size_t (hdt :pointer))

299

300

(cffi:defcfun ("_ZN3hdt8BasicHDT6searchEPKcS2_S2_" %hdt-search*) :pointer

301

(hdt :pointer)

302

(subject :string)

303

(predicate :string)

304

(object :string))

305

306

;;; requires a virtual operator:

307

;;; alternatively ZNK3hdt13BitmapTriples19getNumberOfElementsEv

308

(cffi:defcfun ("_ZNK3hdt12PlainTriples19getNumberOfElementsEv" %triples-get-number-of-elements) :size_t

309

(triples :pointer))

310

(cffi:defcfun ("_ZN3csd11PFCIterator19getNumberOfElementsEv" %triple-id-cursor-get-number-of-elements) :size_t

311

(iterator :pointer))

312

313

314

(cl:in-package :spocq.i)

315

316

(defun hdt-boxed-term-number-role (term-number)

317

"derive the role from the term number range"

318

(ecase (logand term-number #b11)

319

(#b00 :graph)

320

(#b01 :subject)

321

(#b10 :predicate)

322

(#b11 :object)))

323

(defun hdt-box-term-number (term-number role)

324

(+ (ash term-number 2) (ecase role

325

(:graph #b00)

326

(:subject #b01)

327

(:predicate #b10)

328

(:object #b11))))

329

(defun hdt-unbox-term-number (term-number)

330

(ash term-number -2))

331

332

333

(defun hdt:%term-literal-term-number (%dictionary term-literal role)

334

;; when string-to-id returns a zero, the term is not present

335

;; in that case, return nil

336

(let ((id (hdt:%string-to-id %dictionary term-literal role)))

337

(unless (zerop id)

338

(hdt-box-term-number id role))))

339

340

(defparameter *hdt-term-literal* nil)

341

342

(cffi:defcallback hdt::%set-term-literal :size_t

343

((ptr :pointer) (size :size_t))

344

(handler-case

345

(let ((literal (cffi:foreign-string-to-lisp ptr :count size)))

346

(setq *hdt-term-literal* literal)

347

size)

348

(error (c)

349

;; log- macros di not exist yed

350

(warn "hdt::%set-term-literal error: ~a" c)

351

0)))

352

353

(defun hdt:%term-number-term-literal (%dictionary term-number)

354

(let ((*hdt-term-literal* nil))

355

(hdt:%call-with-id-string

356

(cffi:callback hdt::%set-term-literal)

357

%dictionary (hdt-unbox-term-number term-number) (hdt-boxed-term-number-role term-number))

358

*hdt-term-literal*))

359

360

(cffi:defcallback hdt::%intern-term :size_t

361

((term-number :size_t) (ptr :pointer) (size :size_t))

362

(let ((literal nil)

363

(object nil))

364

(declare (special *spocq->store-term-registry*

365

*store->spocq-term-registry*))

366

(flet ((cache-object ()

367

(setcache object *spocq->store-term-registry* term-number)

368

(setcache term-number *store->spocq-term-registry* object)))

369

(handler-case

370

(progn (setf literal (cffi:foreign-string-to-lisp ptr :count size))

371

(setf object (case (char literal 0)

372

(#\" (parse-term literal))

373

(#\_ (intern-blank-node (subseq literal 2)))

374

(t (intern-iri literal))))

375

(cache-object)

376

term-number)

377

(error (c)

378

                (cond (object (warn "hdt::%intern-term: error caching term object: ~a: ~s" c object))

379

(literal

380

                       (setf object (spocq:make-unsupported-typed-literal literal |http://www.w3.org/2001/XMLSchema|:|anyType|))

381

(cache-object))

382

(t

383

(warn "hdt::%intern-term: error: ~a" c)))

384

0)))))

385

386

387

(defgeneric hdt:map-indexed-hdt (pathname)

388

(:method ((pathname cl:pathname))

389

(hdt:map-indexed-hdt (cl:namestring pathname)))

390

(:method ((pathname cl:string))

391

;; rely on the intermidiate listener to avoid that matter

392

(hdt:%map-indexed-hdt-progress-listener pathname (cffi:null-pointer))))

393

394

395

(defun hdt:%hdt-statement-count (%hdt-environment)

396

(hdt:%triples-get-number-of-elements (hdt:%hdt-get-triples %hdt-environment)))

397

398

(defun hdt:%hdt-pattern-count (%hdt-environment subject predicate object)

399

(let ((%cursor (hdt:%create-spo-string-triple-id-cursor %hdt-environment subject predicate object)))

400

(when %cursor

401

#+(or)

402

       (unwind-protect (hdt:%id-cursor-estimated-num-results %cursor) ; (hdt:%triple-id-cursor-get-number-of-elements %cursor)

403

(hdt:%id-cursor-close %cursor))

404

        (unwind-protect (hdt:%id-cursor-count %cursor) ; (hdt:%triple-id-cursor-get-number-of-elements %cursor)

405

(hdt:%id-cursor-close %cursor)))))

406

407

(defparameter hdt::*pattern-count-mode* :estimate)

408

;;; (defparameter hdt::*pattern-count-mode* :scan)

409

410

(defun hdt:%hdt-read-id-pattern-count (%hdt-environment subject predicate object)

411

(let ((%cursor (hdt:%create-spo-id-triple-id-cursor %hdt-environment subject predicate object)))

412

(when %cursor

413

(unwind-protect

414

(ecase hdt::*pattern-count-mode*

415

(:scan

416

(hdt:%id-cursor-count %cursor))

417

(:estimate

418

(hdt:%id-cursor-estimated-num-results %cursor)))

419

(hdt:%id-cursor-close %cursor)))))

420

421

(defun hdt:%hdt-read-term-pattern-count (%hdt-environment subject-literal predicate-literal object-literal)

422

   (let ((%cursor (hdt:%create-spo-string-triple-id-cursor %hdt-environment subject-literal predicate-literal object-literal)))

423

(when %cursor

424

(unwind-protect

425

(ecase hdt::*pattern-count-mode*

426

(:scan

427

(hdt:%id-cursor-count %cursor))

428

(:estimate

429

(hdt:%id-cursor-estimated-num-results %cursor)))

430

(hdt:%id-cursor-close %cursor)))))

431

432

(defun hdt:%hdt-map-triple-strings (operator %hdt-environment %triple-string)

433

(declare (dynamic-extent operator))

434

(let ((count 0))

435

(let ((%cursor (hdt:%create-triple-string-triple-string-cursor %hdt-environment %triple-string)))

436

(loop while (hdt:%string-cursor-has-next %cursor)

437

for %triple = (hdt:%string-cursor-next %cursor)

438

for count from 0

439

do (funcall operator %triple)

440

do (incf count))

441

(hdt:%string-cursor-close %cursor))

442

count))

443

444

(defun hdt:%hdt-map-triple-ids (operator %hdt-environment %triple-id)

445

(declare (dynamic-extent operator))

446

(let ((count 0))

447

(let ((%cursor (hdt:%create-triple-id-triple-id-cursor %hdt-environment %triple-id)))

448

(loop while (hdt:%id-cursor-has-next %cursor)

449

for %triple = (hdt:%id-cursor-next %cursor)

450

for count from 0

451

do (funcall operator %triple)

452

do (incf count))

453

(hdt:%id-cursor-close %cursor))

454

count))

455

456

#+(or)

457

(defun hdt:%hdt-map-triple-strings* (operator %hdt-environment subject predicate object)

458

(declare (dynamic-extent operator))

459

(let ((count 0))

460

(let ((%cursor (hdt:%create-spo-string-triple-string-cursor %hdt-environment subject predicate object)))

461

(loop while (hdt:%string-cursor-has-next %cursor)

462

for %triple = (hdt:%string-cursor-next %cursor)

463

for count from 0

464

do (funcall operator %triple)

465

do (incf count))

466

(hdt:%string-cursor-close %cursor))

467

count))

468

469

(defun hdt:%hdt-map-triple-strings* (operator %hdt-environment subject predicate object)

470

(declare (dynamic-extent operator))

471

(let ((count 0))

472

(let ((%cursor (hdt:%hdt-search* %hdt-environment subject predicate object)))

473

(loop while (hdt:%string-cursor-has-next %cursor)

474

for %triple = (hdt:%string-cursor-next %cursor)

475

for count from 0

476

do (funcall operator %triple)

477

do (incf count))

478

(hdt:%string-cursor-close %cursor))

479

count))

480

481

482

(defparameter hdt::*pattern-scan-mode* :triple) ; :matrix)

483

(defparameter hdt::*pattern-scan-size* 512)

484

485

(defun hdt:%map-repository-statements (continuation %environment quad-pattern)

486

(let ((subject (aref quad-pattern 1))

487

(predicate (aref quad-pattern 2))

488

(object (aref quad-pattern 3)))

489

(let ((%cursor (hdt:%create-spo-id-triple-id-cursor %environment subject predicate object)))

490

(when %cursor

491

(unwind-protect

492

(ecase hdt::*pattern-scan-mode*

493

(:triple

494

(loop while (hdt::%id-cursor-has-next %cursor)

495

for %triple = (hdt::%id-cursor-next %cursor)

496

do (funcall continuation

497

                              (hdt-box-term-number (hdt:%get-triple-id-subject %triple) :subject)

498

                              (hdt-box-term-number (hdt:%get-triple-id-predicate %triple) :predicate)

499

                              (hdt-box-term-number (hdt:%get-triple-id-object %triple) :object))))

500

(:matrix

501

                (let ((%field (cffi:foreign-alloc :size_t :count (* 3 hdt::*pattern-scan-size*))))

502

(unwind-protect

503

                      (loop for count = (hdt:%id-cursor-next-matrix %cursor %field hdt::*pattern-scan-size*)

504

while (> count 0)

505

do (loop for row from 0 below count

506

with offset = -1

507

do (funcall continuation

508

                                         (hdt-box-term-number (cffi:mem-aref %field :size_t (incf offset)) :subject)

509

                                         (hdt-box-term-number (cffi:mem-aref %field :size_t (incf offset)) :predicate)

510

                                         (hdt-box-term-number (cffi:mem-aref %field :size_t (incf offset)) :object))))

511

(cffi:foreign-free %field)))))

512

(hdt:%id-cursor-close %cursor))))))

513

514

(defun hdt-continue-map (continuation %triple)

515

(funcall continuation

516

(hdt-box-term-number (hdt:%get-triple-id-subject %triple) :subject)

517

(hdt-box-term-number (hdt:%get-triple-id-predicate %triple) :predicate)

518

(hdt-box-term-number (hdt:%get-triple-id-object %triple) :object)))

519

520

(defun hdt-continue-map* (continuation %subject %predicate %object)

521

(funcall continuation

522

(hdt-box-term-number %subject :subject)

523

(hdt-box-term-number %predicate :predicate)

524

(hdt-box-term-number %object :object)))

525

526

(defun hdt:%map-repository-statements* (continuation %environment subject predicate object)

527

;; accept boxed term numbers, return boxed

528

(let ((%cursor (hdt:%create-spo-id-triple-id-cursor %environment

529

                                                       (hdt-unbox-term-number subject)

530

                                                       (hdt-unbox-term-number predicate)

531

                                                       (hdt-unbox-term-number object))))

532

(when %cursor

533

(unwind-protect

534

(ecase hdt::*pattern-scan-mode*

535

(:triple

536

(loop while (hdt::%id-cursor-has-next %cursor)

537

for %triple = (hdt::%id-cursor-next %cursor)

538

do (hdt-continue-map continuation %triple)))

539

(:matrix

540

(let ((%field (cffi:foreign-alloc :size_t :count (* 3 hdt::*pattern-scan-size*))))

541

(unwind-protect

542

                      (loop for count = (hdt:%id-cursor-next-matrix %cursor %field hdt::*pattern-scan-size*)

543

while (> count 0)

544

do (loop for row from 0 below count

545

with offset = -1

546

do (hdt-continue-map* continuation

547

                                                   (cffi:mem-aref %field :size_t (incf offset))

548

                                                   (cffi:mem-aref %field :size_t (incf offset))

549

                                                   (cffi:mem-aref %field :size_t (incf offset)))))

550

(cffi:foreign-free %field)))))

551

(hdt:%id-cursor-close %cursor)))))

552

553

(defun hdt:%map-subject-numbers (continuation %environment &key (distinct nil))

554

(let ((%cursor (hdt:%create-spo-id-triple-id-cursor %environment 0 0 0)))

555

(when %cursor

556

(unwind-protect

557

(if distinct

558

(let ((cache (make-hash-table :test 'eql)))

559

(loop

560

while (hdt::%id-cursor-has-next %cursor)

561

for %triple = (hdt::%id-cursor-next %cursor)

562

                     for term-number =  (hdt-box-term-number (hdt:%get-triple-id-subject %triple) :subject)

563

unless (gethash term-number cache)

564

do (progn (setf (gethash term-number cache) t)

565

(funcall continuation term-number))))

566

(loop

567

while (hdt::%id-cursor-has-next %cursor)

568

for %triple = (hdt::%id-cursor-next %cursor)

569

do (funcall continuation

570

                               (hdt-box-term-number (hdt:%get-triple-id-subject %triple) :subject))))

571

(hdt:%id-cursor-close %cursor)))))

572

573

(defun hdt:%map-predicate-numbers (continuation %environment &key (distinct nil))

574

(let ((%cursor (hdt:%create-spo-id-triple-id-cursor %environment 0 0 0)))

575

(when %cursor

576

(unwind-protect

577

(if distinct

578

(let ((cache (make-hash-table :test 'eql)))

579

(loop

580

while (hdt::%id-cursor-has-next %cursor)

581

for %triple = (hdt::%id-cursor-next %cursor)

582

                     for term-number = (hdt-box-term-number (hdt:%get-triple-id-predicate %triple) :predicate)

583

unless (gethash term-number cache)

584

do (progn (setf (gethash term-number cache) t)

585

(funcall continuation term-number))))

586

(loop

587

while (hdt::%id-cursor-has-next %cursor)

588

for %triple = (hdt::%id-cursor-next %cursor)

589

do (funcall continuation

590

                               (hdt-box-term-number (hdt:%get-triple-id-predicate %triple) :predicate))))

591

(hdt:%id-cursor-close %cursor)))))

592

593

(defun hdt:%map-object-numbers (continuation %environment &key (distinct nil))

594

(let ((%cursor (hdt:%create-spo-id-triple-id-cursor %environment 0 0 0)))

595

(when %cursor

596

(unwind-protect

597

(if distinct

598

(let ((cache (make-hash-table :test 'eql)))

599

(loop

600

while (hdt::%id-cursor-has-next %cursor)

601

for %triple = (hdt::%id-cursor-next %cursor)

602

                     for term-number = (hdt-box-term-number (hdt:%get-triple-id-object %triple) :object)

603

unless (gethash term-number cache)

604

do (progn (setf (gethash term-number cache) t)

605

(funcall continuation term-number))))

606

(loop

607

while (hdt::%id-cursor-has-next %cursor)

608

for %triple = (hdt::%id-cursor-next %cursor)

609

do (funcall continuation

610

                               (hdt-box-term-number (hdt:%get-triple-id-object %triple) :object))))

611

(hdt:%id-cursor-close %cursor)))))

612

613

614

615

#+(or)

616

(

617

(in-package :spocq.i)

618

(initialize-spocq)

619

(cffi:load-foreign-library #p"/opt/spocq/libhdt_lisp.so")

620

(load "patches/rdfhdt.lisp")

621

(load "patches/spocq-classes.lisp")

622

623

(let ((hdt::*pattern-count-mode* :scan)) ;; 3 seconds == 16M/s in c++

624

(hdt:%hdt-read-id-pattern-count (repository-hdt-environment (repository "hdtaccount/hdttest")) 0 0 0))

625

;; 57153765

626

(let ((hdt::*pattern-count-mode* :estimate)) ;; immediate

627

(hdt:%hdt-read-id-pattern-count (repository-hdt-environment (repository "hdtaccount/hdttest")) 0 0 0))

628

629

630

(time ;; 6.504s = 8787479.0

631

(let ((count 0)

632

(hdt::*pattern-scan-mode* :matrix))

633

(flet ((nothing (s p o) (declare (ignore s p o)) (incf count)))

634

      (hdt:%map-repository-statements* #'nothing (repository-hdt-environment (repository "hdtaccount/hdttest")) 0 0 0)

635

count)))

636

(time ;; 32.426s = 1762590.8 : the repeated net calls + the spo slot access v/s array access

637

(let ((count 0)

638

(hdt::*pattern-scan-mode* :triple))

639

(flet ((nothing (s p o) (declare (ignore s p o)) (incf count)))

640

      (hdt:%map-repository-statements* #'nothing (repository-hdt-environment (repository "hdtaccount/hdttest")) 0 0 0)

641

count)))

642

643

644

(defparameter *hdt-repo* (make-instance 'hdt-repository :id "hdtaccount/hdttest" :hdt-filename "data_1.hdt"))

645

; #P"/srv/dydra/catalog/repositories/a6d45140-e11e-074a-8396-e96bf13408f6/"

646

; (repository-hdt-environment *hdt-repo*)

647

;(cffi:foreign-slot-value (repository-hdt-environment *hdt-repo*) 'hdt:basic-hdt 'hdt:dictionary)

648

649

(flet ((print-results (s p o g)

650

(print (list s p o g))))

651

(let ((s (iri-lexical-form <http://example.org/subject20190109:089147169>))

652

(p (iri-lexical-form <http://example.org/object_1>))

653

(o "\"20190109:089147169\""))

654

(hdt::map-repository-statements-x #'print-results *hdt-repo* s p o)))

655

656

(flet ((print-results (s p o g)

657

(print (list s p o g))))

658

(let ((s (iri-lexical-form <http://example.org/subject20190109:089147169>))

659

(p (iri-lexical-form <http://example.org/object_1>))

660

(o "\"20190109:089147169\""))

661

(hdt::map-repository-statements* #'print-results *hdt-repo* s p o)))

662

663

(flet ((print-results (s p o g)

664

(print (list s p o g))))

665

(let ((s "")

666

(p "")

667

(o ""))

668

(hdt::map-repository-statements* #'print-results *hdt-repo* s p o)))

669

670

(let ((s "")

671

(p "")

672

(o "")

673

(count 0))

674

(flet ((print-results (s p o g)

675

(declare (ignore s p o g))

676

(incf count)))

677

(dotimes (x 1000000)

678

(hdt::map-repository-statements* #'print-results *hdt-repo* s p o))

679

count)) ;; scan rate 4782400.5 / sec

680

681

(defparameter *c* (hdt:%hdt-search* (repository-hdt-environment *hdt-repo*)

682

                                     (iri-lexical-form <http://example.org/subject20190109:089147169>)

683

                                     (iri-lexical-form <http://example.org/object_1>)

684

"\"20190109:089147169\""))

685

(hdt:%cursor-has-next *c*)

686

(defparameter *t* (hdt:%cursor-next *c*))

687

(cffi:foreign-slot-value *t* '(:struct hdt:triple-id) 'hdt:subject)

688

(cffi:foreign-slot-value *t* '(:struct hdt:triple-id) 'hdt:predicate)

689

(cffi:foreign-slot-value *t* '(:struct hdt:triple-id) 'hdt:object)

690

691

(cffi:foreign-slot-value *c* '(:struct hdt::triple-id-string-iterator) 'hdt::dictionary)

692

693

;; wor with allie dataset

694

695

  (defparameter *allie-repo* (make-instance 'hdt-repository :id "hdtaccount/hdttest" :hdt-filename "allie_pubmed.hdt"))

696

; (repository-hdt-environment *allie-repo*) (repository-hdt-handle *hdt-repo*)

697

698

(defun test-repository-scan-times (repository patterns &key (verbose nil) (limit most-positive-fixnum))

699

(let* ((triple-string-count 0)

700

(triple-id-count 0)

701

(empty-string "")

702

(%triple-string (hdt:%create-triple-string empty-string empty-string empty-string))

703

(%triple-id (hdt:%create-triple-id 0 0 0))

704

(%hdt (repository-hdt-environment repository))

705

(count 0))

706

(flet ((count-triple-string (%triple)

707

(incf triple-string-count)

708

(when (and verbose (zerop (mod triple-string-count 100)))

709

(print (list :string triple-string-count)))

710

(hdt:%get-triple-string-subject %triple)

711

(hdt:%get-triple-string-predicate %triple)

712

(hdt:%get-triple-string-object %triple))

713

(count-triple-id (%triple)

714

(incf triple-id-count)

715

(when (and verbose (zerop (mod triple-id-count 100)))

716

(print (list :id triple-id-count)))

717

(hdt:%get-triple-id-subject %triple)

718

(hdt:%get-triple-id-predicate %triple)

719

(hdt:%get-triple-id-object %triple)

720

#+(or)

721

(progn

722

(hdt:%triple-id-to-triple-string %hdt %triple %triple-string2)

723

(print (list :returned-id

724

(hdt:%get-triple-string-subject %triple-string2)

725

(hdt:%get-triple-string-predicate %triple-string2)

726

(hdt:%get-triple-string-object %triple-string2)

727

(hdt:%get-triple-id-subject %triple)

728

(hdt:%get-triple-id-predicate %triple)

729

(hdt:%get-triple-id-object %triple))))))

730

(declare (dynamic-extent #'count-triple-string #'count-triple-id))

731

(loop for (s p o) in patterns

732

while (< count limit)

733

;; do (print (list s p o))

734

do (progn

735

(incf count)

736

(hdt:%set-triple-string-all %triple-string s p o)

737

(hdt:%triple-string-to-triple-id %hdt %triple-string %triple-id)

738

#+(or)(print (list :constructed

739

(hdt:%get-triple-string-subject %triple-string)

740

(hdt:%get-triple-string-predicate %triple-string)

741

(hdt:%get-triple-string-object %triple-string)))

742

(hdt:%hdt-map-triple-strings #'count-triple-string %hdt %triple-string)

743

(hdt:%hdt-map-triple-ids #'count-triple-id %hdt %triple-id)

744

;; (print :wild-object)

745

(hdt:%set-triple-string-object %triple-string empty-string)

746

(hdt:%set-triple-id-object %triple-id 0)

747

(hdt:%hdt-map-triple-strings #'count-triple-string %hdt %triple-string)

748

(hdt:%hdt-map-triple-ids #'count-triple-id %hdt %triple-id)

749

;; (print :wild-predicate)

750

(hdt:%set-triple-string-predicate %triple-string empty-string)

751

(hdt:%set-triple-id-predicate %triple-id 0)

752

(hdt:%hdt-map-triple-strings #'count-triple-string %hdt %triple-string)

753

(hdt:%hdt-map-triple-ids #'count-triple-id %hdt %triple-id)

754

)

755

;; do (print (list count triple-string-count triple-id-count))

756

))

757

(hdt:%delete-triple-string %triple-string)

758

(hdt:%delete-triple-id %triple-id)

759

(values count triple-string-count triple-id-count)))

760

761

(sb-profile:profile hdt:%hdt-map-triple-strings hdt:%hdt-map-triple-ids

762

hdt:%triple-string-to-triple-id

763

hdt:%set-triple-string-all

764

                      hdt:%get-triple-string-subject hdt:%get-triple-string-predicate hdt:%get-triple-string-object

765

                      hdt:%get-triple-id-subject hdt:%get-triple-id-predicate hdt:%get-triple-id-object

766

                      hdt:%create-triple-string-triple-string-cursor hdt:%string-cursor-has-next hdt:%string-cursor-next

767

                      hdt:%create-triple-id-triple-id-cursor hdt:%id-cursor-has-next hdt:%id-cursor-next

768

)

769

770

  (let ((triples (with-open-file (allie-triples #P"/srv/dydra/catalog/repositories/a6d45140-e11e-074a-8396-e96bf13408f6/allie-pubmed-1000.txt"

771

:direction :input )

772

(loop for line = (read-line allie-triples nil nil)

773

while line

774

for i below 1000

775

for triple = (split-string line " ")

776

collect triple))))

777

(test-repository-scan-times *allie-repo* triples))

778

779

"hdtSearch allie_pubmed.hdt <<EOF | tail -n +2 | shuf -n 1000 > hdt.txt

780

? ? ?

781

EOF

782

"

783

)

784

785

786

787