Coverage report: /development/source/library/org/datagraph/spocq-shard/src/spocq-server/shell/run-history.lisp

KindCoveredAll%
expression0203 0.0
branch00nil
Key
Not instrumented
Conditionalized out
Executed
Not executed
 
Both branches taken
One branch taken
Neither branch taken
1
 ;;; -*- Mode: lisp; Syntax: ansi-common-lisp; Base: 10; Package: org.datagraph.spocq.implementation; -*-
2
 
3
 (in-package :org.datagraph.spocq.server.implementation)
4
 
5
 (:documentation "(re-)run a colection of http requests from a history folder and records the results."
6
   "the directory /srv/dydra/history/ is the root for storing
7
    records and new results in the following tree
8
    history/
9
    |-- requests/ the request stream images
10
        |--aa/ arranged by task uuid, with in initial two character index level
11
    |-- responses/ the response stream images
12
        |--aa/ ...
13
                 ")
14
 ;;; sbcl --core sbcl-spocq.core
15
 ;;; (sb-ext:save-lisp-and-die "run-history" :executable t :toplevel #'spocq.i::main-run-history)
16
 
17
 (defun main-run-history (&rest args &key
18
                                ((:directory *history-directory*)
19
                                 (or (getarg "--directory") *history-directory*))
20
                                (verbose (or (getarg "-v") (getarg "--verbose")))
21
                                (title (or (getarg "--title") "history"))
22
                                (sort (or (getarg "--sort") :chronological))
23
                                &allow-other-keys)
24
   
25
   (handler-bind ((serious-condition
26
                   (lambda (condition)
27
                     (log-stacktrace "Error in main-run-history: ~s -> ~a." args condition)
28
                     (format *standard-output* "~a~%" condition)
29
                     (when spocq.i::*exit-on-errors*
30
                       (spocq.i::stop)
31
                       (exit-lisp 70)))))
32
     ;; need to generate the task id as, to this point, the store connection has not been initialized
33
     (with-command-line-configuration ((list* :task-id (prin1-to-string (uuid:make-v1-uuid)) args))
34
       (setq *start-timestamp* (iso-time))
35
       (spocq.i:initialize-spocq :title title)
36
       (dydra:log-debug "run-history: ~a" args)
37
       (let* ((host-name (dydra:host-name))
38
              (host-package (or (find-package host-name)
39
                                (make-package host-name :use ()))))
40
         (import *response-functions* host-package)
41
         (with-package-iterator (next host-package :internal)
42
           (loop (multiple-value-bind (symbol-p symbol) (next)
43
                   (unless symbol-p (return))
44
                   (export symbol host-package))))
45
         (setq *spocq-acceptor*
46
               (make-instance 'spocq-acceptor
47
                 :name "spocq"
48
                 :port *host-port* :address host-name
49
                 :request-class *service-request-class*
50
                 :response-class *service-response-class*
51
                 :taskmaster (make-instance *class.taskmaster*
52
                               :name (concatenate 'string host-name "-taskmaster")
53
                               :max-thread-count dydra:*service-request-count-limit*
54
                               :max-accept-count (max *http-accept-count-limit*
55
                                                      (1+ dydra:*service-request-count-limit*)))))
56
         (setf (http:acceptor-dispatch-function *spocq-acceptor*) host-package))
57
       (setf sort (intern (string-upcase sort) :keyword))
58
       (run-history :directory *history-directory*
59
                    :verbose (string-equal verbose "true")
60
                    :sort sort))))
61
 
62
 (defun run-history (&key ((:directory *history-directory*) *history-directory*) (verbose nil) (sort :chronological))
63
   (let* ((requests (request-directory-pathnames :directory (request-history-directory) :sort sort))
64
          (timestamp (format nil "~/format-iso-time/" (get-universal-time)))
65
          (from-wild (merge-pathnames (make-pathname :directory '(:relative "requests" :wild) :name :wild) *history-directory*))
66
          ;; disable further transcripts
67
          (to-wild (merge-pathnames (make-pathname :directory `(:relative "results" ,timestamp :wild) :name :wild) *history-directory*))
68
          (*history-directory* nil))
69
     (loop for request-pathname in requests
70
       for response-pathname = (translate-pathname request-pathname from-wild to-wild)
71
       collect (progn
72
                 (when verbose
73
                   (format *trace-output* "~%~/format-iso-time/: ~a"
74
                           (get-universal-time)
75
                           (enough-namestring request-pathname *history-directory*)))
76
                 (ensure-directories-exist response-pathname)
77
                 (let ((result-code (handler-case (tbnl::process-asynchronous-connection *spocq-acceptor*
78
                                                                                         request-pathname
79
                                                                                         response-pathname)
80
                                      (error (c)
81
                                        (format *trace-output* "~%~s: ~a" request-pathname c)
82
                                        500))))
83
                   (when verbose
84
                     (format *trace-output* " ~s" result-code)))
85
                 response-pathname)
86
       finally (when verbose (terpri *trace-output*)))))
87
 
88
 ;;; (main-run-history)
89
 ;;; (spocq.si::run-history :verbose t)
90
 
91
 (defun request-directory-pathnames (&key (directory (request-history-directory)) (sort :chronological))
92
   (let ((files (directory (merge-pathnames (make-pathname :directory '(:relative :wild) :name :wild)
93
                                            directory))))
94
     (ecase sort
95
       ((nil) files)
96
       (:alphabetic (sort files #'string-lessp :key #'pathname-name))
97
       (:chronological (sort files #'< :key #'(lambda (pathname) (file-write-date pathname)))))))
98
 
99
 (defun response-directory-pathnames (&key (directory *history-directory*) (sort :chronological))
100
   (let ((files (directory (merge-pathnames (make-pathname :directory '(:relative :wild) :name :wild)
101
                                            directory))))
102
     (ecase sort
103
       ((nil) files)
104
       (:alphabetic (sort files #'string-lessp :key #'pathname-name))
105
       (:chronological (sort files #'< :key #'(lambda (pathname) (file-write-date pathname)))))))
106
 
107
 
108
 #|
109
 (in-package :spocq.i)
110
 (initialize-spocq)
111
 (let ((*history-directory* (make-pathname :directory '(:absolute "srv" "dydra" "history"))))
112
   (run-history :verbose t))
113
 
114
 
115
 (translate-pathname
116
   #P"/srv/dydra/history/requests/96/968c4d16-4d61-a046-8692-f76b62d6a6c2"
117
   (merge-pathnames (make-pathname :directory '(:relative "requests" :wild) :name :wild) *history-directory*)
118
   (merge-pathnames (make-pathname :directory `(:relative "results" ,(format nil "~/format-iso-time/" (get-universal-time)) :wild) :name :wild) *history-directory*))
119
 
120
 curl -X GET 'http://de8.dydra.com/james/foaf/sparql?query=construct%20%7b?s%20?p%20?o%7d%20where%20%7b?s%20?p%20?o%7d' -H "Accept: application/ld+json"
121
 
122
 curl -v -X POST 'http://de8.dydra.com/james/test2/sparql' \
123
   -H "Accept: application/ld+json" \
124
   -H "Content-Type: application/sparql" \
125
   --user "${TOKEN}:" \
126
   --data-binary @- <<EOF
127
 construct {?s ?p ?o }
128
 where {?s ?p ?o }
129
 EOF
130
 
131
 curl -v -X POST 'http://de8.dydra.com/james/test/sparql' \
132
   -H "Accept: application/rdf+xml" \
133
   -H "Content-Type: application/sparql-query" \
134
   --user "${TOKEN}:" \
135
   --data-binary @- <<EOF
136
 construct {?s ?p ?o }
137
 where {?s ?p ?o }
138
 limit 10
139
 
140
 (ensure-directories-exist #p"/srv/dydra/history/results/96/968c4d16-4d61-a046-8692-f76b62d6a6c2")
141
 (let ((request #P"/srv/dydra/history/requests/8a/8a016e73-cb12-8f4c-b809-e64e04898166")
142
       (*request-history-directory* nil)
143
       (*response-history-directory* nil))
144
 (tbnl::process-asynchronous-connection spocq.si::*spocq-acceptor*
145
   request
146
   :output (translate-pathname request
147
            (merge-pathnames (make-pathname :directory '(:relative "requests" :wild) :name :wild) *history-directory*)
148
            (merge-pathnames (make-pathname :directory `(:relative "results" ,(format nil "~/format-iso-time/" (get-universal-time)) :wild)
149
                                            :name :wild)
150
                             *history-directory*))))
151
 
152
 (format t "~/format-iso-time/" (get-universal-time))
153
 |#