webstatlib.lisp
No OneTemporary
Actions

Subscribers

None

File Metadata

Created: Tue, Nov 12, 15:28

webstatlib.lisp
View Options

	;;; webstatlib.lisp -- library with httpd log file analyzer to gather
	;;; CDS usage stats. Another functionality is to parse old Apache log
	;;; files and prepare Detailed record page views statistics for
	;;; rnkPAGEVIEWS table.
	;;;
	;;; This file is part of Invenio.
	;;; Copyright (C) 2005, 2006, 2007, 2008, 2010, 2011 CERN.
	;;;
	;;; Invenio is free software; you can redistribute it and/or
	;;; modify it under the terms of the GNU General Public License as
	;;; published by the Free Software Foundation; either version 2 of the
	;;; License, or (at your option) any later version.
	;;;
	;;; Invenio is distributed in the hope that it will be useful, but
	;;; WITHOUT ANY WARRANTY; without even the implied warranty of
	;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	;;; General Public License for more details.
	;;;
	;;; You should have received a copy of the GNU General Public License
	;;; along with Invenio; if not, write to the Free Software Foundation, Inc.,
	;;; 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

	;;
	;; Auxiliary functions come first.

	#-gcl (in-package #:cl-user)

	(defpackage #:webstatlib
	#-gcl (:use #:cl)
	(:export #:print-general-stats
	#:print-search-interface-stats
	#:print-search-collections-stats
	#:print-search-pattern-stats
	#:print-basket-stats
	#:print-alert-stats
	#:print-download-stats))

	#-gcl (proclaim '(optimize (speed 3) (safety 1) (debug 0) (compilation-speed 0) (space 0)))

	(defun parse-integer-or-zero (string)
	"Parse integer from STRING and return either the integer, or zero."
	(let ((out 0))
	(if (string-not-equal string "-")
	(setf out (parse-integer string)))
	out))

	(defun hash-table-keys (hashtable &optional (sort-by-value nil))
	"Return list of HASHTABLE keys. Optionally, sort the key list by
	hash value."
	(let ((keys '()))
	(maphash (lambda (key val)
	(declare (ignore val))
	(push key keys))
	hashtable)
	(if sort-by-value
	(sort keys #'(lambda (x y)
	(let ((x-val (gethash x hashtable))
	(y-val (gethash y hashtable)))
	(if (equalp x-val y-val)
	(string< x y) ; when values are equal, test by keys
	(> x-val y-val)))))
	keys)))

	(defun hash-table-values (hashtable)
	"Return list of HASHTABLE values."
	(let ((vals '()))
	(maphash (lambda (key val)
	(declare (ignore key))
	(push val vals))
	hashtable)
	vals))

	(defun hash-table-key-val-stats (hashtable predicate
	&optional (apply-predicate-to-vals nil))
	"Return the number of keys in HASHTABLE that satisfy the PREDICATE.
	Another returned value is the sum of values for these keys. Suitable
	when values are integers, as in histograms."
	(declare (type function predicate))
	(let ((number-of-keys 0)
	(sum-of-values 0))
	(declare (type fixnum number-of-keys sum-of-values))
	(maphash (lambda (key val)
	(if apply-predicate-to-vals
	(when (funcall predicate val)
	(incf number-of-keys)
	(incf sum-of-values val))
	(when (funcall predicate key)
	(incf number-of-keys)
	(incf sum-of-values val))))
	hashtable)
	(values number-of-keys sum-of-values)))

	(defun split-string-by-one-space (string)
	"Return list of substrings of STRING divided by one space each.
	Two consecutive spaces will be seen as if there were an empty string
	between them. Taken from Common Lisp Cookbook."
	(declare (type simple-base-string string))
	(loop for i = 0 then (1+ j)
	as j = (position #\Space string :start i)
	collect (subseq string i j)
	while j))

	(defun split-string-by-equal-sign (string)
	"Return list of two substrings of STRING around one equal sign,
	stripping any whitespace. Return nil if the equal sign was not found."
	(declare (type simple-base-string string))
	(let ((pos (position #\= string)))
	(if pos
	(values (string-trim " " (subseq string 0 pos))
	(string-trim " " (subseq string (1+ pos))))
	nil)))

	(defun string-substitute (string substring replacement-string)
	"Taken from c.l.l."
	(declare (type simple-base-string string substring replacement-string))
	(let ((substring-length (length substring))
	(last-end 0)
	(new-string ""))
	(declare (type simple-base-string new-string)
	(type integer substring-length last-end))
	(do ((next-start
	(search substring string)
	(search substring string :start2 last-end)))
	((null next-start)
	(concatenate 'string new-string (subseq string last-end)))
	(declare (type (or integer null) next-start))
	(setq new-string
	(concatenate 'string
	new-string
	(subseq string last-end next-start)
	replacement-string))
	(setq last-end (+ next-start substring-length)))))

	(defun string-decode-url (string)
	"Return string where URL is unquoted, that is %22 is substituted by
	a double quote, etc."
	(declare (type simple-base-string string))
	(setf string (string-substitute string "%20" " "))
	(setf string (string-substitute string "+" " "))
	(setf string (string-substitute string "%2B" "+"))
	(setf string (string-substitute string "%22" "\""))
	(setf string (string-substitute string "%23" "#"))
	(setf string (string-substitute string "%25" "%"))
	(setf string (string-substitute string "%26" "&"))
	(setf string (string-substitute string "%27" "'"))
	(setf string (string-substitute string "%28" "("))
	(setf string (string-substitute string "%29" ")"))
	(setf string (string-substitute string "%2A" "*"))
	(setf string (string-substitute string "%2C" ","))
	(setf string (string-substitute string "%2F" "/"))
	(setf string (string-substitute string "%3A" ":"))
	(setf string (string-substitute string "%3D" "="))
	(setf string (string-substitute string "%3E" ">"))
	(setf string (string-substitute string "%3F" "?"))
	(setf string (string-substitute string "%5B" "["))
	(setf string (string-substitute string "%5C" "\\"))
	(setf string (string-substitute string "%5D" "]")))

	(defun get-month-number-from-month-name (month-name)
	"Return month number for MONTH-NAME."
	(declare (type simple-base-string month-name))
	(cond ((string= month-name "Jan") 1)
	((string= month-name "Feb") 2)
	((string= month-name "Mar") 3)
	((string= month-name "Apr") 4)
	((string= month-name "May") 5)
	((string= month-name "Jun") 6)
	((string= month-name "Jul") 7)
	((string= month-name "Aug") 8)
	((string= month-name "Sep") 9)
	((string= month-name "Oct") 10)
	((string= month-name "Nov") 11)
	((string= month-name "Dec") 12)
	(t 0)))

	(defun get-urlarg-values-from-url-string (url-string url-arg
	&optional (return-empty-value nil))
	"Return list of values of URL-ARG in the URL-STRING. For example,
	if URL-STRING contains 'search?p=ellis&f=title', and URL-ARG is
	'f', then return list of 'title'. If URL-ARG is not present in
	URL-STRING, return list of empty string."
	(declare (type simple-base-string url-string url-arg))
	(let ((out nil)
	(position-pattern-beg nil)
	(position-pattern-end nil))
	(setf position-pattern-beg (search (concatenate 'string url-arg "=")
	url-string))
	(if (not position-pattern-beg)
	(progn
	(when return-empty-value
	(push "" out)))
	(progn
	(setf position-pattern-end (search "&" url-string
	:start2 position-pattern-beg))
	(if (null position-pattern-end)
	(setf position-pattern-end (length url-string)))
	(push (string-decode-url (subseq url-string
	(+ (length url-arg) 1 position-pattern-beg)
	position-pattern-end))

	out)))
	(if return-empty-value
	out
	(remove-if #'(lambda (x)
	(declare (type simple-base-string x))
	(string= "" x))
	out))))

	(defun get-datetime-from-httpd-log-datetime-string (httpd-log-datetime-string)
	"Return request date in numerical format YYYYMMDD from HTTPD-LOG-DATETIME-STRING."
	(declare (type simple-base-string httpd-log-datetime-string))
	(let ((yyyy (subseq httpd-log-datetime-string 7 11))
	(mm-number (get-month-number-from-month-name (subseq httpd-log-datetime-string 3 6)))
	(dd (subseq httpd-log-datetime-string 0 2)))
	(declare (type fixnum mm-number))
	(+ (* 100 100 (parse-integer yyyy))
	(* 100 mm-number)
	(parse-integer dd))))

	(defun get-datetimee-from-httpd-log-datetime-string (httpd-log-datetime-string)
	"Return request date in the format YYYY-MM-DD HH:MM:SS from HTTPD-LOG-DATETIME-STRING."
	(declare (type simple-base-string httpd-log-datetime-string))
	(let ((yyyy (subseq httpd-log-datetime-string 7 11))
	(mm (get-month-number-from-month-name (subseq httpd-log-datetime-string 3 6)))
	(dd (subseq httpd-log-datetime-string 0 2))
	(hms (subseq httpd-log-datetime-string 12 20)))
	(format nil "~A-~A-~A ~A" yyyy mm dd hms)))

	(defun get-current-collection-from-httpd-log-request-string (httpd-log-request-string)
	"Return current collection found in HTTPD-LOG-REQUEST-STRING.
	Examples of input: 'search?cc=Books', '/?c=Books', '/collection/Books?ln=en'."
	(declare (special home-collection))
	(let ((out nil)
	(url-string (subseq httpd-log-request-string
	(position #\space httpd-log-request-string)
	(position #\space httpd-log-request-string :from-end t))))
	(if (search "?cc=" url-string)
	(setf out (get-urlarg-values-from-url-string url-string "cc" t))
	(if (search "?c=" url-string)
	(setf out (get-urlarg-values-from-url-string url-string "c" t))
	(if (search "/collection/" url-string)
	(setf out (list (string-decode-url (subseq url-string
	(+ (search "/collection/" url-string) (length "/collection/"))
	(search "?ln=" url-string :from-end t)))))
	(setf out (list home-collection)))))
	(if (string= (car out) "")
	(setf out (list home-collection)))
	out))

	(defun get-search-patterns-from-httpd-log-request-string (httpd-log-request-string)
	"Return list of search pattern strings (p=,p1=,p2=,p3=) found in
	HTTPD-LOG-REQUEST-STRING. As a side effect, increment counters for
	NUMBER-OF-SIMPLE-SEARCHES and NUMBER-OF-DETAILED-RECORD-PAGES when
	a simple search or a detailed record page is selected."
	(declare (type simple-base-string httpd-log-request-string)
	(special search-engine-url
	search-engine-url-old-style
	detailed-record-url
	number-of-simple-searches
	number-of-advanced-searches
	number-of-detailed-record-pages))
	(let ((out nil)
	(url-string (subseq httpd-log-request-string
	(position #\space httpd-log-request-string)
	(position #\space httpd-log-request-string :from-end t))))
	;; url-string now contains URL without leading GET and trailing HTTP/1.1 strings
	(when (or (search search-engine-url url-string)
	(search search-engine-url-old-style url-string)
	(search detailed-record-url url-string))
	(if (or (search detailed-record-url url-string)
	(search "?id=" url-string)
	(search "?recid=" url-string)
	(search "?sysno=" url-string)
	(search "?sysnb=" url-string))
	;; detailed record page happened:
	(incf number-of-detailed-record-pages)
	(if (search "p1=" httpd-log-request-string)
	;; advanced search happened:
	(progn
	(incf number-of-advanced-searches)
	(setf out (append (get-urlarg-values-from-url-string url-string "p1")
	(get-urlarg-values-from-url-string url-string "p2")
	(get-urlarg-values-from-url-string url-string "p3"))))
	;; simple search happened:
	(progn
	(incf number-of-simple-searches)
	(setf out (get-urlarg-values-from-url-string url-string "p" t))))))
	out))

	(defun wash-httpd-log-line (httpd-log-line)
	"Fix eventual problems in HTTPD-LOG-LINE, such as backslash-quote
	that some browsers seem to produce. To be called for the input to
	wash it out."
	(declare (type simple-base-string httpd-log-line))
	(string-substitute httpd-log-line "\\\"" "%22"))

	;; HTTPD-LOG-ENTRY stucture that will represent Apache log hits, and
	;; functions to create it.

	(defstruct httpd-log-entry
	"Structure representing Apache combined log entry."
	(ip "" :type simple-base-string)
	(datetime 0 :type integer) ; YYYYMMDD (FIXME: get rid of this)
	(datetimee "" :type simple-base-string) ; YYYY-MM-DD HH:MM:SS
	(request "" :type simple-base-string)
	(status 0 :type integer)
	(bytes 0 :type integer)
	(referer "" :type simple-base-string)
	(browser "" :type simple-base-string))

	(defun parse-httpd-log-line-conses-a-lot (line)
	"Return list of elements read from Apache combined log LINE. The
	elements in line are text strings delimited either by spaces or by
	double quotes or brackets. (Spaces within double quotes or brackets
	do not take their separator effect. Double quote preceded by a
	backslash do not take its effect neither.) WARNING: THIS FUNCTION
	CONSES A LOT, A FASTER REWRITE IS AVAILABLE BELOW."
	(declare (type simple-base-string line))
	(if (string= line "")
	nil
	(let ((current-char (char line 0))
	(position-term-next nil)
	(position-term-offset 0))
	(declare (type (or fixnum null) position-term-next position-term-offset)
	(type character current-char))
	(cond ((char= current-char #\")
	(setf position-term-next (position #\" line :start 1)
	position-term-offset 1))
	((char= current-char #\[)
	(setf position-term-next (position #\] line :start 1)
	position-term-offset 1))
	(t (setf position-term-next (position #\Space line :start 1))))
	(if (or (null position-term-next)
	(>= (+ 1 position-term-next position-term-offset) (length line)))
	(list (subseq line position-term-offset))
	(append (list (subseq line position-term-offset position-term-next))
	(parse-httpd-log-line (subseq line
	(+ 1
	position-term-next
	position-term-offset))))))))

	(defun parse-httpd-log-line (line)
	"Return list of elements read from Apache combined log LINE. The
	elements in line are text strings delimited either by spaces or by
	double quotes or brackets. (Spaces within double quotes or brackets
	do not take their separator effect. Double quote preceded by a
	backslash do not take its effect neither.)"
	(declare (type simple-base-string line))
	(labels ((get-next-char-to-search (current-char)
	(declare (type character current-char))
	(cond ((char= current-char #\[) #\])
	((char= current-char #\") #\")
	(t #\space)))
	(get-char-offset (current-char)
	(declare (type character current-char))
	(cond ((char= current-char #\[) 1)
	((char= current-char #\") 1)
	(t 0))))
	(let* ((out nil)
	(position-max (1- (length line)))
	(position-term 0)
	(position-term-offset (get-char-offset (char line position-term)))
	(position-term-next (position (get-next-char-to-search (char line position-term))
	line
	:start (+ position-term-offset position-term))))
	(declare (type (or integer null) position-term position-term-next)
	(type integer position-max position-term-offset))
	(loop (when (null position-term-next)
	(when (< position-term position-max)
	(push (subseq line (+ position-term position-term-offset)) out))
	(return (nreverse out)))
	(push (subseq line (+ position-term position-term-offset) position-term-next) out)
	(setf position-term (min (+ 1 position-term-next position-term-offset)
	position-max))
	(setf position-term-offset (get-char-offset (char line position-term)))
	(setf position-term-next (position (get-next-char-to-search (char line position-term))
	line
	:start (+ position-term position-term-offset)))))))

	(defun create-httpd-log-entry (httpd-log-line)
	"Create HTTPD-LOG-ENTRY structure from HTTPD-LOG-LINE Apache
	combined log line. The line is supposed to be washed already."
	(declare (type simple-base-string httpd-log-line))
	; firstly parse Apache line according to spaces and quotes into list:
	(let ((httpd-log-line-list (parse-httpd-log-line httpd-log-line)))
	;; secondly transform into entry:
	(make-httpd-log-entry :ip (nth 0 httpd-log-line-list)
	:request (nth 4 httpd-log-line-list)
	:datetime (get-datetime-from-httpd-log-datetime-string
	(nth 3 httpd-log-line-list))
	:datetimee (get-datetimee-from-httpd-log-datetime-string
	(nth 3 httpd-log-line-list))
	:status (parse-integer (nth 5 httpd-log-line-list))
	:bytes (parse-integer-or-zero (nth 6 httpd-log-line-list))
	:referer (nth 7 httpd-log-line-list)
	:browser (nth 8 httpd-log-line-list))))

	(defun create-httpd-log-entries (httpd-log-filename
	&optional (exclude-ip-list nil))
	"Create list of HTTPD-LOG-ENTRY structures from HTTPD-LOG-FILENAME
	Apache combined log file. The log file lines are cleaned to make them
	valid, if necessary. When EXCLUDE-IP-LIST is set, then lines coming
	from those IPs are excluded."
	(declare (type simple-base-string httpd-log-filename)
	(type list exclude-ip-list))
	(let ((httpd-log-entries nil))
	(format t "~&** APACHE LOG FILE ANALYSIS")
	(format t "~&Filename: ~A" httpd-log-filename)
	(unless (null exclude-ip-list)
	(format t "~&Excluding search engine hits from ~A." exclude-ip-list))
	(with-open-file (input-stream httpd-log-filename :direction :input)
	(do ((line (read-line input-stream nil)
	(read-line input-stream nil)))
	((not line))
	(declare (type (or simple-base-string null) line))
	(let ((httpd-log-entry-item (create-httpd-log-entry (wash-httpd-log-line line))))
	(if (or (null exclude-ip-list)
	(not (member (httpd-log-entry-ip httpd-log-entry-item)
	exclude-ip-list :test #'equal)))
	(push httpd-log-entry-item httpd-log-entries)))))
	httpd-log-entries))

	(defun filter-httpd-log-entries (httpd-all-log-entries url-substring-pattern
	&optional (status-code 200))
	"Return only those of HTTPD-ALL-LOG-ENTRIES that satisfy
	URL-SUBSTRING-PATTERN and are of status STATUS-CODE."
	(remove-if-not #'(lambda (httpd-log-entry-item)
	(and (search url-substring-pattern (httpd-log-entry-request httpd-log-entry-item))
	(= status-code (httpd-log-entry-status httpd-log-entry-item))))
	httpd-all-log-entries))

	;; Functions that analyze the files according to the typical use cases.

	(defun print-general-stats (httpd-all-log-entries)
	"Read HTTPD-LOG-ENTRIES and print some general stats such as
	the total number of hits, etc."
	(declare (special nb-histogram-items-to-print))
	(format t "~&~%** GENERAL STATS")
	(let ((nb-hits (length httpd-all-log-entries))
	(datetime-last (httpd-log-entry-datetime (car httpd-all-log-entries)))
	(datetime-first (httpd-log-entry-datetime (car (last httpd-all-log-entries))))
	(visitor-histogram (make-hash-table :test #'equalp)))
	;; build histogram:
	(dolist (httpd-log-entry-item httpd-all-log-entries)
	(incf (gethash (httpd-log-entry-ip httpd-log-entry-item)
	visitor-histogram 0)))
	;; print summary info:
	(format t "~&There were ~D website hits dating from ~A to ~A."
	nb-hits datetime-first datetime-last)
	(format t "~&This makes an average of ~F website hits per day."
	(/ nb-hits (1+ (- datetime-last datetime-first))))
	(format t "~&There were ~D unique visitors in ~D days."
	(length (hash-table-keys visitor-histogram))
	(1+ (- datetime-last datetime-first)))
	;; print visitors histogram:
	(format t "~&~80,1,0,'.<~A ~; ~A~>" "Visitor" "no. of hits")
	(let ((visitors-to-print (hash-table-keys visitor-histogram t)))
	(if (> (length visitors-to-print) nb-histogram-items-to-print)
	(setq visitors-to-print (subseq visitors-to-print 0 nb-histogram-items-to-print)))
	(dolist (visitor visitors-to-print)
	(format t "~&~80,1,0,'.<~A ~; ~A~>" visitor
	(gethash visitor visitor-histogram))))))

	(defun print-search-interface-stats (httpd-all-log-entries)
	"Read HTTPD-ALL-LOG-ENTRIES, analyze search interface usage and print
	interesting statistics."
	(declare (special search-interface-url
	search-interface-url-old-style
	home-collection
	nb-histogram-items-to-print))
	(let ((httpd-log-entries (append (filter-httpd-log-entries httpd-all-log-entries search-interface-url)
	(filter-httpd-log-entries httpd-all-log-entries search-interface-url-old-style)))
	(collection-histogram (make-hash-table :test #'equalp)))
	;; build histogram:
	(dolist (httpd-log-entry httpd-log-entries)
	(dolist (current-collection (get-current-collection-from-httpd-log-request-string
	(httpd-log-entry-request
	httpd-log-entry)))
	(incf (gethash current-collection collection-histogram 0))))
	;; print summary info:
	(format t "~&~%** SEARCH INTERFACE COLLECTIONS USAGE ANALYSIS")
	(let ((number-of-interface-visits
	(reduce #'+ (hash-table-values collection-histogram))))
	(format t "~&There were ~D visits of search interface pages."
	number-of-interface-visits)
	(format t "~&There were ~D visits of non-Home search interface pages. (~4F\%)"
	(- number-of-interface-visits (gethash home-collection collection-histogram 0))
	(if (zerop number-of-interface-visits)
	0
	(* 100 (/ (gethash home-collection collection-histogram 0)
	number-of-interface-visits)))))
	;; print histogram:
	(format t "~&~80,1,0,'.<~A ~; ~A~>" "Collection" "no. of visits")
	(let ((collections-to-print (hash-table-keys collection-histogram t)))
	(if (> (length collections-to-print) nb-histogram-items-to-print)
	(setq collections-to-print (subseq collections-to-print 0 nb-histogram-items-to-print)))
	(dolist (collection collections-to-print)
	(format t "~&~80,1,0,'.<~A ~; ~A~>" collection
	(gethash collection collection-histogram))))))

	(defun print-search-collections-stats (httpd-all-log-entries)
	"Read HTTPD-ALL-LOG-ENTRIES, analyze search engine usage and print
	interesting statistics with respect to collections."
	(declare (special search-engine-url
	search-engine-url-old-style
	home-collection
	nb-histogram-items-to-print))
	(let ((httpd-log-entries (append (filter-httpd-log-entries httpd-all-log-entries search-engine-url)
	(filter-httpd-log-entries httpd-all-log-entries search-engine-url-old-style)))
	(collection-histogram (make-hash-table :test #'equalp)))
	;; build histogram:
	(dolist (httpd-log-entry httpd-log-entries)
	(if (or (search "c=" (httpd-log-entry-request httpd-log-entry))
	(search "cc=" (httpd-log-entry-request httpd-log-entry)))
	(dolist (current-collection (get-current-collection-from-httpd-log-request-string
	(httpd-log-entry-request
	httpd-log-entry)))
	(incf (gethash current-collection collection-histogram 0)))))
	;; print summary info:
	(format t "~&~%** SEARCH ENGINE COLLECTIONS USAGE ANALYSIS")
	(let ((number-of-interface-visits
	(reduce #'+ (hash-table-values collection-histogram))))
	(format t "~&There were ~D search engine hits."
	number-of-interface-visits)
	(format t "~&There were ~D searches originating from non-Home collections. (~4F\%)"
	(- number-of-interface-visits (gethash home-collection collection-histogram 0))
	(* 100 (/ (gethash home-collection collection-histogram 0) number-of-interface-visits))))
	;; print histogram:
	(format t "~&~80,1,0,'.<~A ~; ~A~>" "Originating collection" "no. of searches")
	(let ((collections-to-print (hash-table-keys collection-histogram t)))
	(if (> (length collections-to-print) nb-histogram-items-to-print)
	(setq collections-to-print (subseq collections-to-print 0 nb-histogram-items-to-print)))
	(dolist (collection collections-to-print)
	(format t "~&~80,1,0,'.<~A ~; ~A~>" collection
	(gethash collection collection-histogram))))))

	(defun print-search-pattern-stats (httpd-all-log-entries)
	"Read HTTPD-ALL-LOG-ENTRIES, analyze search patterns and print
	interesting statistics."
	(declare (special search-engine-url
	search-engine-url-old-style
	nb-histogram-items-to-print
	number-of-simple-searches
	number-of-advanced-searches
	number-of-detailed-record-pages))
	(setf number-of-simple-searches 0
	number-of-advanced-searches 0
	number-of-detailed-record-pages 0)
	(let ((httpd-log-entries (append (filter-httpd-log-entries httpd-all-log-entries search-engine-url)
	(filter-httpd-log-entries httpd-all-log-entries search-engine-url-old-style)))
	(pattern-histogram (make-hash-table :test #'equalp)))
	;; build histogram:
	(dolist (httpd-log-entry httpd-log-entries)
	(dolist (pattern (get-search-patterns-from-httpd-log-request-string (httpd-log-entry-request
	httpd-log-entry)))
	(incf (gethash pattern pattern-histogram 0))))
	;; print summary info:
	(format t "~&~%** SEARCH ENGINE QUERY PATTERN ANALYSIS")
	(if (null httpd-log-entries)
	(format t "~&No valid search engine hits found. Exiting.")
	(let ((number-of-search-engine-hits (length httpd-log-entries))
	(number-of-patterns (hash-table-count pattern-histogram))
	(day-first (httpd-log-entry-datetime (nth (1- (length httpd-log-entries))
	httpd-log-entries)))
	(day-last (httpd-log-entry-datetime (first httpd-log-entries))))
	(declare (type integer number-of-search-engine-hits
	number-of-patterns day-first day-last))
	;; a - total number of searches and patterns
	(format t "~&Found ~D search engine hits." number-of-search-engine-hits)
	(format t "~&First search engine hit log is dated ~A." day-first)
	(format t "~&Last search engine hit log is dated ~A." day-last)
	(format t "~&This makes an average of ~F search engine hits per day."
	(/ number-of-search-engine-hits (1+ (- day-last day-first))))
	(format t "~&There were ~D simple searches out of ~D search engine hits. (~4F\%)"
	number-of-simple-searches number-of-search-engine-hits
	(* 100 (/ number-of-simple-searches number-of-search-engine-hits)))
	(format t "~&There were ~D advanced searches out of ~D search engine hits. (~4F\%)"
	number-of-advanced-searches number-of-search-engine-hits
	(* 100 (/ number-of-advanced-searches number-of-search-engine-hits)))
	(format t "~&There were ~D detailed record pages out of ~D search engine hits. (~4F\%)"
	number-of-detailed-record-pages number-of-search-engine-hits
	(* 100 (/ number-of-detailed-record-pages number-of-search-engine-hits)))
	(format t "~&There are ~D different query patterns for ~D search engine hits. (~4F\%)"
	number-of-patterns number-of-search-engine-hits
	(* 100 (/ number-of-patterns number-of-search-engine-hits)))
	;; b - empty searches
	(format t "~&There are ~D empty query pattern searches out of ~D search engine hits. (~4F\%)"
	(gethash "" pattern-histogram 0)
	number-of-search-engine-hits
	(* 100 (/ (gethash "" pattern-histogram 0) number-of-search-engine-hits)))
	;; c - phrase searches and patterns
	(multiple-value-bind (number-of-phrase-patterns number-of-phrase-searches)
	(hash-table-key-val-stats pattern-histogram #'(lambda (x)
	(declare (type simple-base-string x))
	(position #\" x)))
	(format t "~&There are ~D phrase searches out of ~D search engine hits. (~4F\%)"
	number-of-phrase-searches
	number-of-search-engine-hits
	(* 100 (/ number-of-phrase-searches number-of-search-engine-hits)))
	(format t "~&There are ~D phrase query patterns out of ~D query patterns. (~4F\%)"
	number-of-phrase-patterns
	number-of-patterns
	(* 100 (/ number-of-phrase-patterns number-of-patterns))))
	;; d - onetime searches and patterns
	(multiple-value-bind (number-of-onetime-patterns number-of-onetime-searches)
	(hash-table-key-val-stats pattern-histogram #'(lambda (x)
	(declare (integer x))
	(= x 1)) t)
	(format t "~&There are ~D one-time event searches out of ~D search engine hits. (~4F\%)"
	number-of-onetime-searches
	number-of-search-engine-hits
	(* 100 (/ number-of-onetime-searches number-of-search-engine-hits)))
	(format t "~&There are ~D one-time query patterns out of ~D query patterns. (~4F\%)"
	number-of-onetime-patterns
	number-of-patterns
	(* 100 (/ number-of-onetime-patterns number-of-patterns))))
	;; e - one-word searches and patterns
	(multiple-value-bind (number-of-oneword-patterns number-of-oneword-searches)
	(hash-table-key-val-stats pattern-histogram #'(lambda (x)
	(declare (type simple-base-string x))
	(not (position #\space x))))
	(format t "~&There are ~D one-word searches out of ~D search engine hits. (~4F\%)"
	number-of-oneword-searches
	number-of-search-engine-hits
	(* 100 (/ number-of-oneword-searches number-of-search-engine-hits)))
	(format t "~&There are ~D one-word query patterns out of ~D query patterns. (~4F\%)"
	number-of-oneword-patterns
	number-of-patterns
	(* 100 (/ number-of-oneword-patterns number-of-patterns))))
	;; f - wildcard queries and patterns
	(multiple-value-bind (number-of-wildcard-patterns number-of-wildcard-searches)
	(hash-table-key-val-stats pattern-histogram #'(lambda (x)
	(declare (type simple-base-string x))
	(position #\* x)))
	(format t "~&There are ~D wildcard searches out of ~D search engine hits. (~4F\%)"
	number-of-wildcard-searches
	number-of-search-engine-hits
	(* 100 (/ number-of-wildcard-searches number-of-search-engine-hits)))
	(format t "~&There are ~D wildcard query patterns out of ~D query patterns. (~4F\%)"
	number-of-wildcard-patterns
	number-of-patterns
	(* 100 (/ number-of-wildcard-patterns number-of-patterns))))
	;; g - punctuation-like queries and patterns
	(multiple-value-bind (number-of-punctuation-patterns number-of-punctuation-searches)
	(hash-table-key-val-stats pattern-histogram #'(lambda (x)
	(declare (type simple-base-string x))
	(or (position #\+ x)
	(position #\- x)
	(position #\/ x))))
	(format t "~&There are ~D punctuation-like searches out of ~D search engine hits. (~4F\%)"
	number-of-punctuation-searches
	number-of-search-engine-hits
	(* 100 (/ number-of-punctuation-searches number-of-search-engine-hits)))
	(format t "~&There are ~D punctuation-like query patterns out of ~D query patterns. (~4F\%)"
	number-of-punctuation-patterns
	number-of-patterns
	(* 100 (/ number-of-punctuation-patterns number-of-patterns))))
	;; h - anyfield queries and patterns
	(multiple-value-bind (number-of-anyfield-patterns number-of-anyfield-searches)
	(hash-table-key-val-stats pattern-histogram #'(lambda (x)
	(declare (type simple-base-string x))
	(not (position #\: x))))
	(declare (ignore number-of-anyfield-searches))
	(format t "~&There are ~D any-field query patterns out of ~D query patterns. (~4F\%)"
	number-of-anyfield-patterns
	number-of-patterns
	(* 100 (/ number-of-anyfield-patterns number-of-patterns))))
	;; print detailed histogram:
	(format t "~&~80,1,0,'.<~A ~; ~A~>" "User query" "no. of occurrences")
	(let ((patterns-to-print (hash-table-keys pattern-histogram t)))
	(if (> (length patterns-to-print) nb-histogram-items-to-print)
	(setq patterns-to-print (subseq patterns-to-print 0 nb-histogram-items-to-print)))
	(dolist (pattern patterns-to-print)
	(format t "~&~80,1,0,'.<~A ~; ~A~>" pattern
	(gethash pattern pattern-histogram))))))))

	(defun print-basket-stats (httpd-all-log-entries)
	"Read HTTPD-LOG-ENTRIES and print stats related to user baskets."
	(declare (special basket-url
	add-to-basket-url
	display-basket-url
	display-public-basket-url))
	(let ((datetime-first (httpd-log-entry-datetime (car httpd-all-log-entries)))
	(datetime-last (httpd-log-entry-datetime (car (last httpd-all-log-entries))))
	(httpd-log-entries (filter-httpd-log-entries httpd-all-log-entries basket-url))
	(basket-user-histogram (make-hash-table :test #'equalp)))
	;; build basket users histogram:
	(dolist (httpd-log-entry-item httpd-log-entries)
	(incf (gethash (httpd-log-entry-ip httpd-log-entry-item)
	basket-user-histogram 0)))
	(format t "~&~%** USER BASKETS STATS")
	;; print basket usage summary info:
	(format t "~&There were ~D user basket page hits." (length httpd-log-entries))
	(format t "~&This makes an average of ~F user basket page hits per day."
	(/ (length httpd-log-entries) (1+ (- datetime-last datetime-first))))
	(format t "~&There were ~D unique basket page users in ~D days."
	(length (hash-table-keys basket-user-histogram))
	(1+ (- datetime-last datetime-first)))
	;; add to basket:
	(format t "~&There were ~D additions to baskets."
	(length (filter-httpd-log-entries httpd-log-entries add-to-basket-url)))
	;; display baskets:
	(format t "~&There were ~D displays of baskets, out of which ~D public baskets accesses."
	(length (filter-httpd-log-entries httpd-log-entries display-basket-url))
	(length (filter-httpd-log-entries httpd-log-entries display-public-basket-url)))))

	(defun print-alert-stats (httpd-all-log-entries)
	"Read HTTPD-LOG-ENTRIES and print stats related to user alerts."
	(declare (special alert-url
	display-your-alerts-url
	display-your-searches-url))
	(let ((datetime-first (httpd-log-entry-datetime (car httpd-all-log-entries)))
	(datetime-last (httpd-log-entry-datetime (car (last httpd-all-log-entries))))
	(httpd-log-entries (filter-httpd-log-entries httpd-all-log-entries alert-url))
	(alert-user-histogram (make-hash-table :test #'equalp)))
	;; build alert users histogram:
	(dolist (httpd-log-entry-item httpd-log-entries)
	(incf (gethash (httpd-log-entry-ip httpd-log-entry-item)
	alert-user-histogram 0)))
	(format t "~&~%** USER ALERTS STATS")
	;; print alert usage summary info:
	(format t "~&There were ~D user alert page hits." (length httpd-log-entries))
	(format t "~&This makes an average of ~F user alert page hits per day."
	(/ (length httpd-log-entries) (1+ (- datetime-last datetime-first))))
	(format t "~&There were ~D unique alert page users in ~D days."
	(length (hash-table-keys alert-user-histogram))
	(1+ (- datetime-last datetime-first)))
	;; display alert:
	(format t "~&There were ~D displays of user alerts."
	(length (filter-httpd-log-entries httpd-log-entries display-your-alerts-url)))
	;; display alerts:
	(format t "~&There were ~D displays of user searches history."
	(length (filter-httpd-log-entries httpd-log-entries display-your-searches-url)))))

	;; Special-purpose entry points:

	(defun print-download-stats (httpd-log-filename file-pattern stats-frequency)
	"Read httpd log file HTTPD-LOG-FILENAME, looking for FILE-PATTERN.
	Print download stats for intervals of STATS-FREQUENCY, which may be
	day, week, or month."
	(declare (ignore httpd-log-filename file-pattern stats-frequency))
	(format t "~&FIXME: Not implemented yet.")
	t)

	(defun extract-page-views-events (httpd-log-filename
	&optional (exclude-ip-list nil))
	"Walk through HTTPD-LOG-FILENAME and extract detailed record page
	views events. Useful to input old statistics of page views into
	Invenio's rnkPAGEVIEWS table. If EXCLUDE-IP-LIST is set, then do not
	count events coming from those IPs."
	(declare (type simple-base-string httpd-log-filename)
	(type list exclude-ip-list))
	(flet ((extract-recid-from-detailed-record-page-url (url)
	(declare (type simple-base-string url))
	(if (search "GET /record/" url)
	(or (parse-integer (subseq url (length "GET /record/")) :junk-allowed t) 0)
	(if (search "GET /search.py?recid=" url) ; compatibility with old style URLs
	(or (parse-integer (subseq url (length "GET /search.py?recid=")) :junk-allowed t) 0)
	0))))
	(format t "-- APACHE LOG FILE ANALYSIS")
	(format t "~&-- Filename: ~A" httpd-log-filename)
	(unless (null exclude-ip-list)
	(format t "~&-- Excluding search engine hits from ~A." exclude-ip-list))
	(with-open-file (input-stream httpd-log-filename :direction :input)
	(do ((line (read-line input-stream nil)
	(read-line input-stream nil)))
	((not line))
	(declare (type (or simple-base-string null) line))
	(let ((httpd-log-entry-item (create-httpd-log-entry (wash-httpd-log-line line))))
	(if (and (or (null exclude-ip-list)
	(not (member (httpd-log-entry-ip httpd-log-entry-item)
	exclude-ip-list :test #'equal)))
	(plusp (extract-recid-from-detailed-record-page-url
	(httpd-log-entry-request httpd-log-entry-item))))
	(format t "~&INSERT INTO rnkPAGEVIEWS (id_bibrec, client_host, view_time) VALUES ('~A', INET_ATON('~A'), '~A');"
	(extract-recid-from-detailed-record-page-url
	(httpd-log-entry-request httpd-log-entry-item))
	(httpd-log-entry-ip httpd-log-entry-item)
	(httpd-log-entry-datetimee httpd-log-entry-item)))))))
	(format t "~&-- DONE"))

	(defun read-conf-file (analyzer-config-file)
	"Read ANALYZER-CONFIG-FILE and initialize variables found there.
	E.g. line there saying `detailed-record-url = \"/record/\"' will get
	parsed as (setf detailed-record-url \"/record/\")."
	(with-open-file (input-stream analyzer-config-file :direction :input)
	(let ((inside-apache-log-analyzer-section nil))
	(do ((line (read-line input-stream nil)
	(read-line input-stream nil)))
	((not line))
	(if (string= line "[apache_log_analyzer]")
	(setf inside-apache-log-analyzer-section t))
	(when inside-apache-log-analyzer-section
	(multiple-value-bind (lhs rhs) (split-string-by-equal-sign line)
	(when lhs
	(setf (symbol-value (intern (string-upcase (concatenate 'string "" lhs ""))))
	(read-from-string rhs)))))))))

	;; Main entry point:

	(defun analyze-httpd-log-file (analyzer-config-file httpd-log-file)
	"Main function that analyzes HTTPD-LOG-FILE according to
	instructions presented in ANALYZER-CONFIG-FILE."
	(declare (special profile
	nb-histogram-items-to-print
	exclude-ip-list
	home-collection
	search-interface-url
	search-interface-url-old-style
	search-engine-url
	search-engine-url-old-style
	detailed-record-url
	basket-url
	add-to-basket-url
	display-basket-url
	display-public-basket-url
	alert-url
	display-your-alerts-url
	display-your-searches-url))
	(read-conf-file analyzer-config-file)
	(when profile
	#+cmu (profile:profile-all)
	#+fixmeclisp (mon:monitor-all)
	#+sbcl (progn
	(sb-profile:profile PRINT-SEARCH-PATTERN-STATS)
	(sb-profile:profile CREATE-HTTPD-LOG-ENTRIES)
	(sb-profile:profile GET-SEARCH-PATTERNS-FROM-HTTPD-LOG-REQUEST-STRING)
	(sb-profile:profile PARSE-HTTPD-LOG-LINE)
	(sb-profile:profile STRING-DECODE-URL)
	(sb-profile:profile CREATE-HTTPD-LOG-ENTRY)
	(sb-profile:profile GET-DATETIME-FROM-HTTPD-LOG-DATETIME-STRING)
	(sb-profile:profile WASH-HTTPD-LOG-LINE)
	(sb-profile:profile GET-MONTH-NUMBER-FROM-MONTH-NAME)
	(sb-profile:profile MAKE-HTTPD-LOG-ENTRY)
	(sb-profile:profile PARSE-INTEGER-OR-ZERO)
	(sb-profile:profile GET-URLARG-VALUES-FROM-URL-STRING)
	(sb-profile:profile HASH-TABLE-KEYS)
	(sb-profile:profile HASH-TABLE-KEY-VAL-STATS)
	(sb-profile:profile STRING-SUBSTITUTE)))
	(let ((httpd-all-log-entries (create-httpd-log-entries httpd-log-file exclude-ip-list)))
	(print-general-stats httpd-all-log-entries)
	(print-search-interface-stats httpd-all-log-entries)
	(print-search-collections-stats httpd-all-log-entries)
	(print-search-pattern-stats httpd-all-log-entries)
	(print-basket-stats httpd-all-log-entries)
	(print-alert-stats httpd-all-log-entries))
	(format t "~&")
	(when profile
	#+cmu (profile:report-time)
	#+fixmeclisp (mon:report)
	#+sbcl (sb-profile:report)))

webstatlib.lispNo OneTemporaryActions

File Metadata

webstatlib.lispView Options

Event Timeline

webstatlib.lisp
No OneTemporary
Actions

webstatlib.lisp
View Options