Login

Logging and statistic middleware

Author:
feuervogel
Posted:
February 15, 2008
Language:
Python
Version:
.96
Tags:
middleware stats logging statistic user-activities
Score:
2 (after 2 ratings)

Reviewing some statistic-middleware-classes I think you might need one wich is working correct and high-performant.

Please comment it and have fun with it!

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
#-*- coding: utf-8 -*-
from django.conf import settings
import thread,threading, time, random, base64, time, os, datetime, gzip
from xml.dom.minidom import Document
from django.db import connection

class LoggingActivities(threading.Thread):
	"""
	Author: Julian Moritz, public@julianmoritz.de
	This middleware is logging the website-activities on your django-project-website.
	It does NOT save the users' ip-addresses (according to german law) but a unique ID for each user.
	Once added to your MIDDLEWARE_CLASSES, you can control it via 3 variables in your settings.py:
	- STATS_LOGDIR - the directory-name where it puts the logfiles into (default: logs)
	- STATS_BUFFER - the file-buffer-size in bytes for your logfile (default: 10MB)
	- STATS_TIME - the time the logfile is flushed to disk (default: 12 hours)
	Note: At the moment there is now import to database and evaluating script written. I will do this in time.
	License: http://creativecommons.org/licenses/by/2.0/de/deed.en_GB
	Output: Have a look at http://www.julianmoritz.de/dl/logs.xml
	"""
	def __init__(self):
		threading.Thread.__init__(self)
		try:
			l = settings.STATS_LOGDIR
		except:
			l = "logs"
			
		if not os.path.isdir(l):
			os.mkdir(l)
			
		try:
			b = settings.STATS_BUFFER
		except:
			b = 1024*1024*10
			
		try:
			t = settings.STATS_TIME
		except:
			t = 60*60*12
		
		self.buffer = b
		self.logdir = l
		self.time = t
		self.lock = thread.allocate_lock()
		self.__rand_id = base64.b64encode(str(random.random()))
		fname = base64.b64encode(str(random.random())) + ".xml"
		self.logfile = open(os.path.join(self.logdir, fname), "w", self.buffer)
		self.xmldoc = Document()
		self.logs = self.xmldoc.createElement("logs")
		self.xmldoc.appendChild(self.logs)
		self.start()
	
	def process_request(self,request):
		request.STATS = {}
		request.STATS["start"] = datetime.datetime.now()
		return None
		
	def process_view(self, request, view, args, kwargs):
		request.STATS["view_name"] = view.__name__
		return None

	def process_excepton(self,request,exception):
		self.log_request(request,exception)
		return None

	def process_response(self,request,response):
		self.log_request(request)
		return response
	
	def log_request(self,request,exception=None):
		id = base64.b64encode(str(random.random()))
		now = int(time.mktime(datetime.datetime.now().timetuple()))
		
		try:
			start = request.STATS["start"]
			stop = datetime.datetime.now()
			diff = (stop - start)
			diffsecs = diff.seconds + diff.microseconds/1000000.
		except:
			diffsecs = 0.
			
		try:
			view_name = request.STATS["view_name"]
		except:
			view_name = ""
			
		user_agent = request.META["HTTP_USER_AGENT"]
		user_enc = request.META["HTTP_ACCEPT_ENCODING"]
		user_lang = request.META["HTTP_ACCEPT_LANGUAGE"]
		full_path = request.get_full_path()
		abs_uri = request.build_absolute_uri()
		query_count = len(connection.queries)
		user_id = base64.b64encode(self.__rand_id + request.META["REMOTE_ADDR"])
		
		if exception == None:
			exc_msg = ""
		else:
			exc_msg = str(exception)
		
		xml_dict = {"timestamp":now,
				    "user_id":user_id,
				    "duration":diffsecs,
				    "view":view_name,
				    "http_user_agent":user_agent,
				    "http_accept_encoding":user_enc,
				    "http_accept_language":user_lang,
				    "full_path":full_path,
				    "absolute_uri":abs_uri,
				    "query_count":query_count,
				    "exception":exc_msg,
				    }
		#create xml
		self.lock.acquire()
		log = self.xmldoc.createElement("log")
		log.setAttribute("id",id)
		for k in xml_dict:
			e = self.xmldoc.createElement(k)
			v = self.xmldoc.createTextNode(str(xml_dict[k]))
			e.appendChild(v)
			log.appendChild(e)
		self.logs.appendChild(log)			
		self.lock.release()			
		
				
	def run(self):
		while(True):
			try:
                                #it's bedtime now!
				time.sleep(self.time)
				self.lock.acquire()
				#write xml to file
				self.xmldoc.writexml(self.logfile,indent="\t", addindent="\t", newl="\n")
				#flush and close file
				self.logfile.flush()
				self.logfile.close()
				#create new empty xml-document
				self.xmldoc = Document()
				self.logs = self.xmldoc.createElement("logs")
				self.xmldoc.appendChild(self.logs)
				#create new empty log-file
				oldfilename = self.logfile.name
				fname = base64.b64encode(str(random.random())) + ".xml"
				self.logfile = open(os.path.join(self.logdir, fname), "w", self.buffer)
				#finished writing
				self.lock.release()
				#gzip old file and delete the old xml-file
				r_file = open(oldfilename, 'r')
				w_file = gzip.GzipFile(oldfilename + '.gz', 'w', 9)
				w_file.write(r_file.read())
				w_file.flush()
				w_file.close()
				r_file.close()
				os.unlink(oldfilename)
			except Exception, e:
				if self.lock.locked():
					self.lock.release()
				break
		
		

More like this

  1. logging tastypie middleware by lalutium 4 months, 3 weeks ago
  2. Log Django exceptions to Apache error log in mod_wsgi by simon 5 years, 6 months ago
  3. Middleware to log username into apache logs with mod_python by kahless 8 years, 1 month ago
  4. LogViewTimeMiddleware by majgis 2 years, 8 months ago
  5. Request time logging middleware by mpasternacki 5 years, 3 months ago

Comments

Please login first before commenting.