Welford's Algorithm

Computing means and standard deviations for data can be hard.  Naively implementing the formula can result in large numerical errors.

A better way is to use Welford's Algorithm, which is numerically stable.

Below is a gist for doing just that.

	import math
	class Welford(object):
	""" Implements Welford's algorithm for computing a running mean
	and standard deviation as described at:
	http://www.johndcook.com/standard_deviation.html
	can take single values or iterables
	Properties:
	mean - returns the mean
	std - returns the std
	meanfull- returns the mean and std of the mean
	Usage:
	>>> foo = Welford()
	>>> foo(range(100))
	>>> foo
	<Welford: 49.5 +- 29.0114919759>
	>>> foo([1]*1000)
	>>> foo
	<Welford: 5.40909090909 +- 16.4437417146>
	>>> foo.mean
	5.409090909090906
	>>> foo.std
	16.44374171455467
	>>> foo.meanfull
	(5.409090909090906, 0.4957974674244838)
	"""
	def __init__(self,lst=None):
	self.k = 0
	self.M = 0
	self.S = 0
	self.__call__(lst)
	def update(self,x):
	if x is None:
	return
	self.k += 1
	newM = self.M + (x - self.M)*1./self.k
	newS = self.S + (x - self.M)*(x - newM)
	self.M, self.S = newM, newS
	def consume(self,lst):
	lst = iter(lst)
	for x in lst:
	self.update(x)
	def __call__(self,x):
	if hasattr(x,"__iter__"):
	self.consume(x)
	else:
	self.update(x)
	@property
	def mean(self):
	return self.M
	@property
	def meanfull(self):
	return self.mean, self.std/math.sqrt(self.k)
	@property
	def std(self):
	if self.k==1:
	return 0
	return math.sqrt(self.S/(self.k-1))
	def __repr__(self):
	return "<Welford: {} +- {}>".format(self.mean, self.std)

view raw welford.py hosted with ❤ by GitHub

Python Email Notification

So, I occasionally want to run a long simulation, and not really pay attention to when it finishes.

Ideally, I could have python email me when it was done.  Even more ideal, I could have it use gmail to send the email notification.

Well, I've done just that, along with using netrc to hold the password configuration so I know its secure.

The Gist is up at https://gist.github.com/2150894, or copied below.

	#! /usr/bin/env python
	""" Email Me.
	Usage:
	emailme <message>
	emailme [-s] <message>
	emailme [-s] <subject> <message>
	emailme <toaddr> <subject> <message>
	emailme <toaddr> <fromaddr> <subject> <message>
	emailme -h | --help
	emailme --version
	Options:
	-h --help Show help
	-s --sms Send SMS
	--version Show version
	"""
	#Python script to send simple emails
	import smtplib
	from email.mime.text import MIMEText
	import netrc
	from docopt import docopt
	SMTPPORT = 587 # TLS for gmail.
	DEFAULTFROM = "youraddress@gmail.com"
	DEFAULTTO = "youraddress@gmail.com"
	SMSADDRESS = "YOURNUMBER@messaging.sprintpcs.com"
	DEFAULTSUBJECT = "PYTHON NOTIFICATION"
	"""
	Note that by default, this looks to check your netrc credentials
	to use this feature, create a .netrc file, so that only you can read and write it
	touch ~/.netrc
	chmod 600 ~/.netrc
	and then add the information for the gmail smtp server, i.e.
	machine smtp.gmail.com
	login yourusername@gmail.com
	password yourpassword
	This way only you will have access to this file. Do not use your actual account
	password, instead create an application specific password for your Google
	account at: https://security.google.com/settings/security/apppasswords
	For SMS support, use your service providers gateway.
	See: http://en.wikipedia.org/wiki/List_of_SMS_gateways
	Notice that this can be used either as a command line script, as documented
	in the main doc string, or as a python module. To use as a module:
	import emailme
	emailme.send_email("message")
	or
	emailme.send_sms("message")
	"""
	from functools import partial
	def send_email(message="",subject=DEFAULTSUBJECT,
	me=DEFAULTFROM,recipients=[DEFAULTTO],
	smtpserver="smtp.gmail.com",tls=True,login=None,
	password=None):
	""" Send an email using the gmail smtp servers, and netrc to hide the username
	and password information """
	if login is None or password is None:
	secrets = netrc.netrc()
	netrclogin,netrcaccount,netrcpassword = secrets.authenticators(smtpserver)
	if login is None:
	login = netrclogin
	if password is None:
	password = netrcpassword
	msg = MIMEText(message)
	msg['Subject'] = subject
	msg['From'] = me
	msg['To'] = ", ".join(recipients)
	s = smtplib.SMTP(smtpserver, port=SMTPPORT)
	if tls:
	s.starttls()
	s.login(login,password)
	s.sendmail(me,recipients, msg.as_string())
	s.quit()
	send_sms = partial(send_email, recipients =[SMSADDRESS] ,subject="")
	if __name__=="__main__":
	args = docopt(__doc__, version="Email Me version 0.1")
	toaddr = args['<toaddr>'] or DEFAULTTO
	fromaddr = args['<fromaddr>'] or DEFAULTFROM
	message = args['<message>']
	if args['--sms']:
	subject = args['<subject>'] or ''
	send_sms(subject=subject,
	message=message)
	else:
	subject = args['<subject>'] or DEFAULTSUBJECT
	send_email(recipients=[toaddr],
	me=fromaddr,
	subject=subject,
	message=message)

view raw emailme.py hosted with ❤ by GitHub