@@ -0,0 +1,1798 @@
+#!/usr/bin/python3%0A#%0A# This is a tool Josh uses on his box serving mailinabox.email to parse the nginx%0A# access log to see how many people are installing Mail-in-a-Box each day, by%0A# looking at accesses to the bootstrap.sh script.%0A%0Aimport re, glob, gzip, os.path, json%0Aimport dateutil.parser%0A%0Aoutfn = %22/home/user-data/www/mailinabox.email/install-stats.json%22%0A%0A# Make a unique list of (date, ip address) pairs so we don't double-count%0A# accesses that are for the same install.%0Aaccesses = set()%0A%0A# Scan the current and rotated access logs.%0Afor fn in glob.glob(%22/var/log/nginx/access.log*%22):%0A%09# Gunzip if necessary.%0A%09if fn.endswith(%22.gz%22):%0A%09%09f = gzip.open(fn)%0A%09else:%0A%09%09f = open(fn, %22rb%22)%0A%0A%09# Loop through the lines in the access log.%0A%09with f:%0A%09%09for line in f:%0A%09%09%09# Find lines that are GETs on /bootstrap.sh by either curl or wget.%0A%09%09%09m = re.match(rb%22(?P%3Cip%3E%5CS+) - - %5C%5B(?P%3Cdate%3E.*?)%5C%5D %5C%22GET /bootstrap.sh HTTP/.*%5C%22 200 %5Cd+ .* %5C%22(?:curl%7Cwget)%22, line, re.I)%0A%09%09%09if m:%0A%09%09%09%09date, time = m.group(%22date%22).decode(%22ascii%22).split(%22:%22, 1)%0A%09%09%09%09date = dateutil.parser.parse(date).date().isoformat()%0A%09%09%09%09ip = m.group(%22ip%22).decode(%22ascii%22)%0A%09%09%09%09accesses.add( (date, ip) )%0A%0A# Aggregate by date.%0Aby_date = %7B %7D%0Afor date, ip in accesses:%0A%09by_date%5Bdate%5D = by_date.get(date, 0) + 1%0A%0A# Since logs are rotated, store the statistics permanently in a JSON file.%0A# Load in the stats from an existing file.%0Aif os.path.exists(outfn):%0A%09existing_data = json.load(open(outfn))%0A%09for date, count in existing_data:%0A%09%09if date not in by_date:%0A%09%09%09by_date%5Bdate%5D = count%0A%0A# Turn into a list rather than a dict structure to make it ordered.%0Aby_date = sorted(by_date.items())%0A%0A# Pop the last one because today's stats are incomplete.%0Aby_date.pop(-1)%0A%0A# Write out.%0Awith open(outfn, %22w%22) as f:%0A%09json.dump(by_date, f, sort_keys=True, indent=True)%0A

add a tool for me to see how many installations are happening by scanning for accesses to bootstrap.sh