#!/usr/bin/env python
#
# Gather revision statistics from version control system.
# The process is not optimized for speed or anything. It
# just uses command line tools and updates working copy
# from revision to revision..
# 
# 1. Get a fresh clone / checkout
# 2. Run gather.py inside of it
#
#
# --- dataset 1: size ---
#
# - size of all files in revision
# - number of files
# - number of dirs
#
#
# status
#
# [ ] subversion traversing
#   [ ] getting list of all revisions
#   [ ] processing one revision at a time
#     [ ] update copy to revision
#       [ ] stop on error
#       [ ] save progress
#       [ ] rollback bad revision data
#       [ ] ...
#
# [ ] mercurial traversing
#   [x] getting list of all revisions
#   [ ] processing one revision at a time
#     [ ] update copy to revision
#       [ ] stop on error
#       [ ] save progress
#       [ ] rollback bad revision data
#       [ ] ...

import copy
import os
import sys
import subprocess


SET1 = {
  'totalsize': 0,
  'dirsnum': 0, 
  'filesnum': 0,
}



def runout(cmd):
  return subprocess.check_output(cmd, shell=True)


class HG(object):

  def revlist(self):
    """get list of revisions from oldest to youngest"""
    output = runout('hg log --template "{rev}\\n"')
    rev = []
    for line in output.splitlines():
      rev.append(line)
    return reversed(rev)


def process(path):
  """calculate SET1 directory stats for given path"""

  # unicode is critical to for non-English local names on Windows
  path = unicode(path)

  s = copy.copy(SET1)
  s['totalsize'] = 0
  for root, dirs, files in os.walk(path):
    for f in files:
      s['totalsize'] += os.path.getsize(os.path.join(root, f))
    s['filesnum'] += len(files)
    s['dirsnum'] += len(dirs)

  return s


if __name__ == '__main__':
  # safety check
  if len(runout('hg status')) != 0:
    sys.exit('Error: Working copy is not clean, can not continue')


  # get API to repository information
  repapi = HG()

  # CSV header 
  print "revision, size, files, dirs"
  for rev in repapi.revlist():
    line = process('.')
    line['rev'] = rev
    #print line
    print "{rev}, {totalsize}, {filesnum}, {dirsnum}".format(**line)


@@ -1355,24 +1355,35 @@
 process(path
+, ignore=%5B%5D
 ):%0A  %22%22%22calc
@@ -1423,16 +1423,92 @@
 ven path
+, skipping%0A     directories mentioned in ignore (e.g. '.hg', '.svn', ...)%0A  
 %22%22%22%0A%0A  #
@@ -1679,16 +1679,109 @@
 (path):%0A
+    # filter directories%0A    for ig in ignore:%0A      if ig in dirs:%0A        dirs.remove(ig)%0A%0A
     for 
@@ -2180,18 +2180,18 @@
 ze, 
-files, dir
+dirs, file
 s%22%0A 
@@ -2216,24 +2216,56 @@
 .revlist():%0A
+    runout('hg up -r %25s' %25 rev)%0A
     line = p
@@ -2274,16 +2274,32 @@
 cess('.'
+, ignore=%5B'.hg'%5D
 )%0A    li
@@ -2366,20 +2366,19 @@
 %7D, %7B
-file
+dir
 snum%7D, %7B
 dirs
@@ -2369,27 +2369,28 @@
 %7Bdirsnum%7D, %7B
-dir
+file
 snum%7D%22.forma

Add actual mercurial copy update from revision to revision, put dirnum first, ignore '.hg' service dir