#!/usr/bin/env python

from __future__ import division
import sys

current_ngram = None
cur_values = []


def process_values(values):
    orig_ngram = None
    entity_counts = []
    # prevent zero division
    non_entity_counts = [1]
    for value in values:
        ngram_type, num = value.split('|--|')
        if ngram_type == 'lower':
            non_entity_counts.append(int(num))
        else:
            orig_ngram = ngram_type
            entity_counts.append(int(num))
    if orig_ngram:
        print '%s\t%s' % (orig_ngram, sum(entity_counts)/sum(non_entity_counts))


# input comes from STDIN
for line in sys.stdin:
    # remove leading and trailing whitespace
    line = line.strip()

    # parse the input we got from mapper.py
    ngram, value = line.split('\t')

    if not current_ngram:
        current_ngram = ngram

    # this IF-switch only works because Hadoop sorts map output
    # by key (here: word) before it is passed to the reducer
    if ngram == current_ngram:
        cur_values.append(value)
    else:
        process_values(cur_values)
        cur_values = []
        current_ngram = ngram

process_values(cur_values)

@@ -229,9 +229,8 @@
  = %5B
-1
 %5D%0A  
@@ -483,32 +483,102 @@
  if orig_ngram:%0A
+        if not non_entity_counts:%0A            non_entity_counts = %5B1%5D%0A
         print '%25
@@ -1152,24 +1152,29 @@
 r_values = %5B
+value
 %5D%0A        cu

Update reducer_lowercase.py