#!/usr/bin/env python

import re
import json
import fileinput
import dateutil.parser

MAX_WORDS = 200

word_counts = {}
stop_words = set(["a","able","about","across","after","all","almost","also","am","among","an","and","any","are","as","at","be","because","been","but","by","can","cannot","could","dear","did","do","does","either","else","ever","every","for","from","get","got","had","has","have","he","her","hers","him","his","how","however","i","if","in","into","is","it","its","just","least","let","like","likely","may","me","might","most","must","my","neither","no","nor","not","of","off","often","on","only","or","other","our","own","rather","said","say","says","she","should","since","so","some","than","that","the","their","them","then","there","these","they","this","tis","to","too","twas","us","wants","was","we","were","what","when","where","which","while","who","whom","why","will","with","would","yet","you","your"])

for line in fileinput.input():
    tweet = json.loads(line)
    for word in tweet['text'].split(' '):
        word = word.lower()
        word = word.replace(".", "")
        if len(word) == 0 or len(word) == 1: continue
        if word in stop_words: continue
        if word[0] in ["@", "#"]: continue
        if word.startswith("http://"): continue
        if word.startswith("rt"): continue
        if not re.match('^[a-z]', word): continue
        word_counts[word] = word_counts.get(word, 0) + 1

sorted_words = word_counts.keys()
sorted_words.sort(lambda a, b: cmp(word_counts[b], word_counts[a]))
top_words = sorted_words[0:MAX_WORDS]

words = []
count_range = word_counts[top_words[0]] - word_counts[top_words[-1]]
size_ratio = 200.0 / count_range
for word in top_words:
    words.append({
        "text": word, 
        "size": int(word_counts[word] * size_ratio)
    })

print """<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<script src="http://d3js.org/d3.v3.min.js"></script>
<script src="https://raw.github.com/jasondavies/d3-cloud/master/d3.layout.cloud.js"></script>
</head>
<body>
<script>
  var fill = d3.scale.category20();
  var words = %s

  d3.layout.cloud().size([800, 800])
      .words(words)
      .rotate(function() { return ~~(Math.random() * 2) * 90; })
      .font("Impact")
      .fontSize(function(d) { return d.size; })
      .on("end", draw)
      .start();

  function draw(words) {
    d3.select("body").append("svg")
        .attr("width", 1000)
        .attr("height", 1000)
      .append("g")
        .attr("transform", "translate(400,400)")
      .selectAll("text")
        .data(words)
      .enter().append("text")
        .style("font-size", function(d) { return d.size + "px"; })
        .style("font-family", "Impact")
        .style("fill", function(d, i) { return fill(i); })
        .attr("text-anchor", "middle")
        .attr("transform", function(d) {
          return "translate(" + [d.x, d.y] + ")rotate(" + d.rotate + ")";
        })
        .text(function(d) { return d.text; });
  }
</script>
</body>
</html>
""" % json.dumps(words)

@@ -38,16 +38,30 @@
 rt json%0A
+import urllib%0A
 import f
@@ -969,16 +969,29 @@
 nput():%0A
+    try:%0A    
     twee
@@ -1011,16 +1011,41 @@
 s(line)%0A
+    except:%0A        pass%0A
     for 
@@ -1168,15 +1168,32 @@
 rd) 
-== 0 or
+%3C 3: continue%0A        if
  len
@@ -1203,12 +1203,12 @@
 rd) 
-==
+%3E
  1
+5
 : co
@@ -1312,33 +1312,31 @@
  if 
-word.startswith(%22
+re.match('
 http
-://%22
+s?', word
 ): c
@@ -1420,24 +1420,39 @@
 %5Ba-z%5D', word
+, re.IGNORECASE
 ): continue%0A
@@ -1743,17 +1743,17 @@
 ratio = 
-2
+1
 00.0 / c
@@ -1786,16 +1786,69 @@
 _words:%0A
+    size = int(word_counts%5Bword%5D * size_ratio) + 15 %0A
     word
@@ -1901,49 +1901,140 @@
 e%22: 
-int(word_counts%5Bword%5D * size_ratio)%0A    %7D
+size%0A    %7D)%0A%0Awordcloud_js = urllib.urlopen('https://raw.githubusercontent.com/jasondavies/d3-cloud/master/d3.layout.cloud.js').read(
 )%0A%0Ap
@@ -2265,16 +2265,91 @@
 script%3E%0A
+%0A  // embed Jason Davies' d3-cloud since it's not available in a CDN%0A  %25s%0A%0A
   var fi
@@ -3303,16 +3303,31 @@
 l%3E%0A%22%22%22 %25
+ (wordcloud_js,
  json.du
@@ -3331,14 +3331,25 @@
 .dumps(words
+, indent=2)
 )%0A

embed d3-cloud ; adjust sizes