# Miscellaneous natural language programming functions.
# TODO: it's kind of stupid to have curses be a dependency just because of one
# function

from curses.ascii import isdigit
from nltk.corpus import cmudict
import pylab

d = cmudict.dict()

def tokenize(text):
    """Tokenizes a given block of text. See /doc/tokenization for details."""
    #1. Replace hyphens with spaces
    noHyphens = text.replace('-', ' ')

    #2. Split all words by spaces.
    noSpaces = noHyphens.split(' ')

    #3. Remove all punctuation.
    punctuation = ".!?;,:\"\'()[]{}"
    noPunc = []
    for word in noSpaces:
        for char in word:
            if char is in punctuation:
                noPunc = noPunc + word.strip(punctuation)
            else
                noPunc = noPunc + word
    return noPunc

def syllableCount(word):
    """Return the ESTIMATED number of syllables in a given word. Returns none if
    the word is not inthe dictionary. Be warned that there isn't a deterministic
    way to count syllables, and that some words with multiple pronunciations
    have ambiguous numbers of syllables. For words with multiple pronunciations,
    this method returns the number of syllables found in the first pronunciation
    of a given word found.
    """
    if word.lower() not in d:
        return None
    else:
        return nsyl(word.lower())

def nsyl(word):
    """Return the max syllable count."""
    # TODO: Near-indecipherable one-liner from stackoverflow; should probably replace
    # this.
    syllables = [len(list(y for y in x if y[-1].isdigit())) for x in d[word.lower()]]
    return syllables[0]

@@ -140,16 +140,39 @@
 nction%0A%0A
+import pylab%0Aimport re%0A
 from cur
@@ -227,29 +227,16 @@
  cmudict
-%0Aimport pylab
 %0A%0Ad = cm
@@ -527,16 +527,111 @@
 uation.%0A
+    # TODO: Deal with apostrophes better. We don't want to strip them from%0A    # contractions.%0A
     punc
@@ -641,17 +641,20 @@
 tion = %22
-.
+%5B%5C.,
 !?;,:%5C%22%5C
@@ -660,12 +660,17 @@
 %5C'()
-%5B%5D%7B%7D
+%5C%5B%5C%5D%5C%7B%5C%7D%5D
 %22%0A  
@@ -721,178 +721,72 @@
     
-for char in word:%0A            if char is in punctuation:%0A                noPunc = noPunc + word.strip(punctuation)%0A            else%0A                noPunc = noPunc +
+print(word)%0A        noPunc.append(re.sub(punctuation, '',
  word
+))
 %0A   
@@ -972,13 +972,11 @@
 e is
-n't a
+ no
  det
@@ -1108,75 +1108,46 @@
 es. 
-For words with multiple pronunciations,%0A    this method
+We cope with this by
  return
-s
+ing
  the
+%0A   
  num
@@ -1195,20 +1195,16 @@
 nciation
-%0A   
  of a gi
@@ -1211,22 +1211,16 @@
 ven word
- found
 .%0A    %22%22
@@ -1355,10 +1355,14 @@
 he m
-ax
+inimum
  syl

Use regex to strip punctuation