@@ -0,0 +1,2605 @@
+%0Afrom map import mapping%0A# walk through the os and get all files%0A# read each file in tern and go through line by line%0A# print lines that contain smell and the report name%0Afrom os import listdir%0Aimport nltk.data%0Aimport json%0A%0ASMELL_WORDS = %5B'smell', 'stench', 'stink', 'odour', 'sniff', 'effluvium'%5D%0AREPORTS_DIR = '/Users/deborah/Documents/scripts/python_work/project2016/Full Text Online'%0A%0Aglobal finalResult%0AfinalResult = %7B%7D%0A%0Adef addToDic(d, report, rDate, val):%0A    d.setDefault(report, %5B%5D).append(val)%0A    return d%0A%0A%0Adef getFileNames():%0A    '''Retrieve file names'''%0A    fileNames = %5Bf for f in listdir(REPORTS_DIR) if f.endswith('txt')%5D%0A    return fileNames%0A%0A%0Adef processFile(fileName):%0A    path = REPORTS_DIR + '/' + fileName%0A    references = %5B%5D%0A    with open(path) as f:%0A        for line in f:%0A            report_tokenized = tokenize(line)%0A            for scentence in report_tokenized:%0A                for word in SMELL_WORDS:%0A                    if word in scentence.lower():%0A                        references.append(scentence)%0A    return references%0A%0A%0Adef tokenize(sentence):%0A    parser = nltk.data.load('tokenizers/punkt/english.pickle')%0A    result = parser.tokenize(sentence.strip())%0A    return result%0A%0A%0Adef saveObject(results):%0A    '''Save results dictionary as file'''%0A    with open('processed_results.txt', 'w') as outfile:%0A        json.dump(results, outfile)%0A%0A%0Adef performAnalysis(fileName, references):%0A    '''Create the resuts output'''%0A    # splits a fileName into :%5B'Acton', '1900', 'b19783358', 'txt'%5D%0A    splitReport = fileName.split('.')%0A    bID = splitReport%5B2%5D%0A    year = splitReport%5B1%5D%0A%0A    try:%0A        region = mapping%5BbID%5D%0A    except:%0A        return%0A        # print bID%0A%0A    if region in finalResult:%0A        nestedDic = finalResult%5Bregion%5D%0A    else:%0A        nestedDic = %7B%7D%0A    %0A    nestedDic%5Byear%5D = references%0A    finalResult%5Bregion%5D = nestedDic%0A%0A    # if nestedDic%5BsplitReport%5B1%5D%5D:%0A    #     val = nestedDic%5BsplitReport%5B1%5D%5D%0A    #     nestedDic%5BsplitReport%5B1%5D%5D = len(references) + val%0A    # else:%0A    #     if len(references):%0A    #         nestedDic%5BsplitReport%5B1%5D%5D = len(references)%0A    # # nestedDic.setDefault(splitReport%5B1%5D, 0).__add__(len(references))%0A    # result%5Bregion%5D = nestedDic%0A%0A# print(result)%0A# for k,v in result.iteritems():%0A%0A%0A%0Adef main():%0A    # tokenize(s)%0A    fileNames = getFileNames()%0A    # f1 = fileNames%5B0%5D%0A    # processFile(f1)%0A    fileNames = fileNames%5B:100%5D%0A    for f in fileNames:%0A        references = processFile(f)%0A        if references:%0A            performAnalysis(f, references)%0A    saveObject(finalResult)%0A%0A%0Aif __name__ == '__main__':%0A    main()

Add script to datamine the reports via NLTK