@@ -0,0 +1,2189 @@
+from scripts.rake import *%0Aimport json%0A%0A%0A# get review texts aggregated by asin id%0Adef get_rdd(base, input, num_part):%0A                base_dir = os.path.join(base)%0A                input_path = os.path.join(input)%0A                file_name = os.path.join(base_dir, input_path)%0A                # load data%0A                rdd = sc.textFile(file_name, num_part)%0A                rdd_j = rdd.map(json.loads)%0A                rdd_j.cache()%0A                return rdd_j%0A%0A%0Anum_part = 16%0Arevs = get_rdd('data', 'reviews_electronics.json', num_part)%0Arev_texts = revs.map(lambda x: (x%5B'asin'%5D, x%5B'reviewText'%5D))%0Arev_agg_texts = rev_texts.map(lambda (asin, text): (asin, %5Btext%5D)).reduceByKey(lambda x, y: x + y)%0Arev_agg_texts.cache()%0A%0Aprods = get_rdd('data', 'meta_electronics.json', num_part)%0Acateg = prods.map( lambda x: (x.get('asin'), x.get('categories')) )%0Acateg = categ.flapMapValues(lambda x: x)%0Acomputers = categ_.filter( lambda (asin, cats): 'Computers & Accessories' in cats )%0Aprods_ = prods_.join(computers)%0Aprods.cache()%0A%0A# (asin, (%5Breview%5D, (d_prod, %5Bcategory%5D)) )%0Aitems = rev_agg_texts.join(prods_)%0Aitems = items.map( lambda (asin, (reviews, (d_prod, categories))): (asin, reviews, d_prod, categories) )%0A%0A%0A# 1. RAKE: keyword. use rake algorithm to extract keywords and take top 10 keywords from each asin%0Arake = Rake('data/MergedStopList.txt') # TODO: add more into this list%0Aitems_wk = items.map( lambda (asin, reviews, d_prod, categories): (asin, rake.run(' '.join(reviews)), reviews, d_prod, categories) )%0A%0A# 2. NP: noun phrasee among these keywords%0Aimport nltk%0Afrom scripts.np_extractor import *%0Aitems_wk.cache()%0Aitems_np = items_wk.map(lambda (asin, pairs, reviews, d_prod, categories): %0A                               (asin, %5B(NPExtractor(string).extract(), score) for (string, score) in pairs%5D, reviews, d_prod, categories)%0A                       )%0Aitems_np = items_np.map(lambda (asin, pairs, reviews, d_prod, categories):%0A                               (asin, %5B(toks, scr) for (toks, scr) in pairs if len(toks) %3E 0%5D, reviews, d_prod, categories)%0A                       )%0A%0A%0A%0A# 3. output%0Aimport pandas as pd%0Adf = pd.DataFrame(items_np.collect())%0Adf.to_csv('data/computers_kw.csv')%0A

Create extract_computer_and_accessories.py