@@ -0,0 +1,1391 @@
+%0A#Indexer assumes that collection fits in memory%0Aclass Indexer(object):%0A%09def __init__(self):%0A%09%09self.inverted_index = dict()%0A%09%09self.forward_index = dict()%0A%09%09self.url_to_id = dict()%09%0A%09%09self.doc_count = 0%09%09%09%0A%0A%09#assumes that add_document() is never called twice for a document%0A%09# assumes that a document has a unique URL%0A%09def add_document(self,url, parsed_text):%0A%09%09self.doc_count +=1%0A%09%09assert url not in self.url_to_id%0A%09%09current_id = self.doc_count%0A%09%09self.forward_index%5Bcurrent_id%5D = parsed_text%0A%09%09for word in parsed_text:%0A%09%09%09if word not in self.inverted_index:%0A%09%09%09%09self.inverted_index%5Bword%5D = %5B%5D%0A%09%09%09self.inverted_index%5Bword%5D.append(current_id)%0A%09%0A%09def store_on_disk(self, index_dir):%0A%09%09inverted_index_file_name = os.path.join(index_dir, %22inverted_index%22)%0A%09%09forward_index_file_name  = os.path.join(index_dir, %22forward_index%22)%0A%09%09inverted_index_file = open(inverted_index_file_name, %22w%22)%0A%09%09forward_index_file_name = open(forward_index_file_name, %22w%22)%09%0A%09%09json.dump(self.inverted_index, inverted_index_file)%0A%09%09json.dump(self.forward_index, forward_index_file)%0A%0A%09%0Adef main():%0A%09parser = argparse.ArgumentParser(description='Index https://www.reddit.com/r/learnprogramming/')%0A%09parser.add_argument(%22--stored_documents%22, dest = %22stored_documents%22)%0A%09parser.add_argument(%22--index_dir%22, dest = index_dir%22)%0A%09args = parser.parse_args()%0A%09%0A%09%09%0A#are we involing this from cli%0Aif __name__ == %22__main__%22: %0A%09main()%0A%09%0A

Create Indexer.py