@@ -0,0 +1,846 @@
+from optparse import OptionParser%0Afrom pyspark import SparkContext, SparkConf, StorageLevel%0Aimport json%0Afrom etk.core import Core%0Aimport codecs%0A__author__ = 'amandeep'%0A%0A%0Aif __name__ == '__main__':%0A    compression = %22org.apache.hadoop.io.compress.GzipCodec%22%0A%0A    parser = OptionParser()%0A    (c_options, args) = parser.parse_args()%0A    input_path = args%5B0%5D%0A    output_path = args%5B1%5D%0A    extraction_config_path = args%5B2%5D%0A%0A    sc = SparkContext(appName=%22ETK-Extractions%22)%0A    conf = SparkConf()%0A    extraction_config = json.load(codecs.open(extraction_config_path))%0A    c = Core(extraction_config=extraction_config)%0A    input_rdd = sc.sequenceFile(input_path).partitionBy(100)%0A    input_rdd = input_rdd.mapValues(json.loads).mapValues(c.process).mapValues(json.dumps)%0A    input_rdd.saveAsSequenceFile(output_path, compressionCodecClass=compression)%0A%0A

add code to run spacy in spark