#!/usr/bin/python

#-----------------------------------------------------------------
# @file: crawler.py
# @description: wrapper for Gobblin which handles config file
#               parsing and glue logic
#-----------------------------------------------------------------

import sys
import os
import config
import subprocess
import time
import glob

command = sys.argv[1]

if command == 'start' or command == 'run' or command == 'gogobunnies':
  # generate Gobblin .pull file
  pull_file = open("gobblin/test_config/simplejson.pull", "w")
  pull_file.write("""
  #########################################################################
  ############### Autogenerated Gobblin configuration file ################
  #########################################################################

  job.name=GobblinCrawl
  job.group=crawler
  job.description=A Gobblin job for the metadata crawler
  """)
  pull_file.write("job.schedule=" + config.schedule)
  pull_file.write("""

  source.class=gobblin.example.simplejson.SimpleJsonSource
  converter.classes=gobblin.example.simplejson.SimpleJsonConverter
  extract.namespace=gobblin.example.simplejson

  # source configuration properties
  # comma-separated list of file URIs (supporting different schemes, e.g., file://, ftp://, sftp://, http://, etc)
  """);

  pull_file.write('source.filebased.files.to.pull=')
  for d in config.directories_to_crawl:
    pull_file.write(d + ',')
  pull_file.write("""
  # source data schema
  source.schema={"namespace":"example.avro", "type":"record", "name":"User", "fields":[{"name":"filename", "type":"string"}, {"name":"timestamp", "type":"string"}, {"name":"namespace", "type":"string"}, {"name":"lastAccessTime", "type":"string"}, {"name":"lastModifiedTime",  "type":"string"}, {"name":"creationTime", "type":"string"} , {"name":"size", "type":"string"}, {"name":"isSymbolicLink", "type":"string"}, {"name":"isRegularFile", "type":"string"}, {"name":"isOther", "type":"string"}, {"name":"isDirectory", "type":"string"}]}

  # quality checker configuration properties
  qualitychecker.task.policies=gobblin.policies.count.RowCountPolicy,gobblin.policies.schema.SchemaCompatibilityPolicy
  qualitychecker.task.policy.types=OPTIONAL,OPTIONAL
  qualitychecker.row.policies=gobblin.policies.schema.SchemaRowCheckPolicy
  qualitychecker.row.policy.types=OPTIONAL
  qualitychecker.row.err.file=test/jobOutput

  # data publisher class to be used
  data.publisher.type=gobblin.publisher.BaseDataPublisher

  # writer configuration properties
  writer.destination.type=HDFS
  writer.output.format=AVRO
  writer.fs.uri=file:///
  """)

  pull_file.close()

  # clear out old work directory
  subprocess.call(["rm", "-rf", "gobblin/test_workdir"])
  subprocess.call(["rm", "-rf", "gobblin/output-tarballs"])

  # create work and output directories
  subprocess.call(["mkdir", "gobblin/test_workdir"])
  subprocess.call(["mkdir", "gobblin/output-tarballs"])

  # build path to output directory so we can setup a watch on it (a bit hacky)
  dirs_to_create = ["job-output/", "gobblin/", "example/", "simplejson/", "ExampleTable/"]
  path = "gobblin/test_workdir/"
  for i in range(0, len(dirs_to_create)):
    path = path + dirs_to_create[i]
    subprocess.call(["mkdir", path])

  # start a monitor to check for new files in the job output directory
  inotify_process = subprocess.Popen(["./inotifywait.sh", "gobblin/test_workdir", config.data_mover_location])
  pid_file = open('.inotify_pid', 'w')
  pid_file.write(str(inotify_process.pid))

  # run Gobblin
  os.chdir("gobblin")
  subprocess.call(["./run_gobblin_standalone.sh"])

elif command == 'stop' or command == 'kill' or command == 'bunnydeathtoall':
  pid_file = open('.inotify_pid', 'r')
  inotify_pid = pid_file.readlines()[0]
  os.killpg(os.getpgid(int(inotify_pid)), 9)




@@ -472,16 +472,124 @@
 ll file%0A
+  subprocess.call(%5B%22rm%22, %22-rf%22, %22gobblin/test_config%22%5D)%0A  subprocess.call(%5B%22mkdir%22, %22gobblin/test_config%22%5D)%0A
   pull_f
@@ -599,17 +599,17 @@
  = open(
-%22
+'
 gobblin/
@@ -639,14 +639,14 @@
 pull
-%22, %22w%22
+', 'w'
 )%0A  

make test_config directory before trying to open .pull file to write