#!/usr/bin/env python
# Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import uuid
import os

from django.core.management.base import NoArgsCommand
from django.utils.translation import ugettext as _

from hadoop import cluster

from useradmin.models import install_sample_user
from indexer import utils, controller


LOG = logging.getLogger(__name__)


class Command(NoArgsCommand):
  """
  Install examples but do not overwrite them.
  """
  def handle_noargs(self, **options):
    self.user = install_sample_user()
    self.fs = cluster.get_hdfs()
    self.searcher = controller.CollectionManagerController(self.user)

    LOG.info(_("Installing twitter collection"))
    twitter_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../../../../apps/search/examples/collections/solr_configs_twitter_demo/index_data.csv'))
    self._setup_collection_from_csv({
      'name': 'twitter_example',
      'fields': self._parse_fields(twitter_path),
      'uniqueKeyField': 'id'
    }, twitter_path)
    LOG.info(_("Twitter collection successfully installed"))

    LOG.info(_("Installing yelp collection"))
    twitter_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../../../../apps/search/examples/collections/solr_configs_yelp_demo/index_data.csv'))
    self._setup_collection_from_csv({
      'name': 'yelp_example',
      'fields': self._parse_fields(twitter_path),
      'uniqueKeyField': 'id'
    }, twitter_path)
    LOG.info(_("Yelp collection successfully installed"))

    LOG.info(_("Installing jobs collection"))
    twitter_path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../../../../apps/search/examples/collections/solr_configs_jobs_demo/index_data.csv'))
    self._setup_collection_from_csv({
      'name': 'jobs_example',
      'fields': self._parse_fields(twitter_path),
      'uniqueKeyField': 'id'
    }, twitter_path)
    LOG.info(_("Jobs collection successfully installed"))

  def _setup_collection_from_csv(self, collection, path, separator=',', quote_character='"'):
    if self.searcher.collection_exists(collection['name']):
      self.searcher.delete_collection(collection['name'])

    # Create instance directory, collection, and add fields
    self.searcher.create_collection(collection['name'], collection['fields'], collection['uniqueKeyField'])

    try:
      hdfs_path = '/tmp/%s' % uuid.uuid4()

      # Put in HDFS
      with open(path) as fh:
        if self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.exists, hdfs_path):
          overwrite = True
        else:
          overwrite = False
        self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.create, hdfs_path, data=fh.read(), overwrite=overwrite)

      # Index data
      self.searcher.update_data_from_hdfs(self.fs,
                                     collection['name'],
                                     collection['fields'],
                                     hdfs_path,
                                     'separated',
                                     separator=separator,
                                     quote_character=quote_character)

    except:
      self.searcher.delete_collection(collection['name'])
      raise
    finally:
      # Remove HDFS file
      if self.fs.do_as_user(self.fs.DEFAULT_USER, self.fs.exists, hdfs_path):
        self.fs.remove(hdfs_path, skip_trash=True)

  def _parse_fields(self, path, separator=',', quote_character='"'):
    with open(path) as fh:
      field_generator = utils.field_values_from_separated_file(fh, separator, quote_character)
      row = next(field_generator)
      field_names = row.keys()
      field_types = utils.get_field_types(row.values())
      return [{'name': field[0], 'type': field[1]} for field in zip(field_names, field_types)]

@@ -1399,32 +1399,24 @@
 tion%22))%0A    
-twitter_
 path = os.pa
@@ -1557,32 +1557,32 @@
 dex_data.csv'))%0A
+
     self._setup_
@@ -1667,32 +1667,24 @@
 arse_fields(
-twitter_
 path),%0A     
@@ -1710,32 +1710,24 @@
 'id'%0A    %7D, 
-twitter_
 path)%0A    LO
@@ -1828,32 +1828,24 @@
 tion%22))%0A    
-twitter_
 path = os.pa
@@ -2090,32 +2090,24 @@
 arse_fields(
-twitter_
 path),%0A     
@@ -2133,32 +2133,24 @@
 'id'%0A    %7D, 
-twitter_
 path)%0A    LO
@@ -2252,24 +2252,16 @@
 %22))%0A    
-twitter_
 path = o
@@ -2514,24 +2514,16 @@
 _fields(
-twitter_
 path),%0A 
@@ -2557,24 +2557,16 @@
 %0A    %7D, 
-twitter_
 path)%0A  
@@ -2622,16 +2622,518 @@
 led%22))%0A%0A
+    LOG.info(_(%22Installing logs collection%22))%0A    path = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../../../../../apps/search/examples/collections/solr_configs_log_demo/index_data.csv'))%0A    self._setup_collection_from_csv(%7B%0A      'name': 'logs_example',%0A      'fields': self._parse_fields(path, fieldtypes=%7B%0A        'region_code': 'string',%0A        'referer': 'string'%0A      %7D),%0A      'uniqueKeyField': 'id'%0A    %7D, path)%0A    LOG.info(_(%22Logs collection successfully installed%22))%0A%0A
   def _s
@@ -4596,24 +4596,39 @@
 haracter='%22'
+, fieldtypes=%7B%7D
 ):%0A    with 
@@ -4861,16 +4861,16 @@
 lues())%0A
-
       re
@@ -4901,16 +4901,67 @@
  'type':
+ field%5B0%5D in fieldtypes and fieldtypes%5Bfield%5B0%5D%5D or
  field%5B1

Add logs example