# -*- coding: utf-8 -*-
#!/usr/bin/env python

# $ rsync -HaXxv mongo_backup.py rate.txt cmpitg@125.212.201.146:/home/cmpitg/Data/scripts/

import logging
import sys
import os
import time
import pymongo

from datetime import datetime as dt
from pymongo import MongoClient
from bson.objectid import ObjectId


log_formatter              = logging.Formatter('%(asctime)s : %(message)s')
log_progress_file_handler  = logging.FileHandler('mongo_backup_progress.log',
                                                mode='a+')
log_progress_file_handler.setFormatter(log_formatter)
log_stream_handler = logging.StreamHandler()
log_stream_handler.setFormatter(log_formatter)
LOGGER_PROGRESS = logging.getLogger('progress')
LOGGER_PROGRESS.setLevel(logging.INFO)
LOGGER_PROGRESS.addHandler(log_progress_file_handler)
LOGGER_PROGRESS.addHandler(log_stream_handler)

SOURCE         = MongoClient("mongodb://adsinapp:aDsinApP123@125.212.201.147/ads_in_app")
SOURCE_DB      = SOURCE.ads_in_app
DEST           = MongoClient("mongodb://adsinapp:aDsinApP123@125.212.201.146/ads_in_app")
DEST_DB        = DEST.ads_in_app
MAX_DOCUMENTS  = 500000

LAST_TIME        = dt.now()
TOTAL_DOCUMENTS  = 0


COLLECTIONS = ['adv_user',
               'app_category',
               'apps',
               'apps_permission',
               'banned_partner',
               'config',
               'config_sdk',
               'dev_sdk',
               'follow_revenue',
               'info_adflex_app',
               'info_domain',
               'log_campaign',
               'log_campaign_status',
               'log_fake_install',
               'log_install_daily',
               'log_money',
               'log_open_date',
               'log_pub_postback',
               'log_revenue_source',
               'log_traffic_total',
               'log_warning_rate',
               'log_click_info_2016',
               'pricing',
               'pub_manager',
               'publisher_postback',
               'rates',
               'users']

COLLECTIONS_DATE = ['log_traffic',
                    'log_traffic_country',
                    'log_traffic_hourly',
                    'log_country',
                    'log_install_info']

COLLECTIONS = COLLECTIONS_DATE


def create_file_if_not_exists(path):
    try:
        with open(path, 'r') as f:
            pass
    except Exception:
        with open(path, 'w') as f:
            pass


def print_collection_size(name):
    """Print collection size"""
    coll = SOURCE_DB[name]
    print("{}: {}".format(name, coll.count()))


def read_config(path=None):
    try:
        if not path:
            path = os.path.join(os.path.dirname(__file__), "rate.txt")

        with open(path, 'r') as inp:
            lines = inp.readlines()

        if len(lines) > 1 and str.startswith(lines[1], 'stop'):
            LOGGER_PROGRESS.warn('stopped by user')
            sys.exit(1)
            
        else:
            return {
                'rate': int(lines[0]),
                'stop': False
            }
    except Exception as e:
        print(str(e))
        return {
            'rate': 1000,
            'stop': False
        }


def report_collections_size():
    LOGGER_PROGRESS.info("collections with MongoID")
    map(print_collection_size, COLLECTIONS)

    # LOGGER_PROGRESS.info("collections without MongoID")
    # map(print_collection_size, COLLECTIONS_DATE)


def milisecs_passed():
    delta = int((dt.now() - LAST_TIME).total_seconds() * 1000)
    return delta


def sleep_if_necessary():
    global LAST_TIME

    delta = milisecs_passed()
    if delta < 1000:
        time.sleep((1000 - delta) / 1000)
        LAST_TIME = dt.now()


def build_query(path):
    if os.path.exists(path):
        with open(path, 'r') as f:
            res = f.read()
        return { "_id": { "$gt": ObjectId(res) }}
    else:
        return {}


def backup_collection(name):
    global LAST_TIME

    LOGGER_PROGRESS.info("processing collection %s", name)
    LOGGER_PROGRESS.info("source: %s; destination: %s",
                         SOURCE_DB[name].count(),
                         DEST_DB[name].count())
    LAST_TIME = dt.now()

    source_collection = SOURCE_DB[name]
    dest_collection   = DEST_DB[name]
    current_docs      = []
    config            = read_config()
    # docs              = source_collection.find().sort([('_id', pymongo.DESCENDING )])
    docs              = source_collection.find().sort([('date', pymongo.DESCENDING )]).limit(MAX_DOCUMENTS)

    def insert_to_dest():
        LOGGER_PROGRESS.info('rate: %s → batch inserting %s documents into %s',
                             config['rate'],
                             len(current_docs),
                             name)
        dest_collection.insert_many(current_docs)
        sleep_if_necessary()

    def doc_exists(doc, coll):
        return len(list(coll.find({ u'_id': doc[u'_id'] }))) != 0

    for doc in docs:
        if doc_exists(doc=doc, coll=dest_collection):
            continue
            # break

        current_docs.append(doc)

        if len(current_docs) >= config['rate']:
            insert_to_dest()
            config = read_config()
            current_docs = []

    if len(current_docs) != 0:
        insert_to_dest()
        config = read_config()
        current_docs = []


def init():
    milisecs_passed()


def backup():
    init()
    for name in COLLECTIONS:
        backup_collection(name)


def main():
    # report_collections_size()
    backup()

    SOURCE.close()
    DEST.close()


if __name__ == '__main__':
    main()

@@ -1,142 +1,26 @@
 #
- -*- coding: utf-8 -*-%0A#!/usr/bin/env python%0A%0A# $ rsync -HaXxv mongo_backup.py rate.txt cmpitg@125.212.201.146:/home/cmpitg/Data/scripts/
+!/usr/bin/env python3
 %0A%0Aim
@@ -1062,1084 +1062,8 @@
 0%0A%0A%0A
-COLLECTIONS = %5B'adv_user',%0A               'app_category',%0A               'apps',%0A               'apps_permission',%0A               'banned_partner',%0A               'config',%0A               'config_sdk',%0A               'dev_sdk',%0A               'follow_revenue',%0A               'info_adflex_app',%0A               'info_domain',%0A               'log_campaign',%0A               'log_campaign_status',%0A               'log_fake_install',%0A               'log_install_daily',%0A               'log_money',%0A               'log_open_date',%0A               'log_pub_postback',%0A               'log_revenue_source',%0A               'log_traffic_total',%0A               'log_warning_rate',%0A               'log_click_info_2016',%0A               'pricing',%0A               'pub_manager',%0A               'publisher_postback',%0A               'rates',%0A               'users'%5D%0A%0ACOLLECTIONS_DATE = %5B'log_traffic',%0A                    'log_traffic_country',%0A                    'log_traffic_hourly',%0A                    'log_country',%0A                    'log_install_info'%5D%0A%0ACOLLECTIONS = COLLECTIONS_DATE%0A%0A
 %0Adef

remove redundant code