#!/bin/python
#Copyright (c) 2017, John Skinner
import logging
import logging.config
import traceback
import config.global_configuration as global_conf
import database.client
import util.database_helpers as dh
import batch_analysis.task_manager
import batch_analysis.job_systems.job_system_factory as job_system_factory


def main():
    """
    Schedule tasks for all experiments.
    We need to find a way of running this repeatedly as a daemon
    """
    config = global_conf.load_global_config('config.yml')
    if __name__ == '__main__':
        logging.config.dictConfig(config['logging'])
    db_client = database.client.DatabaseClient(config=config)
    task_manager = batch_analysis.task_manager.TaskManager(db_client.tasks_collection, db_client, config)

    logging.getLogger(__name__).info("Scheduling experiments...")
    experiment_ids = db_client.experiments_collection.find({'enabled': {'$ne': False}}, {'_id': True})
    for experiment_id in experiment_ids:
        experiment = dh.load_object(db_client, db_client.experiments_collection, experiment_id['_id'])
        if experiment is not None and experiment.enabled:
            # TODO: Separate scripts for scheduling and importing, they run at different rates
            try:
                experiment.do_imports(task_manager, db_client)
                experiment.schedule_tasks(task_manager, db_client)
                experiment.save_updates(db_client)
            except Exception:
                logging.getLogger(__name__).error(
                    "Exception occurred during scheduling:\n{0}".format(traceback.format_exc()))

    logging.getLogger(__name__).info("Scheduling tasks...")
    job_system = job_system_factory.create_job_system(config=config)
    task_manager.schedule_tasks(job_system)

    # Actually run the queued jobs.
    job_system.run_queued_jobs()

if __name__ == '__main__':
    main()

@@ -8,16 +8,17 @@
 python%0A#
+ 
 Copyrigh
@@ -79,16 +79,46 @@
 .config%0A
+import argparse%0Aimport typing%0A
 import t
@@ -126,16 +126,28 @@
 aceback%0A
+import bson%0A
 import c
@@ -367,16 +367,109 @@
 ef main(
+do_imports: bool = True, schedule_tasks: bool = True, experiment_ids: typing.List%5Bstr%5D = None
 ):%0A    %22
@@ -576,16 +576,311 @@
  daemon%0A
+    :param do_imports: Whether to do imports for all the experiments. Default true.%0A    :param schedule_tasks: Whether to schedule execution tasks for the experiments. Default true.%0A    :param experiment_ids: A limited set of experiments to schedule for. Default None, which is all experiments.%0A
     %22%22%22%0A
@@ -1198,52 +1198,168 @@
     
-logging.getLogger(__name__).info(%22Schedul
+query = %7B'enabled': %7B'$ne': False%7D%7D%0A    if experiment_ids is not None and len(experiment_ids) %3E 0:%0A        query%5B'_id'%5D = %7B'$in': %5Bbson.ObjectId(id_) for id_ 
 in
-g
  exp
@@ -1365,22 +1365,22 @@
 periment
-s...%22)
+_ids%5D%7D
 %0A    exp
@@ -1435,50 +1435,95 @@
 ind(
-%7B'enabled': %7B'$ne': False%7D%7D, %7B'_id': True%7D
+query, %7B'_id': True%7D)%0A%0A    logging.getLogger(__name__).info(%22Scheduling experiments...%22
 )%0A  
@@ -1738,108 +1738,146 @@
     
-# TODO: Separate scripts for scheduling and importing, they run at different rates%0A            try:%0A
+logging.getLogger(__name__).info(%22 ... experiment %7B0%7D%22.format(experiment.identifier))%0A            try:%0A                if do_imports:%0A    
     
@@ -1927,32 +1927,71 @@
 ger, db_client)%0A
+                if schedule_tasks:%0A    
                 
@@ -2271,16 +2271,43 @@
 xc()))%0A%0A
+    if schedule_tasks:%0A    
     logg
@@ -2358,24 +2358,28 @@
 ks...%22)%0A    
+   
+ 
 job_system =
@@ -2423,32 +2423,36 @@
 (config=config)%0A
+    
     task_manager
@@ -2480,16 +2480,20 @@
 ystem)%0A%0A
+    
     # Ac
@@ -2520,24 +2520,28 @@
 d jobs.%0A    
+ 
+   
 job_system.r
@@ -2558,16 +2558,17 @@
 jobs()%0A%0A
+%0A
 if __nam
@@ -2590,15 +2590,851 @@
 _':%0A    
-main(
+parser = argparse.ArgumentParser(%0A        description='Update and schedule tasks from experiments.'%0A                    'By default, this will update and schedule tasks for all experiments.')%0A    parser.add_argument('--skip_imports', action='store_true',%0A                        help='Skip import data for the experiments.')%0A    parser.add_argument('--skip_schedule_tasks', action='store_true',%0A                        help='Don%5C't schedule the execution or evaluation of systems')%0A    parser.add_argument('experiment_ids', metavar='experiment_id', nargs='*', default=%5B%5D,%0A                        help='Limit the update to only the specified experiment by id. '%0A                             'You may specify any number of ids.')%0A%0A    args = parser.parse_args()%0A    main(not args.skip_imports, not args.skip_schedule_tasks, args.experiment_ids
 )%0A

Add command line interface to scheduler. This allows us to only do imports, or only schedule tasks.