@@ -0,0 +1,1743 @@
+#!/usr/bin/env python%0A%0Aimport logging%0Aimport re%0Aimport subprocess%0Aimport time%0Aimport urllib2%0A%0ADIST_TEST_URL = %22http://dist-test.cloudera.org%22%0A# GCE bills 10-minute minimum, so if we've started instances%0A# more recently than 10 minutes ago, we shouldn't shut them down.%0ASHRINK_LAG = 600%0A%0Adef get_stats():%0A  page = urllib2.urlopen(DIST_TEST_URL).read()%0A  m = re.search(%22Queue length: (%5Cd+).*Running: (%5Cd+)%22, page, re.DOTALL)%0A  if not m:%0A    raise Exception(%22Bad page content%22)%0A  return dict(queue_len=int(m.group(1)), running=int(m.group(2)))%0A%0Adef resize(num_nodes):%0A  logging.info(%22Setting num nodes to %25d%22 %25 num_nodes)%0A  subprocess.check_call(%0A     (%22gcloud compute instance-groups managed resize %22 +%0A     %22dist-test-slave-group --size=%25d%22 %25 num_nodes).split(%22 %22),%0A     stdout=file(%22/dev/null%22, %22w%22))%0A%0Adef get_target_size():%0A  output = subprocess.check_output(%0A    'gcloud compute instance-groups managed describe dist-test-slave-group'.split(%22 %22))%0A  m = re.search('targetSize: (%5Cd+)', output)%0A  return int(m.group(1))%0A    %0A%0Adef main():%0A  logging.basicConfig(level=logging.INFO)%0A  last_size = get_target_size()%0A  logging.info(%22initial size: %25d%22 %25 last_size)%0A  last_grow_time = 0%0A  while True:%0A    try:%0A      stats = get_stats()%0A      logging.info(stats)%0A      new_size = last_size%0A      if stats%5B'queue_len'%5D %3E 0:%0A        new_size = min(100, last_size + 10)%0A        last_grow_time = time.time()%0A      elif stats%5B'queue_len'%5D + stats%5B'running'%5D == 0 and %5C%0A           time.time() - last_grow_time %3E SHRINK_LAG:%0A        new_size = 1%0A      if new_size != last_size:%0A        resize(new_size)%0A        last_size = new_size%0A    except Exception, e:%0A      logging.warning(%22had error%22 + repr(e))%0A    time.sleep(10)%0A%0A%0Aif __name__ == %22__main__%22:%0A  main()%0A

Add autoscaler script