from celery import shared_task
from mrbelvedereci.build.models import Build
from mrbelvedereci.salesforce.models import Org

@shared_task
def run_build(build_id):
    build = Build.objects.get(id=build_id)
    build.run()
    return build.status

@shared_task
def check_queued_build(build_id):
    build = Build.objects.get(id = build_id)

    # Check for concurrency blocking
    try:
        org = Org.objects.get(name = build.trigger.org, repo = build.repo)
    except Org.DoesNotExist:
        return

    # If this is not a scratch org, ensure no builds are currently running against the org
    if not org.scratch:
        running_builds = Build.objects.filter(status='running', repo=build.repo, trigger__org = build.trigger.org).count()
        if running_builds:
            # Requeue this job to check again in 5 seconds
            check_queued_build.apply_async((build.id,), countdown=5)
            return 'Queued: checking again in 5 seconds'

    # Queue the background job with a 1 second delay to allow the transaction to commit
    run_build.apply_async((build.id,), countdown=1)

@@ -3,125 +3,877 @@
 om c
-elery import shared_task%0Afrom mrbelvedereci.build.models import Build%0Afrom mrbelvedereci.salesforce.models import Org
+ontextlib import contextmanager%0Afrom celery import shared_task%0Afrom celery.five import monotonic%0Afrom django.core.cache import cache%0Afrom mrbelvedereci.build.models import Build%0Afrom mrbelvedereci.salesforce.models import Org%0A%0ALOCK_EXPIRE = 60 * 120  # Lock expires in 2 hours%0A%0A@contextmanager%0Adef lock_org(lock_id, oid):%0A    timeout_at = monotonic() + LOCK_EXPIRE - 3%0A    # cache.add fails if the key already exists%0A    status = cache.add(lock_id, oid, LOCK_EXPIRE)%0A    try:%0A        yield status%0A    finally:%0A        # memcache delete is very slow, but we have to use it to take%0A        # advantage of using add() for atomic locking%0A        if monotonic() %3C timeout_at:%0A            # don't release the lock if we exceeded the timeout%0A            # to lessen the chance of releasing an expired lock%0A            # owned by someone else.%0A            cache.delete(lock_id)
 %0A%0A@s
@@ -1378,233 +1378,246 @@
     
-running_builds = Build.objects.filter(status='running', repo=build.repo, trigger__org = build.trigger.org).count()%0A        if running_builds:%0A            # Requeue this job to check again in 5 seconds%0A            check_queued
+lock_id = 'mrbelvedereci-org-lock-%7B1%7D'.format(org.id)%0A        logger.debug('Locking org: %25s', org)%0A        with memcache_lock(lock_id, self.app.oid) as acquired:%0A            if acquired:%0A                # Run the build%0A                run
 _bui
@@ -1654,17 +1654,17 @@
 untdown=
-5
+1
 )%0A      
@@ -1673,142 +1673,219 @@
     
-return 'Queued: checking again in 5 seconds'%0A%0A    # Queue the background job with a 1 second delay to allow the transaction to commit%0A
+else:%0A                # Queue next check in 5 seconds%0A                check_queued_build.apply_async((build.id,), countdown=5)%0A    # For scratch orgs, don't worry about locking, just run the build%0A    else:%0A    
     

use cache locks to lock non-scratch orgs