from __future__ import print_function
from collections import defaultdict
import glob
import json
import os
import re
import subprocess
from urllib2 import urlopen

DEBUG = False
GIT_EXEC = "/usr/bin/git"
REPOSITORIES = glob.glob("/ssd/swinbank/src/*")  # Everything in w_2017_8
JIRA_API_URL = "https://jira.lsstcorp.org/rest/api/2"

class Repository(object):
    def __init__(self, path):
        self.path = path

    def __call_git(self, *args):
        to_exec = [GIT_EXEC] + list(args)
        if DEBUG:
            print(to_exec)
        return subprocess.check_output(to_exec, cwd=self.path)

    def commits(self, reachable_from=None, merges_only=False):
        args = ["log", "--pretty=format:%H"]
        if reachable_from:
            args.append(reachable_from)
        if merges_only:
            args.append("--merges")
        return self.__call_git(*args).split()

    def message(self, commit_hash):
        return self.__call_git("show", commit_hash, "--pretty=format:%s")

    def tags(self, pattern=r".*"):
        return [tag for tag in self.__call_git("tag").split()
                if re.search(pattern, tag)]

    def update(self):
        return self.__call_git("pull")

    @staticmethod
    def ticket(message):
        try:
            return re.search(r"(DM-\d+)", message, re.IGNORECASE).group(1)
        except AttributeError:
            if DEBUG:
                print(message)


def get_ticket_summary(ticket):
    url = JIRA_API_URL + "/issue/" + ticket + "?fields=summary"
    if DEBUG:
        print(url)
    j = json.load(urlopen(url))
    return j['fields']['summary']

def format_output(changelog):
    # Ew, needs a proper templating engine
    print("<html>")
    print("<body>")
    print("<h1>LSST DM Weekly Changelog</h1>")

    for tag in sorted(changelog, reverse=True):
        print("<h2>New in {}</h2>".format(tag))
        print("<ul>")
        for ticket in sorted(changelog[tag]):
            summary = get_ticket_summary(ticket)
            pkgs = ", ".join(sorted(changelog[tag][ticket]))
            link_text = (u"<li><a href=https://jira.lsstcorp.org/browse/"
                         u"{ticket}>{ticket}</a>: {summary} [{pkgs}]</li>")
            print(link_text.format(ticket=ticket, summary=summary, pkgs=pkgs)
                           .encode("utf-8"))
        print("</ul>")
    print("</body>")
    print("</html>")

def generate_changelog(repositories):
    # Dict of tag -> ticket -> affected packages
    changelog =  defaultdict(lambda: defaultdict(set))
    for repository in repositories:
        if DEBUG:
            print(repository)
        r = Repository(repository)
        r.update()
        tags = sorted(r.tags("w\.\d{4}"), reverse=True)
        for newtag, oldtag in zip(tags, tags[1:]):
            merges = (set(r.commits(newtag, merges_only=True)) -
                      set(r.commits(oldtag, merges_only=True)))

            for sha in merges:
                ticket = r.ticket(r.message(sha))
                if ticket:
                    changelog[newtag][ticket].add(os.path.basename(repository))
    return changelog


if __name__ == "__main__":
    changelog = generate_changelog(REPOSITORIES)
    format_output(changelog)

@@ -1611,217 +1611,37 @@
 def 
-format_output(changelog):%0A    # Ew, needs a proper templating engine%0A    print(%22%3Chtml%3E%22)%0A    print(%22%3Cbody%3E%22)%0A    print(%22%3Ch1%3ELSST DM Weekly Changelog%3C/h1%3E%22)%0A%0A    for tag in sorted(changelog, reverse=True):%0A    
+print_tag(tagname, tickets):%0A
     
@@ -1677,23 +1677,23 @@
 rmat(tag
-))%0A    
+name))%0A
     prin
@@ -1702,28 +1702,24 @@
 %22%3Cul%3E%22)%0A    
-  
-  
 for ticket i
@@ -1731,29 +1731,18 @@
 ted(
-changelog%5Btag%5D):%0A    
+tickets):%0A
     
@@ -1790,20 +1790,16 @@
         
-    
 pkgs = %22
@@ -1818,22 +1818,15 @@
 ted(
-changelog%5Btag%5D
+tickets
 %5Btic
@@ -1832,20 +1832,16 @@
 cket%5D))%0A
-    
         
@@ -1923,20 +1923,16 @@
         
-    
 u%22%7Bticke
@@ -1978,28 +1978,24 @@
 %3E%22)%0A        
- 
-   
 print(link_t
@@ -2071,20 +2071,16 @@
         
-    
 .encode(
@@ -2089,28 +2089,24 @@
 tf-8%22))%0A    
-    
 print(%22%3C/ul%3E
@@ -2103,24 +2103,388 @@
 int(%22%3C/ul%3E%22)
+%0A%0Adef format_output(changelog):%0A    # Ew, needs a proper templating engine%0A    print(%22%3Chtml%3E%22)%0A    print(%22%3Cbody%3E%22)%0A    print(%22%3Ch1%3ELSST DM Weekly Changelog%3C/h1%3E%22)%0A%0A    # Always do master first%0A    print_tag(%22master%22, changelog.pop(%22master%22))%0A%0A    # Then the other tags in order%0A    for tag in sorted(changelog, reverse=True):%0A        print_tag(tag, changelog%5Btag%5D)%0A
 %0A    print(%22
@@ -2794,16 +2794,69 @@
 update()
+%0A%0A        # Extract all tags which look like weeklies
 %0A       
@@ -2904,16 +2904,110 @@
 e=True)%0A
+        # Also include tickets which aren't yet in a weekly%0A        tags.insert(0, %22master%22)%0A%0A
         

Print master first.