#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import os
import md5
import json
import pywb.warc.cdxindexer
import ipfsApi

from io import BytesIO
from pywb.warc.archiveiterator import ArchiveIterator, DefaultRecordParser
from pywb.utils.loaders import LimitReader
from pywb.warc.recordloader import ArcWarcRecordLoader
from pywb.utils.bufferedreaders import DecompressingBufferedReader
from pywb.utils.statusandheaders import StatusAndHeadersParser
from surt import surt

IP = '127.0.0.1'
PORT = '5001'

IPFS_API = ipfsApi.Client(IP, PORT)

# TODO: Check PEP-8 for indention guidance

def main():
    textRecordParserOptions = {'cdxj': True, 'include_all': False, 'surt_ordered': False}
    cdxLines = ''
    ipfsRetryCount = 5 # Attempts to push a WARC record to IPFS before giving up
    ipfsTempPath = '/tmp/ipfs/'

    # Read WARC file
    loader = ArcWarcRecordLoader(verify_http = True)
    warcFileFullPath = sys.argv[1]

    with open(warcFileFullPath, 'rb') as warc:
        iter = TextRecordParser(**textRecordParserOptions)

        for entry in iter(warc):
            # Only consider WARC response records from requests for web resources
            # TODO: Change conditional to return on non-HTTP responses to reduce branch depth
            if entry.record.rec_type != 'response' or entry.get('mime') in ('text/dns', 'text/whois'):
              continue

            hdrs = entry.record.status_headers
            hstr = hdrs.protocol + ' ' + hdrs.statusline
            for h in hdrs.headers:
                hstr += "\n" + ': '.join(h)

            statusCode = hdrs.statusline.split()[0]

            if not entry.buffer:
                return

            entry.buffer.seek(0)
            payload = entry.buffer.read()

            fileHash = md5.new(hstr).hexdigest()

            hdrfn = ipfsTempPath + 'header_' + fileHash
            pldfn = ipfsTempPath + 'payload_' + fileHash

            writeFile(hdrfn, hstr)
            writeFile(pldfn, payload)

            httpHeaderIPFSHash = ''
            payloadIPFSHash = ''
            retryCount = 0

            # TODO: First check that IPFS daemon is running, how do we do this?

            while retryCount < ipfsRetryCount:
                try:
                    httpHeaderIPFSHash = pushToIPFS(hdrfn)
                    payloadIPFSHash = pushToIPFS(pldfn)
                    break
                except:
                    logError('IPFS failed on ' + entry.get('url'))
                    retryCount += 1

            if retryCount >= ipfsRetryCount:
                logError('Skipping ' + entry.get('url'))

                continue

            uri = surt(entry.get('url'))
            timestamp = entry.get('timestamp')
            mime = entry.get('mime')

            encrKey = ''  # TODO

            obj = {
                'header_digest': httpHeaderIPFSHash,
                'payload_digest': payloadIPFSHash,
                'status_code': statusCode,
                'mime_type': mime,
                'encryption_key': encrKey,
                }
            objJSON = json.dumps(obj)

            cdxjLine = '{0} {1} {2}'.format(uri, timestamp, objJSON)
            cdxLines += cdxjLine
            print cdxjLine

            resHeader = pullFromIPFS(httpHeaderIPFSHash)
            resPayload = pullFromIPFS(payloadIPFSHash)

            warcContents = resHeader + "\n\n" + resPayload


def logError(errIn):
    print >> sys.stderr, errIn


def pullFromIPFS(hash):
    global IPFS_API
    return IPFS_API.cat(hash)


def pushToIPFS(path):
    global IPFS_API
    res = IPFS_API.add(path)
    # TODO: verify that the add was successful
    return res['Hash']


def writeFile(filename, content):
    with open(filename, 'w') as tmpFile:
        tmpFile.write(content)


class TextRecordParser(DefaultRecordParser):

    def create_payload_buffer(self, entry):
        return BytesIO()


if __name__ == '__main__':
    if len(sys.argv) < 2:
        logError("Usage:\n\n{0} </path/to/file.warc[.gz]>\n".format(sys.argv[0]))
        sys.exit(0)
    main()

@@ -475,16 +475,84 @@
 ort surt
+%0Afrom requests.packages.urllib3.exceptions import NewConnectionError
 %0A%0AIP = '
@@ -2440,16 +2440,201 @@
   break%0A
+                except NewConnectionError:%0A                    print 'IPFS daemon is likely not running.%5CnRun %22ipfs daemon%22 in another terminal session.'%0A                    sys.exit()%0A
         
@@ -2708,32 +2708,77 @@
 try.get('url'))%0A
+                    #print sys.exc_info()%5B0%5D%0A
                 

Handle more gracefully if the ipfs daemon is not running. Closes #13