#!/usr/bin/env python
# -*- coding: utf-8 -*-

import sys
import os
import md5
import json
import pywb.warc.cdxindexer
import ipfsApi

from io import BytesIO
from pywb.warc.archiveiterator import ArchiveIterator, DefaultRecordParser
from pywb.utils.loaders import LimitReader
from pywb.warc.recordloader import ArcWarcRecordLoader
from pywb.utils.bufferedreaders import DecompressingBufferedReader
from pywb.utils.statusandheaders import StatusAndHeadersParser
from surt import surt

IP = '127.0.0.1'
PORT = '5001'

IPFS_API = ipfsApi.Client(IP, PORT)

# TODO: Check PEP-8 for indention guidance

def main():
    textRecordParserOptions = {'cdxj': True, 'include_all': False, 'surt_ordered': False}
    cdxLines = ''
    ipfsRetryCount = 5 # Attempts to push a WARC record to IPFS before giving up
    ipfsTempPath = '/tmp/ipfs/'

    # Read WARC file
    loader = ArcWarcRecordLoader(verify_http = True)
    warcFileFullPath = sys.argv[1]

    with open(warcFileFullPath, 'rb') as warc:
        iter = TextRecordParser(**textRecordParserOptions)

        for entry in iter(warc):
            # Only consider WARC response records from requests for web resources
            # TODO: Change conditional to return on non-HTTP responses to reduce branch depth
            if entry.record.rec_type != 'response' or entry.get('mime') in ('text/dns', 'text/whois'):
              continue

            hdrs = entry.record.status_headers
            hstr = hdrs.protocol + ' ' + hdrs.statusline
            for h in hdrs.headers:
                hstr += "\n" + ': '.join(h)

            statusCode = hdrs.statusline.split()[0]

            if not entry.buffer:
                return

            entry.buffer.seek(0)
            payload = entry.buffer.read()

            fileHash = md5.new(hstr).hexdigest()

            hdrfn = ipfsTempPath + 'header_' + fileHash
            pldfn = ipfsTempPath + 'payload_' + fileHash

            writeFile(hdrfn, hstr)
            writeFile(pldfn, payload)

            httpHeaderIPFSHash = ''
            payloadIPFSHash = ''
            retryCount = 0
            
            while retryCount < ipfsRetryCount:
                try:
                    httpHeaderIPFSHash = pushToIPFS(hdrfn)
                    payloadIPFSHash = pushToIPFS(pldfn)
                    break
                except:
                    logError('IPFS failed on ' + entry.get('url'))
                    retryCount += 1

            if retryCount >= ipfsRetryCount:
                logError('Skipping ' + entry.get('url'))

                continue

            uri = surt(entry.get('url'))
            timestamp = entry.get('timestamp')
            mime = entry.get('mime')

            encrKey = ''  # TODO

            obj = {
                'header_digest': httpHeaderIPFSHash,
                'payload_digest': payloadIPFSHash,
                'status_code': statusCode,
                'mime_type': mime,
                'encryption_key': encrKey,
                }
            objJSON = json.dumps(obj)

            cdxjLine = '{0} {1} {2}'.format(uri, timestamp, objJSON)
            cdxLines += cdxjLine
            print cdxjLine

            resHeader = pullFromIPFS(httpHeaderIPFSHash)
            resPayload = pullFromIPFS(payloadIPFSHash)

            warcContents = resHeader + "\n\n" + resPayload


def logError(errIn):
    print >> sys.stderr, errIn


def pullFromIPFS(hash):
    global IPFS_API
    return IPFS_API.cat(hash)


def pushToIPFS(path):
    global IPFS_API
    res = IPFS_API.add(path)
    return res['Hash']


def writeFile(filename, content):
    with open(filename, 'w') as tmpFile:
        tmpFile.write(content)


class TextRecordParser(DefaultRecordParser):

    def create_payload_buffer(self, entry):
        return BytesIO()


if __name__ == '__main__':
    if len(sys.argv) < 2:
        logError("Usage:\n\n{0} </path/to/file.warc[.gz]>\n".format(sys.argv[0]))
        sys.exit(0)
    main()

@@ -2081,16 +2081,17 @@
 unt = 0%0A
+%0A
         
@@ -2094,16 +2094,84 @@
         
+# TODO: First check that IPFS daemon is running, how do we do this?%0A
 %0A       
@@ -3596,16 +3596,63 @@
 d(path)%0A
+    # TODO: verify that the add was successful%0A
     retu

Add TODOs to make the code slightly more robust.