# -*- coding: utf-8 -*-
__author__ = "Jake Schurch"

import SecCrawler as sc
import re
from bs4 import BeautifulSoup


def GetItemsFromSGML(SgmlHead):
    items = re.findall(r'(?:\<ITEMS\>)(\d{1}\.\d{2})', SgmlHead)
    return items


def CleanHtmlMarkup(markup):
    markup = markup.replace('</font>', " </font>")
    markup = markup.replace("</div>", "\n</div>")
    markup = markup.replace("Table of Contents", "")

    return markup


def CleanTextMarkup(text):
    text = text.replace('\u200b', "")
    text = text.replace('• \n', '•')
    text = re.sub(r'[^\x00-\x7f]', r' ', text)
    text = re.sub(r'^\d+ $', r'', text, flags=re.MULTILINE)
    text = re.sub(r'\n\s+\n\s+\n\s+', '\n\n', text)

    return text


def GetCleanTags(HTML_Entity):
    FoundTags = HTML_Entity.find_all(True, recursive=False)
    for _tag in FoundTags:
        if _tag.name == 'table':
            for descendant in _tag.descendants:
                try:
                    if ('style' in descendant.attrs and
                            'background-color' in descendant.attrs['style']):
                        _tag.clear()
                        _tag.decompose()
                        break
                except AttributeError:
                    pass

        elif (_tag.attrs is not None and
              'style' in _tag.attrs and
              'text-align:center' in _tag.attrs['style']):

            if _tag.string == '' or _tag.string is None:
                pass
            else:
                _tag.clear()
                _tag.decompose()

        elif _tag.name == 'hr':
            _tag.clear()
            _tag.decompose()

        else:
            pass

    return FoundTags


def FilingTextParser(filing):
    _content = BeautifulSoup(filing.FilingText, "lxml").find(['document',
                                                             'text'])
    _CleanTags = GetCleanTags(_content)

    _CleanHtml = CleanHtmlMarkup(
        ''.join(str(_tag) for _tag in _CleanTags))

    CleanText = CleanTextMarkup(
        BeautifulSoup(_CleanHtml, 'lxml').text)

    with open('parseFilingTesting.txt', 'w+') as f:
        f.write(CleanText)

    return CleanText


if __name__ == "__main__":
    testListing = [sc.Filing("amzn", "10-k", totalFilingsWanted=1)]
    sc.SetInterimListing(testListing)

    FilingTextParser(sc.G_filingListing[0])

@@ -149,20 +149,33 @@
 SgmlHead
+: sc.SgmlHead
 ):%0D%0A
-
     item
@@ -280,16 +280,40 @@
 p(markup
+: BeautifulSoup.find_all
 ):%0D%0A    
@@ -514,16 +514,21 @@
 kup(text
+: str
 ):%0D%0A    
@@ -781,24 +781,24 @@
 n text%0D%0A%0D%0A%0D%0A
-
 def GetClean
@@ -813,16 +813,36 @@
 L_Entity
+: BeautifulSoup.find
 ):%0D%0A    
@@ -1835,16 +1835,27 @@
 r(filing
+: sc.Filing
 ):%0D%0A    

Add type annotation to Jake_Testing