<jupyter_start><jupyter_text>Canada's Monetary Policy Report: if text could speak, what would it say? This notebook adds to the appropriate folder and cleans the most recent MPR. Before running this code - Download the latest MPR (PDF) from the BoC website - transform it into a text file - remove charts, tables, footnotes, appendices, and any non-text elements at the beginning or the end of the MPR. This workbook should "just work". If it does not, or if you find an error, please email the authors. <NAME>  <NAME> ---------------------------------------------------------------------------------------------------------------------------  Set up the environment  Import required modules<jupyter_code>import spacy
nlp = spacy.load("en")
import re
from pathlib import Path<jupyter_output><empty_output><jupyter_text>Define helper functions<jupyter_code># Function to clean MPRs

def clean_mpr_text(org_text):
    text = org_text
    
    # convert some common Unicode symbols to the corresponding ASCII
    # https://en.wikipedia.org/wiki/Quotation_mark
    text = re.sub("\u2018|\u2019|\u201A|\u201B|\u00B4|\u2039|\u203A", "\'", text)
    text = re.sub("\u201C|\u201D|\u201E|\u201F|\u00AB|\u00BB|\u2E42", "\"", text)
    # https://en.wikipedia.org/wiki/Dash
    text = re.sub("\u2010|\u2011|\u2012|\u2013|\u2014\u2015", "-", text)
    # https://en.wikipedia.org/wiki/Bullet_(typography)
    text = re.sub("\u2022|\u2023|\u2043|\u204C|\u204D", "*", text)
    
    text = re.sub(r"\n\s*\n", "\n\n", text)
    text = re.sub(r"([^\n])\n([^\n])", "\\1 \\2", text)
    text = re.sub(r"\n\n", "\n \n", text)
    text = re.sub(r"\+/\-", "±", text)
    text = re.sub(r"\s>\s", " - ", text)
    text = re.sub(r"U\.S\.\s*(Federal\s*Reserve|CARS|Residential|Real\s*GDP|Treasuries|Treasury|Midwest|GDP)", "US \\1", text)
    text = re.sub(r"U\.S\.(\s*[A-Z])?", lambda m: "US" + (("."+m.group(1)) if m.group(1) else ""), text)
    text = "\n".join( map( lambda par: re.sub(r"\s", " ", par), text.splitlines()) )
    # remove URL
    text = re.sub(r"\s*(\()?(<)?http[^>\)]*?(?(2)>)(?(1)\))\s*(\.)?", 
                  lambda m: "." if m.group(3) else " ", text)
    
    text = re.sub(r"\s*\((?:\w*|available on the Bank's website.*?|(?:See|Chart|Table|Box|Technical).*?)\)\s*(\.)?", 
                  lambda m: "." if m.group(1) else " ", text)
    
    return text    


# Quick check for the sentences (very short or very long sentences should be look at)

def check_mpr_text(text):
    found = False
    sents = []
    text = re.sub(r"\s+", " ", text)
    text = re.sub(r"\. ", ".  ", text)
    for s in nlp(text).sents:
        sents.append(s)
        if found:
            print(fname, ":", s, '\n')
            found = False
        nwords = sum(t.is_alpha for t in s)     
        if (nwords < 3) or any(tok.text in (".", "?", "!") for tok in s[:-2]) or (nwords > 100):
            found = True
            print(fname, ":", s)


# Save the resulting MPR

def save_mpr(text, year, month, day):
    fname = "mpr_text/mpr-%4d-%02d-%02d.txt" % (year, month, day)    
    with open(fname, "wt") as ff:
        ff.write(text)

        
# Date format to be enter

def date_from_mpr_fname(fname):
    match = re.search(r"(\d{4})-(\d{2})-(\d{2})", str(fname))
    if not match:
        raise ValueError("No date found in the given filename.")
    return [ int(g) for g in match.groups()]<jupyter_output><empty_output><jupyter_text>Run the function -- need to enter the path of the latest MPR text files<jupyter_code># Clean, check and save the MPR for a specific date 

################################################
new_mpr_files = ["MPR-2019-01-09.txt", 
                 ]
################################################

assert isinstance(new_mpr_files, list), "It must be a list, not a string!"
for mpr_filename in new_mpr_files:
    try:
        year, month, day = date_from_mpr_fname(mpr_filename)
    except ValueError:
        print("Enter the publication date for", mpr_filename)
        year = int(input("year:"))
        month = int(input("month:"))
        day = int(input("day:"))
    except:
        raise

    print("%4d-%02d-%02d" % (year, month, day), mpr_filename)
        
    with open(mpr_filename, "rt") as ff:
        text = clean_mpr_text(ff.read())
        check_mpr_text(text)
        save_mpr(text, year, month, day)<jupyter_output>2019-01-09 MPR-2019-01-09.txt
StarcoderdataJupyterStructuredCleanDedup