<jupyter_start><jupyter_text>Carregar/Atualizar Parâmetros Após preencher os dados de configuração e executar a célula, os dados das abas `subsituicao_simples` e `substituicao_regex` da planilha ficarão armazenados nas variáveis `df_simples` e `df_regex` no formato de `DataFrames` do `pandas`.<jupyter_code>#@title Configuração do Projeto
PROJECT_PATH = "/LIAA-3R/PROJETOS/ia-dispositivos-legais" #@param {type:"string"}
PARAMS_PATH = "/params" #@param {type:"string"}
SPREADSHEET_NAME = "cmlima_substituicoes.xlsx" #@param {type:"string"}
USER_NAME = "cmlima" #@param {type:"string"}

from google.colab import drive
import pandas as pd

ROOT = '/content/drive/My\ Drive'
WORKING_PATH = ROOT + PROJECT_PATH + PARAMS_PATH

def path_exists(path):
  output = !test -e {path} && echo 1 || echo 0
  return output[0] == '1'

def file_exists(path):
  output = !(ls {path} >> /dev/null 2>&1 && echo 1) || echo 0
  return output[0] == '1'

def mount_drive():
  if not path_exists(ROOT):
    print('integrando Google Drive ao ambiente...')
    drive.mount('/content/drive')
    print()

def read_excel(file, sheet_name):
  if not file_exists(file):
    raise Exception('Planilha não localizada.')
  return pd.read_excel(file, sheet_name=sheet_name)

mount_drive()
%cd {WORKING_PATH}
df_replace = read_excel(SPREADSHEET_NAME, 'replace')
df_replace = df_replace.fillna('')
df_replace = df_replace.replace(to_replace=r'^\"|\"$', value='', regex=True)
df_replace = df_replace.replace({'is_literal': 1.0}, True)
df_replace = df_replace.replace({'is_literal': ''}, False)
df_stop_words = read_excel(SPREADSHEET_NAME, 'stop_words')
df_stop_words = df_stop_words.fillna('')
df_stop_words = df_stop_words.replace(to_replace=r'^\"|\"$', value='', regex=True)
print('Script concluído.')<jupyter_output>/content/drive/My Drive/LIAA-3R/PROJETOS/ia-dispositivos-legais/params
Script concluído.<jupyter_text>Exibir DataFrames em Formato HTML<jupyter_code>#@title Para exibir, execute a célula
from IPython.display import display, HTML

def render_as_table(name, pandas_data_frame):
  print('\n' + name + ':')
  display(HTML(pandas_data_frame.to_html()))

render_as_table('replace', df_replace)
render_as_table('stop_words', df_stop_words)<jupyter_output><empty_output><jupyter_text>Exibir DataFrames em Formato JSON<jupyter_code>#@title Para exibir, execute a célula
import json

def render_as_json(name, pandas_data_frame):
  print('\n' + name + ':')
  parsed = json.loads(pandas_data_frame.to_json(orient='records'))
  print(json.dumps(parsed, ensure_ascii=False, indent=4))

render_as_json('replace', df_replace)
render_as_json('stop_words', df_stop_words)<jupyter_output><empty_output><jupyter_text>Salvar DataFrames em Formato JSON<jupyter_code>#@title Para salvar, execute a célula
def save_as_json(file_name, pandas_data_frame):
  with open(file_name, 'w') as file:
    file.write(pandas_data_frame.to_json(force_ascii=False, orient='records'))

%cd {WORKING_PATH}
save_as_json(USER_NAME + '_replace.json', df_replace)
save_as_json(USER_NAME + '_stop_words.json', df_stop_words)
print('Script concluído.')<jupyter_output>/content/drive/My Drive/LIIA-3R/PROJETOS/ia-dispositivos-legais/params
Script concluído.<jupyter_text>Processar Substituições<jupyter_code>#@title Configuração { vertical-output: false }
ALGORITHM_TYPE = "all" #@param ["all", "literal_only", "regex_only", "all_literal_first", "all_regex_first"]
INPUT_PATH = "/input" #@param {type:"string"}
INPUT_FILE = "" #@param {type:"string"}
OUTPUT_PATH = "/output" #@param {type:"string"}
OUTPUT_TO_FILE = True #@param {type:"boolean"}

!pip install PyPDF2
!pip install python-docx

import json, re, os, PyPDF2, docx
from google.colab import files
from IPython.display import clear_output

clear_output()

def stop_words_algorithm(text_input, params):
  for item in params:
    text_input = text_input.replace(item['search_expression'], item['replace_by'])
  return text_input

def replace_algorithm(text_input, params, algorithm_type):
  for item in params:
    if item['is_literal'] and ('all' in algorithm_type or 'literal' in algorithm_type):
      text_input = text_input.replace(item['search_expression'], item['replace_by'])
    if not item['is_literal'] and ('all' in algorithm_type or 'regex' in algorithm_type):
      text_input = re.sub(item['search_expression'], item['replace_by'], text_input, count=0, flags=0)
  return text_input

def split_paragraphs(text_input):
  return [ para + '.' for para in re.split('[.!?](?![0-9])', text_input) ]

def paragraphs_to_text(paragraphs):
  return '\n--------------------------------------------------------------------------------------------\n'.join(paragraphs)

def print_in_shell(paragraphs):
  print(paragraphs_to_text(paragraphs))

def print_to_file(file_name, paragraphs):
  with open(file_name, 'w') as file:
    file.write(paragraphs_to_text(paragraphs))

def get_text_from_pdf(file_name):
  pdfFileObj = open(file_name, 'rb')
  pdfReader = PyPDF2.PdfFileReader(pdfFileObj)
  content = []
  for num in range(pdfReader.numPages):
    page = pdfReader.getPage(num)
    try:
      text = page.extractText()
    except:
      text = ''
    content.append(text)
  return ''.join(content)

def get_text_from_docx(file_name):
  doc = docx.Document(file_name)
  content = []
  for para in doc.paragraphs:
    content.append(para.text)
  return '\n'.join(content)

def get_text_from_txt(file_name):
  with open(file_name, 'r') as file:
    content = file.read()
  return content

def get_text_from_file(file_name):
  if re.match('^.+\.(pdf)$', file_name):
    return get_text_from_pdf(file_name)
  if re.match('^.+\.(docx)$', file_name):
    return get_text_from_docx(file_name)
  if re.match('^.+\.(txt)$', file_name):
    return get_text_from_txt(file_name)
  raise Exception('Arquivo inválido.')

def get_text_from_files():
  contents = []
  for file_name in os.listdir():
    if re.match('^.+\.(txt|docx|pdf)$', file_name):
      info = {
          'name': file_name,
          'content': get_text_from_file(file_name)
      }
      contents.append(info)
  return contents

def file_name_without_extension(file_name):
  return re.match('(.+?)(\.[^.]*$|$)', file_name).group(1)

def process_text(raw_text, file_name, algorithm_type):
  stop_words_params = json.loads(df_stop_words.to_json(force_ascii=False, orient='records'))
  replace_params = json.loads(df_replace.to_json(force_ascii=False, orient='records'))
  if algorithm_type == 'all_literal_first':
    replace_params = sorted(replace_params, key=lambda item: item['is_literal'], reverse=True)
  if algorithm_type == 'all_regex_first':
    replace_params = sorted(replace_params, key=lambda item: item['is_literal'], reverse=False)
  print(json.dumps(replace_params, ensure_ascii=False, indent=4))

  new_name = file_name_without_extension(file_name) + '.txt'
  text = stop_words_algorithm(raw_text, stop_words_params)
  text = replace_algorithm(text, replace_params, algorithm_type)

  paragraphs = split_paragraphs(text)

  if OUTPUT_TO_FILE:
    print_to_file(new_name, paragraphs)
  else:
    print_in_shell(paragraphs)
  

# # #


DATA_PATH = ROOT + PROJECT_PATH + INPUT_PATH
RESULTS_PATH = ROOT + PROJECT_PATH + OUTPUT_PATH

%cd {DATA_PATH}

if len(INPUT_FILE) > 0:
  raw_text = get_text_from_file(INPUT_FILE)
  %cd {RESULTS_PATH}
  process_text(raw_text, INPUT_FILE, ALGORITHM_TYPE)
  print('arquivo ' + INPUT_FILE + ' processado.')
else:
  items = get_text_from_files()
  %cd {RESULTS_PATH}
  for item in items:
    process_text(item['content'], item['name'], ALGORITHM_TYPE)
    print('arquivo ' + item['name'] + ' processado.')

%cd {WORKING_PATH}

print('Script concluído.')<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup