<jupyter_start><jupyter_text>Set up<jupyter_code># !pip install bert-extractive-summarizer -q
# !pip install spacy==2.1.3 -q
!pip install transformers -q #==2.2.2
# !pip install neuralcoref -q
# !pip install ipywidgets -q
import pandas as pd
import numpy as np
import os
from pathlib import Path, PurePath
from transformers import *
# from summarizer import Summarizer
import logging
import torch
from numpy import ndarray
from typing import List
from tqdm.notebook import tqdm
# import spacy

# set data paths, this requires local drive to have a folder calld "COVID-19" with the metadata.csv file
# returns a string to the local path setup
def setup_local_data():
  from google.colab import drive
  drive.mount('/content/drive')
  drive_path=PurePath('/content/drive/My Drive')
  input_dir = drive_path/'COVID-19'
  print(list(Path(input_dir).glob('*')))
  return input_dir
#read the metadata file into df
def read_metadata_csv(input_dir):
    metadata_path = input_dir/ 'clean_metadata.csv'
    metadata = pd.read_csv(metadata_path, 
                         dtype={'title':str,
                                'abstract':str})
    #set the abstract to the paper title if it is null
    metadata['abstract'] = metadata['abstract'].fillna(metadata['title'])
    #remove if abstract is empty or contains only one word
    metadata = metadata.dropna(subset=['abstract'], axis = 0)
    metadata['number_tokens'] = metadata['abstract'].apply(lambda x: len(x.split()))
    metadata = metadata[metadata['number_tokens']>1].reset_index(drop=True)
    metadata = metadata.drop('Unnamed: 0', axis=1)
    return metadata
# read preprocessed SciBERT embeddings
def read_summary_data(input_dir):
  summary_path = input_dir/'AbstractSummaries' 
  summaries = pd.concat([pd.read_json(f) for f in Path(summary_path).glob('*')]).reset_index(drop=True)
  return summaries
def create_custom_model_tokenizer(pretrain_model):
    # Load model, model config and tokenizer via Transformers
    custom_config = AutoConfig.from_pretrained(pretrain_model)
    custom_config.output_hidden_states=True
    custom_tokenizer = AutoTokenizer.from_pretrained(pretrain_model)
    custom_model = AutoModel.from_pretrained(pretrain_model, config=custom_config)
    return custom_model, custom_tokenizer

# from spacy.lang.en import English

# class SentenceHandler(object):

#     def __init__(self, language = English):
#         self.nlp = language()
#         self.nlp.add_pipe(self.nlp.create_pipe('sentencizer'))

#     def process(self, body: str, min_length: int = 40, max_length: int = 600):
#         """
#         Processes the content sentences.
#         :param body: The raw string body to process
#         :param min_length: Minimum length that the sentences must be
#         :param max_length: Max length that the sentences mus fall under
#         :return: Returns a list of sentences.
#         """
#         doc = self.nlp(body)
#         doc.is_parsed=True
#         return [c.string.strip() for c in doc.sents if max_length > len(c.string.strip()) > min_length]

#     def __call__(self, body: str, min_length: int = 40, max_length: int = 600):
#         return self.process(body, min_length, max_length)
def extract_summary(text, custom_model=None, custom_tokenizer = None):
    model = Summarizer(custom_model=custom_model, custom_tokenizer=custom_tokenizer)
    return model(text)<jupyter_output><empty_output><jupyter_text>Main code<jupyter_code>local_dir = setup_local_data()
metadata = read_metadata_csv(local_dir)
metadata.info()
#instantiate custom model and tokenizer
sciBert, sciBert_tokenizer = create_custom_model_tokenizer('allenai/scibert_scivocab_uncased')
CHUNKS_COUNT = 20
#extract summaries from the abstracts
#chunk processing due to long processing time and possible notebook runtime shutdowns
chunks = np.array_split(metadata, CHUNKS_COUNT)
# chunk_dfs = []
for index in range(3,len(chunks)):
    chunk = chunks[index].reset_index()
    summary_list = []
    for i in tqdm(chunk.index):
        summary = extract_summary(chunk.iloc[i]['abstract'], 
                                  custom_model=sciBert, 
                                  custom_tokenizer=sciBert_tokenizer)
        summary_list.append(
        {
        "cord_uid": chunk.iloc[i]['cord_uid'],
        "sha": chunk.iloc[i]['sha'],
        # "scibert_emb": bc.encode([summary])[0],
        "summary": summary
        })
    summary_df = pd.DataFrame(data=summary_list)
    summary_df.to_json(local_dir/'abstract_summaries_part{}.json'.format(index))

!pip install --upgrade git+https://github.com/zalandoresearch/flair.git -q
from flair.data import Sentence
from flair.embeddings import BertEmbeddings,DocumentPoolEmbeddings
import glob

emb_model = BertEmbeddings(bert_model_or_path="allenai/scibert_scivocab_uncased", 
                           layers='-2')
def get_embeddings(text, model):
  sentence = Sentence(text)
  document_embedding = DocumentPoolEmbeddings([model],
                                             pooling= 'mean')
  document_embedding.embed(sentence)
  # now check out the embedded sentence.
  return sentence.get_embedding().data.numpy()
CHUNKS_COUNT=3
files = glob.glob('/content/drive/My Drive/COVID-19/AbstractSummaries/*.json')

for index in range(len(files)):
    df = pd.read_json(files[index]).dropna(subset=['summary']).reset_index(drop=True)

    chunks = np.array_split(df, CHUNKS_COUNT)
    
    for chunk_idx in range(len(chunks)):
      chunk = chunks[chunk_idx].reset_index()
      emb_list=[]
      for i in tqdm(chunk.index):
        try:
          embedding = get_embeddings(chunk.iloc[i]['summary'], emb_model)
          emb_list.append(
          {
            "cord_uid": chunk.iloc[i]['cord_uid'],
            "sha": chunk.iloc[i]['sha'],
            "scibert_emb": embedding
          })
        except RuntimeError:
          emb_list.append(
          {
            "cord_uid": chunk.iloc[i]['cord_uid'],
            "sha": chunk.iloc[i]['sha'],
            "scibert_emb": np.nan
          })
      emb_df = pd.DataFrame(data=emb_list)
      emb_df.to_json(local_dir/'AbstractEmbeddings'/'abstract_embeddings_part{}_{}.json'.format(index, chunk_idx),
                     default_handler=str)
      del emb_df<jupyter_output><empty_output>
StarcoderdataJupyterStructuredCleanDedup