code: 
  
  
  '''simple docstring'''
  
  import os
  from functools import partial
  from glob import glob
  
  import faiss
  from datasets import Features, Sequence, Value, concatenate_datasets, load_dataset, load_from_disk
  
  from transformers import DPRContextEncoder, DPRContextEncoderTokenizerFast
  
  def _SCREAMING_SNAKE_CASE   (A						,				A=100						,				A=" " ) -> int:
  
  
  
  
  
  
         """simple docstring"""
  
  
  
  
  
  
  
         lowercase__       =				text.split(A )
         return [character.join(text[i : i + n] ).strip() for i in range(0						,				len(A )						,				A )]
  
  def _SCREAMING_SNAKE_CASE   (A ) -> str:
  
  
  
  
  
  
         """simple docstring"""
  
  
  
  
  
  
  
         lowercase__    ,lowercase__       =				[], []
         for title, text in zip(documents['''title''']						,				documents['''text'''] ):
                if text is not None:
                       for passage in split_text(A ):
                              titles.append(title if title is not None else '''''' )
                              texts.append(A )
         return {"title": titles, "text": texts}
  
  def _SCREAMING_SNAKE_CASE   (A						,				A						,				A						,				A						,				A						,				A ) -> Optional[int]:
  
  
  
  
  
  
         """simple docstring"""
  
  
  
  
  
  
  
         lowercase__       =				load_dataset(
             '''csv'''						,				data_files=[csv_path]						,				split='''train'''						,				delimiter='''\t'''						,				column_names=['''title''', '''text'''] )
         lowercase__       =				kb_dataset.map(
             A						,				batched=A						,				num_proc=1 )  # if you want you can load already splitted csv.
         lowercase__       =				[kb_dataset.shard(A						,				A						,				contiguous=A ) for i in range(A )]
         lowercase__       =				kb_list[process_num]
  
         lowercase__       =				'''data_''' + str(A )
         lowercase__       =				os.path.join(A						,				A )
  
         lowercase__       =				DPRContextEncoderTokenizerFast.from_pretrained('''facebook/dpr-ctx_encoder-multiset-base''' )
         lowercase__       =				ctx_encoder.to(device=A )
  
         def embed(
             A						,				A						,				A						,				A ) -> dict:
                lowercase__       =				ctx_tokenizer(
                    documents['''title''']						,				documents['''text''']						,				truncation=A						,				padding='''longest'''						,				return_tensors='''pt''' )['''input_ids''']
                lowercase__       =				ctx_encoder(input_ids.to(device=A )						,				return_dict=A ).pooler_output
                return {"embeddings": embeddings.detach().cpu().numpy()}
  
         lowercase__       =				Features(
             {'''text''': Value('''string''' ), '''title''': Value('''string''' ), '''embeddings''': Sequence(Value('''float32''' ) )} )  # optional, save as float32 instead of float64 to save space
  
         lowercase__       =				data_shrad.map(
             partial(A						,				ctx_encoder=A						,				ctx_tokenizer=A						,				device=A )						,				batched=A						,				batch_size=16						,				features=A						,				)
         dataset.save_to_disk(A )
  
  def _SCREAMING_SNAKE_CASE   (A						,				A ) -> Optional[int]:
  
  
  
  
  
  
         """simple docstring"""
  
  
  
  
  
  
  
         lowercase__       =				[]
  
         for shard_address in glob(str(A ) + '''/*/''' ):
                data_shard_list.append(load_from_disk(A ) )
  
         lowercase__       =				concatenate_datasets(A )
         faiss.omp_set_num_threads(96 )
  
         lowercase__       =				faiss.IndexHNSWFlat(768						,				128						,				faiss.METRIC_INNER_PRODUCT )
         concat.add_faiss_index('''embeddings'''						,				custom_index=A )
         concat.get_index('''embeddings''' ).save(
             A )  # since we load the index in to memory,we can directly update the index in the disk
  
  
  
  
  
style_context: 
  
  
  '''simple docstring'''
  
  import argparse
  
  import torch
  
  from transformers import ElectraConfig, ElectraForMaskedLM, ElectraForPreTraining, load_tf_weights_in_electra
  from transformers.utils import logging
  
  
  logging.set_verbosity_info()
  
  def _SCREAMING_SNAKE_CASE   (A						,				A						,				A						,				A ) -> Dict:
  
  
  
  
  
  
         """simple docstring"""
  
  
  
  
  
  
  
         lowercase__       =				ElectraConfig.from_json_file(A )
         print(f"Building PyTorch model from configuration: {config}" )
  
         if discriminator_or_generator == "discriminator":
                lowercase__       =				ElectraForPreTraining(A )
         elif discriminator_or_generator == "generator":
                lowercase__       =				ElectraForMaskedLM(A )
         else:
                raise ValueError('''The discriminator_or_generator argument should be either \'discriminator\' or \'generator\'''' )
  
         # Load weights from tf checkpoint
         load_tf_weights_in_electra(
             A						,				A						,				A						,				discriminator_or_generator=A )
  
         # Save pytorch-model
         print(f"Save PyTorch model to {pytorch_dump_path}" )
         torch.save(model.state_dict()						,				A )
  
  
  if __name__ == "__main__":
   lowerCamelCase							:     Tuple								=			argparse.ArgumentParser()
   # Required parameters
   parser.add_argument(
       '--tf_checkpoint_path', default=None, type=str, required=True, help='Path to the TensorFlow checkpoint path.'
   )
   parser.add_argument(
       '--config_file',
       default=None,
       type=str,
       required=True,
       help='The config json file corresponding to the pre-trained model. \nThis specifies the model architecture.',
   )
   parser.add_argument(
       '--pytorch_dump_path', default=None, type=str, required=True, help='Path to the output PyTorch model.'
   )
   parser.add_argument(
       '--discriminator_or_generator',
       default=None,
       type=str,
       required=True,
       help=(
           'Whether to export the generator or the discriminator. Should be a string, either \'discriminator\' or '
           '\'generator\'.'
       ),
   )
   lowerCamelCase							:     Optional[int]								=			parser.parse_args()
   convert_tf_checkpoint_to_pytorch(
       args.tf_checkpoint_path, args.config_file, args.pytorch_dump_path, args.discriminator_or_generator
   )
  
  
  
  
  