"""Opinosis Opinion Dataset"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os

import tensorflow as tf
import tensorflow_datasets.public_api as tfds

_CITATION = """
@inproceedings{ganesan2010opinosis,
  title={Opinosis: a graph-based approach to abstractive summarization of highly redundant opinions},
  author={Ganesan, Kavita and Zhai, ChengXiang and Han, Jiawei},
  booktitle={Proceedings of the 23rd International Conference on Computational Linguistics},
  pages={340--348},
  year={2010},
  organization={Association for Computational Linguistics}
}
"""

_DESCRIPTION = """
The Opinosis Opinion Dataset consists of sentences extracted from reviews for 51 topics. 
Topics and opinions are obtained from Tripadvisor, Edmunds.com and Amazon.com. 
"""

_URL = "https://github.com/kavgan/opinosis-summarization/raw/master/OpinosisDataset1.0_0.zip"

_REVIEW_SENTS = 'review_sents'
_SUMMARIES = 'summaries'

class Opinosis(tfds.core.GeneratorBasedBuilder):
  """Opinosis Opinion Dataset """

  VERSION = tfds.core.Version('1.0.0')

  def _info(self):
    return tfds.core.DatasetInfo(
        builder=self,
        description=_DESCRIPTION,
        features=tfds.features.FeaturesDict({
            _REVIEW_SENTS: tfds.features.Text(),
            _SUMMARIES: tfds.features.Text(),
        }),
        supervised_keys=(_REVIEW_SENTS, _SUMMARIES),
        homepage='http://kavita-ganesan.com/opinosis/#.XeTZopNKhTY',
        citation=_CITATION,
    )

  def _split_generators(self, dl_manager):
    """Returns SplitGenerators."""
    extract_path = dl_manager.download_and_extract(_URL)
    return [
        tfds.core.SplitGenerator(
            name=tfds.Split.TEST,
            gen_kwargs={"path": extract_path},
        ),
    ]

  def _generate_examples(self, path=None):
    """Yields examples."""
    topics_path = os.path.join(path, "topics")
    filenames = tf.io.gfile.listdir(topics_path)
    for i, filename in enumerate(filenames):
      file_path = os.path.join(topics_path, filename)
      topic_name = filename.split(".txt")[0]
      with tf.io.gfile.GFile(file_path, "rb") as src_f:
        input_data = src_f.read()
      summaries_path = os.path.join(path, "summaries-gold", topic_name)
      summary_lst = []
      for summ_filename in tf.io.gfile.listdir(summaries_path):
        file_path = os.path.join(summaries_path, summ_filename)
        with tf.io.gfile.GFile(file_path, "rb") as tgt_f:
          data = tgt_f.read()
          summary_lst.append(data)
      summary_data = b"[SEP_SUM]".join(summary_lst)
      yield i, {
          _REVIEW_SENTS: input_data,
          _SUMMARIES: summary_data
          }

@@ -1328,16 +1328,39 @@
 MARIES: 
+tfds.features.Sequence(
 tfds.fea
@@ -1363,33 +1363,33 @@
 .features.Text()
-,
+)
 %0A        %7D),%0A   
@@ -1996,19 +1996,16 @@
 %0A    for
- i,
  filenam
@@ -2013,18 +2013,8 @@
  in 
-enumerate(
 file
@@ -2018,17 +2018,16 @@
 ilenames
-)
 :%0A      
@@ -2580,26 +2580,8 @@
 a = 
-b%22%5BSEP_SUM%5D%22.join(
 summ
@@ -2587,17 +2587,16 @@
 mary_lst
-)
 %0A      y
@@ -2600,17 +2600,24 @@
   yield 
-i
+filename
 , %7B%0A    

update tfds.features.Sequence and fname as key