


"""simple docstring"""




from typing import TYPE_CHECKING

from ..utils import OptionalDependencyNotAvailable, _LazyModule, is_flax_available, is_tf_available, is_torch_available


__SCREAMING_SNAKE_CASE					=						{
    """configuration_utils""": ["""GenerationConfig"""],
    """streamers""": ["""TextIteratorStreamer""", """TextStreamer"""],
}

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			__SCREAMING_SNAKE_CASE					=						[
			    """Constraint""",
			    """ConstraintListState""",
			    """DisjunctiveConstraint""",
			    """PhrasalConstraint""",
			]
			__SCREAMING_SNAKE_CASE					=						[
			    """BeamHypotheses""",
			    """BeamScorer""",
			    """BeamSearchScorer""",
			    """ConstrainedBeamSearchScorer""",
			]
			__SCREAMING_SNAKE_CASE					=						[
			    """EpsilonLogitsWarper""",
			    """EtaLogitsWarper""",
			    """ForcedBOSTokenLogitsProcessor""",
			    """ForcedEOSTokenLogitsProcessor""",
			    """HammingDiversityLogitsProcessor""",
			    """InfNanRemoveLogitsProcessor""",
			    """LogitsProcessor""",
			    """LogitsProcessorList""",
			    """LogitsWarper""",
			    """MinLengthLogitsProcessor""",
			    """MinNewTokensLengthLogitsProcessor""",
			    """NoBadWordsLogitsProcessor""",
			    """NoRepeatNGramLogitsProcessor""",
			    """PrefixConstrainedLogitsProcessor""",
			    """RepetitionPenaltyLogitsProcessor""",
			    """SequenceBiasLogitsProcessor""",
			    """EncoderRepetitionPenaltyLogitsProcessor""",
			    """TemperatureLogitsWarper""",
			    """TopKLogitsWarper""",
			    """TopPLogitsWarper""",
			    """TypicalLogitsWarper""",
			    """EncoderNoRepeatNGramLogitsProcessor""",
			    """ExponentialDecayLengthPenalty""",
			    """LogitNormalization""",
			]
			__SCREAMING_SNAKE_CASE					=						[
			    """MaxNewTokensCriteria""",
			    """MaxLengthCriteria""",
			    """MaxTimeCriteria""",
			    """StoppingCriteria""",
			    """StoppingCriteriaList""",
			    """validate_stopping_criteria""",
			]
			__SCREAMING_SNAKE_CASE					=						[
			    """GenerationMixin""",
			    """top_k_top_p_filtering""",
			    """GreedySearchEncoderDecoderOutput""",
			    """GreedySearchDecoderOnlyOutput""",
			    """SampleEncoderDecoderOutput""",
			    """SampleDecoderOnlyOutput""",
			    """BeamSearchEncoderDecoderOutput""",
			    """BeamSearchDecoderOnlyOutput""",
			    """BeamSampleEncoderDecoderOutput""",
			    """BeamSampleDecoderOnlyOutput""",
			    """ContrastiveSearchEncoderDecoderOutput""",
			    """ContrastiveSearchDecoderOnlyOutput""",
			]

try:
			if not is_tf_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			__SCREAMING_SNAKE_CASE					=						[
			    """TFForcedBOSTokenLogitsProcessor""",
			    """TFForcedEOSTokenLogitsProcessor""",
			    """TFLogitsProcessor""",
			    """TFLogitsProcessorList""",
			    """TFLogitsWarper""",
			    """TFMinLengthLogitsProcessor""",
			    """TFNoBadWordsLogitsProcessor""",
			    """TFNoRepeatNGramLogitsProcessor""",
			    """TFRepetitionPenaltyLogitsProcessor""",
			    """TFTemperatureLogitsWarper""",
			    """TFTopKLogitsWarper""",
			    """TFTopPLogitsWarper""",
			    """TFForceTokensLogitsProcessor""",
			    """TFSuppressTokensAtBeginLogitsProcessor""",
			    """TFSuppressTokensLogitsProcessor""",
			]
			__SCREAMING_SNAKE_CASE					=						[
			    """TFGenerationMixin""",
			    """tf_top_k_top_p_filtering""",
			    """TFGreedySearchDecoderOnlyOutput""",
			    """TFGreedySearchEncoderDecoderOutput""",
			    """TFSampleEncoderDecoderOutput""",
			    """TFSampleDecoderOnlyOutput""",
			    """TFBeamSearchEncoderDecoderOutput""",
			    """TFBeamSearchDecoderOnlyOutput""",
			    """TFBeamSampleEncoderDecoderOutput""",
			    """TFBeamSampleDecoderOnlyOutput""",
			    """TFContrastiveSearchEncoderDecoderOutput""",
			    """TFContrastiveSearchDecoderOnlyOutput""",
			]

try:
			if not is_flax_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			__SCREAMING_SNAKE_CASE					=						[
			    """FlaxForcedBOSTokenLogitsProcessor""",
			    """FlaxForcedEOSTokenLogitsProcessor""",
			    """FlaxLogitsProcessor""",
			    """FlaxLogitsProcessorList""",
			    """FlaxLogitsWarper""",
			    """FlaxMinLengthLogitsProcessor""",
			    """FlaxTemperatureLogitsWarper""",
			    """FlaxTopKLogitsWarper""",
			    """FlaxTopPLogitsWarper""",
			]
			__SCREAMING_SNAKE_CASE					=						[
			    """FlaxGenerationMixin""",
			    """FlaxGreedySearchOutput""",
			    """FlaxSampleOutput""",
			    """FlaxBeamSearchOutput""",
			]

if TYPE_CHECKING:
			from .configuration_utils import GenerationConfig
			from .streamers import TextIteratorStreamer, TextStreamer

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .beam_constraints import Constraint, ConstraintListState, DisjunctiveConstraint, PhrasalConstraint
						from .beam_search import BeamHypotheses, BeamScorer, BeamSearchScorer, ConstrainedBeamSearchScorer
						from .logits_process import (
						    EncoderNoRepeatNGramLogitsProcessor,
						    EncoderRepetitionPenaltyLogitsProcessor,
						    EpsilonLogitsWarper,
						    EtaLogitsWarper,
						    ExponentialDecayLengthPenalty,
						    ForcedBOSTokenLogitsProcessor,
						    ForcedEOSTokenLogitsProcessor,
						    HammingDiversityLogitsProcessor,
						    InfNanRemoveLogitsProcessor,
						    LogitNormalization,
						    LogitsProcessor,
						    LogitsProcessorList,
						    LogitsWarper,
						    MinLengthLogitsProcessor,
						    MinNewTokensLengthLogitsProcessor,
						    NoBadWordsLogitsProcessor,
						    NoRepeatNGramLogitsProcessor,
						    PrefixConstrainedLogitsProcessor,
						    RepetitionPenaltyLogitsProcessor,
						    SequenceBiasLogitsProcessor,
						    TemperatureLogitsWarper,
						    TopKLogitsWarper,
						    TopPLogitsWarper,
						    TypicalLogitsWarper,
						)
						from .stopping_criteria import (
						    MaxLengthCriteria,
						    MaxNewTokensCriteria,
						    MaxTimeCriteria,
						    StoppingCriteria,
						    StoppingCriteriaList,
						    validate_stopping_criteria,
						)
						from .utils import (
						    BeamSampleDecoderOnlyOutput,
						    BeamSampleEncoderDecoderOutput,
						    BeamSearchDecoderOnlyOutput,
						    BeamSearchEncoderDecoderOutput,
						    ContrastiveSearchDecoderOnlyOutput,
						    ContrastiveSearchEncoderDecoderOutput,
						    GenerationMixin,
						    GreedySearchDecoderOnlyOutput,
						    GreedySearchEncoderDecoderOutput,
						    SampleDecoderOnlyOutput,
						    SampleEncoderDecoderOutput,
						    top_k_top_p_filtering,
						)

			try:
						if not is_tf_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tf_logits_process import (
						    TFForcedBOSTokenLogitsProcessor,
						    TFForcedEOSTokenLogitsProcessor,
						    TFForceTokensLogitsProcessor,
						    TFLogitsProcessor,
						    TFLogitsProcessorList,
						    TFLogitsWarper,
						    TFMinLengthLogitsProcessor,
						    TFNoBadWordsLogitsProcessor,
						    TFNoRepeatNGramLogitsProcessor,
						    TFRepetitionPenaltyLogitsProcessor,
						    TFSuppressTokensAtBeginLogitsProcessor,
						    TFSuppressTokensLogitsProcessor,
						    TFTemperatureLogitsWarper,
						    TFTopKLogitsWarper,
						    TFTopPLogitsWarper,
						)
						from .tf_utils import (
						    TFBeamSampleDecoderOnlyOutput,
						    TFBeamSampleEncoderDecoderOutput,
						    TFBeamSearchDecoderOnlyOutput,
						    TFBeamSearchEncoderDecoderOutput,
						    TFContrastiveSearchDecoderOnlyOutput,
						    TFContrastiveSearchEncoderDecoderOutput,
						    TFGenerationMixin,
						    TFGreedySearchDecoderOnlyOutput,
						    TFGreedySearchEncoderDecoderOutput,
						    TFSampleDecoderOnlyOutput,
						    TFSampleEncoderDecoderOutput,
						    tf_top_k_top_p_filtering,
						)

			try:
						if not is_flax_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .flax_logits_process import (
						    FlaxForcedBOSTokenLogitsProcessor,
						    FlaxForcedEOSTokenLogitsProcessor,
						    FlaxLogitsProcessor,
						    FlaxLogitsProcessorList,
						    FlaxLogitsWarper,
						    FlaxMinLengthLogitsProcessor,
						    FlaxTemperatureLogitsWarper,
						    FlaxTopKLogitsWarper,
						    FlaxTopPLogitsWarper,
						)
						from .flax_utils import FlaxBeamSearchOutput, FlaxGenerationMixin, FlaxGreedySearchOutput, FlaxSampleOutput
else:
			import sys

			__SCREAMING_SNAKE_CASE					=						_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




"""simple docstring"""




# Copyright 2022 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from typing import List



def 						__a      (  a,		a					):





       """simple docstring"""







       if name != "training_function" and name != "main":
              raise ValueError(F'Incorrect function name passed: {name}, choose either \'main\' or \'training_function\''					)
       _a   , _a          =				[], False
       for line in lines:
              if not found_start and F'def {name}' in line:
                     _a          =				True
                     good_lines.append(a					)
                     continue
              if found_start:
                     if name == "training_function" and "def main" in line:
                            return good_lines
                     if name == "main" and "if __name__" in line:
                            return good_lines
                     good_lines.append(a					)



def 						__a      (  a					):





       """simple docstring"""







       return [line for line in lines if not line.lstrip().startswith("#"					) and line != "\n"]



def 						__a      (  a,		a,		a,		a = None					):





       """simple docstring"""







       with open(a,		"r"					) as f:
              _a          =				f.readlines()
       with open(os.path.abspath(os.path.join("examples",		"nlp_example.py"					)					),		"r"					) as f:
              _a          =				f.readlines()
       with open(a,		"r"					) as f:
              _a          =				f.readlines()
       if secondary_filename is not None:
              with open(a,		"r"					) as f:
                     _a          =				f.readlines()

    # This is our base, we remove all the code from here in our `full_filename` and `feature_filename` to find the new content
       if parser_only:
              _a          =				clean_lines(get_function_contents_by_name(a,		"main"					)					)
              _a          =				clean_lines(get_function_contents_by_name(a,		"main"					)					)
              _a          =				clean_lines(get_function_contents_by_name(a,		"main"					)					)
              if secondary_filename is not None:
                     _a          =				clean_lines(get_function_contents_by_name(a,		"main"					)					)
       else:
              _a          =				clean_lines(get_function_contents_by_name(a,		"training_function"					)					)
              _a          =				clean_lines(get_function_contents_by_name(a,		"training_function"					)					)
              _a          =				clean_lines(get_function_contents_by_name(a,		"training_function"					)					)
              if secondary_filename is not None:
                     _a          =				clean_lines(
                         get_function_contents_by_name(a,		"training_function"					)					)

       _a          =				"train_dataloader, eval_dataloader = get_dataloaders(accelerator, batch_size)\n"

       # Specific code in our script that differs from the full version, aka what is new
       _a          =				[]
       _a          =				[]  # We keep track of the idxs just in case it's a repeated statement
       _a          =				iter(a					)
       for i in range(len(a					) - 1					):
              if i not in passed_idxs:
                     _a          =				next(a					)
                     if (line not in full_file_func) and (line.lstrip() != _dl_line):
                            if "TESTING_MOCKED_DATALOADERS" not in line:
                                   new_feature_code.append(a					)
                                   passed_idxs.append(a					)
                            else:
                                   # Skip over the `config['num_epochs'] = 2` statement
                                   _a          =				next(a					)

    # Extract out just the new parts from the full_file_training_func
       _a          =				[]
       _a          =				[]  # We keep track of the idxs just in case it's a repeated statement
       for i, line in enumerate(a					):
              if i not in passed_idxs:
                     if (line not in full_file_func) and (line.lstrip() != _dl_line):
                            if "TESTING_MOCKED_DATALOADERS" not in line:
                                   new_full_example_parts.append(a					)
                                   passed_idxs.append(a					)

    # Finally, get the overall diff
       _a          =				[line for line in new_feature_code if line not in new_full_example_parts]
       if secondary_filename is not None:
              _a          =				[line for line in full_file_contents if line not in secondary_file_func]
              _a          =				[line for line in diff_from_example if line not in diff_from_two]

       return diff_from_example
