# -*- coding: utf-8    # This file contains Unicode characters.

from io import StringIO
from textwrap import dedent

from jedi._compatibility import u, is_py3
from jedi.parser.token import NAME, OP, NEWLINE, STRING, INDENT
from jedi import parser
from token import STRING


from ..helpers import unittest


class TokenTest(unittest.TestCase):
    def test_end_pos_one_line(self):
        parsed = parser.Parser(parser.load_grammar(), dedent(u('''
        def testit():
            a = "huhu"
        ''')))
        tok = parsed.module.subscopes[0].statements[0].children[2]
        assert tok.end_pos == (3, 14)

    def test_end_pos_multi_line(self):
        parsed = parser.Parser(parser.load_grammar(), dedent(u('''
        def testit():
            a = """huhu
        asdfasdf""" + "h"
        ''')))
        tok = parsed.module.subscopes[0].statements[0].children[2].children[0]
        assert tok.end_pos == (4, 11)

    def test_simple_no_whitespace(self):
        # Test a simple one line string, no preceding whitespace
        simple_docstring = u('"""simple one line docstring"""')
        simple_docstring_io = StringIO(simple_docstring)
        tokens = parser.tokenize.generate_tokens(simple_docstring_io.readline)
        token_list = list(tokens)
        _, value, _, prefix = token_list[0]
        assert prefix == ''
        assert value == '"""simple one line docstring"""'

    def test_simple_with_whitespace(self):
        # Test a simple one line string with preceding whitespace and newline
        simple_docstring = u('  """simple one line docstring""" \r\n')
        simple_docstring_io = StringIO(simple_docstring)
        tokens = parser.tokenize.generate_tokens(simple_docstring_io.readline)
        token_list = list(tokens)
        assert token_list[0][0] == INDENT
        typ, value, start_pos, prefix = token_list[1]
        assert prefix == '  '
        assert value == '"""simple one line docstring"""'
        assert typ == STRING
        typ, value, start_pos, prefix = token_list[2]
        assert prefix == ' '
        assert typ == NEWLINE

    def test_function_whitespace(self):
        # Test function definition whitespace identification
        fundef = dedent(u('''
        def test_whitespace(*args, **kwargs):
            x = 1
            if x > 0:
                print(True)
        '''))
        fundef_io = StringIO(fundef)
        tokens = parser.tokenize.generate_tokens(fundef_io.readline)
        token_list = list(tokens)
        for _, value, _, prefix in token_list:
            if value == 'test_whitespace':
                assert prefix == ' '
            if value == '(':
                assert prefix == ''
            if value == '*':
                assert prefix == ''
            if value == '**':
                assert prefix == ' '
            if value == 'print':
                assert prefix == '        '
            if value == 'if':
                assert prefix == '    '

    def test_identifier_contains_unicode(self):
        fundef = dedent(u('''
        def 我あφ():
            pass
        '''))
        fundef_io = StringIO(fundef)
        tokens = parser.tokenize.generate_tokens(fundef_io.readline)
        token_list = list(tokens)
        unicode_token = token_list[1]
        if is_py3:
            assert unicode_token[0] == NAME
        else:
            # Unicode tokens in Python 2 seem to be identified as operators.
            # They will be ignored in the parser, that's ok.
            assert unicode_token[0] == OP

    def test_quoted_strings(self):

        string_tokens = [
            'u"test"',
            'u"""test"""',
            'U"""test"""',
            "u'''test'''",
            "U'''test'''",
        ]

        for s in string_tokens:
            parsed = parser.Parser(u('''a = %s\n''' % s))
            tok_list = parsed.module.statements[0]._token_list
            self.assertEqual(len(tok_list), 3)
            tok = tok_list[2]
            self.assertIsInstance(tok, parser.tokenize.Token)
            self.assertEqual(tok.type, STRING)


def test_tokenizer_with_string_literal_backslash():
    import jedi
    c = jedi.Script("statement = u'foo\\\n'; statement").goto_definitions()
    assert c[0]._name.parent.obj == 'foo'

@@ -226,24 +226,31 @@
 rom jedi
+.parser
  import 
 parser%0Af
@@ -245,39 +245,38 @@
 ort 
-p
+P
 arser
-%0Afrom token import STRING
+, load_grammar, tokenize
 %0A%0A%0Af
@@ -397,37 +397,23 @@
 arsed = 
-parser.Parser(parser.
+Parser(
 load_gra
@@ -659,29 +659,15 @@
 d = 
-parser.Parser(parser.
+Parser(
 load
@@ -1132,39 +1132,32 @@
        tokens = 
-parser.
 tokenize.generat
@@ -1618,39 +1618,32 @@
        tokens = 
-parser.
 tokenize.generat
@@ -2347,39 +2347,32 @@
        tokens = 
-parser.
 tokenize.generat
@@ -3088,23 +3088,16 @@
 okens = 
-parser.
 tokenize
@@ -3724,22 +3724,31 @@
 d = 
-p
+P
 arser
-.Parser(
+(load_grammar(), 
 u(''
@@ -3778,23 +3778,26 @@
         
-tok_lis
+simple_stm
 t = pars
@@ -3810,33 +3810,67 @@
 ule.
-statements%5B0%5D._token_list
+children%5B0%5D%0A            expr_stmt = simple_stmt.children%5B0%5D
 %0A   
@@ -3882,42 +3882,43 @@
     
-self.
 assert
-Equal(len(tok_list),
+ len(expr_stmt.children) ==
  3
-)
 %0A   
@@ -3930,22 +3930,39 @@
     
-tok = tok_list
+string_tok = expr_stmt.children
 %5B2%5D%0A
@@ -3977,57 +3977,83 @@
     
-self.
 assert
-IsInstance(tok, parser.tokenize.Token)
+ string_tok.type == 'string'%0A            assert string_tok.value == s
 %0A   
@@ -4065,42 +4065,42 @@
     
-self.
 assert
-Equal(tok.type, STRING)
+ string_tok.eval() == 'test'
 %0A%0A%0Ad

Change the tests of @hamatov a small bit. They are now working with the new parser.