[SPARK-28132][PYTHON] Update document type conversion for Pandas UDFs (pyarrow 0.13.0, pandas 0.24.2, Python 3.7)
[SPARK-28132][PYTHON] Update document type conversion for Pandas UDFs (pyarrow 0.13.0, pandas 0.24.2, Python 3.7)

## What changes were proposed in this pull request?

This PR updates the chart generated at SPARK-25666. We deprecated Python 2. It's better to use Python 3.

We don't have to test `unicode` and `long` anymore in Python 3. So it was removed.

Use this code to generate the chart:

```python
from pyspark.sql.types import *
from pyspark.sql.functions import pandas_udf

columns = [
    ('none', 'object(NoneType)'),
    ('bool', 'bool'),
    ('int8', 'int8'),
    ('int16', 'int16'),
    ('int32', 'int32'),
    ('int64', 'int64'),
    ('uint8', 'uint8'),
    ('uint16', 'uint16'),
    ('uint32', 'uint32'),
    ('uint64', 'uint64'),
    ('float64', 'float16'),
    ('float64', 'float32'),
    ('float64', 'float64'),
    ('date', 'datetime64[ns]'),
    ('tz_aware_dates', 'datetime64[ns, US/Eastern]'),
    ('string', 'object(string)'),
    ('decimal', 'object(Decimal)'),
    ('array', 'object(array[int32])'),
    ('float128', 'float128'),
    ('complex64', 'complex64'),
    ('complex128', 'complex128'),
    ('category', 'category'),
    ('tdeltas', 'timedelta64[ns]'),
]

def create_dataframe():
    import pandas as pd
    import numpy as np
    import decimal
    pdf = pd.DataFrame({
        'none': [None, None],
        'bool': [True, False],
        'int8': np.arange(1, 3).astype('int8'),
        'int16': np.arange(1, 3).astype('int16'),
        'int32': np.arange(1, 3).astype('int32'),
        'int64': np.arange(1, 3).astype('int64'),
        'uint8': np.arange(1, 3).astype('uint8'),
        'uint16': np.arange(1, 3).astype('uint16'),
        'uint32': np.arange(1, 3).astype('uint32'),
        'uint64': np.arange(1, 3).astype('uint64'),
        'float16': np.arange(1, 3).astype('float16'),
        'float32': np.arange(1, 3).astype('float32'),
        'float64': np.arange(1, 3).astype('float64'),
        'float128': np.arange(1, 3).astype('float128'),
        'complex64': np.arange(1, 3).astype('complex64'),
        'complex128': np.arange(1, 3).astype('complex128'),
        'string': list('ab'),
        'array': pd.Series([np.array([1, 2, 3], dtype=np.int32), np.array([1, 2, 3], dtype=np.int32)]),
        'decimal': pd.Series([decimal.Decimal('1'), decimal.Decimal('2')]),
        'date': pd.date_range('19700101', periods=2).values,
        'category': pd.Series(list("AB")).astype('category')})
    pdf['tdeltas'] = [pdf.date.diff()[1], pdf.date.diff()[0]]
    pdf['tz_aware_dates'] = pd.date_range('19700101', periods=2, tz='US/Eastern')
    return pdf

types =  [
    BooleanType(),
    ByteType(),
    ShortType(),
    IntegerType(),
    LongType(),
    FloatType(),
    DoubleType(),
    DateType(),
    TimestampType(),
    StringType(),
    DecimalType(10, 0),
    ArrayType(IntegerType()),
    MapType(StringType(), IntegerType()),
    StructType([StructField("_1", IntegerType())]),
    BinaryType(),
]

df = spark.range(2).repartition(1)
results = []
count = 0
total = len(types) * len(columns)
values = []
spark.sparkContext.setLogLevel("FATAL")
for t in types:
    result = []
    for column, pandas_t in columns:
        v = create_dataframe()[column][0]
        values.append(v)
        try:
            row = df.select(pandas_udf(lambda _: create_dataframe()[column], t)(df.id)).first()
            ret_str = repr(row[0])
        except Exception:
            ret_str = "X"
        result.append(ret_str)
        progress = "SQL Type: [%s]\n  Pandas Value(Type): %s(%s)]\n  Result Python Value: [%s]" % (
            t.simpleString(), v, pandas_t, ret_str)
        count += 1
        print("%s/%s:\n  %s" % (count, total, progress))
    results.append([t.simpleString()] + list(map(str, result)))

schema = ["SQL Type \\ Pandas Value(Type)"] + list(map(lambda values_column: "%s(%s)" % (values_column[0], values_column[1][1]), zip(values, columns)))
strings = spark.createDataFrame(results, schema=schema)._jdf.showString(20, 20, False)
print("\n".join(map(lambda line: "    # %s  # noqa" % line, strings.strip().split("\n"))))
```

## How was this patch tested?

Manually.

Closes #24930 from HyukjinKwon/SPARK-28132.

Authored-by: HyukjinKwon <2680965ad5aae29275c8fb10150cb10f5554e78b@apache.org>
Signed-off-by: Bryan Cutler <060dd14c05a8ffa8f201883ef57d0676b7c9c04a@gmail.com>

vinodkc/spark,rezasafi/spark,srowen/spark,zero323/spark,mahak/spark,kevinyu98/spark,hvanhovell/spark,xuanyuanking/spark,taroplus/spark,HyukjinKwon/spark,milliman/spark,darionyaphet/spark,rednaxelafx/apache-spark,chuckchen/spark,mahak/spark,srowen/spark,darionyaphet/spark,cloud-fan/spark,WeichenXu123/spark,apache/spark,dongjoon-hyun/spark,maropu/spark,maropu/spark,rednaxelafx/apache-spark,spark-test/spark,goldmedal/spark,apache/spark,rezasafi/spark,skonto/spark,kiszk/spark,HyukjinKwon/spark,kiszk/spark,jiangxb1987/spark,wangmiao1981/spark,caneGuy/spark,nchammas/spark,holdenk/spark,milliman/spark,apache/spark,spark-test/spark,BryanCutler/spark,actuaryzhang/spark,actuaryzhang/spark,chuckchen/spark,maropu/spark,skonto/spark,zzcclp/spark,shaneknapp/spark,bdrillard/spark,jiangxb1987/spark,ueshin/apache-spark,techaddict/spark,darionyaphet/spark,gengliangwang/spark,actuaryzhang/spark,shaneknapp/spark,dbtsai/spark,JoshRosen/spark,gengliangwang/spark,wzhfy/spark,WeichenXu123/spark,wzhfy/spark,dongjoon-hyun/spark,taroplus/spark,ueshin/apache-spark,zzcclp/spark,zuotingbing/spark,wangmiao1981/spark,jkbradley/spark,pgandhi999/spark,HyukjinKwon/spark,wzhfy/spark,ueshin/apache-spark,zzcclp/spark,jkbradley/spark,kevinyu98/spark,ConeyLiu/spark,xuanyuanking/spark,vinodkc/spark,bdrillard/spark,taroplus/spark,pgandhi999/spark,shaneknapp/spark,zero323/spark,gengliangwang/spark,zuotingbing/spark,rezasafi/spark,kiszk/spark,jiangxb1987/spark,holdenk/spark,gengliangwang/spark,vinodkc/spark,rednaxelafx/apache-spark,shaneknapp/spark,BryanCutler/spark,apache/spark,caneGuy/spark,apache/spark,chuckchen/spark,mahak/spark,witgo/spark,holdenk/spark,cloud-fan/spark,dongjoon-hyun/spark,taroplus/spark,zero323/spark,wangyum/spark,zuotingbing/spark,wangmiao1981/spark,dongjoon-hyun/spark,hvanhovell/spark,srowen/spark,holdenk/spark,bdrillard/spark,bdrillard/spark,bdrillard/spark,holdenk/spark,zero323/spark,cloud-fan/spark,maropu/spark,HyukjinKwon/spark,wangmiao1981/spark,wangyum/spark,witgo/spark,gengliangwang/spark,pgandhi999/spark,rednaxelafx/apache-spark,kiszk/spark,techaddict/spark,maropu/spark,darionyaphet/spark,pgandhi999/spark,gengliangwang/spark,mahak/spark,cloud-fan/spark,shaneknapp/spark,skonto/spark,rednaxelafx/apache-spark,vinodkc/spark,BryanCutler/spark,taroplus/spark,dongjoon-hyun/spark,taroplus/spark,pgandhi999/spark,taroplus/spark,holdenk/spark,HyukjinKwon/spark,WeichenXu123/spark,shuangshuangwang/spark,shuangshuangwang/spark,jiangxb1987/spark,rezasafi/spark,BryanCutler/spark,srowen/spark,chuckchen/spark,ConeyLiu/spark,rezasafi/spark,kiszk/spark,spark-test/spark,techaddict/spark,dongjoon-hyun/spark,zzcclp/spark,nchammas/spark,techaddict/spark,ptkool/spark,chuckchen/spark,shuangshuangwang/spark,pgandhi999/spark,zuotingbing/spark,shaneknapp/spark,hvanhovell/spark,wangmiao1981/spark,wangyum/spark,zzcclp/spark,mahak/spark,shuangshuangwang/spark,zero323/spark,ueshin/apache-spark,jiangxb1987/spark,holdenk/spark,jkbradley/spark,shuangshuangwang/spark,JoshRosen/spark,wangyum/spark,srowen/spark,ConeyLiu/spark,maropu/spark,goldmedal/spark,dbtsai/spark,zuotingbing/spark,maropu/spark,goldmedal/spark,milliman/spark,ptkool/spark,darionyaphet/spark,wangmiao1981/spark,nchammas/spark,xuanyuanking/spark,dbtsai/spark,apache/spark,skonto/spark,goldmedal/spark,actuaryzhang/spark,jkbradley/spark,ConeyLiu/spark,JoshRosen/spark,matthewfranglen/spark,milliman/spark,milliman/spark,rednaxelafx/apache-spark,jkbradley/spark,jiangxb1987/spark,zuotingbing/spark,hvanhovell/spark,xuanyuanking/spark,kiszk/spark,HyukjinKwon/spark,WeichenXu123/spark,witgo/spark,xuanyuanking/spark,zuotingbing/spark,cloud-fan/spark,actuaryzhang/spark,mahak/spark,cloud-fan/spark,ptkool/spark,ueshin/apache-spark,chuckchen/spark,spark-test/spark,kevinyu98/spark,goldmedal/spark,hvanhovell/spark,ptkool/spark,witgo/spark,caneGuy/spark,zero323/spark,caneGuy/spark,JoshRosen/spark,bdrillard/spark,darionyaphet/spark,dongjoon-hyun/spark,wangyum/spark,srowen/spark,bdrillard/spark,techaddict/spark,milliman/spark,cloud-fan/spark,zero323/spark,dbtsai/spark,mahak/spark,ueshin/apache-spark,kevinyu98/spark,WeichenXu123/spark,zzcclp/spark,darionyaphet/spark,jkbradley/spark,wzhfy/spark,jiangxb1987/spark,kevinyu98/spark,actuaryzhang/spark,xuanyuanking/spark,shaneknapp/spark,nchammas/spark,witgo/spark,gengliangwang/spark,JoshRosen/spark,goldmedal/spark,zzcclp/spark,witgo/spark,apache/spark,nchammas/spark,rednaxelafx/apache-spark,wzhfy/spark,skonto/spark,goldmedal/spark,vinodkc/spark,pgandhi999/spark,techaddict/spark,nchammas/spark,actuaryzhang/spark,BryanCutler/spark,nchammas/spark,JoshRosen/spark,wangyum/spark,ptkool/spark,witgo/spark,ConeyLiu/spark,kevinyu98/spark,spark-test/spark,srowen/spark,WeichenXu123/spark,hvanhovell/spark,JoshRosen/spark,jkbradley/spark,wzhfy/spark,ConeyLiu/spark,caneGuy/spark,rezasafi/spark,ptkool/spark,milliman/spark,techaddict/spark,kevinyu98/spark,dbtsai/spark,HyukjinKwon/spark,caneGuy/spark,spark-test/spark,vinodkc/spark,ConeyLiu/spark,dbtsai/spark,ptkool/spark,skonto/spark,ueshin/apache-spark,wangyum/spark,caneGuy/spark,hvanhovell/spark,kiszk/spark,chuckchen/spark,skonto/spark,shuangshuangwang/spark,rezasafi/spark,BryanCutler/spark,BryanCutler/spark,vinodkc/spark,dbtsai/spark,spark-test/spark,xuanyuanking/spark,wzhfy/spark,shuangshuangwang/spark,wangmiao1981/spark,WeichenXu123/spark