[SPARK-12090] [PYSPARK] consider shuffle in coalesce()
[SPARK-12090] [PYSPARK] consider shuffle in coalesce()

Author: Davies Liu <davies@databricks.com>

Closes #10090 from davies/fix_coalesce.

panfengfeng/spark,setjet/spark,highfei2011/spark,holdenk/spark,sahilTakiar/spark,janewangfb/spark,ron8hu/spark,ajaysaini725/spark,rezasafi/spark,tengpeng/spark,vinodkc/spark,gengliangwang/spark,wzhfy/spark,huang1900/spark,kevinyu98/spark,lvdongr/spark,nilsgrabbert/spark,facaiy/spark,aray/spark,wangyum/spark,nchammas/spark,zzcclp/spark,maropu/spark,liyichao/spark,WindCanDie/spark,wangyum/spark,witgo/spark,jlopezmalla/spark,UndeadBaneGitHub/spark,ptkool/spark,ddna1021/spark,markhamstra/spark,icexelloss/spark,gioenn/xSpark,BryanCutler/spark,shaneknapp/spark,techaddict/spark,byakuinss/spark,byakuinss/spark,poffuomo/spark,MLnick/spark,ron8hu/spark,holdenk/spark,shubhamchopra/spark,JoshRosen/spark,metamx/spark,vinodkc/spark,facaiy/spark,dhruve/spark,milliman/spark,shuangshuangwang/spark,WeichenXu123/spark,aokolnychyi/spark,rikima/spark,bOOm-X/spark,cloud-fan/spark,pgandhi999/spark,panfengfeng/spark,jlopezmalla/spark,milliman/spark,taroplus/spark,adrian-ionescu/apache-spark,publicRoman/spark,shubhamchopra/spark,apache/spark,byakuinss/spark,tejasapatil/spark,skonto/spark,liutang123/spark,darionyaphet/spark,spark-test/spark,big-pegasus/spark,goldmedal/spark,dbtsai/spark,saltstar/spark,LantaoJin/spark,debugger87/spark,actuaryzhang/spark,milliman/spark,jiangxb1987/spark,jiangxb1987/spark,brad-kaiser/spark,BryanCutler/spark,bOOm-X/spark,rednaxelafx/apache-spark,bOOm-X/spark,highfei2011/spark,panfengfeng/spark,aokolnychyi/spark,witgo/spark,rekhajoshm/spark,sachintyagi22/spark,jrshust/spark,stanzhai/spark,saltstar/spark,gioenn/xSpark,skonto/spark,holdenk/spark,sureshthalamati/spark,JerryLead/spark,poffuomo/spark,facaiy/spark,kiszk/spark,huang1900/spark,dbtsai/spark,wzhfy/spark,gengliangwang/spark,map222/spark,aokolnychyi/spark,skonto/spark,icexelloss/spark,wangmiao1981/spark,holdenk/spark,ahnqirage/spark,apache/spark,patrick-nicholson/spark,lxsmnv/spark,yanboliang/spark,actuaryzhang/spark,aosagie/spark,xuanyuanking/spark,byakuinss/spark,aosagie/spark,taroplus/spark,SnappyDataInc/spark,maropu/spark,LantaoJin/spark,kevinyu98/spark,Panos-Bletsos/spark-cost-model-optimizer,mike0sv/spark,setjet/spark,tengpeng/spark,big-pegasus/spark,actuaryzhang/spark,lxsmnv/spark,icexelloss/spark,dhruve/spark,mdespriee/spark,esi-mineset/spark,hhbyyh/spark,JerryLead/spark,jlopezmalla/spark,michalsenkyr/spark,zero323/spark,rezasafi/spark,zzcclp/spark,joseph-torres/spark,ericvandenbergfb/spark,gioenn/xSpark,maropu/spark,Aegeaner/spark,wangmiao1981/spark,esi-mineset/spark,setjet/spark,sachintyagi22/spark,shubhamchopra/spark,byakuinss/spark,apache-spark-on-k8s/spark,shaneknapp/spark,saltstar/spark,Aegeaner/spark,tejasapatil/spark,hhbyyh/spark,zhouyejoe/spark,andrewor14/spark,setjet/spark,aokolnychyi/spark,tengpeng/spark,SHASHANKB/spark,mahak/spark,ueshin/apache-spark,poffuomo/spark,alunarbeach/spark,sureshthalamati/spark,milliman/spark,metamx/spark,tejasapatil/spark,jlopezmalla/spark,Panos-Bletsos/spark-cost-model-optimizer,WeichenXu123/spark,nlalevee/spark,vinodkc/spark,akopich/spark,SnappyDataInc/spark,andrewor14/spark,caneGuy/spark,mahak/spark,jlopezmalla/spark,map222/spark,guoxiaolongzte/spark,markhamstra/spark,yanboliang/spark,jrshust/spark,tejasapatil/spark,aray/spark,mike0sv/spark,WeichenXu123/spark,ibm-research-ireland/sparkoscope,holdenk/spark,narahari92/spark,ibm-research-ireland/sparkoscope,wzhfy/spark,liutang123/spark,maropu/spark,jkbradley/spark,nlalevee/spark,alunarbeach/spark,1haodian/spark,cin/spark,nlalevee/spark,icexelloss/spark,hhbyyh/spark,bravo-zhang/spark,mdespriee/spark,apache/spark,dongjoon-hyun/spark,adrian-ionescu/apache-spark,SnappyDataInc/spark,SHASHANKB/spark,liyichao/spark,zhouyejoe/spark,darionyaphet/spark,ueshin/apache-spark,sachintyagi22/spark,jrshust/spark,skonto/spark,publicRoman/spark,michalsenkyr/spark,jkbradley/spark,holdenk/spark,milliman/spark,rikima/spark,yanboliang/spark,srowen/spark,BryanCutler/spark,1haodian/spark,WindCanDie/spark,ioana-delaney/spark,HyukjinKwon/spark,hhbyyh/spark,saturday-shi/spark,kissmetrics/spark,yanboliang/spark,ajaysaini725/spark,techaddict/spark,map222/spark,matthewfranglen/spark,poffuomo/spark,actuaryzhang/spark,LantaoJin/spark,cin/spark,brad-kaiser/spark,icexelloss/spark,akopich/spark,ddna1021/spark,stanzhai/spark,rikima/spark,facaiy/spark,apache-spark-on-k8s/spark,caneGuy/spark,eyalfa/spark,zuotingbing/spark,milliman/spark,guoxiaolongzte/spark,vinodkc/spark,sahilTakiar/spark,icexelloss/spark,apache-spark-on-k8s/spark,aray/spark,rekhajoshm/spark,spark-test/spark,sahilTakiar/spark,ConeyLiu/spark,maropu/spark,dhruve/spark,gioenn/xSpark,lxsmnv/spark,taroplus/spark,liyichao/spark,bravo-zhang/spark,dhruve/spark,aokolnychyi/spark,shaneknapp/spark,bOOm-X/spark,loneknightpy/spark,spark-test/spark,saturday-shi/spark,saltstar/spark,zuotingbing/spark,srowen/spark,ajaysaini725/spark,lvdongr/spark,aokolnychyi/spark,wangmiao1981/spark,nilsgrabbert/spark,bravo-zhang/spark,BryanCutler/spark,jkbradley/spark,SHASHANKB/spark,someorz/spark,BryanCutler/spark,jkbradley/spark,stanzhai/spark,UndeadBaneGitHub/spark,patrick-nicholson/spark,wangmiao1981/spark,zhouyejoe/spark,big-pegasus/spark,aokolnychyi/spark,map222/spark,wzhfy/spark,kissmetrics/spark,someorz/spark,darionyaphet/spark,srowen/spark,brad-kaiser/spark,rezasafi/spark,nchammas/spark,sahilTakiar/spark,hvanhovell/spark,metamx/spark,goldmedal/spark,JerryLead/spark,ahnqirage/spark,zero323/spark,jrshust/spark,michalsenkyr/spark,sureshthalamati/spark,nlalevee/spark,hvanhovell/spark,yanboliang/spark,stanzhai/spark,vinodkc/spark,spark-test/spark,lvdongr/spark,aray/spark,Panos-Bletsos/spark-cost-model-optimizer,jrshust/spark,mahak/spark,nlalevee/spark,goldmedal/spark,poffuomo/spark,JerryLead/spark,WindCanDie/spark,bravo-zhang/spark,kevinyu98/spark,kevinyu98/spark,icexelloss/spark,witgo/spark,hhbyyh/spark,bOOm-X/spark,spark-test/spark,minixalpha/spark,apache/spark,ahnqirage/spark,ajaysaini725/spark,mdespriee/spark,tejasapatil/spark,jlopezmalla/spark,bOOm-X/spark,jlopezmalla/spark,janewangfb/spark,nchammas/spark,kissmetrics/spark,publicRoman/spark,mzl9039/spark,1haodian/spark,janewangfb/spark,map222/spark,ibm-research-ireland/sparkoscope,eyalfa/spark,ddna1021/spark,sachintyagi22/spark,janewangfb/spark,rednaxelafx/apache-spark,debugger87/spark,maropu/spark,aosagie/spark,pgandhi999/spark,mzl9039/spark,mzl9039/spark,kiszk/spark,jrshust/spark,MLnick/spark,hvanhovell/spark,zuotingbing/spark,mahak/spark,alunarbeach/spark,ptkool/spark,facaiy/spark,guoxiaolongzte/spark,mzl9039/spark,zzcclp/spark,cin/spark,ioana-delaney/spark,poffuomo/spark,lxsmnv/spark,facaiy/spark,vinodkc/spark,kissmetrics/spark,adrian-ionescu/apache-spark,goldmedal/spark,joseph-torres/spark,markhamstra/spark,ibm-research-ireland/sparkoscope,alunarbeach/spark,ueshin/apache-spark,witgo/spark,caneGuy/spark,rednaxelafx/apache-spark,HyukjinKwon/spark,liutang123/spark,szhem/spark,alunarbeach/spark,wangyum/spark,zzcclp/spark,someorz/spark,shaneknapp/spark,kiszk/spark,ron8hu/spark,ConeyLiu/spark,holdenk/spark,shubhamchopra/spark,esi-mineset/spark,witgo/spark,ioana-delaney/spark,eyalfa/spark,apache/spark,big-pegasus/spark,shubhamchopra/spark,narahari92/spark,UndeadBaneGitHub/spark,mdespriee/spark,dhruve/spark,MLnick/spark,cloud-fan/spark,chuckchen/spark,mike0sv/spark,dbtsai/spark,ddna1021/spark,saturday-shi/spark,panfengfeng/spark,LantaoJin/spark,techaddict/spark,wangyum/spark,dongjoon-hyun/spark,bravo-zhang/spark,tejasapatil/spark,shuangshuangwang/spark,narahari92/spark,srowen/spark,lvdongr/spark,LantaoJin/spark,aosagie/spark,ConeyLiu/spark,tengpeng/spark,ericvandenbergfb/spark,minixalpha/spark,publicRoman/spark,BryanCutler/spark,szhem/spark,xuanyuanking/spark,kevinyu98/spark,zzcclp/spark,jiangxb1987/spark,SnappyDataInc/spark,JoshRosen/spark,nlalevee/spark,saturday-shi/spark,akopich/spark,zhouyejoe/spark,aosagie/spark,ron8hu/spark,wzhfy/spark,yanboliang/spark,map222/spark,dbtsai/spark,nilsgrabbert/spark,adrian-ionescu/apache-spark,panfengfeng/spark,ericvandenbergfb/spark,JoshRosen/spark,publicRoman/spark,nchammas/spark,xflin/spark,andrewor14/spark,highfei2011/spark,guoxiaolongzte/spark,michalsenkyr/spark,rikima/spark,witgo/spark,wzhfy/spark,JoshRosen/spark,zzcclp/spark,metamx/spark,alunarbeach/spark,liyichao/spark,ueshin/apache-spark,debugger87/spark,caneGuy/spark,mahak/spark,jianran/spark,shubhamchopra/spark,bravo-zhang/spark,darionyaphet/spark,andrewor14/spark,SnappyDataInc/spark,dongjoon-hyun/spark,rezasafi/spark,pgandhi999/spark,Panos-Bletsos/spark-cost-model-optimizer,jiangxb1987/spark,alunarbeach/spark,esi-mineset/spark,markhamstra/spark,ConeyLiu/spark,ericvandenbergfb/spark,goldmedal/spark,rezasafi/spark,aosagie/spark,HyukjinKwon/spark,mike0sv/spark,cloud-fan/spark,caneGuy/spark,kimoonkim/spark,ioana-delaney/spark,WeichenXu123/spark,1haodian/spark,bdrillard/spark,Aegeaner/spark,jkbradley/spark,aray/spark,rednaxelafx/apache-spark,publicRoman/spark,byakuinss/spark,shuangshuangwang/spark,esi-mineset/spark,gengliangwang/spark,kiszk/spark,sahilTakiar/spark,taroplus/spark,WeichenXu123/spark,JerryLead/spark,rednaxelafx/apache-spark,kimoonkim/spark,kimoonkim/spark,Aegeaner/spark,liyichao/spark,techaddict/spark,mzl9039/spark,panfengfeng/spark,Panos-Bletsos/spark-cost-model-optimizer,tengpeng/spark,darionyaphet/spark,michalsenkyr/spark,setjet/spark,ajaysaini725/spark,bOOm-X/spark,cloud-fan/spark,map222/spark,ibm-research-ireland/sparkoscope,someorz/spark,mzl9039/spark,dbtsai/spark,Panos-Bletsos/spark-cost-model-optimizer,1haodian/spark,Panos-Bletsos/spark-cost-model-optimizer,kevinyu98/spark,jianran/spark,highfei2011/spark,highfei2011/spark,rednaxelafx/apache-spark,gioenn/xSpark,hvanhovell/spark,ioana-delaney/spark,mike0sv/spark,spark-test/spark,ajaysaini725/spark,szhem/spark,hhbyyh/spark,SHASHANKB/spark,rekhajoshm/spark,cin/spark,liutang123/spark,dhruve/spark,liutang123/spark,sureshthalamati/spark,gengliangwang/spark,brad-kaiser/spark,minixalpha/spark,andrewor14/spark,markhamstra/spark,nilsgrabbert/spark,mzl9039/spark,dongjoon-hyun/spark,nchammas/spark,gioenn/xSpark,szhem/spark,lvdongr/spark,zuotingbing/spark,sureshthalamati/spark,wangmiao1981/spark,guoxiaolongzte/spark,shuangshuangwang/spark,bdrillard/spark,xflin/spark,WindCanDie/spark,sureshthalamati/spark,pgandhi999/spark,esi-mineset/spark,setjet/spark,spark-test/spark,goldmedal/spark,JerryLead/spark,huang1900/spark,debugger87/spark,zero323/spark,tejasapatil/spark,kiszk/spark,darionyaphet/spark,patrick-nicholson/spark,minixalpha/spark,bdrillard/spark,mahak/spark,ahnqirage/spark,xuanyuanking/spark,taroplus/spark,patrick-nicholson/spark,mike0sv/spark,apache/spark,jiangxb1987/spark,wzhfy/spark,taroplus/spark,wangyum/spark,patrick-nicholson/spark,narahari92/spark,JerryLead/spark,jianran/spark,rezasafi/spark,vinodkc/spark,ueshin/apache-spark,markhamstra/spark,SHASHANKB/spark,techaddict/spark,aray/spark,shaneknapp/spark,metamx/spark,gengliangwang/spark,eyalfa/spark,hvanhovell/spark,huang1900/spark,aosagie/spark,srowen/spark,wangyum/spark,michalsenkyr/spark,zhouyejoe/spark,someorz/spark,skonto/spark,Aegeaner/spark,WeichenXu123/spark,Aegeaner/spark,ahnqirage/spark,adrian-ionescu/apache-spark,poffuomo/spark,kissmetrics/spark,nilsgrabbert/spark,gengliangwang/spark,nlalevee/spark,WindCanDie/spark,wangmiao1981/spark,guoxiaolongzte/spark,xuanyuanking/spark,ron8hu/spark,pgandhi999/spark,sachintyagi22/spark,cin/spark,mahak/spark,shaneknapp/spark,LantaoJin/spark,stanzhai/spark,joseph-torres/spark,ericvandenbergfb/spark,chuckchen/spark,huang1900/spark,kiszk/spark,jiangxb1987/spark,facaiy/spark,jianran/spark,zhouyejoe/spark,zero323/spark,HyukjinKwon/spark,szhem/spark,sahilTakiar/spark,joseph-torres/spark,highfei2011/spark,nilsgrabbert/spark,ericvandenbergfb/spark,loneknightpy/spark,cin/spark,loneknightpy/spark,wangmiao1981/spark,sachintyagi22/spark,saltstar/spark,nchammas/spark,ioana-delaney/spark,SHASHANKB/spark,ConeyLiu/spark,rekhajoshm/spark,xuanyuanking/spark,metamx/spark,apache-spark-on-k8s/spark,dongjoon-hyun/spark,mdespriee/spark,setjet/spark,debugger87/spark,ptkool/spark,jkbradley/spark,narahari92/spark,kissmetrics/spark,ioana-delaney/spark,zero323/spark,shuangshuangwang/spark,actuaryzhang/spark,nchammas/spark,lvdongr/spark,minixalpha/spark,rikima/spark,akopich/spark,witgo/spark,kimoonkim/spark,chuckchen/spark,esi-mineset/spark,sachintyagi22/spark,xuanyuanking/spark,someorz/spark,narahari92/spark,debugger87/spark,michalsenkyr/spark,kissmetrics/spark,lxsmnv/spark,MLnick/spark,hhbyyh/spark,tengpeng/spark,cloud-fan/spark,1haodian/spark,WeichenXu123/spark,bravo-zhang/spark,minixalpha/spark,bdrillard/spark,ddna1021/spark,kimoonkim/spark,lvdongr/spark,xflin/spark,MLnick/spark,xflin/spark,chuckchen/spark,SnappyDataInc/spark,saturday-shi/spark,rezasafi/spark,akopich/spark,shaneknapp/spark,ahnqirage/spark,rednaxelafx/apache-spark,zhouyejoe/spark,xflin/spark,rekhajoshm/spark,zzcclp/spark,dbtsai/spark,kevinyu98/spark,guoxiaolongzte/spark,huang1900/spark,liutang123/spark,liyichao/spark,patrick-nicholson/spark,bdrillard/spark,jianran/spark,ajaysaini725/spark,janewangfb/spark,lxsmnv/spark,chuckchen/spark,big-pegasus/spark,zuotingbing/spark,maropu/spark,ConeyLiu/spark,brad-kaiser/spark,actuaryzhang/spark,loneknightpy/spark,akopich/spark,chuckchen/spark,rekhajoshm/spark,stanzhai/spark,ericvandenbergfb/spark,WindCanDie/spark,liutang123/spark,panfengfeng/spark,eyalfa/spark,techaddict/spark,patrick-nicholson/spark,loneknightpy/spark,ptkool/spark,brad-kaiser/spark,apache-spark-on-k8s/spark,byakuinss/spark,apache-spark-on-k8s/spark,rikima/spark,cloud-fan/spark,stanzhai/spark,mdespriee/spark,ptkool/spark,jrshust/spark,wangyum/spark,ConeyLiu/spark,ahnqirage/spark,dbtsai/spark,rekhajoshm/spark,jianran/spark,gengliangwang/spark,szhem/spark,loneknightpy/spark,sureshthalamati/spark,skonto/spark,big-pegasus/spark,dhruve/spark,MLnick/spark,aray/spark,sahilTakiar/spark,akopich/spark,eyalfa/spark,taroplus/spark,cloud-fan/spark,dongjoon-hyun/spark,metamx/spark,lxsmnv/spark,ptkool/spark,jiangxb1987/spark,srowen/spark,ron8hu/spark,caneGuy/spark,liyichao/spark,BryanCutler/spark,UndeadBaneGitHub/spark,saturday-shi/spark,tengpeng/spark,darionyaphet/spark,UndeadBaneGitHub/spark,zero323/spark,HyukjinKwon/spark,pgandhi999/spark,highfei2011/spark,jkbradley/spark,joseph-torres/spark,HyukjinKwon/spark,skonto/spark,actuaryzhang/spark,huang1900/spark,markhamstra/spark,xflin/spark,HyukjinKwon/spark,gioenn/xSpark,JoshRosen/spark,eyalfa/spark,1haodian/spark,ptkool/spark,kiszk/spark,mdespriee/spark,debugger87/spark,JoshRosen/spark,nilsgrabbert/spark,ibm-research-ireland/sparkoscope,SHASHANKB/spark,dongjoon-hyun/spark,adrian-ionescu/apache-spark,ddna1021/spark,mike0sv/spark,zero323/spark,hvanhovell/spark,bdrillard/spark,ron8hu/spark,big-pegasus/spark,ibm-research-ireland/sparkoscope,ueshin/apache-spark,xflin/spark,ddna1021/spark,kimoonkim/spark,saltstar/spark,kimoonkim/spark,UndeadBaneGitHub/spark,narahari92/spark,ueshin/apache-spark,someorz/spark,shuangshuangwang/spark,adrian-ionescu/apache-spark,UndeadBaneGitHub/spark,chuckchen/spark,bdrillard/spark,loneknightpy/spark,saturday-shi/spark,hvanhovell/spark,pgandhi999/spark,andrewor14/spark,LantaoJin/spark,szhem/spark,techaddict/spark,cin/spark,joseph-torres/spark,JoshRosen/spark,janewangfb/spark,xuanyuanking/spark,minixalpha/spark,shuangshuangwang/spark,srowen/spark,joseph-torres/spark,apache/spark,milliman/spark,caneGuy/spark,apache-spark-on-k8s/spark,saltstar/spark,SnappyDataInc/spark,brad-kaiser/spark,janewangfb/spark,Aegeaner/spark,rikima/spark,zuotingbing/spark,goldmedal/spark,zuotingbing/spark,shubhamchopra/spark,jianran/spark,WindCanDie/spark,yanboliang/spark,andrewor14/spark,publicRoman/spark,MLnick/spark