misleading task number of groupByKey
misleading task number of groupByKey

"By default, this uses only 8 parallel tasks to do the grouping." is a big misleading. Please refer to https://github.com/apache/spark/pull/389

detail is as following code :

  def defaultPartitioner(rdd: RDD[_], others: RDD[_]*): Partitioner = {
    val bySize = (Seq(rdd) ++ others).sortBy(_.partitions.size).reverse
    for (r <- bySize if r.partitioner.isDefined) {
      return r.partitioner.get
    }
    if (rdd.context.conf.contains("spark.default.parallelism")) {
      new HashPartitioner(rdd.context.defaultParallelism)
    } else {
      new HashPartitioner(bySize.head.partitions.size)
    }
  }

Author: Chen Chao <crazyjvm@gmail.com>

Closes #403 from CrazyJvm/patch-4 and squashes the following commits:

42f6c9e [Chen Chao] fix format
829a995 [Chen Chao] fix format
1568336 [Chen Chao] misleading task number of groupByKey

setjet/spark,facaiy/spark,rezasafi/spark,HyukjinKwon/spark,aokolnychyi/spark,WeichenXu123/spark,zero323/spark,shubhamchopra/spark,aray/spark,jkbradley/spark,janewangfb/spark,ioana-delaney/spark,witgo/spark,dongjoon-hyun/spark,jlopezmalla/spark,ahnqirage/spark,debugger87/spark,nilsgrabbert/spark,sureshthalamati/spark,gioenn/xSpark,chuckchen/spark,jkbradley/spark,szhem/spark,pgandhi999/spark,markhamstra/spark,joseph-torres/spark,huang1900/spark,janewangfb/spark,LantaoJin/spark,Aegeaner/spark,debugger87/spark,ueshin/apache-spark,nchammas/spark,setjet/spark,vinodkc/spark,yanboliang/spark,lxsmnv/spark,huang1900/spark,esi-mineset/spark,bravo-zhang/spark,maropu/spark,ahnqirage/spark,metamx/spark,dhruve/spark,setjet/spark,rednaxelafx/apache-spark,cin/spark,ptkool/spark,ericvandenbergfb/spark,HyukjinKwon/spark,taroplus/spark,andrewor14/iolap,yanboliang/spark,someorz/spark,goldmedal/spark,spark-test/spark,sachintyagi22/spark,hhbyyh/spark,mahak/spark,kissmetrics/spark,icexelloss/spark,spark-test/spark,alunarbeach/spark,jiangxb1987/spark,jlopezmalla/spark,UndeadBaneGitHub/spark,guoxiaolongzte/spark,1haodian/spark,UndeadBaneGitHub/spark,spark-test/spark,nlalevee/spark,poffuomo/spark,WeichenXu123/spark,map222/spark,kevinyu98/spark,BryanCutler/spark,eyalfa/spark,tejasapatil/spark,loneknightpy/spark,cin/spark,joseph-torres/spark,adrian-ionescu/apache-spark,shaneknapp/spark,kimoonkim/spark,loneknightpy/spark,apache/spark,zhouyejoe/spark,minixalpha/spark,Aegeaner/spark,narahari92/spark,holdenk/spark,xuanyuanking/spark,ptkool/spark,kiszk/spark,aosagie/spark,facaiy/spark,wzhfy/spark,cloud-fan/spark,goldmedal/spark,taroplus/spark,SnappyDataInc/spark,szhem/spark,michalsenkyr/spark,brad-kaiser/spark,metamx/spark,spark-test/spark,hhbyyh/spark,bravo-zhang/spark,UndeadBaneGitHub/spark,jiangxb1987/spark,maropu/spark,lvdongr/spark,patrick-nicholson/spark,ericvandenbergfb/spark,1haodian/spark,mike0sv/spark,LantaoJin/spark,loneknightpy/spark,ibm-research-ireland/sparkoscope,lvdongr/spark,minixalpha/spark,andrewor14/iolap,mzl9039/spark,ddna1021/spark,matthewfranglen/spark,zzcclp/spark,ericvandenbergfb/spark,stanzhai/spark,SHASHANKB/spark,dongjoon-hyun/spark,facaiy/spark,skonto/spark,cin/spark,ioana-delaney/spark,gioenn/xSpark,ioana-delaney/spark,shubhamchopra/spark,WeichenXu123/spark,saltstar/spark,publicRoman/spark,ajaysaini725/spark,jrshust/spark,WindCanDie/spark,michalsenkyr/spark,sachintyagi22/spark,publicRoman/spark,cloud-fan/spark,saturday-shi/spark,byakuinss/spark,ibm-research-ireland/sparkoscope,kissmetrics/spark,lvdongr/spark,caneGuy/spark,1haodian/spark,UndeadBaneGitHub/spark,kissmetrics/spark,joseph-torres/spark,mahak/spark,dhruve/spark,mdespriee/spark,ueshin/apache-spark,lxsmnv/spark,MLnick/spark,zero323/spark,dbtsai/spark,gioenn/xSpark,eyalfa/spark,panfengfeng/spark,nchammas/spark,aokolnychyi/spark,mdespriee/spark,tejasapatil/spark,dhruve/spark,jianran/spark,mdespriee/spark,tengpeng/spark,shaneknapp/spark,cin/spark,sahilTakiar/spark,jiangxb1987/spark,wangyum/spark,holdenk/spark,zzcclp/spark,bOOm-X/spark,gioenn/xSpark,mike0sv/spark,ueshin/apache-spark,JerryLead/spark,akopich/spark,bravo-zhang/spark,skonto/spark,techaddict/spark,tejasapatil/spark,apache/spark,ioana-delaney/spark,UndeadBaneGitHub/spark,zuotingbing/spark,stanzhai/spark,facaiy/spark,liutang123/spark,michalsenkyr/spark,zero323/spark,publicRoman/spark,actuaryzhang/spark,liyichao/spark,zzcclp/spark,aokolnychyi/spark,srowen/spark,witgo/spark,highfei2011/spark,big-pegasus/spark,wzhfy/spark,highfei2011/spark,brad-kaiser/spark,byakuinss/spark,BryanCutler/spark,michalsenkyr/spark,lvdongr/spark,aray/spark,ajaysaini725/spark,ibm-research-ireland/sparkoscope,aray/spark,JerryLead/spark,markhamstra/spark,mzl9039/spark,rezasafi/spark,narahari92/spark,techaddict/spark,zero323/spark,sahilTakiar/spark,srowen/spark,hhbyyh/spark,shaneknapp/spark,metamx/spark,byakuinss/spark,michalsenkyr/spark,gengliangwang/spark,adrian-ionescu/apache-spark,SnappyDataInc/spark,sahilTakiar/spark,highfei2011/spark,ron8hu/spark,saturday-shi/spark,jianran/spark,wangmiao1981/spark,wangmiao1981/spark,map222/spark,markhamstra/spark,vinodkc/spark,wangmiao1981/spark,kevinyu98/spark,JoshRosen/spark,brad-kaiser/spark,gengliangwang/spark,jiangxb1987/spark,caneGuy/spark,wangyum/spark,mahak/spark,zero323/spark,huang1900/spark,rikima/spark,alunarbeach/spark,apache/spark,saltstar/spark,yanboliang/spark,hvanhovell/spark,andrewor14/spark,apache-spark-on-k8s/spark,lxsmnv/spark,BryanCutler/spark,ioana-delaney/spark,shubhamchopra/spark,rednaxelafx/apache-spark,zhouyejoe/spark,icexelloss/spark,milliman/spark,zhouyejoe/spark,vinodkc/spark,kiszk/spark,ahnqirage/spark,aosagie/spark,icexelloss/spark,ddna1021/spark,zhouyejoe/spark,patrick-nicholson/spark,dongjoon-hyun/spark,saturday-shi/spark,WeichenXu123/spark,narahari92/spark,LantaoJin/spark,liyichao/spark,markhamstra/spark,WeichenXu123/spark,SHASHANKB/spark,vinodkc/spark,big-pegasus/spark,skonto/spark,jianran/spark,saltstar/spark,BryanCutler/spark,LantaoJin/spark,kimoonkim/spark,JerryLead/spark,poffuomo/spark,ahnqirage/spark,jrshust/spark,kimoonkim/spark,shuangshuangwang/spark,nlalevee/spark,cloud-fan/spark,andrewor14/spark,gengliangwang/spark,dhruve/spark,JoshRosen/spark,lvdongr/spark,gengliangwang/spark,esi-mineset/spark,huang1900/spark,facaiy/spark,jrshust/spark,mike0sv/spark,lxsmnv/spark,liutang123/spark,mike0sv/spark,loneknightpy/spark,sureshthalamati/spark,rikima/spark,kiszk/spark,goldmedal/spark,Panos-Bletsos/spark-cost-model-optimizer,brad-kaiser/spark,milliman/spark,wangyum/spark,nilsgrabbert/spark,poffuomo/spark,cin/spark,hhbyyh/spark,darionyaphet/spark,narahari92/spark,zzcclp/spark,brad-kaiser/spark,ron8hu/spark,liutang123/spark,someorz/spark,ueshin/apache-spark,jrshust/spark,eyalfa/spark,shaneknapp/spark,publicRoman/spark,gengliangwang/spark,Panos-Bletsos/spark-cost-model-optimizer,ptkool/spark,bravo-zhang/spark,aosagie/spark,yanboliang/spark,apache/spark,nchammas/spark,JerryLead/spark,rekhajoshm/spark,aray/spark,Panos-Bletsos/spark-cost-model-optimizer,pgandhi999/spark,kimoonkim/spark,techaddict/spark,ibm-research-ireland/sparkoscope,shaneknapp/spark,tengpeng/spark,haowu80s/spark,someorz/spark,zuotingbing/spark,shuangshuangwang/spark,haowu80s/spark,pronix/spark,tengpeng/spark,HyukjinKwon/spark,sachintyagi22/spark,publicRoman/spark,kevinyu98/spark,mahak/spark,jkbradley/spark,SnappyDataInc/spark,dongjoon-hyun/spark,patrick-nicholson/spark,holdenk/spark,hhbyyh/spark,vinodkc/spark,saturday-shi/spark,maropu/spark,debugger87/spark,MLnick/spark,aosagie/spark,srowen/spark,darionyaphet/spark,SnappyDataInc/spark,WindCanDie/spark,adrian-ionescu/apache-spark,bOOm-X/spark,ddna1021/spark,goldmedal/spark,bdrillard/spark,wangyum/spark,zuotingbing/spark,big-pegasus/spark,tejasapatil/spark,huang1900/spark,icexelloss/spark,dbtsai/spark,joseph-torres/spark,ron8hu/spark,wangyum/spark,kevinyu98/spark,map222/spark,srowen/spark,mzl9039/spark,Panos-Bletsos/spark-cost-model-optimizer,saturday-shi/spark,ahnqirage/spark,witgo/spark,alunarbeach/spark,narahari92/spark,shubhamchopra/spark,vinodkc/spark,jianran/spark,hhbyyh/spark,cloud-fan/spark,nlalevee/spark,MLnick/spark,map222/spark,xuanyuanking/spark,shuangshuangwang/spark,debugger87/spark,rekhajoshm/spark,xflin/spark,brad-kaiser/spark,spark-test/spark,byakuinss/spark,panfengfeng/spark,haowu80s/spark,jiangxb1987/spark,andrewor14/spark,ron8hu/spark,kissmetrics/spark,jianran/spark,srowen/spark,ron8hu/spark,SHASHANKB/spark,akopich/spark,hvanhovell/spark,1haodian/spark,kissmetrics/spark,mdespriee/spark,bdrillard/spark,jianran/spark,ericvandenbergfb/spark,sahilTakiar/spark,rikima/spark,Panos-Bletsos/spark-cost-model-optimizer,aray/spark,liutang123/spark,kiszk/spark,guoxiaolongzte/spark,witgo/spark,zhouyejoe/spark,pronix/spark,loneknightpy/spark,sureshthalamati/spark,sachintyagi22/spark,rezasafi/spark,liyichao/spark,akopich/spark,tengpeng/spark,dhruve/spark,hvanhovell/spark,rekhajoshm/spark,caneGuy/spark,lxsmnv/spark,mzl9039/spark,taroplus/spark,mike0sv/spark,panfengfeng/spark,ericvandenbergfb/spark,SHASHANKB/spark,jrshust/spark,JoshRosen/spark,szhem/spark,sachintyagi22/spark,actuaryzhang/spark,wangmiao1981/spark,ddna1021/spark,haowu80s/spark,ptkool/spark,andrewor14/iolap,andrewor14/spark,rekhajoshm/spark,ioana-delaney/spark,JoshRosen/spark,dongjoon-hyun/spark,ioana-delaney/spark,sahilTakiar/spark,SHASHANKB/spark,apache/spark,pgandhi999/spark,xuanyuanking/spark,wzhfy/spark,kevinyu98/spark,map222/spark,szhem/spark,nilsgrabbert/spark,shuangshuangwang/spark,bdrillard/spark,pgandhi999/spark,skonto/spark,alunarbeach/spark,setjet/spark,mzl9039/spark,panfengfeng/spark,zzcclp/spark,guoxiaolongzte/spark,maropu/spark,bravo-zhang/spark,joseph-torres/spark,MLnick/spark,panfengfeng/spark,caneGuy/spark,ConeyLiu/spark,pronix/spark,maropu/spark,holdenk/spark,metamx/spark,liutang123/spark,rezasafi/spark,xuanyuanking/spark,dbtsai/spark,zzcclp/spark,rednaxelafx/apache-spark,stanzhai/spark,wangmiao1981/spark,xflin/spark,highfei2011/spark,actuaryzhang/spark,WeichenXu123/spark,ibm-research-ireland/sparkoscope,skonto/spark,skonto/spark,apache/spark,rezasafi/spark,dongjoon-hyun/spark,apache-spark-on-k8s/spark,hvanhovell/spark,wzhfy/spark,adrian-ionescu/apache-spark,holdenk/spark,chuckchen/spark,minixalpha/spark,ibm-research-ireland/sparkoscope,eyalfa/spark,icexelloss/spark,jlopezmalla/spark,witgo/spark,srowen/spark,liyichao/spark,actuaryzhang/spark,szhem/spark,jkbradley/spark,wangmiao1981/spark,guoxiaolongzte/spark,dbtsai/spark,esi-mineset/spark,ajaysaini725/spark,rezasafi/spark,bOOm-X/spark,nchammas/spark,adrian-ionescu/apache-spark,maropu/spark,mzl9039/spark,chuckchen/spark,zuotingbing/spark,dhruve/spark,LantaoJin/spark,LantaoJin/spark,jlopezmalla/spark,nchammas/spark,WindCanDie/spark,rezasafi/spark,zuotingbing/spark,rednaxelafx/apache-spark,MLnick/spark,jkbradley/spark,hhbyyh/spark,techaddict/spark,chuckchen/spark,sureshthalamati/spark,MLnick/spark,actuaryzhang/spark,liyichao/spark,liutang123/spark,chuckchen/spark,rikima/spark,ericvandenbergfb/spark,big-pegasus/spark,jkbradley/spark,tengpeng/spark,BryanCutler/spark,setjet/spark,apache/spark,MLnick/spark,hvanhovell/spark,aray/spark,ptkool/spark,michalsenkyr/spark,ericvandenbergfb/spark,shuangshuangwang/spark,JoshRosen/spark,apache-spark-on-k8s/spark,ddna1021/spark,Aegeaner/spark,ConeyLiu/spark,jlopezmalla/spark,mike0sv/spark,patrick-nicholson/spark,SnappyDataInc/spark,wangmiao1981/spark,mahak/spark,poffuomo/spark,sachintyagi22/spark,andrewor14/spark,goldmedal/spark,aray/spark,markhamstra/spark,minixalpha/spark,stanzhai/spark,publicRoman/spark,ahnqirage/spark,chuckchen/spark,stanzhai/spark,ptkool/spark,metamx/spark,lxsmnv/spark,kevinyu98/spark,ConeyLiu/spark,milliman/spark,caneGuy/spark,jianran/spark,mzl9039/spark,someorz/spark,big-pegasus/spark,apache-spark-on-k8s/spark,pronix/spark,spark-test/spark,zero323/spark,saturday-shi/spark,lxsmnv/spark,ConeyLiu/spark,sureshthalamati/spark,chuckchen/spark,bdrillard/spark,rekhajoshm/spark,hvanhovell/spark,WindCanDie/spark,nchammas/spark,pronix/spark,jrshust/spark,aokolnychyi/spark,caneGuy/spark,xflin/spark,eyalfa/spark,nilsgrabbert/spark,milliman/spark,szhem/spark,kimoonkim/spark,rikima/spark,rikima/spark,jlopezmalla/spark,metamx/spark,wzhfy/spark,ajaysaini725/spark,janewangfb/spark,xuanyuanking/spark,nilsgrabbert/spark,dhruve/spark,SHASHANKB/spark,apache-spark-on-k8s/spark,michalsenkyr/spark,bravo-zhang/spark,ueshin/apache-spark,zuotingbing/spark,goldmedal/spark,apache-spark-on-k8s/spark,bOOm-X/spark,jiangxb1987/spark,WeichenXu123/spark,icexelloss/spark,gengliangwang/spark,ajaysaini725/spark,techaddict/spark,akopich/spark,tengpeng/spark,sahilTakiar/spark,mahak/spark,big-pegasus/spark,shubhamchopra/spark,highfei2011/spark,JerryLead/spark,bravo-zhang/spark,dongjoon-hyun/spark,rikima/spark,andrewor14/iolap,someorz/spark,markhamstra/spark,yanboliang/spark,andrewor14/spark,liyichao/spark,Aegeaner/spark,map222/spark,WindCanDie/spark,patrick-nicholson/spark,taroplus/spark,1haodian/spark,bdrillard/spark,rednaxelafx/apache-spark,UndeadBaneGitHub/spark,kimoonkim/spark,Panos-Bletsos/spark-cost-model-optimizer,guoxiaolongzte/spark,lvdongr/spark,bOOm-X/spark,setjet/spark,rekhajoshm/spark,esi-mineset/spark,ConeyLiu/spark,big-pegasus/spark,goldmedal/spark,ajaysaini725/spark,xuanyuanking/spark,alunarbeach/spark,BryanCutler/spark,byakuinss/spark,ddna1021/spark,jlopezmalla/spark,taroplus/spark,janewangfb/spark,narahari92/spark,cloud-fan/spark,cin/spark,stanzhai/spark,yanboliang/spark,actuaryzhang/spark,jrshust/spark,darionyaphet/spark,cin/spark,map222/spark,rednaxelafx/apache-spark,someorz/spark,publicRoman/spark,minixalpha/spark,kevinyu98/spark,aosagie/spark,aokolnychyi/spark,highfei2011/spark,hvanhovell/spark,caneGuy/spark,pronix/spark,Aegeaner/spark,andrewor14/spark,shaneknapp/spark,maropu/spark,aosagie/spark,JoshRosen/spark,darionyaphet/spark,sachintyagi22/spark,huang1900/spark,eyalfa/spark,tejasapatil/spark,andrewor14/iolap,wzhfy/spark,adrian-ionescu/apache-spark,nlalevee/spark,dbtsai/spark,shaneknapp/spark,SnappyDataInc/spark,kissmetrics/spark,dbtsai/spark,zhouyejoe/spark,milliman/spark,mdespriee/spark,dbtsai/spark,1haodian/spark,joseph-torres/spark,poffuomo/spark,Aegeaner/spark,liyichao/spark,nilsgrabbert/spark,xflin/spark,taroplus/spark,ueshin/apache-spark,aokolnychyi/spark,eyalfa/spark,ptkool/spark,ajaysaini725/spark,brad-kaiser/spark,wzhfy/spark,HyukjinKwon/spark,narahari92/spark,aosagie/spark,Panos-Bletsos/spark-cost-model-optimizer,xflin/spark,someorz/spark,HyukjinKwon/spark,poffuomo/spark,mdespriee/spark,SnappyDataInc/spark,taroplus/spark,zhouyejoe/spark,haowu80s/spark,akopich/spark,milliman/spark,holdenk/spark,WindCanDie/spark,saltstar/spark,aokolnychyi/spark,pgandhi999/spark,srowen/spark,shuangshuangwang/spark,guoxiaolongzte/spark,darionyaphet/spark,sahilTakiar/spark,1haodian/spark,setjet/spark,nchammas/spark,bOOm-X/spark,patrick-nicholson/spark,gioenn/xSpark,nilsgrabbert/spark,haowu80s/spark,darionyaphet/spark,janewangfb/spark,mahak/spark,zuotingbing/spark,HyukjinKwon/spark,JoshRosen/spark,vinodkc/spark,zero323/spark,tejasapatil/spark,kissmetrics/spark,cloud-fan/spark,byakuinss/spark,apache-spark-on-k8s/spark,andrewor14/iolap,witgo/spark,metamx/spark,stanzhai/spark,gengliangwang/spark,debugger87/spark,JerryLead/spark,minixalpha/spark,debugger87/spark,byakuinss/spark,wangyum/spark,kiszk/spark,gioenn/xSpark,cloud-fan/spark,mdespriee/spark,tengpeng/spark,andrewor14/iolap,ConeyLiu/spark,bdrillard/spark,saturday-shi/spark,saltstar/spark,Aegeaner/spark,rednaxelafx/apache-spark,UndeadBaneGitHub/spark,milliman/spark,JerryLead/spark,BryanCutler/spark,sureshthalamati/spark,ron8hu/spark,bdrillard/spark,actuaryzhang/spark,ahnqirage/spark,techaddict/spark,panfengfeng/spark,lvdongr/spark,gioenn/xSpark,alunarbeach/spark,joseph-torres/spark,poffuomo/spark,SHASHANKB/spark,saltstar/spark,akopich/spark,loneknightpy/spark,yanboliang/spark,HyukjinKwon/spark,alunarbeach/spark,ueshin/apache-spark,darionyaphet/spark,jiangxb1987/spark,highfei2011/spark,esi-mineset/spark,ddna1021/spark,saltstar/spark,kiszk/spark,tejasapatil/spark,kimoonkim/spark,mike0sv/spark,huang1900/spark,nlalevee/spark,markhamstra/spark,loneknightpy/spark,esi-mineset/spark,debugger87/spark,rekhajoshm/spark,akopich/spark,icexelloss/spark,bOOm-X/spark,panfengfeng/spark,shuangshuangwang/spark,pgandhi999/spark,minixalpha/spark,facaiy/spark,szhem/spark,WindCanDie/spark,guoxiaolongzte/spark,shubhamchopra/spark,kiszk/spark,techaddict/spark,adrian-ionescu/apache-spark,ron8hu/spark,jkbradley/spark,pgandhi999/spark,facaiy/spark,holdenk/spark,esi-mineset/spark,ibm-research-ireland/sparkoscope,shubhamchopra/spark,janewangfb/spark,nlalevee/spark,witgo/spark,xflin/spark,xflin/spark,patrick-nicholson/spark,zzcclp/spark,liutang123/spark,skonto/spark,sureshthalamati/spark,nlalevee/spark,spark-test/spark,xuanyuanking/spark,LantaoJin/spark,ConeyLiu/spark,janewangfb/spark,wangyum/spark