[SPARK-6506] [pyspark] Do not try to retrieve SPARK_HOME when not needed...
[SPARK-6506] [pyspark] Do not try to retrieve SPARK_HOME when not needed...

....

In particular, this makes pyspark in yarn-cluster mode fail unless
SPARK_HOME is set, when it's not really needed.

Author: Marcelo Vanzin <vanzin@cloudera.com>

Closes #5405 from vanzin/SPARK-6506 and squashes the following commits:

e184507 [Marcelo Vanzin] [SPARK-6506] [pyspark] Do not try to retrieve SPARK_HOME when not needed.

metamx/spark,dbtsai/spark,JoshRosen/spark,zhouyejoe/spark,mdespriee/spark,publicRoman/spark,HyukjinKwon/spark,techaddict/spark,haowu80s/spark,narahari92/spark,hhbyyh/spark,markhamstra/spark,dbtsai/spark,akopich/spark,apache-spark-on-k8s/spark,rikima/spark,hvanhovell/spark,xflin/spark,lvdongr/spark,kissmetrics/spark,apache-spark-on-k8s/spark,bdrillard/spark,JerryLead/spark,setjet/spark,facaiy/spark,ueshin/apache-spark,BryanCutler/spark,huang1900/spark,joseph-torres/spark,pgandhi999/spark,michalsenkyr/spark,rekhajoshm/spark,1haodian/spark,xuanyuanking/spark,shubhamchopra/spark,ibm-research-ireland/sparkoscope,gioenn/xSpark,kimoonkim/spark,haowu80s/spark,zuotingbing/spark,MLnick/spark,mike0sv/spark,JoshRosen/spark,michalsenkyr/spark,minixalpha/spark,bdrillard/spark,esi-mineset/spark,eyalfa/spark,ConeyLiu/spark,milliman/spark,UndeadBaneGitHub/spark,facaiy/spark,byakuinss/spark,michalsenkyr/spark,UndeadBaneGitHub/spark,SHASHANKB/spark,skonto/spark,1haodian/spark,brad-kaiser/spark,pgandhi999/spark,nchammas/spark,kissmetrics/spark,aosagie/spark,gioenn/xSpark,lvdongr/spark,akopich/spark,dbtsai/spark,xuanyuanking/spark,ConeyLiu/spark,nchammas/spark,someorz/spark,ddna1021/spark,tejasapatil/spark,ioana-delaney/spark,actuaryzhang/spark,dongjoon-hyun/spark,BryanCutler/spark,ConeyLiu/spark,patrick-nicholson/spark,spark-test/spark,jrshust/spark,xflin/spark,jkbradley/spark,ioana-delaney/spark,ueshin/apache-spark,metamx/spark,lxsmnv/spark,jkbradley/spark,Panos-Bletsos/spark-cost-model-optimizer,wangyum/spark,kiszk/spark,ericvandenbergfb/spark,chuckchen/spark,shuangshuangwang/spark,liutang123/spark,saltstar/spark,MLnick/spark,spark-test/spark,nchammas/spark,apache/spark,tejasapatil/spark,hhbyyh/spark,saltstar/spark,michalsenkyr/spark,srowen/spark,zzcclp/spark,ueshin/apache-spark,aokolnychyi/spark,kiszk/spark,Aegeaner/spark,brad-kaiser/spark,hvanhovell/spark,BryanCutler/spark,apache-spark-on-k8s/spark,panfengfeng/spark,gioenn/xSpark,kevinyu98/spark,apache/spark,jlopezmalla/spark,jianran/spark,yanboliang/spark,techaddict/spark,kissmetrics/spark,darionyaphet/spark,narahari92/spark,brad-kaiser/spark,zuotingbing/spark,zuotingbing/spark,milliman/spark,srowen/spark,byakuinss/spark,ibm-research-ireland/sparkoscope,ibm-research-ireland/sparkoscope,saltstar/spark,matthewfranglen/spark,dongjoon-hyun/spark,spark-test/spark,actuaryzhang/spark,someorz/spark,witgo/spark,bravo-zhang/spark,panfengfeng/spark,Panos-Bletsos/spark-cost-model-optimizer,apache-spark-on-k8s/spark,tengpeng/spark,ueshin/apache-spark,pronix/spark,andrewor14/spark,sachintyagi22/spark,shaneknapp/spark,jrshust/spark,actuaryzhang/spark,big-pegasus/spark,markhamstra/spark,stanzhai/spark,cloud-fan/spark,michalsenkyr/spark,vinodkc/spark,WeichenXu123/spark,huang1900/spark,joseph-torres/spark,cin/spark,zero323/spark,mike0sv/spark,metamx/spark,mike0sv/spark,brad-kaiser/spark,JerryLead/spark,nlalevee/spark,gioenn/xSpark,kimoonkim/spark,aosagie/spark,ConeyLiu/spark,janewangfb/spark,rednaxelafx/apache-spark,rekhajoshm/spark,cloud-fan/spark,jrshust/spark,akopich/spark,shuangshuangwang/spark,loneknightpy/spark,ueshin/apache-spark,adrian-ionescu/apache-spark,nilsgrabbert/spark,nlalevee/spark,bdrillard/spark,UndeadBaneGitHub/spark,skonto/spark,wangyum/spark,saturday-shi/spark,tejasapatil/spark,jkbradley/spark,jlopezmalla/spark,kevinyu98/spark,Panos-Bletsos/spark-cost-model-optimizer,apache/spark,jrshust/spark,apache/spark,zuotingbing/spark,rezasafi/spark,xflin/spark,caneGuy/spark,UndeadBaneGitHub/spark,jlopezmalla/spark,nchammas/spark,big-pegasus/spark,skonto/spark,joseph-torres/spark,liutang123/spark,facaiy/spark,mahak/spark,bOOm-X/spark,tejasapatil/spark,caneGuy/spark,minixalpha/spark,JerryLead/spark,cin/spark,ptkool/spark,metamx/spark,mike0sv/spark,taroplus/spark,brad-kaiser/spark,pronix/spark,aray/spark,liyichao/spark,zhouyejoe/spark,kevinyu98/spark,byakuinss/spark,someorz/spark,tengpeng/spark,rednaxelafx/apache-spark,stanzhai/spark,jianran/spark,mdespriee/spark,SnappyDataInc/spark,maropu/spark,gengliangwang/spark,joseph-torres/spark,cloud-fan/spark,dongjoon-hyun/spark,panfengfeng/spark,Panos-Bletsos/spark-cost-model-optimizer,nilsgrabbert/spark,lxsmnv/spark,patrick-nicholson/spark,aosagie/spark,highfei2011/spark,bOOm-X/spark,JoshRosen/spark,lvdongr/spark,byakuinss/spark,rikima/spark,andrewor14/spark,sachintyagi22/spark,panfengfeng/spark,HyukjinKwon/spark,minixalpha/spark,bravo-zhang/spark,LantaoJin/spark,map222/spark,apache/spark,gengliangwang/spark,eyalfa/spark,JerryLead/spark,aray/spark,gengliangwang/spark,yanboliang/spark,bOOm-X/spark,xuanyuanking/spark,janewangfb/spark,actuaryzhang/spark,tengpeng/spark,MLnick/spark,guoxiaolongzte/spark,srowen/spark,wangyum/spark,aray/spark,spark-test/spark,HyukjinKwon/spark,SnappyDataInc/spark,saltstar/spark,rezasafi/spark,eyalfa/spark,bdrillard/spark,andrewor14/spark,szhem/spark,metamx/spark,goldmedal/spark,nchammas/spark,kissmetrics/spark,stanzhai/spark,pronix/spark,shubhamchopra/spark,WeichenXu123/spark,pgandhi999/spark,bdrillard/spark,nilsgrabbert/spark,cloud-fan/spark,srowen/spark,zuotingbing/spark,alunarbeach/spark,icexelloss/spark,mahak/spark,someorz/spark,andrewor14/iolap,andrewor14/spark,ibm-research-ireland/sparkoscope,janewangfb/spark,aokolnychyi/spark,liutang123/spark,srowen/spark,jiangxb1987/spark,liyichao/spark,cin/spark,icexelloss/spark,debugger87/spark,bdrillard/spark,rezasafi/spark,haowu80s/spark,aokolnychyi/spark,nlalevee/spark,tengpeng/spark,LantaoJin/spark,mdespriee/spark,markhamstra/spark,holdenk/spark,rekhajoshm/spark,highfei2011/spark,apache-spark-on-k8s/spark,aokolnychyi/spark,sahilTakiar/spark,big-pegasus/spark,kiszk/spark,aosagie/spark,witgo/spark,nlalevee/spark,michalsenkyr/spark,bOOm-X/spark,zhouyejoe/spark,mzl9039/spark,mdespriee/spark,aokolnychyi/spark,saltstar/spark,loneknightpy/spark,szhem/spark,LantaoJin/spark,huang1900/spark,joseph-torres/spark,ddna1021/spark,kevinyu98/spark,esi-mineset/spark,SnappyDataInc/spark,bOOm-X/spark,ConeyLiu/spark,poffuomo/spark,jkbradley/spark,panfengfeng/spark,hhbyyh/spark,JoshRosen/spark,narahari92/spark,mike0sv/spark,gengliangwang/spark,hvanhovell/spark,dbtsai/spark,szhem/spark,WindCanDie/spark,mike0sv/spark,bravo-zhang/spark,andrewor14/iolap,jiangxb1987/spark,saturday-shi/spark,HyukjinKwon/spark,liutang123/spark,yanboliang/spark,minixalpha/spark,chuckchen/spark,LantaoJin/spark,wangmiao1981/spark,rezasafi/spark,shaneknapp/spark,dhruve/spark,shubhamchopra/spark,milliman/spark,ConeyLiu/spark,ajaysaini725/spark,lxsmnv/spark,jlopezmalla/spark,sureshthalamati/spark,kiszk/spark,haowu80s/spark,holdenk/spark,ptkool/spark,saturday-shi/spark,tejasapatil/spark,UndeadBaneGitHub/spark,mzl9039/spark,wzhfy/spark,nlalevee/spark,cloud-fan/spark,zuotingbing/spark,ptkool/spark,icexelloss/spark,rednaxelafx/apache-spark,rednaxelafx/apache-spark,rezasafi/spark,wangmiao1981/spark,skonto/spark,jkbradley/spark,ajaysaini725/spark,minixalpha/spark,adrian-ionescu/apache-spark,mahak/spark,metamx/spark,rednaxelafx/apache-spark,aray/spark,nchammas/spark,huang1900/spark,milliman/spark,guoxiaolongzte/spark,ibm-research-ireland/sparkoscope,ron8hu/spark,mzl9039/spark,akopich/spark,hhbyyh/spark,techaddict/spark,MLnick/spark,highfei2011/spark,maropu/spark,WindCanDie/spark,liutang123/spark,rekhajoshm/spark,alunarbeach/spark,taroplus/spark,debugger87/spark,patrick-nicholson/spark,wzhfy/spark,rekhajoshm/spark,debugger87/spark,kiszk/spark,alunarbeach/spark,zhouyejoe/spark,map222/spark,spark-test/spark,SHASHANKB/spark,wangmiao1981/spark,ericvandenbergfb/spark,jlopezmalla/spark,lxsmnv/spark,sahilTakiar/spark,zzcclp/spark,loneknightpy/spark,xflin/spark,adrian-ionescu/apache-spark,rezasafi/spark,HyukjinKwon/spark,Aegeaner/spark,zzcclp/spark,aokolnychyi/spark,brad-kaiser/spark,aray/spark,cin/spark,liutang123/spark,ron8hu/spark,alunarbeach/spark,loneknightpy/spark,hhbyyh/spark,aosagie/spark,zero323/spark,ajaysaini725/spark,liyichao/spark,wangyum/spark,jiangxb1987/spark,ioana-delaney/spark,kissmetrics/spark,hhbyyh/spark,eyalfa/spark,rekhajoshm/spark,Aegeaner/spark,zero323/spark,setjet/spark,ron8hu/spark,maropu/spark,zzcclp/spark,gioenn/xSpark,Aegeaner/spark,ioana-delaney/spark,zhouyejoe/spark,rekhajoshm/spark,liyichao/spark,mdespriee/spark,map222/spark,metamx/spark,sureshthalamati/spark,zhouyejoe/spark,hvanhovell/spark,poffuomo/spark,skonto/spark,liyichao/spark,1haodian/spark,WeichenXu123/spark,saturday-shi/spark,taroplus/spark,pgandhi999/spark,jrshust/spark,lxsmnv/spark,janewangfb/spark,cin/spark,ptkool/spark,actuaryzhang/spark,skonto/spark,sachintyagi22/spark,ajaysaini725/spark,rikima/spark,yanboliang/spark,jrshust/spark,ajaysaini725/spark,MLnick/spark,nilsgrabbert/spark,wzhfy/spark,techaddict/spark,Aegeaner/spark,sahilTakiar/spark,szhem/spark,kimoonkim/spark,zzcclp/spark,rednaxelafx/apache-spark,BryanCutler/spark,andrewor14/iolap,caneGuy/spark,narahari92/spark,wzhfy/spark,ahnqirage/spark,nlalevee/spark,holdenk/spark,witgo/spark,sahilTakiar/spark,akopich/spark,bravo-zhang/spark,adrian-ionescu/apache-spark,sahilTakiar/spark,gioenn/xSpark,brad-kaiser/spark,big-pegasus/spark,hvanhovell/spark,ron8hu/spark,kevinyu98/spark,nchammas/spark,darionyaphet/spark,taroplus/spark,poffuomo/spark,janewangfb/spark,1haodian/spark,alunarbeach/spark,narahari92/spark,jlopezmalla/spark,panfengfeng/spark,ptkool/spark,JerryLead/spark,milliman/spark,ron8hu/spark,hhbyyh/spark,bdrillard/spark,sahilTakiar/spark,sachintyagi22/spark,ajaysaini725/spark,icexelloss/spark,vinodkc/spark,sureshthalamati/spark,kiszk/spark,MLnick/spark,kissmetrics/spark,xflin/spark,tengpeng/spark,zuotingbing/spark,rikima/spark,yanboliang/spark,techaddict/spark,highfei2011/spark,facaiy/spark,shubhamchopra/spark,dhruve/spark,bravo-zhang/spark,SHASHANKB/spark,nilsgrabbert/spark,publicRoman/spark,SnappyDataInc/spark,wangyum/spark,vinodkc/spark,goldmedal/spark,darionyaphet/spark,debugger87/spark,chuckchen/spark,jiangxb1987/spark,icexelloss/spark,panfengfeng/spark,zero323/spark,UndeadBaneGitHub/spark,ajaysaini725/spark,sachintyagi22/spark,WeichenXu123/spark,wangyum/spark,nlalevee/spark,ioana-delaney/spark,poffuomo/spark,tengpeng/spark,wzhfy/spark,mahak/spark,shuangshuangwang/spark,UndeadBaneGitHub/spark,kissmetrics/spark,shubhamchopra/spark,goldmedal/spark,mdespriee/spark,andrewor14/iolap,milliman/spark,saltstar/spark,highfei2011/spark,esi-mineset/spark,shaneknapp/spark,jkbradley/spark,JoshRosen/spark,ibm-research-ireland/sparkoscope,highfei2011/spark,lvdongr/spark,shaneknapp/spark,andrewor14/iolap,ibm-research-ireland/sparkoscope,andrewor14/iolap,dbtsai/spark,srowen/spark,akopich/spark,ddna1021/spark,michalsenkyr/spark,pronix/spark,cloud-fan/spark,chuckchen/spark,guoxiaolongzte/spark,ahnqirage/spark,zzcclp/spark,publicRoman/spark,big-pegasus/spark,yanboliang/spark,loneknightpy/spark,JoshRosen/spark,loneknightpy/spark,lvdongr/spark,dhruve/spark,haowu80s/spark,jiangxb1987/spark,skonto/spark,mzl9039/spark,HyukjinKwon/spark,apache-spark-on-k8s/spark,big-pegasus/spark,goldmedal/spark,BryanCutler/spark,apache/spark,Aegeaner/spark,sureshthalamati/spark,eyalfa/spark,joseph-torres/spark,minixalpha/spark,kevinyu98/spark,1haodian/spark,ron8hu/spark,milliman/spark,SnappyDataInc/spark,jrshust/spark,wangmiao1981/spark,lvdongr/spark,aosagie/spark,SHASHANKB/spark,map222/spark,jianran/spark,adrian-ionescu/apache-spark,WindCanDie/spark,pronix/spark,pgandhi999/spark,bravo-zhang/spark,ioana-delaney/spark,rednaxelafx/apache-spark,guoxiaolongzte/spark,rikima/spark,jianran/spark,mahak/spark,map222/spark,techaddict/spark,ddna1021/spark,sachintyagi22/spark,markhamstra/spark,maropu/spark,Aegeaner/spark,mahak/spark,zzcclp/spark,xflin/spark,markhamstra/spark,JoshRosen/spark,kimoonkim/spark,HyukjinKwon/spark,setjet/spark,andrewor14/iolap,guoxiaolongzte/spark,minixalpha/spark,janewangfb/spark,ericvandenbergfb/spark,srowen/spark,darionyaphet/spark,caneGuy/spark,publicRoman/spark,chuckchen/spark,debugger87/spark,joseph-torres/spark,xflin/spark,JerryLead/spark,someorz/spark,cin/spark,byakuinss/spark,saltstar/spark,ptkool/spark,dhruve/spark,tejasapatil/spark,facaiy/spark,ericvandenbergfb/spark,poffuomo/spark,rezasafi/spark,xuanyuanking/spark,highfei2011/spark,sureshthalamati/spark,ueshin/apache-spark,ahnqirage/spark,WindCanDie/spark,big-pegasus/spark,shuangshuangwang/spark,holdenk/spark,facaiy/spark,liyichao/spark,SHASHANKB/spark,bravo-zhang/spark,aokolnychyi/spark,SnappyDataInc/spark,alunarbeach/spark,andrewor14/spark,zero323/spark,liutang123/spark,stanzhai/spark,vinodkc/spark,mdespriee/spark,nilsgrabbert/spark,dbtsai/spark,zero323/spark,poffuomo/spark,alunarbeach/spark,ron8hu/spark,aray/spark,spark-test/spark,darionyaphet/spark,shuangshuangwang/spark,shubhamchopra/spark,kiszk/spark,ahnqirage/spark,techaddict/spark,lxsmnv/spark,ddna1021/spark,loneknightpy/spark,wzhfy/spark,ericvandenbergfb/spark,guoxiaolongzte/spark,shubhamchopra/spark,publicRoman/spark,maropu/spark,gengliangwang/spark,lvdongr/spark,shuangshuangwang/spark,debugger87/spark,goldmedal/spark,guoxiaolongzte/spark,rikima/spark,setjet/spark,cin/spark,hvanhovell/spark,goldmedal/spark,WindCanDie/spark,xuanyuanking/spark,wangmiao1981/spark,ptkool/spark,aray/spark,taroplus/spark,mike0sv/spark,holdenk/spark,WindCanDie/spark,poffuomo/spark,shuangshuangwang/spark,dhruve/spark,sachintyagi22/spark,ericvandenbergfb/spark,zero323/spark,saturday-shi/spark,mahak/spark,map222/spark,icexelloss/spark,huang1900/spark,tejasapatil/spark,gengliangwang/spark,esi-mineset/spark,aosagie/spark,janewangfb/spark,kimoonkim/spark,rikima/spark,huang1900/spark,xuanyuanking/spark,someorz/spark,witgo/spark,WindCanDie/spark,bOOm-X/spark,ioana-delaney/spark,LantaoJin/spark,haowu80s/spark,dbtsai/spark,xuanyuanking/spark,LantaoJin/spark,map222/spark,debugger87/spark,stanzhai/spark,chuckchen/spark,adrian-ionescu/apache-spark,jianran/spark,someorz/spark,stanzhai/spark,WeichenXu123/spark,jianran/spark,eyalfa/spark,maropu/spark,wzhfy/spark,dongjoon-hyun/spark,publicRoman/spark,patrick-nicholson/spark,1haodian/spark,byakuinss/spark,ericvandenbergfb/spark,WeichenXu123/spark,jiangxb1987/spark,caneGuy/spark,ConeyLiu/spark,adrian-ionescu/apache-spark,dongjoon-hyun/spark,actuaryzhang/spark,dongjoon-hyun/spark,taroplus/spark,Panos-Bletsos/spark-cost-model-optimizer,huang1900/spark,markhamstra/spark,narahari92/spark,setjet/spark,icexelloss/spark,shaneknapp/spark,ddna1021/spark,vinodkc/spark,SnappyDataInc/spark,caneGuy/spark,kimoonkim/spark,facaiy/spark,MLnick/spark,saturday-shi/spark,wangyum/spark,sahilTakiar/spark,caneGuy/spark,dhruve/spark,actuaryzhang/spark,pronix/spark,publicRoman/spark,szhem/spark,vinodkc/spark,apache/spark,pgandhi999/spark,darionyaphet/spark,setjet/spark,shaneknapp/spark,saturday-shi/spark,yanboliang/spark,andrewor14/spark,ahnqirage/spark,sureshthalamati/spark,stanzhai/spark,markhamstra/spark,wangmiao1981/spark,goldmedal/spark,akopich/spark,narahari92/spark,zhouyejoe/spark,ddna1021/spark,cloud-fan/spark,gioenn/xSpark,hvanhovell/spark,dongjoon-hyun/spark,jianran/spark,mzl9039/spark,Panos-Bletsos/spark-cost-model-optimizer,maropu/spark,SHASHANKB/spark,mzl9039/spark,SHASHANKB/spark,setjet/spark,esi-mineset/spark,tengpeng/spark,wangmiao1981/spark,holdenk/spark,andrewor14/spark,1haodian/spark,sureshthalamati/spark,witgo/spark,kevinyu98/spark,eyalfa/spark,ahnqirage/spark,LantaoJin/spark,shaneknapp/spark,BryanCutler/spark,patrick-nicholson/spark,jlopezmalla/spark,patrick-nicholson/spark,bOOm-X/spark,jiangxb1987/spark,witgo/spark,pgandhi999/spark,spark-test/spark,BryanCutler/spark,witgo/spark,liyichao/spark,mzl9039/spark,nilsgrabbert/spark,jkbradley/spark,ueshin/apache-spark,esi-mineset/spark,gengliangwang/spark,holdenk/spark,lxsmnv/spark,darionyaphet/spark,taroplus/spark,byakuinss/spark,chuckchen/spark,ahnqirage/spark,JerryLead/spark,WeichenXu123/spark,esi-mineset/spark,kimoonkim/spark,apache-spark-on-k8s/spark,szhem/spark,vinodkc/spark,dhruve/spark,Panos-Bletsos/spark-cost-model-optimizer,patrick-nicholson/spark,szhem/spark