<filename>docs/latex/AutoFeatures.tex<gh_stars>1-10
%% Generated by Sphinx.
\def\sphinxdocclass{report}
\documentclass[letterpaper,11pt,english]{sphinxmanual}
\ifdefined\pdfpxdimen
   \let\sphinxpxdimen\pdfpxdimen\else\newdimen\sphinxpxdimen
\fi \sphinxpxdimen=.75bp\relax

\PassOptionsToPackage{warn}{textcomp}
\usepackage[utf8]{inputenc}
\ifdefined\DeclareUnicodeCharacter
% support both utf8 and utf8x syntaxes
  \ifdefined\DeclareUnicodeCharacterAsOptional
    \def\sphinxDUC#1{\DeclareUnicodeCharacter{"#1}}
  \else
    \let\sphinxDUC\DeclareUnicodeCharacter
  \fi
  \sphinxDUC{00A0}{\nobreakspace}
  \sphinxDUC{2500}{\sphinxunichar{2500}}
  \sphinxDUC{2502}{\sphinxunichar{2502}}
  \sphinxDUC{2514}{\sphinxunichar{2514}}
  \sphinxDUC{251C}{\sphinxunichar{251C}}
  \sphinxDUC{2572}{\textbackslash}
\fi
\usepackage{cmap}
\usepackage[T1]{fontenc}
\usepackage{amsmath,amssymb,amstext}
\usepackage{babel}



\usepackage{times}
\expandafter\ifx\csname T@LGR\endcsname\relax
\else
% LGR was declared as font encoding
  \substitutefont{LGR}{\rmdefault}{cmr}
  \substitutefont{LGR}{\sfdefault}{cmss}
  \substitutefont{LGR}{\ttdefault}{cmtt}
\fi
\expandafter\ifx\csname T@X2\endcsname\relax
  \expandafter\ifx\csname T@T2A\endcsname\relax
  \else
  % T2A was declared as font encoding
    \substitutefont{T2A}{\rmdefault}{cmr}
    \substitutefont{T2A}{\sfdefault}{cmss}
    \substitutefont{T2A}{\ttdefault}{cmtt}
  \fi
\else
% X2 was declared as font encoding
  \substitutefont{X2}{\rmdefault}{cmr}
  \substitutefont{X2}{\sfdefault}{cmss}
  \substitutefont{X2}{\ttdefault}{cmtt}
\fi


\usepackage[Bjarne]{fncychap}
\usepackage{sphinx}

\fvset{fontsize=\small}
\usepackage{geometry}

% Include hyperref last.
\usepackage{hyperref}
% Fix anchor placement for figures with captions.
\usepackage{hypcap}% it must be loaded after hyperref.
% Set up styles of URL: it should be placed after hyperref.
\urlstyle{same}

\usepackage{sphinxmessages}
\setcounter{tocdepth}{2}



\title{AutoFeatures: PySpark Auto Feature Selector}
\date{July 07, 2020}
\release{1.0}
\author{<NAME>}
\newcommand{\sphinxlogo}{\sphinxincludegraphics{logo.png}\par}
\renewcommand{\releasename}{Release}
\makeindex
\begin{document}

\pagestyle{empty}
\sphinxmaketitle
\pagestyle{plain}
\sphinxtableofcontents
\pagestyle{normal}
\phantomsection\label{\detokenize{index::doc}}\begin{quote}
\end{quote}

Welcome to our \sphinxstylestrong{AutoFeatures: PySpark Auto Feature Selector}!!! The PDF version
can be downloaded from \sphinxhref{../latex/AutoFeatures.pdf}{HERE}.




\chapter{Preface}
\label{\detokenize{preface:preface}}\label{\detokenize{preface:id1}}\label{\detokenize{preface::doc}}
\begin{sphinxadmonition}{note}{Chinese proverb}

Good tools are prerequisite to the successful execution of a job. \textendash{} old Chinese proverb
\end{sphinxadmonition}


\section{About}
\label{\detokenize{preface:about}}

\subsection{About this API}
\label{\detokenize{preface:about-this-api}}
This document is the \sphinxcode{\sphinxupquote{API}} book for our AutoFeatures: PySpark Auto Feature Selector \sphinxcite{reference:autofeatures} API.
The PDF version can be downloaded from \sphinxhref{PyAudit.pdf}{HERE}. \sphinxstylestrong{You may download and distribute it. Please beaware,
however, that the note contains typos as well as inaccurate or incorrect description.}

The \sphinxcode{\sphinxupquote{API}} assumes that the reader has a preliminary knowledge of \sphinxcode{\sphinxupquote{python}} programing and \sphinxcode{\sphinxupquote{Linux}}. And this
document is generated automatically by using \sphinxhref{http://sphinx.pocoo.org}{sphinx}.


\subsection{About the author}
\label{\detokenize{preface:about-the-author}}\begin{itemize}
\item {} 
\sphinxstylestrong{<NAME>}
\begin{itemize}
\item {} 
Sr. Data Scientist and PhD in Mathematics

\item {} 
University of Tennessee at Knoxville

\item {} 
Webpage: \sphinxurl{http://web.utk.edu/~wfeng1/}

\item {} 
Email: \sphinxhref{mailto:<EMAIL>}{<EMAIL>}

\end{itemize}

\item {} 
\sphinxstylestrong{Biography}

<NAME> is Data Scientist within DST’s Applied Analytics Group. Dr. Feng’s responsibilities include providing
DST clients with access to cutting-edge skills and technologies, including Big Data analytic solutions, advanced
analytic and data enhancement techniques and modeling.

Dr. Feng has deep analytic expertise in data mining, analytic systems, machine learning algorithms, business
intelligence, and applying Big Data tools to strategically solve industry problems in a cross-functional business.
Before joining DST, Dr. Feng was an IMA Data Science Fellow at The Institute for Mathematics and its
Applications (IMA) at the University of Minnesota. While there, he helped startup companies make marketing
decisions based on deep predictive analytics.

Dr. Feng graduated from University of Tennessee, Knoxville, with Ph.D. in Computational Mathematics and Master’s
degree in Statistics. He also holds Master’s degree in Computational Mathematics from Missouri University of
Science and Technology (MST) and Master’s degree in Applied Mathematics from the University of Science and
Technology of China (USTC).

\item {} 
\sphinxstylestrong{Declaration}

The work of <NAME> was supported by the IMA, while working at IMA. However, any opinion, finding,
and conclusions or recommendations expressed in this material are those of the author and do not necessarily
reflect the views of the IMA, UTK and DST.

\end{itemize}


\section{Feedback and suggestions}
\label{\detokenize{preface:feedback-and-suggestions}}
Your comments and suggestions are highly appreciated. I am more than happy to receive
corrections, suggestions or feedback through email (<NAME>: \sphinxhref{mailto:<EMAIL>}{<EMAIL>}) for improvements.


\chapter{How to Install}
\label{\detokenize{install:how-to-install}}\label{\detokenize{install:install}}\label{\detokenize{install::doc}}

\section{Install with \sphinxstyleliteralintitle{\sphinxupquote{pip}}}
\label{\detokenize{install:install-with-pip}}
You can install the \sphinxcode{\sphinxupquote{PyAudit}} from {[}PyPI{]}(\sphinxurl{https://pypi.org/project/AutoFeatures}):

\begin{sphinxVerbatim}[commandchars=\\\{\}]
pip install AutoFeatures
\end{sphinxVerbatim}


\section{Install from Repo}
\label{\detokenize{install:install-from-repo}}

\subsection{Clone the Repository}
\label{\detokenize{install:clone-the-repository}}
\begin{sphinxVerbatim}[commandchars=\\\{\}]
git clone https://github.com/runawayhorse001/AutoFeatures.git
\end{sphinxVerbatim}


\subsection{Install}
\label{\detokenize{install:id1}}
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{n+nb}{cd} AutoFeatures
pip install \PYGZhy{}r requirements.txt
python setup.py install
\end{sphinxVerbatim}


\subsection{Uninstall}
\label{\detokenize{install:uninstall}}
\begin{sphinxVerbatim}[commandchars=\\\{\}]
pip uninstall AutoFeatures
\end{sphinxVerbatim}


\subsection{Test}
\label{\detokenize{install:test}}

\chapter{AutoFeatures Class}
\label{\detokenize{auto_feature:autofeatures-class}}\label{\detokenize{auto_feature:form-frame}}\label{\detokenize{auto_feature::doc}}

\section{Utils Functions}
\label{\detokenize{auto_feature:utils-functions}}

\subsection{Data types}
\label{\detokenize{auto_feature:data-types}}\index{AutoFeatures (class in AutoFeatures)@\spxentry{AutoFeatures}\spxextra{class in AutoFeatures}}

\begin{fulllineitems}
\phantomsection\label{\detokenize{auto_feature:AutoFeatures.AutoFeatures}}\pysigline{\sphinxbfcode{\sphinxupquote{class }}\sphinxcode{\sphinxupquote{AutoFeatures.}}\sphinxbfcode{\sphinxupquote{AutoFeatures}}}
Auto feature selector for Machine Learning Modeling with PySpark. This class has four selectors:
\begin{enumerate}
\def\theenumi{\arabic{enumi}}
\def\labelenumi{\theenumi .}
\makeatletter\def\p@enumii{\p@enumi \theenumi .}\makeatother
\item {} 
unique selector: identify the single unique features

\item {} 
missing selector: identify missing values features with missing threshold

\item {} 
collinear selector: identify collinear features with threshold

\item {} 
low importance selector: identify low importance features with Gradient Boosting Machine(GBM)

\end{enumerate}
\index{dtypes\_class() (AutoFeatures.AutoFeatures class method)@\spxentry{dtypes\_class()}\spxextra{AutoFeatures.AutoFeatures class method}}

\begin{fulllineitems}
\phantomsection\label{\detokenize{auto_feature:AutoFeatures.AutoFeatures.dtypes_class}}\pysiglinewithargsret{\sphinxbfcode{\sphinxupquote{classmethod }}\sphinxbfcode{\sphinxupquote{dtypes\_class}}}{\emph{df\_in}}{}
Generate the data type categories: numerical, categorical, date and unsupported category.
\begin{quote}\begin{description}
\item[{Parameters}] \leavevmode
\sphinxstyleliteralstrong{\sphinxupquote{df\_in}} \textendash{} the input rdd data frame

\item[{Returns}] \leavevmode
data type categories

\end{description}\end{quote}

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{g+gp}{\PYGZgt{}\PYGZgt{}\PYGZgt{} }\PYG{n}{test} \PYG{o}{=} \PYG{n}{spark}\PYG{o}{.}\PYG{n}{createDataFrame}\PYG{p}{(}\PYG{p}{[}
\PYG{g+go}{                    (\PYGZsq{}Joe\PYGZsq{}, 67, \PYGZsq{}F\PYGZsq{}, 7000, \PYGZsq{}asymptomatic\PYGZsq{}, 286.1, \PYGZsq{}2019\PYGZhy{}6\PYGZhy{}28\PYGZsq{}),}
\PYG{g+go}{                    (\PYGZsq{}Henry\PYGZsq{}, 67, \PYGZsq{}M\PYGZsq{}, 8000, \PYGZsq{}asymptomatic\PYGZsq{}, 229.2, \PYGZsq{}2019\PYGZhy{}6\PYGZhy{}29\PYGZsq{}),}
\PYG{g+go}{                    (\PYGZsq{}Sam\PYGZsq{}, 37,  \PYGZsq{}F\PYGZsq{}, 6000, \PYGZsq{}nonanginal\PYGZsq{}, 250.3, \PYGZsq{}2019\PYGZhy{}6\PYGZhy{}30\PYGZsq{}),}
\PYG{g+go}{                    (\PYGZsq{}Max\PYGZsq{}, 56, \PYGZsq{}M\PYGZsq{}, 9000, \PYGZsq{}nontypical\PYGZsq{}, 236.4, \PYGZsq{}2019\PYGZhy{}5\PYGZhy{}28\PYGZsq{}),}
\PYG{g+go}{                    (\PYGZsq{}Mat\PYGZsq{}, 56, \PYGZsq{}F\PYGZsq{}, 9000, \PYGZsq{}asymptomatic\PYGZsq{}, 254.5, \PYGZsq{}2019\PYGZhy{}4\PYGZhy{}28\PYGZsq{})],}
\PYG{g+go}{                    [\PYGZsq{}Name\PYGZsq{}, \PYGZsq{}Age\PYGZsq{}, \PYGZsq{}Sex\PYGZsq{}, \PYGZsq{}Salary\PYGZsq{}, \PYGZsq{}ChestPain\PYGZsq{}, \PYGZsq{}Chol\PYGZsq{}, \PYGZsq{}CreatDate\PYGZsq{}]}
\PYG{g+go}{                   )}
\PYG{g+gp}{\PYGZgt{}\PYGZgt{}\PYGZgt{} }\PYG{n}{test} \PYG{o}{=} \PYG{n}{test}\PYG{o}{.}\PYG{n}{withColumn}\PYG{p}{(}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{CreatDate}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{,} \PYG{n}{F}\PYG{o}{.}\PYG{n}{col}\PYG{p}{(}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{CreatDate}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{)}\PYG{o}{.}\PYG{n}{cast}\PYG{p}{(}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{timestamp}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{)}\PYG{p}{)}
\PYG{g+gp}{\PYGZgt{}\PYGZgt{}\PYGZgt{} }\PYG{k+kn}{from} \PYG{n+nn}{PySparkAudit} \PYG{k+kn}{import} \PYG{n}{dtypes\PYGZus{}class}
\PYG{g+gp}{\PYGZgt{}\PYGZgt{}\PYGZgt{} }\PYG{n}{dtypes\PYGZus{}class}\PYG{p}{(}\PYG{n}{test}\PYG{p}{)}
\PYG{g+go}{(     feature       DataType}
\PYG{g+go}{0       Name     StringType}
\PYG{g+go}{1        Age       LongType}
\PYG{g+go}{2        Sex     StringType}
\PYG{g+go}{3    Salary       LongType}
\PYG{g+go}{4  ChestPain     StringType}
\PYG{g+go}{5       Chol     DoubleType}
\PYG{g+go}{6  CreatDate  TimestampType,}
\PYG{g+go}{[\PYGZsq{}Age\PYGZsq{}, \PYGZsq{}Salary\PYGZsq{}, \PYGZsq{}Chol\PYGZsq{}],}
\PYG{g+go}{[\PYGZsq{}Name\PYGZsq{}, \PYGZsq{}Sex\PYGZsq{}, \PYGZsq{}ChestPain\PYGZsq{}],}
\PYG{g+go}{[\PYGZsq{}CreatDate\PYGZsq{}], [])}
\end{sphinxVerbatim}

\end{fulllineitems}

\index{get\_dummy() (AutoFeatures.AutoFeatures class method)@\spxentry{get\_dummy()}\spxextra{AutoFeatures.AutoFeatures class method}}

\begin{fulllineitems}
\phantomsection\label{\detokenize{auto_feature:AutoFeatures.AutoFeatures.get_dummy}}\pysiglinewithargsret{\sphinxbfcode{\sphinxupquote{classmethod }}\sphinxbfcode{\sphinxupquote{get\_dummy}}}{\emph{df\_in}, \emph{index\_col=None}, \emph{categorical\_cols=None}, \emph{continuous\_cols=None}, \emph{label\_col=None}, \emph{dropLast=False}}{}
Get dummy variables and concat with continuous variables for ml modeling.
\begin{quote}\begin{description}
\item[{Parameters}] \leavevmode\begin{itemize}
\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{df\_in}} \textendash{} the dataframe

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{categorical\_cols}} \textendash{} the name list of the categorical data

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{continuous\_cols}} \textendash{} the name list of the numerical data

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{label\_col}} \textendash{} the name of label column

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{dropLast}} \textendash{} the flag of drop last column

\end{itemize}

\item[{Returns}] \leavevmode
encoded dummy variable names and feature matrix

\item[{Author}] \leavevmode
<NAME>

\item[{Email}] \leavevmode
\sphinxhref{mailto:<EMAIL>}{<EMAIL>}

\end{description}\end{quote}

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{g+gp}{\PYGZgt{}\PYGZgt{}\PYGZgt{} }\PYG{n}{df} \PYG{o}{=} \PYG{n}{spark}\PYG{o}{.}\PYG{n}{createDataFrame}\PYG{p}{(}\PYG{p}{[}
\PYG{g+go}{              (0, \PYGZdq{}a\PYGZdq{}),}
\PYG{g+go}{              (1, \PYGZdq{}b\PYGZdq{}),}
\PYG{g+go}{              (2, \PYGZdq{}c\PYGZdq{}),}
\PYG{g+go}{              (3, \PYGZdq{}a\PYGZdq{}),}
\PYG{g+go}{              (4, \PYGZdq{}a\PYGZdq{}),}
\PYG{g+go}{              (5, \PYGZdq{}c\PYGZdq{})}
\PYG{g+go}{          ], [\PYGZdq{}id\PYGZdq{}, \PYGZdq{}category\PYGZdq{}])}
\end{sphinxVerbatim}

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{g+gp}{\PYGZgt{}\PYGZgt{}\PYGZgt{} }\PYG{n}{index\PYGZus{}col} \PYG{o}{=} \PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{id}\PYG{l+s+s1}{\PYGZsq{}}
\PYG{g+gp}{\PYGZgt{}\PYGZgt{}\PYGZgt{} }\PYG{n}{categorical\PYGZus{}cols} \PYG{o}{=} \PYG{p}{[}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{category}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{]}
\PYG{g+gp}{\PYGZgt{}\PYGZgt{}\PYGZgt{} }\PYG{n}{continuous\PYGZus{}cols} \PYG{o}{=} \PYG{p}{[}\PYG{p}{]}
\PYG{g+gp}{\PYGZgt{}\PYGZgt{}\PYGZgt{} }\PYG{n}{label\PYGZus{}col} \PYG{o}{=} \PYG{p}{[}\PYG{p}{]}
\end{sphinxVerbatim}

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{g+gp}{\PYGZgt{}\PYGZgt{}\PYGZgt{} }\PYG{n}{mat} \PYG{o}{=} \PYG{n}{get\PYGZus{}dummy}\PYG{p}{(}\PYG{n}{df}\PYG{p}{,}\PYG{n}{index\PYGZus{}col}\PYG{p}{,}\PYG{n}{categorical\PYGZus{}cols}\PYG{p}{,}\PYG{n}{continuous\PYGZus{}cols}\PYG{p}{,}\PYG{n}{label\PYGZus{}col}\PYG{p}{)}
\PYG{g+gp}{\PYGZgt{}\PYGZgt{}\PYGZgt{} }\PYG{n}{mat}\PYG{o}{.}\PYG{n}{show}\PYG{p}{(}\PYG{p}{)}
\end{sphinxVerbatim}

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{g+go}{\PYGZgt{}\PYGZgt{}\PYGZgt{}}
\PYG{g+go}{    +\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+}
\PYG{g+go}{    \textbar{} id\textbar{}     features\textbar{}}
\PYG{g+go}{    +\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+}
\PYG{g+go}{    \textbar{}  0\textbar{}[1.0,0.0,0.0]\textbar{}}
\PYG{g+go}{    \textbar{}  1\textbar{}[0.0,0.0,1.0]\textbar{}}
\PYG{g+go}{    \textbar{}  2\textbar{}[0.0,1.0,0.0]\textbar{}}
\PYG{g+go}{    \textbar{}  3\textbar{}[1.0,0.0,0.0]\textbar{}}
\PYG{g+go}{    \textbar{}  4\textbar{}[1.0,0.0,0.0]\textbar{}}
\PYG{g+go}{    \textbar{}  5\textbar{}[0.0,1.0,0.0]\textbar{}}
\PYG{g+go}{    +\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+}
\end{sphinxVerbatim}

\end{fulllineitems}

\index{get\_encoded\_names() (AutoFeatures.AutoFeatures class method)@\spxentry{get\_encoded\_names()}\spxextra{AutoFeatures.AutoFeatures class method}}

\begin{fulllineitems}
\phantomsection\label{\detokenize{auto_feature:AutoFeatures.AutoFeatures.get_encoded_names}}\pysiglinewithargsret{\sphinxbfcode{\sphinxupquote{classmethod }}\sphinxbfcode{\sphinxupquote{get\_encoded\_names}}}{\emph{df\_in}, \emph{categorical\_cols}}{}
get the encoded dummy variable names
\begin{quote}\begin{description}
\item[{Parameters}] \leavevmode\begin{itemize}
\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{df\_in}} \textendash{} the input dataframe

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{categorical\_cols}} \textendash{} the name list of the categorical columns

\end{itemize}

\item[{Returns}] \leavevmode
the name list of the encoded dummy variable for categorical columns

\end{description}\end{quote}

\end{fulllineitems}


\end{fulllineitems}



\section{AutoFeatures Class}
\label{\detokenize{auto_feature:id1}}\index{AutoFeatures (class in AutoFeatures)@\spxentry{AutoFeatures}\spxextra{class in AutoFeatures}}

\begin{fulllineitems}
\pysigline{\sphinxbfcode{\sphinxupquote{class }}\sphinxcode{\sphinxupquote{AutoFeatures.}}\sphinxbfcode{\sphinxupquote{AutoFeatures}}}
Auto feature selector for Machine Learning Modeling with PySpark. This class has four selectors:
\begin{enumerate}
\def\theenumi{\arabic{enumi}}
\def\labelenumi{\theenumi .}
\makeatletter\def\p@enumii{\p@enumi \theenumi .}\makeatother
\item {} 
unique selector: identify the single unique features

\item {} 
missing selector: identify missing values features with missing threshold

\item {} 
collinear selector: identify collinear features with threshold

\item {} 
low importance selector: identify low importance features with Gradient Boosting Machine(GBM)

\end{enumerate}
\index{\_\_init\_\_() (AutoFeatures.AutoFeatures method)@\spxentry{\_\_init\_\_()}\spxextra{AutoFeatures.AutoFeatures method}}

\begin{fulllineitems}
\phantomsection\label{\detokenize{auto_feature:AutoFeatures.AutoFeatures.__init__}}\pysiglinewithargsret{\sphinxbfcode{\sphinxupquote{\_\_init\_\_}}}{}{}
Initialize self.  See help(type(self)) for accurate signature.

\end{fulllineitems}

\index{corr\_selector() (AutoFeatures.AutoFeatures class method)@\spxentry{corr\_selector()}\spxextra{AutoFeatures.AutoFeatures class method}}

\begin{fulllineitems}
\phantomsection\label{\detokenize{auto_feature:AutoFeatures.AutoFeatures.corr_selector}}\pysiglinewithargsret{\sphinxbfcode{\sphinxupquote{classmethod }}\sphinxbfcode{\sphinxupquote{corr\_selector}}}{\emph{data}, \emph{index\_col=None}, \emph{label\_col=None}, \emph{corr\_thold=0.9}, \emph{method='pearson'}, \emph{rotation=True}, \emph{display=False}, \emph{tracking=False}, \emph{cat\_num=2}}{}
collinear selector: identify collinear features with threshold
\begin{quote}\begin{description}
\item[{Parameters}] \leavevmode\begin{itemize}
\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{data}} \textendash{} input dataframe

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{index\_col}} \textendash{} the name of the index column and the other columns you want to exclude

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{label\_col}} \textendash{} the name of the label column

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{corr\_thold}} \textendash{} threshold for collinear scores

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{method}} \textendash{} the method to use for computing correlation, supported: pearson (default), spearman

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{rotation}} \textendash{} the flag of rotate x-ticks

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{display}} \textendash{} the flag for displaying plots, the default value is False

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{tracking}} \textendash{} the flag for displaying CPU time, the default value is False

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{cat\_num}} \textendash{} the number of the categorical feature (helping removing binary features)

\end{itemize}

\item[{Returns}] \leavevmode
The name list of the correlated values features above threshold

\end{description}\end{quote}

\end{fulllineitems}

\index{ensemble\_drop() (AutoFeatures.AutoFeatures class method)@\spxentry{ensemble\_drop()}\spxextra{AutoFeatures.AutoFeatures class method}}

\begin{fulllineitems}
\phantomsection\label{\detokenize{auto_feature:AutoFeatures.AutoFeatures.ensemble_drop}}\pysiglinewithargsret{\sphinxbfcode{\sphinxupquote{classmethod }}\sphinxbfcode{\sphinxupquote{ensemble\_drop}}}{\emph{data}, \emph{index\_col}, \emph{label\_col}, \emph{task}, \emph{importance\_thold=None}, \emph{cumulative\_thold=0.96}, \emph{missing\_thold=0.6}, \emph{corr\_thold=0.9}, \emph{method='pearson'}, \emph{rotation=True}, \emph{n\_train=5}, \emph{top\_n=20}, \emph{dropLast=False}, \emph{display=False}, \emph{tracking=False}, \emph{cat\_num=2}}{}
Ensemble drop (based on essential drop, that is to say it has included the functionals of essential drop) is a
method to identify the essential drop features based on ensemble ML model (GBM).
\begin{quote}\begin{description}
\item[{Parameters}] \leavevmode\begin{itemize}
\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{data}} \textendash{} input dataframe

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{index\_col}} \textendash{} the name of the index column and the other columns you want to exclude

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{label\_col}} \textendash{} the name of the label column

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{task}} \textendash{} the ensemble model type, supported task “classification” or “regression”

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{importance\_thold}} \textendash{} the threshold of the feature importance if missing will be auto calculated by the
cumulative threshold

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{cumulative\_thold}} \textendash{} the threshold of the cumulative feature importance, this will be used to determine
the importance\_thold when importance\_thold is missing

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{missing\_thold}} \textendash{} threshold for missing values percentage

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{corr\_thold}} \textendash{} threshold for collinear scores

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{method}} \textendash{} the method to use for computing correlation, supported: pearson (default), spearman

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{rotation}} \textendash{} the flag of rotate x-ticks

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{n\_train}} \textendash{} the numbers of train for average the feature importance

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{top\_n}} \textendash{} the numbers for plot top\_n highest feature importance

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{dropLast}} \textendash{} the flag of the drop last column during applying the OneHotEncoder

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{display}} \textendash{} the flag for displaying plots, the default value is False

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{tracking}} \textendash{} the number of the categorical feature (helping removing binary features)

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{cat\_num}} \textendash{} the number of the categorical feature (helping removing binary features)

\end{itemize}

\item[{Returns}] \leavevmode
The name list of the to\_drop features with low feature importance

\end{description}\end{quote}

\end{fulllineitems}

\index{essential\_drop() (AutoFeatures.AutoFeatures class method)@\spxentry{essential\_drop()}\spxextra{AutoFeatures.AutoFeatures class method}}

\begin{fulllineitems}
\phantomsection\label{\detokenize{auto_feature:AutoFeatures.AutoFeatures.essential_drop}}\pysiglinewithargsret{\sphinxbfcode{\sphinxupquote{classmethod }}\sphinxbfcode{\sphinxupquote{essential\_drop}}}{\emph{data}, \emph{index\_col}, \emph{label\_col}, \emph{missing\_thold=0.6}, \emph{corr\_thold=0.9}, \emph{method='pearson'}, \emph{rotation=True}, \emph{display=False}, \emph{tracking=False}, \emph{cat\_num=2}}{}
Essential drop (included: missing selector, unique selector, correlation selector) is all in one functions
to identify the essential drop features.
\begin{quote}\begin{description}
\item[{Parameters}] \leavevmode\begin{itemize}
\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{data}} \textendash{} input dataframe

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{index\_col}} \textendash{} the name of the index column and the other columns you want to exclude

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{label\_col}} \textendash{} the name of the label column

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{missing\_thold}} \textendash{} threshold for missing values percentage

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{corr\_thold}} \textendash{} threshold for collinear scores

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{method}} \textendash{} the method to use for computing correlation, supported: pearson (default), spearman

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{rotation}} \textendash{} the flag of rotate x-ticks

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{display}} \textendash{} the flag for displaying plots, the default value is False

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{tracking}} \textendash{} the flag for displaying CPU time, the default value is False

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{cat\_num}} \textendash{} the number of the categorical feature (helping removing binary features)

\end{itemize}

\item[{Returns}] \leavevmode
The name list of the to\_drop features with essential  drop functions

\end{description}\end{quote}

\end{fulllineitems}

\index{importance\_selector() (AutoFeatures.AutoFeatures class method)@\spxentry{importance\_selector()}\spxextra{AutoFeatures.AutoFeatures class method}}

\begin{fulllineitems}
\phantomsection\label{\detokenize{auto_feature:AutoFeatures.AutoFeatures.importance_selector}}\pysiglinewithargsret{\sphinxbfcode{\sphinxupquote{classmethod }}\sphinxbfcode{\sphinxupquote{importance\_selector}}}{\emph{data}, \emph{index\_col}, \emph{label\_col}, \emph{task}, \emph{importance\_thold=None}, \emph{cumulative\_thold=0.96}, \emph{missing\_thold=0.6}, \emph{corr\_thold=0.9}, \emph{method='pearson'}, \emph{rotation=True}, \emph{n\_train=5}, \emph{top\_n=20}, \emph{dropLast=False}, \emph{display=False}, \emph{tracking=False}, \emph{cat\_num=2}}{}
importance selector: identify low feature importance features with threshold
\begin{quote}\begin{description}
\item[{Parameters}] \leavevmode\begin{itemize}
\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{data}} \textendash{} input dataframe

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{index\_col}} \textendash{} the name of the index column and the other columns you want to exclude

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{label\_col}} \textendash{} the name of the label column

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{task}} \textendash{} the ensemble model type, supported task “classification” or “regression”

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{importance\_thold}} \textendash{} the threshold of the feature importance if missing will be auto calculated by the
cumulative threshold

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{cumulative\_thold}} \textendash{} the threshold of the cumulative feature importance, this will be used to determine
the importance\_thold when importance\_thold is missing

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{missing\_thold}} \textendash{} threshold for missing values percentage

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{corr\_thold}} \textendash{} threshold for collinear scores

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{method}} \textendash{} the method to use for computing correlation, supported: pearson (default), spearman

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{rotation}} \textendash{} the flag of rotate x-ticks

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{n\_train}} \textendash{} the numbers of train for average the feature importance

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{top\_n}} \textendash{} the numbers for plot top\_n highest feature importance

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{dropLast}} \textendash{} the flag of the drop last column during applying the OneHotEncoder

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{display}} \textendash{} the flag for displaying plots, the default value is False

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{tracking}} \textendash{} the number of the categorical feature (helping removing binary features)

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{cat\_num}} \textendash{} the number of the categorical feature (helping removing binary features)

\end{itemize}

\item[{Returns}] \leavevmode
The name list of the dropped features with low feature importance

\end{description}\end{quote}

\end{fulllineitems}

\index{missing\_selector() (AutoFeatures.AutoFeatures class method)@\spxentry{missing\_selector()}\spxextra{AutoFeatures.AutoFeatures class method}}

\begin{fulllineitems}
\phantomsection\label{\detokenize{auto_feature:AutoFeatures.AutoFeatures.missing_selector}}\pysiglinewithargsret{\sphinxbfcode{\sphinxupquote{classmethod }}\sphinxbfcode{\sphinxupquote{missing\_selector}}}{\emph{data}, \emph{missing\_thold=0.6}, \emph{display=False}, \emph{tracking=False}}{}
Missing selector: identify missing values features with missing threshold
\begin{quote}\begin{description}
\item[{Parameters}] \leavevmode\begin{itemize}
\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{data}} \textendash{} input dataframe

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{missing\_thold}} \textendash{} threshold for missing values percentage

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{display}} \textendash{} the flag for displaying plots, the default value is False

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{tracking}} \textendash{} the flag for displaying CPU time, the default value is False

\end{itemize}

\item[{Returns}] \leavevmode
The name list of the missing values features above missing threshold

\end{description}\end{quote}

\end{fulllineitems}

\index{unique\_selector() (AutoFeatures.AutoFeatures class method)@\spxentry{unique\_selector()}\spxextra{AutoFeatures.AutoFeatures class method}}

\begin{fulllineitems}
\phantomsection\label{\detokenize{auto_feature:AutoFeatures.AutoFeatures.unique_selector}}\pysiglinewithargsret{\sphinxbfcode{\sphinxupquote{classmethod }}\sphinxbfcode{\sphinxupquote{unique\_selector}}}{\emph{data}, \emph{tracking=False}}{}
Unique selector: identify the single unique features
\begin{quote}\begin{description}
\item[{Parameters}] \leavevmode\begin{itemize}
\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{data}} \textendash{} input dataframe

\item {} 
\sphinxstyleliteralstrong{\sphinxupquote{tracking}} \textendash{} the flag for displaying CPU time, the default value is False

\end{itemize}

\item[{Return unique\_drop}] \leavevmode
The name list of the single unique features

\end{description}\end{quote}

\end{fulllineitems}


\end{fulllineitems}



\chapter{AutoFeatures Demos}
\label{\detokenize{demo:autofeatures-demos}}\label{\detokenize{demo:demo}}\label{\detokenize{demo::doc}}
The following demos are designed to show how to use \sphinxcode{\sphinxupquote{AutoFeatures}} to select proper features.


\section{AutoFeatures Essential Drop}
\label{\detokenize{demo:autofeatures-essential-drop}}
For example:

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{c+c1}{\PYGZsh{} simple test}
\PYG{k+kn}{from} \PYG{n+nn}{AutoFeatures} \PYG{k+kn}{import} \PYG{n}{AutoFeatures}


\PYG{k+kn}{from} \PYG{n+nn}{pyspark}\PYG{n+nn}{.}\PYG{n+nn}{sql} \PYG{k+kn}{import} \PYG{n}{SparkSession}

\PYG{n}{spark} \PYG{o}{=} \PYG{n}{SparkSession} \PYGZbs{}
    \PYG{o}{.}\PYG{n}{builder} \PYGZbs{}
    \PYG{o}{.}\PYG{n}{appName}\PYG{p}{(}\PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{Python Spark regression example}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{)} \PYGZbs{}
    \PYG{o}{.}\PYG{n}{config}\PYG{p}{(}\PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{spark.some.config.option}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{,} \PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{some\PYGZhy{}value}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{)} \PYGZbs{}
    \PYG{o}{.}\PYG{n}{getOrCreate}\PYG{p}{(}\PYG{p}{)}

\PYG{n}{my\PYGZus{}list} \PYG{o}{=} \PYG{p}{[}\PYG{p}{(}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{a}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{,} \PYG{l+m+mi}{2}\PYG{p}{,} \PYG{l+m+mi}{3}\PYG{p}{)}\PYG{p}{,}
           \PYG{p}{(}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{b}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{,} \PYG{l+m+mi}{5}\PYG{p}{,} \PYG{l+m+mi}{6}\PYG{p}{)}\PYG{p}{,}
           \PYG{p}{(}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{c}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{,} \PYG{l+m+mi}{8}\PYG{p}{,} \PYG{l+m+mi}{9}\PYG{p}{)}\PYG{p}{,}
           \PYG{p}{(}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{a}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{,} \PYG{l+m+mi}{2}\PYG{p}{,} \PYG{l+m+mi}{3}\PYG{p}{)}\PYG{p}{,}
           \PYG{p}{(}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{b}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{,} \PYG{l+m+mi}{5}\PYG{p}{,} \PYG{l+m+mi}{6}\PYG{p}{)}\PYG{p}{,}
           \PYG{p}{(}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{c}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{,} \PYG{l+m+mi}{8}\PYG{p}{,} \PYG{l+m+mi}{9}\PYG{p}{)}\PYG{p}{]}
\PYG{n}{col\PYGZus{}name} \PYG{o}{=} \PYG{p}{[}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{col1}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{,} \PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{col2}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{,} \PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{col3}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{]}

\PYG{n}{df} \PYG{o}{=} \PYG{n}{spark}\PYG{o}{.}\PYG{n}{createDataFrame}\PYG{p}{(}\PYG{n}{my\PYGZus{}list}\PYG{p}{,} \PYG{n}{schema}\PYG{o}{=}\PYG{n}{col\PYGZus{}name}\PYG{p}{)}



\PYG{n}{df}\PYG{o}{.}\PYG{n}{show}\PYG{p}{(}\PYG{p}{)}

\PYG{n}{Fs} \PYG{o}{=} \PYG{n}{AutoFeatures}\PYG{p}{(}\PYG{p}{)}
\PYG{n}{indexCol} \PYG{o}{=} \PYG{p}{[}\PYG{p}{]}
\PYG{n}{labelCol} \PYG{o}{=} \PYG{p}{[}\PYG{p}{]}

\PYG{n}{to\PYGZus{}drop} \PYG{o}{=} \PYG{n}{Fs}\PYG{o}{.}\PYG{n}{essential\PYGZus{}drop}\PYG{p}{(}\PYG{n}{df}\PYG{p}{,} \PYG{n}{index\PYGZus{}col}\PYG{o}{=}\PYG{n}{indexCol}\PYG{p}{,} \PYG{n}{label\PYGZus{}col}\PYG{o}{=}\PYG{n}{labelCol}\PYG{p}{,} \PYG{n}{missing\PYGZus{}thold}\PYG{o}{=}\PYG{l+m+mf}{0.68}\PYG{p}{,} \PYG{n}{corr\PYGZus{}thold}\PYG{o}{=}\PYG{l+m+mf}{0.9}\PYG{p}{,}
                            \PYG{n}{method}\PYG{o}{=}\PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{pearson}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{,} \PYG{n}{rotation}\PYG{o}{=}\PYG{k+kc}{True}\PYG{p}{,} \PYG{n}{display}\PYG{o}{=}\PYG{k+kc}{True}\PYG{p}{,} \PYG{n}{tracking}\PYG{o}{=}\PYG{k+kc}{True}\PYG{p}{,} \PYG{n}{cat\PYGZus{}num}\PYG{o}{=}\PYG{l+m+mi}{2}\PYG{p}{)}

\PYG{n+nb}{print}\PYG{p}{(}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{essential dropped features:}\PYG{l+s+si}{\PYGZob{}\PYGZcb{}}\PYG{l+s+s1}{\PYGZsq{}}\PYG{o}{.}\PYG{n}{format}\PYG{p}{(}\PYG{n}{to\PYGZus{}drop}\PYG{p}{)}\PYG{p}{)}

\end{sphinxVerbatim}

Result:

\begin{sphinxVerbatim}[commandchars=\\\{\}]
+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+
\PYG{p}{\textbar{}}col1\PYG{p}{\textbar{}}col2\PYG{p}{\textbar{}}col3\PYG{p}{\textbar{}}
+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+
\PYG{p}{\textbar{}}   a\PYG{p}{\textbar{}}   \PYG{l+m}{2}\PYG{p}{\textbar{}}   \PYG{l+m}{3}\PYG{p}{\textbar{}}
\PYG{p}{\textbar{}}   b\PYG{p}{\textbar{}}   \PYG{l+m}{5}\PYG{p}{\textbar{}}   \PYG{l+m}{6}\PYG{p}{\textbar{}}
\PYG{p}{\textbar{}}   c\PYG{p}{\textbar{}}   \PYG{l+m}{8}\PYG{p}{\textbar{}}   \PYG{l+m}{9}\PYG{p}{\textbar{}}
\PYG{p}{\textbar{}}   a\PYG{p}{\textbar{}}   \PYG{l+m}{2}\PYG{p}{\textbar{}}   \PYG{l+m}{3}\PYG{p}{\textbar{}}
\PYG{p}{\textbar{}}   b\PYG{p}{\textbar{}}   \PYG{l+m}{5}\PYG{p}{\textbar{}}   \PYG{l+m}{6}\PYG{p}{\textbar{}}
\PYG{p}{\textbar{}}   c\PYG{p}{\textbar{}}   \PYG{l+m}{8}\PYG{p}{\textbar{}}   \PYG{l+m}{9}\PYG{p}{\textbar{}}
+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+\PYGZhy{}\PYGZhy{}\PYGZhy{}\PYGZhy{}+

Unique selector \PYG{n+nv}{took} \PYG{o}{=} \PYG{l+m}{6}.319664716720581 s
Missing selector \PYG{n+nv}{took} \PYG{o}{=} \PYG{l+m}{17}.472286224365234 s
Correlation selector \PYG{n+nv}{took} \PYG{o}{=} \PYG{l+m}{28}.78574252128601 s
The essential selector \PYG{n+nv}{took} \PYG{o}{=} \PYG{l+m}{65}.23012638092041 s
essential dropped features:\PYG{o}{[}\PYG{l+s+s1}{\PYGZsq{}col3\PYGZsq{}}\PYG{o}{]}
\end{sphinxVerbatim}

\begin{figure}[htbp]
\centering

\noindent\sphinxincludegraphics{{results0}.png}
\end{figure}

\begin{figure}[htbp]
\centering

\noindent\sphinxincludegraphics{{results1}.png}
\end{figure}


\section{AutoFeatures  Ensemble Drop}
\label{\detokenize{demo:autofeatures-ensemble-drop}}

\subsection{Classification}
\label{\detokenize{demo:classification}}
For example:

\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{k+kn}{from} \PYG{n+nn}{pyspark}\PYG{n+nn}{.}\PYG{n+nn}{sql} \PYG{k+kn}{import} \PYG{n}{SparkSession}

\PYG{n}{spark} \PYG{o}{=} \PYG{n}{SparkSession} \PYGZbs{}
    \PYG{o}{.}\PYG{n}{builder} \PYGZbs{}
    \PYG{o}{.}\PYG{n}{appName}\PYG{p}{(}\PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{Python Spark regression example}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{)} \PYGZbs{}
    \PYG{o}{.}\PYG{n}{config}\PYG{p}{(}\PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{spark.some.config.option}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{,} \PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{some\PYGZhy{}value}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{)} \PYGZbs{}
    \PYG{o}{.}\PYG{n}{getOrCreate}\PYG{p}{(}\PYG{p}{)}


\PYG{c+c1}{\PYGZsh{} from PySparkAudit import dtypes\PYGZus{}class, hist\PYGZus{}plot, bar\PYGZus{}plot, freq\PYGZus{}items,feature\PYGZus{}len}
\PYG{c+c1}{\PYGZsh{} from PySparkAudit import dataset\PYGZus{}summary, rates, trend\PYGZus{}plot}

\PYG{c+c1}{\PYGZsh{} path = \PYGZsq{}/home/feng/Desktop\PYGZsq{}}

\PYG{k+kn}{from} \PYG{n+nn}{AutoFeatures} \PYG{k+kn}{import} \PYG{n}{AutoFeatures}

\PYG{c+c1}{\PYGZsh{} load dataset}
\PYG{n}{data} \PYG{o}{=} \PYG{n}{spark}\PYG{o}{.}\PYG{n}{read}\PYG{o}{.}\PYG{n}{csv}\PYG{p}{(}\PYG{n}{path}\PYG{o}{=}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{../data/credit\PYGZus{}example.csv}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{,}
                      \PYG{n}{sep}\PYG{o}{=}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{,}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{,} \PYG{n}{encoding}\PYG{o}{=}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{UTF\PYGZhy{}8}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{,} \PYG{n}{comment}\PYG{o}{=}\PYG{k+kc}{None}\PYG{p}{,} \PYG{n}{header}\PYG{o}{=}\PYG{k+kc}{True}\PYG{p}{,} \PYG{n}{inferSchema}\PYG{o}{=}\PYG{k+kc}{True}\PYG{p}{)}
\PYG{n}{data} \PYG{o}{=} \PYG{n}{data}\PYG{o}{.}\PYG{n}{fillna}\PYG{p}{(}\PYG{l+m+mi}{0}\PYG{p}{)}

\PYG{n+nb}{print}\PYG{p}{(}\PYG{n}{data}\PYG{o}{.}\PYG{n}{toPandas}\PYG{p}{(}\PYG{p}{)}\PYG{o}{.}\PYG{n}{head}\PYG{p}{(}\PYG{l+m+mi}{5}\PYG{p}{)}\PYG{p}{)}

\PYG{n}{indexCol} \PYG{o}{=} \PYG{p}{[}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{SK\PYGZus{}ID\PYGZus{}CURR}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{]}
\PYG{n}{labelCol} \PYG{o}{=} \PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{TARGET}\PYG{l+s+s1}{\PYGZsq{}}

\PYG{n}{task} \PYG{o}{=} \PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{classification}\PYG{l+s+s1}{\PYGZsq{}}

\PYG{n}{Fs} \PYG{o}{=} \PYG{n}{AutoFeatures}\PYG{p}{(}\PYG{p}{)}

\PYG{c+c1}{\PYGZsh{} correlation selector}
\PYG{n}{to\PYGZus{}drop} \PYG{o}{=} \PYG{n}{Fs}\PYG{o}{.}\PYG{n}{corr\PYGZus{}selector}\PYG{p}{(}\PYG{n}{data}\PYG{p}{,} \PYG{n}{index\PYGZus{}col}\PYG{o}{=}\PYG{n}{indexCol}\PYG{p}{,} \PYG{n}{label\PYGZus{}col}\PYG{o}{=}\PYG{n}{labelCol}\PYG{p}{,}
                           \PYG{n}{corr\PYGZus{}thold}\PYG{o}{=}\PYG{l+m+mf}{0.9}\PYG{p}{,} \PYG{n}{method}\PYG{o}{=}\PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{pearson}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{,} \PYG{n}{rotation}\PYG{o}{=}\PYG{k+kc}{True}\PYG{p}{,}
                           \PYG{n}{display}\PYG{o}{=}\PYG{k+kc}{False}\PYG{p}{,} \PYG{n}{tracking}\PYG{o}{=}\PYG{k+kc}{False}\PYG{p}{,} \PYG{n}{cat\PYGZus{}num}\PYG{o}{=}\PYG{l+m+mi}{2}\PYG{p}{)}
\PYG{n+nb}{print}\PYG{p}{(}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{corr\PYGZus{}selector::}\PYG{l+s+si}{\PYGZob{}\PYGZcb{}}\PYG{l+s+s1}{\PYGZsq{}}\PYG{o}{.}\PYG{n}{format}\PYG{p}{(}\PYG{n}{to\PYGZus{}drop}\PYG{p}{)}\PYG{p}{)}

\PYG{c+c1}{\PYGZsh{} essential selector (included: missing selector, unique selector, correlation selector)}
\PYG{n}{to\PYGZus{}drop} \PYG{o}{=} \PYG{n}{Fs}\PYG{o}{.}\PYG{n}{essential\PYGZus{}drop}\PYG{p}{(}\PYG{n}{data}\PYG{p}{,} \PYG{n}{index\PYGZus{}col}\PYG{o}{=}\PYG{n}{indexCol}\PYG{p}{,} \PYG{n}{label\PYGZus{}col}\PYG{o}{=}\PYG{n}{labelCol}\PYG{p}{,}
                            \PYG{n}{missing\PYGZus{}thold}\PYG{o}{=}\PYG{l+m+mf}{0.6}\PYG{p}{,} \PYG{n}{corr\PYGZus{}thold}\PYG{o}{=}\PYG{l+m+mf}{0.9}\PYG{p}{,} \PYG{n}{method}\PYG{o}{=}\PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{pearson}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{,} \PYG{n}{rotation}\PYG{o}{=}\PYG{k+kc}{True}\PYG{p}{,}
                            \PYG{n}{display}\PYG{o}{=}\PYG{k+kc}{True}\PYG{p}{,} \PYG{n}{tracking}\PYG{o}{=}\PYG{k+kc}{True}\PYG{p}{,} \PYG{n}{cat\PYGZus{}num}\PYG{o}{=}\PYG{l+m+mi}{2}\PYG{p}{)}
\PYG{n+nb}{print}\PYG{p}{(}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{essential\PYGZus{}drop::}\PYG{l+s+si}{\PYGZob{}\PYGZcb{}}\PYG{l+s+s1}{\PYGZsq{}}\PYG{o}{.}\PYG{n}{format}\PYG{p}{(}\PYG{n}{to\PYGZus{}drop}\PYG{p}{)}\PYG{p}{)}

\PYG{c+c1}{\PYGZsh{} ensemble selector (ensemble selector is based on essential selector.)}
\PYG{n}{to\PYGZus{}drop} \PYG{o}{=} \PYG{n}{Fs}\PYG{o}{.}\PYG{n}{ensemble\PYGZus{}drop}\PYG{p}{(}\PYG{n}{data}\PYG{p}{,} \PYG{n}{index\PYGZus{}col}\PYG{o}{=}\PYG{n}{indexCol}\PYG{p}{,} \PYG{n}{label\PYGZus{}col}\PYG{o}{=}\PYG{n}{labelCol}\PYG{p}{,} \PYG{n}{task}\PYG{o}{=}\PYG{n}{task}\PYG{p}{,} \PYG{n}{tracking}\PYG{o}{=}\PYG{k+kc}{True}\PYG{p}{)}
\PYG{n+nb}{print}\PYG{p}{(}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{ensemble\PYGZus{}drop::}\PYG{l+s+si}{\PYGZob{}\PYGZcb{}}\PYG{l+s+s1}{\PYGZsq{}}\PYG{o}{.}\PYG{n}{format}\PYG{p}{(}\PYG{n}{to\PYGZus{}drop}\PYG{p}{)}\PYG{p}{)}
\end{sphinxVerbatim}

Result:

\begin{sphinxVerbatim}[commandchars=\\\{\}]
   SK\PYGZus{}ID\PYGZus{}CURR  TARGET  ... AMT\PYGZus{}REQ\PYGZus{}CREDIT\PYGZus{}BUREAU\PYGZus{}QRT AMT\PYGZus{}REQ\PYGZus{}CREDIT\PYGZus{}BUREAU\PYGZus{}YEAR
\PYG{l+m}{0}      \PYG{l+m}{247408}       \PYG{l+m}{0}  ...                       \PYG{l+m}{0}.0                        \PYG{l+m}{1}.0
\PYG{l+m}{1}      \PYG{l+m}{153916}       \PYG{l+m}{0}  ...                       \PYG{l+m}{0}.0                        \PYG{l+m}{0}.0
\PYG{l+m}{2}      \PYG{l+m}{229065}       \PYG{l+m}{0}  ...                       \PYG{l+m}{0}.0                        \PYG{l+m}{7}.0
\PYG{l+m}{3}      \PYG{l+m}{282013}       \PYG{l+m}{0}  ...                       \PYG{l+m}{0}.0                        \PYG{l+m}{1}.0
\PYG{l+m}{4}      \PYG{l+m}{142266}       \PYG{l+m}{0}  ...                       \PYG{l+m}{1}.0                        \PYG{l+m}{1}.0

\PYG{o}{[}\PYG{l+m}{5} rows x \PYG{l+m}{122} columns\PYG{o}{]}
\end{sphinxVerbatim}

and
\begin{quote}

\begin{figure}[htbp]
\centering

\noindent\sphinxincludegraphics{{results21}.png}
\end{figure}

\begin{figure}[htbp]
\centering

\noindent\sphinxincludegraphics{{results22}.png}
\end{figure}
\end{quote}


\subsection{Regression}
\label{\detokenize{demo:regression}}
\begin{sphinxVerbatim}[commandchars=\\\{\}]
\PYG{k+kn}{from} \PYG{n+nn}{pyspark}\PYG{n+nn}{.}\PYG{n+nn}{sql} \PYG{k+kn}{import} \PYG{n}{SparkSession}

\PYG{n}{spark} \PYG{o}{=} \PYG{n}{SparkSession} \PYGZbs{}
    \PYG{o}{.}\PYG{n}{builder} \PYGZbs{}
    \PYG{o}{.}\PYG{n}{appName}\PYG{p}{(}\PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{Python Spark regression example}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{)} \PYGZbs{}
    \PYG{o}{.}\PYG{n}{config}\PYG{p}{(}\PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{spark.some.config.option}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{,} \PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{some\PYGZhy{}value}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{)} \PYGZbs{}
    \PYG{o}{.}\PYG{n}{getOrCreate}\PYG{p}{(}\PYG{p}{)}


\PYG{c+c1}{\PYGZsh{} from PySparkAudit import dtypes\PYGZus{}class, hist\PYGZus{}plot, bar\PYGZus{}plot, freq\PYGZus{}items,feature\PYGZus{}len}
\PYG{c+c1}{\PYGZsh{} from PySparkAudit import dataset\PYGZus{}summary, rates, trend\PYGZus{}plot}

\PYG{c+c1}{\PYGZsh{} path = \PYGZsq{}/home/feng/Desktop\PYGZsq{}}

\PYG{k+kn}{from} \PYG{n+nn}{AutoFeatures} \PYG{k+kn}{import} \PYG{n}{AutoFeatures}

\PYG{c+c1}{\PYGZsh{} load dataset}
\PYG{n}{data} \PYG{o}{=} \PYG{n}{spark}\PYG{o}{.}\PYG{n}{read}\PYG{o}{.}\PYG{n}{csv}\PYG{p}{(}\PYG{n}{path}\PYG{o}{=}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{../data/credit\PYGZus{}example.csv}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{,}
                      \PYG{n}{sep}\PYG{o}{=}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{,}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{,} \PYG{n}{encoding}\PYG{o}{=}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{UTF\PYGZhy{}8}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{,} \PYG{n}{comment}\PYG{o}{=}\PYG{k+kc}{None}\PYG{p}{,} \PYG{n}{header}\PYG{o}{=}\PYG{k+kc}{True}\PYG{p}{,} \PYG{n}{inferSchema}\PYG{o}{=}\PYG{k+kc}{True}\PYG{p}{)}
\PYG{n}{data} \PYG{o}{=} \PYG{n}{data}\PYG{o}{.}\PYG{n}{fillna}\PYG{p}{(}\PYG{l+m+mi}{0}\PYG{p}{)}

\PYG{n+nb}{print}\PYG{p}{(}\PYG{n}{data}\PYG{o}{.}\PYG{n}{toPandas}\PYG{p}{(}\PYG{p}{)}\PYG{o}{.}\PYG{n}{head}\PYG{p}{(}\PYG{l+m+mi}{5}\PYG{p}{)}\PYG{p}{)}

\PYG{n}{indexCol} \PYG{o}{=} \PYG{p}{[}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{SK\PYGZus{}ID\PYGZus{}CURR}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{,} \PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{CODE\PYGZus{}GENDER}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{]}
\PYG{n}{labelCol} \PYG{o}{=} \PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{AMT\PYGZus{}INCOME\PYGZus{}TOTAL}\PYG{l+s+s1}{\PYGZsq{}}

\PYG{n}{task} \PYG{o}{=} \PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{regression}\PYG{l+s+s1}{\PYGZsq{}}

\PYG{n}{Fs} \PYG{o}{=} \PYG{n}{AutoFeatures}\PYG{p}{(}\PYG{p}{)}


\PYG{c+c1}{\PYGZsh{} essential selectors (included: missing selector, unique selector, correlation selector)}
\PYG{n}{to\PYGZus{}drop} \PYG{o}{=} \PYG{n}{Fs}\PYG{o}{.}\PYG{n}{essential\PYGZus{}drop}\PYG{p}{(}\PYG{n}{data}\PYG{p}{,} \PYG{n}{index\PYGZus{}col}\PYG{o}{=}\PYG{n}{indexCol}\PYG{p}{,} \PYG{n}{label\PYGZus{}col}\PYG{o}{=}\PYG{n}{labelCol}\PYG{p}{,}
                            \PYG{n}{missing\PYGZus{}thold}\PYG{o}{=}\PYG{l+m+mf}{0.68}\PYG{p}{,} \PYG{n}{corr\PYGZus{}thold}\PYG{o}{=}\PYG{l+m+mf}{0.9}\PYG{p}{,} \PYG{n}{method}\PYG{o}{=}\PYG{l+s+s2}{\PYGZdq{}}\PYG{l+s+s2}{pearson}\PYG{l+s+s2}{\PYGZdq{}}\PYG{p}{,} \PYG{n}{rotation}\PYG{o}{=}\PYG{k+kc}{True}\PYG{p}{,}
                            \PYG{n}{display}\PYG{o}{=}\PYG{k+kc}{True}\PYG{p}{,} \PYG{n}{tracking}\PYG{o}{=}\PYG{k+kc}{True}\PYG{p}{,} \PYG{n}{cat\PYGZus{}num}\PYG{o}{=}\PYG{l+m+mi}{2}\PYG{p}{)}
\PYG{n+nb}{print}\PYG{p}{(}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{essential\PYGZus{}drop:}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{)}
\PYG{n+nb}{print}\PYG{p}{(}\PYG{n}{to\PYGZus{}drop}\PYG{p}{)}

\PYG{c+c1}{\PYGZsh{} ensemble selectors}

\PYG{n}{to\PYGZus{}drop} \PYG{o}{=} \PYG{n}{Fs}\PYG{o}{.}\PYG{n}{ensemble\PYGZus{}drop}\PYG{p}{(}\PYG{n}{data}\PYG{p}{,} \PYG{n}{index\PYGZus{}col}\PYG{o}{=}\PYG{n}{indexCol}\PYG{p}{,} \PYG{n}{label\PYGZus{}col}\PYG{o}{=}\PYG{n}{labelCol}\PYG{p}{,} \PYG{n}{task}\PYG{o}{=}\PYG{n}{task}\PYG{p}{)}
\PYG{n+nb}{print}\PYG{p}{(}\PYG{l+s+s1}{\PYGZsq{}}\PYG{l+s+s1}{ensemble\PYGZus{}drop:}\PYG{l+s+s1}{\PYGZsq{}}\PYG{p}{)}
\PYG{n+nb}{print}\PYG{p}{(}\PYG{n}{to\PYGZus{}drop}\PYG{p}{)}
\end{sphinxVerbatim}


\chapter{Main Reference}
\label{\detokenize{reference:main-reference}}\label{\detokenize{reference:reference}}\label{\detokenize{reference::doc}}
\begin{sphinxthebibliography}{AutoFeat}
\bibitem[AutoFeatures]{reference:autofeatures}
<NAME> and <NAME>.
\sphinxhref{https://runawayhorse001.github.io/AutoFeatures/}{Python Data Audit Library API}, 2019.
\end{sphinxthebibliography}



\renewcommand{\indexname}{Index}
\printindex
\end{document}