@@ -0,0 +1,2624 @@
+package ai.h2o.automl.guessers.column;%0A%0Aimport ai.h2o.automl.colmeta.ColMeta;%0Aimport water.MRTask;%0Aimport water.fvec.Chunk;%0Aimport water.fvec.Vec;%0Aimport water.nbhm.NonBlockingHashMapLong;%0A%0A/**%0A * Check to see if the column is actually a uniq ID column (check uniques across Vec).%0A */%0Apublic class IDGuesser extends Guesser %7B%0A  public IDGuesser(ColMeta cm) %7B super(cm); %7D%0A%0A  @Override public void guess0(String name, Vec v) %7B%0A    if( _cm._response ) return; // dont care%0A    if( _cm._nameType==ColNameScanner.ID ) %7B%0A      assert v.isInt(): %22ColNameScanner declared this column as ID, but column not integral!%22;%0A%0A      // need to gather up some info on number of uniques in this column%0A      // if the number of uniques is ~nrow then want to ignore this column%0A      // for smaller datasets, it's ok to MR collect the grand set of uniques to a%0A      // single node... but as the number of rows grows (think 100M) then this becomes%0A      // prohibitive. As a rule of thumb, cap out number of possible uniques to 1M%0A      // after which we must try something else (heuristic based on unqies/chk)%0A%0A      // Assume that ignorable ID columns are %3E= 0.%0A      // Using the pigeonhole principle, we can exclude any cases where the RANGE of%0A      // values is less than the the number of rows.%0A      long nrow;%0A      if( (v.min() %3E= 0) &&  (v.max() - v.min() %3E= (nrow=(v.length() - v.naCnt()))) ) %7B%0A        if( nrow %3E 5e6 )  // for %3E 5M rows, use a different strategy%0A          _cm._ignored = nrow-(new UniqTask().doAll(v))._size %3C= 10;%0A        else%0A          _cm._ignored = new UniqTaskPerChk().doAll(v)._fracUniq %3E= 0.75;%0A      %7D%0A    %7D%0A  %7D%0A%0A  private static class UniqTask extends MRTask%3CUniqTask%3E %7B%0A    NonBlockingHashMapLong%3CString%3E _hm;%0A    int _size;%0A    @Override public void setupLocal() %7B _hm=new NonBlockingHashMapLong%3C%3E(); %7D%0A    @Override public void map(Chunk c) %7B%0A      for(int i=0;i%3Cc._len;++i) _hm.putIfAbsent(c.at8(i),%22%22);%0A    %7D%0A    @Override public void reduce(UniqTask t) %7B%0A      if( _hm!=t._hm ) _hm.putAll(t._hm);%0A      t._hm=null;%0A    %7D%0A    @Override public void postGlobal() %7B _size = _hm.size(); %7D%0A  %7D%0A%0A  private static class UniqTaskPerChk extends MRTask%3CUniqTaskPerChk%3E %7B%0A    private double _fracUniq;%0A    @Override public void map(Chunk c) %7B%0A      NonBlockingHashMapLong%3CString%3E hm = new NonBlockingHashMapLong%3C%3E();%0A      for(int i=0;i%3Cc._len;++i) hm.putIfAbsent(c.at8(i),%22%22);%0A      _fracUniq = (double)hm.size()/(double)c._len;%0A    %7D%0A    @Override public void reduce(UniqTaskPerChk t) %7B _fracUniq+=t._fracUniq; %7D%0A    @Override public void postGlobal() %7B _fracUniq /= (double)_fr.anyVec().nChunks(); %7D%0A  %7D%0A%7D%0A

move ID/Uniq checking to new guesser